From b317cbf558f0ec8985eb23d96a9f9610f7a15387 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Mon, 4 Sep 2017 23:21:50 -0700 Subject: [PATCH 01/94] fix typo, rewrite graph --- .../images/multigpu_allreduce.graffle | Bin 0 -> 5294 bytes .../framework/images/multigpu_allreduce.png | Bin 0 -> 114947 bytes paddle/framework/multigpu.md | 60 ++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 paddle/framework/images/multigpu_allreduce.graffle create mode 100644 paddle/framework/images/multigpu_allreduce.png create mode 100644 paddle/framework/multigpu.md diff --git a/paddle/framework/images/multigpu_allreduce.graffle b/paddle/framework/images/multigpu_allreduce.graffle new file mode 100644 index 0000000000000000000000000000000000000000..2659baf8ada1c7ed09cda326fbb6e56dacb8fdcb GIT binary patch literal 5294 zcmV;f6jAFRiwFP!000030PS6CbK6F;{T%-aUVquDyAm|-N7nI{EZMPh9b0S3_Qs{E zTQCVpSVIH@LYAFW{`>ZT6iK}J7GG+)%A|xIU;y-)KHWV%^W>l3uA|1+AnAs2`=`g$ zBaa(FyBW8__T^8H-=DqMH;@1M^wE?5IevR|_S?zxMkflpY2)Pmv)8YV8jtr64h}n= zC^$GcK09ulync0h)_`*y96W#XxbgTZO*=mx9NgaCdcKqLn(?(0>>ivXaVJR9@2}xx z`_RB^rLD(sncmrR-GRTh!e;vP(GO3)1mB+?Hq-EH@Y??#B(K`7;M2X|lZn!vTgrk}zoKlIK#Xyhk}@4o!-q?;yi%TJ+)UVPmSFO&GDRuOa()h6TpPOzvxOAG_1P6D=pZU!%?vm{m zT&{B!HV0z;d+&=miSFI8K{x*Vlk7FfUwe%|sg!iDp*8>OuUvm!gQLjrmfN1jQ8>J4 zp~0;n9)9z2nD;e>?2fuw!rDj&=tg=-^4* zeH(u1J;HAADUPqFP>3VH{nhWjO~T7?s0w$_GJBMxZ*th;H2fnd$qyfSsZku=T(=LS z@UmU%6fJiulRoYEO*q9sJ_-EjDF@}tzTAnzFHW0}{j}5{MyTE=W}le`V?R>8WM94&o4yF6;Ou*+ z)XWE16lF7AJdT?;*Fifi;&t%&Sn2uob-3$>lnQ+3m9hY&5hPK;rDq8fiZE-85qz(q ztaRYAfOIMiWt1CXtrYOyrOG3%3Ti{FUHP4$3T!2`4ewMcn*&@G;!2q6Uiq!~b`(@Y zfojy%!EOY1HFz$c`zTJ@LGsRTg*W+2WIAjHx|~1z=vUX~Eclkz-(mjf@co@9=>Cv9ND}er&-078ou;3o^9yo*(GKCWdF3bFASDe@|F1y`*5XZY z6a3>ZkZs0MoPh8@&Xb_Uq&&Y&f}ribe2#7c_{^mK_dNL4@xdUqhVsqsC1~GylREej z5IgVqN$WiQCKZAIDP`G*GVX&hk$o8AJ~+=rxXna_$V8y;q}SSgD4K~V4I=KmVbb=~ zo5YXKe{Z@!q~YaN8niF`CZL==dX(L4m$Sxs^MW)Eqv&1Gx@iV~O&VoqfDWT{-C!{Y z76i+(!SXDL{Z`ZOrb`5iz}!4ywd7V4N~zRx!;mBpNrGzv zJS8=(LgnkQJqDDcCmKCd!BX~wPvUNvjs#2x@FAWMFjh%LjnbNehU5X|#NWkf7FHgO z)Q(=06#8C3-bN2D^PC%pmQ@ZdM|DF=@9};F5SP)He|xoqPtEY)fLA{_z#DnhzKHAJ zxRa=O6;8AcH2j@U-UANJ=bMLO6oNJ#DlCa%BZ)M$@8t4EwS9 zqmO%uhe?v_z~KBZ!YB&v4v$`_Zykb5zxh4a@WPL}S?>6UQ4>-awrg^YLsQ|WX^^xl zTQE}D7|h1_bcfp5^>^5ZHSD4LGe(2AwAa|z)T7|6 zbIQ2^mt8PZ>(^H_pVYc!bGJDRBlJ9+NZR#;VXTY7P|ZP13dK0HSr}%UInlG=k1$Ab zC%>C^kGyF^V>1+3k!J;`0!9o_R{}~bB=;mxCZ-IxR+wI5m6L7fP0}olH|bebv0wXJ97{Zo?N$RtU7!6%00V~z~5VqPg%xWvC5|qd+I0_&nR0~e16ogAmz}_E( z*NR7uc)TAx%DM5#C2>v+Axse~7&WxClhyldFO0xl+SdUW9B{keCDjiu868@$wCyE+jtkFA?4iHOQ z^g&Qe2Vgn?(*X}R9U$gjqk#+KW*}%|t&mIt|8TJh>v01w)Vd{h;E=3~Y`F27Idu>& z89=#l<2BVDWy-(+M}f~$tzKt6YR zjw$K97DlvNz?Eu4dw3jm<^@AM?j|0^GAjuzymxTTgi=frskq})PCpFrYALKGL|MHX z3lj0A71r}v3+t8PR63kW38*6i*I7pY7Z*L;O)@~|Dl6pkSL#dZF$?31g`wJjcJ=bT zo-h)4f+UK-!P@C|^H2#S18aAv=L&UK&^}h8eJjiMaNH3KE!n0|6A+wxR%Qi*oNGlC zfJRB}SxdDQ6ozeh4R$y7A>ndq{8-G$k15y~4liB_w^tu`#t`{2@?+eGWC<&FG9^6e z*61Xj;Uhn+HekEoZ!)RHPOZyH*j%(-%pJqC%yS_YA+;tPh&r2Tdla{pi0rK&B5^fN zT#a+mYMjdq8Zyv5?dBE>!3_fk(QR|+mLHLt!Y^)Z1<>Ce2^#XHks>xDDbnDWSuGfG z8>ImAv`C7O6k*VCDWu41XFZkm1Ux~hB{)9}{JdI5)RW(z&|sGc(HN64cD!WsCpATn z(I`(Vuq&E#@USUYwV}&77!;BtBu6DpH_KeA+%2cWnP$R*_2w`lyPaz#^?)zIkRg?j zR9e0x;*=128W@e;BjJUaF zZgVZdtu&M)BSJ=m@uDSdduKFQTIw7-Fv8Lpu)h0JdYUm~DA7c@)dXs%SyeTk)Ox4V zYx7N!XJ$Ry-tGPrdE`;sw+o);%CpAJkpUHEpoA{%radl*RYDMO2`pv1F&Bv~Z7vd< zbuN;&;M!^K*eWr^a->m6qi`z0prKFNlt-z^p;bY_#$SW{m zg}kC>NST3GfHqYocaT>gub4q^;<6#(jEq|P%aX6N}FIMf1Rr+Ijd{y~mRUOLl+i{Vzs?+Bi zkF>2O>_rbP|0YV&U#5Ay7OsAXGcM z1zI_lSZ&qE6Qgus~Bb&B!tV!E8ZOMy;x`Qd8MGG-cao$|T`) zn=-`g5-`gb#B7Gdy#{3hG5cUJOAxc`#H?6`%ZP}5NDiaCM~BgfHF%2>y+!mE7xNZ> zo7-EQj0wj1T7oF58W!6%K-Xn5Dsvf#rA@%nuQw6KUKZno6KM4W;Opi1ag!1W_3j?> z!fo{AD0*^>qIQwwpIH#K^((ec4Q|BkHsY2cZr6!hyNXN)q84)WyOciIv zI1KuZv-pnp6o%=8Vm*PFD(ixQ7zES3gK3dC%vd;ant2-QnH*NdchSrsG8U*tSQaj* zRO>?pM^qpx7Ar`&CsYW%*XWgH3Ri8QiCq~e#N5nlDV#`dFemEXhTU$JmhYg3GttxV zpHJR5jv9&I{?cIMlzsTLPa?k=TnG6;R^5Ub`C6(%X?ta9nJ`tGl>X``mtniE@+Q-$ z{9ZlTS0>AUs}SREvE1Bx1ci=<-P4=Tmkw+r9T=?LhjmHYImt}3`9qwVE*9bxl)2EQ z9>4EOvBAWRS}^drjnmJxzL&sfe=&%X%B+0ajYjtri8$6St=rt`1hoh-RE4w^lUEB{-spWyEr(OzQp|QBzd0HIzkZ zg^nmXqMKM2l|h7Yz5ZB@#+f;YD=d-PN+M|%n6$0UQryB>avD>6l1L`E(J{m#jD_*M zNQQF3A`GTPuXak!if42VcN5RJ6xBNMY{-gJFoagH8dSzJapY&$1`UFw3i8Wc8YzROE0t5lFf#q3>?llyza$uyZ z+tNgOnk%ZAQbZa9hgO9ciM3l;yOpoqI?QUf$YzvoDSfY{TU4s5;w|Q}92~Kd>!HO} zC7(a8Di__BBijd<>G6wI)ovDy(bH{=o=S5s(Nn83Zcf*+HnK#273nw@X|3CmnFTD_ zf`uKbw+kYjY{)9D9U8JNG-OI*Qh%M0U9fsXYuTGY#}Tp*2C_J>Zmp12%kUQwu@A{x z)c5Bt8omZsQKPGfuHq(^QDu-|d@Vb4OM`Q15K34%wUvxeEif}%+NC&g6H*BEt{!s2 zZS>->nqpzhE|B$83szGY$$HgMbPCL(C$}@G#igs(iduUA84koOrZ}dWM$0ojSx5Xu z;d-j*G;U&LQy&v+3cx6i0T{&6?i4c_%6i32YnczGusj2Sv5ECdN}Bb7LLw*-6q{JH zBc?U;N zJAM-$_u!=cc;2up{d@mUoH+3#r%^-Ue9@C%E?puXEf+0)u8=EBT}~yHOeNKYyR0az z>L!<;M}y%x;W@h&Y|ro_%#R~fgwnPq$EyD;{O)}owN9~>5uec5UuM4^!)EZGQ9FY zO$)lVLQC`)8TDacO(;S0t1YH4co_IH!Y&l6kwUjo(3?&`e_)q3*y8h1^FoJ=JPg; zW97cI;m-Oewa0caj+YO3uf(S=wNyqcfzUU^I(H6eyKxG z8En7$m+c^(MZOE@IlT&kw4nF{RQwvZ!V56duGhySZ~E?}j8MH#>^_kVqJE_MM`jv- zcTcW#t?Wc2SnVG^j@y@?uEML>4MX`RZ{jqV{$}1LAsEiw;~k6#tTY~gG$_l|CwHUL zGk8N^x>LwHAlF0Y&FQl^P2=mH*&S-ah+XWT?;?u*G!N+hcR}*t9S&B#E4+-8@Q=9d zN6_!ET~N95g_?28jpY>=dUlxp(R%qy^xH>Df`31YKD~MwABulH``4?}=U0bUZ{HJ1 zKY#osYF-|}@2$7*>8GpDFF!6$tQ*dE=uxi2W*UB-hH+($ol*AN z&0xm1@7v+;H^I=?7<2dUyTS8z8Ybf&`-f`sS024ThV52-TR+Ag-6RP-iR?+1$!*?` z${IZHJC^_7d&DE$YG@Dp^9zzuS~rY4xP=U|wSy>gkvPq0KJ3tcwfSC;dOCN<-`zw( zQY!$j=d(ZQ@W-$<%4r7AkyP9{K<+g1KfeqdwhO$3^8$_O+sk*sWp3meroDtGnc+lQ zVy)0H2<rt~7q4sJcRewgZdBSM_yet5OtTOOB&pPdMR`1H|#0RRexa|LMu02u=- AUH||9 literal 0 HcmV?d00001 diff --git a/paddle/framework/images/multigpu_allreduce.png b/paddle/framework/images/multigpu_allreduce.png new file mode 100644 index 0000000000000000000000000000000000000000..7741bb37f47e10f13f92c732cbbe9d31f7762be4 GIT binary patch literal 114947 zcmeEu1zS~XyY2)g(%s$NEh*jICEeZK(%q#rDAL^uB1lRpjS5H#DlH&!-sxK3xA*?e zI)C6?dtKL3oMVhR-g@f3?}r3Ubp;GmQd9^8f}x}+s||rXz=A*!5Ru`*CzD>sRp2i; zZ*2u>NW&EQKKKWUhoX@;1cHVK`yUQc_>36bFz={q=wqm+Dq`*K%4uoiZe`0E;OYVH zhCsvuM8J=(wmz2B0j@4?-Xa0ww14gp0YAe&=AxzkbBm9&IIW?YCbf*amo2pbCqE}Q ztpqAHHMN+Rjh%?Lto+}PgMW$BI{5f_h;VWF`}=eH^KrU+*>mv-3k!2`^K$X>a)3KH zyaU~QECV>)yy^b>l7D@VtgW}Tm!pS|qq`e5?0YS(+Kfg=E%h49h`p?rme?R@-Kl}TBF)rA||Jy?R zwaR}!1l)DkrUx_W9t&~ zDd9iH;h}a`FD*w-bR@vnM2FH@z@J)4!{M#d=D%%-LgCqkJU~zSa<%;Bg3Rohko)r& zy}OfCGaqyw`~G|x^y%@h&pqvDKl|@rXH%glO2Xr&M?mSR;r{!NH&PZP!vB0pk($1= zomqAx@;{&XbNhg{aS}Q7AHES>wS~43<}@EE{?iK;C7m;@|8XdarO6Qs<-CNHO8^p9h+aGh?j#ZjkQ3&+EVLtd~puA2a^hmCf3UpBAliSq z6>5EG*!|bjFOMY1InDa=!>t+027!aAQnJ{u$}_Vou)XwO2G@>Q)bO1+fbCG91NUF` zI!RCS&ymTLg{;oF_upO|H1>xby+PW2<~{Y){k5?D6!{dr5|V9B1awZN)a+G|$?Y)- zugz%CLH&rqp~AkN(4#^`zi_vo0dNm@zTA9t=79ODZ;}x2@N-N^y_|39LDgR#EV2(5 zSyc{=h%@BwY8OlVY=&>@VTE~r7>9A&KvKz~U!5u#h-50X$*1pV5G_sOdXMp`3vcN4 zS&{!|qbFMD%umREPd~c9*@&Ym4*t=74XGK~ZUj#ZLA&DIU!5WM8C; zG%)v|Q(=unkm)ZkQJoFZu#rxDeSi16+k3ae?NiWrHcRmLc7rMgaTah(=^pgCdeO?O z;9muN_HJ#n^5+MO{z}GG;<(okyXoT4%Z<3N2ut46;^%O>om$9iGsLFcc#91M)O40_r3Kp@KwyZ|plk#gUIh7_fcUzF}|l zSMZd4gS@_awf~~z_3g!K;OWly-k+}#oktQGKq%abgIMwoNqc?e#nGseY$m)tnc$L5 zHuIU4BQo`f+$hh{KB9Zjd9rq_AyNPTKhTiW5Di))bXmf~_`J!2{>#tA#BgQ`CEHApQ zg}Ujq$6#Y0iiV`g>pQo)1lPAO1f9WyxcN!oP1LmMrSj zFxw+P#>yNVu`OAwelK89RGP6X`sShM1O}!VDe9CVM{g$KyD2oD3c4;n_OHJ)Hk)kd z`Bw97;?nsV*<%qkkE2qfx5oiWqUu=p@dJ`3QL<9s#qF;zTtg;MG-tE$TVHuy+yZ8A zTG%-}dd&ZFq5Si!;NEK?90YY5*jm!3g3Z$q{obadA>PiM8F%-ryv?mb3wJ%GTub)w-eU}TM+T=0ChqghPZ8-gX`%2`l4bYT%1}G)kA1tv!3&T5ALpJnHRG=$ zrjqb6rr$&&QCc`+VMeTfXm?%&H>8&Q=s9Sp?^t2g5pAH52npCBNevWfdLvx>5liey z@)_LK3epn`CrZNY71Vy(klm@GUf%;j&rR727mVrP^dR^$1nP_h+RELkA@tIhRL@zN zVMc5SRs1ymgwg7)+NBF;?m`C{Pb8UpUxv_m*zGkU+DtxVE$sLAI)z$h`Ch;8f~zjA zBz$(dH!rXGRMl779Ceyi^Ee;hT}_AMw#eqsjZlS8O6(ofcYgjH{PPyru}*$Yz5<2s zN87x|9b=47v9qTxm%e^9nG$7n%J~VE@pL{B!89 zPm|(K?Pa)63)9GiQnSBT`rCx#Wp>%?PduQ3g@6nN%7jq+p?I9boZp*CT$G+eIFqQv z(Ur#i-#b>DXvMGhDpJUVBz~=VzjgNH;tL!Y9cJv^P{I`P*^A&KuhrjCN>hoB5NfCq zG3M3FP&x?wQTq_$_c?G^QcC})7uW68Zpp;HNq^XFMVds=#o;pXmx){5eFOc=7P|t; z*>wZAiF~gGL+g~u!L_j~%^fg=9kB}v>D+jCQho3#af@TWq+0aQSz2c{sw6&=)jz>pN0qwIpX2yjI)2PmmiM{4wup-GA2Z32Y;t)T zBfXW%BVdLb%#Fhvem1Mnm4~2}@T2?J=inz_fq%u>f9u(5_dbe`p4$peLa=b%S@8a( zT*Cs#I^azdB3J(n?<~ffJI0OnL%cy~L?e3)EEbJ=MAxIzv4pv+!wlQ7koX1+*0uxN zbXaz?=`E&pF`v`N>h2ojQ9=ZZ_67zV2~d z?O0o7yE8sUzqp-&C%93r*6Owr(`@Md(eoDT(KABxp#3TZ%D}DHzkhtCoa*UVXVjqI zUw`vtB$Yx1f?pe}ltt8`UWwb>FeGXF`p6W-0x{$${mYl{`h2&yj-5r2>8Fi{#1{jm ze9$G@nH2`aJ;&jWYoUqi5lomR%s`1jmTMx1m>)y%Ky`M+TV4rD(4=BUi)k zZwnqhuLJv}fcl6EO#N7ql|#o2Caq98}q^Grzk&XbhZ5KSj=i z*dmK=77WHk&~55H$#Tg{B}m|PZs@kN!?Q8pWYfSlMH8#x?7umgG#BT@fln|^c-~$u zD{sGFZ4@u=2_N*3Gk`tjpz0t#n?<=>HDFSk^VONQw24t)ghoZ`d2ly$Q+{AXZ7qo2 z_T*^%?(BmGQpP>8?b2ZKGyDP(Uh#GMt7Kj^zC%y>I!(i52kNP4b2iF#al3A#?ll>F zG)9iLsY2d`(&qW8t+A}P6MQD<8g8$zcDlL}IJ~~LrY9i5QVDo?Hjo6c*4CXIwoX4V zR}vhGW9YFg^v;dQ_>zLPK=tE7N?UmHwg3~-P``&j9ka$aEtWMF_+zb`K&Tfz6f(ip zFZPZi`L5o$eLTb$3tjSK*8T$vB1BX8lJL7gyW2O%;diH0+Rm+(@IRt41eZ#!;HE2! zC28yBUj8H?U}u3}D@E&M!c3}K#vy+p6`5WfxKrSAJkuW@esYn~F!&u8b!gcsNsb!* zm^l~H@zf1I{QmbBp6&F<@V)d;zQ24?Y~?GtBOReUzx%yJ4?Z;u z0r{~7hw5@0@)@|!bo9<&>-6$fNAx>k7rizyQUS3yecK@rXjluP)Q%|18!3D=E*AfR z#0BhC5*j}&XZC-sR6DLvnDaXxHFWP3S}gN9KNY5C%#Fqm5M!3P?LBPxW3*kR;ylHea`icZMz9V z;d+zK1~XMJC2TfXGbgu|rGzf1iN>%>KsOV?EI7<*s}Vkn>`K*!v3y%f3>-KpnU~lt zL(&hPW8~zbYiyDYj7jYr9wq-dFZ2+&7Sk*a2N2T8ty}D=nn^oxt#66SWQc9K7miS9 zg*w-~CRB?LTje#GmJgzS1s;mJ^uC+QDX`^D3!L8NOi@c@54Z)M!&q~byR-myo{9^4-5l6E*ZV@JdTl-XBoNm&I| zNADgv&e5Y{+bTa~6R?)H`bnnjMbk>O!!34{pWD=l$rMZ7!ZJIfrs&Ab%r>-UgBeS%Psvb~GCHI4YwiV$m|Z!M0J(lp zIW(^r)QG;Lh&n?N-F#JuX5^#aOe{~e0I$QxQyfxVZ(7hgE%DUiew{wJqcGkD12APY?Lm2%u+n}pn3yI!68Z$h1W&QlguFu%e}_$ zdU`ob+ANnWhRj-eJkzkEBwg|x8s96novmYt82bt704v6R>e*O--M3Ah{q`kW!;9f$ z2ogkMMR@|-Z@e7N6fGbyK@PRmHZEpNmx9uNf6n&jt6z*{6g1V($hj>bMn@8$RA_?e zA&yw~&|YkB;SsJ=hg?mA4Cy5d40IB2c->5=St{=7A%z_pbaOm%`jCbkD-V8$Mz96t zeAhAhWqhN}U@)i`(}Z`3$tT2?1~G6RVxsafy(z~hk1IHeH`pAcp5eMo;C(0`W&PJQ z|El=XB~frKY$Y)lLhPXPt(!T;%w^vuqWe#BDf1H_%CKw0yvfJJh`=PLn2AB$(MOT+ z-xFjka9%Qxst$EH0B;~Vg+W-P$11+8F&24!^cJgiJlSD>#z@PWQH)YLngzF&R5!_! zHBQ00iZ)RDKT(%NfS+pNY^=6H@6se`E`}`CNR(1xd37!uQV9a)!_u0fS!DiCAWJ|- zqkW9XQfm}%qBde?x6&YQ z*~-*H)=^i)bHqs^nNU0t$| zvk7A_k+@j0R!TEyEKV|EgFj9FpBPDGG>&1<^}{G)rzZoGT4SA=567&Lbuz5vV0r2N z?a%P$J{v9qqbi=Wbh5@DcBCC&Y2LG*vl})=Y%dg$q}Jpi6^v~u=xTkkt$s}to&&0M z!u zrlJ0N`UFej^2^ap@#Z~Q8C!-@`oL!orVmceCklU;;PogD95n*GT>r*j5}<~{upC(( zpfEo{3OCOLk1&La&oyK>!J1Sq2l?}G<2_#9C**B>+{&vgTqIzo8L*2+zf#jC!NfnFVOwV49HP#Zce8wuk6i7U+TQDqO{Tu9l}xIcU|%M znj+M|!$P2Q3^LqT*WYS2B(ApJ3w=pGc4&Nc@eNdyAblj`cfeL~&0&dza$m!$Cl{6L z;}NRXPf!`3kJVar*i7!mDJ$vkvo7geRFkKzQ-x!1q~gfEM$6(Tzj0Oxl($NvBJUw6 zN-;e*iK+YzD7B&=$1nZ;j6cYtn2eLs%HJ;c8DC6Bq#>G zj^MCIUSO4SQf##A;Z*M-IIMQKx`slMr(cwWi3CwW{Li^3+84@edX`TA3|?~%7$HN? z?~e<9B?X_IL3HV*+6X9nrwdqkfsYPWNR2DmTMV>J_}7XDI(($ z2t23K?=H8@3#~jJ$g~0E0?}>x+Xq$PpQM9-ZReSTP0vcW{ju2i#WIef1EdB7>KW!? zSKA$+hB<7R808nU8jAnk`Q^2TtC6<|GPfP){nu6icNO_A7{z?n^_hAmEE9*x(Dx(x zkwna;H*o9kNB<0W<*pK|v}1^!iXk%l3erX>CmevgKzPTD%fFsXi@?mbam7u5we$IA zB5w9AQ~onPGJ8w0eFD#hO)5GlfB#f-bhFNyB64~~lk7v%}DOO=cjD>O6dNCNE(Xl?~u)+msM(-%X_{r#_`>W}? zyI}~vYAczLn`zF>>F%_>5#xt*Y6RxrK43vPx9-F)*Pmbzfzi7t#rTHMq0Q6NW^ z_0eJ80-h@1uz4i2V2|qlQh_21FUEJ_TLkr5Bmz?A{2s_2f39_V*Eh6#d_xHVs9Ul| zCaPuQGC;!;%YSCG=roaJkS5T8618?m3??51$Njl7$)#DB#lEMW$g|uI~HTt z*jFd>2If4BnGAj|RSYWR%h;a$IC)ZbNTw|%p%+F|b6y+*npo{xOaeOLQ|*CeP-;nh z@d+hvBe&M~g6zKnE|%?=%bA6EO-HwNrS)_b+jWag^vv{7p#d`%aQ32~N3*w=Ra=%o zEjwgyTAz=nsMz-){G&~A!+cB(b27SOl-Q%YOKX~Mom2F<)Wo@3v%UWArZdAiRB+e` zme?%jnI8R{?}op#nG-yJN%`}e@BVwR*AOH@M)j!i18t3|#W~J;I)uzE(*&aDIS@|T zLAoQ8M*zbZp7T7}8bgjCM$S?o`ICpObv`a7e3ZaAKus;qR7QBXv{Ipfh3nTcrUG)Y z16=9p_K%X{ALWZas@}GNz-m`@Bu6vQdwBQzr^XM%n_RIM_opTI>6?}S5j6E%w)M`6 zL+gH~T%CyeGWQ`}VQJYK>ly1G+!@)^wrF30Ds)I2F^Vyf6N{G`q@ll;EPZL&dtLS$VPz z_-)L~85t~!puIAiu(67@zor*ikwf@uY$wB`r?Jkw?Nf zaTAC``@%P*O`ce`2gGC?ESj%$`I(9k_`av#`%b5>A>`=I43cg`bH{KF9Esl_vd3HW z;jQ_92{m!+W*Ccw;a~vBwN7AHRJpIszIKd>xo%VJZTk^CcE5BP&Hyzm{akCnwHb_N3tW~8Hz_Iy8sn1 z$7c#EC!974=w>F695!;*drnhWp;agk8LD>0$OQxdEu-Y#`H$Y8sX|e03pheWq28mh(XzzN&2AS4 zM!bm7e66$ZtNs9gqR%z6j3L;3i4#Rbx!##szqEygQt+l#GTUbdi#z1)Q_@ao=s87D?p#ud;dUVVqnOJnG4&|i4mP+Kw zXTp4n_zFRSxom_6{!q!}^92*Ee_02;W(K5}YvYQVNApHaF7haj!jD)qnwBrqKzD?1 zpKc&EKsGDmd1sfqsZo@i;zC99y)_Ji?HjVf9Tf$88>rwE6G*Q)E&*t)KypRTy2SUR zB7TRLfGIg*m3-HgA0JQ8uCntc!o@9*FiV{!m-zl>_mL8@tRI$y+!uDnhzR1#=nSMc z*o}rZE(U-kQH+?ha*-H(t`3}%#jv?`Ff781CU4ovC%IB>t8fdFzWpFMpgviV$a^mL z5jer<8DoE?iJ?tT8<642#;(MA(fu`??vw%>(_s@9vS|07{pT*hI3@3`;QeYFdRSq0 z76t(2F-T4J6{E-P`C^IBoRWt4xo(|c5=xsaJ(P~~Cx})Pou3x~Iku&^4sun|TK(k_ z?vaBee-d6g;S(Yas0rsoIB6Y#Y@Yo7+5f1cJF!MOVgQdxUG@_G`x{fe-r~Fn3CM`# zBuaJ{Y{w=??0;-FKNhDEgDRK;(xZD(dz~!<10_xDWE8&B`rMOVlGo9%lwONGq3SF1 z7i|?9Vos}E)z@IVj&5Z0zxeUtcD9IoY)@4-R}zaGzI znDMKvH;wTrN7WW9uU$Nh7|p?&Y$a8m<41v^)XT*X8w@vQA)a<61fA`@p|ly!dv1)9 zo_;6?Sj=vdce)cSu8%TxPM56HY|S^+1efWjdR)OVJ)8kWxe2w$vu#~Ll*(er!Am3z zQXhb{zq1*0!W_sTznX=A?A5iVVx=AtM~c}*96N3|jj^ExC)d(Hl!;r71`djE*=~Cb zV<{G7&^QTId@@n<9_!J~%sJhEP_Sv$FmF~5=;WI+8hRIERmGU0U^!w+ zor(hKOR=aUUbZu*Jz*u(v-C3p1hiowC7PoX-B3P{H6fRoYJ8Y0XN$`qa(e5Nf;m7$ z#CdWXmD6$mu4u*+WTjW}s>%V74mPTU#Q?;v3FO#5bG}kxZN$P}Hurb8?=iEYKBMbW z3-eDSU-YTO$p`;@<)6i#oCI_{zSR!c%2&4FqqXp(2-5ZDDlnCd{nW54C?uCJ-q)^vI4^rfBnJaW(n zSd<~_kw}8>pTz`3%Fsy544n8`ZRJLL;w=%K?Fy+&F_jB6dDTKm*&d(44b_O^Kc|Ir zi@1isvl}O|Vy0CVZri}z|za0lqGl<4YOIpW^mCq~-cELSQXsfgE3vzVe6Bs22*bl5m zDRO~A3RR=W`K$*3?QC~j1yCHrHiub|8HNJ(ok4MjU2eEkiIEY1vZZgBKAi!a38SbM z!)9EKu#=Rr;%nJ>wpxmS8O13Ac%uymr z2C`QS*x~Sp!4aOh_>Yc>1dvc>(Y~*((jfU9cSoY4CjSt3Qx$A~_vcw-T}(yV{!H@gAv_J6$Zb$4+&ZI?slH(pKXQ06e z?yV!T=dopZyhmYr77?YCu1kU^1Kw)YQ=^MGkEGeK+fJB+kU83T7}2p)=zU$ zz<5RzN8a*|)S2GXzH_{d<=oqa&11#!#fGvK!l6%-B+f@E79nDFg}53y7ZlZe?#RBA zW`f2aQ5i4XZ!V908%IbJrPqOdhk=6uQlekwZIn_K)v;04BvK}VTC`yD*@_BG25O;@ z?Q3;sEn(`h-^!3FKjrSfed@d?WW>OqLiP<`ERhso{O8tNkyP}OCvF-fi_f;yr5!KodYo&~?@SUchTusIR1*qx)fbr3g zeHKv@TL7;wj5J-@6~@+@vlFPDhG zmTSR0gVS9x2hL_VGe65v@&m@%S0*&*PoUr_z{2`CxtCi5c;33Pu& zL+Le{;9>ke5>-eOP?9XZ%%wL?PIX9pt%R*lGJ$;L^LEbQLj08iDgTGnPA%u>Skz9N z9R_4zby=&b+N-tM#~vy@aA0RStj%ZQtJTlx&av7_=YrB zh9o2KR0aHEsWjm49KXbca&R=hZhd^Ili;ivTj!gOOEKGqNf6-ZiQL3=29O;XK1KNA zs@hmiO53>`&QG_`^@T1nx2SC@X=KNzUVn49?rj(~?dAF%5D2wk#GvZd{}Dp-T7--X zIv=&56oeKvNGjl@JEPCbX6`$$2jbGOU>6Kg0`x6k_-1}+$iwhe*UlW#lDx(nnx-lQ z*4-*WHWnkW2g)4>S>;_|U4oUJwW(ivWLfx-*JZ7^a)67od#18M zg9;+o8uKl_Up*^9j-<^(`NrG0*2*&&0jBE@YF_}3Jn}vP;FRcTNMtIC7EedDqzjs^ ztIgNeXE%=lM7Gv_3KHKNP~gTr3WWtEMk%YuR*&CiU5R_d@8sD#E)Xu8r$|K1kz4qF zG^Uc&g8^fJK4mO5{h>g3)XJdgE7Tb1Hi#yM=Ys+VBjS6WGt~ASpf; z%Z85FmwOdyf-4Rc$Ue8Aeoofg8)GU#&&!4px2d{$e!7yo?ksWYOP0Xe)2;R5gkn{5=XwbcnNvmK7k zxQfDXgABO! zXLOylUR%iPXq1gG$uw(}NVu-F0cDKpL@tLYD0cwv98-g1ICKf)g^+un6JJ!oAv}og zC)^yMF-ESb(zk~SE?~a|PACS+2shp$gz+O7DDwmjey`PVVjdeRGG7ECV(??qTcF&7 z3D!XR)ySDczis?c)<{=9abTP!@)FOm%VPs%ALIqk)C&29Z_hp)|I9dw@);z0?4xC) z_90#8nAvf+sy-@py0QNbw!J?AeZWnPLNQ1@1V5(f<)v=@1xV}7KrtK74fCgEI#3Z= znM=wo)zTTk1=TkIyPG?DNCqE6SuzGok;m~?4i~Lshqw&-OKrR79OC11liDLltqzelE$9KkB&LBJ#1Ux5&N2Dl8xB%m* zv0ECtegRygP~oSTujy=C+y5-nNYrT9c(K}xM5P4b=B$%ioWR3FNpNVR(+9e+e6L4WK4eFW990P9Lnf3~> zDtP1l0mqx0uMvf1-@ZPgwueW&t$nG>3Vf^8u}u|tTmtVq0sxHnBa{Zlm`m2=rU?vY z9<4)t{NUx1EkfS08a$KXk9?w=VXYR?|8lbJb#rd})LY%Q%r2d%N4$Kj$S?-MftnJPWFnFtr5@YfrE1$BY$495QcsSX4C{PD z;vkF%7)uL{=H79^nyKgVOe-M4Bu_5EmNm*gZ1VAcf+5I^WBZ}ZPHKe7d1h4pV}JNP zu-ujy`0ln;fJBy02E3i}ijQR$8!eEOa;+0q(JMt0Xd*;**{I?8^TtxMjZzbDDU@|v z)-yulf)k|f&4#YcL{P%9T;E#Fv~`)atKw7I%dGE~gfl`OKFt))M)4lTpb*KPP#T&_ z`VFvQ5`BV0q7OBCY!^v1D9mQd()Z1!?5uI{0YD=25vZ34b^wWBG7^OA8ECVrPah>u z8;${{)CTgH%1wXlPt6Un&ULyu`OqGx1-xP_apvPDVN~pSO5L6uNF1U zvE&+1ZS>t>TaNS}rd@pU|560txq|oYF3Xvr2%`Q!o&~mGkHCbI81>EXNMowkL{#D= zwSIgX9z1yb%5clut#jrp9+f1hb&lX*xIcxcA51WuB629j#d zY7tm+s7ETCjJl$nl5?83cUhN_iReVHxwjlp1^-uW%Z59Hq?w`Q@qARHv}WieB;}7a zJU=h5d$wq-tClu&`U2Ra>~3O`eK<=F&*m|@F=AVCD%U?q&!B#Pk$BGHV9VBFBb$qJuC|0})v1kC07Y@bD^M%z~t z&z9|WEDLwE7^GH*Mn)~nplr}heEj#j0ABx8)ZYyr`aUim2u0tlJxwdDqoajCBYLG@ z`p&A$JXxkqGSs!bO0^WI)2gt0BU2rW$0>?5Kkc`J-~m{;hh-=BEP|-z&t>>kObKga zA%e#kLTl-tylPmoQUR4h-$tdfRsZDk%N&?pN{^gs8z!N?vXGE)XweuVpej%?#Wmd? zA|&86cd7FlE@K?;f%8j93!I-8h5Ni(49-lS0a+(1h-XSKHa!orUAAbV{XY3Fs@;Ah zLe%0jgcM^CPAHa=!ZYAz0A{a{2Ya`KxP%>j5fo)mp)YHb0zk``a z<`he6%w0}fRlAc=@zC%)(c8ZfuR25!unPXg^E4Eh>$*oM>vX z*8&2RAN8F!e~dCK7`)Q=?zwamP(p?+80%&zSxvzJ8bfVdI=6IVp+@8H+Id`M3Fl4=<)cDy{QUIH-(5}d+5-kH%2RdxAf zFoAHnBS&Lvz%fcwSd;T&LtQKLYv!122qHjO?lc(5;PfLjCPejYEXL$sMg z9`7q7X!u^AlOPJo6Ipz%=r@YAFJbm<>BMJRL(&K`1 zCN7KKhr?D*)JOJ%v6=m`A-2W2`?q>IdOeyt5yCD4X?REaCZIp3@SfxW@>i7tuYbKy zvk1WdcR+i63G^!;Dn-AQ;@4CuvG^vSfW_t6M;@-u2Da~I_{SQW8XNA^4C8Y0yFmJP z1$c)$Ao6&j=dSzvV%6OrROxN3)n>p9BJXa01FiEbuGg1VChGi&NqxH#1S~H310E)DG+Upv5!%ive&%5dU25J0>O|Gz zJtM_rGl#fTqwID?a+D^Cq$2;36fz=(jT0KYnzGV~nUgnjBJ`RJk$MJ?eP>FgRLy~g z^cO1{!!AZ{2rOv~mt2!lue%7mWRc<}| zjB_HD3Jd}nlBOu>3hMxT=>>`GS5^y-t)9(9Ho%zY6^jQiY?3CudpZCU9{`fS#A}?z zB?#MH$vGF305+PMow?j}Np-N_ThZ^6%IUyz*wlmW&5Vx-@~42R@AudktYp=US#$6K zuI&8#YG>jX@gj8it!E;8rBCFOi8IhWo#_~qDFW6b59(Ug0fJfwrB6qLqrD86D+0ny zpuaNXTpZaU-D0ZQ16WNyzoWKGw01f$GXVDh-$Gm-SY2DKR|=n4s;*VlbbN+4gOq+J zq23aE4H^9W!5F_>@tf4wcPW4g-@`^A0B}t#2E5*lrD|_*l5h(B?xlF_)fi@ zK2E(s895DgKe$TOEv?efD-^==W6h@;U<-Q+^>U8H)1XZS?)5m3O|WB(-;})tiBT#z zV5BL^Mj0GHx(Ik?B_sE8Oyx3TmS1%}D%4i+buNzqoH`}E`z(I-y@9u4>W#x}hpWlS zU=yJ=sGURi%HmG+`p@z6AQV9W_@WFwn??CVh0%;dNibQhw-%`@QV&{2ie9T%Kfi|T zJ*<~Qp?IYd{|fK|XTX5~;K*##CA0tRVDTr6D+yvSLb2td!ytb%4 zrCO*yS}_p=E6$B6I^qH-Wm@2Pjo$&GAK(pLFtOq#La*uBeonvo^*L!0tVqESi2a=a z_)YmW>)NsWj=DmXcTD`L|G|pFBoJs96fe>ya~@1SkdkM= ze>0M*{Uzj7jpFI)NjV<1Q!++Amhe{k{oSvAIilzCaM&%DZ)5xX9|K*m0AWW2^hBL{ zRstfnb`3d^px@NDO!y5(IqWLA!#NEiY28u+JZ>yb390FM@K#%5-s*_kRzLY+JS?FH#Nh-RSccq?)Mh%jL;6p%DOfN4y~ z_{P-1>)?nArdJ^s)H>ROsLU|x0Mw|zC(Q4x(zOIiagXu#gFRD1Kn)0DP*OJ*)^Whs zZ_BN_AS^^KB@EQ2@55|AG_;TxSQD&V)_W4THJZU^H@TisqFJ9F0@5@W0pE=nBK0DX z90wLNE0|6vx~5`j8@cIY)Yi5OO7_g{EcWm9Cs?lmnxa+t%qi@z&G)zj@X^rgXnKUi zuKmNOGJ@p78e+%L^~E3*aVs!yU>3zd-vO-Ne$xo0x`p zvmO*EHk%)G@pm_LQCs$5%~a|Qsa1!dRLfrTG)dSJJ27I+nmNLdRu7Pa74)<4pO%<6 zfab=1wtN^IGy8NchN7ZL9eOUXkMH;Q=o6>S34d_uDwZFOvqzrctn)C8>XX{f#&u)*N-n)n>DX%-u^}o@>E#%^%xaNrMacZE363L^APh z7AX)s8hO({{ zAn71Bx&WqKS+^lQr6%+VzaXQJwt{;!-rsHK8v$AXqY)H*OA zT=HE0L_$EMUXn%RWl~M_6~n-j9e^;|dgXj^s(tCYF&N_xIw0uM#5VaFNR{<&qm}YP zi|ra zcN|FJu<80W`2P}j$6y3v;~q6omRHZe0c1h68s3xNXk2}owPdO3%KeZD zmh~?=#k+vaN%(6rUyG8>lb3tlyT#ToN!?aqU9=s_EQW&61&~gF4%TOrg$NJON1ul@ zkE4@6#HLwZu>%kxM=p8?8yt#_OvGaxm>i=6fvJB9u&{yl<2yBd*<~EYu=D8%BoOpm z^AB>4vgt=1-U}?!52NEHi!He5fzCdf!LY)<{N`V5$2B;OR;9wERgOUGQ{tfI_sVr! zvJ|*Yn_K%Tdi6~mbz#Kcc9=OIZ*fU0loJTze$FvQcS-uD-Up@Yq z4Aie8xC%L<%qFlfses zz&FffgCsHraTQ~lNDz72eS6!fBFq@f-3G(B+x+x*4mF*=YwK8{l>8MUnWt-V0g&`G zzdHg6eGpKyqJ9!lo56lR>kxTI-la!sqO11^{6SE_J{=C~52j}x@H;0!G0KfJc~a9u zRKa0v3epB3D*0w+*-eVfJ801cS{>FvYPlVD;@pCq{vt}j8^8>Pf;d%Mf2oE7-Sc|E z&Li@?8g7~c!qQIw+y}kLO#6b?43zr^>E8fOtuVNPDxZ$CT4gD@JwRQtGMA)xC9q)l z6?DP`0rf5rmQFn8l4zkpr{&p|6^l+zvrwi5NRDEJ0zxc7aga^LnnOhj3?!1aIj&H5 zdj4nr^gr6=m%nL4o_qudb1YIkH60_d@5^!LBt;8Qdmeo0fAo8xWgz|b1V;J43shA* zWy2-V=E(T~#h4{enez!VX%>VDvn~pOd8!qO5cJ-F_RSQ;IA0+|SD1XN!y#h;!&o7< zuTwF7_GsfREY<1(Nzw!rU%_YK9 zw=XWY^VcEc{^QjpnQ0>r|`R9?&DovtZ(j|lkJ@2zt)rpZk zPA7|x^He2guRE6{mQA!A(kZ+azw0?V6X;;R`=t9ST8uFj2uL2??rXU8j@8Wdyx=c? z@U2YnTjfr9^bf0o6_?ic=)5(r5AcrJ95wTkaEHca;_8?K1`Q_-iN^jVw;MkwcC3;Q z{Bs}(7_5_OXtUm40Re;+$RuefcL9{lBqxa}pc{_jtIc=*0o7H{{GoY2u8j{MWM}t= z67Wv-UYvveU}qcvn}=f~jWnn&2^|FRHCDs_y}|@O`v^cF+`$~!T@FYOa`EM7G&lKc zlA!t2)6-qPEfrlcv(du=cLt_flMRO&v9?HA76jl@KPJDM291wBrJeoXd|x<1E#hfa z&lzsp@xkvPQC8#PGH!!)xLF$n&$VOczx`$~C(qsgo-sz)mRTvVVbj>Dl+vkUiuMGB zk+^Q>Zuil>kt~mkh&fd8>*xgf20D5Z$Q>|1vp(pnG~yJlB#!Bh{#%52%3zXm4zTA3 z84Kl3>suFD5)o@%fqMKQi*sl6dl63EH+X^2Y+Sz?&!FH{Q#S(gER;{mc1 zcAsSJO3+dhXYd{Damk?@Onedcl=0zxJYDJlciKc=Xws=kce|Xt1(2uN*!Kf;A3WT(Ks>5_YCX*d}%t&?^cJTNmn0kYw;Xy@!-NYlq6mMTv3c*L-fCR=nt|B`SYHwqrh6M4H5i84g4+quZy*<_STWNgbROAm9 zpvSsd?hW)SpuD$pKBCCG{I2GVvJ|rlX?dZ*Epe5XA{#i5!miLYLR>!kG;Z7x{IUe{ z^vOn_f`xKb>cB?}Na_BS`uXPYD1XSApWKROuIQ$IQRv!g)GHh6z1+%7`&{qCGz^aGFal#892MsfEQjRpMo626Cw@fTD@mMpwMH z@_sy&#Vs<a6Ayq({HPk=oeW9sKdlx^`zQ^G_QbuJi0Z6^t3Ccdr`o!I zTywu=O2;)|I?r-C27wc4B zuW7mnj$t#z7A#rUX~7!zX9e^kh;e6(@!zVmR4Y~re6mgxKk_(omm*`8-QLoo9o+-a z3{x|Y5jWawwC}q}kUu^=lfK{u=Xn|DS>CJpg_TS>2puMw z5N=#Jz!z|KCnZ8ee{|6mT#Xefc~u+lOaI%gfWULdtR^>_tdTEdXLfC*C;J0yPWz)W z4%$c}uc2N*Bf%b5kc9q zk0mhb6{L^!f?5$x8LPieG*9RknDCWj4-|d^SFD*zT}9Z<_aqyGJ%Ne6yW$*(;5=oMh|HIvzKSKGw?c~Mdbzj$co#$~L z$8oX6oS$*#rAZq~m8@-z9}D5bu-RtB`V3~JsE$}k9-B97$Y&54?v`|r*ReCgxduXg zjdzW@tUB}hOzm^@%wn_xNKO9wc$=8J7Au7Ud8~m8Nhj(`bzV5Xw{8lriD`}5 z`;AF@(X02XzMC?kF9F+mKrya>ErIpWjiaI4JTZlL#|o+UanY+DDGlL`q%FC6*uzkF z?lgV*7oa5d3La3AzitzM@@(sX3^gGg|NCu^u?A@{iDIw*&wZE}k$Ufmml3@BR^tHC zu+olz3PCd?e9-r44euQ>YjD=mul&-N$lg8bal>yyEmrf-@2Q;|?myg~=00w`#x;@c zSPeaxxYwj$eNnSW<26u;%1S>85=s%d_JdMO?_G<(5bjy~Jzzhpyxf%jC$oz=xNQ;_wa1BqjTi+<)7nyhT3~(yqgiV4ugIe&L&uMU$oojCpjx#8ub0H za)Arap=5%7-ZsGXzaa@=tc-O=G?4w_6z?{0l7rD>ONzrqF;*u>L18k%cEG!iGAA*u zy-w0Xms#ilj9f#JYtD{QSXeNF^2%9;IByUqY0am%?WJCWOgKY}Im4Q)3ZzjFD;uot zp!1sFh&kYc=@8lqyDXgTSZ9-u?S<_o&;2BmJ)*q44xpebcF>uj;s&ha&zl=w?36#b zgiF(D3tjq>KejGio#Vv2$^jcPA`$aMxF13Hbd#3=C`Z;$G@mcn{G^4P1sKu+^ah(* zf90&&>?(Ts?qm^4cr{~iu04#;2_^lj?+StTgNE`so?=qy1)@{rxY#VE3^}2^?}IfC zwNrx`U2+Vh8Z^7w@1;gGd+K(&n;z4#w=raWU{7`@)O(>&ucS_JIAC@dG@j(JJ;_S| zqY^|yrk*kaWG-CeR7%^g`6c3sSBWsLn>SwfbX%>bz3kDutp?Cl-SFYj4W)ES>fR`<)MSiU6+i`Ap6OU*pEPAW)^11sGJMLDheKtk`(E zkNaYx?DU%`Zb&y3?#tcvdXr2jd0ASa>+T@ZLLL<@O+vqG6ZBgTdyLK@l>KrO+f+~@ z;shyaHT$+#i$*>WE|>F}@!#~Wd%gjoUkwyM9U%4-;eBTzit`n;5ncD^4$PNw()hde zqdQ)a`{R>CbA1v=4@c6Dg>lc3L< zyD!yWq3z59|1N<0*Df1iHuN&_|1D`cumkC~>KeSdNUC8iF?E3?yzBaq>@%F}%lj7s zA`|)4efW2^c&;czDSNQMK)76oLN^oCwMomgh=-DZG18{`O!3U%ATN*`7({M}okGDJ z1&tj><}JL2PtR^T49dMcNxn7xa0KnQWjT6LjOW_<1k;p`$VhX zX&L=s2eAj_yLZ(I!}b=cM`C20Pb2^ys zWu#;K#~mPbzaf;ct;g`eT|9lQv@@FW#(7kGFHMJe&S{*dFy z*nn&;3gns5BC07w3pCtuyIe$K>pQ^m-WqCKn>417pGY8w!mn99#RO^#VZ-bcSMv;H z8W8nhNQ_4(9SlA;v%F%+ek@q`()5XVBP~GFhV7g4dE=5#3H;R2%ZVqqxK zrsnhwQ`f%z&5s!zuS^l*alFdmYn2`ngZze7ViLe5>kjtdQVt~ayMu6cFMFoNuc{j4 z6{H~x;_GP3WUy_E!2wwJdNRfy$ZG~IBRygN11m?aCKhJ4x>vVyDb%&WLI##3?30r) zxcg=FgU(jvlW*k@52X+rWMQjNPb!~Z`2ZT@DFXjhoVmZ7a{sO^v#N}wxN1B81zKh zVZ!eqoU7c~_xIC`Z3Pi49o@nxtkfgN5*X~g`#H2yMu6{N(O8flz>4S;*E3i zUZ8Zp$mC=VD39T1a3|+>UhAvWkea(rIbmH%VT2O6*AOuIdIw@7XtOz`m{`0N-BUj zVQcuB>s2eAZW8yS<1_Sa7(_^@X>_EkHdt4AiSik{A$M~dnW4;+R$hWlEU!3nSP>OJ z1kR=LMxfRVx14$R5P0QnuV&5vMWRw1#IzIThU^O7R$%D`(^tPw^cFHh;~`Nnb4^>8 zh0r+J;mN9k0)p=N`r-=!^Y6izASC6z?2l`&GVQw6t{sDifkb4gn3Mp}03!C@tbM;l zwn`u%(}WMjb?r2?fl7EXiZd{=_Hy$xvZWPj!JKXV5V4Q@ z;!-^*sEdKu0jqOC(fAA`FwAgYrNsxlEHu1Ko+gU(hHJp&kNrAIr@TOjOr!~aqfDOo zgc!80<2q8(bzLKrAb@WSgIF4Q*j*9yTp=UPUfauL^rG=5qaotQxD4cAHrI8ee6wOF zAv+bUA6IpFDCnXXkpVcY%ocP88(aJN$fJkTfv1`oJGhw+JNmYjO*-1254V>cSW|12 zRkbqdCdZi`?2b(uyy(3-i5Qs~KpzEu@{$gg3$8zW_pi;rmmPE<9%#OCm@#Gm*xz)Ktpst+FY6ZlM3Xf z#6|>uQrD2;zAEoA;-`#AJPuMdnXvajh$Qb494giYJgiwFErHN*aaK|#yd=Elc~yhNIm?Kxl{P0nWe0d(VVv#mLxDg zZUM~v21eVn-TLwFY>kAZ16biL$MO{RvY#?uMeI06y z`|5i`1Xq3lyO1=au^;*KMAQYLPBQAUKQa*!HmDwU4BDMB%&;$;qz;FwfFs1SmmGpA z=U{c3{92y0z3X!s(2?u`k#A8oyY`RgB(@S1^EG6wX0z+XAj-F7Dq2_1p$O*cBCR2L1Lg$)IEy2NYshWuM|L`FuVo>!WzIZUOFs8* z%Vk&?Aj2(+w8gCpAWg9TTh_#21>MZ_f&C#*@UiBd=WXpS5 z09uc1_$3*^4W1Bc1=Zm}`0iPIneg4y;}tZZWZcpl zVC@Zjmf80&ZDKHR1P%|hsiCn9EP$_IYIr8a#QCD>BjnwQm(de$M8XE3`kMr5OfSex z)wVZgL4qpK-V0FRddJS+vo%#KsDIPG7=zS8-dVtpSBXdj7DSBJKy4j)+JWyj50O1# zjMGCR{+OjmEc&w{Y_E*R6~n)d0`tOzrJUW>%&pi0Nf?MIKjBVv2^8T_5A2pWmST~K zEo06J0PrHj`7+s!bD8EA(}oi&+MCGReTmQ&-^w2RzzkYT%$rI@8iBrD`Nvjmxu!_P+5tYp{$a28;EnAa~IhpmcRioqrmE8}KKAk^V) z5y?oL1spUQ?5L2agfPlxp5KDS^L*bRAv^*HNwxI5vOVhBKmdciRb9mPHu*k#;)sfm z+~40<@_I2NV6)r+9;2cG&rT#GY}kuI_J~6*0Shyi=|L9cf)DmFLBTHw0pZvU)ts-R zh{B{x;q+JX@DswF8DW8lluO5 zwdi6YUr78OKn)BV?wMDW=35bouv#zg@bJ{_r3N3?M2`Be+`DUifMEn4;yv;4sZ>LXM!%C5C_(A%2=>2d0{_!^|H0och1X5aDX zQfhX@e+Ei9sxO0#Q#@kG%rB&a*ZGpt{;MPq4OQ%~UneSKqSUn^H+ctc&AA?KdKD2# z0EmFugotWiKDHwS`4j|>{(MJ7U=4Djiw)V;fQj@Ebqp}rdLXNKt^n4?eg2vz4TNjD zT8h?>fH@3WA_i=oZHsqK1O!Z@Bfy)nEdR8THJi4ek(oin;x-UX&;A1~ApE@3IIIkt zzwNyg9$%J*t+HL|@$)Kl$e~~{@OZ0t>1#Oo7TP_Q=kO(*#h_s@TgKJ=K3}&NW~dBI zcbX(1D?^pgfp$LT2=sZvH~G{2FqyQ(N7S|90%!a0K?yTm4^DMNe?;2o1XVz1{v>Ey zA%;%Fhzhpwweig9>cxa;}IGG2X&6tZIi;rPr(WxA34kqb@(46&n~ zG;sb*xJW^wm6k~NBc?>kyS2aE>%ajH71#K z{kB=21HjqCs^5n~|9)`*orCPVvX3mTb2y1t(pSB(XTsg&;>3|VUw^!+Qi3wV6YhGjk^Tk!p`3At zW!!$N)pLVM3GXa_F6iGXJ@>=^vYK-b#wP@=HWahmzYqLQ3A-;X9qM8v>x6d!gN6~Q0{W;t`ZBt}TZ@KarzZiFv5=a_H?g$b!N(^cLF zax=zCjUUc{vi;nkkZhY{uw=KSuPp2U&D?caw_$;ptms~#lF)ebShp_L}6kji09M4H=(MG&xh+YWLTe+=6&2M^Zgoi=Jbp-&Mw<5g#MO&(lOUW)NM<1dF_ z33`TS1+|PL677NK2@YNLC*Q)3_)S@#>m#n_$P3%a&>EN8R^_yHe{2SZ^7<^o&lD_P zQm5vUgWeT`iNdhwUPjsm2S(SPhe6fO#JAwCb=Tv?-YGq$0j~e+`iS*}Id4rSa(#0) z#(?*BesPPHrC3C4-KaGK(jv#jLnh6xGK0scuDbh7+}B5y8GcAQ{GL$Sp@k_F-MSvz zuxgp!JcMQs`ux%Zn8=7c0`YtRB1?3SHjuprncnl}>UFyS>wrk*l@u!usUi5OYvex2 z^IWv4`B+u?K;#+!3d}oAnXVo^dyKxTt1HK|E7G^^C=9*#J;CWJ1dr+oGe_JAI<5Bs zjLrmJE+ia!=s26*@#o_qYkOvMh2k8KV!M6==MOY|T-Sv% zH)QGN=U8v;E9P~WuvqxA_2bJY@6IpvTH9wG2r(P0(bpDt{SZ z4TpuU+*k>T1}A8hfj>@rTvnTp&YlKBfo}n&8$Fo$op*F>UtE{ zDL-Jx;E`o6Q_zj*|9+e4jz|Xd@|l-e)5EUlr^bToc-M&jr6cc59|F}z-g6#TgHPSF zoAus6Tb)zznP`=&!|;O8?-RJa`ODlu9n3^KABVR*q*B4uQ}vP>CkDa6#FZPWx6;&7 zmw?BDx9^Vf9Rzqw#m$M(vI*}|(dw{t$hte{Ebs=klqCL5Q1T|O5%v))ZrizUvQV9> zcohMO1$H1FA;0)KUWl~sBcNXpQNtkiO=yF`&_pZqj#$8B<_>y&b5um4t|%#GYkN`{1q5v_dz@DDn0uJxP2*mf4o#j zsv}W6;90h3^X`fq_Icbf_*4<#3qf&8JsRn~N$j#Qcij{c!qL-Ppj!Qysca}egOix1 zEdGpqy{QJ6c;xN>?sV1I45HdKDYi1L!>`~60>=9270MMSw>ZST7DAYu?a}Ywpot$% zA9k!sKTb;`5H{+4R`p_@e6?0VIvzCNVcI*2(WEstTpWSd4JcFA&u>#l3#V}sPfdzm zldoo2s1+Qj??(?lh62Vj$d2zx(aKO`(dukAre!DnIWdjtBlLh&e!M)PH-qzhwhCbH zde^=~XgAJcUUsqxGC4&dk7+Bule%vFbl&4*l^gvf%_2uG7YQ)vGEg#vrroW8tP=#N zGJW9-)jD4$+GN{{1oaq^LmS9*11VL>-ysO5g*Itd5m$pH4B))V&_5BnfmHfDuq8rS z5Iyc5&1_*+=co?G^bJ+rrn?{9%{`Fs032doBHrbWNIE)A133>uWqTEoYo_L0j@Dzn z3#vl_(v{s=@P9sp06f4PcAWuyj-Z3e+g=@5bv}eND*k0PyLWdU>4d0msf+%PfByG1 zL{%Wg#OoXt*@a&JYg~4J97_$&#nlDsGV(Y5w=0wXa}}|LUb|vUvs)Ma&j-TMLyu%;gkGQbe|+P=FPc$D z%B}zZJ}i+JyM)+rgK-NK9%2gqK=WQZ0qDcQZL{SefKd(uf!<&EuUPmcyw$Gqopc6F zC>UM*(~l|Kw)zfahh8J#qI*BSIamNauxAhgjF(Ft$XEk(L4X*MAt;jYuf7#XIgbLy z{RZIPvadFv2@0FSjUicX1A-75=)i{|n)r8306F>NFl_c^dQd8O{JhxEbo?HoK5OXB z8%1C;tAWayE$%vWbP&#}c9U7dKWv~3%Fw}#d+ATGJZn1(yh|vo&T9|a~g#}yxS;7_Oqdc&5%w?YVxTc13)x73LN2S z%h%|j#ZkdOpTp5>x*q`20}VzWUGPJpWf%_R8OyFw8bL7%S7$wye@VXkZ0nxkQ%-b&aRxnrlJfg!R;WjZO{;eLZ zp8^BIu(Uo_nSTVi{1{zJF?Rm*RB_SW>9>r!6Q8HvRxL2Y=5hpEH+MD2ZF)UVH%XID zNg(fQ*_vonke=i1GB|cdi7yO}DPZq%Jo|7(QPoihvF4$!*Fc8R2FRq)BZwxKY711p zv^`No&^iOcBXe(tFQzJVc9d~2l`FibX8Y-J38ZzlN1$^(@0ABQCwiUpyv4RZY@v8Db$&Ng%WhITT*D;78F_nv`vIRIS}VhRJt&B2!^07y?r z+3BMNm9W!e87Orou@ivpcwf*KD}H5RTLViku?c{&Hs|K-6p`=1U$OL*;4=ya*!dzb zCuPC`4qXJ2NrhFUn*bb=0^PEy$E%i7nTS8yD$jXRO3Hy+c49*3q)?VX%HXRGgUW@& zumLKgqE4=0-(3V~yAB=g5GTNwWU|r>r4o3T0>1StZiBEqyHkH|KHSo9XJ?1tI>m<- zwKa?9(;K`VFCO=^RK!$lB*IjL$wW)V#HOO6bWfZ>rrIe35&c+<_1dP|!};F@Q$bl5 zo=vC(%;(GA`NbpuYbYnc{>~@hY309p+w5gIdPw=M@8;r2$<^N(b{JGNp$_7z%VX`% zAZLcvL;g46^oPKuu~db3&0bf}_cLpN7tOh!ixlxWs0Yt)Gf=+y{R=j!bWEXG zYSu$jyzhfh?BIuti#im7M>+6B9&o~ZsV#Fy5ali;WXgxZpgb^+PyGOfC}SQpQYX-$ zV_gbz*(Ew8fByY+sKX6(DV@e>Q)Y^~FgijeK9FMBNQXL%gF)qpn`InFXImb}&nGp9 zSR;s6FpA0)uG+ER#!0w*I{B~{Ov)776Q@TgS3YfPQFq?_{$hl(@Xi(>O8S{%Bwl5T zPb|t5zwq*`&bFm{-MT~Y-yFL44w?GbtZz35R!2#w6LB~+nv(q2I;nGnvXw^nGM0I8 zNC*C#ASM>YY(|k>QIVoe6WT)Hruk-X^PGipb^BGYdzxl;=KGET_9N3~|K}_IebW$$ zSW22tKG~F!@at^ThW?jrT-~$ZZ+O7J@BjH-bhsjZ?9q>ig!FjAWlgEnKXi&8C}u_I zyH1S+N#9Cjr3^9pu`Q81)8*MSeAa1Bcy$qXbqT}}fe%8z$H=_;efXQUDmQJa!!`T4 z>-uQ67r<>>YZJ9FG)e4$YS3q_E?lwMRWbftv}Iht>xXGSH3XBUsv~_YoJM1YA{~4# zgVO4}VeB=ddCu$RlaG5`Xxf6j@9A=Muq2+`{f)n{)Q8h*-Ko7Fp&3|4agH2%=;v~V zb8cD~UG($Vu*}nYcE|7E!;}B&j}JRvd@7nm^S}J;-+=3ip!g*87drR7iwGkSV2b=N zf4L$9Y3-D%@!#Lmf4>e6%@7~f^S^z~6&&I7AFw>p@0o$}{UbD&D^1JdZ{8|R{?El6 z45$cAfa&D>3frRZF&#ln)*ve7;OIM5EN^FfVVdRhnj&B)_}#Sc5xZ* zlJ9^w_;9NPF5rT`jXvDHLt-l?mX2^#m1L8_y^zh;tmh*n^#$Ue3G>T;{Ytk$WF{p9P>$F$^bZpFafG(EVA*)x+1V6 z&+yHU#lz(N^MS|+VlmzdYH{c$BgeWy$L2L`R5s0?-ggjU`+6;6e^p$BpZ_;zi5e92 z(O?{6amOAxj!k%K9AaL*geUCoAm;o)wTVA>QiZ=nk(1m=zVs**4i z16qY=ryjnzWwZk^|HQbP>;vGWAu#amXn%?SOu2=)PB5v%)2>?HHZ4OoOr#T(NrRZH zFOXWRgjY2(|Gf|k^m9tX0_C8wIwz>|;5@Sz2$#^_z|;SV`g9TzJ~>nWD_GbaCn z9*N%R`&uufIdXMsx4Z|8P4@Ar;~8IHEBEdUO$$RpaSnP#JKU;I9=@b;g)!k=mX= z5odi@Y-m`j1)(Nh`vXk;uEYhHYXOyr7lNR{b-NaCYk1Q5D^{NWwdni^E09(g-cV&% zgIfUui{+slbvaJI#K?8>lp^-Sl{>TD0r7N4J*VUZFS#5pc(aU*-xyL^F3j?ug9rNH zfI)1o06uiSnbd?C{{t%ehe2N~oSE_U3&2%$!){xi+2sANaSx{mui^nN60lfqfY++~ z4KU4z1D;eC)N_sh{Z@)FG7}jUL$ASN;yDiSQ&*MjyaphGDnKnLn<=oiLfY=`cn%x? z9!`)b0|7WVNtKa4lj2`gDa4};u8XTrtAYomjO<%ZW;HfHz#>r*_#+ShtQpX{kjzTIr(n;JqsxXm1s4%k$3e4-6&M!(<2kV8 zlxTYFUeX7ue~CR~V%$^ukdVTJ24wappnb*H1Qqd*xL{)51+TPnX$K&5AWB}3=(Q97 z*+XLS;f5|&N~9h|&e3M_6fGj8aNr^vvE1}9$1O!94EWn{I1dv^*HSy2t|SV2`rV62 zr}|ZuC4yT(rFZc6(%kUgX*=?F2H=PTiD{8&H`qCP*;sTu0^C{k|IO3}eu8q?QmdAO z*l6*B^mT4DWdIJ$EAhX**D#KEuZ507SoEO#s&N}`WGs>*xpquKNbC6iTPt4AK0JOD z$Kx{QyO<`*A$emomwc<%kW~;ZtTLG6n{?Svh)&&#VSYHEXv)P zN<`1tB!*$EB)BI;tQfh7OcnReY+0Vr%kW~AYr~<_Vi99Pu>QCPGtL{7KG0gUhR_6} z*Mr5y0>}4E16>dsj>LV4=Ob~&7?1E18%cFk26??~KSvD?)!4PkET(5k*!Egh`%K;& zTpDqWw^6Q%s;R*FqN`1sYpvTM7`YOkQzBw`s1*gX=#;}C%+*2TE~oy)zD2$-jWVRh zF22#?>N?#bg5yzCYu-taq6ilRjsg+!!KKYLAUOPlj(q*iGtACR6-)!N<3c$|4`?2|xO)Bz5T?m;Gbrx{EV2X9zd=3ilD ziX`{=7$zNRijEPvv*uMcdwNntbAx>1&X`zO(TZeP`R%1mQxHJT3K6Lt5S)OO1}&B< z2(dJ}>7tbsu8SJDI<{~ysyy@l{wA2yIK5I{?P2^nn{k41rVh9gnB!k(C7F`t4##AQ zatw+X@MG0Gnd?Axn^&fv*^ud_L%jkkH=eHZV(bW|uL9C9tcAh}<(wg5`?~#ML_56g z2yx_Xvl)vFDAKZ(w(9)>D1}A&^~)<{D$4lPJ1x-2VRZn` z@I2d7D%K-gYg$Ir#ed2vv4qIX>gfLfe<-ZNuwuMkjKRdjB40Y@@!}E`?`j~X-H1m{ zny{{VY%g*`?k6t}MXLdmVDEkltF>UzWt0%6|LkBSTw=a(Ni90-G4yAl`9}k!3QuMC?n<@ z;zk710521HLXmlN5$n34laosqSD*Bn{;B8S%Y?yp~05OolZ~x_R z&q}WIMq`<)BP1w z@3h9Q(W_ary5nrL)& z_-#`PSMirGL3?L)CpI^CE;x>4dMBp+Iam%jfiBN2(aE;Sy=at8AuzbuDY+WJMc?eL@#)1a^#2q*TC!$Z~6 zEj_ZMNoNd4rIL7UCg5|4(YZczi$r?VL*Pmv=Y}8$f=r2PUFq>>FiEA;gX98ElAqfT=Ya;reFEN{eGkl0f32dS@7OS1;8 zN`KKPC7n)!L)9LeZ-dn%@0@TB$4*f9bslj^$U!qNNn7`&Uskw&3XJ_KAOiX4K#B!L zY%>^ESRd>&hq*5~SLUGJ8G^?35Ke6XIdrTwpoq7F`JlxgC=(&-P!%&toV;hINi)lb z_1gDvsdE&@K|6^9X3jD!zW9-|%95O=4zLC3YQn_z2+RI)B9P*AQkIb?^JC`~y(F`J zP=mzpi_hNkqn=ja7_IFYIBwwUGteFaujAGALqATeLkw5X#B&iY?Gr3B30%>C5Nne6% z0VD*Hy><6I6q~yAWAW&;7z*VuSfQ!1`+A#SQPx$eC{sj8F6z08zYvlj^aA~(CNfJ? z+<5HG?J*YiqvhFWaWAeCp2IYpDcHcAKbrfS`mZnLAz!qEuiGC?bRR`n;PS*KXibEz zX;CFehy;qAazZ@kYi6Sn*fXU^HxHeVyU#8mfklPmJ6S0r2Z7$8T+DYv_=(lW!Az>d zOn<3>n4($!ZN2f#I|Ttd7)c>+opm*@H<9?>y0=9_Xe=rY5+aJvM)~(Sfoi8r(cANi zGLb`GOjp9=a}GPmzZrc5;uo&Z#Hxl_*k6{JuS)=|hY zLa^4oNZBzxQo(>#^S)r0yb2sg;;r!CMKOUPDRXKhHSVEVmSVoi(-*nz`uQOpOJSx| z^^F_;2}MEsxwsPRB8@%2ZXIErD|A%yUz}!=gY$Bav&N0KD)-g_fcF71z_JeMY85Mg zBr6@n5vhp^2QED~9HDg7p;Anyb6lNyERZwq*VmKdQ1{}-;j`Rs=`g+QfP0JX5jlCU zsX>XK>$Vn@vSK3^ldKmP07sk23;<$8<#Kz)wl0fiCd^hRA*K^?TtZJkn@MHOW4LJa~$>% z@ETA$%_gm_Q3|PDg^nYDx)0!#>Hb>N2G;?kLPa4*xczY5Sj68k7qGsxgkzqz3SWpi zH4*X(L-PsdGaYsW)EvW(PP7K%40z_v94S*BoURZmnJ(14JLYO0(Ng6wfC1b`0L5 zdyCLirI{WJ?w{otm46S~MRi-L;)gD@7s2`)*Qt@{l*A^(Yl?eL12vDkJg1~|F_mwT z?pm~Ng7<;H&@7q)agR@-;+Up)>8++%Od;FncB-&c6R7E=V*Cvl{KJd3<=@zyfD%&U zF5e+rC|#jlVVbIkq3bFq=k+AM&ck#i%rmV@@_MlrbNtd4os>_%UWOKjOK;+%v?&Sc zzkE;GSaR^ie9|%Um#w0&s1wy-E*f8JJ9n*^YNc1$4zqVTU%h zpl?H*1C7N7cscF=K&ET6 zh&_|!de_&i#sT2uE9K$1qZ+ZoOsoDFRwqVV#&>*-I!_Z$+vOm3BN1T?wf(E#;Vq=S zp<12fzqt&9uCt@nqDg6RAI?lI&pE#rL&MAa0QeM%()h~L&dpa=AG0db$5)7mB$_%v zhMgAE32hk9SCu4Zv?jZb**erd0dg`KSNd68tOELui?E{Qv;dD(1(veV^3F-Fu$a!h zf_f-tG)Y|P2g$~D+A2M`K+6Rx_W>J>RBr&^R5m zto5FHsOHZnsVF6G-B1d={^GR#du%-rs9;GutdoUbJCX1(ug%+^bH+I8f?4Is8vIhB z?-j@bhmZXBPLsm5a=45EkGxCI+zI`cJQEk?AY*AYwzysW*F8O3Es#@QJwocvYryLXh zr#NmsrA}*jQ)OxNiPqf%{b_3)EUf%L4o(6>&?e??KAfE3YoJ(=`hw$@tiO9~4F7w| z(Uk+#dfsO-s>o4Bz59}59z3aBmKIr{82iItBlUI?n?=_^%Nrq_TZA+kNWpJrvei!4SK`=KQGipP0wmZ6suab_bCEnrBl5W_~4I5hw)%?&8W} z6Pgm|Xtjx|BedN2t4m$(`2(M+)ik?0XkFY@I0R-OBihVpEy5HzDT(1%vXrUF1XMoF z=U1KWP$V-ZChgmY$Ux0VxP;7Add5Ux^R#~8g9N3t9FNdsKch#g55Fl+tp`t_QroAlUeohS zDq<6aMja5=S5uAMp8=7XHP>pLc1*A6aBqn#fPChaW};0eFW8gI&{Dpzt`^91dhWLD zrO;r7Y2pTSx0mt!iB7;v^AXUB!fSok+8YCBg^>P~H*@gwYUAptAkI~A>cAgb?;bwv zVSIW+TCg<;2=rr@ms?A2Yjh%|Rd)plT$kZq<$WBX;%tYG^R&@bHN3uy-Dl8r3eUFU zBFHcZgv}vYYnuVuLT2+l46{+~KEQK__MD!hjiXoe(PX3&uGsR+>SeQ#Kd+$PGI z9odGq9dd-T6+_X%8}OVU*ombQE9T48BwR*7+8WFlevB_55S(|a5nCeL8$-5(`aNtG zWH5z*X@yQn7@*g0BR5;t<_$Z@GFT_W`C-oElRxpLa>1kd-?lg6VdOqi4YRS4uE=8G0#_Q+FOH2%cwc7 zO_M6e%|ux*K@*={{sSu=W`2unl;&j(`fnSsdF38&fsrQ+g8FYrMR}y1dw}&>2d^K@ zBHvOzY{}l|gk*AZ=sMZ2AXQm@(30!P4Q(f2LYx4z0JG$3w(d$T+bsqFFyzqHZzu`3 zr|nXDn9S;nP$pwircgFBGYa=dfssH$T= z{RwC=jwY2rO4sBwiLjq~Z@9x&vX50#U@)>qn0>O7H3Y%BO)$9L6~+~vi?AP5z)k;;*HRIo8X)X zAY&>fRchkAAy?%gew&EMZw?Off7Q?bL0=Wqmfh^V8UxZ67Q|!*GcZt`T}4w z;(EQh#GHZ*K7T8w{y(B+F7vpi@I4k&96Di4x_X-MyaY^Cd?%RuN0;8146dm8{BDejUX|G6 z#hI_ki+8bVum|;ND2|T6yuZyNGE6t& zkH20328mQs;4T z(8m}paOp*oM*xFr!?l8z6Kwse5!Ha@o`X*=t}m#UjzUvnuJ2^7^b(L=2)%WiyrM=B zgoq1Bcjv4wi8|$Q;F_Zq269|+0rR`F=)1j3r=U%5JbAZk8)Kus5MS{Ze_iqX+pDvT$!o)r3q%f7Wd4F}qmp zSm_;u--;!J_2KsyDAp*BamS$$h$Or+UP1Z_uiYsqI?V%Fym(^c>$Y+f&e$xN)jjJ? zjHQ^CuxUc7YhPvl!~1VXYkt!u&nqNLkEEDk@K&Vn-5iK-XC&OAayORaThv2|~$ z>wSj!OHY=E%p5o9Ed{RN5&1hBQ`dHW1y*@K8bF^0i@n$fR zUpnRog^mvr;eB?md+(`UkuVN=r8MX)mQ1YAvEU19g@VG@enUxSQe`1AbAj~;1<)#N zAV(M5?Jh{dQNCc-SrPE{=lnrVIS+O6{#k)1{5=Kb0V6Ynl2J&EoQ@mx-ygDdrp=3s z%RqLj`Q0n#8kz9k(M6tAKL;(^+@fZG49tQ{&WYhY| zjmRM$A8reXU4c*oM~-Lzu}Q!6?@_x*6td5N6o#AtG_Bxe=LP4H^oF>DVWH&5Y>w%R z_E1Ae#{@~I?t}O7oCo%6DeYpa=b^lto&XE=Nn3n>q^&UX3 z(4s=lXQt-J04so!YA^yALJ+BM$WbR*ato+Of|=wk+xQwqxPS|A&-_Fj76vVUan36=o08YfK(??|2bvK zEqCzo>bbe64C=in;{tEQ)7HlDxOc^4R|#yCB5VibJ)@Eg>-wrICZSCMDdHb6boepw z6d}5T9;g5idbFzZ&A%Wog0*jS-R0(jo;-Q-XYhvkmcm{+K`YEJ_u?=dgD-Rxk zeRc4_JvQ!3VB8DV*`lwf*tT958U#PH!Zv%nh$)aecvdprS2<77k1zSOont~2)mxxx zii$Z#vX<&|vjdcr@z-Gg7~X@FpUx1rvKFwM8@ixD_hW_Lk<#%^k`fiQ8FSa9gzD_~6A}W6u3W zb48bcFb19a#+UPO%1$6qkS%KMa;l}_7~4_BhP z{ihvbfI&IkUmb}YCTJ3{pBteR2-tqt*L=P>N4osDuh)HB5=z6K?PU#At_8sGLAoqX zU1*f@2W%q8Pei@a^&%0^EJmK!&V3D3>)F&SQZ+)j6)7bnj`G?p`aP39A-x;{hV^;Z84Q1 z^bO?L#d-O)(J32eq0QaLwbK_&%|)Tj3rAfcEn-f7J?nOschJLdx9r~7SMnGrJl`q2FNkjky*GZtSP2N3Robdtu5Gavs$r(8f-Grf4}k z#l3fIG-e#>NaLQRZh!v<>+C+10G|D6x(RZe8G&gC@PQZ;KwZ6fm|^obNW&1hEQr5; z1Av+8jVjeufw#{X7bUez19*f%df$!XAdc&G&zP9hu&pcfsPNbBGsBjTcLR%@s9 zO##Q8>aYnkztviyb2U03ewW;UFmh6%4R|(1zfVY_DUacREOm}Z3+RfBNf4o%i3@h) zz-$S7ImJgcNSUJcILW^BY6xN&%O&ls6-~puqUiO!@DV~mU_i`-ULG%%Ca)_rB{N1q ztZvWvQ{TeiBqz%no9IMZ!A71!1doD57|y2YHad0NMH&U-?f@wMxo&E0p8@tO@Kv0p zJGye?cEh>Q@Vi!b1rjx3w7B(j?9`RGMWQ&>Xz%aKo%ce(h5sv@T=07d0DQ)&5I}DM zYlf!d&e6~wbaS$clL!yGKjYl$(NNVTySc%FETb=LiR4^~2;}5`5ySo)#%xTc;kL%WXs?0_6(jPul0BmML!iW?la zKS}p(iR)2d?nH!1W2`vG__CzE!(Co9d7QXE;@2lx$8}*M5Up6?U-7}=1v`FALX%=~ z2Xt}cwYI%kPmbFVX;lxvVV(ImD`FLtqL2t;g(=_5AMiUZVU7@XzTYG-Q^Z7r$HatV z8+cn(S|vBpcW)@1;h2}@7-5nYb$eN4L@KM|)bHFS zZ@^zsw#QFBZWM?Yuz)0c6v{+rah>^;Nj8v^B$e1*WYJTuuD9Cc3z3W zNRIvwyULfp2_uxW9CESGy|1?ovasX2T_*lcbP@I)Dn(M^c7~%r%91lNDf4;QFtONH z>Mg14&vE4gdzvQxjN~h%gj-tQmKVUDoJ$4Exft5 zmCClC9mT9GmH1nHbm)@Yn3};&+ym80aBMsgJ8Pju0!{d9Q0I@W>!=a`{{D3oI5oyq z`EYIn>E~lgVC0t}JaPz*5GF#p2nd=nHoewo>`W285FU#6_w$2Hkx?ts(R57Pyw+~K zf!`t3YSEbTqTJquR%SRqS>^z*svWr4>aKMH6LsXmH{>J~I5B5cXia~HsauJ$jdw<1 zZ*G`6*b{dNX4z@P6abXtKph+vGe_vm7jVWOIPdj3Nsg-%B#(Tfl$=moK_5!{-ATyn z{%Qw2!YMx4vt4N?;Kso5NR(#=*la3b^GgpohP-C)m+p|{JG3_)b=W&+zPY0|CK|1s zMO;1|_fBx;i4aHERcZc4RG*M`xwf4l;XmP`v_Q($11Um6{sLrThoU5o6PEIUPHUsf zzgE47`4ESg_tmd8d{|>p%85DVroV6vPf7PmFV_F*@l!xQN!Z@hRPFIx%9ZeaBnf)Y zZ%o>S!?jFB-4KoEm(OE<#;sti^?-b)SX~}NZ*V|GWxZk#M|Zgs!}_NvEF+kiQKGf2 z=})(AsC^%j1&X$Y&#)sNU1VCO~W!tncAk&_5gg+NaFf?Yr6eJXA3_NTK@$I{Bp zKB1cfM#}?07tbq*M4KMPp|Y7(H=psOwFA`1TQ{plQ@X{sjCEFbHAK!V$J3r?m##sI zNGO~Ni5X0$Ic28BcArow=Opak6;}jNluB$P`ARZZ-ag96!4p9g`Nx<5Rc+v7BAhJe z;iAxOXcLS^ttxL=LH&!3WO{!jYI`^mRqPcW^cl+^fLjH$M%N zv65rld*nM!2xm@-ZbHT*?m~Xf=h-;S36!dfuKCaL4Qi*RYmYpEz%bneP4MU!!-M_p zSJhdR-*?%jvkZ}t14scOarKn$8~Ugr@&--WUPkghIHKokOfu`Sk0Pk#o0*uPnH%}a z9*d*$FhB47NdU(d=X^`qyYfv&rF*YmKGu6!gncPrD(3Nxe(4z4o~^li>(o`AR$o)3 z50b0OznCyf8QJf#{||c~m_Jd-pB6QAyqkP;pF*|!kyMqGIOaC0x5t;Ped({BF!kH? zQ2)H46?Qi!P*?FA;hN8ADV8n~+TcGnt99giLeWFcLXNz#6ht`LFo@ z_KrLOflI<*JrtGQSLxl1*jcvaNRe^|vw`74PtW8$mAq(2H{cVe&Pq$8;%&$}UR5NJ znB^}Mo~99_G1AiFkPZk)NeYsRf=DW1+;7f(?)~>XM<4ghp1t31y=(mz z)ZX(f{&O2pB^aCIK7?kiI-bS~Q+>T@dOOD*`Wk9Lnf0xL-p$Vej=bX0!d2d<`vD=- zr%fwbuPL-xQ%!23i*cUH&ZmNR@$S z@_Cq)u`oPH)huS5Y3#9JdrMU1*NKC00Db1eVZ`$x8Vir3mEGraQO>u*^MApINRfTH#-Z!JLe#n1A70G0eZ$3ljs{g8Jt2rK(~?m=5+c_j`>vV*Iil$)VISd zzvmMpexA^eP^li>ybA4h-L@_{?7=jCHtVCJbNm^2tqHW8F2T#Y@St1 zk1c@~VN^1R>r=gGm89?cYti5mJKo1v63#V)s;s6KLz*&y_p~ZcJJ|3d7u+zV%PDD$ zEx#+w)RWsW&IBKHQuZ{1ao8#|!}M*y4Jv?4Mcf}MO0gu~Uo*HO<9Gb?b-ElMH^6e0 z#OI!l*$FBIXBKSmn8Z!{EBpf9slVY6qcvRt=Td?I|6uV-b5CSwMYLA)?FohD0=b1J z2dqfuR;^Q!m$Cz_7YwiJY063UH&py8P!zSqB>3g%Q9;Jl0q?f=TI%T&&t20klAZ{? zh~Jnb2*tbi9&90-fY;Ah`yyMRa_r4Pbuq4M#)$a1R`HC>vk4D*>C(PpTu4^E34Eh3 zDIY4Qx-FYt^E%1HnNe!EDK4E1+$_3X?Y`uAsG)^nA*VMnY!zUSJX{Hb*D=4Xz|OWn z^9=o1*0I~>VeCM*K+``WThfy1P#C>K)+uPnBi!Svm_IEM^q1EOf?H2V^%LxY$r(j) z+|}(#CpN_7`OIokVN zK+$uxq;YU06p}v*P5Je8E-^Ef@z>^csmyUVn~TL4^AY}hQn=7~{Op_Ymc~bL^KuvU zny4=?ILV!c3~j}nE{8Fu&eid1AAWyoaAV*iLFrh$=pM4@- zNiF{TY#cu&cYe9NQvhz|KV~xi_3mxypICS_^N%=~Ovs--dL$SHFxDCm2FhdB;vel! z|Cm+ibu8TV#w;fcrN-pTmIsyhtL{`4H@39lONvj|@kX&N=fLfETs3Oc;cxsuY2@hv z<(qk7uS&C#A}Z}hj59DZGvmDgk9i#0R_|TW!Odb;!B)FbNe^<>2S0<>G-so|;yxs? ziD|kF6*tu6xNG9zyC|n{HSH`l*eAtU5iKxk3~IT>^*Y@pJ-y{(@u*sJD{0iJ=#)~Q zri#rYx>*?Jvja>WY41EZc!LB^A(SVwY16bfK8q8L;v~NIH1aa&QiVKn6Dr=?VM&?8 zl~(N&LPU$tsM$wBNyn8s^!n~$nn6?dW3Yp)JpRlWYf$oAB*G6V14Ttk&>ingC(}@G zfry=2ivfWi7Zkz+&XrmQk4;xAW~!xDs&*$1tGF}>LS1Fw!1P_cU}0S0w+0`>EgV&B zs~^?=R65I-IA`7co9xQ2pbKwtkb5WCGEDJ<|2zR4BzRjm^uPQ zG7%lf2A^4k-Iz%;f+ua62mKu&Bn!F3K4f02wwE%WF`Os9?Lr2e&|xsCXPZ$sfz z`mij|?uHi~Mcih8l3*^_0WV$zZs8SASI)pLwpm>i^xEfa?X>S5R(UzyCTE&Cr@s4c z9M5UTP;_6w$g#>0{Uenm`;kJ}ZGx+^`R`#eS?$Eo_5gAXHI2|Z#;`>4)ER6L z$VeWRHBf(Oq=}?=CC=N;prwm24iI??<2$9IGEE;mlqnC-^myr;aWQR zH}@>EIVzN}Xw`g_?!I$qj4{VUOtY9OKn77i)j=Rma7qx5yFL@dUnc!ir`_jw>jZ+6 z=IsoksIr#_Yo5tJziNuT(A`UCB?$?`Ji0)f9A+{xpXUf>2^DQJE~l&eU_1s95Jm+cf0Y0OxZ~?2;ALbS;H%P1r@(>s z{rlcP9Dlh+&8iaF<*FQiEb`6WWgNv=@W>*snVknmT32*e!;I@7U)wpCZGvZs4OwgawQ=M3JR zF(SO*aM;s`$MRDpZgKSy zl#^H1yx_a>lEj7q?_+Zq*{&h2C-SWi9F!!Z1-8X|y)r#V zDr-wqLi(PtjH3zm53WYK(^yLvEKfCb)RHn1Or|OCC|<#nB)o?>G(L!HeUM)0;CY`3 z@3USC)<~wE+rhI-MHEI==fh{F5SZpDH-V+yyARFx_Sn(T;93xDRzKvNC{GMsExu)` zh>fIP9(6kdkgDPowTnuSesC|x**Vahbj~V_*EK*YCG`Pf7T&=JxzHfqi0dslt2Qf=zB+cubiLu8 zi0xm|Y!g&(rNUM%%~OVMahEEynokeZcnwOGOJoNQ+Qe2#-lfXfYRUZry1;Gb@-oJ| zAN4pVUH7QoABp8G;Zj3LU}bn>+FfN|r*hQL`IZJ-;bx9uOo4}AsAh;XLU7s|oEZKUZiaepX~jhwq|SuwG2?Nlf1whH z)93gtVXZMy#$#A$z9IN~kumNUetl-W8W$`Lxk43Dk5)1^o)u*pnnKX!ePcBaA4 zXXf>7+jjtozb!b$mK@(f5gVK=Pnd@5;t{E6CIO#KNt%Vjw^Tcz0;&kEp{T;9=ToE2 z-S+KRowl6I`(Y#LRgSY@!Xwx`TvV?s9xBFndG+?@+5ErZ4Q%}>=hiRezw%5OWcior zTgiDSOd?zeJR|P<@JkvI<4g)>alQFmYubgYBW$V0|7AZm%_a54ws9Av1v>a#-rEl0 zt2IySd44o|0g8W5{Me1wxRqv18pG_8{Dz8Dj=uKIkRPjJR3^xEdwe@AMH+j)38Wv( z290Dz4De#O5ZrMB%tqquG*7R9dF1@kVebQw`Xzk&3apo8R4w+t+bnRyC)#T3a6;?9QgggQ4HfqKZBCdPOy1Ks}dsMop}f#Z;QjF*%YK(LL4h zj8HXz%A@g2BMrR4WS6FIeK$SJSZ7N@vgerRC0VkBfUuepH8r??lPXQNW(MK^u07On z9(>z3^#iwMds9#mo)(Bx&MKpPQPo@ZPB5bPPQhasU+1q%8Ac>XgS?}@CsB-#ko13^ zURKtaJS`We?+YZXy5J@*t42ir={^+ICUX< z&2vl`wgvi&NtCqT#tJwwB5Mh34*IYI5Lk7^`@(?p}DotPFr+G;blTFlE6p z6hF9Nt>hmf(LN6;GK*6J$Hm4JK|?83;a&mRMC{4n18l!Hqe^@yjXqiEFh|+JTCdCl zc#5tuvxfCfv?Xj7K7(6@=iN?$c8nWV@IcBjSDjSz}}0QU^Cm(LPOG-s zc1ZGPf+Jpi`D3BjC19lZeKDHKR7pf-7QSid@<)^5rEEve(+KckahU!o+AGzmV`9Z4 z(oMl+t1PVUl_7du5sC3?;Q^_RNoc1yF%hW|XfV=f6~r&)&fG z#`d5>Cq+@k?G*iy1CGlVX8x`(e#s~phhVrFEEP6d_57_j<8tMuYjQR&23zK7y@o2X zVcWopEr25W&>1Uqv&In%Iw@bJ6(3*60nO_{NN9mRL;7AgfKc=ywZoE_GADdz)xj z)Q%WEtl~HI_aE57PYl8M7W+)^r{|J>{+T2{(kwJ!Bt6}s@u%zJD>5qFl97<~%~1Vw z4Nqtr%PNy3Z=}PA7Zg2&JF+ISgPCioEbgH?FriV58GGInQ#El`=>-rQaZoFQ?|7|t z2n8krH{$iq%($0<+r(NxyQI!VOgxkz=x@@NVS8ZCcZYF^Cv+>Y;(8TVHWh7z>}x*d zDSHksaUJEz^%z9v9xsaR3ghLlDTQb1E1zBL_Q0aXMDI)fIk!*_*q($3LrpmT^dMulB^nO}htz#$kdd3^w?S)o=qa=$>2NNo9C+X%W@5q1*| zO@dsF9QwTb!LYN7r+M2{j^%lRR$Kp|MPA}*VqaYjJ)b*?&;&tv=(ycB!N^@;r<3Fd z*%J{O>`KU;_F!+HS51U29YYGfFb`#+=473h%#9?PzDyL*n4dpeMksT$2*|QRfd{5i z>&lo;p1anIC&^Lg zfj{Q@+S?WejR4qF50>RHs^1_%N1f&eZn`kVhD}ajSsu0@_*ayx;fb8Vg#^So_7AE> zXH&Wowq%Nmjxq*<-CxbpwoBj*4&ZOK1+q8dN7QDp8{worRUniBtuanUOyA9``WIvt z>_W}H*+lsu5(&@k^fOvG%HFis0{Jj+*CviNf1Jf{F3nLwPESa6FRFGAJ4Omg{=ts> zTsW;6P4fW*o(*9AQO8Yh8&u07kE;r6>rb*`qV)vX!s5X)>{B>2%5&=_aB*Wz9t#jw zS1@p^(XGqiNDh-C5pN4_m?^(Q}M72!*6k6-rh zH;3==>HGW9D9H?|ahQDf5EjBx)zX!ITE0tNb4c6`0A`yhgSH{Y4os&TXkR1JG#AWb zeSyaeRh|R4EjtfdnO&@bIVhJQb8%X*^^b8%#_idPd6}7OelSLF(3t;sx)4q4m3sWs zr#+RPfZjm!`EQR}Tom$Yb?ayQ+hBM4(-ZF~o@;X!0FZWDzXl)$J)r^$5zS;W6UR8_ zlR}nJNEK*(tzy8|FwoPS$71YS2-(_HKn3%)1RUdEPISo^4lCsUMA%?Jr8e@C|HIS97g(2cT?bgJ9d$l^fPBeQ#Y}0Taf+!;ee~-2_q6vnsLi0k)JHR13??%^nAN*>a~^e!_2VaI#a#GY$_ z0IdW%-jh*o9)yBI2ohhu<8_xNK!`kr-yRS)tJYMQ=6iGFD%K%?xNU-R7aOR;^`jTD zH&Et`4V};wAmWdBL-ITjbf|o;kImR^b{s-<>sXe8w_#<=gvH$O2{`$Of&I}&l-W9{ zcyNQ0)k$*x9MmARF!d_WUCbvbb6^7XED3V-q8K*jN0>XxmsKufS@g}Ef;(ysz#XV^ zGWDg#vc;+UTbxb0N(6{=0|IH?{Wtx;WEw%_EKUvJv2PA?t7I=gTHZcO|5M9zrlCiq z#~QOjZOAcezvWfT56}v|TSF;fCd;j^+i=da$T3p$S6&ti#I7modxD`dm|um#4|Im9 z44hXF?Fy<-K=fWKGVMjjqe+7G^EdU|J5wE*Wf;KsNO4tZS>(P0(e;!qD)VVDE{|~~ zcXLgw63!8^=@8tvt0ZetQwh86JWAf0z%;3IcWc>p4Y;eph%gzxcs2-_3C@P%K zW%&Uatf1esI|~n_KLp$b@@^S8(HhoX3(a$^E!C1B6MJseW#!bMwjkv5Mz=5b?%T7{ z;@h7V=sl3vH<`XCXou}lcM;+4QRH2RfBNs(jC}gv{p0}dCxr-t@!=}Y*!+Ou zYuE*-1&2%EQ*=iP+%!)d;t?KY96bbEbRy*U(s}H8&1D~85`IN;Sy;DMKC-C%QL%>} zPC&6}1tGN*neU+%94Cd`?*U}-eV3mAIk$oLI1vQrmw5$Z?nmrH@{&YXDNLX&<)45NvOJy=> zMw4L~U^4K-9e5LaGZzUA;9D7;)#A`EHgM*L3|&#N^>bC{zDnhcP!1S*{z`#l4-Rh#teUw z_Z01$NAKvOIgjr|a-FFDuQ2qcJz#)oXeXHWZ$;cQMpO4+0pQet+Xe6lEd^*RyrqR< z6Y`=)Ee5%X5b_|=UP{eS4X5|II*NZjiTtA-q$t8UxcGe=E=3sKOtrC*CKO=#^6N6i z{Z6jX3M-LdhY+!QEwRp(gKxr(I?BwSfPA23SY1 z;Vv%a6dU`&#=4vEwqtS>g(h>5HM*|3j1F&@4E^N8f~bt}yeOndbU$zDQFMLOkpca! z(h?Z>y}g@89Es%3gph9m3!NU7u0rVX23`PTF}99s;Y|`72VXC@JdUgJtv$$?td1?{UP-@Ry#i8#(su5>3JsD6e+~P{ z%%0f>k_7Yy$nVcqlZCx9*nv|`0)NlIF>xf;UACvnDM0fnXP2Zq7CKMFO?~KJFp#6P z7kjmsDDqc6I;om#meqz`P?y4bdPooLJo`p_JXW(OfZhxzg$;KRnae5#=yJcZ&{ z0J|n|v$V#FFalRm2Wwxbf3*4@Fsf&azrB)~!6%UF)F3V+a(8|0&=mE)jRMBBok;5XLu_cni{%q|_f%RakBprWfbW`E7ju($at`1Mx#&?H5JY9{? zTr6jSofjwhHEGDe&IPCHcMu|HB@I579&9%lMB}!eQds}v8C59RvjaY7O4Tj`2SY1h zCQNbuBA%e2^#&=y)0@2hQO5W^SN0BR zp$5Z3rk@@PDmj(C)5?Z~wVXAUnUVYFw+|1QO+OnK#^g4GNN~y@LBv~vSdALMBtZ4s^}hEWAqr9DlP!bDGD(^4mS=Eoc4$KSjPCkRbzf8D-}W$cxW`j-bvj z9|b(O8hDvWSabl`_J<&Ry;f;ch5&r;6}JeVBbMl>_Vb>G{7?3vaU)L?>OrloAXzYf zlr=>soy+*H#M!1w`U6u{dD7W+Inx{J)ECXR=Vr@g@?dP`xl7nmKwcq7RoATb4NDp) z4A&DN!{XHfclA;p!s;++o{5^grQkHuxFz~r0gKk8I@>-p50mA^VhpaU%-}D1h8KCc zSh@`XZu7O_ydwsD|3iFnn)u=YrqtXT)^Xevb%%p7!s{Ursdygrl?s{y1L96F(&Rg6 zp&a@ae9k0wU`KtT=HMY!rP)l6n|^9mE4u zR<%IJXL~5v760~@c0X5N59?du34k?k%HhKz2YAHRrD{|(gVjxbjZ`i zk1!r3LN($q_dBPj$r7Q2!}o!2D?P8sIM?%13DmNtKfUiYt1=}g%K(xmw`~WDVQtrj z{8 zdTG^`u;UL9T5)$Ec6bm~es;}!Gxr@Jd+OqR`3I%(&Z?(h`~?2^ zkj7TqZ0e=_^}YR}kpT{$L7%%+RmM~h2Hap(jbb6-59*q9^ZgFamX51AZl^l1`<8e% zZcn&Asx8G?{rlGB|2U&Z3S4Ay7t_Fe8Fl`$Sug`0QYjcX_fmOm z6W6=8h~$|0?<&(sBaBSKdyBWO1}H>7;JRF%>|!j`ANdR)73kC+7zjqv*qLwl0EUe% zMtfdljHy{jPmKbOqp^1c(wnQ$%yimnOEWFWTc$-NAP~;(v}N$Nmc(Hr(~~+yDYw~n zkzOx_N|=l5ii2Oy|KXHW`$7&)S>*SXS0E(Ro=}NO&BQ_cIRntB)^(|tBXCgQ{?=dU z?3;Rl#$JA^q|V})DqwH7M)5C+WKK4Nh@}rdOWfDjz(Ec~Inxe~Ar#?BJnA(2^y&P# zJMV*gUWg&UVKW(czSH87YL($9GACb#01Mu{Ir8{!qi`Ui3K45`}wK*D-XP629c>tLNhC+w{R*8 z%ttzR|8x@?G#Yu5HBS@C(Q0hX)w1=QypR83CXHiP^h$Q~Zq0PD%zq36zei!W8&_ld zk_e&~((3v4q9?g9zLil%E9A*I;eWvxduz|^@`ErHs%WvyH;EWVqNFK4|8?vIC2Je z$zOHN!j}FZ%;lc)X}CFRyq9&<1bh&cFE(Ql|0>v~B}v3NhO0@+ycI3J9|fy^j`z3S zP1U=pJyM9%FEvbaHq;b9TLGd_r(jq*9qRk~5Hp47uKa2nI``fqPryO|HtM94Vrmh; zj@f(q2>3@yNKS~0L>vAgAo27ZRBO6Yw`5t&z#B_fM=Ea%9AJVBSf#vZndI8;rdMP- z%L!`F1RnU-GwDfJHs;_yg}abqOJd<%ziiY=5^e+7NfI+v0(EpHVP(=D7ip4E-bgB* zkuIg;5gOd8SWac{3g^Ut5vU({!~(tC3<%ju;>Av`Q?hu1o)(J#*42~b#$MAW{|_nO?diO{l;$Vo5nZQ~pc zjz5OaGq)1S?A(}zOQS5SZZYclqzGoxg9-ZY^kxr$Ie{3S1OWaHhk-_Rd?$`Lh@VI* zr#&@}P79C^&H4;{k}XlU0l%&m7q~8afQbkkloTqNns}n3w}+6@O|5_14Xr_d zP#%RjN_#)<+ZErUjT3Je5?Yl?@=WBVwYv4NoUP$}%MueWEuMEWVZs%Mt{$X{mzXu! zc)qr$Wy|SoE5R`T-19w?6vu4;8l;&`U@4)-30uyp42pILoE&Vh$bjl~Q-9x27v8Qzy6g0Q`4eU?WWzux|DP@MiIMGCW z9e0=GR7WO~5KXBB{-A02+2S0p+1!uwd8v!4A?+7sF# z{gx#tRp!0p+6NdO9}PvAo+XRt62(WTDa_OpssIalZ{$G1fIC4r7ZTU^J%67@^wUe4 z5yC%_fLUqMj_NW@hbNy>*RUnCxLJe;458wLO%>l}Etk-!Yl!{mn>s^KBoEbZHGU|q z%n?JJ_rFiGhH*hnVt$jP(_}kmt9c_5jrB8c-FO_T?B!LZDVl;dNfU3P8cvRoChaEK zI7;+2@%mBsX&Rid@K_Js5Q(*;&89Tq$#UkQi==mqVKp%#`NG`Uyrh%jiSK;>&N7tKQ>fVkgiDWK6bB}oO+9J`FKoXO zMk!p;{iTMU;hmmlL+iK&&1kvU*cu9`3Rux4iE=OQSsNP>XcOgNzRw&FQlPN3#KDik z#orm+lUIOD0)DPHsEe7!_JxiwEJ@C@3yI zkuC6IyWHd$ml~J(akaVr>LRt?htI?gF66XvvDl&;(C_NwcsaT`tr739E#&P@&fV+M zH5I;CHpoXr!7qAUEcqhJTzwk=(F%-nQ_F6h;J1*{2{fxZv2aRJ1-UTMze#F&uk;@Z zV3Rt^OYVI4MrS~$)DuZu4^~0TC0u$YcCwNbe@h0)+4(#{Sp`RF3A?m!Ha-8}=Sjbs zN_Fq*uoBU}_J9dpgwc;R-h@V7Emd*Qsl^RGm2}s@|%5|Ejeb?P?W_BTQ>z zyHSV^WYB!)>FZTNqO#}kc8kiS%g^P?voLDDRLl!6zZ&@HAHS%$;$MGb0=vG`doV`J z*x0rq55;A%XIU!&m#-G=-Q}Rsc{51NWsxnOscN;ql}wqxP zpO-sui^5WNCstVzJoLpA7cGqMnitX@jdCho&{@{?Ew)Ye%5umPxzD!7BOY(gkDtXk^)Ix7|Dtg~ zir9RZ1@cl*Cx;37=@{70#oZ?I0R?HxBJH8X<9GBz)?ncZgibS^-()T_e%%rgkgTPl zu6USzm(ecxmB0pb@0%t6q6Bg82l>BmkF<=qI|cspb$eNt#Ipf7v)eS;xoT~s0f*SN zg8GX~z_s7ox4ZosO@N~JsnCM(fS@kB=!Mm{BMw@+W@#dkB6bVfgF;+kw*s{WW%w>q6ZFH}yk~B(1X^~m zPlN?ix?DzpN4eSBrkJI})1tfx8mG)*aKx0zo-)>B@kFjYaN<*;n}x*k$w#T-%zw8c z3I+*Z(%M;78k{1~*h?sQYeVLnFycmHw%guo|4|KN^tVYFEcAp$IU;n0LV(^7r;!oU zY}(cqChAwz4w<6!z8JY)bPlJBO!*GQ^&oxl0z&*#;24F!McOQAU$*6{}ve{V{QR%8u?S?*ax7WoCG3XDq;>D|l&i zR{Yx+U>q&xzNh-5#T$>xzw+S`(_bxIj`-l{a-8nm3PxGwen7IN=Oq+3YpDgx`U3k* z7Pqby>?XD3(qL@ch)v)v zWcGymd3gOitn1ST*b`ccmh#R6)?4X^TPIryR)kvq<=gx>3K2NnWApB<8H9Y`N-cZt zPV*DxA8?2n-kZL6V29>W|N8r;f*2un6cfLI$4?@{4(H)*f)9-r4SC=jKEQ|lcapco z6OHDbWe-gBFPdDjJ}AUVHzLh~y6!5SsdXHa+tNCIWidt8Ro8t|b02sbd+`zW`Lr&r z&Cq&Sv|(mmZX4SBt0}u-@ZXMTBM*K(PkP4oBP}Z8jwruD=&;tUyC`%sgn3>kg-U+O zu_C6OFFE>__+1vSJj<=_K@@8}-F6;=fO>-K&iS1dM%<-f+7}%s7`fuPu9qLHPY8?l zRs$^2w6|Yh(w3qHsAqZ*zYj{SY6Z`=PZ>$~PY}_PU=v60C{odzF zPMG_3rrhU$wB|Wyi{m14z=Gn(?bI?b1{>5@(2&*`8nObQ3F^+RR>>{H%cNm>(sFg4 zYk~lWC)&Rg`$p5Y{lVrB9>U$c5B>MsRhcfFB*7elQG0OrNo?(1(cv4fKf#?xMs`Qi z$|DP(Y47#vgA9Kx4pA_Wq4e;WSuo%8=*4n$spK63@BEeWsk`V{fqoK#{n!9nz(tPp z51&6pjMsqz|F5EKqY9}`hd_NO_+gE}Di|J^fgO})Dz(TB(s15UGS3y^z9Y5LC(ArS z@G1J|p(YnBZg|X(35<1GIZ0K(9)pC}AH}j(inKp^#p6v*4Zmlv{uc?dr@p{4-uS?} zieOsy5Cy5JIMh_V4*jPEwXVik1~h+W5iv_NPzArIcvnKS9zzTceF(vklZ2NHt*B1d zr)(qnF(+v&l<`FUa?3D;GrE9e8mqv>2~sf0K565BWSTyzW4UfeGnZLH0_8jL8$b{* z{|07e84_@34I!y6Ic04)!wx{iF^n7#boD+b9&f*W4+`VZlA)zg^7Pvaom}~~N5}B6 zcEjFJVBW1*MLbbO6yl_61-Qt8Y5qxw3wUs_LcZvtA|p=j>%3axjA|9w{Nb#fUu26P z{awB>p6hDT0ycbNz>lA%phhg^=MfD0kH!|-g(0%*<(|rlJajd>!<8r}!(*S})xQ|;!F6hzq zvssoy+wJ90|6j1ghpeEi&UOEc!rIO2Oof;Ca6~z!WZ3S5-OHt-)elWsKqaUMxmX+p z-?LLcThN5qc=61~Mb133|NUhz@RP%#Lvebl8Q|C>w^|9Jl+`~Ug4z=h7rp&l< zm&UE>Vffdc{fd3Hmc+>!0IeWrRpsF|n>XJ2lbII7>dAxuEK_ENl4s@vfXN0gY0Zn@ z!ij+~zs*7*m^4~c0uW_8z{17z5;p5XeQWi#23kSO!UN9~8sbFk?CVl-DPg7eHtnYN zfjhx&oq<(|p+Gy}ic?F^ypf(#^MumFMfxu)T-A$mSy3!Z2dGXI|F8&oLs}f-9~GTr zW7C~)AMxCw!Vh&JW4L-?uObZUTcZ8Pwqv565-gP>|6-R{f58rw2j%#9tFR(pGlIjb zNYe8Jl$`=ZkkGUlk;Uj1s~3~uj+5}RQ3pqGQLQ_<;ZY(hZmG{iv|Wv0Z@#!I>1XP( zFdAp}elAG*_e@1NT_LEB0omSc=J*>>{6?idso~q2ZgKg!rC@u4cw(=qhcgS$8h8;B z9Yk4(BbW+K2D#Rar&5i7mJpjak{=NBdM_x&@mCEk6ns|m;H;oG>jeJIzc*XVtaCCY z1fq@B_;CIvenb@_jEr@7G=9G&J0Mz;_ReoLO`%~AC~IABMN!4N$YeZsMUg0k!&P*o z%%9Qk3-R80P9ED_$NLv%Y619uVB-zOs9VF*ue_42R=|j&_zRiR9T!l6UMVd_j6vEx zE1FY>V7svzb75)GTu(?lT&t4U`vZ9IyT>f>TmIJ&;LV2o;cXr7PaR$Zbe|TuE7wZvw7Wpq{$PFY-|13T$j8WNyd<4~~ASKZh zv?JUn_0l?u-x4qkitd_gYLgcSLXsN)FxspG0uCrSF_CJ!$8Xbt(jUWGO8sULp@vnK zzw%TWfL5SXLvVMrhfUgtku*fHpXMB>@4vv0SmdwOV{M=00$uQTV)090kUVMw19;!b zaJ$=D$b8&L3d5J6-+)_@BWx%C3Yt2CsZTvO^eQ)81nf44T%BzQn0dlzo#ewOr52m# zUsxdi@S45>RC~H@kpxOFW7^Y{ti9w-V}`jZ6_9T*EbRO!%XmN(O%iZfWvosou0Af-lkW9rh+ z7@T!Fhr7sr7S^#aX}@8Qaio&|GoK?r3PUHl0Y%H_Nsh_Xt&HuL&N=s7UH~5nTruHK->Z?wRT(^ksJ$O5-ZIa}&^8;A7l5J-#g~*L9KHbodf>7C(}jyI zns%~iyt}0YMy@Stf@+v=2IE+4_rz1+1r&cE$Ia#C@DyQlw~_R79;Z7@l;+E^Bt2MY zOZ%i0@0K*nTC#ZtCOaBHkB-F#*HGZpp!4hEqz3xquSSK&ncufA?giVjq5;>p?~5=T zJ^DT1^wO+-AhutkSz=2!MdtbIjRvuUu9};ROi@HKgd(wh)is;&**#_(IqU~EXru&A z=A%efgac#Tq)LX4t11UA!P0RVy)zAso_C=no&vTLp~?QWWVtpjli>LhdgxV5zj;Ws z-z8Ybs!@#+mE@e(+0h)o{X=qnumY5p?L5Zdfn(>hy&0eem8eQG=n5ogdaS+6z4feJ zrbWl4qA^NS9VGpwetjCFB%p8w+l!dq-)#4O(jpBI>?_s!3@$8KS4fkzeLLXqIY8f4 z;qvl0F^0*8eFU379Rpwh%)Uf@$-SP-1PPyPVqw?M$S>)cS{zuo;GnW%V=EQ z$sR^fr8OvDI^WBFOxyy(ydn5 z#vXp^K+q$8Vk_CKJG^69saZs9yF?i@u5xHPm2K}4rFtWRuu9<9A?MAUOq_JabrfBJ zVKvWvCnRq(k|I)cN#8B&n_UbsWhw8(+rEZ^hYZig0M#Dzr3h2+*X;`>Ht#5+PfUxG zid$|Tv*g*#SXK}=pLAZ|U z^K^}eS!7q9{1rzOU%~Fou=y)1u{bCySyX2%d~r){&?Ymx8c-FDhKb2!f%YWCf@GqR za&x1O0-Vatj{^hbO4Ec4woy%q*(oa;qgb_7cB1e-%Ucri8d*}FQ!r@|IT^lR|BGef zlw?tnM6r^Ggf6T5v}M*P#(br+PumWLrM*zn-x2vubRqNd<6f!EdMGE%wWWV-#|fO> zYH>w9(8p8{lnv7Sao?fnHAxbA+9|ATwES{keXOLCAhbkh7slpmOU3$$N#=JSn@nvR zw$IU|4RCfWYz)NSU>;Ga+IoWS5TEAMy2nF|OV2XAD_%%OxfO8NT}iL)18*D73`=P8 zE@kW;^*hjF0&FUpay_j-ek8K29ZyT**047MC*coQ@m60w%r`8PfD`OQJgw7n6<5TL zq2h2yGg08-phnpG&AV4?QE^ z=4>qlM}Gm6@hgYT`23`}qF>Iyaf*fEEOz;?1bYl1Z)Q%3uscn(md9AHpTd=9ioHNW z{mVOdmRL>dRQmm{?R5M^xc7=VuJ%h%U|^8#;8~8ZUr*T2?KvqvO|TTS?)7^nbN7XX z=oarN9;UCA2qGC1NeOo(U2vGLO#dj(jJvKpsWcPC&=xlP%70XX*$d&GLc+Njj+-Ap z1_nih6|Rwr8?r=K$Wsd&!#5&>u!{ev$;<3yy!UFJTD{5SCKbbw@fK^V+~f(~mSxG? zB8(~jbB}JV+nKzDa~RO0;C&SwI1>e33V6zM6!RKB8m=+_D)twc>G@C5S!2wkk#|QX zPWJTtMQdw=i7w)7|ic^a*bxE zfg5W9Dnnjld%&LhO^3#tW0P*6Jaug5x$FR8 zy2>2U!*Ko`sOwBaKPDnUU9oT$|4~oPU*uC*Tg_)~f){A%$I#OmCN8|`|V*Zd}4|! zDXtm59?^SamAgyq-5``jGWQs`IPVn8&hUzX2L(I<`F`Bmf-{sDHvsy(n~e%Pe|DDw ziWZ8lM3Y*5JY+r-Sft)x8@h<6wDU1c0h6-_SfO-tNVTwhl}nBW0UgG09&RR?u{DJH zH};5yQU7>_fNDJ61u5q*61n+LBj)G(4tw?yYw31s(}$KHvf)B_%xX`IilP*KJ;rE< zlm%akn%qv}wgObO!F;QshoCKF+1}mtOUJ5TtW^hDTvtEXg0(J)Rc9XL=2vB@PSD1@ z{u&2e8sp@C@u)5Eo(#c-Wg+-W#e6gxbTgjfSR=g2J7k z#}VKQnjz>1HA#L+PzDyROq%jLQ|Bh{#zz2pX}oVTTz^$9(D=x%(YRc#rt_Vl%$mj^ z>AlD*Au?e{M6y58uzlHoxfS8(1q@;O!0eq#Qe&oukQw~4jV@}4dAK-uK>V~ ztIo?0O{=1;ZXgU8cLbKrx5Fk>h4?UsymrXK#rh_0yTd??iqEJe69bn~J<0Dzq%qdP znBMMz^)rfX2pku0a%dA{IHZ#Ug7F?s0)Pi4k3g*>zLJis+TF`vevp;Wy>nf)3E0{HyaW zFOhLy&c%=8%8P!Pe^LhBl#wiGN;TTJcRq0Ct0`_q;4yUkL((8>=^f&briaDsm zoutzd&`Y1<5{I&BMs0qx;@b73!%$9AD7>8RDh2<0Pn~jV>BRCp1BY1#Ax8^F4Nvs?QWPpqL!e7RD#t{+tIBwv3hSD^T!^Y1ba zT8h8~hvMv3jEnt2lSxB6`KGh0ax9N31XJ>YkDU2t)am&HG(I$_YzgruhG zd>b#?V9H{aT*PUI< zKMsgdBSK<_P9QU8IeT@7W}(gf4jC$tu?BFGs7*Sbj=o~H!hp|pI#~e5BaeLm@d@E& zf|knT57QxyYK%LoF2T<38t>Ar5hsUIT@H;fKubK3vU8E*MsPlE-t;4d>7y@8p;uFc-_fk zCVGo0Vk)VZ;MnR1Pe&$KPK1$tlHAs3 zy9r|MD4m-l77KLBg=`5Xa_OV-M`w%8le!ynX>JXxP{%PjzGFQM9`BV3srJE#EG5=( z{m@c-nwd|lgMe%SxMBC;c1M=I^%tCqxbrc}h~UpGotRvD`M;M^h;^v?LtA}pEfUp# z{$Fx!zFGLnofwJ@a9A5%5PzUgpJ6inogS|e5QI}Xl@??!UkV`uhtnDMb@0B?G5&le zqAVQev4;7ejz@qpKE8{k9=}@2Bhbu}K}M@PRM}7q*|YG_wTL5{%)6w8%V*(k_<5a- z!5khCd5Nw1Hvr#G5a)tIq~MfXKLMY?d^&;9K z=g6tTo%ZNhbG)=l%9*3e=UVl@$)UKF)tVbodQ^`Vu`KL(dqnfj8m)^8ITu|;!$j|+ zb4sE#l5eW8BHX-5U;?@Z{E=U^H3#(aPj@sj=aFL3;?ltmJ#F6jg9A$rDM;K7#! zeZk=6d7SkYcEgZvl~RZF*MDJaOaix11sRBYZ1`*mr9R43Xnrkd%ts^qgFtiCHfQ61 zX!`1)D7*LXWm#ZJm!+GfL1HOEy1ToiJ0t{YSyH;CK{^Dbm6VhcL6HWL7LZ3$1V#KE zzVEy<&iIcyD|_GPT-PTeh*v0N?gKbSU);^?i<;x|Bz{@|i*?U| znA7?r=|0AcYKM!XiDY5;@b`ol zdhBpVmi}=c5IeLO>TsT#g5CXmQo>#1*3k>(FZhGA=X4GN^rW}VlayGCN4BYn!44qT zQjLh*q&2uLx_jxi;jqKJP1NeP-7?vY@xMyxC}tp>bc=m_?Rcnnc1w@m2}w?dXiM9~ zKSh`Z0Ium$V2^nzhV7fSQQBjFX_oSE(DCJy*}quC8t%_<_U_AuXO0ij~TC@SM|MlD0tl{OUQdaa}ce;3ZtXQogp;mxZ zvOIp4naqSv%s>Zx%QfK@j`S)BK8&I{1%YkAWi?Ryld{Aui0X?>muTK(;k_`9;G)jL zKtck>VYNKZ*hNR4z4v?kK7E|>wud~DveXixN)50xvR7nS8vtE9|C~V@aBTeet>RT# zS+eRFK&olflae158{%fXV>qcoi#*g{E^T@#zcoyn(Hg$YhavcJ!nN6}S(66P3Aedd8mt2x%T zEmR77nEALOEOk%Yu`atqxl@HxX@%tt;ndpLDrn`m zwXMx#5GpeRg*P=VqWGeyOiBPnULJU`*j&*vmzlbksh5p9g zby@xr|D@bygf=z{wPG*h*zB3aYX8A2N7vxVDBBwxSt{7$cM*|88?$wWtl`{n{b8bk z=_vJ?kn-LAIyjptIqLVp;0I{s?mJPV*7^&2x>sw)e(%TH^F+(|fg8`k-3 zXcL+J4Mfv%7HfcnW&-`9Q{H?IQG5&1`4~IdNCDIdM&vqO4*B>`I%i=?Q`}0&W6w;= z`|A#_a#70mA4I&CMP~ekMRchAl>*h_<}aQ+JlkL;;AV;iRp8J$e}C9IRtmQ0f#J0e ze%I6SaR2akD?*Pg0X?BFhds)fnPI0HYn}c@vtww~B+`owYwUiP|9{KR$Y}`v7m=V< z7+L-URy06E_Xkvm8zQYgHo*6K%NN7 z#{VMG;}rn5wf_=rO3NZUMKY{5UcvA@4Q&7t&PfXBbaZ=eNZq!SEYACeA_dX3jQ#?# zMH}HHFQ2=o%O?`38Msv)K1F{yIJTI|6RZH{Y0z=^T}Sf2ZTekx^Fe}oyBrZpd)USB zbDpp-F7Hz<4Bl*Tr8C;fOQbe=k7u&u+@sjs`d`>cXN6Tfucl!>e36sSFKGKb%9r3P z?Q7aQ(&u{dC!%hNxNbQk`-8?>PWB##>dBUH2xk?^R~hs-)($XTG-22c=*keFz99J( zUH&Za{Ed-N*OxafFEkY+Y+%nlrxfhKYw3C1`prE{LgH@ z8Z6m{r8RCK-aU_ZpKUXRMwmmCQIV$}Dv7G~i8+%n?|$r6LJO<3fRKM-U}#SguAqJ+;f3A%)4$Q^)pASU~0v6=P5lb+!F zpx#GU8+YFn@Vlo_m5O#bvF+AK%@1lfDe82!x>y4y}qu2$g;v-caN-?2iA4a)H z&RfF#&BweZym>}%cGBXJ^!7ooYQGLM)miqxFnVPrraoMx0JFl^cx5GC#g2 zczsfEa3d_z{8%UA{r3dgE0?s=-bAwB3k z!*L>^EKkR)iX4JatKD2NJaNf7;>q?5#a8`%*$A0N$_&ncB3fwrpaM?&qi;HAe(5#- z%sNk^l7onD8xzFq&BKW9B99mkz4AJ)nMMoCQ;p55vj9d2-5&qg5+dgLuw9N1;j3nxM--CVU6F z_~YbJy1D1$!AH+$cZaJIm$HsE#RNXdOTi$C+TL>Ndmv8Mhf5a=F0crWbKsk zS%;Y0HYE6HwkrC4`_=zH^%ypp6M-bu@r9u70@5#tt4fJPU`4jickqk#O6U~>8b+T&3{hU~>vmJ6uv%hY> zp_ZhZ7xAq7XO=Z(|9R-G@QfqVQemoaJVc;2?@rT>yj}_^q8ocf&l7QtqPV~*DZBRV zH$-yy*;@uR$M9wywgwS0OIqxPgGo@lSGd;Vw_@vh_f=Q|_c3TY(%3V3E)#D5#N!4D z%*WVK;)@k6goB-9Y_xq!Tcdj!HY}P|4a$PwGO`+;zcNqKqUq2*JjLoykMbHrBb%X! zcnt`v6A1UEMmK*L8vykNCzYTyfw}_wE~HP~(%;a7;_c@j+xD z-XoNHBMjAYXzM-k0(#WfQ`0otzm%LCKVrVJm#hqHYCctv_u=}FH6kHf@*7qnUh2NB z^k}z3!vXK1)T1Ty4QyMUgo7$#!|~wiBM@eAy+ZY^ODr`snNEGaoH}djErCJcYf`gR z+f@IlkNr_eb<|0?e<-ZenknC=r29LM>AXuS`tqSUpp|X zhvj^}!n@SLm z9v8?bhL=K{R7~V6*Yln z1|0cytR&!21{!1>{0kM5-9fSDo2PxfXGSSL&D5vFOB%SKk=$6D33rR(@F`X7kPjI6B%>AKrEN+aSvE}mJB#u^G{8|KSwkm_xj(AVU7k> z2p$%r$S|edYM?gTg?qtvp-ff^nW?(ZdTkAdU4h<_nM2O4+}0O`n!g3ZKH9KB7iZ7|nc14I*l4><=H{$Q zYCoC$&as^Nw9fSR`!TP;)w*Bw3k%m?{QKWp_U&S{(s77x8{3UbG5Q`@= z*FN#dv7XOdug?X`$aq88kXbb;cyTCdF6(rQ-$QkHC}vn#3gm*+U^c7m4p_=f_)wb3ffjOW`oNXh25O+Jm} zY5n0FL`;Hjale$1i5kt&uwn;6{?ltsoL_khLrSh_)l-37vh}CEIfP|gBNo}rv}k8< z37S|r0Mz7D8MIVPao&t6B{Rt~z7^!%yR{zWFYYT@AT6|*xf;}C1e$x0{lVZ`a0tO5 zp{?ZVE=nksacm|SrDES_X(jHRK~)1${O&3?*xm^X)@M$W`-_xU-1=23iM)?mkw)xyIE!$@T;3n-*Z(wWj7^MK z9yimFGGzI2p74hO!Ikg9aVl-2JY;HXc=Jk`K;2|CQ@7orj@( z#ZQ-!5K;5FgomWw`ZK?fagVLV%Th%4Ch*mngVMyQN<}_yXY@05zTKW^?i?<6L}Oa` z)~PbjgSwbypdWh%{!sk;)A1ESFzlCbm9~>99{-6u6W+G#i(`Q_@l3H<%dxjV>A&OlL7hnH| z8%y@s(m8#92;2DviIR=NB>%l@I8t47cDr`aRL^>{3>VmN36q?#Y566nF1)c*k()0^ zPs?N@c$c#)?Y^nzx#yLQy^u`u0Y$RSTW56Rzh6$VsX&~y!ZbnuyZY*+>et(|tYxvb zW7>~Fml0WylGS-#Zql>nzeWW|49K++ep2i?vSqWUD_ReKn#Wq6QWRv2a?M_FF%S&H zvT#=b0a*zjvDPG^$Bs%k=1B(K#5VyH=E`{+k(r!W5n;dHeB$!9CNQ=ohbO<^vypJw z#C}hI54!U8V4sXtqMAt|riokbW)u-qbS!W$U^m0eYLa-87nfs_c#&ZI!Qi!NZ%Dvi z{gs%tBaIkt0KrHq1D%rvM?(T!calC|?o*?~E6X=iT%Qq!Sk-X}6^;<$_?6)%#HX1i zdVLefUF4%kJFf44v1>%FmaWy)xG0wr>%jUhp>~E4a#rx zIPkB;pog<>JeQ(kap;OZ#~)k7@BHz8|5Y}EI-F>QNs)|JB|+mmcg(Qm;G@VYRs-j} z>Ajd=h~7}8!OTl)XS_-2AbT<~%?(S1{*FeqHJuHM2S_B$k&j?psnFft9@D*X8(ly~ z@K~64YS&aEUIBAyZl<5me_HBIL`!n0B;K4rn1qzvVj73pg|2@u_cIp+tHE84U7E+) zYt58)4gRlITkBd#z;Qw&6fCTJ~X{nqLNSAEQ0g(+eWY!u}Bwt{0qBw=WPbsku zg8JXnSo~D17q4cC{wf>}D~)A06;vB`-*>RDdI~Y^%5EQ^fbEaK2ox~K^dJ4sw9M3zR594_MT>X3V~i|IR(zD7!0B*3bI28Vjc%3R)NG6 zDD0SpXlEyOP~O-9q+zv2s4;O8#3OXtu%RBJprDfEih9QM5deo{7%lDYs7LVrMCib{ ze#uLCi3i+iomZk*GHWOE7E77~8CHWtZU$Y7>nSVaIg76kUY7uB9vGK9?oSGN_p%v_2?O75ASe8>4iUz~Z{y*R=1hXWe0I`M2324mmNfl)~3A)q*4mX`4v&OmPc8OsQ#E=Z> z-xyss0gDS&P`9O2On@~U9(m7eSqk)zHcUpbh{8(WF#p9OGVET>splf>b(v0 z1lWx9va#{luH-vD*Ur)DXerc-{(|h7TNyGvm)`FsC-(7D`bVOqUm@f)k$u;6s*i^) z$7;TO_;PQBh`B#1FH*ml21$iTWiI(X<lJ@lexYay;-HD`Hgp3%*Wf z7&Ji+c*;Rx*!Fcb*1Ya;&1BoYwgl<6mB7>!3Pk@C=7l9tXn*=J}{rCQnPOxKH#mXr8 z;Zuq-4>dJ4fLcnG+@*W+v%d!5;w_+ z{4OmOcWlV`_2A7zmm{}xoCnzBX^v`@pHzab#fXqSXC9q0zl2?qIgW#DGBrv4Uz?(@>PSP z2cNTlpgRr}Qlm!Gb8&QieOEzY6gYuEQ)GQ$=~)3A_&;x+?8T*T)olL`5Ee(kU?4Tn=K25tOJI`Ec*J*)zse`ukA}sd|=IKN|WeUfgTY4@wDI?J(g>;$v01Y@mqg*{5+?_@MHRe~$MJ z;bhR?FurO~GuuUl^l7Hgh{C?q*IU&^(~kty{06@9b{nH=d1 z2?YmrQ_l=Cc>Qz-ya!kuL9q+v04sT5{0f(kjovC16KP3#zngQPf>CSrNK@f69KY?c zPYve6>@OB&a}V_4LD*RNiB@?}K|$@B|7UGS>uz+;Q5U zg#Arbha2Y|ES<8!ip3=Y%V z%^+4e23J$ldkb=$co~U!2s^I&7w{7g1+w#m)ki*TM*$yL?S|t?Z=G^5@eImK@dHP;6A*l;SnlKQ>H_O3R? zMYGxn$IIyRDX<>nuE~@?8j6PJj{^?6d=ZNW8@e)jyjPN4O;abR6_i2X#wn`p&I^-@ z?+4NfX)fa+rSg(}uQy-fJlLdoEI*y!R+RxNRu{PO@21RgEW+3M*}d5pB{aGl;61mL zG6*wh)g^MC_wJ3rbr4fWz0Vc+^|tn*#w9SVBZTH(J&+W``!HtKLg@s%NGF*vxdF{C z?5zyZq`R3-6y;rjwFX0@W-%dkaBUdrOd)gGvQf)z{aWEYFXZc)Tm?CBkW<4EWQ_<0W3)+7cUKl4O>hHcOksgaD z9By@mS)+fb?>OK-$#LL;k84hz74hno%Jnpc`7$1;P_Etl{qsk`Vj-po)JKtLfo@TM zuVF1>HV^&k^gCZVK;Q>!q1u6N*?paRR$g{z*oGACz;zGibC9N@+LRmngQTex-DWZy zCl2Dxc-xH0z{wEP--p1Q9@;$rIS>G=MSKOC8Gu)_OA3{+d1`^=g_K~_lS79z>3cJU z#HQ!KgDpP}U)H-H9~ti~v=Vv-`QbA55)kf(+9zz!{)6jPQ~+DU>%e*v=~PG@_VoZG zl@P;8sRwiRF3>eIX zMzpP*0fRnue67J}3nc@Tuj0orM4rQM(+;h!|RfpzCM?d;J5>bT`Q$vFi^x#A1<|G4d;^WSAHbm^+P`R$p?$VKyY`!ahI4} z$;?G&#jxIG4Z;Ay(Dbr?OcQ>!U)>%L6A_x40)-owGRem*tD1na=mn@*vdjinjA#zD zVE(C3-wt4ICUOOs-i>a*rKS}OhXL_k^7AzER$z2TNZ{pT#$J3@oePxSdqb3kTLMY` zIsc76y#xa$2x6UH5(Wvg(SZd%@Lc>nzfuWmP=KW?EL3$$W|rJ}-7U~5cc?NucVlZjy0*~y z$i=;tx{?oI!cT`ofO7vf_d=|ue#RAjhZ$4$5!=Ag0M^2*p29wk;HdLKiBAT z&OEa+Z#@B)r6C&e?MY#JITB+a%p(ai5_JYU4oD&y@2H@T2x9BWs68+P=ZqFgu7HPI z?eao3jj5qO4+HyPW%U{(5 z5me#X^>LI79Ms!JfSxb1aaGoI8FSH!mWa+Gy|Bp(Fgqkx7CV@FiQE1)5XDuNYB{xk zHJF}z*i+jJ);JQB+zdZE>DV7;-e->K>1YiJSasnSMnGP~eB_|8HUb#~hnUJjJ4PPR z+zSX!QfiM+AQ>5ax1*1UO(dCm>w&~HgQZEyxEsHFM1uy4C1LSd#u3Q92zJmBXC$pq$1JO8iy>#e>t%McU=;!KS$6K9CYb8vjH#X{cG&b61m zsRt!GPGjnZ#s{hf5a%9WaOWo=ExP>cl-1Gbxd!tryrPUaXR82E+lAPS{5y6YWp>Ed z`eE@*3q+S+&*vA%R;qATj+H8JRTqQd8yv)iKw4cHqURvXf?@v+q=anlL^|cbaS%-q z7sPR08(%17NdF&53u^nZe2Z!`=}(2x(J1+qxA)>^d3y;Mj3gUT^RfgrZ6G zICa|z#cs(4g&~L;B(gj3y_8w(@DBv*V)g1Wmo!9ZuCZ@xlBWq+DasnkTA-Ep`Wife zLYgytU~@@9&KXa~oKy>9QOOjPA~wsi^wF%fZm%;F&Ev z#_^XI;u=4aMtho${35JU_DEszdoB)I!z>qjKsrrCcaMmb)DGh)8&o9pl@(H8kFY!k z{bL5DBT`wH%;*vbqsm89eLBJBw^)Bn=F2hKwX-K!&>lk4qO_ON<`#JV4g)(jtUYi- z(kJM=!6hxl2aefo$wUjkl;`q-&vuIeYaOF{WxLB(4qmuaR8zor6RBQM-Ufz@Wl&E& zD6zk&cd*C~gZuAPMNN8yQ6Fd1Jxqovt6<8MK61S`a+(B zplSk%V-hF}&&8Bg2M7Kgs>_;E1y=v~+OSDQwo>VzvxW_PPt1+2h9)-U%vMWD=|$bM z-(8%-%wQ9R9~}8kcYD-c=rPM}-kl;As(R`1{_~Gy;VJY>EWGrz z!Jqn8qH!;$-&znvLRjUho`GGkuq%do0K37t)`%5(0hXFSpigfjx-~PHI03cfuK$VY zI04~*4@4RD$*BgrDNxjIZBRuHv2yB9f=&ngw6`;}a+q1?Cut2Ra(6Zs+maw(f%MiKHHSG4TqVHqpMkV0B}JfG-4 z+HH~!yM#nXeAYnh7jRmo*S}5$e-snb-LnEwan_UuE#QqV|~bWp-nV2Q7{pn z)GV$q<;sLmHBbUbNNB9W<{P2cDYrVhO79EaLf}Icl|7JygDZO#fG5Ziei2V4$csi$ z1Y)1iI^%+t!ph>*<>62;`?nrtg(3#%S9Au=w zm^w^9RW`y;4!bo5;Zx;QF+D{6};(QRWL1n)^#5x^EbW&yMN z%w1Ci{Zps>?u+MZU~dGWb0aHhUEB#HNT|F5#9mZEC#o{TUyzi!A#UAdGXWqp(uq$B zzTL;Pm4lUYyF$5yXiYGco^c0sa~mETokghx<;64YFElc7E5n>Z<=Mv9Nns%1TaCIx zH3t6~bx?N#-CwYOV>br4VkhO9nSOIf4!bIn?dq4DKAT7z`j=No|5QVkznB^KB68bALhhpWAAOp!X@8$!!6vIeP?X z_+}OwEy?^@-(Qq3>Oooc)&2z&HNzLvn8~+bP3}~DN^-j!gF7!_1zeR@K%D#~SBce` zGV)7z;OO0@S2ND}1+-6|frjC`?E=E=7|E^8w5OOxu;2e6`qr)wUhn1POD#tFciDTJ z@;`(-r8)QmxRS`Z!QDij_IuUmJ<-gz|wbKd)Bj8JsiwAVp zoM7Qsd3K@cwF$)c9*YCx?i)6ns$6a7pu=V0DRL@guFj4cQx8w)bV5NAW3*#Sq(c=% z7-u_KD>HU#fl%D8(8SONhzhv;(_v7_z#qd#O5&5il#xE{O z&*Z<0Y|bPtmCc0tMjH^KcGUW&0RT4Ur=mMD0cAD_%oLFd;R_EMcB^Za*V`hb;$R@X zozmcDAB5#4`g^r09rH}tz=`QE+g_9So8qp(R^}iSSOhAz4%5|U z@8om|w&~mI6Y2SDz+pI86BSC&h6xg#XILM=K(wQd)F~LB3m&R6c0U2&qSL>tvplP} zq)&TVK?I+k&7c@k^J5z+2w-IbRksAWII~7z{4P@?9N@k!eO_j$k0xT4d$kAr=j{N~ zAxi#lJN@!?jdo#N``EaA&)S;?W$Wjt#_`+04E=_`z$#35CBRu6oIqu%^gNq_`kv<^ z*a5S3{xSAlfOj^s%2G5=T1FeG3$ILuIF%(*@)ex!bbTy@AG<%TXANIkludu+rD?iP z=wN!sFVajNslaj}Yhv+28$-Izhw_ z7CaH&~eu|_3tNIUs8|@&A8Mr_8ayC&K*Xl(Q zFoQJ%j)n-kObxG73sVX7TWE^j;t`leQlO@itR>HXOE65D-mJVE@cTh<EA|;y4?C8H=f6nlA~H0v?MW z*@ij^VuQdwtC~u${-rI5`1;A3&0PTzgC@^@^^L$q_C7=tLu5lcgBOzm0}wW7tda8= zjZp>z#9fhCbdAw#pKq&J@AW3=js8$9892gz(gFEMMIUnbeaX4 z>hhI;SMU3ev1m`hX9PF9R+Al%dCRuwf)$yR1)c**;Nc?Hkuh-BT?C8Pj4A!XkC zLg~Oy{s*WKdpu+Cn8U`vedLv)C*T2g`2$&Uw8JyL#BULm4s7>sG9S`Q>HC zJ7{Gv;TF^TS#~8fk-2KbdiW3N`|d#h4en@8IYL@t7LSa;hni^`gLkngJ_Zfdh?lD> zpFcALM9@?YFL9I^^mVjp)bBLe8*rggg?)>zvIXhkpeDMt@u3L*U6`i|hMM4Dpxt51OM(LZ`|l;aOKv7!>R|m>$-m*GJE^3OqD&r9 zx`QLiB128KT}@fJWAGtPC}W%?1ca9xk_#-=7kTsn00Bg27eAW1KSjKeTLOTS_gYRW zQvgo+?}OOaZDK6v?~`hX4*kIYmSZhtEZfIUK~T!X+weavUi%feXWyn>mfFkT)?`mS zm;DEz)5POj`UH+8MrzjLh<_A0Ep@26OA%YMou86ClDd<~nIRMp(MR2Le&MV~N?a@h1&zVy z0BY3=B4w;T4}EWY=Mv(ri*Nl@4BMtyFA39 zBjxg%FMPJYPc-gX@@~b7R{3&LKY*uz>wBd__0Hnmx&S|*NL*Rz3ppc)4#PV}_V7pf zo{%Q;hZ00q#th`ngBjJ<)W4h`&mAey|2jq__#mH0RI;v+jVR>=s?b4$t6fz=0Rz`! z4%mClTb<3_4nN#WpB#KUy!*9qvy=B(gEBw%tBt`_%)3PvTb7?kAfkQ=KrU!zRvmZH z6J<1TTp{tHP>k~@u2>6GZ0@82vnb50AxL?9chp6WuMkY<)$?huPD!Dh#r_6ha<(Gt z0ExSWV&1!-)|cZiNVaXL)Lx6rasOO&NRt;+T>A#l{ibCKv7Q2^(d(!t z*H0`qs!&{Sm7WID4dq*7Em2!;tQPYMz{JbSFj@Tag~ws+9{Ru&UJwGsN@`Xf(}2!` z*vSN0S)9&snkPWqX4vtuQ?piZQj+tNJ9u`irlrte+Xb!|zTLUo5 zKppl0$v`h-u$Mo z=tJxT-h%nboH*zr0E|dJ;0KrbY6>92GSMu`i9uFs3ExOz2Mdk5>P%dsQY^YH_OL~= zbYAkd+;%RUkNyb=HBu}t6rkW8*vURnLp(l#D2E#WN3cm+-{Pp$rFjo--|mt_CnP5P zfjNdb33O{q#H4jmK-c6kK_ehh>3rN~d2Dx$2X=)lvf}QqoOVherGFeavUK~&1%jxod?Nn{ zxeh0@%IR>~t5=1E0}r8USY<&4N*7HhUZ}L29+KNJvz~xFi<$Kq1j03)t5W zD$lz>tvBGk81DI~KiFv>!K&qzi@~v;2eD?M#Fpc%utR0cVJjf)R#s|mUovvs>K6>n zuNX64fICErK7)rl813Wu(Stbh?vVoMNzm4Cp1$`M0NG&5)57tX*5DeTSHPSPQQS4r z8(jyxeE;v@Nx=|7fA~K7F=6N@5F{Ss^+k1JO9S=mHP~@wGMC)++quM3n!JCzEGW;O z+y(&^W5~bSx|r)lc^ofVKD-H3jDHf(S5EF^U>f{Fun{Yrbqyp8m1OeZ*kvc@fB{_z z4#zrT?suQ$$criz=sAuGoLD)DnxIo|*-%E?PXcwm;o|R4I0_t}z~fcGJuLF725|{s zJe$z}L@%)x&xVZN^$lZ&wEO|5AEKdyzvThD#lRa5z~QK6V6lFG%yu0r%G6wv1<%Z$ z9C*m`f z^SlJQMCW6g-B(?-eCu#I8wMZZsrQgrPv{-fe-JfZf_=yi?qa~wM9VfMp1FWkB0Fo6J#2&83 zzkD5bbt8|epk#1*r{ofYd4uRrAAP;p zx1k(q9*im)Jbb6~-tyPeN>b=JtIys?6qGo#Psj8UiRHNp_&OG?XOPfM%ieJ-ohc|X zgj1G(K#R;w367hriN73lT;K2G7mBbeX9sQ!QTJK3 zmt<70Sb|#RBB20#4+J8mxXZI?Sx)VO?GMQ3o=D)s}Dtr&~c;EaK)G@RQDf z|3qVhBv18(djo~8)9xZc$F>d1dA!nh%ebk3cun{x_P&P=iy!AZuxW&ZlS5JR{gnX# z-+bF=M51Mzfh9?TXHZa=T zxhW*o4U*c6bB7*-SlYSPDT#W}jmS8BR+*(k08uW*i~QNH24Ky9Ai6C(X79WDdHXRC zJg|5KErM|@j_9O?B|TWXSmN2RmH=Ts=@PkRHtC{hEkcbJE62hhSXk{Kzin8R!O3w9 zuX+Rxznw{&h9-=%2S1c*0=171F&0me0x!W3*R|4Nc(@%8vKl72 z^h-b=eVb}uGvkYn>>p){m>ks9Sm+A)4uH$Y{=kJ61 zFH~t7-#`UgW!M~unBBbDqy6+AzOGSunw`s@?TJz?1(b{tSbZ6Q3WR zi+2h%i4FGCi0j<1>CGGFvp5kTq>#BKEn-^uNu(q{(pY$VduzVzx!9~*i;-)ABM5R2 z+1-xsVml9v8pq{{?^aH!$ftp^bdUVN6X&#Qy-pB?N~ZjuY#trwJl zDrPHl#2>N!f`(+GqT#kBfIP7Pfb{1KJ22-S>xF?tjkQ23R<`YVa;oFEwfmFtuZYo5 z*!cke^wx;z&~K1@dJS5!4wu^rM{K0GpkW^OYiQpP`zxm}ONhmMmiAtG{StWB!F>Dp zGna!hcT;Th9{BUal}=UR5?*;qt_fGHa>=$V6TSwV@PAlT0sEv$T~)(V?0t1OKCb zR;`u4Id#}**36Ghz?|D>UAjNw?6 z;2m+Z5dj`*-3A6ugp&J@cGvHm6%Z`ip?(}YS7{2^vU_AuIm-*dn6Q#LDxH>)WpZmr z;6^y|o#JlUOAzk`@J_UB`-~nu2!iz@Bvg(dff;rv4>>3YPv$iU{~3D?`gqWFdito9 zoP9AL{x=0`Z37S}*E=SjJdPb8s7l`|^xREZ_%;Z{0!yu6WWC5M4Jdi}S*pDpf zvJp1`H9@{}z`-&gGoyIfrq%ZWY}@!L!}=LdOd}8F=gZ}E>az)ipybkZR-LCHEw8s> z8&I!YL-GN10)*U4F>(;Z?WMx}cE?K?(xnz#iGVjHq{lRZb656a(P0Uy(99VX4j!Fn zUhhTd(kq+5dLuOD_dtd-K}PnX4kQTya+G|Ro1Ok0jAxP)IIJ_krjBN1{(_C^ z2lM0=7}g_>2k@2-6hzoHe*mnc!0Y`vo%lC=HJf_^=gj}_=LnD7Mg)OB^^}O_6wGiO zf%!=S!wh!+@%k)N|l^7>{O(RWL#`G)7M0F=EX;s+G;k@C%I`H6Z|-c#zMS zszdM_mKktsz6I)wyL_MBh757B=Rt7)a~;lsw}2-}ItB?_m50;whADimbJbpeqcyn^|Z#S^afwFz~r z6dwD-Etm6h^GCyf$IfkFwhy;o@|-*I#s*8iWS|LqACRroRd4Kwlh+s2K7c`Er~r3B z^!9iBBkTJyUXc%;tSeS0Sn=tVVoc?7={MvcjH@7kDO6F702?kKQjkc49Gjh6}y9PpD3ne~jz~3JVx`Tn3gN=+&`>aUO zSN*leOA@duJJF}nl2ST;KP%Aq)CQt8eVz}Q#(o&v|1{T!7~_TZF!b+y?Th&5`L0x) zqgnady2UGB0Xy$yO8v|)SS7lvk3XY*%;xM_Tbr2=cquhuz?$%h<7EQ>r5N;>;hx(J zjPYZ{^VGiyiJo|O7?b}Ne{nINU;6sAtfe1tlUiN@-6w_-3e5aagrK$00FJKI2~4&b zPMNU z(&TZf7;u-J7d@J~0wJR_``XRQ`=wOL9>G7IEJ;v=p=@jU0L#~wBE7ak8|!~SQ%h(N zqSE=GtN)muTyB6Jk;`43@dq$bJ6PYQi^wHw69r6qf4-l9jfwZV%1G+r_pSNaoC=k7A zT<#7g#I;dEnK5soD>Y3#32ZXWb@9i7aA3*pUoPAML5*HWr4>!2DB*e##q+9jw*Y-< zYta1{OcS&ZWjbB}MWPY~pLvT?U7k$d%WNfH{c@PhG?;d0C+2=Q{$s({zXQr!S}8JS zRe)x}lN20Qc-~ak&AH{)NW3NMa$I?Zb@0NyvqG|^@E$i|2a$kia4O`$Qf5)xrBm+ojEssP~ga@;x8H6Iyd6cnZn1opi^yzRyBuDVy3xx$$^J z5xY~XQceGXsOv|}VP|dDZ)!i7EeFe|xE5;EsQdvT7B<$JZu4_{&Orl*qT9oEEAr8j zQ?C9@WR>P&hR_{Wz-lwyq%ZTk=BksG=>< z1a|uEB>wWkOb?mKcXAPl-1pl?&u*I{rWs-12&ctYJopMnxPOnHT?0jtGyn;187z!l zu6rE{^LZ-lXUf4qPh`c^_X-4@6Y90x7WqLsQ->-|P0G)hT49`grO7V3*9nBM6$^jEH_v0OIOL$t$Xo_Y z21KDW^3qENR7;^x^N#+(C{Gpu_-dbY8(JEws_zYyjK^NB!v>Jq3<8Dz`>>|x$j=Zr zeE>Qn>x*xlE~mDST@JJ~WwCdGJJV<6*rLeEJc$>&ljfWzbqdFMhL0Mc{C+2a-Dw$) zUdUdg2Kgg0x<$`D%{K$~^@M1zyKno2>_I?Ex)r4P-tdskw^$4y-4&R*VLopfta}Td zQl+}Ggd=(s(l3&&x}!Qq(>Pu?28!lrVo0R?j@A&OFO^v*6wx0Iy#C zr|%wV`Sp8?hD;%Xxiy(REk69B9Y#ZHsWev^6u#1(e@_+lEAJTZhJ7&J56woBp6>aC zE)HAaYV+O4IQj7XJOJqdV7!m)=eh@sJP+ahTL*yFez4_sk6h-U91lAZ4jZDkGeLjN z?oZRbH`J9j?t2pMUSyd1!KcTLlNL0b&Z?P&=oLa(ZDGEnMtYI>w~HGp9fIX5!Weoe zh#W9mA-nEyUwB0#n*Nr%mkuFJcy+I!Xc9USj+zNxX)^ z5EME)VV7TV!|;9&Z1ocqJ$`aHv}Xn#n`uUcWQSTj3_5Z(*a~vEa&atx6acr_x2pxNG8Ih403PmC6 znAv+PGDEh=4p|L}6dDpL6=f%h-~H+J{(OJ8TmN*syeahH^TcKoNMv6 zIko|?h+m7ct&ii`_ym2dBQ$?3hwEn|KH8?me7(lXf0GKg_`a=4{n$2#g`T>t28p&C zq#Lskho_}&JNX}skcl?@w2uGs;?A=l+H-C;j|8s9PG-WQtj7U*m%^v-LWlmPfk7>a zOit{4D%~Gt{N^1Z>2jEb@?-vtyI1&5hO?@u=9hMJ!U9*Ac2%ViyK(w%%vjIt z8jL+cVZgzkutE;VfRu zOC2ULo+xOJX-J@)RPe&B^^m;9+o4dA6-T% zT{ByT3A~U+L9qKoJozDvNj=*@gurLwzrZ7xXXI+mi+YZiK3Vznm|5oJ-`H!uW)PpQGgMSO4nJNE{U{%p4z|&76dqI|C=4+Auw@;t8 ze0O56%A!(D)WdV~Z2;?931$0TT=1Vz&$Kj$22E|Jnn-uLx2ZW#PU#kXOC8*m63%{c zv_aFtU?`7xLojHY2WNbKDv4NWR_1HSTtpmT!z92$3l1O5(r;^~AXDjQ*nY1I|CHVk z4DP?2Xvj0pKQXcs#*-8>k{miHvs$}kDAtAes7>5cVu$8mT82=-L zSSkd)3j35a`5ll&<~ZoBjqPAqW0Oz0W-wKw`TnWPh>4bHMZ)P_m9eDBz?RZaZ8gek zw(zlYocS(jT(A_bw7_#|s-7HmBtg~1999GKje+ql+2khZuyJ*Ujb9tsOsCk3TLFXQ zST%gIb$3e*zk&hg{;fmiv#&G4V{v!Uy`Uo+F;jr$cv~-Hxv2lzE_a73KeB(5)E8tO zv;7x(wfy5=gHH`uJGm7u2imwj5d{4N={EUZUS0oq^xLx z{C6>4tQ#7?cqa3W)JL*KqeSr>bPzV3pC_41fFFC|H)~q#Rz@D0l3lA9CU2=|Fx4;- zy%PWBjrkRRr|>Yiq~08D^LUeWHdU|0E6Ez-%t?275EqgRlIF*|qKk%IGlBDoK!|P= z;d*dUs#(a0y6H}P4CiC-f37k9SBj5k{lcbH4qr2OE9v3qMbJ}3`v4D1q?7fsIj+a^ zx9j|evRWoSBAZmwr!)b5vrkDD|6mcsuaA%R!7VZRrbc;j|I2T-NIg_^nd&`oDHrom zdVB5s6K@E`y_GTF>S*+B&hn&B;mo@24hzPU(U4$oaAsnQ&Ko{4J3otkfxEGeg@HGz z0!%x7J}4Y@t8I1h;cq^CFz<(i*bXx;^dHy+V7U1!p>pEqv&n}K9z}kHiL$QH9XTLz z2_Lz2h|3Z|qHzS)#eB(kG*XF~fc z7L*=of{jPgm#G=f4}zS6rn;fh-DN!tr9l ztF9|6e6h1(Tg{v18z)MJ)ivsDM_qN!qJbi&exC5**c~4K8Yog)@bF;8rGl%{O>E

R?RP1l?cU2H;W8T*Pkx7CHXSc9zLexKjRi-c#c@4xbidA`{c&`KFqNPYY^$ z{K91i`b1%v@nbOxC>D7!&^8RKrtGNo&|!u-W}|HCt*C6`mj=wcXu=-h>c#4H?EJbr z3iD3-AQByLOrN=yrtkOzV=`9-EgG2N+e5!vWU~CB4FfIusr{#tPY&uh z6;0sX6dK*`-;zLeYDqYBbEm8B5aPrnLg_#J`UH(w32hqkWVCVepalZZQdd$S2g4&> z5l?qmVrenv`+IH7F3?uyhmwD%_As5nY}18-bzn6d{=GCC__*OVP$akw@*IuIS#k6V zDHGHOpfLOvFHqaMA?^R@7OYmQwte^$x~hPi9*K)slm?xeLn3R=BV=E%Ml2JxL_zpCTql5XJAk4Xa(}Mp?(h25JFoSs)>EkJ1Q*MUuYa@aD$2?1I9w z=a}Ix)t^^kkBkveKdHf79DZmtft<k{|Kt^SpR3(_e=* zn{1mvE&RtcxVtO4w6X*09#9OJ1P}bR{2E1uNpaYRj%PLIm}NCPt`)f5+W;ciLka`d zIh%-?2KGQM6-k=JrZf+G$r6Hg6MF#A6VA;zb?ws4`@G*?spFw)1_c)67|Y-fi=Gd0 z+Yn0M^O(*5p$!96W3t-jl@*~wXbJXvSGgBn_xm3SYLK>55s=%IjOS-_&0PMH#e7OQu&oCyLx}Bn{QV7^IA6m9vq&fSw@Fv{O3381{+$Lns%MC)eebqe1 zQsxc{nvd|dQh${jpb&8_1v1C|2=E~c?QlN7DEtKGOtpKnd-D@g;1OamO_hs}_W%6? zRJI9jFda0|^y-8P0>d|Uos&#YsPjuHvZ3Ig-=-k1`LLHGq$`4sBt62) z!~7FGZ8clt9`!RcJutZ-e_sh!pFDiOsGeYq^|*H|9!Y2iKN|%?gAAWNx0f;IAQC0 zN5FDB#TXxT**31eSZv~Vx6&vpyo55P4MV|5PWrQ z0TK5xW@U{J-ava!R5;zd2O7bH;~|p>M?SZJJQ4X7c*~b~rQBa|%fL3H%XB`4iA~pk zUoxu|PSlhX+J$;^X5l~g6K>LPY0i+m_%>6I#ARh&ee1Pqb-y8m> z!??xqvS*M)7c`&nr~h}^{c5u?*a(3bAo2?_%uEYqb2P`XHCU7FhW>;CGvRmU3&2yt zfO-H6uB5oqQ$^V2m3;y+%MfKmAZHH}vdf!QbDZkdCbT3gG3b`2^YU zL@i)_Ac4j4hw!o)k*vKIDiH)qgo@JZWA{JlA#QW1>{)ik1P(rNUaSSW-61-#P{-~=v?l^~nX0aaZI=GG7|OFx7GPF+@hApYj*C8jG1`WHrzKbaej~ zS$Q!GBM>#mj)STUYtd2HU!d<9+Bdh@rwj8y-vBl$q?v~oUPYfOEF{A1p9F3@)Y~A2 zd}hINC`;wP-|sn%8U9eprHVZL*<&bn8DvDX#9p*?ByJ8)-)Ye3LANYzNrEsR%|s<~ z2aSTCp$-{^V`n_aStu6@?5a>gWxqdb7?c_}_ICehFd+jU`2GoL9?+4K%VxIIR!}h` zXC;z`+$O$H?%nu73d3U1dw)OfgC`?J_O^}ton|nUjfq1QWo7tshXEdPN+sHv0x_S_ z7_F4cbS9&2a{lgP2a>-U+lK%yU50cVURnA83?Pt>(H(0dcy_!sZHK6Ey zNR_YCTEPHZT;&Gkd>9=2d%6u!eqg@Sq1Dzu@JZ3|4r zEKj&-qs_i~twKty=MV#wwq?G*%ZU&KnUeV-&B{YiUUGyXJg!Wb;x-+7eGq0CkgPo3 zW2P3BoerlV4^1vrH3M6(S>}4Lr~6;iJ~H4n3Jq;RU(^B9f)^`_-un4Le|#Etr|NG_ zLkY6{Lq65S&`v93gP6|7X;Gc`Hk~IRWBqrCA(C`Pla$Eq^KmxuX$3gXc#t@S8FJa4lyE!! z7QkU-Z{BbUukP>7QA&U$nTvEQ&)vT-lhlKZKB4V%wWFtjQjkQ(QUi=kL5_PMfN^?h zrC6RoMmhg!!|>0V!MT{{b)9Rp0`Kl-wgF8C3Z3HsV8Z`@`(+NW$8Y=g(d05c_nP1P zdU`)sF+kM59gt|FJh>-cKonzqI0VcsJ06S22X(-0BC&u>$i(oTF&NsY0{-~oaofmz zZqAC|{9p}C?_dV`EL|vhY!X1EXSKCpk-xNFWm!X8gZ~QH`iS&G-j~3XOS9~^$~I$# z3Oqh`cLnVMS+B~zOOg9FV2PhWJD#wP2%d@`xIVitos<55UI1(}sgSfRn;IBolE?~X zzysO+|86~~1Dpu>mQT_@)f@81I(Fm&N5?0(wu`Ou;%*|lbhykqjMq+q*M%{ZpA7ZJnp^Xg&N26;ORnqj)9@jA zLWi6@2xUe#@=gE2S0fx^2F^Iw1FA+Q;TMnKHcl#@{|XclE0eIfRBaw7b2T$%W4}wG zCqj^ZCg43xs&pYKiKgoHoBDk0{vdxE+#hD5bPTuvZ{*4VA|5C2oByciR%ovHh2j2} zbks{r(hR|>On|?jl1WP%VTI>ASoml9C9lynKOcp9_PvqPKcF{1^8aS0<_3Vf+hLZl zCF;H}Fh^pU=cFn@w*1O8y1l%snv>~7<;V#AXd>_vp<9?D3!ClF_vz-i#hu~q(!@k^ zbKCl~`E!frFf9-S8%8`bf90O zPU+g@(I>mnlER@4xyk;D(o6FFSXeSOyhME+D>BC(ogk}~pRBSIlZ@z8qj~0)Zb;=d zJo+>bg4r{HF#DVHji&}9y*H?*1B zFkIiltbFd=3sOCGNoVeGMcdCjsnh>9m@@{oEGoSsJnudy^hEcLTtC+Il(Xbs+r&QX zk3my^vXBjq*nc-AvYPsjH*6*nlk2I5ec_8RN!0xd84Z{x8~fhj76|XnPy8Ad*=HeySct%i0 zY7hN*y-v|z(@49_`sv6EZV=OUG~Ne8L-snzN7H(SP~OZajHDV$V$7 z5r`aiF_jpmLm_tHCr!a&f9xvP!X0IBz_|T2=OzfHB9iGwKXJI<4qH!o_Yn3S4W1c% z1m&Y!$z$byAVo?+aqqNmC;MCDJpLqHPl{MB$ZdR$b z#yMH}&5|LtS`RI%>X;*?z~tK=J34Y3mVs!mlE1Ea`vqib^GHWbK{lSapnjq$ji3Dv z=6DQ-S?;BkE-#q~>+R%D{n>!|WX|TL40&Cl!GEOb;U%n)tn}=lHuvSgQvFQP0S>m@ z3T121q&~mDa{7wXg7*5<7)76`^H#EwH{SV1Svr!D!yNG`ubGKS3IH4icq zDO4K6`VaLuxO~5qA%t%JDX@~-c<#?eF~Wmcx>>n@GX2OBanw^q^}$+uxQqQkgrCK4VRkmmMt+8E zn2nIlh1JIXVVYZ8>}F>;e|CNO#bK2GumNw+v(wz+xtt$%T6Y(wJJ0&mWzk@Jgbb#! z6?Z`Re_96TThH-^a6Y^>>#k1|^;NsbS6oNyy0*?Bwv5v;Xa(uI^oojI!icciT+tMCbk*F9)Gk)VW#43g@kCS`FFojX~ zdFoglPQoQgRfcmZ_JPyi^Gwe9AJa}7Z(!dLWcN7ue)L7r(L9!Yw@3sQMGx>w|8)af zH*JTmDPq&>^+6*w>xUlx`{mOQ9B=GZUhkCPa+}YqrU5P0eb<-i>zGs6dNuEy^szTT zad~Jewgy`IP?jgI-nT)m;Th85@`Nek5Q%*Slpp32HL%fS(8~0bT+_)(21Kv@M*OBc z=94rjFh9qCQfrcUP+$GRI|ieKZ7=9)e)fA4@?Nc6sK@S4i4u7v{u9mJb^+BAD4m7m zA1`T8Um)C_DlR2g0>9Y0kNgH{GH!MTk+`taOBPedE|c1BZ5$1;Nntj%j5whs%0cZb zZPt!D>s6JH^JF=GB19`Ey%7#5qrLnfb=AAWS>(wz!yavQ;t!o8N5V4X~C#lM0? zllx*&b%jKdzR{4pU+g~qX>7BiV@#?v%@87Gd)cce?9L@zM;Y)&_#xHey3VdFjtGsX zT-`!*F7~{A{dCC2k4eT!f{T_$L3$cUish_Zr^p?3ik1J8F}-Y!Z7MI~=xR83SCpCy z^LQ5>Taxq8;C4XDF;wMicEPrY{FKlwzGNk}5ywksPs~M%NGnF)^j2q2XchmHEvzt? zDL;EWwp9OpX>q>p)g-ldJd)S|F}3^MEQ~y;>W8U3W3wWQl!=%hp#D2qHc}kv^f8Je z+1Hz!y)|1}=+n6KQO#No4RXLC2+KiP4PP9^`}$(;_#sYB@hkg)GD}pQ6jE8EANbWr zl~&{f;t_9Kop^~QE*jJ9)M1+EukS$%+QLUBBp2_l6y2P0i;g6Ocl{9R%lm4+?lfvN zFrRXIW1yuhz4|nT3;IA!R3XPw=t(Z?U`*w1mvrBJlhPpJE~ z1A{o-oPj#p*yD{=7~Vqi@+R3<>F`mO7Kc1Zf&mF+P}Ab&%Rbx zQDu@8L2oytUnQaR61y3grw8lget{<`NKJ9J0|s4nz4TG>$B`+&3yTn;KRT* zrf$+}045B>^npJ-VtwAN&|*i!Zsp6#8LX84USll}qRq(f$`mn=R-#$w+m)JlDR;Wu zHzluIgnnU7V9{0L+nhM+Q2u_aD#E&MGQVN5n|ri|+qdqpnB{Xl{pKeuDBq(dsnT%g z94Eb8$>Wbl+t}M=aD-dTR?qFVm+eWLRmfe%{{bY(o&w zY2~pJ`5GIeqKd~aokLz1sZ-DCib!dyB|91Qy}p$g>yyl_aNhofDvNV8t%*iB^DpaM zkJXg1Al?)Vf(t%gO0Uo?f27~mNEUAXmhnC|>pJ_%Aa;$a^VNNW9P}m8;-?$K{_6~{ z@)GsDK|Wq2Me3TWRT1A(KV$CwIq%Lytye%8W2GKDDLiE z^U|rQ^TNN##O##fGKU9SxVZ|*z(1;sqkwE(fYO-EOtX1xXaXBEsMu`ppNYk=qO8kp zjym02tA69=8n!Pcz58I3^Knkx-S>oN^nA*{^s!7qmWKPZT4xV!aJcAhTTzOoF%LZT zYRSy<55^|J1hqcau}9CI^+Se8?Yyg9lsNTdPR6__0rs=H`b#BY4NMdRH&#BXv5dRzglyC82Y2;;qY-N@pw2|S@KAnZJ>>@(a%;CkKKW9;9%N2 zD}&nJTFIErk=~EG{16Zl3yzXmvr?&6-EK(!tt^j!{_8nx)Cm>tupXrL9)_t`#w z)IY9vGBv>Bt%)YJ`53a_hJflGt-k0l=!o%1qBv1xHspuKSlpN7~pu;?5!K&|BY|XX;E25Xr9a zn3IXqtuxJg-7J`M?SRE}*1RHy=fJ=9rYLWvY&03j_fB2=@)6JujZ{8)A?DjdfY+$p z{`cZgQNe%62moN@>^`qvY!zG{+;uxLPFQ(%j^&c->%(2c#&TKy z4R+dw?JBfk-q;nD3#KP%HA0TmHIjLuFL-=Or=sOwm!I=IQXWsHY73NJgNNA$4vdV1 zvJoHHiUe*!*I?^(m^7;^=Oi{UIe`B6m;PAh*m!O!N78b6G2l&AFk0tDSyUZNqaumc zICh20{3o6UzBMmQHN4U31BR9^Fb6A!31R|PCz6;_tr@#9aqR`)OIUsBdHE)wU@sUy z4`UEE}X2_TYqN z<46-eP#LXKg#udtZTcrvCNS6@-3Fe;>*|T&84qd5!&|_kfPNs^ z@L${HA%81u7cfxsgU|5ye7yWje2C|I_4UVIrFZdUO~`ssQ9s?Wh zX3O(0h3ox!@XF|l-<1C?#0gKbW2w>q`q*ZXHfcq%L#p*hZ3oOF+z+e#r<1QPe7;K%pF)sIFJQRsR$(@hao ziqK3IREG1=B@`>ef(cBrZu&jKj#CDS9~F72;xe4Facku)BfwH1oe)QVmm&|+Yk~WF zOt}s;yg)rsnt5$B5JX@Q{m4Xhw0&iF>m$(6j%iHAV#b8fKCpqXUcvg8s`&qft=4F= zF?CRNMJ7moi3vQl9seiOh?ZOu8DC|)Yc~ebg~3p2$QDIu%K4m`WxU*A3P^68l@0(R zhv!{T3-SpugfqPa4A|1A9H;ldUZ9g>HN($^AJi(2I~U$QsLZ^N47uP8!=MyA;vFAd zdQ&|Om?a|G6`QO|{BNd$A=kzst3hF~cAmi$TJH~8n;PyRM6Mi2$2 z3&zf-z-Rw9m{t`;FQz8OrCOi@U2?#W$M64agL6uq26lcc{8kBt z2?oK=pYc-Sh7a0N0Gw+rKr@GW?y~QbxT*W!v`8j_jJ(EeGx&Q(7Z_KxUHN+Ad&Zec zvw;muH*38^l4vDl?9Y>fj)jkxcnn$@Kzo*fe-{i&i$nQi(8}Ea zZT?`p(bqKfyRb+*qgekZXEPhB0SoiVFwGl;_3yxB?p1md_J6nj4!8X!-$&=-iKWjKx0yc;`HU z`1)=D0}MO_V*i+g$Hv#3?SiRM%htUYeepd|Y&4#Y2W?mnWA3AsIl{>QUiB4>iDslm z6MPsQz*7Q1v*$zxXkeblmZV0J3bETATFr$E%HM_2D!_e$iP(7dd*QKV$<0@(C%kvc zBQmvX553C}D3d%i;*w~foq>=<;-(CLoC-lri zO7vmHqD#t*Baq|UrLv-RNW|If4&e_GRg%F7o;{ex({(>SoOtx$&R`{hdBT-X)cOKskhRhSF!5+&CM@|VXym1w( ze+cF>_7q}Yrz2sDnIot+QV`w0vFm&_srv-(Y?bpz zCpJb6(Rl*DaQ@2$;Jib322e~fFunAk|3W2@weZeG6E3Q=i1zNNr^-B?a)7y~a_|N| z*}!y{yp@_I4oA{WFyjSY9h)GuMh$F5l}y1UKW~82lt<$(RHNTgTvFx#^iVK9hs|&h ziQof4W=qro89Lnuc^7^gc=wM%5V-~fk3NC@33Fq_8mK$PK`od(4Df5*yQ$8gD|6hi z6zdw;KjUBLNIO6HyVkb^1gcp%6uQ{#7Wy5QWg_}w3~>S|9qCp~%f`2y+qC5`orVxD@z`A3JiwsfE(;P3!Jev1q_ec~F!`6WFUYR* zjs*G?xwZWSXXm<9lj|-aoVVGK!a~!0BOn;R36VC2jQ0dj##60Su+n%4gX<=%lJ6XC zJYe(DYM^Kzk@O=V-P-aggy}dkK#ODSFvVXl@=HUAQXgQE~0TXMtEb+7bId;)|twb$n%jv zGQDxE?hN`!&$skV2F!Dg2OYEFbT2JH(|E;Ef8mv2apu2@aMs*J6Z-SDk5@V{9nCRZ#SkWe!5P_w zxlh}W0o;kfKo+022bU>aI}bfm<{@j;a&jfak}7~EB%>lyNrSI#o$#d_Ju^{pwmT#! za3iz4b%vOYmB{0TCJI*9d_twC<#+03h_YHg7vY=dpt^rgeZ3LHN(S>?!JC&W z_PslR^4x-r>I|z}e#={UYI5RhtQOf@IMJ@I8VTB{Wpel5t4sn71w4}r<; zyZd!l9sZLqE1@wAagB$ZBE^9Z@NFx}7ozi#fyT$l9pH|F()=s4A_V5FzYnxsoBaU7;wa8uUt)sHAv)`)wJreBo2 ztQ48gq9xiE(!g#_q38x!?9Mprwj^@qRh;bm=;#>_DHk$d4hAbH=R8Qy7wiG&8QOX| zZ}lbb1}wZZBmJ(}Oa^bD*^<5E9kI9t_5`#+i9aaNfz@u7daZL$v&5f0%U6H5@!a#U zLc3`T4fLR@H1~$gxw%e0R$@w23T$gOMxBVjH|s+EiIskAP(Hz zJ683o*~=8=3 zlzPR{Qk0Wh`edte3(=@gD^gTnE6d{0z)u_>lZokwWzsoLLDB z%di(=JEzkqj1t}qPIr?J&8-2y%MkvftQLfa*G1K@>k^)% z1)+5we>xKcq@YfHPgub zLzJwh(5^Zs*+PKM`HHxw`CkO^!uiJibhVy)7%Wkn*Uh@3A$`#H9aLJJqZSM1H3!t& zJ$Pq?n~Bh=zN8J2_t5(^P7jz-NCxPMww7>fGQ!3Yl8Z1#}Eo zAAkSHtebl=yyp>i#B_zzD0%WXb);v9pP#lPE>Am>_AMPp zVEi@DM{`7tv@^skWW>J{aacSi5?M@2-VZ`^7OzzCa>8vfN?I0Qoxp$}IJ7XcRxROr zNA+5(kEj}z{G`b1j5F+DdZ?;K;&xga*2_A^VJ;FLn^3s$KL>{%w!=-i01Uu&DJAtt z^JE*((Lno9e|7oniH{r{XCvIu!Q z)16KeH_O`ir`-MG;h*PUU%da({p-&4U&luO%!84r+m~a>SAM=%7!B&OcLx)sgg7Y4 zK`zk~@S^bS6+xrhF^n3Xli#bzaE`D4RKI33egX~j`>XFRqF={Zc>wQ-LB%>bdv!%v znI9d~SpXi@fr+atMm%+xvUn0EhRB9l{HjEkaz6Y$EQW@=01HCM1{{?!b!aLZdIKj#&@A%1tbQbW#Jj(jAfSY-|f|To1+h* zwSL~Q*4}Cii9F|Ck%+K-!^3}RcizXS%YoDT{n~cXQv=ibye$p?O12{ONyC=gLdD13 zi)etdo%iUb^FP58gu@eHaEY`(#z(<9xqQfG5yhH%9M#!54GwQW4V^zb2fP&A=-Bhk z5P>-4;ca7d#AzK0t2Dnc^<3ktTmR!9Ke4%oe+haCov8hwx3b3OP4$~dRk z`z)M#t}nof#4ZaFezQnS3bYBVK%smjV8XPmCPI%q?M;ql;Cb*#bchuLt-GyU%oKTq#p*;) zQ~#Z_;)YqIiDDPssGmv+*Fg7aIQt24PWvdo%h6ylL~sOVA0l{EQYT$D`MyeG8>ccF_a8W3OJss4Fy)ly5=+?)?Z% zD0J<)s(MTe3FDGZnqzZKZ+S{fMNTPBt99D7JPW`XJ#5$UCo-lnP`fZS0^Tl8S5hj% zYU%+7LXWoR=tA<+s!2xAWWt~ny$Ow$jl_iLPhjpgZ8bC7Aza`Xt03c2g5W5`|{YUY)qEu(WA-xUAepKRr!=mw8WlwVcg#( zuoXSm#GgVl9b@Yq6(Hc~42usT`1*SujxE>`Xw*5YF$AB4^ zJ&Qi_ls61oycuR2sXu~P#ecwWBs$<3et`Nc{gH|LP9&L%3{-0bEgnUjzBB{76xp)m zzjkSXnT~69UMd3aGTdx_U#O&gR0Dh8GlNf4^Q)%q$ctTwg%|soCfkVN zG`V+<^uXwQ&;CzB?7M6rEp9NFfdVG`9g zz+vIZlKar;h_7YcyG*A(@d9xIQ1eN)orUs#V7;?3;W2|td@MQJJNM5F!)7Iyc@y8OL1bU{j&7Q)a zxr4=^;@QOGtVY!t6!>R&4?;2t6R1!0Ons=7X*)}U{gN)eGGan8j(40W?sQ8rtsJ(9 z#x%Fq>WDw}nq2s{3%YGbt^UFBK)V?CIZ#4`GVNSjof`m@EFhLi!1@dK6TI>A_!9+Q z=;_MAH75W{cIW}L_;-L`$lRIOv$hCDC6P(yE%qjo?SaVMc$s+~+m&{l4Gd~b?-Ijq zKhTVdBkrI9<963-Z8n+)1AC^?gx`pd$RqjBcoYHtqzJ;1x`3Ma41aOr>)A+%dLA+* z`X;CiR+`+{UDd8xm<#%K_mNbYHa|*|d;sTa?LohO$ZpicT}e`+j^pQa6ZP14D5in| zo2sz2mz@05Mj-8|yATL@0!j6M0YqkwiW8A*7*9N*Va?|6Gkiu^~_Y@$QoJs9Yl&!RQ(M06<|Wkr+488~9zSp0Wt-uvhD-@>G%Q4Nlj zU9y{Es{B&`64eJBd$SF)SyOjc8RtHqP{j4U4ElQKH#kS)GzNVxF&x`XP>VEt3Tv;< z0R=wB-%vhp+HqgYww~)P3{R9kH$XF%p!{WYb@y4Hs(KAZnf#}ztlRT?7Q)XmEiN#x zVryww$_Dq%4eSIF`7SZc@VuP98#ZgAHF_ve^1=1T^1QSVI)?o`N86Y5%LK;v@+~^z zdK`|=qd+*>JLLYR+6LR(y ztrqxo1pLUd9ZCW1_#R-e02FQL^5413czA+v`Xu>e6QF1V^1^@G3UGB?Su9|w)&I>K zCUG!f@h|g(KE|1AF<2*-fW@O0Ri|bMRBdnmKm_(Pv?@koNDPrk!{6cIMa;kFc~F7a z)7%|S;8A|NS@m8|(;_yrCALomWW>x>BPYR00N*TJt$hD#ED2Mp;|M(g z;6BSBwuqZ_-^zZXz|=GJ%0vB)!+SCsk26rb0PrlU`{lZ} zaF&6kYEV3JkOm~XoL?<7Ox_syiqlKVQg!~|Jt#j&7>z_o?G>A?Z^cjqUosTD7y)T7 zM&C4szp+GcKi>KMtBxb7aEG5gU>cSxLS>&MK*@E&aV*rw50-dS!&1{F54&kBk|8EG8l{J}@9LyO_OV&bL^v~4ki zG7wf_j|+BT=5cXmCdMU^EF#zBhti<++n3O@aEyAWE6vpZoB{RokjhMk&HXf5CM25U zFpBWSs{T@a{gDd=D$B2JwD%~Kw#wLo9?R9Y>4?(WO-jFGw7av+I+!r9XZm-B$>UY| z2&}48!+W~7KyXKBYt#(fF%}arJ^_cqk3mUh8a?}HQ1RPe(kZ0JZ+- z5cJvvDW^1g9zrPj)8b{(P|ZZ-_k7>Fs(JVv42kc8aOwd-5jNuXQJ=%fjD&A-U)GV( zv&;>`uK-vo^SV&RRS!Ty*d%xRmRQ7<@8Hdjz5R5jp?sUy9*_kH##d{&rK_CBA?-g? zn+4+VAWY5cg$VdDP%EB=c}dQz4O4W8<}7GH9&fzf{IgL1m;rHn8LY3{(a$_6AF$XI z^&1#PQ$w9>vR_%bqW()$t1zt=>{|sB+gBr|lYa}4aS;>x(n{#&V+Yml*bp(0v7~|i z9Qb61f5n0fMpAeL0f$vv5~q*c&j;<0obcvvTjl+oACh+7ARc2fZ-DrNb4@5R14}b_ zNkiNLH+BKE;w z96%n)QM^#@QmIs-cI`kKrX(3KoKNcD|56XbGM9V)jQYLQk$L&_;M6UBV>eLiNG+k? zVyiyG#BD&&<=JnaYD2fHmbG@V9x;8koq>LNoxloE3Ek-gW%Hs&h(@*-99d{k=BAM? zLl{{?3(O)Mz# zfSmhC96Ai{LrF^ zl{_08ECX?!MX`J%2eD7QTS9 z+$ck8Ftc35_2Ygq;I((_cdo)RurkhT%3k^a$?`eOWqGcM3u@*Z8Q}k3f^oS;_7>|S z0x`_Go?bUY`iWjhEh|h0GQHZFG~p1#AG{EpH@4eo<+WkHW4;AXH_6-Fpi6<^ zKG6I0^jZEl8BkC(ZhsyvM2ykiQ+0Ly5{La+05`4*_VjQ4gb%kMA=KnOPw*lyeeQ!WqkD!T}`&Qm|xh>w3C|eS(D}GV(5$T{Ey7J=JE$nTdn0tQoFBPylESK znWHMYk@X-BV#({E`&^4FU84~tk1PT{Ktn;&Yf=>5VHKF1qKy-CQ>=wNgs|v@@7EC5 zQy6Yr6A(|*u!^gtqOsCC*6;p8JI+o+A4bs}^9WjWZm$P_?{(R0TB@Zap8P8i?XQ%~ zC0{CaL8zR)IRF0cLd}cMg_q0>$N6VD!N!Vuwb-uAgj*t|Gri{$hWd$flER>eI=lW1 zU=JSZGzK>2!ur`rzDH9238=xJ3SP|odoy1Pef!LJqg3CCZ%bjR7pZW2y`?rB~3 zgK}(jq$ICHgL`4Ge<-%Dg#pMLA3zqLYg1 z>dR=m?1$I7^#1SM&fbJ`yZceQ%87~wPqX^{l$-0NtPcv{ml*4}4y7`%ZzUrJ#D35p zxETjkc3k?Pi269V;Z3Dyo<)scflqb4{2!T6)T-1^q31ly;_Bke_X>=<1o{J_U1_nDo-tyrRciQRzLU&tqs{CY(7NoH%k z%#qEM|M%g(L^Ck>FP{m*>D(gwMG>^^toj*7&jSmuD$Rvtpc=(#wNF>ER$_aqLZE!E zwX$!&c?DXoLS|yViQPNzEnL{S>zm(?!{E-FjkCWIk*8KI7Y5Ee(Y8F#alMeteOZ=! z5tB4irk|Z7iU9T1HEih>$)fv1&Q`c~o~qGD~}%@E#gdbVOB(@Cd4gvg8p z0~ibh7le|tH6msbk1Ut_MnxsD4><%;J(j;k8>`OtY*Vr-hTSpBdbhYgtY?G|=~#{B zf{YCxz>DwbH#7jy@Tw#uW_p>$M{p(?Vu}AdSU8<@sH^12zh8!;I0f6o@fD~PNLrYE z8}WDmOaiRI$6ZM*_Kp8y5l35tdEAflU$+UL+BoDra%-jN_-z`xJ`JXY0M&y}M!Wo3 z^P13Y?2bjaz>{5On+q_ohQ0*jnl$X1kfMlAmb&=M zknM$;Ty+_d1nXltb9a!65vKY>r@rSEd>+=zv1=%J3&SzfcK-_r^H3;VhGBNA<@16V zp|?=7#_GKR=LqO?U_2kkjX6E^-FxhT*n`KhvwLu1@xfDd4yW(EtP{4vaWi^0{+CUV zf-2~lc^eeUF0o`r$}?I0#U}vw&V3lDW#+070y zE(Ow@vcnx5M_%DzchXiAKqisII(z-O4=iW=P!(ky(vwh?jVR!;*CE=#sE)(_*?E5?xS5Vg43mMA?OkQ5`M+Ve+Hfw^q^v@yLL zqK8|!Rdt@+AA;L?pYVU0w8w{BDQ?|;_}gb{VD@Pw&lnJU;R)VLv5k@JO}h`q%oVJV zN_Hk6?R^f?w^2+~LTFoXPGqQ)9?GAL(^b@TS7>CSt98<3wHe7~8qw5t?0_a$uu$FqMgP5o*B@aG@eI@yM<%N8 z=eWHy{cKrhpO~01jm*N00G%3U)0So1e)e9`fh8kqSPwj;bHSpQ03~&Ny-5d*(gY21 zQmhWQ$?^WUyw8lczR^kYvcBykka3oo<+JGVOYM3G*_w!^4~q!2IK)OL`kZSFS;b!a zrFX9SZ5+uMgVh&SEwwMSkh%35tfwhYs?V!y{C|DD2RxPiA3uyFWSnDVk7Mt>GUJ%Z z%1*M0WEZ83?7degBT2G~QnGhOL`Jf*SE!UC&*y5~_wWAypXa$>>eXH8T<1F9>-+tD z-s>X?1o~PwdJMIlQ(G#~h^BqEFf@IcRM7ApmB^3&^;!|~G8)@z3wV?bg^rs#}6 zuJx#P|7KsF!jHqY{J_K8CH!Nf`=35Og63!}49!n$!^zm_32ie+2Pz#4bqUv|%7asH zjAyx&KYxWj@;OB2W;rhcauj_{(SOI@$akl6CenHN*?5odpz6dVA1GUOxmPv z(@E;2_Bw@vPh51&{09YTxBjei{9M=;?|!|zDvW2t6=V#D4|U6A%jbz*2p4DjjGE~f z63!%pDCV_7QaUjX&nZhWn6A*av|N&lqifSGv%DRD0Jlp8COo@td>v~vXHl)ZwH|qt z#_?#mTawXyG-q3svudcI26OdKifUrxRG!7{;euW$j+{F33oOkgp0NKq5LofF04z=A z;K}8JE$|2))aL`I9yIe(7?;2efcl`ulHdAy-ROmWK6T5QswGS zfUX)aRO67KX_@}Y@7I&-!ahjoXs&Okt;Z!B+22PO-v}Uq!Fy&98#$_;CU|ZaVb6tL zJYh3c)COc6K}Umhuerqb|@PBEi%%IRcCp&<29u zs9HZULaj3O2Y3T>)^w-^EvTG$ROtQ9OqL(p+SbEEgygh4!jwgQ{bOso zhp%9Svn#O2nYZ^U-qt-p4a($64uPX&= zXF;Z%0~7xiFRfR4nS*H;rci}~+!5)#_7&zL*{jVmSgC!x*dmZ1Kut-L^M|)_);c?5 z{09nBw#g2zGJogzv+OY_S)>#iU2yS`hhNMkX(=KQ|FhX}hO$pt_B6s)Ij`^I`?Iyg zr(M31Sw;B?!)3s-fxm^5Eow)gn2hA^09uvs+(xPNpq_$O(uK-V%rDDE&*-t^nFsC! ztl>uP>(el60ETD%s5czhyUDIRzXJQ-*l6V;R4VZ3%vtC)94}w_yHb$#@<5IL2xcn~ zAKErGD%$63b?LTh^PaL%6N# zCghEAP@4x(tXEplQ7|~VmM_UeyrD|RpE35+h+_zUR67Nc%{eobazrMN@;i|@+$|-JW2Zf!Fv^@R94s-SUlW5!r zG#ywI66-}i9Y+xR^2!==ZYmTA1j3x7F%%oc0eH0q&b+U(>x3_t<_B$)w(+`|BKaVq z5{=qc++7|#;tJ_Ks#y(?>I*(NR1%_=D?5zTgqRZ_`!IbeskS&sOiBP&*10SKwIJxW z47jxhgKbSU>goA2l>b~GJzSr;>Z1@mI~n*y$8dH6Se@* z3?wZHRM5^aK1Vm8e2E|ZA`0pN3db($cDJYR7qd6ByBOa|Dg;DK31~xn#TQHUWzjfx6tMyz zRm#gkhM}nT5$j&8fn}S0@Kv3PJH7Xnp-6lBlP1%r?T99OR&uNP#aPm8@%y@*&M+g_?4Yl`Q(t7-NIuy0*Emk9Y` z5T06`D$0kny{)px{n=3O8)z_5SDZ&uTuvu{aHaVe+Ix?Z13>gU57nSBae6P^(E<5x zW<+HAnMNr8iPrYIpZlNmCfQQRZ{*f7ckB0eW#F@&gsxaTwXyn4FwU*<(7Wma@n4xz z(XPbSzEOWG;(t5-t9bbF&7t$qbeTAllfNj1JzA!i?HInPph{ztS`w|Q$umZKR@(wf zKSH}FP*5r`mqY&EJYOoJFYZtE1-jLYw*8@%?hw&fs0x;YNoqYRaW>HlvE4xk7ws{1 za@VlnA#goW3xA7=IJzLhUr_XdzG9(L+kpQX6vyfeVu6id&;w94`{+6X8kG#{U;pPh zP(gTd=jVo`+u=c?!kFQB>2cnS!2J(k#cSg1yX-da#8);$^O%{=%79$qGb$V>-N=4b zuLu0o$-vJ*bL}-^mU=>Ee+CD7oJPx$W~)sY>9qa9Zs8-;p1EEaY0m*&bfNJlv|7z49YFC!exY>) z^Jim21(%?qR;Xb%y-u#I>@tXp!|CbQ5laX2U}msB>ELj%y>S+{X{bOrJuhzRoqFdo zSP=G-+3)lLD7fKTtGTmpVCZBi4^l$Jk<&Y%fTk)yZo7W)GBTA+=UqzxOvrc=1)11Ja3 zGK*MGU<@Hhtv{=r>fdW1WC_02I?7R_i9f`qO!5I{vE5HmO|94%EIhW0-HmIK0FzBQ zs3lk{k+K@L@p5B_=If(AO5D{R46Z{=;C>l@z%h6Z>2lUn7c6>^gHxy$WQzFJB3fz^ zyTB1+?|Zn_P$+Q2Mn{n4>91={hloOcec{K$)7s>l#g}7__w5l0v%Jh>2Yjik_II1! zDm;irScozkDsLoZN^1|gQM;kNZ_w@AN-gMo)gura`yMWVTe zM7Y$Pa(zv9`4e@cpn+2H;8>TQb+@GM_7O{1uUVczCKbBkbwDi^4sL(j|GYXb8iw+| zlO%!uG8v-}@-Ura+`6UAbTU!a()dq1^p~OT1QnyR4v~OR$D|in8)(_bMo@U! zOt?Pv^0rxBn|wzJ){4ld8)%xDnE35-iqML@=YgsOw)sNxwZsq$>5>q{(YE?=?2$D= z%h1Qr#v7o70=Q-sKET~_b-LathsLBD%UE*xt)PMX-`mzWY2O_F2t51< zAzJLqNDwEIW9lupwkrr%`V?|My!72d=#Y!M{}8#Iz?R-7$8 z5_(c<+copO?2+-%+k__M24>`hx5<8JkM%A^v5MQaMf&B&C=7xQA>{m#6ywE1<5HvT zMup?M`iN{Y-IE7fe7@r*pB2L+pE+YWg`Kj(pX1w&MxUWJqmaC#yb_6xWE#pPm+|{% z(jjgox0q@7T46B!^%e5W1l`}4^!w+(IN**x+U(^uNl|?d+N!5Lb(-l1&Y8OxWs(## zqhJ=JDC0(Sl!U#zQE#>RnQV!um8(Cx-zuhKbxa;5C{DAbl1rV~)t6bej8a451~DEm zunOt(`ux`jkB7;dzPRF6R^VloHOv>J`x!V|jCau$|KV~>0ByLbz@?>$Es(bT6?MVW zgnY45;!_E+Q5vN==@VI(2E|3)IN2v5iK$g9o9e@yf>~Dr!PX%Yf$`U-LBtUMgpd=J z832Z#-I8--H0@`t5+f=D*(fMY)L;|rm*uit@)&i2K~JC|N{ z<{3c>r_kP@SMt%%FzG;I1EK8ye%S}ZE+?RDfY2X1I`~1yBC+)d|o(eZVUgzDtaYqC9v8nO@ zJk_Xc@Kmczx4T&95#7BlB3HvT##Sq6E2Ue+Z22&9PDdsWdNTQeO0OHu07p6Y=VYx} ze0#nRWe4jsm^K_~QLX;sWw~EY4bTlB^Z~t@SwnyBL?&`rFt3Vhew= z&FtijE2W)1gJm_vhdR5%e#A^7bflK>j@;sWnfw7%g$UeZTJlC9L*}duB=~AA3kSlz zhF&jL%Km{G)R8S`5JxZIo}C|iwZ9%AUi}1cj5+b(=KB|j1d*ob)BTZi_(x2=Ph-#R zgCnubVV&rzG5AnvDxDsP(a?Mj{OqX|>pgPn58wt1EI=1}g8@0skymuMCSocgh^f^h zy4##n?CcigGn+@!r;rAuW==R1CkiY?9~z}=6MKn<@?(Mq;lz%>S0%ZN=(QM7I^B_^ z^H1+}?hT!X3Kg1^7r9@N0TRl&Fcnro^!5)(a^i@o(a@)@W^Du=7PeN5co0G}^)v+v zNs$THKrX9aj&D(krXk(1X_X}t$s+d(IQ>3Wcbd7TVsUWee zdI+TmvbcCgT-%1w;=X5j=*(hhQp)(DUooDpjZqds(iCb%F7XgR71lD=_Oymnzn#!C z&SDsLOVFdRPxUz0x%ESqsHY$jgFF>h);9`S&~UfK3B1nnYk!24(Ehc=6=HedAKMsO zLVIfNLjG{qzs&Qm74?t$qz?5BxKXoux^rpJGFGE#!%2j=!}y6#6anRLtJbm$voq0e zvWbfV{3l!% z_;LybXKsRPCW)Ewm3CJiP}QiCG#sJ)X=kpud)>Veo~rEDl@-tLSin|F^z<3fis51R zpy@hR8l5N&(GQ5U{m>9CqL3PmQN-?Do?F0r9-h^ zi58bOu1>6f&yN{-iH3XDERF?gqp#O#6Kx4K><- zj~H~2eGri8@k1IvGfZ=yNEX(!kPQ%zJlB3uVeaTp`UtB0 zcL!7;%0=^ln4a9!Ld!rzCO1y7vPc#A0f*-^=( zE|K*5bW8;h_NmNf%RV8!izQJi!eFGi$2A&GlDZMvMD|&9}jrOS`zDOI~SC?7f18ApHcn8Ax3Fn{SrF)_3{-KKB{b&q@M6x5)=0 zmZ3l52I0CWnmHaO>^6f4!kjujXnI%WM#Dgh$B%pYht8_djds$IyG3fMfX{$#UytHeJI58T9e58~x|5nfISjHByy$ zrK;Nw>{6t1B_aMwl&qP(@wq#wnHni?YWd0l!#g>qg%FbKcQZI_@0>dSV`!_ve@V%5 zCpLX>IL=pzY^rzTrdcQ#@zFLCSi6&ht!yVj_<=_O%ZscFcoyOUqu?+4@NvS; zzx;#^FSwAD3FsQd;c(mU2K{N~-sQ1u6$x4tuTdkYGhQXj63Fe06u?!hi@!9gac{P) zrQcGN@6nm^+!N=03_`4*b}x9&JSL_Q_%?HkocUXQVqWkHjSD)-6RtGueJ7;PZlCdX zdl+6qc{+qG`r4elk`<&Uoo@gD2y~p8TV@Su(PpLKb@JTK)(6Etv<^tvFJVPj1(=H3 zX_^iVlRV4q_ca8KEOnIVm$VBOaXA(f6&KTk!b@f=*s|YwmJA_VThkJ3YJ2s!j9&T> zq4v@u@_oFCM!(>ImCkLVD$u)+*u4w}e7#_f*2Tm$gZtqVpbH4j`Q|5xI9-6$7VU$u z6WkV!QMT_Oo3ufh0?HWBvz{T7WT63b0Gu{{T!~=C0)i87`V#Lb@jSo$YV~-sY+ZN;t0vC z2hr;{6f$n>iX1zs!IFpvnZer~Iq*;?y^A~@=bh!-4N5sAS#_<@{VGfgL4Dk#5SIuK zb$88EX`i>GJJRP>9;-FlX^DP~iesx_qd$UWp+q$aC8$wH7I*$&Akp;CV>3IqES;@o zcTtSAv&D3=fJj}lpZsFvH971(nuP$(hj>|>K^e!~<;BjazxFytXeR$aQVK1`s-0{P zOB#keL|+PJV9g!`6f7>_F54b!iqQzUz%&B@<1aCVTH=(7? zU2T7evmH)+LWuFBbJx&<<;wbN{Qk85l(giFa7T4p;0KqsCk5-oiF{gMZht}mKl*F* zXcgMOqc}?g~$gt;|(~Ss)IF+R>jsH6QZs?2jqxN9yrnR44)zr>Exb4g1DwjEI z{&4hDanb=S%c@t|u5i(EL~k=YA3&JvzIfPnC^dzc>O!Zz$0M7VxxsF#r~z;Aa}El~rZ*jrxJ z(++oNjnV$?^v8w*lNT)aSZD%2%I5MPDi$2dvchK(TF$m=UqB^?Tb5tNH+N67?H;BT zpYUVD52262P;=jst7A&t^;T0>9oC&{WxIxoFWxOXXw(ZMGBSWFw`pYZ@RF*z>Y_V6 zUw@vj?Kre!P&-}1763`PMd=5VA7F-MNc&7~27kLMUVp=u$fp^{qv^b7i!+x*9^=zq zu#bI@Ky)p~Sf9@3rLVj-h=9E24QSgwpkc5MsFtNtqlS2qWG@i%LP6``;ciJkRBoqg zW?41QVQ^zMTEPK2l7KxgF79k8x#sW><+?8-H)GiZb%pw+)TI`(UCU>af~?h(IAA3dsbl`(uO1kLv%x_HH)OAy4>siJcx4vAr}(iOm1Z%I|BwnDi68tS zUXDzUW#1JuXdx+9iVp#c*qkuGQ^kr?J{~JT-rqF*P%*|UbjM%}Po*R0p zz~>zGK8201ri;jg7lR_g!}7;s;7Nl8o3@+0`y~k_Nl*h zGczafU|W8Cj}#bA&ep9$dposcP>Jo`D=3Zlw}*qOIy3~nP2Z=Go(aTCaSaVmx`T|f5dSbL&38bLy zs{Rs||K&a4PD1p|N(4-K808E7?N-p8Up8XVT|kz{oL+Md^dPqx^WK~LFfbq500rGVNl zO651*%#UEURdTfUHU{nwN;L&Yu>Mh_wF!ZT>(FHB?ARk@@~0$zBIFEMnYRam;bmlh{wE6_j3ajk z2=_}CD=!eFFkHEqYlMb!V?GB5gKkY0mHk(Le5}l{w`ucr{*l2zm~h}rx_+eK4bgw! zktf~`Hs%|S{-^W5lBbI3TMNMdddK&zIiwk^PQ8R!0w#@-j}m@%+0Hs-h! zBtlH)JZPL{tpvUes=zeA0Z=<_1I-LW%W_m+Y*n{Oth`n|{K(l1KroWE)UihdQ&j+u zvnY(uK~yu)Qj4vcWle3S$$~XUeH6N|%V{^eIWuAzWrqSRLH-LL5N!&?Thy(VK73no z!96z|+fJwiv}~fo{WG^6@40e^;M-1eEF3a4i=yswaeq8Kog?|`;rn%ut8Tzc0e)cU zT(KW`HmLaY?>{mz?WQV|m5aur?I9~5Bp4@jv`hDmnAL!u43Olsi zz&r#^!X99u@qD4__F=>rdi2@v(7(nrNXK9Ngyarzb$mlXSbi#zk_(wR+ky^+;WsPOk$?b5i}(AErJVq>2~?GdiGHbVW1*#n3RL+0 z+$ahYgUlS@3d5QB12;m0nqcTItY~cw%B*FWX_>!v)|txYk(omou%4DNh4-SawDEn( zq(^TMk`B+0HwFgUvKbH;l4U%FN8%9Rk42QPwTL+3qoKV9AdbQl);+orA&f9b_6mY4 z!=+@40}pwzL0I!)>=yK%$h2YFO>8+NI5yYXFP_EGO8`J_Z#XJHeOD_qVOJiY%{; zI~0jL0A(-B6-MCJ8J;)64H?LX1{5O;lL(T#Wmk?6kcd3K5h_T`8FuUqAV93lgN!M4 zKf}XuT7#y9&p($CbeV2YIN}IpUgj3ZW!96$M7Ci)-Fq*W>sT)3N?h_+_632a7* zZM0BDdaTzsD_V;}eGeaw$QDugS9m*86JkE;g9X`2GrWwKeNnWE`~Qx5_^)UDx(NFH9$ zSp^g}jDf46BVd&FWwG%5Aw!L5cZJ*cpN*57GwJb%!U-&hlJqD`r}t#A|EdV@&S6cL zv&LVzwur_dq;!Xb78gTolV>Nj@0>X;=36C9*FWDZu>y^o3=EOq=o4qr;zpk)=IX|G znyryFB$LwSu;p(HcS}0pD>#vrz%`$Ez|+CBsG{2x7Qw5q9aWOQ{go}LFq3I^@Yatb zDFLhDA8vP_2~^bzqC5U69h{)dYRPefAr@^oa@KaDtIc0}FKOz2%Z5&t5hE}d$#jbp z?Gisu$cIHoCOBq}AF~H}UQ|xnGXb$S`~murMSB9iTXe26incaiuNsh68mR==Xh>@3 z)WwJA;e7XV`Prx`TfX@mb-M5HDvXVuG~GK1S*ZlaWhC_N%sNLkem3T_M2M1pZaX|d zly@mgylZ-RKTy=osv)4xSQ;DHA(gF&+@?xr?j?MxM4_egcZoUllyO&*o!YZ@u&*eE zav6#6O2o0~>Ijor&c_#J>EpiW4it&RSVvf=k5Sr)VtZKyZipy$8Xmb$9GgHW7~6al z_rm(RKr1mj3HouX7@6%IExDf#X56#&s)_7*=Dc4)!Bk+vMajWYs*yMLlv14GHkr;e z*u2R*H73gJ7FG( zKPkhF5^K+Va>sF68eJjUrc4_xOPWS3t{~O##6mrCa~7AH=STMhISa1dKC+Cwj_LLt zf;Xcpd-6J|N5=+Ol3EmL7L6W^g zE`stxHdbZjlUCV7s60l$)P;LgA&Ms->=EOa1TWXu%S$!v$g!*sBk0xWbg1Tfq!t-4bv8<1fYJ=%iBk;>pkFx1+ebh(fw$Pw~f{N|e=h zrln#Lh>dh=^TG2x)`}^@!n04jXu7tk!RybnGSvVQrud}D4deCHP-~+N*7cD6}4mI8R-uPY(@#Ayo*8%pS zw?DS9MKHEhJ|+JE$plFu{ET#7spem&)zao(g+Z9A33?)m_0O$tNwG-}Is6EdL_)>6 zGCr8}p3m;{qkMHJ?V>8-aCl*AllJgJK?_?0d&Xm((tPg2>5{kt!rF}K8zzblt#@hV z&ZzKgS@*t|3E>&GpQf0Z%?k3NU2g9RhCB{8IG8-&Q_ca1ENfZ&Z zaZw~$cI-6P)y-#%&8vwInR+8VAchD}m?V0}K}L;L(?IKzYUaFt`a-Sah@~7ZXHx|| zJ?ZK9@75A*$Ud$i^B7NeRSm0&=&9Ez{N)rd2cghS335GZbv_V!&c`GHYUq@azUQ98IE9-(jf%skuLt|%eD(zNep0k#H z*998OR->LC@l$TTJZ<>(JBPl$qi4nsGj8*Jp!{;joj4)cgi&bkFx-#lxs)J$5V3>L zq^oBY!{fR%F7Ugex|n2IMwPw0brQbfBR_II<(lh|5y-^KjI2a<}4p)Z?+W7T>H zZ%Tk0%qqyPrb~{dg+z3kt5+nZ$P9uLTaIVG_{2=`AqEH_ks5WPxn&>>;Mq<#1my}R zsko+4=V!DXX*CmM@)9Ht-v^Wx;s;qKa5MBHAok$ym2FLzg^Ju6il+I9n@;;?1+gPE z1PtB@K)iNpYdIbhjm%&pR26U^$iz-IPY_Z{Jz-E!?GE-KP}(ar#W+BIVguvvLUN)T+kLb4Jw`^ijP3y~0zqH{*7wRc|q56=L(w&$Q2X(St&h3Yv zO+g{u_Or2LcGT8k?+`=n>CQ2)KZ;QA2w$-;85dcTL{-q?3T$Z9r5k46$;6h|>>6|uJvw9FyT6ga zQFn(;Kr6lc%{soC!qc0w8YlIcVyrTOepEvqmikYV`7iUS=l!9 zifJE?K4~4@epS$lQ$bIoR=KF@_y~EQUY&0ge?nO+5sdvQQ3xcUw#9TzamPn$Brro)md_1clF@zwR*qv$aZFGaXq0YK&9{N$CePmBJDUGy_ z{aJpdZNeLO66EQ3JTBh*R2G%QWFV-Sf6daBNxRa{-6-UZ56^LYimyjiBPsn3FI5md zm7I5vjn|qN?vGmJp?^+C6sc_)cC_k%&q_E+Gn0e@mVnW^=%d1%?&Hilx9Q%I$Xnr= zNA$YlJmE8ZecS9+%SrpU(@htWsfGp*9;tRrM>NX&&8)`o=lBvv-6`16V}Hx`L)TVd zSKMy5E|KBH(~3-=8==ZFo2wEW+gl>iSEerS5J&f6`R3L8oF!U=800?Q^ULl2B+k2d zC2qD}wr6JkXYWBelZj`0c}hjNtUXjuiY)#cq2twGN{M@~SmXy=yVYkz89od}OWk%za>aeD~fGdnf> z?Cr!7K}0U01KlUmHHwW$k*Z0z^%G*{wAQ^g>rsw(s(PBd721eMyc&-7Km4{g zpD2EwcaUQy?NInE;r$mvnrF8T7h;;?a`%o{b;z%Zp0ieL{Y2-@uy?5Ll6r0V(Dib- zBT*MrN&r*DouRZ^bxwVAU+*c&@-iCvD`o@*Rm7SnA}>c@;-pR#YA?SUt)-}U%2&O- zutu@se(4e=9z9<<6PrM}nOAa2gGc-3F@fPzwQA)rQ)EOvr?X67=h(TG%$vE)9J*G= zRS9CyT7)s4?D{*p30>wLPe{E9(v`&bDpk8^FTL7x?rrwq5m!{?NvpT-sBNha;Zn`R znDIAhZZNh>Qk>=-iaO`CFQjEpiaWK!HvWOQhmDa>uqsb_?Au{-R|nowJwefBe0ee6 z?VpBKei5m84tPZ0@%S|*d|sdvjitrKqE^ypsC{DH&=lmYBu{-(hziKlom2beoUA+< z((RA7~XQvx$|FiGIxC`Z+U%FeUU| zMK>xdkC?-bU5m@a?$eXjuxi0*4FN+?$*UK}s!j>oupDFj^yO5H^j)1^YZz$Lg0twy zLBWdWezievvb8GDI%wOW`C25?=FslLuF1)S44E+0xeY+1CCU6Jwt=PxpP7FnUEO>MGBv=P!36PM$XY*IPtr zAA+NID1Voui|)9a)DI$cqL#C<5jS5La1R`e@yxCSok>jcGE)y4mmCPO?7x&1SI8JG zx;h_4I?XsKXpX`O?iO3M55x80oD|oR5OO#I3E&^n#J*F z@zN*QJ#Hlr_9U(?rJ_tE<4i?~=j_4&ydF8j2m^P@RUt(LJy_9M;EHP^KPWaiX2Uxi=w zgk$f}V);o5SuoXpU=}m&gRUA&g1WU(+=Up`asasStCevcP(6sm{U{LAEi05BDf2cT ztZ@t%nU9>rd`0__ZBRdmENuHMvE=d8>EHsT^;nTL<#(jsJ_tNmyRcrlSqfb;Uf&Fz z29th>2m_l%TwT>UIzxhY%V}bVe0!D}e%5A!dB#TtG${v${Wh1`Y#4L%vJNBT z=(TINHFfmU^;&fEp4B$L^xVt;Sif;X&x?{qPvly1)b|!@yB@P=s8jeCWcgJhTz?Mg zkaV8UZ8+`k$FX_UN%>^|(o(`*5fkN;Ay?32?G{c!l$J+^vO;ojS2U-AlZ#h^KYAiq zKNXuHdJ->!XOKCu-#)E(n#4w{*g2O{mG$Y3SrSdBQt-6HK3?3i8{{Jz@ogD?tG{_% zP!D&-u(idwL~Sv>oM*Iu%HV%Cggp{7%s_Ih3BmNyZe7xXD_YaBcAiyl)% z?b~j^iTPFojwly=%Mad3)SgLvCS}Xn+_)=amgGsa?ukxHthsYE?Q86RVu$_@7#d_Y z;M$J1t$5h?9~k)`ko>=N!#}_M_ZJ3;$1wV3xzztk#{PW`<=+;T-@#J<{`OyO88!(( zaGu?3MH4#9-)4lcx%qi`gl1x?b=7_R2q!4|ixd3M_x+m>rqDc)| z(~~cB`qOl|WOW8nzAd9DU$MW-V;B<`oJc9Nztsxz=xe|pZgEiU6mt#Avx3lwoYh)u z&L=mS6yel+5KbNUeTMhF#~%z^n&=J>J~lsbfr3-)kwOf;eaWoHVz%7++WNB^%8T=#GMJ z;>%K(PQHnZDTGy%Zd8Clgx8X};dHF`LL{2KW%Nl^k)nJ|?XlhuOzWBU~f@>+zFo8<`7Mv?|0@B9vcuC6dJ2HbYv`i>u{i(eEtQ#>ni z!7%C0F<2zwC3iZVc6?L3@8zcD@NY;2R9CRMd?X|_kR#pbWWVctL02r`D#S>xfL&gf z&^hvC)paqf!#Us(Q(sfe;BKcMoO3=`5`Hu%DBQV8PJiT32~YD5GU+`w1oiBv?5bC82U5+ZaaWC%mD-!r1Boychj?E4%_D&4& z8u>xu9#{qlE&C31jD3GIZ;@<4_g5kK?pvMg=hp%+&G$e(CcM&i-fo2VfcQa>h!DN%Xh<_@4nDW8=%K+;4MoHaoqq%hd#k$lX9iwVuK9`v>~3m>%>`T z!cS^&J>j|xXaUBXU4B`z|MRrZG!pVmWHjeHUpSN$*#CS=Rf1C0$R+7c$FtSnEA{XF zg{%09e9+Om3ylVQ7bJ&JUjibv1;1CIi^>1F!he3DiLl7W<5btfSho@Euk*4=1D|SPLL)c76t=K$X+Z^++{;<7v30ty)e&hlAsz1Z z)U{ixZ>1!EL&(4PssH_%l?g=Z`PJmKOF%V}$wNGUhk6`iEeRfc@2H zdBx)Y{f`M5fTy)tGpPChc=;hjsm65e`TuqL|Gv{L>R_rFyJob3gcJSWU%@^g;M9)D zEB_m1^54G{409SRsA`#wum9(B1J9cB%-vgLJF1dKzf@_Oi2j)B?h3`9Z|@?+?c)~< zG0p-XhJA9-Wy|g5XdQVm#1pY4j=MI+S<^M}zUcXH+6WsN^yOUu*e;QQk1B$uXW>4E zXa&iQ08^~}{?R*Ak#|Ys7T}y}e*VyjrpgQXM^z^rlxQk_4U-wQBex`}-qm07!7zFBBxU<_|= zj&i0P6!pGqxH5?@XuTn1c2Zu^hp*M~zfZOzwIG($%$s|4TKpOER+swtfoJ^!p5Ucz zm)u`0|2UTj4AKS^iZ#;gY|N$6rs;)KT!BCLK~^bnuC{D*_2(x4h%P>MCXh(yjv`Y{ z&|Sy=y@i*qNvI9EKD|Df4>%EK02)USz1tw{{zE=05#s$%KX!i0Ye1E|G+xX*gHEz%+Ss zpe3K*V`(|B82JdvlU34%!z5$R74<^Tmj+oGN|ZC5I}q~B1a?#<(cX}1F4}{E!~Fq> z=>ppK0DNP%<5;G*jaAt}ywU*>tw3ciMY6}w z=sNTXd7V>eOzV$(|4ho;bmu0y1K}3QW^_FM0=vGuJhoAfo&AtUG8fX zI4}IyrgE>=Cld2kTIZoV^Ra zQcK7kLjtVb7`_JUoNyVq%L)M_4 zf%h*UBaH`Z7Ud-_Y*<7n?SJ}!yvPmB`mA+_q%S-P%^+tOV@S*L{Lf0nH0iM`rE4(x z8UZCW82jzzlbZsMyv1JH_3>vKiPX%DlGbQ`iq^3+{RxXwi$Z2*7}Fhq@uio(a`}TF zT_NRm##6kvPt@G;Fp?C*X6P+)T>f6XkJ?eso758}Q`r?K?%cdpN<;U0(x%TXroVy*!5`uOc|ka#0j%HSuSjbx>}#;(uW5+? zNZE^YSQn+m76OLq6ImpsiYYIxi!{%;XZs_uGq`BYJ2=u>M!r8gZd(*x2=)5Ci*6AG aAClWSZV)j(d6|O)|7oe|V#`&mL;fGq_pbl| literal 0 HcmV?d00001 diff --git a/paddle/framework/multigpu.md b/paddle/framework/multigpu.md new file mode 100644 index 0000000000..61ff1ba204 --- /dev/null +++ b/paddle/framework/multigpu.md @@ -0,0 +1,60 @@ +# Design Doc: Multi-GPU support in Operation Graph + +## Abstract + +This Design Doc refers to the multi-GPU feature in paddle. We propose an approach to support multi-GPU both on a single machine and multiple machines. Every device only run sub-graphs which our framework issued. We use `Broadcast`, `Allreduce` operators to join different device sub-graph to the whole graph. + + + +## Motivation + +Paddle supports training with multiple CPUs and GPUs, refer to different physical devices. We need to support multi-GPU training in parallel for acceleration, in detail, there are two aspects. + +- GPU Data Parallelism + + Suppose to we have `n`GPUs, every GPU has `1/n`part of training data, and store a complete model in GPU memory. + +- GPU Model Parallelism + + every GPU have part of a complete model in GPU memory. + +At the beginning of training, the framework needs to issue the same sub-graph to every GPU in Data Parallelism, or different sub-graph in Model Parallelism. + +During training, we need the operations of peer to peer copy between different GPUs, aggregating gradients/parameters from GPUs, and broadcasting parameters to GPUs. Every GPU only need to run the sub-graph with correct place information. + +Besides, it needs interfaces to synchronize model update with each other, and issue/merge model from different GPU Cards. + +## Implementation + +As mentioned above, we summarise that several kinds of operators are needed. Currently, we need to issue parameters to different GPUs, named it with Broadcast operator. And also synchronize parameters between GPUs, called it with AllReduce. + +### Graph Converter + +To be compatible with parameter server design doc, the graph converter converts the user defined operation graph into sub-graphs to be executed on different devices. + +1. The user-defined operator graph will be partitioned into sub-graph. + +2. Control operators between GPUs will be inserted into the graph. + + *Broadcast, AllReduce in a single machine. And Broadcast, AllReduce, Send, Recv in multiple machines* + + + +After convert, the graph as shows + + + +Operators are added to the sub-graphs. Every GPU assigned a role of `rank0`, `rank1` etc. + +- **Broadcast**. Broadcast operator distribute initialized parameter to all the GPUs from the GPU who owns it. e.g. from`rank0` GPU. +- **Allreduce**. Allreduce operator synchronizes parameters/gradients between GPUs. AllReduce implemented in the Ring-Based communicating method, avoid of the bottle neck in a single GPU. + +These two operators need the Multi-GPU context support. + +Need to notice that Allreduce operator force GPUs synchronized at that point. Every device only need runs sub-graph in a loop style forever, the whole training process in asynchronous or synchronous mode depends on the Allreduce point in the graph. + +### Benefits + +- can easily move the optimize sub-graph to parameter server, multi-GPU feature can be compatible with distributed support design. +- easily plug-in with NCCL2 library. +- GPU Model parallelism becomes easier to implement. we only need to replace different GPU's sub-graph with different part of the whole graph. From dbaaa4979b7cef401a28491906b4954acc9f9b3b Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Mon, 4 Sep 2017 23:33:25 -0700 Subject: [PATCH 02/94] fix typo, rewrite graph --- paddle/framework/multigpu.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/paddle/framework/multigpu.md b/paddle/framework/multigpu.md index 61ff1ba204..c8501725f5 100644 --- a/paddle/framework/multigpu.md +++ b/paddle/framework/multigpu.md @@ -53,6 +53,10 @@ These two operators need the Multi-GPU context support. Need to notice that Allreduce operator force GPUs synchronized at that point. Every device only need runs sub-graph in a loop style forever, the whole training process in asynchronous or synchronous mode depends on the Allreduce point in the graph. +For the simplest implement, when each GPU compute the gradient of `W`, followed with a `AllReduce` operator, accumulate the `dW` to full batch of data, then run the optimize process individually and apply the gradient to its `W`. + +In fact, in the way of every GPU optimized full batch of data, wasted (n-1) GPU compute resources. We will enhance it in the next stage. + ### Benefits - can easily move the optimize sub-graph to parameter server, multi-GPU feature can be compatible with distributed support design. From c11718550b959aeeba28d0f7cd35d6a9f2f872ed Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Tue, 5 Sep 2017 09:57:16 -0700 Subject: [PATCH 03/94] rewrite graph --- .../images/multigpu_allreduce.graffle | Bin 5294 -> 5704 bytes .../framework/images/multigpu_allreduce.png | Bin 114947 -> 118413 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/paddle/framework/images/multigpu_allreduce.graffle b/paddle/framework/images/multigpu_allreduce.graffle index 2659baf8ada1c7ed09cda326fbb6e56dacb8fdcb..588f79feb9a3f996b6d6208e556d61f0a9602aa5 100644 GIT binary patch literal 5704 zcmV-O7PsjiiwFP!000030PS6CbJR$({XG1NzI@q;yI8LGW0qmV7-nED11vVMLpb7w z=$36Qa?7KZdCUa=eY4cQTURhRN)p3JPQdic+;7g6VPkPO1O|I>Zy zk^7yX-;H}=|LmvxZ;yZ8Gxz`b=-$KsdGhM;`1jXOJC{*7NIS3JK7R51uycQJfB)d} zG79$hpBz8wyngZg=(q#-*x!Hp@_y(3d757SxWE79%NNggt-Nl0;TjJ1UnlWpkfh&U zz|HocgV#%Y_u(Y#r^!{Ye|`P2-~Hqs z+3&&QF3-d6xY_8tkD9p_gX?SKVf^(MdC(!>A9VhrQql(z?r*f1(TN7Lmpm)TgIOzS96z#*PkT*m&vdu1AF-@KOC6Q z@l0HQntT{e;UM@B#}_jQ#G&8+>Y%dLN0)vVZZU431b+00gYab^uA9PtoHRf2)5>@lp?Z&)J!U$L{Yd$b zoOk|C;7?_A?j~_k?*A-^!hYwy@5WH<@FW~uM*g>>t{+vN3P9~#rk$TU*@v&?u0MxS zaQyAE(#;20l-p*q_$2OLT?GBKjMwq>vC907WO(3IN)5h?LRkV*4oOr{=4nESRaOZ~ z8N6J8w#O8g+Hz*8<-$0uG4`OtNfBsuNSi?I+`kO!uy#F3>2dHit@iu<=}}GVDE?Ay z`X(4;GFon(1M*}NrQroE51YOS`!}F9f1~`JidtdR_;ox8)7 zxt|PzlypEnd=65urZ0o5;2+R~B$LK16pINK_Jqf;E z`e3kollI-g8KCo|OPzWm(Cp;WPkJZmSE&g63)stkDC2%GCbA!fxF4KnO}NdP2$3~` zv6EqM_e0UFiPF%--8W47etMPo(a9fO_lq<!;fF9FKC?cgX zpsDtx^fY6}P@;(fb0zurlB|s_T21$9y(fI_L(g200!B@QowOx+=Flzi7WOic!x0zeam2DF9 zM%C5Kn=A!C$Z#b&F^mv!ha7LBp5#Ju5KCz_AJQhe^|Xn$Mw`f-w22x;m@+Vx+8CH^ zjkF1A(+1I|$4Hy*E^QKeA=*S1rA^lG0&5}$coQ`q%(l@WzeW?rS<~yXjzv+;wIos~ zXlOGUg}e!Q(=^_sTjWiYS>sWwEFs!}?l2s96O$fgoWn0bZwo@`Zp53kZHqW*GbeF! zQwd%Xu?iHl1p0);35gTNn_3}GR(leZmIY(Xh}2SY$*h!3hIen+es4;LouWp{EJBUS zT5gVr5|_lYf)nr-i4|Z=9&`j@9y3-@&ImZ;jEhDl%Q-miL|dYWvP@V#ome4LLZ(y= zq_Q@dlF-7F8U#=iFyK~#R4PF|E3NyVR7@L?>D^#Tx;3Vx=447nSPLGNAd160GlN74 zi4qc}X5FyOti&$FO4Q$b!{J{gl%gP{E*8U$XSvW?0h1O2{FU8EVrkhL!w~Z_49P%=z^gNiN?;ib zDF#vuOk(XO#aJjOR8}c-AykW$ zLs;&CoPkQC7G#%b7fF@c}HK9fi*Kr6Xjkbg;P8GE~V#r%t0ISK|>o_E_S}@}5I8iQwiG~~v zImQlf9mh(-8ET<$#==WnLEF_3mRU0o_pvu}H{6wz`gkOV1Lw zN}gE|PT{gFHj8ms4jf7v%4in2FT5%~BGY!o zB^$^D*5uMh(VL)m)eXK++s9&_#o{Kj(4?%w*M}X0*g@>H&Z3jp?{)n_+9Gx=S4FEs zHw;M{k%UOv1}mi9DlSc|A%ZGunA$X(q!3Anq;0TCLOUa5j7J2wL{SZXJ=g1lju0H7 zgV5OqBLqawBUVdpHKCNsY~jQvAxR*T5J_8MgpBkUf$-8kx9VBD-vQI(f;y0~ z-N;-j)>h_HQCxzAnM+)PgsYybK&{f~nbL%cF21fQtEAGvh%99cM1@N(ZSz7VT`cd; z>bq5$JK;j6yPWf*ZSD?8xfO0d0xC2iAX@|^PDt(nEw@Y=Zmlpww$916a|~(P;uuoT z$uWebS_{oxA-v^8Ajd$Cu}K``{mI{|3uKXJ+@MSJ;O$F0UAYug2+~?GhA3Di3VQF`!~mxi3c^)|f~*b$3^9Nhm>&0Q5dt9Tt|Sgv zlNH9gYHOEDV4emzNTq~S)Y=`%B(zp%EeU24Fq43pggcx`Sp1lsSq{p{6&<_G0nyZ0 z3vRA9ZqXR9bx#Xwh)`NdLl`bD*rC*c&={(`m7}qQUDGy*yLU9!PRAw}*pUzcI1>QP zdN%Gr0B#uo+uR?+Ik=UV&-m(sIsz+^(PP{6Y=D(jh67)r%y3o*8ikQiM8Rsa0ftsd z#yT@%LLvqb11rr2aAG}5IhWLMtEhBjNl5secDwRYBaAy7Za2~aqP5~hK`|YG=>SXz z+}(75Sh&`Z3*%NGXk)FAOalLKu?g#O125FNQgOJ{xM{4@*UX87aLEA5jnmgudz2{y z3XV#*6FaVg+@b6M$Kd4RQ*ck%nHLQ4xSMzsZB`Okc<X5R+6P9R0=5*c(A}U!%(g{|n3Y?PkHX)7-IDVuyn8jPQ@{`i+1pQOQaQZ$gQO<(h9|> zM>VtHiBQEUs58ealn26Qr4?3z+p;TXXtUNbR&3skm0^raNhPIPfm7ka$H*&?S75>l zc}0UtnS)n=G}R_|kXInDm_u%5t1Dr2cF0j>r-qdk`A~~nx}o7)?Y4%DRZAKj21+H3 ztHXdm3?K$pd&rSk@t)5_VlgK|qHRe0+y5K}Nn7Qdza;V1WnB(Biu}t#&?`D7d14P> zA~gUSVWi13s7PFKSkry5MlY;*^IA`AsxMaWjn(>NMSRuyWOV~7@Y|arXLYwPcFy&! zC+uYpt@tJ?(O;%{#Y>whL|kyk>a|?mR8%xwsZoQrYu0zADQnPqwNI&a(PnYd8r`(A zqjr-M)#yf*g`gr4MQ3V0fm1gLT;X#iM_)3rr2}Qn@c7K;T@$83zuDMM146yCTcVXW z63fkz-6l=guO*10>S3{63c4wiQJc#^ENudoe!Ge=_Ocu&oIZ}U}Vh~KX4yI+|FgL=9)6COY&*HE;zKdoCk+DQILR+|?Qmqda z98rO&Sgs)9mQW$|gU+xlQ@CmaP3+1*Ar@v{t#Bf_!JKIDB^(Uuq-x)I~SR0ws?qB)8#^(f-)Dn)Hk2I zQtmMIpcV{#ZsYPZt#2jpIob@OA~GvqcB9cfMIw&1OY1gwdI@v4Y#jt8vskaW?IW7I zEi`vVV0M0;SYEb1!sy`|H{&rxGol&S$gLI4vIR#Jv5Z*Ggh}0=BWlVbwuZ7Otl_&A>b5kI zp5}^brWBFJz@=3oMq=$2)@~JRw+^z}EwUMYpyLSHI|ErEfC+mp6DBMpK zoyJY9Y#L!=O#v9iF#v;D+MQx1L)oyHX)Wu)6qaWoFgCG%NlCLlP)Gy?f?^YEmV|xK zk!EtD6kQ!A6jmi+RZ_7k>GAMHDY_YDNy4oDL@CwEiBgk(qw=7zmyZRGg?D`Q=+f`P zbdPU3iswy+GWzcSi4)iS&`H#!ak1#hua+(mk5-G8KGw9WwmO@sRWehn0es7v!m2@X z_A%c$SI$-V)|E*bOG;i&CNCkI+^et`3|_?DPr*3yS!-u=t>1-%@Z&@YUN`^!=~21; zv)BGvFqXyfcnS{fP{1#s;UO4;G)OAVX{Gu91;OQUeAM-GqZmi)h5Rzvkl{27<`#}+a{dBXydE2 z8EFUo@WM|gU$cH7S<@~s!1%YY|0EowWmK9HEVH&p0cco14dZ^PIq^uTeUuIJX&>}h zbu7I&tqi6g_-X9&4KITJS$bZ103*Xc%o@Tt*p1Qyv;K1oF#Qr=hBLJIzr^@DVt_5(oA(RmQ0CBYw~;ZL=!|JffD$iuM51-K&FFr4o&q+AExH2gdZ#_IvS{W*vKYl_W3|*-Tc+ZPAa) zIy@aYmj6GT;vBb{*u&BKf+SDta_1H>SFvofBq;d5F7jYX`=N~9ZgJpwXXpC*o2w{D z8XLeH`sh!){66gEY0Y>lNzIoB)E!0s$7g|qcZr&CUScvMlldk%D=4q=JnT^^9(48? z$@Yn%MiGx|Eu^tjDap)(4yA^AlrsvRH>X5eT{BG|;`pM%k|eY4kEYyMtwF_=#^yS` u^P{WE>#Mr8&4~J8rsw4vaZ>oh>%HBoHk$mri~xR*?)?`7r1L9NkO2S$Mingp literal 5294 zcmV;f6jAFRiwFP!000030PS6CbK6F;{T%-aUVquDyAm|-N7nI{EZMPh9b0S3_Qs{E zTQCVpSVIH@LYAFW{`>ZT6iK}J7GG+)%A|xIU;y-)KHWV%^W>l3uA|1+AnAs2`=`g$ zBaa(FyBW8__T^8H-=DqMH;@1M^wE?5IevR|_S?zxMkflpY2)Pmv)8YV8jtr64h}n= zC^$GcK09ulync0h)_`*y96W#XxbgTZO*=mx9NgaCdcKqLn(?(0>>ivXaVJR9@2}xx z`_RB^rLD(sncmrR-GRTh!e;vP(GO3)1mB+?Hq-EH@Y??#B(K`7;M2X|lZn!vTgrk}zoKlIK#Xyhk}@4o!-q?;yi%TJ+)UVPmSFO&GDRuOa()h6TpPOzvxOAG_1P6D=pZU!%?vm{m zT&{B!HV0z;d+&=miSFI8K{x*Vlk7FfUwe%|sg!iDp*8>OuUvm!gQLjrmfN1jQ8>J4 zp~0;n9)9z2nD;e>?2fuw!rDj&=tg=-^4* zeH(u1J;HAADUPqFP>3VH{nhWjO~T7?s0w$_GJBMxZ*th;H2fnd$qyfSsZku=T(=LS z@UmU%6fJiulRoYEO*q9sJ_-EjDF@}tzTAnzFHW0}{j}5{MyTE=W}le`V?R>8WM94&o4yF6;Ou*+ z)XWE16lF7AJdT?;*Fifi;&t%&Sn2uob-3$>lnQ+3m9hY&5hPK;rDq8fiZE-85qz(q ztaRYAfOIMiWt1CXtrYOyrOG3%3Ti{FUHP4$3T!2`4ewMcn*&@G;!2q6Uiq!~b`(@Y zfojy%!EOY1HFz$c`zTJ@LGsRTg*W+2WIAjHx|~1z=vUX~Eclkz-(mjf@co@9=>Cv9ND}er&-078ou;3o^9yo*(GKCWdF3bFASDe@|F1y`*5XZY z6a3>ZkZs0MoPh8@&Xb_Uq&&Y&f}ribe2#7c_{^mK_dNL4@xdUqhVsqsC1~GylREej z5IgVqN$WiQCKZAIDP`G*GVX&hk$o8AJ~+=rxXna_$V8y;q}SSgD4K~V4I=KmVbb=~ zo5YXKe{Z@!q~YaN8niF`CZL==dX(L4m$Sxs^MW)Eqv&1Gx@iV~O&VoqfDWT{-C!{Y z76i+(!SXDL{Z`ZOrb`5iz}!4ywd7V4N~zRx!;mBpNrGzv zJS8=(LgnkQJqDDcCmKCd!BX~wPvUNvjs#2x@FAWMFjh%LjnbNehU5X|#NWkf7FHgO z)Q(=06#8C3-bN2D^PC%pmQ@ZdM|DF=@9};F5SP)He|xoqPtEY)fLA{_z#DnhzKHAJ zxRa=O6;8AcH2j@U-UANJ=bMLO6oNJ#DlCa%BZ)M$@8t4EwS9 zqmO%uhe?v_z~KBZ!YB&v4v$`_Zykb5zxh4a@WPL}S?>6UQ4>-awrg^YLsQ|WX^^xl zTQE}D7|h1_bcfp5^>^5ZHSD4LGe(2AwAa|z)T7|6 zbIQ2^mt8PZ>(^H_pVYc!bGJDRBlJ9+NZR#;VXTY7P|ZP13dK0HSr}%UInlG=k1$Ab zC%>C^kGyF^V>1+3k!J;`0!9o_R{}~bB=;mxCZ-IxR+wI5m6L7fP0}olH|bebv0wXJ97{Zo?N$RtU7!6%00V~z~5VqPg%xWvC5|qd+I0_&nR0~e16ogAmz}_E( z*NR7uc)TAx%DM5#C2>v+Axse~7&WxClhyldFO0xl+SdUW9B{keCDjiu868@$wCyE+jtkFA?4iHOQ z^g&Qe2Vgn?(*X}R9U$gjqk#+KW*}%|t&mIt|8TJh>v01w)Vd{h;E=3~Y`F27Idu>& z89=#l<2BVDWy-(+M}f~$tzKt6YR zjw$K97DlvNz?Eu4dw3jm<^@AM?j|0^GAjuzymxTTgi=frskq})PCpFrYALKGL|MHX z3lj0A71r}v3+t8PR63kW38*6i*I7pY7Z*L;O)@~|Dl6pkSL#dZF$?31g`wJjcJ=bT zo-h)4f+UK-!P@C|^H2#S18aAv=L&UK&^}h8eJjiMaNH3KE!n0|6A+wxR%Qi*oNGlC zfJRB}SxdDQ6ozeh4R$y7A>ndq{8-G$k15y~4liB_w^tu`#t`{2@?+eGWC<&FG9^6e z*61Xj;Uhn+HekEoZ!)RHPOZyH*j%(-%pJqC%yS_YA+;tPh&r2Tdla{pi0rK&B5^fN zT#a+mYMjdq8Zyv5?dBE>!3_fk(QR|+mLHLt!Y^)Z1<>Ce2^#XHks>xDDbnDWSuGfG z8>ImAv`C7O6k*VCDWu41XFZkm1Ux~hB{)9}{JdI5)RW(z&|sGc(HN64cD!WsCpATn z(I`(Vuq&E#@USUYwV}&77!;BtBu6DpH_KeA+%2cWnP$R*_2w`lyPaz#^?)zIkRg?j zR9e0x;*=128W@e;BjJUaF zZgVZdtu&M)BSJ=m@uDSdduKFQTIw7-Fv8Lpu)h0JdYUm~DA7c@)dXs%SyeTk)Ox4V zYx7N!XJ$Ry-tGPrdE`;sw+o);%CpAJkpUHEpoA{%radl*RYDMO2`pv1F&Bv~Z7vd< zbuN;&;M!^K*eWr^a->m6qi`z0prKFNlt-z^p;bY_#$SW{m zg}kC>NST3GfHqYocaT>gub4q^;<6#(jEq|P%aX6N}FIMf1Rr+Ijd{y~mRUOLl+i{Vzs?+Bi zkF>2O>_rbP|0YV&U#5Ay7OsAXGcM z1zI_lSZ&qE6Qgus~Bb&B!tV!E8ZOMy;x`Qd8MGG-cao$|T`) zn=-`g5-`gb#B7Gdy#{3hG5cUJOAxc`#H?6`%ZP}5NDiaCM~BgfHF%2>y+!mE7xNZ> zo7-EQj0wj1T7oF58W!6%K-Xn5Dsvf#rA@%nuQw6KUKZno6KM4W;Opi1ag!1W_3j?> z!fo{AD0*^>qIQwwpIH#K^((ec4Q|BkHsY2cZr6!hyNXN)q84)WyOciIv zI1KuZv-pnp6o%=8Vm*PFD(ixQ7zES3gK3dC%vd;ant2-QnH*NdchSrsG8U*tSQaj* zRO>?pM^qpx7Ar`&CsYW%*XWgH3Ri8QiCq~e#N5nlDV#`dFemEXhTU$JmhYg3GttxV zpHJR5jv9&I{?cIMlzsTLPa?k=TnG6;R^5Ub`C6(%X?ta9nJ`tGl>X``mtniE@+Q-$ z{9ZlTS0>AUs}SREvE1Bx1ci=<-P4=Tmkw+r9T=?LhjmHYImt}3`9qwVE*9bxl)2EQ z9>4EOvBAWRS}^drjnmJxzL&sfe=&%X%B+0ajYjtri8$6St=rt`1hoh-RE4w^lUEB{-spWyEr(OzQp|QBzd0HIzkZ zg^nmXqMKM2l|h7Yz5ZB@#+f;YD=d-PN+M|%n6$0UQryB>avD>6l1L`E(J{m#jD_*M zNQQF3A`GTPuXak!if42VcN5RJ6xBNMY{-gJFoagH8dSzJapY&$1`UFw3i8Wc8YzROE0t5lFf#q3>?llyza$uyZ z+tNgOnk%ZAQbZa9hgO9ciM3l;yOpoqI?QUf$YzvoDSfY{TU4s5;w|Q}92~Kd>!HO} zC7(a8Di__BBijd<>G6wI)ovDy(bH{=o=S5s(Nn83Zcf*+HnK#273nw@X|3CmnFTD_ zf`uKbw+kYjY{)9D9U8JNG-OI*Qh%M0U9fsXYuTGY#}Tp*2C_J>Zmp12%kUQwu@A{x z)c5Bt8omZsQKPGfuHq(^QDu-|d@Vb4OM`Q15K34%wUvxeEif}%+NC&g6H*BEt{!s2 zZS>->nqpzhE|B$83szGY$$HgMbPCL(C$}@G#igs(iduUA84koOrZ}dWM$0ojSx5Xu z;d-j*G;U&LQy&v+3cx6i0T{&6?i4c_%6i32YnczGusj2Sv5ECdN}Bb7LLw*-6q{JH zBc?U;N zJAM-$_u!=cc;2up{d@mUoH+3#r%^-Ue9@C%E?puXEf+0)u8=EBT}~yHOeNKYyR0az z>L!<;M}y%x;W@h&Y|ro_%#R~fgwnPq$EyD;{O)}owN9~>5uec5UuM4^!)EZGQ9FY zO$)lVLQC`)8TDacO(;S0t1YH4co_IH!Y&l6kwUjo(3?&`e_)q3*y8h1^FoJ=JPg; zW97cI;m-Oewa0caj+YO3uf(S=wNyqcfzUU^I(H6eyKxG z8En7$m+c^(MZOE@IlT&kw4nF{RQwvZ!V56duGhySZ~E?}j8MH#>^_kVqJE_MM`jv- zcTcW#t?Wc2SnVG^j@y@?uEML>4MX`RZ{jqV{$}1LAsEiw;~k6#tTY~gG$_l|CwHUL zGk8N^x>LwHAlF0Y&FQl^P2=mH*&S-ah+XWT?;?u*G!N+hcR}*t9S&B#E4+-8@Q=9d zN6_!ET~N95g_?28jpY>=dUlxp(R%qy^xH>Df`31YKD~MwABulH``4?}=U0bUZ{HJ1 zKY#osYF-|}@2$7*>8GpDFF!6$tQ*dE=uxi2W*UB-hH+($ol*AN z&0xm1@7v+;H^I=?7<2dUyTS8z8Ybf&`-f`sS024ThV52-TR+Ag-6RP-iR?+1$!*?` z${IZHJC^_7d&DE$YG@Dp^9zzuS~rY4xP=U|wSy>gkvPq0KJ3tcwfSC;dOCN<-`zw( zQY!$j=d(ZQ@W-$<%4r7AkyP9{K<+g1KfeqdwhO$3^8$_O+sk*sWp3meroDtGnc+lQ zVy)0H2<rt~7q4sJcRewgZdBSM_yet5OtTOOB&pPdMR`1H|#0RRexa|LMu02u=- AUH||9 diff --git a/paddle/framework/images/multigpu_allreduce.png b/paddle/framework/images/multigpu_allreduce.png index 7741bb37f47e10f13f92c732cbbe9d31f7762be4..52d3eacd55834880d09f4efbee804b5c567a57fd 100644 GIT binary patch delta 87925 zcmZU*cRZEh|37{X2gitWY%-6Xk-f85LPWBXJ)+EHcFg10GkcFh#W6A>*(2F9GO|M| zJ4C*h-tW)n_xnD+|HL1+bME`PU)S^X9IvaeP@=e8BHVQ^^sc$iuZs(|nqQpeA1m*q zCYLxB_A!1jpyqo%W^=R9sg(N7`aMQh$8;a|{b9;YSuHW$b zD&{yOFzA;>G0Ag>;VC zGeltVpvxvN9W(~2VuCJC27a0PX(NQGLyQO@0tNV5_Aut)KjWj&0!&x@@XXyJLie99 zFz24%2O`)KS@Gh@A(_bK03RgmrRtSM@d)x@#RgMv$XS!(D0JuBnYTA&w>3qOquYYi1|z0v%iCvgmR*0QaM0RR3;y?LL1upITL=UU231!2_ag}D?hUEL zyYoOLLUjM~cWg9?H33a9$F*xENgeru*vO9NblN&~r55yC=B!nOL+NifPt0v;n*Z-3 z{`*N@NQ84zlP#U#+Vc5Tv|#JlJ4M*`{@=sih6&35=VE0xJ(!RC`^DuFl-5^-4(A7a za_Khdk74en-uFN8t*8et7`{ixylBKRR_~kBpWFXc|N66}Rt`Z@y2Vt9bMT8RD1kFWp_vd0SQTUjOEb)c1ny zyBT%t`-|!3`GI@HEL6)YDKe4vcb|&vW~e-7&K64UsUomV|IZbQ7iYD#CH!BY=E5Q2 z5#q264N9o^6{-^kBp&z$WmRhW-{=1S|ENK+ z@l3(`7#N$B=>J=HZ~-+sBpk)&@t=#>*x;aKg8%!26^50)_+Isavidmv!LyUim-*L> zUvGTlE9dy1tLWo`r)e2(1gfeozNyHksvk>s>{eV#OcH>I?pP$XY z3^bXa9E@syI9bnTf66^bjQ-CTmZ?LSA!8SAAI^OSvm}1K|DE?`NOw!OM4dY>ZRFbj zeBr;lLrCDE|DJqH6EtuA^HZ({?xp(_eSSzcWA*A@)k6KF*|LE2BZB{wR)@NhH{@iA zZ>;FdJ>}*9*4w|;+Ey#D?TvRv$Nm5xgN?TITcg>I9edn=`@d^Bh(s_eJ;Uc@ZTWxy zRzMB%H9Uz!^XI$om6cX-1B3VehYrJeard$!$VhMfTfyo0;0?&$RqG_$wa9FFuK()^ ztZ4866g(O&>L%tv=j;qheT`+7qYXiBvtN66yd3jVFZbQJo}6j-8{vOTtt5&7x3aJz zFjmx3m_kGlqQ-%H__^Y$TT8Xy{`!@xlCST+GG6=me8!U1Z#8M-{-58!@|~Q*{r>wP z#fUal>yB_@aMAnaX#T#Z^*%q$Tw@p|9vW2`@u82Eo^{=y8tUU63OD)p7nE_}&Pka5 z+7KvQPap*lB8uGj{A#LpSm6(Vo|iSl4*m%@jvALRSMQLnnf=c`9-#^Wcfv-$Torh- z*YNe(pObI-`PXRtf4rR=(HRunhlf;&^ ziPf{vj;qYU0^6n@0|L~uMLIeqYT)yuH5P~DehFB%_&$7Z(;c1U>f0SnvwBreFx}<9 zkEpC91*5%gIalXw_}WJ1)~9?$^g@eY#>W@Y(QLr15)2DFN z8Ci3}ljmy#Af@XX%4Rg%t+k)>tm@02>i;f1J{0es;@M7Tu|}Ni;e_F+yp+?3RV1^| zNSJt`%73n6yA4w#O5}0>%R=Rxq?l8Z#+K*{YyErr9UaNy@AN4{1XkDjEiVA9vwSFR zVIi01r|MpoX7?qPsXnn?ePYOeSB{Y#=QN6<9`aiM#4d*a;MGdJoF2z=IQ4&DZOdh$ zTH*)-w3cZJI&VEB#UoY=elOvm2CB0&b|X$|p{=9#+4*UiT@n>?!~f6G4-@pW=rZ0A zDpeEOfB5VO6UJ~oRxj0lUiogTLTpHSW#IaUE@5LebyPd}lDr4OUokY$S6<}`$DYNmxU1#fml)ubVVZG&!3YC-J>*EOc+2qK$ zFX1wI_6vQHZOp4k07d`z*6ERb;+W@#YO$T{_d(GIvmr?E%Aq3C*^9flDa(L}u^RDY zQYhHXDjFXNLRL{bpYJ_b@|N`3p02q2ikx1!;PL#snqj@4u85(3;3G@~P5s0xqvbwX zV}C2w+9Opm(x=eqzb``$uL!>{{?2OW&A zj|ioLVhFwgU)+eEw^3Q0a3}e2L}tm9b1Z_nk}i~4w6x6khWPSc6KSBMP`e0q(?YG7 zRAoVCV!TsSaMjH5?h5h7&bNH@g7>(to$Rb_!i@v2d%RR&rDu`S$BSW6mVW4I>&Tg* zjf-RR56{k(iI`Jidc})K+i4_+h80=B>E_EY5Cj#E zv;TfJqFnnzpeqCm;Y96p(>?J1KKTR75uYLBwVueXYW&9RN#_+QY#DEqp|US6%v)u# zi|TeD0C(>qUKO)N>Sr9=ZRoKVBAEBvH3Qd-%9Ku5xo)NH&;TIM6Acj0EK1Ykx8}bF zV}iOEy>XeW1hB+_@P^#*Kad91-d`Fzq0Hphjzzf1O|Qp_ zuP#m!HXwL$HQZ1UyJ!byiE6=*Br((^c(f>-&yO#a_nVi!*K>3fz#4&e7+u!akwlvY zgYF~S)&qAI+rQ;LWjQ)q_ex2@Z|rPK@q4CvBE}|o8y$2_%uF7EV%*hUK z#G3^0-weR9-t)%>B+q|~ZxrfFAtDPI;;wps^hILILRBettn^mE)IIlmFPw}M)zN0D zG_?36Tnv%WFlwo&4`+L%GlM2EF*N*`@tKmFDL%80r0muDsqYfL_2z_6N4EhWX}tLR zXQe6FD8jdghTmAtw?rV;gSu=PPp499nE>&eSVGQN9d+$vv{(;LI`_uuMzLe=6WRyW zUPCfGE^ntv?W21XU@F1NaBT4B0(5&c)@T%}275j1u&-|zRoBN;|K+ZiO*a>3Gwwpk z;!B*N4(yHPd&Z!{l?26SM>CEIOfoeNgIQa|NqiM7UYiC890gR54m3bJX0K$>)fznj zwyoDhjdCS>r)e&8u)&aPUhDd;2V=L`Y5{^i@4;o-9bZr4+hXM}=Sz_ZWCVVs%G)Ji!09&T)D}UtM-XGJ0m)j&N3Qw{p#Uo-`_M;g7pM%99ha6 z9e-BGXE;HjeMi#~hO?f*S>xwleZ4<1()8rJaqM)`5lUO@SZYXd(e6oC>U*#=|FRkw zrpyUdYi)gYr}MvFi5A%g;2$gETI(=q#Mp@$((}GjBeYpBtu)4y9~kPEq@^gkYZydb|v(#dQpdIM!GQ{e-;z8OkPPM1|l;iB;kv&M$hsbxB^uGT;7LrItwy zvg43Rhf^y*tQ6#rrfsLGenx(LcXFRL!28M6y1U-f6=p?iGGG!JC2&QPl_J6d*D3Dyr_g zZBJ~w3x(n9X?1lrc0=*_O|~EsJYp5Alf5;b9Ctj2`Yy4txJNo;!g1_obIvbwF%emh zS2vX?@LHR^zFE4r6THlSCYy3Klo|F@MwBT#M@4_;NZzWcop zV0(}qPfE0T4}nIHyBb&oHHFL1(%36w`0A2<1cjiEn6Dv1IzP^o@`AhKSwgN1(I@=8hGz$4}>Zu!4YeWrEAmGwn^Rket^=I z-6_>hx?NtS|He$B2JZrZSlP$X6bfe9IzVN%`bW?enz7a=zDciZL9e6l+a+klugJk}i&pr0}(YODCU0fE0O;Yq62$*v2gAC}*bCATE6REv(!Meb<-Wp;pX_ur@t1qKi3tV161dT*%`mBIA3PK z@diGgNZ<2#^0ld*Z08#8(QN&OxVz#+q8>$Z%geIfFuyq!qmo(I;V@FaOHx~d3MLX7 zoq2X~{`9wyhxkC6;P<~1-$I$F1vG_jfeA(jS37G3QeC{s1k7c|vqc z=u*ZofEDj^2&2g2lZ9+ML3&4_0v3&loMLi6Go<%PeZByGw&Yp^2-5e;=J;TGzy&jE zWH^}1)s#xquZ_I?{sj;8Qstos7`!x zu!@KZ$9}%b^Lq955`E|1E~DbJv!-u(e~woY@;!POjP|qZJ1Rxtet!5OyW8RrT?qm_=PW%XMN2l=pb?vjWT}}Hw`q5E1`OvAz75__!<;AS%w283)`9p`ZR2~HQqQ^fF6i1e;6;;o-$k&fjPiGo5^e+@1VzwQj>-fX4W5mA~YSw)Er)cNYRQ zmjRC8YELTMFJ0l@4Hm7~xTDmCo&>A>7l4L?^{<&)9>Ndr^l72Yz-np&y)Z#$onGKW zmXurT<9XTFZL#~=@eq3FJ7A43NqI1xEaMy%b1-2Tf`=F-7)^D~d&-^eUS;_nPS{mB z5%nS}sx=R(gbdvV7#AF@3N=kUxk-M^D=bo^o4iiOB2S;+{tSJ7ifqn`?Qm_RkXc1= zVChi*n65qs(*D=&{oy2d4WRm|9L4|MoqRG9|IT|ew2)WGnSLs*@ZdbQ*pg=7vURZqUIW@b0Lq4;OJ_WaN9r}*9NEH)fB zO4ZdM-TLUS%#hgF%9cIt*uuHwa0{z4p!=uNh3;pKyy+Xb)PF!$8QD>WhT-@wg#tjh zow!b?2pe2S;cL*=jinZ2dH7fG^Jbc2nefD5?5pj;M7E8p(62KrSIEjMc_%%304Cj58;{}@wA(g=c_U8Pkx3FvCxa!!96DGxFo1Mj9;6KzRH zzu8m)ZghAkS;!-1Ig8%lFs7K#n%hJCiR_4}5iEkKlS$U^_DEwMu+D%cBKAltQqlBl z#&#lRmFy)EDN^N)6M|i@)ozMvJ}3TUlxkG z7X4Jea-e=d{~xccw$IiL_vY^ zasYk%8IT+bKph`z@z#C&@b{EdSHPceyJQ4?b;SUf2nj~7O{zzH*8bzKB}YR@X0s$* zD8GJTguw6D0aXAPf=f5D2M0PMDL7ko8P~H+F<0(G_@69v$~D(aQ3@Kc}!D?rjHaFFox`F{AUw zTef8UjPV`p&uK5$5Fg6_1QPDMLhZ8@LFX@Ad7K98(kE+tKA(Bb80RTWVizs24Frw& z4NYQ<0WTB$y%Cy?@fz!;QmtxQXB3DUp&16x-5yHN1*I`a`pBNxdik-&ux+?&l8o%P z94|dUka27MEv2v3Vy&P5vn4`frwC z3z@>@LG$K}s^w^W!o4RQc(+V{sp3^7Dq3xQUh7sDz2qI=QV#&hJ4$W-%u2r&-rtx` zIG}Y*aUOr+A-QDKF%uq4&yA^xqeeN)^WxcSO{(yf?sPV>#kmlG90l?Bd7D>fBt>smRGU|cA;KUD zZ}rVDl@fJHvR$g%X!+eMb;=5up>My!Q%NVO>;%6xwD};9E&Cg5jm& zQee4TiCBEC@)d%F3)PrX{O$$y18|KJ4QDGjRa0=Op!^8Tda~B*){%CBiz6?ijj+P}5jz%{a!0s7dh1U*EdiYiUvML?a1?#$H_7 zu~AAY-jJ1^I8`Ej;98-ed?le&h_KrJdk=!Rpduv7ck&K3lw4z4c10G2+TYBa-L15k zftZCQ_Wkq&qG&@UOrHALJn(X9n`rbL%d7(qHbNBA=IgQ19Ykhkm*i2IjR3m3_jJxf zGm4Vy&d2Q*PC>>|bPORm6TNIeW)Cj;wU~Eesq$qpMS+6G34kT|6;B>~d-~Uhi#$K6 z)Z~d^`W$UeHVxh~ZH^`cNL_^NKJoH29(F}h0RzlZtnZ2n*NZ!PMg>6Z5&8+$y}Okr z;JnZx^Klw%x|XwHR>Ct$2KNX|syOFAE>i}VM*}{twKel7@=HZ)+zrsI)LMGlWvD4d zFkIi&-#+}@HxyOHKokWuOgjHPq12vI$I*6B^~U2 zz4)8HI2rWk=&eMnX!gA9lP#-yM&dvjMlQSw;2pj)e)FZ#W5w|Hjpw``t4?z0&5{&P zqjZ~pp9%{RMjo%FKftcrdboer!3jU$%9BE8HsC#P?7=0-?_udX2}AX+UFHWRfZ-J& zeAS8c;r@HBA>ixFlprgCbFGV$nNGkuh5ULDdr!)qvU(4f16 zu?Hztt*6)ehZrSYdG#DgdQIQ`3lGtYXm}tHOiu73_;RsKYj`8~50ND8YPZAN74)5_ zz1k0fU8`gPo|cVbRh=l2$Z1cD8ayp8iP7D$sY^9hokt}2kIzx)UYrrw8 z4GM6!p68Tpp~2*!Nx7KJKieOa$`D|FcDxjhr@j(0ZVE`}5s2S#U($U*Nnd8Ip(tVd zkJM!HQN45r$}U?}KN*`Vsy_Vvl46n~pknHnd$(BQ%CaAw0>rIlx99miOVfJSnZazS z>lTgfN82-HUO$f3`I*F>l`o+oqzuV+ZV%`;CC!{H)fh;0tWtVXxa@p^lhxq%b)n)e z{}0}nWZgrctxIJSX0e>}+OwT@q6E30jb~>6o6mKqL&S!G zu)9}fd1KBY&gG5caGnXpV7}tlQYJ%=z+noxZQ+#Lsodx+34Z5wVc;0I98Q`8jM)K9 zmXI;^$@j^^*DP{A+n46fuLYnwv&A>?+PR+pN3A3PVySi7>ba(Uc>nw43J@l;uG7ME zyJ^=S{GBv+^Uy7-TOZ86ZdSK3ZJY2p&aCjC)(yFQt!>JL%xwUW637>~YllgEv`3(c zFP$dJ3{w84GNOTFbH~*#^oE1_4}kN}KtwsLc+qnEu_aI`AjCKWu*$uN{IgAG>ATdj zKa{dI&EtRPKOk!lXJgs~{X;a&+%YuPxY(rH1|%S!vDW=hWprUj+KJMsEjFkce5-qe$*t9E`9*FLF`wq8+mjXd_R@({(}Z*3&Kixr09MBS7g32Tk5Sq=}+e zGfJi5=lf6n6Tvuf5n?3HJvgu5wntjy3u5y`rQYW(C-$;P?$w*uO1Xz~%@dJ1l4R%{2=!xW;g&U9@ z4?1Qg(QhAuEMj=F7;Lv-4L!|wIoVbtl;^*c8@>c2@&ZukmEHuN0{pzTGV6TbWnuyO zL3)G2OHmIC>EZ|`;2r8X%U0qP#xRT6pkaUEpDXTm($glHe)$~)Xqu*6XoP^?2B3@TMbDm6^W|5^OB02E1&ZdBt zS&6@>od=<)K_7c0(;m=TU0Qc0E4%AK#`1-xrd9mm#73&2-e`OcS{nWig!(sW!$gpJ zj$m*`jcUL;EQ1aZaa-|{|D@`BYQb%^IsD|Z8wK+DQu`wE$Ub9bLJTU-`s8OiuK`|R za%L>aXKPe&vcTtni0@#CAaW+8jtd?h0-CIrgZpNYxmujz&i|m1Ov7rd6C&-Y}9JB0#z--2%ge2tW-3Vena8xS?Kl1|s% zqITlPLEyGF`5qsjZcGFK*#6#^!~wj&m|j!h_)EG@2PZmxOyw)|LB;(G_SoQMN^Axp zf_;xM_-|dvngMFr+3{wcLxL_r7TB<$`#rmMi#Y~4jRrm`u+O)SIrrRvl=hjir;9nE znbf&-*oyFlJr;V@Yk^ysMB6Q)dUkY6hscH@QF7rI{^V!#W}im@#-|;JbLG;HC;5$E zJB`1%@@M<@vtQjl~P{=4dsr)+1|q0?B@3!8XZ93qF zQ(sCUH6r~k>ZI8VjW%lLHu8+hvuw|KwDJxxyR;ZpR(OZ;O{H(_c2PY5oYhW4zy_@Y zbD0|)c}=;Qa**Oozu+}|188jY5ZGne#zKlzSjaej!md%zXe;Gk`L}A71R+Xrw({QC zq+#7^n`luL+${I%($?o!->0hFcBRNyvVe@)t9pZs0nzjKjxY*^KI#sZdJwWG+(tI7 z^DqYJ#GWk3ED3wvP4WDWpko>`jDz+c%5CjAp3UA}<2lR_F zcpvTu(1yROWzZkV^HaHrq4E-UhHc{yO_L!AO9H4Z0w;}A7qXqk>wV#|(&Jma5`W)H zZm)3@?k>Bg-~Qo+80x?r114{PvdH=I&OWedyM&;tt=4+6E#xFB+9^f#c(FaK8liJV zWBrJWu3s6ln^-I<2gv6klBRU&`dE>;lKAOO_wVIs!=YWT?wTu1=Ij+x*+T$ZNnb+U>SVIx5jVK zW(=B52v;c(t;1zo!9UR%Zw?Qqpje))7hV*s;lQS6F888D(Y6a5DbxETYeIGr`Ajg4 zzeJcTrvTXjZKk+L(9-2cw)r^%vP13?uI5qo#?Cn7PH?bqQ3SatX7DmVibg277fP|* z3S*idE!#gwtqSWo!pm=nqnQZqVBZVg^y!UfHosirb-2}?^BHi@AVfcl5fjkkA7WJ& z0w&${*ObQ`*$dSdXUw2woW-Vd?5gdzResM~| z21xrg+=~9F{s&Hy=LQt|KnDVoQ+ndsUEm7UTqiG!p}YR<(yG1w;L@G$VEYR?+w{_2 zuf7c=Vj~2N^z`rm?FFG0W_4#;1IYIF8jc4^CY$=u0CB>YMw&}DewAI`B3|LtX|FAf z|K3oFN8VL@5V5BuKJma`c9pZBlR9m4eu|efb`KKHiuXaT=Hqs>Sel3B} zG*|8BYHkO7V&*x@+oaYZu6c!hnx})qjthviWn31cg21EgZS+VntrJ-`=H^H~X!7XA z5F!M<{f&&J<==n{4Q7boH_3#lBf^J269esfz9gDIQ?6au(>w;g#I{53@zxAH%L zTuWpm_PSi+<%nHTjL`B-0k!tI65PsoXF?IJ_4DCGSx8&_Z{EbMm_{BfL6XS(oeokp z;YIr{bluVABLSZa-{6jxeVTTZXD+GI!du@*F-4Qm#CqTT^+6!92O@%%*UB%$dlGMm zt0|HsQD{7Y9Kty>m#h-v%N+Px>hT;_nZ%7%1^w-oE`pHOz3ZbB!8AT-=b_!<(l~6g zsil1D5Mf-BX$j@;pf7dK@!XJcs}e=@#`JNA=!Tvdq_ycxhSgrvbJewS`g(O4C4P2- z^#0`}0|+gb#0q1B)@cr;MKzOX`ND1}X-XG}up?f;e0vfrwrh7OI-FR;K1)9O(zaL0a-Rx?v}fnmw*L&UZw+fhum(_k24 z)tc;T-T0?d?7qHc?Z>7DX%=B@&SYD@hwDfIRNpqO1LpJnxXG$BdY!5XKhq>huHZ*k z9>H&SX)c=deA<%u+b~}jEp%k~KKb4Te^m=5QIGcXYbSw|#f-~}cn%^|IE327EjF!X zCi~VX7Qxl0F`3+BMHVC7C_zd<)t36qDr?Ex*nPoVi3tv*VZB;0S}D4V1idYEh4{pX z8qq`GPMzDL&iHP#YikI{4MD8EBgvVC)xjN8Vs9D9gSc4Hb z`^(=KSfs?LUlG#&iRx@W=IV)Fqg}DSx>c8h)SN@9L@i>mxNo`bgF}+w#8woJll#>|^^HRq4Mh3L5JvrKw9DAlrRN$CoD{A~ zx?-WXOyYQ&m?afA@RC~D4^{nWN3XUYu$2ovv|cCVMpKr%*-$UmTJFZH#JHk{kWQEAABIT^XQ@&L)9mwB;AKwl0@8PeZ zMCru2%5sugdR4BHN@opys{A&X9f=pr-sX__T2m{eHeO{tERzd|8G%+wrFkD5sI_yv zF?W#QL-Qe^l5xSG9z&-XTtkpoX{HPQNRJf(dn2Y;Nb*BYu?VY|o4QAP#ZRHl^?r-( zG}?A%1N;n_0a>8MBh)bWf?S7N-N%exO-;gO^C6zmx7P(P&6Xkr?zUMtOHsh@pG<6~ zaY$J;sVuOb-fB<1KgKf^yR39w2~2Vv%<4v`NBod*bPFX{i!>=nL*9SuG>XSBxGFu( zP#r~f;9KyqOk>n`pTJQZ>5e#(7+JX^Lj$cF?2FnVF$H1GLqD~t@CTQ{W9Iv5EOOww ztVD@cjJ$?^TpiXDIEEr5nGJQHq0%Lk{n%rPmL}{I+`?eVheXQBeBc34A|l_SbK86( z#JPsw!P$a3sngjx*7&z*8{`j2DOK4^Zvd);B+}#&(|%4H#lz?X`_tKylACEe-@AY; zXLk|57>lNw3qc#al0{bFVC1XZAGd=26NM2ZuAf4ynOh|PxQeg#ZJVx}JfEKOG=neb z1oOBbzBYTi1rQ(aNw{K@SQh$h!15!5{*j45v1=EDu1|mL3)}tv0fu0~n={2T24N82~$8Bxg<9s;Kzejy@f z!uDQyXs!@LNKkKeW63;roM8a|ff<#FGWT@>E0~5G&p_{4D@Df|)lI06zE35rsDevPmWsVZPZz@|egdmeuVU){Q)Xt7+-VDBN<>@( z5`PKN|56wo9ZVm=(H4nGxd0l(6lLOCm@-AXsp_52;z0gYdJ?Ad3im3R_lX|`>aR`T zLuJZYIG#;ezk6iEG4sLu>V3=YFZsOCs;HS1M8O-M*VQkEJ*2~DCEEFvy{@$mOoMR-KSZ4OeA`mYzQ zw$s;X*(RAM*?M7&`GI)2@}-cz_`baz>cb6k5nAg`dVLM5b|z(61Q&uN-otL&H<9G5 z`r&@XPH*xcY3EYN&S7e?g}oAkuEkbO55wVx*<%_k#BCW^;xd6YC73$+83Wx521fcW zbdA;lgW5Z*mW*(tMI4|xehDGPc=fGuj1nEMr~c>)Bx?E1F~WS$ebUXQWC0E1TyXy{T#R43v>!V}eR4!Brc1eNPlM&Ytw3(G7L0R4c~w zii`Ic>K;*B4`F-t2E_A9h#`P{2+7P_Px>YIA#&OoAzMaqOo7y&rk|)yC5etLwS){* zjrGu7!;iJ7V5n%mdh$Kb?YmWK?E?5NWGHJ6v5xy9{1GQKH5JWcs9iPN>Ou5LWGV3- ziaI5ebcEy{n1S!K7f(!)dF~fZu26iw*GfNvYzxggFX!9B+oE>}yd=iwROy$Q)? zTcGVy%5-AC-UcUFV=?Di;I}gY(c9(d`lzp0tnv%JzjO)>ecABIw#;Z>Pp58TVh>>3 z6qs=1aV5iHRL}YEalg2ME`Kz-F~<04%+ulN9ZD2li)hZ*#8T0>iSm^1X_jszRR|Xd zj`in|7!`{q;Ij6m<3^~jtou``mRMwoGC~!qX}bhbR!r49!UEAArA02}Wrd+}*ccz<(l3#%Gk%u}%#{H>#wt2e)d!M{ioqS#R_MzJlbggs7| z{uECxqTlN7)0gNXt(Y48fj!M=5{0tH^vhuzKO&l2k8&^9-kH`~`wPh}d6`eSqk^EL66MvRYsimU%QD9RY;^zMy^HOsHc1*-WyMfZ`#ot7ec{`0b((|R_G5((E# zW01o4%?`kj31zWSKF5t@0&EB;eBANN&P;a|NvRMIt0-0N?pUN`I^~|M?OK=SB=O@P zX}wRBEv^Y9_^8BwD{%E9x`5ZdTV(yUppxotoHyGUms zpz)-jLIoH7Rn)v^qv#RAGRsvVO*cY6SR%Uj2YgoO_3xR6CHwi8xAXNU3QMZHb|`wq z!*`|6Yn)%$`;)0v7!5^0P839rRqCRx;yH5omxk^tI>}ra6NnopU>?0cLR->$2&Wem z>?n>FV}Fs9B``4L8bYrXD%=+n^cN8m>69w4=)ur=sum1;m&1jnr(Ee5PBA)jaTi8= z7j?}0e}9unLunbRk1-fGOvOxwzafP_o^FZQk9W}EFph?1dal6zTg2S+U=ZWA81BB^y%L(t?Pnv| zOAu@hHoSKlv6~X~Xv3L+sWM70;v1fR;#49U$xh&TZ-PXY8Ebt!-EH$O0+lPwV34AA zNiukSRA!_!b#Zl7WKgeCbr5Q`7i*iRc*4Wvu||@q<;+bzIHq+#PU6tD)?4@IE2E-< zuZ8Z6y+f{#sJ1*Bq7lfOvha!vJf0KD5_2SXBo-}b;(B)y!lX zK^JHy&p!Z^cUr|?i46A%74@$^Df`A#5TiL}@_!-Gk>n3h6csJdAB9)7;JA7w*& zHOc8ZUADL8OYbXCjh3h;`lLMsQrO5$4JSW& zIybVu`A(##iZq8unR$ay(~-RI%^|_i5}fzO2O_0+k%p2dZxEWa$Nx&hzqT08^*Ct* zBCjNQiwMHO^a6ysJ(|KkziKVYgv{!&g1caf!Sk+`gN(={VuzxpC7PKTwi+cDi6q){ z7tBg&6#6QXP*g-?w$KG0OqQ!!h7bzb8FoU7gk?#4*o4&XJg`M9P?V`ukR*kRYdlol zDaZS*q}7%Y}(MxzYIo=k@Su+?AoYUjHDsO7U0do(8b_2;H*8;0@K^a~OCF>Sm{v>IIn ztKV7OMhq{g!m1WZ(=D~n6!sLU_qat5nEcg8lW9)$OVq*9iD#wl8g$-_|VLHB^=T`ACd9`h9lMx*F(Cs{S!es9ND$;LyX7F$b>#bVP zqQYMt)DfArhL|o%7=SnKD$pOdvMp4zS9+IyC?Qey*_rQ~ zvginU`Iu2Ij)u6>-}16tYNDnc`$tflId!vx2R0b5Jk7?oc7+5H>;{qtZBO>t!#YCz zdbr-vOh$X{2?yJ6kO9>Kmr%tl=5%JJ)^VQ^GhlGbQAI9AE+^mLoK z6z{__wy;O=y-2n>!Ps*=8@4ES580Ne>f$65yqxnKZPIp?A)1n>c5m`T?W0XsBC^6i zB_mjcW;EzK*KIO8*^AIc;K0jjMS-JCfT2!_>2M0eK~@*tYE%FS84j74DJrnmYDJW; z)%(w=Os|2wfvzLs^;R99Y_Pc1AP^R@Aa=v=7jPCDKOQ^cp~YVfWuwgoxft8_O!&9)%G)ZF zvHdALujB;?A_mG6EC>ydro;N**(E<#CGqB=o+*QIhe8s%m%Lx3-s^Jvffy{)c_97z zR2IExB$ z`0n)VyX0ne@r31po9MXLu)zk8nq*w?{f5WP-=y~nB0d*<^Z|p|K`U`m>((ks-p)u# z(bdY1Gqp7*f*+)NY?KkVM~FyYCumcNd7?}D&IX=@yeapaixSi&Sp^n^^M>UB8%TU9 z6MPR@Q$Qe!+dpbLXK(WU%rWYL7GVP*Rq+=rO~xbz!0< zldTB~M{iB`y`7J|i9ls()OVs`L=ZWH19OL17TojrcXB~r(A(9$`b!wDU36?j709`g zZxR*%k$(XOD&;W(Zq7*rvcdA438qQn_lXvT)jOWVPDYJpKGz&1^}ux>B{KcU8fTF8 z>oe64VY^B3-d^I0=QH+S<6w3q)F<7^8Tm@`EaekF@58O{s;id=5cAJX^dHVFEqw^) zLiSLb&=k&+eNnMEyc{h-=ddIcn!SDaX#SmqkZAOFgK8wXUF)qzF2`%yIAloWO;e$=ar?qOFo$!7VJy$ks%`?(}M>zg2e{ z`Rz*JP0ts{uQ%ZCv=4_GmH3`sy+QlegWCcr;F3Mtr^tal0f#TqslC~}VMkG=@ddYG*LR#&7*Fy_`S6F; z0LFP5jL*=QxlHo^UZiqeBbm@udyd?*j-mG+e1cWx)}B}bF}kLj`Nm-KG-)UY&6>?u zWgS#%wLq`HM+fwKitx{N+;i@oSJdh!(E;e;gd)T+G4;a)PhGeww^6@y5pn*U1PW)1U+UX93=f*$IC zhjh5YF$1O+hnwU1o+1mtdAB%=y~5t>Q&*rmn3JN-s;a%Ht8TP~F<89KZ@S*I)Xqty z>g(z(fbS9&c#$vbNY{>vHV5oMQG`xr3U?)?oh-2HuiRoAR0vq^ic+Ih8k+xrMw7i=UVfGM>;#LthAQJM zs>7WXkrwt|)G|fFNr9zVi+}yDs|Ewu-j5suJ26 z*(32fT;oqcDY!hua@) zjGF#pyr%`r2g9s#ZMP8Wg}X{P4IBm@~M0I~NXI24;5N5EUltj4GV| z;;O2tvH&MptO(}vP=IcAZ~rQzlAio-7jR+`U!k3wpGdOO*;Y|P!BMGn^mlr?J>7gXc5=V{U>24RAT(=Q49I&{sPhJX6g=N1z6y;IvJunbYYL z5L{?M>{uU^uY`_If;=_d%%)u;lB9?VK`r_{JL9=>60LniHy_!xf3!S*1oWtGR{i9f&^zV<&{FY19S-}}EC=pZJ@9S6KVb9Q% z*(4Iov8$T<;4GYZofDYX0r72YPrLP3;yMgJ-mQ45a?l_dC=)~{%PL9w= z^;28T-ZV3eQZ*T;(Pf%EI&Y(>MT=_@`aEZ>Ine|-{{i?xmkbYVfC zd4!>^(e}q1LgV21o9$slws_n7NOWz5$UQ%(-<3hsEXCX7V-7*>AgcHIl+H;+3d*J6 z&oosxe^c6(^z7o+Z(PYns*( z><#q&IoDLqDo=)8&}VrF{yBoWM-J08oL<-l<8WXvsIA)o2D|1%XIb0oB;gCis+W(cY;`pfV8|AJ)@zP?AbEqd3i4nJ;hwm=b% zxrvq^IMe?Nk^>hWM(FW z)B*SR!V^<5wZtFmYwQQidaR6nw6;`Ez-gA61&MbEepu>nFf_{Hk}(=<$CWqu84R?6 z;eqtS%FT$&altIRzC`wgSX>3ircavpK&kqwz#JG~r1x}FBYQUs#@NE`>9<~veo~2b zqd1trY!P>@BlGp8DN>-#_3vPks7)?)$pl*Zcjtj7&3J`vZcC zvA{(fDs??S$bU<;A6(y+lG#yZskECDII5g7)?-iE{N;9H{reTC55NbKa?ickaFHB& z!<3D+A1*EM!^-y|diwZP>?hlY-^_gH+}~<+_g=t8?_W(lKHDy-Fd=%E=EM*&)l02X zTm0JMEq1>1(TxS&c(IT1twXX#<(lkH>-N_-F*RdyM?V!ugp}mY5T`5aaTt_cIaDr} z$!F-N$wILL>rG$D0g0f>!XPP}=fCpzS0|%9*7mJlg}=G0r%T!1y<~pEph?1a(|JQ8 zFHw4Rs?D*P%EL>(V;8upxZZH6gefu1D+c^n#4acs!LL@Wwd?hw7lLpJ%TP_>sK9ZV zxwbO54WgC*BwWOe<=qg6^@Jw%u80NoS8UL<2Zc##EeQ#X+!nnfh2m{&Ya87}?#~m& z-VT`?!IRz`Y}#vOdiJr)F+L>kl`Y2%q#qPvY@owMe>U^I`PTK%QuRvE&%dQzrf zJ8!NCz1}oIxrIyFVY>4wte10rUyp!CpI+8L#=+bqge=NFUKUaWGHe z_bEeGXu>D)|2h6CAhXT#`N?@fqkW^bFNJ)pdw+J7Tj2g|?E?7X53rs4Sy+)SmDz7q z`#vc@Q)2{W2py8xsWmDiUhzKhQ2SUJwDjm+ z(&>Z7j|RQDddc(4n4j6^Ufl*c$YPE3fdYh&kO)X4925B1H_MDAlgm)_@QuCY{bgG$ z#Hp0oKXUKhkl2SSKWXGuBG#peE$TYnK#Q8DvQuhPWhJhBW*e0I)4P-eIszm?RVJV@ zsY|6y=>vll{TaOzLykCXSwxFK{qQkh#FRHSNHLlSnWGaezMANEzJB!-0pDSI-APET zG?|a3J?xp`eP*M+=v6QQX257WQQCGW>J&g@e6AwC17jHp_VFZcq)*o6k@^n|q**OK zTG7T;!iD;>`0vk^#1BBk-!Z5$yiZGT`2LSr-Qaore4i3ck94~N4~#IuH`;k>qa8ZZnF$~3x;E5Ck=n~&+yd7MW^tqx-Bim=Y^O3 z@4dTlh$6;P4xPJZ8gBuPM6;b~pS3C-<|N#=?bn5tg|H|RN1gusNh!ugCL<}$uJ2N` zLQxF$O=UC3>F7V-6Y{o=ARCwudo{=L=mCa5nY@5Rw54Eo;Z3D}Nyn#wJ7K%PAb+PO z{G&O`qlD*WFfucW7e5C{!yR8o+Dgd(MH^m&T(EHjww=imqL~st;rpFYvmfG*@9;kc zp2U3(m>9l=iTO+O@zr@*DjVk{UKt50$Vw@LlgFn@18oR`8{hryfLcbU_usYQFM^`0 zR|BN1oHOBd?-?w(*=cuszO-5l=f`GBAh=zzbUYnN-Ty#mG^K4|zi?@}raJ_S5BEiMt}wD`;n0WRz`Qp6V> z$bbkqZqcX`l*XSWAqY~fvuHpX8HN&gl{Del7(6`Di9P%O=)De&S}Rny!)ax?bC^vDZiL)UQ!e+@o)%U& ztjvxMdjiIy-oU6l6}>mfa(9tan;RT6J;YOjx4oa4*r?3oS15#w*uJG}ci&D)W>lsn zeOegb?_HYcZo^PYh$FwiUGmBkwqP}NhSVV8P`39Mn=Fb1)9Iyj9wRtcP4_p~QECmn zf*nQ~dTd9!m}nFS4=+jGM0@OPo*%)?Yuk<%P|D2uWIct`;58K4 zwRl?2cJLvTfqM(}?XG`qGMs}kT0bW_op?t<9_ll1j*uYwL)6F=1a8Az(U>TScc7N- z0A1b)!a1i|^?h1knBU)2tBd{hgWHt8=Oe$aLudL=(KX&q9wt{f_%$ zX~v=71^n+L7epqR@IDDF7%cZ6h#d|FhNQ`+b56cBq9?jlp9^V!KsTwAuLf8B_yLe` zh@J?2xJ0W1)D)a5q4ir3izuf(=jSfq4)z1(>E;}U-Js<8Pj8Vz?LU&r1olS4MZmCM z54^MGpmxzx8(GG&ur9%^o&bxek#6F)2Z-fyO*Img#WyhcqgZG}KJ3m^6B;t6QZq^Q z6&BD7nPM{M`fRY*NU5Y76Gy$BJAp$Jxc0Igo8V6$?`MlxN^uQ1q7#kHVwQ%su@8X> zmz4B`Bwym?KO8g$!3q1OBO@%wYSQuW@D5rgdsWhDz(X;FIb2!adeKp>(SE zkQIKh;u|y~Sh=6M+OI~~;u)XGHy}ht!+VSp=Om8tQnZR2Nr*XBU$`s9nh7A3C9Tb7_Uj;y7mnh`W=S1WNGd@#bV zd7vAu%q$p`LomyF>8(+5dWIM$`!i|tQM_IOfP#z6-9cHbP9~qI{rk{?BqrOApm&$M zFF_!rFpr2yfOS3c36*-gNgJKdD8?NC7Bx%6-ukS20devMF67eWb%I`a4Z zSRfr+L58@%aeJ5Uny*hK}W zk$UyqBZ4cPQHT%m)j5NHq6*;_nAAEG$Z>Q*Y901T<;IM@cV`}6Hmaz)xpbiX=+`4A za&MkjM2LgTI||i6xxFL*y&J@uI|3bNAYHXiuXkxO_j+!+!L1mGHYup`z%vmwDuwi9 z*_3&x$-`Nh43VwcgXD_!AV2tq)W5^0#Xjev9MQ<)(2l)tei{? zyZUpr#dVo3DJ#GTN?=rA`fld}NDTmRbKP}jJO~$<6Zqb_7Z)-I-tEQ`RA77)E@I7o zkgxwDyCG(SB~F8uzK5;3_LXGTe#VNzo^2Fa8KYcz*S`OB0f8}PG9V7!Q#X@}tQ(^4 z8_5|Hgz;H&1u`0*mButJZ&~tmc%krioiZa$p>Ue7 zQontJoXA$g9W`CFJDmx{P;~35-afMyzDaaz+f|e8=Byu|iob{)k5FXUz#jO$MfNpM6o4mpjO_d?^L=G z7-I*jNHN(7O#%mn3A>fS|FmNERSp{YO)zZ}k$WhJPq1&mL+}8|re3W?!oPWq^{3f} z(7C30{TUHE4@UVC9OLLoJd(KIS2gtc$5>jJ6Tc3~vVhQu!<|*JAwc0tr)@|?s>p7B z0x^ZhAY1Zu<;;)m10 zF+>=fgEVbkvN+$;At&CMH}^hzzrP@E@C1Fei2%sKFI!3Upzs%*2%SQZQm zjrRPQ%sd(ID`5oM+d=6yr#WjS zr^veP8@h7M?H}_RPCn~ZFa~kEB=5>dTbbLsidf>TRBfOOKdhBQ*tnM6NTDbl39gfP z>FN*L+2c#zOT|-8HPSZlSHt*!dKdB=sFE|U?Y|55_251!OL4Eacr~NNG3E^h9_9Px zH}8mP%IHWK$gMyE@`vb*Ur?Y^P2S1flB&#@Z9{W+T&IgMOVVT0G;GC-*SbO>zR zU76YOebIawk8A~p8uM4TxsJutzO`*k9C4u?sn-y(!7B<+uWxwh-7tN@B^7YVe{eYY zM>mKliXZ;3HMhBfFE9K$mujGAPBt#o*M113DC)vZz?8}EwFS^XTp*=6xHnl;!b07} z8}1Xx+5bE0rB+@exIS5nvAVvdL<5Qg(BcpO{bGUVnQRrf%z1{>p%b?WxP7!oy0UTG zclXQFWiYHMCCX_h;#9t5RH6y8lvCq~;x!kFcTY@Tvp1|do~_1O-gV z;JnFtC$Hpp4-=MF4gaO=_r)kGh5LOtxSL{?g*1MfBWLgJ@rljuwg=Nlxx2M142Slq zJ7(viE!vP`!OlXYHRi^_mvW5s(0c^6oE_%g>)0lAGC_7{KHLqOh~iu%6G`cNN8YM$ zb(%u2O>~cS3Ueb*Xsx|RB++2|>UPvMxOa#*-3HW@*qg7E5{f_&R-~pX z#h-9(R<*3R+bwxbuQ^LH4CWVB{R-;;=%0~l27CrgKem>JrliXYfVjOkY&IFPpN9uo zyw8RWjs0MRhvu@y2&j6!5Fk!TJp{hnUY&?BQ;`}rg@9rP9&M^-@Ya=5}R>_EI zjOvrNRkLq+h@VgK;?#S;j}DFJT3%JWz(O~l0#Bti?En-m@Xy^q9led`$Pwv7+68%z z#tyOdJhf4yOc(UnI z0phwL3*RcsV8QdplAqJ_6ltF~Bq}@?9{W;aCFvE9=IquxG5trE2oKlS_H`FdE$Ed` z{yJhSWM3cuo-K|@@7QFN@|~*{R)Gsj@!~uG!n%tSISQk__Z?Lfj=HNKfH&pv^1X`| ztp|K&Y`4SwJ9q5W#^|%V-@Q?{kmEW-+K+%h{+|n7R2d)T4p;(t_%1a(!na@w!h=sqwH~CK$T-3S zWvf3!*`YcL?ciN!vuSiDE%Mthe1)ZTvcIGG|F=6DPs+EzjMgzs4zyc9{gy+W ztv;v$Zdjbrst5Sp!@giH{jZ=}-F_qfwOq4rp@&wStf*H{)hQK^OeYA1Kh!rXx&rwz zAK%71WqZ`j=RrY(ylbcNHyt2Eoj|y{&f6;BoucU4L;^-=*N(0!HJmSDQb=IKY7I|L*Y^77C?k5gx_sH0A0`<$13@N`U zJ3wVA7af&pL}cBNzK*tlt2Xlp)Khk#>3sr?qzM;!s=ql|qT$_faI=;qO{%_lh&v5dPM=jIb+dW-&MvzzNn*o!>F6fM^Wv%ZfeAz5FKJ z#p6ZaK|!p)mcMUP9EyTgi0GDMa?h_SfK01?a&32`yXn28)GSfzp`iZk z$M;leq*RZ4cBkLeLJU2VFqMI{O*Me5(;^hv4=^90Z`kSAvpHd61gtCj%GCsO0G^N! z*t%j~%q`i`CC+uLlYPjYw|k{~>J4$@*eMX5l4hk-;gOWjEe&&Lwjv=?oYo^2X(e z2z7d|F-sXBU;4td44;o=xy(TV3nd%~9BBXZSQE)YIB=~F2se0!f12+eN$}j zjBU)mw&!rnw~`_9n19|@Az(Fy5R()!0r~MOAn)`%FfW>|4~}EhqHWL7eVQo>{l>S; zq(OV-pxwpK;*WVII4XAj*byt1ULY&^YehG5d7f&R+qeX9O#rAYq}pg6Cb|5j*IMJs z#90exc_ANmbw0mkRu175MKXoDGsyieChh8}0?KdohJyXtm(*m&rQ-7-pGMbj8ae6; zo`r5=(KnIql7U~F+}1w86<=r`KR5cX%pRjpQ25pekIeJ-*Fkt;F*Z#Pcb>{HPg(!! z0C&z$!LPikdhwd$@i<5}UN>)KDGrQ09goh`x6}dMKnl)T&RnCL_%j=XRU_5r$JW0! zX3+57Hm$SCeL5ob-XUk(&%$*JP@TgID{PX_P- zINJe;UUw$)vD=im$c|D%Lp;AFTg;!-p$?T!2KD3{%xQam~S9z z>EnFdP0Ph6_>{oy`W;B$&s6=+=cR1?9ove0{fOQnlW|OQR{9-*I!U7amzk+YJ1?FU z%FvhexnYzh z^|OE<^NHzLN2S0IdThB+>kbCQpigG4as4F(;%q_j|3J08sLu?49Z$Engq7M}1s7*e zR@aEd94O~Bx>E)pmM--LZ$00}TkTizIK=5o-f&xJ+yGk5bgku$FmP67-hG}kTl+O7 z&j*AAHvF#u0f7xn>8g`4B3kAywcf_$58eXE4bMvyv-W)gpYbx-rzyaMfBr)GmHNI) z`sFu({Aa{Uh8iaJVT!2Z-wFpUMPh zF3U4HMNlG0_M(5X=~K=tju$5jnx!=t_Mfj)rv$`h6fM*oImkSoCF{#K0}MsQwotnI zODvC%P$In}T;2Sh@q6cC8rQ$qzxQZVkjsehkH%u7pP57}t2}u`u3TzmgAYhgI$E!b z97gwdpP(~MD4Y_)Q&ZtTTmc0_`|aQ}+2r8g{4~6!4^0L|8#W2PQHUvqEun~~Y^Oz= zL?drGrdl3~?5@}N*r7ZA@hafy(nxV*I~LYWP(E_y-^B7o+B&D_N7{J@Y+*sWQ`#KK z>HA>VnWTX(f0X9{yXmxU^!APj!G5q|VncibDgOKp2otBW-|}=E$!ExZ?76)1r6bSx z;gl#fhcc@gBTInsMCG(WAIES0{Wl&TfCBT4CQ3{$tT^)=;7BM-+tJ37BI?*Zy9&MB z@^^roNE=8b!VG}U#qIy@gmL-^a34b#H-5J@R^pr^EfsVE(|rL_Fy}6dIKZiWZN#}u zhttwZCMgvt{b=7}=ivif{_)V+Ga3X*0S-(m{z+@0z1Du{biuf!PRg1TcCi`nQaS_C zKU^mYNOeFn%JH0}&Quti$sYPCMd<_`D}I_i#T(78O|o2jqnlCkj@3Y2Df%nYlc zdcb!p6LaT-1TGJfRRJyWedr@FGGIr##n|}vC)We3bIH#qO95lRY+e2f?Vd~ zp+kBJbrHN(anuaFwj?TM9QWm!E7%Y5x_Y8TGdxNIDqj z`p&Ar>Nr>Yr&Y9+sPr8xaEu)P$n%rTcqqT0wjBKT>f%vr@xI#XY~0>RxG$w|3hb8^0G(DtI3O-kIP~Nm1ukC2lV+V0;b3`n&dorYPrT@^#`O#vHJr{p{M%gPKX9PBX2vOu%iT+fDBdg$!0yPB2`v$9&ms4|7_>DxQ8Jt+q^3ci#9Zc@m z*yJ%c5r2-suk@I!i{X}XDjgWpBajH(H7!t-%PApwyBtY=^wY=BD1YcWR}>*9vh>jg z*XCQ$g-IyGNPX{vrH{Q9tle4ossi{{7q{xROzNY$sl+?@)*nGWadk2e?!b02flIM=Qk@G2{;&GPiO9NN@Qu!mgQ4ZoFZ0vKS9O*SgkiAp;Nuq_9z+OEfrIcIb7vT$f zTqLpEjbSKe-k?Wt;$=`c9(QnR#fC*gR5P-@i;N~el1n3&&hitE49$uT2%icuPZ4`} zHVr^DZy2=GoF5~|C%DdKFwCCaC6NsPXiQea1KOGo_Mn&xfCd|+2cW)aV%^t7e1Q#O z_iO7Ruv3_fj=!^eA1WW;TYqb8gn(-odKq;Lx6}uq_@@FcPwJ1<$51`veXE#*;W(F#zd|o>>JxqzDe@pEO9!<^ zotiIA*6757vjRqiMhy14ymO;ZI*+e{syI;V-q++CC?bYhjn3#zy z>HDk%5Sp;f+Lb z8~p@s@Xn_DQ>9?5NS}i5h14;@YimIL_HkUM`O7z;2$BF;93A%8XU0zOD6H1NJih|M z|)jc^@9q_f(JwM?phP=Z?SQ4M;-Ou$#x5mGA^MY|mXeXu2k+x_? z!$qZvI29FyXn6;5-6NVj?u5pc4Vsd&EaJ!a(-*N^FUJq{op*09mfC#cHe5tj!h=N0 zJqIU*@8(xzl?{Za3*5vkLBaP=0=AF*tSdVadqiQu0Wh6qjh_TAd!&f`OOOmfZ#Sz_ z_6H2cRRe^c>jn(sQ@-Ra<2ES4!(6VE(L!&T2g-V--(|m4U8}5Z#JWv^13c%G+av3z zQ&=V=?Rx({xOSHn>R@ki|GKcgO;jj2mfAwarcmrZ6nYJ#;>{&^?XkAk1 zF2@Bzn(m}^`wW;_A;*#oWD~>?Gc1p`c*9+&?fShLsOY^qf+U{=o|S9z`*5Ht@`1A! zPxi0gX0O0`Z_d}wQH9_0hm65q7#pX&xF%c$*o{GMpt8Z}M)RWIKwl=%g0Z1(r7z{N zg?_5hnffFJtM}bkRvT}8r8#O7l!gCPYpG^{pzf%_IBZV|1oA0Zb8+fePZJjtG=-cZ zw-8xPvXhxXbQ7+-YbyjJ@uTAsIF}-pJawmWFS_aCw-PqkJ5*5G;*;c z^mvr1^M=tqGq?X}c+Oy7+pAb<&?J|V`#)8(O8F12R6+VF1lXh@KqErVoQ{RoCs@tZgx#hups+u}79Hz4Sl&o{|+<%q11NeKd9adu(Jvi}i7Ab$SMC zQZ)YwGQr`olixS_I1N~=r26Zrb%ph24AdE}Ys_fLH?bZC=moobE?WRSA8pw}9^D3V zh#hZcC11eoCG?DkIbF%RmjZgA`|If$pE@ktU$PK{is(0FF0XLNUgIaea&r|-Lf!+W zhF3hCft4qXw!~oKr_tz0Kd=xFbdEpP{iX4gv5G&iw~kWCNPn->nRsh3Kzn9=8H@Wg!j1c5V*C08nI|2F&*QyR&9Q(!#0 z_T=yztm$}iFEcu)l_Kztc|ZLYtKJJaZYH?oG6>W-($K6HW4mP2wSZ}1PH)uO3KdmK z=wqZwpgTf$qTqY!vb6}ZV6@!H*Dky^nV*N>zi*7<$dM78Vx(^?N{svc$1ArB%3$3; zK{JwI$hSAmEN<8tkG{0Hh{vZB=lvUUudyUSBOtBlW{TOjZ`t##twTgv`V{?0A$`|HB{1Vx*W;00Q_ho9&L{iI*+V~PBh;1-Vv7m!4Lie zuNvw#jC3SMIh%L^A)iL05)*t+3z}3AEl5m?>d9+5w<`=@jqkhV;&3lzB`Vv@JaSKx z<^^cq#B0OKJV=T}807&&M$UA-VNSg(dM6+NyB_mdDq?p&9&jOZS*O__oCJ0SQ8*NB z`VvoFIrR)I3Sa3vpJ`fUdEBN$ik6N?@8RX#WfYx#c{lYl4c9)989Kkew<#+&koTqV z@j!}qDDylr=_%E&IsqbJ@ZHhJzsGKTEKq#tHEVr`%i*Wajz3V9o$5adtWk_sjFyFA zlUjpmxoFBT+z>6!;l`wBVebd5sr~A?ZE8HKcHidnFHSCN1i=o`*%MMl5T5+ z8fw^LZPD?b2P$QV4j-zBA8%sqV1F|i^|Wpf9M^}lM3!i9I+Lh}r@ru=Mlu58D0K;| zj4h`t{awBgI{wes5GLOb3IT&Ftf(E8B8gR6crF!nv5l#G1&2k+W=*k%e2_6(YeHeK zmY9-xy;ZkcZ4FGQK1V#Wk>RtONd5d@$>VZYdCuG3JYnK>I=5J2MM&wRJRByt6S zFUtp7ZiU0RBAQ5z-`i8%Z}=32`Dlz;0fVKSNCU0bZ!c zLonpl=8sb%!v5R1kMFSg&vUs83O`}9SvGqUBYD9_Aq z06)Ul$%JkRZ~e{4@+qxwpDIl>lf&q%VN=j! zvZ!_`OY`QXL74i=Rn<|jR$!T+V{d(WHmyJc}sR+V`?b4vW75&O(DO-fStoW-aL<Db(+hTef>u_}8Wqg4x^ z19VUSL%17!m-}a)GfMMZt6w{9rkc2qY;RflnZ$wrKm(v^r84;451#rdM|1K@{==l9Bw^Jm2gmB`ENB) z9hK4Zb*i4oY@iNo2m>M|;PW|kuKS`jO!|KjX7oIv)wEidY;@2`+y-v~yIg)S`5N0g zm8Up4-AMGPqFTBn18>{&k9eP#-mQ6QtycqpqvO5uu&;;u)=Li`WlHXF@B>1Xr2R_} zvNSit)^X81vxV|~i2XZLQ+WhJF4yZFS+;?HraJTC4mim zZkGWj$m(gZ4Lcm-s1}~NlEieQ{`bH(8w<%zt%fwU>!2veuctvAe2q$L?Ne~Twn6z) zasQ!&c7JCB*r>)sO!jg?=}Y&wePFNPSrj?uv!Dl#m{9A2`Qo7tjIm>6f;x7B9WLB`6}^X2 z-Tu=2vM42~D(ZFPDccA&rHhif9S{!DR&OMuTDvPJUKPcA&R4IGRF(~D7AWF=48n-KUPg6rim8^NgUC0sQon)f zucQ@n?`Xre#Bl@LdGwc)mD2TPn#s`z4e%f4eRec;_aj}&`+5n(zojM1s(X%P~*yfiGse5ah)wJ)6t9 z-wW+oNePBFzi`8-9@+BwJdvo!g+b;yl!nL!J9Zw9ce*0>Fg0ca5IB z3C*~M%UAu`V8q7$BZ%u2G^s#c8pBF)Op!&pKrfK6!=3%_Os>nNjcLNYw_#Hi*~#q; z#M9dcBF`-bPUGkCc#(c6O=uEME{a0zje0q65`$}MUq_c-l5`4V4JMAUNxO3wHI}W=YObfWWC}}N(HR^O^`uu1WYY*PK;38jiygJKZ&2tcg>{W4_=J|Ci z*!NE^nb4kG%0^mcqlw0!9w`n>!cp3lPq@O0%J&3<9|3Jkr_v9gA`zu>8bnw#q`}+deVo`-Xrd8(Ai8GOn zPbiX!sl{}gXyIM7VyGwmycSJ(L76U~_@dKGmi0lFY_KUFZG}qJTl%WP=(|BnsM?S& z%5;u>>~%f*yqb z618d)_<$%4CUfK@?U8<%l^FTZ?UQDgvRisDb0tHP$@y&kXGZ1FVBE2c8^B%w3Pq2Z zUVEC=BlsGEpi8H^x0QPA0vfx)FdzZXgi`v)kkL;Da%a8;rria6C$lVt$lA<4?>nNB zmkNCh-TYLnCkuOP*ew04#D9ZJxg<-bubu#ob|%GCcqxr$Je0|J*a9i&gflG7YDY6W zXFKoS%mt#Pz^PJdyi|pdHDn$$o`kWP|avwE% zUMv51W>TtI|4SXMp35D_nk`|G+13{jF@>*Ay`7-fofDID%Kc3VU5ZJH5p3Ba84yor zxeLnuD`AMckuH+8(Jac~WuTFn+Y>XOB4 z25Kv_ytG1!`46|o7ptr^v4J;xI5RZ<(`|jrxqR0nJwoLs)p>^C->$c5oca|E=S0Ri zO6}3)?FGLX>|_+mt*I#m#8hkZ+`Hj6Tl7 zSZu1Q_i`>cC`%hN3|G3Ez;8e4x2<5xB^Pu0k-su;y1YtYL9@HSaiR)X>6VJBiU#Y*GI-Ped_XlI`XBKXwjh`^I`#>}?X)d#rpo`V`86(~K{~lSwmd-)5IZ4a{qn}hSEVC#WUV?TM z+~BE=Da=yog-L|PFUnA81)Jnd>-4dJ6{+q@v8LukLF$Gp(0huO0B*X(40-W(Ihni} zQ5ctT^=gVFVi<-m#lk!IB5Eq)Nn7`qZ7D2sEwM!bfL><=Xjjw0NJ8~Uuat5o>Svh8 zK~aaQ=b(;yL{CMzswN>M$>Jz3`>_Mtb$JL-k^P!o$marT(T~7U{fW*Ato)450(%M+ z$HQhi3!1$F-nZlo3tYB24!7Vm(WJ}wM*EVT6n>$InM~Or=ywqeu}qsFc#93t7UGkY zO7dY>^>&`wC|a$r;O&N$iN9ic=&y8=lC{ zLVRlON79}t;(TGK8zUq3`x&;7u%K{;c=LgMwWuU@w}sj|!+5Z_dYyLF zv%4Qqd?ZzP5_~7&xw}IV5qQ5xn=K2Aeu3x(a9#BsW*uyR>{EXeb}TYM;;P(>MEF6r zMkjnI5c4L%x{yu$*M0Um;HIVNANZsz;RH7KWbT?X^C#jxccr1rL7WfYp!d;84k0_X zG`5v|;Ujxo?d0ex6>-3>LXz9>%HR20=fO=~dXvdaMg&+CNJwRod8A8yi+j)rjwgUH zLOotZN0C#%WTjohg5aRUe@8FZg-Zvj%)19Bk6=gQbqsuDOQVbT!(0X~6R?uZ)6a*C zydP=Pu6tb3*ji*U>nc0lX7HH$;C%*2q{9SGccQKH=ra^QkUZx{1g#6K46;aI@VIUm zaVW}@64XZHeynNUFYPvnd5ZA#*<@P7Gd&)KN9n%`2=x-dmiI8X9r0_rx4)9Rw6H$a zU7JgxL_QIwu4$0&bK9TOCi+PF6IK0pfF>Gd86+dNDB=WVbum<0p4gxyn=AGn34Kdk zEKahj34?eTZNK9N(dH4GYD}+7vSVltHt8v)KXc2>hgzMvH`@;1nA{tw5A;)Rp~5MJ zHVHi+!Aza7kiH*XQgkd3j`E)OU5Z)MB%QntNdA_XWM1c7=c4{RJj*G+@BDf>;h(-5 zQi@XvWB&d${6rV)+Vk-AKN>UjTF+=NP)m#Mwr+|aU%Qd~Qwnikqy5wK?_bd*bCe-J zpek-RAT-~l7C8S67`JjB>GF4cMvWIs=cy5!j$^`xtC|up3XU>L4~ECT;wJ%ndwMbz zKuw-RHHLS!nsY0ssbSwq=H@N@SdDKI3NYG{{q9-9L#j>n3_DVF=P%t$bkziZ?0*UN zC3Y_)J;U`&Y^qb3rHgLpsYB`;8cR6kva(A)phgN$3HmQT9$@Y!K8$;`6fAf;;o=fQ z9d3=)F%^H%_EhhD@jU?%+0`h+)?3+>dr2fZwVFm=vU@@TES6dr`AGR`c@08Zu?j~K z=GwdS>`tNy$mD`cdx%8f8}tT=YXM(9t_V51@bAIT!w(YU>W$%Ch(m24@RzpuuJUw< zjGacyfvsECIceiu^^~;EeR>a`>m1-wg=S-a{-p|WxZC@O;vXZLrl_AqOs00#rvF#G zh5Ed;o5qx3#(gO5wKag0wo8C%OZ`H~5DJ;JqO1BNJW>&kr-)FNt4j7*ap}rdTk;~a zeH2Lgw;X5!^>Zh;@djLYHCSOG@~gfy3C94cZsZ+-rENK@iLgjqj6Cn{yP{sTJDfn{QNm%g($ zfs;zmgy4!B=BK}p2Y0NLbKRXY%+x2(N_i^#;}TW$lU z!+rf{8^hZP)HPGJUKW(a6a7Lco)BQ(;reb`@BC`@p$)@vaJ8D$JlHB+8l_`B5SQ?9 zHu#f6oa6I3P56uT_2258IIO0EN)X4s!PAWIbg`D-3Z_49QC~zO!$m1+MczxPvxJjo zLgEW~`qNHHBTGbIarkpR`$TFx#r17k{trH}qukzqXx$-22OK8Epo#t0v*>V`SR(y0 za9@QpB_xrqJ`s4Faa2>z40A@0+eZ2wo6+>45cFIf>4K{~gm1pA>0=8_x757G5)(!q z&VReOIO^q+LNuC5X?+@uzEi)gwxxE&z|I(pDYj5^-=XHeb-mSn7K-8E(kvV@f7M*C z8@%Kt;iZ0ZKLDpfyvg06P{}c5Lps;m&^uIT%4r=jKKZy7_MVcPjnkMx9&A&>;hD-f zPo@>#CZmx&Sy#nWW3$3NSm5R*!UQZ0IT~1v3Icht#el|-oUfb`QFihWW+>?ksyZq9 z^3QjOS09k}MLl0kSpPD1P(i8ljIh!vvBw7!@*()x@x*anPs*w1Olz9VSp%ZjcPxEE zPkCR>TEthFCIWBs~T5{4H z{&G`&hpL9#gdd#%gBL0xZsWP>;LxcbpEAJ}4giI;IPD4*0xcvnV+Nm;*9Z)MMQEIY z|BI`S*t(y>$^kCo5v9+*aJ38>bR7m_6>Pr{*C#974mSG!aSjV5#u-5Ub*k@m@K>nHimExL178hE4O&vmJvL9GDC*ZJ9qS?n`S#V z=YxGbqZt}dR36!1ij0^nO!jUsybP{3qEW<35zA+Hb?zkHwXP-hvxC*+@WS+mQk~3Z z$ZW^0_6_r@Q}Mh!OKoxNjvq7}Tx=>EekqEiE*#P5eg!HhcPEtWK?hxQsAvXnN2*misn(!4K+Ddl_FrbGc z=J}=nT=kxI#;gpw9IXG{PDmY+n*J~EeuG2=Z4?`s*S-bidgnhF=TC|Xc_h~P)qc5! zX=Z9{5&fs>qm|x;{c~#AmNXKE@91bwg^1&9jP7H&{>^dMvE3ZGBthF*)oSk4ThLMA zqM42~VOp-P+l9!KI~Eo==3a)8N`M9ewFP;J?)kHg!`IYW9KFBdE>b5Y_V;|r$f9?F zMjQeUyDP$0veh7|#x#VbG5^J0Qs1XCK=Kug%q$)RpQgjQ`OTx`wuT4dd~Vlj`f_Hx zhC~!|bzM)}ubyENfegY0Y)9SOLzAqz|^q}m3`k}C}8&n+nSDH8OG-O9P@i&ftfss)5Zl;w1q5jIoKB$cnpOPXgg`2JbdYU)G`AEwlc*d@-sMg|WS zvegBqm$qc(-8XVb&s^d@0{eK>;hM*77CX&LiC9K!g4N&k2)JFJhhh_bpU(RGnCIzh za!^b;e|~DhbKfuJSM&plIpv%Ux7$hD4Z%K@s|Y+g&9uDqyD=J14b3I#EjwRbr}+Pn zWi+n%GzG_(jiV0%Sb~*o`(Q?zo=x8GniYS&mC0Bx4XqxaYWxhn)`j%BJaZc|BgDQ8 z!_W_!LOxWA?lK4%hTCY~l$uxe+>zic0b+a`jxXL|kue z|HfN$&Hl=ZROjJWW)&GPIr-`$ip;W=$-o)P94Oog+SCuS${Z`KZ>7-?gadq)%qTyMdH}8as#sB@#3$r`B?mty4A3U$9VQ3HB z`O24*3RRHl__={^wpA56h?tN+K(AMHvW2eC^A@sbK1P6TYc6m`AXu14Co0ht#Tbdj ze1#0sdMlhW8@J8Y=~{>yo@6|g^`_iAV+~67ddFe+AK3OW0k#{s{zqeVo7u|Php0qS zbG*t*GtCN?MbD-W!B(~Do7tZlXj2z&hTq8A`8H*AH6#Dy;Cji#)1B(6MX_0=%B`2m zQ!!s0uD3!`>wPi(*3+wFi8vjA)fu=^^eB$s@Pm;@ID1eG?($DOYK%Dq#x#NdiaSz1 z>Ak;R2KE6tS!GmqJyT0Z39+fgE=8V^NSql4yZ>fY3dW|@vy}=d(Q&U#0^PHR#JtgY zI9gPL<$hBcl-@e!KrZ8!S1fHjcCx3!u_5>DQHlHC9@Zec=CDgapwtc~R zm*Oaldtd+^HsFprlX$V5%%jp8!LiMc6?k~6cseLG(+17!*FQ3x7c$f~QK;DZqGCig za<&8u_G6finb{dPL!xg40%3Rf5@RwM;VixX5Nvlhj(Z_PB)U!mw%(vA@f&Dg#9D8g zfzUDUzr+~t|D)-<2_n)6T&T(Du_v`hXAB>M@>k+zN1@cQ}=*cPjW%Fr1b4lCO zA6q)JF_|(Mt4siyV1{Z|zWx3C+6q*t3e?z*oiA(oWO88DsAug<-bc&zw~FN2ha+BZ|E+H_}BYV!zSGvDOHm z&v%vAbpL-Pp9oeKu~EOs>3d-N)F8{^in8_#^&X#sE!!F%{jO%?-%ePZ$}U9Pbde?n}QGO0t`iS=MJe$Unahc68^hXMVZ%-)Z{xM3`>tZ|9 z{1;k`4pWIMvgXgg8TgneyhVjtCQ=xqQhaLlQoeK1^?nDNtQ&2Q(8J##%YBFtS=B%N zM2uW$(473F2dwT+S(sT1JG##X>T+9xXUta$-3ZAdPfkDWe!0G?6~+!Vz`5sG(^Sj4 zQt5eU_#J`uQDZhvjiE=KV~60c`>c`BWHt0OXVnwp7T4LkoUJl4-!wH;xWEMs%AYG2 zbbeMq$C$c1;tXwC4tTk_3ZUW<-}ar_zi}HSei6>6F&c3@g07T{#-HV^0_hFY*MOjN zOZYwmQR!FF?@7yFI6`)^D0(44a`sX0`jlbzSS)!+hr^u4de|N4EXlz?>D@+HH(v^mQ5Uq^G-*E7>-OeQEnj||*i}US9IyZ*nYL~g zNd)4T-{ib8rQkhC+V~Z{aKYW<{I9nxj;{QfS4Qv1+=v$xPE@Bn@e8OejXz&?KXTr< zIVd&uzWBsxYYyDI7@EV+%j6AD@~amfq5T5>gC~VzZ>s1l=$^LF^Hd$JGZijp!kS(? ziiaKz22pMOuhEK5yRw$(KF=>?YNnirlVJM&|zcMzT>6apqYJMB6Cf! zrtVS|+@Xz^NTnIvcUAjWzGiqt&)Ad0AipiX=v3IcGDyr&+MBVf-7w$Q+k~*8R<}E| z1yv`l81RD<$It!LeU>9pN&+ZHe)4oRha!|+vJelzCuU-8?AY>Goa)#7$4^&zw%ZiG+coYR@C&7x~` z0{+MiUP$CPd#7kAN{_TIP`7g_YS_xK?D6p-uXE6krfr?PPzP#IB^R>~rOAD3Sl(E0j-SQ&xWs)KpJS0v+ z$2l|3d!DYjE?KJzonE&e9BHrUOJAuR1A9CY-;=pNI@C#dT`YtP1$OSsr%-hJl!!6sGN=zh1)xti1Gm@z*j7{Abo zm`hem*jM29fxkg?h+g|4#-KdM(Zk`c#XEq<5O45Yz`tXpGf32dW+r!bbtGOPTH15H zYDd2^>UQJi_M1mdzRQi)0-0M|A%|Xf3)U>qHbM7QPddfX)zkDMKKjJ3K%rZI<;q?f zZC|_A1ej~-jjL-%&_G5z50YVa=n)!KEqqDQMx|H03sZ8H;% z`Y&H1?5OzT(&H-%ppZ5V`Ci2OzUs8^+7?8@uuk>~uP0M8RhEvI$u|TE#4F}HW%lc6 zguOX{Y%WRSLT=4*3nkYO?QfK}EZyL>hq-6*di%YMyOLp+B&w9E?rNoBh|=+{YW-`@ z&iS1B9{M*V>cMI>7>B7;i#(6x;O9l#r6Hi`ggP*ucI%tSm#>0eKKsy=_*AAWOQvTf z$tqOI^#$2JDkR{q`<8G>&TaXfeKXJCU%K9C21_~_`xue zrjzsbBpY4D`Qk@V^<&)SmM%hY9%c@fnI%$%W@zdY*)K;|I7epqDx|eFxqh|`@$pab z;vK3FMeh9)e)v;cb1MWL263ijQ8|r0=C4xP!GBL~ZN?xZEgt#HU8Gg;u`=TA*v*SG zBRv^X#}qH8e`SzL>kOO`-_wH~GG?(OmL!lKHeG8FK5$ef?qX#^_MjFfA0W<(|nDv!q;MLH|~IQ9LWVWRkU?DLVBBc04Yr{X$Ue{Gx= zZ>9ATu*()~PBV(oh@~nES8lpF5kq^J#sB545Rdo`p*yNSbP;Bn`xT>HbGj#a!7BkY zYOE-UP6}piYJ8=q&XCgNpaiDt)Ki+*-(jD+R!G5oh}b-RlQqxI=oL3pYIS20fS4GDq0gQ z`mcth-TKU~Y4B(%+`IZL?Q;m?nO6k?Ok^LQ#ZIq$vuQhg@1E2sx-qC-6cKoA6BMwH zM~AtQtJ0xjX!emGlg|0R04X9W>(Y%Ap?6vCF~Ac1lQE_M8l>10hA6Q?l2i(<46El& z$Ap>1Yy>amh&dnqTGPm;~A_{CLg$c(TI*i>uYl}j}-;v$p-C8 z9D{7Y3iY5<%Zn$e(WQ@ZEp+m)G< zb1sMyZSc?;@TRmy>{c6WbIFB=vp#WRMu#qIA_wFxEhDw)4OFl(ZkV*SEX2a?aV1gr znc4&RULd;QjaE(b(RJiCgkx5yW^7-)IK-9F%uVjebj0G^wP_gd{mm0V44*+~ES#L~ z#>6)UMA86HBr^XCW}f_JU9<4{U&@F#JMl&^-1>I(DL+0y5Pkhoyn#V5K3k|5$GLsD zK+CYv4(ORTjglT>X%xp!!|>4^Gkx0c?;zxGe`yjWI|KiZWO;PDDoJ+jpyrsGhZkj$}#;@p~T^U53XYrb| zRZE$?(BZ-Kg6&7qz(2vLlyeH`jRJdl$=@YUs7D?J4}kQ>YVo+kOe&>AN%K)r?=f7#3 z!Uh)wjVfW|ROv-XIs6Ee)g;4gHH_Un8s|!Xa<;^9Zp}b7q+7-yqy1IRj)Y~^$%9HRR$SJY(ajmZil zcC1_016lEteianOLePXufD@wYo|@P!(PPP>yKty+Jc-fJycW92o{}FouI|K%rMA`q2jopr z>O=K#_HHN1PflFAe!6c%SVygcugo2nc{lFmCB^69F)t7v=)NtE35n9lpmR~+U4&?I z>m4F&$l8?rqH1#wlR11Ce+&N&KK5S;IBUZ44r@{%e@uw%n~Cv6xl=tMLuv zDF)LlhfIsr#GeGoSt7FR@xnqM@do zCuf$S8lF6O14a!vb2sbNjLL?@HBaCz)-BdIk}J%3CmBe)8S(}JJ~%{5_P zF-Dq9j~rvEWRs$Chw8^Ge6et4eZ4Vcq%Rs&*+OCcMMg7`Tr>N3O!@J63}_pz9D|HLd0@HJ7ai2dktSudUU5M`4pJQ zDYMgOKk8OLS}Z4c=f@juE5qn{E;{x;emy5Wx1@X_-)FR0_FhN3yrua0N~6==$eL)c z4DZjB=#t-W#?mqfT6&taK19||Lq7HRn~?Lxo{psy{H+~6c21u{VT5q&1N0c@__oM% zCG+e(WEy;e-4_oONEQ3=MRUYu{mXBO#ilfhv;0ZlI=e+oKau3jdj13@vw86kF&wg0 z&J1YGyp&&DwYu+Om-P$M)-u>+5R2HtjN2x@!Rai? zH_9m#iEY8+qx}yrqAYt?AEwE(9|Zmg?(Ze%6W%Dp+a&WFmsBzY-hK+DsCsI)RL89j zqxi)g-n;h*Q_HaJHB{;dM5Q&J8`l-AJi@1OrB}bic|@^Al0JFl2*P>a>p`k>uuKI- zrdHE}S&M$)5?h`)4x77|5qZHZ;M3yHEy#|)79b_Z-O>NmC`6!?wVpwlq(C!g7=9v< z?!;j$Sm2UKvxA-wWbM!qMm2gneE?3b z_MOLtSEyr>)_G=L>!-C?v@cBj5a)Zd1V&79`IH?KlCLFI{F_{o8<>=?mPGOZudgU6 zQ+8m`e$YGkNSiKYrMSpztg>&0~6qDH)nwVl^2bolqH4m@i+kKtTX5=c77c#%DSh`ESyP_UcJq!eOf6w?>?y*N zYzMG&!7gJya82twK_M-%?{#Vy!d{bA`J3Y7=c=*!6@zT<0IQ~W(bOQ_mmjD}epg+M z<^(Mx37xEU)RnNO)TU6p=qRif>`yZ~o5_P$!8nR;^0pX0&7btW*6P%WX394R@E`u- zDP?o3YY<%I9M|P;j;IPfB^U5AqCQ2st#?Nlh+%q)4tZ3FvDnjMOGl8p`@qLWoDT2-%4UYXcIxN7wu+B;>hq!O4QI2R9@cRrq)-Hkn$pVITHIkh--I(@P}CY0BS zL22?+baL{GoSzgO?Xzk`EqA3;cqw9TGg{sRP%Pto`TOIfn`F<|dQD2j?()%WZuhsa zAOE}-DDIyekiVD5s~(?gqi1!ChL}#5L9l}Q`XP1}1TSpmy|N{IdiQeZz3scRN{3hz zS=tK8mA~k6xW9UBi4E8AgI~tVxvCFmnsAQmRn1nr_C|Qc@E7R{BsmoY(G?uiuCvd) zty@m6Vb;fV;Fn`whOly(%8%el`Tb^Pa@fbScfm#l@o?TEmfnhqWqJ9Kto8&~=3|+T z{24pu2c(U`#_h(BzYPq3vhd$v*r{iga#??R`8Mp3!y#FK1Hpjw*_0>{p9#_FhrNq} zOD0dnyyhI=FI?_i;u;tuH}Wo)%*wpz6gr5-?v*QrG}nCHcp^Dly?c}{ia|L;?1hlz zr2q}gbXEUah6Lb1ew}>9waIs_kU_bd>a$0>D5AVPRKe^I!v;Z$#$LP0hiI`iCTwLQ zgEzo^#U=jOdU=u6@TV0che`s5E(l-n3+7g{$LJ3k0GF^|KkhI(}{(*yz*mfX7Pc0+ zl;cPjSp(N#YWK-sJ8z%nw~PH0WBZ6CQAC}Rb+K}LBDz|gIj>Q`B(Vi^hs_OEsH<&- zpH-#L@CQTxIi!nlK}{gNd(`sz4b#5O<$-HMCz1=_7(@mbAxFMJ{GtBI&pn`Uapuo! z{2~)w<6q$;UaOo;n;t_sv=C+NZ?u?Yy{tH+iV5^ zgg{*P+H>{+I_Hn`Bj**}ODV-KE0ayOA|em-HJFGL zGVG5l3dJqv0sTsuCt$?*)5ttcVXtioCjjw`Gd_p zmVag7O4Mg0zGa@cU)!Pu>+-+t9He$?U~R7Ac;unt(^-LZ59Adg0%gRSWgCNv##KsA zgFv`0TaB15h3)8mFyj@K!dAZWXL{^q=Xt{W#mzC+1+_23{A}1ORf*;nC>Gb4#Hg)EdT@84DbGH zB{w0VentGi zOQu3U=?nx^_~&2cb=$X08-#`gTT@;6v%>YIW|N?yDD^uRwt^c_9H81mqVF5F==Z%6 z`|?&TVjX+=4>khnq>`X^YJu%9Q!FMCeg4Wt^- z6dzF*v)R&WW5S%v)bb56Ty7T6_`4Na*#Edlq9hSBl5S*V$x(Frb)U6v)@z?Jb~F9g8Z0OCu|XoG+;?`viBq%_pDgrO{S_KTl(%_Ne{s$l&KKO^Sm} z+#E-Jb;H5_o~`cfn-F8|^z6#9!8OouCg3y5Yr&|CZe5%?B(F~@>D*Iiu4A)y0->CKmBE!h2 z_>i_F>g9oi_TQJG14sgRtm%JwUcGaXd6R_XmA&reul3W`O2DL5Rh~aGE|ARd_5OQS zks61DXuTmPtf0cMmZ>!ezRd(X<2xdFZ=gqVMo-_ERBky^bYprUPNn=r>s)ry%i&X& zrRhVwbA&&}ySW|$YZtmWJR$B^kVdFgyoz+v?2XFCK+o&XNw}Y(r=Gl z)BFd)wd=#U|2{#)8pl`5YcQ(NzJ0L3KN!5l*Qz$-3mvpQZ<#W*iXZiazoCb9rlZc_ z$X7SnEf;GKu{ww%u<8GRZ7;MVULvdrqiIbMoTpi<6RbxIw$Z zjtu>BB*4F3dg9cLK?pFs7-w+j0u(@XiUm&Art9WQ6s4yQyoZJTP21~;CsOnq60ZX@ zD6ILb%9_gq@lKsplLHXU->B(Hiw1}967=8b#Nv5{PpIrRGa0I**qx?!dwQrdoibnP zofzl@IrTJb9rGKzOm6n+URz>zjaE7vNZ8^7VGA|Cp<+(Ou*QcSv{)m#a$3R@Tp&^F zNa)8}(Cnjrm?W8y6H4l@5R2g8O*S9>5Kz*M&8GW9WOloyA7QC-zXT%CrQVCHG_4@@ z(uV^XWd^&F72-aYnFQVvVYobYVT~#|%=RQT=;*QYMX9IgwM%Uv=BUB^GIFZo#b=6q z=lK$D@ieo)+&#Gp#J4aq#4vPVTaKYweKM``rw7X9{UGhxF*N0$fO@#ir@8h9numzR zhg}4~IitJxvswx!h}Q|Zyl69rV0_%(Hiy-ddN_zSzKb{pkp=;gw}JaWIf>EJ25zS) zn(CC8B{=nAI{tiJ_l**Ha!!)rj19(k%|I;#rLUUZGP_TP{zXWLzQ*xSr$>9 z7M(#LIkp1m1)&rCZ`Xlk2z=ewZD1j*FFS2VmycaDJr6Q)ZL6Vy?aR)8SpUhdikGLd z0gIKsMvCEhR!w}jQ=ioO8eBTpz$MFdx(9W)gYC@LV$lM{BYR+QzO6rIvNt-8aNsas zIp&L6m~q25tqS;*J0Ad3f8*|?L#s( z{^K!khbKvO@+u!ef@!{FRIz4PYAFv080$lqf4V`#LFWe{iQbz(h0xPSnQ0>by>>qT zJvZ-o6LGc|p-Wgy3DHp_hljx9ql%$@-B*@sz-|rK z_!$T@Hy&i>>W76}rJ-cEMH0Q9%y_7+C6-v~j`a+w>Us*t!gofl0G=a~)kwP3`snB?A5ai<2!R@X)`bS)q77=^C2fcbj;Rr`02T3}+=qFGdFQy-%8 z1{|Tziy$GA)>5|sEFQIjNL5Y zL|5~9XNu~VP1~C&p5mf7?hUv>QNB2pmyv-g973;pBfrOgtE+5sB?7y1Pmi=vsxM1* zOmlCAG8)D|?UROi!Lj&D?9gwJ=Xm=u6j0XGCdQVHrus7VvoYpxkBzj!rO$1n z=k6&hXE~Ngs1!p_I0lLLrd{Pk!$>Y7L&E63J8IZlW{co*KFYg(hm3ERfnr7+*sgCc zEj=v$6BF}gDFFb3g?T?;$6H@Sv7%3|9FJVriV#-D;yMhRxP=Vo>A?8Y?$qg>+!p$( z2Y(_YkEow?;e!81PP8B=1G_AV#Pdmj@t&4V%J&~XZbhjaITDa^dvxi9@LiZj2bX2n zJ0)k2A^oE%Wph^>FhgSF?zOyd_^Xnhh(6;F+gUe8AGz|$Bi?}q_2(wYw+LR%n0cn< zkM!XYUD(_kq~wq5k4EyY4H9)^JzX=Vg#We&Ki7*aEKp5Is5|@~=Q-3sjD_P_H1G$Q zh+ctm;ZV0)ogEFc=*p{}$r}Vx-af3gJcvqEW3;I3@V6G1&`=-Ko|lBZc8AvTtOSFY zlXpZ25!kcOje_&;#+rmCkp!>`A&Z4?HNL;#-gy1kg5O*)^g67M7W8ZG%Yrpqkjn+- zOyl1_XNe0?qhka}M>rX7yuNFKh!Y2;J7c%kz~c1!R>%oxr?h9@mMOz7jRXJk2!skn zMH9G9a?c0&g`u%2glh5n_nVX$0YJE#YTQfM2d(u;zOxz>H^>K0PZ6flUQ z&7;=`Suv-9PKNu*Js%f>vQ;rgCF-Qmp1k?q{V^p19q?$h8CaUQoKRn-=_5 zx@QI5VA$CSCr%gMY{LW>hC|i6*u<#|emB!TCZN0Yg7blxQPDG@@h=?@7(WU<_TLy3 zog>4rHCU@vW^ZeDgH;jlt;_eC(&(wp2vvjDonSkSdL2ENC-&xpXh2L`7cJ(F7Qp3G zh@F-6PY(1jOGeYE-%o{v=(IezQRDajI&D_LIs}Q!L(yKHcSW9#PQR;*gUTai_4vp6 zLRDJP@yuMmL5`H=892+LWT#P1&EpvkOo*C4kEe|H%xRuO#jgwdZY(V=6~jn%5_}=z zXBi-I3XWQfYZ@Uku9KGuo~mdB5Z|u5mK#>#7^r756?V7j1FmQaLJUuwEA1y4`Q50y zzV~6f`~u5&ID)n>$9>-d&k-ct6a^=hN=@2qoP)=yu)W!-vs#MBMC6_}g(EQ>xo2(2 z9F9Vn{VU{+^dk6R*)><=1>&c>@wV)vDJCUiMs-6%-cmNJ(mjY2rDQ&CTHd^odvi-k zROJ%6bTN3LE06l5n^SB|rTk6uCpdpKu9ucP9=K!AuI4XcP%rTW=4U@N>U2l&p36qu zxRVNQqs3i38L<780C^^Jb4&f@BtNsP=O5%T zb}SmVq2q}cE@C|^By8)=V9lD!OUz}B6@_?;H%U3ic#ik7^7G%W?;-71*H?;p*$#*F zZj>2o8}$mBW6~N(ZWVJ`L8;L_4De9COhA7BxKA(erAHyuLssT}XJQ-H=0w8Ga;MMX zcYBnF>3aeSJJD?OLrw}{N;Y$GSZtU|y^K}md|_}` zieE1iv+&TF$D@7f&z-!TCmh;?(imCxzaVA{bWwsX?F7*4G%Ja9DDT@BwNig4Mw8aVVxC-I!OQpOWIoc3}IKTiF!Pv%I}J5 z>bJ`iM%*mVj#NcI_m-O6|B?PxTaQ#NB4kp?4qVpQ7>L6z%5mG>wssy>IQiiF)$?^# z3MY}=Csc_{A{JWbo}9L=#B!oFl<@gq?ZLQd((cBq$#46X6rOe&?o$cKSFo-y5xwx3`z#Y4bCaRq}7$g3BiP>1431 z*AQmSc*>Xek`Uz;y`EQOxzNO+%_f5T5%5B47?5_kk&n+~ZB9Rb)R+<2%um5U|I6R( ziv#8kL_4ubx%k$dR0)ZFBD=pil{jzA8f03bqD+H#ve(ai(VdK7kV{i*SsnB2-eL$*my$*O&>WqOGcts{I^r#%k>kcTvwA<$~T(CfJ@ce>LnMhSTmJ zZ=HnK`)601>iL7Rqr&YrM>o>}d5V3@!sypIQ;V#-7vdJV}eNQ~tpdwQQy~mC!wXB-_>7X5RIP4c$O;N?p{7CRt+Op|Tv6upB zVX;R%!BVTMCaA<-e74#*wDZ(f=L)Dr%CVr0dJ&jf6Tdce#>iPZR;xA5K>rp5b~*=? zo#(2Lp8cuaD2e1k_EF&VhMZFJR;5Qldux zfQ*gIZ|t3;*jpf{3+!SHy8m7QD;d#U-E8&?8#pfG+$H$&_QsXkY*flBz7k@=p@_$} zJkIO!L)JQhUf1iw=->qBV%^ap&UE!}yBo+orjO=K%{5fIy;w#=M{Rw%^ob6v}vLq;;lOW$%Zmh=fz%^0G#XmkB2XClx0*q!xn6 z%?q1COSc9lUap>uGQ%MUYRMBMi z%CuDblUn|^bW?h-11U~=lzbt%#E4RpKkUz5__)yd z$q9@q`rGuvfHx9Z9$KXCD-GqK@4Os;S{_j4Q&9}ZKa<9ucqGJQs> zuqMX9_Ba1`#EyAfMEj-^j!@AE#~Fm=iX@au`qVpHzKcora(LvRPZ0oj&tsM_V%lRq zqD76!#|6I7#>ttB#aCj!yUB;AjK|TQ>Iou0%(r_57apQhCFQ@-ucaIS6Y^A3 zmw`b2u!Ys{34U>YKfJLU!QPhVz*R5>jpxB8wHz=$$INx< zr+;gnvf#lxK}K0u<$7F3z@|*qIbz&F(LEYTTdhQ9Gen z9s7)JDg4%)i&B>DZqEBS7_LJ5w++6%d+KK8kvc((x)Sql_Ly13Hx#|93_^pxuUBbZ zW6**0UYy)m!>8-F(ddendEb(cyy%h{dqYdr{B5r161{{AEY370D&2|H`9=Zkfoq3N zKh@^H>(b%WPn3OMNGL5xehFaxQ4IdKUM#p*?l7u8hlp>3ze-OF^g6E|k4&|A)h?M^ zzkbU#+w$@qPgBRqbOJ7e;#=?V#Z)xo;2VT6*Z%|*X!gaSy>D3`xEokm>U<=gy@reY zXX%OOYcx)ms^06oTVoWep+)){a$`83VH(eZLA}Qfo;NY0tknlGyQ0DBr6-Z9v&1)t zpA1_Re*hYZdz{@p^Ne$gK5OD+4M!1PtkDchealH*kX3BTfOOV8Jqw2X)o;VU5E$ki zO6Y}Z?hl-3bx|A!r^un00*WrBTny@jt>paMet2EL^{N$!T{-RC=}jdg!~H^Ue|W(f zcLGxq7FeU_{vED(^D;TAWiF4akg7BMwd<;K>{r5<1#*DItrB?<=_o6w^V96f6F#0W zBpH(%nJZZfIK0*uU!6v<0mH2P4^Ni38Ju#JrHn1;@tD1AVmgBV05Meq>LyZOKS)S7 zr}?p=eT$F2#dkjMEB?yro!MBk-6l4C=&VJcI9w9Y;A6d`5@5p(bPvZSTCmo+g zKWLYxl^eJ(uf{&GPyni}AMFs|as9~gO_-5xUO%?+h!t~*@VU{=*i*;OX}y+!JB;#I zI0Uddkv;P46+TBuS**I~SAuW2?mHmYqFwMU&8-QjLBdsZ6lt!T8WrWiMW zqa*0ygR4VtH(Zl0a5(CzsP`aSNTzv@=B-QqY-o5*PV(#9y#PCdm*cg`99HbgGjDT* zW6aH5S?EW3qq|kG0kW?23*6RLkP|ks8_S8rVy)aZYfpJQ4Cht)$rgAMaxf@z!or5*kj%EHSo)@|$ z0Wf3or@~j58`?0jKJ_VKz#^xsz0;pu7z+!>kxuD5t*@;Jcr>L`Q1_d?1M=72R;{+$ zmGDr~Oeb7WAwr;Z_03J=kJq7Ea0_%`KLrJITl-?7j(SV&|D4jlbm&kCOk?YyB_=cp z22);{7WiQAGnsd(DsJD{KmtfgCO!BQz4?c}oA2H1;tgl_XKCw3-&CF7}ZE~ozD!q5UJBTeMW8v%@Z=#d# zGQ&;{RbM4O60;ab)pF?*8VzH&MAgo;YK=SH18q`Lx9Dt-*x`?lKIVR_4`8vyKc=NT zM3*+@^q&JJ|f&XFo>fO$kd+WC=(P&`A1nR`$ zT|C+6W}j=~i^EJBdcxt~A2j(zq?c8EOOEH+3r*ka2IE<#I)zun1sOaH-Sm)C5?E$& zJM5eSp2jXXanK)T_s+JV^EpC!SsBi$1P9Yk?>kw&TvF<@7tReOQ_sp27AI+i;Rrs3 zN2oPtRvIPCAL>xFVRSM|v!DXw8-Rol1K-X30)+LlnF5mJoOOS3)(<%h=-D`*AHSy+z>Vt2TK} zHkBl)t@FbF1%>9hr0r|_kP|Op4M)Bg^mi_u_31W7eyGh7cdGd_1mfZw0z91Be&@JOQ11FT#`&#Z@F>ueKQA4CNEdD0F7!M~1iFk>>dDLp_JOX(_2xn6k9wS}J_E z_mCMoUo|cN+n)E6EUv0hNrgR2S+@McFM4hDRJ=IG9`^W@2OmIemnKlU_1ItV-m3JV zLlMyG%5M&>N{Ewe-W|u5XJkJnQfRkhScPcIpD)__WFA9LD+#7w4-iL(p)=tCeygz4 zwI9LhwV}y~NN_ZS__y^%{&bG`OP4B#IJ!0Ps|8*fre|e%HVp9QaXWak3 z;_MY(p__>ROV7ZZYxJk?bYmY;nIinQ?+uv2VmpZPm(vZ+03|vhR|0XAzR%x*%}t&M zHWNsk__g75=A{h%noq|yzzR{O6ygFS%++E&2M-GY$-kGexm1^Ydf?l#>k+W$J1bgG zjF4L8+pZ<%ieWuxqJ7{QT$N;UgR?A40y@e$zUUnRcjwSTE9ZM^m)jcoX|M zhie{hOfejCvIddL|20SB$c=C4X{Y{?{{3iUrgy@pXK{RyBdbQ%F%s27vo;}JHR zi^s{o!<%`vn9Wk+5JsDhYieR8Lm<1nD?xDH_I>unS!sbn$7ASUS>*}-!)NPYZ2a({ zNLDG^bd7(cA$((gz9FiP3jH-E{AVXiv9y~DZKA%>9FD-FmD%78Yew$DPDVJ2!#P0+ zQ}&d%g^&8OF(oC+!L{QimNN+<5h&15o$v^JaAaYW)gDvsLdDRd9ImzM{ahf zxmfgl>1)-r^v}3A`_lA@0>0yki=bIc_-GZd;5VHKj{qur5BG&Ar% z(tql(yIO#lO?1R=-fUhCi#Dz8wWV{ zZTLy$Gq@G=@CJOhWDXK?aJ{bGeODe7mFK!1 zg-Qh38OsvF$95>fA6u;?-S>fbQKjUxt(KN|=8)EPKrZGv{_huAG2b0#vII;|c(eac z6Bb?Wk|}u0DDS=D@K*+O>|oma=g>;HFHY^~y3gLXtP4o(Tt*LZam&d@QP}+u_*LO)i7-DI7qnmp?@*N!WURI@Qab4Lv znd5m_PpNX1Xz^{fuseF^CCJ_vf(F|6gKVul%&{{j#qy_Kt6sJ=`rBs33V<`EsKKJ0$ zBMCV$b_4T?Ku63yI28bty$QlIM^_{~B|5FP?!P5(k)JS2+&W!`mBh~HP~e3M&BpSD zGXJ`n=Ij#&b^d|osG+}4(Bnho)Pazg_7IsW>$@}vFqv0MSe%tx>H}rzFB&IK7uTz1 zDEIlC0{x;t46Zb6(*02zWIY49M;qv#K$yw1xsR{8ih59^5wP2>aKT4@Stnc`TA^jF zM~$ge#Ghgk`kXwclY8k0ggx8LSWKO{Ft-;cdjQYi{HG{fg!7G^TC^UQb zBalNa?d#`kuzMS=iPG>m3X{@gi5Z%J9;oBC!TX~SRWHi;|HBB#NePfRndJgD`*9NH2k^4}W0gLREv zO(D~{`rYYH#s^4X?+_)Xa&o0DPPT(i3>%v`2p28z1chO~%28ke(fv5_w!{M*GM_;2 zCbKFuNifGKytdYx+0QqX=D#&s!fj^#y$FI8eq-rCr3ay4pK~MxgfmI?k$0Rj^Y6a+ zFL(okGR5wGn)?D0Q@ENJnrs^+-3VFmB_LWyFytP2MNGIA%~~y(pN+m8>=8!KIa;jzZ%lmR~|Xo z*ZXHNbAyaQ*(>Ck!YPeG3E^NNHBUrS7222h4hlw10=Ob+Ksb5@_a|4c*w8Cpmz1&n>RZ8EiWr&8C3(* ztve}GPrShMHj_bFu7EnyK1?$KkgWPiwk9{7 z6~G`sJRBV$%VF@}htt;Lz`#*oc7i|2KU)j7S8^mpFC+1u05sf2N^O;cro;osnmy}9!%1lCH z9_f1+;`U&j4mV#8dvGLsSNdvTHbtjCbE`58Wbfkr1rET!XPCp=n$<&WG{nNA^7hut z(oF+xyVjmW5w%M0|UCtYISOKcx)$ zcylEG@Hth9c(Mvv91HBqI+#7I^*OJNp%CgI7;oShD+gl_+|nkZO6TIQg;)i_Z5^B* zXB6}K)rQDeWHsB*FUx!^m9#Q{jv2qz{P*iQFr9yw|NZrtJ{2KHz~YW%q}5uv2jVBVFtRM_5F7!xNzFmYbM%Dh*?)lhja-Zd z#0y^Cd(e%wb_{~w(g)$~XuJ)PGXfRo&Z9wP^|C+i8$jfAs&{1US?e$;4`RpJpQoFH z+~POt?jh3_g*Hb}P3iTEJoq^BsT_*+938y&6s4S)C>hFO2T(a2!^&vlQ%Cc-xnIazC5dJ&4- ze!-~;aYXiJ`l!CY0gWz@tear2pZoU4s-^DPUONWNsB5bAsI+s_K;o|^Xlc91z*hkV za+asI`MjsjVIl(Hh!KF>jq9pQM?b{_|9EI~cz_g6`D^x%C2yA>l_;Ku(I$XhL zXxPSt{`Y;+Bm%mZ7c~ogW?j!`+uTgA7*$TQoh;BHRN?uXmRwJp`BQW2x&gmoQ)>iH z9C~hLie5Uk*{iC^nHXorlNWyjzuh))5dSWjETE964NAhFVGU}57j<_!Z@F9jP)sPVK=4tk#P~5Y#myEuW^F-q^C}OP$44da7Ny_ha|1x6G(n z#7TH3M_J-OHZ{okJyQcBc_$y6l^7uk+llm8^{!Q6G#8xe?kbJ$AVdY5W^5>=n!zI0?J!4k1vx^Y9mW zyUCmc5#)kEUgT+|V#??E!KpitE%bZ@hl}bSZgkpuHUX+Rnm=?O+abhe&jjka zyj>XD+W3Qi|7|x{%{1m!^jj$4OAT~XXW<_5n{!C@#y{q%yHXs(Qq>z(g-cQgp*G=x zaWgaL;As`^{-HL%`z&k+hK?eL+l(tf!mQWurn2Z)sxam4jokCU;a8b4cm{PxXv z48F%rdf%0GWdo~Ssyh2}DJX0mvcS75>d!(w0h-DlWHDe_s$oCDly#MEoUXH1>5jc^ zX%`W%f0=M$S2S)68v>nEaKDOWk%^9O4GrY^BqfLX(pcuuyQY7-H=)y*?fdr%JQUEj z1rM*j&AnyrZbG?5Fm@T2QTIXsH>ACujjq`E&?4dJkVa${lnnO)RJ(A3)Ut?lh6C-$ zNFl}bw%?H*F*3aM<}j3(BI3$&w|P*f6kZ+|`^p>??WBl48TH#Ad18L@q#%MD`u^Xd zr$R21>i=+zS1+CkwSV%40Ss9SvG=5}Zb^1iD&Fz@6(ko%>rJlQ%Tzc5Ldb(!xE2^6 zM(rJRI`{qwkq1A07=QSwusD$E{RCi$(c>UBnmw5KBBf#QQaBge-g6u+#k)y~r&OO4 zM5)m8{AV)SU9Mfe)}FPUOE&*Qf%>(=$o-k%!#??AqT(7Vlg;@&;kX8LwYAqDCOb4A z$H#Ie>4zCc|DdxWrRXYi+$TM0s4-mDn+`=iq`}xzoK}>46U}%vP$Yjy=$OqmOy9eZ z-TX=8pC~)eK7uN|+8U?0qIXP4-tUxI%N@8}R3`YQW}ja_PvPZC^Ymrn^v1OIn->fU z1ME#_)*Z|EcWtGeK5%4|p)a=V4qOB4S$_Tip*5ny&OMcw zPkRp%_ulK0r%F?XK%tfu?75<}=EL|cjO{Z|m&`hiAQGqC3Nolx?#74?tGi+U_b@&* zq1uMIEo~zeSbGR_7MiGys37!@LFH5p2kTbY6v>y9L1I~ImGKKp z>}}v4M$SmhLdM6LOF1r&ZuXw$rT)c%Nj{*zas%9mO7DjV9ozcT4ybKm_~N-)Gu(M| z!Jqt^k@C2)JVkR*L`HpLN#hktYSx95%TQQ$UUX7__52RqFAmw+M@pxB(z2qTH(o+L2u&!gL4c_-Ywj86X+A1B zrt2hxu7r-#n&Z#E^1%g{i_4@7$W@{><*QWHSa=J<3*Dj72Ur zGq0-eV_a9ly2ER0}ug@w8}K4&vwcGH`@8sa0r2NjtkAK|)oGmjWl%4VE4kDWB1ZNxCwaN5{Wj<_AW#OrKik)Hy8Pd?a zgG|%2o+XC)9?kOFSaTHd)yk%p39MuM1PyBT=nc#Iz?jC_LA@NIlp=we=U2bd%)Sh7 zuX(Qg_Adc}cTmpr>3ZY-s9&X{)6pz4j8sV}uw6&W+?z`G_7Op|Qv0XtMEi&;-AMmD zE`6j`uERVmHM1=C52x&u;pdU}?^z1Z0OyCuJ#X*I*k?jBY~}Z8i?o`7E7kw5(7~L8 z&cekQS~YrJP|=YRsR(gX4~u&(Dwx_$F5+(oK2o=hA@_^&Vlfg=7`N6dIy=OW$oT#! z<;brfw3;4t?2mgjqAH+iu3-vss!zh5BK*Os4n&*3fkS>e#W_r2T9qLFUs>TCfbjd& z`iPDy*_ok>q|E6&cWmg?oR@Qr%8!Z5LEESz>{hlw`++TuE48RT!Cp{NXf5HiM3DHc zyeB?G&p1ccjZb{6&{RhlfOx>;{?zf;+I%O7DVSSUgMuh8p9*WhMW@}HX#JeQCL9Ml zlPeM-G6hCjSQGJ`LaSUEJw69d}WQ4nTbgcF~v`M9s;X4JDD6g zBQ9FY8fF=#6GYdkPN;>WUxcGqt5bHLFx}>6FTu&2RDs#1NpBwIWY0TJb9PU>Liff&#^?tgJ9)etFE%Y~o zLnlsIj>s|8CI=xt%fH7S@-&{KvSl2WS~ zejE;1B?P_%OuCZOki1pym;MV8dp*hU`9n%sLhK>EJFy<T6lud&^HSW(z77Q}_PVKl!dU^m)%Blszm%nBG(YUr0vg3jT@DkM0M zS1(0;dP>Zu+~7Jz_MZM-!I`hBN|W&;;fnv|)Qnvs(jE{J{o6hqe@>Q&G*?tO&WnW2 zy~OsEXROr~S&TMykvF>PN;boE=ISo!a*dr%mD8y$nLy>&{v)=Ujfup(AgcTYX#wQ`|J?1wp&$;Ki|5#`qKj(S{A~ueNj0D963e*f zJvn2#xcthwIK9xrw<(jRu<+E}l$6^9bja_gGNb=B?)VyQZNa#@_(XDCtw4_J-EZu~ z!}cHa&@zeQ-g$5Vixh;0)jfn!+u_6bh8V3Kj|)~r%ThaDi6J4RIr~=a zU7~sccWU2%t!jypCEdg69oUrSyBw!?DZ2hC5sDj1%}bRkK69JOdG=_D=!WjJdEr1;z@rQ#~wrN_&GSWbPRuw^JNyA)e4TIxH$o=b?M2)4 zQKEG6T20~U;M}3+Q37wF*nvYY{$@A317OXu5x;gJNb79QlFGOoY22vsyDTqd;X*6pn#0)(^PXYkB+sntq+@v9E40nM9U31Nn5}x_Gp_k)=A@U;J*!j5b#_-1jJO5y9~ z(#ga^bc#)45WK~6)+h*(Nwq;vQNYw+rSY6)i(rCbgT@O%*`w(zq&RtP+Q9fO1(7T+ zVpeAHd!bjb8!8vTx-O;e&{O@#`P#kd;U5;H#In~9K~2>?$v$bVYt8$ZV6k9cL?VM< z35QD#^^aKj zoEYyJF1+^*jF+IgV{mIVR1r9r`$d~QX-3@p6A-YJFl>@+uxiLawES@KRIl;b7S4+u zx@X8*MqoU64JFl8O%eeccwBbF(R`Mhb!2RpgxLU>0Vv@h(KXJaIrbn#+?PvqA1=;3^hkkaLzqG)r<1M@k+bunO~hKC}5m zcQ>j>+EX$++CV1Tz|DL$3-OP9-iuV-EUHMVpO~L<7DdJ+0bo<1X(DYRcc8ZvqBF3| zVbVuDHq z(I}<>z-t(IjKjKw6-gBDT}K7%*8pww3E#sg~Z{FhFFB`=gE z0+%J(xZ~;<^-uCYNIxDDLZl}XEA@STp!oPVnDAFevorTpKWMH<)}?Rv>3(%<>2SgM zn7w}T4`>6ZrG-bH7&m4$Q-h3jWguUr?)IS24+TThpqaDjLErF4Z*j+olGonJyG@o- zmCn!%#xRMH7_9&ZHxjgZe(~FaX{y@NQYTH`9ct&Xl0QZJ9%Fe#FG!B>AJ%9%Nzxuc z30`9m!twgqM1F47*{zCi&ob9v3gB8PK{U0d7(}wsn4#&~e3!|LGo-fkJAimM8a(BI zj^woWD$v`<^mM1av%EnhlOF;%x^EzU*~MAIhWhzS#9bA@zK5%XPkvrKNjlX+$?P1- zEcl@k3v4E*+4;R?Xal-9LPPrm9u?){neq240KPFR4v*x!QCGTs0|#TBe+TnDHC=l3 z7=2$FIQUt}C7cEH@V*MQMuHMv=c!xPHw{t_xOfS~oS9mD*!HNIaD)sR(40WJ7#X{{ zYY{v2a};Iw*6*T@?D=;OD-EsMpeT7^*^;t}9n?B+*&*0|Pyi5N+y-@j#}DOma%?39 z>*@n@vuJW0NWHE7!-_TV_q&X{`&D?(UXm8AN4NhWq2Q2xN1au?sMB-V70aV!z^2qk zrvL7Lu(&1V*hAaYS&Zdnk$59g6gn@@ObYqW($~bqfDfkiQEx|b> zhWhpGga5yuGl^pWoo&ED{sEGGD78L=j>r%Z&L7dOGWy@%%Hu8exL>7@J>a%!9u4Yd@v}JHJpuB81J}!=8J3AuIRb2R*5>jjb z0FlOZg4+jh)eWTql(iM$!CC@MDy->PV`3`i)=_~nNc>~_Y`29p`s0lxQeQ}yBE=Q9 zcLUl0i7D4@zm}du4A4&ASPP@JuvcO^{a@1nq1W^{4xiEz2(y|4x8bs?`TMvK>Kj?- zd)EPSD|?}3ObV0l=;)=m(JG3II3O-=)j3l^Jb8PfQJ{h*dM6OrF^Z{{XWHorhA?&; zCEqM@YN)hRbrIw_5yS*H4&_%~DM2(>U3MIpKY}*F@SBPNElO4m@2g$%XDqH6IA^(^lScW0E^TFK6O|ZZ z9D$61FA8YhG*rZoAs8g>&oU_^geo!h^)gj*`qa>;H<*At1O=RW!KP85>hArAX#JRM zp;1^qa)4q0*(zYwD(Nk;dg{$(we(;6e@ypXeR-J?m~fH{>4q4k;>lLOyTvNhHlkP% zE)fq*$MmSS!f_W?Q$3EO87Cw;!Gm<_4x`GzFx#F%k|0|r0lFi$W5C&p!(%=IivB2}e+$r_noe3{ z#lXi8l*QUW;%OrT@YOjDr+Gse4Sby4^|SXE%s2QYJ*tHfmR|c)H;fio065RPrT`&M zU78iHZG~fhO2T#m@eOEw*A&m_&5pGn++ftqjk}0F>fn_-uVNiDBKP>@6KW*8feh6ccyWSJuNP(-P-gzAJG9Zfz zbrgYUeGRl0&$u%DPDoG?i&wEpv3ehVw%<1FYhal4`a?u z7!za|0YWIV^<7dKZk%f<;ww{u8n7RX98%K-pbmbTJ^5L0iT~Ya-SDhHg=)$?3U(hF z$oZ>sSoQ?MY~W}}W$ShDnF14=9@n{->>nL}-`;~a`{g)KxPaN3Sz&`Rjk%0jSD%GD z9cRmvaGJFmeQ$Hp1t>R7Y#PZgtFYz&-bxW3QFW^zg3FoGr*gUmYs*j8k_qiN1nUPP zwAqEMAOyL1H7}l;_`*8WX9VPVHmaY<6sw=PT58!ME?x*H{QJq=DNoQ4*t{yMqd_fW zgh15S_(|u$KQFr$V!%LY5Yxe7r=}@#Y_7ZG-ja zhT0OSP*yBy(uo5QEFb~>GWHVb>^U2iB6#Z<9DN^w@drj?_w2>Vpu6%+{180VPNYx! zgPv$G9ZR+YT$B+J&u%L$0>Zm2EkEVY&~q>Bx!j`zui1wG9wXuI!o{HV_dH?i1G{bB z>RaQp8~IAT#R7s`z$5>fvcpk@m}!%%4Q8mqF#~)@?K!V}j!9rUo~rksZV<-v=)Yqh zu6DzF0cA-LU=X2HjOkGc*gS$Lv!BnRp3jqdSarZkU>oWO!rw@%f;T2zR$X#>1c&P@ zR@^fhxA;K4q_Cok$fMSOeH9fK2f0a&VX=a}(LioK@nThyO3VYQeP05@J&B0+O*KF8 zgd@Q=_$5A3RD+7>&IsJnnae$Q(qUyB2_k_r8-@svm1q5PCsTUIgkVoaW(D*D822Vy z?BBivej0p#UyGZ}BKMnb;V_k_(fSo%ATZrXk&v>{ zav=~v(ilSM!7qkXOP}iLM`W->VL;1uM+FGyBdBk?%|yaHyQUq zsF-}%Pn-pAV9>-gKluIxMhS_^00N2w+xl#W9seJwd9TXX$Z6H;HOYd=*6!T4z|us~ zBN!=O2@Sy}c(ZWUX+L-|!1U zH^MjtQ$b<-EOMd<;j3Jd|B9%Z)wxu=E@&!dzLh_D1gIIQ1qk9)^0BVSZCt)2gQ=KL<25NZfQ6(|r`>(R9KlU(wqv7Bb=-1vpotd@(*h!F&`wVFl8D z3>C9HL^9l$RhJE>b)2uUQv-_A&*0l!?ddn&V37{c)rn(CFiIpo$z!!5RLm9E?LWE)l(k)U3PEkBQyn$f4$GM1daVT z?c++>PUskRn2wLE7;q>={@KW6I*zM0|lwq-m0f+mWlL2ZZ zZ^j@3E4VhYD4#%eb`(H!;1x1OPWnqFtoyrK`&m9DTp`>>YbWnz_VCY3>Q7#aQW=cP zma`?lTgr^?!2Rx{02ZvS|TH@|N4a^8f>W*$no)j6Ow zsU-P?v3(e3B*vPbh@XUQLKGW8+ZmWy$&=7DyXUs+FA&gxq^wUgBL}4sXsM(+W!mUC zXo~Hmo=UqzIQpb0%^~mGC~BdaWhLs|G~Cq2EW7jd9TH!`YG6zg@xp}DuDn+4dIwJ; zg0f?GA7pOnlHQnWg1Yy8`F?Fqo%y!{NC8BQZWeu2K-rQCc@i|xH5=hZ`@iT?-?l&g zTYwq)0GE^Z3*!QG7#?k=-eFE|txR5#%%3zy7azqSZ~2h~|+m02LO8{FrpD>VT{)06$7lBCw2lPmJov4N_Yqm#o6(^0Noj&Z{U zgcFO@EW_oWdhpUZ%VBk@{&BpHOO_0EEY>|^4`_9@kHvOW>fs7o;#IPe=oq*-xNbIj zX*O*vBWOAX`84BkHawBipg7xgF}`{GR?QgndDNMuv{aNtR6A7gd{xBg_auLD$7t@4 z&*=LXNNuY$aB%UBLM3|6Ns#4ZXj=Z^@1Uc}z-E2cO?H#`iLtWFFmXy|ecrHbsOKvk z?qUNZ{GNc*n7wPErfTHTD{#`b5hj5`yVNhM;FUgbmbU zWz_b#DaM#70-l~Oz8dc?<=PbIM;qIN*zy$W0ok-0kltBG`yJn_7o@!5A6)j7t zez9wxE3-?$*3w%+c13CAz?k6oozsU^KPY|SVC?q#nZ0VzrgCeWQ=yJRR%=dfo)>2T zN3N#t5THu_>De7b!!JHzN!B@2pk`t>=`J)$E!W2bWieGY54}OIf(!SqZGLz20Je|2i ze_)=?igQ5H!CncGxi{Qhiu1htnBBq_*)87tH}hHREm7j|=ZOb=**Hk62YZ2yvI%_^ zIw$VnVdmT2gEvb-CXcM%hK(fsu8B!2huMy@-x^o8D#ZTX1=VOt#yk4CIn@MB)Ug)c zyB=&NjWoL%m)$YiShK*z`y$_0ZVTs)NkVCraVgs)##FV!s6G!Ty;c3D)BrLO2xGf+ zrV?CkGQ1!Ssqe=!4-Vc%f`Z>|<6W^i7VRaAe#@c-wTrq%5uld2 zd?@A9a(9scb&q)2ad%Jd0eT-~sc!ZC$@<4tzk^_gkxjQsV>BqC8grT~m|k8FTu}%f zh4tIeK_ZOHB|1g&2r!0#iVPk)4LtTn0dI2^4_J{mVBD8C42e-qfh7#fk&=jkbZ1oR z@t=Fz8?S$OyXB{1ko|Z$(nfG`NuL4vXyjDgo5|Vs;us zsg(x;?J74)TP=f#FR0n;J(Xs`(8M#~bUn68)zf%vjbFJ;tU(OZuhu|n1(b>cDp1oJ z1Xz3sH?`Ef`rGSw}A&GP4TF%UN^bCbYQ~4rRdFD~}go`DrwleYNq~LkxUo z1WhY)nxM(?OAR^5fUl8I&-I2lyw!v3$ixjef__75Q zu}P|?xMp@4)Y{uzz11*iryZV2CS#D8ID`KNbU>8XooRC?hdNpZ0b~_TTz&ujqCtI> zuo5DfEZ^tAp6!8yD^b9E&z5cAzY^;E152uvP{yI!TM*$38t<(PMeTHe+nX>?4tnr2wnAuMU~L}CpVHqFDD9zEC4+XX(p8xGk?zqFu*X3N`s{v#=dJ{i=^s*t4hp(!l_7Q5 zYaWu}{74L?DDoh!yYd};_bdKeHOK${t|U|bMza6OM*IkY@JCi#(k0h@5bosOd#5qE z4vHq3{V|;KKP$0;@AO1}R6+2ik10xGS&J}~E>anO6KOzs>+XoyWwN{itO80{1%Z{f zDu&Fmdm+2m>&0mr;nsn-?aFop@>8junZHM!;%}1OF^hCW^@UF$9IBPa^0Q+u;*CUf zyWig07DKjM_yIVL0uq&~u5ThgQCS=DaO~kwY|Tp{7eHj9hFF0dhL4~)GVZ1`83zgL z<31H+A3MIs97NqKIRM9I`w>OUWy-&%C$=l&!V)&gOHFGqCB7qeN^aD7D&{_0tsak2 zuXC9w0i|%Vn002+kR`oK4g9#C9@x6rc7e;n(1&+I%Xhn<6LN=ELU!* z(Y(RJVAQ5H^Ii`@rvMhk6h3pM7mXnM_bc}D?jddY4ISA0IYsINXLJdc5Z{LbUQB&Z{9lNwjuOeZb*u034Dzgz#l^qv+eM5+gJXBBs8 zR18x#e#13?1#_6?<<0bgW85*s1QF7mo||q1>!MwTb_@p|@0Mo6x-GPVAvJL6FwJL> z`6p&AT=>_ecIP}2lp`hFBg^)DiC~~?DYcIpxyAjzz3oENQ5{w-_HtkQLZ0V-3ce2e zargR$I5kY89_l=O@DB4e8dg%Q!BW0w{9ZqMulOJa;b@!o;;*hdbU^UQESw5A zl|B;D$_uQ1=$M&Gt?!>1;an0?LVP-VLQ?e>9Y1D24|yRznT}t{7~FjsVz`}D)>|dC7S7S6JGUXLrW#B!hC_gSOZTcEs;xPLKkg!h<7pI{oWTiFl+ASp>s^`HsVz}uQ5Q;2yeof;G7lfYF z=OmEQ9kyW-!*4m_;?DB(HXs*n!}VX}RK6fJ ztiSy!Cx7_vNJ^#O-8NN*Sv_m0#k`-gAs@mGx}0NZD}+wsG6+0yfJ&NN^}g{LSc0Jn zfJoSzt&WWaT!D*nwa+whIGvsOeZlbVE13uT!l)!23sqy1*8P=-rtp(+f#WFA1Idjc z?jMmHiq zfwv?$FT!6lToEG#_yxEiB-O4Xy@sX0|H_X;mN7HoSbu;!+EAyEn*- z^d)U7px3r~qVx>>(JSi=W({nYO?M?WHMM|`8sG%48e1IGLrlFb2!j#L@>=$Zj5(CP zkQ62Exv;J=&SOkCF;qG%k*#&Lg;zUyA;JC|&U4g!lpmKeWaN#pmI?r=X9bWK^U<@r zj)SGXCgsL9yO6Hfl%%<_&?GWLlUoV^1owxu)gD6PEda*RMi+i*!Wp2xD`1)O0O{_$ z3wJoRV_J+iESWEIZ(6_Lr4!DY>NE1+Y^;Wo$nUMblzk{dCq#>z>g(arG4Ahoz*?Ea zTxsVqNDF&4M6#_&0663htKU%6nkx-(5H=MijxAt_;t_~>`(!K<8fN{{t@O9t88kjK z5!GRZ`F5k)FAHW-_EBDgmHbMUls-Br0&y{#mBb-NHQJfAS@^-W8uY zZ{JE=YjiKr@EB5TaKL=!-|Cnq2s%^Cxj+`GJMK!D$$hPyj>lMGr0=%h=42Uw1?R6l z@`XOAi09?A;+Gr)@FVmV^b}_z?p%t*{&(ao-a-;F&N^7Ee)`5a02}`KStd{W;0m0u zj-Ex5lw7>>H-cd&kd$ns&Hpz&w_&%&Oh6>NB8We(TmdaFp3Ny)|VgXQdHcFE2hFKIveWq;MkmCnl+Zu zRGOp55{+LcoX0yd0f^R`q5A&s+4{SYcyyz79xxz2->&&U<_`^JD&_o(R@`pH(0z)> zKI^ysfNK}R&zJibAA}e>!j+@n`E!`0jXVX-MMi4S{m~5KR6@>Tp0*FM5hG4v9@74G z2MgjVG&%6xxLrHj)?xiKNh0DMV(0F_lS%|F;*ALe)d{T*H^l?lTZkg(>GL z1R#SeV(u9fD|~ z=aM~As0{5iR6b2uPxv7}{nI{k^kx1ycrIZbCu({6`AY7QzfRcy?KwciQ7PT=;aVpy zYaEcq-@wEigKF+F?UhChjI-GQ?0`N;9~9X;F3+;*f7^UFX{%9LpLq5bV4Q9rf5Eo> z1Y@j~n9A}UP@jgJvRih$C%%eew29iJ6r9OA3i}*RzpGoY37>qBgzJDmcb161YKO>- zB+kQNA;komRs~;DT3%DWs!W6mu!cbN*qj^B@>@&yRku0m%FzrS`l+8_O&eX`zZL;L zKaywtKnTzZS}izADGR&@nra@fr#NN(pEQ)#dRbt+u?1`yj6x>5sWrbb&uPsrf4&#+ z4vHET1ssZx4PPtu{GpVGqVhrk$j)zY`Rrl_>bd+(n+@R{qeYGN6N1FNpKn%{qvL0< zg>!AK*WP)^*Fz^W91cLvHXufiH5XwoITgy4MeyBjhmOLTmv`uWKktuYpfy;OLl!j% zP#mW|9g7uWI=3yb;tt`%J9v>IILBehFIjg9zB7VaJDCMx*HVY|wlEDRyeDgJBHPc< z<7M?TxHs6Bg^7p1g7XLdxOp}MvX^Er87#=CPwyLIMytO(R^WZ+>JWiA?t)V0>RnRg z7dy&~#x%q*7xi_0C&I@uTchAjwu2DdPHs>q&%a`5*ytxnGJYY2N4xuFG|bi4Nf` zzlhv&QyeDy#qAiI1b$tq4exVd7D*GMB;GF?gbeDj^x_dJyCDiNaE(HGo%!OUR&J}} z8_J7OBBPQJ#a>Cp+0KlEbNuo()2%MblY9M!v7k3*duHzwxB=4_vIC#0@qGLO0B>NQ zxo>$zhGQV&FC!52KY+6RVp#xa+yDcMhrw>ey<+sN2A;DY(k}ruQR)zuVNWvl;}inV zDL5Ic@uGga_P_K;#oi#Iw%^fzo*!(6WySajoUB7Qt9dB?$+1sO2C_&gHKe~J)@LnI z_0W??6x+_9dhKo8R$r&HdFLYZqGg|oWNtE7Lo5q`1q4wrIFD9FzQ-w3c(WKeP}w3S z(a?{1L4O8PP%Y9R*>S_J;z(=<4|4Y5iqBvQEob&ULx?gofAZw#)b+{){jSw)1WvPq zFp1(H8PI*xaI&(AB-L|i=tH~X&WTS5V@?84)2_RZY0Af&J@*0%t_(eimp}N<-7KjI z*{L{iryIWU58mAb)9*-0%+_~kZVN2x|L>S89cg1eX21F^J za3^x*!Qi~N_TPml8Pq60oSF#rVNlE{XmX<&#JU-RL#*7`t0XccY1KKckY zAmGxnc3?o?W}*N$Q@Yi)<{>rI>|05838FK&=c+qv1cL`SgX;C`TO+##<8Kk_Gb?aW zfnnuyKWl6&urGT#9zJ)*hUI0lBs>??ldd*vCiFap#PJ}Kvt4iNpsbut1Z?gr?5{Tp zv7MhnstFvW%6*4x@VnjLgq$UaOhE3$ZZ2c#cq-t$m}?OyiG8v7#Qs3zZwTZQ6`e|lQgX1Kg zibyPT94gEuw)Sc{#x^+{TsyN&-WU5b$_*(y)bLWXV2uFo;li2VeYm%$8FCa+v+tYl zKNmb&!`VuGN$PrZ`943#LdD0d`_twXtArdV0OzGFZ+l3K;MgRVe^hm}u(nm%4X55GGOa< zfSc_ldQauX9Mq^ZxDJquv`;zjDft8kLmON$!A6}@uz)^o`@wK;lEKcxQ4TAMC}^( zRHU7YpmPuN{b9?%bL({c_^`aPr8BI)XW2gIZ(u);d$SkP%Jx??x_)ex@_uSQ6V1bb z6EK{wbJ>GnfUG-%+@&cHn)i$McIhKqH~Y#CYJdwAP+h9{sW1+mjP4uH3jV}S9*3UQ ziraMiXfTn(K>4NnyD0R8IR7)6$F0<$Sv_5k-Lke7R=OMIk9&KRQUPuRL&-t<-LG&hzl3;$mon|Z(Rh1KjcAv96y08F*6sqj=C--4503y$f3Q@(e$y|uco8*&Sq;P-c3)r0s zvkb~CWZ&N`XbZo4o7)WYtdeK%lOI{OtCwp=fMhJ{$%mEyxsBG^A&?;0{|I&uHQ-{ zv~i#gM}kpA36sU+V*9WTk+-rz(H0@X*X!X*yoSNI zq{kK|mtx|0pszDzFZl{$?H-sW7L6&?61dfV1lL6@sSADGd`Q_f)vpgA55k5miXEAG zFg5L=Pmy>^i8MaersU*|W@Mff$q~;F>_%=sVRteJ%NT`vx zJMWEzIO|S>n!pyzz;8IY`|550#TKW6aT6FMsboUh-p&?xJqZZH@vz8W_3?lsbT9Bm zXVzwMI!@PqHz8(3zTrBA^a!S#`QL^!o3w(S`|>=-XX+4)>6aw>Z>%Ouj|vl&MZ6@x z*%OY>M(Ii}q3V>}^C=ygM~65=wt)kxHgeaXOYMLbe(r$pTbU5&<%?^B&%a^tn(I)_ z#m=W^wvmV7p6Eqgkev+fe}Dp3Rt;~jlq?UL77!c6mG{4=KLSJ|oO`8`JSWN1@mTx# zaHn0}i*Vds-RoN&q45hecJ>`Uu8Cw@JI^!2ES9?o=p8dMfCc|^4A8qS)696WWo*)8 zx>{qzyR!p*zj zm0S9*u=Anx-s4xfh)j0NI|J;q2sLh^?Nr$~YU;w&*{?N+oL8&n zs&w|1W}d8$vDV++^!6DbO(l+$4p{wgfOy*`P2#R{=usJ1mlw z-)e?cbmu-2FV#JL`|46L{~%pFK(|yoGyz^D2Wj%SYw`0xUJteV5ziOEZ~I#oX^%A> zIT8s-zx5k*g_7V?5f^lNfP5AP6g^F@nk){PMJs=~!o0ROeCPg{;%Tb1*U*p{PsiRT zjT(j^Q$kqADWgj!DsV8+pctPYR-Ypb=rDTaYJ$08UqJhr8<|5X8ODD@j$3kDi7KtZ z%QpN_cen0&#RaG-Un=1FpEC&D?AMhr=NkHeVI+r?f)p58sGa9$wAAt8NC)Sc)I}~% zBW01v`4htNFT`dV$5U4>f;Qv+ri0_LWSj7w{Qj77_Wo=TY^(4Y0%ieRB}VqGB&NF! ztZP7*Va~rb(v@PT*Rm%QM4bGYd7T$*n}l`%TO*sQMQL1^_8|aX!A9m62h{q$QO`n2 zN~Lyj$up;OD4GQqvtHe(O^p`zeKlkk-6d82g(|}ndbFJsMByJOl}F;L8nGYipK8MY zsV6GH!8y??68awP+ihVV>1Y2{6!pRUj@%ltJZI(X=eIu6HOEiZtN2AK5 zm&OB#4;A*W?;F0?^j)@iX|Hidf#x^3t29lLc;zhPcmjsYBG7JKUmN%G3H|gc3>WJ4 z6fj}A-$tmh!xVN=(7dG{^)w5!YK?XA5H@Byhl1p}2a%FLJNiDz)+Sz1jKt~6Y>SpK z`39#WRQ&`~J<`&i>~(gA`bI<1440kX$J(=o6ndbUt8Dl<4}~4G3D24Poo066dNrel zWR_U|Gwm=euRAAseG#fD5piCHk$e3tq4tV*=vV-}XBMuy>Rc>e__)_L zhTnYrnAa@%YiV*m8V6sM5qVJA(q4bwy8azKaih117!_KW2y$~A3gm>w(g1o`q8qq) zCJzqIrShPE*z*oGSEVQvhgT1+8G=GeAt6!Hgbs*^=mq4XGk2ww((`_He5e4bmZ+*M z(&SA7QVWx$lGc?CwGCJJthJ?Q1@E* zAa*0x+ZGP*ER{=yX9?JW`XXq4e6BWRy<9GS2vP{fgD z7J*5xIewedXx2@6JF;7G=&F?gY8rzulJdi@I7i!pt`@1xo%6=d3shOF;D(rrdI5- z06_@W!1U3QkKN52x?G%FsYx6x7HL!%dr?O^X~)vTX-Md}jJ^40?*$aBMGKHgLF(zP z`vHW?Tp?axY`io31;J3yMwOjiD;Ip?ricc2!+8e~3aEIu$OJtME{dK`Ac)6t$B!s3 zonh{3v_4Tn$X_;=`rz>HhZt+1n1Fk?tkS831KaGU2=B3ULu#ksjyhxlTfq1Bzq+{2 zXNmU3L$WMFpv%IH6-r1*|K=O=f`TiLrC^MJ)xsBL=3tO!-7u+LAmh;`)O8D(v6ct7 zla5h7z-9~i+45?d8C%Ve`uIV$38NXNDg$4p#MWWWl`8|`rwuvqCQ6V~VARepDxsYt z%Y;3bb1mi==|R;JUW=2T2Ulwv7xNf$ z)}$a5*@fQRgufNgSzhDP#qca&Udqybz(K-htr&$y!Mh1Jb54IDVBc_D3!$DI{ZKRB znX91C1jvA1C->Ti(uk~rxlRNf^{yoMHEI0?4DO0piV9u87*$6f&%OJ)#a+%%Z@?!& z3IXn9uTwCiLwU(AEqE00N<#}ymk#7`eCx;aD;;( znEJ*6$U396XKK*)VTdX}5Kftg2%j^l2irFnD8@_6w@e)T@;OcA<}nq%1*Z5m>VYKx zS({L*O|8xAw=EQlBNcF(sCr1c-Y6a59w>}tQP^P~7B^QP5k71B*h{Wb8Yq#0@uFe0Z^+}{M9W1 zwWG(vW0SxTy805X{WaqW;G#d@)7*e{ULbF=U2-o{0H+(&5lDQ7@$%JG#hPyiJ0AsV7PCB|Rz z7YzuphOCCxV5CaJzrVk0c?e_69Bxr3!(knFnFu#WU)(qHv!!~Z6)GKba^}%}QqHW! zhsOi<&cBpwUI&&&LfYrsH4(las>OeopZ`+X!-{D1yHm#1b6rSc9kf;&jg%2+2MtiT z`n1g>WLo`T6xX%$4MyNnS@KCa?iYOaK^d@G&U0maqD#OzBI&o7&x|DqB8Rd8Cv1h= z>#w_Z;*n)~%S)J@Tp{ixfcp$gTHGb%Oa4KMFWtq9gO7nz+^gzh{=Sq*E8#?|3K1MG zP*f0#nEv6*-TfhgPZO8HV@Q8kTEph*NKi;TXeW>{Hq2Ri$47?{=M&zEtI*+aAPlMY zHzlWht(@I2Byj(^aU)`Z=W; zQOuq|1W#^TG`|7eXSQbq#PE@t!aU|}CVA$iNzuMb4=%R%UM`PV>>ziB$&-A+g^UkD zlRDeF47#)LP{e>DPq9dLO(myRQyBqIok*tvm59K-wh6hg#Yhl051)5@Zc2b+4Ad@R zT{^l0p31S-ROL$Dbu>N%put6d5cO@^6OoVAfP^76ts3zfK?2Q=1XWbOAY^DdE{%tYlUm|-JSWW zIYUYdAq)_#l@ff#W4wU4(+qM07Jy3p_^OoeL(tT{1@4);fZAH0+D27{F;Yvg?H;hz z^Ikkx8(OorKeoxd{f=v0U2O3un1Bcc`6VjdpD-a#O+yqu9ZXSwsxSexj6bDYE|R=- zE&mTYMJ40Subg$=J~9;4GRBfHTnHkdvUGW`+-8>V-K??2uBAO;Vd8yOhpo|0qP9Zz zGU?1?9{6{_eh&X$#aF*<_yGd;1*lauj6vga1}D&H4f%)_ag^K~-+pS|i^r3yuG@x8 zJ7N_s-aKzWogY~=%@wY%veEN!m zBAiHx1utdYXUN2ubMo}Avo!gEDjMC(7GNtv8p#m)kNSsl2Ulb=*XR5&5j=FAH{GvV=m)&PL%K0aA|Q@=o{uUhikKw$wJqu;LHyyYJu zZ0dGQsenq(s9nAUh|l3l-%Zn)Z(uzs(nu`7{$f<%J}8jY<_{14sPcR(Y)>U| zPVS9v{!c3L|DGQ;;p`&1m`!$x2-%0LzCQyMAwzS9)5|VM^60&~4ZYWhJ&Whsi0QKX z^|!tDTM-$3kLDEjCXMt)>^NLLjnZ#iknk*$27yx_E6{RB{wsh`&ZgUfJL!wge+1L$?g+L1hG}VfGTU{4zA!z#)Hg86E;K_a zbmnh8P|?4^_|ddG$a~7@#cTxOjsOawW;3(%%ocLb;GI%7s{P)%o3=>tW+c%u4h(0J?wS z3}|&fV$EMgmnU6$t@%I20u$R9NO+(dv#-W0I1zwGt4FMaET4|@r$P$trN3dHSV3U` z=P9iVXw=^qY0xC8@i9Gy$0sG13yaawX(}#j)P0g&*K&(?iocA&oVR*|Y6pm=ri10k z4};SokH-Fee)|F^r|sb!K|Y9=W0s#Hr>7jNU<&>rC3Z!0rL+gc%_q=r|0!-Rji8X* z*B=r8_s%N82vvZt?x}q8+CY2vIGN%0vaYROH`_9Zpq!=%gUbl$HFS;{zlUoRm93*x zfBs6(rYJOgq>P^v3yQzpJyMld8W8wTrTaYbg{*p<$NR4PQ0_xzr1%ArCVv150EiM_ zOZA@dJcRmsm9a)beV7py))zeQ24qZtamY(SGmxQs&A^EGo3Q8hx_S}ft!p!n}6RqxQVR2=?zwozr~lyZmg!JBxJ#^G-rhd<~yt@D%^Y^C+kz; zqFNVn!3OB_X$`o4NRDQ(xiGMK7|gHVD}?ATVyXG93Brp&lWd>^4u^%>ut1`DK z#4G)ObT~l=4Z7msIviHuKA8iJA#&@a{~L-=HD<`-UAh; zd07vtYarLO!cyDHRe!>~)fPUZvr0&VZ~;5PS|mt5B3&*=>m{bv{mIyxR?`gAq|)=n z55V&b{qrW{(lwE$ZW_ciO4z0qx6u@tcFuze3(>avwWQNa!HO`Xjp~S z;xReXXiN+_={35MQ)*V-*m%4dz;L4WyO4Yk9Cmr}RL_}*F5aaz3;-G3t*mS)H|}c% zWqb^)ilAllpt4&1^ZU!m{v$ytQHiL2-q9-jeqN{>G;vY#?~N~>1F^wO(@s{7n(;}D zW|Zwcd%*RwQ;R-}d&_1qyy-JVHbfFB!HzULqvyIr=6Fjx4U6*wfNml&{7{a{5#-P)(p|Q&y+PR%Ecv5 z<@Pku9GkC-7sF;ew>M|Y^qd#Vul`c>BjiRekAR2abzvQ1Ry9%5kL3;_8CD;cF;}t+ zk%f%(e@2usRw{W0<-a0pWMQXn-wTDr#n4zfLgO3*(4!wZ4e!ok;!Sg6_#fzkYJ0DeNcs^pk@xOqPY^~=UVOmL4y8=PW z+TP}bHRl{9yYLmO#Rs5Nx-6b2$e9yio_K3uBoz?1pxotFtbF7B>N*~J8Fv?^+nFA= zZqNh+VXg7J^4527bvZ`G!8`2EiyvH}6J=l|;d=J%tpuBASGeQ%J)k@5ZdMah$ruBP z5wsLO9qQm(ZO-o_PF(Ywp`G8C(s^u99&PnckI3G?RraT#+u?T>GVjU<_fY-#^yMFJj zFOsUL1gSdPMm%|uaPrG0>041VP_oKFIsTdD%+cX_Q}pw`eb{Uk@+)Iv_>}ASy*wiy z+W!8$@Ra02)gpBUgsQ`&1y*vZV*24@T;|9Kk1TOT+;oJNPAb6&X#k?2rB88S=G(8g zp~gIvo!`(@8j_y7alt-Yz_yZdAWbcif3yi=+rh<%chIcH_Uw4w_^b38q?6F%E#lKw zuFUn9~Z>cOVc-y$zSPba>nCs z<`~EkbfP#)RQ(drgrIikeDf1RPdo^*tK0dRkwz&cSCFW;;pDWw94Z*PC(g=<>M@wZoy)$5gV7_Tekf zOag_k<+=tUOQ%D12ny6$>i)R-(@aYROxTO|6nBDz8k{jX$7P&%dSH;XId+U zvqlC2c!`;m1~nN|KrzP0{o|U&4L(=WIO#Rl%Ad-&zS*-{UM)9yo&? zeDU6<+IJQfYC`)LE0Cq%>lfnWvhRqp`fO@Tg*t^$!dB-x-hW($MXME=MDL-tDguC> z+mRY7J0pZ(WJ)|kl|DX%eVf$c2U#eRlwN%}Zf&O$3$lv)8#q&-qiBD;U*1Mjwo!gk;vEQJsHT12=x*C0YdNoGq3)w`QQI5@zRwEP=<5+3xmxwHN$>yrDf=8sNldLz1pq65=&D z#X;^h+Wl<#M3fODf9-Azd_2GU4T}>9=U2tK&Gm<#Wk!VhzWEp`F+9Hd?rA(Zl@{l0 z>KMbQjBb|H&z?I{;px#?^iD6M^96g{J0_$1Lc`V@8>v zpu7dOJ8Zb`){n}I7Txw?4PUrci@L=%9D$>fSmFq)nN)o_HGlE9wI4Jppp2x}$_J^^ z{Es_;YhHx|3)`(Rcx}~O@m~Vu-+u;kw@33BVl4HXJ5}ej+ddgiake$&ExpSj;5~~h z{nC8ft^ZPecRmRU8Dh`p^Um}$v_@+aFBrTLL`~X~9Zi$8na7?!KXOi27%*JS6Q5FT zJ6Irl3x*fqrknySxW ziLv1~HRKLtbqOJevI}|j(z52%IV<-`1J1=V%7Zf#h1DjRQ8b5is7$-8%lhs&m>#3) za}UHXCYY0_G+O(afqp7atfNyb__!y6uAQjog-znkrT`yyiB~y@%zH5_GGT~) zv}nda{8y}xrK=o2y>D)k%F1AYmQ8Z)ceQj^$#gl4w$-q?+sot0Y@6+LBk>YGqWMs1 ziVWR^=^>Gk;1^H9hPX$n_mW7H6l7Nz5-Ikkp30A8YGiza^-*G7@r(T`fAc{s+A|hi zaY<%dXb>7tkSsLk`i6@;YH;tVhl{^cs*m|Pk{~k8>AFVaNI2Xw1!VaWbFobWl+RSA zGCzMCwS;O+fE-QhdoF?XTj6Ekyd8t_={j&NJ;6*x$}>mLqkA!$qpFj#7xs^`jb!{c z|14wEZS&)~J0P4K|S@;KBOq(e9qC zJao1b3gMBD$!Vwg0bMzRcGVHx`p!Z#0!lZX&Ie1lN0DY#m>#)}cOvxXJrbiuQ&VG+ z5Q;t2?o!}5JAkCDY&4ROa*_A0=_C!p-VSOk=}hk5w*eXCeGp?>ELWJloB{|Hl(h|| zUtdJ4A?Dp?M54SY&6EJ+bk=X^x56aKxZv9<7BIk8t0`<2C6-$IUx*!++ZOBU0ptcB z&0J3v(P~6atJ&3cDvDOQk8_R#5sOa5Unef@AuLVcvg2qkmS7#2@la7Yy#d4ib?4Wj zTjyZP-*a)gk|XIvO)l%He963Z`m0nBEg~x$a;Uj5JT&M^eesK4i_QYlf#J#icaEmS z*m-<9(4B~<7v0PNs1Ep`I>iLH`c5cLBueoXr~wQbsNB~8@$vS=%Rio%&5r=o76&Yb1fqJBV! z&_}uIM09%2XSC1Oh1saOfSoOc|3<+Z3~zBnu*lS|mKE(p$KFL(WQ0b%jI?Xrpw$Bc z!nD@q?Kb}l7L^L8R5X+1)(exJ-{r<)r%;y^{M4WtlW4}iYF-0|0-Q0@*-w*Gu4bjH z-ZtdKqd!C?Y>vEM>?_yeL_z7>nf3XwjEJf-IA})O!HOgJ-i?i(&0=smF+CcCmRb+! zUBJE)o0$32Dq)of#Tcou4T^A+$7oS3vY9(F_0Os0cf{=r7gyRu-6^u1|?svU5eRck6ZC<%MW_Ju+1 zSrr1S;Wvndx(CbWjKk3$4Y$9UHE^DSyFUFsg_F>pubp`V{AWED!)Z2fHuw)w3T#+# zebt|{0%QQD-`LV5W`9>+AZ7!+7CR|&F?9!J{P|ycN_{pgj+*9WKP_u?kSTJ>hIyK$ zYO|>7lcUTsKua$7y3sVerNJ1(=~0C$BLTQDW&DM?xb=lWUk6OXDbi#7u!yLabH<61 zuJSCu05?hg#i$05L&!uGDlDA7so za$&#`{deRnS2-wpS?_OksIwi+RX+sYbpCGmj>gQN&$Wu_vqHa&vYOPI((m$og=CM% z7b@XtxADN!&VZ*)ihtTl^?^EeMY{#@@|ZSlVKuaYU6}dS^HMcGp#zh@rT2fD0+Mfn zMS|)$ev|2B7UuEA+Y|SULffosVNksGY$0#(ng+qE*MLe_ud|VI8DatBy-rG;rUA#~ zK1~8q-LVHZ`X8!W41w9R6N)01L9F9(`Ftzt^3%poy8R!07w#y*agrMcwVGU?%5NQg z|C)2of$;J{9yi8Ugm*Sa278+`C3KsVB||Gya18vHrob~53P_a3C#VJU%LNZi z++Q?P-`ASTx2V*j3O1m$3mq`{AZnZnRn&i}2ly8hXE0!^3*mRb14iv85b;yHO|Js^ z)W|hy@2rAkzytDzo5& z?VE-5nJ56IO7sLhYv5Ai<>#t4#s9H%|IeKdwxa>_N#(S8opzK0lyvwI29aYZYCs$y z1GZA&4qf0V#k*gPep&gB0rDjS7&$b_Bg~Q95pH;(8}C%XLd^?a-sKeIq=8TCxa)tQ z_Ad|w2i1G9h0TJhV|5>ZVl1dZx&@CwWv-!OWm0Ouv06^?|9B#unqXh_s&Bb@d_n~o z>%C2@x!<@a;0=d&5}0$nrNSxc7esitL5s7Q=YbD2?|@3VV#Q^0O5+|3@T8{*Tl&D~ zCbFKbDS*H@c-UoV!9&$BN+RJN6pq#o9sFW@1~=D9FdI> z4+*AeG2d4BV*r|-%3A_E3R6^8pccFxrlP8fuj~15P68-!d0mDwn35WgGO+$~p*YaY z?^VtG4fUFHN;p<7?|pk#>&Y#Xn(^wG3P7$c;N~$e51w2icLg3!T!nqG#nxU(yxVee z3LY`1?6j&;pxPX;M1jCwfI6~Qj)?;@ydnozU|xZW#|Pjx_AXz3eg!N@8<*vw4zE>O zOUgD-Y>k#E`T=gfX>Bo&H-I1xcf)8m3G6$$Wc1Z3mTINcU2q)9USxp+5+Lo&?08n$ zdhkWV;I(gfTt2E;*o5dlD~#}XmCGMq`4=xh7v#3#5tC9h=~fPH|A|!U6eLMb23W;= zZs}Dh0VRHv44ATSxN_3q4F-}Am{U2+CtRLBxC3;U?}aoK-l?&EY$BJ~#Un`SR4!4v z()}Fl0*@>-Qj}hGEx&}e2g1EB^>7g7Wa_P@jQm)Jko|d~(=(lZ47M76#lU2=06;XH z>NxU)ctY?>E65?>Q0(=3H=qas3VMVf4G@l}lgBqv?}#7z|IzsqL3m~jzWPXf(yE|u z;mNHE25c5-w)67(7ufAU0FjZ7%$?Gltr+@;Pz&;JW2z10S$vR7hnXEeKKMqlmFMBH zx?N|tqq2K@S2MsLg`bvIYTZxT!ci;1kI6Wa$(6VGb{|?byrkPo{cVQWA`lAq7Gp03 zoGjiY6B_-2@93E>4zlNR;wSZZU>iR73P{Sr_JN-9zdf$N{YAnWD5fQBUl1DS5axc2kk)!=lXa|w36n0 z&mMe)a-hm>ot2}rjD&tf*V;^;8+|B+SIRE~%t-c~$&J{MH8EXEq1x>0@BC64Rqr=` z1Cot*t!{!Md27r7dptVt%POCPU_{i@)JT}(@&}5OkpjN< z+t^>)6N~}IKTl%a%}lm%q_*BYbU5}bRn|6>>prmObW=y{Y?|YuEL616+=xkf;%ZCN z=O0jZ9v$N%=l109XC&`-KDLS-c$6u1#9*R&zXwBT%T^$`Aha(ZP)c<#0GenVlf8w# zi7G@h(2Y0=U(S0oFD@*Q!7b}cAU9Qstr5+<0D~cqKR|Gn+nb&APsV}d5tQk!^Bf&-Hte#hp@-rY={0H=23T6A4PD*DHspjr@Jc7w~Iab{bdB}9YXYw zxil^N_yg*!exKU2$j+1-qRW4cKZwhSwxLFN`*llH*xiw%`8z~Q(n>vspD+dZQRiv& z!)Lr8w(?K00cFir5PBTM_^k5M(!#!qu*PBAZi2x{!cqqO58wXb&LI;Ml$#Husg&%V zeliyR<&`v@)EU>@`oLc&Sf|ENoAQ$%Adl%6r@qBNv7h}18+hEfUYwOZ^TsWiIy3!3X>yvh%f9RUF?q$rW8_I+!&$m$L82S{VfqTj&GhMC?~p* z((>DRE)iZ2$QY+FIE9XFI}@4Rvw@0di#f?$;yN35i}%toIXA*tc-SgqN-TLv`Ie%U z%ITrHYQ~>)cb(-+vn$VH=(Silg~z-)`?ASB;tnZIA~+4?pIGWTK6Vt>Tq^H&O61@T z(0e7t@ZJzsYS%WFjgo;j)0^wyS zBGZTUf4mOR1?+PIp&WwWnBYwnkwF#QQcb1#WS^-+9iSi)REZJun38NQ*^VhTc=*gY zI4t9roFyYAeomD;!5HD$_e$L8tkbnZf7?peZ;z~Wd3HA_Rxgd?)8gS#VIO*(7|gN`92=C=F{lzdtZ=&DV4c99*X!FroLL`io&iQN~8-#Yrrhnp@Kaa-H(yO_H%5 zT2IgsUQbR;S7pnJ_>{qs#~9TdBOu~@_kLrpLv1NG@zP3Ja8#>BA19?8p+)*uMxKov zg#jY(mDu0@;^GUfTZFdver>)r-yjK$+Zwv~Y%Orm&UdgsUW$+Ju~V)hS}rd#C2W!^ z$|0O^r%&@M1T18V+7kv+8=pC*)j!8|s%-}9#HS8^n)bUv&U3n5C%>LOW)lzPOX~r` z$g&aZ#W5u9{641fZ<~ss<5uCzz{91oo=BsBfBQrf6qY)s;`v#}9+u{{b(*JRodcdKS&0XW;%(X9Rui23g3k zrBYYkc1LGKfLg0g!AwVGgfxMkqe|2~GnQ6um}Ho+pov)jDQH@<96n65R6c}_h=GPr zZm~a@F*61OEup%pN^{lNTo85S^)7^rF#XfRs4J*fe+$=Wh`Ppkm6lsIh&+hbTb?<9?bqQBymgjX$|;@Q zR5s6tq)_h`Osn5<7(S=bK^I3RJ(RusG+%3H5o4p0ZyW}1e;879UV4t;0oQL1yTwqW z@a>tymsJGqmZ6Vq=_~rwOjWd?N=0+rdgzJYhM2C_X3OO@rF&e-X>i;)ZTVfHrP~_( zA-z|h5V5=n)Y}&O`vD#a)|r+6%8Bi!INbk7%7<~y*&aso1AG-H7k-=~=riN@xf1kj zb|&fI!gv0YFWnJ<+g<0L)51mVo%)Xa_px-h%F!fHgg!8H%1Gq#l~ufo8&5wYRWDqJ zVH#}lTPns4I4*o~1LxeWX-%I`D7Xi40#%1|$I+3L^>m5YM8fQ(fX5DlZ?%Nq+g-nf zqmMlv^CEA*+K)R&rNQV%x)G0~$ozFxjV61lF^IwcgMv=YWO@`17x$b>;wBuAU-bj* zM0yXN65A|L7C3M!ui*GeMJMOZ({xdGw__h$BJY%x3p$KwQI!iCAV~y9FlRf35Ak3=rUM6UK&9`U{B-bEHkBwuaXmUAvv zd91!Z&>c>87~Mr&Y^0ZqE}RgcLWv@D)MI~3uoYnO%CSLWore~8q)b;DoF&U*PUPG$ zSMuOb8TxKp$j=ev^b^Sp-l8>48B^P~m}k)Pj-hMSkxlIHTtVd2pIo?hwO38PG5(oP zCqLOEilH&|AMD%RYk%7if2&^H>|mcW!#^w!MzuI7{^F_xnMsh}?~TbZtW$5B2zTV~rPuNGO3rtxf7saIPb*5)vF zn{7fL1=%uYrBMBez)_ZS+tq1g-0UP1JIy{UnpJo?^Xl%fL`KVEc+j>gzLdRIfL_>u zaIRqIsPjb@2TnG@Oxz6)M-f%0Cgri9eqvPMJX1k(MsACsnaES|R9ep19o4&%mxh-gZk~SrOve<8hRE;I9)B{?y5GYc zb=ye6T_BsBIvw=vz533V_l~|r6^2ldmeBc=*y_wXyQ|i-5%K)OEeUM4QwiS@ zA44+7J}s1FMNhL@_RPWQPpMEBWqIz%a(6=bs@E`k{9tdRoNW2QO?SP_+gpC^JE$%N zCEMMzlAo5oB4T9*24acK#0?O>LDh_P7eWcNsn%o}5!pjArzN{6>LrGa7&%#P!nz^!5k|CEZocMks8G|bd+XxK+Fb-nlAy{S;C2KA%g zzA%fzOfXr zx_D$(XB?`}CM|h~S8`i6i$rtJd02@Ks9b^JN7BW%V;uohkyli0ixzvX^z``oJS492 zw@36n$L(bswf0~Nu}Y_Ost#T*a~FPFZ#@xH?IWEC|Jq{HvgMf3csMHWg&=vN!QR@L z)UQPlS?V?!Gb|k~-vBn35py;geBQ}bw+il%YG0BXWATn_3ACFS)=udj>vFFQT6*qL z1?e{9ha0VzvQnuf&G_STS~4D2c6ONC#09>-GafkIJDi}4<9-cHJmc&X7+H==?&y)^+|SU<^&_AU5cO~_ zByPEJ*Y~3^QQnIMp(Eb?YOl=&aq;f+k7&ooe#@2&m#4P12H0hU*FEtip}VT8{wDTv z3FjGVhsaj(uh@*Ucvc~Q{3jhD^TdNOa%%sFHRQ{H!7*Dh%sw(WGhd*>xoMXuEh<=D zF|0q`#763tvN6-SmD`)=p;Vkd6)~I;oF1sCamu4)0ZqY~#a=Rmiz$W1ab*J{dIJ;Y zYwh3gGU8aR>^9acKbHugHC#3^hgYi=Cqpyl)6KE!Y)or3=+-%n&YKA8C&4)_m4;q= z+w5~GOt`>}{7L`WVavc}T8=G?#d(gn;SuZ0Sy`K_h22*)PHAt%&McFySDz0Fm*-3n zN4Q@ReHHPV{M!TJKdcwd&Uj@zh^){({n?Id(U@-v*>0QsJku0(PoM{#-}`z*jELn9 zXNuD>_ENT$AKpl0SGIULOJ!E0EkAUdo;uStZvBN{Yhy`cklx5E_I(kyr`b`u=257y zWCF8?y~lwd`n1@3MmslgE;Pr2nXzi&OSiV*g*yJ5K&?gOci?Kp&FurFKmJ=6IuH%f zgf*DT6=W_s%yzywe@aU(D4=#;GwR1#tbk5{!PV{X!LJ&na(UK!Cl^DXO%jS+WL@Rm&m6E8ZMu*6&}uW6lm<9x?IJUo@ z-}4*4m{@T4>raf&ykd9lI}&n63}!Oi3*4hMx<`5@1A>o7D$j|T`;gk|%CqE|bZSMs z>Sd>REWWrLRzZJ-_9IVR9o5*JX=%|!s))!l#lujq8$2g8OfRtGH2x1A ztQ;S@u3TuoO-6>cvcE~?WTnC>$ib2`+{|3ekGwBQKY8QKl+MG+h2C;X3Himb_#$fJ zM~yngs>?O#hX)x8?kbaCuULG0Eu87= z>pit%emz$*W7Z>duDbArv!)a)-_{sd>L@;WrM>cgdo9vCbe;0nYt%-2hYObcv}vuL z+NXBO$melY_f(TR1+|7F%{b#C`TY zp`wFaI(~+mc^)q8u`CZKGeWH6QFNVNZ;wb?3ZHGprzG%+-@{aIDI-r+h?R_rh;1ys zi}-R$sK2rKi zZ#*yu?5}0MRaK9~Yy)|E6epyMLibRYoiT>tABd32hoOsIz5p-pJC^`5k;ch8)peA`6O;6@g>C9kHt$h~q`;quiAjwIpO zp;&;g@35r^A5>NS1mP?$3HAB}AYPmw&N1SLv^L=i<9RZFX<|ST`&lGh7jO6Pcf{34 zeC8G<>CMX^la8)OX*>Fg6?-f_waN1=S#zXVKy?>ttVmgN5vJSMA?PTiXbcpZYpv9$ z@@vP|)zUdy5Hwgygf@CEO$YuI)QM94WXN!+^jB=JAK$zfbozL;`RvX=u0e!(|A+rpMKnhGoPeNgtMn`Z|9&L z>$F+N1HiWu2|{{*W~~K-B6S4#eT{IHVz+5*{rUM9*=HsE=h_lP42);;hi^yBEf4=l zlc#CDyIiH#nWP>hGFwp}LHhoba6ixz-d5_7k!alLJsL=LvAkD_n z=loz~*#iBtN^jX-Evc102>}7=u_i(l_{;KD5`yEPPol$L{q9yWRKddeeAl$axWJ zcDpKzcCW}{5$5UHulZmD`X`j(zaKlv4UPw?8!v>5XAxqjmRA1RV{2{P}r$ z(d+rZUe`A?y9(13fwu}s-m^o3zl<&>ECaol3-n$Ap59xiyEz(1r!Qd11;6ei*5C#E zEtKE)soQXr!m{2k4*0EQU4&}W>;==_*rGSrGp|R>1{6qP!Li? z2qTCWG{R1}@6pg>&pZ~8Hj6JL|OCq5LHH0773}E;Ii1@qvOQ8n< zX(i`iK`4bxzj9)=v7}9(JG$?Ppd-smp$kH^^EVRo>u7Hq-X1gg9aGVeMzC1~)S3Ek z%Mv|Sw@1UpcD=9tz@IfKEU9RDGmrmr{Y8V60NBbhdGo|LqAJhzA!+Ulk|nobOMA7p zCQ|2&tIumGSdT|w>C=a(na%lcR3Uy788 z-1I$3aUY;bxOanFi1f4eghJXSgTeqA`e0$o(F{uTWT&F6QEPS_-F?9{GF?j$nd0_R zjNr7T7O~#AMJPw+DZjPte-X?5zzba(eX7C3w8o1{;jqQ+lSqsOSkrA_D6yBRe11Kn zh5fkOv(vTA{k2JT>%1P*#D7X7orb(JCU=Wd-$9h@gynd)7tv(|1zUn zXAn*4*EVg+@Cv0ryH@Fi2oCtkPYxkydDRbzL4NAkOJ(y~ThBwcTzr)?HkO0iK{z1~ zB+`Apzn`>&5 z_5RK8i)GWFc$+i|#X3r^Ix@#a1S4;lb}PK^LCZhdjOmb$J>_KbrN#eMli^hU^>DJB ze{E+dJ}{hhPHvXfK^vw8<~FS$?WugEm!B4LhGz@(hQGc~`@>r0fA#t8q<>S@Uc`kz zz*Ibrc4F>SzZqR!Yx4vdp3rBy&u(->?CboG6nGY0MG=50tk#~%kYRCX5; z#7}*$9Hi8$n*9nu^Zon3ITC~r!Z9FhQ0;DqT=mBim^mo*-$$6H&{+KIWccUcprb^5 zp;R#U*$s}r#a(YP8hSoWH$M`;SJIINp&YaDn-OotPC5b(8FFBT+nwPpo}7_M(*5VM zNI|Nj3!J7qjd>vcYUoMrB;Z?vo(M87|2eJDlf1B#)JMLM`{(n>@jkgs_^nb6agH?q zu-l}R!R`( zzW6;I9t7pYbyGSpl<^_d%3yLBma9eL{$UMUX6g~RT|SoPT6Paw!&m1a94YnO*?mzT z!IwN+X#JIXQ0-sG*~rm3AGbwy9#vg!kXOSB1T=d zsQql01T_GmB2RniY#a6jlFvq&#&^XAe^=E5nY=LnAy1Jgv+bzrkh=y?RpWGm9xsCf z=F*)F3pTOgkm_R5qu*HI(!n|l3aOt!_qA7}new{x{osA9c3*b%{iY!Wcjp#*t`tl# zNuURSe6UqGP%3~+fs{O^=h>5w8WtwaWQbb4J)BKkcnG#7%cei7f#pO=uRZ1~F!>um zdV@FNI*7GFbCl8WC6uxAzWV7*aJWiA@HuvG&$SKQNn@4v%2)F5e~P2Fz(415&h{?q zb1o*T0dMi@;8k#kDL!Rlxegj8;C=;8J;r;sZ;JDp8w8G?m2$@u3pC<(d1MzT*dTPPq2S> zA!a<&ez)s4`Ca&sLXq3V(2JL>r4N=~zc;ade#>Q;^twI36KJ zo~TRKI0G2d*c+tlB5C>Mc=-ePiCqtwUX#)ZhJ~Yq3{UOC6i?B9*(t2;6g?t@D$U!idYsJu{Hu1ofT;BY(V-A%O?6JR5AMtk*%&biCGB)oJw=L4Fb zogZ)Ie2+_QgMA63GfJ5#?@#KF#mv2Td+3KiCp&a3$RH8jdp-@r+=osCd0gZQIX zWOMstb~o3!1Lnj1Z{{9}h_@)WaBDSt`Tfy!?tfo{H(c%5(B#wH@3eR4_Ou?dxd^N5 z`YG!>Ix~zA>FtudVpd%S##2T2yhU_xXT)>WCBa($WtE_@-)+9BL^rwOSfOqGBm~<9 zJ}0TY^=K#E+*Y1tD^mE=mnqH?-n+qf4A%amu<;-?eAi#QJFpll_u)V^XRZrDL-uz? zOkeWzKW}#+UpsrGSx%d@?)e>?FrRp1tg?(2Gv>y}(h5EV(#An1>cxTSM(ae;RH&PvE*X`H~S zIg*pW7k^mEZOA(jX~<1!pekcvQBzZiAJ3jG=cLW8g+rL{9?NyD{w@A~H#hkGR~i4+ zBGG4mn8p8$=KJ4%_CpH%JpK8($E`_0=@&l7>k}1mf3vI*uo!g1-^~t(H)=h1?a6lp z{T}jLwgzlqXVN9UJoktT8lqyl${?Bc(CKZ?-Iu?~>kl$8&W|U!b#EDr~t^Oo;pD|J-Lj$%dL3&p7uie!?#Zb zX(AQF5R%dmM>X8mECyLbDjq&XDGfRQ>O2A6{&B8K(DQI_Y64pN?Bw^0?^@Zs7#3;B z4{B*Fic4PzYRjhY%BZ2=)BW@30d>**N{UD%91i{WvXAMRfVL57JRqbR z8C3`0qj`Cwsf-~hH8p8!MA%j|6VcXfi*ah`&iTiCt{EzM+26W{>EsuT|NlSn-@i19 zQLMzQrdYRRCI0cowj62wI!89V|2xh6-BA1gdKexdiJEzR5Sf^nfPSDNlzvJoIS*Ol zChK*YxGP+pL<0@cJ2>YrSnPH0AM>_b=Gs|D#%;V;xCnTK-HR3Z`1i(^nk*AZx=rfs z z%Es%#>CRw~4lY%1l_AGu?oGb~QTtubmm2il)JfL=dE)GJ@BLUty(@v|YdAUeB+`~% z6Fx4cv+SaZ(14HQ!n2+6|FeSso&LXW5*E z4Id>utgcM?pQmiaVD^@d``d&&|9#7UUs%}2pV@S0aA-3Hj4R#Wdi6)3W20#)MP?I( zuR4Qof4Vx;@yV}eabNSrxDeQXA6QBa4S;A|bRVx4zL)>IQ5^Wq;7hyatN50o$_a9d z|8u70YV%+xzBW0%eepd^o7%2^eG?3m^_u6q0&$suHRRo`$*%tlQLznyBSkC7Vl1+l zy>_$yXWajpWHBOOeUc*?+c+2aozY-ITkutMvB78lW1jykL8%n19B;@wd^sxj|9)=_ zV!lwwz8|;vCb1)uMHO@R*Qa8cK!MwC)FOO*gqJI5EA#K|(f`knHbL;&bak3MF92Kcpjxu`{ za&ULrdK#=j?A2?ZYK^Kbp1XTKBU3o_*{}ce)nBi?47K{-RZCKakj2DPaG5^4eYX2n zkn8&_$YPEB?qAz{^F^HIv3SR*Kx-Mc3P}q4fA7!`10(15IZC8D>Xa1#i!rQP>ixZI zs#@>j?^d_btc&QYL8kjw_kTaHjq&6EpLkh|?-@wGh9cRyRcrXj()-l`2w%NyqF1{k z*Viy@Pq=oJv;Vs%rtop7PO+EDpDWTwK-8I z+%W)o>@tmE*nGa+dGY(@ulm}91gh(5O$*uAZ)JE(KDVm>-@uqdn!OBg+gp9tn6Sdv z=(Es^PfTsgTgb~BIZw`^rIRm$xiJ#M^q<5NVTCBgL!Zss#W5bvIABxD@XVT>Z~obw zV>Cdemg4oq8mnU3$KUeNd@0w_sZ<5IBSCE*gnw_+8VET5$D5sdV;tH!R;`}qV6F*a zX72xcW_;1xgUnc+5?MMqFqT4@z|GNoB{2ryq^SLWVWl_(KDsB*b*xCr6*dlK2>j}c z&1VF=OF^+ko=zRS#1;sqG;f1AJ^#Cv$$8WMVR8#%_H-##q3iB1Q+p~@tGoHT(rEoRT$|X?(%~Czsy4in zik$@+(v9@DaxECiOcW%geuke@RrhN3w#WPVH?QNF z9cFyiGkE9)N+t{bjny2$1w~V{EUJVS4Z07L>F82H*ipgFJfVniQtbKh4-hkYRn2mT zGb%QIF(P248yF*^NTqRW*^jibxvj+$7Yp8fu~!M0f^5kplOxrW?Bd9nQ-n8at4khF z(wbJeQdaQXu}BI>nDH}xWRHUqVdFSMok!F?$jaWMpo`LJ3u?d~1P)`M6=Xtm> zs}f6M4sz7<>D$-K%h`4D7tE34gb|lD0HpSb_;Q3>gwauQ1DY0<5jj+HaVHt2kitJNDvL=<%-0!v}{&{$q^~ z?=0wpUD8nu?@z>nP!f%pM7oBatXG4BgH%=-c7-{Zu@;v2^b$^6`Com=KSos}J(7LB zI`4jdc>*+dQ%7A9URRY`3Om6NltLV zu&>pbH*=Dg2Z=L?U{n|_-A2RU4AARtW4~Pv6fRJVilqjh*wL{%6Ll4wt^X7l5Z>qtQP6WAI(RRGD_oWhI-SfbKuZDRL|;Z*Wb-2x1KQ_p04_5X5zK=^-yGn zgehZ>78AMahdI82BlF^0$PUE2)q>e)@mqfC=!)zBTNcTOz1%ZeeePjN657f(&C``!n<1|O3#&Ra;i554 zy^_u^UL8DaT2(b7HILYodQ9Q<9h)!D{(NVCk6+2#FIsQ-h^Uv%$u6lvk&4FF?-fcg zyhf*Hwvm_Iz`(RfIQGr@#)LYv+)|S0Q9|+EPv+)N-rVLk5<6r;!bp_CPF}%YHC3%l zAVssPRb}tMC3axPYz2BdI z5~lM!X7XP*KsJy;b!o0%lW=78&yeu?+~Nj+{%TYF1V=_dj`|`S8jf1=U)As0?j=Sy zI}Vd)4>aUE9to!N{A9jf%abV;PKP~Oh{Ge5?2#PVTe>lA^yl0Ct;Z%$a0hu$B5$sE z(xob#2ib4V^cz!lZX5EMHH1Gnyyo5M`_bgq-Hn-aZj3>|a`nyp@-Y1jX=%3bt8YQN zJN{~0Tpc?G5|Dj=N(+sFa&NcqIh5-OTh+$hpC5KEa0La+_u?^z&tf~q@B3nt?E=3A zyCm6Jwz!r}(ndiKB$3Q)7s{xsxxB zmQsh}zdiV4Jva6q9Onn|%&2Y#gF&+x ziKVQg9}vL^mFGdc2-cnG{N9vG@w!mlRXrwY;uK2J1TB`(fkr8o#z-5ODn-69_kjQbm zN-tIePOM)(04IRaJvV)7VSatk1jL!Z@*^9io)9>)AMJFsTIfB~=3mCrO?BP$6i_B` zO7Y4m`K@0Xs}L=*5_%AKZS+inDX1mWgN@q+fu1noEiJ%MX``fwMQW%h20+}x^A3yx zAkN9TdhS$j?6)hhC2S)oK?I~rW z7}24`9FGhFwzb|yDyAclEA)?6z*A{&^n+974kcHzj1oe%9>ma#yQfhd0Z^@Ml2Q`7 zHeUAo%Zpoju~<_%dy=<D9ua;2E(H>2Poy(nW~}o_~ZkJeHA$ zL<+2HI`U0%@}mc)^c0>(<*M-M&rH|OP}3J%z5J3tBe+kZfpn^Vd z8y}ZbM9d&0JYe(d@pc~>mpKWn;xZ%MER>K=#3nF&qSBiHl}32-SkOqUla8OC5Eg;z zp@Bq=G&&4Qm$Q~|y)|1I$;P`ud4~7#v5896)_(JyFB0rlU@Ap`v%p-J@OyuvD3nFe?8?3XLeyx^*eo{2D6mDAnBie)VE z+ia0K?8W)n$$`VODyrY#!~XJ{M?VbtSqqMv2PQAJU+A9y>L|+;oj@F;bbw-Y#NV>m zfRMChIgA|B=n{1~>zwm_CaWPf0A$jrpt3&kV}}4!cLc^)n3pNz|EZRzSuVAe@>^e^ z6!TqF)uoA_Y?B^nAOk@Hz|$jaw{V?|ncpR(8$l`(C`w!l9oLzF+N$G8rjXk1EWTmS zpY+@L9LPvLe`E`g4%+3;vPJ ztZ?2krQ7VMOk@NWK_7j|HIRyLQWHrPh36Xsb0rO2H*nY4=0VJG^mCT8Mt`Rt2vA(o ziq>{z!**=bxfJd6ktuMi4B%1#p_yxRP%j$G;kS&HroZ9F?lCOoOe?IWTnSK7jk@jk z2Y+6zR}~J`8#Xx(U$Q)yuI!HLf#cCS>X`|8Mg64++GMn&;XLNgIQM5yI^-kk^o2*ykBG0 zgNk*Um7fTulU&g28=uq*C^N?*?O|>I=MT8H3$~aeRjcw43}xgp;@XFcj?AwYgl@wA zt2(_{a&juYR{+OsgH)8o9Wq>u>}?0bcSI{@!PR5Oa#ITX@kVOOwV^0WEI6YxH@=5v z#=HvL`bK;DQ`sCI;5pm00?2~G&zg#cQ+M#g$-%EinCO_dG|aYA6JWhEi7y$}8#a?m0>47mz8IZr~=AQ(gwsI*WRDZZo9DwouJMNdCY zLbJdhEaZZyqz`sp(f(GZLEhCruL+nv;XD7<`7##KGZ*h}e7AaRTnnh>Qqn;8$V<}C z27plsQ@-K2xaRve5ay*|h zjolcHM9*ty>r>U={ol)Zl}gU2cve=o;1H#>Kd~Kn;pql|774}rX_Na!l%0!DTNwcw zru>3c&=T;!L~4G+bWH?lZ`D6ojA8A$L0Unz*d`^ei8Ebh6E^LEzWe=Pyj00yAnA3+ zcYfhB)_1hj`KWSM;aKqpW-(z(igF+z^;6peQ_JzT2e=FZIYyq}>AN}d?dQiQOIWR+ z7M)vr?;pm7vhw?}g)1}NQ#<%I9ax+r&Ms+{QS?9}ozE;ioq8cajW)%nBkj{aYSBOl zm8Y}rxX0ns6mLUhdL;kgB;e-H3Jpq)dOAaYgoC5p3zjK?1u^Y3^QiVvS=w?DL${jxhjU9z0#TQlz^>>NRR{6Js zSEwSx$?J!9_P|DDOylj$u|3=b>#lyG+NuK8?rd?Js>%<#f*@jHc|}0Rlq8IL>}>x$ zZ08M)Bw0cYH+Pbu4Zx=vu|45XJok@}G4^m3I(6fHKpy=77z@V*v&?(Nst$kv`+FW3oVdMZFAUp#GcWz8 z2ak+|P9$2={c{FAk=^xM^`76m;#<01GVTqv@E&S5wIp;` zVY{av!=iq4u<>K8^;6VFsZSWdUt{urJu`%mZS)Q^TqQxhnkIJevwz{Z79YL_v1g{4 zfcDG73n&_*e1G^O2E>-a@koVI{G5y;;KD$kW0VUKlM4e723}(P>F#(+*Twn$fkxGM z@>A`GD352aav<-Ou1$2G1F;rRhFQD5SX5)V19mV4WOd2y9=rptFZx933L}fQYcuO= z68>&W1IeLYS?jXqFK>9xG&sMb|LW&NU;U0CqiOodExjKku5R!jC$pWwdwwe;bAV{K z^SYXhr6l@jOBSb3(N~N5x#D8kzCkCquMuc^dLv}JK{6U3PKkZ>_VK1L_Znp8ljau%aZ_8u=I z(WQcgq=FssEcL~-us$ZGY)uR{euGV*GDuGIylwX)qQhP$B4Y{g!S&Z^c!RhY%|QO zySvZl$Q1&@kOFS3w;O4A0CYm~KPf_n++P9lcpkK4Zw12`Q&Os7YX$#l9upD#B5`CC zN#9=G?{J$Fx!!LXDa_YYCVls(?Ql#l;fjI*K6rksRAVbL`~kwO3NHv$Ir+1FqjzzH zj6EN3+3-V!#h5wghA7<9z1R$qa{VYYo*c1~($oGwLXpSAnOxlQJ|wUu*==?G z^q!Dd@U zt#Z#Lwdt$*1;8H|thkpgU~L@+6>8W8y*N3PmV?#Ld!OC7mfDb!{{#h)K}SW+Erv-- zsZ8+6m!dnozO>$%dwEyW*22~!Mj_>rPS%|G<_^%BiZR5f@bX{ERRl28pAf{(0MAA* zvEaU~)eb))u8h{_dWWbW`e#vJr740Lx_EEZ zhrXe@g~YJ#c~x@QlU(EJoNNPyAqZ{zinP?GJB(>VJtVPtIEhYFve1Wj!Bbh0ghFLj zjVN7=i4xk#Pn3{XbwDT_Zc@!SN%Y|<{0!UvyY-9Mn-hhLW3UWLe*r*c7PjxddgE&Y z2!Rs0`=6m*t$D)6;94uUZS-nSeKQV=llbSHRjhrH;1mv8v?>IZy(` z;C0hbbnSvw+Q9S3hieeTh1PugC99_3g1t}IRZ_(erM>7PPN$v~>IVc6d&G=Ay`+4z zJ);V;wnbz{PfVXqCRYHM*WxvIvR08gP~r;e3NeiU`$*JNF1ooMnh>xuXafXtAZ64_ z0{_JrgZ%zgpa5LvK2h%&%gEl~kt`l@k5BypFMvQ&Q%uELlP*kb%D0msu{0C?c{QTG}aTm#Gf9jL7PTi=-}wEBhf13%`Tf<(iu zf`BFH!LPKa#F5Sch2ujei`>7=l7@uKgV6%H2)q57;ND)xTY(M|X8F_49?vi`O(5*m zUh@q!u&Jl?tNxmCNPV?4aBXLo`l>pXQc}C{5;d~J2q&OPj55@WiU5iy!Yk8=bQMX) zIDsga8BQC6=6N5&hHk}tJEHZ;o&V%{4v4ffnZS zp#cb0%FJ)QELAZJc>MhT8Db>F?-*3-EIxCQN26ofljW#auGsDQ*JaJQZY0--rBw*b z=_x_jJ-jr>-bPxw>~5|>j-t;16w41(;nBQ`dtIvvvyYB^ES5nGNbHL8K$2girCx#9 zIylxMQD4!<_Pi^q_@7_i^+@}n{GJ1$Yd4!Js65qSWpgTO}9VP7({Ip9ToVd(Tckpze%7{akHZT zk9(j1S0zYe_Nvv3Cq+|TZ8wby*GT*@&!8!Iakhme^pdmffmkkGZ=RL=QRFXVnGEk@ zNG6B6_eK)}Wz+j}%ep34v1*p2Ukf7^d%7d_SiA8bwu)5DLjb@csJhoE*>;!HEbkfl z`@-%~%LpwihZ#T`;)OlYYn4slS=9tkk%=O11wdj-x!zObz{eB8g~kSd6MGfMua+uB zZJ)mh65lgjjL2oV!=-IK*KQ$j~>WIiffm~iMIxZAniuLuB{mYAYVRq zlNU?y7Xuqkag6u$=U()lBP^8G5^By+IRe(7JA?g6a2g87U}BjYb)f7o22}YJJ-Yr+ zoIb}AL;#?UysAX06e>2Di=3{M<22+se0r`)#n8Y%c8VU^QaJ8Wj4>37~um zDv$vt0kW~n9L(f!q*Vwz1;7aup(l(^2IK+200eRwjIH!kdKL(b6{+%CDb3fPb!N=h zvh#+443H|dns;iES^eKJ_F^ES@0;JtcYx}@AmL%<@#_Acfe_b)-WY+EJo<(Ew*k`6 z)X(C?=8JNk3f!A^hJSPJBEC9(X>X<~GM=!;lEkHVpbH&zRuQ{^+RZIO; zNCdyeNtgRa-UI7jpBw;fj=Op-R^}TZ;uizgynSw#w}68vVBWM066&X9i!$|pleGe| z)T4BTu?NUiiKO$RqVI_Uw8o)n)vn7!>6sT<^w?|S?$OCQ4TE(xbC0xh0UA4)$++rXn!)5Iso_+Oy4~~{zj>E$>P4Rz`mBkY7Qwa*vFo{<&p@>DY{hNer z9-{%_SPncq5jGT+6O4lc9D!1PKtyc;efexylO%MDV0+pID+LwiHF>-bjsw_GTvJw;DPhPNVcchXx`GIh;hQI^#z+Yc{n z`+|SgR+BHt>``-Ex+phObzI)d?vZ>>0;`gV14YlgF$0*>VATT@6=9Z=6=}{Sf5C~g z0bmSN(M(bYcCmks8xjG4RGUv39y()SAr8wN)&nK5)$9M9(xRb?5s>#QQvlZYBsS?h zw-=0yBm#ABeL$b#YT0sG=vrAEu$h}btkUpX1gt;GHLbVGuM>Wo!(S-wVG6L*n{_@a z)^_SkT%o7}Kl!tT-2@H~fL-*HRpIQOwtxv(g1`e?`x1zKj<)ECF|2_)BWC)=S3hFb z%3HUCF;Ke6m~%v0ZWP5T_$a*E(++SJwW~{!sOs+@ zZqPXsOy;1GS3uyS^ItzrB*88GgtU5pXSy;o89RGvrXt^`pYLsf(}wK?h}(TALnBd) zAuAx`QgHyLZ3tkq>o>@c|31pS{Sp+c?u+xkKtN3QgjH(DfC!+7j~4omiTxUJ>DxBX zdepZM*fK??0sm`lqCzt!Dv89#O@!M)+PL0IYD)r1m2Puf=-stb&l&hseLj*{;0R<6 zYl(KO*Gxm%Un2oH^d=H??*uB609`}G;4Zq4k`dIrfIR-&Vv?_)UeW@&H_5jv<3hH; z`DGNgSB)W}+LrVfFMWOKpp)M$mKOh@TPghS1i+zG2zK5E{MmOtrT3EUZauY#Nh0BB zW-h&yf&l0|Sexk6bEe@Id__#aDF=HTD#AwHsK6O1zUl008o#f3GAu^21?+er z5qn@y_SfB3c10r)LUP-*dQJ?#uP?ke(q6VeBBCJ~A)~a%Un#FwakIfY?kF=|_FaXY zols+!gz9UpJ0vbX(>EdjkP?suu9*(YhMi@r=lULHx<^=f<6wW@Rxt&s=q+)c7fcbL z<--24X_XG$`Nuf(RjG9I_}7hqtp8NOW^-n-JDL#uzVv0lMgU1*3Vw_mQY7e^bp%z$ zv3Awd4`^S@6R2ibhe2pcwq=dfq zhw$JDpC$zHEHYdeVa71Snk(!X;V{+WX4E<1J1Ee=82mK=E?MkXGjB6RhdSX`fvjH~ z?JUjKTMp#D)wNJ0k-c_>jH&2h4NgKT`ZAN4A*!yh`TxyGTta~mi#8yL`^ zgU%v6)ryDk7bOyRu=J~<^b?fAJgy%)?*6(5`p#I|T`W%l{;E@ib7~Q81HNEJKb@TX zg7j0|oNA9F>jhdx!UrirRH7~QUo?r#Fk5&KPLCAEc7fM-)lvOOq_y}S_$P3w$j@4thJ z@#cMy6uF)l*(sgOud~DnC<(n7Zrn4ceT140+2`=}>}xNYSg`UEc6N{}96UJg~IH z`YD!_QH#U?rh+7t8ILlY6@df6WFK%YUj`?Ich8-^63Rq|!{9T|kvHi?^>iPf`numu zsqHlNWIUI8bpBoeu`gBOXS!n3D{%CE5&8Ay?N=f9a0!ZpOQxzA7htP$*BDgH?5oQu zp8fr!02)K|pA`9x@L&$#K<}Rl8+r(y* zZnmxuy)Skpv5iWw`&8r_i>y_O?5@Uc4`_;<2;Kt1%ELQfT6Z{iwH7J+q$t{17TK>9 zge!3s`SfO{moSoM@ffjALKy%@0alM%W@ia43FxkpDS#Z30pK8>IbXtw_5Rigz>;J8 z9SrIRGFnwncBday^~9nzfS&}T`E}77MW7qgLW6C$ZXGcn-wOPl#@(j*U2>`X*qbg| zYpEzG=|@eR-#c9=(BqJJeyk$f@4ivM@AFz9|0dTKrCRA&J!M%RNS&*jL5RcnZG^cd z=yZ~-tBNJlz9xf}*{{K;@2j#S-~SO!NWl~|4~46llR38?$?c7bHaovl_8PNA!OILE zX)$QbgpmJDEwkgk1n>Y#PSI(_eoH5*KmXwk*d2-3(p%DH&^GZ3$ZtUs+%bjFgEKQb)0`T{?P^y90xLp#_+}F;inBLmhES)Y9L2Ei6i9I;9ebFt^ z7ve2?l~-~e+#I;UO}9}b6#J%RNH`YMQ*)+)1|^wyb*YkMcg3C*-A`PCMh&@LJ*6AG za+Mw;?#!cO`y%Bbu|%D#e7a0pJXJg1XWb2RAF?>*AG7E~r6M>{(G`~uM8pz0AtiE>twEqyVxcWta##vvi^ z0L4Q>4E=TjI*zeT-O5+vg)#?BMgt}7{QD*b&V zZ{0Roez6^!cMwO7FD&JmhiS#Sqbf>K%DP(@C^q-`A$PA`uzNG08~BQnN0g z!THF;_SBglWNmqxwUo7SzCzu0+mBgo27H~iA^Kxb#uvN2C;{a9HA96g7t|}14q3*} za%yKUtSxK$38xxV{uk92f>@g}x$s+6fytaw{4=UZhq25&Bx@VjFX}5%+*cy)huCB# z9fy$gWjyqivvm)Z$9mxNWS_g_4}!(@1z7!+j3P(qhVS#WG=^8Kk8Xs#j>PAN(84^g z42MKhO8(+@l+QDVc_0|KLmV{K_4FuZCto_5Vly*fDBCH-&<3?n@oQ?@MuUi}FH_n0 zUjj`->ZkqQTPgZI;XkI7G<82WGrUSh-*~y9zM{m#HIZ+620wD{dM>_VJ5_s<6dW4h zi0)bmCAa=Nd8x3l6G8u0V%Atf4mXCO86RoOR01kAmjaa31JtWGhJ;^A=zi^?2_7rL zN`;!w-1a}@TvH6d;k(mKfDnAzFsYmpRg5YeQcNXyoa9a^Ct>5GMgb+Tv%!-S$TmVL zen~1qJWB%&kCi#<0XL7bLC`#$O-!b3JS{g;NoW~zfPd6TT*CxFm@3kOwJkxE@}?)@ zZ2Afl%?N(Y>6{Sr*qyD9m*GKuv1VTgJ#T3k?baniCV}NeCt}!D?&bSdr5i0wY4{oN zO1Q62fJlY0TY6A=gj$2bsLX7>)TajL9kozfp7N=w3g|sA7#p5D~7u_9u+~I zQs0P-s1;2(=8n2bOFZl>f+8{=8?erI#^<4=l$In@s$RvrI?#&uKZ{Nw1_7hl)^djg zR@e8bpWnValBpb+ul*G)thMxm<(HTgsDeRXOIuf(UG;5=KT&kOkt}XZvMq>MCVs(p zlT+ZOlWg9&sQ)>aJr)6dOocoI<|zrS5X430Hqq$3ur#+(RZ?Ww)AdL_sxUhM#osg_ zp24$2?cRL=Fh`v!INI!1kpS>jX--HpuSngOv6D1za#pq3;AKm;jK(bVMBquue|Tz{ zOC>MORdHD@fWEj`8FKLP))Z0zv0OvV&xx+nd}~EtPf;0E^}0=3ja~fMq=N0(@I#qss}Hn6zZTL0Z)Oz zl-R`M(b#axG>C(1(D}#BZY+|B_@NrLy=V8ugTV$HKE*njI;9jsG5KS-tx+FdI%E3E z0?*G0Mti>ASmDwn&Pb`q0JIjGl3{6RW(Q4m0i*NMdVL~5@E&p+RD+AhO&x~&PJa%bL z`Od(J$=gQL8Ia@#2J(e7%G^`+eq9zh7ZbQLzBMV+VOOx?F*F<7mlT;NS>|8XrLQ13z!BVUvCfS4X$?f<^ z^DCcdReRN(qdV+PiT18Lngl+=EXEJmV+;4lv?(f?Tq7C;SUi_OM?~^lS2UMD?;e6| zS2B5=^6IL{l>tcHD>q$toi-&zpfi8b;xa?x6jVZ~lOGo$GNF7s97QxD*Q(WdYpCIK z>z(_+wX4`!rH#@lJ{1i0O22`six8#SSsqWIAYZI1^iporforj&5S)M&s_9_w8-6LD zcqIP0a%FV0l3f)Zohf9rb0LpKFg^YI*v_#qmguOO`(80!?Psye0aAs$NH)}6>0QBS3F-fFixkB z{1#zF;}sJuM@x*3GFmNDR@UmbHY?rj6-ekN4#-X4j==1fZwme5)wQd<*|U{Im>DP& ztbh}s-g2fU`0zXUMn($(s>PQ?RarHtmNqz!Bx&OEVu`LaOUCQRn^;{;I<*r)&PpmJ zRj9tNAQtFgyQI81pQSI?ev+>@tA#2j$S>~&ZAhro!*}P|`(X{?LHLDVj0dP}9D*YC z(RYU$gq0Q(jmWVYINUCz7#B^<6}gsU~z zQWy#E1rG%t;dZf`X>{}l+AK+X|2BJ)FL?^Xa&@Gcn$7lQ%No!$Sk!9e)kD~ck$9TO zs`o7m&51TtTGqPqj-1X*kI3!_a4g>)7>Q48CC5^{b2;JK%H;;pJLE_Z1*ajm4BD^HFqTjn?edfbzb~_LJ)I2N6x4i*5)K1 zvm^M^#c`yLHb2lLr-7LhBB3d?qH4sxk*98}*2A0d<aV%D6{0s=ri|ECd)TyS*I{n;fX|J_C&4ImslWf2fM%+ z_%$L$K*s!K4!O=t(EVfPV%srPuW?;u_N??q9~3EtB1~;$S>*crd3=gU(u$&(=skWZ z`Wedwt1{7YeGOHBG_r5IG4V|RF-F=>q?|KeU=6pazY*Q!ol;=yWavRC`CEH1j{`obHC3ETf(DBlmf&yv+UwCwvZL};d=u6RcGudaLC-``~ z-9ZdoEEpqqRP@&l=s@ou0O?3$zg+*DD!3YveGAo1>Wt~skD{=BhTopt60|xO z$~36!sE}E#p}^9#i#S#qRekqX=F)j6Uo820qgnV1;oBrBD>bPLKTRX#1}Ir;{d6^9Ei zn`Dh#Y20*cgWhDMAHqgygx)tIuITzSMf)kr^f( z3_ESv;N4{_stj{o9m~jkSFa$Ild6~W+YpVY-V$jK zYup{R50q{P+mlB>pFG?3V%DAGkee{554Mi?>cbW)Zm)MvknAW8DUOn8zP!_FCRqRS zCvVD*r2m_Y(AQi;>^P2SUV2>-W(w>HXuUW%wq5%O#CuWC&U}lR8%ic?zb!wZDN3vq z@|Mbf@Co7WE(|F`I$UrO>~r7xd)qUqVJtm^lXXR|YaK!u7NCYpIMj`PBaW;vnxeBK z!4^_sRoz0BbWqYgFvBn%4}kWPwJl6jGW0bTMa5q3a6TnY5akt6mLn$zD_~HGVQQce z_&58Yn;*cAU(@zmlI%#V3w1NW?Xgwr<_RfXe{9ck<-L8t@X#HHb-7es!gQUS?al|` zw1Mf=XSsO`ML$piJ`L|3NLWMqv6^&ImR4pg?}*Dl%d+#f=Q(2N0dR;wyLxTfr}D6S zz~bv10D?ttC8B!MEXimx$T9fM5TYF~-)PEfSdRu;C0#BTdCGzaBzEWyr`2G!DkAl^ z-*9Emfw(ACPbxR0`QxuMnm?)5Eq4DDRL05D%-$cT-3l6R+s#&eaM&8xiE>Ok@0$78 zH%bb(PkWA6UUBZ=2QQWcEfUJ94@U9Y)27$JEjKzZArY{!VtiPhNZ$dC`B%;S??On~ zp1Xfk7SS$vE7sL=7XJp)j>f&I;*TLhfEzl~+p;|Y#!5CU_g=rq6O*qW=hA=6%74P# zhdISG<{BX1NsrH9_31LuB)j&b^%$|clh4riSUd)RY-G%o6WP1+L0urK23O}&bAY_I zGOajv>;INi?GTUHFYNvlBSv+JF)e^P@BRubtrWGanjR+6R{=(KZBMa8@iASS$d%d4~&Suam z#=M}=CJyRHmu}2;-0)^i)YPX}wOEcYW0_=~fL6jhT1b2BqWfiB)1~MucOPAdH<62d z9~jo^h1~;<4@FBuQPieUykFd=>*v6Az}z@O%;Y1?%uref(H=#11yzgSP?QFxyb?2C zgFNreQS4AU>z?2=erwgLFITG=Pa2?qkIU}!N6mmHgwvyeB)W<#NTZO2n~rA~nx~Sj zS=dmpTxi3y!nKDQwRDWIE1HQznTnsjtd-(kI^p7CltX-z?BaXuqlN1TswJ1H5i#mZ z2H%xms5- zPm^`VOa?H9_fZl~3tMohxtKDE3--UJ{-Wa7;-TxZw#?$9gdfQ+x1aM-Z0I3I{;0R{1EqUkc_zyeXm94NyYlfx6#umR!kbPbM z2FgTjD30HLTzs14N7pYeYAjg`CppIYZ7qTYk=~wg`JHG&i`e+{u_jISBO-eS= z>S4U}n+E>Zx)S59s%LFS>*?VkX0-T3-F=TsRNYX`3}5KPxWa5dR+GNOUzRkG>`j|X zVsHu4VBsGaGGfkoV5ls@DDes0$*@%}WLd{T_Y}n&0Edi7KR%ge4AKc)&KP|i>0Lx;M z3rW+K-#=P;$r3&+WFvNVU?T6k^o}(EWq>~GjQ1)8y}@NeokJ=X%WIH;F{Pqm=TGDl zP{{@)Rwnk%cqBTHJ+@bQ1eUG#X}!Qa2yg%kHb$A_@QgJvh*kNDA=UZ#Udm+{Y`Fq=9v&UVTTRteop=Ndv z`9J9UGU}!z9cGMmMW6i`&8ILv$eqQ>b1?8qGNM4QNrl>pK9w{_jXU9bGQY_h|KJR| zWIBp~>@bXC3iMGL(4%V=_R^^1Nqb7eX=@#d59s5i$aBEkoL+fYv7kcfK`$YeVKx|f z3hslECwqa&3zmJdzy`X2`)|L#D9l!MOP_!ue538UdX*zG$(-;T(}aWuX#IezS2eTb z=yw$j+C{B^NRHbIUkGrFKJmYvq~2kUSL7zFC#crE9Sd?wv9E^gu%|Cc}l+uw0F``ctnp!oF;$7WA;UkZ{`OEB7LY%7ZOQS=Jfn`%k-mHH0j4Yx{Un$ld5u zMjQNg72pp6Z3`)Khh}e+=4|6ud_jB6m`-yaZR^kr=hLH|D;Q(G49030JxYEG#4a;u zk32J0_zAs#Hci^}IA7D5eqyS^Wci016MuOCx}K*n{zp56k~_rjA9}VVQNyWqFeRt1 zHPE_r?4zdD+X}!KJ#>R@$FMI<-m-bH(zNcI&gX{Wl?iPizA?H3F@DEy7M`w%dR!yx zyVwt8H;3U!>qdO0X2oP9xxb>ULq28LE63a+ECoE@$yz52#OhrNN7b6V%iTX^%3c`1 zh9$v6D9&yOh(A|W^mm&8T{|u<8{*SueUx?P4kU$%?>_jA#{GEz@S%nM;9X!N;XVzL zIs#+qo=?`u8{iB^eN=Rh7oWcLj>^kHYvCT}i5PhkNIn<1@<_q&UZ31&v~OukeI+6PEbRM-Z$&j!Io=u)Rs=vnIu+I4SB*VB)z#W`cm26h4{mlvE4MAgUhCyh%>0CC9L!BQtKr;N z)w-g^82Op}CVKMrqI(e!xc%R;*q0b=Fx%v`#LBiGHcVvv+}-38fm5Q&8eh~X^N$Lr(wD{Yc&TtmH}jV7c#xy3$FD$ z+!O)UfVZ02$MgUh?UyT)tbWv+;9mofwBKO8GI!aoipsI8d8j5zEL>vVW^VsP`4hiq zw`M0c+jMaMVNq$?LAk%0fo1{K)NI80bC2oW;s1}Pvy6)B{oel2L(k9+Gcv_dmykIeB&biOt*S@aLwrh33 zdhmm$b$0|?*5GHQ+i{)qlE9Ue2r){0(d&-D=%0j)zd%BhdR8B;;xCAvH465l(tKCm z<2}BC(4OpOFaS~g<{oeHjf$=92(+u%jR)|wn!K-%v7c&78QueNa^rWF{HS}^z{KSu z$k`{yy@~wvzpgq9^}n0S0o+uw5x6s$YWCOyf6Nu)q9Tz6DT+hZ`-G3tEOCtM?gZV$ zPcSCO&-$-y@zy}&vJaB{zn^fJ+jLevGOs$U+#ALuG%{%fW@*SVST-&>z7cS@+~|II z-{lu2m);ikoxC0NVg-?SVQU_nRH& z5C@{ASuj9)iG$VNlmlO|ForwhdRz6L;&nR@$n_|_37l7_T>*3l|E`Ve*B;qv%QytP z4;(KNsAi?w|5BzBBQ67P?P75QNjF(ZbzCwKCPsm0C^K+yov-%>I*KA8!QLUjH0$Cy z{L)BrbhMJ-K328*a!UM%a<2d6K$@-9t)UK@X}v7WfQ-yX4g2QPIJ&4W;y4jrC93`_ z3T3npd?T)=nP4`w6JcpIOfmE)&(eOx?E=_=mi;wlC#ev(`E4PCHod$=C`q7DKe>9S z8p`RdwwB!APlErbM#+jH2N&0hNO8RAR@l~7nt;Dm)Atp!``sL&mDYkm4k5ypZL!W( zqp!jZy2{NWK@!g@tZInpPr3CueGf=SaevlNjr33>fL+1jbRjAP*D3!6@&y@>eu(|7 z%k*ZQLiVl@L5FQhMtQk<3JoyNB>p>YN13)m{#iY$Y9EtcQlzO}{CprEcH&|`4|0Fo zg2@76OHl>w#VV*LK}fc->JISRxL4CHc=Bbd;4l`R>z-CnBy|@NcZ!YuU~AKhr|6g* zMW)UWXfs^jT276PAs&wWup}gTcToshEVQ4${3!ZG)R7*|ow8B{cdc1}0Pgi0O-eXIzl33Vy`Z;AFu10tEpwm(sl3<8$@y;^!p2Lt*R0ka) zbZ{TkwW($j7WmFO<3yft@5f$AzhAxtT7&#41T# zi+Lc_FIsgE*uOJoCjUszNP!^Mx+Nn1>jMaWhX7i70Lbi18UJ#^>T)79=LEjqq2UDCsN%>_7 z=PwRR!QxWvQ=8tDNK?YCQgQ3PGZ1K>3|(e$dxJdQDI>RElo3wfAA5(iP(6%*$){j` zd8hK%8adFgwzKAPQxd;|PRy9;yQedEr}A1s1~=s|iXT0R8>?1pmh|E*&kt9so}yZa zkZrhjV*%MCyWiN6EtK0j9ahlkL1Q(411uQ@IzX@V*QS$NJRm7K2SS4?uT0c(BAXg^ zX3}k39=HWU(7s-4R4HSl@x(7V1$i7F4jgN!{IXP&`NbMIW8i5_F{-f>C;lERV1BVRnWhpNPXUFcW$~;E(3~LE(^T2LXt{pSbeL;k2vL6e7tZS zKH^)%SHKsYlwd^t)G*g;5{>$~hr5qiB@Cl=47s1G! zbsdoXS%Ud{;wSHD4si@5(U9ch@jFCoIh+Hpn$56o7C-o}SKWiYql9k2lpokmqC-$0 z*AZOucq#8Sf&3rcg+QnVn~p=dW+g6O!? zC!uY-wgLE$Cq3UUsj8CjklU{39D;T)GMh5lMb;&Dz0zL(1(XPFtpeI07R8&3D#JSP zp_;5a5Y@Gv9i!t~uG^^=;=UEOty?6=y2^5#^}io2 zevdQy#KFH;(nCM8SWZ!}Vj6RX$dk>shsVggJ)RtD2^{??1HB>96)Pp@EoBEJX9?q7$sm6tpmX zwiHWv(LRnVJCefu1XR*vYhlbC-CgIdF^+>Nd*%;8B=j z^IGgc5^nTT+6$h&=rrEWmc3?ZYsb zWJdtPz?Y8>EQS@&&L8rZYF-H|VBa^sQwBUZ_nPf`%{<{tJX-Wj{;^}SDhyp;MM@?_uu`i?6V^VihqM1F~Kjl{X=$=xPRvCj4R`GKS;~rn&88E zVg5SDbbzmFIz?5!W9+0JC-HU(tgvaZv(PqGL2J5X>VtId*a(P zA@TYlq1CB`PX(K7RBbTQcKVBL%kgflArk4Z_?Z3q)swL!n%2kPmyPEX*;TwjzUJods~ugl%cS(osHcS%e#1*S^A zb=>%Xz~-SO3)8V;@?0ichnpZwJmIp4khca7WVBebc<)2v2L3D9rxg0sL^+Q4_X&XS z8Fx1J=%*u-Pbq3yl9}Ah!~Mq^AcSU1sXLJ6LTGY{=78pjxBV*|o(%$*nddgF5W-XO7z{Wz!;iRS(s z5rq0rR^a)A@ZAr%tsH;!nyAIO;=A9zwhCqU6lnDTDZu0BB>~B%bB|iV+5e!FP8>^U zf4QL#^SQ58-zIKJJz6?8ww4U81g1hsLYyD>Y>W(WHSu$&{+mA@C2PE8#f`4tE%}eu zO`$&iU;=u`5WLj~ayhGH$E#XIQVwJ z#OuPyHX}YGdH830^W{bA3m+cidq@GNt&5Ls+WteXK8}|Y+wauk{WJxIcE=}>XqJ^HFFXEP0vXzyxO8Dl`@ZLO2`rT5d zZK1`CPc`U)5V#1VnQmr;NBvo@W}{Y#!*tC?mTVNnNR?X|ufPQhqln>nm%EH5;ucKo z<5DtwLCw^YfnRj%9Rj!JG*G=-{Mg1NX7ah*>UJ<0=M~7~s4tb9wTaj|nY)qj}^-OPqUWd$>1}JB1yHxqp4hT**(;6LWJL+|f zcOPB>0?<#xyCvjIS2K(^cI(VM+Qgupry3eYh<}*r~d8Bc2v!+EU$06+r9Fqq9hh;?1D8LZMNx}o!d}<~HNb@Z_<7|Y-pyCWh%3YXgJ~+d z=8i2HI2MyV(?%({04hP=Jq~K!jDf-~A6eq*tJj9wi4|CkOO)iC#zz=-(Slz5$99f1 z`FSIEm^o>%#&vP&QSa;{&&@xp9aK)eO}jh;mc1(wdA&3C>D5nNdTqyG%w7FAl_R55 z(PetJJaVlZ0ejy9C#H}pCP()I!`0%I4y%ueh=+St5g9sEWV>ovlR9;vRJ}FgI<0^X z`uQ^+hkSnyQOSE9(*9xJ{(!WoyEU77kuBX09wP-lnKMc(2p}P*t1tFZ@8rGOrE<*k<2RaAG2Xw6QhLO&8vnzc)vw|B_%gmr z-MGX!a70rY`I(MAYP697tBU|TYjZTzUb#`qK-p@l-;Gx*<=sYG;>6aJG2-sMKB`X! zc>P|v`KS;Q9@Y$=O|?TSfBv3X4RFM&3JLL+2{U+ zR`OmnkBAePVVIyz{93v6S)YzMpW2DI%@(u))WD8;TCnKj*E9k)V9p2RG*hi>Vi(B= zy`mA&?B%hZc!YhA!58qbAV%}ZR74v0snm6n(C9dwg7N*mog_$ zrGv=-=dFaT`uy@mL-UT`#~y!Yr+cv`sM+9^V)JrgGL z=;L%@>^f+1Nuj*yFflzM44qQj7@d=Si6Yth!id!-B|NE+fFPZo088Y~_F|&X1sU%EW-yMKov5NQcQYYvZ94j}r9) zG?qsuN(Hj`pEZ6ENMjP!Ehtk$itk4pSq1SbE>n z2=JXCy&T#5FTGJ8Yx6XZ1A+Km_iQ03!P3qj@42Z@dgY#Qlm2qt3eV>`Pv-HH9Fl^) zGUH30y}uyc;Is%$E?K2DQ5=%;l7li;#Cq3d>m>Mp1OgnL@sG+iuZ5wQI2{E?_mtQE zqdz6*xvA#OK`9@!Jh341;~OwCmT+1sAGUd8!~Cj(QAhMQ8dz-cLD3bE-n>dWDTN`x z^rYd&6ESV819Rd+`%EV3#R+gC*{Gc;l&wmwRjcp(wVirgx!rM4Sh5sHN zhmrRZS#hY*b!^2Za2GLn!XJ2e{rXuypb4NARAg-x-3QEf(tqxp>?By@Y4}y_@{$&z zA-$;SMfdg$JRa~_t@^=~o|n(#7d~E>GiDd} zL$i59KKPaoa1j4%678^sqPZ7X0}}m;->ljk6hYDrh_d0XU*ykJx{gWgs2smDnPk=1 z)PGWOA3?=ltRudj)~9vo+hBwm<`<<=yYT+sEjbON|2_*fbK$(;O3&DRqyZ!7ih3{# z$7tNKXdG^ZGAwGPz+{#kD`Pr&lB4g4STK3zTkZ6YHZs4U-pxnjQp|D~o*TB&VJ!zy zJ?}c1k}i4Q)3jc5f>*q^=5IAjrTB9Fop)^wWz4YApU7-|dxp*2{WC%>RL}>Y z?J#%!uyqq_qfvg-IOKgU?=*G4-h}h~uf`(#LP=aiE>Ob{6;sQ77m#`{Dk+I-_4Szn zj-t`k8N3AyV@?abO~C*AEPvkRhC{`|y9)t_sxQgb8NQDV2v7 zk+F3aLKxD)Pe|wS@h{_P6;Dt_Pc`=jbV2Ys7J?rHbPgSC2BxY1@@a%~wP7+20fsl? zKJ^z}tME?(h{G+=XI|twANu?)rn?H@`~N-qZetPbbZn)=9Xwy-h)UXrreNlwo=PD| zN)*mLLF~DDci>2+EOLcQ069nVEY$dd$qk#~F|LtTJ3ER<38?0T+i&hSH24UrvbOAnX>Y4{mLY0ny9A?9;q5*{yvVMj9x0TgZGh1 z`h-@_%LNMECR<5Da6^q{!7kaQY}vyOKu9i(1mHt;z9t^;PQERbeL5@_S_UUcS6u4m zD5yI+Muv6j_kRLKY~>m{QW;$YN!9Rokp=~K4#j(3=@UI(oHSV#0`UhPLBB!PVI|8t zsydafO;gx!vu$VCXpmOYXSDIHTrOhH%9F1U3WJtCyavf$-<;_8tf*+GoE&Oa3sp|n z&y}~a4=vk#Zx;a`cMss6NCNLv?roq54*X9M#N~uo|F}&{1?{~9~-e)ZJ-o%^Rf-=p7~a*Q=S8%e8Q&Mpq(D@S?$F6LipxT0D9!nQ;{^ z%{%Wf$nSmomHR4fiErisAb^BfnTs1`x@ffXH#04U*^>+BX{O{nIoJFLz`+b!)>ss| z1BpQ}OlG6e^lEKN{^;`0!2HA2gjjH)xU)8>HY}@Q?tyIr4{;)J_I0Ve6t&iwOk>nM zaL4^puV)>i&({g?$P^OuuO#MFJmEA*!J&&vSJmP?=0+y^gU0SgUW_0~11c>J{SQX% z*xX|HV;$Qaem7VfI*p|Rg!j9qwR+Kh)4MTIDBR_;tee~U#to}*16QNtQ^K6<=#hd>wqqj+u9};kTFUiO8R*x+eepT^cucR^U1_relGVSw9ktos2#a-*%y{P7NGKbQ zRMHYRdrGq}z-{=9B(}Al`vzfR4lrwAItpr`I~a+VUdh(0pv;i_ot4rX=U+LzT2_Xh zhIV>Zwx*6D`h5d^tJd45!lTofQ*P-goZs|klVfL8ZlO4`9`~1T*iTrb{MaV%E*z#Z zNmKX^qV5PFS9$$K!JIdZHD^u4+8P!QDC3&~Z3ozo1oD;X48IWTLqKs=b@4AzUGZue0O?lg^ zFLU4rW{K;!M704Lg zg9UV`ezn!_d}UhDm&@1UT1@n@NchkZg~$z2o&#m}H+Yr>Z>%3{`Xm=>@1vHQZoxO8 z9z5y*rS!lFrqgXBWHD|$h4x$EH2@dn-n}RD5}rCf^&<6L->X9ZBfw`luWehTEjrMa4Vp?Hs=+KCpPih)D@hM+kyVEAU_=++|fwGs7E>&b%A;RGY z&?ZM7>!Ow}b|^cEqp>Z@a1C7BRQOe(!%qpHl#(x{tx{Gjo&m{@dhnxT;n9so&;V%tzBs9cAKs`{s+~Rf(Zji9 zS6)2gI{CJEimgw#4|F1}ng_!BrRt@2)N{n1zh9{lIB2W5xkwgABtyv(JJ(!u@Sa*S z*h=4CvQYzh&}vU)F{2&m;@&7_Xt^r0QGv7NxSYnBl2XUJNDN!{7V_R=KPXnAiAB$U zzC1kkGG@puB>KT6nA@qqCh$vhPwVed9#8%yygFC~^2u&KUC_v}^V!}!P+v-w#b~wp z64X65UgzC;+9}zl9-4I(aH45lm~M>( zEfV&ffP&@#e_f5m&E-TFCKa|0=0jR~z&@69+4!B4pG*o4P#QO$shR5&8#z%36_?MN zp23U%I36>H?Pspvq5-mh0;Cn-LyhNoxh}*$T-{D&ojY>f6zPk5x4Q=^3Fe|>pU{52 zY)<8A8`o4$>GE3YCn|y>wVVOp@x6QG`-TrK_sbwxwSn9Es82(Y!rq1rs-t)uuD}y! zct`RJcg0JcT%K*jE6MmQyP#198LQ=4bLE*dji{FTxh{2w6tO}o7LC{v4Y`A1Gn*<#>a|L{*q@)e&@|{@0S9f9HeKE^%oI$5c0EG0YJYMmyA|v#@l(Cgfw$`0 z_c7<~+)PM1-DV?op?(e5eWxt$)-19oLd&{t*+1^Z5RjK~&rS|B6b92iod$&Nsc*$o zFCO&z!U@b!UBxtsX>sClE3MO3T-zBN%lj>_(t~E5+>&?-i$w4Z!gMb&mCybcB*P*!xqI;_+=hr_R?B74&In7q%|G>aYtp;HZ!LN zkO&Ph1jJH6QW9XwqNfyhbEAj?%)ssSkr5Jk)H^&{yQr4LoRn3yiCd9VYPRSt(72v=R2MrLB=zQigG*>Yd2pE|GWa8kSrHSTszSFCs<6 z)NWx!^lAVDU z2;Z!$puNM^hm%^`}gFTTP#YlSC#I{&oh~o6>B$e5UQ0l(WBE z^GhXeeT?#K#Y-fbh&kissms3Rax-un=d8p*Q~7P0axiCwWUWf++JTQtKtcz4?}WvG zz?NH>F-4-|@4cRi{X$+3(p+$4NwbubN?LHUS{JUXFuj1&;0u zydh4{HzMzFmjgGwuIDqqJU18G;hw;r8feFdip4x3M>-NMIlQY%Ul(D(+EjQWKOaTg z5w`HsZ$gy83ypG5A!OeU$0~@Q2Gt|l8p}Y*Evr;pz*FNkrfonBWfnQC&Cl$nvwXQo zq1s~12%{Y{+F@>&eshAoV^um?JY~ZB%%fLBF_XLK{S*Mhd;bXvn2&-l`=hvyVg`{D z;cAO7V}FC{ocDA%dzyjhiN*Nr$)1kiICRjz(?1mT)du(hQSs0}08a6n9Ny5kpY%1) zdKQ(!=lv&_Mww#XolvtSTS7c?gsQRryKb4i-=8mOoPE&>9P#RemEI~Xe}QK1NeW=r zyHamt7LYfb!c2v$?|;x8Q=TH7a2b@n*lsTXwmdTLjY`yS1C%E`6Z9E5bR~t}Xk`zS zsvQ3?T3IYng^wT!_+1>@G6)4uNpYU+co#8Z2=tClhWByTkfUgtLc>gE<~)7cOgK|i znv^Y7V8?M~@)$IRDdci6N$(zrdfC8EZmn``v;_*U;i|w4w(jnV_hHI;_>T|>K;T8H zuj1wN77$bSwyNSJFi%ZgFOsOCTAfBnH~f{xyr%bn#WC^>{+^t*RB5m61-FV#Cz!8( z4YJu>QpzzIn`watCp&@DFdk;;!H6v5LG_C z0Ilk@Su|$hNzYOf*%j&MJm}%cW6Huo5Fiw<4bw{T&alC_%Og?NE)ZDY(v1sfK;K4!r#Glu;+bqTP|8wtW+ot>O*a!V^=sQDf3~qJv8> zs~sU2cN|C5c23I+Q#tgSIu$V*I0R(X;vDlleL|MgRbQ4_dqLWV@cm=p12im=n&%b< z2L>`B>+OtExlE172AV zeK6SRWfN~>`YxRu4Pqa3;ar^blG7V#)gP=8OA~(avi>#L+)Lul-$e5YP!WrZ13&xr z(HrUaRNnox`jCSZz-CZE6&FXz`Fc!KjVbUq2^lLUaaseC)@XtCSTNXuuww7-`mJm2 zL9BHbaa_+3m_xNM3D;yE$3M z0v|_!Z(<&I2yAe_|Fq}I#emye|7(6orZ^CugR052S?ll`#sML19)|cE+ z%NgB4qEyh5qu+LiiyLB|eF1%plKguAqlfa-r?o^4sXO62(O#7jBcH5>75?8zBb`4zT=W@Hcf&lHi=~aAnf2b z0RKSF#aHKuBd_JEVsZKV0dzHV?_67D7Ff8s8+MWd$!bkXKs=gGNraROfk2TFMNoRG z`J@EA=>$f2F&-HRtvN?)hx=Qu7Ru-ML*_91;=&74fG`Ia(x(kg|Gy50jy$6Uzo&R@9kjRtZr4ojyK_{ip!pMZxU_>rws zAlTPoBq3a_D+Vyb$E^?)`(4wd!L8HbAF3y0@8J%HNvGqVr*l$V;u>vR8@GR0bA0in zo+5ukCfoF`r>qsi-dC@X8aoe-xwK+%vHh&1b4|5VN{PSG! z@cHkrFAR8*GfB=-wv=Hj=ZNWI3L5!#&Ic>)E{owxlbA5T*u^B)_*kGtrmLfPr z7aOTT?`|b=j|Fw^JTP7i&7#9&LK}nz$t*lg0FK}?j9x$%=@*efTIkgZfx7_tM2;rq;8Gji*b7{>d+zxiQLQ#k z!X{pE=mEnrg@Fy8CQ^%S^f?~?ftTr#&u1&;uB zCRl-YoOAq@4Fzh-G*)4qCI1mCxl+U)u#b%841QxkJ-KwHggdy; zYV#W5n(|AxeG|)C(JKO(aJ^*LlO(szg4-)nGf_A+Ul4_CHG|i6+kia@8<~YzuTn6H6iwb!jq;-~C5} zT?H_ysqFGgl9x>aSpk@I+I=nTAJlZeo(d}5jq});`k;!9+h}xbam#jnxtfowo-LUb zt?E#1OCey-#6{gEf^ISEk+@s2fONyj@1_f4^MJ~T?z}*~0=R12IG09r3Z(QZ62P@# z232zlg;RQ~%Z|Gmw2Ok-JYWVE!m+*6kM4w?XU!FT?u?E#!%nLrpFgU4rt#vM1ddfz zqrMfT1AFxGmiilSk8b_ELFJ-E!a-fxFx&s=oE$%m@P{&@7%Tq`@P1qYhQ#l>+5@mB z@Uxm*W*m>p4kju7cgjaXxHi#4%`qCfe&CK4^I7%-FQ(({Nd&1r^%}Vr(eM97WTz51 z1uBVw8-L4ZM9d~q1$~bx%~D=_@~}2E zXZSlSH&9Q6?t3rRjaZnYnh_LF??Jg_VgcnNpGQl69)Ydck(7%7i&q3Nr?ABR$YvkECW4p8Pss$i>i~0O}RUwT#rA(>(JG zrQJ~I6EB!8mWbdAE9otML`=kj(JJtXUv}Z!d%btp?-jSQ!vUYHBAv`*wMN`i%4SNe zZ6Hpaz0ZONADM>?sa{r8q-st9y_a4)B{i_r6hCX6`M5?%?1AxWd25sME(V^}5xFXW zhf@5W+l)}^hTP3MnM3dqy?x}eCF%%Dcj}GEcz&esP^=d5AUmWgB7INar9QVuy;p-EWyq|I11T%276QWR=JIki^yb{$$BqpVs#=eqNTNLF9kBn`I#}rMqXS26D}K z)0m4?(@8KLZ)w62jYBjgYVGbE0no<+Hng>M86gluYw-*$KB;{4W4W!W{X=&8Ub?|| znZp{xS|yhW5K~QlYzHV)oK-K&gO=Fo#^c{xiFC{6w5%*Up>6MV5Z4{ zDbtnuib%)8jK=6MNk^U-gZHNQqjSFjOgeX|4xrQ$8JC>!7jlUc+Q66M92KJl5yw$t z*O^KbAy=8)#m`z3)_OvGvT3BZoIR9c)Scdl`L2pB1c{0n&<3eJ(uP{Uc=F(Mo1KW4 zHU3U>5B7(kAlxl1R2q)>57TQu!oI}m$e_saHIWcoAoLq6;Y@O8XF2M|+h=a*_KZzh zMEi1JO-c6!{kNJ&MNjm=c78DiWP(|shmutW#UJGP{BLQ&hMs9#*Sa3M@p4vloFR~-(_IBPP3=VS@UbgeYX ztGY@Q3ryAL*1P-f_!a!)0aFqVSUmp_@q)*A1dzRMQ$I_?RcszA1Pe8Ds5 zAoh0nrPWq=f?U?&lj>C|5Rm>?7+Zs%^$#NdOGpC-q3;>Aa+5px*LH(CC`E;Bk-A>} zTWfN?!#Q14!@=oIoYXEj&qc+lreoHn#b{hXn#HLb*Pd77LhPE7YX(|PoqjXJw}}9E|MKUkRnGBq ziK{0IMLdmTqVVU~u91Wu`XfJ;(iulu4>SoATuC{^NV1uiRM~^GP!V4Cd?G55UrtK; zB$x!LdwL-Gap9Zfmz>y-WARCWe=yk{4H?80Ptp?GwFyz{k(-WnC{{L8>3*=WjNSWB zH}@sai2QEn%PG25S60ZG%rn0Eri#mDBT4^#J&lVyfe0^N><3?1hEyMuzdOsZG>`Pe2o&Mn# z%z|0;3TAIiZzN)Rjv~RhHe1@st3Z=Kd_btcbecpY$H(QeDxc`F-Pfm*1X8l~q_Z!x zR6C3d6;TQ;G+EriB@Eb^qbhhW1HT!Z24p@DVlzNJiAfD5VFp%1sRru^k~? z7B?PAVbBR89?NS10@&R>t{(T9>FJ$&i^+lpK5c=W$@BwNJ=sVAD(jil!q(^*-{fe|qeFW=#cRIHenc6$o4AuZ9%LZ8k78VJ5&R{{T}{{nMh~i zybc9}25uLw&C=(m2XW}1yUHFaBZl#%G(LV`>31P!$FJeBkpQ~573j=P{b4vhgR0)u z#{aCX`|Hz*7G_4ZKNc)z&fe(vQgz)#&nSN+nfy%?fFjxlKJ*A#Pr#&R%0q`sU!y3g znn#9nFM~*jmB=u$Y$QvSQ@!kZ3FkBG^$ugM8BW)2H*~U$OJY9t|EzN6oZgQOi7vRX zt`w(>q9KBH1>~(?l#SBhuzs91BOll`g8CQUvx;l~VN(jO04LkfwmANrgN{&QHd&jm zk)S%18WgRw`J>vg)qfe0#Cz1#joQ@NvnW%Dbb8`_Ljl5{I5ASoRqe#1y;B?vuhe!Y z_pwh;O3H$q#+U!fDox)oYTHOMl?)Tt9)9#A`Vm1OLc0YQ(SCq(csTO{`|#Dk z^VY@TmDH!`3G21JRCV0ewi6X)Kc4^CQAxS7->{Mi(!V>%Pxd=E{@_253tX|@#&O_F z`cXq_IvrMf2twu8Yqa0`B-0B46}migDHE$Lg)YiBm?i2V2D0?7&4YN`c$)RuP2 z1rUwt#aazZJo0FmO-hVp(!!KIZm9D%Il;?85RhZMsRjhc{cT!$qQCxPVL$xb{wHeXkI%Z3rg{6>RIrC0(Pi**l$F5 zZV&aQe}Vq>o)wMMTh>=<{O|4i`tz3*xO6@`sSS8K8L$5SkvxwT%qHn6{yUeb9Mht>p&bS)q|foo zt7k7wkM$$|2PF#_Tf(|d$8h)d_k^4TyLoYUtpH zkk;P|4|B;ze4}qp)X3~XFx|?07~bmQoB1GvLX9;0 zwLKIUbqT_|3kTf0Pdi=|>;4grcz2%zdwD@ za0MlQI0r?}^E8441Ralkrp=$RdIAI+ZbTa77u;kq$>;nfO$g!J_oNg0S}gzl5PHEn zWk>iiL&oz^Q!uk>*bUt>YCpJG4AY{-ni0rSRt@2^lg3Qt;P`{lH{X_wzS&@ibaa{( zYPRAP!ZezRmaFtYRBJy}9ofQ!P1-_uAQ)B`tHH){_W6cFwykqVKqvLBG3Evqm*ihM zEGJ^2ML&j7?I9{myw=70UGQ;C%|l4@MDQu)R^nhjaRtwWO)eXQklRBU`gkRvz!cJ& zv{%h>-%P2cvMRC+3G?sWrb`N!zpJ5aC~WlDcpA0i1ltBEhQq)+!Z|#O0=t7s+g%Nz zF^$8DOQZVgy}TOlmr+d7b>O|TWli7wZo$IL{8V~fo4&}0u8)<*`?fK*q&wf}b zAp~2ke>l>h7H`cVq9#|Jy-~rdZ(F@)m87M)=d-V6p*zp7`hrI7u8h19vM{NWRSJl> z^@pSf@Q20=zbO(z?4>Hw#P(1#gxYMO8B(<6V(&g@4YT$ReV%FS#Zyf-amsp!-Jes>!f8%7M$}sAa_mQE__-JK8yf=o z4E?I^15FuX{9~K98sW$t0s+8bU<|frrDRt!W?8qBTk}ZBgH2eF#4n>FrMAA|$5V#b zx@CSJ!*RQzKq$sUrT)ERI#FA4dfSiC*Dm@XF-7-zM9Gdh^a7GJKXyB6C@obAy;aDC z@vr7qJATtF@G7X7dLf(Q2bN~G?Vd7D|7n`z(15tR z0_a6a@31yyv5#ET@T^lz`blqssjbxuwxhGTu}~4eyXScx+7p>OP(f2)@7Bo0Uc9~jMmAF=3rJG4uT*;B3MDW)vE4x|JS$=#j)GX;T(e%C1Ys6mLQ<6rL=?S-X@$)%v@#i6 zK<-ckMmzHC|HXMOZntW$rH!sHG6=`QV#!YyI2l#z;m1iKES!^~#wVeB2i%7C2lr@( zX~*D-x(h@o`1{S={A*$x1U9}`4hTfh2Hh=C3_6AKH=CSu-195*$p4;63` zUP@vgEOz^>#KhwLxD~wJ zi~9{5e5^K_eL?3&Fe@MGL@BAeZL2ce)1tL$ux)dXf&$k?fM{B+*vrW&s(<@7LQY8( zBFaDa)lvqn5_Ms1Wt=p8QvQ&Hf$Ts{sx65)g+g|hbsE?4@z#E2fVD7a8SZd>r4Py7 zY@4Xcht9&frE|lp&rX;hASK5-|>^t`h7Uj(2+V@W>10a$=Sba+Uz#!QDB05 z4z}nn;+eulN`JjAGvB7@8rvY^6wTGZMHP{553tF2g7XH9-bD?6uwh{zw0)f|R9!cU zk>EwCOWpP4B8k=)`eG1N0fS*jD@QF*q8bU8&%$!Ju_T1TrK_P`biXi1TPl3t-obc- zbcIfU_K+b8k%fyomCEM_Jr1K(8Lds43M}7PR7SaeT1xQwPaY zvB-PSruc|NOQ9b-HJ@xl!?zx3T3HC`sT+6H7I_#)?NwRA_CTjWWlWW!n~AdDr?&2x zidLubhIf+5b+NyS8%bX8o)VQYh=LTdxt!LhYCJ0sOnua9q3o1bo;E~*t-?kEg|Xq( z(}$UZ^$>K3L=kH1lAwY|p)KWIvE#oCB-savZDay?9A2K*)}zCT!7JGSAfQSTRQ@K? zVEqUkeob$4uG%0~i;Vpwii*T7kLR=$*1~vwcXp~tgjn|FzMPsNEthz}?gyfTI~$xY zre1}0P(2t+jIEKTknL|&7fJ=-ev$TXN-VO2C)qwCU>B~E3U^Ih6s$D{@`V@>`9@~e zfuR!^+1NxF7H7U79^wSUBkIf$Q&MEGK%oqbsS&KGsG97OZUJQ(2t?sH7F$PhZ501G z%mA0?x3Y|{R4{qRk{b1j)l15Uk|`emU}%)Y)1*&zD{XB$f4RAOBKUb*qx#RSM&oRy zDD~t_B+rVE-9iLflG`NU$*P87@u>_H9vc`WpRK7Evyn_Cx{i?y?_g2t@;MRHK=OMQ| zlE;X4THePf&@&n3;t+6LD)+oc)qP+z&{Js;{|)(KZD-0_CjWhqiqy|n?H`$%ag~Vc zOzs`axmSD~`F2Z{n2WtN61L%(f@tGXdJ0-tI@`1TxggeM`y|0wvRH}$18*$@R+k?> zQza}~f5A7(hKrq~1meM9W7FpmoAYNXS0E@;wa2pF_AFaxG?WrcC6NUc_8Bgb3B>y% z*++Myf5?U$SiO28Uws0?+Qt!mlI{YL^;<(DCpSyB4{JW<3CJ5gxZO|%d>L5+gLpkT z!?%wM&+wxbvr}AbKbfuYc-;?_3m0n8j~1JNKw(Pk3A`l=SQ<`5Scpt9?4$8n;F|sZ zIw;x|_efE-Iz}0VIiarb*3!}f`lj5oJG6b)#|5KmJV0iA2?o${nGs;w*L5Sv&P?4@ z1LsyU$H}tY3NP7&@>d30-uUq8>(QGB?uVXdc=vFoGhDQ)=QWQ6|AH+_U}?L4T0EqI z&)YbP+ww&iHkK9848dt}Kn4>VnI5 z9S+3~8HlWeh0^Q&OnM(&b0ExK68lOIIQ#x8&XjJ*tR#~{Bki(OM?Di~}N zT$YOKJ0Ziq5L#-oO%$jF%znSqe_&S^*usmL%zTPx=@L(;Jd) zH!Zg3CunBh;_@6;9AAN&uh`e_bs5kn=KruhXv?fP7tOxE+7?7h?t8EC+UFZd&!(HI@Az!+10OZ0O5a5{#i!}X^MWww5g{t|bEgeas zvV0_2)(=kvBRQ10eC zHZxbU|BhaQDsVQ#2EQ&n5_bx(uA^qw4&}Mp^=(Mz_w;)rRs1H8$$MrJ1kT*CR<%Qf zZHN$eL#LI0`3uO`**vT^C%;@v#Mljn%KVKI?e#q`p4^MeunQH$;ne`D<71WeDT zmX#~64aA}gMq#I1u|OnDj9ePt+bYeeVXl)j49z0HA|PwzErL=^>;<-ilHkQq&5F`( zzen#9e0UXw?WPYQEDHAoHnJul_&%HVntuVkD#qu}w;-?ov;o}(c{>Cpdz%TR$DIE& z+$q=ddR#B_#D8?xUe!MaT{cE)?!je+^Q7Y=)~yWgsH05k7nlA*SO;Gldo1=tb~Am& z6Ieq-BWX}V!4y^-iJd9pEPrF!N{GNJI_F38a3)V(6$h6(6Bp$g!i%u>AMW@89aS^Kr%J{8;F2Lx$pYzd=JPULUiX z%+v31o#5ZVQ>43NQhvesoV}m(0kH*r2Pn&-ZwG%1o#wpwFVxLt6782}6Xd{uN~HZF zE!+@sy*(KG{;uQQ?|s3jTFeSiSpePUQ;;g<^)&=Je_9yV&0W_FrE@byR91(eWL6tR zzcIa#7@g=LwG?rH_~JRz2&?y9BiuLM+|Ov!%1TfaJPfI)R!T=C5TEpMFiNq1EA!#Z z`2Z9`gNL+X$ZV~#*w5~w`>Rzb3*MNP#gAEO?zK8nESmEW-E1S#0r{v2Vd$b-cG{1> zo9|NVAp;if+5CbkfU!q?fevWvV=byMCmDR0YLptphAwsmS+o_P|KUbrm;h3UHi6-3 zr``Dgp+mD)tCzNZ)0M;9bm_?@%jqZ*4F#93!d!`2`@rY)u`NtipkSfwkbL9Y0)ed-2< zRcF8&$WRscdU3=zLfGQ|3ev?WWeHrxD7nY!&PjV6+KIo$%U&T#pn7O)aR#-v`PT zUqkmQ=y`LgO}#CcNUJD~x`l9`Ksp{~E|rE(y4h&rkh^4PlWGH-l*mWiKh0MOM**l_ zyC!93dSswr}jAezhHMNu($#Hsgi@g=8LaSGDf zv#&mJ0M9mPqZ;c+4A4KfYdR89GEx&05T9_YoVv&T)&O=K&j4FXRbC^Clf8p1#Le9b z=y?y@o)>xIXPyslb~CX^MxlTumwGpYt_>vP7*2n6QFu1KAO6%1ZBR#LYAjy!8FYkP%r&82yDe7PT};liui!GH zCabR?Phot9Ze+l*oMoV|zzlbV8&b$uL9PmZnr&-FwUDX)E~;Kd0bBNB6G9hmS8MV) z6DgEoS_Ya?nKUUqEiJ6{Nrn$ z-fc`(qJSUY8Vm=5lDgq*Byh%<_nw4@gSjKOVmBo%{7@q0_AMvgpkCM-Z+CQBe{!RA+}@-7^uG z$+fp>`v#(+0cN?)F=;kc>We^}qmHzc^?>OITrNgCE17Viyj@wfJJxXD7;>o=3Jd8T zAL(QO{{hT;p;PhHr%Cg{f=ovt`@V)@%kv|d0SQfyMQ9au-kW-5i1nNV8Phns4Kl1* zyHKq0{B@$)Lg#wFd(sh%nw_CQO9VodzXE|b{?9>=g1?oxbPPrV;ZQvns}o~hP+Z%r zr3A4+MAu(~96{{Y`n_U-u0;7$gFkV$Ru@7`@y!X z$*W17d-yBFcj}#R5aUEUs>741Y$IFQihEpa8aZ|Z%|voliK~^LoYNn3*;87It@|zy z`ZM!(yK0}oh)2z(IwQu19r?{ElIxb4DWNZqswsNM0o#taDSmnf%W8cb{Vu+$$eBYJ zqEBHrie>Vd22We~euuA7T~avntR4uRS=`N<7oUU~sXVzgnCzc*D`sJVCO&^~xjV>%EFZz6KDAs!bUej5=91 zpX2VCEIYk9T$_p$CDt*zWCLjQZQT=BjX0R~2qN-!CBh@p@AlR{+;P zmOLSp|D-mtNXmlkGZ@FTf7w2rFMu$xAvl3LJx&Vq7|!kxY&-Tjl;G}7N3r6VH3Rj2 zUZ$#Qo+_H-?ss2Q9cWD;8%=zDkIHkzXk&D^Mg0B{r@&vfc23=7*FVr&V1iK46`9ce0ID@6o!f&C!fuK->QnEl)-}-yCeN=CQ&f6`~M{ixsemKe8?hs@;5EMO$LxhDLk17Q3pk#@$+ImUG zWen3AnqxuqRaz%@V7TS-or=R$*+sMhp#~{J7ml=EGx>UEmf>EKzKq;9LC3KXaZ5RQ z4j2#ikDTF?0dvL8OxJav?_ATa{ze!{J*BG9lV6{jl3uS7>13+#t8;jJ7^HL6_aFC1 zl!VP}kaCPR9?9u{Ng3{H{u+d{Pus$x61O~E4F|c8wt%fn$$E~G2kVS?Qwc4WTH!$y zx?t=MAz#WIS2m=0`lc^ZaNZWsQ1KH0~z4|Dj9wzA2~b`q>F8sp>|bm#@C0N>4~MY9$cgA?2v^ zL%r(99TJgdX7qnIvzJY1ob0wGk4EsQR_DT?So9>0sgJn9qt=p#<_N}}-;kQN7TcB` zMwP_1=xsKAW-keI{Tw*>^r-5DT~Vj6-B@Fc9^er$8v}cOvW)JTSylY3$IXleE;?^s z?o;(L3BJ*{`H|3k*LZRIpSkpl4E8j-IGIotPT97Kpv*TTwscjCIbzBhh`Xm`%Nuvg zdFTh!GddDQW&g2kQg=EhCA*RFgb?3HvuHm2u`l_hgo^c?J-B%}FqpT(xmZOaaR@Q6P08>y3iqqBd_J&h)YB+u^rtfp8c>6)LN;1N|qI@V9`k z&Oa;va~aqY+MSB2>a@x<3c3}^rzCaTu9z{VPHRs+ zdw%5YL4jOYg`K{iY2BNd@?izCMgi41?u?ZJ8u3*$hRQ%lzdt3h{ZXxOxkf;I|Chd- zvm!!9M~eWUNOJuYq@t?eYtVA87jX)_^V$%Nq9^*~bJ-M&LpM8EigG$JyeaAGC-=*Mi{5x@=#Js0mD_ zoL-er?3-W9v~TcH^xGgk2FiSM(pwp{h$=GFd;L%Z|D&8KNbz$ETm_aLqnJID?akQs zkxufLSln9-NR>xyQO>V~?iC|{*J(X}=lc+phF~x@w3P8gC~1fWsiux_kySU2smi)8 ziRElaSiQ8K1SpPD^3B4J7x3*BFpq9lYL!whOQy4Nyoc~)i#^MO812yf#53FTO=pC( zQ0@_GyrZjF6!xUlK22i%bAmgVByI}!B2LM-u}MMVA`H~daN<{TLyZ$T zL|%AH&}!MHKob_X7;i~^20Ix3ct|1)JcK*s*uD8tad_C%VjV-*;u%=$3mh@4um6D4 z+m*MSlK%U=mk6PlHhegUVCIw>F3tX4-&%&mJ;&T5C6Z<=>xUp^+qH&KkZc!6P zXQ+TlOHeuOfo=RQPm{-kA^P3Zkm0k74`w_H3R!OELip?14smHdyu!wM#w~mk%>Q2~ zKX#bGF@KYKY{^t98s-5=5$H5kJ0UJ)b5n%d&KmY_hDQ;iD)!`vh3r8$>ZG z#44m0>o;W4yth%5N6{!r6%Pc_LO6OP)`t5s9q2f9Zj z=n0zZatmdp_;0C3#d4DOGYVyW)N?cP(@0~}h;%H85q%~d4+?&Jd zzEBPV3%J{<+WUpy|3*z`BbVUlT&;Eq_;ie2$4!TrAFC1FX7cA9r{=$X#>F2H=4WBiY!=LpVjbhVvqf|9ZkjS0Fc72#f<%ph90 z|NhBT5qi)2Ry|MD;(|)%H9rHZZ3}ustZ>N8?=a?)|Xto zMMDpx{1*V1-qVz@3r#%}XlA%rYZ^_?3G)Y90wL`=F}TVk%_!b$YamMiEbvWOAZ97l z(iL=*4`MT0v_1s?x*S%8y9M1*130?^C5aze=<)lzpfUHiNUA|jXMQC`3!OBmx?3=%RA>{umU*t*SsraQTL z^mS#@;3nV!Pi}l>iS%QCh50oEkH(iA3*)5o{h(&mNoT|9y=xDpI=Q**cvf7-yxF%` zZ!KhjuLsl29^KHv?srC=9jFod%XGq)BMe~x&}&f1E_3*8i*A&dK3s=X-|-TKkRiTr zoroz$wus#p4o36!MTjKklA4`P=#HW691N zTbj>sS#~puPkmW^`Ue>;KQ3|Q4L$@E=}-oO(cZzIEKp8X;B+DYuR7*U zv$KXTx$;F#7ta(#mJ5ba46M7C?%1%d=Ed_0e)>sS+V|`oU~unR0D#-$KOCRhqi_=5 zdHYzJk0y$z-HN2z(Dz}|;*`81z!!=(&+cpP1gU+2sJL6Gq1M@_H+J>qa5kP}`D@lc zU@$%=iw0P;Bnk&8F4e#0yPA0o4(fP0cgk;oUj??RLxOSG@!lHvOHw18ghM2 zC{{p~f<>C!CoANtK?Y90|11&xROrAxxe6Vv!(40F9!=a4p^C9aK8&@B`IVva7wRix zVTnR0CG+V8Z))DK zBQG?&@KV9Q^zVyCW1`PhVr%XuW14zg45>q!r%7bFm6>QaYj6xz@6E>(q_Re-!@+&p zf=+y~zSyS+5C$+lF1a>wVS;KcuN1JAmxk`z6M(pU9zVabNk!!GX-pT>X%_PNO}w45 zUB}2in3=rjj{4o|w_SPa;Nzs{Vn;>yy_^?!RsI3Wn!HEYxS)y7W35}=$i7BTDjrZq z(!!!v0SBndJGRTNj}s6z-1GOXRg1gBOa z=*_W$0~nufm52?6$mh}^n~!q9U3AGH5cg()-&oP}p=i`>Z;xz3Zt921CBuq0DZKz< zffReGQs?x-*}C8$;5aTV^@JbLAqUZ&LtA9SqBpV0ViDxgRdIcJbI`atSzWIXBlV^c z?|m326Sh+zEUQpg#7mj>4XNJAe9EZ0k{$&`@(xTyHffpN@aN!0p8S{VVv4F9Iqf!O3In4H-a|eS4Mi<3;BMzMneZdL}+^Or28} zW!(^}wfSM#Q&qGGn)aHxj0gKzq(I4abLgk-X*$Qi{$e$3;keQ7>*=LY0(UMxn?RMb=$v|NP%Am6 zATaF>&yW42q6Bo|x4un0=9+E8z(5!J8mxyM0P!-%856`{#wGGUQwo+9bblGsWTW5a zF#nTY6IP}Y2gV}pb0V%A*J&knhSy(Tg}pvZ=3-V4*utH~yP>7Bp$a_;hNdY1KDzV= z6M!Bo$8u{WhuZ5VeZ-=6=9^6P&IrmXa+|cepcZH|Md;e|Is{3kt|wu1u|$GgU>3gU z%11lSgz%XLpM}2wSnw)15Fl5kvi!@0Q(n63PJ@rSY=h@afoNsy+GruW5%GVf-S{1N;dZQ8d{b-^?@k{51)7F64_FIhc zVdU|i^9d;o5w~eeo~{TuYl7MOC@8s1dg;VN8SMk2e@l4{rt+v73p%edGlCe6n5aS2 z@*y*TUSDUScbUx@h3%1UtqKhy1~;leB-B?#xxh^bhOat&0;$fhEn`0ypm+Iw#}HAi z2u?MQ#95cMtM-XTKL^ISoAF|-6Nbex>Ggx*;^iPWZj2?qR5sdEEFaY;5Ow$ z6@B)rIyl}N>3d9`>ju~xIzK}apGkeeJ~nmy;Q;;FOQyrCFzfgJ3LBFQm%NVNmbgI? zF%5pwaenV~UTo-d{dfoyuyQtT@rqf8pj=wB|1ZyZt=Vq`%bpTE-HMZ^jcJU}-h$oL zzabilHxorG=-kb%!tP5p5oPi$gBMU~wlIlvPU;=#+cnW>qOY92-edgvF-78Cdodv| zmcGTjVAX0}(~@u*1NOhg!mnu(b^d91#0rH5Wggta90S_t81Y#acTBW!Fktz-XAnPN zcpXRq;rhm88$KukWZtj?_oSh@?dpXQr&XlvnU>Pq=xz%9;n92;XEA)V}wEOkVI}X-!zQ;gx^w?wm@ZbrH=qg&( zkv)K_dbtOO<&??pLeNmI^&haDmwxSuIRY&F@1v0Mp;?h-lJo zojzmSn;u+F=EzysA0825Crl-J7Pt`Z&ow=9arj4=-pt{L3q5OE(KcaR#r?a_jbGaR zxK)Kkj`9R-O>rTqvU`lJj%kQ|-@=}c+b|aqv2OQl)ZTakNfR!hBGzX}Ypsbsm1;ot zCUmd9Hy|hiW0#RK7-OQf2YzW5v%dXtWbj60Yu^i12Z8P}+&(nvqYp~L+!6LUIJl78 zACM%3e!2MRv!5?n2rKFzctlsHMV%^xjtD7{WX^SrZugl_|E&CdI`W=yRg#+sEEbkR zvWMxcN4+GSr}b92@Km-pOjgEiy^#lZLcurbQmv^oR1aD2S=~lj#s}P?ZK}~ofa*hl z?UZnQFsUM7RJb`#bJv#!2eP2t^czpkg`xwVs%xFc{XK!rrU$G}C5XX2Q!nXUUy;|O zw@m1sQ-fB%h|<||A*L^WFRmv)dx;v9+beM8)en!NLvpG0Rs{nR+TCMGZRnImq(Dt- zURak}6J+-H$!R`6I^?Wfs(fWepN10*u3Zc_fmHz7hI-37T=oz8s&y&Fr(p11k~eSz z{9s?VO~}_n3LeMcLDg+za)eeHY4rCFT>wD;f!|tA+}0<(!;U?mOmO>zE4ke zJ2T9vL_T-WP@JLwyGUo3gI$oV#ptpS-LW+flh)lSXdl#JcXt~htr?O)6B^a#fftrJ z{o`dg1E38BP!S0itBa5i2%c!}MnyRbBZ~noX*Vp8a(mHv%95NoY>_19@+w418LJ6h zrg1_gp3SKf>^=BI@%j`so1DiOlRcxuojZnp9EE)PIfV6lmeE{AYVVgMwU+Yzk7Q=N zaU`g8`vnAvc2jD|74w)jOr9Vea3zE`w2VSmfu#nN)cuk{ndNrk=H&>bSzto z#zyxJpTUjw8|cqJ3H}7VcWl~{PWZR{So-fHSv<=*KVdxtxAfy*JN54vGxR?~##L41 zYw&hm|GUL9{Sv*ZU$vi;$Cu;FrBen=mVjqir5$&+6YK36c13s5AW~2NG-L{n7xmpz z@Gz+P(Id`-1R3CBC==Ekajjz1iV9QAw*LP928~@u>vhX_5M8KO!tf2&zYD#zgHN9@c~w)ibXq>!oI~8;Okde&d~ktnMQ)+#c~KE5AWt29phei?eab62a$`t# z;1~Ffo;+uC+UMczGmL4{O6dk(W8?OXiwS=1E zTejdI3|6^UN6Gu;YkDR91Y4?+xKML z`Lq4<8;mFkrYIosF1Pzqv&d{g$+F5t95}dK9JX>6*-jRfa-NRH1Yn3dOJWjL$hK5H-n^g4@5RyescX=L){4bo-FxusuoLf zQm$1;6x5k&XHBUH(gI8nQ5h@A>}JWs%-XXGKItVqa`&uZ6SlU3 z18Tvn;uJ!QajFOmp~7%>zxR_yJkgD>!l0XzGUJ-y3RXEIw+O^U!Q+$b_fm#6 zv#J?dHrR;aY8Fk#UILxx1ugCUI`I5~y(--YAwIU}DB&>y&~RtLBrn0k`HmRh3r*%v zXyS2ueWZ&!>N0!=UjR-i^m}>8D)*60*YTYAA?N=-Np$okCKUeGElTEnXzTbx3Q`)M z*}E=iG0Ef*o`6dTa@}tZh4muY(FtLFZ;6BQOyfr%v3-mRYJiVx5~ zJRXkE_(}k)b764CjU!cnxLC9EEp*oGP0)`F_9jd|}MFKQ6-{d&ig6?mqWK6xI`O2+%xtfB> z1>yeNxXkFG>*E9MtZs@;z=g*zG0LJ@k-*;Mc#8ywwa=!qL!`vaYp%YdD}|eSg+m3gQA?WHD(I z=-{ES9Ts1k_z5>?JOmjY}}e>r-%L ziXz|sJ9G1R4F*U-Z#|ALAmDa&Aozx;rW#QOkNLm4kVK46^c8t{aw<%V`!Y|-G7|SW zFnV5MjC?t8nBCIpJ;}x(tqG+xwc6{FEq4d@h{LU{#^G$)tRP$$Axj5VsU_O6r~N6S9#|5>#r(yCZK{Z%fRlv&bgXmkM5v5q|LaL!rd6 zlljv>&|a`yRqnhG97j!hQJYrHx_ssQ#vDx%vkH{*B=o?uFJ6Cf`^QaYb{e8Z+UT=p zwSo2Ei-lW@)SvOBC9lGPhIG-P;*^^DDh|%wb9)-zG?5)7s}(3M6O&~Pwd}Rj-iOZ1 zAL6AOw1&!nxS&7^_0eFrw9ew5u_9Nnaxof+oIcQ4Y9}oKeU9{LkD=CD7j0EXG~$~k z>@5hax9tHp=C&~pH$NSwOToSuQJFWkXX5RN1Y+`IB)5SH?8;Huv5LFW?(&?xBAKuk z`ctC(G##<{7%EYai?un9;9eV8?5hysCCx4?W}vdX%6hevYo|+@Kqf^ZVFAMTJb{I>;V#R1P2qPR*WUic6ohi(=blbzW#SrE+o2#LCR#RWx$7d(Z|+)5xOX?A zuPwUgnaxC3%Q?i(PUt-Ne5i`xjA(>SRn45;Y7^}B@fN9mF`3*IHClb2;urZ6OGrk_ zALGZPL&)xXdDaUZ7Ej4zf(razUT10Mi!n-)*|?6ApN1qzG!r^=FQ$YL9uWO8bv|`i z!wE=+kAvPP|M0_44p^%_7NnH95VR>VyJ>})=5* zl+ad4h>Lx}bwrJ=S_WamjPKt@3d2lI{Q=Y4a@O)*-VxSkbSnihAByp)YuN$tn2 z&eBF1It9uIW;LZnbH>GpTlrsc|4~51Hf_$Ihk)gPcCdnn^>SOqH zI*HHy4O(J8XMqX)PLxfG@A}%T2XC87v79_5WZ<}f?MkLSq6K$uaKQ0nJRX+z>YPHT zfJY4u&kAorTn~2?O_v$Q4wV;RULxRi0zr$wWTstV^&^m=TtdT27TP>dPc8y&kFV3z zF`kxaeU(mwZlW%4PKdL&;#WU~xM|90szi{V-}x$8mlVtfpx#{4V#P#Va{|V5+@Gw& zp1lVo7WebBnRm3^%BNO+cYK_w z`5h^{XF~s2UzlWlX%*x73OH<^xFN>0E&bajh3(iVX zMyh!!A)V1bfc)~$LmEt3K&wf7-U71@;!^0wa8aNhuWAhjqPx$&46V`r@wYQSx_RC- zif!lzYNm!w#o#70ACGQWy=})1a25C4-u*%?^Mi*m{q2_@`r3#b0&o)eP|42U$u=d` z0OJIMAsu7$a_ov-cH+kndS2Wq?AF`IZ5krS%$E8ZcA9wdZjgtpF5p^R7bWewc-zMj z!YyB{Q$9?Ty!@oI?qSozcOz*&7b2Yog7vS)ESA9p|DlCiJslx))+LMLrxNG)5>@$F}MOZzpJ34u|&uv}dRvr2ZG-@$fB(vHNIRFe4% zKG^+U#2Wx0Wxmh6SLakj+e4_~n%Sj7ZSFmYO|cLFv}M2ZqJ~}AyQ6oEr~7m>8_yi< zJ3R{=yYnXMdE9dyfkl0_TzU(qf@GKeirea(QK(j2ump&+_JNzNhvRG`9cdhQWp_*I z{TEX1wj|R_<&?nx!3EeFbiyCru`<&3vHbm`akE{OQHJ~Wna@bY5cAJKuoti_4fmW) zA)-Q?HZwer<$JICCzL&CS*FoN*n6gGbZ>2dR%Xr$Zdn*>RKGY+mMSe0SsZcjv#H-Lu#YB>C}q=?Cl- zogTzD+y^=`OcOXrPgWFb8>B_vR6}?oA|p>)(T=r1hpS;&Hu=LHc?owR>{awy`uq++ zkhIA#dYjn8g2$$aWZ!V1QS0psw{cT#;o4M{6Xogj`Ot?|YmXb1_U+)2XTG?@Z&JD& zqqxa&b)lIE$)2j_W&x3~JkwyHiw(zy8rh+R9HN@6AbZTGo5MybbPF1ATo21clmojPip-17A$^x3Fn&CgF{(CMWYHlh)7}Z(*qq9G@W?JSMmd+}GA6^iLw1 zf7re7gog9N(NRI!dHxVc`1Oqx5sqnRw4|5jRc%5US z;WhQ;>sX|i9x$l+q+QF<6s1is;~@B<%=|{KLtUChrHO_w1>KLJZJ3#E&uVAB{|&<#hnT^>+*kh;f%+F zPB(wv4#xio15h5Hz-A2f6FmNnzJE=voBi@~{!x2R+zPCt^#n>}fDk6gev69r5^(;G zF8noDn;|DU!^P=|&)|PqQCt3yLG3HDF+;2BWSP>4c4`rKd|Bc23lk@-nc@vU{lp#1 zMdi~IK{>jnD@n%6z&0P2u}+^RAv*72n#PRk)22+rcE;Jk zE9b#LP<{6qXQ~_M8g*1B@T*3FwP7g)QU&=f5`S}7?>JY&R3h0kqbGoAmZUPW4>UH+ zq>aL5hw9cAmvW=N;%>>2Ivod}}Sv9*;tmM(*qJb!qlJcxOe&`756v zx!YN5ZrK(3jUeP85KxlfTfc(b>U*idZl@_K@KjwzQLe*`yIkm(cFC;HM+~S*+yb7P zZ8w-#>rdzv^_8ZzMiJ=Cg&wAqcDRIzyDBOT1>NRXhw;;dwznDuFTb|+@5 ztL0eTmu#B_DMjxdZ~L^VA%DwI9^Jgb{Z6EW<9Z`>Ee|>2vlW#Hzkl1*E=5t6mWE#! zZMx6zIZ+GKXYarvnX+zJ?xf$e0 znI;G%0Sx*Al1}a{$R|eBGJmP}QlLkeSL1D(txnmb>^`@6Mi$+JZ9dzqYaiJC^lssK ze}WVUT+o+8vJIR*p-tEC!}JCw_YN@3mYaTl_w@Xn^QXLecTVgA7@l%|&o(FgNCHhi zdFgw&94D$j0T0BqqQLD>r;yM~#;(i%$r_~3Q%(^3q3Y+mkm3;bo<48*b-))+ z)y@_84#CX*){j)ScJP}t-9G(=^XoS=>NCm&Yt{%@dkLUUQhLFs1}1oge=o#G|7-1( zwSc~J*SL7_NDSGfE#}zGc3bTick)@WC>-UxuWK-*Rgz~DJ{%pa9Egqp-qaK4%SUra z)}~M}itTPC?!47Op8)w~V>shSRxiy(^f5&=_!Kt4CGa=f1|D&sSRnE6K8W%)>sCck zd-zY1K@kp{A<3 z8kMXHMb!Ahd%C;iU(*+0ksRX5nYM0$0z2mnusF~B_{riY-^2g9)l4)nCP%L1nipvH zip_9e$5`3(3rM|gx%U1A!FLHR^m}ij9~`hDty3UWjl3lnT! z>D_AF7<%NH+Ls#;_hJluQc~~>hu<`r%Eoe$P%b%9E?k^GW+Il?jnP%n__VnSnX0Mk z0!`IXuL8}L=oUf-Eco=wO|?8ZJcA?*TmRxF+6Nb=?wQ;zy&@#jeGBt{bewP)qn&vP zMj{}${0_wnf};~8lyIl@gv{W%!ywCbKvX}qf<*RSLJ2OlgdKP!doR7Imp@=eQnAPo z(pG?Svn~CY?*aT}GHlsi0lD;Z4KHX8Ov9i5zVtC3fzEXN2eZ((M3GAkOxRA~jUNNb z-~jauDD4bF=9-xQlv#23c}!-tXP*hw0$7l1#(f*ggAtGIfu?N=I{&`g8c3kuGq z@ON9@kFdvZtG?wh-}p%$4YR;gS2Esk(q zt&TsEcK>t%AT?YL1~T0aPXXWkmuS;my)NNz5(GqQBdYqKTx<2?T^zijb3~eP*zZtC zgIWZN1Ju2yGuQuXaaQ09%lcYlh$Luwi3vO46~Gx`*@yN2N*u_yZl=?h*H@I92_GT` z-G-~%>iPHQDKSFC=HD{JaJ7oJEy@bApMOkD)cEdi zzJQ((Dh}U+k%P@CSlViJCie_dx3!yIk?;X?geMfy4QSI=Z$Qecs7r3VxdP6t@{sk= z_Hde+3v)Ue+|QF1xC@fk*D7%b|Gnr89(d8i0=nx2BCiz+Y`4gbVjZjV-l}sBq6rd3 zuDQSdubp`ark}9XHy*u@<9A0h+R?xcH32@)8vyOs%JVgnXg)eY$nV3fi`H@9K4JGtXaKYTMC9R(ZTv>Pe z5y%HER#Uy6!fkwGf0LjlJbmZR%+O7g_iU{*?jDE||2rg7X>jxlN{P}4zXrbPt6UPE z6KqltTogp%e>bfy;>T6<2ZCuji~M(!N)tNsO|2Sj-58p}m^ zPYG14;LspuSuF~v?@Yawxb?EB^3 zh2H-aA;N=!C3?>AGjNW#jS;&wQ~eT3>ZaBx@9qD+?m_icScOEwT_E(8;Om(k#o%Ot zWoWU+Ax@&cK-Zc2BX!7K>t6TLmY^VOdDqy`U3r zbq{_cZ!F>Lg1)e5jNY478?_9+GZf-o0dRT=Vd!tqtiIWU#^z!Mhf=@X@XVqV61Kqj zwCJKmJQQUQd*Xg67 z_qh4@&dm|%sYIaw!@>xv+4bKUW`2aRE)-6KGtf}lFb z)K#^E3k6}E^u(C*+s<@m?;}V?LAu1#w8^`RuX~-{30B}jFKNJ~?EjD@4kQpjyn*Hi zr&Shm+M$$>Skj#qEWw>Vg@i5EHyBdIA8*oB(2cT^Czx;he1i(v%zPFQ-G>{4&z!l# zpm5SYBHMceYER~90w*jNHo$Enum8Jms0l@Srq^6OKJPZ1i9!^S#QkN+bEaGGi@1F~ zM2G#k&m@eyrTy{$ku|t5!l`y{Tw*4+PJ~*& z~QBbe)6p z^}n+WH%eCmUXjS52y2s)U5fQi0Wnpy5v$#KF%OkD03;IcQzG0=UW^^par6aIbiR6bDCiv#+}3TVcBHvWE8`BxwxEbQZWewB&AhtGbgTM%l0d1~O82v{zd-)G@9<2Xtr$SZ z_aIzPJs?P{D%#y&)=TE!Mj&wib0c6_&VmwlsjY=tDjiX}4DQUH|NqU0Jq0fWp3mng zU+52C$2dJL0ItvV@PLCiklz+_SrfQ!$-wmo(-H8JZq8cGkn&`EDwO0nRDn2whA9jT zr5vw~%QEQ)ENSW+7w&+|?j#4L-3+YsxqJGTa3Qdi-M<)(Bq`hf2_R0Ie>sj{JLwri zo!?0slVpt40c_441zp1fn!7QrbJ;&&)NnEb)!X-h!Zn` ztab>b4;H`UA26cCHsgswAEt#w_rB|In3}>> z-F3#=$6pF1RQ?>&<_u|D*7?MGmyL1vCiIO*h_t<6sl4}i_8+9gph(v&anj+j|CSbn zzGeO4ie8RG7kH~iKc;-d^R^bf2F$CYyckyc+gefvT$cDYgPBs ze@ZUK%6xa1d)N(9X^+;8CvzMm1V1`^)@dE$D;IxEyXnmyVs)=QehTg6r`ft#rYfg{ z4=EeNbXUMf<@&@EC0x|6>-NY&Fd{kmHuOf>XHKNMpLuURMSP!Q9BHinD3HrXfigrh zJBWUb{V<6!nrkTI%;fEudi1XBE!{OvqbLie=@Nsfm-aIm8?k@QHJpG9VxL%trlE?o z2N!H&W(ScQteYiDV8l`UF&_t-tk?{S$u(xr;^>3SrAKDCO6{SGLp{(_dQ=W5`7tQ< zC2Q`|Wa-QF-kJlZ;>Z!9?d^3$6MRTIn_7f^^X+i)?5$6dRzOC$o7;6NVN|gyLY-Us zveJFrdQRoUma7X&cfr#RUdP!==25TRbTkEsYmcEn!xwr1#CgYF8jrruv!QszZ@WH- zOSex3^1pMW!PNP_cfj(KRd6Q0u{nZ0aMh+>;P!B{hu;9FEP#}#pC7J+DjafOT^t@Df2?G*_DY>Ix|HYjGl z!z{zHV(yxkcNLvDJ|sVqjKfP!x4O7&Q=VT3@^ET&8w@aS-Jq#13S(Y$gE!&oXx-$X z>Nauw(Um zD%YGhbq*G$iTj0Jk1`Z}IbCm7-zF63XeZ9h*R-nX)TM~_lx;GJjv-oAV&&H8L8@Ly zx1Fmw0-U&3!(>K9q8f%$CFnGB>zo~3X4N*HpMFya5>%A1ekG5g)~sY7wLXeH5Ve_J zK#URB;bePnoG3VSNAFI-&)h&n8kTgSyg6L^FV=4`jQdN)8(6uGFc4j27-8VfW5wuT ze$uS(Z*?;dLDe! zTmQXdLoSSyiE0gJhr7LdMKzK~DBbsx0}w5diB9IcCi?{68iqm^wmqZF(|TiB)3_$A zm|MD<6w7W>yYp`s8rPzgY!#D3%pAA$UuK?9{qYoJ1JX-dszAi(trb!I(+w=(>|YdZ zvEO}O|2Jc;2BJoOe^~p3F3$V)858X@gu$jVGXrtSPZH9;&sCyJ9puSw!V zi}J6}YtO6x*K&R3bP7ib+FjAtdLK9!`BuF~px6Fqr6N%r=Nehr6F#*{Xs89`26i<` zuW*+wR8$cug1K$e3a??dl!v`x95z~I*K$E52>%KG7II*c z`wo%-nS@y9^dMf(?UMXx} zz;N#Z&d81nzV7!B&BW}viOmSUglpJ(?(*M22~g*Gp#=h zjaXzHewz&MJ8?4b#l&V$*92c@mLq`A^KsC?rJk&(Mj#EjVgm^+L@Gw&&L4ge2t?aK0K$shy%f^8_!f=i=m6 zWs|B5-d0r<>D@?IU*ZtQ1fNx}@1{G=foyn`#WB4qv_+DJ{seNv^VQ=Oan398#2Ep; zY)qYbk^*Zpu4l9!Fl!P4P{I8<>`u!EC(-`?#F9XQdsNEaKY*mg?w=P>-lu&2wf|Ih zxj!gWeC>2o#CBQ9%<{5E$i`md?!kcikOv_klj5nE(2;YC0x!Vn{w^wj>(xe+3UV^E zh-C5b`NQhl4KAEq;_ASWk?CG%bm#E&V>_U%sU|dYC&DGSske*Lb^kuQgO*yRD!!vq zB4hwc(Q=K24?;V>UZewTE@k3p7ArT3+#~!V-DidH))U8xi|Chifw(H0deu79AeQ9CqnyvR9b=e_XwJJXQPm2h4G*L*yJ& zNOF#Oo@LH(%tPimAt4lsqSU6$^E{R^hGZ&ABxEc@rVN!NNuf}L#Itty-tX^up4Usg zZhzdDefHUVeb#zU>-jqe))R+hlBgQ0M>PGO0N7d)OFI!8SiU(%k-;QEYiN8Ut~skT zs)skUKQK#CrQqeCLM~+!w}#W1I=?o2?M78p)fvPEu^Tn17u8n9BYeh@!m^XvIDW|# zH~uow@BZ+Fk7%Da%QX0b(8ydF z8N=nGU#rX|K{RQ(Ptyg>Bb2FsaBoYDyplOq;+vGyEJ!=M$~%8r?8}TOhAwyau{tW~ zeqeshVmtfDeGcEMgTjbauf9QD4->}sh)J>}e0H{@UZ*KzWg~1IZPVDUXU+mro}~3! zUT$yk)DdQE+$sH`<%c1TBOBd8_3t`sc9=2IgqEe-k3*Z8F?^V!fFCxl*6w+JKSI`$ z+I@wxYm<2s7|xwOsxxL&MpUasTu;k9V4RZC&lEd*t;o1IXc3vG{P1~lOd+OP5hXbG zV4;h4eT6d#Nja86Yhz*LW%>ex(wsJ$E{?`_H9?%mR0a#>@7o&Zmp*@WDtI(sbKi`f zpoEr2qO)<^Xk>h}Pa=nci(|h!)2Rp=6Y_P&?>5;UD@lVv#3USwA3k49E!8S{X3$Ve zaov29{vLwQxWalgh?QLKQqk7QMq3ymdaO3|pRw^GQBB_)%WHqT=uO1_Af{tNam5SOhFW)H2~qW$mwG-fNWDm8uvTvBmUvd=AA!{yCio*n1#B zF?#4$FlLza-1?QdNo%x6AqoMM7pv~wivnP7Ld8*6a2urRsO>H$Pg*LHb1`Vk@5tSz zEEU=>?{_MBHJ&Y8Q68q^fP&q)U(pKb8poOfQeE@>oOwdN<6DgBjx56@0ak%M`>Kam zPd&Q2aQ{)aEOdPcC);DDoDlW~u;ADlWGaeAZ^3i$mnxo?!fdQBr%h)`?uMU#1R#n9 zTj8`>k;G%)sEN=^7@Y$S#k1k2to-nG-H1d{q+Ihev+{tp>m~;TXIbpyTFaDqm?aw_ zsHHdb9V9yLBdif;qK*~-l8_@C!N#bO4TvkU(i?Iu_6A6R{TLcM6+YigEhj$_+lw)5 zjboF(?k+mhk$Z*q#SeHYO{jDq8O1e>m4)F9Zz---!qN-CJjz<;HF!3Q^io+X0S#h} zUM_rcdo=@3dlkiJQS7b77<-3J>b^!qc+{5Zcm(JYXxr?2pS2HZ98C_em_$sp_L&c& z`+7*2=7EZ{{(Me2j|3{Ye6ub;EH39x)c1#HhB=N1j#9?4Kev6EnkyTzN#jG2!7cxR zc9(+MYu1*PYJFiW@oScL-r)vTkC+qaiuA{C&C_+qI@BmGaGFzy?*GB?(F@6&Io$|* z>}-gFMkgR^77-$rNk>p1;1^W7H1{3=5ppuOJU`>jF2HnDZ~p10%g_4livCeoWUlsX zIv2f+vJd^lT}cV&p0`P9qtkvLbAHj5P|R%T z?s~(GE#*8eV9?85@^^pvp&r?m;ijd4AWvM_*)sa3W`PX|(D=yT4G*2?BXUR;%~Cn% zv;wmB*c0r2AG=t6hWpp^&UbFR9UdYrzdXToPJQ&?)9&Lk8U8g6Iz~@aX+pj6%c^Hg z0YDTuoftn{`EXx0KSYc{51XhLfJaR7rC(Ums z8(nMif!s}3pw9|`dW^&ChN&f0Xhr`Vxzx}73RYiwo<{&H>!<(JjY9>RmewcsP>}@V zIbhTnoxCoO@xP-~{M7}3xoKJwWyW|QzkkPe`?rbcyixSrh4&1>QDzcCMtq=ITG@LQ z(ET2f*kAYi_lLR?pdYtwA5k5_;al`-IOnfu#$Qi+L``|U9#Rr$I}(llG>p;tTjQUC zt~K|vB~Q!y^DjlaIImP(dG1wohl}C?y6PCGw=nVGvt(CMBQexMj89^g!E=Fwk?qz$ z1&GpzgtTZ2=Q*|i&J;hXI`s%9_(d(QcYWDUhf0qUp|!<{Jb`78%xWK2&a5T}sv2n* z+Hkh^0vu7~>IizUtJufnH6~0_q3Frxo0Gy{4ECtVeL6h+T}R~~{2JyqHp*cBu>J9z zDo-F?p~y{>ug_1J%}K6E{qLO}M`A-f3npUkUfz#Z4%o8G=F3~07mI_9c+(}h=dSB- zJ{&f_;5Y6ci3atFH!jxtUm0A>*9loxY?OGsP}%q*UD*AvQu^QDsEOlH4tynW!e<9v zKV}}6wd$Hyde=>Cx7_Tm=JdTc|GTeExNX3K5Fo7pX)z2WCe2;sPU6-%P`?LVRRo~< z;35$!f`9o7+!4$O_uUk#aP?}sP2ro$!54Vrcf;Yg?_SqD5~ha1RP4|%B`AaR>){hV-}3`x zeS!Ky)E{-QpfUj(`v06dpe8uT z!s5i|q5_pRVmH%`X(+`}&I`$O*QpKK9|}}fmpx{WDSIy?(@=@wIB4eB%T5B~d_Cuk zMvzai5xm=rAgnEV!FFs10uGwkR?_@jdB79nv~_0kVOjb;lp2Fq88}u*xOaMX?sdfk zcwAAlu<%%U{6B9Sj#7t!E)@kn&gFn9gz#k|C$A|$Y_%Y;=I|hHUi!wkN)rmTk$3LNB*@P}zrO#A(xrD$Tf{+a z55`mMiGl~yFw7WF8C{QLlL@y19Cl4GX zJ~95BqIm~4Yo`?$d}FU;!PMY^*$maXM*8s;G}5iguS5U)@NWnkbGbgcXXVPr z4%HKHCPbVq1B3v?f~cClbYYoX76wmd`X`}ga70 zRm(?vum=_M&#BP&!I}R=A|paKL~S~=QQ+N;a`DS3`rVnr`Sf1wb+sdZKO1aSkS_#0 zEnQLn*Skw6Vcdjrv}mNj3*9s|(fxTz36M9PNGe zaJ6dmbQ}L6yo4;Ohva-yzu$S9UM+i}7? z2$v-G1a=p_j}dSE=N`8V<9cr@Jb(ze6*5m}6EvwS&7(mu@tdJ53exh2!=2SbV*1mJ zO@iy(Kj->JpyC8*(t9$+Z@Y*0-f=>JPA&&WJ}fRsFTUMsiyMDsLeM%haz~xYC91&q zYvF&-kuVN_OMg(jzpmBFC1+*29czBfVPtt* zzAjr4BFch>jhF0Bd}7CB{fLr2e^&k^Ey^r{a^W)|jz};BB%wCLfJh;rYG7aKl~jNBt7riO_)Bj3czNL z)(Zi*XAlw%Z#HN-`$k&gsSh-+{3`~~A>iujqAaQRodIpq;qYhb}2_H;m;Fcu%XQfoFPeQ@WAoYA1gvivHJOB3w=|Qb zUW}gv##9Bl{F5uu&~%&f@jj*)LWo|R84+}PEPOCE)v)B3FwgX&%vCtr+jw_J%#Wj4Ma@YkH7-+z znigsk-i$W|U6|p3U0;_VvziMT&5}Iz@b_xlB8ai3WiZ$RvqPuXDt!1Xl1Y8Y7mc4^9Ifl*rm&##D|FiN0E-yD2`XZ9&!OZpy7F&D)#v>W*qo8WB zx@Q$%(TR?cH@kHgM(`d(MS9R*6f%6M`>juBc!C$(6iv~d47uph+4eaARKe%Wi#LMM ztfzuaMYRVh=j{3^t`PeIee&HJzCj#yRkFzy*I@RWENG*lW&ShZGS?c^+$aj79B0}K zZ6p~YJstt={lL2LD_a96aJ_5~8>g@De`NvI+H7;d6ebKThBYXE z`i%W!-E-rgK+i##PR>Vw=r;|n>E+sDGsI5lXW+L9tP#_RHXz}Gj*mVu04Qd7h}*Iu z*_dj_lth*izN`I91cSOW{M_}T@4mA(^DV2vcgtf=bs-6Tb~M9jG|ztfb%iV1cZ6x| zJHHqEofG%l=Qk*TcEQqU43W6=tYFCBo;H#YmY=Hz*CD&njk_nNWf`n*ikioHz0A0# zJWf5;1ZRcDk=(^MIr{>Vb=lNIx`+QyH*<8)bPcWwyWXXj$~Su7eoVQFikQSqiR76B zfvf-eNA!TluXNTUSB0@}bwapWvnE`1SK;Hl{Is|ex8X+~W~)9k%vX^9r}R2=cDxC# zOXc(BMqFcE)S&_>n4k&`dy2b9QT^|cLVl}SN=;h( z+dMq^Y!?G1LXoQCQgH?wnQKIL zP%5f1f(2uRce-Ucp(?D#`=$iK4#F5T9<4yf!guAq*AmF4j-RWLZrSkHT3bWnw?h`|aZzIKn}=^y9jO2_?ux6YQS=SVh*vAE9iGA)+*wZq%u>X=UB} zrFq#vB2b5`JwK3bb^_urK?>9i>(PDLx-DGW3aH@hPtkz0zi8U*6meP(vl#vR*LV6K zii1bbB`&=PBpai>=bzfEO#y2A_qdlM!geK6xSAQ*CTu+gAXS-vzL&-gwQQ-=7>8gU znc?~J0#G-N!Ib$O;=`l|lpg@d7N-LJnl*2?6l*@dX(ENnIPMK990C9UE=J7mA?^X; ztYi+f36XMa^uPN^kBxDZckv;jr{VZsxy-+saNGCq^a~>^lB9}xK{hy|xyFUPsio%N z*FSS_Dm-MDAThOmf1$>y72~O71MFC+Iz{ap2yJV{%3S*9ut+8!F;Pk4(|_b;Z}Ql!t1iw|61D6Ryn;}s#zD|)1!lpw6`^ADCmpe}0Cc(YpH1j*fi+`$^B z&9VSfOdx_AYSsM-#+u}e>&YH1A(U|PT`~>ZrQ5@73bX8~1be8}S)b8hH~+JMZk)Py zFIA)EE)K7zLv*t#tx4^)e*?W2!aiUzYkvQ)q<06tHxkm9Swts}nTgrI{q`3eF>vgI zSMpkcs8H0Gq=Q%D$gpuTKalRG6!g>8foEsj!fBY>%#t3*H`DFKkQq#wsJRI%WI$Zn zbg_pf%kJjo2rfx5k5-oERV*c0$EMw5A;^@+}oBnpj|GuWs@Z9MJoGmxDqq6@08m?(7~piKOY zfYj{XweVra{2l0Mbs0ko^N$-m*qNmdB>bz9<#mn}Y^du>giA0ExXm@MCX{s@F$2)Sow7?VOfQOO2Ij608TRJ}a}8*8zcRq91ZR=uA2{!% zGW^{ywR~GBA%usB;!GM39u?`$fwTzYrbLP>@-%%ZUs4Gq`tKjPsSj&}Y;`g68}n@4 zI4BwwhVEdcY=_i9Ca+}C64FiKJv29kB+-NiwdG(z-eJQ_#hjQ1Gq5dgceE8%;PiEFjrJxT(XmU z&&FnV!wpM$F(51no!!0+BAcHM;v#NwYW?}WhcX2Xq2Jyeo66odW%6ftclbI!MuccU zL{yI$)W_(15X|&#Cl6}aiJURFO|fDUUA^aFEs*R{_b|Cw_Xt66B1P0JU zy`THvE;N6?b>+Lrz~)DYJawBBNxbmwt-?UiQ%85mWr~Y|P8}Q>Edl+v?Jn>c-;AOs zdyaiAryw|uZdQ!49Pfeg`t8LxXR)I(h_weOm=37gBxbHG3#jm5qnh#{T01a)W!ads z3RfH(f{RjP!OeeG-k)SiUdAosTbRTG*R0})xzsp9VQ{$*aTM{&3@2g?y5a1!1=9m6 z?imb9I8_lD01*I+De3sV7(j1OZKSm8U_=MUk97$rvqKCFTnr|2NJ$B*{g9M36o<7u z4m0kAgkNx{MZoR{APIw&{yr_PWi02_pQaB71N&Ou)q=cuB|+!}maK+tQ&-FL_%N?i z*Jgl$>hS5QKmyUxt-LAt_#K#L0jvQWT3||T$|^(e_r}Uc+ar%)^6qV1ee(D?oaW)Ypu}0wnkc>B!2%Ex9M6d;#YM!S4TXv#G86hJD zu6p6$l;bXe+037%)Nj#=#OP3*^?;tE39rQ9P*qkZgjNd^Eo`uL}^JyNDo(9JN^0z5CezpzM<+0+9fjh=1H z><`XKsR%KCDHRI+DFQQ?t=NdrH&6sWSg&JrKt;w;)WB-&a2Z8;x6hMVwm?cn$(w2Wm?N)m39%GgFf-i!~-nLN}4J zOiLlGs^F@=TD_L)QdWh<{u2=b(t-!WUDZPJK~=Q#-b%B| zEU{jJP?!^##T}IkwH~`Q!)nAPCUjHCSfM^Gma_dcM{~70x3USME!_JQ=l&*+CBIjT zO9{&$!qPgxOCszyDxgE*__v&F>#zmUxM`X^QC6)_pd{c!Ll~0mm|i8A*GX$R0r*aC zm=pCXp)qOR;WGjK0X+B8dspp@v=6-8OJtDbwTE6AI#sRl#U{6u>EN4#ptH-I#vXn_ z3$o zoh8{Z@Yb{a13&9F3wWAq45y%@$-GzrCHf=RMrBPa%cN3a5?UOgLtTdi2NKQYlVE7zh#;CSLiA z!D=JOrQraqK|Zx&?_l`+Rkt<(tpc86kAH>Y(ZFd|fWt<$3?D!C4mwd}j7E9~ItrB{ zG64@am(qt?S9CSw?)m-7lp`cImOae}hrK9k0HsW|>yv zvb=@IVPaWCW2^@>4=C_V5&u@_5L}>vnrHGO<#hXLYW!TP=yIP4)ex7{NI{cZl4)7D zMFg(yaiy;43$L-+FWcZ5chYX}913)Za-RXO1#*>P>(a_h2Y_h-(G0vca~wTf$CrkB z6o@eIl|YJ50JQQj8))-vfnt%dDZXWO9y(1m2B}Frf}Z{$60bea_!0j!MV1BPa!l{2 zh2DIq6&|Cug#|?1UHfsJ@dV`WGYrK2LIX{n$$#R)kl-hY4_>S@7-ZxV{L7iU!1c$&vI>hMd$wPOwji`&m@XmBq!X*uG6X*f;nU z`m4(TV3pcCijQqJkV3XJT}h-I-h}GkfH*UT#|X;Q48_ZlZCL!BXYK?Y+7Yqluz>i| z>fYNY4!In;psS6@Q{_IZU=iy2>T!U!&JIiFQo|Op>?wU0^OqgsRrh>I2B}WU}gJdo4V$GoC@VPQ)#!}Y9`XR zV(mkasl`%XvzQ4np+E5>YUsm>V}x7g{MD#V3rlXH!pL8)7Rm9_MkO7NAK4n_QqPm< z-^$nPit4jD^@f92v$f0pb%iaoS_ei8=YQT2CAf}@?2`x4 zVuWyj1#KnIuQ6aD!@esCjO#l9*8-UIQ;+A?9r}YKq+>@Z#~uLK)*&yj*^o!5I+Vc# zNnh<>yn(jJcxM5(D!^E7&R zspa1F{M(W^Qs3#0<5ZbkD}Sr6Wf7fTAKj)3{^hZ+f`fWa0B$kz8Tyv&BZcGiL8eWr z@y3|G>9$H)`*Pse_A-Q=v4E+B6RW4Sf^-YKi^SHSd&gd7g)LTHdN@3D*z#}vic;&w z%^O*FD_ZnH)x{}}oGkFAxT5)HaQB~#SCv&}V* z^t$^ubh)5>mIF6gSM+YKFVF^0&?EP{R!G)D+$J1d@9QSnKO>bkX#(gA;mpfbCs3hs z?_zWau1MDjh6O-Um%z{wGv>Y_mEnEr^h4T%40(%w1e)WgMp6tJTDpckG+&>5OF`{n z3GED^)Y5u$S9An246W3IV%0jS!79xD*(%NCwV|<+@2wrUl~Pek24(j)#-JbnWq^ zFa{y{QefcA;g$5F_f0wQJ6a7Ao40;_uVPEMy~V>CFaet*q2o{B-E&CJLqs`JuKhi% z*Zc7Z&bEzfdkcBXi*fR}YKGf`5fxf5UPZHh7I8XqT!=*fPWLcZ`rkKw5dC0Ju2yAq z;irow>CMpOBB%r$7j%zsYlR?|12S5)R*6TTvw=vr{Z_Y@>QHLKR{CXL!Bu){CmDap zcx%)?;G!76W(y--RHqbFSXfGfpPhPMke-fnjih8ea24^8E%>TgqE zyzWw+O0&I}Lc@R-b^=Zj9$WSA>Z9)uVZhk=xq;>`l`^tk%o6lmrn*5_h{j<|@(sPi zttIBpxQ-pu-%|`8!zF#NiBE&e)4d)F{Q`*K_8s7evKRrz1ws_x2PK%1TXwM!BVY@` z1hCVR6$Nc8rhBdd#{K7H(5MO8rjlD8K}p%H_p+#|U{K@neEWDs>);6>nD2l`3wa1& zg{`P#_^0a>#sZNX=XJ&Ot+Io-mH}6l8oe!bS|89vSd@1&QuxM&uMjMczxiUTreuTY z2+#x6<%{<@B+E|?K^yQ=V;U66oj|%*^OLv+!5?V{6ie3dPg88L*7Va~Q_p{nu5Hd% zKRv-a zV!)nTzPk4OqF3^Vp$?Em-6Nm5Rf=Nk6VC|Q^%I?;J zxPv!1)fmicpb!yG$K{(r*cyCKT^s?|w!bwZ-#LSqTfvZ62By04REC}%_~2+fWW%Oc z&Qg0Z6EK2cMEK@bSvS)c*;y^j_U|)XqzV1lmsi7eM4|+Ay#Ni6m?hu0JUXJpy{x-xTPV(*&{t z)vH@gV5*)c2a_|s;LU}}Wo821n}m}l)dP8P5|Xdj6&s zCl(u7u}Z-kzP)&yYb>?qOtc64r}-fZ zT+`ZX`wgthYB#rtZ{CYF$_6)det|;Y(fZ<;9nb4D=rC$GJ`LPPL)+f&efspfDC`9T zTyll4rG5PyJid8xS54k>UY;k?)`^sL%Y-s%6Q(?^Vk-{oX|cpaeaC;I8Pj^poG3pN zMVrsX^UVCvOl}8e{c#C;!rjv=ShJ5f&sH8$%lx+@;wh^Tm`zt>idLzGD8uqWP*9VX zFd7oR|DY<6uGr)8*-1A19{hN0+}BHJKr2w=HY7AYHMI0%kx;axuFad@Fs-vv(}q&j zMLmNFp2O?m@4HVOwX8If;*b93jqq1aJS1NvafZKywIKK2_U!%sPq)vR84dAFvqQMn zz7?dvq1c2&EU78A;hJtVG#Fy z^lNS&sDx};5ID$=x;Do+BGGFkC9qii(0iv6T@fZuJXzy@`|9gD({_gQM#mKh&*>^H zoifuXwU^qOv>WtW_T83xefWY(s3w#tRrQru5e}J;E4mzs-u z<*Sn0>2n$H6d=+u+V2N6&>-p+ztJJQK>Ig3wTS|9g60VGkVeU$P7$=tfd}N2r1} zPN{za;<{sYxXhe?`!KHgP-jsa^^C8OL;JK)s@zcPml{RjooHc%QbO$EkJxUkfaWAB2c`Yt1Y9D_~CyYs&6-7B^W z2MTJmESba}SD$X#*u$(CddoQD&HQe+zA;yD<4W`)IKkl|=>Lj+O$`_hQ8f__`%5o4 zOfX3kj{o=lBIu?s+b*ub0@mm(xd3&}QZO;JR4FdxN z+8njXT?M0Bea`fg8$9ju{EiH5^8iAbR@g|!)h8uAUWVZ0i%-eXrhIn%c$16LM%dA#c6vAeIT1gr@h^q#f<94%;%74*`)0orHRXbNMMsf_jl zIY7y?9|gJbA8o73gBS#04jDL|9m8FCS$|N`D^oDKIUs1pDroRjPSU9;`AS*Sc<}&M z-w0NiY+;gLKbTn{y}`ce0xU3^@!ZeyJ$UE=+;}*WE98R)wGM*jeh!YwF21PN2;29t zlP|@!*LyL#J2Y5gx{dNI@GWr0#32sb4EWc?0csR<#f4Yc12ml3<$6x5AqLU=%xzi; zYcf9?TF3l*NJfR+Qq$&OS={4s5xIw{!r#ze6Ih| zqLJ-7O%-|P(J!Czj_DU+oP!`3hI@E7$v#TFHRT?JNtZIiv9dMxZ0A#ufvsY^GAaRs zw?%4S!XtU)XpEksmb*eN!~RNVy|6~M64bxn2qGK&He?Eupwo`6HfaO~j?XYF$@*Y}3~}N7E+d!C zwWbg+o14zOX?BcEpM+%3J<OC~bfWRX9%DL8=ipiLe<2by5 zfO?VRZ40T$QAo_*b5!%ACRrBv_n%N6_`kSHLJE3VKCocP)Fe|Nj~>^}Rf}E|WIF5a?x%`M-8e>ZQA9fl#svBE@7b1-KtdCt*ls6B#x%e1=LlH8WlrvBP$~Gq$HpoAry%{tKMy;C!`^}ldFR)iui_YI zz7R_|iJ05ow*8!3hgfvtPyBS}ka*Ss$d6tqq+^rf@tbmxf(;SvpXW<&Q5ZY-$j`4N z{DB*$0eh{zmVS*tvpu3&ePc0d4~yHL%CpH9voXBw(H@#%qB@+lI}EinmT3ZWD04mYT(SI`1(YzrNxm>z9SG$PDSlE+!xkRJy3|>dLb1$Bw~`sa zEUKX7OJ@=OUkCXDs+Cs5uhEO>{6)c-bqrA*mL`dHJq!){4Xal+L1F6I_nTG9@hT5y zk7`O-@cF)K5W-Y`aW-m?5vr5jXrtjFEotzO^tS-f7C9SL>gb`V8#xp~lG;0B_!p%K zeZ>ec@s^ksRVwc?pq(pmC=%={{wipSfQqORRn~33!~U-e#sG;>&BWT;iVPh5Jda_? z;N#g>uO<2E(b+c{EgT%_c1)UlJ&7=M1KetPJ>bg=qR=dgQmPZ_N)mbxrGq}w<{p$Odyi- zKuS+-8m3iG7$4C*?dh8$9Ck1xf^ws>I_)=@5pOOSGK<xz=`eNOW0fO1=Mp^?q=2NhB5+WE{uR_xwV#_5!4{w z%zU*`8J;F~Vhae-^Z$dJ(eGaXpTsYc@_dMbLKV>?t6SJHUA8@I7z3|GkwnlX1{RJm z_s%=qd6BV~w8DvZp%f|gMYX-E7ODRL0_dBt({RpvvHH9%n1x}vRu~8<(V=WxWhRQd z)+UpSIiQ;ui6#~5%37V+&81lA>WU2-C`es7@aGce7Q*xI!%xC1pxx2fCr@8x_>Y-n zJynzvc>E#aP{ogIR~B){o3HL-gNSVj>E|wH)iC}b@tGc3!Y)(fNS@lOnu6v40j8Ds z#Q9kHpC$%YSx+W6slaUK;}#FxjLLlWaz$A9EWU?r39uCGcaz!Z<~44$*)OR6WPGf8 z7kU?XgJ$fGH}9|f^55fvJ~U-o>{O&>1Pddb78C2Xxvv%>%^!yj+k6@l5`Iq3f|uIn zt1GS;=B32AgxDNei6RqSauqJ{b&9vwT!SPu9%kEMhQ(?-HikoPUX`DfVGgMLLNII} z$L?v3B9GG{&L`GDR1WjM#C3t|Q?u~acX?R{U z@CSV2B1Z3OPIbYLXKPveTbBgmEJPw71X+^NE2``BgH&D+C#0F(3^BxDyTd1=;=797 zC|yWg@plb3k&|nWhZLtoV01kyBJrO@&&~tNt#inwt``63gc0NaE|48A&{p&Qb&Ohz ze4(D`YW+Q2zCO;`*1N&%9o&9!rMQy|0PcVQC!_%c)*i4A$T+z2voN7C8X5xzw{GT+ zvya}*<$TNO=6EBk5*!V>jNi~#;Go}rq#2$3_YkNZsf z{;hD(1?T}QaqPEN-3vNO$9kXfYU0t;ku?^4>mo9pisonraEZOMH{$a3>?Az0C?e)F zrQ9~p#ZY4=`@zk8KupToI9n=Q{ETv)z;RaC*OYhn$BjTKFFWV_2o(YY4`~o8gw4EA z6kW94Ek9aU%HN=^?Wp_k*7U@1`d>)(pBJ4Is`@O^@HBHE20Uv{Pm2I(+SAOdaF!zf zVlN3%Y7f;Kg?@<5EQVb1eW*$ccSnju%Xk1}5KcaUz)uXXK7sf_#w;r-`wCWz1} zDc!Wxm%uqyFZUwS1K7e!)yhx6U)mSx5`=Vl-l0Ms_~)tS7bi0cSN-nOo5;?Hp-E#D zQ@UC=W6CUGelTrKif1^fSHwCNacV~0RI`-e;d!R5U4qL|AL%pwop)`fV=C} zgT;azf`QZ<%`>PAv;C>|zLo^OJGwT*fRr1b@{Cj-0XeIkdQm>40WMcO=+A*Uz*Ofj zbJYnnQN|$UqZiBfu-;-u9>Cmh-_wFl#r@sU&Q1uQv!OmMG}4;P9u2HI0Z!i*C+oUB5(hIyNkX`w zsFA(uQ?+T9 z6M~t@_D2M)e=WRy=g!W$3sp?BJm6$=G^JPhiI1SjuyFl)`~`ToQ-ITf+57ycP1-?? z?P&^_ex5muwmN=;Jyk0oPks6h4e71(F9dZX!;U4YdmcO1clCN3Er?I!@~= z^lECJgOn7!eu15FY_Q#Cn(UteCJL9U=1{SrCE@iu^X<6$8_&Ul@TZ)C27f>u4i{U? zpL3POroi|o^|~~Am)0m_|9pO|ueb7-qSQFKuxC6OULKk{oIyqW*}{zfT_Yu7nHdUQ^Vb8Ge7~>~ z7qcXsHNGtnX4&zj-Q}vagglS1-_$Qs452j*j<3gMguL}< zS2F$IMc|Hv-grL7Q>Tj^BC64Tghg|AQ#I47mIe#=t&`W|TV%k?Qwf3#u4>ej3l+W6 z@^tH!G5=%ywZ4bEkfFn1vY9C38MH^|pT)FKb}!I#RIDC^kJwrXJuR7aAehN_eb3f0 zsPIF{+nendWNgWXDog(UU=!e_Z3)O0x@n8qxKGQeEdSs%R_>DPotD=sWicpiQQ^#V z0nNVB+k{c|)X?SzpaCv&pA3H;r;$XmT-qc~SDy1Y+Ys|5mYTWJ9bM?Ti_RZ#D_-V;dx0;u#(o|I)M=p(dtd` zzp(Oa4I-+pKZvims_l>hVOB3~f1#2M;Qji4dXzYg4VOax%i^QM7bIf%h9vm3+;0U~ ztqUg^N5gET;mfml^n;HBbC17N;$q$WB)W{z3)2bhSp>ulxcq;9JtYCQ5WkV7h2b-W zWCB_)OoE22B#`zj_9{m7vEo7d~bo*rSNQmRtSaJB#UqOr+#~)#~79sC1p@ z!mBeZl!bH?V}kZzeTrpPg;O!2M}W=&XXr16sY9)Bg4Q_bLh-_zcG0BwFgRG3{ShXkQPtu^^hEM51mz16H`;g{FV!Z?nB9V}AfDBx$E z^5N9V*9^yBIspGsnQEv!2+1|?A+iDZ5~nDTlpmTWBlOD4Ap>`}+820Phj)J?yWb() zi+dhT3`HYX{YMG_<=?#fc{2s5?BQc;e)oHzgK-tp3Mlg;poA_IVa&$=pap-10P zjlxTH+Zi}mt0(?v3h9XNNRzKiO`orM<#Guh{)Ytor-hT}f={uJT*OF1$RVR?HYGVn zN5d55R=l#_DNs#><>IX9O>O9Lxd(phkN19#=8|@4j|#jMt1<{mhU+J&QZ46pEz2!d zo~i6#H$jz{8GZufk`tZRgd7ju_n)y;QdzAi+?iN;X7Ll+H3r$+YG0zrQHO_a(aQ&J zSanJ}D$Qk`dZjWL@#-Q%pOt9*&qe+7_kV!lstLj7>1A0-6JOSvM=u(5GyZsFt)G!k zR?Uirg_DvTOEJ+hu09sMwYEk%GSOcy{qA|Knn5Kk3k^gh z`NAG9bO!7TTEOiOSGI1vc($V8ykqu=5E5ei6Zl=yg`$f=BbB6J)(C46@6VIVpp^Sp zA@(n7Lgr$`GKmoxEz;dhsGZ6~;-jxB_Q}YS>a{AS`bg@q!b>F7>G~v+aQQvRAQB$p zc=1`mU@TfMwI65cFn3IftVW0oj8tK-6+$#F=_Q5tI;NUC)uEm2J>w*j^vZk9;zo+Mg{Q={sochy;BC$z62N zI|#3|E+9w0EoeQQl4QopVoG=OO9cIqyf<4s0*436pbZdirB-z14|ycs`e0FT{pxKU z$_Lh#|F5?je;M9xjrSd%PI=UKawU}N_{NF#qAqgAMVt=zqGk-`^I%+49;iNlwG}Wc zxBi>i3w9gGlWk~3xE{lTk9^C`)%+*R-9g>zGMmRSC(q`GAS08pU6+YV@TYEQT6{GF*R(kF$@g@O1h&j*(j}~ zG$c?jt#3?^Ew7aMGK5OoMzN>T1*c_Ao-Roe`4qEjk)cm>UNTIW6fy`0cqCSn_71B3 zI)pduiK3f*RNS>WbOPE|m|hxh{X*AZsNI4cxGIvlzad?UC$>(<1Yxb^dIp*r^s?s=Mgp%JEy8cq$_R5_-0 zV~bn=Yb(iI$5Ad4RCWm*lrGI-C4GZzY78r?hVrif z8F0~RTZbK?gy9^^p+6G{`KH4>dp#se)}$O9%miTF9taWS_3Lux#nsT3Gp19MI?C+?+wcix`;2n)V6okA@Cy$4@>aZhigpC-3cb=8yG zc2HHRA3X*>G9tVf=COU{F-b=ue*#s`@0t-LRMKKds^racGoQ%5H!J7{0XIlwlivIO z5$NHX0X>IDsNZf-K@jJoIx-%pbvt;q2LRLP|KE#f%8hWn0VPf#`OR-2!toXR=tw^q z+p-RHeu&DJI23VcV`KWvbw71T18^~ZxuX-@2l?Qk(0N4hYJ?RTgV2(5?!V^8d|1lo{V^CYs#euYH9MDo1)LD;!!^44Ee1No1+ z{QC{&qr<7piL!`x-jJS7Xi3zHW&vDP{H8~chbT{eicmF)3i1?Sop0;2nBM{>sF%dF zt21d#$<*3V^A+8Ql``fT#cslOxByZ~)rcm@IAFOC?LL=XFZS&*#(KxqC$mK~4(VH4x7Bw_YoU zB6#lZHouYP&toDxEA#`A^DsH_gDhT>z==KH%if7e43 zX4Nhh`m<5GnjrR)MQjxka3jH4ghFTEWKE(*8Dnx=!u_vvPH?+3lN%bs>cM z23`vd-?Y~jegIQ9cs?1A+>Xnb8R!KPAfESS5>fL`{QDz}b6A424EdDe;B%rPs7Eji z@QfYk25N?;z#??b1in(Z|K}bOF`dy-*oPtV+~lc!eLu8=Wtm zmqBl6lnA9(LxMLlrc?Z(qOIRSQs}-><*kD!{7tVrJ?i=7H~oNyMPy_820iCSQ&L{& z7Zy)KvL9S(_`5F1%3U$KaQ0qADWk!4V$9_kBP9I?kaFEb+EMTnkD;Hx^C-;S|ARF$JjsX7tB%e%q6MLWrReOeyAem zyzwg?LW^9>XDDx*O*br_`cvaK(|_mcp^wI=0ydX5a>f{wVq3vr z@dMxo6cHWX0b!^oP~wt(kh?2EuK`d&-zNdA070#km~W_y4#u<_JWHW@P~c!cm&1!JLoJ5H4CP# z#vnGvt$4Q~n;gUE0n*3al4jRA=uGK14MbP*CAF*Ef8l3R))B9aM#YsI2c_Q|H;9F) zZ9&Uz-mFYIh|FlpoEhS|p49|e+6_FqN9>{!H=XHc3Eu;zuUD18zn%Oh${_wic0dmZ z@6Z(3g+iZ8u=fQ2eUm|2CL+woD^H_i)`5+x?|fyP-dL9d;T1lfSVd%~A|GMIw}>Tb zF-Mhj{XRsi8u+pkR3 zc^#ayd(E3Q)dUa1>n@IOF!0{`+HsF!C4%Ol7{_;y?x9Z(Ulw0s`_ubV(^JmCJvBz! z;4eOXKL)PJgZZq$`tHC60m84Cu^K{9XGyk92H&vnIWB$RI8zX zv(#!2x#bxaFNLOZj;F!W;Fon7AR)Y9WbLAiA&YdjMTI~4_Wl{lJx*ooAH}|kxD(cx zvBwoG<*cqBzUu1d|H%zN1iX~8Mr`km{U}NP0}p4-I?2c3#6k#jp4VjoM)Y#6tUv32 zKBM1rfRfIX-yQxi7PsRi2AkpAi2UQzo3DFy_9Z3^&dP)0jt&NnTjdL~WTQ)g=pEhk zOu8pUn%@7{ozg*ggKo?yGvjroJ827*0D0PH)Ul+D96l*WF>orvWueYknFw5$jB|pDxzfX>@t(RLq;h@ z=lL9U-}iGrujlvck6tA@=en-%`F=m2_xd2k2lcuSNDTC#_H7uNz`7`#C@;D)3HD}r zTaQ7j2IZ6sIv428O&TAVUjd^!16!vvQ-oTT33}@`MR%s{k0uLTE=XMxeNM<=VH^1o zhvYRKV}Ci5mAo>n7Xl&E>(J57p(2YnA5uH=LQmRr5>CPoHaTrV0IG*XJ!Oo#) zUvn*RbY08zj_ek>R;8i?4v7F_)HF%V2v;atU8SW4&4C-RwpbZ(HCW(+KYd!kY$dpW z1n-sjj7RT9M6V!pL}V$j(=nWu&X~qwQ#d9_?)=~7GSu{hpR8`Q_e)FYl5>xrAbs#Y zSN{CliB4FxRas=ugoSRep+GTo^ zX1>P=_SY7vb&Z*p$=spN7&&Zf&;giSf7!bx19cf%9;vzTBtLgjC7I3>KBYy45(dmg z%TegE?K^W0n4S?YB7_h~bjAap^92w!?@^2;r@p))IXPc$T4in;hS@o>!bliq80?4-FRi2_O9!HacO&mWTmH_FqznP+RG60eFxG3`ChEeiR!zk^~sC=h<`l2;xR2acS|;S#lO zA>Qu*AW-btK=AhuFs+a_7bK)1C9O`e|M|i14>0dJ>%Ro89{wPwC0`jVGQi%TV@BO5 zgn+hP@15c}zp&%C(ZGD(s_# z#(UJ1C!FQrc#NPG@PLst@ z;iZ5W&rcLG+Q@z=p`b#Tq8xC-4N}1QKMK4^a zfMx!LvwIpo`vJTWt|aZGMXr;(===ftR@Le2v-Czdg_I6^b*5b=-Qx z^oxxECb7E$-1{|+T>!XMm?&MqJ48>jKA(ZdiE54)ll`r1#tEB`73+65Z1m?G89-o2 zgE#rwk=%DA|G6ZD%AH^v9q}FS($2NRK&*VG=3KDdW_u5tK%m0v}5dHL1mca(+9e#E#RnO9$UtR*P9hBk}Lhn*v~wT z02<@+pW60FU_uPQcBXmJ894C`G~D8=#%YrqNwNrdfK*4IL%W%DuahSwl0|mV&kS_M z@Q@f1AZ4Rwwdm<^sfBB1FuIjk5h&eQyT7y_+CO&Y3na9j;9kHoH;Pd`;^q5{y_j+4 z?bDBIZntj$nFRoYzGLY=av4)_@pls>&&OWb;hEnhI(itL1tBKzad_a1h>MKX`bxz2 zXiI+#a`A!=Up%i(;|AE2pnKQ@*fd%&DB1P_DJK)^@(<{4qnV|nukYZJ4|vDUuqVS_c-iUi{RH(Ce%7H`TO~?dIYiN&?C*GNO=h#3@iX7i&Wm0q7uZ<2KVZK zZ3>QAbL+&(X-;#eui!vvcrWP^+4qni*XvHJKaJdk&J;IbnPJ03){!rr&@ExUpsQ_k zu`Tp{?|SsZtJ99t*t-Bm5S5G&}VIO;lksoOKSnqUAA2)mbr0ofv)|$3SN*7k{SM~znfMB6%1N6snwj~TV zhyuvj_8qn*8{>dFPmp+t=c$s-GH!v~yBZ2a*`SJ@HfC2iA!Cgq7;aiRwOrxt11AF8 z369IpL6pZu^mXIoLraZ(vs+_!`JzuiG0b+01*m!ER}FAW`f{K<<&=e43h|w?D@TaP zM4u!6K_a9)501S9n27!4082v6ui#+3X3XU2Yp*0loTlo|9kGYfFLjgqCi~d}66*(D zoqI6ao!&iSw0adV65i!XyS$d4p85QVU7%SOgXoV1dbE}fgw50ZFaW}X=0wW?*pYEl zO>n%XRS0XFnmC&72N(H*Mx-W4wDWvQ@4wlEfezv16~0gAmfDJL{@F^Vx!FH(LS2TA#v7(j-7m}eStiD} z6hQfs=}2Ij)Q{u(D7Ny{hLm=76~C6rW0YsCDNA7>l^hqDP$XjT1s3_NS=%S{JhM)7 ze4s`JkK)WgaOQ(fKd@3SUj~vYYOllMr0zh`s8vOj_A!0bS=uDrI1mk6oK?ow8~lnM zyka#^1S4!`TJ0AeWyYc`*{zU9D)IFu+EBN(+C~LSi7kxMl&x}g3ju*x;(&7-$M^zH zTrvF_A=e-dVehq40y|Ye9l72KY_IC550O+LZ=e9toy4EF^^P+% z_;JakvrDsvGX*3bAagaJ2t?0dEwokvuzkSHTi50{!tkcx^gNciQmnJ|#*bh16C2aX z(b&^5Y&d!9QMPu^i2&bKQGso2Wrw30A%fc-0NTx`w+k%VCS$LMv`C!xbkrK~=MlqR>eP)qQQxC6o56KOHL^RWl~ zZLEtIbQ&In2%OsrE6myY#u1m7$~rS}|L2hezt!Nmhn?DpvId6n_)ap|PfsE$@lgYT zrY(4KmNsImjbFQ6)OEh6L!ZmSDLfF$dY>HS6g@^Ph(?9R*r$#ivjw?dSVmH{uy_k$ zAJfpHEs@}TMrRoXYpZX!^~lQ%E(BDnozcvwi4M-f`{CpCt6p8UWaAap#UAW!7?wR@ zw09O_U5Sp%NM5orZXZ$mRgdJbg@}=VX~CW#$#Myk=$LxHkBHqcuk)+9B8^7cq|y~| zcdE>hZ;6mL_SDj~N2J`kO86`B4y|e1=(jYdGFeCn3MJ5}iV(vYp6AzPnWDbx^yQ02 zScX_8kJ4C)p}X0I{Y4ep^^e>ojf^1{iEKQIZ?U!}+)T!Ai==vE^3^T96yL7*Kus!pGs1Xy1b|? z7%d*9BGP0*MnjT|66C z#5t*62R6E)do%cS2>;K{aY{CP-F@6z@;Ii`g9ZObNBYEFa<{g1Fh4aZ(6ek~$*0)0 zCOB`_3Vg@n=GrVShS_b^LvACK9FZfk_5);lSYD!%JPvjxrL*SggH%~8K8XvDX#*9` zKG`EBEDTt#t(BLm+s7}eDoQ8AzpHdMDU6cza{>5jsyQR&BkT`^Ju5! zuy3?WM21!(wKswMd`>GBUk6EGr|fy5sPnP1nvM*#Y{HSD4lP~;?#G%r^Eop*LTQ_2 ziB*eb?aW2fFW&k*c>wm?RR8%Kw|ws3P}n{}!zVx25x|f((F#La0u0cv$_Fe%sKd;f zDVXR7kht%mNpcD&@*R(};S<=;G(ULAebQ=JwOHtVnr(N)no_?rvA3-`o_Ks3UdoTV zT**{jD2gO799gZ26&4?dI`3L{bhnzs@mW;ApKZ{?&rKX5EKOxEDd!*?Audpeh0!C? z_z)G9HS-3CFkTNR6xDXu~tS2a7K?dCK`cuzPk zAl3f)DZ27P>rsJuvcTN(`MF43eTB%Ahy9^8ie8%JTLI1)-E}?#^{R_&5yn+=ngo$F`{{?_7 z1SGPJZG>#L~NWEZIP3#QtB2wJTGvc&^kjYq*7}=mk)|5@DU@O)9E0V$!x4JpB65PUD zj^DS6q-t1OhJvOAFFa58+*jJUgoMu6lqjg4Z89D{{bB^->y}^jZ8Ia*c6(S(I(Juu zdtTAr9YQ2*3DZ5}zhEoH(QyuU`bNl~&<-S&;$4t`?%!7L9leEJxEQiPJd3iRdSURX z@YV%YytI>gKj93W&Ws&{X%^vjVo;L3Ok{H`xp39NCg^$jFgL*)f17}dHVQ6plsU14 zzv_9dQA;g5Se3+=`^fsy@|fFx+erwRQNW6RH2)&aqf>o^`1fXO#3mXtZW*!?GF$!jiRsGnT&6$ssXh?D z(BlJgLhE^) z+WJ?qpOHdD3_A`>9BqO}`o6-jm5x1nnbKKSu8@qCEq#xO*7xCOEyG)Hi^Y}@vhm0S{OM$jHFR$+ zNhl4LwS{_0fz5Y4XsqIEVJ!kNZ#P>P+oJ8xJ>aZQfBHS5AjVrluJ+MFPjXT(dG<(& zUkgRLYMLtzn>*{mwNl2mSLIi$30@@lgr)~m89fCM*dA#KMMbh@NOiXp!n?Vc_BggI zZ)siO$U}_RqM&NZfo#yg+w~MRwR_?|%1YBmJ(&>St4icOQJGK6^pG+NvGHd)9rurQ z)qSe$uA!xi{U$?&6b9P4|x)L=ig|N-A)2{=i@5gs!oY_ma)z^gY8K=pmqWrBTAdbeAO}W z-Cw#d$(ENZ3IiTMEN%f<;4ZIe(9I{gd}F!5s>^K z5K@=)dX0*`A}t{vwvs$e=M{MaMNQdE_R=eXB$qPTF|k+9!Q7o$PKF{!eYV><=q`mj zlN8e;on(klQV)+yt-iSXm8dg>ea*tfR*^C?;m^69zd-&KaYE3C(oU+hEK+V8E)8Bo zo1GWU4=O~$)r9pygs++xsXQrS#deHk=j-_h=|@`KmN1{E32)Jdo0^@#^>&@yM00tT zC1%U6@lGh~#^CPLj){rUu-yi|w@gjPuq0ram8kJjFWg*^?oqFCB|m6>BFrC{3jP-AkG(MG=9&0>L~TR~Pv+FE9l@WP#L z%>2Q5>kpg>R^9B-gop_h zUO=b#s_g@OFP;er4M~X`^B2AdcyRGv1CUJ?y&&6lzu*an1l3~a?zod;KVho$;M)@I zGSz}}#W~*M8zL@ZCpZ^GyNg?%?z-}8Mk+}(baN1i5@~mzl4;=%k`>`I4RO=rVnR^S zB%{@Fj{g3!;sg9+4D*g!ZPA@(t!f@zWhtHg+^eO;wrO~Cu}6)ggwh0 zYx~&enoJ(`2c|byWTfJo`YVWR3B1FW=skUFF<$6An>iNqEE8AaZk}DadX269(ukCR zB|CoQe+64$@{F;#^`yf#so~m4Mnkh%q=D^FBlOfYEdZ=JAU*Ttgyk=|_pqkRFA^vM zX!nt{lMGKfD$R&$J_X?~L@1X43`SU?g!hE@Nhtnj;fPMzQ^}z+!#Uu@W4?a!$XU*B zC?AS-x+kG|Eng&;++I2ySkPFG=3CPIz?I<#$iXLzwX%&O=%ERErWw^(F9nJ+bC^ce zlwW1k=Tt=sGQxr%hzYROblTjoh2~il)o|S(08yNL_(Du2phrQs6${mUodl|`XXtX9 zv~hF!g0x3oFqOR7A(3m6)3Paeqwi6LhyBV87x^AFZthRoM+Alo=L&kP$9Ao{?=a<; zQA;y=&{i&g&ehyn;5QCu6sr|MLsnw5<*9l%+eGiVCs)L(&G2!2>t4#>cgFCfYLvuI#^a#P*}Ec z!?f^6(^cV3g9%QfeC}DwqZyA%y)=}m5oahDC-E-b87Agt$R$P8Q%2133h2C6*iS8b zVx$BkY1o}Rj+QLq2eK3tS@RU;%&uLsau@Hs68@tpsg|wm(y{lP_ik#ju#DWf^H5#u zQnGH7PF8hQ;~V$AoX@rECv-h%=ygT!#E1Q8qO<8TuBJLqXdx?fA;kIDfEHQ%wamJU zzCPR=w;hzu_AV{OJVHbbmCgpJCLJCYoYc;8xo+Jbw(7_B>+P4$tKc^1cw_b+> zbXW}zEGI&E-QU*Hk=}G=!AX2vAucUf7n_~Ez5~<(cPa3N9+v}qDDn+*wpB_QS0m{w4dnbQf zYx(>1DRgE|76Gxbcw$XOj}VUj6Fa2;{sWEwqd`wPLLi(_1IAGn+O;`Ou*lMsD ziJD${N9DcyGPp1yO#fd!3J!I7h(lc<{HzN<0A{#DW-MYP%Rkz#LO>|2P-6L_sDNX2 z&GmnOMF`T#ka4%BAl-G3gUB0PPC<+i!?U96p=W}F6%b1!v%lF4TIoAL6mD|UZWr(l$g_jUh?3n>W7aD( zo*YM_ci~8M{EsP~kDt%Yo2}T!H`Br;`W;>-^83!JiwrUlD+YkL9OrBV24_?_sJ?RR zYG}X}U=9H!b;qDoIl1&`=g2`48t?y&cO?w{tymw9pavyAAP_D`2qK zA$ANsTYgs@ZFd#u!o+^+Y5c9^gR72L3xkhlV1gYR3%449oSb;~tMxlcshMnl2f<`47iceD#Oa;b`bjsR=43yt%wEe>%{!K?+ zN5y(R&JcRXq?A@~j1#)-u?N9CzN zNL4zpor;azfPbyG0=G(j2@SWo*R}Z9v2^|!@(62nY#`= zd4@vddUs>)$^iX8*64I-;Qr62e!8BRe>|lz$I$|NMtJ|rd1Xl&Wdo+`|svU8vXFvp6$I_0=y_Q{+b}s(|vJxA@N=bO=&iapVcc3l>*|z0xZPi0HuY zkrmxf06ux0>Pn(BHuA<0`tZDKP$Q&W7+1^8DS}Vj_ascN8Q*eRKiZ&1r(YE(lqP1x zznl2!zVfit8MXiSfc$ec?5v=69nAU5`Cl#K-$&>nr0fEl+kb1OLdiaHl@FL7Wyjp$ItNb5cVh|@rW4-gr@c;h~npDA2GkV8h9hD*qdpC&}C^I!f zvP$R<|BAqb2UFUrlUMHk=k9|oO=1aB zjNDL82y#;MpgUTSZdXB$wr>J-W9mI zS(@n*5XpOg!k3A4Lg^0TW-w(T`E%bYqJQtF`6?K;izhesYBYpWp~>p{v$ixa0+(tKkA?58R-Pw&E`^^dtp+&+oC`(S zHDvCFRj_2UsHRVzv24Hq9nJ|ad0f|nLYvo8cCD-#C{y6?q(2TRzkB8sN*t7gcl#?0 zu%`qP@zrCtrfUkJ&v1#eiaKyKr0h9Uz1H=n$Fcc}G}GCrX(D>F+=iUbE5X2R*za(^f4X6XQkQ6L_d zLb;-+b?kahd0ahLpVS-mv0BR0Xh|Dy=-suCuKxKyLX}!bQeXeTg@A!(2^h(tViO6A zI={u0Z|pa%0`ioROlh!`f^*;N&ihiZXano7q^KA0N$rX;ZaMqm9*JK@T)58>d7#UW zpQ^^23UKL?#Mx1R46Xx97bG>hfF&BZN;cT^a7-g)h^{UQ85Tu?oY0nJv*Q{VU0_Ru z%ggMif*55u!F@m)TjBeXj1=)(6fVS0jKYQ0y3eWJTm%^AJ@C$;;8VGC$u#EQ6t+5a%6d(BXnY#|$WSIO%|Y#)1=MS>x=)edA}iJv%^) zM)dL{aFCio3K=fz`Udm20Id^Fec8T8t5k4&BSMNwM_rBpCyHoq*p^!i5 zKHko;MLOel7N7%hW9Rs0PyZZ_Say?q*KD}Sgt_}2hL;1I-nO18&ba5fTJ#~Z{FIjyIXox>xEd(=6l3?z)7g5>hO)CI%+Wiaf^}@|PYhFLEYj7i(k2Ok zOh%$|`kExYs+P!_YmSP*N&JmTtenx Date: Tue, 5 Sep 2017 10:08:56 -0700 Subject: [PATCH 04/94] rewrite graph --- .../images/multigpu_allreduce.graffle | Bin 5704 -> 5489 bytes .../framework/images/multigpu_allreduce.png | Bin 118413 -> 110982 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/paddle/framework/images/multigpu_allreduce.graffle b/paddle/framework/images/multigpu_allreduce.graffle index 588f79feb9a3f996b6d6208e556d61f0a9602aa5..cb5bc420ceafe8ba4c87694d44ee4e5e4ad06779 100644 GIT binary patch literal 5489 zcmV-%6^`m3iwFP!000030PS6CbK6F;{T%-aUVquDyAm|-N7nI{EIF}r9b0S3_Qs{E zTQCVpSVM#YLYAGB|NHiU6iK|q7sQv^sfs9}2N(|ebf2D{o_X@mua{BlOOW)!xck#% z>XFB-pxcf+VfXx}$L~&l-ZhW^`Sj6~|2cefaPs@n^VU@q_R`kTyJxRn9<(0s?(OYg zT}8p(-r>n%>*&?X&LCf7ioI+FxFQ4(JTN&4*- z+-w&bc%8KK7#`EVd#*e1*G||@pFaBG$>-qP)BScDehFUr--6_2w-bDQvgiH^7sGBE zoCnEM@?`J!<82docG&h)xaIr)orjJ5B=OycAD;Bm1itc9=%E*1cEj@|zP|F_z^7jl z|LiOZ)RVn|$e1)$)?*yH&$LpCjfp(jyS;V43)gA9-}%pVFC9KQO|FBz+w0H#_GkCV zZU-KBbrH6QVuSBK5OWf}+iSya{Ph>vX_49;5tN#1t3e-UmOe;!>OyWGEinD{rNZjCzj`ptNEU_gf> zar zC_L|$Iz`K!%A}94{5ITTD4zs=^pu10Wgl)u;TNaPhkjb>4_7 z-wFIH_0HWSPKxbc1X0*+eehi$@(m8d-c{s(J8t_?>8SwJ)>Yd2sg-^BT5S4r7zHQa zu1d{(fJIR@)5XKMeSH~p(;{An!^cYFH_+jpGbt7L&MRdBNMlH%f-+ANO02R3j4p{Lc6Qjtf-Vr8>hj{^V+l*UY_(U$CGn zg+Ao3j`aI9Nzdr1-|dCj|JqmQlu35lj@z;CE=eLD{dIa4chmG!bb3Zk&$=P}Y+v|E zFGxuX%)^%;1#kK~xDNjD9c0^a6el43kJBXRFey*ZlOX8251*pz0Dfjt|9cvIz4F0f zcSiE<-Z`N2v`w9PA|Q5pGU#{2Y# z_XB{qjK2Ka8}nHC#x>@_0k76L9Kah%;^-&_ogZ&IG;m|@c$XUO5wK&L2}Ptd1}xRC zl%8hH7)mrz;I1V9UeL9nN2_Q)sr88O+}bDT@XPL5T(6=|qC!Q78dm6C2I?^fJTxZG z7YuDj@kPA|{LakQOp3d0ON}*U71ngB7e<9)o&Zaz6jxvi1@H-0|9S)$o_hsGf|ue- zfVfoL(?%TqlQp0KO3gx0lB|GjMUi zx!?YlYk1~Iz5D|l91FguhQ5vbou8&b(yeU4NM(}~Ka6U>2ASR&z-?w$b7kw)yw-Hp z^Cn9n4l-OxP7EUi!Xd|-s3*CQ9Mn=;&HJ>8Zai(GjnO7DBWtNxsgU0m;Panh6qIsaqDR`+Y-vr*t!`#>$(}n40sJi4Cjn7O$1U7q#Q^&F#WTn zl`(QI7Bu(D7$bzo-3mb^z`<}P*evNEBOyjJN?SmaP{5h&a`bves&U5GUOrdFyw=>--}! zYnRXTAQM;(6WEWUw?XH+9ekfu{yGD6n8`deDLR1FVaFhL5Ic>t=p^<#ZNHZ`h#kvS z-s{jcLy|@$A(A%13u%}9)x;VisG^3cb@NFIk%UOv1fL|db3(>=L~u(K)exC;y*lU! z!4WzLolS5;K;%4Pwd7V4N~zRx&5$GzNrgA<{@qgj+17-QR%eaX}r( zwj^_@SW}rxMRBndW-hnMN#kOvrqYCE8a-8-P|iJ8wX5UyMu9a|iDnH;LOxjV!4AM$4hA3Dj3OXN}!~mxy3c^*1f-DaM3^9Nhm^|;-AOt|u zT~QmbCMyg$14Ugbfq5F>Ae9nQQERs(lh9b5wIrBHz)S*W5*~0SVgB9ur`cbBtmxQf z4v414S_pHsaf`-)t$SKfw>7#}(h!D=3$`eAAT&-@-ph#^!meodr0qLVW2@&T=h(;y z0XP)^=iaw{o~<_!fEx=y+2l@Qw&LY8zPw2tftAP@v8{VHz{)bifv-?xILiZ#!kJJ+ z!E&6yW1YQrYGAe7DO&?~ zaoGXwIa@6x=hT8J$5oJ9lpWwWIXV9j-3ei`q;lbT5RaJM+u%Z81mH9P-Dq%r$@D+eLRi_^MWBBw-b+|$w~qX?;S!jp%jxuD(?7{(+>l@S_*3kQC4rq<~_Wz z!g@YyVZAb(N{3S^0d+*+VX7Ga;^sZuNe1X#WrckHN_|N^X2HH#Fx3XMtDo=ngpt4# zBvAw|)-Jc3he{wBSb0D_*Hm`}?PDd{x3p{z$8DC;ldbzQ0U*VmcFSml;D&)Mblb$b zN?WNZ{Nl<8sos*4fH`lBAF~9~va|W; z?}Qs;L6oNz_!Z5$6~Mw(Elan|bMrAmYE3w>dbVvlZsJLN%a}YAhBr8QD46u5(N=EGBNw&AJAQIBeOS~aDLQ!r;X)q0zdr>#w|9jhtFd{NPpJ&irU}w&!?a?cc2^Kp8%7nC zprjG`U}`pnQ+Fv`?sF+eUvRO7Q;JOgOqTjmoJr8Hceay&P#x?RXyu*6a((o-;%(+V3se4*$jz0 zEy@I9_Q7D5AZAyISbZ*AdRWEXD~Z(CP`m*UiLnlM)H_;U4nBO^oCyMskayc9Gaf$x*0^q0@Vmj;etxFI#h5(1)^fH zf`ofQh0r^#ep#k))drf_rGY}s&Ab}nL~@NeQST<~^{TXd3pJdHzJ~vD^saT#O8oBU z7Q0K?N27fd`R(8`$WLU|ZJ3d-r7D!RSC*CuQ+4>nZ+>zfcIzr{GL6b_OrL#Zvi!FS zG43|Y&Fx1}=xEqGzW#LXz&6%_;of~%m$aFS%ru*Sic{0YrZ}^6I+@mYZ!%s!*mJ_J zmVqZ#(T}|AJN@ypt~~y zv-7LO@}l(-M)%ja8IK{F5zV+pZl!3J4FsZyWyEr(OzQpuQBzd06_iD3g@GsrqU%@| zl|h7Yz52NtjVp5yS6CvokwnrguxMMGM#92bavD=dl1L^uF)+j;jD_*MNQQF3B8+Jd zLY~J2Af7i7&$t!UD)DT{l2b51pO2G`Ocl(2GYBN?GuU}d(njRbKMQV8{-9&*A>jN-7GVqwfK zko8jwR#O7B6t*cgwT6+H(4#X^`IHp=g%QHP$#r%uH{Zuh%T*u0$ z0VY;70i!rhz#x{krl>z^n^*P|>+nB|`+r5ZU=YSeB}9u#)Q=K_byJG^>)<+owDhc_L} z=Z(5D`0oFS6DNM)G-@QAFM9IJrAx%4<)Woe6>?>%^Qokgsib=FEh`GEddc~xapAFY zuH3gS4bo6k#`R?M60*U)4m&~bRowm@3}-$o{p^_Z`>+>&8X3V`@!y{x7v*0Z`RBn< z7l-}H1+;wyzXZVpa0F?Pl*DOC{QrXB>LfmH`(wQr&eruO^!#r@GJ5h*O$)lT}l+TgWh!AcpnBg6K`9GGw5x6oz^35zZ+iq z>F8@#cO(;S0|ShI3%iG5FD;_d6kwUj9tU7y{WOfb1##k$f_#*9^LZESSh+8qI4yOi z8~ADL@(r(o?s@JZls(&QrN5)!zcOzGNR(7K? ztPU<8#@+Le7vV+h%uv3`>o^Uj|1xiq5FBT2c!&0YmFxjXgR)G0b7z&F!5jMAn?lwB zxgIjFkDtY98ejI^?no2H{Nmt#XHo2@c|Z?73z7%#$XNBC@FGsaKjW?+LBGRpLFLL1 zYR4UC%PSuAY(M?8^WxX&_Yaf=|9%#IeEA~Y7yo+pub0QqFZM6qyd#o+`tWPiK0kop zJ8$06j~Aa_ypQ(b`oVK~a>9Q7;6YT@RMA|uj2N-?0UuB|5B8BnQ>5>Fk^^O5N6o?p`jHIb!zc8_ySHQsL<6-crR}q z-M!Bja}rn~$Ti2G0k9<^T7G zc#K<({NZ4IK{8J3#x~rNdZ)!TbrTN||GJq6Q`iq>@OJYZ9}jkJufM&Hf}~ae-q0t1 z(&Z0fXPnjyhmur$c|hH9QT-pMBbbdX?4XgeT?JF5=)ZIx__E-W3>hoSLmB-^xluIORujS n*488HhpC=7Cd5hZAFqyf%hqW0^C|-PJ$>|FMQQRD-+BQ6USy`d literal 5704 zcmV-O7PsjiiwFP!000030PS6CbJR$({XG1NzI@q;yI8LGW0qmV7-nED11vVMLpb7w z=$36Qa?7KZdCUa=eY4cQTURhRN)p3JPQdic+;7g6VPkPO1O|I>Zy zk^7yX-;H}=|LmvxZ;yZ8Gxz`b=-$KsdGhM;`1jXOJC{*7NIS3JK7R51uycQJfB)d} zG79$hpBz8wyngZg=(q#-*x!Hp@_y(3d757SxWE79%NNggt-Nl0;TjJ1UnlWpkfh&U zz|HocgV#%Y_u(Y#r^!{Ye|`P2-~Hqs z+3&&QF3-d6xY_8tkD9p_gX?SKVf^(MdC(!>A9VhrQql(z?r*f1(TN7Lmpm)TgIOzS96z#*PkT*m&vdu1AF-@KOC6Q z@l0HQntT{e;UM@B#}_jQ#G&8+>Y%dLN0)vVZZU431b+00gYab^uA9PtoHRf2)5>@lp?Z&)J!U$L{Yd$b zoOk|C;7?_A?j~_k?*A-^!hYwy@5WH<@FW~uM*g>>t{+vN3P9~#rk$TU*@v&?u0MxS zaQyAE(#;20l-p*q_$2OLT?GBKjMwq>vC907WO(3IN)5h?LRkV*4oOr{=4nESRaOZ~ z8N6J8w#O8g+Hz*8<-$0uG4`OtNfBsuNSi?I+`kO!uy#F3>2dHit@iu<=}}GVDE?Ay z`X(4;GFon(1M*}NrQroE51YOS`!}F9f1~`JidtdR_;ox8)7 zxt|PzlypEnd=65urZ0o5;2+R~B$LK16pINK_Jqf;E z`e3kollI-g8KCo|OPzWm(Cp;WPkJZmSE&g63)stkDC2%GCbA!fxF4KnO}NdP2$3~` zv6EqM_e0UFiPF%--8W47etMPo(a9fO_lq<!;fF9FKC?cgX zpsDtx^fY6}P@;(fb0zurlB|s_T21$9y(fI_L(g200!B@QowOx+=Flzi7WOic!x0zeam2DF9 zM%C5Kn=A!C$Z#b&F^mv!ha7LBp5#Ju5KCz_AJQhe^|Xn$Mw`f-w22x;m@+Vx+8CH^ zjkF1A(+1I|$4Hy*E^QKeA=*S1rA^lG0&5}$coQ`q%(l@WzeW?rS<~yXjzv+;wIos~ zXlOGUg}e!Q(=^_sTjWiYS>sWwEFs!}?l2s96O$fgoWn0bZwo@`Zp53kZHqW*GbeF! zQwd%Xu?iHl1p0);35gTNn_3}GR(leZmIY(Xh}2SY$*h!3hIen+es4;LouWp{EJBUS zT5gVr5|_lYf)nr-i4|Z=9&`j@9y3-@&ImZ;jEhDl%Q-miL|dYWvP@V#ome4LLZ(y= zq_Q@dlF-7F8U#=iFyK~#R4PF|E3NyVR7@L?>D^#Tx;3Vx=447nSPLGNAd160GlN74 zi4qc}X5FyOti&$FO4Q$b!{J{gl%gP{E*8U$XSvW?0h1O2{FU8EVrkhL!w~Z_49P%=z^gNiN?;ib zDF#vuOk(XO#aJjOR8}c-AykW$ zLs;&CoPkQC7G#%b7fF@c}HK9fi*Kr6Xjkbg;P8GE~V#r%t0ISK|>o_E_S}@}5I8iQwiG~~v zImQlf9mh(-8ET<$#==WnLEF_3mRU0o_pvu}H{6wz`gkOV1Lw zN}gE|PT{gFHj8ms4jf7v%4in2FT5%~BGY!o zB^$^D*5uMh(VL)m)eXK++s9&_#o{Kj(4?%w*M}X0*g@>H&Z3jp?{)n_+9Gx=S4FEs zHw;M{k%UOv1}mi9DlSc|A%ZGunA$X(q!3Anq;0TCLOUa5j7J2wL{SZXJ=g1lju0H7 zgV5OqBLqawBUVdpHKCNsY~jQvAxR*T5J_8MgpBkUf$-8kx9VBD-vQI(f;y0~ z-N;-j)>h_HQCxzAnM+)PgsYybK&{f~nbL%cF21fQtEAGvh%99cM1@N(ZSz7VT`cd; z>bq5$JK;j6yPWf*ZSD?8xfO0d0xC2iAX@|^PDt(nEw@Y=Zmlpww$916a|~(P;uuoT z$uWebS_{oxA-v^8Ajd$Cu}K``{mI{|3uKXJ+@MSJ;O$F0UAYug2+~?GhA3Di3VQF`!~mxi3c^)|f~*b$3^9Nhm>&0Q5dt9Tt|Sgv zlNH9gYHOEDV4emzNTq~S)Y=`%B(zp%EeU24Fq43pggcx`Sp1lsSq{p{6&<_G0nyZ0 z3vRA9ZqXR9bx#Xwh)`NdLl`bD*rC*c&={(`m7}qQUDGy*yLU9!PRAw}*pUzcI1>QP zdN%Gr0B#uo+uR?+Ik=UV&-m(sIsz+^(PP{6Y=D(jh67)r%y3o*8ikQiM8Rsa0ftsd z#yT@%LLvqb11rr2aAG}5IhWLMtEhBjNl5secDwRYBaAy7Za2~aqP5~hK`|YG=>SXz z+}(75Sh&`Z3*%NGXk)FAOalLKu?g#O125FNQgOJ{xM{4@*UX87aLEA5jnmgudz2{y z3XV#*6FaVg+@b6M$Kd4RQ*ck%nHLQ4xSMzsZB`Okc<X5R+6P9R0=5*c(A}U!%(g{|n3Y?PkHX)7-IDVuyn8jPQ@{`i+1pQOQaQZ$gQO<(h9|> zM>VtHiBQEUs58ealn26Qr4?3z+p;TXXtUNbR&3skm0^raNhPIPfm7ka$H*&?S75>l zc}0UtnS)n=G}R_|kXInDm_u%5t1Dr2cF0j>r-qdk`A~~nx}o7)?Y4%DRZAKj21+H3 ztHXdm3?K$pd&rSk@t)5_VlgK|qHRe0+y5K}Nn7Qdza;V1WnB(Biu}t#&?`D7d14P> zA~gUSVWi13s7PFKSkry5MlY;*^IA`AsxMaWjn(>NMSRuyWOV~7@Y|arXLYwPcFy&! zC+uYpt@tJ?(O;%{#Y>whL|kyk>a|?mR8%xwsZoQrYu0zADQnPqwNI&a(PnYd8r`(A zqjr-M)#yf*g`gr4MQ3V0fm1gLT;X#iM_)3rr2}Qn@c7K;T@$83zuDMM146yCTcVXW z63fkz-6l=guO*10>S3{63c4wiQJc#^ENudoe!Ge=_Ocu&oIZ}U}Vh~KX4yI+|FgL=9)6COY&*HE;zKdoCk+DQILR+|?Qmqda z98rO&Sgs)9mQW$|gU+xlQ@CmaP3+1*Ar@v{t#Bf_!JKIDB^(Uuq-x)I~SR0ws?qB)8#^(f-)Dn)Hk2I zQtmMIpcV{#ZsYPZt#2jpIob@OA~GvqcB9cfMIw&1OY1gwdI@v4Y#jt8vskaW?IW7I zEi`vVV0M0;SYEb1!sy`|H{&rxGol&S$gLI4vIR#Jv5Z*Ggh}0=BWlVbwuZ7Otl_&A>b5kI zp5}^brWBFJz@=3oMq=$2)@~JRw+^z}EwUMYpyLSHI|ErEfC+mp6DBMpK zoyJY9Y#L!=O#v9iF#v;D+MQx1L)oyHX)Wu)6qaWoFgCG%NlCLlP)Gy?f?^YEmV|xK zk!EtD6kQ!A6jmi+RZ_7k>GAMHDY_YDNy4oDL@CwEiBgk(qw=7zmyZRGg?D`Q=+f`P zbdPU3iswy+GWzcSi4)iS&`H#!ak1#hua+(mk5-G8KGw9WwmO@sRWehn0es7v!m2@X z_A%c$SI$-V)|E*bOG;i&CNCkI+^et`3|_?DPr*3yS!-u=t>1-%@Z&@YUN`^!=~21; zv)BGvFqXyfcnS{fP{1#s;UO4;G)OAVX{Gu91;OQUeAM-GqZmi)h5Rzvkl{27<`#}+a{dBXydE2 z8EFUo@WM|gU$cH7S<@~s!1%YY|0EowWmK9HEVH&p0cco14dZ^PIq^uTeUuIJX&>}h zbu7I&tqi6g_-X9&4KITJS$bZ103*Xc%o@Tt*p1Qyv;K1oF#Qr=hBLJIzr^@DVt_5(oA(RmQ0CBYw~;ZL=!|JffD$iuM51-K&FFr4o&q+AExH2gdZ#_IvS{W*vKYl_W3|*-Tc+ZPAa) zIy@aYmj6GT;vBb{*u&BKf+SDta_1H>SFvofBq;d5F7jYX`=N~9ZgJpwXXpC*o2w{D z8XLeH`sh!){66gEY0Y>lNzIoB)E!0s$7g|qcZr&CUScvMlldk%D=4q=JnT^^9(48? z$@Yn%MiGx|Eu^tjDap)(4yA^AlrsvRH>X5eT{BG|;`pM%k|eY4kEYyMtwF_=#^yS` u^P{WE>#Mr8&4~J8rsw4vaZ>oh>%HBoHk$mri~xR*?)?`7r1L9NkO2S$Mingp diff --git a/paddle/framework/images/multigpu_allreduce.png b/paddle/framework/images/multigpu_allreduce.png index 52d3eacd55834880d09f4efbee804b5c567a57fd..87a1b3e8f6dd4a713ec9df9f0037d1da04e9178a 100644 GIT binary patch delta 81600 zcmZs@by!qU+b=u=0}MSwcMRPjUD6<3A|Tx;DM$%!K)So6rA28$MY=@mn z2!CMWGs`mTqLk^Medei9ElsCl=P=23Z8n)1r9xv)&Wf17{*h0afAi-3%U$2iv9Xs+ z(Dci%V|g#1e;UiX=KIX*wLiysnJ-~ZF_V07yZ%TNODZYW>5n#MQ+I?V7bp4>^JlIz z^ubUvqZ`x=E{=_{Ts7=^cz$gD%FnIpwddliA4_lMUsk5Q^!;jY7J0f6O-%o7=9#uG zd+EYJkjIzNEIsj)Tc`$l_#}K*zO&NXrI@57m1lymkQeR!2I{)Jcn}AxB@5c-L&HCv z=9CcW1sx$vHZ<6TwVS$kBtYP9qB0~%rCfwI?!i7;R$@lG@ zTNYkPZ4`Gc6lGN*;3{ZM6IeHvOqiwUW2YG)&KU3wG@LSPiec_>`Ox294`$;DvR5`?CxQnK!XVZNUm|Z^0Hc zplQokk2Jah;_qr55HY>5_DJj0>8GhivKI)rk|GL)`1fmq8m@u2!pm~VVZwcl8T>u0 znw#0w04`Nl7D6|2IoA<5-1Wds={BPO^wxZ79I2gY@a3S(aisD8{}uoJCShuL8daXG z|EiJTQrGGlAN1^jO1HwUjaP`a!2fkI3Q?wNOZtgWP>c8MH*t^ZzU)pzU*eMm(j1&m z0|lZMh%Nca(>B}qgQsZeu9Gz8p(<>KCa|9`m%5}?^0v+&`?8rc5v4BkzZn+DLJC*j zZu9)&Yh0aLMgz(E@60wi5&A%PM!YG05Y&@>Z3!Lt5fxA0<%YLd*V|5PPWABR;ocDQ zx(KU+LXEGP>3!-9e&gY14j_mOsPWp8{g3L~1Vl=OQE$D`G z$Nq&kpl4g}emeRw-*x&^wh8I08vvajQqS4m{aE|4R3)b2LCz7*e?lxmP0LnOTzNihsw=xd}8ELgkl}q ztj%*#S8{P0?8jgd{U`8E!H3`6>r>>9%zi&7y?*9hSBl>J?;0d3!$_lej4P+>%nwHS z>e-j)Ds>+>2;BMH;#Ol+v4RbqpaZOA*`7#|+yDDStTe~M<{}d*S#P;v+ZIu|3a}Zpz(EV>>?z6)C7DgEdgalrGu%?Ep0fANn z_XH!6G&1o0+%(}2cHgvlulB|r5>WE0Hm!bqdMm?As?qZKe_s&+Uql6df7WKv;u?OF z-z82hfl9!#9c4VHYKTTq^O^2!R~64B-7D4jPsXs2gILpq{KrIw-Tx$wA_ejcT==0MM7+f*{6Z0t?d{xZqs zkPMDvo7OKe|C6UBcvy(%4|f`e)Ki#3ztvQg1$jnX69nM}D3CFF4JQvY*QieVwC@mNiKvFX z5qDOlh5i}>x-^-Oe05c1^@ZEX!bC^8sd&7S23gdSC3o}!qFU_YOQxZ zKJWbd?b>^}g9M?@ZedeZH5t;p3C@2-h8i&e7rQ%CryNCa{rori*UX@U&9osYl=r_8LLM2v04%S+bASf99%@|@yed}N;n?wRFhabKIkt_%Ic#y;m4R?Uvg~L z{3JXV+VjQl9_=o7S-%NxdY`=OjvD{>rNVgd%rS0hP4AgXUHU{jpG=7u|5B6xhMzo0 zFwGS0;|h!b0kZ?Nt%MTB3m-_JZhp_10`osT&S8Yk_BJO3Gp|BM9=Wm=zi6+sG3`yz zbwp%MQ4lB+O`I*5SJtz}^9=v$SyQP1S}2cFGcVOl$W#(w#K-!J39Fym7|FojYPkF9 z`D3rnJRO(Q-L9_A4FV_d0b5xta67KxFNPY7PLS-A%XP1}*#a@l4+9ybh#zSr+G!-x zgo|8d&GRW5(@IyGCp;b&iD5`H0j33EGjc{nn(9{`!MBWWWZtC~dx=7<#dd!!hL)aa zq&Or?JC7GgKf)um1|)J_v$*|dXmDK zLn0G)`2ADBw`m@6|2qm+<)Pp~EzJI>yX|Pe0-Cjj-u#|6);|ZYeyh%Gua{7$0CcgV zW&wt)mBmM<2Ko#pLbRL)bg-oN3a9Q*)RVAWxtVlriL@O*jArpId-oBCypXz8@{QZL zGSU90Q#>!O<*RFqaCAeyLIlAUHDef7YN24q)0%#A-MJD?jfgNR#v zAVi!-GeMctCmfAL{!5b!QaA$cD08{;h!jO|W|LP4U3UFhdQg7BVOvnRgELAw*dSGy zR?=+x%tAA^?4$QE1+kxdg~%z|8&Ov--B$eZ@=U`I)~Rlu5k@1b_8~ccmZ~C3jQ|Hu zh@Ne6GX_cJBjc;b2@+&`qV!R0JU@dQ=t#oMaQAUMV!G)PCG4G(cV9W#@VsNfxe9n2 z8lm@^k@%8gV$oSVcY6ELbmGb|INzzhAB=92f`liT1op7cZK>9@l^~YYSnS6F*?d_` z1C0T9;~EhSyN71Tm#kX`mKoZ|pYPU({#gvr&f={I6U!?jP|OxYz!T7}OC2XKJm!<- z_BBnP8S6<8DCkLxvy#v&gh&;999F;4YSqlBf|J>L%$XVXl3)e@qJ1r zS*fWiB4boMVpYfMp4$Meq-!((E;82emEZQxA>~3APd=4n8hHJIFy$MAcHJy$x@+}a z7h3+;X4XJ;SoM?e_}l^>>K3h`s9M&)#CU=6{-7&KDPTeI9;ckQ|JrklR*1Kk&B-pY zPmz+^_TC$`P*}B2^;{z_nW3RYfk^bg_6~q**H3qvPvQ$7{Cw&5RLz>(SZtpag+-*S zh^WIQHq|VTCt+c?m6v$@C||2wjj8eD^L%nXLC(3TH+XAPo&^|;YhN_+)>AIMg1~X< zuZeWxwR(jz-|*9TYMFeu4IxwmDO`^_iC)r~(KlVnU61 z&i(H*hZnK=`5XRmsz&wEue%@kB-w$m_DZz7{ldLwiblfQzxQtH{Tpoq@N@;x+d>jl z|K`d3Y^tq-?3M8wO0WaBD{{a+Trx<)x+V9i%98`jNMJst`C&9Qj>o)lClpRMFTn~F zjMOc1Rdt7N(DO`T4D87=gm7h#q&5$=RBNH+V(_2t6l4bw1cvo39}lMxC0 zAXd_OYGIDs7!v@0DR?`=#V$VRUuWgAXuXpKd4B!M{P4KctkFKwfkr-9Y%N7Gx3x7* z)xoUO24nwLVFDvCnw;VpFh>|i(!E;}OVTZ5A3Otcf5>Oj_hvwaS~~f=b*VeIh$rJV z5+*%O%2>8C^cA+JDhMT1T|WUTnDB|I(kv|@UQxNymEwcaUkZeMWH+d`X3s69$K0Vf zmvZ{~+;TfQm6rAoda}fX;)}x{!kG{ugjD?PTV)A)*U$Yr&127Ezp$5K7^Br)Q3#AG zj1iWH!7oR#61%-t#|yg`G4FNB0fQ;RZm24m$R9nodOc;Ds!R>{gS9Q(BXl9JixyAOS ztv7Fu=2r81bC?}Xp{_a{0oiuMC1YGV<9&2ZB>qn}AAV-k;wK^FE#5#D{9^Oq+W zaFQpZNK#a1RQeRRTwsNhQi&oW_FmG|y@ht4Vf*S1&~H9f8$Ri-n!Y8qo_KAT;0!RC zQeF*x1;^o3Kp~!TK`Kv~M54c}t2K=S5N${F^l`KbUtJ`P5{oCY1l#q~b4KZATg%&w zu&1w{V6$wc7pba?dZc67%Ym986DX~U6%|?-?~%6W5PyOz5{ksq?WEyfic1Om^W-Ho zK}!7#OXC+#z+j4jHxWz|K$0tro<6`L91M!({0on#%~18Qoz)&iC@7@Lg_ojgj~-7> zrL(Ni!$aW0)eP4kcChiT1vMq!tT7r0P+7fh@Fw`a<6ph&vH?F;dp`nnvE~!G-0Uw; z1D>4mj&dpsiXH%@l8gS4w`j`kI&51VOUCY85eNHgsmKxJYH%%f3$$+1hqJ^vXW=mD@ z{OH=5?`XPEr8ntM@{?|Q6B?#@Hp}A)ZlFPf(@cOdwX3?Eg6gyIx_`3HpfuvtxR~&s zK?B3f-`)iIqD}-=31)7Vj1IE^2{HoA3H_z-d-aHe8sL%=A`q{R%{_SUC>UY>VqxoB zuxvpqt@J$kKZ%7WII=%Z4!-(In7_RD1@r=a#l$DcGFo#5X~`yl>; z$-%2g{s)##G2NwW7pJ=#IQszAl&wT*_*Vz#o7uB|-x^^h!$&`r)HB;N@4Jc_@u2+; zLcw<1wf;DlqM*twa-&=_fF|XxxL*;B_o1a9EFPZH@p?8+nN7oz7GC`DFAu8(nv(Tw zMz|82?iR*UnI4*p0@!1R89Fnh6;=0Z0U8m&`fGkKLQd{YPqyD+Gf#Lg(iba2hPgtP z@>nqp_l7;%7S;W;$T+li0Cz=skMdMibM)4h8<{2(p(uy;_Jh@Zje-cyAvRoNKNudPA1YdHZ?1p~w_g#@ zR!cE4u+tQNPhhGH)-zqr!;mw~eKtUC!;)-QtW~F*6nlEzC|19pqxP06t7dY8(s`Hc z-g$8<71#jG85gVJ-t=uCd>QM@a~S?!lVy$cy^0Q{FmC*0qCz1?$vC`BBefhX%nPW< zZ|cTw#{mi9gL>;X!p9$@nowJ_ouQ{}DRTFE@wseIK7x`|-}#q-Pk8;TQU#`-55$Xb z_bWm}FGIIL&KQRK2*rVq5r`){7yiqc^Gp8Ef{HMUe}$fyuFmhnL|8n^aXIu44uXMT zIAToyUUy7*9NHsY%w?iP>6qvU`u-_p_M;<(b6;pbT7yQZ1I6=?XNV$@N6-*sKqkL= z6X@4cPxk7Yo}c?q1D;5v=5wR3J=!k`=O#Hq(QoqzV6eEOp}#+2wvK)uZ{D&a`jXwh zfu(Xz@7Z4Jw>=fhz#-IOrbx6KN@3QY@XTGy4FH8L zKDzI>2<2a#@M@H>1zy-TH`5RRGD!q)Y)`*H?xHT^4C9=Uc&qNc_-Y*t5o)vjGoU{8 z8@^S)_FM-bIvFKJ{97BxL4YOj%wtb3vK9XmGuoC z5lxr~y4Kmg^&w~tQl#{1!;T-jO5J!oP3@Kz<)HidK3USc;_J(eA2qhC z1Tx(Jj!-}PzVIYPK_>8*?OL^)_8peUXV!0Cg`OS_$F;9?H@MF=0d*%qO9JIyaaA3G zu8VaW0KoWT&&{S0pI8<2rXdvP->Tm5sP_a22iHL!X!a_u+)1=64%xrKbu$Y6n=tLF zGI9aV7#m`#iibKux$1Q@4HmvI;~`LO{I@1nfAx`mDt1@l zQN7?!cvR|8ML#!I0feRfmxh%Gj@wI3PH%x$?fo{(tw4-fb94z1iNb3o9O!~=m#aFO z$#c++P3l-%a#LDXuvcK!PhIbB!T4|~D_ycHo@SpH1AF5e)?a6EJO-426y1^F)8K3+ zCA{uUW;gz*$)9@deyKQQv@*}h%yU2fFf~#n8s}X5g8E3a{j&CQ|EKkq^TQwFcj_i$Hdv6)CH$c;8`qcDY0QvrZ^v^tWxAdRq zuYZbKm>_(QZxr00F?STBVpQBL&4y%g*;tBPaY?N$rbMh>0)%13qw`rNn5v2V%|^O5 z604Rg=wS_jmXtobNVce6571zM(m23jpsM!JF7q$gP8K8$<1FU>v2f4U$EP`uX9QZ6 zTR3R?F!Z%R6U?^cE-<(tzX>j%B#Hkmh@6cHbCL-g0*Vd*tsg_6+zGfPP&~;{DhGB` zNpfja+Y*?dNfidT=tMJW4W|pf`o>XVK!h6*{#NchqEP(*imfTqwrFHZ%Nh2pdNhNEYvHi* zo=yRk@sxE8iE?=u!UyYvNxJV1BkU*$@RWtV#6^z|Ed%}T?(>(p+8Dn9->>6V9f2Y~ zifSh1_mJg6S2;=vm&LkWo;?2YYtii|>cs9rMqWI8Xa_SV18qR?Symtn@I!D(7`C#_ zyjQni6IAOPrP#7yH)0GM9^p0$G&wL_KKe5QI-i>2!w)At!UUge?C)S6U;ovOecw4I zR3iTh`h@NcP|vEc^7cw-xkN-s{41$r$}1pwIc^j%_B=CA8(Mt|wA5QRq2h+@M!V4# zn8Ak9UcMwk2rEL73Q_k7{`=>mEa7SC(wh7Jg714DiB*%{xyUdY&gdp$N{rir5SIPz z4QdhH@8tHdr+TvGl-0NK|N8E}`Ew`!5V|-3C@=h|afnIw&UE!{uQ&IBXes~Ts0BO3 zLxj5bN_O@{+PVb46=+*oy8Ewv}k62b?hq%>dq zrU5CvzwQc*FTmBM_xi~jxhxuNd&<-jV)t#o-%ywQ6~zSZ5=)5{*~oS+M;9y%6hj)? z_v-$4>)`E*>DxWoCEgGJZdU>EKWf_(Ktlsra{DLuzKr%9Vg4(R;Q)Ee5zh7XHqfk+>@Gy85AAh>Jx{jv039q1pLtG!Wuwp0$Y zlTS1p%I;6y;JUxN#3hRB?dFX}5mh!nAMaZI8=LzmhN)BSJI%oPD~H=;thJbJcUbqFHqbg93&S7r6JG%e z4Fm&Ym=jpopa4HuNH2iSDbugRGud(bS4=Q)5iF&fbR&^rnhBz;4(if9&#EB3!oPsP z;6&XAy|rq2up0farwf-Zzt+D5|ogv-lW|3x>J+6v$l zN=ENJWhYDzJ$67uajc1xjEASjitw8YjZ3W)HTkaH9aOR(%M}5VF;TgXo9dNh8oqEP zEPJ+dC~yF$QViO#AjZN77(MdkeslJx>yo4WKIrToN%dpSf6flx54np|9E*x+A*x3* zxIWdHH=8#*R}}^G8mEhI<@4yE{wcR zc!74orIX7t*w9&$F$%6l@m^UXc-E8fjYmat&|@xe`%&_4ZssY6yCw^a zPa?DvMiFhCujPU!)-s09cbeY=DMezh?K| ze=Gz|l;ivh@?Q0@-#Lq~?kC*z{%)fAHk0R(v;gaYkyo6EJ1?gZ2JAup6%?O@yDF$f zbzq@UZie9~w_A3AK>}3PdGz^i>l1#R2+&@n?o4whhTZ$_cqJNKzmSq1fBtflE}|+O zst%lOumkw{-81nZt*Zn9#o7!Zd>{wVBiJ8G_vXRYnWwje>SCw_EH;diYtWrvH5l(_ z()7K`9lkPzwo<~jU^Hx#@U82HQN24ziqVI+e}0RGQHhif6v_s8E{xd$2{s6jAHSaq zClxu?`{LX-p@)>C-+a~w3nBl{ZpS~ZynAqb9*t958kzrlghM21HllnN0PLH~t)B8! zJVuUN6Q%9zp~tH+pgKu>V%NIzX90`^){^RCHD&PC`lE&;!J{xZPQVKQ3*3SMeUzaJ z!3YbA2WFP;nYqE8*@n^rUD0{yA0g)}=V~ z2&nsk8_px?9Jefe)IdMM5A0Vs1$@Gqrv)|Nv4t)Q zSc*4mGaZPuad92hF}zwhT3T06b{kYh#mkHH)(Qh!yXPb;fQH<|vNx$U*@i$1+jIw& zF*7N`FY_AcvYFTX&~nE>GPr&)cCD@p{rj0x&Q4Jy*`4_p0;;VMq_cLW}>22)TuNhCh; zzehj5;*@dTkk37(ew;33!>qOK8u3RM5*=rd6C zLO{FU7FJh&^sk$Af*Kq~NG$`y;JYzA*lCPm^ry)v%%+C=$@N);dj%J1;`>-LKM(6t?`PUE6|E~FEBhxJZd2A)DSfPJ^*W9XK0)3%| z)q_B@7uWyncn{?Ln~T5Sfo=o#glwz8yBFW@$`-&V2Ls2FGL_Y8D+LxLEoH|9N-LB) z+69!0RL5@<(Dd4kQZRnvL(Qlt_q8k3GJz|f*5MnXZSeI?c9xAOr9}WXK7n>{4z?tyrs&c}`Che%7mNj2#5yzMRLCt!{EL!(juT~Q{K ziXSXr$AocMSPhS53c@ftYfI1-tjv(g|4uDh8+0$n3*H5?7(I}#L300+x(nQJ;4GlY zbc+?_C-lli{yh}}PKoU2ZNQm(=Uw8TWCw(dghV2aX6h1sneB4_AD<7mCi}FVY8*tc zuIzq5aT$tLFFb&o1(2SDNLO!u91x&DK0=)1^VO~rSC3Lw?Hv2*VcfPNKlaq%gV63* z!H*?8z;qY==?)Gd6>1sC%O8Ktx49Ib4_}A+5v|D%cHGp6;mm-M>yX@zrQ>K}j`sKc zM38QXctud6M!0o`ixh8vd@hn`XP#VptDeLORCqnl!G-{!XT1eLVe_#UbZ4jtRZazu zOjL`fDAm6hIJNfpI`x>v00zPjz$Un{s^=v=9O!!wY+vnrjo!b+=>fw9Y;MS;Cvtd> zt1TfX`@kkOl4GN4yv`XR{?pmjH179}za#8KR}%r?P60r*y76WIS~OBv{uhnDBf}5A z)%WDu8y2WvQLuC=QYMvB$uD*8j%iWO@c)9 z5{W#K^6mg3$`xTm-|34T<`Q?wC-EK;k|6XMu90ugQ*Pk`<0oFYit_Hsygao z-#K;jS+D{@zcN-+Q;hbw1hQj;v=(V`<{&;?PMqw$L-L_{0Ssl019w0*NHwT&I?T!q zL6xAO1U^|$E_9LIZ{J3G%X~5D>8iYj`F%SYHgy?C&edeqmMD}{SRO_bC_zLc%pz%s z_PB>+7IQ^WMa1if|m6tJwbNF``Vvn=*P*D-s08{)DGgTV(7ee=`{bPi_I^i_rablI*O9Yb4(1-|3>-j)Vs0Fugs5uea7 z4=v+C)-Y*ox>hiUG$S<{W&0W^+F{Fkl#F3E{qE1|yGRt}CdCRL9J3TafP85duA>@} z&W)tK4HL@f`E!r(5r0SOOubEtRb0+rqaiJX=D|CJBBkmAb|{U`Z&hQ81SRH8 z-BLuCP1$qH(8G_Fr~>-J5Y^(qHT($K70G^u$83@0J(JL_yDHH{tkK-ztg(?lzAM$Z z(_$6M+z(z)l9WnrD&G7o`ZJPINfC~VlgmISqtMZBmFfPk>h#%P0U_J%U>UI6|({vhC258m#=%M8i1TVz!<<9WlxXfBZn z1W~Q!%Y6U@wW13FHUHL~Jb_|Hr-$-xu2ewMwk*8TT4PMCulRMs>I9neLS}kIV2ezmS5$8+`MD zGpUd?++4{P#`PH}!=s~bBi?FnIMPWJ@|0dzMo5_qv5tz%W`Pkl{)?}qd#I<4cVBel zympi*>@cPYmw9azNdl%0{Q~3=r>cRGb>)D-GMJM0#c(7QG6qk}CD~;N@16SYZ;YfU z{N=`a(t>x*@~jAqlFT)`Kx~pO4af+C8Y2!8c6a%>^PvghOaMqG0B)P+$^qd zFsaAf6YvHnHEzXMIzlqAR7YQJ0-Uw`zZx%qy)UA<`a@-FH>3u?w;LhDnrq?+S&EZN)C=xHT|K`&zD~rbYYb}n$Se!UX;^BVGAt8_rzCWG}{8!IPLownFJ9kApACB05jTGTO zQcV_dKiGWdD;#||?{+7&KdAbN`Im*;45T!wdUsdeOG(_BMplbx+hM)F-(lHcr*;sx z5Ux`<%sAn1|32b@R6%K zmjcamPAP9$=LAHyB9!c6lD07q#4{r&jr2GTjH(;Z5kwSk!6=B(xQ6YOwscTP94|rf zsaz6hnVl0MV1|wcJG+D~#aGt-;RFsQROMUDK7YAn#I}-=)pYL|u?{=ga=&o5%bPVu zJ3IU*J|E^ynrmgb_ga;`$AZYnA!WoL$HiV-)IgXGQBWiv9x0O!o0!@ovByWo!?d}u zvxoDasub#RN7Wn!U9}9P*@99cC8)f`3Tv!{z>FvkOeO3ci+M^aRuyB^G@h}xEpaKf zwHN>P)^=QkhDF=-9iwzEEO$D1-XhlXYqMBv5~IyZe|*Z-fc3|$Vdef1lGbog*jyr5 zkGQ#LHT5pzC5YPy*)m*V{9c;l>d&f0x*Y+L%dS}>DWoYx{cQ#My?1Raz_eqy8LMq^ z3sPPc@FHo&COS4hq?Z_2(5={?TbYc~>YoPV=#Me)3uSYJ23ET-3PMCGE6EP+9Vuz#jLRGJgpz9xQD@VeVj(VO4$Vbr~dua=lEUw z!)z(S4;s;I**1Z9G{GDrt&;c@CK%1Am~E)}rs5e%H*1!rdVl@uJGHt;hn~FWCz%Zw zep?gaVN!KJ5HH?V@dVAN!M5ou+lgp-Yz*7lzl7VQ(h%_cNP{$(;-sYH@(jybH0350 z`<9tE3wOpCzAGFhYZ9A3Imt0CSRpw}u=p5f5XI3TBxz`hH~lCw+`!ghGsWYwb5nm(C$_;ZK46AbzXf`o~kYvysBd4VxpQ@vx>Lds-*9an~koXEDad z`qp~Ph#Mb;(cW%Ls$;PQB|Z!q8B3}yB8hZsx0HuYLdpBsEk8@(!e{L6u2D7GD3MMA zL1b+bm1dIUFbU)W3EOU%oTtJ3zPT881I~B+W&470Gj-v2iu+NwvpLDB7e$$Lw(_Je z$weYz9|@4Ro1Jwt!YgPYw-1~|3BR-qMyO|Wt~5M{6nXVJK0RtM7(*BmbK_CLn?hj* zUh`{ZLjGB$g95!}4M4wQVE*>0{yhP6QN|?jWPSnh^q$cb!`+VE?%#~h<#+E$qCv)J zDn!{K`iGg}3StFk8cB2mWIgtTg&i+6nR`!v3;P}qHgeiR7ExANsx@zePL`?oZnSL204OA$$|Uv+F}u)sze{lH%|c_!DqG&z7H)E_OWvVZOJM!eiSw|Eij#$7 zi6wmo81k@4r6O<>&u39mSV%&puzZvGHD^Vr7oi$9Eg4;x{P zKO1nRIiZQ;4e~HuVp3Iq)$Eum*X*At2ed>Y+(`^QhAzi8#425((HKm|tLbKbDp5}W z@y~S#GcX}-^rc5{;$}|vR|t9(O!8a3eI|!q(Y0)GMaSqTPN!0ZUtjMPQ^9^@Mir;< zNRv8|IwO^U!8GWZ)UJrPN%u^f+lR{BLuZKmz?9vHsSOuy4at_PB@Y+?aT!cC942^29x^v`fdkX?zKF6!+u(h~B z`^S5v+T`U-uHg-WtnM2sr1Ft(tA)6Hc@L1J`%+006vWP=xP!2mH*V^8blR1W0nP#i zYa0x4AK(&79Rji8M$n|Pbr_mpRQ`ol$F1Rp=Rly*7JP|Fk{qIO{I&tD`X+S5*%BAj z#B#-$7Nnxu7X9htoiGNNI7MMYFfU~PIpZ@`lHjvRPJt?aE)F=Ro3HPL*qqmxaD;90 zHhPJ?h5-9BR^eEUqBwY@r_w1Okh+t$m69TgqZK?)&@1H7U8MyK)*zaYw66?#i}%%L z7y&Kn$=A#k2VGl|%mY+hCy86Eryto3gqKvOjpO^cR0;ma(L}V$2DoDqH`X3`@+;f_ z-pxN&+F_o(XM0+Q-MVwf;HftY(UGoWO=i1V-e3+n^$LP1iKK^lmu23ttkC(JOvgR0 zo8+fWV2L>=lhpw?eeiDNe(Pz zm|ktQVOdApL@@jq<1>YyN@RLJfH%H{TfTuGDz9F76#w7^OqOT(8Z`FLxJ6#|F?1#S zvxfeRzK(&3b-q*gqx$kck8t|=W*0a`pC8lpkw-#UaVpVC_+&N(>BlO{s+o9OstbMM zT-;;&9n58B`XaAE`t$t_5`!T;oObn$Tl9tErIS0Um0$dQ zWuRrb+O~*lWr7ZznzCSk!o7YwB6*8sy$=jUGBa8QH~9&0=+&JFtpJtV2>ezIu z|x z)Frc;Krfcg;yW%gNc2o~iXz6cs03^}8B%yXbVpf<_F0VP3^@Ha==$#E?k>KT9FL9* zxc{1y6RQ^4tNmR)SA}%%h4ih4Kx^N3;`^(ozDb}_f26CFo?pXzQ!=3R3k zL{f<{T448q z!Sek$dqVnjS0VDpei~JN#BlYFxICJ)eHdEU#6q<}ZbqJ1vLsFgI|vo$E2-q*+6Q)T=g zpvR1AEItXUc&Rv!&3-W9Ub}BV5?*1WdHKhj(BSJBiQf;%?soy2t62aw<+!u z&|wcxoCQ`otCmxfSE#A>KXPX?i9p?mhKt+C7be+Vd*fQ)u}`3KW9FL<y3mEFjfgpmJDP=%Ikix zi80KL{kzG$C-h;C5ge)aIu&L$)dOWnA`@WrtUEQ$GdQ!(n!0JGq(VtoGh1JeR;R6| z92mdkf6CgD=)gx5kJG+`F-H>hkENE+J0~2jz54!qohIelE%p&=xd@#IGv(^CRrUjs zy{a63^R>O6?x!9L?Pz+hN~$AB*MKtcG+TT)lIs)cX8V@(0g-*-bx~CJPg;AH8*P#z z(9d!l8|1G>F+MK3zk)i3!_nUp-XZy#|3s|(n)Xwt>8l)IHl(vb8Ap;dgk>^+_$+h( z+e&(N_7f?J^kbqV&z2kC&(;#-6AV%!RfoS`)lI6|gB+?j5*C3D&TlZtHZqChrBA`4 znnh;|Z~qC}h)chyj`C(2>XD1cE60r&X}^n9sv$FVs~vERGj`* zY2nvYlhZf4B4(rNXs_JY5|PAJN>L!wqD`zH%@`0G4t)F#yi@LF#lf+wLL z<{i_a96_u6ydz+5d>bTWSEAWaJLdcrePdxuMAJew-jjbGz-6a38YPK-JdQl^0UC)9 zntSIQsn5pY>DLVPq&f4x7CjJDCYSdOcPf>;y#36g6@zPV@k4bAi%A5@{owH1z~f>2 zwH#bPjFkA9-O{#TWZ~i#cq_unYm6)pPN3*kN(+8WDh)oU+X}9Dxg!;*^ZJ-JEXR1S(`Ol+6xyRex!NmRThA$fSd!7 zH0%NxeJpdYzDJmyY*r?!{p|sfXb!9d^k9Rz)BvlbK6jHC(~*FmwyFdeUEwRkbFb+8 zsaJe-C0LrYE8UP|TfIA&Nb5M&v!xKT%>~MuN_KP0FczaCN%~NWX8v<*?_A8PjQaQf zEuFVdHlw6w?H_0+`@h>JDv*OV_^TCD%kh?)@^PiRQvbLC72x6Pgw~eWoAehqHos7d zA0D#q1I#h@?%|kF>&~xVHr?q+2ZFlz36H-XthIl^(J*zrlpzD)j0fz@qA6){j^?$IPacVU8eoF@PcdYBQW6WkG+R_5l zQ3v84pT%PSwKGNMgYE?u44m7kCOeq0v2Y89jI|3V3F~pnT50LU=eu8`%hR(L~lWHk9}t%2LO9Z514xWp8S|nEwvWF~krdf%!lb zQbhBK#rSV5Te9s=;zYkP`*(gs%wJAA$ygtjT%KAe6S}KRBPx9`;!-|h(2JFL96#(@ zCgM)jGB>qq^eia5fdBXFHtUc*wcd8{bHEXKpHQGP>;rIH&AYFR>-f>s!mMSE?xTj6#0D|(J!TmlDRq=%LJLx+(~au&hnQKQU2gFuho%%>Ns=k0;p zX}80I&1g;ty83FFG~Ce54D+;NT-LT9t4LnqY)IKj^`5<|R{_vR1}~pyc@7Eq4Ik zM}9rAIzxq;rf>bLZ@e~;aD`rewN~o(?;rD$+~H!UJHWakGlxn40j_3Zsi{fGmk&bB zhN^0aEY(b^OdCIi-+@r}I?ApTIJukW=*M?NlSH;1f4UXf3sL$aIVm}uSC!tx)mUl9 zc_O!j1bW+}vnE`bTl$q^(6b|CqVL6{In1kk1{>lEDy@OW1AVIhCe(@!)siytmcd}H zJLOnalHLqAj;PH}U79t*S-KZVycQ%6hPQUS?8$;K0$Nz?doCnb-I<00#pYA@!J17( zqgT{4a>Ii+G45a3E*-0-@$LZUIpl7;Ccd z*e$bR-+idGj)jU0caTN}_{$$C9u$~7hnE9m@ZDocu1@Ue4>goxLQ2DGmN0Uj_snJ+ zMTisdA0Aj;=+=0_n?Na=4iJ!6j|jV%Dg#rm=Gk6oe_P)B?YffDx?i{(5vAB=W#G_{ zbmPSD?+T%RQ(?mjon%8eRnNd_V|Fk@`K~kq+Cm_rbWS-doeav=-0XY^&b9u5Egf<% zF!}ZdO!Lhk8KojvwX8b=8{-^S3!P^-I^10~_jTOU8Ip$m@_uq=4e43@1|wyRfcAx4 zriTe{aW&Q$eCW~i6stFDECUDBqzYTTmX}-^7>)`R@3yCxy%{4;buI3Fj1Nu(EpV4{ zsD$oH$zf)lA3W>)UJ|^)t?lheE-(2=nZ@*ntDr#e;m2FR8fJf+xM&;m#b>fiZCt0h zkLE>pn=_c*!BsQiPi3saqC+cuc!W#bi?C{E??~h^_{&v4U5cUPG<<5 z5H$a_$0q>JHBt#Dy%q-v+ahw5gn8$T*tZE*yGh009b1pQX;LKWqy_Y{M#rA#vsfwSC4)6Z=d@_Yc7sPD zG;dJvGI#Ojw}k0FHS0RF%}E-Hf4@-_P-_cU4|x-}M4yvy!CLaNWebWR6cn~N%6F@? zBbccZiGJ7bY%jgp2M(ZV@;9!(3fNsz{qBIb56&XY^%!EP0kW4f#g5~cvFrl59UDKG z%=q^4Rus=w`y9z>(e#6vr;6=S71bW$GtwYu@DO&K$+6V=KAny48;0!|4ltO5Ksc-Q zR2i#E^wo)6Ysh=DizKl-nObIKy-zUdW%4UB+6Hl~6)>5e+VChVXZt4(Ug39C36s9F z%DVa&82{d95$o3OOA;zug#SI55|aoays$zISj)fyjpWcB zN=kQ^grGyi&@J5}Ao|cDh;)OLfPl1wfRv&b_-?+}?|T0L&~w;l@4fDI-=C$*XXmc0 z>!yl+uYV2JH##iyncXw=eC0j2oK4)n%EG!k7W;9RK$lB{NCE#hM_Hz_ybFRdhN5>9 z+d$^e`JmW`{?Pznk{Yr`u{Rh1o7bGVOTD|p4P(f9Q^03!zdjyc(C3%Pj6)W}!tE=g zd3l;e-Q?VIcy+VzgV4D=1h9?!{C)v`|~ z9i1rs1GbuY))v#Or`||R?X2mRSfmN~a`&DO^hkZlw4Gijm znMip$vxX)6X#rQWCxEiT(o!ZLCQ7{`w;nJlaE(sdqTZB^c1BFKtD0hKa9h$g0Q36@ zqaL3S{#gP~c+`YeODHOVF+WR4LNy*eSO4zUdH6Kiz|`5&fq$|M6Rw?{|S5 znKvb6zx0~Vt+V*JdNB^(u8;lCc#0|M=C|Z2M65S~K**ZT??G>564Y_ZLKx@;N=?Dm z`WJvv?8Mk1k355cuh+@Fugy?uAi5II7Nc5iB^l>KIp?V^s!*I@$%~Q_LzGBPUA6lC zC28mMJR+qb@v#!W{h8mPExGR?2t`F5j%7@k6`v6jnh%)00|NRAu`RF(g@1KF$VXt8 z+zXc+;mNDJghttuy8NiXPuDz=Awa+Q_xp2+%8EUCBxBhRjp`Z}C z?PDrD1My>29O;zcK5wv}m+4Ql_v8D|>cL-*g3ou+>?yPOO9d<`z=8r#mPwX+2thDM z+Yu-oMxvUY=*J0acRtRDKvxx5IgH}$+XZ`Gz$jY1K3)QY*$)MqU~Q@`=3^hzVCm7v zqXRjG#cyBe6^-z%zIo*NN3AJtgbr#FY;yQMRUd`zlpnP8`eD(zR99R1feEl2NsZh{ zemvmX)~!+Z@@sso!`rdNr3;&_P;%5KlBT3;Rf!D6==^V$Zhs_>4%eRCw;s{!b6q>)4Mr2?2MyEepomT_#NKPW ze@o^Phba=C? zviO%=Um9bRnd@DomCq;+#zE#U*t>Po;H=KRuvv%6I>7n(NbyF_{bcV&DgMx2HQ`UM zw#*lYGK5iv!-kfqVhHy&?+Q-#ZaQ`waIb~1%&M*7FNl(2;f>A6x&1AoISr}|nZrOT zVE4l#B|g)-7Tf!u(Bq*5MMtveXz-%#=8_xRK+~5MGdTky@o<+#YeVYn!s#@J%Kzyu zHYwCY$!wFT0hW&LRQuAMR;@uC^wAB@DwMTB8I*)Unw=LKO8Cq8BTg4Q`0KiRQyI=h zzFo#x4zE#euR|Jy>wYxF2nO~_tCb~o2&jmC7~m0)vSYOjlgIp$N8HN!5E${MIcambZ|K+N)87ICBJ+I`?(;*)UvFnqYzan8%wIpQ^D->Kz`}*Tj9$^j zojXZJnVPdbQUPw;C-*pVaiSui=5HUj4Zq1iI@yuCkn4E8QO8m7N(Kg&k3*)MHg?&> z{C_WbxfhC;k+?a)^ zk)Y+8_MvPF3WR)sJE!y0rbI}hmO<2P|IM<1__qOjB*w+^!;q4y&zz>t#!u5xz;Lp? z`7B>NBan0k2bf|B@`PQ-PM(s-sPT6!0ojq>=dr@Dn?salsKvc;qC8$FzP&`^_UT$n zM{h;Y+jWHR%=pEwu(FU&fI?Z<>xPqUnE>ag$?|{(Q1%~P?#$b zB`<|ni@Q}g@W_3Xt*&<>TfU+VtFx3V$lqZc)c!R4baEK-Z@a-GBS zL~}2#+AOwQpP%5*FW`aLX!oYCB3O=+dflmJ2pAs z1bpaYq!)?cg!9UgCvPIDKhz3>=p*GN>vx?1T(9Jvj2L_fOxA0o=VMxAP&M~(p=h;5 z=E}*Ro2wfrxvQ*-9t0~vH{1LK9qat2-Ow`DkCk%{Pis8tWnfK3TTL?#^m&^y34AC)*I--M-3Jt^6j943nWf7e}}MX_Qy|9Z$iE z5YPTGkkvz8=$GUOj0t;2MX;X!+55xXe)(~HEth8J94JI;6Md3Aa}4m0wkw%*E3|C( z+t|+iZq37H#lBMR44K{AU!n<9KV{0b(I8PLiyi>jl860BnZ?Sy{ihx=VQujm=+4i~ z9^Gu+<@nlL>JP@^uMuS*dfcG-8e8nyE7YCU z^J&?{D}}6C0&-42!yfRdyJOs7*5Rf}?<9i%fN%$HU?}1Dh7iXihSl_Q*iS;Mn}|gt zja(iRP9w`$2|{CBjrFbbwAn9!Hbh`$iL|}@A#|Pqjb2KL>S~7F%_CqE!lD@21#F;T zF4tV%`}w?Ir7yes6JQkrs7k5%HMtKNRIG!;w9`*`Yl*0e0VZO2M@oddJisTxp8~v! zL-(I7JMS9t#I>S{@r~Ru59ti<%gH~?2xDb1lc~xYF>2zB^d-5#etYg&(Rv7|7Baw% z|5n)dF=Mls!&f(1>W@dR;yQW!wB(K=)oSSafs?UARLC72fBPy|G(eY8%!Qa%X5j&8RfeMsf5KeAo_9U1wq(`o91Ik~iE!CE zkOaa0!%CGPUJ+E9B=pdDcvKZm75b2!7%h-DoPw#Z1*~I-z*YYTI59d`*{1sMBKaW< zu$yiF;0YAZ3x>ys6*JP9{O5; zr|GI%7YA^N9RU7hkoi<)c2JQ7k9D{~wwMqI9sEF->#{TPA zi0Yi$HUcKmm%*yq?)7mVXo{}lRUg1N5h0zme}A>CT`T!6_|PzRo@y}Z-58gt0g2NK z4y{fh$P;>HscJbSW39K?gc_seqQLY_!ioRIqNR;r%>~4AGWz>c53FNt*1=}ZbSg85 zi(KuX$5>LuS(`-?6HyEPS#eBeR(?D%`JyUa^705&~VbdVY5);ifVz+ zBUGqm{IxQv;@F)v9Y|bY!FO~htRUWt^ijYDayn*@fB>BLF)x}-a-UGgqDG!tkT@+c895vy7X zr_i87wPS_En&RKL9MVtx)Iy#=s0In_uV!TRmD~S?Jr!~s>#zI{04n@x1o(Wis}gs5 zV$lhR1dCjobz#1WWQzAqp}_}3P@^fi|HST9#_TlKcY1z5;C3c}E!4`AR(zseJ-!37?1Q_xJo|2P`r0-bu)R&_J5YnRMNFTCv>R>y-9CRh$58n2ACw!&F?{Hg=u50NokEd9`>H@LNH?wNY}Z%H~|X$EdWfikP%fTsS#8BmIII)mvH!F{RjRxf-Ye8^rLY*v0L4QPlWl-C$%T_4ErIq4$W7 zkJvB`nTsf@RT48+^19Mn=uMz_6nwmLI~?SGtko>2h?s?uB*$tR4UM>&ww=(|5j>P$ zr0XBKshiYTQnEd$g#DX>F3Y#WQWfcpw}9m5RLA^NV0iOe-_iT|^5%*l1wWd^Cd%Li zEV+v0pRbMJ>L`j*(-B&2QX?t`Ug@=6io6Ls1r8RIHjn$(lZE_1R#mp6I5Jfg%f!^F zU|G>;sKNy{UhKzQsb+={^$(S%gv?GmU&w5t)oJBw-WR!_6hD{^xC)m}u*Xr!dQs?K>Fh1|XMvC(9 zJzyXbY?-6gz%_D~H_N}@B7^_^QTE7p4M^E&W7Uf;md36lV4VgAU2s!9q;RlD4Y@NH zzyho~ENiKSU#`56Fb#&&b3LmpWX}glKGhDiF00W*N zIesrB0+7SfKm!dV$3pKreww;NRpg<`Yh0!aDd7JwKc-rJC1#FhzZUkfkEDKW(gjY| zNWafZIXi!24j}>boM#69D6fnB{Ppl6cTcch?t*+bL)Fh`8&rya0}LOf^fb7uGbVI{ z?G=7Z*8s2?vVi+DWP_N)_UpX`8?O8aPP?9Etn#y+TEdi&SeVjX>4y;Knvt1&{Sm^G zK{bb?zLk~}B>9y^}ub@JgP~{Kb(u`AQA(sQNTExfke`tD(@kt*BgAT z!wf2fWK|BAcJdfgGT*&z>YMrJPuN#fvN{m1>=7~1@7)@6UbItf^sy#5|L!l=_@R{d zUMriE+FIkyQNm2e6C#d{-Vq=}Eb!`@*+V(B+XIihRFB1xpKbCl@cpy$O2=2qM#w>;q+o_qkJ;vR7mE@<~*`7B3pHD~?B^Z)tPWqc_UJ_G#5q%g4}9SW(}1 zldDEz`e3r1@w{j!nvf@e4GVoP*z=0>luxsE4;I|s$-o8bv3n{o8;LoL*aUBOfvTss zARcTUK>3ZSixYw&Wr>%*?Mng4GN7g*%&_7~iQ3%-m#4D;V-_*WWDg+k9^s5nQS<2IM&jRsIS_H}*4V7{XIP_KUylK^W4pR3mWP?i4bx;j;CLsb5EH?=%u507G|c>-&YrgIq!X~S9K zzl`Nhe}+5$-q368;Co-o-Yves6zb0)#IV?!uJ?t0AdK&f`e9|)&{^BlDY9sAd5K>f z;%*4_gZ4%G0Q9xLQVdqtr+GxYD5(mb?$_~m@dA{a59D^bhxZt*dn07Z0!~U^p~%^6 zQTl3kU>|Hvh^T+@eMg6<(qI*X9tFG7w@UANa1%;2s(h}jd^Qxys%MqWsV9H!T-2H< zZ56r`w;h;I=(Yz_VjWN8hSaUR;VABF3J5~QKki>HY!J~*v<@S=(U<`_EI^!p)9YRV zSK94tLF5y5Q{$ZO{-sxU%f)mYM)em4_UJN*;#^pR-Gmat_Yfb`dxym2;B|D_J;{Mz#iL|Cg{_a@k)E7lStlu`IakQ8-)Cy)g$ z9ZmDfrmDc&mPda`*SNi^qYA?|jBvR4(Zn{TR}#NqP+5{S_SLczs+m&?h=mw$&j*MYxedMGm)T@5E3!lb=I;9G)dG{^%~UyJ;&;$Y zkObX1=Tuj)xx*FUM{`8Bk^cS#?#3c3E{{A@7kwF2K#6$-k0*tQ83*oO0RBS$`|R&F zu?sH@xI7X%)%co}wT$wezg;rqq5{H`X% zG+09kRt$x+(&Na@>*5Nc zIDIRrbq8Lu$wrSHL z&c$I?c06u@hi_T|u_fO5_%cuu|E7SjMo&YEm zMtj|Tlc{-dtw8N`rGrOGA;(#|{R6V7<$C0C&WB|my^B2QKc}g!5(S-@$KJ_~3ce1O z45m~`q)HsCguf7cM3v_tf32--n;8_bu73ROP?=j+!8~355}&~$;vAW%rOjar{&uM6YV5z+XFJeE=|ewW^?BN;t+3+; zMoH-X7dhL%^Gwn|2tb;hxq^)yD!P^s_vS0bD$o=LWC{KIzfH zQ2%uYpMKd8J=_6_<`Kx~g!wYS!>jeDjfOOA47nYY9zrUI70)+BsMubLE=fgoG-$;D zShGW}CIVNl442U0|y*U<4c{h8hA^=A%`SqG;8Zh8c{sSpI z86JMEJtsUh$c_$zAZujindFR$I}BA?>7S^UzLT~_Nye2dLv~de)vbiMB<6Cib z;P#kaqEu{O#B8+@CfLbYKCk*&yjEdF)>2+^rTqAfMcePSRYUUYP3z!Vi7k!33j$t* zD>firG|O4TU<0se!5~GS&UEkB9U6L1P^6mCUl2HK#Zk9^h6vTIsrgV38o-^kQl$mXH` zA!0(rQ;AnOr?^@+^AGaXq*fOBuV$?-cob%|DY!24zmwyE;1`QRD+7ePa z@kB8l5I=hI9pzBbihqk%aQn<8>+e=#O^N1cx4GhE!x;KH@&K2ycdWFJFu8T@W_%}3 zN?*?G$9lS7S@IwfK z2>|d+H_Y{W1QjTJInYM?!TG`H-xWaXoh{apk7R*9UR3oR$q8g=BOZ$yMf+-khB~ZDU?L>(_hFZ%hw>K|}{Guvk1`k`+b*>))_=p6J zXK#Da5@%fpOjD~5CoJk4;XM~4^q=75Lin%Zu5&|{kUMW@i9MZCqx6JwpQribpI7uZ z!YW>gPWW@W^6g^(xt!-IU9LVqH%2Yw$Tnh`)ELrJlt^!8l8|3jmeVa(lz`gV+JBhx zJg%7w&9uSKUv)E*`J_v)ND?WbLR4Zpnp1}FZQoDfK~zot9u}>Jn7R7SDO%2N&2Nj- z&OAui`{2U%KO0Zs|YV__MlFjcM!Eqok$~aCrke{t}_;STdd8(m!P0-`d%=ZYq zkn&559!WZYYqi~>fths~HqbM679+bn`lcRaP?bNNzDme0N#P+2_;91zL>h-+Vz-CTqpOJO_dN|HJ-tpH4Xb&!&YJCy`!X`O`{dGE>m;^yz z3awaTn6iqz8xhj zff!TwXcN|kdi(ikuLY3WTM3&);y7d>j&;{+{Nsp8Tzn#O0z=6`YfdC~Xo*hPF4Eue zDATToY~{h@1}|la${V}>sN@Emo@!@%Krn64K)3(8^C?Q)jAUqx9I_M^V>3PZHrHxx zWtfN>?)|S7ld6%Q-z^T}L&RRZ(^F4*UxpluEw)Q*qA4rT)8(G#H-r}8Otdxy~fS`}g&p;?O>8*4$`$CJkA8G%87z=hpq?9Q*Ml z903`4NOE9V8nqha)}KiaRTrjq2x`yQ)}i0r#k@Bcn&AO1_PdkM+F?~ha}K#ewoJIG z`vXE3(jJU~QzFv+zn;t{{z6XdS?y#c1u-O39z0P$NTGevHJIAe2h?R9Q_g2{C{IB< z!X>*|y0q9#cI;3*i66)qyXtWb27heV9`>Eo-{Vx|>UN{j6ETR;zrVh_ZTVvo*7w)0 zhJpUtctD>!MuloH-I!I%OUr9n^>sCGC2L|f&1oe23kw@1hL_;4WWhU3Epx1F_1fVNxnN$}-%liCQ9nlul`P7dNXHot9g>MasOd9u} z*DQ)qU~DM-eXeFF=efj>Nx=1*bp(U-q7lM2V#LJXpHB&Gompp5C|4FHat#1h5sg)S1r&L%R{@bE9`mN4(iQom&wfX@q(EVx9|mV{%PGJ zQ@6VV2{*c6$UU`S`NpFzH5BmUclpeBn5=GrD7fVaqLBegk2nR=^#5FBZ3ZpqVRLIW zIL-o);&Y+SEzC)TcRwjgx4%#si?!M^1#;vCrkX4;+?%f&!D(jQ=%u*q+Sbm&p8uP5 zDAh4q^(4t@{KP=G$zR1foONVkjp&NR-J=N;APvBgRH!l1Rv#~e$+N9lm9tz_YurnzX}fW6 zuJ~)dRemtu{G+{87D3jBDtk^yO-K{_#{$?0zY1OS=^J2c0E}b}<0A8nrX}G=F!?G$)ejza zmx(0c8%QkATr@INyrb(DV>D5NtcOz_#Tu$omo2Nv8}=&m09%6fBbHcOL!gY4syzRp zZneWIbP}kqtssz`^qZtwu7OlU@!JOqtwWiypO0fF7%rrbem~(|{H5!-bIb^URD5Qy ziQB6B=+72jkXfE4QS$Adb2is^x2s8x3Ir%|#XEq}L5)cr#71xG=z` z(3pM4ow>!&=Mca&=nV;lfXPROC@)4rf90KO-p#hf3xo#V7>lI#+vlzX0Cb@cI*}c) zA`}wu(advDzs)LxXzBP$&My5J^H^m7#1Zl2moTwi%rW4YsR?oFAPq zATD>~x!91)Cb`d08MEH%f00Fn!B>2KSn8GV#hwYQ#A8DGB;JO(h2AT-u&4_s@r;jJg}R>JYp4G!a40o1h}TDGPN$G zCF7n_-D~Ekd^;0m7a;4ZIIz_r=RVKg!!5k>pY1yZPIsE1gW1ezjZkvdEaV4u{1l}u zXIcWYnN?|~^h|FUdE#55{XjDL^T_CN3S(+0ii}9cYe1!vYt`iYKjP7AZ@RM93124eWMUqoh6R0EJ z!;*-_^#NSQOjK9Dv5t%U?IXO;oMp^Y04nz^80HUg02b_cNJvTtrZHG}O#aGYfuIm@ zazOsA-rWy#VAioX~018Z5KupQjEHDOze`z^sfV#3UsoX}_;zz+8sun%o8R-@?&s z>wq)jgZ8HeO@l$aX`Mo0eUlUER7YX*{<|Shl|Di#t{$ud0b}u@=7|}&N0uDm#pvaL z$w+rYg*@Ek%>I{&{Om@zOwHLkrNo~CHMaX(!S#0!&U(X-Vy**O!pE?x>d%<^7l2?n zpVQvoM+A%g@cQ9zFwEoPgXmHCdC7a%cVA!1rRVn33Q%kSZv1~_U-X_C_b! z4fLK0wj^HmfYO!bA6%l~&}EF#T=7xOzdS1+`-Nk^)5mbRM6*VjM?_jVS~Z*zoPJV+ zN5mx$R?SfiK2J!5ils8is49a=#$(vABRXT51IAdxJa&OqAF~$>K5*1_&G79JBf2Rj z4&^dU?swrxTMbK3If+3?`3Kr80v;z&&fH|rg@G7 zI3m;5U1}1Uk^LGglWMN!5Rw%s@s-OH|96;89+(7}@bE9CnRUNGgKvP9m5Os2qtFs6 zvzl^3>_o^mB08b(1dyT)FtHm9FCIJvt)14@yq{`WQkhYzcGXsJ0o}fQ)1e8LUgO)< z&V23Ud=m2whuNs|{ew4wb$O z3|rlat+jwqChM7%jZ12LcLsk$vC=(V%+|~;!$8=zw&F7w66}1n1Ta&I#~1@yfhHLk zA{ja9uSE|wQp2Ht#l#vlGZA;zwjzu?JvjUyAg9X;t8ioQAv0pRW2cU9vKK*o1PE;h zn`!X^!okv|8E^CbQ}X0V-QDU}o+=pw(g6AWyPd_y5bP{~R*2%vNaf}#5q#|9dOsvO z)S)M4J?hlCRHeY~BNw;mekCZ`f6$c6hM#p7WeT={JH-kuZR_d#846#71tXMaOZ|mb zglwy}d5Xh$T$?Y;sx)W)F}j^gap!8e*;!}*82AkPAKXX1X1xRemIS*?&qIS(j|qi( zf9?QRUz62e6tEUA{wTFd*IW>BbH&|JKX{9iXIVKE_TPVDdOdB_0+vHvbLHsNcxE24 z85Zb&GCeA#EL(`aaX%nqt%N&%OH0M-X8w`4L-ZydGWNd|`!F`mGGn5Qa&jjN7TdL4 zewD+9IdFmi1QT+MbpJJ|pV%feBY!9& ziYfjbiC|K({s7c#5E8D8aUw_fR{k;?d#8bHy~B#?!?DTsHOo*jHysdKu#%y;VK-84 z{zUjo}ldWq5XvO^#?nCina3rj;d>VVs5$ZwrrSr=)bo? z_J)iRxg}>Aw&E05_+cPov+0X^_ctZ5#Dq8}_tV95VR0X$dB4V2%DY3b%$6tf0yv6C zIj>ojIv|3Cq3^qhqTOUI-8flwp3`Gc6iCo3UvFlvd_Wu?n8s`YhGIVq)0)!gJ(!y! z+lL#duF!)gV|0aJk;fc4f2`#}yNTVd72At&mzp@p*g(3}tx$2Wp53B|XWPfTvsD-yfZ1BGW z{vJ3C7R2$vue5zl%98a>7bHl;*9#_KHJyZ!sHN}!prfO&Qpy>OV3=%(8J3v9$q&|z zPP=Nk!q8X!+g$Y58f6(jK18i;4M;tw9pdPT%u_X&shB>=OgkAvdV?fV) z>FU^uRL8uLhn7Q&#gg81znOYE9~IT4Im+xEsprRkvnxn%i295^cOTMEMlM^j@K?9~ z1{;;uivTcb?_C0wqsF0N-_(w z$MEn?@M#i#ceGTd1@C;Nt{4})JM5E4CUht;4_F~N&%g6%xamlWyWYDa_!DT3KFa!l zX&hP{F^9d?mO3f9OE3ACpBf;k`{i;Klb5d>?SnRu4dLh7s(Yg!qiA@xA5CbMfV7dD zkMzBZUkTNzj!%<7^eHA$L>NWK6eJ{5wjEPKn4z3RsA=r}oPnRn!*aJR0)WtXK^Pgt zVW?~mgbgO(56nqgs+*uKb6mHk3dah6;BPfadXmC1sH%7Ie<%KX(Rs}ONtu{kIe$Mb zz`{Jk>V~avbY9dwGloD8l4A06sqq7iLPTifg@k(&u6y3d(ZDBA zVBD1uvs^=}u`jgONtkdN*0Qtli1yrR7@ zjTDO3cq+5kvOF4`@{_tp8bF4WXMSVHGd+vF8~!Lneeo(^fViyr|K4m2Q)%ni5oqcc zWGvS+G=+Dgf=`862L8mRvt_D>+AkP)`faN;sAXL5+*e8}%Am+YHjenCKVw#4Vcf$v z%gM_hQsJJP9YJIcxcyfGPG+`q@?Vt1ZJMu#m*u(ZXZfJ7pQYoxb{!+X`I=CFS2uU~ ziF`)}YNSCmk+)di#s+DWAx>M@m0Ba@)QDGt=JhC$Ak$%aJX)H4FE9H+A1j=ZF6$E* z6pR%fpf@(6PMnoN^|?;oi4A|Uv&DVDY|E+KEl%0CZU^&4zjwwH$swgID+KigGBW*n z?mWU(e9izWRIo=jhABMHUdeoc6JRb6$Y5c`XVjJwbk!DmLBjDfR{Smb;VYMsFS0+! zrlKam7*Elr!CvMLPWbV;-VkYGYcEafK1)d7D2b!_brsEEGp+-jdIsZ?>gc0PGH61Q zri!hz!}_-XQp(y+2vx%4nS>s6-`!M{oxwhb9SABdVf75MO-R4xD8?!nQP*xh& zy0PW7cB}Dz?8rebaXabktv0u@a1frFqE<86b;7l^(8gJt$F5(&0HdpWn}PeqVv}NG zXJ^b3uKNWCHxV}Pi)F;h4qE(Zt82B8<+KNv$e}DQf2h36R_-joTQNoT{NtRk%-Jyx zTC;Q_ndY{{(i9VtxL~WBN?OMrY;kO;g|7i<<*f*$Wa$ z7T)sb4;eU0VhaZ|a zsspwE_5Ju+GL3m%m-by;UWbdbY_dgYHM4#fP`>v_kCSTwa(X1H%#4#WZLLX2)Xgvu z|NKo_TcVsUC?nryCKb`c)cKh*_NV$cI~NB6TXy2L9$DR7Sq24PnpB-}R!`8O{X;^2czfOf=}JF!m=~x@B!ofu-jTE*4=B_E!F&IIvUT zN{KQ#s#%6d-u+5?NC>luw5P%0)qW~ooaATXCt|1vg#?CM?R{oUKXq#*Ua%`n zYKHjmOmTXC$5<4CRzd>I!oB+0{^NXh5L{63nT|{Js*U1iFkULSy;H;z=Ry*8>$iS4XschNMGe z$`lQ17l8?Zd_P{UWD`nSml`!F`n+1oCix-mRfn?5PTQ+UIG72|eiGrk3T!t*wfSU? zpEJwp*jR$`m{gmT&l5~nsDF}tPA7TFF^)|0B@)VGpJe~V|Bjx;pmj>_QMtvpYooUH zg#Gw4)aNrDnY4yB8P3s`D zWL@!I3)QgGmh2pzQi?jhB)IQQXnz?(lZ(ju;cd~6eu^oQ%zM%>(v%DFRQ@C?yG1_n zDaMTKL5MIl{96(^qJ!~7FD&Z(b~2nWEN2Y$af%m~deuJ1RD|8wbg5#0gFlKQ=Lwl1 z<6c75#B^9b2yD1n))57Mti1lcW;vvg(O z-)DbxAc8pwD`5L1CVoB^S4WeK`<}0d$x%QXksAD0DC+B{?yHOm>Z#RKN0_ZzL(4zZ z+Lz5T;pr?V3|0O*h8cj^uJ&GE8MmU^53?>#)EbFppvPPj)>mJza|{>#v?ZuYW?fKF zO4E7W^Wi*vR!ri=;VTD02QY!&$H!QyE|c-d>7yz?1t!Su=* z@(N>UkYHW%Wqr@@t$RtEglE!IoX3aeR4eb9HnP@#vP$o-*b$3}YSlN-163Ai6StcW*_&Tvz~L<~zklDoewH^FOF-Uf zwiB1!LKZUEdK&0{4wRRjH|e2Bihwiy!z32bo>Vwc^$9T}OwKQ3xYEaX;K9|0tf#Et zH^$qsjJ`W^*D%AnBJg`j;E&osT;1rXU=f@jk03~z-r!ESm{f!5HEUi+He&N}P!qcr z6k$Ew%BiJEXu9)0@8qMWdttknX_W5@Z0S$aNihh^6n}2Ta31Q|QE1_UUVc)Du>Ito zW54E*KDiUoYhtI#ewY)OF&%8WbsF=-L{8wX@8H9d@MjWVdzX?sZc(YQzPxw>s&}^5 z^nI~f(S7>S4v9C|77Z5a^u=RGK`LfJnjJwiLU>pr*6I({dPK(Mx*r{n9N*h`yka{) z&$7w&AQD^`Vx|m{x?5+ebJ?w~&WjK(RgbS=vU^6Gc5!}UC4yFY1L_|g6}CF)kZW{c zX$5>N3!H@`?0!7EA;SrUf}m zBJpNygF(n2;-&jt%8qy9_duKy!^X%RO}Uh{Uojv(%=V?yZ15` zKQVF~8=v7N$Qq@?cUBUIi@!->vyktqqzY8N*R&kvgZqDGpGBP;wL^*Z7{w>zGfLOf z1eLowCM|Q$yt3%CXhb6ZBpUaBJj>D%WX;NGV+qb`Ny0ZC7`u`yP!%tyO|@$G8b=rF za|Nj-oK#=^nfY)FB0+B?0N}Qp7*%IwsS(!7aijP{S!LekeQD}T7;ASB-;~bdm+F2-hEapmylK@t0uwn@AL-?i27KNk4dieqsmL zI?-_c3g#jD%k6Va)L=Yw5$8B;&up?1^M`}xNni>equCQ zdJ8ZX)HMCbv!GX545O5ZuGWNW-zn5^Dj6|pt8C#R#Sl{CE81VxjsG_FFQ+n; ztvtzBu1RM(U6IP%(_m*A{9D8`h@5=dppVQqMztIPVdj8-A|*RCOE;iSV|k97d}~<3 z!{T+7{HOU(3K%{EcS*}5u{~yQX3GYS8$5r{Cx9R25N_<7@wnI-b*uMjYP8B4cOsyK z8O3DU^FE-YOn8-98I$HmlU~c##Bb8c19{vRp_}~&_`YmXHuwGLK7TJTOo1AJyAct} zTyOjB;Jy%BFI!JW;gNDQAx%p|2d_nP^E)>*FQS1U=MVf`Lqqs8nHZ~jR8N!U$8?HQ zv94)*HorfBI0a^m66q@LCGGOyk;|!OTxX#yh8&sdC}O&z3Wob)yD+R+n{@L3=>rqt zyxEm6n9Q6tLrjhPh_EzlzaaJv)=yO7_BOhJ?Z{%aAP4f*?mG5x?oW`?!h^TsR$6xC>+=*o$O-bIv1Cp-5&)Y~QcTiD& z3)sp?jk|{*qHwEH^UUbWw=W+&K~u-a+`N`l={e*UusSVuoNb7jBL2Pme0HXoqL6n= zcE$YK9!d9tn;S}&N)&UVhHB_`I7-}1Pep5TGud01rfmLqN`YAP<%D-{Snj&& zt+~Kh@~K|*k}3S`E2`DU^G5M^T(aw_wf`K#-jg4cLwn$ZSpp(Nt#$K$^iQs)r7q|L zRF&5K(ImtZC95QuUrex(O4!Yukpl5bcJsWT)5&qH&!7l%-2*7|v-FKe5x2O(dxgtHIM1k2$l1ScxZcQBxVjPRpDRIhI z54*1cTHPCnFX}@%pJmysJ@_$U?2&b26K4h8U#sIzXDed<4taE?)1)zp)(4>{h6 zS+)5Jw96*gI%bu>F4qZzgQ8hgxAk}E9l9c*&eM%qaWno%Ub@>pu zjlCSkD`}(Fb4GxD&x-2c3k3}Po6smA!aTE;y_EQw82tiA+vaj_`oH+U-xxNCrVJG0 zUILZsG9~^2jiq|$qB-{eqv^ZDss7*pIh}HdI7s$5_7>Tcy(9E4BwJ=iQ6vtb>^-vg zmdsFARz?v?NH*D_kWqg3tIzj$UH#eRtmo@~-uL5vjCL*mu&=O^s*#-7PEw@va``bu zdf|z_(bVQ-qnH^AzvoH(vHA8W?&wo#5;HdY!5D<=sVMSCci+EDTk2BPu4$Y!h5^18 zYr(}3!|;t>t33EJuV3$->$J#Fx<@o@*^qiA@v#vIyl`W(Mm8KT;NHd1D*6AS3A!S8 zun>Q_PuX}mk^g2jr5R+%vUF+4{sKy<@hR}#e=w1>jVD9>Qo{-eGU6b_Xf? zPubLIKRKQ?2B7Gs_a zW&H64@|KgDn=CsNJwmTcG{blNN6uj}5`DR5tjKU|MFI@5mye)P^5nm)2-5Ica|M;+S^Ex#H~qR1qMFzA}zxVA>sNG zO!piP!9>=!lIXQTM2~$Yc3n5R-MBNq-hK!dZY1rOuk-4>n&oDG9!5&rU)U4sE&h00 zz3eL}*wb(EXy;R}c7+A~`8GaVy&36228wNM8pZp@(X^dZl3;^#e@Vwtz=p4>Ts4K# z&XTOS7393TJw@0`;o-8oH_}%I zg3T)JjCXrXmt{F_-)Tla@u{8~Rl0}OAjAG9M~rf!SgtVN(g9*t&xhz)3y5nQVV$>q zTWrdB-7@o{_0!&*k0)%aUr%5IgS&$73d5i8zmD=71uwEvoxz=g=EvcW-!gwwghxpG zp4@y;iuLVE3v6foGH`hp-mw~49dQc&XYKYy*dRYd5;b$H7T~(1&1O2?Jor6B6bjK% z5He5VXZt8Ad4reW5L7CwYN-haV8r4^<^*Jw;k*f`eiT>~Xq?%;`o1*7H+wMvgN;B} z&9!-V6hymMAkbLU=)2LWcVtgBG}xBl=4%}CmO6a@h?Br6w#+k&sgvI+Hc;gAe~(be zlO78$@IQNM9X<`$pVK}Yc4Sa%3z3v%r(p&Btk#;Z?4jvhPye=~x! zLOr;73!%VOyeaY(SL(GrcG54W^yfX5NKWL&*dwNhvbo2RTA%@WEWbm|Ywkq)Hs^k%e2KfkF7#%-numv<38^BK zZQ((u=V5o8LaqAz_M`pH+W?;flYjB9!%Sv-@Y(nq#O%od)dxZbqfx5(7VXSff@&Ju zAnnNexGi`iv7`}O1CZexb5qJ7)>wtB9yoqhL=@;Jy$4$NQ|Ht51}eJUqB`cxJfI6) z8Vl9E0413)8T0(dt)RIPG0wB6wAqCeG4}O9la`r}5aYX1PJ8Wu^A#MAC^XPnEd`J6!_c_ttCQhDHLkL#y2bj@)9v>#q0;~VJH=~z7RQjy*lfX1Le8bL)UJaz{MWo zFNlH!LL}@Jyu|L7)!`#uS`L;#T5mukV~7>Pzh1`Wj?53{dWq%S0>i>b&OjoQHC}vG zuK$ts!#iNeTd7BW+p;C@G@_@}f!o9aER+vQZoKwb2~tMlZ{TW)ulzI?4LMs2-9ASf z^$h$u!cUqHSIcZJ3mO(i2}WCYdB`J|Jre9EK0n zdIr#(NhuL@5U#z&@4aa^QQ@E`mRZl5bJZRepV9gW^d?)rr#DcdoOdjIjKfLO6Wqdr z#3Tm2RjiFyu@P#yYuPFrGUA-HOPXV~F5hH|WJry#V2KINjhAaO!C4^3FhWs8l!HX4 z7Nt6Y3#E4`&G*-4B>F#LXIR^g8bficl$K{;^l({s-4ixN2?{Zv$Fw=nhkk( zE`{w!9_X{z3G)A(B7cE}LoLVdwv=Kwcg_JXHOT?pp+)m5GAqSWU))9 zfX;VFGMWorTE=%G)@$~Qc8qCD5s;tws$0ytS?=OXGgy3(Yat&7uMsX~y53%%zVAh=llb)WR-rw&(!0ato zMq#8}Rp$V3Mj1$B@>j7x+$iCDhx17v(L$@RhM}L#-Rhi-S$hQC%+K$G^;h(of+(_> zYiO|kYMsX5Hrumr*?6I@dmhv_pc!|WaWG5P2WH(SEqhFK768H&$A}=ew3lD%*mhw~* zC^|H*-9Xt0_R0EA%*zp$D@nz~SAXwoXw8)OvASVb_witxbTig&dadG3lw-n~{^dY> z>K^N6CGn@e3NP94@Z&yFU{x5!mFn#Mn7;BUp%^U?lhUKLU}+q__wAwiXu*pSRalVpE8UIx21D^M`4p*~){VOSlipn{>WUdt{F_$jEg4+O7dR^nX$jr*8I zEdpTy7#Y}@UihB#zEwSeO^|y51tsJ1hWm5>^X{W~poK#|?7N3ZD)zhqbDilI3RV(V zFjj=v(f{NkYrnGTQa{@SuT;6U@|wryJa^>NuPxNx&pGq2X}Czptiww}lSMRPO(;rV z0U4y|*jYp869H0Xl_cz4^~Gz$%;Eq9-QO0eejILOG>4r_Fx0J@&S@1S&hGfH)Dyop z`Zp|R2)+eTTukmS0Ct&E^>3Ci5DnF1BHe^@qXul$H1P zUwC$6ebo>J@O{9+w{xxK6{MZFo#sg=OWOs0MqF+%E;D|Pk> z`@^XSFRr^0jU>uu!^+4t%!>L;iN|QI;&{&UUB_IQh?ooq%6>k?Y*D&A^)7}$S0rll zkA)!4$QtY-jKX^yFJyZR2{&z_R4ukk_x4kVmf?>)z&KCT65F+bq>hZ`@}%z3@(HK>VOw?N=5_8-BtC%A7g$Wr{swbTi<|idp!#1U=Cyr}l9!;oi1j zy0{L?sC_V^XnNM(Tm-~NIVxe3+MbqF%d6Bxgp=iEM zj0C2b%J=^AoXN~ha`o2lqP&QGx84#-5OuX(2fNT?_i~X-YvuD%vQceB>*_|~R+jb> zvcIb62IcV3xIKZ6t5 zxF2-s7hAT>E_`+P{gkch^Hak(FWk&ULb=0URKPpj-ZJqZLE*(6ZkdoN)ljxYW0@`AG2V~3+SM_nGn(u6KCc!7RyI*GB;Ox9Z=@7@ zV$5@;3O7J_lruUA`WWVX-h^`G8)ZeUV4%2NZn{q49qGCb-Ip0?14ryC%|6I7><*l} z{CmboyT2eLt2O&~mcg65?LoxoxvuLUpTY->#-ovG$nk6(3E3C6I3sIj9Ki+o+4Vv{PYqaxM#FjD4;?vKh2se}V(WMm;yLv@)co6S(ny9Ryecv3q3kj? zku|U$<5Cc(LOLSsRzw?uoPmKP7zzbHc6)$(scEHAOK(=cO&O;bbV&ia$#DF%!;Zva zf%|{iQO|mnv?Sp22~Z3?v+cXUtdyfBXH}_X-4hjcDC8gmu|d@y@kSlGC$mPxZB&sQ zkCM$El`tiCjHr#~ihL;aAf_c8y@afrWCuScS+QwTKQj>c@jgxR2L!sE8+)($uhXRn zI3}j>-vd5IyzL6`SMZ$yzaRq_JQ4K%eo*%8e9IXFhq?u9I&*T_ZB^RtqiEfZ_J2e( zy!p=l%)1Rbkh$D&p>|3;^J<6cQUHa(vjUs|M?xm*XU>hmnyk)aJ@`L zEn8|)+pUCOaf7aj-H3W{z~dh?r>1057Z$@$M}0o`F$Be1?iykpH$H%mt7>;|ICU(lm9(GBeDGnx{! z4$4%;v^d!q5*UX&KQR!)x&;$VRVL(iHe>XBWKRWg=`;4h3h089{s`aMu@u`nEFLKy ztd!Y~gy(9Gb=|FtglI#R1O$Jlusa7tuqj^WV^8tU$hHqObQfB1#bI65P{makm6$fz zcOh(f4BzHU=-_U0DKdOsJU-u%HU#gqH?`T&RuhbRbYkixsSpZYXFjI=>Ifu8^9J>y*>gucs{mFR=pm3M#=RLhjmHDpPK?*rVA~6Im;vm56 z&O*?sqGh|LAdVNs#x-2q_S>=@^)N5`^_{@f@>EaUfdj=w1#V2dE)iVX&$FE6K7$ZY zq&08phg}^*>k!yOr3gDlQOYbE1#t)zTmsgxdMsW}7Qm0eYfh}#o@9cxir@F+mf5Ch zPNXTKRNRv!xW3SGxjfM-JISdyfjdg2jT%M0&-SH|clcP4XnO?Wft{eGgdbPi!V^>% zKf@k^@NEh{;+WwMGs%Zc*yI=<3YHw^4C#l3KrZ@?a-&%D-o2S53F~ho?^O-c@HHSu zKj44a6gQD9@s+5Su8v-#?F+R2{(V__V{lVrL{^LN!0UO&crT<{WXX%^+OujvU?3e? z0$iv+( z=}fv{$oH5538;o-OrMX6M8SIEvh=+sHqYWFIJhgF$3?4e8nE0He-U@8&+Z4@Dlqdd z1qF8gnF9K-TJtFmjpcQ99qS9GZi>*tXKdVP{)dp_8;D~RQk}6&%Nhg|?9fbsoTbP& zj;jDmPAV{*%H9+hCvbA~67+ND_DU)Mqq9PT15N3(Zd$VY5?aYZS7}*aiY$dJG;mNp%4Czz=;h%>|g zyOX2>fMgNpI~WxMW<&3u??8F@>cH3_(WGnoVB8i)8`AUzN+UEWn?mYiB^pw!n$;LL zWGYL>*vpmQSLeE%+X~2uGmA!#NuLRIHSw-S1~4RP46&Oc;9t09-$OyxarK$PbQ4MJ zDCmTVS>bGqGi%+1v;cf~aZt<(rze6wvO1zgtD`49zuZRWVVh_~wWzlc8gB7Z8qwzb zD2$hazsDV$+U(aiAhZ?Em@m272*m&;kA7~aVEv>L>W1AL@qEdo`p_qn52u$WbaKlY z*^(ih4{=oish+cGpRwB6+~b!1Cd-a9eRE6a^T1*SA4Ds+J7VZf9_$>=7_KCdW1x;8 zq3aaICUty&dtlVa&gdp|{0YMX^X|U6zfDpxZbmo&Ce&t7oG(==-mCrvl( znfvkbI=#Fs3I_aL0H>Njx45n=xhF9C3TY$pt(Mme;9o;m zg8TcS*Vf?YPr+QhJ`%)V7#f1X0({5dABe8uVa%iNc&ZoQiQFmyUZ|#tsu64}_nW(! zEFp~#^2^rIZu0`kNeVs0bsj4QlIR;*xS0!tnE_xjk^)|O(Ku-!rZs}S0Qd|Z0F3i% zd5C>QtJM6S<75P89S_|+=-S>u8i<%o|8ciha|Yz70=RL}{Q5peLS2viE4FoSsMKWi zmzF_SvPh|1oOzS4pppjB*Z~#H8&q1&ab3^c)uI?f-(GYO$Rr=O*FY*41t9~v;?1TE zpX!AD^Fm@jw2Cv;_Fjx?G{_SOCG1NoYJX3UYxQURcj2Dc*{7$l?i*R7ukE!Ho+QTS`lvfF2G zZw#R5`ITeW>S4%Tid$H{fo@_`cn}OCR5Uxb zbH#e-rhq9sA%|9|L9*s?~K`c0A9ag&I6a`8IKb#cG}H`0p#@onxlo{)8{{WbreZi6;uDzq|5yAiA%c-Wc9mC zn_$eIW#i#AhM+2o$>t9n$VLToPPWNO9>m~dW#9NH7uzA{y(gqC1j}xqa>;JwKP78d z(u=@C1-;MEUE;j22VToL&r*LbtvEV+R!;G(zt5NmiX)g(w#4s-kS{)NETy(=lu|X| z68)qPc$tQNlg{A1mw%wifQH~P_^CEtRH@wDsHSOjCkrYX1T@>XFM;=Z14oUgSMAP! z@!%F)0bVT*s;ryI$pLUG^6%1RpFz;dS5E4_c?o>*K*Ljxo#H9shyWlifjEKx=0IeG z#B+R@T$z>Pgfp`UQ1ZiP2IgV@^5$I~tIE^nLhXir!gP7-de^ z{nXc{xIHSJI%xU4Ebr#z)o<7)8bQo$WP)w2tEfu&DY0ABqM~!`AY*Cc*?g1odTlzWQB!W3;nf@ zAM?>jyQAQP;`&cZ_BPJnM;{6*@2L%g6PJfa!`i~JPun^7v)Tkt^_VQ|?H2x3oxx+q zehJPoqOvOhZp4@B{~bJ#xa{`OZ|z0B&Kq?@YuYQ~O`o0ZEljEWL;%eS+8&VV<%mJ1 zF)4L_R@8SH?P;pD!RwxwVxJ+{pz{RiSiSr8=+0N1oACK@zS5s41q)DkA|t(jHsufj zDw096FrS=PoHT}$0c(1j7&94pdo-GyL$I!N9BJTL)M2#%d|*@1pG8UTy!UI z(!EyuR{n^doOB_I!)(yH`BFNDgHwU216ii7ux(HWG@*+>D>I#1{kkA;^BbQD);2*5 zKA>PWjT1A{4EJYsiq*dP>a2j|#o>1^ERP|)FVnUQ{w`$@G8=U*4xC(-eYoSc#3CR2De z+jMT*ZA!}Dt!Oubw?Az`L~`=2Kv10+a&8xaAJbdhUsmyY6#&7mdHK8=9jA3iTSC_a z+kkpd$HD!(eMc%1hZJTZY_`Z27(axsg=zn>o4AWd;AD4|(g9Xj6I3CPl?bWp6 zmWifuTKac9lJq0J$ylYWo}FWgh~qnJNga`N`cekG^%-WQ>9HM=erV17gty@U%-plw zyXojM*37KUu6gJ6j5LQt(Ee!p8RcV8usI&c8PnFRpU+j8r-mJa9}ZTw>V5>72d*s# zcm8$}rIVfk4Pc7?J-nLwbD-3Kz=PZ75K^AcmK_@E$tT8|l{5ZcZrFkw0pcTK6k;N5 z28!KhHaA4s>@Kxhx|kMUW1BQu9eu%YhcmE*AqCq4yi;Uh_k8Xo(C$&GLy~~8X!g## zPM3+=6#%pzACrY^=8_?0+MnR_Cim4;8>-V4v7M*!jNm`K(0*TBvgWGyT$;Y`)94Yq zkm;01X_OQ%Zq@64>n_Mc_JjZv4ggY=d|3aQY?2|k?9xahJKD{tLoh0iy@`~ZQK}Mr zJsK)taAmo*0npJ#55+2nEJ}b|*sMwF&HbyF!xiS?>YA;zHrJTEfGMq z5M?#*^11Z9OR3b@{IuhpIU6gQn2+hd8>p|l3$&jOjL_ztSn$(`wMXVongB3ZPYIg= zi=+JFabyA=kp6B|vdCKi=L~;dUo?OqNtWiRZ$yPBF-m_BrkhP@kJk62*)p9z`}q*v z5`u0E_EnI!o*C%7w87`bPq+hcZ|DR&1hX!KyGiuXh`N|=wFfHpQf$Z~gzMxx(MHUZ zsGaSW)6~L-oFSAQ#Z6K2K5_$ZyT{M~)JoEj4y1uX-ERY5_a!3nBc(H#!N5u~mx%MS zb|PP*L#}eikfxkD#;p;0bkE5KcW8tVooNaoOfgH2*^$DIXMC@XvID#A&UI<)qL_2E zA_PuYF6|;2kC!Y-^}L@}9_&@L8HZ2arge9f`CNlh2~!Ywp$u1IV1buCW4OS&fE2~B zjxGLDuJjXeY}}21(_-b{e4hTgN4t(@XOWJjH=aAot(|zOOs~s!#Nq3!7hhdu8?&E_ zShmB~(HlId7l%VL?h`XC4C-pvVFReq2%tC49 zcKi(h?<43d8P`X*D>RmN@qPR+DW0!*4H7f$@ZE$RPf!LIKEXA%w}yROWu_I~@Odt0 z)!Jb2B+SaEKP*OXTh#guw;}yKJhyDQj%wliXA2Yo_EbC&Xu{0XDa>J!-Od}T+AE^v zhs*RmWyz-$4AgZN_fD)zA4jqE$h#Q90NSn_?G1_qtx73jlLTvcudK1qEdtM`TU0XZ z)2$~k%ox2EQG9-E!)lPm@&N*aFSVZ0-9T>hO6MolG}7X`Z`X(z`0#JzWlJf40PL3U z$?AiBNhm=V(}W~WOEA0{avmwxm-|!Jm&xG#xd=JB3aqXo6Cj6e+@>5Bht=;_mp6h& zPS20u$Kua>!a#mUSBSIX3M_(7honf$#wLapsPO5j-uUy8huP}iUbbiV>3(r<{Gwuk zcKl~OwzP6p-kG313S$nEOc=>&U*oo>7p^0xyU|ed|A>d z`G;lNsQYUb8U@b{E;h%aITMC{v6}Gt;Z47p%Ybt#?Dttip-1m{4T!?szq)dhEsbhT=;A4b*-j|EW#;#Jonw4Ublp ztaKa4{&yFdfPuTj8Z&b%=qImP0pjevS#134PCGA>Vc+Tj;5hs+UUU}P36s!r)#SU4t9Q25W=!nKQFm7ni-cj3f?GdLwQ6MdD46H!z+_tWj&S^D%_I~ z$vx_;X-+>nv6#l^>>LsjF~;O|8sUrYzTLSK^~cLirk4rUJCx zA%|ctVKb;FCB-qXY&XfMZ)M0T@5YD+fJgv#*?eoCJgrzUL0?T&u7YnD{@)h0R_p=@ z>E6IZzft?X`2!%+*TCeMhVR}Fz6ALm6Y4&PL<37Mg^Smx$bdLV7oW%oUShUV#V*#> zF>z42VTYw{0WX=fMf}$O?Pne?Y0g!op#o5QZd zqf~hp&yJ$Q&N#JzsMppbjGib;p~3LNMJ6P@2H7sd?_8KcPicL>{|}%TpI*Ih4!D6b z5KE}z5ocjKPe?@!;YISG=2sB-)uw>`&w8v)GgbrIj2t$2kk8-@1w41SUY`p{y>7}Z z{N*$;J6-s4KTr?ymKYs!Mb;ZOm#`pbs#1@@l^$QtapP#~fF|d{ zc(T`H7AZFi6RxQyeXmw@HJ};sUxJ1Q!s*A@wqW}ge`_WPP&`0g-SgUhwhtIPQr$qo zk_LE>WgUOIeta&w;3>zeko17Exv@T85>aQqm#=a*@w6g?)~pnv0PPJOlNpD<&&o69 ztnJGRJ{pb3^XCy{M*9pwp{w;hdT`1X4-PC|edNvkmom%XhLVVBRiPxrV&n-){xCzp zTT&{5_KHtI%dA}J2`-ZjAO1pNmou8zP0R9Z0eUiHU-N>!*`o#@MxXU62=#(^@8qmyA*TIp6Vl$XhnM;U@q>nGX`6*Or|{SLklTi=X(%EWv#!A|{WI zz(9C@mOq0Z=~BnB*&pv6hvtx~u?HgcS-APvF!+hUCF}pIc$@FtD_$0N*vG@C6@T@v zgsgA>n(H~j)h{mx`P1jx(@_lLeqOtaRp~(ja_rj!aS%5S z%H^Q1KF33Dc<1|n5O^1CQHY_2ff2?{GjGlkGk|+Bp;Zva8~4jzNYqD@~_{gwQooRJIV;ec}i^>%wXiZ_@*SBEHQ@C=iopTzxhrZPBf#jE;-*^E} z{)Ifti91o3uGtc*iu%cs4i9SN5DbH>IV~os_5*6fpZW{5Z(nO2;%)hq?ve0) z=Wf+r5fpHW6<^-Oe|t0T1)bfn(BJ}h{+@b#+wxEz^o!nQD(f&TOuTvMni7`l&8guO zPViSnz!e(xXIqxJ5se=X@42eeZmy1%y0LD+J>#_MB6Ts?de~Nw7V%B**PHI;Z*p(wyZk9X^Kkhi+g)vO zs1EQ?LpvK5|5ERZ?$WfJ>CE0HP);z`5xeRd=}<5uZu!97>ie06Vbhoo0S`l= zd<)kV0$1aT^yf*9wHOhhXcZen+@1;cy| zto8*I(M}Cf?9X88k*F^wQFQpC<*YG+lY^L_0hSS z+2ERHb3a&HLI0 zOa|X1hvy$>ArJuTe>(oOuTQ{NEt9XGJJT!IR=A^t`|c{vPBpCgwQ|xYVmqP;25csZ0re1E65Wa$3@+4|C z@co_a;=h=`3r62^7U4)rkfySz;H?^J#h(Cpp5CR$Y#jGs#cVYf92hbmPI-sGi88bz zg!Gc~e3bpaz7if_8=2ydT-X*-{!glkw9YLiCnD8j+E?gxtJEG!tiuJuwSkEZs z#yTypCFr)2CRDa?KZKz7xN&i|>I&E_q(^KCnzZ`)4z~c$CzRI?=nX)JtKW{|qgQ3<`UFnZy649gzy@Tk~7mKfrTs0V7&wU;dB zx_WM=nqM(qKY=@6D%|l#QTjbCnxgObNFZqdnEXQW%ck#$jdUk9-y~Wq83+J@1Bm!) z)>|`g(NEl6^|lvryL>CN-u)Uw{}nzHaKhSVBg`tA2{Czj#|?NYdn11O&Yx#`iIb~W z&W#(XRP^usJh_A~#=pDk@h*xf{Vl)xuuPwv+PqoN&!R{(F{gIrdQ{lcw` zMStEuBEIK@(MgxO*h8=J03OgQ#KZN2K?4?SF8pWh*@WhH}m{CV4Xnmmjy zHN0m$S16vA9?$jNrq};g$kiq$;mvGNKco-Kj0nsx{QCG$$hTx&y6~gvSXTgsd1w^80FXDjTapmSH?njF3l{pwvy<+BaUqZ0s&*n1q zK;p{uF0q0{u^vjQhkeJ#(zi*}K;mYFlH2a2qP`$Po7Rzw4SDyvz9XYl7G4}fk^Mxt z0590f{}A$l)D^y^UvGlh7ZA6E+_Udqs6wAs0j$X|>^E6(3+CiYYWULnXdr>W;YLN{ zQmW3Nfe*GPia>nH@H$C!<}9)S*|KM>2~Tv$jAquOKbAm<}?e9U|ak< zw*R$c1`l$TW7$L?=ykOd`lKO&GG?#|YjnCQC9;14Ym~*QM7*86)ZbrS(b;zG+@$$M zFVU<4M?HDeCDn0ZS37-&|G+VGp;!huI8pLL{qLB;Sw&6Wuv+y zw+5x?pQvYwUo82o)J%(>s&l&O*P=RB!+)c^r*5{~?4lF4Fc-jAQu<3h<9|P=A!B4v z1cBn4u}yDOVZ_QU*XG5L5{ybw<|D^S$KsV0ewzyq6h4o`+O_rX?n`KO-zD8XRT|QH zj*QUp_0LXb4^JafRzs3Z#U}r)xF@_<-n1e!AO70M^#RXSfd%!m9*XS0NDfDAI9eZv z{|%^%le?AAq`4Xui@mB$|Fdg`rh}zL714R8Dm>=0ts%wgSxM>6K<{Vg@ESi;km>bAVrGkJSJ}WP%IwW|z=4Wvd;np0uwz>Ra$m82-f(6C!a93^sial7e7VKA zL73uaa7i$T8o5))pGs!Hyb$okag}){Y#XAUYMs6e=;$*g^R15Jglu!~@UVBOu1lFx z$t)a&K4|LxsX?F7KY_;Y^oV%x*p!Yyho{Zlqr5%~U!X>I;2OnH!a-z6U5LePyBanN z!S>rpD(+CQTNbAiV7dBF2&B)YK>XyEOJne@#jV*6@263sB=Wwh-g|#OXS918h5SB( zkp(e1GPBiWyaAUVzF>G6N5O?MA;o-hR+}&hVyC--*UR;4cjT+CxL%t_NplAkXV3R6 zHhJ2t&(4c3vu{B+O(Ie~i;O{Hp@_?qNAqiM;*Xg`g&Bta(_^J@WhPBG{O_$`xwl|L z8>bn=V)dl+XMZFmR^`!QSdR)r0hcaeQtUE#Lvdj|Aly|B8|&D|!^Rnb3G(-+Ii-Xb z3Ek#Xk#WU3yfB{pSvI43e5s-N>Yk@oTkgxRr%Fz|P}eneZntr^RIz`}zsqju6Xqtu zR~S2n%8O&~FwQW2_|0=5HH$If>TP75@Bahc6E+OQD`2$zh1SytOv%_?_^f1p{aQQ{ zr(+^4X6SF*(pl8{25%Eu)gxvYhM&% zU(w8x>8k3qe)N$Ox)MKzm1Dlt*MTJf^IkDCdQ*`I9N*yiiOCsFr`k%6^ zAO7-u4-4N(R(c(-fF^#QE?onSjnsi35NvFc!&w8G_|{hVg{d~&@u6*SF+Xf0B!Xn$ zsDvoUKd0yo^d4|`5b%UiAiK+d>-H&l53y6hwU=kpM)K)%9Kt*#H1&oCOffS?BbAoQ zMa)`TtJlX{Oba{G?rZuMdF)JOkMiIk?9>%piD55UwL3pSN1gI5$xzSSfq3xwMQdO{ zIHZ**nxF*m6W@Rz(J)qYw*)!cjZ`146gEk(uCWb|=7K#;Y`bOQ)llu7%F0HEPh80| zH|xELrD&h3RY)Bd^L|zTqp>>aJ|trs)W`oy(kEuRbo`mxaYe1`G&55(-Na{oj2z=d zD;4S{gWFxj1FEb#XlDGeyIudarvy=%EyKFF)8FtV#<8zb=#9``bGYDKdnhu$brr%1 zON#teZC7vSyI4;QY1|4gUZnn2?{f5oDmC{d%*QG0ghl15H!7qu>Gh;$gBVpse|nTT zLt^r`XN7mP?9EF7SdA5rUtikO)7OTp#*ONlOG4AnvZ9wwI+S~rt1}iC!-57n_G7t) z@a-Ski;^#eNfDyBPt^~IeAAi-(lEpS)O@T5_g%T(mryQ>m^z+3gxFigMp zEW_jMZQ$>-U_kLR@k~FS%)5u}S&L@Oi8pueceQgI;f3^?~4NL1AsQi`iMI6$uSfG z{!?SSBG_yvo!m2l1iH^5i=fLU=1oK2`D9t;A3pY9UkUyf)5LO<9UAoCsum#CfX6T0 z9G~YjxrL2!)o)MF1cx}toYovG`KKYKU_}8=W?#)wifPdBKVJ#`Fca(w8l!Ve-5bSz zwLF8%>3ydB5ll%te)f@Y+0L9LeC*wqB)Hc9k^WKw8g*z8+iwamG7Bi#AWP}KP{#5< z4xUJof`aS%j-cpXSRCF$k#pwyFz(mQiUw9QG&)7nb?Pys6)RGki6Sr;avoV_T2xis zDc^i6G2}L=orGfs5{Mn97BqZt3PMSY^K?r_(0P|%>yBBlAr8Blp&siWIGW2dcQ$wc zK=`gpTEtilB?O<++;dFENNKTVYk%CMbN>2jBN2bo49Qo%NJ+*dQwvl9peimxVA+$j z%s;v#2FQ(tQen-O1UVJTBT{pOfWkr^5YJ${Oz%4cHi;n>So`7pULZk$x@l$tvuXqs zF(MT8WkX9YmIifajmf)V)V!2^a*Lf`uE6H_SOo~YbmWs~g`30~ppx-_9$Titj-@ZbD*9si~oSaQ$^`t~iJj)Mj>Vq)zz;J(lbaCHxaACXo z`oU|%sf&oBqE+`{`Cn6RijR_E`KBKi^W$|(0d$Dt)CMmNiO1m(|62zO^$Ywr1RQ)e zs6kr5T?tgA>!sH?a$xVGaRq%)DFcsxEw)S5Smn zZ~JA>qyz%7aD_!MGMzI(q@ZC6WTU_qdlI`0Y&2*Tjo~+`3CjJX!di;f1vneVL)HI| zuB|YAOe8p<30~z(5QP_bO2K1PX#s>@;OaPp$Wg)y7Z0Q>U@t`=yE8`<;Ar``=yP!5 z;Xb%z|B*F8`gz=HSq5%Kim>S?L@CaA)vtc^K@m2CNx;INf?PzJfm5`Ado|x7B&gKD z0Nt)?0ayduHI30+e_bmI-*8~7(O;XOyoDy^cH8OYh6UNQ8yjaP6FgkwIr$R*_-7R7 zQMP>vK>E|YUYnSSqo6?ry&owz7DG>12l35o&~_r*PZ6PqpGj6T?67zHK4jkWTSgKF?Wzc z#&Yqs`5&zo_?0&xyCM3&KX?r!VDdhDY`U-W?Uqbn{^?0l4h>i#mf z|8;Q)MQ9A&({%(m#Ean@kRJH?V|IR0A=!w-a`FhBuS>@#qf|3az-?E1b#EQJtg zf9$Cf>v%31uDry-8lYi0H~ctJF)v)fhM0#|u-ee|$y_x}ipbeGHc~W1ZEORY_)L*% zQ=4P7U| za^In)=_pFQbT6q5dP#u8I=;LecR@OMg-!Irt%QGQJRyLUYDHA4c-V1~To5W>y?N4= zBZ!o8k>^OP1_*(DfI0aMT&odADcdtfE9e2jZoqR59*PSS;REFCgu|X!vo8O6y;f0X z+ISCiTks*xIaSE#<+O|1s4D&PZT88JRn8a^epWfV)THSMH}MR`3=wmEFzcOHXU|z{ z@}VB!iQlJ+m%*gcbY9rFIv@Vd1HH4(vLK3fJ3oG^UzXs>JQ)I|#1=aN_Z{ZJW~i2k z&b10@k zU7;j=(z-0ukP0p<$ICK&NX2NREWYt#0$4})HdMdEb+Mo`0}paG$^`r_?``smFagxL z1~^L*>JeO^r`SDePj@XAflvbpqF|?eO6>J5_wZpg^w=7lR*LgFkc54K-U>t*;n7}XS9Kfud<3QTJ6xfAxF3z5ah-$(i-7LP zKke|p=S-70UmV9F&z~S`JVw0S19%`zf!PA}K}p()j<*iV^6|tE7!nVdhW+3; z5@7LOP!wT-(?&p?o>6Rho0!L}$qTmBHP&-Kj6s(9o~39Lh8I7OD=(a0u*p>SJ|_#O z4@pgqI66EE!vFo5)xc%@uwq~*qYZ|R*S7sM(_JwHNHJkWT|~hZ%O>q>&BETINS6WE zu80oBUx>Z|kUjXn*HDM>2<^+#K=ADCt@KA=s8A!NUjz?$h`$56meOJ(qQ@lRLqs}c zjRJ>b>3fO^B~W5OetHNG{H=1Mi)9Ei88X)MfuxAwdAvpGN+ybXU>6sEH_oZM+h;yf zQX{+4ZZH!T=J~+ltoj48P+S*5=y?z?HX$Ls(-U}jU2A=lCNFpJ=?RsN79V~31MSkd zm;|-tQ0VFZ+8k^McE-KNekivpdICd7NCNSHm9xOSd0WMzOx!QAO>%i}%z9yOiH`Kv z@cMYoJbBf`L+HKk7lZX&{!VDHXUtzZ7S7B%%-#7P@H`j1m&01B;1lqIx01-#4Y&W$ z)?J-j1#JKWXIu~#&RBo|#-5Y)ti4MX&PhKh=tsl+IZC1j8xyd+bX}Z&?nI}@3}}(i zc^-rw$gG7a{eClDJL~!$d<1q6ISFR*xV~iJ=pstL5hyNodsjfG3QAy{$X$?R%VnE- z45=oC@77P_{lZH^D%>P)#O2W{Q%MwG^LPJ<*5n98s%(hADgmA2rwyW{kY$PJWyWJpK`C$Is=;aO-!T#RyIfsAJd~}Ha4U|s00IKB4 zu6hQ>Ns#AZRZ6s}Ox3#7Z;IFjxl!^xR;A91q{cqw63|77v%q%QKj+=VpbYH?ua?<) zSaUrS3?-vv!$u#1zs#+>zd^NSKN)t@X}-=t+NTgQfCV~?4C;Rh>TDWa<4JC@S$oxb4a zyu9-`Bu`RE>7o^w0$MddB?^<@j<$TCl$v_0RA#Yamo z(WXDNViEg>@6k7ZIzLpemB^P8wPx%?z0lpY(O-i-btz_Zl`vHxG*vc$(e)uLIb}ir zM-ez2({xb1MMYJDKl$r1;Clo^zO^R7$+aD|Tl;&UcDNtXXa!GByGSQ^M>Z5t3CJ5BgKNT_4BJ0%K#R}p z(NJhSU%h9c+WwGOkMnZ%(+3-MU%KnRAoA-<%rX2H0_N6+NneyJxpar2l;fW(`}_Mc zmjQd!HdIA~UQfN@^nXCPff%=ivUUa@E}+&wKZPxcD*rQYU@;^Czgfn!djzC`B51&O@A8h&+foMnNb#a(XoiQ=ZSv=lR?3w<5#$ZK zv8SbP~{sgk`rEVRVLlyOx3ut+gcF zRCgS`gsN7vx98L+QW0Jqxai+de;x+#H2r$E*T-3Pg>@rmg5yQEAcXh2&C{KvnP9vg zQ>RGJE>Wp$xf^lP@f9EP$7_7 zo2@?_f3v8HIeq`$J;RfrGL!$Njx)|h!l*NIHh|(K`gi-M5=H#^&?GWVB}Q)}Z|L*f zSaI<<_qaeJ^6m?%^o$E>tjv^I>}BTtw|OMaytTK{%O{H#uIhvfy>oD0KB$go=_Dd{ z_uIqiE;8X!H3pnTmj8do{)fIG*IihNe}8Uo@ISy`pr=eHaGyT@O)r!Wj=lshsW=32 z7ZC3bYk54CYt z1zYhGR>|VqwFrbX%F)-pQ5FpWpu_Y%0&m3q)Y(G5*tAiUP338iwK?el&A8p}k0hw| z-*?med&eHnTL#UO>+vK$Xfz+S+w;dIqg%rug+n=1>u z#368XWQFx6_GC`kdwsyV)Tw%kXYXLV zqAxd59iR8RZ~OSFXKSjs)4ii5EaMef*(W84iyUS_*PrET@b|HIJslqFja`3@)#_?- z10n1o%_2r~+d8Tm^x96H*|Q2GMS3j5Mq7;HzK6enE^S1vk4`-=XyO5Ek%iWoo^*ZQ`MYQ<@8A8iZ6 zO@tlEiJhE3($LDrf4M{2++^`{YY3L~P@1#5goMoCV!cU#eS zF@e}*&XYOLXRVYB3z6*0akd7ciZ*=t6aPbSyc7tuNpfzDewYlLouoz8Z$b}xZ9*XD ze&!nye9&yv2h^GV{^#*V!}HQ%Mq-qw6L|EW>>VfR^AkB?Y1GyuK=sgW(s^hOA+K25^-nHzn?-S2nPJ;)0SK>x)ppU|(H z{Of~i{!C^#5#1}=`o%%pzUqpL5o;8$4geK2o`|(PckyF#v|L_I`P=zA&z@341uN+_ z_B-qXMWYWMoGj(KK44!Jz-lP^pQVI6dLx5c_dN>Tl6=Fr=gNL4p^W@!sP>kv-H5&1 z@ZRxr9zazav!~?2Huq?WpO_pG6Lz#EeC*ziUJefp3${fqm$MkYd$v`t{J+cD{Nj7Y z+Ydd(zN;EEs<6KKD(jb>F6DRpyUD=v56o~Jl6Q|4WFKxidy~hKEP+FOn^<6zXL|d@ zt2#daD@6R0K{=LBxSNudpD*!VW5%!Tg|J7l>f-idk3~j4M)_77x>tg>CHv~937r^RjgmAqg&louu0($uef1a7w-6R#C`LaF;_!N&x}L>; zN_zelC%^mbFZz0af|1T-o!yhq`fG*Dj9ip&u+)Pzb_ zV6CY|a;o}*?B}9GEa4C0p6d(p_Yu^S;xY<+NDFh(pUV^d$Fxa6<6o+6pBFuj)v(F9 zEi9vr#;FqgKdRmY9Lo0X17?~ugv?k%izR!AP(m6q_U!w!wt(4_?N~1c#r^kyKBjt|I%vK1#c~M;;vt2aYj0 zG7bJ8`a;Au@#m;Yeow#B(aMMmwa2f;n!aOoy4Ib39RMQTj5iPXB~Bj1{$(dmNwh`3 zYx@Gb9pC!I@1RiD^1HcxFlDX59NvRaF8l2+>HIIKgW@_Ol)qUDNcmLUajLBzSMa$l zp}CI6_N<1*CoZyUvHVbTS8j~UxYihvH6le2T=cqz75E`{eXe`r&)ZRM21N{H>C$9p z@TBJUku^zNnKs2d=p=^zVPHTJWd}0PBXMlFLSFbO>f6ysynt?;g)8llF~D2JuY1PX zpHSh{Zo#an#tti>?L!I)DuWA0aW?&VR9F|&>x&l)BZ)=0GRtQv2@BBgK7g>BigF_A5RAeE58n*ZD-{Ic= z`SqTy40q%oyic?CV=)DgkF1q^Bff}o|2He7G zD5G?>NH9nz7CY43KH@@q%;t_L(e0e2+-aHbOMmSD!)c8nxSNo(WCxR-X#I2u$nH>A zWc=spf>~%E+h?_}iDt~D_zawccwXW;b~2W`G;kVWI<$e>aat|i z2_xDVfO2z9-l*|s%@s1R%L-#X#<(?lfiP<)$89hK_u+OnhO}X?#KQ~7Kf1FMRo|O$ za`r@Ln;i&8qXv3QEn&MeOao^;rdz?+5neE#zux;)K#&E6o*5`5r~%jv1k$G15%MQd z0cr3m!q8g1*z}|SMdAfHAvH#8z!P5^@32o?#xz~f&XW0A^@9F~DLvgOj3}e@Z1{@-yC_pSqe zx&p%(OeX$yT(qBof{CSW0T~r+K<<#Ix^nRY+2(a>{?9}TUSy76J3HDv{-L7tvIl

Q5N*J=DxH{kqcyns9Z4R` zf{2(Yx#WYo=`v6i(vxKn_b*P+C{%uhj{2;5aX!p-1>L9+zlY`!SV1&_z~sbi^H zA^%jrt^&ws@-MSHEWR01WT^ z7vY_J40>vxAbCEiqDA1y1SEP4NS+2 zI22=nBlALzsgi9aOU~rwcj}?Kkb!o|j6vR0lqGk2ExAvzAis3(Yl+?uywnB)=ifjQ z>F|nJ1yM8d33U3%F*ly z4Y@wPkeTMA44f=-*Qu~**iuj(SDCCu1B|}E+dBYYwL31!on17Xy;0=*M{ybk=v}B8 z-=BSN4jph5QKum9k(+TRDVKD7zYKOF>MKN!*ZE80imDA=tXr1JrC07)!V=2S`Q$+5=X8X-jpSDUBGK^%cMZ@Xba)94oL*V)l$-s-Z(nsft!Z*r1Ar`pf zw;-dhy+SC`FYE`%NIBQ~#y#Edb<|ZdryaWEph%z=`8sXAi*Q*i5{fLJokNg39j5rPG6ESj`h+ zH#eMzloI3w5!|~{AhNN$R`^+b441DI|@um$Z1E=k#B=B zPRgb=2nIJSX+ABphNV--87MqXRcsZTt^yh&iEg=sreH4WNPbUZ85eJy%?Mp_%8af? zag=68FE}Q%3F&|pviI_&-PdH)UJnmKaIB;pAjjr9t*h)x;uh72LYJ#nzel`+djYklKOcj0tuf(~7{ZMJm5y*QY#}>G~-!A%;PWuWQ ziNTn6Hi1wR>5?vSg9s<>BGGZ6Fa4lw!p8TH$kMxFG#71XB<>8TSk~-!0Gwkx&3G*; z_|f9mO@Ulw8G?+~;lsTpV0bW5FXgqoCcD~UG(zcJPosdu&cDm3lq$7_rpOE^ABZ<{X|7FpOHX--U|DJP z{K21OfPD+&c&X*#U3|p#eV8 zS`j)I(2xv>zdXvAMLHr$vc4!P8v$2V97iw^;V0FM0zGO!BBwAgEAf029c=@r7^u^w zNw@z<;bUUI*0~SZ$DUWt>G(f3yJ*v=hc{KHS5Mp8_tAPuX$5E{ndoo(vZI+kATwgn ziFRbWpmfnY^`!Dx1eE*LWMeDTC|Y>cMfDE64<7dd0qUnqr+(VWN$CM&?Zk0}^fxLf z7|W-X>6~NmJ0jFDwMP2)vG;-a)ogU*c0FHDL$53A6}zc}IrzyFt$phc*1*B|OcFa| zaO8}eA5>32%nW{%ouqAj#sw`dRr>>8{1@aGj3bD>dahc*>RUnl-~u$1V_^aHRqm@I zOCVRTj0ZJ(1GNM)Q$h={h@$?6j}McND8`Avsed;;L`?`Vqt?xD<7?NPA>}h(2@QUM z8N{K!m2gTQqePbpd4=cX>mI2p!V=0SP3IVYZa5+jR$rJkdKZI22n{)v!YK1Ley|J5 z>WyVx1?Fder@Q3f2W}%bbUi(Y`a^F`Qhh_JWby?6j5UP3SH>zTPs8L$m#z#MByyas z>@PfD9(y@ND@yVU_Y))ET@53~6h{RDlm{1v15ZPC^ZE-Z@Ld5Z*~xK!cjpIWGqAU) z?2T*;+N}Z~xkactv@ZpYLlRtTHE;tBzoo=nk^FK}pAwuUfg7Yg{wd_F@}+3~_UUtB zJ->VSA1dh={xP_ajn}D=R?$I;s+f&DEfkrAAwcn;@abigE~=jw)Bvx~`IqlVJ6g{{ zJD$!xIApA62h23erS0TiseEoU^Kgw=D*~sDND+0RIEvu==cjHIrGHOi;VE~#WXX9q zNUU_X--rBSlGNdgC4q;>7`8mBPDLc@Li^$hs_M1y^@`(BGt(Yhq54-tH4@7-h3P*T z^61Ck@q51f>J_swsB%vo_<2P*`P+}ErY>g4M%nSOfd0eibqui6T^xhH0@P5vSA+J^ ztCq4`IZ@xG{mRt0291;I*wBwNZ-+v-rh?NmbCAPk%!<28e;E*-%vt|m~RhVI_o%jF^1o)VU`=Q6VV%OK3d6= zB?{svm927*2tz~7n zaL{r0U)0AJXPS>GtX+zKviQ;Kp4#X6mR-WU9!H*Lko(h5hhDO+)n4zM*-9BOrMy~J zbDJ_tI^q}~1}I>GC?}+P%+OZvQsbQBbRXodz<_xD3s;NR&y8dbwr{9~9$*KMMVq=#ba@u4A_Y14hM z=AFsBIiTTeg~!B2plH=emlG`=dVJKWOTfL3_@6n*E)Y!7E6=n$zwpRLA=b!f>lI_d zZkPG1?0{5 z@u%PSAGU}i-`~ruM`<^l()}QnuLZg~s1u?If7npiN13}58XjT-OrF?tiEX|+* z69EQ}F(;F}5p~LmRnzpi+5AnrhD^56LRxg!>S818fxJ$M~X@CLKYSjK12vKWPeZze>z2 z=!Fy2;bo-~V(EczaUcdYslt_%UPfZ$JQz3x^k5UT3#<{42k;zfv<(kfv#ddHb*IwQ zS!xTSb-IrZK*RplRiG0=4?Yna%@45)Yx+Etq-l4|oS5FIyyZc7@ePce?q^d%Cbo4C z%`>(pE`n0ORW-S3+?fp()aSgr2|o|1}#o5k{$^MehJ0Tp`Q-IpnwStb{nY*ewY!m|NQz3!}8BK zE2oN*rps@Rzp1b?X4YNcW1Kc>fM1n{S;*vb$|aWfw?@5Cvxe)uxgiiYm)NrIAOpd+5FWm* z%8({?g~e%FwMd9}moEr^>~I2rQfhKaoRZinP%IE-FWhtDo-oRXAfIc8a{r-y{NA7I zTpo}`vmW9P8D^gck%(Ulugp*=kCg%g7a<@$MikiO@P2$fs2>WoAN~Uoip+3<#Cf!l z1z5-P(W##VUd&d;yGrW|>g*~8!%*%f0-kP$_uqf_KN5{#PDaq4c0cAHo#d(lp8=?k zQgHz{8jUOO5(FT79VCCy3)v3Mm0BXtHeu!gA&Tn*n}nAhn7K2cb_$wNRF{**LVov^ z9Io-0IoG{RBlP3?zbC1|92-6fF?f&6pO;$Sp`P&jbl)I@kH0|obb`YQm|_d4>Y$lr z1=QmW4mE((SKd04NwuC$@s(2g>4VL$pUxjZ!AnjAyiQEeVn;uqr?$tU|Byy?AvC9E zi<&`eu=ZJ1 z^=hVxwMmZp8+SyLB2Err6XpI9OrE$h>qhSW8E)BOzL>Y5I1M~0G_)S@@S-uV7JoeF zCLoP)N8e}TC|5d(9~4NxAu6)we-$94X{f<~Y?U~(D%t=(z!iQ9GH2s@)$Gy~r$M>F z-|GXpV3+W!*Znk5(Gom_v+S$BBO7^wtRm<01fJwfjl#$hb~_h#9)nObI$?=xzz1|l zs1xkmdKF`Q+n);L*-HE7!n14nB=G)DL!*2!XgTf5Eg-@*$PofGI{4v2JbxJ3ni5cR&CHI*=4Tdn|S*;q}zJZvKN? zI~E+BThJGEm?<%9c`+G8O18|;LR$ncF3mdc3dnhb85pGhd~ab8AYhos{{H<1h|hW- zi?a{yO(M0V3N;5i#w2Ragz3R(pBCP#g|6u_bQD`)O+G5+YT&E>1u64j;M76m^*fq_ zDn|j~64uw{e3Wc@LNcgB61>J;eJ^jMpE~_d7P0}mVVkmTQn3Jo+(k|YnB6ffupR`P zcyjMxa4K_RQJt>c<6_tafM6goL)SMg+?vM8EGjKYY0O-eT+AKSJh01hEu!yE%p0?ErvCYX_yuNR-B+Tzp|6Tn4h0k8et5P-Nr&ZvWyZ z;g!i~tRCp8oqA6qU;bU9IKX|HQTMVJOxyrC#eZ_`sEmgBCayIk+MY?FH;Nmp@zy_i z@Hh%-ho25+m>5ZZD)8!Xl{fS!s){)PGx5l3UvK?x2>^r}q@t{lVDCcI#~SB=@3;VP zV1N#z?Je8r(+_VQ_Fq5dw7pNC;l)>DjNF5fDJB)H}B{q_WY_~5kW#iB&YN+4`XjxYQt?V_J z+fCw0w1K2OG%3PYXfAYA!J;h0 zK=8AOdL#+`nM$0k*s5+h&}lmfwPsL7j2O<2^^JO3l)&r+MUD&_ub5wD^v8HShAnRt zCnevJd+`pn5lDd!kXsGVjd2icF_nbb{5==u3leMeF6JRR=7UlB2JB>j{LxD#Hn^2< zugsIYolwOC5lf5k5-TPE(O?zI?6{sp4Ad};p(pnbGID;RUEyMN9Xe}x5M68mW;|58 z3J^i&`CGmKP3=CG{k_ZOm#12hwuwqkh^o{&Gu{wxDR*3_pZfyyMGtpl9<#v2QnW?n zw&A8!uLC@AK&gi(5f_@&3ieC2Kn}gn`AEu1Y`qhlMV}Gdd#_>3X?Zb@I;plh*+r)i z{hQMmO3Xc||FX92SeoodV$9+-WhBvQ<5cuN{`bd*7ERBz0!3F>_$M+a(~Xb(E3q=s z@FGt%`~zs%;;59jC941G;G;b*m$r({@am8hhA)ZOhpg9Iu$ku z2R0cP9Oz8s1}9(7MWiS`S2AP{-eQ#YU+4M3PDKVx!*wx++T9lsVFj0O6eVO(&V_OH z$=%Ko(VHNPjhm(Z&ZJN$LojU(H2C0yS;!h$cw+by&g_KJG&I&eiPHa49YGfj@pIQe z(+9(ARc;$==$`am=EbiI#gC=rAW#m9nr=wG>TLUQ|E7W!X<`AQzL+{H$ekClDN6_KI(6YQ(s!!^h4l8#3a;;dm-EVhOZ5DlV{hI z@-zw!Ci6<&pcf%GS6{qGKQLMP4E=en3^oUpck)d(okSmlDh>z~xL>|NsPvbyC#4Le zOF?Jpd_{_Ma@*le7k)7==Ji8ig4>@QHu|Bl?+taaby=mZB4Y-P z{eCy1T`yl)6D9qI=W8p22sA`*^2uL78!fy_j(Pybhd6(_uE&)y;DhO+y*;_5lp}U* zCa9}of4x1RE;5lj;wmzLz~t2roPWA_wded*F0}+@ik}R16ntg3L0k+f_{RXVLsF1{ z3#NE_4(|tQC+FKPQAoltMqcn7@^1%xp4CDZo5tbTH^JjK$YQEVmIYmd9{{>l1VtqH zBwPBl!rjidXoQG`?Ns!>8mUt+;glZ*g`4;n2TsOT{@0Z-`$={g!NWOPZGV7%jAG6` zZTwb2dLaR=??mjZIt4FYVJJM?$$(^JDkrLoaJgL3IH;<0V9U;)%>1({NSYtdywDz1 zAadO{Pc=W309_eH9SJKUQ9wS|bw1ee%7+R@p?fr52PgydCb+`$SD7h4Jlc}>VQVgN zP!xJ$@*KRdPzG#SuF*IQRltNV?i~w@CZH{%kHaLAVI0NsEM0NNxD8%DTi5aliIEMz zb%}Eo`p2+lai4xd5_@bcWD^u>B^&C)g`dheQtx?!U`NN!$senlIe;Pjl+whQaI2IH z(LBaMI!--F)SWZp@C}TTF!we5S~>Mh&%NjSOfGef39RfdrGRJb9cLAseCR>(3dF{4 z+B4GZPl3m?DRL~1)Vz!26YD`#+`m&Gbp3RMWLSaj(IN27C#t${to=mw)UnS;V>xBj#~nF2CSY{# z)>#fkN9yn!8QKX^L8qu!ALU-nx;iJwbn5bz3a&WgW=QB!=mCkuDGW?LK zRi*45{;*#LCH;fnVynCW`G{(``}C|_A^1Vsf;V}oUyE#*vS>BrC`Ax*PYsRI5;(Uy z_a8WX# zC?DmJy00$+m|%O?WzSa=&qmeF=O*hC=fpXK)KND18Q8wR`7;aX1g#~VzXMr#J3Q0tJ7&>c8E72Fm)rlK?r00`c)GU@~`F^Bsr%R6O1GM-1u zx;emzOpjI?sJ+TTScww5h8N+L80C`UC$`JhM)S+ zSCks~A4nY$lC=ylRqFg0{HM3bl$$F1?cz$+{r2rlHDtIfk6q-KWwN3AMH!*$PuP6yv8IHs%qmO zkWcgmlKGxp7-#fZ=E*5~qQ#f5YM#W$!utpt+?Ou#ee6k;Npk;KP0g#X*MT+3Dooll z{UEykiFh^Wg>`ctU$eDOg}5O2PA}D~>4A*hfcP;}Yam(0xvD@t54E9#=cMD_-}L15 z0Q9>Cj};vkxNEYiJiH5&I^mr=gyrp{qHWEUt|-{%^qLY6(1}VjRv06voiMzMb}Vtu z813^vncS&qHZq)75HOboY^`o1x5W;&z={^JKn1$jfxK<*gX!M5>5gI83t$y8+B6aP z1&yNuj%)&Qzc-bRMG?DD6DK5bzhZl%a&E7e#o#1?8-Wpm3CvrM0kXu0U4}&}FjoS- zsEtuK#y9hLh*B|4m9S!du*95rzQFSfVYv)eu43h&`=urqoeGbGeN3?>OIDKOBL zSAdcgCT}(eEKg7?P+-U6$dz0;XKEP)do&BgLaJPtpZ$SNCbsSC(9H+AP?;b03;>+k zznUdG@i(v%w_%b%#UF3fvMqweKct;?Kc1olcu^zhy3PJeIty<`% z6sV!v$@&#$6z@l&5HX4{PX_XPD^IuPbnPv+ce zH8i#)z2(^Xiozqk0gyHPdiD75^-oC+Zc(#XqtU%th#NP3^8-LhkHv-Fv;eJ_I0*BT zQc{+kW2UqxCNB-1PBMS{wqU0#2jb{+<$Vb$noDzv{p_{nFHfEbe61XBBHDm$0*>xS zh}^jz_7@F%1>Ztf3OI)Nj0wqmc2#bMW0ZDpsxO}O%@dBs%h06ymz6;1Je%(V>xIAo zZ-2gDqApPhz19WLRD}~auz==1@msT7d6OcN(Gw;nI?}?>8jBn--1xoJUyEZnkR{0L zsmvsCT13vJ5JKLxY2>rcuopDmK~uHEiP}ltd0yI8I$&?dvUAe_(()0N$=F@bl}NCX zYAer~MQ;dBBP=YuyFd6a%gXEXD^m7IB!MM)*eAqNe!cqybf)(W6{vev#>k<$Mha*# zWI*gOiTv7K@AhkRXduCL;gcPs99O~5DlRJg`SFX)dXSn3#&R)Gn77PWun22DF(3su zZA?mGaYDgt^B{qM``sTx75GVDAjBCsL@Jvrxt*eA{dn_hIXtrbq~pwcM!jY(RLGJO zqV3?LLqM$^Kz^Ps;VyMXe8q@ip>jr%_vrIQ)^+UF3{aF~iv!~vHbPkr1J9;`oApP( zPjqEgQo2v=d$zEGIXq$BanHy zg8$j-Dn`v9hkN`QY)SlmU+sO^*2+E4d%d^k{bZO}C-8|j`HnVjd7>flT%ER1o~x2- zP^TBI%84eyo>a!y><7)=qA+u)>;jaSZKFZnx9Jio{s}7a(*BrbYFGxPXPfH-5Y5>X zCaoO7bVeP1I$`v1n@v+8341`UA_Iu;sK&3X zvQL01l_9nB6H~9yayTJ2Gcm2OmZTP1NuDIxK*S$vlYHioAQ{13b?etPyDYwv+L5Fm z0H+cb+Sv|Cxlc6#^2496>lUf@+`^-LeMPx&%~^EzI|3Qkvd7Pz)W)ld1p|##>o$5F z#D%+CFE0=*ydQK{pJ;=-nNKtAEd;=T_ip_5&|s5B#DCC(3xQSASqCh!@~}G<80CC_ z1q((XNnR{HkdVil<-a_BYZ*YV@GD|K~IBqWH!g-qc_L=ZILfausK=G&cTW+IH zxC&PYYdm{~V81Ymx^$pYtwH+c`RLBKCj9rtwC6e_mD7Mi)uuiHSK^t6#jx4t{J_bV zgDn091w5B0sB=W@kUKBx59eXf*6tFgqamry_fSvfIj{cAbCZ+K30l}euAtBTfFeJE zeFtYAsVxkKW{Lvo^H5QNH#D7Q&!|^ReN@&rTAH@0RIHm{+>PN{zd$tbjITs#O#YbH z$irCH?fY74*hKKSOdS7%d+ZwIuGFAf(%R?J2@>#67HUDqI>h5ZO?8@Hyn;zbF@mT} zZL_;Om#AG+o}pntVk?hb*$WfLZ$(kqvgr&Nl$qeIKk<}AGTbxqyLgwD>w{EeE@ZTR zZw=lDy0fdGPZiSL0L7G-|q%DOzmr9$9964DZHpNKym(03tl8 zt?3e%O^{pL%lCeYayK*qSGG(@wkS4e#C1fp0@VFRnFT(+y~++)Q9Kd$8Ymxq~GvD47>!UtV8Sc!b(a$uz1AvZ!LT`4Qrqu4gg9iT_$UjXmjBo4f z;tbXP1Fs5lDx!xxBM7_`b-PYVlh9y>d%#9OMzMb9L6+Z5>4xX=(i*MU6Po*3(N+ww zW*tKHj}uRn)oQPQR;TkWe!Ecu#4du~4Uvpu+Y3VfY(kzX0FLqXU2)>H;wRlq_>6K& zwh&eq!Q--;9XRbk5ZFV)+zR(QGbplbXM>@SVHr2LBXKSbQ#eT8&FMsD?=l16TZ z$X56Qb(0z$<3$S^hLKtq#+k+NrF9@YZ$52&I_V1mjzdi+Au#pj4JR1ys$abh`^^v= zG}&e;kcMWIqAzOi&#EM)OO)JX071SB8*YZaN5H%acKsHT)yCVLT!U$57Z?zY6ydd` zX=L}rGPHNk0jP!^;h(l^BL4seS>wv2ToT17Hjnb>TGAt^< z`90G{|1n(jH&v{_^-s<10R>@N+U<4?a`{b6RzJj7F&YF!N6sjCLFhoIfP~{7elC8XYCcyypHQ{NPt3W}WISsfqwXOX3?#Ls&wE2sB$I;e2Kc&i z#jY3U*oOPSaU;B&2bU4eC$onZ?S$Jve7Zb?+Xga_(0JMnl}@I%#hQY&Ih#A$uQ~^0qY~uC)ccxbinCowCx@ley(uJN9 zf%Y&@Ze5i1!#J#=Z9DPdnPIrJF1&zbn+C3Q_d=y*zZrnJ`k{yH%W8`69FgbaeGs?gPdIg6ki4hJXz`_FB-YwX}i= zokASt-Is7}KAUhjq<>5v2%-)3JOSr;V>9*ZkwG`8k0R0@L5CH+zgiV5{{K+FgXwI8 z5M}ssUIaQFAzi^WXl~XNJ2Vi}hsFKCsj#U)>^Oax#=S6|hy!-VNWrL7d55I^;-~Vy!s- z5<_Y<^QR0pQkU4`HK;ApglWS~P_f6@Q;NT4=N*6|@DJt&cEkAOv)?sf8{r5@TI>qy ze^coHf|0M@v|Y?G@v3fGjyD>0{)RdLoEhj=xDmQ<;E}H``2#WYtKpscW9kFJw{S;6 z)(EIJSPWkx5=XR&o;kCTb7F)gw%nXiGX)DhgiP!|vH018sJXLr) z+FoYSkf7=%rqB>u7v2IU2hK54jR484qLvg-82SlEGoH|T=3Z)n>UZ@IKY_2wUxZX! zDx4041swW~Y&p55)(PZrES2JR5X{!Ae&m5vRFxE%0l6Lp)o*1}m~pY+ggxP#&s}gV zLtw_(99d5}-9BpKckU8U(a>l?LImYF)wxrvb~GtUFv*_3FnILb^?K_-zG~2oomS`G z+;Rq%vbr=vsbcm0io2|uGagoiVU<0AOs&&P2n>>P2Cq~!t6cr>-va|42ORSDch;^ch-G`mqM}ehOQL3YTey zwYI*WB6u(I+4Xfp=Q~Cw9elUQ@*avtHboxn5xAc`JGQTu)U5lrm+|$JNzD-6dlW3o zP0nKn^pQ~(-a-gG5yriJ2uP+&mui1Ul4xO-i|!;)9>X3CL&e;5waKD86}#(kI<;>v0VQ7rREhFdb5<F=UY&z-McJ)G8_y#$3TG`M7gg>)j}gy zZQhOR530T3Db2skeN}fbgNUh*p*Ui83rx@ z%$#+*!*_U*O6H*zDGd)Dk0b~593@Pf?fz=nos2KT;#c`8Ye~+kllo6l5DsyumZxRc z6f=phGniY!zZaxF@mPwFF)riAAwJgr-v2;aHvXUnUcUQmZ}`Q}F-Ro+4Hqnh>L~C5 z>`!8zvmN60UFiNAag0M%5+HtDUQ`mEJ>RBGF8fjoN5souNBmQJN~)dVIasQFW_{%k zpeZOy3vkGXKBVKd6zu6)ftE>x4!_orzFjS08I0L)Ay9Wl(;KB=w(!&Ce}VCjwEicU z`~#NJZNdnjeuH+C!$u_}siav5eX>+BW$ZQ0$=w7A6C?P({+^u|Zorf$cXRcQFQS8K zzY}vzb`w-a^||TV*>e8j+^;^;4yw+6gZ`Hn-yC^arrVH$;uHoABk+}?v^$U^_CWr& zjFxx>i$ObYEJdkghrlJP-!mzi_1qX%pjJQVwT0fg++O)fMWq#gCZQ@-lgT7tIMI2i zH@X7W2l2_L*BnL@}6J8=;SlA3~+XYtLBdwg=H_mG@V@$9tuOB$2m%<;1*g^ zM7b$uNFsSNE=<{JO5NHWelWbA$s_$~GcX_Jn+dfMe8j%PXHwL?i-q2tz~U-D)df{@ zb>s1$;9|3g)4w*~8#a*r%1(?BS&7oUxxGVrUqscZm}7^ur=%OxluPOAjBKiDlpinwj-PE_q)`*AmlAaE%m+{8$a@H~yYdQDDB~xO z7&USf?=?Y*FiE@EMflr6wFkb!p(gA%xCU5$a1%0f-J%Gz7M;BJTcMYYIM{0R3|7#Y z!xA6c!)2BR0Rcp*Vqrt%(d>vahRcO+dMISHo6kr}|8TipLF6drgpumS1usaqh7?u0 zk7tbnQI8=&YWJG0F2nG_$vI9Q@!dB?g#P*1xLopHJ}{O6W&;Y<%x1!cwz0{CstMM( zuHu|f$CKlGfL{+!YjzaG?=8cmsDD+lTe?#-4LMXt+S)>6vcNZY{^o|NlYP;0Ev_hv zuPkb1?8#T~V{pxKMj9v{ngZoL%rRWbdS!}+P=9ieMxmDm_dlT@E+8G{SELLokllT{8Y*CPj*mkw~S<+u4B&hXn(z4RZ zKEtOO;|p+GwU(URIhWj6JSAW9+O-kitWgjVsW9U-C4J=b{Y-IJE`Z5Ss`Nug{tmhlCwrUkK_9^L8nU8pC%s zMO}kzLbMteRly!Bkot4_omq<*Sy_dEx|EV`EvuJnoSeS~9sG&*8okmx5U~_?%}7e8 zhF!q>*YP2y9cBys4^*@lVfJX{< zE;?+(@5NBKFHE=U$~$6QgxbD=4q6LZ>o%Rj>y64jhn_Tdg__)_7I)u(xriA?omA_< z2#Cmi6~`6-mc#Rv9OdZg67!3VZc#b8u@9-mz5^kV{O}5^|0IZ*Uw3)Le=L;c=4=&q z1YB9EYqa_HEwJf&+9-dCoBxZzuOQ6Th&IWY!v{3bq-T&v-jiQ}hMniGaaC`UC`O#$ zW9PO_5v)Hf@>Ls=9NHN%PlSYDld`y`)8KPtQ)9t%;!kk z-H~rojc8akRFcQdO8`0|L>tKULo?Ll4^D7KA91LkOVD_5<|<7}$H5m}^uV&Tw&lZP zz^|1lhA9fhuyaSyF>yvTLp&+YSrgZ~(k>qS{0jB+$bMM7?Ls+770{_!2GnQAIlBn_ z!H-H^jH*GgCRN-;e8~l`4CLzC3iH8FZu4j9EJi6`C2JXZYUWrbPeG~Cwf=Sz&sYz1 z+Web`5OH>v=&cNc>)A%cN0fmg`vR8mzM4%J<;A;I$J^Qa*QK|iG>CPde*YRKFBmQY%zO-c zOR;lSOKuI4lNwp6oq}Oo2p)0?Y+`~VW??DOJEB&9&sEMsIdy!n65(yybpE$~@mzSC z$~XbxDF-&u`&HSr{$1*;PhosDA<-t)b&n4H*Q2fR5~t0NyFg^DoX0dVKx5r+CA4^D z>iPECr3>&8!LWlHPP%_)7mOLbix z_y|>6czU$@D~>UWHgDNwmRG&yIPDN_>9+r2%}~y7kBP6qXLs4Ns+l_+D78oHzv@xS zodH}&V*PrP5kc7mEc$Pf1c!cKzEEEV=|AN( ze~2dcG!@408(<870nHVT#_n7l%#S$2lSB*uIr~JACwajkrS-or^~H27wiz)U^u`QA zW@gr63ifMTQdi)8B!*hPlf@;pxgSU45_MWp`_~@di0g=Ig_wV)%Ch9jJ)UUKege^r z)Hc^+vT-Li;I3yFFl{T$6k$u8)*oa!*CzD;#Wex<`R`Bbb1Y8v1^{83ivt(o%kMM6 zmq(hQT;q>Bp+f&O zYSHvJV1RBzY96WLJ(NteAO_wCK<6xlp=o5QF8rH==<5T3ZuC*&H2POrXc!Ja!_-Mx zQp1!g)0{P{x*9B8{keuqn^rxaPL(j{~IWC{{V35;@-~jubTV>^zV@k z?^=Es+Q-o_-CIvrykvvH9WiJOLBLpi$WfY@(2+UmTf81{EQ+DB79Q z1A*>oFHz72bk|JtozO<1?+5b#zAXo^HCs9KB_TM6I3nnV+#5c#oCf-*54wYUhd59d z6=;^8L$D@W-?_G@Eg%R3jtYH&1dEDD4T@k;RRgDlx>6uqx^Xu0WV_(N(uhS9AYTR~ z)NTUdfWyGndR!Zp`k%wXiP;O=<-`bT5XZ<#S8p2bv|#_22fqcqbVbwpD`<_K^{$E! zC=62P@Q}?1`B6lhj-?`QAcr z>Y*pgK#r7o!~ID^V3lx+{`{|x@`BpCdI2xw_vh65!MFq7OnxtZ?b10LnWYRo0h}D1 zui<2i-@D3`m!5_bGjUkIUnkv2nbIfCZ=mNp>l=4*g7P#J{{T=3!MprYj*W}{xLi2$ z>JcPy7TrfTbw#uDKLHo_6ZemSpi_4ad&_H@8{&8Pvie7WPT*&VUk!x#%-0>P9ONq% z4R@(8j2*2}OqKxTxb!ZFKVkotMB)!1o#E|Yg*zT{ZF2-4)acNy`m~2qH{B$&$^a1% zyzhVx*P2j7H8q+0I6D@p;AbEh>853Z876w)N#xj8P%Frt4spn9USv!!g7|UC+f2#a zW$)ol_IhtP&Cm`^b&?lHWKPT~S22dD}2sOJvPtowpK<=BP5iA+T*8Hud7 zLRlYsQN55ceSO#$(dM{CO_+{0(5c_zH>mP5J{~MV3~&17Du#grDR?G3`3}7jrxZ1` z3Slp49)g_g-a7Jon|5+ppbAzwEq%->wej*z7Ue zsfu2Bq%AZkz6VM;|4{GRmOI(eRB&r;_}eFi@h4d?heCQ7Uk|DUeqH@rYh-=r&>$+x(7L#M7po3-lz{4&+Lv5+Bk;WI!Y^GIlSw?+=TYlnw3xb4LnV(TFdzcrgsGkl(h(mfWk%)fS!_|~3{>;&kB8Av zUzRC(VSHPO(~&yo;{v~ZNo?f9W3eU2@Mp>eEk zaj(fSw@Ucb;Bx>G)bwO!6>~i1-OuB^oa5ZFXEHltcqz+7OvXOnnYwq#>$=sJ2U05T zC^n?L8&VQpX`(sZYi~db=$txmNXqUI=<^QBo}fJ$U9t#5x7LQ}{RiHiwYbZ>Khfa| zeUQo#UlzU}AZyKi*a)Tgwei38n~@cf`}oF3ymN#U4~2=LEJ4IksN@-CE8rlm;Nd$< zu2EyLax*`h1e1LiG(=QR$_AE<{hC{SM@95MB4#kxTI%<|BN zzSn}GecywJ{t%~Q>XxIk^We9d8JnGfA6WdS!y}(c<Z7 z@6Xa);WCp06uA?6y2cmg}ZK{yvRia&mybyuQC<;>BKn%Dk( z)xh+bq&*dTV$aueZz`lNoId56qxe$W&%OYib2C@Egp;HZB@^;YXe=Fm?C1}!A_h`U zfaGM~6j!g zb}g{*vq_vmzIO!_H=h`#m|S{2Fko`!CSTYMXZlo=$CO79yUUl^j95Dv`ro8@GQ89r zGhKJ2qg*&<_1XV&fd$_oRmal`VWEC=5w~Tm+?lV)BJI|+`zXoyfgVpMC}F!UE%<#hjA~gGjy|etENbI3uuV|L{f%c8;0^6M+3f{$YHp<0PFLi#RV5a!P{wdbY6)q3&W*|;7 zDo2>tkItm_zDcQ+bF4b&IH!`q*k_x5SqFE>nVZ+vH*_{FH;C3VLCCUk18PN2V%p8c zLcc$(;#;3Z&FiA=z1526DPOl93ETY+KGVMG2uEZMUH53$Nt5E$_lRwMS1L;#A+_dV z?Fe<;jS3lt3pTbVC7o5qvX&{o^j#F2JSw(u8S>ces}oI+$Er4Ivd+97$zJ(p)k3Tz zS-C6paNNQ)&E-aMX~jug;ak9#OHwJ!3+LLt-LkP&k~?AbNO#4?q~j@k|eV^z1T-Wo*dNHrazP9{#zL=}vDOUch3p*)3>Kd)>7Qf#80oiA_jBeHxMHCcyI z354%MXkU+BsWS!u)R&u^v#4E4GyfO?tufh;2bFl5{Q##$?PAMlaB3u{a_qi31#3U~ zo8NIM$}D#0{jchJ5`s-sj00;;KgLp94~Rby{8w)Ib#Z03;dn1f(|c~Y*&2V6WdE&c zr8X`V@CG!s1J19I^m`YW-^8g0S6tI*Ji!d*ADs$iKXNGUF1mT%ZcOpf_Oif{!*k7l z?F2P#Vhu?PIcZ-&rF*CAY^ChTKzCbyA+OK+FeRsL=EZ6D*if(E58p1hF3w&CHwAw0 zuwSzo_kcb(2YH`S;}3M9}Gp9`eT=Ts$G zr|zug>eI_^=!~E+rBK z*QgeQMmLr}Y9r`6H8)H4t5yk)mXT-^U4Fr4{*FcS%#~($Yz=7&a{f_la%}b1=2w^O zsNYmI;#2P@Zz=Z3D9LaL3B#(}xR=Pg9bHTe_as-NQ4;e}m^Wp$leN;#X#w(=9MAh%AG4e>nDXww}$k*i@@>q5@r-TVd-(815OETP(sRz3((V zZISaKcLFc_XSXWgx(4S@4=JOUPWx4)K_c{PB%P)frWnp%mly|;UKy)xSzjHrz5@Lk z{t%k@C8PMYE1Wii3FCx1YT%MQa^OA6w0z9W&ro?>)is*WLsWI$R-WU@_}@D-RS`j; zv+eAah!%SIDO@+=%)mJ7eflr`+2p_^9<^OORv~+(KNc}NPE@>@qAz#*DEXmnO9EHR zQXeUNc~2HjIE6DDLfq2!L0BQ{E(#hbIK^g@GPq|OnpKSu9lRs60a}E;VxV|8sGcAT z_yM|!kd9n$rkTWnY%PKI?mLT!Q`O+va#DH)o)>&8!AL;Oz=abJobXJ->J>3IT!A5j}YrCS$8Ii#6!?czz5?ZI&-9s`f2%ELy+ z-;AY%wLXgip{S0!Qs_GIc9}oIo2BvWx5m{DE*T>zWoex7Cjittwi_4VI(=%ftIFJ9JN=IuUu zqb;$R>-eWrWCiTg8=728+`P}WiQYUudDyL2*-W}lcqZG4t=6Jprj&6oP zQ1XP{ZYEyy-6HQ8j9!_-*6J_^G6T1qr_2Px&9#TfOPBKJY|&G@Z~EtD6kkOS-nI8O+DZ20PQU$FJ@fq76WuNp(m}5H!rK>tr^>cN{E%&LR55B} zGqP?nyTG)t8jx2PZPPB*$fWCMD?O-v#yhK@>t*elV@I5_@x+DoFt36|$XQjT6aLZ8 znh-FQ!y_ToW{tBLh!AD%eC&9x?{xl8^@nR=MeRkuWNPg;Tv6@&<7fS0)EsII_D)z3 zp{@Puu@A;QU*`e1Bntl|o)Ql_?Wn`5w!4U-DbW$5L9)x5X03_Ah)>L+_1#Wp)~On# zZihzu^}BOER(j*sMas?4mNQQ`HKoHfmDTSvdLMl`<+K!I_+omgf_kPNF;EuON4LtX zmY>syYe+dBPTNbhnm{f=252Z?RL*0fi1UP(qWG}Bt4=np)I-Ml{-&>`q;-TSFF66p zw}vn8H@dTv*s(ZFDAi}h1hZwO@%xFErY6sQP<3;!K3M4`eJ)=Gy{aDaT8nhYc^S|9sgLlApV?)%Wl!m*X}+rnizB_kMD_$X8VAOBdX6P?C?rs_ z8*#2)j*FaaAn!{qfL@2^-i-)amC8KYq#NStq3!HY{W_rWKzE*8A8AFCI!^ofeN58r z-~>uo`3Pamg>CfNclWU&@p43DKLRILA?+gTj7!wOVJ{ogEA?Q_dpBaFss3D?+CjEz)YW}xAZ zuq5K@OMf~*OuodP3HA0yN>_3^1*U3GY3J^czGT7-;M_%?(u&n1me%$msuJl~aE;;P zcHtcb)Mm!}$isEVPPPjk}75&D^ITIUhtSJ2&>eo?j*SHUiJ^$zm z=q}fLb?bA}aohFE@|MJ}9DO>d7tsOKh0n@KycpFK6;nNbXc!f;Cz;&EMk6O;Ga^|v zRH)S-tv<`uDqco9sgi+}GA^<+eKS!bm1Vxz799b#>*dn%OL#eB^@?ZEhM)Ziw`qaU zpfUI!DpzA|?lR7#weMG*x{OM2BS z)E%YJPK07hU9TEV5k7?~%+>7nVOf=9IWmvu9I}c}(^m=J;3~ID+7m=mPsZ&i8=F8m zs^+&Y%yjSU8kuOmf=6a+5(}z)yx&Q}o%v_pqU`DMQKWpi_0THoSqaf1*JXl)=I+^% zK8~i|>GY5DVgo#99`bbyF8sJGt8JJvS2tYBb@*z=TVDB)?!IcVrjvf+2X*;m4=FGQ`E zrm2=UhiqpYggxh~ajGZ7tviv7WXL=nBYgWWZdD zj+YwZZrY?R{jrdhB>MJyoC!beo2uG6eU6P=;>IV!(2XaBY1FkH=Q*xYZz7+c{e#;J zcM-t-dJ)Y%F99!%bY>@!3A2XiITc4{m@^tZdEzlGH}?ZCrKmJpAq<*v2Sp!DU-mkG zrP>>|!`F-(55p(6e`rE zyuifbXlI0dbTsVbJt;}ByW&#sfKywBv)G1hJ|u~sey$(yRn-$jkNDWHpnbP2fEb#X zG!=E1TwSr5Mj8q$UGE%4GW#=vGYTw>*rcS74V4LU0#1#{bvIoOfrKsfk zT6|j?6qGkHzWUM#rs*t=E~P=QWWqkZ9{XQJAXy5=SW=dzpS(!@-No|X|04fd zn@J%)U(@!upvWxaK>!iJpLivj;p;6ekGTU4pglzYz&d)8{FpL8spo(=S+A(i%JTnQ zYHdC{Kp}|KEbP7p(c5vbYkcJm|I{QYnMD1V<-QJ9!|oN8k=?67?f;o3b=*wX1%D@W zw|Php9!YGugH8=r%XA6d!TiWqNUCh5WL7Oj)_?ZNKGcw5F z0KeIT%N`_kNI)71EHRPXKyu=hPX1=K)P^fS#s94vCvq%7u&qd}t(#D!2JjdYZA^m9 zpBK@ZfR3Xk`%dIkRzAU9w^V0xfT>^C1e=+!V z;U=>&kW+!R>^7+^TxM7Uatffl01B_{yv5lI8-Pxbz*&FBXBHzLB~r+Dw0n{;gn#EE zzU2Lp|JN%jgRs6>dWiQH==)1=2%U#3Wl%>Gi3^`&K>8(7$;5X2)BV>voAN=QP2qv3 zf)(>B-|N5D0EY4MFn}0aiHNc1Q-$U`(E>9h20#&_EetC_%yS0TnJVB5e=8wqh+t#J zMOp$?6`00N$G`@nNdM@AJyvJJc6jB%cv==jMFfi1YXVPMlD2h3<}HeXGrgXM%WbCG zN(x`_PFPf=1eG|JJRX2=&}%ONiKR-jK};A7NRF(qX%rqR0fHr^cyKf2*^xYrO#TkL zt+WskYnf)D6VK1hutja&0S>l56&=aYDjVM^42Q9~by`7;6i z!GZLuhGD5;07yv_zNlNI51F}%8GQ$K_^S`B*euzCuO8j=Szt=N@hCroX7=yuJqc4~ zjU;0j&vv}?xmy!@CuLvfLjEA|ZjmSri`cS3iil3L4&%aP@0>8!r(Ws~SLk?lGr~2Sq@V(T<&}VRA^qD{ zRzMaNJG~A58N${ljy%q}FgUP1oi2d1yeMi; zVItE_>RI-vtITvmcz|O}uadBZB?`Ba`-G&u3NoH0fV7XpNC z>=R{-wPkL9vtC7fnnkCQN@UMsg!tt02G6O5|5Z%>DJ+vh5izDfVG)N?2DziJa42Q( zS9lR;7IIQ?UsGvWJvJg`WoFu@g#`&6j@>#fP@hOXy6ikHn0x%zC*w4V0iV6+xhMt_ zt^aw^e}4l`Tr+5;EQvuF2Zc5h5OYmr84SYMDOz(t!S&199Ds39#S4Q4D2}f@eO!$Z zK%*xIYS#O#8t_93q#-zpwX?MmJ1C!RZNY-a+OT6o@n4jEfhrG}4DDI$0)cqC z^fa-$KchkXhj3p;c-k-IK1}5UE5CpoC$Qk|4Uh?x1c5TC#8lS@EUoX~8{9QK{``C~ z2U_Vn^a&4rUTc66$M-Hgmxo51nf z@Rt2+)*aW$`$4$&=mw&R4Fn?yI`U*)_DC=};U3Ujx}9s& z<>EoF#b-BrT7;Tb>Vn!GIZuXqEpA!i`jq5`nQhPgAb_*K33GR31Veu>4*0al!mPgr zzE~30YSRvDRH$tQIG3m!T$pXYMBECbA)w6U!~jEnXG2M(WaN|KC`6F!8zN(sx)j$OS3f+jyLQ9h;9^8MW9^sIIr6K0(dDv#x56LkNDb_9G6l*v zp*)YJd(NZcz}};NbISB)#(ZtphP2vLR(-~qrff+sZ3N5ViTbQX1VALfL#RWJ_ zB_{ZTe5Du~eoKge{{w7zC%BZyIVR=ghHlh113taHVDTixH>dM@{k~u$agVm)JkW}U?2P^_8NNAv9FYGA}N<-@)eO8uNr_y}D?j010DA@u? zAA}FT1xbfsXhjW32LSm6E~O3bY=bN>e9PqOIN!Kqg00{T#wG3VW^}(Eo7GBwfv_Sl zp;Qehj(|GFZ8Z0F$6>)R!>WqvKnD;D0mFT9IduN_Yv9-dS-oqFD{Ue7j)fTWV(`-v zI{_HW9_nqNb0(6=FB<6A^1v~(4@5R#ioiFJF~I{uR{$LZlFAtnkpbLNH(al>h*JSi zQfdQP*oqV|{sIL)P_Sb>;{YvZ6EWetP;)7sOY$lFfeOtGMDYWN1)!~WEh$~ZS%84y z{RDzh>`y&O!E&%*om33A)gWU5_YsA(#GCK~R}OBMXNq;=B2a)&gL((%bevD(cnnxE zU^HunGavG;9|Nr}Lu43}w0R*2vB3iZoOLOX%I{q(ea{n<8-GT`>m*e}KBG9$K^Ox` zqI_Wq>Q9k-w7oc){M4Hm{IN|5TcroTmaGC!+8{|Yx*)ET%a&&158j>ja+fTL%O1f0 zL`QdQ=tF6cKjb=d?mDf$sTHvRGLXZX4sdfI59qvBbcRUOH6UR|ElX~8p7c#5=|T*v z$@AiHd;Xv2%bKLtw+CsY=xzBjg_3YsW?AJ8_+I`KEo~W=Y6x1~VYbuHOb4>n(oA7#qjxdw80g zh=$nJxY{2r4z=Eiw4NAixzDoHy8-m!*yCId^t=lEwDL6#=xc{O{g~_EizQr)XMx{Z z{K)6->jW?Rv?`2g8`vtCIn7jYy|lBplS2$^nIjqM>Cu&%Kr7##`Z7uCkf)Qxvl}0b zY^3rObnW}fa~SW9wz(-*8-)p)rkHb-#AV@wn=&E7FqnK}iYv=SeRtN$cu!DU`2;0vo`q+N5)@%DcK Dcuj(D delta 89144 zcmZVmcRbbaA3u&C931-`n~dYw86kU*VEGmCTNL9D8SH7b=dC5y>89lTpY@ zR%J(#-=)|4^Z9+hx9>mn$J05_=kb3E6gVMXc9!q-@uPkML2{yx_ce?C)=J(Hy3e%DjZd1aPah?TM zZv!8`vUpt{!;{`3TPdOShKuURRgCHN3VWvsIXT zE32Mse=*atAb5|AgKl{xO)lE;)+6!VEOig|T+!6tYGV7$|E^FuKdq}P75YSADHawP zB?-;aq9u@|qB~(h5Q2Z8Y|704_uT)_U$rRYQx|jF6m>OKLO49l(~psoHa1izj#hrB zvjK@r)AwVhFbC^oWNKDw`0oNHOjsle>GglEfWZjELy^dTzc`^d`SY#nceFLfnRg$b zY&jGsSvcjh;gBlYv`ucx1e^*8lP zHTe>rj9&gfU-<9t;8KK`zbD_Gh*-A$`Jqq?^U;5VIXh&UwRv&7dZFR|=klP;BjW$+ zt5Z|e7jm*BI9`0}nfCnu7Tv!k+eR1K_a(Yw;=hBB!Npn!d}Y{8>z=sINYl5Gm( z*XUeD-v8GV*f0)KdkHHf%93z=kGJiT_L#%W zw1xZub&}lrWODU1yz>%!sPMMA*9T$x&&7IrrD*W^v09763O|LdTmA37wd;vZarf_u zWmvsrAd>0!zeiM4m4Px|v7W1UHGXL)cYUxx1+&l^koEpqY%KD)oroq+F`(8XokHn< zwcQL?B~|@h$DJs+PJ9&U=>c_Ukl;!q~vbEO}ykA_{1 zaFJbRCfzW;SgEjNgeqyM_}EMHx>0B$k?d|ahVH5Yc)NUj3%n?iyhv@8HL0 zzrOGr$!}v8aewd$6XDqs4PceY*eZt|1xZe=Enzg?AZ4(GUBq4RT#2y8B`!QNe(F zIgYTm z^q6aMpRLW6d9aoz5p?A4O^<6X=Ufk;LVPYR#9a$fLkhy&KR0}@c>YwK^SUQ0TtRWK zpF8MBvoq_yl-tzlPt19)selSG0p@?V&Wz}n#AXOM>|Q zInHDcIT%|T6-@`l5UK=QT#uc%Q(v9*B!72AY0Z{*ERLnh6wNMPS{8goc5$!C398b>Na zK7}TQ6IXiV{{3h|`}GsCz9=+|2es3~bk}!l>N}1*F-y*8EtyNh^p(ZKE-D#Zxj>Au znm;4dSADUY?nW>Gci&MXL>9>b06q$Q8xKH%M+q(q(Yz! zqgUi1#7KoZ$G44R_;YI2QWBvABJktG>ARrAU&|X7Q|sPv2iV*4v#o_Ooec4B8=Z0z z!U+3)|8|gaBIz;5?<5q?F@Qr}JG22lg}fV+d~oZ91#x$>V9rC2nEQZ0Pzt1*d67`2O7pax?tKub z8sMD+JK@nnz;ly|o8&raf|U@54Rtu4IgC#~$Nf|N#;693<45_g>BieD>Be-+_@%Ds zu8I)MTvTVuS}$HgQ-j6Pm4^$~L^uW7M+*Sg2Ji3dkf(v0f@1{!G~)7*VZQ3ON1vpo ztu$2Q$IGqihW{KHc)M>*3{T!Jn&IeM&IagTAtU%NlaHlBDwZXx2H2O|I5yC})!_MG#ZT z;j>``$I}oPFo6c@!0wX@x?+UdFwgx#xT_g5Aj-``>i$DHo={YPx~S7fG*BS@$$n!R~z zIDBs5EAm8EuJPxi2QloqS+U_v*0F#})QmjddevA378#TgqD&w*@w8shc#=l(B`Ub|a({BP`Qesn{7lLbN>}GtW>{s>;bC|BTd>{#avB+@&573NY<~1)3cOM! zz6}~mytsRv(~t>k7k1ddm#S8Dqd``6oWCGA{6UJ2%2iC?9n(WSJplqnU&xiV<7H4o z?&AdTW37eoLBx%Z1V~X6IyiS)yecnQLGjDY3tD_N`}Lnydb#9aZcPiQuAbdqk4@jx z$YAj%E<7UTaC+stjk4m=jQtGVcRvE;h^C<>&=xdqbTDh**< zv(Gnc&ko|}1&`&w2%{f&d(#@FP-}2Y!U)BI<_pzJf!@TWU|-((At%8Gfv7^Rq18-E zh*ae|IR1>NoDW*4k>7!i68Qd@%dR6c)nraV2$3d{iZeaj72i@X)pSaj#=I-Us+!>%Ws!lD_MZ>0u{oA;%KNI!9;VwEhnek<>t4fcs%wnUj$0;{=D-z@=S zNzv7^yK3!$PlwW%SvqM-QU=A>E%f*l!5!CPkJyQBLdTr~v-|H@kS zb*9EXrN=M;kk#^=Wx6RhDyj`%SxD6qo&$I;e?OK+!!BR1kI9wZ92l+Cj<-GWPkC7f zx;S&!=Ch?yav?4)xDG4fI>e&ULw{J9dD@h;z4FI81oozsKcz-5w?344L~%c!+M2pX z)%NalJ4=VtEW0)}J@@T7L2bb}vV0Fu?!F8MH~{j%8OCwF|~7#*^&61C#RB%eu#> zyV{QZ%wuB!1V6oYi=LPT`od6qg)hNDM6QpLyH^KC54hVQY#Y3v#yM|_*;~3mhX7fv z(JuHrulOmYiqi7ySf02f_F@#95Ii3rZ&}T%wshJ_aRtdGzdv}DML6CzMiW(xQlZn@ zgb@y)+c9GV_@_)lN9PI5dSe9JZ+|T?Gx9q7JY|Iwv4;v0h^L-dMD@SciBpFfmpt>d zLY3?QGayD58*;fuS_<{!T!%{;ky&nyh(gMBk{q7@;Q577x6mt4&fh7BME4l~1OzQ` zv*I~VthG0Pq^Bq#@GLMmZ%0bee%T7&Zm0yNa{Z=i_Y_#=zW_9xY=6$y@sqrJW5`Ir z4xF@R(592*)|iFg<;Xl}^O%=^*&e^2n+Rccy$RO%LT-dI$+OPUv4@gG!1>8gBC&L5 z0>Ak(J*%zX!brNSCS#t(#I!v{s3O9*0mg-fY7m$wpIoCp77!CJ)=yodHLq?JuMa5SbfMepQ{zUu#Qp=^O* zG;yE}kH8CFNFac4I|*GbQL-Ke`n1IiCpz&Ao~k}UoMJZZ0Q72BA6?gbr*jpn=@eP6 z6K|`}X+ZG5J2Bq(KYONOY8!!<(3JXy@#mk@{pVZ-I&pKb z!sO_DqqE^kx^&zI2Y*iP?$QDYRcv#u-tQ^rBeS?le>E84K3hS!1COeGs7AF zCMx`J`OroHGZ5+1iK%C(>aaa>v%z`v@Lx7{NkNuIA z6=pr^^A+c`?5ob+(p-u#XF3Cc<*#tjPQC`d22H-i9HyZr2|!5Ppai^DDJ$P@wFL)M zlPRc;i~4Z}(#mahXTy@YXb5I9Q~>BX-l<{OH_TyUW!ME>hV(Ry>|E%Fem zFe=?tXg4g@%zHDgzQ41E@4I|V@7iGY%FJfT!Nikm(r2r29S?tmk#aCgIKaFn>v^T< zJ567jjXl%9w2HR!v@xr9F-qgpNAq4HZy3jcyO>FUgo$V+vRw<7@tIwf#dH_J_RCFL zEVyUDMxFB+`B-%)mxmHs)ysS|z+d2d^A@w(JNU(04@4p5LudLleIYe8VQbKBo4FIV zaPcA8#ZaR7602Dgq}t%rE6cuk7LB9=%wM@M{)~t0^(CB-kHfqa0kJvBv>}z5^CTj8 z8X;YbIRid4;8Oct7%UaHon{Lld=2-5M%73+ZZs6pqcG_`9iuf93GQp-SgK3Ma{?0q zwZ7GN-#rGJr0MH>NrPM(9!Hg*t^2;ip-;3ts|tYGA!7V&Q0doP+KX`0uiH(q#>fd4 zxmQ5QL}P&g!j7bR=at-jgl*ry5aF$hE;B7JH#s80G1Q?$D&yoO^ZA||ND3YaUn^e4 ztdHMKEmSFo`9LL1GjB9kf*TzfP8IcvTh3uNI*cn3wB_@Xd?-I^ZUT*B>td4+xG~!F z6qtfQ?U8vsDOT4GXu);CKPNDY7XvYp6?i9bqwIZR{`-Io>VF#>M^12>zPqGufw+Ys zCI8%`SZUSlU9P1gU5)?Tey1j1bzt{k;>KOyF-_(!Q(%&L((V1{iZ&zK<$eLCBV%C# z^*O$l7qFJ?Aj!zC`lINH0tq)TUrA`JO~=LWGy_b&Bd9keAIMls_ZE11WQ3Lc0csV2 z=V!lSg0z_h={>GG4VpE1a$!WnP?yQcOD35v{EznacE;?kXfmylr%Lnn9@z}bA|1?( zvw*j}#;>X`MeY@phodgXJ~AwQq}Hws6`mka1S+o}c74F`jH~>Y|b-!_*@9~61 zLJFK4HWLOw#v~Cyj7jnRyLXpkoJO)}=iXR0dTc8{nj16WTKxOxXOitCP+-Tux2iA6 zh+)29&d=7zN_L-jJ%yEti#}BNUZ3V=1?%`Hm0hABYuUoD97slED+yz~I`3S7mG#>| z9x)O8>rT!-X2$1NP0pIG+fxd$CU$aVFkY-75rNP{NTmM?z28z4Q=nMY@e^pbA!Sp> zeD-Biytf#+L#*&NkXn*T+GJQGi~t+#tqGF#GUtT`uWnvr6^>w`; z6fmU><(6c>eSj-)FZK}gcYprTm$0K0Z4No_)qVhw0pCBpo^N^7NECcuoT(G61JV^{ zUX@l8ch?@1&EV%@NUnb z<t3-T<$}=arTs=?oR<>{fuGL*2;as{fDU(EvwukpN?`&;qzqW@Hc}3Mn*!w!u_82ETEOj)0RXNs5|PNlW3-K`g75k$=tDP@ zSjJ2Kg&$;Ui-5{-fXxl`q>IeE7T|2*5ZoS3OC_QL^k3YLGnqySeeIYBo2^yq^z>h-_2$qJ^!6WJhZr=^zB@ic9yZQ7`OVvHD zbys{*wByz6>8&cOS%^hMa{rG2Ad1%2BNXW$&jXj5v6;cZx!gABU_DIv$$SGYwv*Jt z;(|P?A&CiA_a4o8X~)p=-h97}Y2^`N9g8EOW@DBQ%I?LdzP#$2T&8wWOfjIK@xovU zekM|f-kACM?)+(hOnT~MD07}J4^m5@T$j5U4#Ff+hd}}Uro-+SI$-TtU+t$-=Y4k5 zz@!j}J(2*?`Zw2~2)iz{%DtZfo38bAgp=e{n#D5;n=Z)%3I*hpC zY)eoA%?jPx+aX6!D~{#e(tLgIWB+hWH4AAB&@h=ow?)%?%Mkzg$DYQ&c<0IQ^mWM1 zv1B(X7_MH2YB=R!@5}k$%*CmYKS!^n+9Y!4rH2>0%llLYlif&|(bS7iQryEa zcaek}T)s#~N&&+wg!^lf8o~ni+{3`vm#acng6G;6r?OpuDGd2}H~#k32T|7QdMMhO z8aL9J&*0sFBxR+$7Q`u$3n`KqfeS(hEL@7P77!=p0fPx_b>Ps{wgS{dU(T0#VEuTm zjHT~_UbBpP2ZmXt-31Js$@|RTVE%=pix^li5KJzxV%TztTw7!l z-*@p8-I@o7H!7LCe)s9#14gu}6?j@C&8jA8GO5d+4n25U0dkXD^)b5?OP7UiRhAal znf*~UiTpS+Sz)a4JUm3&2@Qok0K^%frKdc%3{^#x#!nsAe^m&|D+01i>#m5QQ9ia& z$q;z~tXCpsRjAlrS0?yMwQE40gsW~}P-sASZm4q=!l7sj6Ifu4(jU-6(u7|EHd13$ zh`0AXqiqilr3UTH&20Yh{*X+TF#F@(cC!)((1U~Q?ae)soN1h+W$5TfmIVX1rk_r-69t`+ga2~i^C&`ib*G7o*f zfW1Oq{-i(V&iS`Ww5XN?n3N$8T6cRhZd;o-xX%vd%3QH(@;us}E%*6;v?jzR>8f@C zHeuxmB=hpzqil{q(zxT6SmR z@ERX{S*W}v^j#n>71+}x32bt-+=Zv|vq1j?b)}nF=T_B>U98}_{CKB>G)dt{)2YS( z4LdtEAy-F$mb_hUtu*J9;P%RS(!nT+=_MvzR1z4O>PLLyr zX%!QVVv5P57GU)IZ6c2zYCz(c$Crn5q=a18#DGEHdN^eXfNuxTYZBJ69_N^Z|SDHL%5QY=6BGM9)p+|xL~a}JgkA90zuL#XaIbR zh(Fs**8WSa`@?BpXE6MMH~;GgjxZ$K255B>5th#3@unqaHFh9|@{YG1Xb&CKL2G8K zUc1NwUHdg}aDQ+0smQVOo73s&??7+-1IPAJ<@Q#&IWX+7Z2S8qEC{^NqGa2yjs zsx+|9WBh}6Q;Rfy0zVaawvm1g8{=glwkxNOB%}IAlc4HglT>(md8>U%qP78s*P?rP zUzc^z_Y4cQDq@2BV1-ckqxtgjP?AQ2=tY(D4(vX-S^myPYZwvTw%?mVa|ivtM?ifP z4Vk^Z#}LDeHc4j?68vwgl0td#QCG=bd+`7~HuYuZeLsjau8V1@NBnDfVHYZ#f=M2N za%BVgGbx)Vt-%f3&@Q>^}_afMujA-@kQEE|8YIOi2{-6z-SB=G|_ zIq)P~0c-+Rs7XRXyUyk_DDE5b`F$#I&6+& zCR48!twZv>m@w?B65rm1SjF+@u-I=x8+%)B@gP^D)aE}`7(WMm_Z+bOmA)kYLZYYb z<+cU>%VffeL(E1+7ybh@teZQOSYWvGG*^{J49hNV2mK5CSb3|9nK8xu)2|RfD76*B zql69C0jV`63?1_=HD9v709ky70SF0@xMPtkoR!PJcsg0 zGJ=~>=X4s#u9d{|x_OXH8})NVv+V&{*sXJOs;Z{}1YMsPYTG35O|GXK8;m8^%EI1& z!HR2)5#k5~XE4Epru$$Uk;McggS}+wf0GS+=+Jh?JRxfN^+Lr0nSG47BBI|^jRcEI zus!*aDPTlcl$sq+{?Qf{nkxJ;C~6B#azxLD)$_t4!vG)9aq`?k5!nGl5eUd=f`)oq zX<*oib^DZ_pI!{e`~=PvYc0<>ii(sszx2_UkYCx(Pd7d{KKPx0{`>fA#_g)6m&3OBGR;(7s2^Xx&K`%D0gf(+8#a%5_8tJy z?>Fnn6nDZfrFY>4788kiE%a*E0betPu}4bd^ys=CsT~U@S$g3o(bPxF7Qg$TKg>9f zokg89}4mPv6PW z075C{8;IsLe%2tp_>%>>B{82{Y2I51fje&`pWOrSynHt#%ke2)b_O#W@D-jbo4Bf9 zQ#XA$`s35LV&hJ z5w3rEhj~^G>&JfwD)v{6+tC{xc*jOzuW1A3DgPB?*zxY!Rn&nc7K}du5tR4+jbm{0b}4apd!6lKd)P@#tV^24 z@nT0r4P1{(Ywd`aX+RCKn_MET0C@5tf}w2b%6PG)s^sr$o?8I2?37Co`iQh}t+zhk zhPBzeps7I(Jr}z2^|W=%XOEVzn+^WE%g4`Z?xX7Ts^o8Cp2DH9_2e9T!WYFr;qyj5 zvNeW_!NOwb_&sK8djOMV?kZl8{a!zTM{|Eu(yDeY2=T+MgwvJ*N&z-q2=2bFaW7Y> z91-hlkN~o1H^bd!z<2{pz770{&3bisI8A`#&w1uU!x;$z1@;ObS`=f4@R1s`U#d1_ z7XvR~gKGUH#ZvtS$PO4j&3TfJz7V24z!|V8YPU$VSM@6g)2y4Jq5j2D)DqYsevp{O z!j(OXWH_!zu+5K^@1LPo#SENb6-tt9#5ZwoMXvevC9+#yEb$u5=H}TfxMeWNa2j`2 z*g$BQQ%w}uiC13I9&_g|)SP2Z*+I`ZjnCxXMeny4)YL7!V`gs;B_M(gIc2|aPazOj z?42JGp{(6e2O)40UTD|&?xtA`#I-FOQXY;ADx`_1c-uEIi>f#8 z>BIL+(vsG}#D$h8=_z)`*(cHyB};Pe{`zQ9H=I`tnt~@%637tIz!Uo{kOUYr85 z{V6c}4Lh3{jK<`?)D71*(|lb3G+|El&1b$J}qmB$yR@|C+6K6)qe zFKBEtOM93;Lpw;+dKd=b?d1j93mhF`b8|)uI1C-N+;>yVHVg@ZBuV0$7%qgvB`!s) zMCIQud+kqzZVw0i1W`r-h=X?d_H>u&yX5Gaxt4iIazfQWy1U1lX{CDfi&)Mf;`{xYZ##K(Vy=CvUjHC(*c+LK>WATipQCjwL_3| z8O+Pd=Rl5Vz69@6Y4mP&1twNg7T9RBv@id3Av~U|Gq)44|7PGGXU7>X48_0D_#^K; zoEOU=3hPQ5GHNO)!Ih%pi;^ zauJ&FNI({SkhWXbwpFkgY|$JbCBWgl;Vi%ss4da7S?~^Idg7z;R}_*j1}BT-M3-j^ z>2=RkVK$~alPVaUANMB9!`c&n2_$dEHSyz!Q^eoybW)(j79G3mkHBkH0Tv&T(M=u4 zC%Q4-`4p-Pul?`G6;BZ)H~8#k`dESIDP& z*F-+ZgiWc+`pj{s#JYh6q^j?+=wd)2*a=1NUDEm3Zg?Eb}g3dn1ht`^0ItT7zO zN@%BG7zHDgRJCOb#kt_mp#HtdmD_c@G@UM-5g(=Re`?>bjb=BM{r)n(HHg`vXhoz| z%C*A^D(|-~3sU-Lj6Q=_qLJSEaJ;2OcLQxcl=K!_b6XQu+jWhBLDNuN=%&RW3Lj&0 z{{9^-9){`-kn*wA_=CP@Gdt}N9Ff{FOlFTU+m2zgp8+!}tG1Mv>L&(&)tmv z%5sR|@}}Agyxd0%2@LI@bVB{Mj+sH&rG=kf>BLS}f71or8~>7?l|68E(-6%moaYb6eWGk0c>Iu*RZrK=^D zFbuKvjizQ7(*$=+kBx#nI6K&wn7vzs?Rv_pM$;nP?Wgt`^rSCe0=Go8Swf%uh!v`` z!Dt?26mtG5JiA#SYUMhH@z=l%{?>>K&tj7vM159yjWZgBaXtTajzdU|1r#G3A8LFa zz+O5rXmThq)U@e#l9_WVld6L+mh>;TzjI0vncR$l_{7xZy}uRp;A*t!@Mg;6m37C_ zLbY->FvO*7*9|*Lw_UnnEU8l`X~5(%KKEpHySDFZ^7a#E-;P3jSFbN7_D?2ORBQWjwlImK!@L%#)ADfJ+K3%_p}7u+WLijrWG}C6^Ae|68=s_^e z;vR;$#4uC%M|QjzI6HAAqSEj3O2j#R9%y=XRQ?d%SR1g~e!__9U^gPlf*MfbAY3=`^c*1#W6< zW252$1+4~GnG&E*2vTkStGXGCG5oA9&_7+RX_)+M`}w{FLD&KdJ{lC;loIk z=BqiUAFSWA7#^7km$-Mc==%-CKeOK-_`nj1tN_;Ea4wD|g-$YY#pP2jP8w^ZajLtH zX)1hRmmYcjb1$-8-d%qVbtJUT6?n9LO_3FqTInVH8$?f})NHsus(Y5oaU{eIHcCsD z@e?c`V9!`|+B`puJ1iGl_C}B?UHyhZVc~@q1hzol*n*ivFX`2?c1n(t2rzsixIUdD zdTg@9bTNBP8MmWrIKVWe2d2gYF~>{MF%Nq-iZ6VWpJJ}bb&Ygb4PG~`gPtmh@g_|( z`lGGsq!kTtt=UqEkHqiA2$~rXz*NwYh&88I)m4>xk#_jtP2$Gs5zN##4<3$4P z(a;*sON82xu)>raG+dQjCsbTg@%-MeFDkI8s2kiA;tgNU+w5nqFe0ber;vS6)`DO{ ze8o~&e`5dM4*lUehFYA_wu{+Ni>`xBO&-n*Cr|Wp*!E8*Kdrg9U%AtlIz-X66t;7i zUSj2_%A#+zS=-BUxNhOWfP*|(262E~uw5y(9(KyY^o)g-xw}^9fCc@=rZp?lWDyT& zQ$SK!31LH90;^Q#%jrM*!pS-TbF5JRv+v1_a;TsMra4TvSra`CJcxBKsP-%SRj2sB z6_;c`)|vM`piV>S`6tH*zu#Y#e4nHM*h6m(4N$)dzY=2nqJ;$ zv*8z{_f3Q8nOOo>mi0+G)g*Ua#Fae>)C5<`-KGbq>t#A|-c)YB zsxV`K_!tQJ>}zl4CC_1M##vE&R!MB3%%A2Ts0~%A&P{Yu7OK{E_?GeeI#eiuM1f}N zZSU<{)#wgkA~#BuE%(*VJK{o7Ck(Y0BWSy5`g8g~&d%YeAAsrKwxe{Yu_GBPjV0~MO_5%Q+OpmZ5RxA~qa zRoow2A7{Nk?(OvGCM}AvRU+?8a+$>IWJTJy3` z?_gK{XddM;yLQ#O0!j^aYtpNi(_QLXcHU2}n5wE0p+`5RYB;g+>w#o1932V7_xQ3NGCUU>=UaT# z8#&OD-^z(*ihFc5j%f3!_0qLZ5wK6PBq%Oat4VyT21&2Wh2zDa4=-%!@gr08)nQ_wz5--j&f-_Y?VbMY%ysN~Jb%j^bQ5Pz7VA+8RD||ME&~ zwYZ|1K(f5UOcrvtM=(Z(2aOI0HRx8d@(J%e?LOy+QsRG>|yI%<0KLsIk4 zL2=duk8fWhv_)}MAs8L-sGl<7cNx6*znAU$q<5n@ndGZ^9761l#Q~VFp)D~f;8sc{ z#)a{~CY(R*%=Tca$;(9fIVI@ocE_WoGimqa?Z0+wPmy_if716*&FZpnE|QFj+|SG_ zr6}NCKP>BZ`DpCqy>bUA1JhJf|DRznM1WsFr!WS@+t81rv`Ir@3+G24BRWq#VyKZS za}oue$4rM;B^*uT2U{**pP`|H=0|GwsTNSenVHo2Qs>ap!#C)~jp= zTI+B_tkHyVI(91Z6$OFEOl#C0Uu=E9eM7VmU~!?OlRrebeIeAZWhLQ!44fZLcpE?9 zep5UX>Y$2gqIKnv#too6^$G%oo0Mos4fD43@mSiofG^)CX*;^=6!Pq{`)tWHsc925_kaif?}@>+7)3*4-GCR z+EM)sW;?qG$HZ`6MMVv%`+_+p_unaS(5YfNM6AqYO?qiPdT1U2xz;5OqJ_2{RxA?Zf)hVwis+t(9GNde>qdIr`Gv2pd9r*x zO`2f&>k=7mXXYi8AemXDq1Xj9E~yTZCo+`0n{6rlnTDH<|KZ2m`!RMDmr`7=Fky0i zwV(S^5ooo>G&85{!4YC67HGWUloFLpVTA@!06gZwgL}4<(}ApAfu2KE^!d>ZEjOdR z)n&Q8$}Jm3n~xOzuMLZYm*Rcb-;t`mi8hu#c?H*IJpNZ2fpsM?UXPS@h@z_Wby5g7 z+cOYS_iBsz{j9Sp7qzIziR@y)Scdm49Va>Q`(#eV%}Wfkv&dRiH>niHGdJu?SNuClZsdZ0dr-Ja8s@kNs^0BcJ(krIGrnex^Xxh z>8w^clq;2z=w4f({H3n7z%lE`CyvO{G%e2-#Ro+vf-{X@wU<=e7OJz7QEY$>uTw+C{dN&}2*_x@V(`>iMWgr8@*9E=? zhu~A)M+A@{J2SSDBOY?I++T;&+|y z*?xiEDr$US9--&&eP)vw_q47ZITgiCM6msoizL-Ih>qf$0mcFr3E{k6=UrU%vy(o` zy(w%azD6SG3M|n;-#Jp;-SiYuwo5FoxelSwIT3&L%Dj$G)YV*0SDhr*8cs@=#B+o);})-noB7U^k~}ad6iT>+@8*sl7uvNgTJ1;CH0-@Uo4Pyb@_}G@I2? zcfyVQ1%jeF;3cYC#QmO~t#jUI1tB3v8Y=INy|}@(m6Wdx+}x|w4hH%yyefB~S(~^c z*j_YpP9*+}&<+{n=_TJ9Q&W;+MwoY&r%Ta+Q6FX~edO@!sf1&!`ASqy5~ zxogcXyNj#17#vMntt@nw3o_O#H6KZ1Imqc|T8#+;NyH%=J543d7F|s1UbFw0&ipb6 zC73#+UT)S4%7;qY3<0qa4-z|~0by6siR1Ajenz6za3o_c2+xq)7w5tDArnRz;*h8r z;xBgN$4%-5sv>dq>lkd@ah%ogJs#aDA}6L-VB9j zwx!(a5TtGv%j`A;^EiXdk3V2~2WfzL>MqY&~5vt83?xP;~1jgP7Z&T{HC(vm-l z&%l7FB3Af;YYMS^s3K32d5Yv6(nT@N&WG_+F=N>o+Cvmx_@1L=w(mI;Eb;;U=7wR& zYcy{irKr3gbN!qEvoPU)nJ%t~7t*I`gF*uLHn%iZF&Ae=3(m|8@69ePy$j_<^wOI# z6n&=rq;7S1F)NeDkyK>y`riHdH&UV!vD=Lr(bNuY*PD2qFYD^Lr%t6|g^yGquQM$Sd3CDJ&KB3BT^pfBd8ft$Lu@w5T}OW)16!bA-y)*a(yUuh9aDs;j7&jH9b^% zjc}jvd#ATFkslrS<~+No&<#_Vp!pZKuSg&-f{3CKx)oLjT~6;9@s7X^7gQk0&@l4 zdkAbslt#Iw%Pw&qP@E%#B$$)0OeWavm*H(O!Wp!U0@Kj`8x}nB5E&$PLmt9M;_Y*E zeh~(6P^&esW&HMjX(HBmV^R=%pUkf|tbup4ujfV*bh4{c!0?#e^pHn4F6-;eaG~5T z(F^5=qaw(@m`((C+i6v{O9MmhDz5ePH-0;fHgyAE;{p6Ox=-|6x51cBvVvZY-uV_Vq6iuQZFO5 z{uGu`%g47TYWFKnGe2?2?`7$F=A9Fu|ogwgpxHRtDVS5Rl{6O zn1?c69D8xdLBFPNV3k$XKykMlxU29l1YH}0?|A^*tFb$+K7S(o~2 z(Wri$3n#`flhGH&zNh2UE4O%k`HAohgXN+)#U!vnRJg+_neHVg6*CoQ@15CPV%J;= zY#4Co`NLP~q%^R9gF(1mr%+}LvvSi|2r8M$m=8i&tq;~d2~v6;Ia0m&<{3SazF_|! zbIldJepbJJ%;&m{qTDd^Jj~NKJ0{(D+BH>bLEPmyw(c6FBp$8 zUpffm+YLC7{oGDmg?07oz&6*u=c?~MKN&+{+-f(&^~shp_r1lG+tC~tI2jE5aG7AOh6g%j#@AY6ga0r0pBIDP3c6w#J|+fH*NaKW6gHq`iHcRnQQ!b8(sfQmJ$;WW z2z31Tl2)B?vd|!KE{4&3Xp|gdW&x-BWVrlx;QvR{d&g7#|Ns9S2gi1dWAD93WM&>5 zA$zZ5Co(cpih6{|9<~>QJ$Uic-|lP`}KCc{{@dh zT>+Tbtf{=?T?*ZZTlZb@QCY|0%|`+w)J2@taQ8`Q~P zSdcT_PRvP&`nf+-g2y;vBdsUg12C=UBGv@tvfIZX`@-9pdiR#2n68NdPX)s2Nm4Ej zaW<$O9CMs-F_A|lzOmPU2m)FI2V8}IHBsmdL&$w{BQ+7qes!;X8#Z_Yq5w|<0Iz~E zn|1H59;k7wgnXY`rz+O=h+5jz83{F^e#*&%S3$1&FpihfUCdW{`X>J;tJIxCmnCW| zFDj{a^1N0B(14E53|eKQrXwClhP=`K9fLuV_DQuKuVz^aDqn672$@=}t-^Xy$JK!E$-z z%qBYXyqNk3XgPowJpWm@AAN&A<~fh1a;zrS*Kq9zX8TW6Xlx0u034J45yl%y+W8qk za#1ey2UW9^YKd+Luk59o)Buvt-c;TdA0__dA6Se5RCI{s%}gM_ z6aOfvzBf6e>!qdQK~BJgQu1WKJz2}Ay9xF0)}49)ERyW@px9uAg!7gO3-b_MivOFH z?_<>5+3T1OwvWG<`Yt}ebgK6aTvATlzn*>iW3Qyr82@dm6J7Z10EKdG@f(ZIrS2!U zmUZGp-^aC$${3bwusUto-@J(_=##pJ$l1arCH61GeMSphky?M9vex&FkrDt*1mhPXGcLKl**j32#;kYlY zo659Ocy;jVqC4RT#GMObc9DKN;aUP|_uV+Y7h%%Y8X94f{mHYG`%Tw)UQNv~E}>FZ zN}VNT=BvfQ&!-@Bz$Y(ReonjF6#>aU{Rq%|#@hp!8-OTT;geh?Ox0~NC)_vQrA85n z&WT)p#^29H6?{)^WB*_1Wff)M5)?hM`wtEx*HO_pgx;QU0}u4x2VQjY^ifYeEx~I_ z=!j=wW}^N3(XSD|;sC%f1LI6TO=k$FYTozW0m(0-C)7@CjI+3dO3-zxuo>Kcj(-Y> zZ84cXKiMy-v~INxCX-C|9W88d@!wylT?SwIV@&r^rf{$F!k3pvKFL2*q6MVog7x#H z;>?>h5)j~yM3N-UUU^hGQ^ZyX8x`NLH7p}o_dfSfeP0#0`ot~q`@_cf`UAPTNlRduqH(9esSF1J-jorVmC@tfCy7aqyc z8j0qEg<5w)F;<&3N=zr1c$J+u{MGw`qdFxri~F+A(o$?mXwA>Xf3m7RF|;%%(P$ zWXNrt6M3Y?$)W3|bWR>0s`Ry>2rk?Qox`f>-QIsU$36*&Y+Mf$F|*Hy)xD#$;9{lr zO_+boqLV_8*=wBTRuj1cN5YnZ&MWw z5K;3&pONX-_V{jUZaJ2RcLB&july^;)I}CrF?zeEXq@09gRBpGgCKUwnQ|%|6)kPV zM@lcwK4WZTNy!fl2k&Z&S`uvyg4+(aTLvpVOxOoo`R(6z>*;Q9;NKqW?e9U5@^``C zVHP`NTWYTX-opzE`{G?nuFflu!=lL{%$ORNk} z)LAre8XAN^yh@s}EcDvUAfWn<8lwJ+LnJ~kM>;d~pR-eu+B_aoqwBae|6f(&s~cV$ zP`jhDjdx&y(`q0c@$02ipNOKJ9Oj&7U9TBkFO0?|K|)>u4Db`gF5VgUw!?x_nNI8( z0GH2;Bgq^8bnTE!>U8x^GZ~W?!?J#g$~{IAEOL{N7H>B z#rj1`>r^|WxnH~;+p*h=dB8?Bp!dFtUQd{kNbD8glXatL&uHef^;QORj7RP9jRr`q zzq|N5c-b`RzZA(t0AoeomJfjCGWB7kuL%l&g#Y^g4Fi#~gZ;H7KmPDBWlez0p*9XU z6RADYYY>ZHmH%E0AUB%k5-;Wdm>&)d^|5>Qdzw)K-587x*IyM`AOg~n?NhD~Ut|9L z^~|61WEI%Lkm#T%YEz=DVfo^R_W#j(ofx)Ns_cbP%W&l|nm}#^-%V92f21`hq-Ic+ z9ToZvj7+@&6+IhuIKyPV!mh;yPOE-`S%JIW&y8)AmvAfP!-Q=+X*wS4B`47Y#{(@n%&UAJJL zUK*E_SyIojp^7T|@ROQ5ygxb$$;Qn?SU24fvyk@)GXKW5s}+<)3qF}ya4MXJBD+>k z%Y`mpgc3l-kYCLGYLnn>^ig^_NofT8@^UbrB{PIL{vZ5ChCm=>5Q&Zye+x?Du4kA( zzs3>HIV~#hQUgLC{Y|mDJlZ;Oe}l$ciU6VZs4THX+v zZp*)&Jo?2aKs9>!+i0N`DQDmVp!f>V>F;Du6%n?+=8eHOSv>}k4ez}lA*bLTfc<1? z%BI%~5d85=d}GTyBb=0`qbsy-KA2=f>fYY8^YnS@Q1(UxEAEDkYtn9kfK+OS_V7;` z_1l#*1u}(TKs|IUdNL9aoGO#XKGSJPi+`s+7uxZVW=1<-6|VBW2Y8b3p9%I{q11qg z3{D>3_9c)>gx#L~V=us#AAyp1XA#Y+U-IIow{W4>9|2lLf<2J|Tu43dL|1+;=D=$iJ; z(Om*I&8SxFD$4uBMBOTil+I1+cXhp}g%3yoxFviYtPdOSVi`6BVVSb3^+@c$Qq~zM zB1!ha3Zh~UMbl%>tG3H5Sq**C5DUH36F{Bqqla9p73 z2>g4eWg)`urd(8)awo+@-j4IPdPaYNfD$*-)fAnTa3xfo#f#%=Njt?@Ry*I1Phz`Z zjgLW;@^Xv{{OyL_w`%o_GZ}Rkx6{L!EG7RjkoRzoQz;iFyG(THu=+zj$h?#`%H;#Y zSutA{2`PwD50oWKw=I#U`+p!0q?gYA^75gtbRRJ|xW364QooEQmFA2wDL850BoDzZ z?cFqCFz^Ret4UX%Hg47AmH3l>DP-ZO#0VwQl`N!6w{v)>A{rcqIBok3fKMa58HQLo zizQzKb&ze1Ni}exa87ptz8#Lof%FM>U_ML7QMt?P4qqRj#FyVhRYrqry9B2@d{W-; zV?=}B+5AC&xH*G1_|h{T08CMX!{8p5HJctzOl+VqIXF{wXy!`6ydVLCe;rL{r$QzU z(tJM;7a#A<^f62Y-F*AmlAE7o{j7R8R|@b)Vj;EE_HP`f_+S&Yw-1YEE_K+2Jdqoxa(J~8u>hIXvQW91h>4D(3SSrY7DyvapNi1h!TCYkq+br zsPWWc#!1ej8^P3v@uWXvOmKWUY%HUUm+mGxyJxUbk{^Q>FX@|gpuo6XkZq&a8fnF< zZs~DE*wLW-g-tB4c{OpZif_~EBoyoU zwRgq5FgaN>8u7apl3jj|SpJ>D#az}|RbLC!m6j{NOi6-<)!!Y4LQ&kDcT<)7d>tQo z8r>&5qkKU0sKSBby_Q~5Gt3n(TVuFG8)1?5UXw!xm-%nK$lc>RKbfqmjd+R&zL1_# zexx^h0NTNMtRez)oVxW~;{xm5k%*qS>YR~BA{hBF3v^1IG4w2|Af@hE)%LuecXu96 zHnOO?rF6Lb^w$#x5^wI;_=w|-d-Bzw{eDmGYae*;bp<%iLwjqT-W*V69QNOKhg;F% z?~sz`foCFOSPJdWv?=pYm4h=g7;v^}jgTnRgHUS|kzbcjt9{O8Iiit?tJ`Bf{4g1F z=a+@TAjSt*OthR>bmQkntLqv~Vy3?#3}RSd@^=3c^oxKG>ZWy&{xFPxk^gJ=VQlas z_zE?aAOqr>vEiHcBfLYG*$vU#OtI?JwEZkCwXY>IkJ8uW4{alf%jjjxdyo9)3Lr*g zNx;zMre-P`Q8!94IG!ysK@jJQNt{=iOmnfg<{ei04ctb5D5G?6YPRQ}%d8B4S@sggw1oN);gz{Q)1kfA&D2;Aj+qLBG z@ zZ**tAo+Mw^ms7BAk94pe(-XK;1I(-ORZ5HF*^uWhv$*XBNZ)V|$b>D8NL5R994}5u zBuj!ia=kNczsenAgc*KGgwBp{=07fsKd1`&rx|^ueB8)qjBcNfI7C8yf_(iSg9ku7 z`+7YB{>5u@DAhKY#x>RJ&$#GC5YiWNhNUI+NaT9=vSH9K+7e@7M)1Zz(*jB(26t8o z25q`D>V^bPWtp81;PUYl#E7!iPwanJB$n69J3g;;okcrr_3BrO(6Cxi$^G~)=j06M z5V)w5yHl!G{CF-X8Xs+QoT|k`9P3*;>clhO(hu&$foD!*DV*c zcl+HaG8%<3shm?Hq-POiM0l|hdElji_1Rv`biC=moip+N(|6!|>B{#8JOo#OO~Jzi z9)A3(yvHhSSuiFz(f@6MW1}T_O2Z}(XCV->Q)!eAYNgMmU?SSFR>ll}lnBmBvm~6) zc;P5-2gNt+X3SOW!dtdYH02t5-*7E!t`>z6n{n!sml3ZJ@ zUM(mwv{{3`NBL3t?R%mc(%Rzsvg=U){81XC3Q|uIAA55nyP+!v z89H(!3Iy&H;4FzXmtL4U63LhL$d-4gF?)TN<4i2|OZ)cpDF@1tViOS)v@V}@Q(nQ$ zL-hn!l|hBo<8c0uW&}|bH}+p!Zc8O^Uf4}oT3^?UcuH`v;{>!5DGGOh=O}x?7C1Vw z0c2*NU|>=X4S68ea33OzvHy2EK%uk=xj9;kb$m_mhZTll#XbN1VuI%x?-sZ$dWO-U z680cmK3e0w+1Nevqw+LqG;?x^QtG)Fxi2xfNW2W$>=dGS(}ncyGm|&04I7R>HexKz z-Pz<9=Ze|aIKylQfVZbw)8YeY;hUj7jaF92`sIAqKuB?(8>1DemlN&s94B9tlS z8au_mT|LLyII?k9|3c%Pza<+{Tk6*9Q3j%~#cQ`1MYnTbYsmPFIxL5Z@6O&y-Tn?* zFQ|an5p?)}53SDiu&PutAgre=uP)$Qt`XQ1SAt`T;vy{t`bJ!XHaCVDfQktg-zC(; z@eDm1qN}gPgnfP4#eSRlUS7%XegRHLaqU z^77sW3!WI-IB`d-d+vz@kHnWLj3oTQROWj*a+oo^r*);J?~6{ZlDf|c+2S=W8dE|P zqp4k;_N`Q_=+TXlgHpDt_6?75^GPeddmjzbpm1EvUluPj(JUpyQ>ab4fEo_`cQ;T( z?csn(h7VCM^aTnt%F_SvwueW92Siq00Hv9k^?Qb7dH9x(VN``5N~1EnG898ni2sjl z!!I`Hf4>iWpjOj(H&N?owdY%1=-?YT^6RGDzU!sl>;$JO%0(4GHougeE_gSV?%rq1gL$&rSZ(B*4S; zw0s{3r4$UvC4C;Z6|}F%#JyvQ#n(MEo*;YcY9-GBBdz$#x4h-zM1n+X9eza?g(1yB zxistW>Ya-wwFi8D@}SG?D_6|*_QZ4ZuinTzoKxDPT2H`~!GnmKj2Rx^uGwPlp@&W-w%gbR-cF{2(Uvg9{bKSy&&7l!rcd;1;ZlILhtbeQlD z*N-yT2l(nMGk-)R%?dV$l15K}JM-D)_Hz~p&7j|qf%4P@m;Wk|H1w+C;B|sg7B?;# z`BtD#0{u8QjHUhu(k?}W_N$q~Mi|+|7G4ihDo%4d^}lJr*y{um|8?G0{%;jTHmBp!f(Ldq%_(8L z@iT%N!|{>QyFpKY4odfh;_|wQs-ihKcknU2cj+Q<5P{HB$6F6%Xv9(N_KQXHcU>)O z4JK=r8wO$Q;kiUJ^^QZtQNrX5vd@|X9vdYOqbyc!sXwF^62bY;z4`#UbUhCS<3!$$ zM+0e;mt3;eTu+YXDoCD?Dg+!LcVa=dE(9dXpp|`p645@_4exmRkkKi0V`UFNVg3Wk zP5LjIMV0x$NAQ&7Cwfve6lSOis=aF*WAOo_U8 z*YWLI!c@un;`L7h8)3}3E#!Z$ZZ1jpn_NHg{bY)r0ke}%XG|GhZ{qw)PzVroY<+ir zOScX(QrJ)H_DpX1&DhJQE50KFewu#vDHfvtI}I$W#IZKbZ$hqHnSbC_|5IZ#sB6(8GG|Jo(a=E3U z4;ZZo@fGHD2}C)(9lHm(<->x`-j_W#!-1+RVtL^3TUf{N4NBE95OmeDQZFClHfs%z z3wxYcGXpIz-5n$!{xpjG?$(&81guAcp%_hr7vq^Oi_m~VaR-P4^?x3l!kGvMj*VfV z2G6h$OMT;MefNZ#kN83zEFN+ruVY`r{GL5C+C|7d_o+^h;T{3@hkf}n+R=pHnq}!g70+FhI-A_AamjZ!fjJd7NWLFiM~?Mq9H!qFT#tEYDkDrk z9n05}kjEaiPPHVR590zYz5{5K-kZ+fBf~~?mEsxTJX*Cy|4A9`0{Rc;E>zDgkOOP} zS=WrcUp4rD?t)NVFFI!zRhpZWW7}XO$x4d_frDa}m=rE-!r(hs;OvT8tV#oTbSF2$ z3O0VOkFPP-GB&VF^cRrf^>IGyqvqfh$bz`vdMcZ#Fv0%Q{OptcisuDO+SSoJcHb)o_jon0B<8^>I!+?iO({)QzYIvvm%(Yj@WV}@ zUZBqizRewJ9Y)@{p=8GO3qYX+8Y$Y!1&=VWGt1@L2LLJnH+kk7*I$Al_!t=X4;1Gs zdW`TlaWsdkl6xEAE#05lJ8rQEibwUnLj$VrkP8vw-L$1PT+as#=Jr@YTbwU zOqEf7m<1ajUQ{T(Ry$HoyZQpQdeCDeLJSfHi*UCUqJIn;8l0T~jG_RM%^lb{z=?Cl zlLpmxqSIxHZJ)1IUO5KIenNxq>`ZZDTF4Y)#7chwt^y4Sy+As?Gk@(+_y3;Ws;+aA zZwI=D>%SrTIT49(n25Kof?5arY+(G4&`kS95?RmSPB#~113N8Q2GU_`HW%pW_-4={ z{Wk6LJ}`_-AgV?m44Z}tSlmiJ>Lzln<}0RXsQd1v`k8Rqhf90+LkvPb3-ySo;@dL` z&Yd4wR(+X=U|mAmhcbcrtMUwXVWco8YtcWMv{~nM$IElfvPNmmrTxd7I5UKxjI@=4 zEr&CYd)4}?X_&64*cL`pf5q)1m_X|YS2J@ndgnYw<@)#L=RKJa;4sAdqrTGk$W*+I zS&k$;S2m@p!3U&r9j&*7PojqU&QTf0q)zc+DJk%8uD~v$)j9H9CMoDQ9~DpOW8)E# zh8@Tk45_F>=qi$6)^<*$StR0)V~XX8@WED%j~%M(ACEka4wWQV*YcJz(nq%Zi)g-Z zd-vSZcn8n0EoI=rtQK2R+7XzKC#s{$pX51E-geqD?7SxoISMjJXozbd!d==2A@vm2 zJD!f?`E=P&J=fMhb>;a!o)w{BQ({)7XYx0~Ojph653>E{J8JUi0S%uoRFR^xp~V>& zK(s?v+JQ2X5LP?1tJKXce+%TH)B*UyjKJ1j-0|;TD7z0o*BNYO`*(X|71lYzQeHbC z&6ku_F~^)q3>XDJH)37p!lQUF=F4dXL+JnwFuX0 zZ}rhj+_M_4D@A?g^u#Y@w;uN0%|PGl0WVUe!+g8fK@ zF2E$=h_vbuP$qgE0P?v{M^9)m;%dS;8)C?Lcx_2!^c42fb62ne=1uiP zAuTx1K9AG5^SMF?L-)&p_xxLwgX57Sx6<6?{RD&v=VXcAY^5W!z`i^Ml#-`OF)yd) zwosFJgejar!1Bb(1{Fl|sIl2&aXRh-ja%ihSQpJD=~OyAsS6PgI4~(tkj*I}>|Bc= zIsNJL$S{BOCSDdUE4=#N2HWmi(2I^QLrZ=gq)eN9BT$RkU;eBN9B-F*>UNFmBm2n3 zx_GyqKtFJFGmh+2?sK2hh2mW^Ip#OWd<2k&+PLGh4DB({OEbhn&xCDqtcK(;u0ygd zMdGQ*_3pDYwkEC!dUR-UxDwZg@ktX(q&y(_;&@-&Z?mjmB>#RDA7iAJ=j(7hLPn|h zaMaMBu!k6zVy(nFU2=%9EOx?0ijH+~vv&Gvrt2$xSKLy$Q<@1jf})rgoU%kW9!=J6 zt!D`qUi^pkh~!c(;a_+awAcs&_giBy^pbwR!1SxYFdVL+l*(<3hR7Dqj$UG_{0I*9 z7#hnDSYi}2%0Fy2*eqG}?TrLHTu7rCJ^M-`8h{&|xP}|tqI&E> zHRlft(oge8e$sfbDeMc#nS;-5CxE~)o|t-T_bx;(ZlM0o z$DN0tnr9Ru1ZuB0vZAA>Go|h`L!eZlJGpL4MGRU`82fH!@`+1%Snj0dTIE^g?;{V2 ze%7f*FP&2VExI0rhEFq|3~XbDAnx$)=KHgyU}w;vyziyt8RU&MaPs>&u2KEv9aaG8 z5iAYn*GKwp@Fc7@0RmD9Q6^WGow{G~CwX zz9%=C-hOM))ybV0m+hY5R{exAlUw!(%RHwTzob{%0L`o$gX<IhYMyKx4!D|Xo*%H4quyb{ObO3)@8^1> z+Tz~0dr`89Yb7RQ7KvIlqTnKuMeGXlg48@C*uHTM?gw~AmJJ#bGE8D;_tRE5UQM0o zIUn3zDYf~)Ww63o1rHQ1_Z*oPGS9EfEE^6>X*d#E8(OI?A&>od^D^-rQ$F6LdSmum+Tc!8J9k$1P6V!UPcYOV-_Iem1fR* zrLtL7+lXoYMLu8Eyf{>-_WToQX*FvJ{rjap0A3&whPFPf4rq;1+9_oUmI{ibXU)- zg}7pJ{>>9-+fx5tpEuSYTFh~Q5~Vq5-u(giXXu#(9q}{))RbFGtic>+yX7_y6LG79 zB>Kd2uiva?7znr zR24(|deQ+F^bPIngUL@V^im9esLhZvW4z5@TWvS_O0m_(D+&Fn)>KIciR}sfDau1d zAZ8$C&c&*E8oL;yNM#kc1c_^soJ{qjnz4PoyQ#1QO&zFl6-{C}3zf_qnD`@!wfa1gl`eiLx4}t@_~FYp^Q!&&GF%HioJw%X2Bby0#5dKEntl zO-HQ1Nc=yEGjF=kNYe_XIsoWh-^6q*4;UEJds2|KN(1L?40y zpnoq%ypKUAKkbRv^B z1~y6A$nNrqlmf(RD`Flq_^5oBp8!;=u6wzed!pQrxqd}pPA!=RXul0+5f@l2CKkRo zar}E`XiSaqacpyXPT8zr_7g?(nkfGg(Ov)l=vQ>CWq`(cRRTQ4&ot9tEgn zJ2#hIumd8>vXvyN9pq8FnieD~DDA~{jmFqriThUl`(a00n6&f{U6viMm>?m-dJQ5F z`H(yqlV^cE?n4d)1Ld2~ds=rX?$z$$*o4EAhi{B0pSd^+W}xqY$HglSPRGohN?oEq z{nKz_dG8%U|HsPz+@hn9CL$0~g`qz2f4Frm)$T-Y5x{R+vJsKaV`f$9<| z_C_W|vlLsWPm-knbC7&Uc(Rp+G)d`Q&Bw5mS``3tb~WHfPeM6Gg-{>-2j9nhHH$F7 zvexlF7E$R1KP&9}+X1}$pzx>+OCh1+`v?tPMDZKfT1aSFH4l(06w~`;1x+udO+%b1 znQ|pFkM6>+(4`TKxA~`{YR^x;P&S{<9A-r2w2=nfGaI7aW!9~b`W>I&-+6oz2isxgf0nr(!IhXgna@k!0xkGBMq#NS1UHa$A*RR_XSQ5@~ zc2T0v$`Yd?zp1Ldf-=g!pWqS`Z@_yv$0%md9f!KIxQxT45##wA?ABNkukN2(bUWGf z%a`mW=C)D%Og++u(G+~JaQ33=+$o{yg=}W)X)HR2AUfCsrF;=0R8_OfLtdMzSVb4T~7H-l%1h%>JTT#oL*$%m3hiX zDvl%2pXmPj&ZewbU(T1*$AeR>ONsl5aet}S%X84H3^Jd1`g`)$`vQf>UJKUuI2?Xz z@B4w~w^RLl{!I+&MCC+TXkuFsH3wB0nhUDQKGv8RCFK2(IfYNhYH$_*dk6S<@C|F5 zp%(!r|HQMO_{jYI2+^K;h=IB-#ugRld8}N9=<*>C|JD>^NBK8{URU!L&*MgMdBOO@LY0=VjB~= zN;ZpLwVZeGGqYBgKxxDDTgID-c+!LVvN&-z4vd zq^cj1a*1p0CSHkT%B10Im1}hH>W9y9=IP~;j4X=J{N9O2J# z#^mbm8%qM0WcFi%I?Dg=NnQAM;t+u+9}wU6j4f#-|ZunmJ5Zeg?>jR~@ti5KMHm4{wfW95-Fqpt@o;$NZWF3Z>8f_{;G=m;)hSF-4U&yMY!jG11p#|Dy56z zp4(n?eB#i$(yOuP@4V-Ia|>`PTsF~laNc@{e`>$U-Z+C!Z(pY9tG8{u#^OQ@A=7bP z&@;;u-quv^Oiai&}?vTh!e^aEtP%XysZYJ6wZ(47-n zoCvT8&mWtThjO=PgHC24!a0WL3QXcUUS&}W4XRK@F*r=T9y0;gY9^IlB`Kb~R4Bt> zxr!Q+vJEgkG^}k|Kj!2KDS2x3L)Xe~?Y3*wN$C+_$lr3s9JFTxwHzJCyEw3GhyE*w zk2V)US$fqe?nJ$ISK>7`Wp039<+!M#dCXe;>Gr&+PJN{WpxD|T`29VeIDAVRcK5S2zyKHsmxIN zyJ7}SF(7aPgN<3h>2T$L{{gZljsnfXid$*9%?$&?Nun?9xeYfge2!85LyzI^@B^-& zdCo|UOMJR06G&1S#q4R>3%QYUZeeI*HUbwSsJdeN%A`6%O-xvq*qu<4ogkff))b)@ zw&4~z4E@QTGKDkY3NT3E{vb?wFz$sZ`M%5U+fT&DPMj<5ptkp{GG3WU&DVD@e?z8|4)|`nF>y}p;F4T$xePNvH@*kiu)?8^ zs$m)Hi43>ue-H1mFkuL9Yc`~+-bAp0?jaTG;A>b~YoCk-%nsQrg`>yfT0`9pV67xK z0r9J4#ZP@-jsRN2y&`DS>mWs+5|H?agWmp^||wpIlz8sivgCacXu+ z&L}r(K#fZI51)lQKXt1+?qD{#PD}T46^N03*wN?vm19!sxc-T#6Wn}$RqfI!Bx&Jf zbe19RUo?qhBabyO{~H2LS$zcBJyOkwtB+dRS&?liUSz!Ad2x!g zSut+kcD_VJ5K2(-T65gaCBBmEBw0Dprljv0pQebJW`zs&T}K@vRrWr$yedjgd>Q$s z@jJ^n1(}PYnjL7cqO52V4htl2SCrMlPAkNvybKu?JUwP)eUV$9MS(hf`tlVXuo;ms zo$_u!?D*%CkCEce3!czr%!s3TLUEBX876{)G#KnTF+&VrQ*Ap-VxMLYE(MA;O%mEc zqqKT+;#8147rga)oJul*E&N4XPeIO?$IGw|PCDB-^&QcvPgu)Ji1`FE+iQ>ctGhwB#9rsaxr>aV}Tq*;1t z5oNFq!uv&Flw=d2UJp<=Ils)mdiTIUT&8QJoWm zb4b--NR$`$H|#PuP^PqKFSkK+-FdV%ha$I0q|Uf%8o0N(4h)|)g{0r?p?LXcn?B|r zBz8c+xDt6~L|KYuh$zwlT@`UV?2rE){dKw2(aqR*HY_T_`%Bpt|f%m{pr(N|3xLOgVvfGxz|D=hm;0sX>%Hd)hXJJy; zsVY=obw)Gv^eqM++9=%CPGCNjkYexSY;e8lUb}ZPFjkSsmv(VS37aanU7o_DGgJD( z6g~QZygcMIY&e|pr$A^s^vl53>hUY@ukQ*dS1?7D_UJb0C+E)iHa;Po3=FL%d-%(5 zqZC3sX_qvq!V1cCfW{k@R;_#gmYt*{$ zxdwIE;%(8-iQcS9(5F$`jv&hzRCT^vcWgnlZcF>JFv>h|6VY|GI2eP`m<6YR6V{+KvjfHGob7yoxt$A|xBTA~ zQ{p6}rAFu9W*)r$?f3cjx8QFKzvbL0JdXb>P+sbJwqSSoF?K&JDQ9a2dL5Gy3!PXLbKk4O!9bCM&p!ND_eP3wvd$?0fb8H%kBgFfP@o|D}df z%jJq?&K9@GXdm*mwNayTA5Q^vawa2oKHH->@ z!pI+V_8}7QB|K`u(U8xwsXow1k_$Q8+R^r=z{1tJYc*PBDN0uU&q3VAC?_SKZ1&B) zv0{UnDkk7|KYP0Re>!cQ>}%#8Y2nJZ$uH6c{`PjJvg=jSUEmw#D0W1VbQJuivy+xD zx27QD7ge^U_fW)mG=1ZH9oH}4P3+>X?uun4bWfU-Mrt_r!=|MIihMw>iwjvnRyN>X}?6fRzr^x@x82FT+5$0Y3 zCVw=zU?*A1{PW+^hB{0CJ+Y)*y#ROe83HCs+8MRNG7C(>*ea~6;1+jnbYZ6Q0Ht_n z+=?`*W{`2toc16n0Zkq##+aDl2dWvY!`u`sfn9i&5n9o?mPFEmD2z?NemzSVK1PWv z$;2~K5jh+FtiA8ko+M)}fkgpu)u;PwRnt%shUgMqE9Oks&ofSeLJ(QcMiKdhmYi%u zRa{U4!{jI?^S%qydvyYYqC*NjXv_9%d!Ng~lMz#_aJ~C(3PL8gU z;m54XgtF`UlzTufu%2TmAI$q+{iIw}IF2h&x7zxI?7h^@V-2$MJ(hvSwZn}Qq>q&5osZByfZ zX`g;{7Q)kKhhY=PSsrTAH#7nrmpRxUvPvb$yYuPIc>+d?7i^*i1&x9yy8l(o@ zj~2D?-xK{rR{tHQilW3=21<*r2s^=;T?`c0rnkw67mK~eLplkH#RxYvD4`yPdvCdb zHDKJP8a?2WR_{qcT;i^^EcYRkp}M+m6^7Fsr-yB^P2pNBO1Y-@m3xVk_h9K~~;vKxn*8DRBPl zKV{`S-s|W1oB}6?##22e4a*RwVgjKTn4p&$35$EpM+i22^=HU~nmmzw66bm&=T1&@ z!;zEp?K`+J>X2^eF&Y${LX{sTA?0*RiCiE>QKF1D;@2HWQ zripCpszK`;8cWz^GqX#2kmH5lAwyU1kJ07{k7J*#1_^whc5#WO2(#8U5qsF4rF*gR z4uVg7JwdnIDUW(W9t}hL}X9Yd36$e#Kd+Em^y(&l;rPhf&`U z6hzd$7=DIbwM#(*7<9&p=H(xu@yak91%#69%OsC=m)>mERWD-OCjmr%%RvjIe)0S+ z&aexQIx}Uk+=eez{2AD4VCWr=p>F+A6K;{P!G03g!f(R*;rxC(uJ9GkQne$;hPp2C ztaeNqjOy#mi|wUn(9@d6-yBaf+i8EvuIOFwVJr}`?_=#bvta|&y$`Zq6E0v#$%5WE zUE&yoPc$GJj)&xX69<1JFTVsReu2NeDqA8-NvS%-Q*FRM?wZGObTiw5+F*t$7G2r0 zXYcFs^x1_3j*Lt%N5ei_8GGNyC~ry^bg-Q$%bZAIM&7gv>250LyCcM1)Q9}OAI}Z6 zFspu(S>3y^Uk&+v$aXf$@BEGYFy2I~G^zjIc#$vV+HUg)<@Pj{C65*x=BQ?XkUhikjJ4 zFAK8b=^;TRcQAmQIKGy;|jTlQoj>LU`j+@ zv-xp6|3G9r%kgDS?hh`3qwL{-D4kIS2P`^RznS&d^QbUN(FEExz{G_y#3vGMJmY_w zep*w`Na>84vW<9jW=b`PM9^|{r3q|s<28NS)GILARrQ)oh#!Bv^yTvMbbvz=(P%2E z`C%mLUj3fxuIecrD}4;Q*h2NeK8Awt&dm~_1sIx*L!)ri>~%}MPSC2CxR=`beSfSn z!46lKd==ZM4bfs-!@y{r3A=Up)XdXb%6DX3EbK;fa$xZq7RN-|c_y{+E-{tF`IZX0 zIy20J32s&*1Yv5(QOBrPLgYkO{2Sl1zjjJM+Q~r~VMG<=b&|B@AMX)hHXafUM!r~y z-}*FpTuG+=9IwhLq2C7`+!J)>c<#8QE9un#LvxPUSskh{cqVmDOLkw=X+_IsblYFe z{GL{O)Y!>{ayX6XM`u&6W~vOb>c30=7Qd@3*QhhtE(a!Cb5b4ta*=<9sf5{tot^_k z4jCW2{la8q^!p=TCCD`v16*C2>~@9n0TvP&(Id~wYxu`LBh#f;IFa+S|5{2JacfTFB*(1%X|48+f8U-g&@D-Q%oyKWkABMj!JNzk zzD~OTyX+B#fnmLVLZWIi&#f04W+@HtOxG2SD+Uomo$zaU|2TRcjD&JF;*s5B=;%av zUgSez4|JooI81ZZi<2oQPu!tc*jd>3SBF4xpJv?tooJGuol*13hYzT7nF0o0MvPA$ z?Bsu`kQXx}^?cCE0gOBlw!wjSD&&04X`VJ?e|fmW*Y0fCT&$m!;3|tNZC#z`bJIXX zg*Gfv_FFPz@AdWYF#(zJ8iM)%oO$?a5-3VkGUL$_ca501zl&=c#*$dG$_Od#@?`L` znkbd~3_QkLXXZu@QiG;YVyLde=@{h?WT#k0^piPRwD1eI3}&e#>&sgR2rwJm?qNTW zu6J~O4%wz$NxBKn^W+pUEwB}QUQzX8bG7{b9CitZItycF+g-K-{Qd&Kh;x8u!%waD zGWyX-N7a#^W0KG=*yF=K3ZsmtSii&LzJ}n|l^!*S+P|kO)x6NXvlQgx8AaECB=^Yv zRAk6tVSIRZ`BhN0A(et8fn0WP_kN;9^tN@Jm5BW%-m!=QHTea&^OqA9m41g@`mm%Wc01etZwMIV?S;*TfTw z*q3b6(UTH-eCsJwX%xz%|8#h2N1e-VHw0_v(BJZo9J|_8@E-+4*KY~t`Ss%mzGms1 z&|=%(%CO56j^ejl_H!%kDND~Tru^I_#2vhP%}^LP*n@5nF^1mw3VKq1wTAy)77<3u z;5$%#0~6MeHB+pe!+UT-#KEIt*Rfq%JqpH&Q0&oXfBsHoN>EH zzr_$57ER90{1V_1OjGcTr_>dwri?j$`Yz?D|F$fx{^zQj)g8S) z;&Qz8cP~C=RC4aWy!#E};na~V#9l`hWLw?;DA|9KR?2ZYzdkCrFv&=XDWd%_ce*~X zd~`uUxhus+rk`kSZ=i4W5C^Xpye#Ks<%!nTy`VtdI9rQ)w2LC^rzB``k z{{5fh$T8v^dpnN3ciDUIy|O7gTPVE|G9sH)viBZE5eZ2u*;G~kF5NjXF@KB{ z=C8cKst83G)`R_~jQ26_CyK$=e~As0)z?1A+SE1Ix)A%@m~rKY z=m=Cc>4kXFM`4b(kG9S%dKscqDT*seffi|+dFZrr%VQIkQ5>!R0R?YkS4`pcryKV) zD~$=C8|snv@yGR;p%`s`_A{^QS2G%fMa9(>je3|}UieVdwNY2}%;pLHb8gl%N__k| zyItHZbH0rb^XMj5YgBIK?SfK2E6?2g6}~+v$B#e4FrMV6%I)#?@ zGE7eWEyRF~UsQRW>!IB-=~Yh+KSO27obIBU|)foy12LVao&;y%VyF>Oou5_?!H8TlA)Brv)7Y9ke6nuu&Zny{mv}fCC{ur z`*PjM$@=0TjcGUd)8+L@8qeqDUc1(S(-D}N^0PEk2)7BU&oo5WQYBu$q|Zftp6YvU z3>B3&V7PONG;SuvQrCr0#+J3GJn~MTBIrQQXW+-Mn zjbjDG!bUz*mHt+Wm1N3G^dp84I@aKh3bpQ@L;~b)ruo&7+S( z&tf}MK7)Wy_)q!0>>1Ph#M;0?!c@Ig6_G;B)fMH^N?l1hq>w(ej0yP8sTz~h&Q&R@ z%E-4q1-R!hg?*>b<75dP4!`ay4AZIXUA+XX9N`mGdeZ0}A)pQGi2oI>ur0jQ)N{z3 zL4qq_te{hS^C1lCmC7%91VlwLB9Z3WY*OWl3bB=qB9rf)*ZyA^L5Cjk@lQ$+K#%u6 zyQy-grgq|F(tHKf)7Vbhxmow1ljWiS*L{5CVKbHJBfoR5>&#Q@%QXt*5}Wi;I%RmA z6zO27Nj>$!4pQ3${$-|wd@xeV*2VX{3gDH<(3Ms6&F0Z~=A5>pH?eYVYdU2*MiCEk z$_$mPJi7-EwB*T2%Sb1>dXJH-pPxQ2!oIegc0{W?&wW}lq<1~F+IDK30pFo#^xDaP z=NQT~?JUik3D(|(Q1rgrxzqChXtXb|MIx*PIT6kR;Yan>4%LRZJf54}CP)9j4;aBn zBi3q{*?s=nJk`%KzoDr0Lao<3f7hmpORuxh=$9h~t9%HdQJo3mw)NN*anu}$@dqyc zi8Fj_QYb30zts^s^GEg7Nyin3R+ok}LIDR6gZRF)RF$SWk?}xMYH5fNiM;ST%MzH| z`MeDCjsd+EIrNpPxY6P;OfPHp%nGCKlRN?Ho*ZctT3PAQR$Gl~*rhixd7t}v(sC6T|slXHml^Y3OLqfU5_ zU4eiXWBy#KljFPyn>1~BTUf4jeetB!&S%==1wnVF6motWrs{UG9BcdoHDVOxq6*A; zb8rSeCJ645ldcj-j7U?wt94Vpa?tdB1;4W^b+5qKFHkR!Ap~Z%?eB)pR48`dgon&- zj#=n=b6c8E`f9Sf{O8Qp3S9BYLQl>ouu_cAE?Q*%R{ zy3Ux{<75}gKzZQPc2NB%cB|Mo-03V@J@$a#g>2dAldNR`twH(*kb181U+17$0AnsK zfjF;%o(kYpzV|Lqe^nOP{&3v0deabX&pc&fbgZf=crHxc)H=;5f%V894jkDY z>j`iYXXBmrYQ=Awt%OCZ@!RegH=gNpeRHguCofjyrrZS}0Y)=zT+Ng4gwMZ-L4GHK zen5uJ�wKZtfR>{61=sTsMBMbg2zH*W4g;LmL%3QRy>TvJcAQ=fzUD_^WEJRJcF~YW%IB>iqn>KxFIp6 z^WTEb7+%>@i#rsyEGIcp#ibGrRE&X>&5L%=hquqZrt$-Y;<^XiQ{1TfovBv_%&dRq zm{Le3=PRW(H!Ub|8cyB*(t1LMNym1bM`c@~B^MJeRdoc#DS5uZtV9ZqDB; zT#3|8*yO7@xDq*HX>jea(y-?R=qJ>&N?xh~(X8S#CI>WfZJ!j+zm=2x+k4a6#lvF6 zL}_%9!QY14wpmX^M~Fo*i{$ZQbyUvE5_F`-{6h7}3CUd(7s}vDV8^##FlE2*;gaK= zwJzV`dFSErM$3w%pX^k45U;Cys9{66N)719zP&*}YF1@j^EE3+?cQWYzd1d0ub$ z45ET{+m6u(<~od1gt-(Q03Ji=;<|);M^B@ls140%Zmf`X5hdxdS#h9O9(kvJXaCKk z2A|b>E56L#-QZ)dd-$uCs2iYqt~Z@TW*s?8E99+5*aO<#YL6p(WxQ?URs&$JC0#c+ zo>reVg9Oli>|c@oSE&jgq1FOicXeI`KU+E}%6c^m^d5ds+BY>;ul@Wv+?Jd-Ha)H^ zAG)j4z7{gSuQ=p?Bg4VJW9d z$^3*%u+|qc8>SveK!CYtxvA}b#(;RJMFM$BMNj345eTXBs%ZIR#>)Pj@;>SpWL3gy zG!%<2SBW*`^_&e1$qbt~RUg&wKoy-bc`0`c&##mQ?Rrl4ZyVmls4&uK=H) z&RhI3S=ZHfc8y#^e`t`tD9FLIWiZ_g!;l6<7C`<_#j~1Ir*T7}2^x-0`_n8mWfzMc zL8+Eer)#p|jQcVS_SK3iZLRVvQf}Cna*p*grL1bb~UIm7!sJ$5E&( zG*qmEG{bQ9 z81idjibzBER5UdOli$nR!R~Qe0{2wD=^%_$56Z?l7IaQ?gQo;|)EMCe8VQ)SDRJc< z+QTQNhs4mGXP#2M{t64#tpXCpV}!=pZsuHD!&g{&9`w7Czf8?~P;*VOO%YccCzy%AJ5biHeOOPxzo z{{GEpX`g~Uaw{O(4Dulr%@6wI8co{ZefOXwrI~=*Qz5>`)`9+;I8>-BgeqP{ppt9; zkuQVx#k>9z1k$W4U8h0@nC{cT68xP$Iv*Qhx(4!>xTa;K7QTTBMEWh`*5;)cxIS(q>O50@2=5C7O1#mm zYLwYTUPJC|nM%g~<;%ky8I7F8o>~VC)=i6w{{Ej_K7(h_1q}zMn-SrSJ|SU{D*{>k z2{TV#qmF6V;!j1yi19Uvp%HZ_%gK*gbMOgNIihNCjdRyRU-qee` zi=>g9JPX4|PvF8C2UVYOY)o3r6&T{={HE~sBHCItYZ6@!)sUR8V1F4>gv^@Tk|N6X z2p?Io?*m;=|BFQhHAJSra4PnS^^)lP+imiM5x1syer<^KB9rHWjVh8deW~4@;RVaL z!oh$1ktrACQCs*8g(yD|=*TDC%Gn%h90YLWKnJ&Y{{o3KFOr(z~UN-+U2T@wMGwbE zt7G(;i;_ml5kK8xWI3(8(PUsYj@Vi3D8rP`mct=>1cOCYhVF#4pe^&R)nHcKv~L+n zkpR?Fy*qvYL5(;U`i@O)>0*;8$C@gt0A`R z8d8zgDl`Uag*a5|QcmRL*NQgq7?FN3_f9(HUDy#aT}^4F4Mx#!Vaj8*aP%VkNpjOu zSDMb=-4fJRZRfe>hRqy^eR)OUIXM6G1qXZXh@yidwKHg(<++z3_224%02{Iv8E?Xo zSNcUrg-K)Q(?5W;$*Lq?P>*b28)dhKV*EDHzic>LKw>ek_DB5-+Cv0}SvIL=%cIABeMIP~GTpBlNhBK= zPTE&KUgL>@J8Q3N*icV6u)LYX>a&zaBC+QA&xGRR$!LfN%fFAUGyC9h@c`{%%y+h6 zvB{D=9)TCOy6RZqPHZh}`)6JBCt6{cMJrjWga1EF~W z6?StHxt8xO(KzOzatZBvq5YNAPYjCGAR&!rajBg_M&_U*`3zY5$+FYuKk8IIS}w(V z<;5B9E5hh`Au8qpZu4q-PI2i{p7(g8^!@fWISbK?<%VZ_HiY}6xWA`F75{oOk(Pni z)YYK&CNOsx@TkRgL-ZYMI);q@msZ%s1w9f(eiVQE17tc~^=X#lNap%6#xV2*b0iwT zmn!n%v&N|N=I38x%MEEH=XsO9bo2 zs4Lgzi%hG(FsZ{|ew;`BMxS1>)2KqTIBoLiarOtE4^y3jq{>J#H5-;poAm-#SaLuH zHLwTG3XEV{Ym*6fofs^oO6c0qt&2yL>WX657IIkgT9}#tklupyR>-xx_Bam=~gPOFm1it0$+ozO1JD?4N-+dqF)UMDcc(ogQkAG|GY(=(+y8Vn}oxj>; zD>YO}bt@nIB5kQIf|H|2QB3E@Lxw##^77rdoXBRWP|kO7fw}W{ERq+vNr%0hXrEn^dvsTj zzsl2-_bbi21#tVbv(WqKda?Wa-FQ)-L$N;uHf(F8U6@<{4rJ8%*|sblec_WJCsn}Z z+`3-+U4Xm+Lo4cjUv4@@ngv8*YA(=2~s=YnO%Z19%m zSG;^$;@#J&ovc+^<-bTieyW&QT+`3y^tWt?6HX1(dHI1d!S|+%;R3%!1iphA68Kys z5lnRo$&2=aO8)*d!}FP3IAyeh@D6vg!PC5HpIa@C9Vmu8eSg1^&mI!iw>yWxZO_&u z+dZlx@RV49O;5TV;kwxqt}lY_Ej;F4Ccke z1CDCFF+0cXwNyL4##b)lLEONXi^P?m!yN8SLouj z@c3QJdeYU-bP`Vm^c{MOZUDtHE|$JmO6Vq9<2I{O$_`g$vN_$}!hZbyR)DBqvVY!> zTyC|v9BW<6+f;;fnhd;UmHzQzRwkUFrPtc7;Mv3LA@}zO=1&}BPGo8=08!C4suZrT zJ{v;aE!@!OiBgVAgd)?JeNwk#zS5;H+%uZDP=_zcu`rM(|D;xpUFIE~Qeq9d9=aX3 z8vQbunZrbG6qk_KZ+cA@^LTy$>}C*F=p|&~rI2_nHxFX+ayEWm+aK>B2Y5cuLUp_PEbt1zf|`b_j|_Q|7y z)s7X8!3ofn-dHZ4mwM45@HYnYqx3{@W7Xc)6Y=@VLm8S#I>ii;7Xso}{MFI375y6- zVt@nrcJLHcCl6Q-$#$#FAMYfI^mJE&OavG<@DfyZS`FR=^W6zSOJgaVK6VeA_-*&) zWrW$_yCprFay*(Q5SRZG23OO^s1F%{m#{XS`px!3N!}|Ut;X=i7two~8)+@?9H4iV zUB(Oj>72cRZVe1I0f2=ix2(H%iLvRLjp{WzV!DHtrtdl^*b9ZL)1m<79N{i*9=HeJ zkr7+3Cz&%{pWMxEDDH!04o=oP$ft~I!9mAN>xX8oM=U4^jTG!pIqIdF@3#I)dlTRV zco%5$bNW|7#YT}h|G<)SW)U)_+Hkz*7XRdF5C?B^^a$!Y$}hmJGh|ggBiH*EA8GL3 z+5=Dr8~5g=PN$KRnL8^e7~3>T{X4QWm3w+I>4MT(=|x{qtHuT)nI)IU1NdulH>!UW zNy`IVj~`!4`!pM#t5;QZ^+Bax<`wHFMp=1$2DH;8OT+w8g>G47pD3|`xo%gRw5K~I zWli<2sve|idu`#V$~pk9J83!%bB$Jk&V{JO*ulU4A*gnDEBXO6sTmHFy?$zjqu~DH z8Fu!2)QjJnBV&+<$4`2gvdR196pk(Tc&hBf4QA{=ID7ULoyKnx6-6hB-zL8?DMk1H z{XPG~GRdh;S%D@#l!^49TKtD5&djb{UQ75^ru!u|IL2G5$WvjnFI+rFmiA zx>-;CUWH^s*SXJbf{eqSAX43#&QWJiHM%TdRjoax>|uydq0aF=dkB{d+m1=D;y4?O za8vB0h8v#sFPzD7>b$7h9$}GRn2|;Ps@(pfc4UCva_AR)0>#U_BC86rZ0SN9;I2&V zIsNnC?bEzAk?$fbACpKT&qzC4x;_zJugqLj&u0+Zg}KAJiy6wDTi{n!=+XT~(|!u> z#9u;G`OYe`73knw}-rV}nQc+cYRSq7w zXGO$(XDc0Z_-1O#qlK^1+tZ6Viz(He{O?ayAjb1=G~eOy)`YQ88N zKo+hC0CFD)lImAKo~K{^253q<^H!7gQC?<`a6HYWxrYnxh*`%s7PlDQwZR*apW>Pq z2}jk|DMK=V*e0g_@yt(f-Fq)}L;TznQ3!Zq2dgmei^ zd(Xo;&&U*((zV~S6E8b1;@>atOfWC0ejedv!Q8Nfjq$)m|8&&G zCNPE}`G(2UQ+(FMqcIQ@37fHT?pP>%!3FXJd+2YB;Bq?j-t`3Lsiwt@J9lRL- zl47{|Hl!H6y!W;7At2T2yi9i;Iv@8U=6nfV*G7Y3wajD`>AdhW{2NX@73zHFAjrcn z?QahVy^!N z8{};mA687NTF|{d7B>`eLbF|v2n`&rK$+etCI-x>smbdev$lh&`cy^8aYYg9UCmYo z^o2}KpJ0R4M$wEvyCDVrj|;_$6VapThK3ewg=b&iwbIFY?LDC;9L0NDj0nq!J9REf zGR-yH$URcyjgDuO$%GlP} z&g;Ou0`FYU%qHuTa1F^m#TjS4?OnMt-op3$X|%)-N~EVW2FgHBzIHJ-F<^SrQ-XHb6$BXnty>B zrLLV|M=LfA(PTdjg{%ZfQT^8^nf;wJG19S62osi7!sU!;c_Xoc` zQ=#2ImEU_)A<(|guDt_s4FC0Y+t>T@bRiu!!POHER@U^i!Y0VtKS9nT{8o;s-cSGS zwZq+Wx*1w7hSsO3eykOz@C&C<-@J$Z9u1en%^0t*2k5k(@jl;44Yh|-2@Umd1r`_{ zUbW};*G8@_CZ{h=DTDYR^<cY#9zzMlEbh|w>^sAu`fBB~ z!Z+xt>K3w5QJuceSAddFjx)NX$vbqBgBi)h7Z+Ini+*0F_6vqd3wQIP?~Gc;Sp#W* za5LwRWEbbiLfBMEAzUHu=`J7$+Z6}F8bOp8Mc>O7VY;iLTa{kGWEdJ29n%sgefd{R z>(9%O!FaHu)Bg0hdG9jgP6Cp9?R6h-wXc?DJUXqS{NnLRzGS++2k)7Ms_f&V5Zz%% z48Qz{rim2@zD@Z%;@ZQxyHKN7$Io_6D>ff5?3!JQRW3c%vXGtha^#FfN%}DN0{*wr zVU9cB#--3-n2$d<^DTlpm`Pdu`_OvUg0fB@TKf--Gd2Xy7e?%+-ulvN zQUj3<*8LwaZ3VW3EBG}*6g9KB2ah5&oagOofDawwU|u|Zaaw!fgHr&1xMY0n^p#I(cG+>8XS7VPAjK`a1lH1TC^9_BtSg#EQ4#T4QMd z&atCndJsA<>NPy5QDApofodu37#z3YDdodP1_L!xR>zsc-d@T~$IMr{rv^JfPdy7; z$Kuu@gR5P-=dOrty`}b6=<-8Bb5-BrB6fw)`muIujG=5PHU0@UfS`Fi?f~ z$6o?4Y7H+sW+bt6-kIeE?9dYsDYyCOX^Bh~RLR|V;C3ak4FV_FU-RMS5J@zIanwce zm%A$dXybX=qNk}FW7Xz1AWYUG;sQ~g>hg@VDgWx%pf2lv8jJ$H)gAa8N#cuOO-0v3 zj2}C4kxHj?t%~nj^YzD$EnlvIuI_s0@69Lkm#N zkj=sSb`wa30Oa-E9({9ZJ?U9nnmo*g$wiQXYgrBt?q7HM&HPVpU9>cn1!%1Fjf7~n zXO)C^2enBpufe5r3rxWrXM0Hp+F8!+E*CD5JhB4@=ga0}20KF~HY|FNZLzSK5j)at zna`ux@eqjmu7PR$7U#k|b<2Romo0$Bu2vl4F^+m9#%Y%*=$s$x z%Ef`h0!*wBqW<2V;)jJwz^4t;+LG6ZdCYH(r6>F(Dz;8TT(2>m_-BEe*|8P{$ESOQFUfE7CD$2ee^Nut%PN2^9T-GaxGC-NC&OG}+{Kko|O(wJ|d z1NvKHFGS8DJ4flxeTQnZCE~ILqFKnb=6626IWnYuQg-xt1NO_*z#MD(Y(T{GWg50T zF$c-45oW7=?S#3Hus`MB9jO>2jyW{(%PGOys>F~LczjgQ)UWSeOVww!f@}QTr&mTp zj2!*2a4XjpA2y4lHk0X(wKm5PO589W!4;iP;aK=e&*9H?T)YxN(j}%p z)sR`>3z#oo*CJ*dqKzHY)&iJmn1NsS+C zWKl5wX`MF64T`~CVTJ0DT!%Z4A@6)wg-SO4S((~4lb39(H73XAA0(0u(G8XI)QbKU zIL3LV(ts!4RzyjUwFO6PvRGdu+9p;^)jDbEoBVx z(l#fRN8C^*2q4^x_L;; zT1+J3%SBLA4uPV5X*bzXFp>*NC7^ZQ$r$uqv%y~dDChDO^4*;Wi|Dan?!L3KGFJ3E zI{NcUJm3XOi@rV%w?7MGgrD3%lp;1Y!vz&F*miwKP62~O8ZiE}Id*s@w}!mx#hnVy zCFmtxy5#qf9mUU1$10tWz_rase_zu&oIMrk?`}|U;XRQ?^diIRN7}IP zPE1Z$Lh{GWN29s7h6viy9xfR(f`8hAp6f;wdjxweEE)At!qOvOnw>*JyiIyv;(xT+x0GhZ>qnoyCxj2j zoPTZ@lsgb(9FmlP@L}YGmkZvie|^EZ_4=_nuNi+x6YP)Xv>R@#{8hVSC}=P^GxjH}X5NQwzR z6v@_vp22oz&rVwx^O1)GX4lN?mhN?TF)MAeTK((5Qj(KOV5R-JASAjmWvDjiKBc&< z{P4P_@fWz5hKH%o*xQ&XVbO=$32~Wml;YE1D^Lv#q6XXdW#V59Q5qtL*f`2WL_^xr zV}?@A|FaBJ@%mEyX0E{v+LuNKL9)4RARmOYv~DK7Fwlr&)4Qf-DfL`FoiNHQYIBGg zeHQp+xS!neu)(A@%0{G#+9_1I3aCYe;y32*Xxk zEY}&mtXTEeg}ipJ-)~5xr8LD?3|V!6?KJXr)IzSvn-9YN(XE}-=zE#~m(SQ*N`7ZU zjWA_2jQf6^rg$p(A=#S$LQJMuY*uznt9Ros7$7kOmHL=hprl=P8I8UHLGb)ae z;}^)0vOEIjnNHZMmr`c{0FsZn2g1cQSD=S4XQ=JB1i0FA@Ee4#m z=C{;?rCg@3<2_VRfcSO}SZrB_p`rN9M9|IpE>J}?EvHVNx=_-eVCdUb)AZxRe(5Eq zudk54i2duaUw6TC1VKQBK}jVN)7D!T;BhKwYjo_Wl;AQJy01mzKnO$bTNyBhk&>GJ zE8vds#Cu~{HP++!;%0krHmu_*#>FCrHNyg464vXIy%J>NdDvMwvwF^*-4nvfSBPz< zK}(&vl&4)CW1`FDx`{8~;?3AT>hB252X2`2>v=0^(w8_qk5xlX>}h$| z@uE(y4A}mPfjyJz-c@@!&C4k5u`Ot(I2TdQ%2%!cc%$^pmv2_goTsow=GHnMlO-|@ zuEGwKs84v!*J{lC2HBipzP&yMKcU3yWXOsOsOo&AXtaNQ$;+^@B*h`bWsFCem`$xItXVU;i8;(M!VsJBCh6))E~P$ZUfw&ky$MH^wdEq7HY34(Ti1-V z4Ey-a&}nrEu4M~ZfvHiwbnsBVgaPLPE#K!}`4D$$srMa;tr+W5@pG#k-W2bCkWtX~ z`WJMdSQdvJ<-wGUm^#}p*Uh9}$Ea}gphhr)m7GFRNqf)hX6hmqWbNo*=nqKn>Sm&s z#+%RP9)rZpsqmS+6}YPJC96VOs04|<&jdXJUBGf@|q#tI}m^aRDj@fgK4 z5}D0j3eR%W2pU7-#bawLrd>QNl9yi*jgj}|nG)T0mysT)5y<;_Mb3X31JDl^ElwDs zvIw-n1>_y*8#a&Hsp!`L?dF6QdU^)nPMAuXB5DcERvagS63jS_sZq_qPS3Wws?f|6 zH5#B|C{$poNpEyHizCh8@M`E@bOZ>MiZBJAA)DNpH98x!6FcpS39j{#4W%2*ZxI0P z^~$TS;d7W#tY(H!It8>7`&WU6whDVD()l4>F&=`RrnSx1GsEH-{yYe@@MP2LCgIhz z`%L05)_BB4TAvBdB%c%xXlFAOwY3c;rGFpZl+GXbd;d`z>_PQvG3SR*7~+}$nJezb z?AS=w3+mKg9XfvU)gCCe8WE{FLt)H__8KI+#Djt*Z6kdSf2i6(Ihj%6dqXAl%k?Qk zPNrwaE25rzNlYJoOW)JdO{f$SFfIU@Pxr(Gh{G<+a@yXpavGOE{qXC}i!~MUr*oc= zCo%|`YhHMA)}|c8j#58?%iFU9A2;B5a5?$jp?&1QB0fpfSf%A3?>JMeI1`b)UW*c-*ieD#ao&uat@#- z_;IU4N*0<&oefi$95ZcusR@_xIh-hX9OGeQhNNSKw$p%jI}Tn*b^U}x&X3Pytj|7wRG$&h$V);;`_s?#vpxDA zB(ktbIQ!I`Rt}DNBE571@@sCIBfsNY0EUW3^i&XnW{?&5yNOYa3UgJP4u2R5{n`K3L{GNT2NVh~G~ly*8|F^DeZVXHj|t}@v*X)e>OQ68bDT=y!fAtPo8IWP2Y_pb z!(1a=g#M05RM&a#W7OweZK?H0^sfU}pIR2S!J5<$jAfj{3pG%Sn0DDx`goct28(a2 zj_Uu+C@_VS^3|I$j*uI0q59_OibTm7YN zMlDcX8Sgpoiw5bJWeTBwu#%QoR7`)j*NR3cLVps~6cwzDkN9t=t(puMiO44`E%%DX zTWEGx1s2lQ89GVEXtt#3>)nQ6Qz!px7ddL9 z=D%yzi|0VnQ$SPQ)e_=TrAtB>gZl}XpORaPzkhxqhHYfB7|Z?R4TW6~=_|Jdkg=Ze zjh#~@YcqtFfnSVH=ie(}CBu6v8%=*=0w$%LI{81|*}74kO`5WfD~GIhsDZL6jrDvy z#$3bK=h7sI3W|3s(itCSPgnZ_gj~8%(k<#*(MGmL5wKEnpwl>r<%hv{Ucc+pSN-P> z3lV@o!U0CypO7E>Z>lm1q}OM%2bj2(fM z+k;at*H1?p;D%)M$n8(C!X+6IiQjs~fKGEev%Lr(STeWCJKc%>i%Q}1_=mhuR zjCgR8#Z+E5kPe4GTMlJC+*BU4S$8f~o16)va?f+Jpd5E3M_=iC{Vm|MW2W?ib#p4h zRwnG4B+4%Dx0X#ehSsVggN5wF837?;gQtn@3wR!1nU0#8>-yj1MiE=O)Sm+e_uBa6 zC!c==N-MgnYTrtja;vjI8O^Wj0ZPyv$sg@SNsp2*B^Mi#Y4C>r{t-4QaB+IPQnEd) z=j`j!ddG*hBX*3o*-&zC_%+*2&^sbh4-YBYQm(f4F-dGlde;9`l?8pUdCg ztLx5tC!Fs_idg9_b@|f$GyfaSMcNqw6T{X>d-Y$CVi#>FB9toU{A9RJNZM$utif05 z^3CG^kxIl;`i$d7R8yTE$xNc#jMB4z@_@oaf4QH3X$+MrF876YBjqnJAx|}Q=4!c8kR=Sh@y7G@wl}LkMf-vGSYAz(H(N6N|90?5gt-nOuNY_?dOy&WW2?26U-zC;DU^+*P z&V_S?lz^BA=7*p*{aGgG#x-e&_`f?V)c_fGwDb3s10AFhIz)a z5_WsR`9zk@;nnxCFkFT7@9Teg_te$WJ#~tj^hWf%`IDyMUr1?Hq<~6#pj&QLrQhCn zd3tLDm#)`Jr6YXJ>$Y6PW#`P8E=?7)w>cgwv|`S%H&Yoa_asv086y6y0UH!1+tqpR zIRjRE`0J$jYrkc6)P=fu-|-Hw||5vk^{+Qf5eH*dRS zTU@{AVd5~Ij>l$@eCZpxoQk3!dV^o+e*y|LyP}Z3x6BXS^erv4KN8O0!bbeF@WAoZ z8>LHB{^%H}G7M4IL=yIbyGHWpW^qht()ZY*i^is8)p{UiSI}R-@+3lKp74g^$%uK; z2jG!7Ct2Mx&p9>gF(*z}u@&M(>P<0}w;k2^nMG#w6V98ZXTg-e{$=DR%<}fdv;tKR z22Zs(D~y0qdkS3~ z8c?O{_7$#pvum>Cs~qk(AysE&uk)s2%pU&p5-~yIcCj3YbQBfSd8vNL;Xj@-NH8Ln zotHD`vw3bVzdDQYA7SPlJ6-Imf5t_cEGEC#eg3+!$tdmv30BZWPZX6JWl%{D>zhr&JHM2DV>wklO9xgHYU^DZ91A7yX*!!gp8Xon2kS@(C z)^T26k9lY=4@_G>%08~W>A2Dk%t%Og)5)zz%;+olPxY=w9@@5!o7H$M1*xC>F@V(x ztP$sL@HjxIWW{CQVq9I5&!B9xUO^(jm`vHsAlA&S)jgc>B`~f0*pHJ;$E-1-;8gKu{~mBS9u{jeq{$My(j-@s*uG#H&pGy8 zQMQ^>!{j2H0H0y3nCsm`P|jf%YAWUQ2`;PE!?+V1zLuBo6M9AL-THL+jPLB@JhFmE zpkzvLl?v6!tGULZ`vPe+Iz6rxFfwATu!FXN}{|J4A)f^Vi&vG1IHfT z9Dcjyl5~m9L04I=7uiKJ&3ZL%U-4r>!Pn$8ub%A-@H2Qi+?u}1j9Gi;WyUt)YNjag zt+d|Fvg9_1?XINUhvoQ0fUk4r6?I%WNU#`L+QTrT_Xs$|B8*Z`wGihwQ=6qD{zEiS zqLU3{`QQ#n=fE_3X0(}B5aiV0oKl#}eUyVP-^fhvfqrJEAdQWanZJt`1#XK0&X~Mi z@Cx0f1rzI2?_xU4*-EeUCzmEd!>|cw^c*)g*7)2T(n%=$P2U0eYiFZcUFm`!6F1ck z_=w*8Lk;BFkM)p~%3eGfysSHgwc6Gvkf|>^&J*y49<`GMp#_=UML4xN;sSz}z7R9=gGU`-!_qrQQ`9qdQ18o?j%XiFIgS{lb(luFpx zw>cK62eX?#UvidH-r02?$yQ7qCe$$6I$5Q#*jN#t)(s3upKpr0dPUe4$yHqH1Bf1! z(d|~`<1N8E5)G}OY^<$bZkqZrD9OlFy>EQ1q$0gC!!r4Onv7;O?a2e}eyE_2xiF^Zm^`&8OgN1Bo{9H?!>qYLo;D5^S8PV zkAs2IN{qL*Mb}Y6?lpmv-#_ z(jcPY896y=``}55jCb=laFkCmlATLrq4m%#fui+g#?e4~`)p zRZYJ;9uJbtZz_{Zgg#4IwfM~|d~5wooG983*7%f%A3$rD##ge7Joe+izb-jsUkJ3i z;+tdZVxkGw@02j58QG5sBwB4~W&!Hb=gT(UnI}=RC-~Ew{6$frs7yG3-^w3!9)+`e zZfVeyCZS-=u>%6L$O!P3Ex2Y1KO2Vd9%!5D<%T|HdyVr5yASIlj8DC7Y3npY=Xb9+ z-;}1wvZO&|^QYu@#N7X~`32&-=bIM|XAj=C9JgzPe_oKFw>p&{z9WyaHR^v~cK!yp zKsVy|(j(yNE!s2pdN7a3O%Pt2_xgCdtUK9!vIgTb>#cO71{4o0R{`u$o`WrN%JP~yUG~GRzI2NkQ|NRYb z7FDBnN(jSfEgFuQskIEg?9$G7{zaSj*_Y=f`3f9PqJAP-rSX0v=WAeW{P3Yr`b4(L z2JdKH*w*4=U1SY8YA-tMdk0gIq^mP^qMqRb7SE-bS?2|7M$X@Zj4)C*r+5MMwWr+8 zJe1ds$jC_TUD~@b?BF(mU_$_azz0XvMOjQ?eKjvEItg2UCS2s0yMMg9C(YUX?&rHL z8WxCO#{K!1CQsyX?N6NfUBdiVFejx352sBU1km^Tu3wsPTpnaFG(d~wJ~-SB?F4!U@zJw=Z;P5*?^03C!S|7NyWQrJ z(O@Y)^urYJ7Fh&A&Y=Q>UWg{Ix*$~7ubFTZxx6dPIZCO?e|CXbd57TJVMhfB6xzz3 z^L+eaO8p9f@W*9=6QBX}eO^&tg+{?4o3x=EZ98+PfJ>pI6|<=4ocwo~sqT$Vj#9)sIOaVqOH^Y!vkx*Sb1@zg zrOHvF`z)dNbT5h#y)Srml+VY<7g%{J+e0ofoEquU`b3oc!0GynSUKy_OLNKjFC)nC zwT8IRe?bbT>A3#P@kBOZp1|>-pcS2nAiN$`J%%K6jHG!l03uR6_sM1cL1`o2>oP$T z8ORn*ThzHBR!cwBrs%!pR@|Jn=^Mu{A*D&Ik#j7MZfdbQsJb}zVt|nKH z@I%`F{vy9V*Y)4!cmEJE^H)GbYnaHD-MumdIGJZkXso4c>O)1z&+4bn7S*a`DBkrx z1Kgq?Os-TclKqieL_q_wEgtxu0GP?MIh8hCgx!g}TSbimajjAqxrN>l2o;X{{JEvSfSs{uTo17txSbdUr)Ne)Ai{ z=vQJQ;2-7 zUQfEC(cgf+VKQ|2^jd4IbUTd*CMIzRP!ymE3PODpBf$cq^Kt5Ju{$_qw!a|HRHW7g zrtxNI`PWvub4Ph5lDxOai#biLz7|5T!Y>RB$n+o-?9){-KEcd{+K79OnRx>*{tMnf zlbT=-w--Kx#uTpRr3RZiaaVj6ya}4e(NKBDp@0svq}r$kGxQ5bGAzx(_rZSyFpw2A zdx=^1$AqCW!BmJmxWT+luN1<9QXXTLaOxb9{93~U6McL35YPfKU~sawA3YM7o)$xo zIBzl?$hQ9XScwvHXbD`*C>Z4?g8Tldphky z@ioXWh zQn7~ZgF}lX!drxe61vVfcK_Q(v`h7;$nNEXzA@5d)n=@)dZD~br0*e%n8&mZ{a)m?>s4_&drUm339L1TJM0inbs~(n zp?$TX^DDI|an-?J504_4#fJwC<80Na4xpR%*Lm;v>`c@RY;8V_@J67jG4EglibZLZ|(0Jo5W~ij1K&`fS zV_MlN)$o5mqnD1jB*`a)W8Pz!@0Efl;_fTMeJu>hrDVtyAZ9Rg+X!Zxs}ijJq3;H_ zLoTP@rEvz0XT2|#@bE#nuKjG8$u8x8i>M6@eCNBf&7w`?SGIuOd8gz%e( zd!Kyg>t4ecQ2gF=i3iik|0!jN=i4>nhrg2Hy+u6kc+fzvtbyCZ0C%oS z{lCAqb1v6~nbA~d5>qpF@n$b!it4~qz){$)rPgU{XiUc`hD4`pAp-AHfoH;H=M z=~Hp5&_@5M$hLvB1f(gR01^vc=8TvUQ6zBJ=daNASUEjCT+b&fC2AIkX7d{;=&Mb@ zgyY%JF+{I{R>+YH{R!=3JG5w^#CuhH&39wE^r}eaA~GWYi{1z)I3;Xde{jU|V1WMc z!__!?cwNkB6gmu^#Elo_8adH0-tG_%O^72h!E574p8R#MvVdgWJ36?OkIgH3%Ap;` zXHg?oQ3fLd=G9Nb4y$3BUG&+@MPN|=kSs4yiS2C6y!+pZZ;a^$6&w$=G>3W-Ge;+kNa28-KLb|1ei(OpeS=fuvuspu2&Q#DkWVFe{xR z6!St+aTz9hUJ$ZVfae=)$mYw%<-z%44~k{iE2;>})(;2XAeK3XNs@!U_vhLGm2xUo zy0=Ze32&F9u?iU6Wji7U_un}Zu3(6^f=t^p=S0=FsB!1ODX@87k&I59b4qqEL4yim zz7CLYoR7-Rq610xRubT2hv9_GG1nwoF0)+$ghnPYHH zS7I*3uvEXZ^3ir9yP#|h1cza{IyiACFGwOfC}iiMTMII$c8>l7{C30B<6G;IMEwMB zxo0@$CcS4J?9;*Dnv+D!Hgm~2Jo4@jloeo(Cbgy}XS{iLL8wix`ENFA5eq3xl=0$O zKDYZIuJaM`(FFbe=k9u#42nORJg1HhQRCAjniB<*{!0A0D(4h?AD|Ux7GYLSRo&;g z>JLVAIQ~$o5 zKXi5Ihq9h{=fR$L0{g&*C+I=zwyv*Fj7mR*Qt9M4u?UcivYNf%3yN~dA$#|W&_yBgNoO<*3=uO3=JI;yPNnIAcB0&ntH=WJnO_1rgydzvW2DwsYn7839D`y zUti!XUaqnEW~Y5B|69Y6GjjhNl24-9X|ycQxM}V1=Nn&h90()+#PK#TLR?t2VTGCc zPU!^Ga8m>M?ja0W-|qa@6N=v`jYe+|>xaMLpgr@#m`IM!+byI)HAnNC?p+V0*zB1= zU01RT!)@EY^Y2d$a5c?iA16*h1z%;Lqq+e1P{?cNY=6QXo|elMNh}RR2@M(QpwlKk zFK%Y$A_A?#jo;MfHy*_7z-(58za2amGTKkJA}tA-!`=8_@3ctcuG!wn#F1(I=o$>! zH{UV%mNMsmL)L>0tajPz?5kCvuyrl~^R8$p3-v6BDhH7j0A{Ji`~Xwd6}lO^z9FUS zP7YQ5B)p;3!uR_Vr*ToxI7N=ASeKjV=r++noKHq(bTEx|5xqz5`+!-U_QK#lm*JtT z^yBYEPOME8+nRfsP;C>9UB=baLlD3XX|HDz>o(tZO1L=ZkeEee!hHbME*v4vEFyie zKsz#0N^wn%xUeIhPi#LsjOL|`ySzGJ9?>U7kWYSW4uWnBPA3Rxz$hP7C zx9Bm*RWkkWE~)AjucGlzcb_qUBWo!+K>EtI6 z1;$$mdk1}PL%*Zs!B8K=AA2Gu1!Q_Z02pHQKFCgFk0iZ7Wf-{<%f+@=OH27;PU0Ts zW11)iy~uw$ugCrBrK>#!Uy8{WzbjBbRd^owD)Mkx{+Ohq36C+^S}YMxX+t;K`Tb^c zMDuaHt6`G9m1p!9IvX;|{%V(fvVUz&hO2t>(WqNASSQL;ijvO~8Lxzkl#UA>vpK}* z``>3bzuW#h!O>@!s0weiQ~aQJOibSIlUXkwy;NQ;_^fH4U%y1*(T6AVmq>Hlb9$`K z8{7|bGJUmycd6#%Y4uVfH_n<);eWo_0Oul^+IaoHrnGAG#)aqAF_~6lw*LJ|C3UgBZWYuzlp|m)W2ZL@IZI3aX2{J+9a0M$G>` zjJF-A?g?&dhj<0nLE@N&4r(I|Wd7k9R8BN;ux`h^Jo)M0@voAmiio#Ul6HRzs(Fsef`{GY{x5TYwu;>CHH?W81jb3&kxAsXV)2 zcFtVz2ft>#Jbt=Fk?7e}R5*=VP2p;?V;)t1oh;!vH!>h18gjXY_1q5KPY&6I+e#;batab_+b^PSMyK6xLV&BW;PnH} zQ+yaXrfVmOT?rkP9nXz@ozn26NF8ESx;|MX%jujNj>yz=;fk2?F{(e4P@cr%Vx3mN z6U6w(^BkD?m>3E8cKR`SWS}yqw)VfA^mg(apn!N?5@80D>nquBlLaFe{A)`($Am$8 zKiH|rI>QW!slk1`(X^4yafYNinJ?CifJGW+l~YyAPT1b{LYUru2^W_WRFuOT@G3-2Hj1a%*Rvww}iPOd@r>dWg~HQuFAP$K!ZxW zC>rGaiz??iMXoFl6-%z|s2dd&iP%xMo&?nTehA&Ms{qB{S=pd{LgQG~U)HmyN2+W` zH5Bm*R6BCW#?j81Eu8Ph8g$|@IVs=e}= zX5mq6PgAY()E^=O@2p%B)c?%u_J~SvpNm<144INrc#n>hxj#nN_lT%jY5v{+Px}ay zYovcJ_j#9GuX#*%es%J1PT7}+AD_Q@!*c&LaDIs5b50(N!zMHnHX*mS$r>5(T-l*F zgbrRi>nvSJqE(~k1r;3`iHZ<6^@O-VPx9KpMZVde=t6lNuxRaw_zV>r#Qu#i6DdG#N9JgI4&~xBG<3TN|PqY`*-M5o) zT_H;RKD;44P0#q6e1MRCN1>yII4AOe$LpTU&-KMVP*X5>u0=#pT{;odmY5ST+x0Pz zO*jQMCRaQ{tWXw!VdWe_&Wal8U zinEW&g){Dgom|OXQf`&F5l3qZknASCd};|BC*zC6>*V#n9Q`r9mrZ&2=%sp>lz*QU z3>}GWMBT_u)f^(RO2}v+%;#;nM4r2sSO~hF2@SKns#%h2n19s5i4SAZYmHgE518#p zNxEKFzhuM@G3b;^tjd^Wz>K0>-&Y5R{DdY9e4;A&IS;eUq~_ln|KbJKXjVeY?`Xjk zl`bBRKt~GqkTHiVkfV2A7<0-+uUbKEs;hYVlR#fyRB>-hzNAEor`&laSGL{!^CunM z=8eq;{OVI9R1pRMT$eW4UBp#%fK%sOJQwwIywIDb&G3KW60Ee08R`)ntdtcQG`C9o zfs!;=r53;Y@9!^GM0I!_b<^h+6&2%Fn#d}dD=qunaJfrkr=Q~SKci4w8(dw1xjK3X z3Z3=nUkuKD%5o$w(e_0M@mF5T2}cf7awL^0rKh|KOCFNR-5+lP1f$1jK%Ry~G++a- zzo*&yJd&!p&*eMGC;A3l(Mjx7$a}pUp%qAgaUdnI#Qv;1p|0?OB}rRZ@Q? zKIN6J{$xN0K-z@z>8v3Q76L=e6Fi-UH8zoyBokxTbLK}(4KokBogG<4Ru*~x0``Rc zTQJv^w3wUu)^++VW0De2ocJZY&SrJ1?*52a1Q)`>XmL-#ak9@M@*YO!0~l1A=vV$l zoE58+;5=Tt82A1jDVuVe$4l}z^e@Uze^ynROMM=z_+L!T^hFZwQ6bSk_@0T>v$7;) z#iGI~eka+yDjiPvB->m@PAxn~I{$Wf^FpJXQGG;p1K^UOYku^3!>Nh)i~3r?y-`|a zTXEwB%bi(_#(U3LW=G%UmZQz3M;V?JG}4jYMRDU%L34SE6#F0b<~I&p%x%uw_Dv05 z%gDKMJm3}iaZXbA%lm+R9lW>7*@svP#p=9y0p+mJ;xFmr5galX&az!Wmp8G6+K+u9 z)4ZokD&tjh_q6Hq>SMQx-22|a9r-l(@1J=6Qp$4{I^;Jm^ArEJ?*todZ^Kl))O3nw zfg+C^U)f11PNXN+;J%RGxni(l#t}9doU7_x`Avh?Y1QHPi)*Bx*s{&>Z>78{Y&5i7AcF4X}JXx$zvx7ZAn@?-sf#dR;70Q z(xaltiuP@K`bG7^t~bB=+|ZdMOSY$XU|&`0ew^E{=<&N!C}ljmBwMQDv@gbO;b?{A zOyI}V@n@XXSAypQA790DR6LkWT)A3;y+3wN#gx8REbL;s!rv;fs1O#WEun5pz3Kj`$l~!QlSJY|u@{eC{>5%~9l)AnBYy4spslk%Lnh;Xq+!t% za!Fo}u-xJzeXopCIr+N@G&wPx6}_}%a#2!b*Di~5^vFTE5F&NTK7q-Gt^`e|@z=45 zz8asbqC|(m_uBt~3$2>#HQq^>_cS9XMSe>k9mCThp)cHDqElnS3so&X;-4J7`ZvVq zRnRBov9POrexwP9vXgFJFa&9J7>}NTL@5aDx;B2fTtMnIjwauQ_%p|y?`4yvIrA|dEce?9P8#7Rf8mr z#)V;@V{!C^*rPEX-mv{fJa~(y#$Emk&R7lfu)$LoCXm=*&sc0?9oxiZFpS z80YhmEhurIT|M53irK{;GTBD0O4TgI-|_h`V+L4Mk(eLYANUl@a^sU>;8UULAnTxT zrneTNGjJ?o*5r_uq;#3cA?bJn++Vd*&BJQ6xX)H?9zEXKrw&cTVBx7js2+CO{3_t< zZy{8V4J{c<7@(JmZP@>!hC=6+#C(IOf((<%sw)?ABkblG*Lh{9fJcg;p|tTZlvPg@ zpS_2_wZEb0ZqGW^;J4E67H1vNO(SsA*<{{8Xd+|@>1N)dbv-shJm0(BdRgXy=ntS8 zFFmNZ&5l;VA}TCk;OzkeW$-I%yDM5>mO5xAkJUp&<&>$fJnaL;uLlplcW~$;sevb4 zt0$5osszy}rZ9kO7i8_-m7e`FpBwH9ttUmFqG^zL6^4WN`3ety@uz2FHos7ODIma4S)jtPBNFf@&LbtX6BE8%Dgk3UYA zy!t}kbFKmTmHIanyt37(nnKro!V`>@^|^Z_jC!#!;{B> zhfNx;lC(!qf;SmNaXfu6TUy+3X1ng|gZzz00$o&~nOawjIJw!Lr|I$J29p_QRCCog z!0;^EeB^+RlpSX+?Dlt#gg9vn_sb$8RSr>=d3xS;~r_e6v6zaLlr zWt`}wVs?vX7JOTe13Ht_?A+cev;qAb(b2;Kx66wO%!I%?fN#tyV&nNNTB^QS!1U{1 zk$m?|S03L%2TB76zW}j>3jiJls!(gBDG_w;c@})tAmf007f)p7%+yk2zD$}4$H|}p z%?YPVl5u=}HExIgHDz_c#tqbw6aVgEy`fDv6eSO>JF~WMV_N5|dj)$A$^arv*`yxn z{jPjgj;*q6Lw)r10-6G(-gcoe6&i%Ve&c}=6`nH}rA1rOJ-<&%}*E~r@68NB3y z<54nTQ^F6E>%aIPE^bXVeaj(x0c(9pB-Mxvg)S*E(*x8bBSkfG`T~hVD@d1Kw)xsV zn$>y7#UBvU5s1@^A_ zUujxKP8@9qw{#qd)FNyLuPbz^p2S>^i;7ga5d?%Y7J`VK@qCKFmOA!xx-(>5PEa#W z$q{uo?@2cbSI|W7gabK7`LgqY zc5a#>3>insKan^wUe%|%j0b&A94XO`L-mZ>Qe4`w~dD>2*E%de>_{m4o`8*a0h}ROR z^k*y{%C;|X6eu_hb|Go<|bzuuYdS2U}+t4N$epL~mMailWg0(AuPp3Qq&ROoqoKb1G zdv`V3L?sDAi4>^ddjgm@4+ZfZ$OXyyy-LOinMzE{AF84 z;i|iVx6t}Yg+i0CeB=Pd0J2rUrc2UaV(rAUOKQ14_kWx2dj#_`Be1b2&*vIql`7`C zLT*&3P}_^*K)6IYI-fMC+6CubOjF|w%?vTf37(_tKnx}i%j|F(c@o(*2gn_%BLm(+ z90t26aY^uOx>K>@HFqjy`Ty4%MiS#*mU*tok!XJ)C8u?J-_uM@Zx#G4JBKgnvw!7M`l z1BLcA80#Io_utQnWF@A*xTN^k3r9V~Y~RkJeufRWq_J$E_{1a=?}eCC?}E4JaDR)C0JMWu(m zP+5kDTp$}FZ1O_sPcC?o^sSEzgWPR2Bhh%Xqj^U3{f{mL#(#H^Qrnu)?71B)ne%ihka@Q~eNTDfho2wa9|I zqt$H&`zWjcgF1>sbUg)Li)Tg|{;0>>Ct|58_E`>3d3+w))ZjQ4YYYF~`oQv_M={4{ z`ltWpJBLZ9XN`$6j4&aT*~TuJ3^(2*8u6K_Ozkv1o{y13YQ7BQ!S@R1KI*OTzxb#d zTM(|$NL50~9z+8f~P^(xWQ1!LHS3695(oDx|N~=`HH9b;IU?JRef;KJZ8%XCa3%$fy|96@m zB_MndcN~X-J^Z9*6o99Jw_pc2A;hRNint_{qE%kOCrEgy8f!Y(>qO;!mCoDU!?G|% z4_w$wkU2#h7a?>tZ8yd!w@VW0sVxZeuQ-MSxCqdv^@q#-#cLZfsxofb73yx zhCCBL1W)yl=+l0uCmGAdk?#N(WkkZW+Xah&_%f?%PJ}WH2E^3L-9GSJX#4Lm67Q}g zwEmtCY<)P?F$ZkN zJ@t`OZNdZ|{TJ*Ljh+NQ;4Fy(3=))zF+C~`S3(qJj__Hw^7&9tsE*nQeEILFqe9WD z5RA!IRaabZ!|D2%^#P4%YPeoTO!)=mQP9q9Ah(!)p&>&h=_Y1B zm`HHr=Mag(+2#j6a3(lJJ|d)xYG6pNKZk2Ne|7MBF070rK_rl76OiHY;lar3e=qfp ziOHUd%nIlwFz(HDI!(O*ej0p#pDQ}d;`g7}V(U+#_3J)CUb^Aj{E3bFGa8c0U_pl5 z-C7{yEjyJqO`&*kfHv0~Pn(+2O$73{U(@vXo$t(S2v0y5tJrVR(5q1x{GV*hP#N*= z7w{|}fl=4{a=k+n-s_wiA+5yNQXcs9-(_);K`hg?*B-KKWqB>N&ooe4Z*We}V{L|1 zCOZj_$dyZ~X$v*Js1i+=g+U&##!MPm>=w4I&rf#X!>SHcGr25(17Q&gjdtvm{{WyJY7flQ`1f>d8 z2&}E><_}QW4s5g43i3`Z03v<`k}i?I>qN8u2d4%n5h5Mfx=2AJ6byn>NbZP;_~id5B~PlvS6Fq;*+ra#RC~GsNKQ>*iC>2EZa6rpLF-0|W2| zln4Pl2S0$-_cmqbOHFin^gE99+Cs?4y1FH^xGcyEiEtq?PF9^OD)SF=6AtCt`WE9- zosxPy4)&z;d9uQ2@|*nzOi(0+S8jaX;Cj))T*84Aa<1aeu{Qh*uNa#7D?Y|kXm0y`Oev&C&DT)PaS#MdVa-(;@*9RGY%VhtMm8QMGbvVG7o>@Xo|IJ9?iRZF*y{(i#`>$U`aS%lNy55h~2r+q(~ zQN<)9Mx6t<=EBrYKAVOJtjOm0@=_wx*;@w9fnU^13bLOn_?UrjYCUK8kZ^@?8?FC) zBeO@St!(}GR1{+{BU{6k`C_@$iHC(Ov67ds`xFR9Qk-A_Ly|9R*7rTf*BdDr^ppEj?aGGNw!&0TpVp*cz!)kTF1hk=WeJ#SR0b6 zHQkH?ltv(?lIoM`rsJTgaFn_y?FHfJbD}hdyi*C(LQShm)Wtb3Hy7C5uI-!*7HkB< zG@T$!Jnd?l#jf@8q$8+$clSZ%mMiIxy(*}S4-Dq}x%Kk&-wyy4K*Z=)`DX=`1DTKy zQ3Kty8EdrvlP-Iz=gyxp?DMy9Ir%>{Eos+~CV*&qvfouQ)Ow1F@gkpH z+AvQz`Z)Zsy*27JSWkRdEU8G-DqQ~A2aoIuoY!8q-bvMQFOZ>5#(CxK0k5w1uEG(c z9;>h|-XJT9PJ)Yr%c|W^vtx4=LDM_NrFP@`aY|-m(Xe}b@G~9nBrd0Cocw+=6x8hW`Sv{DNu_y_K(u}GPhmTip%&RbWQ-aX zt&U5Jq>mbhmgx}oPp6Gh_nnR;W2QKGc)El}f|ry>M@k5uHhB=S?ISculuZNa-3>1o zwen@x%rwYH?XHKSbrt3(yY|^?$243sy$ys{RK<@@3x3-5Be<)Gadz zIW0J{<;We{&sVw@hJO>HsqWQyiG-ab*bbS?aYLgXZ@-?rY9!o?&fPTmHLA*?Qshfaflvnb`VP)ER``bjxM`2iyy{YovdIQ zTgvo;MfPoth-!W^JB6jvDgl9ZgC~`PmO$Q;Tm3u}fr2UHO zaK(ac@G(>~_t`*v*$Il+3{}%7>@ukJsbamg80e;*A4q0mk=PW2|K@W*lGpWlb64jU zT4w=d15FCPbtJHSOdlnzgvcZ>4LWdQyXovf686TY^9%4_X{{sSm5oX$<7n+|$nXV? z_jaD5b}qo}9ayH$u2OD4c3$T+9i)4EslA3M*fla#7HGtLmKfPFT9i=nj8!i1@M<8p zttDap01RlmAs6Ii3&XEKeO$EG{X`q3g73ZlU)n^dySYI508N+~@&)~%ut6O0j15*G z$Stm&T_w3W9_JJ{vUI6*huQ%{%{~)@VO=VGr^V`Sp3liX36PJ_l~7n!X-|F^$8+cv zB_PW{P7UT-+1(M=e;V`n^JI27vl0aoRwkMAT(k{|iEOzl%=}2NL`IWB{w8DSrKG!BAJu}p>Ma?|kHk`mA`fx~>R-T@zZ|;Vbo}?vhfJzpNcP`_i0@Gl z{>bV;w&Jl5!kyxP7aDULAZe1>pH}{BBR2Ytp5(VGa3T8G@(h;sI78`jm5FEZ22{2; zo{L=~FDb(*poCQrc$~aV1!R^5MD1Q{6{l&3TL<2@2ipB=CirM9tk6Y8b$6t3X=T%ZCkev@xQZoBKZOSAj zubB%MrAr=Sh6^oJ;K2A9j3)R^3e2xrSpIL)vljIf^?*k`}wL9UphD_JFA=$p4)Q&Z*RNP z^ftyciapxbK40RqpG9cFz1zLEDNYU3sK@(GUAC&!W;%Z_U7LwjzunKiD+~s&t=}Tk zUWDYExX0)IST@B5*3caM{_*)NlW^q4M^+5lOn8?tgnvx1(S(v>6OQT)p~N(oz!IF%i6!ZDu>ch~W|8MAJl=w^;%lD#+c1J4WrBz9R1Daor{ zAUTtqrsaYWLY4Cy>QRNcA;~Rhb=b7ulPMp7y!~U-5OJm*94fk|Di3TrgS zQ&$H@7Zgi_MEa=(cp&0YO$A-KGH|T1Sp@R@Q zgUcZNz!@rO3e`a4)35}i6#$WNGFzLT4!aB&<=U`m`a~`}^P94X-N!OF_k~dzI+m)& zC%g7P+%koK2^TmOdNi{=%BwQzYwQBM`{~N_BAN)nQmFr2TVmRCecyF42@ZmDk>{&9 zJN|b#V(cSCr#M8+gEG1u`5wF_ktK1Vnz4#lApkJI1tF;}W@rujQiXW}_4C{N%ZPst z?(zLg(I&qnQ884C8Ja~aX+Fu1 z&l)%K$J$ASfz-1OxQoTagnsdA07eh30NzWmAK3xinSHo$z9+V;*9^_3koJ;UVV6j_HC*3q-fkdjMP{z^3Lgp-j!?+*P%&uuuM@hU9-Kl9B*%t225 z_Z{snIBc6F0-9o4nk~9-1;SC3B)wGyRW`f_?k~6T0gm(NW$8tjMFtS-qqlTfDqa^b5=yvTAU_gStUJZiK z9~!D`szrz$iyIH5yc4ux3rSG65PrTovV1ej&;_m>{k|U)C%Y-K&|KtX1_STReDPi1 zqt0OO^^CKTB2Hj$(f;uQ3*s6yIq=}PJ^H#gVEr=_NHN0;DE`(g60`86`V7PW+Y3mc z#_?9JiHl}|3?!Hs)KsK3e_)s1l|dIV$uydvWWnH z`w=`a1|S%`Pslmc9HAL9{S<(FrM3jdmTcnI|qN!gz@Osi4>N)uHfMVtW6nmFSl$dBy+y z$NqQE84`}F=#CH9`|t%b05tvrCgvDKbC>Y6m)o&0&Sn#^1NtI;kYw+;KPaUCW&h2j zyGdnZ_Q4e3I0HeUg5CRR#yA@>mDTH@K8-rzxat`ozJ_A7PuQXod{uB1^D&lwSGQ~n zKKdY|7k=&n34g;5iP^~%Z-b>Q6X;nLg2`xkP5Bz~5h_3$0?%WAcBUW%rWOXP+yCn? z(u^Gcp`T_)o7g(C9tS?ZlMhCK5TF&bS#pt57I*_R)goX|Da!gkXsGP;3cz^d0N^qh zg-mp_n|@(yY0WNu3ub=d3s+;gwl|uu-{KpjlbMJGAm+5>;5>k5^%syv($WS~E_!Sln zIY)#GEs2l5i4O7IT|jPoDo&99|c0bj>fDyxF&>OQoa0&|Fgy{=~ z;rG;d-hBdmH+>0)(oI>C^1uJ7CK{RC7>8Jjr*fT`JmlyOO7ssB!x?+3}PS$aoMJWC$aQ93` z3&^N6q(7y%7OY?f=_%qW92QSJ^*8QrZPD4fegS&XYW(|h87t;S$YlYvfT#)v=h27f z-%^w*{aK8hF%C#cH1uPB(4T=6REsP^cE+&3A|BVvgIu`vA!sa%ma{Ox5V8!-@813K z@>+eGe*an_LbI1Rh~h6f(0%i8u(F9{v~p?aL%ZX}Nl1%f&Hzl)abTF~WspBRzW6)} zt_(eiN8kC*T2(eh?bIE3(M?$Vg_k!A`3F!kIoD2jWzRwyjTZyc*Ilze7sWh`!5j$8 zm@v476BN~RA)YPzPqyQH($MT-2_?tSA)vkeZ}pFBb)9NLr@lO(h(rw#92$UiFc!E1RS&I5|q5G2C6#slT|LX|Vw8&`07K;~& zw%H%bi_&lsM=$r?Ehve4F#4i)li1XO<(e2|4D6MB(y;#zJ~PVM-#ydxBrLgz(9Nc? zb?bK2uHj%^&e=FRubANP4h%fDr&4Do>mbfZ<|``yM0%6kH$UuF8Vcyq~q6W$8saD*!orAE_jf7BS?R=f}Co#{r zzGp}*LV(nP`@i7(HeS6Xs+V){Ty=yXdW%I#u(Z@sGnom~;0meTXPq94?cJ<3XR3XD z#t*RfoO}7Q#P~M{W03%w*IVEhjpsq%F9RioD*NKUK-dd_$TOp&RM}+C$Cu3U$u3iR zJqQNqPK8+pWfrv`Y8J6g*!AT$!#=3z+4ZewWzTNKbEQQkJ%cuy$3H&C_brT|G5QAp z-Dyu&_xSLc^NlLO(DVx_GpFzy+$9!nc5}RpkAuM!yF+#ktnf8=myc_DT#kJnGibH} zkg@0SwJJ_~vV1xo?N4W!l5C7qQ5Ta()o2v22!2m9imPO@yi?&6(01_~s;{bkz6p5{ z_G}64$o$Lqs(&6=c@o;BW+&untW$Cva^4g`rQMW!O5rKd6b^~3fMeh{oZEeTBaCvJQ^B|c43Zd`sP3tSivGJ{5j-sNSAx9Z z1l(QVuM~Df;0W1#E8XOru3vR zNp;*K3ai0bLLo|5as}0*d zbpxz&E8i4$-c|+Nd0dRhXUAJ#A7x)asBx2g$(Bu_roNw@*`{F1|5DR^y_lzI;d9d= z=aq)n@*HK6obla*uef3TpWGc!7<{981PY-pI)w_TIL)+5fP4z3g522)*((sPOpIpU zp42!B<-icLz)Zc!=(7>xDF0dIA}+m_l$!+oYx1Ba0~mIbKjr;R{@3uY z;I8r6!Uf;0i6u{PP33(0nX9YxZyf6v$h4&Mlx?}FK*tbfMFxd!C19em5iubi0>s>R zJgkxrztjvH=+3?)U1_;D_4r~1{}^2=V7HhZnlQhU2RZWitEr3MpN{v0kS>Yx@g?g_7V?5f^m5iF^0PT;iSDNN2&Xh@9b zlLN^TCLqU@n3ZwD=%R@V91NhpV5~iCJWHI=Vf5I;1Z&|`M*EQ)Swtln!*3zSE%`+W zlhfws5PPV*+frM19*W9G3IzUI1_7%PT?uop@wZq;3P>r)f{}&VB_T#Dy>E|nd@}ke z#A&1~^ErP&HvakKeB)H=`el%21a3LI97}c!@5uv5S-8ItQH;O)JN6Y3vjDOZD?25L z9q0z{8t`SQbl||8WXygp-tYJET`1q6c&pE0D=1Ib){+dKRs6n4xeJf{Qu%pAe> zYDNvwEXn+*-?X^Hgy)hBq?DuHKvCw#GL%yy;=Bsa14dY)ofNOru>g9{EZlI#twR3( zogs%Le)E}QUbD>4Rhgy4Z_0=~sPF7)J!jYYf}XV9-$V?97AAq%9Op7Qq3Ile9#-f^ zFPtv9i~sw3r6%Gp?woVe6)8%^iM2yJhKQ&th)9$)p#vf!aS3_*^bIMc+>#%?Z|i`n zC26RRH+hzZ)WT+{C$5)dTo41M!^1qhY}!yi)5WtyNUD*#YX z_v*kHZZp~60nY9Mm5anj3DAN1@+RmXgRZm{JX)=|1`udWrnO zt7frF0z@HL8`C>$K6X!YXmdTQOj2O2NTXud%X@RldRK1ELqNwRoYk|vhfuGUFF_^+ znUBBjTd2SPaz*)lviHv)5d=d$8>XB~-u>(y2qBT_kV=r##D4v$yf4mh6TU9|$BkkBdErgI zgTz?B8I;T+!$UQBzr2G!F-rJYX_}kkuQ#>evP2^X;JgC}1ysD-CWm$wkmW7?ZFCiiQi*MWy3a%2CvS}hz3*VEOgF%{g(*)nVM9!m2tm~Gr z)15bcGmcR~z-9}DIPhwk8QZ)f3krd16Gk(0;3*& zQ3>rLSti`sqN_>ANN-GMY_t3#m|AlfPqYyUpxB2quho$`G|(8zfXIMK;Y-X7b6xwI z3LZnwrYt-{kzMGS75r9!XL*gQmSYQod8w*K00)U#uwfLM1oI}`%ta&j0sDsYS_pOb z=)0Qn&g(i#O@IvOb&9XPt%@rsd)5R`HyTtn}mj;p8%OQ@x@RQWMojB=I+?#EC zchra9T;L{%Tr$+%#;dM}hzv8;LxE(l*>k|-ohq28p#&U!jDlzw@f$UmH?M<5KSp*2 zj&M)}Q(GK>tTRFTRTJ7N2GI}#$|>_G;j<>KVEg6*$#_-Gwuy5{DW|F2BDOBL%#_eg zJ(>}^U>}Xy(%QP_YpGZfub_!)g{14vs!{II`|&IaJIoW}=IYOd&zRovlWX2;O1lXp z-^aqz>u7Ws9`NcssUQWO4{aWe$mRMGFOsO0sy5+r3+b72<-b=)$cCiRS24U;jUzOWb*-rp0g7SF}me@`fiI(z!^bgZ0V=dR{Pzay8 zasM>B&JU%+o~3X1pwClx z*R-FBL_j@|U;HY@qMTUrLPer3<9gONqjg02fM-zvSx3=$)h=5*EGw#;jhN)(1{*Nj z)%Ne@d-M?|__Av%>|36aWJE_F9PcZ2p-UDtjqZSU`0H`c;NVSlphbL4`NU{G5~Cpo zvXVj-f6{;uYusjh9cHSu{r&T+nTI&G%-NPY6AtT?OC+!9D~6x{=xlu61(l9D1@q)S z8E3)rt>aNAx1UP(Ps6L?A?$N%U4-wuD!$^+YVA*zJ)DTfh!<5#E7$o9)-gM!$#@xo z9?$^As?XcsM&`7RC2(Cm*JcDRm6f2Z;}O9JZC%OWhBa(ib`Sf&}AacAA za6%&F?WLW5WS!gj2u3H@iMtBmKLU}KauNB6e~j|eK*jRd9iSBV8v2>>-&WFTC0uD$ zA%epliV9*8Gcs{$U?fWLUivC{4CxQ6n%F#Chzg0DJw!6bp1DBp_~;Pwd}90Xbvhi* z#39w82Gw!YcEcy!KvM}y+A6D#U82%zS+kHjtc9#GLh|GbU>{{EueiIGo!`?{XvKxN z<`_{f{KG>APjPo5zX9Dxwg*JW@V1)5B6cdDBLCu?=y0dRk#egDDv0QdtrKnj`83B)-M5hgvj(BPxb$*)>`VGd3b1t=}L@35U z?IO;-cOdMZ9BWfUvD6J0<3rH-#cASat|=pGUCP|Pvn*}Kau%sy@Vq27d?~c#i0dlpnA}@Q-l#%wCL2SShFp2LTSMj}#cp0$7ZS@cz93%|rm`s65+*x$V zSplO;3N6%v|AB&4JWlp=T{E*g%Bc=&{ZA16x~87lKIo@m))o96ifespdsP+2crC&1 z0AQ;Z{dlglbrpPh*DW*kt!r($Gn8loBG%&uPjo^zLliT6PZu1PzC z+6Fnqq_c><8QKdQI`&%wU+b#jTS(X!pjOo|29e8Yfp$BHN32Vs;^z4JLo*i5R$s@Xq`4m79aH0&ub!qxyaI{;F`q$tU0(#;O4O(23()Ld zo|w@O>?HJW+PvSaSD^ZM@*8EmM1MjAoizw4dQ59w%gh%zh^#BDNcR`zWSZEcTywDL zD~^e9A|;mmlyx5<)0581(|at?l!mKl46Is$s|aZ%L+n2k@c-{~ZveSEcER%Fwmfnm z#2Vs=fEoSE`9z1B#^cj+#J}^3#gh752|LUGUtwb?_B{Q?IQb6kjS2#9ssQFmLC`_0NEAaWx9~2X` z^0-sQC3-X1N(S)0`fB&d=uk{63=wm%Z8n4iqUVH-}r7&<<0v zeqrFktQR2S2OA%i37RzWYDLa~nwsTK%P0GvpAR2Yp4p0-Jo`n80G{~Upd;F(m_r@{$+Cw%RPG5$p zHZEOr%a;IwQx7Zfa(^E!14KEMZVm3J(>HQu33)u$U!v~_e8h*E?h{$@?V4|(E-0nc z4#K{70jLnH7~!SsLqjQPL^^fX*Mw)|ptD_0FoH^1Y9;=i>2w=*kSy#?Ogw^%fr;GL z10U{$kZWPK1)G1vxHN6aAJu@q&^~48=^JMkoFow>{8I~5^lvbJH0<{C;_1Jgiol)` zz#&v@sKHI%CPYxY-^^a7c49C3w>R-mpf+Wnd<(kC z)Kt!1rfxa>1jfq3$J@O*g)7|vcn{}WSD}4~;zb#(xyvK96*9oXmFADI+1ru~l07R( z%CEBC0910;%M`(rDrJYx0{@f38_EwqgLZbSfRz4)`98V)cky@Qp}`ZUNdFlRGD&?T z$|w4_TA|lP<@Z=ouits;%}~fbjt!(pC8^ z80{S<;xdO_#@V&ZHYp~qyuvc)&7Ppz0cNS;a4GV0|EEy@k(1RAZ*OS0Iz|xn2i#z~ zIYC54GEzz#vLk-=vgmSg7igQ$pxr($Xe^E(zOk=^84^1gGf&V*=Az2G-aKj1)!02v zrgmS_vexcoTLKZ3!vuD685X{V*75o8!Ky@g%P6H^2kBYFd3yfx0n=B5F_UQ0{hv156XQgjAXw;(&P@H001cQHI?rf%s{5E$1B!Is1KhW(K^ zJ^mCh65L)%O-aavx6*_a8knE3s7PmmI9ZqNBa`_10>zC%4Lh=9rMDvKPAvB3(N3!abkHCy4yjh;zeiw4u7|sM z*1Lt|Q@BK~YoUW0jB2N@X3vMuK0)e`!O84+yuYD@FycXV$u)n|wr7=8LsUt%yfN4T zxo*LK-=|%>&EL>Tj+jO9nKw^uHiV~Kb|u9?w5@hE;q1zwDkCV)Sn_;GNdi5{jNd)H zO38WXWbkVsnfEK}S!^qo0#y7Z|CloHj6L8j-)&d=c(AKbS`&m3X7ryY!rj@kzkImB z@VKA5Xk{QZ_QXffJ9;kT+|yAE96x{rN$V+*c#!gDKq8(%$-AkPEm))7W&=g27N0!INlI95_jxe*&5VPLO_0t89-%n01z*0NLKJ)yw z7Gaik&xO=&s$B&_Rb|7gP}(K1wf+$=bI=V{RGoMJU_x$R*j6T=JH`n zsvhkVK*CulS_bU6B;TT&K@^P~?7%s{Ou@13E4YXcYxTQ|A*rx%K?^ASMwAa#Ohc-% zn4Dpxb<_pI+myrSluSEgv0eb(iRvG`(gAST<;0S{q8qsSkV4lNRCMkcnNV%`s|BQe z3aJRgP%yfZntuKJ=gZl_Qywuvk%(T7kqYd7p061&aaHux>5E%mbRg}lovE#AYyyoD zNo&_WaJ@`qf-hr+7>x(Fyk>|72*O2}kVY4^99M{J-PKd3d;p)Dhz~uIBDDqi_eOqN zFdVGb1nT?FT1V{x_!Xn~cEd|h78`>um_~|>tuGXR3>ASV<}&)&L+?$b#!}jt|KPyg zVc`8rGq25$v=c8p9hO;>7{(jNaAbECO`U%0t?fM(O9%lK4(y;_te7I_5?FZulVG` zwh$%f9G-ke>{2j?8}Xe%*e5A|(~Zl-Zy+1O?mqa+=NXx4!(f}(3Pt>&v>TWzJgx6J zRQcTMm(#9>{*!=`r#>M=7>$RJGpYC^$=p39n3hF5%_vP&bH6SIi<#3tw^Z6%+q8D$ zyH)Kx8FOH#R1JBRgLxR8b?o@D{`V7IWXZ|Um2;rBbA#|UVdMj(AIR#+eM$>=i3<;E zPb@MaT=|u8h>ou@Z+UXVv}PQlu!Tp5ZHOY%w2yxOx|&c@tr#!rLw_HaNP8-6p=b{h z3u&nS4$E7cD&**t{tmB_gpIm&KNtcRgQF>N^|N#gSN1R;ZuXc(US|5!pUP)4R-`=_ z$=}Fj@TRc8P&45Cg$yld9jaiFTNk!HXTv<6`SCF{Za%vBwmIhHp0~hSVhEk>KoDp_ zg375k?9#HDNHS;(_kscmrPf|U(l=MA^L?3D6r|-TIQ@_1Oy7ftAZmhUZO^-#$)F^+ zxsIA#2xD)%DJUO+8ei861z$5N-WIwH4bHBa;MV$=vWJ$=-0Fg)+aWkqo6U|&$MpE~ zwP8b{elw)wp$wk@1BYI~Q@ht8SLaoPuw;-xyPBmyulWG;0oPYXQPskLfR;3 zjLL(ifV#(G&cHL4GCkxnO;EgiZ(J9!F}DP2(TPfe*W+T8$$K1{>59b{Un^kcR+fFo`#pDG`tvEO!0Wr z0vV}N1qJQFGB}310PszRjJ_gr9)FJ-+eB^lA(n5dQGM;mX+s86I$3PmM)1%-GVh44 zUHcr24&|gVEXKf+qeD)$7fKh6|*HRD&l|>BGYqc1irQnAtEjjbxl*LSIB)Q;%%l-1px@ z!TRhZ?cdR)Dxz+CF@av&z_t=HBMnWEf7NlKTfxSNwa|Qx?%MTmI+gnj(n;v?ma%aw zw<(lBycuOMIBh~56$C&d+@C#p^=V>S{Et>gZ@wAj_$d9Xejyn1<4>1g2m38`egRp{DF z13$jo*^Yk5;@MygoIGXvn!nBu$Y;fTC#(d!3fn-C~ArM{%6 z!;O`giK&ng(fSrzd%1jCH+JH5B#aSXcP#%U@BZD2!Q>{U)ZkSD2y)lsNG$PpVR-Y^ zM_Woi=;_IDtzNG|mVUQSke$PUEz0z>p#>@G96}CLnQiO;tOA2pDlmxLM{Sq+0zJ1Y zHc)br7ePZCe~~nGY+xv%$rg?ac@-jcG)ddAUY{N{;bax)eBnxC9Sl>JN`ry~P8ky8PYDb?NtJe(flA_}zdxwVx&MXwgcSpR5bbcVH%c>>yC6fq zUDIbbNfh{*b9hWf7<};$fenw{4_@r(QMHfTR{B==3t8WPzEU@5?H6`kqk8bYncG$0 zv?i{*vQ+C+1Xup3ds}kuw?Z%YOsK(B^Oe5>lPfMrcRHp*!n~Zm=3F|F!(a!-8S7(B z9PxByeo&{(dqja*3ee+$4cMDJ7;UYTj|<|M80 z@lVsMZ^jguSdr!7pWMY+ZBJ6TcgUwc zC*UZdhBe6X^japEZkszr=7z67VnUaHUBbI4B?x$o1m@m1WMz=JVT32aB^7Bd>k(Qg zDBYoWhYk0`#&K!Evhx8fVu9P$D0kMuuqjfJ73PqtX~j1)i&y`c`9Pxr%1AP`T#za) z{=5&s=1n-TI#v2_f4wF9F9GuJ{{z`tBiVJWO|eK7nCXz=>PYidsfUW2-5Tf;O(`iYy@&#d!hI@DKlgl806 z4wnc;75t?k)Y;nU1C)oVwekGqppXigdzVD=E5&`Q@LXkc;Mkwf8~mgy83EnM zXtdJj3ZW9mt7*S_zaF1 zb1p+&Ha`Z3Ae;!xpto;Ks?J?Db(z*-SuPXUV_3>vl(KS!?^P)9#q zHloHq#R&D#ca&nM_bp75n(8c3Fp94KsTA)hnk|LVwkqZiyV>37EHdq!;;&&NnvdjW zh|mqzT?7&$T*671AeV6EZUS+Fyv#CPJlXE#bGhNP^)&CjScz`kB;F z;@2O&reKu`kfW*n$i^`n;#&pI+cppzuLIZ86-ZkkKX?2pvfEm9L~&YD;NUn@PvXJW zUr@T`#%dKzw2VcjLQ`GKG6ifeqNU~+Oa z5<;;DS{?Fi=lhT(<@I{f5f0LxRqccU_X6sR+7o*Z%s~d}4`NK?)iR?uGk~Ciu(q!F z+v`Xr#G>;Ye}pHA5k?L$PJ8XTUQ_FM2?uODMGpqpN+p@Cg7{*y2LjQ9QaeH&T>#x+ zqnT?-!{wSBq@uut-rUl@9g69s75J z+m~U;-z#CtqGR!RRaUc^T+y5js+**cEFvk12|7|;8XV|%B)j@Wt4U)CX+!<|;0JR< zeDoqV9_W79i>uB?fK>aAE6&iut-c>T#UCN-4r&0MI#QQ)Kzuyiu=0;rMcLo{CjwDZ zGM3=7#0pU330+9K&(UHvJO}TQ>IYA)Y@UG`YRPg8x^x&*Gl9xE7jYMZx_F68jqEWc zLL23%5!UXunASR96JoCD0Cu({E~mWr)*OXlf&4RjYNiwuZTnXpkzp#a65@_E{ics` za8sIBcUm3@7?;bOQ&3Hi+Q?6||BxL$gSsZ;qXfm6NF(NL<2n!&V2lyZe36)RGb2^; zo-PX({lP0@wB_)?FsJ&;7qiE+N)mEXHZwP^fO2#LHsFgo{~Z4`2yS-9%RH;Nla6q-x#1>ONJ@Or=yD zy&Y>w>wp!uJ7rla0-4SbLq44P;V(%otq*=l^TKLxsIc`w7hp{ zBZ2i^F{e|%QU0DNP!eb@{%BUFVi$~Nm8vo}a(oz9$(|6X zJu8A>Is5^&P-lPXf_^C4weH?`qdJyzaMx!)Cb8gJakkQJg8!_`cre8r&V~mAB;1?E ztlzX3OaU2y={FYS@tHs5m+%<@ubt+f6nb)v~RiUX~6XGdfz^tj>5+b)NUPUPgmbL+V5JZ;%?S+JKED{bAP&ZmUdu?)Gj{i;~WhxgFnwAlZbAutFA zNKhTeZqc62!#KWJEBvkz=$bW+bqd#CF6H#!R>67u4p8aJ4Msv%T?_^=-aCZ&DJpPI z9+1c3)ttDpq5q+}g%H;Jmb?-85{L~fE}v^kR(f9ldFO+tKls{;rdTfM2Q(YpnaOP) z`S^}y!3OvGVGf(MH$TTb49;Rn3SMeCtQQaFdI+d*unB4I^{&GSZ&-FCFxsUzrkk*aA8AN$izkWCBqB@U3v@gZ5WYjiuxlz!LPZ zsCG|?bqHVc{6Fg+mN*WCUv2?fs}4W76$HE5^}b&_3>v`E327m_SiQ(7LFH!~j=^hb ze?L6G%A1$K>wOVDUSb&e{unGSiv8GF7hoEmCjsq!0fOz8i!m`nW-gSv;3R_gMqwUY zu|4y!TI0E)aEY9tV(?puzy3VA>$y@(_-c?brsV;6s?Sm5J$xl*4t{O}q1o z;GsZh!-KJ*Hozb~Fdgf`Ou{fV;{@)2ri9XEI0wx^=kLiX0Vfmcrpp*MpyeaT)4&NQ z`K>q)1PE1rd-h{M^+l3d!9%%o1ls&G-e1>fkRZZ)V6BU_o4_=iGq|@{nZMGz?Mh6i zC^zQx%!~~%e*ymHZMkD{wPJvc!nPE$l}tsgJHr4m82ygL&^D4AEZcA&rlP%Uk@XN3 z2^MN0?`HTi(AO`dO@J$fnJQ~g8r}<0P*lWL`CNCW0VBA+Awd#ILWad07=AktAFAed zE2jT}^36UeR0@;xu{ERl%-x`BtTL($fNW#9d33A&XV)$`0xKu1z|`MlVZ|%lX)-+n zkC;VrR#DGSX#qGRKj1PzE7>hY%M6KL;r(kcy+FZr9GH&%>(^i107ug1b!jNetL4@c zGIeB|BSi{+g12v0UC8x40E&a1Fz8L>14b$leRGDsQZ9K9{6~^k>7kwkggZSmmO-)> zT+%Rx?K>8wk0|6b!1HH-fgW$OxkAfNv3hhKHgk3%F9N88Dfg&b=)5PeLS%u+hx@Z#}(AjiA~QWQ95MtkEitU_0hKVVMfi{x%} zz5?^WQ)87RxwjpwZ=eZ+tgmZb%y?PpTI)%}KUX1hf06h6T)Pjog^Euh@ET137Y!!c z4*$fT;JMKZ>IgU^yFEVi$wHih7A`0S_+lxgu`SgH{3j3oYWxkuy|@m)`fzN*S03;D zGxsv;3=CnW{p!XS*zG_Lk(P?go>84I8~BG`3wY2%TkXfb{5YG^dVcKW@H_E#j;rg} zof^wsg}r-ws=g0U*x6~tX1#<>%+(@X)@jEQ+0w?}{h@Kg3cO8~hthFZInqX4G{ z3m0p@kVdMr6aWmEFZc=zJXS0pLfbwdZSIe3I%F{p-I2 z<;JmIW0Ti;EMCIJVrz4)$GH=#fGD!W0sf{<;eGwOIWP64pzG>?^T2A;48w&uRK#d= zDQ^RK36#yopz+{Kxh3JjJRRmm4dr`YL;BQVG6CqEFJC!rc)}uHB!|OHm%qS*4d-#T z-usPFpP=#8|8*AaVq~yAMQGvKMLETkp~%o;aK{VwoKCW^-K|?w5ylE?Xf}}I;8&U$ zz8Xhay0(q+U*4DYPfOfue`Xrp_cUGXnA$+mzsnlef-#S0iT6O-x0v*iFSOKCv{uGe z28xi>Ksjv3cRlC*qA(wK8k?jyj?_$f6-F>!0LDa~j6;r=^ZVV@Nqrwfdco=LLOes# zhg>0kFy2OGr5-%6w)emi<;^bCK$@&sxoz$3H6q)J8i_aV%_U5KXhooK{k~DT_ZtpI zlJ#pxIrR$XLxhv&osXFt2QVk!Ee1}Ha>&{%kHc7FWc2&|DQ^mM?qSaUd>Qt{1mQiQ zD^AHg8An~x?ooOf-k#(nxO%ETE-WF~f*R)N)htqAazT#d?&7V8%XR5aTIXRWpQlg_ zUi5$v%fE#>Bvso1=rPddGf0bz^Lfj{8i#GW0mdltiRo}X8T!kXMI^){wHQKPF4{T! zd^GgSTX9OUi;mf~e!tJ`F!jOarDs1uEYmAY=1xs~kogzmcT&Gmn2|Z)A=Sf>R!5x3 za%S-?iWpm=j}e>3&*y0V@FFmMDJwakdAW8e{|bR=mIC=y-N9{t`TL)Oui4*|Vb zI{fa5)qx!B74VhuCTVZ;6T{f?1@@5+RwoEUVlg!~Yu?UhxBg;!o5IX(-#?F&k9Q!X z;IedE!M)>~Hb$;<4jtWcF+8(t6BWxCb(X%udMU=8#Z2|M_pK{9Xpg5!X`|_-LL{O@BH3krErl+6H=5IV!XaHAB4VNb?_>1W$nVbLrYM2 zhEHn$dK{wjm=?H$nR&j`!kfx3fy#4NOeUYSO4lIsl@SRjw-$1p5p6Eoi7M22^3pyq zB<;792@MHWMdE@pihJp!92+|0d^PWBOYw#i;lAhJXv5vJtF@W=dmCR`Fj}WAX<~W2 zqNzKn^Kt{Zn-GTHLQKUFCX<5R{;n?IqWCaatY(1Q<>|Q}qR-6{8+t(;+SN%O8%sJQ zGLrHXFMDf@Qj9f+Mn|uw-cLcWV95H0g-Qh@m@{86EHlNVS68hJ(BTeCHuvuht2Jma z#`W6<5ukT^$^YE9wzv5hsN6y8kV$G15`!m?(?z0N&GO^1j$^K>tzZ8JS)T!wE9t2< z4T>Z>Oe;oY+3=Pbr621#AC3eGvw@8`4Ze-U_*6y4jIhZx<{X-c#wc!nd(1=s`fQu( z;`nQ8C4mvm#yu<~Qn<#c+i5xGQp7rloVP-!y@iDW&D*#ZkA80r-P$DZi`gEy`f}Z` z-_pCkH&%?3^O;??ELtijJSk+FG{Pnnceh9N8zd^E3tHj&k?CJNq0qj}dah*-O2HR4 zKB`uG0rm@;9cRB^K4TOP=ES6Q0Y_v48L$K5_8+6_Pg@jtY`62@te!bHW}Zd_)FAJm zvCDa=#9PaXSJbg+#)<`REd7~fnYiWf2P*zM-PW8{QjGDw>4NPU6*nelPo4z2{DW40 zd5qh<@-mWHOULDt#xVM}6H(BhiCjm{PFs7JuTrx{-dtOGm^dYlmaRg>A{~X&Ai*GR zUIV`N3sAA7GNmCXG0^lW8!9wbj4lL_h2QBwNbpg;IEuJ|dV89`PA-Lu9Bh9^cukdF zFY}_n1r5OjA%O^F(ED@RR8=+SZ=`QrmJ~|AXfEg&<565{TF3u5R%>xJ&j9lfc*X*CIcYC%=jqhzR{29+n8+5L$dc59)%S)D1B(~#dg>oLOuM!~K3Le-zW0>8iKb$60nJ@ber_j}e#G zjewV!`MHEcfgfCFUpm78nY+Wbpf(k;f9?nJ-{;WTEJYqi9Q@eGE-jwjTT=Gs)L808 zv0AWvDsmT& zyl>jRmi*m^FYwKmNb+o0vhS`tPJNTsdqU*xakK4SSGuWEwx-8mv{HCqkXT(+jwX!@hNuMH z(v?XPjPF0P-neggOv!PP>+uvaWZrStHZ=S$8D0TaN<19C92Nex%jB5zBikObKqT>I z!;X}Fp~5rejlRxMqNB(T{6amgY;^tvHz`UGp`jf8M}#pCs}1YN*LY%lU(9f=&R(=6 z>P*Vn`bG|Xk-F!evGf8?7N!?ZqTijuFlkh2$9R!i%`=L!Swk|uw|xzfReL6I`)0S2 zbbahguXZk?r^ExJ=)ahuz1ycPh|`MKciWg2jIa;Ol}0IsjE|P?C-TA!_b-0)49Ylu zcI!KQuDT_iK96zGB-7Nc6-V1oqD)2rG(ii{nHsMBm?C+3sl=iMBdru(iY9k!Rj5^3 z9aLx5cb;!R9|u^_WF(RP4VxkP%tC^diu@1L6Nkk<vY!&P_{$yopAJ<%*Y>#* z@iu&(>?&J?^XvhgWY=z^W#!RxrL=ow2$dg1OjtzeP4JOz&Um{$=hB{1*Jju&0e6v1 zyCTkGPU=MF0}80J3SMM1?cV|`KZ#6DgvsS&lf4PPn;4Hlrr7@OdMU}$!@Dk8>G!sM zT6a+$GIADsmqaI5z9FI|`ud{rjD&R%-T{>~H3Gpn>ZI$EG>FWBsPm#7#I+)WdNeHb zPFc>l-T}`U3l5;07=9ji1g2d40qeh*#J8-jJ!S!`-ZBgiXyTQ1MwFjSEsTj4RD~*> z!ozgJP_xRwX~VV3aQ3A)=z6!LwBO>r0$@+AasL(zeY6k!)G$a}ccj}xJaeb}(W9AQ zu{!1BKi*MB&+8N}%n+a168p9aO;vvTko0X_Ut4oR9COgK%XPr7Cm7l}8MRIs@6eE{=PP+F zeC|7a9)p8xdS%*?@_fRIXJ|Q_Nh5zG+k&g9V4uPb7;hwAXffL6OB#Md(V}3v>qb|X zkJl6YiU(GRo>x=*nR?A#*7sS%-n4C-R(~`i?SUY8 zu7Y7|Zcpe{!wD~To{kz6kCd(hC(H0HMsjSV$xY`nwxCLHf+~ICw%c(OTh-7bimK} zA~QnsRs<>}5y$9B_lY0qI*qqqQY=hd4$d;Bqp4W>(y7iPP{VZ>J0mc+ZK_)MmX)9Q zr+;fb8@vJXkUB%Dyz~{D`S#aW&Z$WS_*O5fM*O_Qt>LS4b0@U_n@X`%j@ka%)!>)Y zxWYHiz5t(O`7tC_f=&X{7-1u=L_TnK?Vu-@g0z<&QP*yiFXZ@@qBiLFwF@(Sb=S{X z@D1kJ3e}bK1QRompWg9aDAPHgA8E*8k}cPiTb5^bB~^dPL{7$x-Ax{)eupoct>ns* zbsd6GxVM(e?K_u{P+;e~Nt%})(R)^HaaqGUb7?MlE|Dso!(G$9n81_a^2-OJ?Lz48~H%tS6{cj^z#$RINXGPNe&hRYq7WVP)AstN_m~$yHaY6I>5mbmns&N4qw!>Vsk@X! zM0$BNwtx)(sqKhXVI7;Bf>*2HGiPHyMBdT%gwVW(m*rP#&5t5w!nocp$MWMAMghL3 z3F~QJX^u>8s<@ci3g>Iud{>uBEY=&msMS`gNPg>edo5_NN=f3lBX|Qg$C&J?8#ra zQ?@*GJCyd@yM3iXE-gnQeTHLnwlbfnc*v{jJ7ECgaT#rHOw0}$D}k7n&x=Dmm`Z5 zL!{0Ps_LVa#=eT?Kj}YquBTZ?OQdH#C(7a_znmZa^inz=E91-OWYnbXRoE?M-RZ$; zXHADb(W4YDi;!lbnd`*l)GtV7GxHSqHOmVIwok!f4LQ#6vrw2$hvY{9(k@z zsAz;=Xmj~P*q3W!?ZsyrQ;1;}9AFX7l8`+gZ7u()}9M^Uug$c#r`xJ5Y7M|1QwJzz?Bts22bn;odIk+(F1XOKn8p1 zMI4v!^&Ygvq{`;QjLH}0Ic`Pkw!}-F`6_E-W~`~Cg$N1~CR{Oc9ht^zQ{|c5>d_4) zEXq%bQAaPNsEayVZ(IGKWM`2*DJK(_)K$vr-{3uXnHB`CIKJ;Q(p;=9hvux3k~`F9 zllQJ08-s)BZV#pzkT|4>uoXf)@)|wjeA&B$+kbnf1TIdB74c=xObsnSyVY*KfyHp$ z<8gwze`Tx(g2Z$f7<-jag+k}q+AEmF*O`|@9xSxP@$2Z%2Bag@-rp4q@`4nq7=Y@Fvmrgc|N?lj_!%LQ^ zRu#HS_Nxg^wFz)=2v1ZMWDO)+3~M~hXfRegMl%8U`FMp(R1K%l(_kNUDtRG-l#rQ) zWFf!@B&@nOQ?C>vXi>MY*1ZT7>eNxVM^OEQVKx7`seFY9y8qkPF#q#?hZ!=EObiK5 z?wH9o*9`S+8Te!2k#?*GkRXu{{&CD!)u2FL)v42tJ(vTan`!0uhpRcN8rng^OoAC}Oaa%&$-h-pK_X zy0)&-k&nCpTv9;SUK-%}t#>tU6^OcQAnNk4MBP%&-H{kdZEh1*_;H^wI%lDwr_m7%cg*CoRtMv;<=42H!(UQdp#k_#^*FRCc??h|+O1%UO@l57rDwcqe z6jsABlIE|^{V?81O1O%>Cl_i6c0*eMmX`+Z1n|vR-4B=0EE|6M3t=Lggsm|YJ;a2+7Lv%+Ls14T&C-C6>G9^`@GhT$)% zgq*SF@ZU8qjy$z^F%tg*jlJMs|JMt~990=55Riec!3PWjFh&9N_?^AQQ0#+BlBGW{ zm{_7$KE6_4)V#+9-E)l7mf<1O0dd&5n{nDT6!&!RjT-!kDyvJu*(w0qOZksUkrspV z)4@W^?sq<5wVLJ=l{LAW!*#v(s!ozG>{;m?IYP`4<(GSqEs0kU9!JC#c&4Mtqn`Zg`b26;@0#=*{u2JX-e*ZJ zedKYEoY;5?KdVp3q+HX<_m!XuH@L(8ZDG>Rb)0JV<4Znw%&3qm031ZVuyxJq)FDYiW$F#++ZC zucqs*PN=y+F&gf}g*(jz@*AClpJ|ps+8~|dzx26_)2<#%ce1>wuI>CU$+>+I(U5w3 z%e(|DHu}3~nwpPbhVPuT2|`ONj>iZ1C}XabEUIn43f^|`mQUMU4QvGsgftL9{+xaX zG_TnH;phUT$XoUA0Tm?Ww)^$NSbx9v&6AMnZ6N=AH%3MHKU{EBJXeji-FzKPuL6M# zmKK66Nu#i2Rb~`ppfb)tvyV+~prvqn-`MW+Iy$bO7dmyqFMFm7+wEeq4b3w?zW;Nz zWOkCHK_y?Pt>~sLT})UY(#fz>=3^IH`sr3wn|Sm&JA*Gx58gKD&g9++CCd8OL5AW4 z?^yfvR!I$PIBH;1Qwva^$whkjs3GUrw?R4h`^W49SgTxbKMzemXsFl^6Zi`(!?Q>` zx_0Hek(Jfv&yk@?jf$xi1qdoqGIE`5?v9?%3;G}e-MS~i`|lxOc@=^G!t2IiQnjMd zZ(lSg_FpNPcoBS2piWTiY=v<2XCl@MP|be~GfW~k{@0H1&rw0~g+%6-*Ir<(J6o;s zNbr>u&D`*S)DieFLfZ`NCd5&;t005@bP9r2dw6-L=Ohv||G64;r6!P&F66iTQZE)F zs0b`Y;8z7XP#?!XS4GH#JIw+6M{W3PDeOm45tgUJ4-~75u_gbPj3x+lAqVH#Y^P!N zKc9%1(g4aU!&JHQUtidNx5LBaR@w6mE4N13>)>F9X(m$L&)AIm7 zRV^*-Sp($v)`0G*%BjiG^qs4BzuT>z&J}?t4 zSTb3x+u?^eD5wp)wgIu%>zWc@jXu$0{N{>` zkSdG4tJs-2U~0-G@hqg8PHElRWzsLVY?HruqZ;BY=_Y=-@v3s-vTR%6LNuJBY3G!k zhwZN$Q^`~dPMzN+wK<|2%C-$S{=It@$TVE6vXztg#T2HBR?Tyb5<4sfJe>P!10)W1 zypUr{faop;Yw|vf8h9EeQscq?wbd3kMV?rE1qj#8nVQ#1zgk6sy2q7eZ!Mm0!JI*A z*$C~}o>2dvidtZPmlnr!Wbx8ljw_DXs<3K>DUE<9U~C7L#;GuH=Rz)F`@Tkr2GVsLld(F?`kUr7M+9-_Uz!U<9a#0Lc2(Z0%@ zep1NeS#Ca$2oX-+*@C1cn-X1$RJa4RWEhx-Y>r?f%N= z)4`l72ASpi`xwU-a2Ac0Tgl(Z^`DF(GsZsWQkK>ZvdazzilsO($E6%}Oozm_`kH%_ z78uw>uxd}N>>Bh%?|~!c)-Uvj-vOv`C9&7Wsx{`JLs;k~F&8YGjh$5hrG?u}wW|AA+g4{u7WP)#J`YI8P3AW{e(qc* z4y2IBZ1-cPk584>J~ZRb<^petzCbF$T1bVh4=|7ro^EjH>BL!iR|2*#kW8;#9ONyJ z5&yg0p`OGQm>t$aFG)DsR$LfaV@Vqz$J<8Wy;-dQ3=Ngx2g)LRzP^xNTllV4_7iZ~ty z-q%_Gj~vDoSNB72^aVAethf}H4dO-TDQzcBXg_Ty}23OCAX5!bx3YaZhD guM(&{r1cE%QxGXPKF69r4*aL8sHIRQXBqT=0K#T@vj6}9 From 1e5302c9a20ce62486c6490accd5387f57c7330e Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Wed, 13 Sep 2017 13:40:14 -0700 Subject: [PATCH 05/94] "redraw the graph" --- paddle/framework/multigpu.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/framework/multigpu.md b/paddle/framework/multigpu.md index c8501725f5..1c843326ee 100644 --- a/paddle/framework/multigpu.md +++ b/paddle/framework/multigpu.md @@ -30,13 +30,13 @@ As mentioned above, we summarise that several kinds of operators are needed. Cur ### Graph Converter -To be compatible with parameter server design doc, the graph converter converts the user defined operation graph into sub-graphs to be executed on different devices. +To be compatible with [parameter server design doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/ops/dist_train.md), the graph converter converts the user defined operation graph into sub-graphs to be executed on different devices. 1. The user-defined operator graph will be partitioned into sub-graph. 2. Control operators between GPUs will be inserted into the graph. - *Broadcast, AllReduce in a single machine. And Broadcast, AllReduce, Send, Recv in multiple machines* + *Broadcast, AllReduce in a single machine. And Broadcast, AllReduce, [Send, Recv](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/ops/dist_train.md#graph-converter) in multiple machines* @@ -53,12 +53,12 @@ These two operators need the Multi-GPU context support. Need to notice that Allreduce operator force GPUs synchronized at that point. Every device only need runs sub-graph in a loop style forever, the whole training process in asynchronous or synchronous mode depends on the Allreduce point in the graph. -For the simplest implement, when each GPU compute the gradient of `W`, followed with a `AllReduce` operator, accumulate the `dW` to full batch of data, then run the optimize process individually and apply the gradient to its `W`. +As it shown in the picture, when each GPU compute the gradient of `W`, followed with a `AllReduce` operator, accumulate the `dW` to full batch of data, then run the optimize process individually and apply the gradient to its `W`. In fact, in the way of every GPU optimized full batch of data, wasted (n-1) GPU compute resources. We will enhance it in the next stage. ### Benefits - can easily move the optimize sub-graph to parameter server, multi-GPU feature can be compatible with distributed support design. -- easily plug-in with NCCL2 library. +- easily plug-in with [NCCL2](https://developer.nvidia.com/nccl) library. - GPU Model parallelism becomes easier to implement. we only need to replace different GPU's sub-graph with different part of the whole graph. From e0a8b5915dd45d8c732a28d09d1026e78e41e341 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Wed, 13 Sep 2017 13:40:39 -0700 Subject: [PATCH 06/94] "redo the graph" --- .../images/multigpu_before_convert.graffle | Bin 0 -> 3056 bytes .../images/multigpu_before_convert.png | Bin 0 -> 33557 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 paddle/framework/images/multigpu_before_convert.graffle create mode 100644 paddle/framework/images/multigpu_before_convert.png diff --git a/paddle/framework/images/multigpu_before_convert.graffle b/paddle/framework/images/multigpu_before_convert.graffle new file mode 100644 index 0000000000000000000000000000000000000000..6c35ab1b21fb76ceae82d3693ed0d085b5bc0855 GIT binary patch literal 3056 zcmV~Z*3sslx7VKXm4}fV_;1#51Fb3C zw*$xZdvDgx&)#mBYkzLN+Wgn<@y^+&lRahNxlyc~oNpf<>?ms+&1P#b@OZPid$y~b z93HgK6d0q~+&fxR*7|Wgc-?H?+}t#nplaB`wIGa|Ct)z)VSIN8gKa>Eh7&t$Fikpo zjve^caqW2P)o+_u{BEmd$L?2t$nJP};5+tlRatzS$g+WT>%jH3$XviJ|K{ z$&hR|$3v$q4CA2X{4LVJ-!VaCT`^30f1l%5blCA&RPNgjJa;;2q07PUlzzOZB)h&TSv$V-_>^+Ya@q(( z23OP8Omo`48?u`zTT{l4jH!gwu{ zrOfW+v`5?h#!K|mB`Sur^+#_>AAi z<$j;`+7fCnxG9sK@+gtfB6SAH(?!(OJeE_^L)V{y*6cvpk#K&bl=viw+&Hs+;`>C7 zQA4$`VWfgi`0UZ;yX zUEhVH-DhFMW2At3_{w9jrbm3p|F{QQJMe-Ki2vw>+##CQ>4lv8;^fjBayTYb|LO4C z0Rw~WOzG{Y2k7kBSg0oiWSs#EolbnK=?MN~(5&L1o8mx-njB0d4wNP&)JjNHH6a1s zQQBJ^bexdr29Stx-Oy+8Fl1imi!Hv0-CjTDewW#x1<|Wl$;C#LC>^_tl#9;4Eqq^L zfzlYS) z&?Q)+OHKm#Dtnm|eVh?Vs+tW;1x;XCszx-^)Ylaa94l(VhFTizF*IrS1>PsUk#NC4 z96$zC9aB^2Yiu+y(M?crSTi*Xmv(Z>$IDk1`)2&plsKMKHO=CfhklM~;_VgG>{z`p*Q?|SEw000n5Auv@P_1?|5rqt*!>kQ(M$-V zheBAz-~y}2XduhNga8_;Mm4JvvkyRgjoIc)0U)+&0R9jFQtP1rB)TS|7J>{_)hISh zoZ=CnWmw>Ysk%v2QseR0#G|@29;@SDY{9-mXr}5q(U7K7;ZF(qFu||ZRLeq0w+s>g z68y8rztmh+{7a1`<6qsRSQ9waG(a6VuXX%e$GPDar;Qg?E_P9qIJbw zp{cKscwHj#FItjzg3#yTDRbOmeu_kaS{(#ooDCq0kdj_6YQpVL+`hP-9^JO}rICz*G|_ z`srjGX|*?0Lt@pC_?vxodAKw#Dg^tT_?4Ul27%|XLBySWfISUR5gV)aDoFDRd3?!o3o?ds`Gt(GPz_QFDoexXJ7@Z0X+T=c zG~{=pc<+xGM#IplNeoIY6HtEsH6c4L={@FS`QYMvO*V1 zXOZR^7RfYJ<{FCGhFQsj%Je}|I7*U|zX`FNAR4m-F?U{^wP<=JL03s+s*LeG^-yh) z=YfSt!BTKNXF8uRN0uc!pSP|Oo{AK;1%7N`wgg0KbGL|=nS)^ZICFd`gVIL@r9URuGs4JA$o80|&i2&6dU{|z zCfPF!T5uTxS});8ZziqHPmPjQK8dj+j2P&J07R_;QUm01&t?1&farK#(WxweQIkh7 zbhBYl9UDZ~k$C=FeKN#o5BAnp;zwyO*pG^C+$fSk;1?8sqNd{i?&Mr)DIxQ(&@3QK z7yHCxHWw9wrSerwFy$UME0n5`K7FU^vg#WjSlDy@s)v&jfwFQ>xh1DWD}N^;!=~0C zIWXqxkQBIfqxSH!CxDmXW?WJ{TOvDGhWuUVI@v5_zPh+*E!3YL^EPE=v=ILUp&;K8 z(p6RYo0T8^LJeGg=<8BKSElMML=`SX6~R?X9!!hE-epAvo7`2G7HJ|jxvJE;^5pM^ zuEV3lz`o*6u?%+3B4?;Sx{-TXF1DSohn1%NPFRmer83#HR3-c18VGm5WW+pNpeXr& zIUk$_`I6t0Y+#x&6qWd+xdpeyh0iM(lu{nCYsuCm(xKuNUP;420nTz@xH)_kHd3b0IQ+Z?nl%at7)7ac$?E~xiJYI%A0`#=|oW}*hpP=GH;JDq8EH^rC+t=Rr=F`c&SYP zZs7Mm_uYO_uG4=M#C-9KISyU0qPgXjzv^L34Z^bU;6f{H!wy|V3(yK6S3~Bgy&c4H zaGl!VsU&3V;%L0C7qB=F=+U`~^5C5+t8|9_AauV4KJ$QY*Dr`%c_KS-gf5p%wB3ro zIs5OuPZt>Rzqh^52m3)w{cHQLgZ5s()jvK*8os=E@7cW_`0N~?Yqh@s&D&e=X6M~(_78bCe$4U+eKKqO6L9v!+`UH0@a6i8V^3Je zq>(5G-9W@NktjE1Soum~l1YVFoO1C>7%q|3pJ884h3oqxH|8uD{D|T6IyKpdW1kC5 zSNH;_oH@t7cQ+sGr@_sl??nO9g&v-W2 z8MmJfJswsPzz$uqdCeEDlXYvRMf)Td52$N<>~fzAco$p~N(-LM$Yh@KUQT&U)?q`V zWL?>yYFb(sV!#HXhN+p@pcFpABB_Sapqi-}h+>GgHBGWi*%u&9yd;fS;?EE-M=%Uf yaRuH)r;p4VmbS05#H0jKzb!OhhKQovAC*VDWoYybfN&izdQ literal 0 HcmV?d00001 diff --git a/paddle/framework/images/multigpu_before_convert.png b/paddle/framework/images/multigpu_before_convert.png new file mode 100644 index 0000000000000000000000000000000000000000..9c8f7711165d80a2fa3911280fdee91855a401b1 GIT binary patch literal 33557 zcmeFZbyQW+`#wlq;8K#3m+tQFkZyzSl8#Gv!<7c*q7y-^{EvYt5gt){JYhTmC@=aVTa%YuD6CM91RVh4E2MKR#;30PWa^Vz{Jl)S4Z03 z%Y)z6!OPB(Kgh!yoQ;Mi8zc>W^>FmFWeoCg_k>Fa$ua*sLmK>!IxN7<`0o@yH#ue# zT?0lHFJDJSF@8~gL1uY8Mn*=7UlzI@WF#U{cMByJmD<=Y2<%zN7WH-@9X02=i=qbh-%l?&dc9Vj+q&C zqyPE$pMLteIQ_3XdBXquSl|H#P~Qj$@e2z4Put*9S=3Q!6)z8OUq?6`v@b6v`|r&E z)3N_`pZ}b1@Yu`G3ygxVi@m0&pQA6h*v}T#J9**%Uj2W6#{ab~eP0(x@YMgV7W(hi z|HothJzrJ;_2B=9f%wlT|2qnXSsqVT;D273JYK_&@ERJLBATYE(t{xMuO-2Kx(|O2 zZ9JK-l2T>mDkO+k%f#b)g551j%7m*Ko5iVINPw$JmQ$5gk)*~3S0yKXUKOm@JGFBC zZvWlMyQ%QWcR!2aKjnY^zUlAy{nfVa+q!d!|GZUbUCH0SzrRHu>O-OEOpFj^C7XQ% zwHZcbMbzh01_&#>V@2I2%AtN9)iq^CS|S!8&ryf8kTDDLl_IKr=r2P=^uOtT^`x)Xw0Sy!lF#fx ztGlWCS2HenI~r*9&KxTJn~<3Q0r(@1OpSBKygq0@72!xCwHH@{r)j+ zupD{)IWLFD*v{mXn%18KHUCd(qp=BTYE2q3Ud4Z{^!k0Vx)FK%r{+!1Yl8|T=qopE z{l=KeAJBWyV$$;V`{Xy~Z6{HfcNYB@ERSd5w>RhdReZNvo4>k6=SskmL1&Dr@_kOR zw!#5#^V+zBiO%?ppHFc!UE!}ui|gGo7m8_a(s)Gz3=KX9N~|9v7R3~>kM@f*ju#}G zJJ`K_e|~QC-b&)Ald~UAy%evSmvDh9nTXy;(c&5JYuXe!oL=v&Y0Y7(07<7VR$c6r}XmwxWC<{aOT1WozU+eVuA@dVwd%5&XJ)I%) z=B&Cb;@9Z_QMGQ7xaVqn`yN+>{Pk}35co=DN_OW*Zly1F8{^mR^T}uDfB*dMJ^Mf_ zu^wcPUGh@X0mPWp@ds)JBoAf`Z*Pv47Ui+wyW?exqF_lKFP}mWV`S-L6 z(R0tQoC@GQU{P8`DGmwIziu~w7jk*B)|E~zqW0R8*WfyUub{|F#FMReEKfk;pgflQ zQSE_Iy3p2Mkr)5{7b)i37QCiS4%>r>BxL*dn21WT$nyya6U;&hufKEm_1`~Kng4k> zixd0dJ*9O@@@Jkm-NM1&9ey#$he=`@7VHJC`BS=zJrSSB+^wC(3~-i~J?+CZZ*gsL znxGkcdp}~)HcOM#9ICvdG@%bczLeElI3a=5_(^jF!pC7znvbV+biTJW1tJ*gogdZ&V@OoT$ zpM6Q_H%L$4a#Cp`xEa8gGa5XwQ9_j;e(t= zpC4bBEKp1CaTAyb{yg4FoJ9N12vRn|$BcQIe;e02JH?BF(p27eee3xRMoSSJ8Mt2( zxDHn4Vh>!p0%HKIKK=lx<^ElakMMKXP7g}l`DFO~ig7c=Vo3Q$$N zYd98v+v^9fwoTBAT{S#v>nE?Zt04@SRi|;dWGB715nt=_tRsRegZt?)Xir)_R;C?% z9w}7 zcJ@9-{)Xf%NUQv@i1f#ev)_p0iFY4Tl09WW@aqZx?YaE+a1}lkkGJ~1eCwp^TiUaJ zkLSs69}l05j2H+J-<)J^8Tt2$?b~w z%SX9T6&&Txy}FN+DeeE|K^yCjO8EX{zQ8HI=j;pJl~xa*8zcpZInzm})w_`4$FREw z^*Dqyaj6)iOYj}(`WF#{t&6XtZmIejL1Pa2%t!Sc-9F&u#W}p*;%8G*M?Wycd;vAGvzk(Yjw6 z>Pp1-x*4Y1^W%D7fN#B_&2CY*MpG%Lkwyk156=bH;>&r%nX$nX_Y?!f!F;%^gQ6N> zj9q$wX;!DAIZ?=Zr2`a<#78f=ST{u?r$5oZ<{WzHnZYY#-n*IpYKhu)W@S5lta^(r{MTVVO8=T zp&-VMkvx07Gd|{0U6Rmd>)AhDkhHq(HGKe`*UV;rck_MBtLK`1=$)imJe5BM&we`| zv)`XzmxJ66Gb}F<3fk^@_w}Sc7sDsF!t$cI)s!q=u7*J*-TzF=e3SiiF$>a-^w1lr z2Hn+=v>#I1b0jSY3&t?Ij_!bCGo+jM<0AWLQC3A#bv&up8H=n*S&JehSshA@?)S@7 zSC|=jwW;4zm}<(?#yNG~x#b@yo>X?ohNCZ~E^FtjGEb4_e5fnOXUgEmfU&SD3u%)( z<~Gni##H^vDrwmH$Dr(d1Eg+MDJq2sS#H6j>^?P^+jx1++rq)o)yT6S4M*uw5yh3U zwq3*G8D<%*N$81oBrW=~XAqj`bk&cv;U7i*h~U{@FLrc}l0gjIaqC8mDREPwB>Msj zej06%8wV)%_qOx)^$cSTzJ0C+eP7MJ54z6S3b@|)3aNYkTqDi8gM6^%g1nS(Q({O` zNvIb>RX&Y7L-wZ1(l5)EF~5#PNJoEa1B4H6R2HgIyrFX@bA zq;I}9PkmW<6FFz7-dK=9_Jlfbrg2C~Qx7^_p1^Vn;w@QF>8`JokeU#uiVB*;75-C) z`re@B8CqLg&0R6SgXw9%#Hw~dX;DSo3)Z@Fu#xl^MH~ zY>mRk`_qan+mS)+84oM8^Y~en4(993ooIU^YzGgf(b(e9I6?yLCe?efjT9)D6zNlU z(qz)~LwR2_1(8&#VS|R;D>5LquXvojjfEvJr`=tVTUf<7Nx2hN6dkSvOmteO`{B2L zepV3T5msn9>_VdV%OZPBo1LnvcBVo(e<^DE`z|RF#VM8c+H3CpO0G72x5amKmAWH% zw8t$Gr`ja0cvt?|V7QT4wRJ|k6AbM+%(iBN)xFX+y-`26ire8!Qf21+6$N6`ddg7B3h zr|nwPjbpU~zNGMdUEs?)oQVYuy`Ss*(JN zk=2w6k$R?31NTC?AB*;JTEc0PUJw$M?`J~{`;BuEg%W+dHSwAk0AjXZ&YSZ+lS!~h zO2ncLOE%t~gY#;%`McN~)_ML$6zEmfl z)VZ<^OF&>UIYQ&3IaM@W{wSkHwnP0T51nI1Gs3~Pp){M0dNYhhGra1{Nz#*?UX6Bj z8$wbc6U}gR)qy)Y{buMt@p*H{`G{%EJvYB3MdJ!km_vVPaGa>V>?{cH35F?uLMqv8 z_C*8ii051@s2DH1YpUT6HEG2v=2_ z>~{^B;fW-z_E>eviI*)jpQ+GE^T}q|hg2c5vW8wZv%eYtbB8@pVg(J+sON4#`dNy9 zt=0U(heek~{=I(pbKPyaTxpZ#vmY9j!x*}goDoyO5eX+*O~s#*fMpkUqdgv7R-OA* zLe?aNubKnbgA=PhzfYzla_NQFLg-J{qH@ijl6}u2I1XBy{vE#OUU{D)cM-g-CQymFeB-c%d zXT=jW#=Zpk=)n|5HXCLeGAFq^%%~piysk5A8LgaSH$~!9pB6cCxY7$_=$O8h2(py5 zo&Ejm>A1g7^VgW&sol|*lmnfS`1=gmmd;x7%g>|H4pn>q{4nP+(s|MU5dEPoXw%6c zXPgi1-cGVgc83G=5-9tO>n*5pG@G3!et>Zo=)DoHmrW`InYf(haarvY02L%PGSh~3 zGAO8yD1j;<%Rz}?{FSf1EbP1H=!i0@@yVEeNP=m)YqUeXK>mI?S(ZZLv;B2Y1W85= z-IHg-`GPOd-8XuNgNV2o>2=3oKBIt{c==ZfOGecKz+U_XCKB$8o9n+o0Fa8b=#2;u zb6Hc46zCQ(9I=SZBzhx}YbZGs3OW++&PIkfy3&<@S#Q}j4}QXb|JV}j237nGNNrmw z&ke(r7~ftNXdK59$tP`(kotQu#f&s0Qhd$5ir5NAXzl&F6QM!Z^rB$T8=GZ;>yG(a zS0KkW0m~g)arcQrF&6bS)>omDkUp-P1d&{|GpuOh`Tg*pYl*jBpeQgYzQX8oVDKGg z0=F#@b{qwjlk(P=Pc_$%Sg3T%Lnqhs9&1dc;3x9mC^N>19cMk6SA1~ursvm_;e#Rx zuakv08!Zb~=5=ObZOYS>>y&=ePhUo+Lm}_vO`kZhvUp!DHrgM}y@F3l?E~l~!0*A? zQ9H$N*||BdD^}$_{R-oqz)VursCVOJ8NU4Kfb5n^9`rO*z=ss9L~hS3gGnkK@HMHA4fYsHGHXkbYZ+o>p|SS@ZRo&!z1Q)^E>=RS z2nooG9qZm-7mNDS46o@&48*=cA)WFd-9gztkL3j(F1fvV_hT*4`e!wKD~Ri@=bAKX zHm>Krbnqm;N}O>ncaz4#TnxCsN_dd_6N`=*{foFI1jDk-p(CNxW2Lp@0g0_tTPYvv zf_fRZT+58WEM2~_q{*iiU|@92Yb@^&EAnoa-kt6nq6>t{i-3YDYej)#W!3EF9csP> z&hvI{t|tFvSH!#=H*u0SssQn0FNXu3ir zu{bQc<7tR>WmX6O<6KKdxKGxgC3hH<@eS)aircMBRdT zz0v5@g?%(GdaGM*7Z+@gb&+3rbww)5VNNRCgNYD|uBi%1|k1Htp{$U5# zsn98f*}iQe9mOibBfxJcogrOitkeJKijpA#)Tk$gZuN+GjNY~0+rLH6`WoYvESlxt z;p72^)D91M0M>Zny2@;#l0k;O_h=Gvr&3yZ&mVQhI5xAlj}5;PRFow0LZ1b0gy)g@ zV6CO{X@B)hg`y&~^f1KqY0({lzmjUjh*V8RVn|Vr#&Z`VFWi<&J zHJjgvQ<|?1)E67ENUkm(Zgs!k4b}|BxdGLoH%_{@naRC136khXb!PXh_DGFs^2EL! zXzzs#jSd#-w9;y=s{U$vf#As97eDfxQ}QHcoM+~@0A$M4#A$yzo*Qgb^7GUC-9&5! zM|3=7Zr_i^Lw|hTykokT>wFc{b*l1ZH}{A$b&^to$%#JvVun-b?ihY;e|g{lhDRXM zS+W)X{0+iE=|RQohc#yE!HSEd^03=eKq1JK|AjB|CCcE^eec60+ReZwEMdd24m-no zHSFW6lkz*qaZ1WdkIH&791wVweup1lA(S}pry9J3E)Vn?~F{X|W~! z;wd?c_ZE-1n(OG7P8WU*2XDq(`$3qy4bBU@?E68*XiF5b7><5oOO%|GADSa5_Tg#TsL zdw@{&6ei(WjP*8z@eD*W4z#0krC8ayhj;ObufXS}SSEfYQM1`H$sXr9es1 ziiJO4LrMDk;!tIA@A`Z{^p$&1Nr8wn5+6pbS3P{5hSqclNG_+|g;26=Hzf3H5P7># zipJgtbh{&cqY46g*>(Ur>9f;m#yBnlPgV<3DP<{A*o@*FqtIr5q1dUu5?Gm!MKvHF zo}xoVGshp(TL`~VmL{IVY7!n*mryMY25aOBeja#JX z7%b(mE0fn+<$wK-moE_x5^h-%bRR zmGZ(|Y?yeoKHgoFdLtm$HqDhU8!Y_8K<_WP*=#W$ggw!TQ=e56&x@J;QQpnX`IP&w z9T$5_%Jyx$%a|*IVKp`FsfjQ59E0qc$JQc#A3K)@v;U1v$o`UnQ&L?Or=cb!qk+RX+5I)e zn}!t4dpe*?KP(+1Lg&;V)P{Sf8{)ZUk6Bh3m{_{0u87;efjEcDQ5>~*bas>8n361r z(c`_G@#kfcXXl86s3+i)lN?yu97eMKP8T%pPCisQ>~LXHa2ltQZOh{|&3aVh&8>1* z+=5dlP*Q!%W3|1JeBG}?q)tAO15_VDFbIkcpM4x#1=uJhdQ~S)TBKP>y@D6-d1%M> z6CV2D-Aur?11OB-bgjU74V}P@KO{{+CTpu4FSm4LAdC1z)IOmIc)&qlf}zPUxa$@x z^;U_1^(-J!Ht8a}f%kLZ)x`6Cr>JYd=K*3{@6>DPQY@A}%pHT!07K3o$Fc44Xp$wl z)%nD{sV~j(-s7i>Iz)hxrL+p_hv+Tw$5Ls0*Z!^MpgnbY{?0MN8j{N1t6875^rFZ$Lr#^0XCbO>_ChwDq&o}4WRB7x|3jJU`f z^{KcvEi+M#DP8>f_R+yKCbey_b|w}qR_n?mg_(DRrA4pg=K&8z(gld?EaZ_)t8IYB zchSKUFl*F(Gmcl<5~V~3ggBib&?th)L^bcOkos_r$87Ls3qnA?qf(~nJe^FyE~|3x zyI_||&Xo5klEbRd!632pTl>zP(t<|&A*X2z-Ek$fvj+eX)~dm@2t51SZ|oFp7}u8w zw1{2XR)_FMEExLVw(zX_LDdpr-B;VmhNPHwv89J)hE&NUm^5sy=4yjRIx)Ch-Hi>4 zL4Q7K3p6JC4>KiZU~rU&Lpz_t6Ku*Ut;2*QpR&SZ2j-hfN2J6ZhKav#!2IV7|NQ>> z^Qc|$&-8!jAg%oNa_Y;Y1${iP{CpgVc>r(KCmY}WSU$?q=C>8;%hTnPuHccbGMDD~ z;^%!KBR4c^@fm@Z8g@I7b&moTvm<*xE~25CW>^fW3zYsK|NH9>T_FH1TMtFN*1IJ( z!r*{i122}TkxiyieJ*WDEU|=B19Zz1{#2 zp8x)kcLqRZ)&P17goCH|qFS}{Q|q~h&hMIV6KKYm_r8IV*v{`}2_qTI;eO~#A&2K% z|6qBp9Pr$*_`I*K#e|tikovk#z84Nre=w~q)=8t8f$8Nh`FglMJqt%kGXgZsPN%`)YI+r*;AXm!Muv8@l zoV12s&8<#mK16Qe_$MGX{U4!c2}D@C|P+J6aZfStc@ z|6s&Z<|T+sN@B)SZn??GRBkvxd?OWS9sE>_k`>eG<&y49P94rxleHw{e^>1YWbuwh zhVG@fqLJ!y41(y_bg1e3fE8q_6`E!A!BSPmUZ0XE&fmm|=n=5sOyY$RfF_FX$*o z070B0H$+2EhtrmMr~K46N||Hwe)RKDdO+`AqGi3_l>ir3t*LFXm4wD(6IPw>{q!Bq z4INo^CwcH&RtYWsXIBSR6EGgE9C%hX7H21KR--M*IVmL&oY`vcojJOH1ps6}l z^UcMy(-M|owE5(5LBdEU%ukp~E1|YUCm+^MzJ8$J!2knQZgl0ZVPl9v5JEK?DPxO_ z`*nGwy*05ApMkvrSemHkuaz0OK$nu%*mVmxKQhKkaH(?pfr0a{n2skNhM|+M1HG}i zZI81X#3`4(@K=si;M>%Tizgl4Brh#Gnx37OdLwa(DLyxmM0>-mnzQsaU z-*WUi27ra}XbQ-$cKDwwMPMk?MZf8@T}p6ISP3IW2&ff@7r5djdS588wPkg1ZO!O1 zWuSV*DvCsEb9c<@D_G)ry--xED{=6gb9!(?72gMduin);s9-d=#9Xa>HBuZr@ANNB z2S=mfd9``2fj9BqA=Ji5cmY%6H8Z41t?yxwj|<>laU8ZLXbjC|GR9l{_f*^bQ8VV| z`kX_r*aTb#N-VRX80aKQ%K0kvfJm8<5W@ZNwI~S4fLJXLI9~p_5!p+=F4~Z+`Vg!v zooD$Ccwl^v>bp+X0yh9I{0P(&_V|U@OJlhv|E57GQwBA7ly*^o{gm;%oY2gcwS*Tw zBFL5u6>Weu;LCDLP0eY&X)`;X1tRe&&`@8*sY}EwoI?NAHXk@eKE1D(iF7->(B3nX_F&i#a z8wK!Unw_VB+DOPxXcWX}Mv>J)p_{957ej_I3K=>fwg4;%IBCc(z+cmO9wlaiw?iYv z$E*bnQbWl$X~^UfTfqJ{orb9KOWb9H!(+SD?qbL@Z*q3yQ}E`Cd(ecs0q;g;dPxmW zBv><-l=_$K5Z?qeyl;Tn2snJ6=j*p!8hUaKm>XZUC)oU`AQ00yL&s!83;G5?(cp92 zUl)h@V6D?gMrUpjsR#V}=xLjL|z^TsM*$q`lpkmcQ{=%TsfglG@^xgfr&V7LH$YRAY&u#9+0S z2-#!v6Y?0>6L_=wAEQSz(|>vS1Q)=?OjqEyxF=HlizA(blL~-|`;bao^Qg|1?z2>>YgxjO0q1TwQW$3qWu$q+ zQ-1|idca#)15XIdfOs(@5#~D#CN{;2+_xOydk|@5HC~Vv<9SreY2Nx3j1J@l9vq7r8m|ITTgTu&Et8E5Mm<$hk{ASyJd5rpC%(=*YM8kY zFLkJwD1M*~RV9!1{XA;xpm*)q{3GU?kt=`5;7WU$Vsthc^$3y(ShU1hEEGz`jvP*Z92Jdw|U?YD~al zw-jD@nx4EvN%%2l}?RiWL3~$o=Ulo8DbF+LR}}2 zcLaA$qVJ)-V1FnqAtoQN`Ug<1nxjU&5q|)hr{pq<-^U9MEYO}?*L?qwf?zFdIG9B8l{Yhk*;9jWMpCgD z(CM1CjpZ57H@bu2_|VJPK?*)Iog(px^wPORA}Db*2=2f%0#=O~(CUxY-^rAavWJ0R z8{fGhK2&a5Y43F!m;@+g<$$J|Nt(xtNGD#@N6v+v?3n0{T0e6rtxw8^a?h&A+xs&hAi4sF1ejRc^>d7b) zQ^iuctm`9Q=((4Z?YKvCiP@AEZJg zaN)_V)mnD_n2^|51>v;mKzLghWFh%}JAG&zqR;mBCSI!2-+rXkX6-4$23S0K)j$^9 z9w9pfb`29?39a|P&mIH%*8swhS$XfVv~|dY=OW zc@4z>Cp>fdOq-M#V2Qzi-|Jl}wMc@PX`|iy!BXq+%ZHxF0G1=#wzWWn@7zgOkgpe< z26HzW>E^hPGT#9%gHHPwigf|2@k8!`8I&rf4^_+lSn(PnIltOTAIix;2U)zhLogO{ZiJDBlb{?5e{pOQp_3iVCM zIn2@FcwkrKHc|Z~RN7h^S(y!S8I@41# zpE9e%w$fwBgx8K2caaR{+Y6e@vkE~VO};2xZN`EHwamBF9efUAN5f!cXd>D|i|VvG$?LuN3kgj^h) z0h?)4Bnq9a1MM2nDC)e)z($F)VRDkU1iTm3`epRh%!Bh&*DVg7lhu8a7*|OHS)2TJ zZxev^Zg7mE!KXNhaf~Dgz`z+bfv&H@c|bj+!LDbx-%$PY@2?=f?h??T|dG zZGJ*2An!Oz9#P3$PV%k0xg)Di19tg~3!;dojbWAH{@3oU+}DNy5Xf?9axDrz7~Ox~ z)ym@ChmO;S1hzLpM#*mXjh;V=3=v5?HIHX1DJ}g-tex`1&u9mTNsO6JoIUeuI=o^f z^DjhnAPuY*z`e!`p0P^n1OVqApv31@0qVGG;Xf~G)n}-p@%zZ<07?4~DUzYUIVaEz zTqK0F;?B65?5dW2*LdirlT>P5aB!BCcu3PqH%Be=T7&f zCZC-T6~fH};=o@sNUzM3IZjCEI);rbjd~bup9wOy9&5?k?ncCK!pIT8l{C!E0IThw zG&c#x63Z!YiEidbAp!vO?>9`zL7COq1CC%i`!^0o1!Mdcz{FDDzK!SJ59T3_T!^0? zQAN&I^1xA%x6vZ>P`5j|m;aF=KJapQ3^R}MVWdo|p0c*LwpRK5dYBK7_+!T!`RmMQ zw1LBBj2lS7;zjn4N3VY5@>@!mb7}Mov^}adiDsX~R|$QLzQypz(^O3sb7ucOj3x@L z%b{@{zrZlANGh_R<+3ry9bZ1?RsBCleLN#$fjLyMlkcA?nFIt|hW-b}|NQ#Oppxc; zFs^}Us|dYH0x>z=<~0w9)p3eUAmCD9WO#z0%m@hs9L$2c;HMlAah1!7L0ra1*JuN= zn;5`tpfj-Kf6+o&qYa=K(ut4$b1y|B$S{Z6oT`z;*%0D};r5zEcM|Y>GjFVW0fo!v z@m%@8I}e1Fq6X7cP1wsl-0ioSi1(X;b+<9TGwq&YYrKi5AFA8Nkl{PjMf;gffCys} zm;@ktWN))2@1};MpePc~pB5j*0L|#c&6z(&2#NwQYu4_)#gqu!ruf1Po_AGM=K(W# zq-ZFy#Py6iLmeO?fMlYj2II)u2hmk)^$qYvV^MN7D#o~X0aUn>g0N!BV1a0O!BK{m zKKREfO<+3M2Z3W`Eb1*=_AIIZKKKP3)e7oWOQN+7^}5Y$X7P%8#6jQ!Lzi^TSPEoar8n5f*;*Sas^_20uh5Ae-l8gL^PrY<04b4HH2Yv zH&kG>Zt|UdxFFq3ik@qG6Ghp(FgD6$^x#S0CgH`k@Y)Q1%_g@%rt_K=QkjZ2AB%EW zN8`{luIvCQ{P&`L`ukte_lkaIbYzbcQhO6}K*p7A00|0+1>)A_Y<>A)AZ`=Vhzch6 z3>&hnDDS>_K!A(x1syLC*#{}%jo~xw{lK$}kga%1R_}LAc@OyxJ52klKPdM}L!c@m zwn)f0^orI-Eod9T5wPMRan#S9)3h$V$q;^ghD-T-E8#GhY> zEyf-S#2Fs_MC9-poRfp=9ZFnay1>t85#9rcV;)$2Kw67#w~kW{`g^^vLfmdvscD>k z0d!6i0}V8kp_~q+T8imHF$56(>+6&2-d zh2*Y>|FB^L2uDBa(1^Q#b@{A1?9d*_2mo!dh$Df6U)pzf7_hs5u;AB;#-B#r zw#dQL$b^@3x9>N|A7~r;IBNNzau9dzd&kqsmkQYT+u|frLfriE43sysbMpMv}TI^;te; zByLxr#t1xKzAJ(RDO4?;K9z0{RDp^k6H-Os8LAaS9|LIo3^;UiL9Wv+5O$n~03pN^ zXXenL!|%|@iknqBZq{%4^F2$^7t|t74GOI`{TS{o4ueA8Di5_qN6h?Mtj+ly%#(_^ zDv!?&!@-^ivNQ*_sX;ZfyWXQ;z6KZ$whQ~b13*gN^g(1at)<`nwS{4|sUg1F-eJ|^ z`VjCFh&vHy?!I)XF~dGJ@$?;Be*{$A=4+Xv?_yAfS_2yHuetl^ZTIzAL1L@nGc})^ zNWfFSFmS@A1B^iTUZLBt`CkA?ND>m-t-H%uy(jXEYhMDRGoHLX?KzhjXaHnV#yM|V z)bZVS;ZC(3o?@p>b6!*5`6R_5 zKlYCF0i#H@=7Z~GkQRG`Xl7{ds7Yda3zC0hhIRtSehMQGHAxWfW?z($!(xLitD@*c zoiO!?X^xL_8!#nlnDaMaY;g=7niP;UA?I2@e2^ld#4DdvCC4L)zK?>8$PxoMh$wft zA8@|atmk%L>%1$K{D%LwDKViKno;GG3U`P3^hEyk*tJ2S@& zJd)#3LQCgal{!1|J1OAxuJ8tY1pHuEidsJGN@~AYh28AylqOYkb5b6jfI?G7j(~OP zR9d&N$%>SH^nJb+4%UrTBiZL|_aoDS*;l16gay&rZ)=nbtb^b58!k zjE9thS`Z{f_S&XS+!=4Yiq-l@(uC?p;OoV!4+~F*BYJ+q^b*@&pd?s#o~@6>lSw`4 z-+@&mYb$fM6J<=R)+-s#wBPwX6=*V7ff`Qy_)ALnk~QE=D7I^o2tYT)GWbaH(5w3_ z!TgadWyI!Zu4g$_B9jM?B5illN%MK2F(u}yCCrla6c08&^hUmUZ||+LCWDZj zD%j(IQmtS?6HL6Bq~lOYa(gw#daO<<3`#k<#zL()MP9Iv#D1pXs<5tE=3BQ2&6KR+ zw6{$=5YSUfh+=MI#FctA>?8hME<}PI9zef%f_?SDj-DC`_*TK3bh#f?wswmlPcTGo zPrbn|oPnp=n@gN8H00-7J9i=! zC4P98unCWF;eIqf?Jo`D;b2pB3kB94C<0MIQHDou-Z}==FDA1SefbGh8Ow#=GL_be z=#<5^HvY4#>d=^`0$RP@-e<8FDvY=oNSt!q!ba9ak;F4ovMj>nW4^Z4m6e2*w7UyFwN|PPNTA6oSUG8N2Y~H3#IQ zIzUkuLL%i@LN$xi)-A3%=x-r<80&9>kbiz}+yt!BAAt|j;Z~)?s zB#5(soYmfRBB~8JO#)Y@D^fp5?Y7;evoRlGER5lB1l)G9lDG1RgCGqPzt3wHaSUov zu(2_wxo)c;CcJ5=TZ_^ap1AyC$b9S^v`5y>z44$K6vik&b<&uzRq*FX@1XYOb|01( zxN*EB3rSOqoC>lmoWvdN9e>hEpTzMDASK$6q{`=#oBMZs9px)8O69bdNc; z3fhX-Qwwa8>j7*iNG(tME|W@MW5q>XtVTrUSqX1;|Oj-XuECJG$^2V$xQp+5j z?^t5}DT-H?<4}uo%5+)@evY9NMRf7955!@2OI`H(e*v z!o7b0%`vFPNJ_{>tD*D(JrT%9?}$15&7r&G(Io~YzFLaQv9};KeN~0)%u0{WI*GlA zCezHlZzSY+9f}avzQPhGm8uRM<*H`%n3CK{xrtXP{+5Imf^b2iXF>bYKj9PIJM56r z%ggN5@Ie8=y^&YV%FZ@mlN)C%U_;n?wt?=Yk`#WQQ}Z<=q??5?v_7!b_kOm7c=hD) z!{(psq5BCKB5G{Zf=a!T3JH(KMj8Qpz*q|arHGRO2REuFs@e~^$8CJ+?Y}($!Fgpt zl5y^%88`5PTCy0h8Wx+Kn;Cw9p?FmD=opwS*bpj{2y%2Obu=%Eo=})@p_tnTETxRl zpC=`7)FmE{uQOr}z(18)7dsru_GZYw`D9s3hJw2)u6a$8HAOF&HWDIFC!tC4YHR zrUsJnJ008#ubCt%&aLN7UsS2kn=m?CBNlg;$7Q3JRYd$q(&%I;+{kHF6F%;fnW{e8 z4Wnx$xgs58igqC8!r&qo-2+HZy&fl~a7XD@j|C0wKYt70*2I7=XCD)zikwMa*j&E7 zAG>6V%U%|EjR-8G2pc(w+lY(UU;IQ%9C2Ad?Xy&Fwdn8#j5Do>6M_6N%xzL}6FFFs zNUz09o6;-gs1dtQkG}n(L-iE>e`Jgs1tI8aGJhvjos$ePmc~8~ZMQv@Ee32?C@>a- zg88iRGp}jDyT8fP;&G;%?W(zdUUqzfp1KSd(bn+ypH%%wui9Ds#R7d{ z#~a1g$s`q%CI?LwO`S4cdbAWb;3ejGE`^8>dSz3cwq~c-JI2Zf_(HP^Wi0nD&68K0 zUc^Ag7XFwUnfloz^s$RRVWI&pYm!u!LgjG-e%wT9&Ltuq5UloUn+BK_^8nbpuAlEN;68Q#u`L-g2RN6v*vy$-B1Jc2 z`|%yodK|rM^YNu2+#dw@XssK8g zd6IollH%v~1y;y0*z@ZP59zyx6_w&gr8+m~beW?ky1q+|Z z*A7p%f~YrtkH-lYRsDT`I)oWKt%M_#K2AsYs*ho2Y;j3N$JSAXro)51vOko9* z+zrWnOf0LzMhb|Y*$Hmteedljz^@aN=dxqLMRW>SQ9bEH)?IcY;E!+VR&W^?4V95y z8Frg&*!~ztp{Cqi^#ZdNZAFMj@|~ zs1hY8wb;nanw@mPhU=}1S#?N$5-u70=E#6L?rgt~*yCxmnUUIxkrx4hw)<#&L7*~+ zy4lSLA)wYharGU0U~(<9hLeU@tsZhe+WR}ksohLzhg?z_@P7sDP~G%EEeSKqxNZ_r z;qbec=ra|kkTS*TQ%vTSd(?25c|yNFJ$qW|ckcaN zJuO5>VtQm}RIDrbrE)io94X>uIA9;7coVadNf7o+vhV!v-KK0el05_-S!>pAd7*EN zCcy#2XfZdJ>#~jP6Nhp22N|A+#Z06%*XA+1{y(RIHMp@v%#FguQgFrREf*uMMq4q& z8H#Nu$+=mej^z}|RZ6oH`HW*7PzztvD?VmjXMXb`(@?`js$*y|egm5gHdRR9(Z0jJ zdw14qqbFSGuY*2^=iFnfe46Q;#kX7dd8O1sze_)1LNlrhyhzB&t-Ku2`YkpEc)Jhx0zyP=2@Qetw9`s)GZeYO3)P&@KlM(ojS`GOw% zbh+FqXiu>q6fz-(YL^bEb_8^T-dP=IFTZ~ncohK(7!}9!zXjAA+G@Sjo&f$(!NusHw8%iXV8?`d>q57`$x4eseEz_dF9j-PxT;7rF7Ut|jWS~5lH zXGL$RLkLqBCwoq(DgK#~WJro|FcE`3o2f?r4uoK|nxjlLBU+#%yA?wOUkt|sQBNy(566ysxpGy+QO+cTidokBl@2G7d-0Y| zrFis)^yy#sN8Zpcu0*pE(6PJ6*sFQlaW}20t0nN6w_d$>BL0(+M1L8F>GiU0PvP}(e=EHAA{H5D&y)n6?kg-#;>sk6^*jlb;E+e$lW%VgaaC{vMav9 zIQQgBG%GZ=8+H-}oV5!MdCEbJTna$#IakQY^n_Z-l6hmOT)jvR_P$j3b-?o7N%<~A zU4N>b*A%bs9g4<}xqRkMG?ZZfX-3|k?EG)Sl;Np=93L>v?n&m-565b3VD=B*$-G!b zyh=JJJ1f|hP0F5^sF+^r@SSAeo#wPACf8x6?>BKmRi80mcAcd_6lZA#DYPS0ofgSR z#M~T4dTzGY#DaV4-X1z6XDr5)HHX{dRl)9vqdSjyHJnUGNGT`bB8G{to=Wrnr_4r_ z1J>Snel)o>E6En+H}a3RRNA(~ag7Z=Q^<#m``D{YuOfYK--C@n~Xq9901h)OAf)OXG6{rMN} z$NifDXV09o&)#dVbzRTpPa7#u9J*)p#BuB8RLOFbX3{>Ya4$2jLNvDy zw>q=%3sFei>CY>j*5&p&#S&Ut{tkTmqlw3~<1yudI)ZBmZV9WDhWZk(d$)XLqAPME zNV#wsu6gmhkRTjnhRtH|bnP83+}cqitUDt`eP-waA;WYmGDYsUO2oRIBZZW^8cJr> zN2+l|k+C?>%T1GlSJ2#D>Z6AYL&mS4oCO5vS#7$ z_dg;<_iaUZ#y^bhv%H<{>q5jJQmk3Xw#ttdcx}&?iq)em=8j{lM7Y**tcPZI;UtxRGthB%cmUbEnnwoYv+Z>2j0j|xZQhKX*{4?F$f6Oq zE+3lY_ab!8pP~b6%#!=waIB>TW(|-L5ejE~%FMGBrMb_Oezs!J^qV`WD}`pI6BN8)vCeh){ zNDkt|&eLjnNTcUF`H5nHxG9F{YF}ti{Vs+#A?@w!XrIbOg_53)V(3XLk{%u_X6-vt zQMX|HmN?=(kf3^>t5|o)?$_g;{F%{fE?Pum6Q##b!1qyRv1SSYQO7-_r?j) ziB)*^PM$xKJgFIV8vd&6lOO)VZ}vFy`{OB<>@|$t16oXqnQP6$^q(Pnu@Vz>Em&*K z0n*io{M|}YE(yuDm}u$xbxu63>XBs6+Q1d!1G*H=sJYFvbcL8+Q%3ILOv=R$Q$FL$ zu39`Y)_5Vt2K@fhShA!#>QHRdTo-eOYI#*#;^-^p*``i} zv2z`-IO=eLbBpjC{~nt!nro^cb1^9ZF-g(t`!H3$I|(Yy_B&BJ%F;XJ4!aeVc7FuG zRU#}vUB(-r92q$BImp7y%$V-eZsmBf*a?J>9E*7n3pKf=OtEE8xba)Cxj&m8Ivs)t zcX@(I&~H66pDxzZs+UD@ccDPz=6VKedhgMKVSNKvgM^f$3;VaNsU$^FADZT|qZ_&{ zH3EP3q}0;~wm(ou$xprC8UW*HJ=B1iw=GSHKS|}pU0;y>MwZpaqlWP9t{apkkocsw z-orKWMKjdm+LNp43zs4jJ@hPye?CttATAC3Stc$1ieC?>9d#}cC_cC2&>N$>qwj!l z7VO4{+L~*#m1ptr$KSmy*hKacfv=GBoN>(UI(?f7SVFvJOTU)@kL1UsZ8%VCxpvo~ zq6Nw*chesWz7j?j2yZvo!(~!=uw~@;eF9&BjbiMv>}PlC4iuzM>oQ5u5uG-CbC#Ds ztE*`7o7H#OP_OV@0rBr;_C!iva#QAp;{UFaUWXjA>O=JmvR{~(u1@z5zH_H0;AFm? zNFI5@S5Ovf()9H=hryWw(D8YPwpsDnjMd^H)_FA)}N$sqfMJn z#4>0`xcm`Gx1I1_%~GI=tNEi3T{6e2XxYR@r?=O z$;#JYA9fPV0g$LD{k<)Pp+Rdvl7MP&}+xqRGu>Ck#FuWh@*$4 z<;s}+S;h$u%s*Mt_YZgrrUXF1^Yj zlmA^m2U3F3B6)^IW}bvIYPBnDE)6?wUDX)Ggm!q=7iQ4|N1S9B^18)D^N z=m@q6ML&rtO4z1N&gHS{(k?OU4-5mrsiSdHb{pWLy8F}INVRkL{rH|_?sXto1K#Lc z!T2eK_TIh}S>DwpR(+=5S?umF)8bq~^c{N6H4Cs! zbLQQ<<1AP-rZf93*EWUaZV(r5_ZL*3+&)LhH3F^m*9Y0lc&s7EU9MypgWu`!r+_#n z=a2mRqPa`HtY#Z~iFV++m&uH%jqC|4TrR%%oUhn8R35Jljvv^Z<3fg^oCGzZ- zSd^$_JX#N%&7_lzU42){L7FYu<59lyeAKMlb{hi4YUYcY=s!$2d%P8Ji-Xy)BRs91 zr*qK3oxGn|Knj7@?ig~@;qggfDN?Kc4oVL8D+Km4@oqA}+qlLY#OJ8{T8Ax+#q4`} zTizYeU+9SgkR=n@#B7^mnAsLnxWpT{!BR-*k`)Oem36)w%SppLg*0}zVkNAOa9Igd z)7pp(!P>2{-l+Yq7OS{et|9c{epZk8g~JN6TWRZTk~my&ik@sU3hC9Gs{y6KuW3d6 zsWZVO51B)FKCF#AIcUHcMVi0;x98eL(7kqvBfy^eA*T@P-t)Dm?5f{OxPn9Hj;;&e zPXG6JNB=G(qrr?Sq4~-airCwohlJO5&g$<|>mhLzw7jn`c6am!z)L~R-w5?Y6t1ce2)mNyP+1>bdXT*qCwf989TEc zX5Gu*GobAQU!8SM(%Kep-oHV(-W1MZwnD8Zx47Yvn>j3Zs2%h7TVoS{o=(kUzr$8^ ztD0bj8fPkX#YDx4i!%1H*Gs8ijTZ?Ngjcc}⁣oG^-x}cKWt7wDs|h&g%{-Ui_Yw zZXV(7y=Z=S(cBs%g%-yOjtP$r%7x%n;&Opi2Z}W@S2&R^uoF4!Gmw-|ehR>rU_5!| zsHH@uGWeR?Om7nb@$#S*vvulsqr*h*31*a+vJPW0%Mmht`wM5$*|op$>7c$u1QFavH|a$-Bo~*Y_LS zEl?}xIlZ1QNo5?12v;U`3-MBf9GF%(AAM9km!X>Z^GC(_xd~Ef|h{AOFQnLAh>|!;>r6Ss<}g`i>T3rfm%D*_Vu?x z+d_ii(o5U{`0v!RT!Z>3Y2dd1@Q+(qC?;{-rXUU9Mqi6hTu&m6Ws_ z@;%heOD5My=tyFeChvau3mrWCmz?{~JCNg1&xtXV^EX4Dq>uViknT&$4%?`0#gU?8 zHKg~7@Tz|qzs_BwB)kIQf-)R!B!)<+_wlz?F#Cdv+d5v{@bVSUIBF|4Q_mijoNnKg z-i(zV8zmS|6C%OSfI^#D&CEufsCNg`f8o8D`IGiL-^U|QH{@!AaFl4S zP$EdR1k$HBvwhXX_mxSGr!JsGyuZfzsO#H(Y0=w@!WmMv+mrWyLs|}WGOKbkD|~(D zNzX<`Gtpcf(E+<$ch%YNCS$2Y4`$}_2vT&@G4|Thni)5wK2-ERVt3dJdSFe!BoEtd zu|S8Gib8<_&%|3(rIYB<(82ibCk5Q17>iZlVR)#mfb1dh^oQ405pwmNrY(EIn&z+`f(@l5F@}v-t~*K8YSOY>$R|<1)TBqqCy4>qrD}Q`{i5$J)fxslJ(S}%R>%YIg-akFs z1z0MC+mh1h4X)V>GI`p$TJE`o?j2Wd7i$z za>gP-Qhk#Sy{&e888?SEu5e8AYXUlmtQF7@QxL3=atJXzw|gWsc4v_%J-cnD|A^%q z@93qQVsqsFv2sd&af)JEf6|{TJos0iZ~;QfqnQd;Yri;#?a-Bngvwd-lwM}E%%E#$ zE2L9t3KEr4kKb|6OE7=)A;(!aFY49HXw`j!SKDS_nbA3dqOT zQ)AmqIY*}L4loAjysjCk(^nVbZ%(+gugU#RC}`{lw~0g>C}(`&M_Q((IQO^g_G#JLa%z=T<)ZEv{JOrD)UX_K@tSKwjb5MP9^-gtWD{DbmlKb$J zlRgT4aGHy!^F>h!ujbJp(~~aq>@bw=&d^%tW=Vr|;cgjQ@nGY1PRsx~f}`oL1f-Of zbgzf@gKXq!Vj}tveq+ChQBi9=rye$X(RT<$Xvt68wGtQ&qcVoWgcb!8f=JIdQqxKc z`+8qSeV{cACVydW7$qyc5y8INg~blnMMvyh7m;|L5;Pa3DslBsNi<2%=pc`vD9feT zc%>VjzOEAzH+16*do+TsNv$l=DbLRS&>bBE!ENe|?AIyBL9mCsjHAu6Vt;!lc;8F-GxO!EdL6+r6%Trw$c|0F2FcO<&?R8N0hmiO*sZ@TtjFUxxq$h<>G3^nJ-K!Qn>T$+hmht1`6B&MVJ+CxGpXs1`G$bRDa-Ss2 z>A7W*J)pe~dUl=El}&hSN}|TZ#bjL!53SwoS*_|~l&48m-k7XKnfr32)myPaR8r_i zTkV@_NRUgZ6oE$1sV;yp`Avi-Tbhcm)ce+?{!*46NUY1)bzv3RjQLR+54`k0?53CW z9wF!x&Y<=}*hoRye=k7uMK(ISA;w{b)3EkBA;an9bff6%d64L%)4y!C=R1>6F6*t+ z(iyOVsmP6gPU<3{I*la^Qe)S01`8?TY-PBbp`5?0qQTWr_#rbN>kcPL(~^y@UDikO z%mrt8l=~BqwrTBrxfTCH+*}Wt^zamqn!7p%&0jLJ@6Dnw>yZ@xfMser%&TqClT;u@ zmRkON9jqLWu1cQR#pZ{fzi>zQCiAQr3xRb?r9iptTJ0L@*G`nOzpPMXx%Toh~YpUtTdwoi~{tq z<_zQ-Jhi};PWtq{*_@z0KV7~b;jqhUd^6)IoRB=m?Ej`?^%y%a_2|9!nv=NF{^#jc z&6O;cHD#w9rW=qTj-bt+G3x!2Hkz6C&8bh~AyKFVD}Il|a3Iq|tmkYRO(`9n7G??y%u$f%Zh-pRPQvVZcRddRn8 zh;q$IH3=UWZIyg$YS6V;Hs1PhChm9i1I1wKmx+bg{wFpwV6JW ztg#Mt+OBDI%^H7=#NJuR$!G}?F}CQ^vr9y{IAiuHFP+H_Ogn=2q;^6?5eKD1)Zt^K zmsh{WhE^L_O#!#-=E1kKU!c~P3jF@&t+n``B9wLsHcIWqOtZwUA?&(hX|LS2s~IX{ zNae2R7*vlH`?2;uv+3ee($1TRMJQwt4Nb_nm|$sM*d}?n&)145LV>VghQN4_6+;3( zmNYF8gKdaWg6**V{p}o0`*)o6@7kq%v|~x{8M)XsW3&z!&p}e;Pzftw6I4|rgbriO z=s7?NG_#y9PiM>#b!iWqVRQAy2Fwv8u85VzM*3YA$?-6hY+eBN#iY;0vFdx><$D;0 z-m1y1x1sk!`Fi^j9>|rE*?Sbp26>}aR~)|85l|!2%@5zNQw;#Nz(Z)ibm?i4vrnwV ze`TLb#BNtp@vh3ZbR{Td+Yu7BFek+!1a$X|X=bhuF<^=|DQb2saoXnAl0iyX53QmO zWJPoPvSjDA52Fhu3$NM}Chp)bX6LqIZ;WFAYa6P}%Zno-JOgC6C-sZ6AO?!$Mz1J< z%8G@j!cqpt+&|Mw_q%?Ya??_Y|_UcaMaiW$79 zG1i+p9*YD^j)SLznaZs1itWEpebm$i)TqBSI%`K-yWYWV7hrJSY~K86wx*>Ubz-3| zUu8&+$g8_r+lq6UYRUQGC`~)T;|Z#5Y>iw{)PDKPnLZfj8vr};9R=}Mg(Px`4sy-hwhuNO*sYmBMrtIE9P z+j#2MZ=mfdTy|q$cStMru$3uQRqm?r1VxaD^}cZ@3nDmPc>cg}@V3ug18KoNIr5m8 zrhH~aiYj&Io|=g?%$KwebH?aLmp&XDHGQ-FBy-5$lMtsbtOQ~OHJU)bwypvlPKf7sdTH~<1)Wki6U~KiaN&{n_GoLhNPiA+$x2j!hYnGLq!H-?v+ioFd|34*I&z!NmT%Zys5>JPY&S zaH7AU&ce@%jcg(_KN6M-eyaY5Kq*2fZdkCL$kAi!M2DC{qvNK_Zh_BRzcqewrsKra z%Z+($OR?Wi{%PINSB}kS!()Bozlz%YSoHzYe1`79nRvR4Z{Glr>S6=$UvW6<@fbr^ z>9J0C(WbRDWX3A$f2y-IsZQk4d{Mk_h14B$rMCF62JBF-@f5d{D>$CH%S2sUjUR2# zE@Jfq4u<86sLg61eH7jfOxh%%+QJcWWO>;mWCLZ z12GZlMF(ao3GhfCP0$iKDCj|9w6FfQ8q-)>-qH3+-*Nge(?HuzC}ZIzdjd3z z@e=3Xihv(w4AV&OKKmp()x?s!$=50mG7A1gh5e_ze!^*OhT{Cfi5YFjjaj-&naE30 zdV#={GV_3tsuG_gRvkBT1iwTFl{2CK@=ghuipai35nt2Z63X&?Kd-`Sgtv(5+Rl*@ z#SmTgNE6tLOc%fOF>v_(h2u@As#qD*_5B~_KdgWpTvobmM}|Qa8=}C7nmoffB4N6P8O5F0hpt7R^hYp(xGuKafu3)^8bLuwTWG|LKGOGxEp zq+p5nwZ!*Vl#8M;Ql$JJ$K6!xVMdv=w9|vKe=$r6+cwB3ML%zG<}?A^EGXbMe3Ao@k>b{ zxDD^}*D(I-8%*<&z0DF~9ZTc`y7r{&y~E;LF_U?ks0|aYATmzBy698+J!mgj^j zugXwg)+|*7yubG2nU6VYce#6hfZxGk+~{D-0KDYx;lF*OI;9?)la~9H2_09c&5uRl>`jVzAbk^ zRhGvFISG2RGM@HsiPsqR)Flmd0xQv`YnhbkJRaJlIM$NIc6HW!FZvG)w5B{w#Cu+Q zH30ud@(pcKJVXOSXB2rgacIQ{u?E6tdXN9XmwGCDWAM+Dd~!D4B#c?vuoQ)4j0F4z znKQAMWKH^!n^I<=&P&>W+u9|Aa4LSXD|5yYkShC2io1zyoiK?sjzMdW)KOL>q>YO( z-vpQjX@+emBsF&9pQSfxwa*zmrmm0q7p6YYbQ0vUerdsTH@VHqw8nypJ$hN0Obb8q zvi*7R7(h=>tg3=o({M0ZR>00DOepex@B7MTtqV>FrUM*Nn%mlVh^sDsr*y(U92f$V z+Wd%<&(Nf<;||?GE4;4yS%t+I;VZqQWAXaA;5wd@;287ieXvt1`9VxK_&QFDMjP+m z*D4ergEPn(e$Q9B99Df2w90os*b3b|+ zE^8vftSo->srx={HCn;`W%1v#S06cfd*>Faj|U-kMOd^rpLAH^nM~)+b7Ode3 zIk1$op%YmP4(Nl;gPr8)fyRMhYV>#`3<5EPykBgoo>J6vX6SZMA+mNL?IXFP2lV`H z1j5IP`c(ulrtjJh>Jird&67a=D7>pJ9c=?uafyB7r=s5+#pu4RClirr$PkyG_-pO}v`(Zcl#Vye0Xn z5l<-ZU`%$r?59GfLC(k@!uK89u3#ES=?a|sxyE!6CZ;3k6ctH!b7m)n_@~g~y#LqR z|1oB+(<6OPDI*izb7%$llWmLxH4wj`DF#t{YsybE#%|j}i*v}~@$a)A`jK!1JV1!q zzGmCPEm)O0unpOIdfXGh6KYat^Vqiqd}0PXPrmu5b=>mDwr;!U-?s0`Sg;%6R1J(f z06=(^FZ$-sX08qPf^{vS!9%KwE>ojWEXt@Bh`PH=vUFm3Gbo`SfIq398+mzST(?hy z6OXOlzV{M`%Fe>gWI-}cjk+~-BVoRhKA zdyzfDzDr#|3~s)v>nwfsZkm4e9wsKE zl)al)K$3kg8{Spn|K8Q*3zTn$M>^Is@SI#+oxYWZtnZJ}AB@1pjbh}u{9wZprX7k# z%i_5T>k^Annn^}`+CV)7Wq;HwX^3GBvufGjN6HedW;!8N*4W?Z=rTJcoU87Q7B8R5E^`mINC$hdtT}- zdY_L|%}5x7dM;C8wL{+Gto)0ww; zKb9Ucte>}-IuZYDrAk>#s&Ldr9+H`f-hkyq8P1j0+3V_#DVanZ`oP*aYM8PgqHsPe zcgf)Q{GEe_i9vna_c8{fyMwkAk(xFyp zb9&2yaO=Lczn$Xb9X8t1`U>RYK?~>+c}8S&N*HgO-%Tzw!{zue0&G<>sFmZL6bg0h zMwR~NWKVoTS03B>qj5On>&OSu_w#a(gJ-yEg3g@|Q@XBYF2)jNn+5u99gLAwbovLg zc!A8DBR`8<=SL{!v!n}shH_6Q_>G_G7u%G_;|$MU+uA4$Ai;lC1W>3QHH!07u+8n- zEH38z59{{bl;)ujF-$-S(?m)KtiSa5O!B$G`tEGM72icE5!FED}t3~u8V@x?D*PBnLa&VZY%j(@F1@lf0O6Hle4=0+1L_2aM> z2tl%{IC(qs>wGB@sd9M>vafy?+d#L9dZSop$tUQ2>9}pm3 zW?!ECiH&WfnZzvrMO)^nuT!m&3Z=~L4CtBTWQ*5RQnEnq5eUZfhSU(lLJ}P3p{zDY zRU|yP>GlLdfB?_CREk5s z^M0h;x0q9FaT~?KUZ>y5!X(;@n_Tw)B~uWdOv(yEoV0miAPrJ3YEg$~>Z?vBZ35LYd57%QIMM2^`J* zrnU2s@kkg9cuv813=jhC*FiUQM2#d+58}#mnExGG;Fa!>j7}Cno-VTz<33dU^YP}J zyXpFW$~V;Qn*oL@;lHM-L-15PHw9);w-}DpLKt8gQUsuHW0@L$xy=;G%*`6BwtX;g zWG;-hPGq+UVgm*=0^zM3j7j5(ZW2~L3Jqm#9xPHyvQIn*JxVjQSy2#X$}hblxP+Io z^rEEA^d17W6wEqcGvTgfv{m0r&!f3+#ELvnsUC@CP8tRkMG96kb^{CgJaE*G7Z$EX|Wv=)|sm z!=#t`TwMdp8-WOM9OI#}OJG%(Z8%aiaNx;#M;< z60DzqJJ0e=h3D8?-wqp8RQyI}@m~=W(<`e!ECGfFfR-5BbLjX)KyUGl1n0PRqYQVS zsgExysVmuX7IEgtUXc6fuwnGZnYz(*!gd#v3w`}uwR-R{duQ~v42(>Y5w7-*J zL2&C;)U2wxE+L+9owH!GUKIJRHTBOL?$%YaFJK20n)5ClkDjK2Ga@NT=xQOxpiG%` zfWlO1CXj*sEVplCkvcAZFJ-i`;1+ z;S6P}3*ZQQ+3+W^y(6bBltA2PVP1%bc;{isnOt%l1@FEQON7l)GvGe zVY~IylB+sL2Pk4MCA*HDjC;(fj7`#DtFVaU_}6G9rzQt+26z>UPZkOx2VZf8h(Tca zMw&`vQdy~1d=fo;;m?LXvE}&Pi4rEoZu^uc$vtbtfi4h;au-5Gslv~6+ZVhj0!~_N z$2%&VSaz1r`8^_yHE~Y0USeD!7!nU1z#$4;^m-|9VBy78;#eq=X#uoM@STZ^WhMsf57sD7SpUFRfHUz*jPzOjC z#(q=*%V~Hhv?T+CH4qx~_1fFR&t5=@!nM#({SxuC8r+3#)XWwfpPLgmF_dnQ58irV zGFs&Sae71sdcWc4-v%BWbe_QusWh!yfW;d2S)iJ=7ZAC;JIx<4etCPig*TC}bN3az zxKDuX69Y@8 z*gK$acK-o-@z*^F3otxkNc|Rr-wD9BHI@oMAArysBEAJ!r4ONF!b;ZP%}LyYen`5( z%KHUslP!&czznG;1ZW1)#x`U;a!iqS|*U zOv5KXqX-3A9Ph$e7$n|Lz!=uPs+%Xg3{^uuqw>KxVus+$GT%VX;SQ|7XL0H58qC0$ zf#~!cbQohbKx1a{-TDrH&e{9iZ$P8crGPlP*1S!6^*MWKBX_~<&@&1R`m%8_rj7o* zT!m@p)c4}%%13kPdZ-9yy5!b7Ll{H#b0GEvoc;h0BRQJT48*o)m|kubBo{XR?>T7& zTb15i4}C(`-XhvMn6XHtZk)+aHu%3gANz3y+f`n3950#Z0>VoTZNx7OU`TL62mJ0d z#!KL6joue=eCM0!aqcz!d>Sk=!s-F?pc1nV`uYlxGTy3!)Ph#=`JvDb&9Tw!r&tAh-O3fFQZ1&I2ea)tFYQy}YV{KNma0Ay`RK__qnC`ZI`0yftPn1BEB{UJING zQ(^+W?ZcIlAAmaGNE&*x8Jikgw@}e|e6Ibr2&-r*iEKC!OxOBb?;W@dy)doe7WG7mVbfUZ{rwF59#!4Z6ThIrm`3~$|9Fm2uT*U`gu z?PKt5QkdE*#wwK}-Z3@}A{rr-k~X;9a1#FWv6dU9Htj${!bxk1Ul*PDcV$D$BuUb! zVDjjSrHle*M1SV4Ac5dzW#yH)sLP~e#&SHtDC088Zq@nSk`-mL+<1mZBCXd?-%ho!G8p8q=un?5N1UibD_!?5?^%_-Kr_hoIz;WnZAgJu)IKd~9v z8z9U&$Lloqy#)L)g!u&=@ij z$Y3Zg>f2h!ojWW5MF5YtPKtb_N)f|r^)}+;mUh`mP024VI3&dqW>uGC`s zky5PcyAeCc^3dv+C)#?C8EqrKYi*o%6-x)i72jPa(7^#8q&nU{83O}@7Wu}F6#aBl zl{X{YNAdFoa71i~QzrTeTyYGg(bHiEOw^k+iOYc zy;XxBW!$kCP~$F|~X<1{f5C!@ryJLmQmd) z19QX*tI|6h8l6El^ioGKAH@fmyDOzy_MYzUjSDGP6pMJDe?9FC)!^fxU_{&OVqwqS zh&4i9%QDgmR8vCqu&OrsiS`%I&HZ*p~&8e$tKt4fiiI>Z1gQGZ%`Qb`Mz>s>kTB)pENRkmc2v{Iu&9qS!w+u`TOb?d0K7D4Z3U6 zL=tCQv-Sz!N)KDq#0pJ&h5Y+R#D=NH15-?y9w`YUT1nZ^Z|r`mvkwNnEQpD_H>*p< z$GTg5rR9c)^DY9U$Lx{PX73T8N#?1*cUROx&$ zKzxK4&Xkoa&vZIwxN`Ee^=E8Gc$lmDsw?Gh6Frq7|KFEDf13aI@86G8Q{ti(pYQmk zn9)>kuzbib7~eP+t7QGQt-ta0>sN3C>xid&9)g5(j4yZw#F8#MlR}7|jX4^%c-dVC&ua+AeA4-aXwJut5T2 z`sNJVk;Nw|JrkGA9e&l)QfJl>W#;O%nt$i=4z-X?QpHihd|e~lra$=Cj4#q`m>Ozo zeuBLxR%IEmJfowdHnZz}v@Uz6m5lNsl=_eFRO&s~>MH;AaeDgJLCGr_6Llkx#@e<* z8^C9xuiZA%)+YP{FY9Z!Nj46t&8EVD+)yIw(RdRIBfGe(-8I=M#=%yZ7B~5+um0eU zU#HWmNnby|C&8u#dJq|)$bknPUZh4$ zZXMTxl=#2!;@F@J+>>c~mp&rSc@#Y%E7OKCt;I93zU$WHg z+K0A+P|jmRQAL(R({_R9rP*TeoQf#Y)Aiu=aR8IAjE!Y9*e63n*^n4d+YX(JTaWbuz+NV-pu~PZ#?=Kwh+*!*D(C=btpyvKR4FKe6Ht%)%iCE zVDI!hG&D7RcwPG63wfj!(OIurXCUyBkg)fIm5yuKTL{n&5AVLOE#^j7!^Bhps>efEN;_2M6Q02~6Lz#)+TPCa4FE!3K0Z5t zcNWnX`vl&ePWABGI3eQa_UQrd1AI8xc?n~2XjWj6g{A`>MdtZbruDjN&!8Q@$1k0+Zbi+?8$zyo{ literal 0 HcmV?d00001 From fac96456c241a1a34975453d1db0b8418446fe2c Mon Sep 17 00:00:00 2001 From: xzl Date: Fri, 1 Dec 2017 20:41:34 +0800 Subject: [PATCH 07/94] add prelu neon impl --- paddle/math/Matrix.cpp | 23 +++++++++++++++++++- paddle/math/NEONFunctions.cpp | 40 +++++++++++++++++++++++++++++++++++ paddle/math/NEONFunctions.h | 1 + 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 88e9180690..be87a4c296 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -28,6 +28,7 @@ limitations under the License. */ #include "hl_top_k.h" #include "paddle/utils/Logging.h" +#include "NEONFunctions.h" #include "paddle/function/GemmFunctor.h" #include "paddle/utils/ThreadLocal.h" @@ -4157,16 +4158,36 @@ void CpuMatrix::print(std::ostream& os) const { void CpuMatrix::paramReluForward(Matrix& data, Matrix& W) { real* input = data.getData(); real* w = W.getData(); + real* output = data_; size_t numElements = data.getWidth(); size_t numSamples = data.getHeight(); size_t paraSize = W.getHeight() * W.getWidth(); CHECK(!(numElements % paraSize)); // this check from ParameterReluLayer::init + size_t partial_sum = numElements / paraSize; + if (paraSize == numElements) { + for (size_t n = 0; n < numSamples * numElements; ++n) { + output[n] = input[n] > 0 ? input[n] : input[n] * w[n % numElements]; + } + return; + } + +#if defined(__ARM_NEON__) || defined(__ARM_NEON) + for (size_t n = 0; n < numSamples; ++n) { + for (size_t i = 0; i < paraSize; i++) { + neon::prelu( + input + i * partial_sum, w[i], output + i * partial_sum, partial_sum); + } + input = input + numElements; + output = output + numElements; + } +#else for (size_t n = 0, k = 0; n < numSamples; ++n) { for (size_t i = 0; i < numElements; ++i, ++k) { - data_[k] = input[k] > 0 ? input[k] : input[k] * w[i / partial_sum]; + output[k] = input[k] > 0 ? input[k] : input[k] * w[i / partial_sum]; } } +#endif } void CpuMatrix::paramReluBackwardW(Matrix& oGrad, Matrix& data) { diff --git a/paddle/math/NEONFunctions.cpp b/paddle/math/NEONFunctions.cpp index 3bf47901f1..0f83149422 100644 --- a/paddle/math/NEONFunctions.cpp +++ b/paddle/math/NEONFunctions.cpp @@ -49,6 +49,46 @@ void relu(const float* a, float* b, int len) { } } +// b[i] = a[i] > 0.0f ? a[i] : a[i] * w +void prelu(const float* a, float w, float* b, int len) { + int offset = len % 16; + float32x4_t ma0, ma1, ma2, ma3; + + float32x4_t zero = vdupq_n_f32(0.f); + float32x4_t vw = vdupq_n_f32(w); + + for (int k = 0; k < len / 16; k++, a += 16, b += 16) { + ma0 = vld1q_f32(a); + ma1 = vld1q_f32(a + 4); + ma2 = vld1q_f32(a + 8); + ma3 = vld1q_f32(a + 12); + + uint32x4_t flag0 = vcgtq_f32(ma0, zero); + uint32x4_t flag1 = vcgtq_f32(ma1, zero); + uint32x4_t flag2 = vcgtq_f32(ma2, zero); + uint32x4_t flag3 = vcgtq_f32(ma3, zero); + + float32x4_t mul0 = vmulq_f32(ma0, vw); + float32x4_t mul1 = vmulq_f32(ma1, vw); + float32x4_t mul2 = vmulq_f32(ma2, vw); + float32x4_t mul3 = vmulq_f32(ma3, vw); + + ma0 = vbslq_f32(flag0, ma0, mul0); + ma1 = vbslq_f32(flag1, ma1, mul1); + ma2 = vbslq_f32(flag2, ma2, mul2); + ma3 = vbslq_f32(flag3, ma3, mul3); + + vst1q_f32(b, ma0); + vst1q_f32(b + 4, ma1); + vst1q_f32(b + 8, ma2); + vst1q_f32(b + 12, ma3); + } + + for (int i = 0; i < offset; i++) { + b[i] = a[i] > 0.0f ? a[i] : a[i] * w; + } +} + } // namespace neon } // namespace paddle diff --git a/paddle/math/NEONFunctions.h b/paddle/math/NEONFunctions.h index 69085e3335..d67b2f47a8 100644 --- a/paddle/math/NEONFunctions.h +++ b/paddle/math/NEONFunctions.h @@ -18,6 +18,7 @@ namespace paddle { namespace neon { void relu(const float* a, float* b, int len); +void prelu(const float* a, float w, float* b, int len); } // namespace neon } // namespace paddle From 5c057f95529656e379ef404a2e388e1be3e88de1 Mon Sep 17 00:00:00 2001 From: sweetsky0901 Date: Sun, 3 Dec 2017 14:40:33 +0800 Subject: [PATCH 08/94] add spp op only can test ok --- paddle/operators/spp_op.cc | 98 +++++++++++++ paddle/operators/spp_op.h | 148 ++++++++++++++++++++ python/paddle/v2/fluid/tests/test_spp_op.py | 48 +++++++ 3 files changed, 294 insertions(+) create mode 100644 paddle/operators/spp_op.cc create mode 100644 paddle/operators/spp_op.h create mode 100644 python/paddle/v2/fluid/tests/test_spp_op.py diff --git a/paddle/operators/spp_op.cc b/paddle/operators/spp_op.cc new file mode 100644 index 0000000000..62fc2112a8 --- /dev/null +++ b/paddle/operators/spp_op.cc @@ -0,0 +1,98 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +Indicesou may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/spp_op.h" +namespace paddle { +namespace operators { + +class SppOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SppOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput( + "X", + "(Tensor) The input tensor of spp operator. " + "The format of input tensor is NCHW. Where N is batch size, C is the " + "number of channels, H and W is the height and width of feature."); + AddOutput("Out", + "(Tensor) The output tensor of spp operator." + "N * M." + "M = C * H * W"); + AddAttr("pyramid_height", ">= 1"); + AddComment(R"DOC( + "Input shape: $(N, C_{in}, H_{in}, W_{in})$ + Output shape: $(H_{out}, W_{out})$ + Where + $$ + H_{out} = (H_{in}−1) * strides[0] − 2 * paddings[0] + ksize[0] \\ + W_{out} = (W_{in}−1) * strides[1] − 2 * paddings[1] + ksize[1] + $$ + )DOC"); + } +}; + +int OutputSize(int pyramid_level, int input_size) { + int bins = std::pow(2, pyramid_level); + int ksize = std::ceil(input_size / static_cast(bins)); + int padding = (ksize * bins - input_size + 1) / 2; + int output_size = (input_size - ksize + 2 * padding) / ksize + 1; + // output_size = bins + return output_size; +} + +class SppOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of SppOp" + "should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of SppOp should not be null."); + auto in_x_dims = ctx->GetInputDim("X"); + int pyramid_height = ctx->Attrs().Get("pyramid_height"); + PADDLE_ENFORCE(in_x_dims.size() == 4, + "Spping intput must be of 4-dimensional."); + int outlen = 0; + for (int p = 0; p < pyramid_height; ++p) { + int outh = OutputSize(p, in_x_dims[2]); + int outw = OutputSize(p, in_x_dims[3]); + int p_level_outlen = outh * outw * in_x_dims[1]; + outlen += p_level_outlen; + } + std::vector output_shape({in_x_dims[0], outlen}); + ctx->SetOutputDim("Out", framework::make_ddim(output_shape)); + } +}; + +class SppOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null."); + PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")), + "Input(X@GRAD) should not be null."); + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(spp, ops::SppOp, ops::SppOpMaker, spp_grad, ops::SppOpGrad); +REGISTER_OP_CPU_KERNEL(spp, ops::SppKernel, + ops::SppKernel); +REGISTER_OP_CPU_KERNEL(spp_grad, + ops::SppGradKernel, + ops::SppGradKernel); diff --git a/paddle/operators/spp_op.h b/paddle/operators/spp_op.h new file mode 100644 index 0000000000..2a2824bb31 --- /dev/null +++ b/paddle/operators/spp_op.h @@ -0,0 +1,148 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +Indicesou may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" +#include "paddle/operators/math/pooling.h" +#include "paddle/operators/strided_memcpy.h" + +namespace paddle { +namespace operators { +template +class SppKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const framework::Tensor* in_x = context.Input("X"); + auto* out = context.Output("Out"); + int pyramid_height = context.template Attr("pyramid_height"); + out->mutable_data(context.GetPlace()); + auto out_stride = framework::stride(out->dims()); + int input_h = in_x->dims()[2]; + int input_w = in_x->dims()[3]; + size_t output_offset = 0; + for (int p = 0; p < pyramid_height; ++p) { + int bins = std::pow(2, p); + int ksize_h = std::ceil(input_h / static_cast(bins)); + int ksize_w = std::ceil(input_w / static_cast(bins)); + int padding_h = (ksize_h * bins - input_h + 1) / 2; + int padding_w = (ksize_w * bins - input_w + 1) / 2; + std::vector ksize({ksize_h, ksize_w}); + std::vector strides({ksize_h, ksize_w}); + std::vector paddings({padding_h, padding_w}); + // pooling output shape + std::vector output_shape_vec({in_x->dims()[0], in_x->dims()[1]}); + output_shape_vec.push_back((input_h - ksize_h + 2 * padding_h) / ksize_h + + 1); + output_shape_vec.push_back((input_w - ksize_w + 2 * padding_w) / ksize_w + + 1); + framework::DDim output_shape(framework::make_ddim(output_shape_vec)); + // flatten pooling output shape + int output_flatten_w = in_x->dims()[1] * bins * bins; + std::vector output_flatten_shape_vec( + {in_x->dims()[0], output_flatten_w}); + framework::DDim output_flatten_shape( + framework::make_ddim(output_flatten_shape_vec)); + framework::Tensor out_level; + framework::Tensor out_flatten_level; + out_level.mutable_data(output_shape, context.GetPlace()); + // pooling + math::Pool2dFunctor, T> pool_forward; + math::MaxPool max_process; + pool_forward(context.device_context(), *in_x, ksize, strides, paddings, + max_process, &out_level); + out_flatten_level.ShareDataWith(out_level); + out_flatten_level.Resize(output_flatten_shape); + auto in_stride = framework::stride(out_flatten_level.dims()); + const T* src_data = out_flatten_level.data(); + StridedMemcpy(context.device_context(), src_data, in_stride, + out_flatten_level.dims(), out_stride, + out->data() + output_offset); + output_offset += out_flatten_level.dims()[1] * in_stride[1]; + } + } +}; +template +class SppGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const framework::Tensor* in_x = context.Input("X"); + const framework::Tensor* out = context.Input("Out"); + const framework::Tensor* out_grad = + context.Input(framework::GradVarName("Out")); + framework::Tensor* in_x_grad = + context.Output(framework::GradVarName("X")); + auto& device_ctx = context.device_context(); + math::SetConstant zero; + in_x_grad->mutable_data(context.GetPlace()); + zero(device_ctx, in_x_grad, static_cast(0)); + int pyramid_height = context.template Attr("pyramid_height"); + auto outgrad_stride = framework::stride(out_grad->dims()); + auto out_stride = framework::stride(out->dims()); + int input_h = in_x->dims()[2]; + int input_w = in_x->dims()[3]; + size_t out_offset = 0; + for (int p = 0; p < pyramid_height; ++p) { + int bins = std::pow(2, p); + int ksize_h = std::ceil(input_h / static_cast(bins)); + int ksize_w = std::ceil(input_w / static_cast(bins)); + int padding_h = (ksize_h * bins - input_h + 1) / 2; + int padding_w = (ksize_w * bins - input_w + 1) / 2; + std::vector ksize({ksize_h, ksize_w}); + std::vector strides({ksize_h, ksize_w}); + std::vector paddings({padding_h, padding_w}); + // split outgrad and get flatten + std::vector out_shape_vec({in_x->dims()[0], in_x->dims()[1]}); + out_shape_vec.push_back((input_h - ksize_h + 2 * padding_h) / ksize_h + + 1); + out_shape_vec.push_back((input_w - ksize_w + 2 * padding_w) / ksize_w + + 1); + framework::DDim out_shape(framework::make_ddim(out_shape_vec)); + int out_flatten_w = in_x->dims()[1] * bins * bins; + std::vector out_flatten_shape_vec( + {in_x->dims()[0], out_flatten_w}); + framework::DDim out_flatten_shape( + framework::make_ddim(out_flatten_shape_vec)); + framework::Tensor out_level; + framework::Tensor outgrad_level; + framework::Tensor out_flatten_level; + framework::Tensor outgrad_flatten_level; + out_flatten_level.mutable_data(out_flatten_shape, context.GetPlace()); + outgrad_flatten_level.mutable_data(out_flatten_shape, + context.GetPlace()); + + auto flatten_stride = framework::stride(out_flatten_level.dims()); + // memcpy + StridedMemcpy(context.device_context(), out->data() + out_offset, + out_stride, out_flatten_level.dims(), flatten_stride, + out_flatten_level.data()); + + StridedMemcpy(context.device_context(), + out_grad->data() + out_offset, outgrad_stride, + outgrad_flatten_level.dims(), flatten_stride, + outgrad_flatten_level.data()); + out_offset += out_flatten_level.dims()[1] * out_stride[1]; + // flatten backward + out_level.ShareDataWith(out_flatten_level); + out_level.Resize(out_shape); + outgrad_level.ShareDataWith(outgrad_flatten_level); + outgrad_level.Resize(out_shape); + math::MaxPool2dGradFunctor pool2d_backward; + pool2d_backward(context.device_context(), *in_x, *&out_level, + *&outgrad_level, ksize, strides, paddings, in_x_grad); + } + } +}; +} // namespace operators +} // namespace paddle diff --git a/python/paddle/v2/fluid/tests/test_spp_op.py b/python/paddle/v2/fluid/tests/test_spp_op.py new file mode 100644 index 0000000000..806d5e7736 --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_spp_op.py @@ -0,0 +1,48 @@ +import unittest +import numpy as np +from op_test import OpTest +from test_pool2d_op import max_pool2D_forward_naive + + +class TestSppOp(OpTest): + def setUp(self): + self.op_type = "spp" + self.init_test_case() + input = np.random.random(self.shape).astype("float32") + nsize, csize, hsize, wsize = input.shape + out_level_flatten = [] + for i in xrange(self.pyramid_height): + bins = np.power(2, i) + ksize = [0, 0] + padding = [0, 0] + ksize[0] = np.ceil(hsize / bins.astype("double")).astype("int32") + padding[0] = ((ksize[0] * bins - hsize + 1) / 2).astype("int32") + + ksize[1] = np.ceil(wsize / bins.astype("double")).astype("int32") + padding[1] = ((ksize[1] * bins - wsize + 1) / 2).astype("int32") + out_level = max_pool2D_forward_naive(input, ksize, ksize, padding) + out_level_flatten.append( + out_level.reshape(nsize, bins * bins * csize)) + if i == 0: + output = out_level_flatten[i] + else: + output = np.concatenate((output, out_level_flatten[i]), 1) + # output = np.concatenate(out_level_flatten.tolist(), 0); + self.inputs = {'X': input.astype('float32'), } + self.attrs = {'pyramid_height': self.pyramid_height} + + self.outputs = {'Out': output.astype('float32')} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + def init_test_case(self): + self.shape = [1, 1, 2, 2] + self.pyramid_height = 2 + + +if __name__ == '__main__': + unittest.main() From 531e7b6fa694e7c15ca3831704097946162248fe Mon Sep 17 00:00:00 2001 From: sweetsky0901 Date: Sun, 3 Dec 2017 17:26:43 +0800 Subject: [PATCH 09/94] gpu test ok --- paddle/operators/spp_op.cc | 28 +++------- paddle/operators/spp_op.cu.cc | 22 ++++++++ paddle/operators/spp_op.h | 61 +++++++++++---------- python/paddle/v2/fluid/tests/test_spp_op.py | 6 +- 4 files changed, 64 insertions(+), 53 deletions(-) create mode 100644 paddle/operators/spp_op.cu.cc diff --git a/paddle/operators/spp_op.cc b/paddle/operators/spp_op.cc index 62fc2112a8..ff607c5769 100644 --- a/paddle/operators/spp_op.cc +++ b/paddle/operators/spp_op.cc @@ -29,28 +29,22 @@ class SppOpMaker : public framework::OpProtoAndCheckerMaker { "(Tensor) The output tensor of spp operator." "N * M." "M = C * H * W"); - AddAttr("pyramid_height", ">= 1"); + AddAttr("pyramid_height", "int"); AddComment(R"DOC( - "Input shape: $(N, C_{in}, H_{in}, W_{in})$ + "Does spatial pyramid pooling on the input image by taking the max, + etc. within regions so that the result vector of different sized + images are of the same size + Input shape: $(N, C_{in}, H_{in}, W_{in})$ Output shape: $(H_{out}, W_{out})$ Where $$ - H_{out} = (H_{in}−1) * strides[0] − 2 * paddings[0] + ksize[0] \\ - W_{out} = (W_{in}−1) * strides[1] − 2 * paddings[1] + ksize[1] + H_{out} = N \\ + W_{out} = ((std::pow(4, pyramid_height) - 1) / (4 - 1)) * C_{in} $$ )DOC"); } }; -int OutputSize(int pyramid_level, int input_size) { - int bins = std::pow(2, pyramid_level); - int ksize = std::ceil(input_size / static_cast(bins)); - int padding = (ksize * bins - input_size + 1) / 2; - int output_size = (input_size - ksize + 2 * padding) / ksize + 1; - // output_size = bins - return output_size; -} - class SppOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -64,13 +58,7 @@ class SppOp : public framework::OperatorWithKernel { int pyramid_height = ctx->Attrs().Get("pyramid_height"); PADDLE_ENFORCE(in_x_dims.size() == 4, "Spping intput must be of 4-dimensional."); - int outlen = 0; - for (int p = 0; p < pyramid_height; ++p) { - int outh = OutputSize(p, in_x_dims[2]); - int outw = OutputSize(p, in_x_dims[3]); - int p_level_outlen = outh * outw * in_x_dims[1]; - outlen += p_level_outlen; - } + int outlen = ((std::pow(4, pyramid_height) - 1) / (4 - 1)) * in_x_dims[1]; std::vector output_shape({in_x_dims[0], outlen}); ctx->SetOutputDim("Out", framework::make_ddim(output_shape)); } diff --git a/paddle/operators/spp_op.cu.cc b/paddle/operators/spp_op.cu.cc new file mode 100644 index 0000000000..a7057907ce --- /dev/null +++ b/paddle/operators/spp_op.cu.cc @@ -0,0 +1,22 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +Indicesou may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/spp_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(spp, ops::SppKernel, + ops::SppKernel); +REGISTER_OP_GPU_KERNEL(spp_grad, + ops::SppGradKernel, + ops::SppGradKernel); diff --git a/paddle/operators/spp_op.h b/paddle/operators/spp_op.h index 2a2824bb31..7a385352a0 100644 --- a/paddle/operators/spp_op.h +++ b/paddle/operators/spp_op.h @@ -42,34 +42,36 @@ class SppKernel : public framework::OpKernel { std::vector strides({ksize_h, ksize_w}); std::vector paddings({padding_h, padding_w}); // pooling output shape + framework::Tensor out_level; std::vector output_shape_vec({in_x->dims()[0], in_x->dims()[1]}); output_shape_vec.push_back((input_h - ksize_h + 2 * padding_h) / ksize_h + 1); output_shape_vec.push_back((input_w - ksize_w + 2 * padding_w) / ksize_w + 1); framework::DDim output_shape(framework::make_ddim(output_shape_vec)); - // flatten pooling output shape - int output_flatten_w = in_x->dims()[1] * bins * bins; - std::vector output_flatten_shape_vec( - {in_x->dims()[0], output_flatten_w}); - framework::DDim output_flatten_shape( - framework::make_ddim(output_flatten_shape_vec)); - framework::Tensor out_level; - framework::Tensor out_flatten_level; out_level.mutable_data(output_shape, context.GetPlace()); // pooling math::Pool2dFunctor, T> pool_forward; math::MaxPool max_process; pool_forward(context.device_context(), *in_x, ksize, strides, paddings, max_process, &out_level); + // flatten pooling output shape + framework::Tensor out_flatten_level; + int output_flatten_w = in_x->dims()[1] * bins * bins; + std::vector output_flatten_shape_vec( + {in_x->dims()[0], output_flatten_w}); + framework::DDim output_flatten_shape( + framework::make_ddim(output_flatten_shape_vec)); out_flatten_level.ShareDataWith(out_level); out_flatten_level.Resize(output_flatten_shape); - auto in_stride = framework::stride(out_flatten_level.dims()); - const T* src_data = out_flatten_level.data(); - StridedMemcpy(context.device_context(), src_data, in_stride, - out_flatten_level.dims(), out_stride, - out->data() + output_offset); - output_offset += out_flatten_level.dims()[1] * in_stride[1]; + // concat + auto out_flatten_level_stride = + framework::stride(out_flatten_level.dims()); + StridedMemcpy(context.device_context(), out_flatten_level.data(), + out_flatten_level_stride, out_flatten_level.dims(), + out_stride, out->data() + output_offset); + output_offset += + out_flatten_level.dims()[1] * out_flatten_level_stride[1]; } } }; @@ -83,12 +85,11 @@ class SppGradKernel : public framework::OpKernel { context.Input(framework::GradVarName("Out")); framework::Tensor* in_x_grad = context.Output(framework::GradVarName("X")); + int pyramid_height = context.template Attr("pyramid_height"); auto& device_ctx = context.device_context(); math::SetConstant zero; in_x_grad->mutable_data(context.GetPlace()); zero(device_ctx, in_x_grad, static_cast(0)); - int pyramid_height = context.template Attr("pyramid_height"); - auto outgrad_stride = framework::stride(out_grad->dims()); auto out_stride = framework::stride(out->dims()); int input_h = in_x->dims()[2]; int input_w = in_x->dims()[3]; @@ -102,26 +103,17 @@ class SppGradKernel : public framework::OpKernel { std::vector ksize({ksize_h, ksize_w}); std::vector strides({ksize_h, ksize_w}); std::vector paddings({padding_h, padding_w}); - // split outgrad and get flatten - std::vector out_shape_vec({in_x->dims()[0], in_x->dims()[1]}); - out_shape_vec.push_back((input_h - ksize_h + 2 * padding_h) / ksize_h + - 1); - out_shape_vec.push_back((input_w - ksize_w + 2 * padding_w) / ksize_w + - 1); - framework::DDim out_shape(framework::make_ddim(out_shape_vec)); + // split out and outgrad ... to flatten + framework::Tensor out_flatten_level; + framework::Tensor outgrad_flatten_level; int out_flatten_w = in_x->dims()[1] * bins * bins; std::vector out_flatten_shape_vec( {in_x->dims()[0], out_flatten_w}); framework::DDim out_flatten_shape( framework::make_ddim(out_flatten_shape_vec)); - framework::Tensor out_level; - framework::Tensor outgrad_level; - framework::Tensor out_flatten_level; - framework::Tensor outgrad_flatten_level; out_flatten_level.mutable_data(out_flatten_shape, context.GetPlace()); outgrad_flatten_level.mutable_data(out_flatten_shape, context.GetPlace()); - auto flatten_stride = framework::stride(out_flatten_level.dims()); // memcpy StridedMemcpy(context.device_context(), out->data() + out_offset, @@ -129,15 +121,24 @@ class SppGradKernel : public framework::OpKernel { out_flatten_level.data()); StridedMemcpy(context.device_context(), - out_grad->data() + out_offset, outgrad_stride, + out_grad->data() + out_offset, out_stride, outgrad_flatten_level.dims(), flatten_stride, outgrad_flatten_level.data()); out_offset += out_flatten_level.dims()[1] * out_stride[1]; - // flatten backward + // flatten backward to nchw + framework::Tensor out_level; + framework::Tensor outgrad_level; + std::vector out_shape_vec({in_x->dims()[0], in_x->dims()[1]}); + out_shape_vec.push_back((input_h - ksize_h + 2 * padding_h) / ksize_h + + 1); + out_shape_vec.push_back((input_w - ksize_w + 2 * padding_w) / ksize_w + + 1); + framework::DDim out_shape(framework::make_ddim(out_shape_vec)); out_level.ShareDataWith(out_flatten_level); out_level.Resize(out_shape); outgrad_level.ShareDataWith(outgrad_flatten_level); outgrad_level.Resize(out_shape); + // pooling backward math::MaxPool2dGradFunctor pool2d_backward; pool2d_backward(context.device_context(), *in_x, *&out_level, *&outgrad_level, ksize, strides, paddings, in_x_grad); diff --git a/python/paddle/v2/fluid/tests/test_spp_op.py b/python/paddle/v2/fluid/tests/test_spp_op.py index 806d5e7736..89b12e885c 100644 --- a/python/paddle/v2/fluid/tests/test_spp_op.py +++ b/python/paddle/v2/fluid/tests/test_spp_op.py @@ -37,11 +37,11 @@ class TestSppOp(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['X'], 'Out') + self.check_grad(['X'], 'Out', max_relative_error=0.05) def init_test_case(self): - self.shape = [1, 1, 2, 2] - self.pyramid_height = 2 + self.shape = [3, 2, 4, 4] + self.pyramid_height = 3 if __name__ == '__main__': From 8368e55be93ea6d2912e3ebebb35e284c5428a28 Mon Sep 17 00:00:00 2001 From: sweetsky0901 Date: Mon, 4 Dec 2017 11:19:16 +0800 Subject: [PATCH 10/94] modify some doc --- paddle/operators/spp_op.cc | 4 +- paddle/operators/spp_op.h | 47 +++++++++++---------- python/paddle/v2/fluid/tests/test_spp_op.py | 19 ++++++--- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/paddle/operators/spp_op.cc b/paddle/operators/spp_op.cc index ff607c5769..026b35de1e 100644 --- a/paddle/operators/spp_op.cc +++ b/paddle/operators/spp_op.cc @@ -29,7 +29,7 @@ class SppOpMaker : public framework::OpProtoAndCheckerMaker { "(Tensor) The output tensor of spp operator." "N * M." "M = C * H * W"); - AddAttr("pyramid_height", "int"); + AddAttr("pyramid_height", "int", "multi level pooling"); AddComment(R"DOC( "Does spatial pyramid pooling on the input image by taking the max, etc. within regions so that the result vector of different sized @@ -39,7 +39,7 @@ class SppOpMaker : public framework::OpProtoAndCheckerMaker { Where $$ H_{out} = N \\ - W_{out} = ((std::pow(4, pyramid_height) - 1) / (4 - 1)) * C_{in} + W_{out} = (((4^pyramid_height) - 1) / (4 - 1))$ * C_{in} $$ )DOC"); } diff --git a/paddle/operators/spp_op.h b/paddle/operators/spp_op.h index 7a385352a0..0f2c43ee65 100644 --- a/paddle/operators/spp_op.h +++ b/paddle/operators/spp_op.h @@ -34,27 +34,27 @@ class SppKernel : public framework::OpKernel { size_t output_offset = 0; for (int p = 0; p < pyramid_height; ++p) { int bins = std::pow(2, p); - int ksize_h = std::ceil(input_h / static_cast(bins)); - int ksize_w = std::ceil(input_w / static_cast(bins)); - int padding_h = (ksize_h * bins - input_h + 1) / 2; - int padding_w = (ksize_w * bins - input_w + 1) / 2; - std::vector ksize({ksize_h, ksize_w}); - std::vector strides({ksize_h, ksize_w}); + int kernel_size_h = std::ceil(input_h / static_cast(bins)); + int kernel_size_w = std::ceil(input_w / static_cast(bins)); + int padding_h = (kernel_size_h * bins - input_h + 1) / 2; + int padding_w = (kernel_size_w * bins - input_w + 1) / 2; + std::vector kernel_size({kernel_size_h, kernel_size_w}); + std::vector strides({kernel_size_h, kernel_size_w}); std::vector paddings({padding_h, padding_w}); // pooling output shape framework::Tensor out_level; std::vector output_shape_vec({in_x->dims()[0], in_x->dims()[1]}); - output_shape_vec.push_back((input_h - ksize_h + 2 * padding_h) / ksize_h + - 1); - output_shape_vec.push_back((input_w - ksize_w + 2 * padding_w) / ksize_w + - 1); + output_shape_vec.push_back( + (input_h - kernel_size_h + 2 * padding_h) / kernel_size_h + 1); + output_shape_vec.push_back( + (input_w - kernel_size_w + 2 * padding_w) / kernel_size_w + 1); framework::DDim output_shape(framework::make_ddim(output_shape_vec)); out_level.mutable_data(output_shape, context.GetPlace()); // pooling math::Pool2dFunctor, T> pool_forward; math::MaxPool max_process; - pool_forward(context.device_context(), *in_x, ksize, strides, paddings, - max_process, &out_level); + pool_forward(context.device_context(), *in_x, kernel_size, strides, + paddings, max_process, &out_level); // flatten pooling output shape framework::Tensor out_flatten_level; int output_flatten_w = in_x->dims()[1] * bins * bins; @@ -96,12 +96,12 @@ class SppGradKernel : public framework::OpKernel { size_t out_offset = 0; for (int p = 0; p < pyramid_height; ++p) { int bins = std::pow(2, p); - int ksize_h = std::ceil(input_h / static_cast(bins)); - int ksize_w = std::ceil(input_w / static_cast(bins)); - int padding_h = (ksize_h * bins - input_h + 1) / 2; - int padding_w = (ksize_w * bins - input_w + 1) / 2; - std::vector ksize({ksize_h, ksize_w}); - std::vector strides({ksize_h, ksize_w}); + int kernel_size_h = std::ceil(input_h / static_cast(bins)); + int kernel_size_w = std::ceil(input_w / static_cast(bins)); + int padding_h = (kernel_size_h * bins - input_h + 1) / 2; + int padding_w = (kernel_size_w * bins - input_w + 1) / 2; + std::vector kernel_size({kernel_size_h, kernel_size_w}); + std::vector strides({kernel_size_h, kernel_size_w}); std::vector paddings({padding_h, padding_w}); // split out and outgrad ... to flatten framework::Tensor out_flatten_level; @@ -129,10 +129,10 @@ class SppGradKernel : public framework::OpKernel { framework::Tensor out_level; framework::Tensor outgrad_level; std::vector out_shape_vec({in_x->dims()[0], in_x->dims()[1]}); - out_shape_vec.push_back((input_h - ksize_h + 2 * padding_h) / ksize_h + - 1); - out_shape_vec.push_back((input_w - ksize_w + 2 * padding_w) / ksize_w + - 1); + out_shape_vec.push_back( + (input_h - kernel_size_h + 2 * padding_h) / kernel_size_h + 1); + out_shape_vec.push_back( + (input_w - kernel_size_w + 2 * padding_w) / kernel_size_w + 1); framework::DDim out_shape(framework::make_ddim(out_shape_vec)); out_level.ShareDataWith(out_flatten_level); out_level.Resize(out_shape); @@ -141,7 +141,8 @@ class SppGradKernel : public framework::OpKernel { // pooling backward math::MaxPool2dGradFunctor pool2d_backward; pool2d_backward(context.device_context(), *in_x, *&out_level, - *&outgrad_level, ksize, strides, paddings, in_x_grad); + *&outgrad_level, kernel_size, strides, paddings, + in_x_grad); } } }; diff --git a/python/paddle/v2/fluid/tests/test_spp_op.py b/python/paddle/v2/fluid/tests/test_spp_op.py index 89b12e885c..b57f4a795d 100644 --- a/python/paddle/v2/fluid/tests/test_spp_op.py +++ b/python/paddle/v2/fluid/tests/test_spp_op.py @@ -13,14 +13,19 @@ class TestSppOp(OpTest): out_level_flatten = [] for i in xrange(self.pyramid_height): bins = np.power(2, i) - ksize = [0, 0] + kernel_size = [0, 0] padding = [0, 0] - ksize[0] = np.ceil(hsize / bins.astype("double")).astype("int32") - padding[0] = ((ksize[0] * bins - hsize + 1) / 2).astype("int32") - - ksize[1] = np.ceil(wsize / bins.astype("double")).astype("int32") - padding[1] = ((ksize[1] * bins - wsize + 1) / 2).astype("int32") - out_level = max_pool2D_forward_naive(input, ksize, ksize, padding) + kernel_size[0] = np.ceil(hsize / + bins.astype("double")).astype("int32") + padding[0] = ( + (kernel_size[0] * bins - hsize + 1) / 2).astype("int32") + + kernel_size[1] = np.ceil(wsize / + bins.astype("double")).astype("int32") + padding[1] = ( + (kernel_size[1] * bins - wsize + 1) / 2).astype("int32") + out_level = max_pool2D_forward_naive(input, kernel_size, + kernel_size, padding) out_level_flatten.append( out_level.reshape(nsize, bins * bins * csize)) if i == 0: From 141a323c3410f9d65b75cd546af69715735db23a Mon Sep 17 00:00:00 2001 From: sweetsky0901 Date: Mon, 4 Dec 2017 11:43:14 +0800 Subject: [PATCH 11/94] fix a bug --- paddle/operators/spp_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/operators/spp_op.cc b/paddle/operators/spp_op.cc index 026b35de1e..5e51e73ecc 100644 --- a/paddle/operators/spp_op.cc +++ b/paddle/operators/spp_op.cc @@ -29,7 +29,7 @@ class SppOpMaker : public framework::OpProtoAndCheckerMaker { "(Tensor) The output tensor of spp operator." "N * M." "M = C * H * W"); - AddAttr("pyramid_height", "int", "multi level pooling"); + AddAttr("pyramid_height", "(int), multi level pooling"); AddComment(R"DOC( "Does spatial pyramid pooling on the input image by taking the max, etc. within regions so that the result vector of different sized From 4f1381eac3708ce92b07a01f6cfc9d4131c996af Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 8 Dec 2017 16:20:09 +0800 Subject: [PATCH 12/94] recv_op use serialized program --- paddle/operators/recv_op.cc | 11 +++++++---- paddle/operators/send_recv_op_test.cc | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index c69e416e10..45222f6b76 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -72,8 +72,10 @@ class RecvOp : public framework::OperatorBase { // FIXME(typhoonzero): do not copy framework::CopyFrom(t, dev_ctx.GetPlace(), dev_ctx, tensor); - auto *block = Attr("OptimizeBlock"); - auto *program = block->Program(); + std::string program_str = Attr("OptimizeProgram"); + framework::Program program_desc; + program_desc.ParseFromString(program_str); + framework::ProgramDescBind program(program_desc); framework::Executor executor(dev_ctx); // Run sub graph to get optimized tensor executor.Run(*program, &recv_scope, block->ID(), @@ -108,8 +110,9 @@ This operator will recv tensor from send_op "IP address to listen on.") .SetDefault("127.0.0.1:6164") .AddCustomChecker([](const std::string &ip) { return !ip.empty(); }); - AddAttr("OptimizeBlock", "type BlockDescBind*", - "optimize network run in server"); + AddAttr( + "OptimizeProgram", "type string", + "Serialized ProgramDesc string for recv to run."); } }; diff --git a/paddle/operators/send_recv_op_test.cc b/paddle/operators/send_recv_op_test.cc index ac03eb3752..c35dc8fa50 100644 --- a/paddle/operators/send_recv_op_test.cc +++ b/paddle/operators/send_recv_op_test.cc @@ -85,7 +85,7 @@ void StartServerNet() { paddle::framework::AttributeMap attrs; attrs.insert({"endpoint", std::string("127.0.0.1:6174")}); - attrs.insert({"OptimizeBlock", block}); + attrs.insert({"OptimizeProgram", program.Proto()->SerializeToString()}); recv_op = paddle::framework::OpRegistry::CreateOp("recv", {{"RX", {"RX"}}}, {{"Out", {"Out"}}}, attrs); paddle::platform::CPUDeviceContext ctx(place); From 986ca03ce24f6d84eb9cfaef64b59fda2298823b Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 8 Dec 2017 19:45:15 +0800 Subject: [PATCH 13/94] update --- paddle/operators/recv_op.cc | 9 ++++----- paddle/operators/send_recv_op_test.cc | 5 ++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 45222f6b76..eed482c1b4 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -73,12 +73,12 @@ class RecvOp : public framework::OperatorBase { framework::CopyFrom(t, dev_ctx.GetPlace(), dev_ctx, tensor); std::string program_str = Attr("OptimizeProgram"); - framework::Program program_desc; + framework::ProgramDesc program_desc; program_desc.ParseFromString(program_str); framework::ProgramDescBind program(program_desc); framework::Executor executor(dev_ctx); // Run sub graph to get optimized tensor - executor.Run(*program, &recv_scope, block->ID(), + executor.Run(program, &recv_scope, 0, /*global_block*/ false /*create_local_scope*/); auto *out_var = recv_scope.FindVar("Out"); @@ -110,9 +110,8 @@ This operator will recv tensor from send_op "IP address to listen on.") .SetDefault("127.0.0.1:6164") .AddCustomChecker([](const std::string &ip) { return !ip.empty(); }); - AddAttr( - "OptimizeProgram", "type string", - "Serialized ProgramDesc string for recv to run."); + AddAttr("OptimizeProgram", "type string", + "Serialized ProgramDesc string for recv to run."); } }; diff --git a/paddle/operators/send_recv_op_test.cc b/paddle/operators/send_recv_op_test.cc index c35dc8fa50..3e2e2051af 100644 --- a/paddle/operators/send_recv_op_test.cc +++ b/paddle/operators/send_recv_op_test.cc @@ -85,7 +85,10 @@ void StartServerNet() { paddle::framework::AttributeMap attrs; attrs.insert({"endpoint", std::string("127.0.0.1:6174")}); - attrs.insert({"OptimizeProgram", program.Proto()->SerializeToString()}); + std::string program_proto; + PADDLE_ENFORCE(program.Proto()->SerializeToString(&program_proto)); + + attrs.insert({"OptimizeProgram", program_proto}); recv_op = paddle::framework::OpRegistry::CreateOp("recv", {{"RX", {"RX"}}}, {{"Out", {"Out"}}}, attrs); paddle::platform::CPUDeviceContext ctx(place); From 578ad6d23251c0fc08cbf49aa1d6bf9daae55a88 Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Mon, 11 Dec 2017 11:21:47 +0800 Subject: [PATCH 14/94] Use PADDLE_WITH_NATIVE_FP16 for float16_t. --- paddle/math/float16.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/math/float16.h b/paddle/math/float16.h index f805cad08b..76ad3a0123 100644 --- a/paddle/math/float16.h +++ b/paddle/math/float16.h @@ -101,7 +101,7 @@ public: half tmp = __float2half(val); x = *reinterpret_cast(&tmp); -#elif defined(PADDLE_NEON) +#elif defined(PADDLE_WITH_NATIVE_FP16) float32x4_t tmp = vld1q_dup_f32(&val); float16_t res = vget_lane_f16(vcvt_f16_f32(tmp), 0); x = *reinterpret_cast(&res); @@ -252,7 +252,7 @@ public: half tmp = *reinterpret_cast(this); return __half2float(tmp); -#elif defined(PADDLE_NEON) +#elif defined(PADDLE_WITH_NATIVE_FP16) float16x4_t res = vld1_dup_f16(reinterpret_cast(this)); return vgetq_lane_f32(vcvt_f32_f16(res), 0); From a6ef875885486eea5573e4da52ec2c1c02f1d1e7 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Mon, 11 Dec 2017 14:05:15 +0800 Subject: [PATCH 15/94] refine conv --- paddle/operators/conv_op.h | 7 +++++-- paddle/operators/conv_transpose_op.h | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/paddle/operators/conv_op.h b/paddle/operators/conv_op.h index 09bff0a68d..66728d75a6 100644 --- a/paddle/operators/conv_op.h +++ b/paddle/operators/conv_op.h @@ -260,8 +260,11 @@ class GemmConvGradKernel : public framework::OpKernel { if (input_grad) { input_grad->mutable_data(context.GetPlace()); - set_zero(context.device_context(), input_grad, static_cast(0)); - + // if is_expand is false, the operation of set_zero is unnecessary, + // because math::matmul will reset input_grad. + if (is_expand) { + set_zero(context.device_context(), input_grad, static_cast(0)); + } math::Col2VolFunctor col2vol; math::Col2ImFunctor col2im; diff --git a/paddle/operators/conv_transpose_op.h b/paddle/operators/conv_transpose_op.h index 1cacb770e6..a43dd5b8c0 100644 --- a/paddle/operators/conv_transpose_op.h +++ b/paddle/operators/conv_transpose_op.h @@ -225,7 +225,7 @@ class GemmConvTransposeGradKernel : public framework::OpKernel { if (input_grad) { input_grad->mutable_data(context.GetPlace()); - set_zero(context.device_context(), input_grad, static_cast(0)); + // set_zero is unnecessary, math::matmul will reset input_grad. } if (filter_grad) { // filter size (m, c, k_h, k_w) filter_grad->mutable_data(context.GetPlace()); From 8d428bd9b89ec59dfcf16eb9e319a9106415b766 Mon Sep 17 00:00:00 2001 From: ranqiu Date: Mon, 4 Dec 2017 19:28:12 +0800 Subject: [PATCH 16/94] Update annotations of layers.py --- .../paddle/trainer_config_helpers/layers.py | 121 ++++++++++-------- 1 file changed, 66 insertions(+), 55 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 4bd94861af..48858f4c34 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1516,34 +1516,33 @@ def lstmemory(input, NOTE: This is a low level user interface. You can use network.simple_lstm to config a simple plain lstm layer. - Please refer to **Generating Sequences With Recurrent Neural Networks** for - more details about LSTM. - - Link_ goes as below. - - .. _Link: http://arxiv.org/abs/1308.0850 + Reference: + `Generating Sequences With Recurrent Neural Networks + `_ - :param name: The lstmemory layer name. + :param name: The name of this layer. It is optional. :type name: basestring - :param size: DEPRECATED. size of the lstm cell + :param size: DEPRECATED. The dimension of the lstm cell. :type size: int :param input: The input of this layer. :type input: LayerOutput - :param reverse: is sequence process reversed or not. + :param reverse: Whether the input sequence is processed in a reverse order. :type reverse: bool :param act: Activation type. TanhActivation is the default activation. :type act: BaseActivation - :param gate_act: gate activation type, SigmoidActivation by default. + :param gate_act: Activation type of this layer's gates. SigmoidActivation is the + default activation. :type gate_act: BaseActivation - :param state_act: state activation type, TanhActivation by default. + :param state_act: Activation type of the state. TanhActivation is the default activation. :type state_act: BaseActivation :param bias_attr: The bias attribute. If the parameter is set to False or an object whose type is not ParameterAttribute, no bias is defined. If the parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any - :param param_attr: Parameter Attribute. - :type param_attr: ParameterAttribute | None | False - :param layer_attr: Extra Layer attribute + :param param_attr: The parameter attribute. See ParameterAttribute for details. + :type param_attr: ParameterAttribute + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute | None :return: LayerOutput object. :rtype: LayerOutput @@ -1632,14 +1631,14 @@ def grumemory(input, h_t = (1 - z_t) h_{t-1} + z_t {\\tilde{h_t}} NOTE: In PaddlePaddle's implementation, the multiplication operations - :math:`W_{r}x_{t}`, :math:`W_{z}x_{t}` and :math:`W x_t` are not computed in - gate_recurrent layer. Consequently, an additional mixed_layer with + :math:`W_{r}x_{t}`, :math:`W_{z}x_{t}` and :math:`W x_t` are not performed + in gate_recurrent layer. Consequently, an additional mixed_layer with full_matrix_projection or a fc_layer must be included before grumemory is called. - More details can be found by referring to `Empirical Evaluation of Gated - Recurrent Neural Networks on Sequence Modeling. - `_ + Reference: + `Empirical Evaluation of Gated Recurrent Neural Networks on Sequence Modeling + `_ The simple usage is: @@ -1647,28 +1646,29 @@ def grumemory(input, gru = grumemory(input) - :param name: The gru layer name. - :type name: None | basestring + :param name: The name of this layer. It is optional. + :type name: basestring :param input: The input of this layer. :type input: LayerOutput. - :param size: DEPRECATED. size of the gru cell + :param size: DEPRECATED. The dimension of the gru cell. :type size: int - :param reverse: Whether sequence process is reversed or not. + :param reverse: Whether the input sequence is processed in a reverse order. :type reverse: bool :param act: Activation type, TanhActivation is the default. This activation affects the :math:`{\\tilde{h_t}}`. :type act: BaseActivation - :param gate_act: gate activation type, SigmoidActivation by default. - This activation affects the :math:`z_t` and :math:`r_t`. It is the - :math:`\\sigma` in the above formula. + :param gate_act: Activation type of this layer's two gates. SigmoidActivation is + the default activation. This activation affects the :math:`z_t` + and :math:`r_t`. It is the :math:`\\sigma` in the above formula. :type gate_act: BaseActivation :param bias_attr: The bias attribute. If the parameter is set to False or an object whose type is not ParameterAttribute, no bias is defined. If the parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any - :param param_attr: Parameter Attribute. - :type param_attr: ParameterAttribute | None | False - :param layer_attr: Extra Layer attribute + :param param_attr: The parameter attribute. See ParameterAttribute for details. + :type param_attr: ParameterAttribute + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute | None :return: LayerOutput object. :rtype: LayerOutput @@ -1712,10 +1712,10 @@ def last_seq(input, """ Get Last Timestamp Activation of a sequence. - If stride > 0, this layer slides a window whose size is determined by stride, - and return the last value of the window as the output. Thus, a long sequence - will be shorten. Note that for sequence with sub-sequence, the default value - of stride is -1. + If stride > 0, this layer will slide a window whose size is determined by stride, + and return the last value of the sequence in the window as the output. Thus, a + long sequence will be shortened. Note that for sequence with sub-sequence, the + default value of stride is -1. The simple usage is: @@ -1724,14 +1724,16 @@ def last_seq(input, seq = last_seq(input=layer) :param agg_level: Aggregated level + :type agg_level: AggregateLevel :param name: The name of this layer. It is optional. :type name: basestring :param input: The input of this layer. :type input: LayerOutput :param stride: The step size between successive pooling regions. - :type stride: Int - :param layer_attr: extra layer attributes. - :type layer_attr: ExtraLayerAttribute. + :type stride: int + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. + :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput """ @@ -1768,10 +1770,10 @@ def first_seq(input, """ Get First Timestamp Activation of a sequence. - If stride > 0, this layer slides a window whose size is determined by stride, - and return the first value of the window as the output. Thus, a long sequence - will be shorten. Note that for sequence with sub-sequence, the default value - of stride is -1. + If stride > 0, this layer will slide a window whose size is determined by stride, + and return the first value of the sequence in the window as the output. Thus, a + long sequence will be shortened. Note that for sequence with sub-sequence, the + default value of stride is -1. The simple usage is: @@ -1780,13 +1782,15 @@ def first_seq(input, seq = first_seq(input=layer) :param agg_level: aggregation level + :type agg_level: AggregateLevel :param name: The name of this layer. It is optional. :type name: basestring :param input: The input of this layer. :type input: LayerOutput :param stride: The step size between successive pooling regions. - :type stride: Int - :param layer_attr: extra layer attributes. + :type stride: int + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute. :return: LayerOutput object. :rtype: LayerOutput @@ -1844,8 +1848,8 @@ def expand_layer(input, expand_level=ExpandLevel.FROM_NO_SEQUENCE, layer_attr=None): """ - A layer for "Expand Dense data or (sequence data where the length of each - sequence is one) to sequence data." + A layer for expanding dense data or (sequence data where the length of each + sequence is one) to sequence data. The example usage is: @@ -1857,7 +1861,9 @@ def expand_layer(input, :param input: The input of this layer. :type input: LayerOutput - :param expand_as: Expand as this layer's sequence info. + :param expand_as: Expand the input according to this layer's sequence infomation. And + after the operation, the input expanded will have the same number of + elememts as this layer. :type expand_as: LayerOutput :param name: The name of this layer. It is optional. :type name: basestring @@ -1865,9 +1871,10 @@ def expand_layer(input, whose type is not ParameterAttribute, no bias is defined. If the parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any - :param expand_level: whether input layer is timestep(default) or sequence. + :param expand_level: Whether the input layer is a sequence or the element of a sequence. :type expand_level: ExpandLevel - :param layer_attr: extra layer attributes. + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute. :return: LayerOutput object. :rtype: LayerOutput @@ -3294,7 +3301,7 @@ def row_l2_norm_layer(input, name=None, layer_attr=None): A layer for L2-normalization in each row. .. math:: - out[i] = \frac{in[i]}{\sqrt{\sum_{k=1}^N in[k]^{2}}} + out[i] = \\frac{in[i]} {\\sqrt{\\sum_{k=1}^N in[k]^{2}}} where the size of :math:`in` is (batchSize x dataDim) , and the size of :math:`out` is a (batchSize x dataDim) . @@ -6161,9 +6168,11 @@ def huber_regression_cost(input, Given a prediction f(x), a label y and :math:`\delta`, the loss function is defined as: - .. math: - loss = 0.5*\left ( y-f(x) \right )^2, \left | y-f(x) \right |\leq \delta - loss = \delta \left | y-f(x) \right |-0.5\delta ^2, otherwise + .. math:: + + loss = 0.5*(y-f(x))^{2}, | y-f(x) | < \delta + + loss = \delta | y-f(x) | - 0.5 \delta ^2, otherwise The example usage is: @@ -6210,12 +6219,14 @@ def huber_classification_cost(input, """ For classification purposes, a variant of the Huber loss called modified Huber is sometimes used. Given a prediction f(x) (a real-valued classifier score) and - a true binary class label :math:`y\in \left \{-1, 1 \right \}`, the modified Huber + a true binary class label :math:`y\in \{-1, 1 \}`, the modified Huber loss is defined as: .. math: - loss = \max \left ( 0, 1-yf(x) \right )^2, yf(x)\geq 1 - loss = -4yf(x), \text{otherwise} + + loss = \max ( 0, 1-yf(x) )^2, yf(x) \geq -1 + + loss = -4yf(x), otherwise The example usage is: @@ -6959,7 +6970,7 @@ def clip_layer(input, min, max, name=None): .. math:: - out[i] = \min\left(\max\left(in[i],p_{1}\right),p_{2}\right) + out[i] = \min (\max (in[i],p_{1} ),p_{2} ) .. code-block:: python From ddf20e589fad724f077b0613ebf3872d2311647a Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Mon, 11 Dec 2017 18:31:14 +0800 Subject: [PATCH 17/94] typo WITH_TEST to WITH_TESTING --- doc/howto/dev/contribute_to_paddle_cn.md | 2 +- paddle/scripts/docker/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/howto/dev/contribute_to_paddle_cn.md b/doc/howto/dev/contribute_to_paddle_cn.md index 6993901452..3eb477eb65 100644 --- a/doc/howto/dev/contribute_to_paddle_cn.md +++ b/doc/howto/dev/contribute_to_paddle_cn.md @@ -87,7 +87,7 @@ no changes added to commit (use "git add" and/or "git commit -a") 随后可以用这个开发镜像开始build PaddlePaddle的源码。比如如果要build一个不依赖GPU,但是支持AVX指令集,并且包括unit tests的PaddlePaddle,可以: ```bash -➜ docker run -v $(pwd):/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TEST=ON" paddle:dev +➜ docker run -v $(pwd):/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TESTING=ON" paddle:dev ``` 这个过程除了编译PaddlePaddle为 `./build/libpaddle.so`,并且输出一个 `./build/paddle.deb`文件之外,还会输出一个 `build/Dockerfile`。我们只需要运行下面命令把编译好的PaddlePaddle打包成一个*生产镜像*(`paddle:prod`): diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index f3a6f1dba7..1e1fcc50dc 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -192,7 +192,7 @@ For developers who are interested in the C++ source code, please use -e "WOBOQ=O - The following command builds PaddlePaddle, generates HTML pages from C++ source code, and writes HTML pages into `$HOME/woboq_out` on the host: ```bash -docker run -v $PWD:/paddle -v $HOME/woboq_out:/woboq_out -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TEST=ON" -e "WOBOQ=ON" paddlepaddle/paddle:latest-dev +docker run -v $PWD:/paddle -v $HOME/woboq_out:/woboq_out -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TESTING=ON" -e "WOBOQ=ON" paddlepaddle/paddle:latest-dev ``` - You can open the generated HTML files in your Web browser. Or, if you want to run a Nginx container to serve them for a wider audience, you can run: From 9f44af9d7c9dd2f1dc775f59d059a87eb9e64fd6 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 11 Dec 2017 18:37:18 +0800 Subject: [PATCH 18/94] Fix #6460 (#6461) --- python/paddle/v2/fluid/layers.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/python/paddle/v2/fluid/layers.py b/python/paddle/v2/fluid/layers.py index b4426bad14..fd8a2ed18c 100644 --- a/python/paddle/v2/fluid/layers.py +++ b/python/paddle/v2/fluid/layers.py @@ -762,7 +762,7 @@ def sequence_conv(input, helper = LayerHelper('sequence_conv', **locals()) dtype = helper.input_dtype() filter_shape = [filter_size * input.shape[1], num_filters] - filter = helper.create_parameter( + filter_param = helper.create_parameter( attr=helper.param_attr, shape=filter_shape, dtype=dtype) pre_bias = helper.create_tmp_variable(dtype) @@ -770,7 +770,7 @@ def sequence_conv(input, type='sequence_conv', inputs={ 'X': [input], - 'Filter': [filter], + 'Filter': [filter_param], }, outputs={"Out": pre_bias}, attrs={ @@ -785,7 +785,7 @@ def sequence_conv(input, def conv2d(input, num_filters, filter_size, - stride=[1, 1], + stride=None, padding=None, groups=None, param_attr=None, @@ -802,6 +802,8 @@ def conv2d(input, conv-2d output, if mentioned in the input parameters. """ + if stride is None: + stride = [1, 1] helper = LayerHelper('conv2d', **locals()) dtype = helper.input_dtype() @@ -827,7 +829,7 @@ def conv2d(input, std = (2.0 / (filter_size[0]**2 * num_channels))**0.5 return Normal(0.0, std, 0) - filter = helper.create_parameter( + filter_param = helper.create_parameter( attr=helper.param_attr, shape=filter_shape, dtype=dtype, @@ -839,7 +841,7 @@ def conv2d(input, type='conv2d_cudnn', inputs={ 'Input': input, - 'Filter': filter, + 'Filter': filter_param, }, outputs={"Output": pre_bias}, attrs={'strides': stride, @@ -875,8 +877,8 @@ def sequence_pool(input, pool_type, **kwargs): def pool2d(input, pool_size, pool_type, - pool_stride=[1, 1], - pool_padding=[0, 0], + pool_stride=None, + pool_padding=None, global_pooling=False, main_program=None, startup_program=None): @@ -884,6 +886,10 @@ def pool2d(input, This function adds the operator for pooling in 2 dimensions, using the pooling configurations mentioned in input parameters. """ + if pool_padding is None: + pool_padding = [0, 0] + if pool_stride is None: + pool_stride = [1, 1] if pool_type not in ["max", "avg"]: raise ValueError( "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", From 7389ea98eaae7ca79442811991cba37f908e5a74 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 11 Dec 2017 03:24:25 -0800 Subject: [PATCH 19/94] "add NCCL multi-GPU design doc" --- .../design}/images/multigpu_allreduce.graffle | Bin .../design}/images/multigpu_allreduce.png | Bin .../images/multigpu_before_convert.graffle | Bin .../design}/images/multigpu_before_convert.png | Bin .../multigpu.md => doc/design/paddle_nccl.md | 14 ++++++-------- 5 files changed, 6 insertions(+), 8 deletions(-) rename {paddle/framework => doc/design}/images/multigpu_allreduce.graffle (100%) rename {paddle/framework => doc/design}/images/multigpu_allreduce.png (100%) rename {paddle/framework => doc/design}/images/multigpu_before_convert.graffle (100%) rename {paddle/framework => doc/design}/images/multigpu_before_convert.png (100%) rename paddle/framework/multigpu.md => doc/design/paddle_nccl.md (83%) diff --git a/paddle/framework/images/multigpu_allreduce.graffle b/doc/design/images/multigpu_allreduce.graffle similarity index 100% rename from paddle/framework/images/multigpu_allreduce.graffle rename to doc/design/images/multigpu_allreduce.graffle diff --git a/paddle/framework/images/multigpu_allreduce.png b/doc/design/images/multigpu_allreduce.png similarity index 100% rename from paddle/framework/images/multigpu_allreduce.png rename to doc/design/images/multigpu_allreduce.png diff --git a/paddle/framework/images/multigpu_before_convert.graffle b/doc/design/images/multigpu_before_convert.graffle similarity index 100% rename from paddle/framework/images/multigpu_before_convert.graffle rename to doc/design/images/multigpu_before_convert.graffle diff --git a/paddle/framework/images/multigpu_before_convert.png b/doc/design/images/multigpu_before_convert.png similarity index 100% rename from paddle/framework/images/multigpu_before_convert.png rename to doc/design/images/multigpu_before_convert.png diff --git a/paddle/framework/multigpu.md b/doc/design/paddle_nccl.md similarity index 83% rename from paddle/framework/multigpu.md rename to doc/design/paddle_nccl.md index 1c843326ee..7c889fdd7f 100644 --- a/paddle/framework/multigpu.md +++ b/doc/design/paddle_nccl.md @@ -1,15 +1,17 @@ -# Design Doc: Multi-GPU support in Operation Graph +# Design Doc: NCCL support in Paddle Fluid ## Abstract -This Design Doc refers to the multi-GPU feature in paddle. We propose an approach to support multi-GPU both on a single machine and multiple machines. Every device only run sub-graphs which our framework issued. We use `Broadcast`, `Allreduce` operators to join different device sub-graph to the whole graph. - +This Design Doc refers to the NCCL feature in paddle. We propose an approach to support NCCL library both on a single machine and multiple machines. We wrapper the NCCL primitives `Broadcast`, `Allreduce`, `Reduce` as operators to utilize Multi-GPU powers in one script. ## Motivation -Paddle supports training with multiple CPUs and GPUs, refer to different physical devices. We need to support multi-GPU training in parallel for acceleration, in detail, there are two aspects. +NCCL is a Nvidia library support Multi-GPU communicating. [NCCL](https://developer.nvidia.com/nccl). With NCCL library, we can easily accelerate the training in parallel. +- can easily move the optimize sub-graph to parameter server, multi-GPU feature can be compatible with distributed support design. +- easily plug-in with [NCCL2](https://developer.nvidia.com/nccl) library. +- GPU Model parallelism becomes easier to implement. we only need to replace different GPU's sub-graph with different part of the whole graph. - GPU Data Parallelism Suppose to we have `n`GPUs, every GPU has `1/n`part of training data, and store a complete model in GPU memory. @@ -58,7 +60,3 @@ As it shown in the picture, when each GPU compute the gradient of `W`, followed In fact, in the way of every GPU optimized full batch of data, wasted (n-1) GPU compute resources. We will enhance it in the next stage. ### Benefits - -- can easily move the optimize sub-graph to parameter server, multi-GPU feature can be compatible with distributed support design. -- easily plug-in with [NCCL2](https://developer.nvidia.com/nccl) library. -- GPU Model parallelism becomes easier to implement. we only need to replace different GPU's sub-graph with different part of the whole graph. From 69b44f2f198509e29b8ab50edab9ab34f56fd1af Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Tue, 12 Dec 2017 15:31:44 +0800 Subject: [PATCH 20/94] unify MKL macro definition --- cmake/cblas.cmake | 2 +- cmake/external/mkldnn.cmake | 2 +- paddle/gserver/activations/ActivationFunction.cpp | 4 ++-- paddle/gserver/gradientmachines/NeuralNetwork.cpp | 4 ++-- paddle/math/Allocator.h | 2 +- paddle/math/MathFunctions.cpp | 2 +- paddle/math/MathFunctions.h | 2 +- paddle/memory/detail/system_allocator.cc | 2 +- paddle/operators/math/math_function.cc | 2 +- paddle/operators/math/math_function.h | 2 +- paddle/parameter/FirstOrderOptimizer.h | 2 +- paddle/parameter/ParameterUpdateFunctions.cpp | 2 +- paddle/utils/Flags.cpp | 2 +- 13 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmake/cblas.cmake b/cmake/cblas.cmake index b21fc43904..13294c0548 100644 --- a/cmake/cblas.cmake +++ b/cmake/cblas.cmake @@ -17,7 +17,7 @@ if(WITH_MKLML AND MKLML_INC_DIR AND MKLML_LIB) set(CBLAS_INC_DIR ${MKLML_INC_DIR}) set(CBLAS_LIBRARIES ${MKLML_LIB}) - add_definitions(-DPADDLE_USE_MKLML) + add_definitions(-DPADDLE_WITH_MKLML) add_definitions(-DLAPACK_FOUND) message(STATUS "Found cblas and lapack in MKLML " diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index fc52d339d7..5d24caebdc 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -67,5 +67,5 @@ ADD_LIBRARY(mkldnn SHARED IMPORTED GLOBAL) SET_PROPERTY(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB}) ADD_DEPENDENCIES(mkldnn ${MKLDNN_PROJECT}) MESSAGE(STATUS "MKLDNN library: ${MKLDNN_LIB}") -add_definitions(-DPADDLE_USE_MKLDNN) +add_definitions(-DPADDLE_WITH_MKLDNN) LIST(APPEND external_project_dependencies mkldnn) diff --git a/paddle/gserver/activations/ActivationFunction.cpp b/paddle/gserver/activations/ActivationFunction.cpp index f5a41b66bf..57c890e488 100644 --- a/paddle/gserver/activations/ActivationFunction.cpp +++ b/paddle/gserver/activations/ActivationFunction.cpp @@ -24,7 +24,7 @@ limitations under the License. */ #include "paddle/utils/ClassRegistrar.h" #include "paddle/utils/Logging.h" -#ifdef PADDLE_USE_MKLDNN +#ifdef PADDLE_WITH_MKLDNN #include "MKLDNNActivation.h" #endif @@ -490,7 +490,7 @@ Error __must_check backward(Argument& act) { END_DEFINE_ACTIVATION(log) ActivationFunction* ActivationFunction::create(const std::string& type) { -#ifdef PADDLE_USE_MKLDNN +#ifdef PADDLE_WITH_MKLDNN if (!type.empty() && type.compare(0, 7, "mkldnn_") == 0) { return MKLDNNActivation::create(type); } diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.cpp b/paddle/gserver/gradientmachines/NeuralNetwork.cpp index be112b4123..68bf37d59d 100644 --- a/paddle/gserver/gradientmachines/NeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/NeuralNetwork.cpp @@ -20,7 +20,7 @@ limitations under the License. */ #include "paddle/utils/Logging.h" #include "paddle/utils/Stat.h" -#ifdef PADDLE_USE_MKLDNN +#ifdef PADDLE_WITH_MKLDNN #include "paddle/gserver/layers/MKLDNNLayer.h" #endif @@ -307,7 +307,7 @@ void NeuralNetwork::backward(const UpdateCallback& callback) { } void NeuralNetwork::finish() { -#ifdef PADDLE_USE_MKLDNN +#ifdef PADDLE_WITH_MKLDNN FOR_EACH_R(layer, layers_) { MKLDNNLayerPtr dnnLayer = std::dynamic_pointer_cast(*layer); if (dnnLayer) { diff --git a/paddle/math/Allocator.h b/paddle/math/Allocator.h index 94ef561f06..17563bf5e1 100644 --- a/paddle/math/Allocator.h +++ b/paddle/math/Allocator.h @@ -48,7 +48,7 @@ public: */ virtual void* alloc(size_t size) { void* ptr; -#ifdef PADDLE_USE_MKLDNN +#ifdef PADDLE_WITH_MKLDNN // refer to https://github.com/01org/mkl-dnn/blob/master/include/mkldnn.hpp // memory alignment CHECK_EQ(posix_memalign(&ptr, 4096ul, size), 0); diff --git a/paddle/math/MathFunctions.cpp b/paddle/math/MathFunctions.cpp index ba86eacbb5..28ab54b450 100644 --- a/paddle/math/MathFunctions.cpp +++ b/paddle/math/MathFunctions.cpp @@ -206,7 +206,7 @@ double dotProduct(const int n, const double* x, const double* y) { } #endif -#if defined(PADDLE_USE_MKLML) +#if defined(PADDLE_WITH_MKLML) template <> void vExp(const int n, const float* a, float* r) { diff --git a/paddle/math/MathFunctions.h b/paddle/math/MathFunctions.h index f6e77029bd..29fe36e3a4 100644 --- a/paddle/math/MathFunctions.h +++ b/paddle/math/MathFunctions.h @@ -15,7 +15,7 @@ limitations under the License. */ #ifndef MATHFUNCTIONS_H_ #define MATHFUNCTIONS_H_ -#ifdef PADDLE_USE_MKLML +#ifdef PADDLE_WITH_MKLML #include #include #include diff --git a/paddle/memory/detail/system_allocator.cc b/paddle/memory/detail/system_allocator.cc index b543b767e8..6a815a1b57 100644 --- a/paddle/memory/detail/system_allocator.cc +++ b/paddle/memory/detail/system_allocator.cc @@ -43,7 +43,7 @@ void* CPUAllocator::Alloc(size_t& index, size_t size) { void* p; -#ifdef PADDLE_USE_MKLDNN +#ifdef PADDLE_WITH_MKLDNN // refer to https://github.com/01org/mkl-dnn/blob/master/include/mkldnn.hpp // memory alignment PADDLE_ENFORCE_EQ(posix_memalign(&p, 4096ul, size), 0); diff --git a/paddle/operators/math/math_function.cc b/paddle/operators/math/math_function.cc index 2e333a8cde..e099a6a439 100644 --- a/paddle/operators/math/math_function.cc +++ b/paddle/operators/math/math_function.cc @@ -132,7 +132,7 @@ void matmul( matrix_b.data(), beta, matrix_out->data()); } -#ifdef PADDLE_USE_MKLML +#ifdef PADDLE_WITH_MKLML // Use cblas_{s,d}gemm_batched if available: Run with 1 group of size batchSize. template <> void batched_gemm( diff --git a/paddle/operators/math/math_function.h b/paddle/operators/math/math_function.h index 5a42854f22..f2b025b78b 100644 --- a/paddle/operators/math/math_function.h +++ b/paddle/operators/math/math_function.h @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#ifdef PADDLE_USE_MKLML +#ifdef PADDLE_WITH_MKLML #include #include #include diff --git a/paddle/parameter/FirstOrderOptimizer.h b/paddle/parameter/FirstOrderOptimizer.h index f157188a4f..5b0c52a30d 100644 --- a/paddle/parameter/FirstOrderOptimizer.h +++ b/paddle/parameter/FirstOrderOptimizer.h @@ -38,7 +38,7 @@ public: real torch_learningRate = optConfig_.learning_method() == "torch_momentum" ? 1.0 - paraConfig.momentum() : 1.0; -#ifdef PADDLE_USE_MKLDNN +#ifdef PADDLE_WITH_MKLDNN sgdUpdate(learningRate_ * paraConfig.learning_rate() * (firstTime_ ? 1.0 : torch_learningRate), paraConfig.momentum(), diff --git a/paddle/parameter/ParameterUpdateFunctions.cpp b/paddle/parameter/ParameterUpdateFunctions.cpp index 1898598e49..d60cb36383 100644 --- a/paddle/parameter/ParameterUpdateFunctions.cpp +++ b/paddle/parameter/ParameterUpdateFunctions.cpp @@ -30,7 +30,7 @@ void sgdUpdateCpu(real learningRate, const real* grad, real* momentumVec) { decayRate *= learningRate; -#ifdef PADDLE_USE_MKLML +#ifdef PADDLE_WITH_MKLML #pragma omp parallel for #endif for (size_t i = 0; i < size; ++i) { diff --git a/paddle/utils/Flags.cpp b/paddle/utils/Flags.cpp index 8f100f02e9..9a7dc0e356 100644 --- a/paddle/utils/Flags.cpp +++ b/paddle/utils/Flags.cpp @@ -20,7 +20,7 @@ DEFINE_bool(use_gpu, false, "Only support CPU training"); DEFINE_bool(use_gpu, true, "Whether to use GPU for training"); #endif -#ifdef PADDLE_USE_MKLDNN +#ifdef PADDLE_WITH_MKLDNN // TODO(TJ): change to true when MKLDNN layers support multi-inputs DEFINE_bool(use_mkldnn, false, "Default still keep use CPU training"); #else From ebd0cf1408f3408551c342c8eb249f4313981695 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 11 Dec 2017 06:55:04 -0800 Subject: [PATCH 21/94] "add manual allreduce" --- doc/design/paddle_nccl.md | 44 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/doc/design/paddle_nccl.md b/doc/design/paddle_nccl.md index 7c889fdd7f..dba451b9b3 100644 --- a/doc/design/paddle_nccl.md +++ b/doc/design/paddle_nccl.md @@ -7,36 +7,34 @@ This Design Doc refers to the NCCL feature in paddle. We propose an approach t ## Motivation -NCCL is a Nvidia library support Multi-GPU communicating. [NCCL](https://developer.nvidia.com/nccl). With NCCL library, we can easily accelerate the training in parallel. +NCCL is a NVIDIA library support Multi-GPU communicating and optimized for NVIDIA GPUs, it provides routines such as all-gather, all-reduce, broadcast, reduce, reduce-scatter, that can achieve high bandwidth over PCIe and NVLink high-speed interconnect. [NCCL](https://developer.nvidia.com/nccl). With NCCL library, we can easily accelerate the training in parallel. -- can easily move the optimize sub-graph to parameter server, multi-GPU feature can be compatible with distributed support design. -- easily plug-in with [NCCL2](https://developer.nvidia.com/nccl) library. -- GPU Model parallelism becomes easier to implement. we only need to replace different GPU's sub-graph with different part of the whole graph. -- GPU Data Parallelism +- Pros +1. easily plug-in with [NCCL2](https://developer.nvidia.com/nccl) library. +1. high performance in NVIDIA GPUs. +1. MPI like primitives, which have low learning cost for users. - Suppose to we have `n`GPUs, every GPU has `1/n`part of training data, and store a complete model in GPU memory. +- Cons +1. Only design for NVIDIA GPUs, not a general multi-device solution. +1. Although NCCL1 is opensourced under BSD license, but NCCL2 is not opensourced anymore. -- GPU Model Parallelism +At the beginning of training, the framework needs to distribute the same parameters to every GPU, and merge the gradients at any time user interests. - every GPU have part of a complete model in GPU memory. +As a result, during training, we need the operations of peer to peer copy between different GPUs, aggregating gradients/parameters from GPUs, and broadcasting parameters to GPUs. Every GPU only need to run the operator with correct place information. -At the beginning of training, the framework needs to issue the same sub-graph to every GPU in Data Parallelism, or different sub-graph in Model Parallelism. - -During training, we need the operations of peer to peer copy between different GPUs, aggregating gradients/parameters from GPUs, and broadcasting parameters to GPUs. Every GPU only need to run the sub-graph with correct place information. - -Besides, it needs interfaces to synchronize model update with each other, and issue/merge model from different GPU Cards. +Besides, it needs interfaces to synchronize model update with each different GPU Cards. ## Implementation -As mentioned above, we summarise that several kinds of operators are needed. Currently, we need to issue parameters to different GPUs, named it with Broadcast operator. And also synchronize parameters between GPUs, called it with AllReduce. +As mentioned above, we wrap the NCCL routines as several kinds of operators. Need to note that NCCL need to create Communicator between gpu at the beginning, so there is a NCCLInit operator created. ### Graph Converter To be compatible with [parameter server design doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/ops/dist_train.md), the graph converter converts the user defined operation graph into sub-graphs to be executed on different devices. -1. The user-defined operator graph will be partitioned into sub-graph. +1. The user-defined model will be a single device program -2. Control operators between GPUs will be inserted into the graph. +2. Broadcast/Reduce operators between GPUs will be inserted into the program, even for the multi-node, may insert the `Send`, `Recv` operator. *Broadcast, AllReduce in a single machine. And Broadcast, AllReduce, [Send, Recv](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/ops/dist_train.md#graph-converter) in multiple machines* @@ -49,14 +47,14 @@ After convert, the graph as shows Operators are added to the sub-graphs. Every GPU assigned a role of `rank0`, `rank1` etc. - **Broadcast**. Broadcast operator distribute initialized parameter to all the GPUs from the GPU who owns it. e.g. from`rank0` GPU. -- **Allreduce**. Allreduce operator synchronizes parameters/gradients between GPUs. AllReduce implemented in the Ring-Based communicating method, avoid of the bottle neck in a single GPU. +- **AllReduce**. AllReduce operator synchronizes parameters/gradients between GPUs. AllReduce implemented in the Ring-Based communicating method, avoid of the bottle neck in a single GPU. -These two operators need the Multi-GPU context support. - -Need to notice that Allreduce operator force GPUs synchronized at that point. Every device only need runs sub-graph in a loop style forever, the whole training process in asynchronous or synchronous mode depends on the Allreduce point in the graph. +Need to notice that AllReduce operator force GPUs synchronized at that point. The whole training process in asynchronous or synchronous mode depends on the AllReduce point in the graph. As it shown in the picture, when each GPU compute the gradient of `W`, followed with a `AllReduce` operator, accumulate the `dW` to full batch of data, then run the optimize process individually and apply the gradient to its `W`. -In fact, in the way of every GPU optimized full batch of data, wasted (n-1) GPU compute resources. We will enhance it in the next stage. - -### Benefits +- **AllReduce2** +If we use the NCCL2 AllReduce primitive, every GPU optimized full batch of data, wasted (n-1) GPU compute resources. In addition, AllReduce will only utilize the communicate resource during synchronization, then update the gradient will be a seperated phase. In fact, we can amortize the update gradient time cost into the communicating phase. +- Every parameter has its root card. That card will call **Reduce** operator and collect the gradients from GPUs. +- The whole model's parameter will be hashed to different root card, ensure the load balance between GPUs. +Then we have another version AllReduce operator. Other part keep the same with before. From 0ca6274451d3693f363f2b8b5d6b29ce722febaf Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 11 Dec 2017 23:15:35 +0800 Subject: [PATCH 22/94] "add global regularization" (#6443) * "add global regularization" * Polish `append_regularization_ops` --- python/paddle/v2/fluid/optimizer.py | 38 +++++++++++---------------- python/paddle/v2/fluid/regularizer.py | 15 ++++++++--- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/python/paddle/v2/fluid/optimizer.py b/python/paddle/v2/fluid/optimizer.py index 719e3b2563..bbdfab2df9 100644 --- a/python/paddle/v2/fluid/optimizer.py +++ b/python/paddle/v2/fluid/optimizer.py @@ -18,8 +18,9 @@ class Optimizer(object): but need to use one of it's implementation. """ - def __init__(self, global_step=None): + def __init__(self, global_step=None, regularization=None): self._global_step = global_step + self.regularization = regularization # Dictionary of accumulators. Some optimizer subclasses need to # allocate and manage extra variables associated with the parameters # to train. These variables are called accumulators. @@ -199,7 +200,8 @@ class Optimizer(object): """ params_grads = append_backward_ops(loss, parameter_list, no_grad_set) # Add regularization if any - params_grads = append_regularization_ops(params_grads) + params_grads = append_regularization_ops(params_grads, + self.regularization) optimize_ops = self.create_optimization_pass(params_grads, loss, startup_program) return optimize_ops @@ -209,9 +211,9 @@ class SGDOptimizer(Optimizer): """ Simple SGD optimizer without any state. """ - def __init__(self, learning_rate, global_step=None): + def __init__(self, learning_rate, **kwargs): assert learning_rate is not None - super(SGDOptimizer, self).__init__(global_step) + super(SGDOptimizer, self).__init__(**kwargs) self.type = "sgd" self._learning_rate = learning_rate @@ -236,14 +238,10 @@ class MomentumOptimizer(Optimizer): """ _velocity_acc_str = "velocity" - def __init__(self, - learning_rate, - momentum, - use_nesterov=False, - global_step=None): + def __init__(self, learning_rate, momentum, use_nesterov=False, **kwargs): assert learning_rate is not None assert momentum is not None - super(MomentumOptimizer, self).__init__(global_step) + super(MomentumOptimizer, self).__init__(**kwargs) self.type = "momentum" self._learning_rate = learning_rate self._momentum = momentum @@ -284,10 +282,10 @@ class AdagradOptimizer(Optimizer): """ _moment_acc_str = "moment" - def __init__(self, learning_rate, epsilon=1.0e-6, global_step=None): + def __init__(self, learning_rate, epsilon=1.0e-6, **kwargs): assert learning_rate is not None assert epsilon is not None - super(AdagradOptimizer, self).__init__(global_step) + super(AdagradOptimizer, self).__init__(**kwargs) self.type = "adagrad" self._learning_rate = learning_rate self._epsilon = epsilon @@ -331,12 +329,12 @@ class AdamOptimizer(Optimizer): beta1=0.9, beta2=0.999, epsilon=1e-8, - global_step=None): + **kwargs): assert learning_rate is not None assert beta1 is not None assert beta2 is not None assert epsilon is not None - super(AdamOptimizer, self).__init__(global_step) + super(AdamOptimizer, self).__init__(**kwargs) self.type = "adam" self._learning_rate = learning_rate self._beta1 = beta1 @@ -436,12 +434,12 @@ class AdamaxOptimizer(Optimizer): beta1=0.9, beta2=0.999, epsilon=1e-8, - global_step=None): + **kwargs): assert learning_rate is not None assert beta1 is not None assert beta2 is not None assert epsilon is not None - super(AdamaxOptimizer, self).__init__() + super(AdamaxOptimizer, self).__init__(**kwargs) self.type = "adamax" self._learning_rate = learning_rate self._beta1 = beta1 @@ -514,16 +512,12 @@ class DecayedAdagradOptimizer(Optimizer): """ _moment_acc_str = "moment" - def __init__(self, - learning_rate, - decay=0.95, - epsilon=1.0e-6, - global_step=None): + def __init__(self, learning_rate, decay=0.95, epsilon=1.0e-6, **kwargs): assert learning_rate is not None assert decay is not None assert epsilon is not None - super(DecayedAdagradOptimizer, self).__init__(global_step) + super(DecayedAdagradOptimizer, self).__init__(**kwargs) self.type = "decayed_adagrad" self._learning_rate = learning_rate self._decay = decay diff --git a/python/paddle/v2/fluid/regularizer.py b/python/paddle/v2/fluid/regularizer.py index bb1ac8911e..d1955b0047 100644 --- a/python/paddle/v2/fluid/regularizer.py +++ b/python/paddle/v2/fluid/regularizer.py @@ -3,7 +3,7 @@ import framework __all__ = ['append_regularization_ops', 'L1Decay', 'L2Decay'] -def append_regularization_ops(parameters_and_grads): +def append_regularization_ops(parameters_and_grads, regularization=None): """Create and add backward regularization Operators Creates and adds backward regularization operators in the BlockDesc. @@ -14,6 +14,8 @@ def append_regularization_ops(parameters_and_grads): Args: parameters_and_grads: A list of (parameters, gradients) pairs that need to be regularized. + regularization: A global regularizer. If the parameter is not + set. It will be applied with regularizer. Returns: list of (parameters, gradients) pair with the regularized gradient @@ -23,14 +25,19 @@ def append_regularization_ops(parameters_and_grads): """ params_and_grads = [] for param, grad in parameters_and_grads: + regularization_term = None + if param.regularizer is not None: + # Add variable for regularization term in grad block + regularization_term = param.regularizer(param, grad.block) + elif regularization is not None: + regularization_term = regularization(param, grad.block) + # If no gradient or no regularization specified, # then we don't need to do anything - if grad is None or param.regularizer is None: + if grad is None or regularization_term is None: params_and_grads.append((param, grad)) continue - # Add variable for regularization term in grad block - regularization_term = param.regularizer(param, grad.block) assert grad.shape == regularization_term.shape grad.block.append_op( From 4ff6bc175a18d03f1159e78aef0c305703adbe37 Mon Sep 17 00:00:00 2001 From: Siddharth Goyal Date: Mon, 11 Dec 2017 11:33:54 -0800 Subject: [PATCH 23/94] Add row conv operator (#6013) * Fix documentation * Address review comments --- paddle/operators/row_conv_op.cc | 257 +++++++++++ paddle/operators/row_conv_op.cu | 408 ++++++++++++++++++ paddle/operators/row_conv_op.h | 33 ++ .../paddle/v2/fluid/tests/test_row_conv_op.py | 95 ++++ 4 files changed, 793 insertions(+) create mode 100644 paddle/operators/row_conv_op.cc create mode 100644 paddle/operators/row_conv_op.cu create mode 100644 paddle/operators/row_conv_op.h create mode 100644 python/paddle/v2/fluid/tests/test_row_conv_op.py diff --git a/paddle/operators/row_conv_op.cc b/paddle/operators/row_conv_op.cc new file mode 100644 index 0000000000..ea0bb99f8d --- /dev/null +++ b/paddle/operators/row_conv_op.cc @@ -0,0 +1,257 @@ +/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/row_conv_op.h" +#include "paddle/framework/eigen.h" + +namespace paddle { +namespace operators { + +using LoDTensor = framework::LoDTensor; +using framework::Tensor; + +template +using EigenMatrix = framework::EigenMatrix; + +class RowConvOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of RowConvOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Filter"), + "Input(Filter) of RowConvOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of RowConvOp should not be null."); + + auto x_dims = ctx->GetInputDim("X"); + auto filter_dims = ctx->GetInputDim("Filter"); + PADDLE_ENFORCE_EQ(x_dims.size(), 2, "Input(X)'s rank should be 2."); + PADDLE_ENFORCE_EQ(filter_dims.size(), 2, "Input(Y)'s rank should be 2."); + PADDLE_ENFORCE_EQ( + x_dims[1], filter_dims[1], + "The 2nd dimension of Input(X) and Input(Filter) should be same."); + ctx->SetOutputDim("Out", x_dims); + ctx->ShareLoD("X", "Out"); + } +}; + +class RowConvGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Filter"), + "Input(Filter) should not be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Gradient of output(Out) should not be null."); + + auto x_grad_name = framework::GradVarName("X"); + if (ctx->HasOutput(x_grad_name)) { + auto x_dims = ctx->GetInputDim("X"); + ctx->SetOutputDim(x_grad_name, x_dims); + } + + auto filter_grad_name = framework::GradVarName("Filter"); + if (ctx->HasOutput(filter_grad_name)) { + auto filter_dims = ctx->GetInputDim("Filter"); + ctx->SetOutputDim(filter_grad_name, filter_dims); + } + } +}; + +class RowConvOpMaker : public framework::OpProtoAndCheckerMaker { + public: + RowConvOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : framework::OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(LoDTensor), the input(X) is a LodTensor, which supports " + "variable time-length input sequences. The underlying tensor " + "in this LoDTensor is a matrix with shape (T x N), where T " + "is the total time steps in this mini-batch and N is the input " + "data dimension."); + AddInput("Filter", + "(Tensor), the input(Filter) is a learnable parameter. It " + "is a 2-D tensor with shape (future_context x N), where, " + "future_context is the future context length and N is the data " + "dimension."); + AddOutput("Out", + "(LoDTensor), the output(Out) is a LodTensor, which supports " + "variable time-length input sequences. The underlying tensor " + "in this LodTensor is a matrix with shape T x N, i.e., the " + "same shape as X."); + AddComment(R"DOC( +Row-convolution Operator. + +The row convolution is called lookahead convolution. This operator was +introduced in the following paper for DeepSpeech2: +http://www.cs.cmu.edu/~dyogatam/papers/wang+etal.iclrworkshop2016.pdf + +The main motivation is that a bidirectional RNN, useful in DeepSpeech +like speech models, learns representation for a sequence by performing a +forward and a backward pass through the entire sequence. However, unlike +unidirectional RNNs, bidirectional RNNs are challenging to deploy in an online +and low-latency setting. The lookahead convolution incorporates information +from future subsequences in a computationally efficient manner to improve +unidirectional recurrent neural networks. The row convolution operator is +different from the 1D sequence convolution, and is computed as follows: + +Given an input sequence $in$ of length $t$ and input dimension $d$, +and a filter ($W$) of size $context \times d$, +the output sequence is convolved as: + +$$ +out_{i, :} = \sum_{j=i}^{i + context} in_{j,:} \dot W_{i-j, :} +$$ + +)DOC"); + } +}; + +template +class RowConvKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &context) const override { + auto *x = context.Input("X"); + auto *filter = context.Input("Filter"); + auto *out = context.Output("Out"); + + out->mutable_data(context.GetPlace()); + + auto batch_indices = x->lod()[0]; + auto input_dim = x->dims()[1]; // 'in' is of size T x N + size_t num_sequence = batch_indices.size() - 1; + + auto future_context = filter->dims()[0]; + auto weights = EigenMatrix::From(*filter); + + for (size_t i = 0; i < num_sequence; i++) { + int start = static_cast(batch_indices[i]); + int end = static_cast(batch_indices[i + 1]); + int current_timesteps = end - start; + Tensor cur_input_sequence = + x->Slice(start, end); // Current input sequence + Tensor cur_output_sequence = + out->Slice(start, end); // Current output sequence + auto cip_seq = EigenMatrix::From(cur_input_sequence); + auto cot_seq = EigenMatrix::From(cur_output_sequence); + + for (int k = 0; k < current_timesteps; + k++) { // For different time steps in the same sequence + for (int w = 0; (w < future_context) && ((k + w) < current_timesteps); + w++) { + for (int d = 0; d < input_dim; d++) { + if (w == 0) { + cot_seq(k, d) = weights(w, d) * cip_seq(k + w, d); + } else { + cot_seq(k, d) += weights(w, d) * cip_seq(k + w, d); + } + } + } + } + } + } +}; + +template +class RowConvGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &context) const override { + auto *x = context.Input("X"); + auto *filter = context.Input("Filter"); + auto *d_out = context.Input(framework::GradVarName("Out")); + auto *dx = context.Output(framework::GradVarName("X")); + auto *d_filter = context.Output(framework::GradVarName("Filter")); + + auto input_dim = x->dims()[1]; // 'x' is of size T x N + auto batch_indices = x->lod()[0]; + size_t num_sequence = batch_indices.size() - 1; + auto future_context = filter->dims()[0]; + + if (d_filter) { + d_filter->mutable_data(context.GetPlace()); + auto dweights = + EigenMatrix::From(*d_filter); // Gradient of weight matrix + dweights.setZero(); + + for (size_t i = 0; i < num_sequence; i++) { // For different sequences + int start = static_cast(batch_indices[i]); + int end = static_cast(batch_indices[i + 1]); + + Tensor cur_input = x->Slice(start, end); // Current input sequence + Tensor cur_doutput = + d_out->Slice(start, end); // Current output grad sequence + + auto cur_ip = EigenMatrix::From(cur_input); + auto cur_dout = EigenMatrix::From(cur_doutput); + int current_timesteps = end - start; + + for (int k = 0; k < current_timesteps; + k++) { // For different time steps in the same sequence + for (int w = 0; (w < future_context) && ((k + w) < current_timesteps); + w++) { + // For dweights (Updating the gradient of weight matrix) + for (int d = 0; d < input_dim; d++) { + dweights(w, d) += cur_ip(k + w, d) * cur_dout(k, d); + } + } + } + } + } + + if (dx) { + dx->mutable_data(context.GetPlace()); + auto weights = EigenMatrix::From(*filter); + for (size_t i = 0; i < num_sequence; i++) { // For different sequences + int start = static_cast(batch_indices[i]); + int end = static_cast(batch_indices[i + 1]); + + Tensor cur_doutput = + d_out->Slice(start, end); // Current output grad sequence + Tensor cur_dinput = + dx->Slice(start, end); // Current input grad sequence + + auto cur_dout = EigenMatrix::From(cur_doutput); + auto cur_dip = EigenMatrix::From(cur_dinput); + cur_dip.setZero(); + int current_timesteps = end - start; + + for (int k = 0; k < current_timesteps; + k++) { // For different time steps in the same sequence + for (int w = 0; (w < future_context) && ((k + w) < current_timesteps); + w++) { + // For dinput (Updating the gradient wrt input) + for (int d = 0; d < input_dim; d++) { + cur_dip(k + w, d) += weights(w, d) * cur_dout(k, d); + } + } + } + } + } + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(row_conv, ops::RowConvOp, ops::RowConvOpMaker, row_conv_grad, + ops::RowConvGradOp); +REGISTER_OP_CPU_KERNEL(row_conv, + ops::RowConvKernel); +REGISTER_OP_CPU_KERNEL( + row_conv_grad, ops::RowConvGradKernel); diff --git a/paddle/operators/row_conv_op.cu b/paddle/operators/row_conv_op.cu new file mode 100644 index 0000000000..e0d7ebda7e --- /dev/null +++ b/paddle/operators/row_conv_op.cu @@ -0,0 +1,408 @@ +/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/math/math_function.h" +#include "paddle/operators/row_conv_op.h" +#include "paddle/platform/cuda_helper.h" + +namespace paddle { +namespace operators { + +using LoDTensor = framework::LoDTensor; +using framework::Tensor; + +namespace { + +inline int DivUp(int x, int y) { return (x + y - 1) / y; } + +// Forward prop (shared memory version, for small future_context) +template +__global__ void RowConvForwardSharedMemory(const T *in, const T *wt, + int num_sequence, int input_dim, + int future_context, + const size_t *batch_indices, + T *out) { + int blx = blockDim.x; + int bly = blockDim.y; + int thx = threadIdx.x; + int thy = threadIdx.y; + int d = blockIdx.x * blx + thx; // index along input dim + + extern __shared__ T mem[]; + T *sw = mem; + + if (thy < future_context) { + sw[thy * blx + thx] = + (d < input_dim) ? wt[thy * input_dim + d] : static_cast(0); + } + __syncthreads(); + + for (size_t i = 0; i < num_sequence; i++) { + int start = static_cast(batch_indices[i]); + int end = static_cast(batch_indices[i + 1]); + int current_timesteps = end - start; + for (int k = thy; k < current_timesteps; k += bly) { + T sum = 0; + for (int w = 0; (w < future_context) && ((k + w) < current_timesteps); + w++) { + sum += (d < input_dim) + ? sw[w * blx + thx] * in[(start + k + w) * input_dim + d] + : static_cast(0); + } + if (d < input_dim) { + out[(start + k) * input_dim + d] = sum; + } + } + } +} + +// Forward prop (naive version) +template +__global__ void RowConvForward(const T *in, const T *wt, int num_sequence, + int input_dim, int future_context, + const size_t *batch_indices, T *out) { + int d = blockIdx.x * blockDim.x + threadIdx.x; // index along input_dim + int bly = blockDim.y; + int thy = threadIdx.y; + + if (d >= input_dim) return; + + for (size_t i = 0; i < num_sequence; i++) { + int start = static_cast(batch_indices[i]); + int end = static_cast(batch_indices[i + 1]); + int current_timesteps = end - start; + for (int k = thy; k < current_timesteps; k += bly) { + T sum = 0; + for (int w = 0; (w < future_context) && ((k + w) < current_timesteps); + w++) { + sum += (wt[w * input_dim + d] * in[(start + k + w) * input_dim + d]); + } + out[(start + k) * input_dim + d] = sum; + } + } +} + +// Compute input gradient (shared memory version, for small future_context) +template +__global__ void RowConvGradInputSharedMemory(const T *dout, const T *wt, + int num_sequence, int input_dim, + int future_context, + const size_t *batch_indices, + T *din) { + int blx = blockDim.x; + int bly = blockDim.y; + int thx = threadIdx.x; + int thy = threadIdx.y; + int d = blockIdx.x * blx + thx; // index along input dim + + extern __shared__ T mem[]; + T *sw = mem; + if (thy < future_context) { + sw[thy * blx + thx] = + (d < input_dim) ? wt[thy * input_dim + d] : static_cast(0); + } + __syncthreads(); + + for (int i = 0; i < num_sequence; i++) { + int start = static_cast(batch_indices[i]); + int end = static_cast(batch_indices[i + 1]); + int current_timesteps = end - start; + for (int k = thy; k < current_timesteps; k += bly) { + T sum = 0; + for (int w = 0; (w < future_context) && ((k - w) >= 0); w++) { + sum += (d < input_dim) + ? (sw[w * blx + thx] * dout[(k + start - w) * input_dim + d]) + : static_cast(0); + } + if (d < input_dim) { + din[(k + start) * input_dim + d] = sum; + } + } + } +} + +// Compute input gradient (Naive version) +template +__global__ void RowConvGradInput(const T *dout, const T *wt, int num_sequence, + int input_dim, int future_context, + const size_t *batch_indices, T *din) { + int d = blockIdx.x * blockDim.x + threadIdx.x; // index along input_dim + int bly = blockDim.y; + int thy = threadIdx.y; + + if (d >= input_dim) return; + for (int i = 0; i < num_sequence; i++) { + int start = static_cast(batch_indices[i]); + int end = static_cast(batch_indices[i + 1]); + int current_timesteps = end - start; + for (int k = thy; k < current_timesteps; k += bly) { + T sum = 0; + for (int w = 0; (w < future_context) && ((k - w) >= 0); w++) { + sum += (wt[w * input_dim + d] * dout[(k + start - w) * input_dim + d]); + } + din[(k + start) * input_dim + d] = sum; + } + } +} + +// Compute W gradient (small future_context version) +template +__global__ void RowConvGradFilterImproved(const T *in, const T *dout, + int num_sequence, int input_dim, + int future_context, int block_x, + int block_y, + const size_t *batch_indices, + T *dfilter) { + int blx = blockDim.x; + int bly = blockDim.y; + int thx = threadIdx.x; + int thy = threadIdx.y; + int gx = blockIdx.x * blx; + int d = gx + thx; // index along input dim + + extern __shared__ T mem[]; + + int xdim_sh_in = block_y; + int xdim_sh_dout = block_y; + // int xdim_sh_dfilter = future_context; + int ydim_sh_in = block_x; + int ydim_sh_dout = block_x + future_context - 1; + int ydim_sh_dfilter = block_y; + + T *sh_in = mem; + T *sh_dout = &mem[xdim_sh_in * ydim_sh_in]; + T *sh_dfilter = &mem[xdim_sh_in * ydim_sh_in + xdim_sh_dout * ydim_sh_dout]; + + if (thy < future_context) { + sh_dfilter[thy * ydim_sh_dfilter + thx] = static_cast(0); + } + __syncthreads(); + + for (int i = 0; i < num_sequence; i++) { + int start = static_cast(batch_indices[i]); + int end = static_cast(batch_indices[i + 1]); + int current_timesteps = end - start; + int scaled_cur_steps = + ((current_timesteps + block_x - 1) / block_x) * block_x; + + for (int k = thy; k < scaled_cur_steps; k += block_x) { + int pos = start + k; + sh_in[thx * ydim_sh_in + thy] = + (d < input_dim && pos < end) ? in[pos * input_dim + d] : T(0); + sh_dout[thx * ydim_sh_dout + thy + future_context - 1] = + (d < input_dim && pos < end) ? dout[pos * input_dim + d] : T(0); + __syncthreads(); + + if (thy < future_context - 1) { + int pos_offset = pos - future_context + 1; + sh_dout[thx * ydim_sh_dout + thy] = + (d < input_dim && pos_offset >= start) + ? dout[pos_offset * input_dim + d] + : T(0); + } + __syncthreads(); + + for (int w = 0; w < future_context; w++) { + T val = sh_in[thy * ydim_sh_in + thx] * + sh_dout[thy * ydim_sh_dout + thx + future_context - 1 - w]; + __syncthreads(); + + for (int offset = 16; offset > 0; + offset = offset / 2) { // blockDim.x is 32. + val += __shfl_down(val, offset); + } + __syncthreads(); + + if (thx == 0) { + sh_dfilter[w * ydim_sh_dfilter + thy] += val; + } + __syncthreads(); + } + } + } + for (int w = thy; (w < future_context) && (d < input_dim); w += bly) { + dfilter[w * input_dim + d] += sh_dfilter[w * ydim_sh_dfilter + thx]; + } +} + +// Compute weight(filter) gradient +template +__global__ void RowConvGradFilter(const T *in, const T *dout, int num_sequence, + int input_dim, int future_context, + int block_x, int block_y, + const size_t *batch_indices, T *dfilter) { + int blx = blockDim.x; + int bly = blockDim.y; + int thx = threadIdx.x; + int thy = threadIdx.y; + int gx = blockIdx.x * blx; + int d = gx + thx; // index along input dim + extern __shared__ T mem[]; + T *sh_in = mem; + T *sh_dout = &mem[block_x * block_y]; + + for (int i = 0; i < num_sequence; i++) { + int start = static_cast(batch_indices[i]); + int end = static_cast(batch_indices[i + 1]); + int current_timesteps = end - start; + int scaled_cur_steps = + ((current_timesteps + block_x - 1) / block_x) * block_x; + + for (int k = thy; k < scaled_cur_steps; k += block_x) { + int pos = start + k; + sh_in[thx * block_y + thy] = + (d < input_dim && pos < end) ? in[pos * input_dim + d] : 0.0; + __syncthreads(); + + for (int w = 0; w < future_context; w++) { + sh_dout[thx * block_y + thy] = + (d < input_dim && (k - w) >= 0 && (k - w) < current_timesteps) + ? dout[(pos - w) * input_dim + d] + : 0.0; + __syncthreads(); + + T val = sh_in[thy * block_y + thx] * sh_dout[thy * block_y + thx]; + __syncthreads(); + + for (int offset = 16; offset > 0; + offset = offset / 2) { // blockDim.x is 32. + val += __shfl_down(val, offset); + } + __syncthreads(); + + if (thx == 0 && (gx + thy) < input_dim) { + dfilter[w * input_dim + gx + thy] += val; + } + } + } + } +} + +} // namespace + +template +class RowConvKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &context) const override { + auto *X = context.Input("X"); + auto *Filter = context.Input("Filter"); + auto *Out = context.Output("Out"); + + const T *in = X->data(); + const T *weight = Filter->data(); + T *out = Out->mutable_data(context.GetPlace()); + + auto batch_indices = X->lod()[0]; + int input_dim = X->dims()[1]; + int num_sequence = batch_indices.size() - 1; + int future_context = Filter->dims()[0]; + size_t *idx = batch_indices.data(); + auto stream = context.cuda_device_context().stream(); + + if (future_context <= 32) { + dim3 block_dim = dim3(32, 32); + dim3 grid_dim = dim3(DivUp(input_dim, block_dim.x), 1); + int mem_per_block = (future_context * block_dim.x) * sizeof(T); + RowConvForwardSharedMemory< + T><<>>( + in, weight, num_sequence, input_dim, future_context, idx, out); + } else { + dim3 block_dim = dim3(32, 32); + dim3 grid_dim = dim3(DivUp(input_dim, block_dim.x), 1); + RowConvForward<<>>( + in, weight, num_sequence, input_dim, future_context, idx, out); + } + } +}; + +template +class RowConvGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &context) const override { + auto *X = context.Input("X"); + auto *Filter = context.Input("Filter"); + auto *dOut = context.Input(framework::GradVarName("Out")); + const T *in = X->data(); + const T *weights = Filter->data(); + const T *dout = dOut->data(); + + Tensor *dX = context.Output(framework::GradVarName("X")); + Tensor *dFilter = context.Output(framework::GradVarName("Filter")); + + auto batch_indices = X->lod()[0]; + int input_dim = X->dims()[1]; + int num_sequence = batch_indices.size() - 1; + int future_context = Filter->dims()[0]; + size_t *idx = batch_indices.data(); + + auto &device_ctx = context.cuda_device_context(); + math::SetConstant zero; + + if (dFilter) { + T *dfilter = dFilter->mutable_data(context.GetPlace()); + zero(device_ctx, dFilter, static_cast(0.0)); + + if (future_context <= 32) { + dim3 block_dim = dim3(32, 32); + dim3 grid_dim = dim3(DivUp(input_dim, block_dim.x), 1); + int block_x = block_dim.x; + int block_y = block_dim.y; + int mem_per_block = + (block_y * block_x + block_y * (block_x + future_context - 1) + + future_context * block_y) * + sizeof(T); + RowConvGradFilterImproved< + T><<>>( + in, dout, num_sequence, input_dim, future_context, block_x, block_y, + idx, dfilter); + } else { + dim3 block_dim = dim3(32, 32); + dim3 grid_dim = dim3(DivUp(input_dim, block_dim.x), 1); + int block_x = block_dim.x; + int block_y = block_dim.y; + int mem_per_block = + (block_x * block_y * 2) * sizeof(T); // For 2 arrays of size 32x32 + RowConvGradFilter< + T><<>>( + in, dout, num_sequence, input_dim, future_context, block_x, block_y, + idx, dfilter); + } + } + + if (dX) { + T *din = dX->mutable_data(context.GetPlace()); + if (future_context <= 32) { + dim3 block_dim = dim3(32, 32); + dim3 grid_dim = dim3(DivUp(input_dim, block_dim.x), 1); + int mem_per_block = (future_context * block_dim.x) * sizeof(T); + RowConvGradInputSharedMemory< + T><<>>( + dout, weights, num_sequence, input_dim, future_context, idx, din); + } else { + dim3 block_dim = dim3(32, 32); + dim3 grid_dim = dim3(DivUp(input_dim, block_dim.x), 1); + RowConvGradInput<<>>( + dout, weights, num_sequence, input_dim, future_context, idx, din); + } + } + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(row_conv, + ops::RowConvKernel); +REGISTER_OP_GPU_KERNEL( + row_conv_grad, ops::RowConvGradKernel); diff --git a/paddle/operators/row_conv_op.h b/paddle/operators/row_conv_op.h new file mode 100644 index 0000000000..525e83908d --- /dev/null +++ b/paddle/operators/row_conv_op.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +class RowConvKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &context) const override; +}; + +template +class RowConvGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &context) const override; +}; +} // namespace operators +} // namespace paddle diff --git a/python/paddle/v2/fluid/tests/test_row_conv_op.py b/python/paddle/v2/fluid/tests/test_row_conv_op.py new file mode 100644 index 0000000000..1ed86e23ac --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_row_conv_op.py @@ -0,0 +1,95 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def row_conv_forward(x, lod, wt): + out = np.zeros_like(x) + seq_info = lod[0] + num_sequences = len(seq_info) - 1 + context_length = wt.shape[0] + + for i in range(num_sequences): # loop over number of sequences + start = seq_info[i] + end = seq_info[i + 1] + curinput = x[start:end, :] + curoutput = out[start:end, :] + + cur_timesteps = end - start + for j in range(cur_timesteps): # loop over different timesteps + for k in range(context_length): + + if j + k >= cur_timesteps: + continue + curoutput[j, :] += curinput[j + k, :] * wt[k, :] + + return out + + +class TestRowConvOp1(OpTest): + def setUp(self): + + self.op_type = "row_conv" + lod = [[0, 2, 5, 7]] + T = lod[0][-1] + D = 16 + context_length = 2 + + x = np.random.random((T, D)).astype("float32") + wt = np.random.random((context_length, D)).astype("float32") + self.inputs = {'X': (x, lod), 'Filter': wt} + + out = row_conv_forward(x, lod, wt) + self.outputs = {'Out': (out, lod)} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Filter'], 'Out', max_relative_error=0.05) + + def test_check_grad_ignore_x(self): + self.check_grad( + ['Filter'], 'Out', max_relative_error=0.05, no_grad_set=set('X')) + + def test_check_grad_ignore_wt(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.05, no_grad_set=set('Filter')) + + +class TestRowConvOp2(OpTest): + def setUp(self): + + self.op_type = "row_conv" + lod = [[0, 20, 50, 100]] + T = lod[0][-1] + D = 35 + context_length = 35 + + x = np.random.random((T, D)).astype("float32") + wt = np.random.random((context_length, D)).astype("float32") + self.inputs = {'X': (x, lod), 'Filter': wt} + + out = row_conv_forward(x, lod, wt) + self.outputs = {'Out': (out, lod)} + + def test_check_output(self): + self.check_output() + + #max_relative_error is increased from 0.05 to 0.06 as for higher + #dimensional input, the dX on CPU for some values has max_rel_error + #slightly more than 0.05 + def test_check_grad_normal(self): + self.check_grad(['X', 'Filter'], 'Out', max_relative_error=0.06) + + def test_check_grad_ignore_x(self): + self.check_grad( + ['Filter'], 'Out', max_relative_error=0.06, no_grad_set=set('X')) + + def test_check_grad_ignore_wt(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.06, no_grad_set=set('Filter')) + + +if __name__ == '__main__': + unittest.main() From 35420cdf63dd1369972c26f70cac2d4d75b1492a Mon Sep 17 00:00:00 2001 From: kavyasrinet Date: Mon, 11 Dec 2017 15:33:28 -0800 Subject: [PATCH 24/94] Updating the Latex equation for Adagrad (#6009) * Updating the Latex equation for Adagrad * Fixing Latex euqations for adadelta, adam and adamax --- paddle/operators/adadelta_op.cc | 12 ++++++------ paddle/operators/adagrad_op.cc | 4 ++-- paddle/operators/adam_op.cc | 12 +++++++----- paddle/operators/adamax_op.cc | 8 ++++---- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/paddle/operators/adadelta_op.cc b/paddle/operators/adadelta_op.cc index 16a7794d5b..29434a0ee2 100644 --- a/paddle/operators/adadelta_op.cc +++ b/paddle/operators/adadelta_op.cc @@ -92,12 +92,12 @@ for gradient descent. Adadelta updates are as follows: -$$avgSquaredGradOut = \rho * avgSquaredGrad + (1 - \rho) * grad * grad \break -paramUpdate = - $\sqrt{((avgSquaredUpdate + \epsilon) / - (avgSquaredGrad_out + \epsilon))}$ * grad \break -avgSquaredUpdateOut = \rho * avgSquaredUpdate + (1 - \rho) * - {(paramUpdate)}^2 \break -paramOut = param + paramUpdate$$ +$$ +avg\_squared\_grad\_out = \rho * avg\_squared\_grad + (1 - \rho) * grad * grad \\ +param\_update = - \sqrt{\frac{avg\_squared\_update + \epsilon}{avg\_squared\_grad\_out + \epsilon}} * grad \\ +avg\_squared\_update\_out = \rho * avg\_squared\_update + (1 - \rho) * {param\_update}^2 \\ +param\_out = param + param\_update +$$ )DOC"); } diff --git a/paddle/operators/adagrad_op.cc b/paddle/operators/adagrad_op.cc index d6686e3ef3..d19602244b 100644 --- a/paddle/operators/adagrad_op.cc +++ b/paddle/operators/adagrad_op.cc @@ -80,8 +80,8 @@ Adaptive Gradient Algorithm (Adagrad). The update is done as follows: -$$momentOut = moment + grad * grad \break -paramOut = param - learningRate * grad / ($\sqrt{momentOut}$ + \epsilon) \break +$$moment\_out = moment + grad * grad \\ +param\_out = param - \frac{learning\_rate * grad}{\sqrt{moment\_out} + \epsilon} $$ The original paper(http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) diff --git a/paddle/operators/adam_op.cc b/paddle/operators/adam_op.cc index 03faa2a7c5..a268d05484 100644 --- a/paddle/operators/adam_op.cc +++ b/paddle/operators/adam_op.cc @@ -112,11 +112,13 @@ adaptive estimates of lower-order moments. Adam updates: -$$moment_1_{out} = \beta_1 * moment_1 + (1 - \beta_1) * grad \break -moment_2_{out} = \beta_2 * moment_2 + (1 - \beta_2) * grad * grad \break -learningRate = learningRate * - $\sqrt{(1 - \beta_2_{pow})}$ / (1 - \beta_1_{pow}) \break -paramOut = param - learningRate * moment_1/ ($\sqrt{(moment_2)} + \epsilon)$$ +$$ +moment\_1\_out = \beta_1 * moment\_1 + (1 - \beta_1) * grad \\ +moment\_2_\out = \beta_2 * moment\_2 + (1 - \beta_2) * grad * grad \\ +learning\_rate = learning\_rate * + \frac{\sqrt{1 - \beta_{2\_pow}}}{1 - \beta_{1\_pow}} \\ +param\_out = param - learning\_rate * \frac{moment\_1}{\sqrt{moment\_2} + \epsilon} +$$ )DOC"); } diff --git a/paddle/operators/adamax_op.cc b/paddle/operators/adamax_op.cc index 867ddd9790..9e7576c961 100644 --- a/paddle/operators/adamax_op.cc +++ b/paddle/operators/adamax_op.cc @@ -108,10 +108,10 @@ Adam algorithm based on the infinity norm. Adamax updates: $$ - momentOut = \beta_{1} * moment + (1 - \beta_{1}) * grad \\ - infNormOut = max(\beta_{2} * infNorm + \epsilon, |grad|) \\ - learningRate = \frac{learningRate}{1 - \beta_{1}^{Beta1Pow}} \\ - paramOut = param - learningRate * \frac{momentOut}{infNormOut} +moment\_out = \beta_1 * moment + (1 - \beta_1) * grad \\ +inf\_norm\_out = max(\beta_2 * inf\_norm + \epsilon, |grad|) \\ +learning\_rate = \frac{learning\_rate}{1 - \beta_{1\_pow}} \\ +param\_out = param - learning\_rate * \frac{moment\_out}{inf\_norm\_out} $$ The original paper does not have an epsilon attribute. From f4f17e539b70a6cdb4245267ae4027cd5202b0fa Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Tue, 12 Dec 2017 16:35:13 +0800 Subject: [PATCH 25/94] skip mkl setting in v1 with Mac --- paddle/scripts/submit_local.sh.in | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/paddle/scripts/submit_local.sh.in b/paddle/scripts/submit_local.sh.in index d71cb84df3..43d2d1b410 100755 --- a/paddle/scripts/submit_local.sh.in +++ b/paddle/scripts/submit_local.sh.in @@ -140,7 +140,11 @@ else: sys.exit(0) EOF -cpu_config +if [ "`uname -s`" == "Linux" ]; then + # only support on linux yet, with mac can use v2 + cpu_config +fi + # echo $KMP_AFFINITY $OMP_DYNAMIC case "$1" in From c175eeb387ee90d8ba8a549a2e29a1680c9f1f35 Mon Sep 17 00:00:00 2001 From: wangmeng28 Date: Tue, 12 Dec 2017 11:08:23 +0800 Subject: [PATCH 26/94] Fix wrong index in dataset downloading exception --- python/paddle/v2/dataset/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/dataset/common.py b/python/paddle/v2/dataset/common.py index e31e501ce9..191d9ecfb1 100644 --- a/python/paddle/v2/dataset/common.py +++ b/python/paddle/v2/dataset/common.py @@ -71,7 +71,7 @@ def download(url, module_name, md5sum): if retry < retry_limit: retry += 1 else: - raise RuntimeError("Cannot download {0} within retry limit {2}". + raise RuntimeError("Cannot download {0} within retry limit {1}". format(url, retry_limit)) print "Cache file %s not found, downloading %s" % (filename, url) r = requests.get(url, stream=True) From f3acdd3af94cb3018134b13893cffebdf09c827c Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Tue, 12 Dec 2017 11:17:33 +0800 Subject: [PATCH 27/94] fix warning in row_conv_op.cu --- paddle/operators/row_conv_op.cu | 1 - 1 file changed, 1 deletion(-) diff --git a/paddle/operators/row_conv_op.cu b/paddle/operators/row_conv_op.cu index e0d7ebda7e..79b7086b24 100644 --- a/paddle/operators/row_conv_op.cu +++ b/paddle/operators/row_conv_op.cu @@ -243,7 +243,6 @@ __global__ void RowConvGradFilter(const T *in, const T *dout, int num_sequence, int block_x, int block_y, const size_t *batch_indices, T *dfilter) { int blx = blockDim.x; - int bly = blockDim.y; int thx = threadIdx.x; int thy = threadIdx.y; int gx = blockIdx.x * blx; From 59c74fb14adcd23fc244275ef9e74b8430cc43fc Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Tue, 12 Dec 2017 11:35:38 +0800 Subject: [PATCH 28/94] change paddle:dev to paddle:latest-dev --- doc/howto/dev/contribute_to_paddle_cn.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/howto/dev/contribute_to_paddle_cn.md b/doc/howto/dev/contribute_to_paddle_cn.md index 3eb477eb65..3e0bf7b397 100644 --- a/doc/howto/dev/contribute_to_paddle_cn.md +++ b/doc/howto/dev/contribute_to_paddle_cn.md @@ -76,18 +76,18 @@ no changes added to commit (use "git add" and/or "git commit -a") ## 构建和测试 -编译 PaddlePaddle 的源码以及生成文档需要多种开发工具。为了方便大家,我们的标准开发流程是把这些工具都装进一个Docker image,称为*开发镜像*,通常名字是 `paddle:dev`。然后所有用 `cmake && make` 的地方(比如IDE配置里)都用 `docker run paddle:dev`来代替。 +编译 PaddlePaddle 的源码以及生成文档需要多种开发工具。为了方便大家,我们的标准开发流程是把这些工具都装进一个Docker image,称为*开发镜像*,通常名字是 `paddle:latest-dev` 或者 `paddle:[version tag]-dev` 如 `paddle:0.11.0-dev`。然后所有用 `cmake && make` 的地方(比如IDE配置里)都用 `docker run paddle:latest-dev`来代替。 如要build这个开发镜像,在源码目录树的根目录中运行: ```bash -➜ docker build -t paddle:dev . +➜ docker build -t paddle:latest-dev . ``` 随后可以用这个开发镜像开始build PaddlePaddle的源码。比如如果要build一个不依赖GPU,但是支持AVX指令集,并且包括unit tests的PaddlePaddle,可以: ```bash -➜ docker run -v $(pwd):/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TESTING=ON" paddle:dev +➜ docker run -v $(pwd):/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TESTING=ON" paddle:latest-dev ``` 这个过程除了编译PaddlePaddle为 `./build/libpaddle.so`,并且输出一个 `./build/paddle.deb`文件之外,还会输出一个 `build/Dockerfile`。我们只需要运行下面命令把编译好的PaddlePaddle打包成一个*生产镜像*(`paddle:prod`): @@ -99,7 +99,7 @@ no changes added to commit (use "git add" and/or "git commit -a") 如果要运行所有的单元测试,可以用如下命令: ```bash -➜ docker run -it -v $(pwd):/paddle paddle:dev bash -c "cd /paddle/build && ctest" +➜ docker run -it -v $(pwd):/paddle paddle:latest-dev bash -c "cd /paddle/build && ctest" ``` 关于构建和测试的更多信息,请参见[这篇文档](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。 From 936f0546e3aaa772514bba721167e897769ec2a9 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Tue, 12 Dec 2017 10:24:50 +0800 Subject: [PATCH 29/94] fix img_pool maxout doc --- .../paddle/trainer_config_helpers/layers.py | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index d0b14cf63c..3a82e858f7 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -2729,15 +2729,17 @@ def img_pool_layer(input, .. math:: - w = 1 + \frac{ceil(input\_width + 2 * padding - pool\_size)}{stride} \\\\ - h = 1 + \frac{ceil(input\_height + 2 * padding\_y - pool\_size\_y)}{stride\_y} + w & = 1 + \\frac{ceil(input\_width + 2 * padding - pool\_size)}{stride} + + h & = 1 + \\frac{ceil(input\_height + 2 * padding\_y - pool\_size\_y)}{stride\_y} - ceil_mode=False: .. math:: - w = 1 + \frac{floor(input\_width + 2 * padding - pool\_size)}{stride} \\\\ - h = 1 + \frac{floor(input\_height + 2 * padding\_y - pool\_size\_y)}{stride\_y} + w & = 1 + \\frac{floor(input\_width + 2 * padding - pool\_size)}{stride} + + h & = 1 + \\frac{floor(input\_height + 2 * padding\_y - pool\_size\_y)}{stride\_y} The example usage is: @@ -2870,17 +2872,21 @@ def img_pool3d_layer(input, .. math:: - w = 1 + \frac{ceil(input\_width + 2 * padding - pool\_size)}{stride} \\\\ - h = 1 + \frac{ceil(input\_height + 2 * padding\_y - pool\_size\_y)}{stride\_y} \\\\ - d = 1 + \frac{ceil(input\_depth + 2 * padding\_z - pool\_size\_z)}{stride\_z} + w & = 1 + \\frac{ceil(input\_width + 2 * padding - pool\_size)}{stride} + + h & = 1 + \\frac{ceil(input\_height + 2 * padding\_y - pool\_size\_y)}{stride\_y} + + d & = 1 + \\frac{ceil(input\_depth + 2 * padding\_z - pool\_size\_z)}{stride\_z} - ceil_mode=False: .. math:: - w = 1 + \frac{floor(input\_width + 2 * padding - pool\_size)}{stride} \\\\ - h = 1 + \frac{floor(input\_height + 2 * padding\_y - pool\_size\_y)}{stride\_y} \\\\ - d = 1 + \frac{floor(input\_depth + 2 * padding\_z - pool\_size\_z)}{stride\_z} \\\\ + w & = 1 + \\frac{floor(input\_width + 2 * padding - pool\_size)}{stride} + + h & = 1 + \\frac{floor(input\_height + 2 * padding\_y - pool\_size\_y)}{stride\_y} + + d & = 1 + \\frac{floor(input\_depth + 2 * padding\_z - pool\_size\_z)}{stride\_z} The example usage is: @@ -5437,13 +5443,21 @@ def maxout_layer(input, groups, num_channels=None, name=None, layer_attr=None): .. math:: - out = \max_k (in[n, k, o_c , s]) \\\\ - out_{i * s + j} = \max_k in_{ k * o_{c} * s + i * s + j} \\\\ - s = \frac{input.size}{ num\_channels} \\\\ - o_{c} =\frac{num\_channels}{groups} \\\\ - 0 \le i < o_{c} \\\\ - 0 \le j < s \\\\ - 0 \le k < groups \\\\ + + out & = \max_k (in[n, k, o_c , s]) + + out_{i * s + j} & = \max_k in_{ k * o_{c} * s + i * s + j} + + s & = \\frac{input.size}{ num\_channels} + + o_{c} & = \\frac{num\_channels}{groups} + + 0 \le i & < o_{c} + + 0 \le j & < s + + 0 \le k & < groups + The simple usage is: From 7d8802938149df6b0dcf8934c0eb7fe0e4e5145c Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Tue, 12 Dec 2017 12:55:58 +0800 Subject: [PATCH 30/94] equation align --- python/paddle/trainer_config_helpers/layers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 3a82e858f7..7e118b24a4 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -5444,19 +5444,19 @@ def maxout_layer(input, groups, num_channels=None, name=None, layer_attr=None): .. math:: - out & = \max_k (in[n, k, o_c , s]) + & out = \max_k (in[n, k, o_c , s]) - out_{i * s + j} & = \max_k in_{ k * o_{c} * s + i * s + j} + & out_{i * s + j} = \max_k in_{ k * o_{c} * s + i * s + j} - s & = \\frac{input.size}{ num\_channels} + & s = \\frac{input.size}{ num\_channels} - o_{c} & = \\frac{num\_channels}{groups} + & o_{c} = \\frac{num\_channels}{groups} - 0 \le i & < o_{c} + & 0 \le i < o_{c} - 0 \le j & < s + & 0 \le j < s - 0 \le k & < groups + & 0 \le k < groups The simple usage is: From 61ec0b951656fb402e61c4dc519e80ba3fbc61d0 Mon Sep 17 00:00:00 2001 From: QI JUN Date: Tue, 12 Dec 2017 14:00:28 +0800 Subject: [PATCH 31/94] Refine device context (#6433) There are mainly following fixes: - take `DeviceContext` as the template parameter of math functors and OpKernel instead of `Place` - remove `eigen_device` interface in base class `DeviceContext` - remove `GetEigenDevice` interface in `ExecutionContext` and base class `DeviceContext` - remove unused `platform::EigenDeviceConverter` - rename `REGISTER_OP_GPU_KERNEL` to `REGISTER_OP_CUDA_KERNEL` - rename `USE_GPU_ONLY_OP` to `USE_CUDA_ONLY_OP` --- paddle/framework/op_registry.h | 12 +- paddle/framework/operator.cc | 16 +- paddle/framework/operator.h | 26 +-- paddle/framework/operator_test.cc | 2 +- paddle/operators/CMakeLists.txt | 2 +- paddle/operators/accuracy_op.cc | 2 +- paddle/operators/accuracy_op.cu | 5 +- paddle/operators/accuracy_op.h | 2 +- paddle/operators/activation_op.cc | 21 +- paddle/operators/activation_op.cu | 23 +- paddle/operators/activation_op.h | 14 +- paddle/operators/adadelta_op.cc | 4 +- paddle/operators/adadelta_op.cu | 6 +- paddle/operators/adadelta_op.h | 4 +- paddle/operators/adagrad_op.cc | 18 +- paddle/operators/adagrad_op.cu | 20 +- paddle/operators/adagrad_op.h | 17 +- paddle/operators/adam_op.cc | 6 +- paddle/operators/adam_op.cu | 6 +- paddle/operators/adam_op.h | 10 +- paddle/operators/adamax_op.cc | 6 +- paddle/operators/adamax_op.cu | 6 +- paddle/operators/adamax_op.h | 10 +- paddle/operators/auc_op.h | 2 +- paddle/operators/batch_norm_op.cc | 14 +- paddle/operators/batch_norm_op.cu.cc | 32 +-- paddle/operators/batch_norm_op.h | 4 +- .../operators/bilinear_tensor_product_op.cc | 11 +- .../operators/bilinear_tensor_product_op.cu | 16 +- paddle/operators/bilinear_tensor_product_op.h | 41 ++-- paddle/operators/cast_op.cc | 2 +- paddle/operators/cast_op.cu | 6 +- paddle/operators/cast_op.h | 13 +- paddle/operators/chunk_eval_op.h | 2 +- paddle/operators/clip_by_norm_op.cc | 3 +- paddle/operators/clip_by_norm_op.cu | 5 +- paddle/operators/clip_by_norm_op.h | 5 +- paddle/operators/clip_op.cc | 8 +- paddle/operators/clip_op.cu | 8 +- paddle/operators/clip_op.h | 16 +- paddle/operators/compare_op.cu | 10 +- paddle/operators/compare_op.h | 31 ++- paddle/operators/concat_op.cu.cc | 9 +- paddle/operators/concat_op.h | 4 +- paddle/operators/conv_cudnn_op.cc | 22 +- paddle/operators/conv_cudnn_op.cu.cc | 32 +-- paddle/operators/conv_op.cc | 22 +- paddle/operators/conv_op.cu.cc | 26 ++- paddle/operators/conv_op.h | 55 +++-- paddle/operators/conv_shift_op.cu | 22 +- paddle/operators/conv_shift_op.h | 4 +- paddle/operators/conv_transpose_cudnn_op.cc | 18 +- .../operators/conv_transpose_cudnn_op.cu.cc | 32 +-- paddle/operators/conv_transpose_op.cc | 18 +- paddle/operators/conv_transpose_op.cu.cc | 28 ++- paddle/operators/conv_transpose_op.h | 58 ++--- paddle/operators/cos_sim_op.cc | 7 +- paddle/operators/cos_sim_op.cu | 9 +- paddle/operators/cos_sim_op.h | 10 +- paddle/operators/crf_decoding_op.cc | 5 +- paddle/operators/crf_decoding_op.h | 6 +- paddle/operators/crop_op.cc | 4 +- paddle/operators/crop_op.cu | 6 +- paddle/operators/crop_op.h | 19 +- paddle/operators/cross_entropy_op.cu | 23 +- paddle/operators/cross_entropy_op.h | 12 +- paddle/operators/decayed_adagrad_op.cc | 2 +- paddle/operators/decayed_adagrad_op.cu | 4 +- paddle/operators/decayed_adagrad_op.h | 4 +- paddle/operators/dropout_op.cc | 6 +- paddle/operators/dropout_op.cu | 12 +- paddle/operators/dropout_op.h | 10 +- paddle/operators/elementwise_add_op.cc | 16 +- paddle/operators/elementwise_add_op.cu | 21 +- paddle/operators/elementwise_add_op.h | 10 +- paddle/operators/elementwise_div_op.cc | 16 +- paddle/operators/elementwise_div_op.cu | 21 +- paddle/operators/elementwise_div_op.h | 8 +- paddle/operators/elementwise_mul_op.cc | 16 +- paddle/operators/elementwise_mul_op.cu | 21 +- paddle/operators/elementwise_mul_op.h | 8 +- paddle/operators/elementwise_op_function.h | 94 ++++---- paddle/operators/elementwise_sub_op.cc | 16 +- paddle/operators/elementwise_sub_op.cu | 21 +- paddle/operators/elementwise_sub_op.h | 8 +- paddle/operators/expand_op.cc | 7 +- paddle/operators/expand_op.cu | 9 +- paddle/operators/expand_op.h | 10 +- .../fill_constant_batch_size_like_op.cc | 11 +- .../fill_constant_batch_size_like_op.cu.cc | 13 +- .../fill_constant_batch_size_like_op.h | 7 +- paddle/operators/fill_zeros_like_op.cc | 11 +- paddle/operators/fill_zeros_like_op.cu.cc | 13 +- paddle/operators/fill_zeros_like_op.h | 7 +- paddle/operators/ftrl_op.cc | 4 +- paddle/operators/ftrl_op.cu | 4 +- paddle/operators/ftrl_op.h | 4 +- paddle/operators/gather.cu.h | 2 +- paddle/operators/gather_op.cu | 7 +- paddle/operators/gather_op.h | 3 +- paddle/operators/gaussian_random_op.cu | 4 +- paddle/operators/gru_op.cc | 11 +- paddle/operators/gru_op.cu.cc | 11 +- paddle/operators/gru_op.h | 48 ++-- paddle/operators/gru_unit_op.cc | 11 +- paddle/operators/gru_unit_op.cu | 13 +- paddle/operators/gru_unit_op.h | 64 +++--- paddle/operators/hinge_loss_op.cc | 7 +- paddle/operators/hinge_loss_op.cu | 9 +- paddle/operators/hinge_loss_op.h | 10 +- paddle/operators/huber_loss_op.cc | 7 +- paddle/operators/huber_loss_op.cu | 9 +- paddle/operators/huber_loss_op.h | 10 +- paddle/operators/l1_norm_op.cc | 7 +- paddle/operators/l1_norm_op.cu | 9 +- paddle/operators/l1_norm_op.h | 10 +- paddle/operators/linear_chain_crf_op.cc | 9 +- paddle/operators/linear_chain_crf_op.cu | 13 +- paddle/operators/linear_chain_crf_op.h | 24 +- paddle/operators/lod_reset_op.cu | 13 +- paddle/operators/lod_reset_op.h | 4 +- paddle/operators/log_loss_op.cc | 7 +- paddle/operators/log_loss_op.cu | 9 +- paddle/operators/log_loss_op.h | 8 +- paddle/operators/logical_op.cu | 8 +- paddle/operators/logical_op.h | 21 +- paddle/operators/lookup_table_op.cu | 20 +- paddle/operators/lrn_op.cc | 21 +- paddle/operators/lrn_op.cu | 30 +-- paddle/operators/lrn_op.h | 10 +- paddle/operators/lstm_op.cc | 11 +- paddle/operators/lstm_op.cu.cc | 11 +- paddle/operators/lstm_op.h | 94 ++++---- paddle/operators/lstm_unit_op.cu | 8 +- paddle/operators/lstm_unit_op.h | 4 +- paddle/operators/margin_rank_loss_op.cc | 4 +- paddle/operators/margin_rank_loss_op.cu | 8 +- paddle/operators/margin_rank_loss_op.h | 8 +- paddle/operators/math/context_project.cc | 4 +- paddle/operators/math/context_project.cu | 4 +- paddle/operators/math/context_project.h | 20 +- paddle/operators/math/cross_entropy.cc | 10 +- paddle/operators/math/cross_entropy.cu | 14 +- paddle/operators/math/cross_entropy.h | 6 +- paddle/operators/math/gru_compute.cc | 28 +-- paddle/operators/math/gru_compute.cu | 34 ++- paddle/operators/math/gru_compute.h | 13 +- paddle/operators/math/im2col.cc | 32 +-- paddle/operators/math/im2col.cu | 48 ++-- paddle/operators/math/im2col.h | 11 +- paddle/operators/math/im2col_test.cc | 27 +-- paddle/operators/math/lstm_compute.cc | 16 +- paddle/operators/math/lstm_compute.cu | 16 +- paddle/operators/math/lstm_compute.h | 13 +- paddle/operators/math/math_function.cc | 144 ++++++------ paddle/operators/math/math_function.cu | 207 ++++++++---------- paddle/operators/math/math_function.h | 81 ++++--- paddle/operators/math/math_function_impl.h | 39 ++-- paddle/operators/math/math_function_test.cc | 9 +- paddle/operators/math/math_function_test.cu | 10 +- paddle/operators/math/matmul.h | 19 +- paddle/operators/math/maxouting.cc | 16 +- paddle/operators/math/maxouting.cu | 33 ++- paddle/operators/math/maxouting.h | 14 +- paddle/operators/math/pooling.cc | 128 ++++++----- paddle/operators/math/pooling.cu | 182 +++++++-------- paddle/operators/math/pooling.h | 68 +++--- .../operators/math/selected_rows_functor.cc | 45 ++-- .../operators/math/selected_rows_functor.cu | 77 +++---- paddle/operators/math/selected_rows_functor.h | 16 +- .../math/selected_rows_functor_test.cc | 12 +- .../math/selected_rows_functor_test.cu | 12 +- paddle/operators/math/sequence2batch.cc | 16 +- paddle/operators/math/sequence2batch.cu | 19 +- paddle/operators/math/sequence2batch.h | 22 +- paddle/operators/math/sequence_pooling.cc | 18 +- paddle/operators/math/sequence_pooling.cu | 24 +- paddle/operators/math/sequence_pooling.h | 8 +- paddle/operators/math/softmax.cc | 8 +- paddle/operators/math/softmax.cu | 8 +- paddle/operators/math/softmax.h | 13 +- paddle/operators/math/softmax_impl.h | 32 ++- paddle/operators/math/unpooling.cc | 16 +- paddle/operators/math/unpooling.cu | 36 ++- paddle/operators/math/unpooling.h | 10 +- paddle/operators/math/vol2col.cc | 16 +- paddle/operators/math/vol2col.cu | 24 +- paddle/operators/math/vol2col.h | 10 +- paddle/operators/math/vol2col_test.cc | 24 +- paddle/operators/matmul_op.cc | 7 +- paddle/operators/matmul_op.cu.cc | 9 +- paddle/operators/matmul_op.h | 45 ++-- paddle/operators/maxout_op.cc | 7 +- paddle/operators/maxout_op.cu.cc | 13 +- paddle/operators/maxout_op.h | 18 +- paddle/operators/mean_op.cc | 11 +- paddle/operators/mean_op.cu | 11 +- paddle/operators/mean_op.h | 10 +- paddle/operators/minus_op.cc | 4 +- paddle/operators/minus_op.cu | 5 +- paddle/operators/minus_op.h | 5 +- paddle/operators/modified_huber_loss_op.cc | 2 +- paddle/operators/modified_huber_loss_op.cu | 8 +- paddle/operators/modified_huber_loss_op.h | 5 +- paddle/operators/momentum_op.cu | 4 +- paddle/operators/mul_op.cc | 7 +- paddle/operators/mul_op.cu.cc | 7 +- paddle/operators/mul_op.h | 18 +- paddle/operators/multiplex_op.cc | 5 +- paddle/operators/multiplex_op.cu | 16 +- paddle/operators/multiplex_op.h | 11 +- paddle/operators/nccl_op.cu.cc | 6 +- paddle/operators/nccl_op_test.cu.cc | 6 +- paddle/operators/nce_op.cc | 4 +- paddle/operators/nce_op.h | 8 +- paddle/operators/pad_op.cc | 7 +- paddle/operators/pad_op.cu | 7 +- paddle/operators/pad_op.h | 38 ++-- paddle/operators/pool_cudnn_op.cc | 26 ++- paddle/operators/pool_cudnn_op.cu.cc | 18 +- paddle/operators/pool_op.cc | 24 +- paddle/operators/pool_op.cu.cc | 26 ++- paddle/operators/pool_op.h | 59 ++--- paddle/operators/pool_with_index_op.cc | 22 +- paddle/operators/pool_with_index_op.cu.cc | 32 ++- paddle/operators/pool_with_index_op.h | 26 ++- paddle/operators/positive_negative_pair_op.h | 2 +- paddle/operators/precision_recall_op.h | 2 +- paddle/operators/prelu_op.cc | 9 +- paddle/operators/prelu_op.cu | 11 +- paddle/operators/prelu_op.h | 16 +- paddle/operators/proximal_adagrad_op.cc | 2 +- paddle/operators/proximal_adagrad_op.cu | 4 +- paddle/operators/proximal_adagrad_op.h | 10 +- paddle/operators/proximal_gd_op.cc | 3 +- paddle/operators/proximal_gd_op.cu | 5 +- paddle/operators/proximal_gd_op.h | 4 +- paddle/operators/rank_loss_op.cc | 7 +- paddle/operators/rank_loss_op.cu | 12 +- paddle/operators/rank_loss_op.h | 8 +- paddle/operators/reduce_op.cc | 15 +- paddle/operators/reduce_op.cu | 15 +- paddle/operators/reduce_op.h | 44 ++-- paddle/operators/reshape_op.cu | 4 +- paddle/operators/reshape_op.h | 4 +- paddle/operators/rmsprop_op.cc | 4 +- paddle/operators/rmsprop_op.cu | 4 +- paddle/operators/rmsprop_op.h | 4 +- paddle/operators/roi_pool_op.cc | 9 +- paddle/operators/roi_pool_op.cu | 15 +- paddle/operators/roi_pool_op.h | 9 +- paddle/operators/row_conv_op.cc | 13 +- paddle/operators/row_conv_op.cu | 17 +- paddle/operators/row_conv_op.h | 4 +- paddle/operators/scale_op.cc | 10 +- paddle/operators/scale_op.cu | 12 +- paddle/operators/scale_op.h | 5 +- paddle/operators/scatter_op.cu | 4 +- paddle/operators/seq_expand_op.cc | 7 +- paddle/operators/seq_expand_op.cu | 9 +- paddle/operators/seq_expand_op.h | 14 +- paddle/operators/sequence_concat_op.cc | 4 +- paddle/operators/sequence_concat_op.cu.cc | 10 +- paddle/operators/sequence_concat_op.h | 4 +- paddle/operators/sequence_conv_op.cc | 9 +- paddle/operators/sequence_conv_op.cu.cc | 13 +- paddle/operators/sequence_conv_op.h | 65 +++--- paddle/operators/sequence_pool_op.cc | 5 +- paddle/operators/sequence_pool_op.cu | 9 +- paddle/operators/sequence_pool_op.h | 26 ++- paddle/operators/sequence_slice_op.cc | 4 +- paddle/operators/sequence_slice_op.cu | 8 +- paddle/operators/sequence_slice_op.h | 9 +- paddle/operators/sequence_softmax_op.cc | 4 +- paddle/operators/sequence_softmax_op.cu.cc | 8 +- paddle/operators/sequence_softmax_op.h | 12 +- paddle/operators/sgd_op.cc | 13 +- paddle/operators/sgd_op.cu | 22 +- paddle/operators/sgd_op.h | 14 +- .../sigmoid_cross_entropy_with_logits_op.cc | 4 +- .../sigmoid_cross_entropy_with_logits_op.cu | 12 +- .../sigmoid_cross_entropy_with_logits_op.h | 9 +- paddle/operators/sign_op.cc | 4 +- paddle/operators/sign_op.cu | 5 +- paddle/operators/sign_op.h | 5 +- paddle/operators/smooth_l1_loss_op.cc | 5 +- paddle/operators/smooth_l1_loss_op.cu | 9 +- paddle/operators/smooth_l1_loss_op.h | 30 +-- paddle/operators/softmax_op.cc | 7 +- paddle/operators/softmax_op.cu.cc | 9 +- paddle/operators/softmax_op.h | 10 +- .../softmax_with_cross_entropy_op.cu | 40 ++-- .../operators/softmax_with_cross_entropy_op.h | 18 +- paddle/operators/split_op.cu.cc | 4 +- paddle/operators/split_op.h | 2 +- paddle/operators/squared_l2_distance_op.cc | 8 +- paddle/operators/squared_l2_distance_op.cu | 10 +- paddle/operators/squared_l2_distance_op.h | 10 +- paddle/operators/squared_l2_norm_op.cc | 4 +- paddle/operators/squared_l2_norm_op.cu | 8 +- paddle/operators/squared_l2_norm_op.h | 14 +- paddle/operators/sum_op.cc | 9 +- paddle/operators/sum_op.cu | 9 +- paddle/operators/sum_op.h | 23 +- paddle/operators/top_k_op.cu | 2 +- paddle/operators/top_k_op.h | 2 +- paddle/operators/transpose_op.cc | 6 +- paddle/operators/transpose_op.cu.cc | 9 +- paddle/operators/transpose_op.h | 29 +-- paddle/operators/uniform_random_op.cc | 2 +- paddle/operators/uniform_random_op.cu | 6 +- paddle/operators/unpool_op.cc | 11 +- paddle/operators/unpool_op.cu.cc | 13 +- paddle/operators/unpool_op.h | 22 +- paddle/platform/device_context.cc | 12 - paddle/platform/device_context.h | 17 -- paddle/platform/device_context_test.cc | 5 +- paddle/platform/transform.h | 24 +- paddle/platform/transform_test.cu | 8 +- 319 files changed, 2624 insertions(+), 2546 deletions(-) diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index daade439e5..b29238432b 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -181,8 +181,8 @@ class OpKernelRegistrar : public Registrar { return 0; \ } -#define REGISTER_OP_GPU_KERNEL(op_type, ...) \ - REGISTER_OP_KERNEL(op_type, GPU, ::paddle::platform::GPUPlace, __VA_ARGS__) +#define REGISTER_OP_CUDA_KERNEL(op_type, ...) \ + REGISTER_OP_KERNEL(op_type, CUDA, ::paddle::platform::GPUPlace, __VA_ARGS__) #define REGISTER_OP_CPU_KERNEL(op_type, ...) \ REGISTER_OP_KERNEL(op_type, CPU, ::paddle::platform::CPUPlace, __VA_ARGS__) @@ -217,7 +217,7 @@ class OpKernelRegistrar : public Registrar { #else #define USE_OP_KERNEL(op_type) \ USE_OP_DEVICE_KERNEL(op_type, CPU); \ - USE_OP_DEVICE_KERNEL(op_type, GPU) + USE_OP_DEVICE_KERNEL(op_type, CUDA) #endif #define USE_NO_KERNEL_OP(op_type) USE_OP_ITSELF(op_type); @@ -226,9 +226,9 @@ class OpKernelRegistrar : public Registrar { USE_OP_ITSELF(op_type); \ USE_OP_DEVICE_KERNEL(op_type, CPU); -#define USE_GPU_ONLY_OP(op_type) \ - USE_OP_ITSELF(op_type); \ - USE_OP_DEVICE_KERNEL(op_type, GPU) +#define USE_CUDA_ONLY_OP(op_type) \ + USE_OP_ITSELF(op_type); \ + USE_OP_DEVICE_KERNEL(op_type, CUDA) #define USE_OP(op_type) \ USE_OP_ITSELF(op_type); \ diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index f1444eeee9..e83d754783 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -22,20 +22,6 @@ limitations under the License. */ namespace paddle { namespace framework { -template <> -Eigen::DefaultDevice& ExecutionContext::GetEigenDevice< - platform::CPUPlace, Eigen::DefaultDevice>() const { - return *device_context_.GetEigenDevice(); -} - -#ifdef PADDLE_WITH_CUDA -template <> -Eigen::GpuDevice& -ExecutionContext::GetEigenDevice() const { - return *device_context_.GetEigenDevice(); -} -#endif - std::string OperatorBase::Input(const std::string& name) const { auto& ins = Inputs(name); PADDLE_ENFORCE_LE(ins.size(), 1UL, @@ -429,7 +415,7 @@ void OperatorWithKernel::Run(const Scope& scope, } OpKernelType OperatorWithKernel::GetKernelType( const ExecutionContext& ctx) const { - return OpKernelType(IndicateDataType(ctx), ctx.device_context()); + return OpKernelType(IndicateDataType(ctx), ctx.GetPlace()); } DataType OperatorWithKernel::IndicateDataType( const ExecutionContext& ctx) const { diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 60861d9293..e60dbfc313 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -276,17 +276,25 @@ class ExecutionContext { out_tensor->set_lod(in_tensor.lod()); } - template ::EigenDeviceType> - DeviceType& GetEigenDevice() const; - platform::Place GetPlace() const { return device_context_.GetPlace(); } + template + const DeviceContextType& device_context() const { + return *reinterpret_cast(&device_context_); + } + const platform::DeviceContext& device_context() const { return device_context_; } +#ifdef PADDLE_WITH_CUDA + const inline platform::CUDADeviceContext& cuda_device_context() const { + PADDLE_ENFORCE(platform::is_gpu_place(device_context_.GetPlace())); + return *reinterpret_cast( + &device_context_); + } +#endif + //! Get actual name vector for this input. const std::vector& Inputs(const std::string& name) const { return op_.Inputs(name); @@ -297,14 +305,6 @@ class ExecutionContext { return op_.Outputs(name); } -#ifdef PADDLE_WITH_CUDA - const inline platform::CUDADeviceContext& cuda_device_context() const { - PADDLE_ENFORCE(platform::is_gpu_place(device_context_.GetPlace())); - return *reinterpret_cast( - &device_context_); - } -#endif - private: const OperatorBase& op_; const Scope& scope_; diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc index 59ddbc7791..b678178454 100644 --- a/paddle/framework/operator_test.cc +++ b/paddle/framework/operator_test.cc @@ -115,7 +115,7 @@ class OpWithKernelTest : public OperatorWithKernel { protected: void InferShape(framework::InferShapeContext* ctx) const override {} OpKernelType GetKernelType(const ExecutionContext& ctx) const override { - return OpKernelType(DataType::FP32, ctx.device_context()); + return OpKernelType(DataType::FP32, ctx.GetPlace()); } }; diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index 38b89b9eb1..5aaaf99332 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -138,7 +138,7 @@ function(op_library TARGET) if ("${TARGET}" STREQUAL "nccl_op") set(pybind_flag 1) # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_GPU_ONLY_OP(ncclAllReduce);\n") + file(APPEND ${pybind_file} "USE_CUDA_ONLY_OP(ncclAllReduce);\n") endif() # reduce_op contains several operators diff --git a/paddle/operators/accuracy_op.cc b/paddle/operators/accuracy_op.cc index 2785a8c6fb..76da21c472 100644 --- a/paddle/operators/accuracy_op.cc +++ b/paddle/operators/accuracy_op.cc @@ -57,7 +57,7 @@ class AccuracyOp : public framework::OperatorWithKernel { const framework::ExecutionContext &ctx) const override { return framework::OpKernelType( framework::ToDataType(ctx.Input("Out")->type()), - ctx.device_context()); + ctx.GetPlace()); } }; diff --git a/paddle/operators/accuracy_op.cu b/paddle/operators/accuracy_op.cu index d2dcab4e54..539a935302 100644 --- a/paddle/operators/accuracy_op.cu +++ b/paddle/operators/accuracy_op.cu @@ -104,5 +104,6 @@ class AccuracyOpCUDAKernel : public framework::OpKernel { // FIXME(typhoonzero): types of T is for inference data. // label data is always int64 -REGISTER_OP_GPU_KERNEL(accuracy, paddle::operators::AccuracyOpCUDAKernel, - paddle::operators::AccuracyOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(accuracy, + paddle::operators::AccuracyOpCUDAKernel, + paddle::operators::AccuracyOpCUDAKernel); diff --git a/paddle/operators/accuracy_op.h b/paddle/operators/accuracy_op.h index d060e6eddd..04104a695f 100644 --- a/paddle/operators/accuracy_op.h +++ b/paddle/operators/accuracy_op.h @@ -21,7 +21,7 @@ namespace operators { using Tensor = framework::Tensor; -template +template class AccuracyKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc index 7f3118f176..63490f0ec9 100644 --- a/paddle/operators/activation_op.cc +++ b/paddle/operators/activation_op.cc @@ -611,16 +611,17 @@ REGISTER_OP(hard_sigmoid, ops::ActivationOp, ops::HardSigmoidOpMaker, REGISTER_OP(swish, ops::ActivationOp, ops::SwishOpMaker, swish_grad, ops::ActivationOpGrad); -#define REGISTER_ACTIVATION_CPU_KERNEL(act_type, functor, grad_functor) \ - REGISTER_OP_CPU_KERNEL( \ - act_type, \ - ops::ActivationKernel>, \ - ops::ActivationKernel>); \ - REGISTER_OP_CPU_KERNEL( \ - act_type##_grad, ops::ActivationGradKernel>, \ - ops::ActivationGradKernel>, \ + ops::ActivationKernel>); \ + REGISTER_OP_CPU_KERNEL( \ + act_type##_grad, \ + ops::ActivationGradKernel>, \ + ops::ActivationGradKernel>); FOR_EACH_KERNEL_FUNCTOR(REGISTER_ACTIVATION_CPU_KERNEL); diff --git a/paddle/operators/activation_op.cu b/paddle/operators/activation_op.cu index 97737857ab..856d3fc35d 100644 --- a/paddle/operators/activation_op.cu +++ b/paddle/operators/activation_op.cu @@ -17,16 +17,17 @@ namespace ops = paddle::operators; -#define REGISTER_ACTIVATION_GPU_KERNEL(act_type, functor, grad_functor) \ - REGISTER_OP_GPU_KERNEL( \ - act_type, \ - ops::ActivationKernel>, \ - ops::ActivationKernel>); \ - REGISTER_OP_GPU_KERNEL( \ - act_type##_grad, ops::ActivationGradKernel>, \ - ops::ActivationGradKernel>, \ + ops::ActivationKernel>); \ + REGISTER_OP_CUDA_KERNEL( \ + act_type##_grad, \ + ops::ActivationGradKernel>, \ + ops::ActivationGradKernel>); -FOR_EACH_KERNEL_FUNCTOR(REGISTER_ACTIVATION_GPU_KERNEL); +FOR_EACH_KERNEL_FUNCTOR(REGISTER_ACTIVATION_CUDA_KERNEL); diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h index ac0e0a3b01..75eefca8b8 100644 --- a/paddle/operators/activation_op.h +++ b/paddle/operators/activation_op.h @@ -19,7 +19,7 @@ namespace paddle { namespace operators { -template +template class ActivationKernel : public framework::OpKernel { public: @@ -32,18 +32,19 @@ class ActivationKernel auto x = framework::EigenVector::Flatten(*X); auto y = framework::EigenVector::Flatten(*Y); - auto place = context.GetEigenDevice(); + auto* place = + context.template device_context().eigen_device(); Functor functor; auto attrs = functor.GetAttrs(); for (auto& attr : attrs) { *attr.second = context.Attr(attr.first); } - functor(place, x, y); + functor(*place, x, y); } }; -template +template class ActivationGradKernel : public framework::OpKernel { public: @@ -59,13 +60,14 @@ class ActivationGradKernel auto x = framework::EigenVector::Flatten(*X); auto y = framework::EigenVector::Flatten(*Y); auto dx = framework::EigenVector::Flatten(*dX); - auto place = context.GetEigenDevice(); + auto* place = + context.template device_context().eigen_device(); Functor functor; auto attrs = functor.GetAttrs(); for (auto& attr : attrs) { *attr.second = context.Attr(attr.first); } - functor(place, x, y, dy, dx); + functor(*place, x, y, dy, dx); } }; diff --git a/paddle/operators/adadelta_op.cc b/paddle/operators/adadelta_op.cc index 29434a0ee2..507811e7b5 100644 --- a/paddle/operators/adadelta_op.cc +++ b/paddle/operators/adadelta_op.cc @@ -109,5 +109,5 @@ $$ namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(adadelta, ops::AdadeltaOp, ops::AdadeltaOpMaker); REGISTER_OP_CPU_KERNEL( - adadelta, ops::AdadeltaOpKernel, - ops::AdadeltaOpKernel); + adadelta, ops::AdadeltaOpKernel, + ops::AdadeltaOpKernel); diff --git a/paddle/operators/adadelta_op.cu b/paddle/operators/adadelta_op.cu index 9fb6185207..eee2d0a2f5 100644 --- a/paddle/operators/adadelta_op.cu +++ b/paddle/operators/adadelta_op.cu @@ -16,6 +16,6 @@ #include "paddle/operators/adadelta_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( - adadelta, ops::AdadeltaOpKernel, - ops::AdadeltaOpKernel); +REGISTER_OP_CUDA_KERNEL( + adadelta, ops::AdadeltaOpKernel, + ops::AdadeltaOpKernel); diff --git a/paddle/operators/adadelta_op.h b/paddle/operators/adadelta_op.h index a8c5f0c8aa..819d0845db 100644 --- a/paddle/operators/adadelta_op.h +++ b/paddle/operators/adadelta_op.h @@ -19,7 +19,7 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class AdadeltaOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -51,7 +51,7 @@ class AdadeltaOpKernel : public framework::OpKernel { framework::EigenVector::Flatten(*avg_squared_grad_out_tensor); auto avg_squared_update_out = framework::EigenVector::Flatten(*avg_squared_update_out_tensor); - auto place = ctx.GetEigenDevice(); + auto& place = *ctx.template device_context().eigen_device(); avg_squared_grad_out.device(place) = rho * avg_squared_grad + (1 - rho) * grad.square(); diff --git a/paddle/operators/adagrad_op.cc b/paddle/operators/adagrad_op.cc index d19602244b..5d00716316 100644 --- a/paddle/operators/adagrad_op.cc +++ b/paddle/operators/adagrad_op.cc @@ -100,8 +100,8 @@ size_t FindPos(const std::vector& rows, int64_t value) { } // namespace template -struct SparseAdagradFunctor { - void operator()(const platform::DeviceContext& context, +struct SparseAdagradFunctor { + void operator()(const platform::CPUDeviceContext& context, const framework::SelectedRows& grad, const framework::Tensor& learning_rate, T epsilon, framework::Tensor* moment, framework::Tensor* param) { @@ -120,7 +120,7 @@ struct SparseAdagradFunctor { {static_cast(merge_rows.size()), grad_width}), context.GetPlace()); - math::SetConstant constant_functor; + math::SetConstant constant_functor; constant_functor(context, grad_merge->mutable_value(), 0.0); auto* grad_merge_data = grad_merge->mutable_value()->data(); @@ -144,9 +144,9 @@ struct SparseAdagradFunctor { auto gs = framework::EigenVector::Flatten(*(grad_square->mutable_value())); auto gm = framework::EigenVector::Flatten(grad_merge->value()); - gs.device(*context.GetEigenDevice()) = gm * gm; + gs.device(*context.eigen_device()) = gm * gm; - math::SelectedRowsAddToTensor functor; + math::SelectedRowsAddToTensor functor; functor(context, *grad_square, moment); // 3. update parameter @@ -164,13 +164,13 @@ struct SparseAdagradFunctor { } }; -template struct SparseAdagradFunctor; -template struct SparseAdagradFunctor; +template struct SparseAdagradFunctor; +template struct SparseAdagradFunctor; } // namespace operators } // namespace paddle namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(adagrad, ops::AdagradOp, ops::AdagradOpMaker); REGISTER_OP_CPU_KERNEL( - adagrad, ops::AdagradOpKernel, - ops::AdagradOpKernel); + adagrad, ops::AdagradOpKernel, + ops::AdagradOpKernel); diff --git a/paddle/operators/adagrad_op.cu b/paddle/operators/adagrad_op.cu index 1c870214b2..585b2d9289 100644 --- a/paddle/operators/adagrad_op.cu +++ b/paddle/operators/adagrad_op.cu @@ -72,8 +72,8 @@ __global__ void SparseAdagradFunctorKernel(const T* grad, const int64_t* rows, } // namespace template -struct SparseAdagradFunctor { - void operator()(const platform::DeviceContext& context, +struct SparseAdagradFunctor { + void operator()(const platform::CUDADeviceContext& context, const framework::SelectedRows& grad, const framework::Tensor& learning_rate, T epsilon, framework::Tensor* moment, framework::Tensor* param) { @@ -92,7 +92,7 @@ struct SparseAdagradFunctor { {static_cast(merge_rows.size()), grad_width}), context.GetPlace()); - math::SetConstant constant_functor; + math::SetConstant constant_functor; constant_functor(context, grad_merge->mutable_value(), 0.0); auto* grad_merge_data = grad_merge->mutable_value()->data(); @@ -119,9 +119,9 @@ struct SparseAdagradFunctor { auto gs = framework::EigenVector::Flatten(*(grad_square->mutable_value())); auto gm = framework::EigenVector::Flatten(grad_merge->value()); - gs.device(*context.GetEigenDevice()) = gm * gm; + gs.device(*context.eigen_device()) = gm * gm; - math::SelectedRowsAddToTensor functor; + math::SelectedRowsAddToTensor functor; functor(context, *grad_square, moment); // 3. update parameter @@ -139,13 +139,13 @@ struct SparseAdagradFunctor { } }; -template struct SparseAdagradFunctor; -template struct SparseAdagradFunctor; +template struct SparseAdagradFunctor; +template struct SparseAdagradFunctor; } // namespace operators } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( - adagrad, ops::AdagradOpKernel, - ops::AdagradOpKernel); +REGISTER_OP_CUDA_KERNEL( + adagrad, ops::AdagradOpKernel, + ops::AdagradOpKernel); diff --git a/paddle/operators/adagrad_op.h b/paddle/operators/adagrad_op.h index 4d4a6434c7..0d77dbcbac 100644 --- a/paddle/operators/adagrad_op.h +++ b/paddle/operators/adagrad_op.h @@ -19,15 +19,15 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template struct SparseAdagradFunctor { - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::SelectedRows& grad, const framework::Tensor& learning_rate, T epsilon, framework::Tensor* moment, framework::Tensor* param); }; -template +template class AdagradOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -52,11 +52,11 @@ class AdagradOpKernel : public framework::OpKernel { auto param_out = framework::EigenVector::Flatten(*param_out_tensor); auto moment_out = framework::EigenVector::Flatten(*moment_out_tensor); - auto place = ctx.GetEigenDevice(); + auto* place = ctx.template device_context().eigen_device(); - moment_out.device(place) = moment + grad * grad; + moment_out.device(*place) = moment + grad * grad; Eigen::DSizes m_dsize(moment_out_tensor->numel()); - param_out.device(place) = + param_out.device(*place) = param - lr.broadcast(m_dsize) * grad / (moment_out.sqrt() + epsilon); } else if (grad_var->IsType()) { auto* param_tensor = ctx.Input("Param"); @@ -65,8 +65,9 @@ class AdagradOpKernel : public framework::OpKernel { auto* moment_tensor = ctx.Input("Moment"); PADDLE_ENFORCE_EQ(moment_tensor, moment_out_tensor); - SparseAdagradFunctor functor; - functor(ctx.device_context(), *ctx.Input("Grad"), + SparseAdagradFunctor functor; + functor(ctx.template device_context(), + *ctx.Input("Grad"), *ctx.Input("LearningRate"), epsilon, moment_out_tensor, param_out_tensor); } else { diff --git a/paddle/operators/adam_op.cc b/paddle/operators/adam_op.cc index a268d05484..cf6ef6dd53 100644 --- a/paddle/operators/adam_op.cc +++ b/paddle/operators/adam_op.cc @@ -128,6 +128,6 @@ $$ namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(adam, ops::AdamOp, ops::AdamOpMaker); -REGISTER_OP_CPU_KERNEL(adam, - ops::AdamOpKernel, - ops::AdamOpKernel); +REGISTER_OP_CPU_KERNEL( + adam, ops::AdamOpKernel, + ops::AdamOpKernel); diff --git a/paddle/operators/adam_op.cu b/paddle/operators/adam_op.cu index 6e34f7818c..c135b37378 100644 --- a/paddle/operators/adam_op.cu +++ b/paddle/operators/adam_op.cu @@ -16,6 +16,6 @@ #include "paddle/operators/adam_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(adam, - ops::AdamOpKernel, - ops::AdamOpKernel); +REGISTER_OP_CUDA_KERNEL( + adam, ops::AdamOpKernel, + ops::AdamOpKernel); diff --git a/paddle/operators/adam_op.h b/paddle/operators/adam_op.h index 7f7fa1da1c..45157842a6 100644 --- a/paddle/operators/adam_op.h +++ b/paddle/operators/adam_op.h @@ -19,7 +19,7 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class AdamOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -52,17 +52,17 @@ class AdamOpKernel : public framework::OpKernel { auto param_out = framework::EigenVector::Flatten(*param_out_tensor); auto moment1_out = framework::EigenVector::Flatten(*moment1_out_tensor); auto moment2_out = framework::EigenVector::Flatten(*moment2_out_tensor); - auto place = ctx.GetEigenDevice(); + auto* place = ctx.template device_context().eigen_device(); - moment1_out.device(place) = beta1 * moment1 + (1 - beta1) * grad; - moment2_out.device(place) = beta2 * moment2 + (1 - beta2) * grad.square(); + moment1_out.device(*place) = beta1 * moment1 + (1 - beta1) * grad; + moment2_out.device(*place) = beta2 * moment2 + (1 - beta2) * grad.square(); // All of these are tensors of 1 element auto lr_t = lr * (1 - beta2_pow).sqrt() / (1 - beta1_pow); // Eigen does not support automatic broadcast // Get dimensions of moment vector to broadcast lr_t Eigen::DSizes m_dsize(moment1_out_tensor->numel()); - param_out.device(place) = + param_out.device(*place) = param - lr_t.broadcast(m_dsize) * (moment1_out / (moment2_out.sqrt() + epsilon)); diff --git a/paddle/operators/adamax_op.cc b/paddle/operators/adamax_op.cc index 9e7576c961..49ce497bb7 100644 --- a/paddle/operators/adamax_op.cc +++ b/paddle/operators/adamax_op.cc @@ -127,6 +127,6 @@ division by 0 error. namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(adamax, ops::AdamaxOp, ops::AdamaxOpMaker); -REGISTER_OP_CPU_KERNEL(adamax, - ops::AdamaxOpKernel, - ops::AdamaxOpKernel); +REGISTER_OP_CPU_KERNEL( + adamax, ops::AdamaxOpKernel, + ops::AdamaxOpKernel); diff --git a/paddle/operators/adamax_op.cu b/paddle/operators/adamax_op.cu index 057ef39025..2d143905c4 100644 --- a/paddle/operators/adamax_op.cu +++ b/paddle/operators/adamax_op.cu @@ -16,6 +16,6 @@ #include "paddle/operators/adamax_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(adamax, - ops::AdamaxOpKernel, - ops::AdamaxOpKernel); +REGISTER_OP_CUDA_KERNEL( + adamax, ops::AdamaxOpKernel, + ops::AdamaxOpKernel); diff --git a/paddle/operators/adamax_op.h b/paddle/operators/adamax_op.h index bf36ed7860..172c179c5f 100644 --- a/paddle/operators/adamax_op.h +++ b/paddle/operators/adamax_op.h @@ -19,7 +19,7 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class AdamaxOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -51,14 +51,14 @@ class AdamaxOpKernel : public framework::OpKernel { auto moment_out = framework::EigenVector::Flatten(*moment_out_tensor); auto inf_norm_out = framework::EigenVector::Flatten(*inf_norm_out_tensor); - auto place = ctx.GetEigenDevice(); + auto* place = ctx.template device_context().eigen_device(); - moment_out.device(place) = beta1 * moment + (1 - beta1) * grad; - inf_norm_out.device(place) = + moment_out.device(*place) = beta1 * moment + (1 - beta1) * grad; + inf_norm_out.device(*place) = grad.abs().cwiseMax((beta2 * inf_norm) + epsilon); auto lr_t = lr / (1 - beta1_pow); Eigen::DSizes m_dsize(moment_out_tensor->numel()); - param_out.device(place) = + param_out.device(*place) = param - lr_t.broadcast(m_dsize) * (moment_out / inf_norm_out); } }; diff --git a/paddle/operators/auc_op.h b/paddle/operators/auc_op.h index e5ac57b038..b80509e2a9 100644 --- a/paddle/operators/auc_op.h +++ b/paddle/operators/auc_op.h @@ -25,7 +25,7 @@ template using EigenVector = framework::EigenVector; -template +template class AucKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { diff --git a/paddle/operators/batch_norm_op.cc b/paddle/operators/batch_norm_op.cc index ac97bd83ab..94a972b7ab 100644 --- a/paddle/operators/batch_norm_op.cc +++ b/paddle/operators/batch_norm_op.cc @@ -135,7 +135,8 @@ The required data format for this layer is one of the following: }; template -class BatchNormKernel : public framework::OpKernel { +class BatchNormKernel + : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { const float epsilon = ctx.Attr("epsilon"); @@ -318,12 +319,12 @@ class BatchNormGradOp : public framework::OperatorWithKernel { PADDLE_THROW("can't find Y@GRAD"); } return framework::OpKernelType(framework::ToDataType(t->type()), - ctx.device_context()); + ctx.GetPlace()); } }; template -class BatchNormGradKernel +class BatchNormGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { @@ -436,8 +437,9 @@ class BatchNormGradKernel namespace ops = paddle::operators; REGISTER_OP(batch_norm, ops::BatchNormOp, ops::BatchNormOpMaker, batch_norm_grad, ops::BatchNormGradOp); -REGISTER_OP_CPU_KERNEL(batch_norm, - ops::BatchNormKernel); +REGISTER_OP_CPU_KERNEL( + batch_norm, + ops::BatchNormKernel); REGISTER_OP_CPU_KERNEL( batch_norm_grad, - ops::BatchNormGradKernel); + ops::BatchNormGradKernel); diff --git a/paddle/operators/batch_norm_op.cu.cc b/paddle/operators/batch_norm_op.cu.cc index 7b2f318700..c7adc3d80e 100644 --- a/paddle/operators/batch_norm_op.cu.cc +++ b/paddle/operators/batch_norm_op.cu.cc @@ -47,7 +47,8 @@ void ExtractNCWHD(const framework::DDim &dims, } template -class BatchNormKernel : public framework::OpKernel { +class BatchNormKernel + : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), @@ -121,11 +122,12 @@ class BatchNormKernel : public framework::OpKernel { saved_mean->mutable_data(ctx.GetPlace()); saved_variance->mutable_data(ctx.GetPlace()); - math::SetConstant functor; - functor(ctx.device_context(), saved_mean, 0); - functor(ctx.device_context(), saved_variance, 0); + auto &dev_ctx = ctx.template device_context(); + math::SetConstant functor; + functor(dev_ctx, saved_mean, 0); + functor(dev_ctx, saved_variance, 0); - auto handle = ctx.cuda_device_context().cudnn_handle(); + auto handle = dev_ctx.cudnn_handle(); // Now, depending on whether we are running test or not, we have two paths. if (is_test) { @@ -171,7 +173,7 @@ class BatchNormKernel : public framework::OpKernel { }; template -class BatchNormGradKernel +class BatchNormGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { @@ -244,11 +246,12 @@ class BatchNormGradKernel const void *saved_mean_data = saved_mean->template data(); const void *saved_var_data = saved_var->template data(); + auto &dev_ctx = ctx.template device_context(); CUDNN_ENFORCE(platform::dynload::cudnnBatchNormalizationBackward( - ctx.cuda_device_context().cudnn_handle(), mode_, - CudnnDataType::kOne(), CudnnDataType::kZero(), - CudnnDataType::kOne(), CudnnDataType::kZero(), data_desc_, - x->template data(), data_desc_, d_y->template data(), data_desc_, + dev_ctx.cudnn_handle(), mode_, CudnnDataType::kOne(), + CudnnDataType::kZero(), CudnnDataType::kOne(), + CudnnDataType::kZero(), data_desc_, x->template data(), + data_desc_, d_y->template data(), data_desc_, d_x->template mutable_data(ctx.GetPlace()), bn_param_desc_, scale->template data(), d_scale->template mutable_data(ctx.GetPlace()), @@ -266,8 +269,9 @@ class BatchNormGradKernel } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(batch_norm, - ops::BatchNormKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( + batch_norm, + ops::BatchNormKernel); +REGISTER_OP_CUDA_KERNEL( batch_norm_grad, - ops::BatchNormGradKernel); + ops::BatchNormGradKernel); diff --git a/paddle/operators/batch_norm_op.h b/paddle/operators/batch_norm_op.h index 4e80134a1a..8d99b68647 100644 --- a/paddle/operators/batch_norm_op.h +++ b/paddle/operators/batch_norm_op.h @@ -34,13 +34,13 @@ inline TensorFormat StringToTensorFormat(const std::string& str) { } } -template +template class BatchNormKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override; }; -template +template class BatchNormGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override; diff --git a/paddle/operators/bilinear_tensor_product_op.cc b/paddle/operators/bilinear_tensor_product_op.cc index c88b2c9beb..217fd52366 100644 --- a/paddle/operators/bilinear_tensor_product_op.cc +++ b/paddle/operators/bilinear_tensor_product_op.cc @@ -159,9 +159,12 @@ REGISTER_OP(bilinear_tensor_product, ops::BilinearTensorProductOp, ops::BilinearTensorProductOpGrad); REGISTER_OP_CPU_KERNEL( bilinear_tensor_product, - ops::BilinearTensorProductKernel, - ops::BilinearTensorProductKernel); + ops::BilinearTensorProductKernel, + ops::BilinearTensorProductKernel); REGISTER_OP_CPU_KERNEL( bilinear_tensor_product_grad, - ops::BilinearTensorProductGradKernel, - ops::BilinearTensorProductGradKernel); + ops::BilinearTensorProductGradKernel, + ops::BilinearTensorProductGradKernel); diff --git a/paddle/operators/bilinear_tensor_product_op.cu b/paddle/operators/bilinear_tensor_product_op.cu index 858d2668d0..0f48010716 100644 --- a/paddle/operators/bilinear_tensor_product_op.cu +++ b/paddle/operators/bilinear_tensor_product_op.cu @@ -16,11 +16,15 @@ limitations under the License. */ #include "paddle/operators/bilinear_tensor_product_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( bilinear_tensor_product, - ops::BilinearTensorProductKernel, - ops::BilinearTensorProductKernel); -REGISTER_OP_GPU_KERNEL( + ops::BilinearTensorProductKernel, + ops::BilinearTensorProductKernel); +REGISTER_OP_CUDA_KERNEL( bilinear_tensor_product_grad, - ops::BilinearTensorProductGradKernel, - ops::BilinearTensorProductGradKernel); + ops::BilinearTensorProductGradKernel, + ops::BilinearTensorProductGradKernel); diff --git a/paddle/operators/bilinear_tensor_product_op.h b/paddle/operators/bilinear_tensor_product_op.h index 1113a4c6f3..ba9a2c5ce3 100644 --- a/paddle/operators/bilinear_tensor_product_op.h +++ b/paddle/operators/bilinear_tensor_product_op.h @@ -27,7 +27,7 @@ template using EigenMatrix = framework::EigenMatrix; -template +template class BilinearTensorProductKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -46,7 +46,8 @@ class BilinearTensorProductKernel : public framework::OpKernel { int out_dim = weight_dims[0]; auto x_dim = weight_dims[1]; auto y_dim = weight_dims[2]; - auto place = ctx.GetEigenDevice(); + auto& place = *ctx.template device_context().eigen_device(); + auto& dev_ctx = ctx.template device_context(); // Create the intermediate variable to caculate the result of // Input(X) multiplied by Input(Weight_i), the formula is: @@ -60,9 +61,9 @@ class BilinearTensorProductKernel : public framework::OpKernel { auto output_col_vec = output_mat.chip(i, 1); Tensor weight_mat = weight->Slice(i, i + 1).Resize(framework::make_ddim({x_dim, y_dim})); - math::gemm(ctx.device_context(), CblasNoTrans, CblasNoTrans, - batch_size, y_dim, x_dim, 1, x->data(), - weight_mat.data(), 0, left_mul.data()); + math::gemm(dev_ctx, CblasNoTrans, CblasNoTrans, + batch_size, y_dim, x_dim, 1, x->data(), + weight_mat.data(), 0, left_mul.data()); output_col_vec.device(place) = (left_mul_mat * y_mat).sum(Eigen::DSizes(1)); } @@ -74,7 +75,7 @@ class BilinearTensorProductKernel : public framework::OpKernel { } }; -template +template class BilinearTensorProductGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -96,8 +97,8 @@ class BilinearTensorProductGradKernel : public framework::OpKernel { auto x_mat = EigenMatrix::From(*x); auto y_mat = EigenMatrix::From(*y); auto d_out_mat = EigenMatrix::From(*d_out); - auto place = ctx.GetEigenDevice(); - + auto& place = *ctx.template device_context().eigen_device(); + auto& dev_ctx = ctx.template device_context(); // Create the intermediate variable to caculate the Output(Y@Grad). Tensor x_scale; x_scale.mutable_data(framework::make_ddim({batch_size, x_dim}), @@ -110,18 +111,18 @@ class BilinearTensorProductGradKernel : public framework::OpKernel { ctx.GetPlace()); auto y_scale_mat = EigenMatrix::From(y_scale); - math::SetConstant set_zero; + math::SetConstant set_zero; // Set Output(X@Grad) be zero. if (d_x) { d_x->mutable_data(ctx.GetPlace()); - set_zero(ctx.device_context(), d_x, static_cast(0)); + set_zero(dev_ctx, d_x, static_cast(0)); } // Set Output(Y@Grad) be zero. if (d_y) { d_y->mutable_data(ctx.GetPlace()); - set_zero(ctx.device_context(), d_y, static_cast(0)); + set_zero(dev_ctx, d_y, static_cast(0)); } // Caculate the Output(X@Grad) and Output(Y@Grad). @@ -137,18 +138,18 @@ class BilinearTensorProductGradKernel : public framework::OpKernel { output_vec.reshape(Eigen::DSizes(batch_size, 1)) .broadcast(bcast_for_x) * y_mat; - math::gemm(ctx.device_context(), CblasNoTrans, CblasTrans, - batch_size, x_dim, y_dim, 1, y_scale.data(), - weight_i.data(), 1, d_x->data()); + math::gemm( + dev_ctx, CblasNoTrans, CblasTrans, batch_size, x_dim, y_dim, 1, + y_scale.data(), weight_i.data(), 1, d_x->data()); } if (d_y) { x_scale_mat.device(place) = output_vec.reshape(Eigen::DSizes(batch_size, 1)) .broadcast(bcast_for_y) * x_mat; - math::gemm(ctx.device_context(), CblasNoTrans, CblasNoTrans, - batch_size, y_dim, x_dim, 1, x_scale.data(), - weight_i.data(), 1, d_y->data()); + math::gemm( + dev_ctx, CblasNoTrans, CblasNoTrans, batch_size, y_dim, x_dim, 1, + x_scale.data(), weight_i.data(), 1, d_y->data()); } } } @@ -165,9 +166,9 @@ class BilinearTensorProductGradKernel : public framework::OpKernel { output_vec.reshape(Eigen::DSizes(batch_size, 1)) .broadcast(bcast_for_weight) * x_mat; - math::gemm(ctx.device_context(), CblasTrans, CblasNoTrans, - x_dim, y_dim, batch_size, 1, x_scale.data(), - y->data(), 0, d_weight_i.data()); + math::gemm(dev_ctx, CblasTrans, CblasNoTrans, x_dim, + y_dim, batch_size, 1, x_scale.data(), + y->data(), 0, d_weight_i.data()); } } diff --git a/paddle/operators/cast_op.cc b/paddle/operators/cast_op.cc index 3082a53ccf..42bff69a1e 100644 --- a/paddle/operators/cast_op.cc +++ b/paddle/operators/cast_op.cc @@ -68,7 +68,7 @@ class CastOpGradMaker : public framework::SingleGradOpDescMaker { } // namespace paddle namespace ops = paddle::operators; -using CPU = paddle::platform::CPUPlace; +using CPU = paddle::platform::CPUDeviceContext; REGISTER_OP_WITH_KERNEL(cast, ops::CastOpGradMaker, ops::CastOpInferShape, ops::CastOpProtoMaker); REGISTER_OP_CPU_KERNEL(cast, ops::CastOpKernel, diff --git a/paddle/operators/cast_op.cu b/paddle/operators/cast_op.cu index fb75ddbabf..4681deaa62 100644 --- a/paddle/operators/cast_op.cu +++ b/paddle/operators/cast_op.cu @@ -16,7 +16,7 @@ template using CastOpKernel = - paddle::operators::CastOpKernel; + paddle::operators::CastOpKernel; -REGISTER_OP_GPU_KERNEL(cast, CastOpKernel, CastOpKernel, - CastOpKernel, CastOpKernel); +REGISTER_OP_CUDA_KERNEL(cast, CastOpKernel, CastOpKernel, + CastOpKernel, CastOpKernel); diff --git a/paddle/operators/cast_op.h b/paddle/operators/cast_op.h index 850dc8e349..a6773f13a8 100644 --- a/paddle/operators/cast_op.h +++ b/paddle/operators/cast_op.h @@ -27,13 +27,13 @@ struct CastOpTransformFunctor { HOSTDEVICE OutT operator()(InT in) const { return static_cast(in); } }; -template +template struct CastOpFunctor { const framework::Tensor* in_; framework::Tensor* out_; - const platform::DeviceContext& ctx_; + const DeviceContext& ctx_; CastOpFunctor(const framework::Tensor* in, framework::Tensor* out, - const platform::DeviceContext& ctx) + const DeviceContext& ctx) : in_(in), out_(out), ctx_(ctx) {} template @@ -42,13 +42,13 @@ struct CastOpFunctor { auto numel = in_->numel(); auto* in_end = in_begin + numel; auto* out_begin = out_->mutable_data(ctx_.GetPlace()); - platform::Transform trans; + platform::Transform trans; trans(ctx_, in_begin, in_end, out_begin, CastOpTransformFunctor()); } }; -template +template class CastOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -56,7 +56,8 @@ class CastOpKernel : public framework::OpKernel { auto* out = context.Output("Out"); framework::VisitDataType( static_cast(context.Attr("out_dtype")), - CastOpFunctor(in, out, context.device_context())); + CastOpFunctor( + in, out, context.template device_context())); } }; diff --git a/paddle/operators/chunk_eval_op.h b/paddle/operators/chunk_eval_op.h index dd88f2553b..9cd758a825 100644 --- a/paddle/operators/chunk_eval_op.h +++ b/paddle/operators/chunk_eval_op.h @@ -23,7 +23,7 @@ namespace operators { using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; -template +template class ChunkEvalKernel : public framework::OpKernel { public: struct Segment { diff --git a/paddle/operators/clip_by_norm_op.cc b/paddle/operators/clip_by_norm_op.cc index f73d55bbe3..0b7975a63f 100644 --- a/paddle/operators/clip_by_norm_op.cc +++ b/paddle/operators/clip_by_norm_op.cc @@ -71,4 +71,5 @@ namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(clip_by_norm, ops::ClipByNormOp, ops::ClipByNormOpMaker); REGISTER_OP_CPU_KERNEL( - clip_by_norm, ops::ClipByNormKernel); + clip_by_norm, + ops::ClipByNormKernel); diff --git a/paddle/operators/clip_by_norm_op.cu b/paddle/operators/clip_by_norm_op.cu index 2593a24ebb..acd7543823 100644 --- a/paddle/operators/clip_by_norm_op.cu +++ b/paddle/operators/clip_by_norm_op.cu @@ -15,5 +15,6 @@ #include "paddle/operators/clip_by_norm_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( - clip_by_norm, ops::ClipByNormKernel); +REGISTER_OP_CUDA_KERNEL( + clip_by_norm, + ops::ClipByNormKernel); diff --git a/paddle/operators/clip_by_norm_op.h b/paddle/operators/clip_by_norm_op.h index b26476cae9..d8db1566b0 100644 --- a/paddle/operators/clip_by_norm_op.h +++ b/paddle/operators/clip_by_norm_op.h @@ -26,7 +26,7 @@ template using EigenVector = framework::EigenVector; -template +template class ClipByNormKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -38,7 +38,8 @@ class ClipByNormKernel : public framework::OpKernel { auto x = EigenVector::Flatten(*input); auto out = EigenVector::Flatten(*output); auto x_norm = x.square().sum().sqrt(); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); auto temp = (x_norm <= max_norm).template cast().eval(); auto scaling = temp + (static_cast(1) - temp) * max_norm / x_norm; diff --git a/paddle/operators/clip_op.cc b/paddle/operators/clip_op.cc index 4ddf24dea3..6092212de4 100644 --- a/paddle/operators/clip_op.cc +++ b/paddle/operators/clip_op.cc @@ -83,7 +83,7 @@ class ClipOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(clip, ops::ClipOp, ops::ClipOpMaker, clip_grad, ops::ClipOpGrad); -REGISTER_OP_CPU_KERNEL(clip, - ops::ClipKernel); -REGISTER_OP_CPU_KERNEL(clip_grad, - ops::ClipGradKernel); +REGISTER_OP_CPU_KERNEL( + clip, ops::ClipKernel); +REGISTER_OP_CPU_KERNEL( + clip_grad, ops::ClipGradKernel); diff --git a/paddle/operators/clip_op.cu b/paddle/operators/clip_op.cu index ca9701298f..bb7dcc671a 100644 --- a/paddle/operators/clip_op.cu +++ b/paddle/operators/clip_op.cu @@ -15,7 +15,7 @@ #include "paddle/operators/clip_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(clip, - ops::ClipKernel); -REGISTER_OP_GPU_KERNEL(clip_grad, - ops::ClipGradKernel); +REGISTER_OP_CUDA_KERNEL( + clip, ops::ClipKernel); +REGISTER_OP_CUDA_KERNEL( + clip_grad, ops::ClipGradKernel); diff --git a/paddle/operators/clip_op.h b/paddle/operators/clip_op.h index ac702e9935..0c40797410 100644 --- a/paddle/operators/clip_op.h +++ b/paddle/operators/clip_op.h @@ -55,7 +55,7 @@ class ClipGradFunctor { T max_; }; -template +template class ClipKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -66,13 +66,13 @@ class ClipKernel : public framework::OpKernel { T* out_data = out->mutable_data(context.GetPlace()); const T* x_data = x->data(); int64_t numel = x->numel(); - Transform trans; - trans(context.device_context(), x_data, x_data + numel, out_data, - ClipFunctor(min, max)); + Transform trans; + trans(context.template device_context(), x_data, + x_data + numel, out_data, ClipFunctor(min, max)); } }; -template +template class ClipGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -86,9 +86,9 @@ class ClipGradKernel : public framework::OpKernel { auto* d_x_data = d_x->mutable_data(context.GetPlace()); const T* d_out_data = d_out->data(); const T* x_data = x->data(); - Transform trans; - trans(context.device_context(), d_out_data, d_out_data + numel, x_data, - d_x_data, ClipGradFunctor(min, max)); + Transform trans; + trans(context.template device_context(), d_out_data, + d_out_data + numel, x_data, d_x_data, ClipGradFunctor(min, max)); } } }; diff --git a/paddle/operators/compare_op.cu b/paddle/operators/compare_op.cu index 6ac8c124b9..596a878bcf 100644 --- a/paddle/operators/compare_op.cu +++ b/paddle/operators/compare_op.cu @@ -14,10 +14,10 @@ #include "paddle/operators/compare_op.h" -REGISTER_LOGICAL_KERNEL(less_than, GPU, paddle::operators::LessThanFunctor); -REGISTER_LOGICAL_KERNEL(less_equal, GPU, paddle::operators::LessEqualFunctor); -REGISTER_LOGICAL_KERNEL(greater_than, GPU, +REGISTER_LOGICAL_KERNEL(less_than, CUDA, paddle::operators::LessThanFunctor); +REGISTER_LOGICAL_KERNEL(less_equal, CUDA, paddle::operators::LessEqualFunctor); +REGISTER_LOGICAL_KERNEL(greater_than, CUDA, paddle::operators::GreaterThanFunctor); -REGISTER_LOGICAL_KERNEL(greater_equal, GPU, +REGISTER_LOGICAL_KERNEL(greater_equal, CUDA, paddle::operators::GreaterEqualFunctor); -REGISTER_LOGICAL_KERNEL(equal, GPU, paddle::operators::EqualFunctor); +REGISTER_LOGICAL_KERNEL(equal, CUDA, paddle::operators::EqualFunctor); diff --git a/paddle/operators/compare_op.h b/paddle/operators/compare_op.h index afdf3ab3e0..a56536e155 100644 --- a/paddle/operators/compare_op.h +++ b/paddle/operators/compare_op.h @@ -59,7 +59,7 @@ struct EqualFunctor { } }; -template +template class CompareOpKernel : public framework::OpKernel { public: @@ -69,24 +69,23 @@ class CompareOpKernel auto* y = context.Input("Y"); auto* out = context.Output("Out"); Functor binary_func; - platform::Transform trans; - trans(context.device_context(), x->data(), x->data() + x->numel(), - y->data(), out->mutable_data(context.GetPlace()), - binary_func); + platform::Transform trans; + trans(context.template device_context(), x->data(), + x->data() + x->numel(), y->data(), + out->mutable_data(context.GetPlace()), binary_func); } }; } // namespace operators } // namespace paddle -#define REGISTER_LOGICAL_KERNEL(op_type, dev, functor) \ - REGISTER_OP_##dev##_KERNEL( \ - op_type, \ - ::paddle::operators::CompareOpKernel<::paddle::platform::dev##Place, \ - functor>, \ - ::paddle::operators::CompareOpKernel<::paddle::platform::dev##Place, \ - functor>, \ - ::paddle::operators::CompareOpKernel<::paddle::platform::dev##Place, \ - functor>, \ - ::paddle::operators::CompareOpKernel<::paddle::platform::dev##Place, \ - functor>); +#define REGISTER_LOGICAL_KERNEL(op_type, dev, functor) \ + REGISTER_OP_##dev##_KERNEL( \ + op_type, ::paddle::operators::CompareOpKernel< \ + ::paddle::platform::dev##DeviceContext, functor>, \ + ::paddle::operators::CompareOpKernel< \ + ::paddle::platform::dev##DeviceContext, functor>, \ + ::paddle::operators::CompareOpKernel< \ + ::paddle::platform::dev##DeviceContext, functor>, \ + ::paddle::operators::CompareOpKernel< \ + ::paddle::platform::dev##DeviceContext, functor>); diff --git a/paddle/operators/concat_op.cu.cc b/paddle/operators/concat_op.cu.cc index ede832ddcd..7b46452d3d 100644 --- a/paddle/operators/concat_op.cu.cc +++ b/paddle/operators/concat_op.cu.cc @@ -14,7 +14,8 @@ limitations under the License. */ #include "paddle/operators/concat_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(concat, - ops::ConcatKernel); -REGISTER_OP_GPU_KERNEL( - concat_grad, ops::ConcatGradKernel); +REGISTER_OP_CUDA_KERNEL( + concat, ops::ConcatKernel); +REGISTER_OP_CUDA_KERNEL( + concat_grad, + ops::ConcatGradKernel); diff --git a/paddle/operators/concat_op.h b/paddle/operators/concat_op.h index c113f19fb5..de4011585a 100644 --- a/paddle/operators/concat_op.h +++ b/paddle/operators/concat_op.h @@ -21,7 +21,7 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class ConcatKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -43,7 +43,7 @@ class ConcatKernel : public framework::OpKernel { } }; -template +template class ConcatGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { diff --git a/paddle/operators/conv_cudnn_op.cc b/paddle/operators/conv_cudnn_op.cc index 0dd8c13b2a..008bf01885 100644 --- a/paddle/operators/conv_cudnn_op.cc +++ b/paddle/operators/conv_cudnn_op.cc @@ -57,18 +57,20 @@ REGISTER_OP(conv2d_cudnn, ops::ConvOp, ops::CudnnConv2DOpMaker, REGISTER_OP(conv3d_cudnn, ops::ConvOp, ops::CudnnConv3DOpMaker, conv3d_cudnn_grad, ops::ConvOpGrad); -REGISTER_OP_CPU_KERNEL(conv2d_cudnn, - ops::GemmConvKernel, - ops::GemmConvKernel); +REGISTER_OP_CPU_KERNEL( + conv2d_cudnn, + ops::GemmConvKernel, + ops::GemmConvKernel); REGISTER_OP_CPU_KERNEL( conv2d_cudnn_grad, - ops::GemmConvGradKernel, - ops::GemmConvGradKernel); + ops::GemmConvGradKernel, + ops::GemmConvGradKernel); -REGISTER_OP_CPU_KERNEL(conv3d_cudnn, - ops::GemmConvKernel, - ops::GemmConvKernel); +REGISTER_OP_CPU_KERNEL( + conv3d_cudnn, + ops::GemmConvKernel, + ops::GemmConvKernel); REGISTER_OP_CPU_KERNEL( conv3d_cudnn_grad, - ops::GemmConvGradKernel, - ops::GemmConvGradKernel); + ops::GemmConvGradKernel, + ops::GemmConvGradKernel); diff --git a/paddle/operators/conv_cudnn_op.cu.cc b/paddle/operators/conv_cudnn_op.cu.cc index bc265dcc4f..3da0a9001a 100644 --- a/paddle/operators/conv_cudnn_op.cu.cc +++ b/paddle/operators/conv_cudnn_op.cu.cc @@ -118,7 +118,8 @@ class CudnnConvOpKernel : public framework::OpKernel { } // ------------------- cudnn conv algorithm --------------------- cudnnConvolutionFwdAlgo_t algo; - auto handle = ctx.cuda_device_context().cudnn_handle(); + auto& dev_ctx = ctx.template device_context(); + auto handle = dev_ctx.cudnn_handle(); PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionForwardAlgorithm( handle, cudnn_input_desc, cudnn_filter_desc, cudnn_conv_desc, @@ -238,7 +239,8 @@ class CudnnConvGradOpKernel : public framework::OpKernel { workspace_size_limit = user_workspace_size * 1024 * 1024; } - auto handle = ctx.cuda_device_context().cudnn_handle(); + auto& dev_ctx = ctx.template device_context(); + auto handle = dev_ctx.cudnn_handle(); if (input_grad) { PADDLE_ENFORCE( platform::dynload::cudnnGetConvolutionBackwardDataAlgorithm( @@ -313,16 +315,16 @@ class CudnnConvGradOpKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -REGISTER_OP_GPU_KERNEL(conv2d_cudnn, - paddle::operators::CudnnConvOpKernel, - paddle::operators::CudnnConvOpKernel); -REGISTER_OP_GPU_KERNEL(conv2d_cudnn_grad, - paddle::operators::CudnnConvGradOpKernel, - paddle::operators::CudnnConvGradOpKernel); - -REGISTER_OP_GPU_KERNEL(conv3d_cudnn, - paddle::operators::CudnnConvOpKernel, - paddle::operators::CudnnConvOpKernel); -REGISTER_OP_GPU_KERNEL(conv3d_cudnn_grad, - paddle::operators::CudnnConvGradOpKernel, - paddle::operators::CudnnConvGradOpKernel); +REGISTER_OP_CUDA_KERNEL(conv2d_cudnn, + paddle::operators::CudnnConvOpKernel, + paddle::operators::CudnnConvOpKernel); +REGISTER_OP_CUDA_KERNEL(conv2d_cudnn_grad, + paddle::operators::CudnnConvGradOpKernel, + paddle::operators::CudnnConvGradOpKernel); + +REGISTER_OP_CUDA_KERNEL(conv3d_cudnn, + paddle::operators::CudnnConvOpKernel, + paddle::operators::CudnnConvOpKernel); +REGISTER_OP_CUDA_KERNEL(conv3d_cudnn_grad, + paddle::operators::CudnnConvGradOpKernel, + paddle::operators::CudnnConvGradOpKernel); diff --git a/paddle/operators/conv_op.cc b/paddle/operators/conv_op.cc index 462e6d9cbc..7ef805fd44 100644 --- a/paddle/operators/conv_op.cc +++ b/paddle/operators/conv_op.cc @@ -235,16 +235,18 @@ namespace ops = paddle::operators; REGISTER_OP(conv3d, ops::ConvOp, ops::Conv3DOpMaker, conv3d_grad, ops::ConvOpGrad); -REGISTER_OP_CPU_KERNEL(conv2d, - ops::GemmConvKernel, - ops::GemmConvKernel); REGISTER_OP_CPU_KERNEL( - conv2d_grad, ops::GemmConvGradKernel, - ops::GemmConvGradKernel); + conv2d, ops::GemmConvKernel, + ops::GemmConvKernel); +REGISTER_OP_CPU_KERNEL( + conv2d_grad, + ops::GemmConvGradKernel, + ops::GemmConvGradKernel); -REGISTER_OP_CPU_KERNEL(conv3d, - ops::GemmConvKernel, - ops::GemmConvKernel); REGISTER_OP_CPU_KERNEL( - conv3d_grad, ops::GemmConvGradKernel, - ops::GemmConvGradKernel); + conv3d, ops::GemmConvKernel, + ops::GemmConvKernel); +REGISTER_OP_CPU_KERNEL( + conv3d_grad, + ops::GemmConvGradKernel, + ops::GemmConvGradKernel); diff --git a/paddle/operators/conv_op.cu.cc b/paddle/operators/conv_op.cu.cc index 546451234a..38615a8bef 100644 --- a/paddle/operators/conv_op.cu.cc +++ b/paddle/operators/conv_op.cu.cc @@ -16,16 +16,18 @@ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(conv2d, - ops::GemmConvKernel, - ops::GemmConvKernel); -REGISTER_OP_GPU_KERNEL( - conv2d_grad, ops::GemmConvGradKernel, - ops::GemmConvGradKernel); +REGISTER_OP_CUDA_KERNEL( + conv2d, ops::GemmConvKernel, + ops::GemmConvKernel); +REGISTER_OP_CUDA_KERNEL( + conv2d_grad, + ops::GemmConvGradKernel, + ops::GemmConvGradKernel); -REGISTER_OP_GPU_KERNEL(conv3d, - ops::GemmConvKernel, - ops::GemmConvKernel); -REGISTER_OP_GPU_KERNEL( - conv3d_grad, ops::GemmConvGradKernel, - ops::GemmConvGradKernel); +REGISTER_OP_CUDA_KERNEL( + conv3d, ops::GemmConvKernel, + ops::GemmConvKernel); +REGISTER_OP_CUDA_KERNEL( + conv3d_grad, + ops::GemmConvGradKernel, + ops::GemmConvGradKernel); diff --git a/paddle/operators/conv_op.h b/paddle/operators/conv_op.h index 09bff0a68d..749258183b 100644 --- a/paddle/operators/conv_op.h +++ b/paddle/operators/conv_op.h @@ -72,7 +72,7 @@ class ConvOpGrad : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext* ctx) const override; }; -template +template class GemmConvKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -141,9 +141,10 @@ class GemmConvKernel : public framework::OpKernel { int in_step = static_cast(input->dims()[1]) / groups; int out_step = static_cast(output->dims()[1]) / groups; - math::Vol2ColFunctor vol2col; - math::Im2ColFunctor im2col; + math::Vol2ColFunctor vol2col; + math::Im2ColFunctor im2col; + auto& dev_ctx = context.template device_context(); for (int i = 0; i < batch_size; i++) { Tensor in_batch = input->Slice(i, i + 1).Resize(input_shape); Tensor out_batch = output->Slice(i, i + 1).Resize(output_matrix_shape); @@ -157,27 +158,26 @@ class GemmConvKernel : public framework::OpKernel { col_matrix.Resize(col_matrix_shape); } else if (data_dim == 2U) { // im2col - im2col(context.device_context(), in_slice, dilations, strides, + im2col(dev_ctx, in_slice, dilations, strides, std::vector{paddings[0], paddings[1], paddings[0], paddings[1]}, &col); } else if (data_dim == 3U) { // vol2col - vol2col(context.device_context(), in_slice, dilations, strides, - paddings, &col); + vol2col(dev_ctx, in_slice, dilations, strides, paddings, &col); } // gemm Tensor out_slice = out_batch.Slice(g * out_step, (g + 1) * out_step); Tensor filter_slice = filter.Slice(g * out_step, (g + 1) * out_step); - math::matmul(context.device_context(), filter_slice, false, - col_matrix, false, T(1.0), &out_slice, T(0.0)); + math::matmul(dev_ctx, filter_slice, false, col_matrix, + false, T(1.0), &out_slice, T(0.0)); } } } }; -template +template class GemmConvGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -256,14 +256,15 @@ class GemmConvGradKernel : public framework::OpKernel { col_matrix.Resize(col_matrix_shape); } - math::SetConstant set_zero; + math::SetConstant set_zero; + auto& dev_ctx = context.template device_context(); if (input_grad) { input_grad->mutable_data(context.GetPlace()); - set_zero(context.device_context(), input_grad, static_cast(0)); + set_zero(dev_ctx, input_grad, static_cast(0)); - math::Col2VolFunctor col2vol; - math::Col2ImFunctor col2im; + math::Col2VolFunctor col2vol; + math::Col2ImFunctor col2im; for (int i = 0; i < batch_size; i++) { Tensor out_grad_batch = @@ -282,18 +283,17 @@ class GemmConvGradKernel : public framework::OpKernel { col_matrix.ShareDataWith(in_grad_slice); col_matrix.Resize(col_matrix_shape); } - math::matmul(context.device_context(), filter_slice, true, - out_grad_slice, false, T(1.0), &col_matrix, - T(0.0)); + math::matmul(dev_ctx, filter_slice, true, + out_grad_slice, false, T(1.0), + &col_matrix, T(0.0)); if (is_expand && data_dim == 2U) { - col2im(context.device_context(), col, dilations, strides, + col2im(dev_ctx, col, dilations, strides, std::vector{paddings[0], paddings[1], paddings[0], paddings[1]}, &in_grad_slice); } else if (is_expand && data_dim == 3U) { - col2vol(context.device_context(), col, dilations, strides, paddings, - &in_grad_slice); + col2vol(dev_ctx, col, dilations, strides, paddings, &in_grad_slice); } } } @@ -303,9 +303,9 @@ class GemmConvGradKernel : public framework::OpKernel { filter_grad->mutable_data(context.GetPlace()); Tensor filter_grad_ = *filter_grad; filter_grad_.Resize(filter_matrix_shape); - set_zero(context.device_context(), filter_grad, static_cast(0)); - math::Im2ColFunctor im2col; - math::Vol2ColFunctor vol2col; + set_zero(dev_ctx, filter_grad, static_cast(0)); + math::Im2ColFunctor im2col; + math::Vol2ColFunctor vol2col; for (int i = 0; i < batch_size; i++) { Tensor out_grad_batch = output_grad->Slice(i, i + 1).Resize(output_matrix_shape); @@ -321,21 +321,20 @@ class GemmConvGradKernel : public framework::OpKernel { col_matrix.ShareDataWith(col); col_matrix.Resize(col_matrix_shape); } else if (data_dim == 2U) { - im2col(context.device_context(), in_slice, dilations, strides, + im2col(dev_ctx, in_slice, dilations, strides, std::vector{paddings[0], paddings[1], paddings[0], paddings[1]}, &col); } else if (data_dim == 3U) { - vol2col(context.device_context(), in_slice, dilations, strides, - paddings, &col); + vol2col(dev_ctx, in_slice, dilations, strides, paddings, &col); } // gemm Tensor filter_grad_slice = filter_grad_.Slice(g * out_step, (g + 1) * out_step); - math::matmul(context.device_context(), out_grad_slice, - false, col_matrix, true, T(1.0), - &filter_grad_slice, T(1.0)); + math::matmul(dev_ctx, out_grad_slice, false, + col_matrix, true, T(1.0), + &filter_grad_slice, T(1.0)); } } } diff --git a/paddle/operators/conv_shift_op.cu b/paddle/operators/conv_shift_op.cu index 95e13c38a8..f7ca82ce26 100644 --- a/paddle/operators/conv_shift_op.cu +++ b/paddle/operators/conv_shift_op.cu @@ -111,7 +111,8 @@ __global__ void ConvShiftDy(const T *x, const T *dout, int x_width, int y_width, } // namespace template -class ConvShiftKernel : public framework::OpKernel { +class ConvShiftKernel + : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { const Tensor *X = context.Input("X"); @@ -132,7 +133,8 @@ class ConvShiftKernel : public framework::OpKernel { dim3 grid_dim(num_x_blocks, batch_size); - auto stream = context.cuda_device_context().stream(); + auto stream = + context.template device_context().stream(); ConvShiftForward<<>>( x_data, y_data, x_width, y_width, y_half_width, batch_size, out_data); @@ -140,7 +142,7 @@ class ConvShiftKernel : public framework::OpKernel { }; template -class ConvShiftGradKernel +class ConvShiftGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { @@ -159,8 +161,9 @@ class ConvShiftGradKernel int y_width = Y->dims()[1]; int y_half_width = (y_width - 1) / 2; - auto &device_ctx = context.cuda_device_context(); - math::SetConstant zero; + auto &device_ctx = + context.template device_context(); + math::SetConstant zero; const int x_per_block = 256; int num_x_blocks = DivUp(x_width, x_per_block); @@ -186,8 +189,9 @@ class ConvShiftGradKernel } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(conv_shift, - ops::ConvShiftKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( + conv_shift, + ops::ConvShiftKernel); +REGISTER_OP_CUDA_KERNEL( conv_shift_grad, - ops::ConvShiftGradKernel); + ops::ConvShiftGradKernel); diff --git a/paddle/operators/conv_shift_op.h b/paddle/operators/conv_shift_op.h index 5a160b0f16..1a70b38a0d 100644 --- a/paddle/operators/conv_shift_op.h +++ b/paddle/operators/conv_shift_op.h @@ -18,13 +18,13 @@ namespace paddle { namespace operators { -template +template class ConvShiftKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override; }; -template +template class ConvShiftGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override; diff --git a/paddle/operators/conv_transpose_cudnn_op.cc b/paddle/operators/conv_transpose_cudnn_op.cc index 0192178ce3..4cb6a2ccff 100644 --- a/paddle/operators/conv_transpose_cudnn_op.cc +++ b/paddle/operators/conv_transpose_cudnn_op.cc @@ -61,12 +61,13 @@ REGISTER_OP(conv2d_transpose_cudnn, ops::ConvTransposeOp, REGISTER_OP_CPU_KERNEL( conv2d_transpose_cudnn, - ops::GemmConvTransposeKernel, - ops::GemmConvTransposeKernel); + ops::GemmConvTransposeKernel, + ops::GemmConvTransposeKernel); REGISTER_OP_CPU_KERNEL( conv2d_transpose_cudnn_grad, - ops::GemmConvTransposeGradKernel, - ops::GemmConvTransposeGradKernel); + ops::GemmConvTransposeGradKernel, + ops::GemmConvTransposeGradKernel); REGISTER_OP(conv3d_transpose_cudnn, ops::ConvTransposeOp, ops::CudnnConv3DTransposeOpMaker, conv3d_transpose_cudnn_grad, @@ -74,9 +75,10 @@ REGISTER_OP(conv3d_transpose_cudnn, ops::ConvTransposeOp, REGISTER_OP_CPU_KERNEL( conv3d_transpose_cudnn, - ops::GemmConvTransposeKernel, - ops::GemmConvTransposeKernel); + ops::GemmConvTransposeKernel, + ops::GemmConvTransposeKernel); REGISTER_OP_CPU_KERNEL( conv3d_transpose_cudnn_grad, - ops::GemmConvTransposeGradKernel, - ops::GemmConvTransposeGradKernel); + ops::GemmConvTransposeGradKernel, + ops::GemmConvTransposeGradKernel); diff --git a/paddle/operators/conv_transpose_cudnn_op.cu.cc b/paddle/operators/conv_transpose_cudnn_op.cu.cc index 494904fe52..f0297f6c40 100644 --- a/paddle/operators/conv_transpose_cudnn_op.cu.cc +++ b/paddle/operators/conv_transpose_cudnn_op.cu.cc @@ -83,7 +83,8 @@ class CudnnConvTransposeOpKernel : public framework::OpKernel { } // ------------------- cudnn conv algorithm --------------------- cudnnConvolutionBwdDataAlgo_t algo; - auto handle = ctx.cuda_device_context().cudnn_handle(); + auto& dev_ctx = ctx.template device_context(); + auto handle = dev_ctx.cudnn_handle(); // Get the algorithm PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionBackwardDataAlgorithm( handle, cudnn_filter_desc, cudnn_input_desc, cudnn_conv_desc, @@ -165,7 +166,8 @@ class CudnnConvTransposeGradOpKernel : public framework::OpKernel { workspace_size_limit = user_workspace_size * 1024 * 1024; } - auto handle = ctx.cuda_device_context().cudnn_handle(); + auto& dev_ctx = ctx.template device_context(); + auto handle = dev_ctx.cudnn_handle(); if (input_grad) { // choose backward algorithm for data PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionForwardAlgorithm( @@ -234,16 +236,16 @@ class CudnnConvTransposeGradOpKernel : public framework::OpKernel { namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(conv2d_transpose_cudnn, - ops::CudnnConvTransposeOpKernel, - ops::CudnnConvTransposeOpKernel); -REGISTER_OP_GPU_KERNEL(conv2d_transpose_cudnn_grad, - ops::CudnnConvTransposeGradOpKernel, - ops::CudnnConvTransposeGradOpKernel); - -REGISTER_OP_GPU_KERNEL(conv3d_transpose_cudnn, - ops::CudnnConvTransposeOpKernel, - ops::CudnnConvTransposeOpKernel); -REGISTER_OP_GPU_KERNEL(conv3d_transpose_cudnn_grad, - ops::CudnnConvTransposeGradOpKernel, - ops::CudnnConvTransposeGradOpKernel); +REGISTER_OP_CUDA_KERNEL(conv2d_transpose_cudnn, + ops::CudnnConvTransposeOpKernel, + ops::CudnnConvTransposeOpKernel); +REGISTER_OP_CUDA_KERNEL(conv2d_transpose_cudnn_grad, + ops::CudnnConvTransposeGradOpKernel, + ops::CudnnConvTransposeGradOpKernel); + +REGISTER_OP_CUDA_KERNEL(conv3d_transpose_cudnn, + ops::CudnnConvTransposeOpKernel, + ops::CudnnConvTransposeOpKernel); +REGISTER_OP_CUDA_KERNEL(conv3d_transpose_cudnn_grad, + ops::CudnnConvTransposeGradOpKernel, + ops::CudnnConvTransposeGradOpKernel); diff --git a/paddle/operators/conv_transpose_op.cc b/paddle/operators/conv_transpose_op.cc index 678b192dea..ca063e94bb 100644 --- a/paddle/operators/conv_transpose_op.cc +++ b/paddle/operators/conv_transpose_op.cc @@ -197,21 +197,23 @@ REGISTER_OP(conv2d_transpose, ops::ConvTransposeOp, ops::Conv2DTransposeOpMaker, REGISTER_OP_CPU_KERNEL( conv2d_transpose, - ops::GemmConvTransposeKernel, - ops::GemmConvTransposeKernel); + ops::GemmConvTransposeKernel, + ops::GemmConvTransposeKernel); REGISTER_OP_CPU_KERNEL( conv2d_transpose_grad, - ops::GemmConvTransposeGradKernel, - ops::GemmConvTransposeGradKernel); + ops::GemmConvTransposeGradKernel, + ops::GemmConvTransposeGradKernel); REGISTER_OP(conv3d_transpose, ops::ConvTransposeOp, ops::Conv3DTransposeOpMaker, conv3d_transpose_grad, ops::ConvTransposeOpGrad); REGISTER_OP_CPU_KERNEL( conv3d_transpose, - ops::GemmConvTransposeKernel, - ops::GemmConvTransposeKernel); + ops::GemmConvTransposeKernel, + ops::GemmConvTransposeKernel); REGISTER_OP_CPU_KERNEL( conv3d_transpose_grad, - ops::GemmConvTransposeGradKernel, - ops::GemmConvTransposeGradKernel); + ops::GemmConvTransposeGradKernel, + ops::GemmConvTransposeGradKernel); diff --git a/paddle/operators/conv_transpose_op.cu.cc b/paddle/operators/conv_transpose_op.cu.cc index 4165eb0c7b..b91ebd7922 100644 --- a/paddle/operators/conv_transpose_op.cu.cc +++ b/paddle/operators/conv_transpose_op.cu.cc @@ -16,20 +16,24 @@ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( conv2d_transpose, - ops::GemmConvTransposeKernel, - ops::GemmConvTransposeKernel); -REGISTER_OP_GPU_KERNEL( + ops::GemmConvTransposeKernel, + ops::GemmConvTransposeKernel); +REGISTER_OP_CUDA_KERNEL( conv2d_transpose_grad, - ops::GemmConvTransposeGradKernel, - ops::GemmConvTransposeGradKernel); + ops::GemmConvTransposeGradKernel, + ops::GemmConvTransposeGradKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( conv3d_transpose, - ops::GemmConvTransposeKernel, - ops::GemmConvTransposeKernel); -REGISTER_OP_GPU_KERNEL( + ops::GemmConvTransposeKernel, + ops::GemmConvTransposeKernel); +REGISTER_OP_CUDA_KERNEL( conv3d_transpose_grad, - ops::GemmConvTransposeGradKernel, - ops::GemmConvTransposeGradKernel); + ops::GemmConvTransposeGradKernel, + ops::GemmConvTransposeGradKernel); diff --git a/paddle/operators/conv_transpose_op.h b/paddle/operators/conv_transpose_op.h index 1cacb770e6..80600b5361 100644 --- a/paddle/operators/conv_transpose_op.h +++ b/paddle/operators/conv_transpose_op.h @@ -52,7 +52,7 @@ class ConvTransposeOpGrad : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext* ctx) const override; }; -template +template class GemmConvTransposeKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -109,11 +109,12 @@ class GemmConvTransposeKernel : public framework::OpKernel { filter.Resize(filter_matrix_shape); output->mutable_data(context.GetPlace()); - math::SetConstant set_zero; - set_zero(context.device_context(), output, static_cast(0)); + math::SetConstant set_zero; + auto& dev_ctx = context.template device_context(); + set_zero(dev_ctx, output, static_cast(0)); - math::Col2ImFunctor col2im; - math::Col2VolFunctor col2vol; + math::Col2ImFunctor col2im; + math::Col2VolFunctor col2vol; std::vector dilations({1, 1, 1}); // convolution transpose: gemm + col2im or col2vol (similar to conv-backward @@ -127,29 +128,27 @@ class GemmConvTransposeKernel : public framework::OpKernel { // col_matrix = filter * input_batch // of shape (c * k_h * k_w, h * w) or (c * k_d * k_h * k_w, d * h * w) - math::matmul(context.device_context(), filter, true, - input_batch, false, static_cast(1.0), - &col_matrix, static_cast(0.0)); + math::matmul(dev_ctx, filter, true, input_batch, false, + static_cast(1.0), &col_matrix, + static_cast(0.0)); if (data_dim == 2U) { // col2im: col_matrix -> dy // from (c * k_h * k_w, h * w) to (c, o_h, o_w) - col2im(context.device_context(), col, - std::vector{dilations[0], dilations[1]}, strides, - std::vector{paddings[0], paddings[1], paddings[0], - paddings[1]}, + col2im(dev_ctx, col, std::vector{dilations[0], dilations[1]}, + strides, std::vector{paddings[0], paddings[1], paddings[0], + paddings[1]}, &output_batch); } else if (data_dim == 3U) { // col2vol: col_matrix -> dy // from (c * k_d * k_h * k_w, d * h * w) to (c, o_d, o_h, o_w) - col2vol(context.device_context(), col, dilations, strides, paddings, - &output_batch); + col2vol(dev_ctx, col, dilations, strides, paddings, &output_batch); } } } }; -template +template class GemmConvTransposeGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -206,6 +205,7 @@ class GemmConvTransposeGradKernel : public framework::OpKernel { // convolution transpose grad on input: // im2col + gemm (similar to conv-forward) // input need to compute gradient + auto& dev_ctx = context.template device_context(); if (input_grad || filter_grad) { Tensor col; col.mutable_data(col_shape, context.GetPlace()); @@ -217,19 +217,19 @@ class GemmConvTransposeGradKernel : public framework::OpKernel { col_matrix.Resize(col_matrix_shape); Tensor filter_grad_; - math::SetConstant set_zero; + math::SetConstant set_zero; - math::Im2ColFunctor im2col; - math::Vol2ColFunctor vol2col; + math::Im2ColFunctor im2col; + math::Vol2ColFunctor vol2col; std::vector dilations({1, 1, 1}); if (input_grad) { input_grad->mutable_data(context.GetPlace()); - set_zero(context.device_context(), input_grad, static_cast(0)); + set_zero(dev_ctx, input_grad, static_cast(0)); } if (filter_grad) { // filter size (m, c, k_h, k_w) filter_grad->mutable_data(context.GetPlace()); - set_zero(context.device_context(), filter_grad, static_cast(0)); + set_zero(dev_ctx, filter_grad, static_cast(0)); filter_grad_ = *filter_grad; filter_grad_.Resize(filter_matrix_shape); } @@ -242,7 +242,7 @@ class GemmConvTransposeGradKernel : public framework::OpKernel { if (data_dim == 2U) { // im2col: dy -> col matrix // from (c, o_h, o_w) to (c * k_h * k_w, h * w) - im2col(context.device_context(), output_grad_batch, + im2col(dev_ctx, output_grad_batch, std::vector{dilations[0], dilations[1]}, strides, std::vector{paddings[0], paddings[1], paddings[0], paddings[1]}, @@ -250,8 +250,8 @@ class GemmConvTransposeGradKernel : public framework::OpKernel { } else if (data_dim == 3U) { // vol2col: dy -> col_matrix // from (c, o_d, o_h, o_w) to (c * k_d * k_h * k_w, d * h * w) - vol2col(context.device_context(), output_grad_batch, dilations, - strides, paddings, &col); + vol2col(dev_ctx, output_grad_batch, dilations, strides, paddings, + &col); } if (input_grad) { @@ -263,9 +263,9 @@ class GemmConvTransposeGradKernel : public framework::OpKernel { // or // (m, c * k_d * k_h * k_w) * (c * k_d * k_h * k_w, d * h * w) -> (m, // d, h, w) - math::matmul(context.device_context(), filter, false, - col_matrix, false, static_cast(1.0), - &input_grad_batch, static_cast(0.0)); + math::matmul( + dev_ctx, filter, false, col_matrix, false, static_cast(1.0), + &input_grad_batch, static_cast(0.0)); } if (filter_grad) { // input batch @@ -275,9 +275,9 @@ class GemmConvTransposeGradKernel : public framework::OpKernel { // or // (m, d * h * w) * (d * h * w, c * k_d * k_h * k_w) -> (m, c * k_d * // k_h * k_w) - math::matmul(context.device_context(), in_batch, false, - col_matrix, true, static_cast(1.0), - &filter_grad_, static_cast(1.0)); + math::matmul(dev_ctx, in_batch, false, col_matrix, + true, static_cast(1.0), + &filter_grad_, static_cast(1.0)); } } } diff --git a/paddle/operators/cos_sim_op.cc b/paddle/operators/cos_sim_op.cc index 312264ccd4..440c427cba 100644 --- a/paddle/operators/cos_sim_op.cc +++ b/paddle/operators/cos_sim_op.cc @@ -155,7 +155,8 @@ class CosSimOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(cos_sim, ops::CosSimOp, ops::CosSimOpMaker, cos_sim_grad, ops::CosSimOpGrad); -REGISTER_OP_CPU_KERNEL(cos_sim, - ops::CosSimKernel); REGISTER_OP_CPU_KERNEL( - cos_sim_grad, ops::CosSimGradKernel); + cos_sim, ops::CosSimKernel); +REGISTER_OP_CPU_KERNEL( + cos_sim_grad, + ops::CosSimGradKernel); diff --git a/paddle/operators/cos_sim_op.cu b/paddle/operators/cos_sim_op.cu index 0cb8fd26de..1cb01f5945 100644 --- a/paddle/operators/cos_sim_op.cu +++ b/paddle/operators/cos_sim_op.cu @@ -16,7 +16,8 @@ #include "paddle/operators/cos_sim_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(cos_sim, - ops::CosSimKernel); -REGISTER_OP_GPU_KERNEL( - cos_sim_grad, ops::CosSimGradKernel); +REGISTER_OP_CUDA_KERNEL( + cos_sim, ops::CosSimKernel); +REGISTER_OP_CUDA_KERNEL( + cos_sim_grad, + ops::CosSimGradKernel); diff --git a/paddle/operators/cos_sim_op.h b/paddle/operators/cos_sim_op.h index 62a4e484ec..fecb5a79b2 100644 --- a/paddle/operators/cos_sim_op.h +++ b/paddle/operators/cos_sim_op.h @@ -27,7 +27,7 @@ template using EigenVector = framework::EigenVector; -template +template class CosSimKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -51,7 +51,8 @@ class CosSimKernel : public framework::OpKernel { auto y_norm = EigenVector::Flatten(*out_y_norm); // compute - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); auto row_along = Eigen::array({{1}}); x_norm.device(place) = x.square().sum(row_along).sqrt(); y_norm.device(place) = y.square().sum(row_along).sqrt(); @@ -66,7 +67,7 @@ class CosSimKernel : public framework::OpKernel { } }; -template +template class CosSimGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -96,7 +97,8 @@ class CosSimGradKernel : public framework::OpKernel { auto z_bcast = z.broadcast(bcast_cols); auto dz_bcast = dz.broadcast(bcast_cols); auto x_snorm_bcast = x_norm.square().eval().broadcast(bcast_cols); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); if (rows_x == rows_y) { auto y_snorm_bcast = y_norm.square().eval().broadcast(bcast_cols); auto norm_prod_bcast = (x_norm * y_norm).eval().broadcast(bcast_cols); diff --git a/paddle/operators/crf_decoding_op.cc b/paddle/operators/crf_decoding_op.cc index 291b23ed1b..1ce189fa6e 100644 --- a/paddle/operators/crf_decoding_op.cc +++ b/paddle/operators/crf_decoding_op.cc @@ -135,5 +135,6 @@ namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(crf_decoding, ops::CRFDecodingOp, ops::CRFDecodingOpMaker); REGISTER_OP_CPU_KERNEL( - crf_decoding, ops::CRFDecodingOpKernel, - ops::CRFDecodingOpKernel); + crf_decoding, + ops::CRFDecodingOpKernel, + ops::CRFDecodingOpKernel); diff --git a/paddle/operators/crf_decoding_op.h b/paddle/operators/crf_decoding_op.h index 57b5e21b3a..f6827b7b11 100644 --- a/paddle/operators/crf_decoding_op.h +++ b/paddle/operators/crf_decoding_op.h @@ -24,7 +24,7 @@ using framework::LoDTensor; using framework::LoD; using framework::Tensor; -template +template class CRFDecodingOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -44,8 +44,8 @@ class CRFDecodingOpKernel : public framework::OpKernel { const size_t seq_num = lod[level].size() - 1; int64_t* path = decoded_path->mutable_data(platform::CPUPlace()); - math::SetConstant()(ctx.device_context(), - decoded_path, 0); + math::SetConstant()( + ctx.template device_context(), decoded_path, 0); for (size_t i = 0; i < seq_num; ++i) { int start_pos = static_cast(lod[level][i]); int end_pos = static_cast(lod[level][i + 1]); diff --git a/paddle/operators/crop_op.cc b/paddle/operators/crop_op.cc index 6752eb8c1c..7c2a0ac7a7 100644 --- a/paddle/operators/crop_op.cc +++ b/paddle/operators/crop_op.cc @@ -133,5 +133,5 @@ class CropOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(crop, ops::CropOp, ops::CropOpMaker, crop_grad, ops::CropOpGrad); REGISTER_OP_CPU_KERNEL(crop, ops::CropKernel); -REGISTER_OP_CPU_KERNEL(crop_grad, - ops::CropGradKernel); +REGISTER_OP_CPU_KERNEL( + crop_grad, ops::CropGradKernel); diff --git a/paddle/operators/crop_op.cu b/paddle/operators/crop_op.cu index f8ee18a1d6..90fd83ca10 100644 --- a/paddle/operators/crop_op.cu +++ b/paddle/operators/crop_op.cu @@ -16,6 +16,6 @@ #include "paddle/operators/crop_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(crop, ops::CropKernel); -REGISTER_OP_GPU_KERNEL(crop_grad, - ops::CropGradKernel); +REGISTER_OP_CUDA_KERNEL(crop, ops::CropKernel); +REGISTER_OP_CUDA_KERNEL( + crop_grad, ops::CropGradKernel); diff --git a/paddle/operators/crop_op.h b/paddle/operators/crop_op.h index 2e72583d68..d531a19c78 100644 --- a/paddle/operators/crop_op.h +++ b/paddle/operators/crop_op.h @@ -49,7 +49,7 @@ class CropKernel : public framework::OpKernel { } }; -template +template void CropGradFunction(const framework::ExecutionContext& context) { auto* d_x = context.Output(framework::GradVarName("X")); if (d_x != nullptr) { @@ -63,12 +63,13 @@ void CropGradFunction(const framework::ExecutionContext& context) { } auto d_x_tensor = EigenTensor::From(*d_x); auto d_out_tensor = EigenTensor::From(*d_out); - d_x_tensor.device(context.GetEigenDevice()) = + d_x_tensor.device( + *context.template device_context().eigen_device()) = d_out_tensor.pad(paddings, 0); } } -template +template class CropGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -76,22 +77,22 @@ class CropGradKernel : public framework::OpKernel { context.Input(framework::GradVarName("Out"))->dims().size(); switch (rank) { case 1: - CropGradFunction(context); + CropGradFunction(context); break; case 2: - CropGradFunction(context); + CropGradFunction(context); break; case 3: - CropGradFunction(context); + CropGradFunction(context); break; case 4: - CropGradFunction(context); + CropGradFunction(context); break; case 5: - CropGradFunction(context); + CropGradFunction(context); break; case 6: - CropGradFunction(context); + CropGradFunction(context); break; default: PADDLE_THROW( diff --git a/paddle/operators/cross_entropy_op.cu b/paddle/operators/cross_entropy_op.cu index 6212e39dfd..0546964588 100644 --- a/paddle/operators/cross_entropy_op.cu +++ b/paddle/operators/cross_entropy_op.cu @@ -53,8 +53,9 @@ class CrossEntropyOpCUDAKernel : public framework::OpKernel { Tensor* y = ctx.Output("Y"); y->mutable_data(ctx.GetPlace()); - math::CrossEntropyFunctor()( - ctx.device_context(), y, x, label, ctx.Attr("soft_label")); + math::CrossEntropyFunctor()( + ctx.template device_context(), y, x, label, + ctx.Attr("soft_label")); } }; @@ -80,15 +81,17 @@ class CrossEntropyGradientOpCUDAKernel : public framework::OpKernel { int block = 512; int grid = (batch_size * class_num + block - 1) / block; - auto stream = ctx.cuda_device_context().stream(); + + auto& dev_ctx = ctx.template device_context(); + auto stream = dev_ctx.stream(); if (ctx.Attr("soft_label")) { auto* label_data = label->data(); SoftCrossEntropyGradientKernel<<>>( dx_data, dy_data, x_data, label_data, batch_size, class_num); } else { - math::SetConstant functor; - functor(ctx.device_context(), dx, 0); + math::SetConstant functor; + functor(dev_ctx, dx, 0); auto* label_data = label->data(); grid = (batch_size + block - 1) / block; CrossEntropyGradientKernel<<>>( @@ -101,8 +104,8 @@ class CrossEntropyGradientOpCUDAKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(cross_entropy, ops::CrossEntropyOpCUDAKernel, - ops::CrossEntropyOpCUDAKernel); -REGISTER_OP_GPU_KERNEL(cross_entropy_grad, - ops::CrossEntropyGradientOpCUDAKernel, - ops::CrossEntropyGradientOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(cross_entropy, ops::CrossEntropyOpCUDAKernel, + ops::CrossEntropyOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(cross_entropy_grad, + ops::CrossEntropyGradientOpCUDAKernel, + ops::CrossEntropyGradientOpCUDAKernel); diff --git a/paddle/operators/cross_entropy_op.h b/paddle/operators/cross_entropy_op.h index 37db0a930a..5623d2ded1 100644 --- a/paddle/operators/cross_entropy_op.h +++ b/paddle/operators/cross_entropy_op.h @@ -37,8 +37,9 @@ class CrossEntropyOpKernel : public framework::OpKernel { Tensor* y = ctx.Output("Y"); y->mutable_data(ctx.GetPlace()); - math::CrossEntropyFunctor()( - ctx.device_context(), y, x, labels, ctx.Attr("soft_label")); + math::CrossEntropyFunctor()( + ctx.template device_context(), y, x, labels, + ctx.Attr("soft_label")); } }; @@ -61,7 +62,8 @@ class CrossEntropyGradientOpKernel : public framework::OpKernel { auto lbl_mat = EigenMatrix::From(*label); auto dx_mat = EigenMatrix::From(*dx); - dx_mat.device(ctx.GetEigenDevice()) = + dx_mat.device(*ctx.template device_context() + .eigen_device()) = -(lbl_mat * dy_mat.broadcast(Eigen::DSizes(1, class_num)) / x_mat); } else { @@ -70,8 +72,8 @@ class CrossEntropyGradientOpKernel : public framework::OpKernel { const T* x_data = x->data(); const int64_t* label_data = label->data(); - math::SetConstant functor; - functor(ctx.device_context(), dx, 0); + math::SetConstant functor; + functor(ctx.template device_context(), dx, 0); for (int64_t i = 0; i < batch_size; ++i) { PADDLE_ASSERT(label_data[i] >= 0 || label_data[i] < class_num); diff --git a/paddle/operators/decayed_adagrad_op.cc b/paddle/operators/decayed_adagrad_op.cc index 640b4e7744..fd29c7270b 100644 --- a/paddle/operators/decayed_adagrad_op.cc +++ b/paddle/operators/decayed_adagrad_op.cc @@ -99,4 +99,4 @@ REGISTER_OP_WITHOUT_GRADIENT(decayed_adagrad, ops::DecayedAdagradOp, ops::DecayedAdagradOpMaker); REGISTER_OP_CPU_KERNEL( decayed_adagrad, - ops::DecayedAdagradOpKernel); + ops::DecayedAdagradOpKernel); diff --git a/paddle/operators/decayed_adagrad_op.cu b/paddle/operators/decayed_adagrad_op.cu index 6fce77fe4e..282b90f275 100644 --- a/paddle/operators/decayed_adagrad_op.cu +++ b/paddle/operators/decayed_adagrad_op.cu @@ -16,6 +16,6 @@ #include "paddle/operators/decayed_adagrad_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( decayed_adagrad, - ops::DecayedAdagradOpKernel); + ops::DecayedAdagradOpKernel); diff --git a/paddle/operators/decayed_adagrad_op.h b/paddle/operators/decayed_adagrad_op.h index 0fe0fc5acd..fec9705cfc 100644 --- a/paddle/operators/decayed_adagrad_op.h +++ b/paddle/operators/decayed_adagrad_op.h @@ -19,7 +19,7 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class DecayedAdagradOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -43,7 +43,7 @@ class DecayedAdagradOpKernel : public framework::OpKernel { auto param_out = framework::EigenVector::Flatten(*param_out_tensor); auto moment_out = framework::EigenVector::Flatten(*moment_out_tensor); - auto place = ctx.GetEigenDevice(); + auto& place = *ctx.template device_context().eigen_device(); moment_out.device(place) = decay * moment + (1 - decay) * grad * grad; Eigen::DSizes m_dsize(moment_out_tensor->numel()); diff --git a/paddle/operators/dropout_op.cc b/paddle/operators/dropout_op.cc index 932c0bf8fb..acd526ae80 100644 --- a/paddle/operators/dropout_op.cc +++ b/paddle/operators/dropout_op.cc @@ -100,6 +100,8 @@ namespace ops = paddle::operators; REGISTER_OP(dropout, ops::DropoutOp, ops::DropoutOpMaker, dropout_grad, ops::DropoutOpGrad); REGISTER_OP_CPU_KERNEL( - dropout, ops::CPUDropoutKernel); + dropout, + ops::CPUDropoutKernel); REGISTER_OP_CPU_KERNEL( - dropout_grad, ops::DropoutGradKernel); + dropout_grad, + ops::DropoutGradKernel); diff --git a/paddle/operators/dropout_op.cu b/paddle/operators/dropout_op.cu index db3578b9bf..10c670751d 100644 --- a/paddle/operators/dropout_op.cu +++ b/paddle/operators/dropout_op.cu @@ -58,7 +58,7 @@ class GPUDropoutKernel : public framework::OpKernel { auto X = EigenMatrix::Reshape(*x, 1); auto Y = EigenMatrix::Reshape(*y, 1); - auto place = context.GetEigenDevice(); + auto& place = *context.template device_context().eigen_device(); if (!context.Attr("is_test")) { auto* mask = context.Output("Mask"); auto* mask_data = mask->mutable_data(context.GetPlace()); @@ -80,7 +80,9 @@ class GPUDropoutKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( - dropout, ops::GPUDropoutKernel); -REGISTER_OP_GPU_KERNEL( - dropout_grad, ops::DropoutGradKernel); +REGISTER_OP_CUDA_KERNEL( + dropout, + ops::GPUDropoutKernel); +REGISTER_OP_CUDA_KERNEL( + dropout_grad, + ops::DropoutGradKernel); diff --git a/paddle/operators/dropout_op.h b/paddle/operators/dropout_op.h index d9a130fdc0..84ad39f0bb 100644 --- a/paddle/operators/dropout_op.h +++ b/paddle/operators/dropout_op.h @@ -25,7 +25,7 @@ template using EigenMatrix = framework::EigenMatrix; -template +template class CPUDropoutKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -55,13 +55,14 @@ class CPUDropoutKernel : public framework::OpKernel { } else { auto X = EigenMatrix::Reshape(*x, 1); auto Y = EigenMatrix::Reshape(*y, 1); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); Y.device(place) = X * dropout_prob; } } }; -template +template class DropoutGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -77,7 +78,8 @@ class DropoutGradKernel : public framework::OpKernel { auto dX = EigenMatrix::Reshape(*grad_x, 1); auto dY = EigenMatrix::Reshape(*grad_y, 1); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); dX.device(place) = dY * M; } }; diff --git a/paddle/operators/elementwise_add_op.cc b/paddle/operators/elementwise_add_op.cc index 432b9ba6f7..a62eeeeb95 100644 --- a/paddle/operators/elementwise_add_op.cc +++ b/paddle/operators/elementwise_add_op.cc @@ -34,13 +34,13 @@ REGISTER_OP(elementwise_add, ops::ElementwiseOp, ops::ElementwiseAddOpMaker, elementwise_add_grad, ops::ElementwiseOpGrad); REGISTER_OP_CPU_KERNEL( elementwise_add, - ops::ElementwiseAddKernel, - ops::ElementwiseAddKernel, - ops::ElementwiseAddKernel, - ops::ElementwiseAddKernel); + ops::ElementwiseAddKernel, + ops::ElementwiseAddKernel, + ops::ElementwiseAddKernel, + ops::ElementwiseAddKernel); REGISTER_OP_CPU_KERNEL( elementwise_add_grad, - ops::ElementwiseAddGradKernel, - ops::ElementwiseAddGradKernel, - ops::ElementwiseAddGradKernel, - ops::ElementwiseAddGradKernel); + ops::ElementwiseAddGradKernel, + ops::ElementwiseAddGradKernel, + ops::ElementwiseAddGradKernel, + ops::ElementwiseAddGradKernel); diff --git a/paddle/operators/elementwise_add_op.cu b/paddle/operators/elementwise_add_op.cu index 7591428ac7..78642bb424 100644 --- a/paddle/operators/elementwise_add_op.cu +++ b/paddle/operators/elementwise_add_op.cu @@ -17,15 +17,16 @@ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( elementwise_add, - ops::ElementwiseAddKernel, - ops::ElementwiseAddKernel, - ops::ElementwiseAddKernel, - ops::ElementwiseAddKernel); -REGISTER_OP_GPU_KERNEL( + ops::ElementwiseAddKernel, + ops::ElementwiseAddKernel, + ops::ElementwiseAddKernel, + ops::ElementwiseAddKernel); +REGISTER_OP_CUDA_KERNEL( elementwise_add_grad, - ops::ElementwiseAddGradKernel, - ops::ElementwiseAddGradKernel, - ops::ElementwiseAddGradKernel, - ops::ElementwiseAddGradKernel); + ops::ElementwiseAddGradKernel, + ops::ElementwiseAddGradKernel, + ops::ElementwiseAddGradKernel, + ops::ElementwiseAddGradKernel); diff --git a/paddle/operators/elementwise_add_op.h b/paddle/operators/elementwise_add_op.h index 921dc5f6a6..069bdaf0ab 100644 --- a/paddle/operators/elementwise_add_op.h +++ b/paddle/operators/elementwise_add_op.h @@ -24,7 +24,7 @@ struct AddFunctor { inline HOSTDEVICE T operator()(T a, T b) const { return a + b; } }; -template +template class ElementwiseAddKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -34,8 +34,8 @@ class ElementwiseAddKernel : public framework::OpKernel { auto* y = ctx.Input("Y"); auto* z = ctx.Output("Out"); z->mutable_data(ctx.GetPlace()); - TransformFunctor, T, Place> functor( - x, y, z, ctx.device_context(), AddFunctor()); + TransformFunctor, T, DeviceContext> functor( + x, y, z, ctx.template device_context(), AddFunctor()); auto x_dims = x->dims(); auto y_dims = y->dims(); @@ -137,11 +137,11 @@ struct ElementwiseAddBroadCast2GradFunctor { } }; -template +template class ElementwiseAddGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - ElementwiseGradCompute, + ElementwiseGradCompute, ElementwiseAddOneGradFunctor, ElementwiseAddBroadCastGradFunctor, ElementwiseAddBroadCast2GradFunctor>(ctx); diff --git a/paddle/operators/elementwise_div_op.cc b/paddle/operators/elementwise_div_op.cc index 7a325199bd..1c3e9e70ee 100644 --- a/paddle/operators/elementwise_div_op.cc +++ b/paddle/operators/elementwise_div_op.cc @@ -35,13 +35,13 @@ REGISTER_OP(elementwise_div, ops::ElementwiseOp, ops::ElementwiseDivOpMaker, elementwise_div_grad, ops::ElementwiseOpGrad); REGISTER_OP_CPU_KERNEL( elementwise_div, - ops::ElementwiseDivKernel, - ops::ElementwiseDivKernel, - ops::ElementwiseDivKernel, - ops::ElementwiseDivKernel); + ops::ElementwiseDivKernel, + ops::ElementwiseDivKernel, + ops::ElementwiseDivKernel, + ops::ElementwiseDivKernel); REGISTER_OP_CPU_KERNEL( elementwise_div_grad, - ops::ElementwiseDivGradKernel, - ops::ElementwiseDivGradKernel, - ops::ElementwiseDivGradKernel, - ops::ElementwiseDivGradKernel); + ops::ElementwiseDivGradKernel, + ops::ElementwiseDivGradKernel, + ops::ElementwiseDivGradKernel, + ops::ElementwiseDivGradKernel); diff --git a/paddle/operators/elementwise_div_op.cu b/paddle/operators/elementwise_div_op.cu index de4d0c3344..502c528936 100644 --- a/paddle/operators/elementwise_div_op.cu +++ b/paddle/operators/elementwise_div_op.cu @@ -17,15 +17,16 @@ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( elementwise_div, - ops::ElementwiseDivKernel, - ops::ElementwiseDivKernel, - ops::ElementwiseDivKernel, - ops::ElementwiseDivKernel); -REGISTER_OP_GPU_KERNEL( + ops::ElementwiseDivKernel, + ops::ElementwiseDivKernel, + ops::ElementwiseDivKernel, + ops::ElementwiseDivKernel); +REGISTER_OP_CUDA_KERNEL( elementwise_div_grad, - ops::ElementwiseDivGradKernel, - ops::ElementwiseDivGradKernel, - ops::ElementwiseDivGradKernel, - ops::ElementwiseDivGradKernel); + ops::ElementwiseDivGradKernel, + ops::ElementwiseDivGradKernel, + ops::ElementwiseDivGradKernel, + ops::ElementwiseDivGradKernel); diff --git a/paddle/operators/elementwise_div_op.h b/paddle/operators/elementwise_div_op.h index 8946ff3d25..d91313db42 100644 --- a/paddle/operators/elementwise_div_op.h +++ b/paddle/operators/elementwise_div_op.h @@ -19,11 +19,11 @@ namespace paddle { namespace operators { -template +template class ElementwiseDivKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - ElementwiseCompute(ctx); + ElementwiseCompute(ctx); } }; @@ -102,11 +102,11 @@ struct ElementwiseDivBroadCast2GradFunctor { } }; -template +template class ElementwiseDivGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - ElementwiseGradCompute, + ElementwiseGradCompute, ElementwiseDivGradFunctor, ElementwiseDivBroadCastGradFunctor, ElementwiseDivBroadCast2GradFunctor>(ctx); diff --git a/paddle/operators/elementwise_mul_op.cc b/paddle/operators/elementwise_mul_op.cc index 8851267a52..aadb95cbe3 100644 --- a/paddle/operators/elementwise_mul_op.cc +++ b/paddle/operators/elementwise_mul_op.cc @@ -36,13 +36,13 @@ REGISTER_OP(elementwise_mul, ops::ElementwiseOp, ops::ElementwiseMulOpMaker, elementwise_mul_grad, ops::ElementwiseOpGrad); REGISTER_OP_CPU_KERNEL( elementwise_mul, - ops::ElementwiseMulKernel, - ops::ElementwiseMulKernel, - ops::ElementwiseMulKernel, - ops::ElementwiseMulKernel); + ops::ElementwiseMulKernel, + ops::ElementwiseMulKernel, + ops::ElementwiseMulKernel, + ops::ElementwiseMulKernel); REGISTER_OP_CPU_KERNEL( elementwise_mul_grad, - ops::ElementwiseMulGradKernel, - ops::ElementwiseMulGradKernel, - ops::ElementwiseMulGradKernel, - ops::ElementwiseMulGradKernel); + ops::ElementwiseMulGradKernel, + ops::ElementwiseMulGradKernel, + ops::ElementwiseMulGradKernel, + ops::ElementwiseMulGradKernel); diff --git a/paddle/operators/elementwise_mul_op.cu b/paddle/operators/elementwise_mul_op.cu index b0dfdee1cc..089451b3e1 100644 --- a/paddle/operators/elementwise_mul_op.cu +++ b/paddle/operators/elementwise_mul_op.cu @@ -17,15 +17,16 @@ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( elementwise_mul, - ops::ElementwiseMulKernel, - ops::ElementwiseMulKernel, - ops::ElementwiseMulKernel, - ops::ElementwiseMulKernel); -REGISTER_OP_GPU_KERNEL( + ops::ElementwiseMulKernel, + ops::ElementwiseMulKernel, + ops::ElementwiseMulKernel, + ops::ElementwiseMulKernel); +REGISTER_OP_CUDA_KERNEL( elementwise_mul_grad, - ops::ElementwiseMulGradKernel, - ops::ElementwiseMulGradKernel, - ops::ElementwiseMulGradKernel, - ops::ElementwiseMulGradKernel); + ops::ElementwiseMulGradKernel, + ops::ElementwiseMulGradKernel, + ops::ElementwiseMulGradKernel, + ops::ElementwiseMulGradKernel); diff --git a/paddle/operators/elementwise_mul_op.h b/paddle/operators/elementwise_mul_op.h index 4469b07eaa..16fa5ec4b3 100644 --- a/paddle/operators/elementwise_mul_op.h +++ b/paddle/operators/elementwise_mul_op.h @@ -18,11 +18,11 @@ namespace paddle { namespace operators { -template +template class ElementwiseMulKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - ElementwiseCompute(ctx); + ElementwiseCompute(ctx); } }; @@ -101,11 +101,11 @@ struct ElementwiseMulBroadCast2GradFunctor { } }; -template +template class ElementwiseMulGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - ElementwiseGradCompute, + ElementwiseGradCompute, ElementwiseMulGradFunctor, ElementwiseMulBroadCastGradFunctor, ElementwiseMulBroadCast2GradFunctor>(ctx); diff --git a/paddle/operators/elementwise_op_function.h b/paddle/operators/elementwise_op_function.h index ca3542e783..7ebfc7df8c 100644 --- a/paddle/operators/elementwise_op_function.h +++ b/paddle/operators/elementwise_op_function.h @@ -59,17 +59,17 @@ inline void get_mid_dims(const framework::DDim& x_dims, } } -template +template class RowwiseTransformIterator; -template +template class MidWiseTransformIterator; template -class RowwiseTransformIterator { +class RowwiseTransformIterator { public: RowwiseTransformIterator(const T* ptr, int n) : ptr_(ptr), i_(0), n_(n) {} - RowwiseTransformIterator& operator++() { + RowwiseTransformIterator& operator++() { ++i_; if (UNLIKELY(i_ == n_)) { i_ = 0; @@ -77,13 +77,13 @@ class RowwiseTransformIterator { return *this; } - bool operator==( - const RowwiseTransformIterator& rhs) const { + bool operator==(const RowwiseTransformIterator& + rhs) const { return (ptr_ + i_) == &(*rhs); } - bool operator!=( - const RowwiseTransformIterator& rhs) const { + bool operator!=(const RowwiseTransformIterator& + rhs) const { return (ptr_ + i_) != &(*rhs); } @@ -96,12 +96,12 @@ class RowwiseTransformIterator { }; template -class MidWiseTransformIterator { +class MidWiseTransformIterator { public: MidWiseTransformIterator(const T* ptr, int n, int post) : ptr_(ptr), i_(0), j_(0), n_(n), post_(post) {} - MidWiseTransformIterator& operator++() { + MidWiseTransformIterator& operator++() { ++j_; i_ = j_ / post_; if (UNLIKELY(i_ == n_)) { @@ -111,13 +111,13 @@ class MidWiseTransformIterator { return *this; } - bool operator==( - const MidWiseTransformIterator& rhs) const { + bool operator==(const MidWiseTransformIterator& + rhs) const { return (ptr_ + i_) == &(*rhs); } - bool operator!=( - const MidWiseTransformIterator& rhs) const { + bool operator!=(const MidWiseTransformIterator& + rhs) const { return (ptr_ + i_) != &(*rhs); } @@ -133,12 +133,12 @@ class MidWiseTransformIterator { #ifdef __NVCC__ template -class RowwiseTransformIterator +class RowwiseTransformIterator : public thrust::iterator_adaptor< - RowwiseTransformIterator, const T*> { + RowwiseTransformIterator, const T*> { public: typedef thrust::iterator_adaptor< - RowwiseTransformIterator, const T*> + RowwiseTransformIterator, const T*> super_t; HOSTDEVICE RowwiseTransformIterator(const T* x, int n) : super_t(x), begin_(x), n_(n){}; @@ -153,12 +153,12 @@ class RowwiseTransformIterator }; template -class MidWiseTransformIterator +class MidWiseTransformIterator : public thrust::iterator_adaptor< - MidWiseTransformIterator, const T*> { + MidWiseTransformIterator, const T*> { public: typedef thrust::iterator_adaptor< - MidWiseTransformIterator, const T*> + MidWiseTransformIterator, const T*> super_t; HOSTDEVICE MidWiseTransformIterator(const T* x, int n, int post) : super_t(x), begin_(x), n_(n), post_(post){}; @@ -174,12 +174,11 @@ class MidWiseTransformIterator }; #endif -template +template class TransformFunctor { public: TransformFunctor(const framework::Tensor* x, const framework::Tensor* y, - framework::Tensor* z, const platform::DeviceContext& ctx, - Functor func) + framework::Tensor* z, const DeviceContext& ctx, Functor func) : x_(x->data()), y_(y->data()), z_(z->mutable_data(ctx.GetPlace())), @@ -188,20 +187,20 @@ class TransformFunctor { func_(func) {} inline void Run() const { - platform::Transform trans; + platform::Transform trans; trans(ctx_, x_, x_ + nx_, y_, z_, func_); } inline void RunRowWise(int n, int pre) const { - platform::Transform trans; - trans(ctx_, x_, x_ + nx_, RowwiseTransformIterator(y_, n), z_, - func_); + platform::Transform trans; + trans(ctx_, x_, x_ + nx_, RowwiseTransformIterator(y_, n), + z_, func_); } inline void RunMidWise(int n, int pre, int post) const { - platform::Transform trans; - trans(ctx_, x_, x_ + nx_, MidWiseTransformIterator(y_, n, post), - z_, func_); + platform::Transform trans; + trans(ctx_, x_, x_ + nx_, + MidWiseTransformIterator(y_, n, post), z_, func_); } private: @@ -209,22 +208,24 @@ class TransformFunctor { const T* y_; T* z_; int64_t nx_; - const platform::DeviceContext& ctx_; + const DeviceContext& ctx_; Functor func_; }; #define EIGEN_FUNCTOR(name, eigen_op) \ struct Eigen##name##Functor { \ - template \ + template \ inline void Run(const framework::Tensor* x, const framework::Tensor* y, \ framework::Tensor* z, \ const framework::ExecutionContext& ctx) { \ auto x_e = framework::EigenVector::Flatten(*x); \ auto y_e = framework::EigenVector::Flatten(*y); \ auto z_e = framework::EigenVector::Flatten(*z); \ - z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_e); \ + z_e.device( \ + *ctx.template device_context().eigen_device()) = \ + eigen_op(x_e, y_e); \ } \ - template \ + template \ inline void RunBroadCast(const framework::Tensor* x, \ const framework::Tensor* y, framework::Tensor* z, \ const framework::ExecutionContext& ctx, int pre, \ @@ -235,9 +236,11 @@ class TransformFunctor { auto y_bcast = y_e.reshape(Eigen::DSizes(1, n)) \ .broadcast(Eigen::DSizes(pre, 1)) \ .reshape(Eigen::DSizes(x_e.size())); \ - z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_bcast); \ + z_e.device( \ + *ctx.template device_context().eigen_device()) = \ + eigen_op(x_e, y_bcast); \ } \ - template \ + template \ inline void RunBroadCast2(const framework::Tensor* x, \ const framework::Tensor* y, \ framework::Tensor* z, \ @@ -249,11 +252,13 @@ class TransformFunctor { auto y_bcast = y_e.reshape(Eigen::DSizes(1, n, 1)) \ .broadcast(Eigen::DSizes(pre, 1, post)) \ .reshape(Eigen::DSizes(x_e.size())); \ - z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_bcast); \ + z_e.device( \ + *ctx.template device_context().eigen_device()) = \ + eigen_op(x_e, y_bcast); \ } \ } -template +template void ElementwiseCompute(const framework::ExecutionContext& ctx) { using Tensor = framework::Tensor; @@ -269,7 +274,7 @@ void ElementwiseCompute(const framework::ExecutionContext& ctx) { if (x_dims == y_dims) { functor f; - f.template Run(x, y, z, ctx); + f.template Run(x, y, z, ctx); return; } @@ -282,11 +287,11 @@ void ElementwiseCompute(const framework::ExecutionContext& ctx) { get_mid_dims(x_dims, y_dims, axis, pre, n, post); if (post == 1) { functor f; - f.template RunBroadCast(x, y, z, ctx, pre, n); + f.template RunBroadCast(x, y, z, ctx, pre, n); return; } else { functor f; - f.template RunBroadCast2(x, y, z, ctx, pre, n, post); + f.template RunBroadCast2(x, y, z, ctx, pre, n, post); return; } } @@ -303,8 +308,9 @@ EIGEN_FUNCTOR(Mul, EIGEN_MUL); #define EIGEN_DIV(x, y) ((x) / (y)) EIGEN_FUNCTOR(Div, EIGEN_DIV); -template +template void ElementwiseGradCompute(const framework::ExecutionContext& ctx) { using Tensor = framework::Tensor; @@ -313,7 +319,7 @@ void ElementwiseGradCompute(const framework::ExecutionContext& ctx) { auto* out = ctx.Input("Out"); auto* dout = ctx.Input(framework::GradVarName("Out")); - auto place = ctx.GetEigenDevice(); + auto& place = *ctx.template device_context().eigen_device(); auto x_dims = x->dims(); auto y_dims = y->dims(); diff --git a/paddle/operators/elementwise_sub_op.cc b/paddle/operators/elementwise_sub_op.cc index 95d7979e39..3e4d19361e 100644 --- a/paddle/operators/elementwise_sub_op.cc +++ b/paddle/operators/elementwise_sub_op.cc @@ -34,13 +34,13 @@ REGISTER_OP(elementwise_sub, ops::ElementwiseOp, ops::ElementwiseSubOpMaker, elementwise_sub_grad, ops::ElementwiseOpGrad); REGISTER_OP_CPU_KERNEL( elementwise_sub, - ops::ElementwiseSubKernel, - ops::ElementwiseSubKernel, - ops::ElementwiseSubKernel, - ops::ElementwiseSubKernel); + ops::ElementwiseSubKernel, + ops::ElementwiseSubKernel, + ops::ElementwiseSubKernel, + ops::ElementwiseSubKernel); REGISTER_OP_CPU_KERNEL( elementwise_sub_grad, - ops::ElementwiseSubGradKernel, - ops::ElementwiseSubGradKernel, - ops::ElementwiseSubGradKernel, - ops::ElementwiseSubGradKernel); + ops::ElementwiseSubGradKernel, + ops::ElementwiseSubGradKernel, + ops::ElementwiseSubGradKernel, + ops::ElementwiseSubGradKernel); diff --git a/paddle/operators/elementwise_sub_op.cu b/paddle/operators/elementwise_sub_op.cu index ec23bec35f..0b2f0f7d4d 100644 --- a/paddle/operators/elementwise_sub_op.cu +++ b/paddle/operators/elementwise_sub_op.cu @@ -17,15 +17,16 @@ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( elementwise_sub, - ops::ElementwiseSubKernel, - ops::ElementwiseSubKernel, - ops::ElementwiseSubKernel, - ops::ElementwiseSubKernel); -REGISTER_OP_GPU_KERNEL( + ops::ElementwiseSubKernel, + ops::ElementwiseSubKernel, + ops::ElementwiseSubKernel, + ops::ElementwiseSubKernel); +REGISTER_OP_CUDA_KERNEL( elementwise_sub_grad, - ops::ElementwiseSubGradKernel, - ops::ElementwiseSubGradKernel, - ops::ElementwiseSubGradKernel, - ops::ElementwiseSubGradKernel); + ops::ElementwiseSubGradKernel, + ops::ElementwiseSubGradKernel, + ops::ElementwiseSubGradKernel, + ops::ElementwiseSubGradKernel); diff --git a/paddle/operators/elementwise_sub_op.h b/paddle/operators/elementwise_sub_op.h index 3f40c1c5bc..731a30c5e3 100644 --- a/paddle/operators/elementwise_sub_op.h +++ b/paddle/operators/elementwise_sub_op.h @@ -18,11 +18,11 @@ namespace paddle { namespace operators { -template +template class ElementwiseSubKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - ElementwiseCompute(ctx); + ElementwiseCompute(ctx); } }; @@ -101,11 +101,11 @@ struct ElementwiseSubBroadCast2GradFunctor { } }; -template +template class ElementwiseSubGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - ElementwiseGradCompute, + ElementwiseGradCompute, ElementwiseSubOneGradFunctor, ElementwiseSubBroadCastGradFunctor, ElementwiseSubBroadCast2GradFunctor>(ctx); diff --git a/paddle/operators/expand_op.cc b/paddle/operators/expand_op.cc index 282775fcda..8b3cddbb94 100644 --- a/paddle/operators/expand_op.cc +++ b/paddle/operators/expand_op.cc @@ -130,7 +130,8 @@ class ExpandGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(expand, ops::ExpandOp, ops::ExpandOpMaker, expand_grad, ops::ExpandGradOp); -REGISTER_OP_CPU_KERNEL(expand, - ops::ExpandKernel); REGISTER_OP_CPU_KERNEL( - expand_grad, ops::ExpandGradKernel); + expand, ops::ExpandKernel); +REGISTER_OP_CPU_KERNEL( + expand_grad, + ops::ExpandGradKernel); diff --git a/paddle/operators/expand_op.cu b/paddle/operators/expand_op.cu index 6744562b6c..99ee584d08 100644 --- a/paddle/operators/expand_op.cu +++ b/paddle/operators/expand_op.cu @@ -17,7 +17,8 @@ #include "paddle/operators/expand_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(expand, - ops::ExpandKernel); -REGISTER_OP_GPU_KERNEL( - expand_grad, ops::ExpandGradKernel); +REGISTER_OP_CUDA_KERNEL( + expand, ops::ExpandKernel); +REGISTER_OP_CUDA_KERNEL( + expand_grad, + ops::ExpandGradKernel); diff --git a/paddle/operators/expand_op.h b/paddle/operators/expand_op.h index 4d7996ad1e..14ef8b0912 100644 --- a/paddle/operators/expand_op.h +++ b/paddle/operators/expand_op.h @@ -56,7 +56,7 @@ template using EigenTensor = framework::EigenTensor; -template +template class ExpandKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -83,12 +83,13 @@ class ExpandKernel : public framework::OpKernel { auto x = EigenTensor::From(*in0); out0->mutable_data(context.GetPlace()); auto y = EigenTensor::From(*out0); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); y.device(place) = x.broadcast(bcast_dims); } }; -template +template class ExpandGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -164,7 +165,8 @@ class ExpandGradKernel : public framework::OpKernel { reduce_dims[i] = reduce_dims_vec[i]; } auto out_grad = EigenVector::Flatten(*in0); - x_grad.device(context.GetEigenDevice()) = + x_grad.device( + *context.template device_context().eigen_device()) = out_grad.reshape(reshape_dims).sum(reduce_dims).reshape(x.dimensions()); } }; diff --git a/paddle/operators/fill_constant_batch_size_like_op.cc b/paddle/operators/fill_constant_batch_size_like_op.cc index 892922cd3a..7fb74e2b95 100644 --- a/paddle/operators/fill_constant_batch_size_like_op.cc +++ b/paddle/operators/fill_constant_batch_size_like_op.cc @@ -100,8 +100,11 @@ REGISTER_OPERATOR(fill_constant_batch_size_like, ops::FillConstantBatchSizeLikeOpMaker); REGISTER_OP_CPU_KERNEL( fill_constant_batch_size_like, - ops::FillConstantBatchSizeLikeOpKernel, - ops::FillConstantBatchSizeLikeOpKernel, - ops::FillConstantBatchSizeLikeOpKernel, - ops::FillConstantBatchSizeLikeOpKernel, + ops::FillConstantBatchSizeLikeOpKernel, + ops::FillConstantBatchSizeLikeOpKernel, + ops::FillConstantBatchSizeLikeOpKernel); diff --git a/paddle/operators/fill_constant_batch_size_like_op.cu.cc b/paddle/operators/fill_constant_batch_size_like_op.cu.cc index 9e7a1eeab8..2e0e15f36b 100644 --- a/paddle/operators/fill_constant_batch_size_like_op.cu.cc +++ b/paddle/operators/fill_constant_batch_size_like_op.cu.cc @@ -16,10 +16,13 @@ #include "paddle/framework/op_registry.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( fill_constant_batch_size_like, - ops::FillConstantBatchSizeLikeOpKernel, - ops::FillConstantBatchSizeLikeOpKernel, - ops::FillConstantBatchSizeLikeOpKernel, - ops::FillConstantBatchSizeLikeOpKernel, + ops::FillConstantBatchSizeLikeOpKernel, + ops::FillConstantBatchSizeLikeOpKernel, + ops::FillConstantBatchSizeLikeOpKernel); diff --git a/paddle/operators/fill_constant_batch_size_like_op.h b/paddle/operators/fill_constant_batch_size_like_op.h index 339d97a30a..66da9d0307 100644 --- a/paddle/operators/fill_constant_batch_size_like_op.h +++ b/paddle/operators/fill_constant_batch_size_like_op.h @@ -19,7 +19,7 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class FillConstantBatchSizeLikeOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -27,8 +27,9 @@ class FillConstantBatchSizeLikeOpKernel : public framework::OpKernel { out->mutable_data(ctx.GetPlace()); auto value = ctx.Attr("value"); - math::SetConstant setter; - setter(ctx.device_context(), out, static_cast(value)); + math::SetConstant setter; + setter(ctx.template device_context(), out, + static_cast(value)); } }; diff --git a/paddle/operators/fill_zeros_like_op.cc b/paddle/operators/fill_zeros_like_op.cc index 95fb5932b8..720c11f5f1 100644 --- a/paddle/operators/fill_zeros_like_op.cc +++ b/paddle/operators/fill_zeros_like_op.cc @@ -54,8 +54,9 @@ namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(fill_zeros_like, ops::FillZerosLikeOp, ops::FillZerosLikeOpMaker); REGISTER_OP_CPU_KERNEL( - fill_zeros_like, ops::FillZerosLikeKernel, - ops::FillZerosLikeKernel, - ops::FillZerosLikeKernel, - ops::FillZerosLikeKernel, - ops::FillZerosLikeKernel); + fill_zeros_like, + ops::FillZerosLikeKernel, + ops::FillZerosLikeKernel, + ops::FillZerosLikeKernel, + ops::FillZerosLikeKernel, + ops::FillZerosLikeKernel); diff --git a/paddle/operators/fill_zeros_like_op.cu.cc b/paddle/operators/fill_zeros_like_op.cu.cc index 1501a17441..9f412306bb 100644 --- a/paddle/operators/fill_zeros_like_op.cu.cc +++ b/paddle/operators/fill_zeros_like_op.cu.cc @@ -16,9 +16,10 @@ #include "paddle/framework/op_registry.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( - fill_zeros_like, ops::FillZerosLikeKernel, - ops::FillZerosLikeKernel, - ops::FillZerosLikeKernel, - ops::FillZerosLikeKernel, - ops::FillZerosLikeKernel); +REGISTER_OP_CUDA_KERNEL( + fill_zeros_like, + ops::FillZerosLikeKernel, + ops::FillZerosLikeKernel, + ops::FillZerosLikeKernel, + ops::FillZerosLikeKernel, + ops::FillZerosLikeKernel); diff --git a/paddle/operators/fill_zeros_like_op.h b/paddle/operators/fill_zeros_like_op.h index 7e7d78eea2..a6e2941f52 100644 --- a/paddle/operators/fill_zeros_like_op.h +++ b/paddle/operators/fill_zeros_like_op.h @@ -19,15 +19,16 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class FillZerosLikeKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { auto* out = context.Output("Y"); out->mutable_data(context.GetPlace()); - math::SetConstant setter; - setter(context.device_context(), out, static_cast(0)); + math::SetConstant setter; + setter(context.template device_context(), out, + static_cast(0)); } }; diff --git a/paddle/operators/ftrl_op.cc b/paddle/operators/ftrl_op.cc index cb7ae69196..b14913ff21 100644 --- a/paddle/operators/ftrl_op.cc +++ b/paddle/operators/ftrl_op.cc @@ -135,5 +135,5 @@ The paper that proposed Follow The Regularized Leader (FTRL): namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(ftrl, ops::FTRLOp, ops::FTRLOpMaker); -REGISTER_OP_CPU_KERNEL(ftrl, - ops::FTRLOpKernel); +REGISTER_OP_CPU_KERNEL( + ftrl, ops::FTRLOpKernel); diff --git a/paddle/operators/ftrl_op.cu b/paddle/operators/ftrl_op.cu index 97b36dade6..abbbe7adbe 100644 --- a/paddle/operators/ftrl_op.cu +++ b/paddle/operators/ftrl_op.cu @@ -15,5 +15,5 @@ specific language governing permissions and limitations under the License. */ #include "paddle/operators/ftrl_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(ftrl, - ops::FTRLOpKernel); +REGISTER_OP_CUDA_KERNEL( + ftrl, ops::FTRLOpKernel); diff --git a/paddle/operators/ftrl_op.h b/paddle/operators/ftrl_op.h index b040162f8d..4eea04cd8d 100644 --- a/paddle/operators/ftrl_op.h +++ b/paddle/operators/ftrl_op.h @@ -24,7 +24,7 @@ template using EigenVector = framework::EigenVector; -template +template class FTRLOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -53,7 +53,7 @@ class FTRLOpKernel : public framework::OpKernel { auto p_out = EigenVector::Flatten(*param_out); auto s_acc_out = EigenVector::Flatten(*sq_accum_out); auto l_acc_out = EigenVector::Flatten(*lin_accum_out); - auto place = ctx.GetEigenDevice(); + auto& place = *ctx.template device_context().eigen_device(); Eigen::DSizes grad_dsize(grad->numel()); diff --git a/paddle/operators/gather.cu.h b/paddle/operators/gather.cu.h index 8d04ecd284..c806aa5f05 100644 --- a/paddle/operators/gather.cu.h +++ b/paddle/operators/gather.cu.h @@ -20,7 +20,7 @@ namespace paddle { namespace operators { using framework::Tensor; -using platform::Place; +using platform::DeviceContext; #define CUDA_1D_KERNEL_LOOP(i, n) \ for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ diff --git a/paddle/operators/gather_op.cu b/paddle/operators/gather_op.cu index 92219d6a43..b37f0576e2 100644 --- a/paddle/operators/gather_op.cu +++ b/paddle/operators/gather_op.cu @@ -49,7 +49,8 @@ class GatherGradOpCUDAKernel : public framework::OpKernel { dX->mutable_data(ctx.GetPlace()); auto dxt = framework::EigenVector::Flatten(*dX); - auto place = ctx.GetEigenDevice(); + auto &place = *ctx.template device_context() + .eigen_device(); dxt.device(place) = dxt.constant(static_cast(0)); GPUScatterAssign(ctx.device_context(), *dO, *Index, dX); @@ -60,5 +61,5 @@ class GatherGradOpCUDAKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(gather, ops::GatherOpCUDAKernel); -REGISTER_OP_GPU_KERNEL(gather_grad, ops::GatherGradOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(gather, ops::GatherOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(gather_grad, ops::GatherGradOpCUDAKernel); diff --git a/paddle/operators/gather_op.h b/paddle/operators/gather_op.h index 8276ed0d3d..1a1ba0c41a 100644 --- a/paddle/operators/gather_op.h +++ b/paddle/operators/gather_op.h @@ -53,7 +53,8 @@ class GatherGradientOpKernel : public framework::OpKernel { dX->mutable_data(ctx.GetPlace()); auto dxt = framework::EigenVector::Flatten(*dX); - auto place = ctx.GetEigenDevice(); + auto &place = *ctx.template device_context() + .eigen_device(); dxt.device(place) = dxt.constant(static_cast(0)); ScatterAssign(ctx.device_context(), *dO, *Index, dX); diff --git a/paddle/operators/gaussian_random_op.cu b/paddle/operators/gaussian_random_op.cu index 315560bf1b..ffce6f7138 100644 --- a/paddle/operators/gaussian_random_op.cu +++ b/paddle/operators/gaussian_random_op.cu @@ -60,5 +60,5 @@ class GPUGaussianRandomKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -REGISTER_OP_GPU_KERNEL(gaussian_random, - paddle::operators::GPUGaussianRandomKernel); +REGISTER_OP_CUDA_KERNEL(gaussian_random, + paddle::operators::GPUGaussianRandomKernel); diff --git a/paddle/operators/gru_op.cc b/paddle/operators/gru_op.cc index 5aa03f8916..311e7edcf1 100644 --- a/paddle/operators/gru_op.cc +++ b/paddle/operators/gru_op.cc @@ -213,8 +213,9 @@ class GRUGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(gru, ops::GRUOp, ops::GRUOpMaker, gru_grad, ops::GRUGradOp); -REGISTER_OP_CPU_KERNEL(gru, ops::GRUKernel, - ops::GRUKernel); -REGISTER_OP_CPU_KERNEL(gru_grad, - ops::GRUGradKernel, - ops::GRUGradKernel); +REGISTER_OP_CPU_KERNEL( + gru, ops::GRUKernel, + ops::GRUKernel); +REGISTER_OP_CPU_KERNEL( + gru_grad, ops::GRUGradKernel, + ops::GRUGradKernel); diff --git a/paddle/operators/gru_op.cu.cc b/paddle/operators/gru_op.cu.cc index 0ceff94ec3..458630ca61 100644 --- a/paddle/operators/gru_op.cu.cc +++ b/paddle/operators/gru_op.cu.cc @@ -15,8 +15,9 @@ #include "paddle/operators/gru_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(gru, ops::GRUKernel, - ops::GRUKernel); -REGISTER_OP_GPU_KERNEL(gru_grad, - ops::GRUGradKernel, - ops::GRUGradKernel); +REGISTER_OP_CUDA_KERNEL( + gru, ops::GRUKernel, + ops::GRUKernel); +REGISTER_OP_CUDA_KERNEL( + gru_grad, ops::GRUGradKernel, + ops::GRUGradKernel); diff --git a/paddle/operators/gru_op.h b/paddle/operators/gru_op.h index 564489d3a9..6d02dff578 100644 --- a/paddle/operators/gru_op.h +++ b/paddle/operators/gru_op.h @@ -27,16 +27,16 @@ namespace operators { using LoDTensor = framework::LoDTensor; using Tensor = framework::Tensor; -template -inline void ReorderInitState(const platform::DeviceContext& ctx, +template +inline void ReorderInitState(const DeviceContext& ctx, const framework::Tensor& src, const size_t* index, framework::Tensor* dst, bool indexed_src) { - math::CopyMatrixRowsFunctor row_shuffle; + math::CopyMatrixRowsFunctor row_shuffle; dst->mutable_data(src.dims(), ctx.GetPlace()); row_shuffle(ctx, src, index, *dst, indexed_src); } -template +template class GRUKernel : public framework::OpKernel { public: void BatchCompute(const framework::ExecutionContext& context) const { @@ -60,12 +60,12 @@ class GRUKernel : public framework::OpKernel { auto hidden_dims = hidden->dims(); bool is_reverse = context.Attr("is_reverse"); - math::LoDTensor2BatchFunctor to_batch; - auto& dev_ctx = context.device_context(); + math::LoDTensor2BatchFunctor to_batch; + auto& dev_ctx = context.template device_context(); to_batch(dev_ctx, *input, *batch_gate, true, is_reverse); if (bias) { - math::RowwiseAdd add_bias; + math::RowwiseAdd add_bias; add_bias(dev_ctx, *batch_gate, *bias, batch_gate); } @@ -80,8 +80,9 @@ class GRUKernel : public framework::OpKernel { // Since the batch computing for GRU reorders the input sequences // according to their length. The initialized cell state also needs // to reorder. - ReorderInitState(context.device_context(), *h0, order, - &ordered_h0, true); + ReorderInitState( + context.template device_context(), *h0, order, + &ordered_h0, true); gru_value.prev_out_value = ordered_h0.data(); } else { gru_value.prev_out_value = nullptr; @@ -99,14 +100,14 @@ class GRUKernel : public framework::OpKernel { gru_value.output_value = hidden_t.data(); gru_value.gate_value = gate_t.data(); gru_value.reset_output_value = reset_hidden_prev_t.data(); - math::GRUUnitFunctor::compute( + math::GRUUnitFunctor::compute( dev_ctx, gru_value, frame_size, cur_batch_size, math::ActiveType(context.Attr("activation")), math::ActiveType(context.Attr("gate_activation"))); gru_value.prev_out_value = gru_value.output_value; } - math::Batch2LoDTensorFunctor to_seq; + math::Batch2LoDTensorFunctor to_seq; batch_hidden->set_lod(batch_gate->lod()); to_seq(dev_ctx, *batch_hidden, *hidden); } @@ -116,7 +117,7 @@ class GRUKernel : public framework::OpKernel { } }; -template +template class GRUGradKernel : public framework::OpKernel { public: void BatchCompute(const framework::ExecutionContext& context) const { @@ -141,14 +142,14 @@ class GRUGradKernel : public framework::OpKernel { auto hidden_dims = hidden->dims(); int frame_size = hidden_dims[1]; - math::LoDTensor2BatchFunctor to_batch; + math::LoDTensor2BatchFunctor to_batch; LoDTensor batch_hidden_grad, batch_gate_grad, batch_reset_hidden_prev_grad; batch_hidden_grad.mutable_data(hidden_dims, context.GetPlace()); batch_gate_grad.mutable_data(gate_dims, context.GetPlace()); batch_reset_hidden_prev_grad.mutable_data(hidden_dims, context.GetPlace()); - math::SetConstant zero; - auto& dev_ctx = context.device_context(); + math::SetConstant zero; + auto& dev_ctx = context.template device_context(); zero(dev_ctx, &batch_hidden_grad, static_cast(0.0)); zero(dev_ctx, &batch_gate_grad, static_cast(0.0)); zero(dev_ctx, &batch_reset_hidden_prev_grad, static_cast(0.0)); @@ -156,12 +157,13 @@ class GRUGradKernel : public framework::OpKernel { Tensor ordered_h0, ordered_h0_grad; const size_t* order = batch_gate->lod()[2].data(); if (h0) { - ReorderInitState(context.device_context(), *h0, order, - &ordered_h0, true); + ReorderInitState(dev_ctx, *h0, order, &ordered_h0, + true); } if (h0_grad) { ordered_h0_grad.mutable_data(h0_grad->dims(), context.GetPlace()); - zero(context.device_context(), &ordered_h0_grad, static_cast(0.0)); + zero(context.template device_context(), &ordered_h0_grad, + static_cast(0.0)); } bool is_reverse = context.Attr("is_reverse"); @@ -216,25 +218,25 @@ class GRUGradKernel : public framework::OpKernel { gru_grad.prev_out_grad = hidden_prev_grad_t.data(); } - math::GRUUnitGradFunctor::compute( + math::GRUUnitGradFunctor::compute( dev_ctx, gru_value, gru_grad, frame_size, cur_batch_size, math::ActiveType(context.Attr("activation")), math::ActiveType(context.Attr("gate_activation"))); } if (input_grad) { input_grad->mutable_data(context.GetPlace()); - math::Batch2LoDTensorFunctor to_seq; + math::Batch2LoDTensorFunctor to_seq; batch_gate_grad.set_lod(batch_gate->lod()); to_seq(dev_ctx, batch_gate_grad, *input_grad); } if (bias_grad) { bias_grad->mutable_data(context.GetPlace()); - math::ColwiseSum col_sum; + math::ColwiseSum col_sum; col_sum(dev_ctx, batch_gate_grad, bias_grad); } if (h0 && h0_grad) { - ReorderInitState(context.device_context(), ordered_h0_grad, - order, h0_grad, false); + ReorderInitState(dev_ctx, ordered_h0_grad, order, + h0_grad, false); } } diff --git a/paddle/operators/gru_unit_op.cc b/paddle/operators/gru_unit_op.cc index 877c969103..705de87be5 100644 --- a/paddle/operators/gru_unit_op.cc +++ b/paddle/operators/gru_unit_op.cc @@ -201,9 +201,10 @@ class GRUUnitGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(gru_unit, ops::GRUUnitOp, ops::GRUUnitOpMaker, gru_unit_grad, ops::GRUUnitGradOp); -REGISTER_OP_CPU_KERNEL(gru_unit, - ops::GRUUnitKernel, - ops::GRUUnitKernel); REGISTER_OP_CPU_KERNEL( - gru_unit_grad, ops::GRUUnitGradKernel, - ops::GRUUnitGradKernel); + gru_unit, ops::GRUUnitKernel, + ops::GRUUnitKernel); +REGISTER_OP_CPU_KERNEL( + gru_unit_grad, + ops::GRUUnitGradKernel, + ops::GRUUnitGradKernel); diff --git a/paddle/operators/gru_unit_op.cu b/paddle/operators/gru_unit_op.cu index 821c8c6421..7c752db494 100644 --- a/paddle/operators/gru_unit_op.cu +++ b/paddle/operators/gru_unit_op.cu @@ -16,9 +16,10 @@ #include "paddle/operators/gru_unit_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(gru_unit, - ops::GRUUnitKernel, - ops::GRUUnitKernel); -REGISTER_OP_GPU_KERNEL( - gru_unit_grad, ops::GRUUnitGradKernel, - ops::GRUUnitGradKernel); +REGISTER_OP_CUDA_KERNEL( + gru_unit, ops::GRUUnitKernel, + ops::GRUUnitKernel); +REGISTER_OP_CUDA_KERNEL( + gru_unit_grad, + ops::GRUUnitGradKernel, + ops::GRUUnitGradKernel); diff --git a/paddle/operators/gru_unit_op.h b/paddle/operators/gru_unit_op.h index 3398c0934e..8fe60c750d 100644 --- a/paddle/operators/gru_unit_op.h +++ b/paddle/operators/gru_unit_op.h @@ -34,7 +34,7 @@ using EigenVector = framework::EigenVector; enum GRUActivationType { identity = 0, sigmoid = 1, tanh = 2, relu = 3 }; -template +template class GRUUnitKernel : public framework::OpKernel { public: template @@ -71,7 +71,8 @@ class GRUUnitKernel : public framework::OpKernel { auto g = EigenMatrix::From(*gate); auto r_h_p = EigenMatrix::From(*reset_hidden_prev); auto h = EigenMatrix::From(*hidden); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); // calculate unactivated gate outputs if (bias) { @@ -86,10 +87,10 @@ class GRUUnitKernel : public framework::OpKernel { const T* weight_data = weight->data(); T* gate_data = gate->data(); T* reset_hidden_prev_data = reset_hidden_prev->data(); - math::gemm(context.device_context(), false, false, batch_size, - 2 * frame_size, frame_size, 1, hidden_prev_data, - frame_size, weight_data, frame_size * 2, 1, gate_data, - frame_size * 3); + math::gemm( + context.template device_context(), false, false, + batch_size, 2 * frame_size, frame_size, 1, hidden_prev_data, frame_size, + weight_data, frame_size * 2, 1, gate_data, frame_size * 3); // calculate activited gate Eigen::array extents({{batch_size, frame_size}}); @@ -102,11 +103,11 @@ class GRUUnitKernel : public framework::OpKernel { g.slice(r_offsets, extents), g.slice(r_offsets, extents)); auto r = g.slice(r_offsets, extents); // reset gate r_h_p.device(place) = r * h_p; // reset previous hidden state - math::gemm(context.device_context(), false, false, batch_size, - frame_size, frame_size, 1, reset_hidden_prev_data, - frame_size, weight_data + frame_size * frame_size * 2, - frame_size, 1, gate_data + frame_size * 2, - frame_size * 3); + math::gemm( + context.template device_context(), false, false, + batch_size, frame_size, frame_size, 1, reset_hidden_prev_data, + frame_size, weight_data + frame_size * frame_size * 2, frame_size, 1, + gate_data + frame_size * 2, frame_size * 3); Eigen::array c_offsets({{0, frame_size * 2}}); ActCompute(context.Attr("activation"), place, @@ -118,7 +119,7 @@ class GRUUnitKernel : public framework::OpKernel { } }; -template +template class GRUUnitGradKernel : public framework::OpKernel { public: template @@ -166,7 +167,8 @@ class GRUUnitGradKernel : public framework::OpKernel { auto d_h = EigenMatrix::From(*hidden_grad); auto d_g = EigenMatrix::From(gate_grad); auto d_r_h_p = EigenMatrix::From(reset_hidden_prev_grad); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); int batch_size = input->dims()[0]; int frame_size = hidden_prev->dims()[1]; @@ -186,11 +188,11 @@ class GRUUnitGradKernel : public framework::OpKernel { ActGradCompute(context.Attr("activation"), place, c, c, d_g.slice(c_offsets, extents), d_h * u); // backward for reset_hidden_prev - math::gemm(context.device_context(), false, true, batch_size, - frame_size, frame_size, 1, - gate_grad_data + frame_size * 2, frame_size * 3, - weight_data + frame_size * frame_size * 2, frame_size, - 0, reset_hidden_prev_grad_data, frame_size); + math::gemm( + context.template device_context(), false, true, + batch_size, frame_size, frame_size, 1, gate_grad_data + frame_size * 2, + frame_size * 3, weight_data + frame_size * frame_size * 2, frame_size, + 0, reset_hidden_prev_grad_data, frame_size); // backward for unactivated reset gate ActGradCompute(context.Attr("gate_activation"), place, r, r, d_g.slice(r_offsets, extents), d_r_h_p * h_p); @@ -198,17 +200,18 @@ class GRUUnitGradKernel : public framework::OpKernel { if (weight_grad) { T* weight_grad_data = weight_grad->mutable_data(context.GetPlace()); // backward for state_weight - math::gemm( - context.device_context(), true, false, frame_size, frame_size, - batch_size, 1, reset_hidden_prev_data, frame_size, - gate_grad_data + frame_size * 2, frame_size * 3, 0, + math::gemm( + context.template device_context(), true, false, + frame_size, frame_size, batch_size, 1, reset_hidden_prev_data, + frame_size, gate_grad_data + frame_size * 2, frame_size * 3, 0, weight_grad_data + frame_size * frame_size * 2, frame_size); // backward for update_gate_weight and reset_gate_weight - math::gemm(context.device_context(), true, false, frame_size, - frame_size * 2, batch_size, 1, hidden_prev_data, - frame_size, gate_grad_data, frame_size * 3, 0, - weight_grad_data, frame_size * 2); + math::gemm( + context.template device_context(), true, false, + frame_size, frame_size * 2, batch_size, 1, hidden_prev_data, + frame_size, gate_grad_data, frame_size * 3, 0, weight_grad_data, + frame_size * 2); } // backward for hidden_prev if (hidden_prev_grad) { @@ -216,10 +219,11 @@ class GRUUnitGradKernel : public framework::OpKernel { hidden_prev_grad->mutable_data(context.GetPlace()); auto d_h_p = EigenMatrix::From(*hidden_prev_grad); d_h_p.device(place) = d_r_h_p * r + d_h * (u.constant(T(1)) - u); - math::gemm(context.device_context(), false, true, batch_size, - frame_size, frame_size * 2, 1, gate_grad_data, - frame_size * 3, weight_data, frame_size * 2, 1, - hidden_prev_grad_data, frame_size); + math::gemm( + context.template device_context(), false, true, + batch_size, frame_size, frame_size * 2, 1, gate_grad_data, + frame_size * 3, weight_data, frame_size * 2, 1, hidden_prev_grad_data, + frame_size); } // backward for input if (input_grad) { diff --git a/paddle/operators/hinge_loss_op.cc b/paddle/operators/hinge_loss_op.cc index 1e13897bb6..373b4d99b4 100644 --- a/paddle/operators/hinge_loss_op.cc +++ b/paddle/operators/hinge_loss_op.cc @@ -106,8 +106,9 @@ class HingeLossGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(hinge_loss, ops::HingeLossOp, ops::HingeLossOpMaker, hinge_loss_grad, ops::HingeLossGradOp); -REGISTER_OP_CPU_KERNEL(hinge_loss, - ops::HingeLossKernel); +REGISTER_OP_CPU_KERNEL( + hinge_loss, + ops::HingeLossKernel); REGISTER_OP_CPU_KERNEL( hinge_loss_grad, - ops::HingeLossGradKernel); + ops::HingeLossGradKernel); diff --git a/paddle/operators/hinge_loss_op.cu b/paddle/operators/hinge_loss_op.cu index ec20b08e30..31a5bde292 100644 --- a/paddle/operators/hinge_loss_op.cu +++ b/paddle/operators/hinge_loss_op.cu @@ -16,8 +16,9 @@ #include "paddle/operators/hinge_loss_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(hinge_loss, - ops::HingeLossKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( + hinge_loss, + ops::HingeLossKernel); +REGISTER_OP_CUDA_KERNEL( hinge_loss_grad, - ops::HingeLossGradKernel); + ops::HingeLossGradKernel); diff --git a/paddle/operators/hinge_loss_op.h b/paddle/operators/hinge_loss_op.h index c0be496f9c..91369cfb8a 100644 --- a/paddle/operators/hinge_loss_op.h +++ b/paddle/operators/hinge_loss_op.h @@ -19,14 +19,15 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class HingeLossKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { auto* pred = context.Input("Logits"); auto* label = context.Input("Labels"); auto* loss = context.Output("Loss"); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); auto x = framework::EigenVector::Flatten(*pred); auto y = framework::EigenVector::Flatten(*label); @@ -38,7 +39,7 @@ class HingeLossKernel : public framework::OpKernel { } }; -template +template class HingeLossGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -48,7 +49,8 @@ class HingeLossGradKernel : public framework::OpKernel { context.Input(framework::GradVarName("Loss")); auto* dpred = context.Output(framework::GradVarName("Logits")); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); auto x = framework::EigenVector::Flatten(*pred); auto y = framework::EigenVector::Flatten(*label); diff --git a/paddle/operators/huber_loss_op.cc b/paddle/operators/huber_loss_op.cc index 938803d5b3..11828d083a 100644 --- a/paddle/operators/huber_loss_op.cc +++ b/paddle/operators/huber_loss_op.cc @@ -124,8 +124,9 @@ class HuberLossGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(huber_loss, ops::HuberLossOp, ops::HuberLossOpMaker, huber_loss_grad, ops::HuberLossGradOp); -REGISTER_OP_CPU_KERNEL(huber_loss, - ops::HuberLossKernel); +REGISTER_OP_CPU_KERNEL( + huber_loss, + ops::HuberLossKernel); REGISTER_OP_CPU_KERNEL( huber_loss_grad, - ops::HuberLossGradKernel); + ops::HuberLossGradKernel); diff --git a/paddle/operators/huber_loss_op.cu b/paddle/operators/huber_loss_op.cu index 317321dc6c..d49a4d9d42 100644 --- a/paddle/operators/huber_loss_op.cu +++ b/paddle/operators/huber_loss_op.cu @@ -16,8 +16,9 @@ #include "paddle/operators/huber_loss_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(huber_loss, - ops::HuberLossKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( + huber_loss, + ops::HuberLossKernel); +REGISTER_OP_CUDA_KERNEL( huber_loss_grad, - ops::HuberLossGradKernel); + ops::HuberLossGradKernel); diff --git a/paddle/operators/huber_loss_op.h b/paddle/operators/huber_loss_op.h index 4e7bc55432..4dd20e8b08 100644 --- a/paddle/operators/huber_loss_op.h +++ b/paddle/operators/huber_loss_op.h @@ -41,7 +41,7 @@ struct HuberLossForward { T delta; }; -template +template class HuberLossKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -50,7 +50,8 @@ class HuberLossKernel : public framework::OpKernel { auto* out0 = context.Output("Residual"); auto* out1 = context.Output("Out"); auto delta = static_cast(context.Attr("delta")); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); auto x = EigenVector::Flatten(*in0); auto y = EigenVector::Flatten(*in1); @@ -85,7 +86,7 @@ struct HuberLossBackward { T delta; }; -template +template class HuberLossGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -94,7 +95,8 @@ class HuberLossGradKernel : public framework::OpKernel { auto* out0 = context.Output(framework::GradVarName("X")); auto* out1 = context.Output(framework::GradVarName("Y")); auto delta = static_cast(context.op().Attr("delta")); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); auto residual = EigenVector::Flatten(*in0); auto out_grad = EigenVector::Flatten(*in1); diff --git a/paddle/operators/l1_norm_op.cc b/paddle/operators/l1_norm_op.cc index 02ebf02296..c0b51202c6 100644 --- a/paddle/operators/l1_norm_op.cc +++ b/paddle/operators/l1_norm_op.cc @@ -69,7 +69,8 @@ $$Out = \sum{|X|}$$ namespace ops = paddle::operators; REGISTER_OP(l1_norm, ops::L1NormOp, ops::L1NormOpMaker, l1_norm_grad, ops::L1NormGradOp); -REGISTER_OP_CPU_KERNEL(l1_norm, - ops::L1NormKernel); REGISTER_OP_CPU_KERNEL( - l1_norm_grad, ops::L1NormGradKernel); + l1_norm, ops::L1NormKernel); +REGISTER_OP_CPU_KERNEL( + l1_norm_grad, + ops::L1NormGradKernel); diff --git a/paddle/operators/l1_norm_op.cu b/paddle/operators/l1_norm_op.cu index 1c206e04cc..fd725f86f6 100644 --- a/paddle/operators/l1_norm_op.cu +++ b/paddle/operators/l1_norm_op.cu @@ -16,7 +16,8 @@ #include "paddle/operators/l1_norm_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(l1_norm, - ops::L1NormKernel); -REGISTER_OP_GPU_KERNEL( - l1_norm_grad, ops::L1NormGradKernel); +REGISTER_OP_CUDA_KERNEL( + l1_norm, ops::L1NormKernel); +REGISTER_OP_CUDA_KERNEL( + l1_norm_grad, + ops::L1NormGradKernel); diff --git a/paddle/operators/l1_norm_op.h b/paddle/operators/l1_norm_op.h index 3c60dc3dc7..ae3878f2b7 100644 --- a/paddle/operators/l1_norm_op.h +++ b/paddle/operators/l1_norm_op.h @@ -20,7 +20,7 @@ namespace paddle { namespace operators { // Out = sum(abs(X)) -template +template class L1NormKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { @@ -30,14 +30,15 @@ class L1NormKernel : public framework::OpKernel { auto x = framework::EigenVector::Flatten(*X); auto out = framework::EigenScalar::From(*Out); - auto place = context.GetEigenDevice(); + auto &place = + *context.template device_context().eigen_device(); out.device(place) = x.abs().sum(); } }; // dX = dout * sign(X) -template +template class L1NormGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { @@ -52,7 +53,8 @@ class L1NormGradKernel : public framework::OpKernel { auto x_eigen = framework::EigenVector::Flatten(*x); auto d_out_eigen = framework::EigenVector::Flatten(*d_out); auto dx_eigen = framework::EigenVector::Flatten(*dx); - auto place = context.GetEigenDevice(); + auto &place = + *context.template device_context().eigen_device(); Eigen::DSizes x_dsize(x->numel()); dx_eigen.device(place) = d_out_eigen.broadcast(x_dsize) * x_eigen.sign(); diff --git a/paddle/operators/linear_chain_crf_op.cc b/paddle/operators/linear_chain_crf_op.cc index 8e079a14e0..896e3657d4 100644 --- a/paddle/operators/linear_chain_crf_op.cc +++ b/paddle/operators/linear_chain_crf_op.cc @@ -261,9 +261,10 @@ REGISTER_OP(linear_chain_crf, ops::LinearChainCRFOp, ops::LinearChainCRFOpMaker, linear_chain_crf_grad, ops::LinearChainCRFGradOp); REGISTER_OP_CPU_KERNEL( linear_chain_crf, - ops::LinearChainCRFOpKernel, - ops::LinearChainCRFOpKernel); + ops::LinearChainCRFOpKernel, + ops::LinearChainCRFOpKernel); REGISTER_OP_CPU_KERNEL( linear_chain_crf_grad, - ops::LinearChainCRFGradOpKernel, - ops::LinearChainCRFGradOpKernel); + ops::LinearChainCRFGradOpKernel, + ops::LinearChainCRFGradOpKernel); diff --git a/paddle/operators/linear_chain_crf_op.cu b/paddle/operators/linear_chain_crf_op.cu index 6fc8995f4c..3b105ec341 100644 --- a/paddle/operators/linear_chain_crf_op.cu +++ b/paddle/operators/linear_chain_crf_op.cu @@ -16,11 +16,12 @@ limitations under the License. */ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( linear_chain_crf, - ops::LinearChainCRFOpKernel, - ops::LinearChainCRFOpKernel); -REGISTER_OP_GPU_KERNEL( + ops::LinearChainCRFOpKernel, + ops::LinearChainCRFOpKernel); +REGISTER_OP_CUDA_KERNEL( linear_chain_crf_grad, - ops::LinearChainCRFGradOpKernel, - ops::LinearChainCRFGradOpKernel); + ops::LinearChainCRFGradOpKernel, + ops::LinearChainCRFGradOpKernel); diff --git a/paddle/operators/linear_chain_crf_op.h b/paddle/operators/linear_chain_crf_op.h index 014bbfa758..694584e79c 100644 --- a/paddle/operators/linear_chain_crf_op.h +++ b/paddle/operators/linear_chain_crf_op.h @@ -50,7 +50,7 @@ template using EigenMatrix = framework::EigenMatrix; -template +template class LinearChainCRFOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -137,7 +137,8 @@ class LinearChainCRFOpKernel : public framework::OpKernel { framework::make_ddim({static_cast(batch_size), 1}), platform::CPUPlace()); - auto place = ctx.GetEigenDevice(); + auto& place = *ctx.template device_context() + .eigen_device(); auto x = EigenMatrix::From(*emission_weights); auto x_row_max = EigenMatrix::From(emission_row_max); x_row_max.device(place) = @@ -287,7 +288,7 @@ class LinearChainCRFOpKernel : public framework::OpKernel { } }; -template +template class LinearChainCRFGradOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -359,8 +360,7 @@ class LinearChainCRFGradOpKernel : public framework::OpKernel { emission_grad->mutable_data(platform::CPUPlace()); if (transition_grad) { transition_grad->mutable_data(platform::CPUPlace()); - math::SetConstant()(ctx.device_context(), - transition_grad, 0.); + math::set_constant(ctx.device_context(), transition_grad, 0.); } // Now, all the inputs and outputs should be on the CPU memory. @@ -384,10 +384,10 @@ class LinearChainCRFGradOpKernel : public framework::OpKernel { Tensor one_seq_beta = beta.Slice(start_pos, end_pos); Tensor one_seq_emission_grad = emission_grad->Slice(start_pos, end_pos); - BackwardOneSequence(ctx.device_context(), ll_grad[i], - one_seq_emission_exps, *transition_exps, - one_seq_alpha, one_seq_label, &one_seq_beta, - transition_grad, &one_seq_emission_grad); + BackwardOneSequence( + ctx.template device_context(), ll_grad[i], + one_seq_emission_exps, *transition_exps, one_seq_alpha, one_seq_label, + &one_seq_beta, transition_grad, &one_seq_emission_grad); } if (platform::is_gpu_place(ctx.GetPlace())) { @@ -441,8 +441,8 @@ class LinearChainCRFGradOpKernel : public framework::OpKernel { copyTensor(ctx, transition_grad_src, transition_grad_dst); } - void BackwardOneSequence(const platform::DeviceContext& ctx, const T ll_grad, - const Tensor& emission_exps, + void BackwardOneSequence(const platform::CPUDeviceContext& ctx, + const T ll_grad, const Tensor& emission_exps, const Tensor& transition_exps, const Tensor& alpha, const Tensor& label, Tensor* beta, Tensor* transition_grad, @@ -481,7 +481,7 @@ class LinearChainCRFGradOpKernel : public framework::OpKernel { auto alpha_mat = EigenMatrix::From(alpha); auto beta_mat = EigenMatrix::From(*beta); - auto* place = ctx.GetEigenDevice(); + auto* place = ctx.eigen_device(); auto prob = alpha_mat * beta_mat; auto row_sum = prob.sum(Eigen::DSizes(1)) .reshape(Eigen::DSizes(seq_length, 1)) diff --git a/paddle/operators/lod_reset_op.cu b/paddle/operators/lod_reset_op.cu index 5244a17c3a..f7c2358980 100644 --- a/paddle/operators/lod_reset_op.cu +++ b/paddle/operators/lod_reset_op.cu @@ -16,9 +16,10 @@ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(lod_reset, - ops::LoDResetKernel, - ops::LoDResetKernel); -REGISTER_OP_GPU_KERNEL( - lod_reset_grad, ops::LoDResetGradKernel, - ops::LoDResetGradKernel); +REGISTER_OP_CUDA_KERNEL( + lod_reset, ops::LoDResetKernel, + ops::LoDResetKernel); +REGISTER_OP_CUDA_KERNEL( + lod_reset_grad, + ops::LoDResetGradKernel, + ops::LoDResetGradKernel); diff --git a/paddle/operators/lod_reset_op.h b/paddle/operators/lod_reset_op.h index cbcbf80adc..b86f8b1313 100644 --- a/paddle/operators/lod_reset_op.h +++ b/paddle/operators/lod_reset_op.h @@ -20,7 +20,7 @@ namespace paddle { namespace operators { -template +template class LoDResetKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { @@ -65,7 +65,7 @@ class LoDResetKernel : public framework::OpKernel { } }; -template +template class LoDResetGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { diff --git a/paddle/operators/log_loss_op.cc b/paddle/operators/log_loss_op.cc index 257e5c8a49..4524229a33 100644 --- a/paddle/operators/log_loss_op.cc +++ b/paddle/operators/log_loss_op.cc @@ -109,7 +109,8 @@ class LogLossGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(log_loss, ops::LogLossOp, ops::LogLossOpMaker, log_loss_grad, ops::LogLossGradOp); -REGISTER_OP_CPU_KERNEL(log_loss, - ops::LogLossKernel); REGISTER_OP_CPU_KERNEL( - log_loss_grad, ops::LogLossGradKernel); + log_loss, ops::LogLossKernel); +REGISTER_OP_CPU_KERNEL( + log_loss_grad, + ops::LogLossGradKernel); diff --git a/paddle/operators/log_loss_op.cu b/paddle/operators/log_loss_op.cu index 6c189ef341..e87ac7d12a 100644 --- a/paddle/operators/log_loss_op.cu +++ b/paddle/operators/log_loss_op.cu @@ -16,7 +16,8 @@ #include "paddle/operators/log_loss_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(log_loss, - ops::LogLossKernel); -REGISTER_OP_GPU_KERNEL( - log_loss_grad, ops::LogLossGradKernel); +REGISTER_OP_CUDA_KERNEL( + log_loss, ops::LogLossKernel); +REGISTER_OP_CUDA_KERNEL( + log_loss_grad, + ops::LogLossGradKernel); diff --git a/paddle/operators/log_loss_op.h b/paddle/operators/log_loss_op.h index 73404fce91..743eddb740 100644 --- a/paddle/operators/log_loss_op.h +++ b/paddle/operators/log_loss_op.h @@ -24,7 +24,7 @@ template using EigenVector = framework::EigenVector; -template +template class LogLossKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -38,7 +38,7 @@ class LogLossKernel : public framework::OpKernel { auto label = EigenVector::Flatten(*ctx.Input("Labels")); auto loss = EigenVector::Flatten(*loss_out); - auto place = ctx.GetEigenDevice(); + auto& place = *ctx.template device_context().eigen_device(); loss.device(place) = (-(label * (prediction + epsilon).log()) - ((static_cast(1) - label) * @@ -46,7 +46,7 @@ class LogLossKernel : public framework::OpKernel { } }; -template +template class LogLossGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -59,7 +59,7 @@ class LogLossGradKernel : public framework::OpKernel { auto* dpred = ctx.Output(framework::GradVarName("Predicted")); auto dl = EigenVector::Flatten(*dloss); - auto place = ctx.GetEigenDevice(); + auto& place = *ctx.template device_context().eigen_device(); if (dpred) { dpred->mutable_data(ctx.GetPlace()); diff --git a/paddle/operators/logical_op.cu b/paddle/operators/logical_op.cu index d41239b2ca..7fef60e0c9 100644 --- a/paddle/operators/logical_op.cu +++ b/paddle/operators/logical_op.cu @@ -14,11 +14,11 @@ #include "paddle/operators/logical_op.h" -REGISTER_BINARY_LOGICAL_KERNEL(logical_and, GPU, +REGISTER_BINARY_LOGICAL_KERNEL(logical_and, CUDA, paddle::operators::LogicalAndFunctor); -REGISTER_BINARY_LOGICAL_KERNEL(logical_or, GPU, +REGISTER_BINARY_LOGICAL_KERNEL(logical_or, CUDA, paddle::operators::LogicalOrFunctor); -REGISTER_UNARY_LOGICAL_KERNEL(logical_not, GPU, +REGISTER_UNARY_LOGICAL_KERNEL(logical_not, CUDA, paddle::operators::LogicalNotFunctor); -REGISTER_BINARY_LOGICAL_KERNEL(logical_xor, GPU, +REGISTER_BINARY_LOGICAL_KERNEL(logical_xor, CUDA, paddle::operators::LogicalXorFunctor); diff --git a/paddle/operators/logical_op.h b/paddle/operators/logical_op.h index 6e78a7d6ed..629388cac8 100644 --- a/paddle/operators/logical_op.h +++ b/paddle/operators/logical_op.h @@ -47,7 +47,7 @@ struct LogicalXorFunctor { } }; -template +template class BinaryLogicalOpKernel : public framework::OpKernel { public: @@ -57,14 +57,14 @@ class BinaryLogicalOpKernel auto* y = context.Input("Y"); auto* out = context.Output("Out"); Functor binary_func; - platform::Transform trans; - trans(context.device_context(), x->data(), x->data() + x->numel(), - y->data(), out->mutable_data(context.GetPlace()), - binary_func); + platform::Transform trans; + trans(context.template device_context(), x->data(), + x->data() + x->numel(), y->data(), + out->mutable_data(context.GetPlace()), binary_func); } }; -template +template class UnaryLogicalOpKernel : public framework::OpKernel { public: @@ -73,8 +73,9 @@ class UnaryLogicalOpKernel auto* x = context.Input("X"); auto* out = context.Output("Out"); Functor unary_func; - platform::Transform trans; - trans(context.device_context(), x->data(), x->data() + x->numel(), + platform::Transform trans; + trans(context.template device_context(), x->data(), + x->data() + x->numel(), out->mutable_data(context.GetPlace()), unary_func); } }; @@ -85,9 +86,9 @@ class UnaryLogicalOpKernel #define REGISTER_BINARY_LOGICAL_KERNEL(op_type, dev, functor) \ REGISTER_OP_##dev##_KERNEL( \ op_type, ::paddle::operators::BinaryLogicalOpKernel< \ - ::paddle::platform::dev##Place, functor>); + ::paddle::platform::dev##DeviceContext, functor>); #define REGISTER_UNARY_LOGICAL_KERNEL(op_type, dev, functor) \ REGISTER_OP_##dev##_KERNEL( \ op_type, ::paddle::operators::UnaryLogicalOpKernel< \ - ::paddle::platform::dev##Place, functor>); + ::paddle::platform::dev##DeviceContext, functor>); diff --git a/paddle/operators/lookup_table_op.cu b/paddle/operators/lookup_table_op.cu index 84b044184a..9431030a53 100644 --- a/paddle/operators/lookup_table_op.cu +++ b/paddle/operators/lookup_table_op.cu @@ -85,6 +85,8 @@ template class LookupTableGradCUDAKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { + auto& dev_ctx = + context.template device_context(); bool is_sparse = context.Attr("is_sparse"); if (is_sparse) { auto* ids = context.Input("Ids"); @@ -95,7 +97,7 @@ class LookupTableGradCUDAKernel : public framework::OpKernel { auto* ids_data = ids->data(); auto ids_dim = ids->dims(); - auto stream = context.cuda_device_context().stream(); + auto stream = dev_ctx.stream(); // copy GPU memory to CPU pinned memory framework::Vector new_rows; new_rows.resize(ids_dim[0]); @@ -129,14 +131,11 @@ class LookupTableGradCUDAKernel : public framework::OpKernel { T* d_table = d_table_t->mutable_data(context.GetPlace()); auto t = framework::EigenVector::Flatten(*d_table_t); - t.device(context.GetEigenDevice()) = - t.constant(static_cast(0)); + t.device(*dev_ctx.eigen_device()) = t.constant(static_cast(0)); dim3 threads(128, 8); dim3 grids(8, 1); - LookupTableGrad< - T, 128, 8, - 8><<>>( + LookupTableGrad<<>>( d_table, d_output, ids, N, K, D); } } @@ -146,7 +145,8 @@ class LookupTableGradCUDAKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(lookup_table, ops::LookupTableCUDAKernel, - ops::LookupTableCUDAKernel); -REGISTER_OP_GPU_KERNEL(lookup_table_grad, ops::LookupTableGradCUDAKernel, - ops::LookupTableGradCUDAKernel); +REGISTER_OP_CUDA_KERNEL(lookup_table, ops::LookupTableCUDAKernel, + ops::LookupTableCUDAKernel); +REGISTER_OP_CUDA_KERNEL(lookup_table_grad, + ops::LookupTableGradCUDAKernel, + ops::LookupTableGradCUDAKernel); diff --git a/paddle/operators/lrn_op.cc b/paddle/operators/lrn_op.cc index e20340e77b..b5b7bc940a 100644 --- a/paddle/operators/lrn_op.cc +++ b/paddle/operators/lrn_op.cc @@ -20,7 +20,7 @@ namespace operators { using framework::Tensor; template -struct LRNFunctor { +struct LRNFunctor { void operator()(const framework::ExecutionContext& ctx, const framework::Tensor& input, framework::Tensor* out, framework::Tensor* mid, int N, int C, int H, int W, int n, @@ -55,11 +55,11 @@ struct LRNFunctor { out_e = x_v * e_mid.reshape(Eigen::DSizes(e_mid.size())).pow(-beta); } }; -template struct LRNFunctor; -template struct LRNFunctor; +template struct LRNFunctor; +template struct LRNFunctor; template -struct LRNGradFunctor { +struct LRNGradFunctor { void operator()(const framework::ExecutionContext& ctx, const framework::Tensor& x, const framework::Tensor& out, const framework::Tensor& mid, framework::Tensor* x_g, @@ -113,8 +113,8 @@ struct LRNGradFunctor { } } }; -template struct LRNGradFunctor; -template struct LRNGradFunctor; +template struct LRNGradFunctor; +template struct LRNGradFunctor; class LRNOp : public framework::OperatorWithKernel { public: @@ -204,7 +204,7 @@ Input(i, x, y), Output(i, x, y) represents an element in an image. C is the number of feature maps of one image. n is a hyper-parameter configured when operator is initialized. The sum in the denominator is the sum of the same positions in the neighboring maps. - + )DOC"); } }; @@ -230,6 +230,7 @@ class LRNOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(lrn, ops::LRNOp, ops::LRNOpMaker, lrn_grad, ops::LRNOpGrad); -REGISTER_OP_CPU_KERNEL(lrn, ops::LRNKernel); -REGISTER_OP_CPU_KERNEL(lrn_grad, - ops::LRNGradKernel); +REGISTER_OP_CPU_KERNEL( + lrn, ops::LRNKernel); +REGISTER_OP_CPU_KERNEL( + lrn_grad, ops::LRNGradKernel); diff --git a/paddle/operators/lrn_op.cu b/paddle/operators/lrn_op.cu index e9a8671233..c6857c2b6d 100644 --- a/paddle/operators/lrn_op.cu +++ b/paddle/operators/lrn_op.cu @@ -69,19 +69,18 @@ void CrossMapNormal(const framework::ExecutionContext& ctx, const T* inputs, const int block_size = 1024; int grid_size = (img_size + block_size - 1) / block_size; - KeCMRNormFillScale< - T><<>>( + auto& dev_ctx = ctx.template device_context(); + KeCMRNormFillScale<<>>( img_size, inputs, mid, C, H, W, n, k, alpha); int input_size = N * H * W * C; grid_size = (input_size + block_size - 1) / block_size; - KeCMRNormOutput< - T><<>>( + KeCMRNormOutput<<>>( input_size, inputs, mid, -beta, outputs); } template -struct LRNFunctor { +struct LRNFunctor { void operator()(const framework::ExecutionContext& ctx, const framework::Tensor& input, framework::Tensor* out, framework::Tensor* mid, int N, int C, int H, int W, int n, @@ -92,8 +91,8 @@ struct LRNFunctor { } }; -template struct LRNFunctor; -template struct LRNFunctor; +template struct LRNFunctor; +template struct LRNFunctor; template __global__ void KeCMRNormDiff(int img_size, const T* x, const T* out, @@ -148,14 +147,14 @@ void CrossMapNormalGrad(const framework::ExecutionContext& ctx, const T* x, const int block_size = 1024; int grid_size = (img_size + block_size - 1) / block_size; - KeCMRNormDiff< - T><<>>( + auto& dev_ctx = ctx.template device_context(); + KeCMRNormDiff<<>>( img_size, x, out, mid, x_g, out_g, C, H, W, n, -beta, 2.0f * alpha * beta); } template -struct LRNGradFunctor { +struct LRNGradFunctor { void operator()(const framework::ExecutionContext& ctx, const framework::Tensor& x, const framework::Tensor& out, const framework::Tensor& mid, framework::Tensor* x_g, @@ -167,12 +166,13 @@ struct LRNGradFunctor { } }; -template struct LRNGradFunctor; -template struct LRNGradFunctor; +template struct LRNGradFunctor; +template struct LRNGradFunctor; } // namespace operators } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(lrn, ops::LRNKernel); -REGISTER_OP_GPU_KERNEL(lrn_grad, - ops::LRNGradKernel); +REGISTER_OP_CUDA_KERNEL( + lrn, ops::LRNKernel); +REGISTER_OP_CUDA_KERNEL( + lrn_grad, ops::LRNGradKernel); diff --git a/paddle/operators/lrn_op.h b/paddle/operators/lrn_op.h index aa7539db4a..44063d3e03 100644 --- a/paddle/operators/lrn_op.h +++ b/paddle/operators/lrn_op.h @@ -29,7 +29,7 @@ struct LRNFunctor { T k, T alpha, T beta); }; -template +template class LRNKernel : public framework::OpKernel { public: using Tensor = framework::Tensor; @@ -65,12 +65,12 @@ class LRNKernel : public framework::OpKernel { PADDLE_ENFORCE(beta >= 0.0, "beta should >= 0.0"); PADDLE_ENFORCE(k >= 0.0, "k should >= 0.0"); - LRNFunctor f; + LRNFunctor f; f(ctx, x, out, mid, N, C, H, W, n, k, alpha, beta); } }; -template +template struct LRNGradFunctor { void operator()(const framework::ExecutionContext& ctx, const framework::Tensor& x, const framework::Tensor& out, @@ -98,7 +98,7 @@ struct LRNGradFunctor { * The upper and lower is the same as forward. The logic of the sum * is also the same as forward. */ -template +template class LRNGradKernel : public framework::OpKernel { public: using Tensor = framework::Tensor; @@ -121,7 +121,7 @@ class LRNGradKernel : public framework::OpKernel { T alpha = ctx.Attr("alpha"); T beta = ctx.Attr("beta"); - LRNGradFunctor f; + LRNGradFunctor f; f(ctx, x, out, mid, x_g, out_g, N, C, H, W, n, alpha, beta); } }; diff --git a/paddle/operators/lstm_op.cc b/paddle/operators/lstm_op.cc index fa8e5f2da8..2db7da30db 100644 --- a/paddle/operators/lstm_op.cc +++ b/paddle/operators/lstm_op.cc @@ -273,8 +273,9 @@ class LSTMGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(lstm, ops::LSTMOp, ops::LSTMOpMaker, lstm_grad, ops::LSTMGradOp); -REGISTER_OP_CPU_KERNEL(lstm, ops::LSTMKernel, - ops::LSTMKernel); -REGISTER_OP_CPU_KERNEL(lstm_grad, - ops::LSTMGradKernel, - ops::LSTMGradKernel); +REGISTER_OP_CPU_KERNEL( + lstm, ops::LSTMKernel, + ops::LSTMKernel); +REGISTER_OP_CPU_KERNEL( + lstm_grad, ops::LSTMGradKernel, + ops::LSTMGradKernel); diff --git a/paddle/operators/lstm_op.cu.cc b/paddle/operators/lstm_op.cu.cc index 610cbb03e8..48519bed6f 100644 --- a/paddle/operators/lstm_op.cu.cc +++ b/paddle/operators/lstm_op.cu.cc @@ -15,8 +15,9 @@ #include "paddle/operators/lstm_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(lstm, ops::LSTMKernel, - ops::LSTMKernel); -REGISTER_OP_GPU_KERNEL(lstm_grad, - ops::LSTMGradKernel, - ops::LSTMGradKernel); +REGISTER_OP_CUDA_KERNEL( + lstm, ops::LSTMKernel, + ops::LSTMKernel); +REGISTER_OP_CUDA_KERNEL( + lstm_grad, ops::LSTMGradKernel, + ops::LSTMGradKernel); diff --git a/paddle/operators/lstm_op.h b/paddle/operators/lstm_op.h index a78f548aaf..14abd4bf0a 100644 --- a/paddle/operators/lstm_op.h +++ b/paddle/operators/lstm_op.h @@ -24,16 +24,16 @@ namespace operators { using LoDTensor = framework::LoDTensor; using Tensor = framework::Tensor; -template -inline void ReorderInitState(const platform::DeviceContext& ctx, +template +inline void ReorderInitState(const DeviceContext& ctx, const framework::Tensor& src, const size_t* index, framework::Tensor* dst, bool indexed_src) { - math::CopyMatrixRowsFunctor row_shuffle; + math::CopyMatrixRowsFunctor row_shuffle; dst->mutable_data(src.dims(), ctx.GetPlace()); row_shuffle(ctx, src, index, *dst, indexed_src); } -template +template class LSTMKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -52,8 +52,8 @@ class LSTMKernel : public framework::OpKernel { cell_out->mutable_data(ctx.GetPlace()); bool is_reverse = ctx.Attr("is_reverse"); - math::LoDTensor2BatchFunctor to_batch; - auto& device_ctx = ctx.device_context(); + math::LoDTensor2BatchFunctor to_batch; + auto& device_ctx = ctx.template device_context(); to_batch(device_ctx, *input, *batch_gate, true, is_reverse); auto in_dims = input->dims(); @@ -64,7 +64,7 @@ class LSTMKernel : public framework::OpKernel { Tensor b = *bias; b.Resize({bias->numel(), 1}); Tensor gate_bias = b.Slice(0, 4 * frame_size); - math::RowwiseAdd add_bias; + math::RowwiseAdd add_bias; add_bias(device_ctx, *batch_gate, gate_bias, batch_gate); } @@ -88,8 +88,8 @@ class LSTMKernel : public framework::OpKernel { // Since the batch computing for LSTM reorders the input sequence // according to their length. The initialized cell state also needs // to reorder. - ReorderInitState(device_ctx, *cell_t0, order, &ordered_c0, - true); + ReorderInitState(device_ctx, *cell_t0, order, + &ordered_c0, true); lstm_value.prev_state_value = ordered_c0.data(); } @@ -121,9 +121,9 @@ class LSTMKernel : public framework::OpKernel { int pre_h_start = static_cast(batch_starts[n - 1]); int pre_h_end = pre_h_start + cur_batch_size; auto pre_hidden_t = batch_hidden.Slice(pre_h_start, pre_h_end); - math::matmul(device_ctx, pre_hidden_t, false, *weight, false, - static_cast(1.0), &gate_t, - static_cast(1.0)); + math::matmul(device_ctx, pre_hidden_t, false, *weight, + false, static_cast(1.0), &gate_t, + static_cast(1.0)); } else if (hidden_t0) { // If n == 0 and there is no initialized hidden state, that is to say // the H0 is zeros, the calculation W_h * H0 will be skiped. @@ -133,24 +133,24 @@ class LSTMKernel : public framework::OpKernel { // according to their length. The initialized hidden state also needs // to reorder. Tensor ordered_h0; - ReorderInitState(device_ctx, *hidden_t0, order, &ordered_h0, - true); - math::matmul(device_ctx, ordered_h0, false, *weight, false, - static_cast(1.0), &gate_t, - static_cast(1.0)); + ReorderInitState(device_ctx, *hidden_t0, order, + &ordered_h0, true); + math::matmul(device_ctx, ordered_h0, false, *weight, + false, static_cast(1.0), &gate_t, + static_cast(1.0)); } lstm_value.gate_value = gate_t.data(); lstm_value.output_value = out_t.data(); lstm_value.state_value = cell_t.data(); lstm_value.state_active_value = cell_pre_act_t.data(); - math::LstmUnitFunctor::compute(device_ctx, lstm_value, - frame_size, cur_batch_size, - gate_act, cell_act, cand_act); + math::LstmUnitFunctor::compute( + device_ctx, lstm_value, frame_size, cur_batch_size, gate_act, + cell_act, cand_act); lstm_value.prev_state_value = lstm_value.state_value; } - math::Batch2LoDTensorFunctor to_seq; + math::Batch2LoDTensorFunctor to_seq; batch_hidden.set_lod(batch_gate->lod()); // restore the output hidden in LoDTensor from the batch hidden to_seq(device_ctx, batch_hidden, *hidden_out); @@ -161,7 +161,7 @@ class LSTMKernel : public framework::OpKernel { } }; -template +template class LSTMGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -187,8 +187,8 @@ class LSTMGradKernel : public framework::OpKernel { auto* h0_g = ctx.Output(framework::GradVarName("H0")); auto* c0_g = ctx.Output(framework::GradVarName("C0")); - auto& device_ctx = ctx.device_context(); - math::SetConstant zero; + auto& device_ctx = ctx.template device_context(); + math::SetConstant zero; if (weight_g) { weight_g->mutable_data(ctx.GetPlace()); zero(device_ctx, weight_g, static_cast(0.0)); @@ -200,7 +200,8 @@ class LSTMGradKernel : public framework::OpKernel { Tensor ordered_h0, ordered_c0, ordered_h0_g, ordered_c0_g; const size_t* order = batch_gate->lod()[2].data(); if (c0) { - ReorderInitState(device_ctx, *c0, order, &ordered_c0, true); + ReorderInitState(device_ctx, *c0, order, &ordered_c0, + true); } if (c0 && c0_g) { ordered_c0_g.mutable_data(c0_g->dims(), ctx.GetPlace()); @@ -240,10 +241,10 @@ class LSTMGradKernel : public framework::OpKernel { lstm_grad.check_og_grad = nullptr; } - math::LoDTensor2BatchFunctor to_batch; + math::LoDTensor2BatchFunctor to_batch; auto ToBatch = [&batch_gate, &to_batch]( - const platform::DeviceContext& ctx, const framework::LoDTensor& src, + const DeviceContext& ctx, const framework::LoDTensor& src, const framework::DDim& dims, framework::LoDTensor& dst) { dst.mutable_data(dims, ctx.GetPlace()); dst.set_lod(batch_gate->lod()); @@ -299,7 +300,7 @@ class LSTMGradKernel : public framework::OpKernel { } int cur_batch_size = bend - bstart; - math::LstmUnitGradFunctor::compute( + math::LstmUnitGradFunctor::compute( device_ctx, lstm_value, lstm_grad, frame_size, cur_batch_size, gate_act, cell_act, cand_act); @@ -307,33 +308,34 @@ class LSTMGradKernel : public framework::OpKernel { int pre_h_start = static_cast(batch_starts[n - 1]); int pre_h_end = pre_h_start + cur_batch_size; auto pre_hidden_g = batch_hidden_g.Slice(pre_h_start, pre_h_end); - math::matmul(device_ctx, gate_g, false, *weight, true, - static_cast(1.0), &pre_hidden_g, - static_cast(1.0)); + math::matmul(device_ctx, gate_g, false, *weight, true, + static_cast(1.0), &pre_hidden_g, + static_cast(1.0)); if (weight_g) { /* backward weight */ auto pre_hidden = batch_hidden.Slice(pre_h_start, pre_h_end); - math::matmul(device_ctx, pre_hidden, true, gate_g, false, - static_cast(1.0), weight_g, - static_cast(1.0)); + math::matmul(device_ctx, pre_hidden, true, gate_g, + false, static_cast(1.0), weight_g, + static_cast(1.0)); } } else { if (h0 && weight_g) { - ReorderInitState(device_ctx, *h0, order, &ordered_h0, true); - math::matmul(device_ctx, ordered_h0, true, gate_g, false, - static_cast(1.0), weight_g, - static_cast(1.0)); + ReorderInitState(device_ctx, *h0, order, + &ordered_h0, true); + math::matmul(device_ctx, ordered_h0, true, gate_g, + false, static_cast(1.0), weight_g, + static_cast(1.0)); } if (h0 && h0_g) { ordered_h0_g.mutable_data(h0_g->dims(), ctx.GetPlace()); - math::matmul(device_ctx, gate_g, false, *weight, true, - static_cast(1.0), &ordered_h0_g, - static_cast(0.0)); + math::matmul(device_ctx, gate_g, false, *weight, + true, static_cast(1.0), + &ordered_h0_g, static_cast(0.0)); } } } - math::Batch2LoDTensorFunctor to_seq; + math::Batch2LoDTensorFunctor to_seq; if (in_g) { /* backward data */ in_g->mutable_data(ctx.GetPlace()); @@ -344,15 +346,17 @@ class LSTMGradKernel : public framework::OpKernel { Tensor b_g = *bias_g; b_g.Resize({bias_g->numel(), 1}); Tensor gate_bias_g = b_g.Slice(0, 4 * frame_size); - math::ColwiseSum col_sum; + math::ColwiseSum col_sum; col_sum(device_ctx, batch_gate_g, &gate_bias_g); } if (h0 && h0_g) { - ReorderInitState(device_ctx, ordered_h0_g, order, h0_g, false); + ReorderInitState(device_ctx, ordered_h0_g, order, h0_g, + false); } if (c0 && c0_g) { - ReorderInitState(device_ctx, ordered_c0_g, order, c0_g, false); + ReorderInitState(device_ctx, ordered_c0_g, order, c0_g, + false); } } }; diff --git a/paddle/operators/lstm_unit_op.cu b/paddle/operators/lstm_unit_op.cu index e192283aa0..291f2c295e 100644 --- a/paddle/operators/lstm_unit_op.cu +++ b/paddle/operators/lstm_unit_op.cu @@ -173,7 +173,7 @@ class LstmUnitGradOpCUDAKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(lstm_unit, ops::LstmUnitOpCUDAKernel, - ops::LstmUnitOpCUDAKernel); -REGISTER_OP_GPU_KERNEL(lstm_unit_grad, ops::LstmUnitGradOpCUDAKernel, - ops::LstmUnitGradOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(lstm_unit, ops::LstmUnitOpCUDAKernel, + ops::LstmUnitOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(lstm_unit_grad, ops::LstmUnitGradOpCUDAKernel, + ops::LstmUnitGradOpCUDAKernel); diff --git a/paddle/operators/lstm_unit_op.h b/paddle/operators/lstm_unit_op.h index 38cb298f92..61705675d9 100644 --- a/paddle/operators/lstm_unit_op.h +++ b/paddle/operators/lstm_unit_op.h @@ -35,7 +35,7 @@ inline T tanh(T x) { return 2. * sigmoid(2. * x) - 1.; } -template +template class LstmUnitKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -78,7 +78,7 @@ class LstmUnitKernel : public framework::OpKernel { } }; -template +template class LstmUnitGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { diff --git a/paddle/operators/margin_rank_loss_op.cc b/paddle/operators/margin_rank_loss_op.cc index d7e8a0ea76..42e8961c0e 100644 --- a/paddle/operators/margin_rank_loss_op.cc +++ b/paddle/operators/margin_rank_loss_op.cc @@ -117,7 +117,7 @@ REGISTER_OP(margin_rank_loss, ops::MarginRankLossOp, ops::MarginRankLossGradOp); REGISTER_OP_CPU_KERNEL( margin_rank_loss, - ops::MarginRankLossKernel); + ops::MarginRankLossKernel); REGISTER_OP_CPU_KERNEL( margin_rank_loss_grad, - ops::MarginRankLossGradKernel); + ops::MarginRankLossGradKernel); diff --git a/paddle/operators/margin_rank_loss_op.cu b/paddle/operators/margin_rank_loss_op.cu index 3a639f25d4..1c2afccc5b 100644 --- a/paddle/operators/margin_rank_loss_op.cu +++ b/paddle/operators/margin_rank_loss_op.cu @@ -16,9 +16,9 @@ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( margin_rank_loss, - ops::MarginRankLossKernel); -REGISTER_OP_GPU_KERNEL( + ops::MarginRankLossKernel); +REGISTER_OP_CUDA_KERNEL( margin_rank_loss_grad, - ops::MarginRankLossGradKernel); + ops::MarginRankLossGradKernel); diff --git a/paddle/operators/margin_rank_loss_op.h b/paddle/operators/margin_rank_loss_op.h index 8d0830147e..9c1f96cac1 100644 --- a/paddle/operators/margin_rank_loss_op.h +++ b/paddle/operators/margin_rank_loss_op.h @@ -34,7 +34,7 @@ struct Heaviside { } }; -template +template class MarginRankLossKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { @@ -56,13 +56,13 @@ class MarginRankLossKernel : public framework::OpKernel { auto x1 = framework::EigenVector::Flatten(*x1_t); auto x2 = framework::EigenVector::Flatten(*x2_t); - auto& dev = ctx.GetEigenDevice(); + auto& dev = *ctx.template device_context().eigen_device(); out.device(dev) = (-label * (x1 - x2) + margin).unaryExpr(ReLU()); act.device(dev) = out.unaryExpr(Heaviside()); } }; -template +template class MarginRankLossGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { @@ -78,7 +78,7 @@ class MarginRankLossGradKernel : public framework::OpKernel { auto d_out = framework::EigenVector::Flatten(*d_out_t); auto act = framework::EigenVector::Flatten(*act_t); auto label = framework::EigenVector::Flatten(*label_t); - auto& dev = ctx.GetEigenDevice(); + auto& dev = *ctx.template device_context().eigen_device(); // compute d_x1 if (d_x1_t) { diff --git a/paddle/operators/math/context_project.cc b/paddle/operators/math/context_project.cc index f82ea5d7be..980dd90df8 100644 --- a/paddle/operators/math/context_project.cc +++ b/paddle/operators/math/context_project.cc @@ -18,8 +18,8 @@ namespace paddle { namespace operators { namespace math { -template class ContextProjectFunctor; -template class ContextProjectFunctor; +template class ContextProjectFunctor; +template class ContextProjectFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/context_project.cu b/paddle/operators/math/context_project.cu index 04eeed543c..934e3df645 100644 --- a/paddle/operators/math/context_project.cu +++ b/paddle/operators/math/context_project.cu @@ -20,8 +20,8 @@ namespace paddle { namespace operators { namespace math { -template class ContextProjectFunctor; -template class ContextProjectFunctor; +template class ContextProjectFunctor; +template class ContextProjectFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/context_project.h b/paddle/operators/math/context_project.h index d853507188..4036614086 100644 --- a/paddle/operators/math/context_project.h +++ b/paddle/operators/math/context_project.h @@ -81,17 +81,17 @@ using LoDTensor = framework::LoDTensor; * */ -template +template class ContextProjectFunctor { public: - void operator()(const platform::DeviceContext& context, const LoDTensor& in, + void operator()(const DeviceContext& context, const LoDTensor& in, const Tensor& padding_data, bool padding_trainable, const int context_start, const int context_length, const int context_stride, const int up_pad, const int down_pad, Tensor* col) { auto lod_level_0 = in.lod()[0]; - math::Im2ColFunctor im2col_ocf; + math::Im2ColFunctor im2col_ocf; std::vector dilation({1, 1}); std::vector padding({up_pad, 0, down_pad, 0}); @@ -188,17 +188,17 @@ class ContextProjectFunctor { } }; -template +template class ContextProjectGradFunctor { public: - void operator()(const platform::DeviceContext& context, const LoDTensor& in, + void operator()(const DeviceContext& context, const LoDTensor& in, bool padding_trainable, const int context_start, const int context_length, const int context_stride, const int up_pad, const int down_pad, bool pad_grad, bool input_grad, Tensor* padding_data, Tensor* col) { auto lod_level_0 = in.lod()[0]; - math::Col2ImFunctor col2im_ocf; + math::Col2ImFunctor col2im_ocf; std::vector dilation({1, 1}); std::vector padding({up_pad, 0, down_pad, 0}); @@ -258,8 +258,8 @@ class ContextProjectGradFunctor { Tensor out_t_sub = out_t.Slice(k * context_length, k * context_length + padding_size); Tensor w_sub = padding_data->Slice(k, k + padding_size); - axpy(context, w_sub.numel(), static_cast(1), - out_t_sub.data(), w_sub.data()); + axpy(context, w_sub.numel(), static_cast(1), + out_t_sub.data(), w_sub.data()); } } if (down_pad > 0) { @@ -290,8 +290,8 @@ class ContextProjectGradFunctor { (down_pad_begin_row + t) * context_length); Tensor w_sub = padding_data->Slice( up_pad + padding_idx, up_pad + padding_idx + padding_size); - axpy(context, w_sub.numel(), static_cast(1), - out_t_sub.data(), w_sub.data()); + axpy(context, w_sub.numel(), static_cast(1), + out_t_sub.data(), w_sub.data()); } } out_t.Resize({sequence_height, context_length * sequence_width}); diff --git a/paddle/operators/math/cross_entropy.cc b/paddle/operators/math/cross_entropy.cc index cf238a58e0..6011a196d4 100644 --- a/paddle/operators/math/cross_entropy.cc +++ b/paddle/operators/math/cross_entropy.cc @@ -24,9 +24,9 @@ template ; template -class CrossEntropyFunctor { +class CrossEntropyFunctor { public: - void operator()(const platform::DeviceContext& ctx, framework::Tensor* out, + void operator()(const platform::CPUDeviceContext& ctx, framework::Tensor* out, const framework::Tensor* prob, const framework::Tensor* labels, const bool softLabel) { const int batch_size = prob->dims()[0]; @@ -35,7 +35,7 @@ class CrossEntropyFunctor { auto lbl = EigenMatrix::From(*labels); auto loss = EigenMatrix::From(*out); - loss.device(*ctx.GetEigenDevice()) = + loss.device(*ctx.eigen_device()) = -((lbl * in.log().unaryExpr(math::TolerableValue())) .sum(Eigen::DSizes(1)) .reshape(Eigen::DSizes(batch_size, 1))); @@ -53,8 +53,8 @@ class CrossEntropyFunctor { } }; -template class CrossEntropyFunctor; -template class CrossEntropyFunctor; +template class CrossEntropyFunctor; +template class CrossEntropyFunctor; } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/cross_entropy.cu b/paddle/operators/math/cross_entropy.cu index 651c08f740..2132d49c93 100644 --- a/paddle/operators/math/cross_entropy.cu +++ b/paddle/operators/math/cross_entropy.cu @@ -95,10 +95,10 @@ __global__ void SoftCrossEntropyKernel(T* Y, const T* X, const T* label, using Tensor = framework::Tensor; template -class CrossEntropyFunctor { +class CrossEntropyFunctor { public: - void operator()(const platform::DeviceContext& ctx, framework::Tensor* out, - const framework::Tensor* prob, + void operator()(const platform::CUDADeviceContext& ctx, + framework::Tensor* out, const framework::Tensor* prob, const framework::Tensor* labels, bool softLabel) { const T* prob_data = prob->data(); T* loss_data = out->mutable_data(ctx.GetPlace()); @@ -118,16 +118,14 @@ class CrossEntropyFunctor { const int64_t* label_data = labels->data(); int block = 512; int grid = (batch_size + block - 1) / block; - CrossEntropyKernel<<< - grid, block, 0, - reinterpret_cast(ctx).stream()>>>( + CrossEntropyKernel<<>>( loss_data, prob_data, label_data, batch_size, class_num); } } }; -template class CrossEntropyFunctor; -template class CrossEntropyFunctor; +template class CrossEntropyFunctor; +template class CrossEntropyFunctor; } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/cross_entropy.h b/paddle/operators/math/cross_entropy.h index 70ed9ddd55..677adb5ada 100644 --- a/paddle/operators/math/cross_entropy.h +++ b/paddle/operators/math/cross_entropy.h @@ -33,11 +33,11 @@ struct TolerableValue { } }; -template +template class CrossEntropyFunctor { public: - void operator()(const platform::DeviceContext& context, - framework::Tensor* out, const framework::Tensor* prob, + void operator()(const DeviceContext& context, framework::Tensor* out, + const framework::Tensor* prob, const framework::Tensor* labels, const bool softLabel); }; } // namespace math diff --git a/paddle/operators/math/gru_compute.cc b/paddle/operators/math/gru_compute.cc index ae4e47b014..d570c68cd4 100644 --- a/paddle/operators/math/gru_compute.cc +++ b/paddle/operators/math/gru_compute.cc @@ -19,14 +19,14 @@ namespace operators { namespace math { template -struct GRUUnitFunctor { - static void compute(const platform::DeviceContext &context, +struct GRUUnitFunctor { + static void compute(const platform::CPUDeviceContext &context, hl_gru_value value, int frame_size, int batch_size, activation_mode_t active_node, activation_mode_t active_gate) { #ifndef __NVCC__ if (value.prev_out_value) { - math::gemm( + math::gemm( context, false, false, batch_size, frame_size * 2, frame_size, 1, value.prev_out_value, frame_size, value.gate_weight, frame_size * 2, 1, value.gate_value, frame_size * 3); @@ -36,7 +36,7 @@ struct GRUUnitFunctor { frame_size, batch_size, active_gate); if (value.prev_out_value) { - math::gemm( + math::gemm( context, false, false, batch_size, frame_size, frame_size, 1, value.reset_output_value, frame_size, value.state_weight, frame_size, 1, value.gate_value + frame_size * 2, frame_size * 3); @@ -49,8 +49,8 @@ struct GRUUnitFunctor { }; template -struct GRUUnitGradFunctor { - static void compute(const platform::DeviceContext &context, +struct GRUUnitGradFunctor { + static void compute(const platform::CPUDeviceContext &context, hl_gru_value value, hl_gru_grad grad, int frame_size, int batch_size, activation_mode_t active_node, @@ -60,13 +60,13 @@ struct GRUUnitGradFunctor { grad, frame_size, batch_size, active_node); if (value.prev_out_value && grad.prev_out_grad) { - math::gemm( + math::gemm( context, false, true, batch_size, frame_size, frame_size, 1, grad.gate_grad + frame_size * 2, frame_size * 3, value.state_weight, frame_size, 0, grad.reset_output_grad, frame_size); if (grad.state_weight_grad) { - math::gemm( + math::gemm( context, true, false, frame_size, frame_size, batch_size, 1, value.reset_output_value, frame_size, grad.gate_grad + frame_size * 2, frame_size * 3, 1, @@ -78,13 +78,13 @@ struct GRUUnitGradFunctor { grad, frame_size, batch_size, active_gate); if (grad.prev_out_grad && value.prev_out_value) { - math::gemm( + math::gemm( context, false, true, batch_size, frame_size, frame_size * 2, 1, grad.gate_grad, frame_size * 3, value.gate_weight, frame_size * 2, 1, grad.prev_out_grad, frame_size); if (grad.gate_weight_grad) { - math::gemm( + math::gemm( context, true, false, frame_size, frame_size * 2, batch_size, 1, value.prev_out_value, frame_size, grad.gate_grad, frame_size * 3, 1, grad.gate_weight_grad, frame_size * 2); @@ -94,10 +94,10 @@ struct GRUUnitGradFunctor { } }; -template struct GRUUnitFunctor; -template struct GRUUnitFunctor; -template struct GRUUnitGradFunctor; -template struct GRUUnitGradFunctor; +template struct GRUUnitFunctor; +template struct GRUUnitFunctor; +template struct GRUUnitGradFunctor; +template struct GRUUnitGradFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/gru_compute.cu b/paddle/operators/math/gru_compute.cu index 0252bdbdb6..dd518cd1e4 100644 --- a/paddle/operators/math/gru_compute.cu +++ b/paddle/operators/math/gru_compute.cu @@ -19,13 +19,12 @@ namespace operators { namespace math { template -struct GRUUnitFunctor { - static void compute(const platform::DeviceContext &context, +struct GRUUnitFunctor { + static void compute(const platform::CUDADeviceContext &context, hl_gru_value value, int frame_size, int batch_size, activation_mode_t active_node, activation_mode_t active_gate) { - auto stream = - reinterpret_cast(context).stream(); + auto stream = context.stream(); dim3 threads; dim3 grid; if (batch_size == 1) { @@ -39,7 +38,7 @@ struct GRUUnitFunctor { } if (value.prev_out_value) { - math::gemm( + math::gemm( context, false, false, batch_size, frame_size * 2, frame_size, 1, value.prev_out_value, frame_size, value.gate_weight, frame_size * 2, 1, value.gate_value, frame_size * 3); @@ -62,7 +61,7 @@ struct GRUUnitFunctor { } if (value.prev_out_value) { - math::gemm( + math::gemm( context, false, false, batch_size, frame_size, frame_size, 1, value.reset_output_value, frame_size, value.state_weight, frame_size, 1, value.gate_value + frame_size * 2, frame_size * 3); @@ -87,14 +86,13 @@ struct GRUUnitFunctor { }; template -struct GRUUnitGradFunctor { - static void compute(const platform::DeviceContext &context, +struct GRUUnitGradFunctor { + static void compute(const platform::CUDADeviceContext &context, hl_gru_value value, hl_gru_grad grad, int frame_size, int batch_size, activation_mode_t active_node, activation_mode_t active_gate) { - auto stream = - reinterpret_cast(context).stream(); + auto stream = context.stream(); dim3 threads; dim3 grid; if (batch_size == 1) { @@ -124,13 +122,13 @@ struct GRUUnitGradFunctor { } if (value.prev_out_value && grad.prev_out_grad) { - math::gemm( + math::gemm( context, false, true, batch_size, frame_size, frame_size, 1, grad.gate_grad + frame_size * 2, frame_size * 3, value.state_weight, frame_size, 0, grad.reset_output_grad, frame_size); if (grad.state_weight_grad) { - math::gemm( + math::gemm( context, true, false, frame_size, frame_size, batch_size, 1, value.reset_output_value, frame_size, grad.gate_grad + frame_size * 2, frame_size * 3, 1, @@ -155,13 +153,13 @@ struct GRUUnitGradFunctor { } if (grad.prev_out_grad && value.prev_out_value) { - math::gemm( + math::gemm( context, false, true, batch_size, frame_size, frame_size * 2, 1, grad.gate_grad, frame_size * 3, value.gate_weight, frame_size * 2, 1, grad.prev_out_grad, frame_size); if (grad.gate_weight_grad) { - math::gemm( + math::gemm( context, true, false, frame_size, frame_size * 2, batch_size, 1, value.prev_out_value, frame_size, grad.gate_grad, frame_size * 3, 1, grad.gate_weight_grad, frame_size * 2); @@ -170,10 +168,10 @@ struct GRUUnitGradFunctor { } }; -template struct GRUUnitFunctor; -template struct GRUUnitFunctor; -template struct GRUUnitGradFunctor; -template struct GRUUnitGradFunctor; +template struct GRUUnitFunctor; +template struct GRUUnitFunctor; +template struct GRUUnitGradFunctor; +template struct GRUUnitGradFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/gru_compute.h b/paddle/operators/math/gru_compute.h index 58ea59f68e..ca1343cb2c 100644 --- a/paddle/operators/math/gru_compute.h +++ b/paddle/operators/math/gru_compute.h @@ -40,19 +40,18 @@ struct hl_gru_grad { T *prev_out_grad; }; -template +template struct GRUUnitFunctor { - static void compute(const platform::DeviceContext &context, - hl_gru_value value, int frame_size, int batch_size, + static void compute(const DeviceContext &context, hl_gru_value value, + int frame_size, int batch_size, activation_mode_t active_node, activation_mode_t active_gate); }; -template +template struct GRUUnitGradFunctor { - static void compute(const platform::DeviceContext &context, - hl_gru_value value, hl_gru_grad grad, - int frame_size, int batch_size, + static void compute(const DeviceContext &context, hl_gru_value value, + hl_gru_grad grad, int frame_size, int batch_size, activation_mode_t active_node, activation_mode_t active_gate); }; diff --git a/paddle/operators/math/im2col.cc b/paddle/operators/math/im2col.cc index c10c44c520..707ebf0596 100644 --- a/paddle/operators/math/im2col.cc +++ b/paddle/operators/math/im2col.cc @@ -25,9 +25,9 @@ namespace math { */ template class Im2ColFunctor { + platform::CPUDeviceContext, T> { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& im, const std::vector& dilation, const std::vector& stride, const std::vector& padding, framework::Tensor* col) { @@ -90,9 +90,9 @@ class Im2ColFunctor class Col2ImFunctor { + platform::CPUDeviceContext, T> { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& col, const std::vector& dilation, const std::vector& stride, @@ -149,13 +149,13 @@ class Col2ImFunctor; + platform::CPUDeviceContext, float>; template class Im2ColFunctor; + platform::CPUDeviceContext, double>; template class Col2ImFunctor; + platform::CPUDeviceContext, float>; template class Col2ImFunctor; + platform::CPUDeviceContext, double>; /* * im = [input_channels, input_height, input_width] @@ -164,9 +164,9 @@ template class Col2ImFunctor class Im2ColFunctor { + platform::CPUDeviceContext, T> { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& im, const std::vector& dilation, const std::vector& stride, const std::vector& padding, framework::Tensor* col) { @@ -235,9 +235,9 @@ class Im2ColFunctor class Col2ImFunctor { + platform::CPUDeviceContext, T> { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& col, const std::vector& dilation, const std::vector& stride, @@ -300,13 +300,13 @@ class Col2ImFunctor; + platform::CPUDeviceContext, float>; template class Im2ColFunctor; + platform::CPUDeviceContext, double>; template class Col2ImFunctor; + platform::CPUDeviceContext, float>; template class Col2ImFunctor; + platform::CPUDeviceContext, double>; } // namespace math } // namespace operators diff --git a/paddle/operators/math/im2col.cu b/paddle/operators/math/im2col.cu index bf78942439..a88e837b03 100644 --- a/paddle/operators/math/im2col.cu +++ b/paddle/operators/math/im2col.cu @@ -58,9 +58,9 @@ __global__ void im2col(const T* data_im, int num_outs, int im_height, */ template class Im2ColFunctor { + platform::CUDADeviceContext, T> { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& im, const std::vector& dilation, const std::vector& stride, const std::vector& padding, framework::Tensor* col) { @@ -96,9 +96,7 @@ class Im2ColFunctor<<(context) - .stream()>>>( + im2col<<>>( im.data(), num_outputs, im_height, im_width, dilation[0], dilation[1], filter_height, filter_width, stride[0], stride[1], padding[0], padding[1], col_height, col_width, col->data()); @@ -160,9 +158,9 @@ __global__ void col2im(int n, const T* data_col, int im_height, int im_width, */ template class Col2ImFunctor { + platform::CUDADeviceContext, T> { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& col, const std::vector& dilation, const std::vector& stride, @@ -203,9 +201,7 @@ class Col2ImFunctor<<(context) - .stream()>>>( + col2im<<>>( num_kernels, col.data(), im_height, im_width, dilation[0], dilation[1], filter_height, filter_width, stride[0], stride[1], padding[0], padding[2], col_height, col_width, im->data()); @@ -213,13 +209,13 @@ class Col2ImFunctor; + platform::CUDADeviceContext, float>; template class Im2ColFunctor; + platform::CUDADeviceContext, double>; template class Col2ImFunctor; + platform::CUDADeviceContext, float>; template class Col2ImFunctor; + platform::CUDADeviceContext, double>; template __global__ void im2colOCF(const T* im_data, int im_channels, int im_height, @@ -260,9 +256,9 @@ __global__ void im2colOCF(const T* im_data, int im_channels, int im_height, */ template class Im2ColFunctor { + platform::CUDADeviceContext, T> { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& im, const std::vector& dilation, const std::vector& stride, const std::vector& padding, framework::Tensor* col) { @@ -310,9 +306,7 @@ class Im2ColFunctor<<(context) - .stream()>>>( + im2colOCF<<>>( im.data(), im_channels, im_height, im_width, filter_height, filter_width, stride[0], stride[1], padding[0], padding[1], col_height, col_width, col->data()); @@ -358,9 +352,9 @@ __global__ void col2imOCF(const T* col_data, int im_channels, int im_height, */ template class Col2ImFunctor { + platform::CUDADeviceContext, T> { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& col, const std::vector& dilation, const std::vector& stride, @@ -409,9 +403,7 @@ class Col2ImFunctor<<(context) - .stream()>>>( + col2imOCF<<>>( col.data(), im_channels, im_height, im_width, filter_height, filter_width, stride[0], stride[1], padding[0], padding[1], col_height, col_width, im->data()); @@ -419,13 +411,13 @@ class Col2ImFunctor; + platform::CUDADeviceContext, float>; template class Im2ColFunctor; + platform::CUDADeviceContext, double>; template class Col2ImFunctor; + platform::CUDADeviceContext, float>; template class Col2ImFunctor; + platform::CUDADeviceContext, double>; } // namespace math } // namespace operators diff --git a/paddle/operators/math/im2col.h b/paddle/operators/math/im2col.h index 24fd9a06e9..38f2c9fe0a 100644 --- a/paddle/operators/math/im2col.h +++ b/paddle/operators/math/im2col.h @@ -79,20 +79,19 @@ enum class ColFormat { kCFO = 0, kOCF = 1 }; * \note The caller needs to ensure that imShape.inputChannels is equal to * colShape.inputChannels. */ -template +template class Im2ColFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& im, const std::vector& dilation, + void operator()(const DeviceContext& context, const framework::Tensor& im, + const std::vector& dilation, const std::vector& stride, const std::vector& padding, framework::Tensor* col); }; -template +template class Col2ImFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& col, + void operator()(const DeviceContext& context, const framework::Tensor& col, const std::vector& dilation, const std::vector& stride, const std::vector& padding, framework::Tensor* im); diff --git a/paddle/operators/math/im2col_test.cc b/paddle/operators/math/im2col_test.cc index ae197a97ed..256f3bc9bd 100644 --- a/paddle/operators/math/im2col_test.cc +++ b/paddle/operators/math/im2col_test.cc @@ -16,7 +16,7 @@ limitations under the License. */ #include #include -template +template void testIm2col() { paddle::framework::Tensor input_tmp; paddle::framework::Tensor input; @@ -59,18 +59,7 @@ void testIm2col() { memcpy(input_ptr, arr, 6 * sizeof(float)); auto* place = new Place(); - paddle::platform::DeviceContext* context; - if (paddle::platform::is_cpu_place(*place)) { - context = - new paddle::platform::CPUDeviceContext(paddle::platform::CPUPlace()); - } else { -#ifdef PADDLE_WITH_CUDA - context = - new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace()); -#else - PADDLE_THROW("no GPU support"); -#endif // PADDLE_WITH_CUDA - } + DeviceContext* context = new DeviceContext(*place); if (paddle::platform::is_cpu_place(*place)) { input = input_tmp; } else { @@ -83,10 +72,10 @@ void testIm2col() { // Im2Col paddle::operators::math::Im2ColFunctor< - paddle::operators::math::ColFormat::kCFO, Place, float> + paddle::operators::math::ColFormat::kCFO, DeviceContext, float> im2col; paddle::operators::math::Im2ColFunctor< - paddle::operators::math::ColFormat::kOCF, Place, float> + paddle::operators::math::ColFormat::kOCF, DeviceContext, float> im2col_ocf; im2col(*context, input, dilation, stride, padding, &output_cfo); @@ -119,10 +108,10 @@ void testIm2col() { // Col2Im: kCFO paddle::operators::math::Col2ImFunctor< - paddle::operators::math::ColFormat::kCFO, Place, float> + paddle::operators::math::ColFormat::kCFO, DeviceContext, float> col2im; paddle::operators::math::Col2ImFunctor< - paddle::operators::math::ColFormat::kOCF, Place, float> + paddle::operators::math::ColFormat::kOCF, DeviceContext, float> col2im_ocf; float col2im_data[] = {0, 2, 2, 3, 8, 5}; @@ -168,8 +157,8 @@ void testIm2col() { } TEST(math, im2col) { - testIm2col(); + testIm2col(); #ifdef PADDLE_WITH_CUDA - testIm2col(); + testIm2col(); #endif } diff --git a/paddle/operators/math/lstm_compute.cc b/paddle/operators/math/lstm_compute.cc index ad3a59bcdb..2c2e8bb82e 100644 --- a/paddle/operators/math/lstm_compute.cc +++ b/paddle/operators/math/lstm_compute.cc @@ -21,8 +21,8 @@ namespace operators { namespace math { template -struct LstmUnitFunctor { - static void compute(const platform::DeviceContext& context, +struct LstmUnitFunctor { + static void compute(const platform::CPUDeviceContext& context, LstmMetaValue value, int frame_size, int batch_size, const std::string& gate_act, const std::string& cell_act, const std::string& cand_act) { @@ -42,8 +42,8 @@ struct LstmUnitFunctor { }; template -struct LstmUnitGradFunctor { - static void compute(const platform::DeviceContext& context, +struct LstmUnitGradFunctor { + static void compute(const platform::CPUDeviceContext& context, LstmMetaValue value, LstmMetaGrad grad, int frame_size, int batch_size, const std::string& gate_act, const std::string& cell_act, @@ -72,10 +72,10 @@ struct LstmUnitGradFunctor { } }; -template class LstmUnitFunctor; -template class LstmUnitFunctor; -template class LstmUnitGradFunctor; -template class LstmUnitGradFunctor; +template class LstmUnitFunctor; +template class LstmUnitFunctor; +template class LstmUnitGradFunctor; +template class LstmUnitGradFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/lstm_compute.cu b/paddle/operators/math/lstm_compute.cu index b2122f2a5c..92b1f4228b 100644 --- a/paddle/operators/math/lstm_compute.cu +++ b/paddle/operators/math/lstm_compute.cu @@ -21,8 +21,8 @@ namespace operators { namespace math { template -struct LstmUnitFunctor { - static void compute(const platform::DeviceContext& context, +struct LstmUnitFunctor { + static void compute(const platform::CUDADeviceContext& context, LstmMetaValue value, int frame_size, int batch_size, const std::string& gate_act, const std::string& cell_act, const std::string& cand_act) { @@ -33,8 +33,8 @@ struct LstmUnitFunctor { }; template -struct LstmUnitGradFunctor { - static void compute(const platform::DeviceContext& context, +struct LstmUnitGradFunctor { + static void compute(const platform::CUDADeviceContext& context, LstmMetaValue value, LstmMetaGrad grad, int frame_size, int batch_size, const std::string& gate_act, const std::string& cell_act, @@ -45,10 +45,10 @@ struct LstmUnitGradFunctor { } }; -template class LstmUnitFunctor; -template class LstmUnitFunctor; -template class LstmUnitGradFunctor; -template class LstmUnitGradFunctor; +template class LstmUnitFunctor; +template class LstmUnitFunctor; +template class LstmUnitGradFunctor; +template class LstmUnitGradFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/lstm_compute.h b/paddle/operators/math/lstm_compute.h index 9652399d4c..5f74e27358 100644 --- a/paddle/operators/math/lstm_compute.h +++ b/paddle/operators/math/lstm_compute.h @@ -67,21 +67,20 @@ inline activation_mode_t ActiveType(const std::string &type) { } } -template +template class LstmUnitFunctor { public: - static void compute(const platform::DeviceContext &context, - LstmMetaValue value, int frame_size, int batch_size, + static void compute(const DeviceContext &context, LstmMetaValue value, + int frame_size, int batch_size, const std::string &gate_act, const std::string &cell_act, const std::string &cand_act); }; -template +template class LstmUnitGradFunctor { public: - static void compute(const platform::DeviceContext &context, - LstmMetaValue value, LstmMetaGrad grad, - int frame_size, int batch_size, + static void compute(const DeviceContext &context, LstmMetaValue value, + LstmMetaGrad grad, int frame_size, int batch_size, const std::string &gate_act, const std::string &cell_act, const std::string &cand_act); }; diff --git a/paddle/operators/math/math_function.cc b/paddle/operators/math/math_function.cc index e099a6a439..2b35e4532a 100644 --- a/paddle/operators/math/math_function.cc +++ b/paddle/operators/math/math_function.cc @@ -21,13 +21,11 @@ namespace operators { namespace math { template <> -void gemm(const platform::DeviceContext& context, - const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, - const int N, const int K, - const float alpha, const float* A, - const float* B, const float beta, - float* C) { +void gemm( + const platform::CPUDeviceContext& context, const CBLAS_TRANSPOSE transA, + const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, + const float alpha, const float* A, const float* B, const float beta, + float* C) { int lda = (transA == CblasNoTrans) ? K : M; int ldb = (transB == CblasNoTrans) ? N : K; int ldc = N; @@ -36,13 +34,11 @@ void gemm(const platform::DeviceContext& context, } template <> -void gemm(const platform::DeviceContext& context, - const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, - const int N, const int K, - const double alpha, const double* A, - const double* B, const double beta, - double* C) { +void gemm( + const platform::CPUDeviceContext& context, const CBLAS_TRANSPOSE transA, + const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, + const double alpha, const double* A, const double* B, const double beta, + double* C) { int lda = (transA == CblasNoTrans) ? K : M; int ldb = (transB == CblasNoTrans) ? N : K; int ldc = N; @@ -51,35 +47,32 @@ void gemm(const platform::DeviceContext& context, } template <> -void gemm(const platform::DeviceContext& context, - const bool transA, const bool transB, - const int M, const int N, const int K, - const float alpha, const float* A, - const int lda, const float* B, - const int ldb, const float beta, float* C, - const int ldc) { +void gemm( + const platform::CPUDeviceContext& context, const bool transA, + const bool transB, const int M, const int N, const int K, const float alpha, + const float* A, const int lda, const float* B, const int ldb, + const float beta, float* C, const int ldc) { cblas_sgemm(CblasRowMajor, transA == false ? CblasNoTrans : CblasTrans, transB == false ? CblasNoTrans : CblasTrans, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); } template <> -void gemm(const platform::DeviceContext& context, - const bool transA, const bool transB, - const int M, const int N, const int K, - const double alpha, const double* A, - const int lda, const double* B, - const int ldb, const double beta, - double* C, const int ldc) { +void gemm( + const platform::CPUDeviceContext& context, const bool transA, + const bool transB, const int M, const int N, const int K, + const double alpha, const double* A, const int lda, const double* B, + const int ldb, const double beta, double* C, const int ldc) { cblas_dgemm(CblasRowMajor, transA == false ? CblasNoTrans : CblasTrans, transB == false ? CblasNoTrans : CblasTrans, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); } template <> -void matmul( - const platform::DeviceContext& context, const framework::Tensor& matrix_a, - bool trans_a, const framework::Tensor& matrix_b, bool trans_b, float alpha, +void matmul( + const platform::CPUDeviceContext& context, + const framework::Tensor& matrix_a, bool trans_a, + const framework::Tensor& matrix_b, bool trans_b, float alpha, framework::Tensor* matrix_out, float beta) { auto dim_a = matrix_a.dims(); auto dim_b = matrix_b.dims(); @@ -99,15 +92,16 @@ void matmul( CBLAS_TRANSPOSE transA = (trans_a == false) ? CblasNoTrans : CblasTrans; CBLAS_TRANSPOSE transB = (trans_b == false) ? CblasNoTrans : CblasTrans; - gemm( + gemm( context, transA, transB, M, N, K, alpha, matrix_a.data(), matrix_b.data(), beta, matrix_out->data()); } template <> -void matmul( - const platform::DeviceContext& context, const framework::Tensor& matrix_a, - bool trans_a, const framework::Tensor& matrix_b, bool trans_b, double alpha, +void matmul( + const platform::CPUDeviceContext& context, + const framework::Tensor& matrix_a, bool trans_a, + const framework::Tensor& matrix_b, bool trans_b, double alpha, framework::Tensor* matrix_out, double beta) { auto dim_a = matrix_a.dims(); auto dim_b = matrix_b.dims(); @@ -127,7 +121,7 @@ void matmul( CBLAS_TRANSPOSE transA = (trans_a == false) ? CblasNoTrans : CblasTrans; CBLAS_TRANSPOSE transB = (trans_b == false) ? CblasNoTrans : CblasTrans; - gemm( + gemm( context, transA, transB, M, N, K, alpha, matrix_a.data(), matrix_b.data(), beta, matrix_out->data()); } @@ -135,8 +129,8 @@ void matmul( #ifdef PADDLE_WITH_MKLML // Use cblas_{s,d}gemm_batched if available: Run with 1 group of size batchSize. template <> -void batched_gemm( - const platform::DeviceContext& context, const CBLAS_TRANSPOSE transA, +void batched_gemm( + const platform::CPUDeviceContext& context, const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, const float alpha, const float* A, const float* B, const float beta, float* C, const int batchCount, const int strideA, const int strideB) { @@ -157,8 +151,8 @@ void batched_gemm( } template <> -void batched_gemm( - const platform::DeviceContext& context, const CBLAS_TRANSPOSE transA, +void batched_gemm( + const platform::CPUDeviceContext& context, const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, const double alpha, const double* A, const double* B, const double beta, double* C, const int batchCount, const int strideA, const int strideB) { @@ -183,8 +177,8 @@ void batched_gemm( // functions of Intel MKL are not available. In the future, this computation // should be parallelized. template <> -void batched_gemm( - const platform::DeviceContext& context, const CBLAS_TRANSPOSE transA, +void batched_gemm( + const platform::CPUDeviceContext& context, const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, const float alpha, const float* A, const float* B, const float beta, float* C, const int batchCount, const int strideA, const int strideB) { @@ -192,14 +186,14 @@ void batched_gemm( const float* Ak = &A[k * strideA]; const float* Bk = &B[k * strideB]; float* Ck = &C[k * M * N]; - gemm(context, transA, transB, M, N, K, alpha, Ak, - Bk, beta, Ck); + gemm(context, transA, transB, M, N, K, + alpha, Ak, Bk, beta, Ck); } } template <> -void batched_gemm( - const platform::DeviceContext& context, const CBLAS_TRANSPOSE transA, +void batched_gemm( + const platform::CPUDeviceContext& context, const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, const double alpha, const double* A, const double* B, const double beta, double* C, const int batchCount, const int strideA, const int strideB) { @@ -207,55 +201,53 @@ void batched_gemm( const double* Ak = &A[k * strideA]; const double* Bk = &B[k * strideB]; double* Ck = &C[k * M * N]; - gemm(context, transA, transB, M, N, K, alpha, - Ak, Bk, beta, Ck); + gemm(context, transA, transB, M, N, K, + alpha, Ak, Bk, beta, Ck); } } #endif template <> -void gemv(const platform::DeviceContext& context, - const bool trans_a, const int M, - const int N, const float alpha, - const float* A, const float* B, - const float beta, float* C) { +void gemv( + const platform::CPUDeviceContext& context, const bool trans_a, const int M, + const int N, const float alpha, const float* A, const float* B, + const float beta, float* C) { CBLAS_TRANSPOSE transA = (trans_a == false) ? CblasNoTrans : CblasTrans; cblas_sgemv(CblasRowMajor, transA, M, N, alpha, A, N, B, 1, beta, C, 1); } template <> -void gemv(const platform::DeviceContext& context, - const bool trans_a, const int M, - const int N, const double alpha, - const double* A, const double* B, - const double beta, double* C) { +void gemv( + const platform::CPUDeviceContext& context, const bool trans_a, const int M, + const int N, const double alpha, const double* A, const double* B, + const double beta, double* C) { CBLAS_TRANSPOSE transA = (trans_a == false) ? CblasNoTrans : CblasTrans; cblas_dgemv(CblasRowMajor, transA, M, N, alpha, A, N, B, 1, beta, C, 1); } template <> -void axpy(const platform::DeviceContext& context, - const int n, const float alpha, - const float* x, float* y) { +void axpy( + const platform::CPUDeviceContext& context, const int n, const float alpha, + const float* x, float* y) { cblas_saxpy(n, alpha, x, 1, y, 1); } template <> -void axpy(const platform::DeviceContext& context, - const int n, const double alpha, - const double* x, double* y) { +void axpy( + const platform::CPUDeviceContext& context, const int n, const double alpha, + const double* x, double* y) { cblas_daxpy(n, alpha, x, 1, y, 1); } -template struct SetConstant; -template struct SetConstant; -template struct SetConstant; -template struct SetConstant; -template struct SetConstant; +template struct SetConstant; +template struct SetConstant; +template struct SetConstant; +template struct SetConstant; +template struct SetConstant; -#define DEFINE_CPU_TRANS(RANK) \ - template struct Transpose; \ - template struct Transpose; +#define DEFINE_CPU_TRANS(RANK) \ + template struct Transpose; \ + template struct Transpose; DEFINE_CPU_TRANS(1); DEFINE_CPU_TRANS(2); @@ -310,10 +302,10 @@ void set_constant(const platform::DeviceContext& context, #endif } -template struct RowwiseAdd; -template struct RowwiseAdd; -template struct ColwiseSum; -template struct ColwiseSum; +template struct RowwiseAdd; +template struct RowwiseAdd; +template struct ColwiseSum; +template struct ColwiseSum; } // namespace math } // namespace operators diff --git a/paddle/operators/math/math_function.cu b/paddle/operators/math/math_function.cu index 3018e50a4f..1b560a7e2d 100644 --- a/paddle/operators/math/math_function.cu +++ b/paddle/operators/math/math_function.cu @@ -22,13 +22,11 @@ namespace operators { namespace math { template <> -void gemm(const platform::DeviceContext& context, - const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, - const int N, const int K, - const float alpha, const float* A, - const float* B, const float beta, - float* C) { +void gemm( + const platform::CUDADeviceContext& context, const CBLAS_TRANSPOSE transA, + const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, + const float alpha, const float* A, const float* B, const float beta, + float* C) { // Note that cublas follows fortran order, so the order is different from // the cblas convention. int lda = (transA == CblasNoTrans) ? K : M; @@ -39,19 +37,16 @@ void gemm(const platform::DeviceContext& context, (transB == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; PADDLE_ENFORCE(platform::dynload::cublasSgemm( - reinterpret_cast(context) - .cublas_handle(), - cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, lda, &beta, C, N)); + context.cublas_handle(), cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, + lda, &beta, C, N)); } template <> -void gemm(const platform::DeviceContext& context, - const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, - const int N, const int K, - const double alpha, const double* A, - const double* B, const double beta, - double* C) { +void gemm( + const platform::CUDADeviceContext& context, const CBLAS_TRANSPOSE transA, + const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, + const double alpha, const double* A, const double* B, const double beta, + double* C) { // Note that cublas follows fortran order, so the order is different from // the cblas convention. int lda = (transA == CblasNoTrans) ? K : M; @@ -61,51 +56,45 @@ void gemm(const platform::DeviceContext& context, cublasOperation_t cuTransB = (transB == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; PADDLE_ENFORCE(platform::dynload::cublasDgemm( - reinterpret_cast(context) - .cublas_handle(), - cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, lda, &beta, C, N)); + context.cublas_handle(), cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, + lda, &beta, C, N)); } template <> -void gemm(const platform::DeviceContext& context, - const bool transA, const bool transB, - const int M, const int N, const int K, - const float alpha, const float* A, - const int lda, const float* B, - const int ldb, const float beta, float* C, - const int ldc) { +void gemm( + const platform::CUDADeviceContext& context, const bool transA, + const bool transB, const int M, const int N, const int K, const float alpha, + const float* A, const int lda, const float* B, const int ldb, + const float beta, float* C, const int ldc) { // Note that cublas follows fortran order, so the order is different from // the cblas convention. cublasOperation_t cuTransA = transA == false ? CUBLAS_OP_N : CUBLAS_OP_T; cublasOperation_t cuTransB = transB == false ? CUBLAS_OP_N : CUBLAS_OP_T; PADDLE_ENFORCE(platform::dynload::cublasSgemm( - reinterpret_cast(context) - .cublas_handle(), - cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, lda, &beta, C, ldc)); + context.cublas_handle(), cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, + lda, &beta, C, ldc)); } template <> -void gemm(const platform::DeviceContext& context, - const bool transA, const bool transB, - const int M, const int N, const int K, - const double alpha, const double* A, - const int lda, const double* B, - const int ldb, const double beta, - double* C, const int ldc) { +void gemm( + const platform::CUDADeviceContext& context, const bool transA, + const bool transB, const int M, const int N, const int K, + const double alpha, const double* A, const int lda, const double* B, + const int ldb, const double beta, double* C, const int ldc) { // Note that cublas follows fortran order, so the order is different from // the cblas convention. cublasOperation_t cuTransA = transA == false ? CUBLAS_OP_N : CUBLAS_OP_T; cublasOperation_t cuTransB = transB == false ? CUBLAS_OP_N : CUBLAS_OP_T; PADDLE_ENFORCE(platform::dynload::cublasDgemm( - reinterpret_cast(context) - .cublas_handle(), - cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, lda, &beta, C, ldc)); + context.cublas_handle(), cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, + lda, &beta, C, ldc)); } template <> -void matmul( - const platform::DeviceContext& context, const framework::Tensor& matrix_a, - bool trans_a, const framework::Tensor& matrix_b, bool trans_b, float alpha, +void matmul( + const platform::CUDADeviceContext& context, + const framework::Tensor& matrix_a, bool trans_a, + const framework::Tensor& matrix_b, bool trans_b, float alpha, framework::Tensor* matrix_out, float beta) { auto dim_a = matrix_a.dims(); auto dim_b = matrix_b.dims(); @@ -125,15 +114,16 @@ void matmul( CBLAS_TRANSPOSE transA = (trans_a == false) ? CblasNoTrans : CblasTrans; CBLAS_TRANSPOSE transB = (trans_b == false) ? CblasNoTrans : CblasTrans; - gemm( + gemm( context, transA, transB, M, N, K, alpha, matrix_a.data(), matrix_b.data(), beta, matrix_out->data()); } template <> -void matmul( - const platform::DeviceContext& context, const framework::Tensor& matrix_a, - bool trans_a, const framework::Tensor& matrix_b, bool trans_b, double alpha, +void matmul( + const platform::CUDADeviceContext& context, + const framework::Tensor& matrix_a, bool trans_a, + const framework::Tensor& matrix_b, bool trans_b, double alpha, framework::Tensor* matrix_out, double beta) { auto dim_a = matrix_a.dims(); auto dim_b = matrix_b.dims(); @@ -153,14 +143,14 @@ void matmul( CBLAS_TRANSPOSE transA = (trans_a == false) ? CblasNoTrans : CblasTrans; CBLAS_TRANSPOSE transB = (trans_b == false) ? CblasNoTrans : CblasTrans; - gemm( + gemm( context, transA, transB, M, N, K, alpha, matrix_a.data(), matrix_b.data(), beta, matrix_out->data()); } template <> -void batched_gemm( - const platform::DeviceContext& context, const CBLAS_TRANSPOSE transA, +void batched_gemm( + const platform::CUDADeviceContext& context, const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, const float alpha, const float* A, const float* B, const float beta, float* C, const int batchCount, const int strideA, const int strideB) { @@ -176,15 +166,13 @@ void batched_gemm( const int strideC = M * N; PADDLE_ENFORCE(platform::dynload::cublasSgemmStridedBatched( - reinterpret_cast(context) - .cublas_handle(), - cuTransB, cuTransA, N, M, K, &alpha, B, ldb, strideB, A, lda, strideA, - &beta, C, ldc, strideC, batchCount)); + context.cublas_handle(), cuTransB, cuTransA, N, M, K, &alpha, B, ldb, + strideB, A, lda, strideA, &beta, C, ldc, strideC, batchCount)); } template <> -void batched_gemm( - const platform::DeviceContext& context, const CBLAS_TRANSPOSE transA, +void batched_gemm( + const platform::CUDADeviceContext& context, const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, const double alpha, const double* A, const double* B, const double beta, double* C, const int batchCount, const int strideA, const int strideB) { @@ -200,68 +188,58 @@ void batched_gemm( const int strideC = M * N; PADDLE_ENFORCE(platform::dynload::cublasDgemmStridedBatched( - reinterpret_cast(context) - .cublas_handle(), - cuTransB, cuTransA, N, M, K, &alpha, B, ldb, strideB, A, lda, strideA, - &beta, C, ldc, strideC, batchCount)); + context.cublas_handle(), cuTransB, cuTransA, N, M, K, &alpha, B, ldb, + strideB, A, lda, strideA, &beta, C, ldc, strideC, batchCount)); } template <> -void gemv(const platform::DeviceContext& context, - const bool trans_a, const int M, - const int N, const float alpha, - const float* A, const float* B, - const float beta, float* C) { +void gemv( + const platform::CUDADeviceContext& context, const bool trans_a, const int M, + const int N, const float alpha, const float* A, const float* B, + const float beta, float* C) { cublasOperation_t cuTransA = (trans_a == false) ? CUBLAS_OP_T : CUBLAS_OP_N; - PADDLE_ENFORCE(platform::dynload::cublasSgemv( - reinterpret_cast(context) - .cublas_handle(), - cuTransA, N, M, &alpha, A, N, B, 1, &beta, C, 1)); + PADDLE_ENFORCE(platform::dynload::cublasSgemv(context.cublas_handle(), + cuTransA, N, M, &alpha, A, N, B, + 1, &beta, C, 1)); } template <> -void gemv(const platform::DeviceContext& context, - const bool trans_a, const int M, - const int N, const double alpha, - const double* A, const double* B, - const double beta, double* C) { +void gemv( + const platform::CUDADeviceContext& context, const bool trans_a, const int M, + const int N, const double alpha, const double* A, const double* B, + const double beta, double* C) { cublasOperation_t cuTransA = (trans_a == false) ? CUBLAS_OP_T : CUBLAS_OP_N; - PADDLE_ENFORCE(platform::dynload::cublasDgemv( - reinterpret_cast(context) - .cublas_handle(), - cuTransA, N, M, &alpha, A, N, B, 1, &beta, C, 1)); + PADDLE_ENFORCE(platform::dynload::cublasDgemv(context.cublas_handle(), + cuTransA, N, M, &alpha, A, N, B, + 1, &beta, C, 1)); } template <> -void axpy(const platform::DeviceContext& context, - const int n, const float alpha, - const float* x, float* y) { - PADDLE_ENFORCE(platform::dynload::cublasSaxpy( - reinterpret_cast(context) - .cublas_handle(), - n, &alpha, x, 1, y, 1)); +void axpy( + const platform::CUDADeviceContext& context, const int n, const float alpha, + const float* x, float* y) { + PADDLE_ENFORCE(platform::dynload::cublasSaxpy(context.cublas_handle(), n, + &alpha, x, 1, y, 1)); } template <> -void axpy(const platform::DeviceContext& context, - const int n, const double alpha, - const double* x, double* y) { - PADDLE_ENFORCE(platform::dynload::cublasDaxpy( - reinterpret_cast(context) - .cublas_handle(), - n, &alpha, x, 1, y, 1)); +void axpy( + const platform::CUDADeviceContext& context, const int n, const double alpha, + const double* x, double* y) { + PADDLE_ENFORCE(platform::dynload::cublasDaxpy(context.cublas_handle(), n, + &alpha, x, 1, y, 1)); } -template struct SetConstant; -template struct SetConstant; -template struct SetConstant; -template struct SetConstant; -template struct SetConstant; +template struct SetConstant; +template struct SetConstant; +template struct SetConstant; +template struct SetConstant; +template struct SetConstant; -#define DEFINE_GPU_TRANS(RANK) \ - template struct Transpose; \ - template struct Transpose; +#define DEFINE_GPU_TRANS(RANK) \ + template struct Transpose; \ + template struct Transpose; DEFINE_GPU_TRANS(1); DEFINE_GPU_TRANS(2); @@ -277,8 +255,9 @@ struct TensorSetConstantGPU { template void operator()() const { - SetConstant functor; - functor(context_, tensor_, static_cast(value_)); + SetConstant functor; + functor(reinterpret_cast(context_), + tensor_, static_cast(value_)); } const platform::DeviceContext& context_; @@ -294,27 +273,27 @@ void set_constant_with_place( TensorSetConstantGPU(context, tensor, value)); } -template struct RowwiseAdd; -template struct RowwiseAdd; -template struct ColwiseSum; -// template struct ColwiseSum; -// The ColwiseSum failed in debug mode, +template struct RowwiseAdd; +template struct RowwiseAdd; +template struct ColwiseSum; +// template struct ColwiseSum; +// The ColwiseSum failed in debug mode, // and only failed for this case. So reimplemented it. template <> -void ColwiseSum::operator()( - const platform::DeviceContext& context, const framework::Tensor& input, +void ColwiseSum::operator()( + const platform::CUDADeviceContext& context, const framework::Tensor& input, framework::Tensor* vector) { auto in_dims = input.dims(); auto size = input.numel() / in_dims[0]; PADDLE_ENFORCE_EQ(vector->numel(), size); framework::Tensor one; one.mutable_data({in_dims[0]}, context.GetPlace()); - SetConstant set; + SetConstant set; set(context, &one, static_cast(1.0)); - gemv(context, true, static_cast(in_dims[0]), - static_cast(in_dims[1]), 1.0, - input.data(), one.data(), - 0.0, vector->data()); + gemv( + context, true, static_cast(in_dims[0]), static_cast(in_dims[1]), + 1.0, input.data(), one.data(), 0.0, + vector->data()); } } // namespace math diff --git a/paddle/operators/math/math_function.h b/paddle/operators/math/math_function.h index f2b025b78b..8cc03c2ba0 100644 --- a/paddle/operators/math/math_function.h +++ b/paddle/operators/math/math_function.h @@ -62,53 +62,51 @@ namespace math { // Then matrixA: M * K, matrixB: K * N, matrixC : M * N // For more detailed info, please refer to // http://www.netlib.org/lapack/explore-html/d4/de2/sgemm_8f.html -template -void gemm(const platform::DeviceContext& context, const CBLAS_TRANSPOSE transA, +template +void gemm(const DeviceContext& context, const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, const T alpha, const T* A, const T* B, const T beta, T* C); // gemm wrapper with stride args for matrix uncontinuous in memory -template -void gemm(const platform::DeviceContext& context, const bool transA, - const bool transB, const int M, const int N, const int K, - const T alpha, const T* A, const int lda, const T* B, const int ldb, - const T beta, T* C, const int ldc); +template +void gemm(const DeviceContext& context, const bool transA, const bool transB, + const int M, const int N, const int K, const T alpha, const T* A, + const int lda, const T* B, const int ldb, const T beta, T* C, + const int ldc); // matrix multiply with continuous memory -template -void matmul(const platform::DeviceContext& context, - const framework::Tensor& matrix_a, bool trans_a, - const framework::Tensor& matrix_b, bool trans_b, T alpha, - framework::Tensor* matrix_out, T beta); +template +void matmul(const DeviceContext& context, const framework::Tensor& matrix_a, + bool trans_a, const framework::Tensor& matrix_b, bool trans_b, + T alpha, framework::Tensor* matrix_out, T beta); // Batched gemm -template -void batched_gemm(const platform::DeviceContext& context, - const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, - const int M, const int N, const int K, const T alpha, - const T* A, const T* B, const T beta, T* C, - const int batchCount, const int strideA, const int strideB); - -template -void gemv(const platform::DeviceContext& context, const bool trans_a, - const int M, const int N, const T alpha, const T* A, const T* B, - const T beta, T* C); - -template -void axpy(const platform::DeviceContext& context, const int n, const T alpha, - const T* x, T* y); - -template +template +void batched_gemm(const DeviceContext& context, const CBLAS_TRANSPOSE transA, + const CBLAS_TRANSPOSE transB, const int M, const int N, + const int K, const T alpha, const T* A, const T* B, + const T beta, T* C, const int batchCount, const int strideA, + const int strideB); + +template +void gemv(const DeviceContext& context, const bool trans_a, const int M, + const int N, const T alpha, const T* A, const T* B, const T beta, + T* C); + +template +void axpy(const DeviceContext& context, const int n, const T alpha, const T* x, + T* y); + +template struct Transpose { - void operator()(const platform::DeviceContext& context, - const framework::Tensor& in, framework::Tensor* out, - const std::vector& axis); + void operator()(const DeviceContext& context, const framework::Tensor& in, + framework::Tensor* out, const std::vector& axis); }; -template +template struct SetConstant { - void operator()(const platform::DeviceContext& context, - framework::Tensor* tensor, T num); + void operator()(const DeviceContext& context, framework::Tensor* tensor, + T num); }; template @@ -118,17 +116,16 @@ void set_constant_with_place(const platform::DeviceContext& context, void set_constant(const platform::DeviceContext& context, framework::Tensor* tensor, float value); -template +template struct RowwiseAdd { - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, const framework::Tensor& vec, - framework::Tensor* output); + void operator()(const DeviceContext& context, const framework::Tensor& input, + const framework::Tensor& vec, framework::Tensor* output); }; -template +template struct ColwiseSum { - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, framework::Tensor* vec); + void operator()(const DeviceContext& context, const framework::Tensor& input, + framework::Tensor* vec); }; } // namespace math diff --git a/paddle/operators/math/math_function_impl.h b/paddle/operators/math/math_function_impl.h index 4dc17a4e52..3e6d833865 100644 --- a/paddle/operators/math/math_function_impl.h +++ b/paddle/operators/math/math_function_impl.h @@ -20,16 +20,17 @@ namespace paddle { namespace operators { namespace math { -template -void SetConstant::operator()(const platform::DeviceContext& context, - framework::Tensor* tensor, T num) { +template +void SetConstant::operator()(const DeviceContext& context, + framework::Tensor* tensor, + T num) { auto t = framework::EigenVector::Flatten(*tensor); - t.device(*context.GetEigenDevice()) = t.constant(static_cast(num)); + t.device(*context.eigen_device()) = t.constant(static_cast(num)); } -template -void Transpose::operator()( - const platform::DeviceContext& context, const framework::Tensor& in, +template +void Transpose::operator()( + const DeviceContext& context, const framework::Tensor& in, framework::Tensor* out, const std::vector& axis) { Eigen::array permute; for (int i = 0; i < Rank; i++) { @@ -40,15 +41,15 @@ void Transpose::operator()( auto eigen_in = framework::EigenTensor::From(in); auto eigen_out = framework::EigenTensor::From(*out); - auto* dev = context.GetEigenDevice(); + auto* dev = context.eigen_device(); eigen_out.device(*dev) = eigen_in.shuffle(permute); } -template -void RowwiseAdd::operator()(const platform::DeviceContext& context, - const framework::Tensor& input, - const framework::Tensor& vector, - framework::Tensor* output) { +template +void RowwiseAdd::operator()(const DeviceContext& context, + const framework::Tensor& input, + const framework::Tensor& vector, + framework::Tensor* output) { auto in_dims = input.dims(); auto size = input.numel() / in_dims[0]; PADDLE_ENFORCE_EQ(vector.numel(), size); @@ -59,14 +60,14 @@ void RowwiseAdd::operator()(const platform::DeviceContext& context, auto out = framework::EigenMatrix::From(*output); Eigen::array shape({{1, static_cast(size)}}); Eigen::array bcast({{static_cast(in_dims[0]), 1}}); - out.device(*context.GetEigenDevice()) = + out.device(*context.eigen_device()) = in + vec.reshape(shape).broadcast(bcast); } -template -void ColwiseSum::operator()(const platform::DeviceContext& context, - const framework::Tensor& input, - framework::Tensor* vector) { +template +void ColwiseSum::operator()(const DeviceContext& context, + const framework::Tensor& input, + framework::Tensor* vector) { auto in_dims = input.dims(); auto size = input.numel() / in_dims[0]; PADDLE_ENFORCE_EQ(vector->numel(), size); @@ -74,7 +75,7 @@ void ColwiseSum::operator()(const platform::DeviceContext& context, auto vec = framework::EigenMatrix::From(*vector); auto in = framework::EigenMatrix::From(input); Eigen::array shape({{1, static_cast(size)}}); - vec.reshape(shape).device(*context.GetEigenDevice()) = + vec.reshape(shape).device(*context.eigen_device()) = in.sum(Eigen::array({{0}})).reshape(shape); } diff --git a/paddle/operators/math/math_function_test.cc b/paddle/operators/math/math_function_test.cc index 983c9fdcff..7c6f098ca9 100644 --- a/paddle/operators/math/math_function_test.cc +++ b/paddle/operators/math/math_function_test.cc @@ -21,7 +21,7 @@ TEST(math_function, gemm_notrans_cblas) { memcpy(input3_ptr, arr3, 8 * sizeof(float)); paddle::platform::CPUDeviceContext context(*cpu_place); - paddle::operators::math::gemm( + paddle::operators::math::gemm( context, false, false, m, n, k, 1, input1_ptr, 3, input2_ptr + 1, 4, 1, input3_ptr + 1, 4); @@ -55,7 +55,7 @@ TEST(math_function, gemm_trans_clbas) { memcpy(input3_ptr, arr3, 8 * sizeof(float)); paddle::platform::CPUDeviceContext context(*cpu_place); - paddle::operators::math::gemm( + paddle::operators::math::gemm( context, false, true, m, n, k, 1, input1_ptr, 3, input2_ptr + 3, 3, 1, input3_ptr + 1, 4); @@ -74,7 +74,8 @@ TEST(math_function, zero) { auto* cpu_place = new paddle::platform::CPUPlace(); float* t = tensor.mutable_data({2, 2}, *cpu_place); paddle::platform::CPUDeviceContext context(*cpu_place); - paddle::operators::math::SetConstant + paddle::operators::math::SetConstant functor; functor(context, &tensor, 0); EXPECT_EQ(t[0], 0); @@ -110,7 +111,7 @@ void GemvTest(int m, int n, bool trans) { } paddle::platform::CPUDeviceContext context(*cpu_place); - paddle::operators::math::gemv( + paddle::operators::math::gemv( context, trans, static_cast(m), static_cast(n), 1., data_a, data_b, 0., data_c); diff --git a/paddle/operators/math/math_function_test.cu b/paddle/operators/math/math_function_test.cu index d5d6f0c73b..32e96d9487 100644 --- a/paddle/operators/math/math_function_test.cu +++ b/paddle/operators/math/math_function_test.cu @@ -21,7 +21,7 @@ TEST(math_function, notrans_mul_trans) { out_gpu.mutable_data({2, 2}, *gpu_place); - paddle::operators::math::matmul( + paddle::operators::math::matmul( context, input1_gpu, false, input2_gpu, true, 1, &out_gpu, 0); paddle::framework::CopyFrom(out_gpu, *cpu_place, context, &out); @@ -55,7 +55,7 @@ TEST(math_function, trans_mul_notrans) { out_gpu.mutable_data({3, 3}, *gpu_place); - paddle::operators::math::matmul( + paddle::operators::math::matmul( context, input1_gpu, true, input2_gpu, false, 1, &out_gpu, 0); paddle::framework::CopyFrom(out_gpu, *cpu_place, context, &out); @@ -106,7 +106,7 @@ TEST(math_function, gemm_notrans_cublas) { float* b = input2_gpu.data(); float* c = input3_gpu.mutable_data(*gpu_place); - paddle::operators::math::gemm( + paddle::operators::math::gemm( context, false, false, m, n, k, 1, a, 3, b + 1, 4, 1, c + 1, 4); paddle::framework::CopyFrom(input3_gpu, *cpu_place, context, &input3); @@ -161,7 +161,7 @@ TEST(math_function, gemm_trans_cublas) { float* b = input2_gpu.data(); float* c = input3_gpu.mutable_data(*gpu_place); - paddle::operators::math::gemm( + paddle::operators::math::gemm( context, false, true, m, n, k, 1, a, 3, b + 3, 3, 1, c + 1, 4); paddle::framework::CopyFrom(input3_gpu, *cpu_place, context, &input3); @@ -208,7 +208,7 @@ void GemvTest(int m, int n, bool trans) { paddle::framework::CopyFrom(mat_a, *gpu_place, context, &g_mat_a); paddle::framework::CopyFrom(vec_b, *gpu_place, context, &g_vec_b); - paddle::operators::math::gemv( + paddle::operators::math::gemv( context, trans, static_cast(m), static_cast(n), 1., g_data_a, g_data_b, 0., g_data_c); diff --git a/paddle/operators/math/matmul.h b/paddle/operators/math/matmul.h index 6ba9a0ba9a..7048e11e6f 100644 --- a/paddle/operators/math/matmul.h +++ b/paddle/operators/math/matmul.h @@ -26,13 +26,12 @@ namespace math { // // Both a & b can be 1- to 3-dimensional. Higher rank tensors are not supported // yet. -template +template class MatMulFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& a, bool trans_a, - const framework::Tensor& b, bool trans_b, T alpha, - framework::Tensor* out, T beta) { + void operator()(const DeviceContext& context, const framework::Tensor& a, + bool trans_a, const framework::Tensor& b, bool trans_b, + T alpha, framework::Tensor* out, T beta) { auto dim_a = a.dims(); auto dim_b = b.dims(); @@ -108,13 +107,13 @@ class MatMulFunctor { if (!batchCount) { // regular matrix multiplication - gemm(context, transA, transB, M, N, kA, alpha, a.data(), - b.data(), beta, out->data()); + gemm(context, transA, transB, M, N, kA, alpha, + a.data(), b.data(), beta, out->data()); } else { // batched matrix multiplication - batched_gemm(context, transA, transB, M, N, kA, alpha, - a.data(), b.data(), beta, out->data(), - batchCount, strideA, strideB); + batched_gemm( + context, transA, transB, M, N, kA, alpha, a.data(), b.data(), + beta, out->data(), batchCount, strideA, strideB); } } }; diff --git a/paddle/operators/math/maxouting.cc b/paddle/operators/math/maxouting.cc index c9003962d3..fea86675f7 100644 --- a/paddle/operators/math/maxouting.cc +++ b/paddle/operators/math/maxouting.cc @@ -20,9 +20,9 @@ namespace math { // All tensors are in NCHW format, and the groups must be greater than 1 template -class MaxOutFunctor { +class MaxOutFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, framework::Tensor* output, int groups) { const int batch_size = input.dims()[0]; @@ -54,9 +54,9 @@ class MaxOutFunctor { }; template -class MaxOutGradFunctor { +class MaxOutGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, framework::Tensor* input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, int groups) { @@ -91,10 +91,10 @@ class MaxOutGradFunctor { } }; -template class MaxOutGradFunctor; -template class MaxOutGradFunctor; -template class MaxOutFunctor; -template class MaxOutFunctor; +template class MaxOutGradFunctor; +template class MaxOutGradFunctor; +template class MaxOutFunctor; +template class MaxOutFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/maxouting.cu b/paddle/operators/math/maxouting.cu index c3fabcae08..6056ad251c 100644 --- a/paddle/operators/math/maxouting.cu +++ b/paddle/operators/math/maxouting.cu @@ -78,9 +78,9 @@ __global__ void KernelMaxoutGrad(const int nthreads, const T* input_data, * All tensors are in NCHW format. */ template -class MaxOutFunctor { +class MaxOutFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, framework::Tensor* output, int groups) { const int batch_size = input.dims()[0]; @@ -98,20 +98,18 @@ class MaxOutFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelMaxOut< - T><<(context) - .stream()>>>(nthreads, input_data, input_channels, - input_height, input_width, groups, output_data); + KernelMaxOut<<>>( + nthreads, input_data, input_channels, input_height, input_width, groups, + output_data); } }; /* * All tensors are in NCHW format. */ template -class MaxOutGradFunctor { +class MaxOutGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, framework::Tensor* input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, int groups) { @@ -132,20 +130,17 @@ class MaxOutGradFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelMaxoutGrad< - T><<(context) - .stream()>>>(nthreads, input_data, output_data, - output_grad_data, input_grad_data, input_channels, - input_height, input_width, groups); + KernelMaxoutGrad<<>>( + nthreads, input_data, output_data, output_grad_data, input_grad_data, + input_channels, input_height, input_width, groups); } }; -template class MaxOutGradFunctor; -template class MaxOutGradFunctor; +template class MaxOutGradFunctor; +template class MaxOutGradFunctor; -template class MaxOutFunctor; -template class MaxOutFunctor; +template class MaxOutFunctor; +template class MaxOutFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/maxouting.h b/paddle/operators/math/maxouting.h index 2d9069b0b3..68f4743db0 100644 --- a/paddle/operators/math/maxouting.h +++ b/paddle/operators/math/maxouting.h @@ -23,20 +23,18 @@ namespace math { #define FLT_MAX __FLT_MAX__ -template - +template class MaxOutFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, framework::Tensor* output, - int groups); + void operator()(const DeviceContext& context, const framework::Tensor& input, + framework::Tensor* output, int groups); }; -template +template class MaxOutGradFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, framework::Tensor* input_grad, + void operator()(const DeviceContext& context, const framework::Tensor& input, + framework::Tensor* input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, int groups); }; diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc index 135984586a..150de6fd59 100644 --- a/paddle/operators/math/pooling.cc +++ b/paddle/operators/math/pooling.cc @@ -24,9 +24,9 @@ namespace math { * height and width, respectively. */ template -class Pool2dFunctor { +class Pool2dFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, std::vector& ksize, std::vector& strides, std::vector& paddings, PoolProcess pool_process, framework::Tensor* output) { @@ -84,9 +84,9 @@ class Pool2dFunctor { * and width, respectively. */ template -class Pool2dGradFunctor { +class Pool2dGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, @@ -152,9 +152,9 @@ class Pool2dGradFunctor { * height and width, respectively. */ template -class MaxPool2dGradFunctor { +class MaxPool2dGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, @@ -213,25 +213,29 @@ class MaxPool2dGradFunctor { } }; -template class MaxPool2dGradFunctor; -template class MaxPool2dGradFunctor; +template class MaxPool2dGradFunctor; +template class MaxPool2dGradFunctor; -template class Pool2dFunctor, float>; -template class Pool2dFunctor, float>; -template class Pool2dGradFunctor< - platform::CPUPlace, paddle::operators::math::MaxPoolGrad, float>; -template class Pool2dGradFunctor< - platform::CPUPlace, paddle::operators::math::AvgPoolGrad, float>; -template class Pool2dFunctor, + float>; +template class Pool2dGradFunctor, + float>; +template class Pool2dFunctor, double>; -template class Pool2dFunctor, double>; -template class Pool2dGradFunctor< - platform::CPUPlace, paddle::operators::math::MaxPoolGrad, double>; -template class Pool2dGradFunctor< - platform::CPUPlace, paddle::operators::math::AvgPoolGrad, double>; +template class Pool2dGradFunctor, + double>; +template class Pool2dGradFunctor, + double>; /* * All tensors are in NCDHW format. @@ -239,9 +243,9 @@ template class Pool2dGradFunctor< * depth, height and width, respectively. */ template -class Pool3dFunctor { +class Pool3dFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, std::vector& ksize, std::vector& strides, std::vector& paddings, PoolProcess pool_process, framework::Tensor* output) { @@ -314,9 +318,9 @@ class Pool3dFunctor { * depth, height and width, respectively. */ template -class Pool3dGradFunctor { +class Pool3dGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, @@ -398,9 +402,9 @@ class Pool3dGradFunctor { * depth, height and width, respectively. */ template -class MaxPool3dGradFunctor { +class MaxPool3dGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, @@ -473,25 +477,29 @@ class MaxPool3dGradFunctor { } }; -template class MaxPool3dGradFunctor; -template class MaxPool3dGradFunctor; +template class MaxPool3dGradFunctor; +template class MaxPool3dGradFunctor; -template class Pool3dFunctor, float>; -template class Pool3dFunctor, float>; -template class Pool3dGradFunctor< - platform::CPUPlace, paddle::operators::math::MaxPoolGrad, float>; -template class Pool3dGradFunctor< - platform::CPUPlace, paddle::operators::math::AvgPoolGrad, float>; -template class Pool3dFunctor, + float>; +template class Pool3dGradFunctor, + float>; +template class Pool3dFunctor, double>; -template class Pool3dFunctor, double>; -template class Pool3dGradFunctor< - platform::CPUPlace, paddle::operators::math::MaxPoolGrad, double>; -template class Pool3dGradFunctor< - platform::CPUPlace, paddle::operators::math::AvgPoolGrad, double>; +template class Pool3dGradFunctor, + double>; +template class Pool3dGradFunctor, + double>; /* * All tensors are in NCHW format. @@ -499,9 +507,9 @@ template class Pool3dGradFunctor< * height and width, respectively. */ template -class MaxPool2dWithIndexFunctor { +class MaxPool2dWithIndexFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, std::vector& ksize, std::vector& strides, std::vector& paddings, framework::Tensor* output, framework::Tensor* mask) { @@ -564,9 +572,9 @@ class MaxPool2dWithIndexFunctor { * height and width, respectively. */ template -class MaxPool2dWithIndexGradFunctor { +class MaxPool2dWithIndexGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& output_grad, const framework::Tensor& mask, std::vector& ksize, std::vector& strides, std::vector& paddings, @@ -602,10 +610,14 @@ class MaxPool2dWithIndexGradFunctor { } }; -template class MaxPool2dWithIndexFunctor; -template class MaxPool2dWithIndexGradFunctor; -template class MaxPool2dWithIndexFunctor; -template class MaxPool2dWithIndexGradFunctor; +template class MaxPool2dWithIndexFunctor; +template class MaxPool2dWithIndexGradFunctor; +template class MaxPool2dWithIndexFunctor; +template class MaxPool2dWithIndexGradFunctor; /* * All tensors are in NCDHW format. @@ -613,9 +625,9 @@ template class MaxPool2dWithIndexGradFunctor; * depth, height and width, respectively. */ template -class MaxPool3dWithIndexFunctor { +class MaxPool3dWithIndexFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, std::vector& ksize, std::vector& strides, std::vector& paddings, framework::Tensor* output, framework::Tensor* mask) { @@ -692,9 +704,9 @@ class MaxPool3dWithIndexFunctor { * depth, height and width, respectively. */ template -class MaxPool3dWithIndexGradFunctor { +class MaxPool3dWithIndexGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& output_grad, const framework::Tensor& mask, std::vector& ksize, std::vector& strides, std::vector& paddings, @@ -735,10 +747,14 @@ class MaxPool3dWithIndexGradFunctor { } }; -template class MaxPool3dWithIndexFunctor; -template class MaxPool3dWithIndexGradFunctor; -template class MaxPool3dWithIndexFunctor; -template class MaxPool3dWithIndexGradFunctor; +template class MaxPool3dWithIndexFunctor; +template class MaxPool3dWithIndexGradFunctor; +template class MaxPool3dWithIndexFunctor; +template class MaxPool3dWithIndexGradFunctor; } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu index ca3560f264..0243cf8316 100644 --- a/paddle/operators/math/pooling.cu +++ b/paddle/operators/math/pooling.cu @@ -155,9 +155,9 @@ __global__ void KernelMaxPool2DGrad( * height and width, respectively. */ template -class Pool2dFunctor { +class Pool2dFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, std::vector& ksize, std::vector& strides, std::vector& paddings, PoolProcess pool_process, framework::Tensor* output) { @@ -183,11 +183,7 @@ class Pool2dFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelPool2D< - PoolProcess, - T><<(context) - .stream()>>>( + KernelPool2D<<>>( nthreads, input_data, input_channels, input_height, input_width, output_height, output_width, ksize_height, ksize_width, stride_height, stride_width, padding_height, padding_width, pool_process, output_data); @@ -200,9 +196,9 @@ class Pool2dFunctor { * height and width, respectively. */ template -class Pool2dGradFunctor { +class Pool2dGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, @@ -231,11 +227,7 @@ class Pool2dGradFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelPool2DGrad< - PoolProcess, - T><<(context) - .stream()>>>( + KernelPool2DGrad<<>>( nthreads, input_data, output_data, output_grad_data, input_channels, input_height, input_width, output_height, output_width, ksize_height, ksize_width, stride_height, stride_width, padding_height, padding_width, @@ -249,9 +241,9 @@ class Pool2dGradFunctor { * height and width, respectively. */ template -class MaxPool2dGradFunctor { +class MaxPool2dGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, @@ -281,10 +273,7 @@ class MaxPool2dGradFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelMaxPool2DGrad< - T><<(context) - .stream()>>>( + KernelMaxPool2DGrad<<>>( nthreads, input_data, output_data, output_grad_data, input_channels, input_height, input_width, output_height, output_width, ksize_height, ksize_width, stride_height, stride_width, padding_height, padding_width, @@ -292,25 +281,29 @@ class MaxPool2dGradFunctor { } }; -template class MaxPool2dGradFunctor; -template class MaxPool2dGradFunctor; +template class MaxPool2dGradFunctor; +template class MaxPool2dGradFunctor; -template class Pool2dFunctor, float>; -template class Pool2dFunctor, float>; -template class Pool2dGradFunctor< - platform::GPUPlace, paddle::operators::math::MaxPoolGrad, float>; -template class Pool2dGradFunctor< - platform::GPUPlace, paddle::operators::math::AvgPoolGrad, float>; -template class Pool2dFunctor, + float>; +template class Pool2dGradFunctor, + float>; +template class Pool2dFunctor, double>; -template class Pool2dFunctor, double>; -template class Pool2dGradFunctor< - platform::GPUPlace, paddle::operators::math::MaxPoolGrad, double>; -template class Pool2dGradFunctor< - platform::GPUPlace, paddle::operators::math::AvgPoolGrad, double>; +template class Pool2dGradFunctor, + double>; +template class Pool2dGradFunctor, + double>; template __global__ void KernelPool3D(const int nthreads, const T* input_data, @@ -478,9 +471,9 @@ __global__ void KernelMaxPool3DGrad( * depth, height and width, respectively. */ template -class Pool3dFunctor { +class Pool3dFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, std::vector& ksize, std::vector& strides, std::vector& paddings, PoolProcess pool_process, framework::Tensor* output) { @@ -512,11 +505,7 @@ class Pool3dFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelPool3D< - PoolProcess, - T><<(context) - .stream()>>>( + KernelPool3D<<>>( nthreads, input_data, input_channels, input_depth, input_height, input_width, output_depth, output_height, output_width, ksize_depth, ksize_height, ksize_width, stride_depth, stride_height, stride_width, @@ -531,9 +520,9 @@ class Pool3dFunctor { * depth, height and width, respectively. */ template -class Pool3dGradFunctor { +class Pool3dGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, @@ -569,11 +558,7 @@ class Pool3dGradFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelPool3DGrad< - PoolProcess, - T><<(context) - .stream()>>>( + KernelPool3DGrad<<>>( nthreads, input_data, output_data, output_grad_data, input_channels, input_depth, input_height, input_width, output_depth, output_height, output_width, ksize_depth, ksize_height, ksize_width, stride_depth, @@ -588,9 +573,9 @@ class Pool3dGradFunctor { * depth, height and width, respectively. */ template -class MaxPool3dGradFunctor { +class MaxPool3dGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, @@ -626,10 +611,7 @@ class MaxPool3dGradFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelMaxPool3DGrad< - T><<(context) - .stream()>>>( + KernelMaxPool3DGrad<<>>( nthreads, input_data, output_data, output_grad_data, input_channels, input_depth, input_height, input_width, output_depth, output_height, output_width, ksize_depth, ksize_height, ksize_width, stride_depth, @@ -638,25 +620,29 @@ class MaxPool3dGradFunctor { } }; -template class MaxPool3dGradFunctor; -template class MaxPool3dGradFunctor; +template class MaxPool3dGradFunctor; +template class MaxPool3dGradFunctor; -template class Pool3dFunctor, float>; -template class Pool3dFunctor, float>; -template class Pool3dGradFunctor< - platform::GPUPlace, paddle::operators::math::MaxPoolGrad, float>; -template class Pool3dGradFunctor< - platform::GPUPlace, paddle::operators::math::AvgPoolGrad, float>; -template class Pool3dFunctor, + float>; +template class Pool3dGradFunctor, + float>; +template class Pool3dFunctor, double>; -template class Pool3dFunctor, double>; -template class Pool3dGradFunctor< - platform::GPUPlace, paddle::operators::math::MaxPoolGrad, double>; -template class Pool3dGradFunctor< - platform::GPUPlace, paddle::operators::math::AvgPoolGrad, double>; +template class Pool3dGradFunctor, + double>; +template class Pool3dGradFunctor, + double>; template __global__ void KernelMaxPool2dWithIdx( @@ -747,9 +733,9 @@ __global__ void KernelMaxPool2DWithIdxGrad( * height and width, respectively. */ template -class MaxPool2dWithIndexFunctor { +class MaxPool2dWithIndexFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, std::vector& ksize, std::vector& strides, std::vector& paddings, framework::Tensor* output, framework::Tensor* mask) { @@ -776,10 +762,7 @@ class MaxPool2dWithIndexFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelMaxPool2dWithIdx< - T1, T2><<(context) - .stream()>>>( + KernelMaxPool2dWithIdx<<>>( nthreads, input_data, input_channels, input_height, input_width, output_height, output_width, ksize_height, ksize_width, stride_height, stride_width, padding_height, padding_width, output_data, mask_data); @@ -792,9 +775,9 @@ class MaxPool2dWithIndexFunctor { * height and width, respectively. */ template -class MaxPool2dWithIndexGradFunctor { +class MaxPool2dWithIndexGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& output_grad, const framework::Tensor& mask, std::vector& ksize, std::vector& strides, std::vector& paddings, @@ -821,10 +804,7 @@ class MaxPool2dWithIndexGradFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelMaxPool2DWithIdxGrad< - T1, T2><<(context) - .stream()>>>( + KernelMaxPool2DWithIdxGrad<<>>( nthreads, output_grad_data, mask_data, input_channels, input_height, input_width, output_height, output_width, ksize_height, ksize_width, stride_height, stride_width, padding_height, padding_width, @@ -832,10 +812,14 @@ class MaxPool2dWithIndexGradFunctor { } }; -template class MaxPool2dWithIndexFunctor; -template class MaxPool2dWithIndexGradFunctor; -template class MaxPool2dWithIndexFunctor; -template class MaxPool2dWithIndexGradFunctor; +template class MaxPool2dWithIndexFunctor; +template class MaxPool2dWithIndexGradFunctor; +template class MaxPool2dWithIndexFunctor; +template class MaxPool2dWithIndexGradFunctor; template __global__ void KernelMaxPool3DWithIdx( @@ -950,9 +934,9 @@ __global__ void KernelMaxPool3DWithIdxGrad( * depth, height and width, respectively. */ template -class MaxPool3dWithIndexFunctor { +class MaxPool3dWithIndexFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, std::vector& ksize, std::vector& strides, std::vector& paddings, framework::Tensor* output, framework::Tensor* mask) { @@ -985,10 +969,7 @@ class MaxPool3dWithIndexFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelMaxPool3DWithIdx< - T1, T2><<(context) - .stream()>>>( + KernelMaxPool3DWithIdx<<>>( nthreads, input_data, input_channels, input_depth, input_height, input_width, output_depth, output_height, output_width, ksize_depth, ksize_height, ksize_width, stride_depth, stride_height, stride_width, @@ -1002,9 +983,9 @@ class MaxPool3dWithIndexFunctor { * depth, height and width, respectively. */ template -class MaxPool3dWithIndexGradFunctor { +class MaxPool3dWithIndexGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& output_grad, const framework::Tensor& mask, std::vector& ksize, std::vector& strides, std::vector& paddings, @@ -1037,10 +1018,7 @@ class MaxPool3dWithIndexGradFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelMaxPool3DWithIdxGrad< - T1, T2><<(context) - .stream()>>>( + KernelMaxPool3DWithIdxGrad<<>>( nthreads, output_grad_data, mask_data, input_channels, input_depth, input_height, input_width, output_depth, output_height, output_width, ksize_depth, ksize_height, ksize_width, stride_depth, stride_height, @@ -1049,10 +1027,14 @@ class MaxPool3dWithIndexGradFunctor { } }; -template class MaxPool3dWithIndexFunctor; -template class MaxPool3dWithIndexGradFunctor; -template class MaxPool3dWithIndexFunctor; -template class MaxPool3dWithIndexGradFunctor; +template class MaxPool3dWithIndexFunctor; +template class MaxPool3dWithIndexGradFunctor; +template class MaxPool3dWithIndexFunctor; +template class MaxPool3dWithIndexGradFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h index 19fbd8b4bb..2759f06cb6 100644 --- a/paddle/operators/math/pooling.h +++ b/paddle/operators/math/pooling.h @@ -84,62 +84,58 @@ class AvgPoolGrad { * This is different from average pooling. So we rewrite the max_pool_grad: * MaxPool2dGradFunctor, MaxPool3dGradFunctor. */ -template +template class Pool2dFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, std::vector& ksize, - std::vector& strides, std::vector& paddings, - PoolProcess pool_compute, framework::Tensor* output); + void operator()(const DeviceContext& context, const framework::Tensor& input, + std::vector& ksize, std::vector& strides, + std::vector& paddings, PoolProcess pool_compute, + framework::Tensor* output); }; -template +template class Pool2dGradFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, + void operator()(const DeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, PoolProcess pool_compute, framework::Tensor* input_grad); }; -template +template class MaxPool2dGradFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, + void operator()(const DeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, framework::Tensor* input_grad); }; -template +template class Pool3dFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, std::vector& ksize, - std::vector& strides, std::vector& paddings, - PoolProcess pool_compute, framework::Tensor* output); + void operator()(const DeviceContext& context, const framework::Tensor& input, + std::vector& ksize, std::vector& strides, + std::vector& paddings, PoolProcess pool_compute, + framework::Tensor* output); }; -template +template class Pool3dGradFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, + void operator()(const DeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, PoolProcess pool_compute, framework::Tensor* input_grad); }; -template +template class MaxPool3dGradFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, + void operator()(const DeviceContext& context, const framework::Tensor& input, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, @@ -153,38 +149,38 @@ class MaxPool3dGradFunctor { * In pool2d, all tensors are in NCHW format. In pool3d, all tensors are in * NCDHW format. */ -template +template class MaxPool2dWithIndexFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, std::vector& ksize, - std::vector& strides, std::vector& paddings, - framework::Tensor* output, framework::Tensor* mask); + void operator()(const DeviceContext& context, const framework::Tensor& input, + std::vector& ksize, std::vector& strides, + std::vector& paddings, framework::Tensor* output, + framework::Tensor* mask); }; -template +template class MaxPool2dWithIndexGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::Tensor& output_grad, const framework::Tensor& mask, std::vector& ksize, std::vector& strides, std::vector& paddings, framework::Tensor* input_grad); }; -template +template class MaxPool3dWithIndexFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, std::vector& ksize, - std::vector& strides, std::vector& paddings, - framework::Tensor* output, framework::Tensor* mask); + void operator()(const DeviceContext& context, const framework::Tensor& input, + std::vector& ksize, std::vector& strides, + std::vector& paddings, framework::Tensor* output, + framework::Tensor* mask); }; -template +template class MaxPool3dWithIndexGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::Tensor& output_grad, const framework::Tensor& mask, std::vector& ksize, std::vector& strides, std::vector& paddings, diff --git a/paddle/operators/math/selected_rows_functor.cc b/paddle/operators/math/selected_rows_functor.cc index 514f2adef2..ab758d1e7f 100644 --- a/paddle/operators/math/selected_rows_functor.cc +++ b/paddle/operators/math/selected_rows_functor.cc @@ -19,8 +19,8 @@ namespace paddle { namespace operators { namespace math { template -struct SelectedRowsAdd { - void operator()(const platform::DeviceContext& context, +struct SelectedRowsAdd { + void operator()(const platform::CPUDeviceContext& context, const framework::SelectedRows& input1, const framework::SelectedRows& input2, framework::SelectedRows* output) { @@ -67,12 +67,12 @@ struct SelectedRowsAdd { } }; -template struct SelectedRowsAdd; -template struct SelectedRowsAdd; +template struct SelectedRowsAdd; +template struct SelectedRowsAdd; template -struct SelectedRowsAddTensor { - void operator()(const platform::DeviceContext& context, +struct SelectedRowsAddTensor { + void operator()(const platform::CPUDeviceContext& context, const framework::SelectedRows& input1, const framework::Tensor& input2, framework::Tensor* output) { auto in1_height = input1.height(); @@ -88,7 +88,7 @@ struct SelectedRowsAddTensor { PADDLE_ENFORCE_EQ(in1_row_numel, input2.numel() / in1_height); PADDLE_ENFORCE_EQ(in1_row_numel, output->numel() / in1_height); - SetConstant functor; + SetConstant functor; functor(context, output, 0.0); auto* in1_data = in1_value.data(); @@ -103,17 +103,16 @@ struct SelectedRowsAddTensor { auto out_eigen = framework::EigenVector::Flatten(*output); auto in2_eigen = framework::EigenVector::Flatten(input2); - out_eigen.device(*context.GetEigenDevice()) = - out_eigen + in2_eigen; + out_eigen.device(*context.eigen_device()) = out_eigen + in2_eigen; } }; -template struct SelectedRowsAddTensor; -template struct SelectedRowsAddTensor; +template struct SelectedRowsAddTensor; +template struct SelectedRowsAddTensor; template -struct SelectedRowsAddTo { - void operator()(const platform::DeviceContext& context, +struct SelectedRowsAddTo { + void operator()(const platform::CPUDeviceContext& context, const framework::SelectedRows& input1, const int64_t input2_offset, framework::SelectedRows* input2) { @@ -143,14 +142,14 @@ struct SelectedRowsAddTo { } }; -template struct SelectedRowsAddTo; -template struct SelectedRowsAddTo; -template struct SelectedRowsAddTo; -template struct SelectedRowsAddTo; +template struct SelectedRowsAddTo; +template struct SelectedRowsAddTo; +template struct SelectedRowsAddTo; +template struct SelectedRowsAddTo; template -struct SelectedRowsAddToTensor { - void operator()(const platform::DeviceContext& context, +struct SelectedRowsAddToTensor { + void operator()(const platform::CPUDeviceContext& context, const framework::SelectedRows& input1, framework::Tensor* input2) { auto in1_height = input1.height(); @@ -175,10 +174,10 @@ struct SelectedRowsAddToTensor { } }; -template struct SelectedRowsAddToTensor; -template struct SelectedRowsAddToTensor; -template struct SelectedRowsAddToTensor; -template struct SelectedRowsAddToTensor; +template struct SelectedRowsAddToTensor; +template struct SelectedRowsAddToTensor; +template struct SelectedRowsAddToTensor; +template struct SelectedRowsAddToTensor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/selected_rows_functor.cu b/paddle/operators/math/selected_rows_functor.cu index c1dd323ba2..c44577e00a 100644 --- a/paddle/operators/math/selected_rows_functor.cu +++ b/paddle/operators/math/selected_rows_functor.cu @@ -20,8 +20,8 @@ namespace paddle { namespace operators { namespace math { template -struct SelectedRowsAdd { - void operator()(const platform::DeviceContext& context, +struct SelectedRowsAdd { + void operator()(const platform::CUDADeviceContext& context, const framework::SelectedRows& input1, const framework::SelectedRows& input2, framework::SelectedRows* output) { @@ -64,16 +64,15 @@ struct SelectedRowsAdd { reinterpret_cast(context).stream()); auto* in2_data = in2_value.data(); - memory::Copy( - boost::get(out_place), out_data + in1_value.numel(), - boost::get(in2_place), in2_data, - in2_value.numel() * sizeof(T), - reinterpret_cast(context).stream()); + memory::Copy(boost::get(out_place), + out_data + in1_value.numel(), + boost::get(in2_place), in2_data, + in2_value.numel() * sizeof(T), context.stream()); } }; -template struct SelectedRowsAdd; -template struct SelectedRowsAdd; +template struct SelectedRowsAdd; +template struct SelectedRowsAdd; namespace { template @@ -96,8 +95,8 @@ __global__ void SelectedRowsAddTensorKernel(const T* selected_rows, } // namespace template -struct SelectedRowsAddTensor { - void operator()(const platform::DeviceContext& context, +struct SelectedRowsAddTensor { + void operator()(const platform::CUDADeviceContext& context, const framework::SelectedRows& input1, const framework::Tensor& input2, framework::Tensor* output) { auto in1_height = input1.height(); @@ -117,30 +116,28 @@ struct SelectedRowsAddTensor { auto* in2_data = input2.data(); auto* out_data = output->data(); - SetConstant functor; + SetConstant functor; functor(context, output, 0.0); const int block_size = 256; dim3 threads(block_size, 1); dim3 grid(1, in1_rows.size()); - SelectedRowsAddTensorKernel<<< - grid, threads, 0, - reinterpret_cast(context) - .stream()>>>(in1_data, in1_rows.data(), out_data, in1_row_numel); + SelectedRowsAddTensorKernel< + T, block_size><<>>( + in1_data, in1_rows.data(), out_data, in1_row_numel); auto out_eigen = framework::EigenVector::Flatten(*output); auto in2_eigen = framework::EigenVector::Flatten(input2); - out_eigen.device(*context.GetEigenDevice()) = - out_eigen + in2_eigen; + out_eigen.device(*context.eigen_device()) = out_eigen + in2_eigen; } }; -template struct SelectedRowsAddTensor; -template struct SelectedRowsAddTensor; +template struct SelectedRowsAddTensor; +template struct SelectedRowsAddTensor; template -struct SelectedRowsAddTo { - void operator()(const platform::DeviceContext& context, +struct SelectedRowsAddTo { + void operator()(const platform::CUDADeviceContext& context, const framework::SelectedRows& input1, const int64_t input2_offset, framework::SelectedRows* input2) { @@ -163,18 +160,17 @@ struct SelectedRowsAddTo { auto* in1_data = in1_value.data(); auto* in2_data = in2_value->data(); - memory::Copy( - boost::get(in2_place), in2_data + input2_offset, - boost::get(in1_place), in1_data, - in1_value.numel() * sizeof(T), - reinterpret_cast(context).stream()); + memory::Copy(boost::get(in2_place), + in2_data + input2_offset, + boost::get(in1_place), in1_data, + in1_value.numel() * sizeof(T), context.stream()); } }; -template struct SelectedRowsAddTo; -template struct SelectedRowsAddTo; -template struct SelectedRowsAddTo; -template struct SelectedRowsAddTo; +template struct SelectedRowsAddTo; +template struct SelectedRowsAddTo; +template struct SelectedRowsAddTo; +template struct SelectedRowsAddTo; namespace { template @@ -197,8 +193,8 @@ __global__ void SelectedRowsAddToTensorKernel(const T* selected_rows, } // namespace template -struct SelectedRowsAddToTensor { - void operator()(const platform::DeviceContext& context, +struct SelectedRowsAddToTensor { + void operator()(const platform::CUDADeviceContext& context, const framework::SelectedRows& input1, framework::Tensor* input2) { auto in1_height = input1.height(); @@ -216,17 +212,16 @@ struct SelectedRowsAddToTensor { const int block_size = 256; dim3 threads(block_size, 1); dim3 grid(1, in1_rows.size()); - SelectedRowsAddToTensorKernel<<< - grid, threads, 0, - reinterpret_cast(context) - .stream()>>>(in1_data, in1_rows.data(), in2_data, in1_row_numel); + SelectedRowsAddToTensorKernel< + T, block_size><<>>( + in1_data, in1_rows.data(), in2_data, in1_row_numel); } }; -template struct SelectedRowsAddToTensor; -template struct SelectedRowsAddToTensor; -template struct SelectedRowsAddToTensor; -template struct SelectedRowsAddToTensor; +template struct SelectedRowsAddToTensor; +template struct SelectedRowsAddToTensor; +template struct SelectedRowsAddToTensor; +template struct SelectedRowsAddToTensor; } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/selected_rows_functor.h b/paddle/operators/math/selected_rows_functor.h index d6dc6c03c9..1149075abf 100644 --- a/paddle/operators/math/selected_rows_functor.h +++ b/paddle/operators/math/selected_rows_functor.h @@ -21,33 +21,33 @@ namespace math { // SelectedRows + SelectedRows will simplely concat value and rows. // The real computation happens in dealing with LoDTensor. -template +template struct SelectedRowsAdd { - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::SelectedRows& input1, const framework::SelectedRows& input2, framework::SelectedRows* output); }; -template +template struct SelectedRowsAddTensor { - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::SelectedRows& input1, const framework::Tensor& input2, framework::Tensor* output); }; // input2 = input1 + input2 -template +template struct SelectedRowsAddTo { - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::SelectedRows& input1, const int64_t input2_offset, framework::SelectedRows* input2); }; // input2 = input1 + input2 -template +template struct SelectedRowsAddToTensor { - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::SelectedRows& input1, framework::Tensor* input2); }; diff --git a/paddle/operators/math/selected_rows_functor_test.cc b/paddle/operators/math/selected_rows_functor_test.cc index a3649b6875..8c74cab0a1 100644 --- a/paddle/operators/math/selected_rows_functor_test.cc +++ b/paddle/operators/math/selected_rows_functor_test.cc @@ -23,7 +23,7 @@ TEST(selected_rows_functor, cpu_add) { CPUPlace cpu_place; CPUDeviceContext ctx(cpu_place); - SetConstant functor; + SetConstant functor; int64_t height = 10; int64_t row_numel = 10; @@ -47,7 +47,7 @@ TEST(selected_rows_functor, cpu_add) { // simplely concat two SelectedRows out_value->mutable_data(make_ddim({7, 10}), cpu_place); - SelectedRowsAdd add_functor; + SelectedRowsAdd add_functor; add_functor(ctx, *selected_rows1, *selected_rows2, output.get()); auto out_height = output->height(); @@ -85,7 +85,7 @@ TEST(selected_rows_functor, cpu_add) { std::unique_ptr tensor2{new Tensor()}; tensor2->mutable_data(make_ddim({height, row_numel}), cpu_place); - SelectedRowsAddTensor add_tensor_functor; + SelectedRowsAddTensor add_tensor_functor; add_tensor_functor(ctx, *output, *tensor1, tensor2.get()); auto* tensor2_data = tensor2->data(); @@ -112,7 +112,7 @@ TEST(selected_rows_functor, cpu_add_to) { CPUPlace cpu_place; CPUDeviceContext ctx(cpu_place); - SetConstant functor; + SetConstant functor; int64_t height = 10; int64_t row_numel = 10; @@ -137,7 +137,7 @@ TEST(selected_rows_functor, cpu_add_to) { // simplely concat two SelectedRows out_value->mutable_data(make_ddim({7, 10}), cpu_place); - SelectedRowsAddTo add_to_functor; + SelectedRowsAddTo add_to_functor; add_to_functor(ctx, *selected_rows1, 0, output.get()); add_to_functor(ctx, *selected_rows2, in1_value->numel(), output.get()); @@ -173,7 +173,7 @@ TEST(selected_rows_functor, cpu_add_to) { tensor1->mutable_data(make_ddim({height, row_numel}), cpu_place); functor(ctx, tensor1.get(), 3.0); - SelectedRowsAddToTensor add_to_tensor_functor; + SelectedRowsAddToTensor add_to_tensor_functor; add_to_tensor_functor(ctx, *output, tensor1.get()); auto* tensor1_data = tensor1->data(); diff --git a/paddle/operators/math/selected_rows_functor_test.cu b/paddle/operators/math/selected_rows_functor_test.cu index 7de9291c17..777caf5635 100644 --- a/paddle/operators/math/selected_rows_functor_test.cu +++ b/paddle/operators/math/selected_rows_functor_test.cu @@ -24,7 +24,7 @@ TEST(selected_rows_functor, gpu_add) { GPUPlace gpu_place(0); CPUPlace cpu_place; CUDADeviceContext ctx(gpu_place); - SetConstant functor; + SetConstant functor; int64_t height = 10; int64_t row_numel = 10; @@ -48,7 +48,7 @@ TEST(selected_rows_functor, gpu_add) { // simplely concat two SelectedRows out_value->mutable_data(make_ddim({7, 10}), gpu_place); - SelectedRowsAdd add_functor; + SelectedRowsAdd add_functor; add_functor(ctx, *selected_rows1, *selected_rows2, output.get()); auto out_height = output->height(); @@ -90,7 +90,7 @@ TEST(selected_rows_functor, gpu_add) { std::unique_ptr tensor2{new Tensor()}; tensor2->mutable_data(make_ddim({height, row_numel}), gpu_place); - SelectedRowsAddTensor add_tensor_functor; + SelectedRowsAddTensor add_tensor_functor; add_tensor_functor(ctx, *output, *tensor1, tensor2.get()); Tensor tensor2_cpu; @@ -122,7 +122,7 @@ TEST(selected_rows_functor, gpu_add_to) { GPUPlace gpu_place(0); CPUPlace cpu_place; CUDADeviceContext ctx(gpu_place); - SetConstant functor; + SetConstant functor; int64_t height = 10; int64_t row_numel = 10; @@ -147,7 +147,7 @@ TEST(selected_rows_functor, gpu_add_to) { // simplely concat two SelectedRows out_value->mutable_data(make_ddim({7, 10}), gpu_place); - SelectedRowsAddTo add_to_functor; + SelectedRowsAddTo add_to_functor; add_to_functor(ctx, *selected_rows1, 0, output.get()); add_to_functor(ctx, *selected_rows2, in1_value->numel(), output.get()); @@ -187,7 +187,7 @@ TEST(selected_rows_functor, gpu_add_to) { tensor1->mutable_data(make_ddim({height, row_numel}), gpu_place); functor(ctx, tensor1.get(), 3.0); - SelectedRowsAddToTensor add_to_tensor_functor; + SelectedRowsAddToTensor add_to_tensor_functor; add_to_tensor_functor(ctx, *output, tensor1.get()); Tensor tensor1_cpu; diff --git a/paddle/operators/math/sequence2batch.cc b/paddle/operators/math/sequence2batch.cc index 5b3bde02fb..88977be1f8 100644 --- a/paddle/operators/math/sequence2batch.cc +++ b/paddle/operators/math/sequence2batch.cc @@ -19,9 +19,9 @@ namespace operators { namespace math { template -class CopyMatrixRowsFunctor { +class CopyMatrixRowsFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& src, const size_t* index, framework::Tensor& dst, bool is_src_index) { auto src_dims = src.dims(); @@ -48,13 +48,13 @@ class CopyMatrixRowsFunctor { } }; -template class CopyMatrixRowsFunctor; -template class CopyMatrixRowsFunctor; +template class CopyMatrixRowsFunctor; +template class CopyMatrixRowsFunctor; -template class LoDTensor2BatchFunctor; -template class LoDTensor2BatchFunctor; -template class Batch2LoDTensorFunctor; -template class Batch2LoDTensorFunctor; +template class LoDTensor2BatchFunctor; +template class LoDTensor2BatchFunctor; +template class Batch2LoDTensorFunctor; +template class Batch2LoDTensorFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/sequence2batch.cu b/paddle/operators/math/sequence2batch.cu index c5d968aeb2..452ae89510 100644 --- a/paddle/operators/math/sequence2batch.cu +++ b/paddle/operators/math/sequence2batch.cu @@ -39,9 +39,9 @@ __global__ void CopyMatrixRowsKernel(const T* src, T* dst, const size_t* index, } template -class CopyMatrixRowsFunctor { +class CopyMatrixRowsFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& src, const size_t* index, framework::Tensor& dst, bool is_src_index) { auto src_dims = src.dims(); @@ -59,20 +59,19 @@ class CopyMatrixRowsFunctor { dim3 threads(128, 8); dim3 grid(8, 1); - auto stream = - reinterpret_cast(context).stream(); + auto stream = context.stream(); CopyMatrixRowsKernel<<>>( src_data, dst_data, index, height, width, is_src_index); } }; -template class CopyMatrixRowsFunctor; -template class CopyMatrixRowsFunctor; +template class CopyMatrixRowsFunctor; +template class CopyMatrixRowsFunctor; -template class LoDTensor2BatchFunctor; -template class LoDTensor2BatchFunctor; -template class Batch2LoDTensorFunctor; -template class Batch2LoDTensorFunctor; +template class LoDTensor2BatchFunctor; +template class LoDTensor2BatchFunctor; +template class Batch2LoDTensorFunctor; +template class Batch2LoDTensorFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/sequence2batch.h b/paddle/operators/math/sequence2batch.h index 73295ddbcb..a5c43a2c7d 100644 --- a/paddle/operators/math/sequence2batch.h +++ b/paddle/operators/math/sequence2batch.h @@ -26,7 +26,7 @@ template using EigenMatrix = framework::EigenMatrix; -template +template class CopyMatrixRowsFunctor { public: // If is_src_index is true, @@ -34,12 +34,12 @@ class CopyMatrixRowsFunctor { // If is_src_index is false, // copy the input src to the indexed rows of output dst. // The indexed rows are based on the input index. - void operator()(const platform::DeviceContext& context, - const framework::Tensor& src, const size_t* index, - framework::Tensor& dst, bool is_src_index); + void operator()(const DeviceContext& context, const framework::Tensor& src, + const size_t* index, framework::Tensor& dst, + bool is_src_index); }; -template +template class LoDTensor2BatchFunctor { // Calculate the length of each sequence and // sort sequence index by the length. @@ -56,7 +56,7 @@ class LoDTensor2BatchFunctor { }; public: - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::LoDTensor& lod_tensor, framework::LoDTensor& batch, bool is_cal_batch_lod, bool is_reverse = false) const { @@ -65,7 +65,7 @@ class LoDTensor2BatchFunctor { PADDLE_ENFORCE_GT(lods.size(), 2UL); PADDLE_ENFORCE_EQ(lods[1].size(), static_cast(lod_tensor.dims()[0])); - CopyMatrixRowsFunctor to_batch; + CopyMatrixRowsFunctor to_batch; to_batch(context, lod_tensor, lods[1].data(), batch, true); return; } @@ -143,22 +143,22 @@ class LoDTensor2BatchFunctor { } batch.set_lod(batch_lods); - CopyMatrixRowsFunctor to_batch; + CopyMatrixRowsFunctor to_batch; to_batch(context, lod_tensor, seq2batch_idx, batch, true); } }; -template +template class Batch2LoDTensorFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::LoDTensor& batch, framework::LoDTensor& lod_tensor) const { auto in_lod = batch.lod(); PADDLE_ENFORCE_GT(in_lod.size(), 2UL); PADDLE_ENFORCE_EQ(in_lod[1].size(), static_cast(lod_tensor.dims()[0])); - CopyMatrixRowsFunctor to_seq; + CopyMatrixRowsFunctor to_seq; size_t* index = in_lod[1].data(); to_seq(context, batch, index, lod_tensor, false); } diff --git a/paddle/operators/math/sequence_pooling.cc b/paddle/operators/math/sequence_pooling.cc index 5913c99fdb..8fb92b1a13 100644 --- a/paddle/operators/math/sequence_pooling.cc +++ b/paddle/operators/math/sequence_pooling.cc @@ -20,9 +20,9 @@ namespace operators { namespace math { template -class MaxSeqPoolFunctor { +class MaxSeqPoolFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::LoDTensor& input, framework::Tensor* output, framework::Tensor* index) { auto in_dims = input.dims(); @@ -60,9 +60,9 @@ class MaxSeqPoolFunctor { }; template -class MaxSeqPoolGradFunctor { +class MaxSeqPoolGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& out_grad, const framework::Tensor& index, framework::LoDTensor* in_grad) { @@ -80,7 +80,7 @@ class MaxSeqPoolGradFunctor { const int* max_index = index.data(); T* ig_data = in_grad->data(); - SetConstant set_zero; + SetConstant set_zero; set_zero(context, in_grad, static_cast(0.0)); int64_t num_seq = og_dims[0]; int64_t dim = out_grad.numel() / num_seq; @@ -93,10 +93,10 @@ class MaxSeqPoolGradFunctor { } }; -template class MaxSeqPoolFunctor; -template class MaxSeqPoolFunctor; -template class MaxSeqPoolGradFunctor; -template class MaxSeqPoolGradFunctor; +template class MaxSeqPoolFunctor; +template class MaxSeqPoolFunctor; +template class MaxSeqPoolGradFunctor; +template class MaxSeqPoolGradFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/sequence_pooling.cu b/paddle/operators/math/sequence_pooling.cu index 5ed951402f..4c9e6b375c 100644 --- a/paddle/operators/math/sequence_pooling.cu +++ b/paddle/operators/math/sequence_pooling.cu @@ -46,9 +46,9 @@ __global__ void KeMaxSequencePool(const T* input, const size_t* starts, } template -class MaxSeqPoolFunctor { +class MaxSeqPoolFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::LoDTensor& input, framework::Tensor* output, framework::Tensor* index) { auto in_dims = input.dims(); @@ -71,8 +71,7 @@ class MaxSeqPoolFunctor { dim3 threads(256, 1); dim3 grid(num_seq, 1); - auto stream = - reinterpret_cast(context).stream(); + auto stream = context.stream(); KeMaxSequencePool<<>>( in_data, starts.data(), out_data, max_index, num_seq, dim); } @@ -91,9 +90,9 @@ __global__ void KeMaxSequencePoolGrad(const T* out_grad, const int* max_index, } template -class MaxSeqPoolGradFunctor { +class MaxSeqPoolGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& out_grad, const framework::Tensor& index, framework::LoDTensor* in_grad) { @@ -111,7 +110,7 @@ class MaxSeqPoolGradFunctor { const int* max_index = index.data(); T* ig_data = in_grad->data(); - SetConstant set_zero; + SetConstant set_zero; set_zero(context, in_grad, static_cast(0.0)); int64_t num_seq = og_dims[0]; int64_t dim = out_grad.numel() / num_seq; @@ -119,17 +118,16 @@ class MaxSeqPoolGradFunctor { unsigned int blocks = (num_seq * dim + 128 - 1) / 128; dim3 threads(128, 1); dim3 grid(blocks, 1); - auto stream = - reinterpret_cast(context).stream(); + auto stream = context.stream(); KeMaxSequencePoolGrad<<>>( og_data, max_index, ig_data, num_seq, dim); } }; -template class MaxSeqPoolFunctor; -template class MaxSeqPoolFunctor; -template class MaxSeqPoolGradFunctor; -template class MaxSeqPoolGradFunctor; +template class MaxSeqPoolFunctor; +template class MaxSeqPoolFunctor; +template class MaxSeqPoolGradFunctor; +template class MaxSeqPoolGradFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/sequence_pooling.h b/paddle/operators/math/sequence_pooling.h index 35dfe26de1..13ffb2ebef 100644 --- a/paddle/operators/math/sequence_pooling.h +++ b/paddle/operators/math/sequence_pooling.h @@ -23,18 +23,18 @@ namespace math { #define FLT_MAX __FLT_MAX__ -template +template class MaxSeqPoolFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::LoDTensor& input, framework::Tensor* output, framework::Tensor* index); }; -template +template class MaxSeqPoolGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::Tensor& out_grad, const framework::Tensor& index, framework::LoDTensor* in_grad); diff --git a/paddle/operators/math/softmax.cc b/paddle/operators/math/softmax.cc index 3e2f15d6c2..72f10f35f4 100644 --- a/paddle/operators/math/softmax.cc +++ b/paddle/operators/math/softmax.cc @@ -19,10 +19,10 @@ namespace paddle { namespace operators { namespace math { -template class SoftmaxFunctor; -template class SoftmaxFunctor; -template class SoftmaxGradFunctor; -template class SoftmaxGradFunctor; +template class SoftmaxFunctor; +template class SoftmaxFunctor; +template class SoftmaxGradFunctor; +template class SoftmaxGradFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/softmax.cu b/paddle/operators/math/softmax.cu index 4dbab51d46..9e73f6a371 100644 --- a/paddle/operators/math/softmax.cu +++ b/paddle/operators/math/softmax.cu @@ -21,10 +21,10 @@ namespace paddle { namespace operators { namespace math { -template class SoftmaxFunctor; -template class SoftmaxFunctor; -template class SoftmaxGradFunctor; -template class SoftmaxGradFunctor; +template class SoftmaxFunctor; +template class SoftmaxFunctor; +template class SoftmaxGradFunctor; +template class SoftmaxGradFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/softmax.h b/paddle/operators/math/softmax.h index fe10746502..471f44d340 100644 --- a/paddle/operators/math/softmax.h +++ b/paddle/operators/math/softmax.h @@ -19,19 +19,18 @@ namespace paddle { namespace operators { namespace math { -template +template class SoftmaxFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor* X, framework::Tensor* Y); + void operator()(const DeviceContext& context, const framework::Tensor* X, + framework::Tensor* Y); }; -template +template class SoftmaxGradFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor* y, const framework::Tensor* y_grad, - framework::Tensor* x_grad); + void operator()(const DeviceContext& context, const framework::Tensor* y, + const framework::Tensor* y_grad, framework::Tensor* x_grad); }; } // namespace math diff --git a/paddle/operators/math/softmax_impl.h b/paddle/operators/math/softmax_impl.h index 05793eeb3e..82f597ff79 100644 --- a/paddle/operators/math/softmax_impl.h +++ b/paddle/operators/math/softmax_impl.h @@ -32,10 +32,10 @@ struct ValueClip { } }; -template -void SoftmaxFunctor::operator()( - const platform::DeviceContext& context, const framework::Tensor* X, - framework::Tensor* Y) { +template +void SoftmaxFunctor::operator()(const DeviceContext& context, + const framework::Tensor* X, + framework::Tensor* Y) { auto logits = EigenMatrix::From(*X); auto softmax = EigenMatrix::From(*Y); @@ -56,19 +56,18 @@ void SoftmaxFunctor::operator()( .broadcast(one_by_class)) .unaryExpr(ValueClip()); - softmax.device(*context.GetEigenDevice()) = shifted_logits.exp(); - softmax.device(*context.GetEigenDevice()) = - (softmax * - softmax.sum(along_class) - .inverse() - .eval() - .reshape(batch_by_one) - .broadcast(one_by_class)); + softmax.device(*context.eigen_device()) = shifted_logits.exp(); + softmax.device(*context.eigen_device()) = (softmax * + softmax.sum(along_class) + .inverse() + .eval() + .reshape(batch_by_one) + .broadcast(one_by_class)); } -template -void SoftmaxGradFunctor::operator()( - const platform::DeviceContext& context, const framework::Tensor* y, +template +void SoftmaxGradFunctor::operator()( + const DeviceContext& context, const framework::Tensor* y, const framework::Tensor* y_grad, framework::Tensor* x_grad) { auto softmax = EigenMatrix::From(*y); auto softmax_grad = EigenMatrix::From(*y_grad); @@ -89,8 +88,7 @@ void SoftmaxGradFunctor::operator()( .eval() .reshape(batch_by_one) .broadcast(one_by_class); - logits_grad.device(*context.GetEigenDevice()) = - (softmax_grad - dot) * softmax; + logits_grad.device(*context.eigen_device()) = (softmax_grad - dot) * softmax; } } // namespace math diff --git a/paddle/operators/math/unpooling.cc b/paddle/operators/math/unpooling.cc index b57d3dc141..ecd3a647e0 100644 --- a/paddle/operators/math/unpooling.cc +++ b/paddle/operators/math/unpooling.cc @@ -17,9 +17,9 @@ namespace paddle { namespace operators { namespace math { template -class Unpool2dMaxFunctor { +class Unpool2dMaxFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, const framework::Tensor& indices, framework::Tensor* output) { const int batch_size = input.dims()[0]; @@ -48,9 +48,9 @@ class Unpool2dMaxFunctor { } }; template -class Unpool2dMaxGradFunctor { +class Unpool2dMaxGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, const framework::Tensor& indices, const framework::Tensor& output, @@ -82,10 +82,10 @@ class Unpool2dMaxGradFunctor { } } }; -template class Unpool2dMaxGradFunctor; -template class Unpool2dMaxGradFunctor; -template class Unpool2dMaxFunctor; -template class Unpool2dMaxFunctor; +template class Unpool2dMaxGradFunctor; +template class Unpool2dMaxGradFunctor; +template class Unpool2dMaxFunctor; +template class Unpool2dMaxFunctor; } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/unpooling.cu b/paddle/operators/math/unpooling.cu index 37c3c8b689..ecbde0f6a7 100644 --- a/paddle/operators/math/unpooling.cu +++ b/paddle/operators/math/unpooling.cu @@ -67,9 +67,9 @@ __global__ void KernelUnpool2dMaxGrad( * All tensors are in NCHW format. */ template -class Unpool2dMaxFunctor { +class Unpool2dMaxFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, const framework::Tensor& indices, framework::Tensor* output) { const int batch_size = input.dims()[0]; @@ -83,21 +83,18 @@ class Unpool2dMaxFunctor { T* output_data = output->mutable_data(context.GetPlace()); int threads = 1024; int grid = (input.numel() + threads - 1) / threads; - KernelUnpool2dMax< - T><<(context) - .stream()>>>(input.numel(), input_data, indices_data, - input_height, input_width, output_channels, - output_data, output_height, output_width); + KernelUnpool2dMax<<>>( + input.numel(), input_data, indices_data, input_height, input_width, + output_channels, output_data, output_height, output_width); } }; /* * All tensors are in NCHW format. */ template -class Unpool2dMaxGradFunctor { +class Unpool2dMaxGradFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, const framework::Tensor& indices, const framework::Tensor& output, @@ -116,19 +113,16 @@ class Unpool2dMaxGradFunctor { T* input_grad_data = input_grad->mutable_data(context.GetPlace()); int threads = 1024; int grid = (input.numel() + threads - 1) / threads; - KernelUnpool2dMaxGrad< - T><<(context) - .stream()>>>(input.numel(), input_data, indices_data, - input_height, input_width, output_channels, - output_data, output_grad_data, output_height, - output_width, input_grad_data); + KernelUnpool2dMaxGrad<<>>( + input.numel(), input_data, indices_data, input_height, input_width, + output_channels, output_data, output_grad_data, output_height, + output_width, input_grad_data); } }; -template class Unpool2dMaxGradFunctor; -template class Unpool2dMaxGradFunctor; -template class Unpool2dMaxFunctor; -template class Unpool2dMaxFunctor; +template class Unpool2dMaxGradFunctor; +template class Unpool2dMaxGradFunctor; +template class Unpool2dMaxFunctor; +template class Unpool2dMaxFunctor; } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/unpooling.h b/paddle/operators/math/unpooling.h index 7077d7c227..0f0ff1371e 100644 --- a/paddle/operators/math/unpooling.h +++ b/paddle/operators/math/unpooling.h @@ -18,18 +18,16 @@ limitations under the License. */ namespace paddle { namespace operators { namespace math { -template +template class Unpool2dMaxFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, + void operator()(const DeviceContext& context, const framework::Tensor& input, const framework::Tensor& indices, framework::Tensor* output); }; -template +template class Unpool2dMaxGradFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& input, + void operator()(const DeviceContext& context, const framework::Tensor& input, const framework::Tensor& indices, const framework::Tensor& output, const framework::Tensor& output_grad, diff --git a/paddle/operators/math/vol2col.cc b/paddle/operators/math/vol2col.cc index 99eb7fd46d..d574ed9234 100644 --- a/paddle/operators/math/vol2col.cc +++ b/paddle/operators/math/vol2col.cc @@ -25,9 +25,9 @@ namespace math { * output_depth, output_height, output_width] */ template -class Vol2ColFunctor { +class Vol2ColFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& vol, const std::vector& dilations, const std::vector& strides, @@ -111,9 +111,9 @@ class Vol2ColFunctor { * output_depth, output_height, output_width] */ template -class Col2VolFunctor { +class Col2VolFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& col, const std::vector& dilations, const std::vector& strides, @@ -190,10 +190,10 @@ class Col2VolFunctor { } }; -template class Vol2ColFunctor; -template class Vol2ColFunctor; -template class Col2VolFunctor; -template class Col2VolFunctor; +template class Vol2ColFunctor; +template class Vol2ColFunctor; +template class Col2VolFunctor; +template class Col2VolFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/vol2col.cu b/paddle/operators/math/vol2col.cu index dae3be858e..b029442fe4 100644 --- a/paddle/operators/math/vol2col.cu +++ b/paddle/operators/math/vol2col.cu @@ -68,9 +68,9 @@ __global__ void vol2col(int num_kernels, const T* data_vol, int depth, * output_depth, output_height, output_width] */ template -class Vol2ColFunctor { +class Vol2ColFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& vol, const std::vector& dilations, const std::vector& strides, @@ -117,9 +117,7 @@ class Vol2ColFunctor { const int threads = 1024; const int blocks = (num_outputs + 1024 - 1) / 1024; - vol2col<<(context) - .stream()>>>( + vol2col<<>>( num_outputs, vol.data(), input_depth, input_height, input_width, dilations[0], dilations[1], dilations[2], filter_depth, filter_height, filter_width, strides[0], strides[1], strides[2], paddings[0], @@ -196,9 +194,9 @@ __global__ void col2vol(int num_kernels, const T* data_col, int depth, * output_depth, output_height, output_width] */ template -class Col2VolFunctor { +class Col2VolFunctor { public: - void operator()(const platform::DeviceContext& context, + void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& col, const std::vector& dilations, const std::vector& strides, @@ -245,9 +243,7 @@ class Col2VolFunctor { const int threads = 1024; const int blocks = (num_kernels + 1024 - 1) / 1024; - col2vol<<(context) - .stream()>>>( + col2vol<<>>( num_kernels, col.data(), input_depth, input_height, input_width, dilations[0], dilations[1], dilations[2], filter_depth, filter_height, filter_width, strides[0], strides[1], strides[2], paddings[0], @@ -256,10 +252,10 @@ class Col2VolFunctor { } }; -template class Vol2ColFunctor; -template class Vol2ColFunctor; -template class Col2VolFunctor; -template class Col2VolFunctor; +template class Vol2ColFunctor; +template class Vol2ColFunctor; +template class Col2VolFunctor; +template class Col2VolFunctor; } // namespace math } // namespace operators diff --git a/paddle/operators/math/vol2col.h b/paddle/operators/math/vol2col.h index dc64d1d977..dcd80370e8 100644 --- a/paddle/operators/math/vol2col.h +++ b/paddle/operators/math/vol2col.h @@ -63,22 +63,20 @@ namespace math { * \note The caller needs to ensure that volShape.inputChannels is equal to * colShape.inputChannels. */ -template +template class Vol2ColFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& vol, + void operator()(const DeviceContext& context, const framework::Tensor& vol, const std::vector& dilations, const std::vector& strides, const std::vector& paddings, framework::Tensor* col) const; }; -template +template class Col2VolFunctor { public: - void operator()(const platform::DeviceContext& context, - const framework::Tensor& col, + void operator()(const DeviceContext& context, const framework::Tensor& col, const std::vector& dilations, const std::vector& strides, const std::vector& paddings, diff --git a/paddle/operators/math/vol2col_test.cc b/paddle/operators/math/vol2col_test.cc index 62c3152304..f46db3c567 100644 --- a/paddle/operators/math/vol2col_test.cc +++ b/paddle/operators/math/vol2col_test.cc @@ -16,7 +16,7 @@ limitations under the License. */ #include #include -template +template void testVol2col() { paddle::framework::Tensor input; paddle::framework::Tensor input_tmp; @@ -24,18 +24,7 @@ void testVol2col() { paddle::framework::Tensor output_tmp; auto* place = new Place(); - paddle::platform::DeviceContext* context; - if (paddle::platform::is_cpu_place(*place)) { - context = - new paddle::platform::CPUDeviceContext(paddle::platform::CPUPlace()); - } else { -#ifdef PADDLE_WITH_CUDA - context = - new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace()); -#else - PADDLE_THROW("no GPU support"); -#endif // PADDLE_WITH_CUDA - } + DeviceContext* context = new DeviceContext(*place); /** * input = [[0, 1, 2, @@ -88,7 +77,7 @@ void testVol2col() { output_depth, output_height, output_width}, *place); - paddle::operators::math::Vol2ColFunctor vol2col; + paddle::operators::math::Vol2ColFunctor vol2col; vol2col(*context, input, dilations, strides, paddings, &output); float vol_2_col[] = {0, 1, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11}; @@ -113,7 +102,7 @@ void testVol2col() { CopyFrom(input_tmp, *place, *context, &input); } - paddle::operators::math::Col2VolFunctor col2vol; + paddle::operators::math::Col2VolFunctor col2vol; col2vol(*context, output, dilations, strides, paddings, &input); float* in_ptr; @@ -130,8 +119,9 @@ void testVol2col() { } TEST(math, vol2col) { - testVol2col(); + testVol2col(); #ifdef PADDLE_WITH_CUDA - testVol2col(); + testVol2col(); #endif // PADDLE_WITH_CUDA } diff --git a/paddle/operators/matmul_op.cc b/paddle/operators/matmul_op.cc index 5a1a615420..ee0bc0c370 100644 --- a/paddle/operators/matmul_op.cc +++ b/paddle/operators/matmul_op.cc @@ -206,7 +206,8 @@ class MatMulOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(matmul, ops::MatMulOp, ops::MatMulOpMaker, matmul_grad, ops::MatMulOpGrad); -REGISTER_OP_CPU_KERNEL(matmul, - ops::MatMulKernel); REGISTER_OP_CPU_KERNEL( - matmul_grad, ops::MatMulGradKernel); + matmul, ops::MatMulKernel); +REGISTER_OP_CPU_KERNEL( + matmul_grad, + ops::MatMulGradKernel); diff --git a/paddle/operators/matmul_op.cu.cc b/paddle/operators/matmul_op.cu.cc index b7e66382f0..6a3772c004 100644 --- a/paddle/operators/matmul_op.cu.cc +++ b/paddle/operators/matmul_op.cu.cc @@ -15,7 +15,8 @@ #include "paddle/operators/matmul_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(matmul, - ops::MatMulKernel); -REGISTER_OP_GPU_KERNEL( - matmul_grad, ops::MatMulGradKernel); +REGISTER_OP_CUDA_KERNEL( + matmul, ops::MatMulKernel); +REGISTER_OP_CUDA_KERNEL( + matmul_grad, + ops::MatMulGradKernel); diff --git a/paddle/operators/matmul_op.h b/paddle/operators/matmul_op.h index 1e4aa48b70..de9da487b3 100644 --- a/paddle/operators/matmul_op.h +++ b/paddle/operators/matmul_op.h @@ -27,7 +27,7 @@ using DDim = framework::DDim; using framework::make_ddim; using framework::vectorize; -template +template class MatMulKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -38,8 +38,9 @@ class MatMulKernel : public framework::OpKernel { bool transpose_x = context.Attr("transpose_X"); bool transpose_y = context.Attr("transpose_Y"); - math::MatMulFunctor()(context.device_context(), x, transpose_x, y, - transpose_y, T(1), out, T(0)); + math::MatMulFunctor()( + context.template device_context(), x, transpose_x, y, + transpose_y, T(1), out, T(0)); } }; @@ -68,17 +69,16 @@ Tensor CombineBatchAndM(const Tensor& input) { // Reshape a rank-3 tensor from P x M x N to M x (P * N). // (Warning: This requires transposing data and writes into new memory.) // Identity op if the tensor is not of rank 3. -template -Tensor CombineBatchAndN(const framework::ExecutionContext& context, - const Tensor& input) { +template +Tensor CombineBatchAndN(const DeviceContext& context, const Tensor& input) { Tensor output; auto in_dims = input.dims(); if (in_dims.size() == 3) { output.Resize({in_dims[1], in_dims[0], in_dims[2]}); output.mutable_data(context.GetPlace()); std::vector axis = {1, 0, 2}; - math::Transpose trans; - trans(context.device_context(), input, &output, axis); + math::Transpose trans; + trans(context, input, &output, axis); std::vector out_dims = {in_dims[1], in_dims[0] * in_dims[2]}; output.Resize({in_dims[1], in_dims[0] * in_dims[2]}); } else { @@ -112,7 +112,7 @@ Tensor CombineBatchAndN(const framework::ExecutionContext& context, // // To handle this sort of scenario, we reshape X : P x M x K, dOut: P x M x N // to X: (P * M) x K, dOut: (P * M) x N. -template +template class MatMulGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -178,24 +178,23 @@ class MatMulGradKernel : public framework::OpKernel { Tensor Y = Reshape(y, make_ddim(y_dims)); Tensor dOut = Reshape(dout, make_ddim(dout_dims)); + auto& dev_ctx = context.template device_context(); if (dx) { dx->mutable_data(context.GetPlace()); const Tensor& dOut_for_dX = (x_dims.size() == 2 && y_dims.size() == 3) - ? CombineBatchAndN(context, dOut) + ? CombineBatchAndN(dev_ctx, dOut) : dOut; if (x_dims.size() == 2 && y_dims.size() == 3) { Y = transpose_y ? CombineBatchAndM(Y) - : CombineBatchAndN(context, Y); + : CombineBatchAndN(dev_ctx, Y); } if (transpose_x) { - math::MatMulFunctor()(context.device_context(), Y, - transpose_y, dOut_for_dX, transpose_x, - T(1), dx, T(0)); + math::MatMulFunctor()( + dev_ctx, Y, transpose_y, dOut_for_dX, transpose_x, T(1), dx, T(0)); } else { - math::MatMulFunctor()(context.device_context(), dOut_for_dX, - transpose_x, Y, !transpose_y, T(1), dx, - T(0)); + math::MatMulFunctor()( + dev_ctx, dOut_for_dX, transpose_x, Y, !transpose_y, T(1), dx, T(0)); } } @@ -205,18 +204,16 @@ class MatMulGradKernel : public framework::OpKernel { ? CombineBatchAndM(dOut) : dOut; if (y_dims.size() == 2 && x_dims.size() == 3) { - X = transpose_x ? CombineBatchAndN(context, X) + X = transpose_x ? CombineBatchAndN(dev_ctx, X) : CombineBatchAndM(X); dOut = CombineBatchAndM(dOut); } if (transpose_y) { - math::MatMulFunctor()(context.device_context(), dOut_for_dY, - transpose_y, X, transpose_x, T(1), dy, - T(0)); + math::MatMulFunctor()( + dev_ctx, dOut_for_dY, transpose_y, X, transpose_x, T(1), dy, T(0)); } else { - math::MatMulFunctor()(context.device_context(), X, - !transpose_x, dOut_for_dY, transpose_y, - T(1), dy, T(0)); + math::MatMulFunctor()( + dev_ctx, X, !transpose_x, dOut_for_dY, transpose_y, T(1), dy, T(0)); } } } diff --git a/paddle/operators/maxout_op.cc b/paddle/operators/maxout_op.cc index 44bf402e95..011616e615 100644 --- a/paddle/operators/maxout_op.cc +++ b/paddle/operators/maxout_op.cc @@ -101,7 +101,8 @@ class MaxOutOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(maxout, ops::MaxOutOp, ops::MaxOutOpMaker, maxout_grad, ops::MaxOutOpGrad); -REGISTER_OP_CPU_KERNEL(maxout, - ops::MaxOutKernel); REGISTER_OP_CPU_KERNEL( - maxout_grad, ops::MaxOutGradKernel); + maxout, ops::MaxOutKernel); +REGISTER_OP_CPU_KERNEL( + maxout_grad, + ops::MaxOutGradKernel); diff --git a/paddle/operators/maxout_op.cu.cc b/paddle/operators/maxout_op.cu.cc index decd43913d..2904f0ff96 100644 --- a/paddle/operators/maxout_op.cu.cc +++ b/paddle/operators/maxout_op.cu.cc @@ -15,9 +15,10 @@ #include "paddle/operators/maxout_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(maxout, - ops::MaxOutKernel, - ops::MaxOutKernel); -REGISTER_OP_GPU_KERNEL( - maxout_grad, ops::MaxOutGradKernel, - ops::MaxOutGradKernel); +REGISTER_OP_CUDA_KERNEL( + maxout, ops::MaxOutKernel, + ops::MaxOutKernel); +REGISTER_OP_CUDA_KERNEL( + maxout_grad, + ops::MaxOutGradKernel, + ops::MaxOutGradKernel); diff --git a/paddle/operators/maxout_op.h b/paddle/operators/maxout_op.h index 44a0d073dd..e8b12552b9 100644 --- a/paddle/operators/maxout_op.h +++ b/paddle/operators/maxout_op.h @@ -23,7 +23,7 @@ namespace operators { using Tensor = framework::Tensor; -template +template class MaxOutKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -31,12 +31,13 @@ class MaxOutKernel : public framework::OpKernel { Tensor* out = context.Output("Out"); int groups = context.template Attr("groups"); - math::MaxOutFunctor maxout_forward; - maxout_forward(context.device_context(), *in_x, out, groups); + math::MaxOutFunctor maxout_forward; + maxout_forward(context.template device_context(), *in_x, out, + groups); } }; -template +template class MaxOutGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -46,14 +47,13 @@ class MaxOutGradKernel : public framework::OpKernel { context.Input(framework::GradVarName("Out")); Tensor* in_x_grad = context.Output(framework::GradVarName("X")); int groups = context.template Attr("groups"); - auto& device_ctx = context.device_context(); - math::SetConstant zero; + auto& device_ctx = context.template device_context(); + math::SetConstant zero; if (in_x_grad) { in_x_grad->mutable_data(context.GetPlace()); zero(device_ctx, in_x_grad, static_cast(0.0)); - math::MaxOutGradFunctor maxout_backward; - maxout_backward(context.device_context(), *in_x, in_x_grad, *out, - *out_grad, groups); + math::MaxOutGradFunctor maxout_backward; + maxout_backward(device_ctx, *in_x, in_x_grad, *out, *out_grad, groups); } } }; diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index dcc5b4286f..8932d700c2 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -76,8 +76,9 @@ class MeanGradMaker : public framework::SingleGradOpDescMaker { namespace ops = paddle::operators; REGISTER_OPERATOR(mean, ops::MeanOp, ops::MeanOpMaker, ops::MeanGradMaker); REGISTER_OPERATOR(mean_grad, ops::MeanGradOp); -REGISTER_OP_CPU_KERNEL(mean, ops::MeanKernel, - ops::MeanKernel); -REGISTER_OP_CPU_KERNEL(mean_grad, - ops::MeanGradKernel, - ops::MeanGradKernel); +REGISTER_OP_CPU_KERNEL( + mean, ops::MeanKernel, + ops::MeanKernel); +REGISTER_OP_CPU_KERNEL( + mean_grad, ops::MeanGradKernel, + ops::MeanGradKernel); diff --git a/paddle/operators/mean_op.cu b/paddle/operators/mean_op.cu index ca089938c0..93062bf540 100644 --- a/paddle/operators/mean_op.cu +++ b/paddle/operators/mean_op.cu @@ -17,8 +17,9 @@ #include "paddle/operators/mean_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(mean, ops::MeanKernel, - ops::MeanKernel); -REGISTER_OP_GPU_KERNEL(mean_grad, - ops::MeanGradKernel, - ops::MeanGradKernel); +REGISTER_OP_CUDA_KERNEL( + mean, ops::MeanKernel, + ops::MeanKernel); +REGISTER_OP_CUDA_KERNEL( + mean_grad, ops::MeanGradKernel, + ops::MeanGradKernel); diff --git a/paddle/operators/mean_op.h b/paddle/operators/mean_op.h index c99286a5b9..351b345959 100644 --- a/paddle/operators/mean_op.h +++ b/paddle/operators/mean_op.h @@ -27,7 +27,7 @@ template using EigenVector = framework::EigenVector; -template +template class MeanKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -38,13 +38,14 @@ class MeanKernel : public framework::OpKernel { auto X = EigenVector::Flatten(*input); auto y = EigenScalar::From(*output); - auto& place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); y.device(place) = X.mean(); } }; -template +template class MeanGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -56,7 +57,8 @@ class MeanGradKernel : public framework::OpKernel { T ig_size = static_cast(IG->numel()); Eigen::DSizes bcast(ig_size); - EigenVector::Flatten(*IG).device(context.GetEigenDevice()) = + EigenVector::Flatten(*IG).device( + *context.template device_context().eigen_device()) = (EigenVector::From(*OG) / ig_size).broadcast(bcast); } }; diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc index 4684c20208..27f0c8de20 100644 --- a/paddle/operators/minus_op.cc +++ b/paddle/operators/minus_op.cc @@ -102,5 +102,5 @@ class MinusGradMaker : public framework::GradOpDescMakerBase { namespace ops = paddle::operators; REGISTER_OPERATOR(minus, ops::MinusOp, ops::MinusOpMaker, ops::MinusGradMaker); -REGISTER_OP_CPU_KERNEL(minus, - ops::MinusKernel); +REGISTER_OP_CPU_KERNEL( + minus, ops::MinusKernel); diff --git a/paddle/operators/minus_op.cu b/paddle/operators/minus_op.cu index a8375cc630..3b202ea92e 100644 --- a/paddle/operators/minus_op.cu +++ b/paddle/operators/minus_op.cu @@ -14,5 +14,6 @@ #include "paddle/operators/minus_op.h" -REGISTER_OP_GPU_KERNEL( - minus, paddle::operators::MinusKernel); +REGISTER_OP_CUDA_KERNEL( + minus, + paddle::operators::MinusKernel); diff --git a/paddle/operators/minus_op.h b/paddle/operators/minus_op.h index bd9a2790aa..78e1e1be6d 100644 --- a/paddle/operators/minus_op.h +++ b/paddle/operators/minus_op.h @@ -19,7 +19,7 @@ namespace paddle { namespace operators { -template +template class MinusKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -28,7 +28,8 @@ class MinusKernel : public framework::OpKernel { auto* out_tensor = context.Output("Out"); out_tensor->mutable_data(context.GetPlace()); - auto& dev = context.GetEigenDevice(); + auto& dev = + *context.template device_context().eigen_device(); framework::EigenVector::Flatten(*out_tensor).device(dev) = framework::EigenVector::Flatten(*left_tensor) - framework::EigenVector::Flatten(*right_tensor); diff --git a/paddle/operators/modified_huber_loss_op.cc b/paddle/operators/modified_huber_loss_op.cc index 28528848af..f0a42491bf 100644 --- a/paddle/operators/modified_huber_loss_op.cc +++ b/paddle/operators/modified_huber_loss_op.cc @@ -115,6 +115,6 @@ REGISTER_OP(modified_huber_loss, ops::ModifiedHuberLossOp, REGISTER_OP_CPU_KERNEL( modified_huber_loss, - ops::ModifiedHuberLossKernel); + ops::ModifiedHuberLossKernel); REGISTER_OP_CPU_KERNEL(modified_huber_loss_grad, ops::ModifiedHuberLossGradCPUKernel); diff --git a/paddle/operators/modified_huber_loss_op.cu b/paddle/operators/modified_huber_loss_op.cu index 8854e166cd..40a8447da4 100644 --- a/paddle/operators/modified_huber_loss_op.cu +++ b/paddle/operators/modified_huber_loss_op.cu @@ -71,8 +71,8 @@ class ModifiedHuberLossGradGPUKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( modified_huber_loss, - ops::ModifiedHuberLossKernel); -REGISTER_OP_GPU_KERNEL(modified_huber_loss_grad, - ops::ModifiedHuberLossGradGPUKernel); + ops::ModifiedHuberLossKernel); +REGISTER_OP_CUDA_KERNEL(modified_huber_loss_grad, + ops::ModifiedHuberLossGradGPUKernel); diff --git a/paddle/operators/modified_huber_loss_op.h b/paddle/operators/modified_huber_loss_op.h index aba75efad9..157ae0682e 100644 --- a/paddle/operators/modified_huber_loss_op.h +++ b/paddle/operators/modified_huber_loss_op.h @@ -46,7 +46,7 @@ struct ModifiedHuberLossForward { } }; -template +template class ModifiedHuberLossKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -57,7 +57,8 @@ class ModifiedHuberLossKernel : public framework::OpKernel { out0->mutable_data(context.GetPlace()); out1->mutable_data(context.GetPlace()); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); auto x = EigenVector::Flatten(*in0); auto y = EigenVector::Flatten(*in1); diff --git a/paddle/operators/momentum_op.cu b/paddle/operators/momentum_op.cu index be0c8ea071..00f1253465 100644 --- a/paddle/operators/momentum_op.cu +++ b/paddle/operators/momentum_op.cu @@ -74,5 +74,5 @@ class MomentumOpCUDAKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(momentum, ops::MomentumOpCUDAKernel, - ops::MomentumOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(momentum, ops::MomentumOpCUDAKernel, + ops::MomentumOpCUDAKernel); diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc index 3c39ae10dc..bc4a5fdf0b 100644 --- a/paddle/operators/mul_op.cc +++ b/paddle/operators/mul_op.cc @@ -149,6 +149,7 @@ REGISTER_OPERATOR(mul, paddle::framework::OperatorWithKernel, ops::MulOpMaker, ops::MulOpShapeInference, paddle::framework::DefaultGradOpDescMaker); REGISTER_OPERATOR(mul_grad, ops::MulOpGrad); -REGISTER_OP_CPU_KERNEL(mul, ops::MulKernel); -REGISTER_OP_CPU_KERNEL(mul_grad, - ops::MulGradKernel); +REGISTER_OP_CPU_KERNEL( + mul, ops::MulKernel); +REGISTER_OP_CPU_KERNEL( + mul_grad, ops::MulGradKernel); diff --git a/paddle/operators/mul_op.cu.cc b/paddle/operators/mul_op.cu.cc index 66dc3d6d10..6095de58d0 100644 --- a/paddle/operators/mul_op.cu.cc +++ b/paddle/operators/mul_op.cu.cc @@ -15,6 +15,7 @@ #include "paddle/operators/mul_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(mul, ops::MulKernel); -REGISTER_OP_GPU_KERNEL(mul_grad, - ops::MulGradKernel); +REGISTER_OP_CUDA_KERNEL( + mul, ops::MulKernel); +REGISTER_OP_CUDA_KERNEL( + mul_grad, ops::MulGradKernel); diff --git a/paddle/operators/mul_op.h b/paddle/operators/mul_op.h index 0eb9df41e9..1b467dca83 100644 --- a/paddle/operators/mul_op.h +++ b/paddle/operators/mul_op.h @@ -23,7 +23,7 @@ namespace operators { using Tensor = framework::Tensor; -template +template class MulKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -46,15 +46,16 @@ class MulKernel : public framework::OpKernel { if (z_dim.size() != 2) { z->Resize({x_matrix.dims()[0], y_matrix.dims()[1]}); } - math::matmul(context.device_context(), x_matrix, false, y_matrix, - false, 1, z, 0); + math::matmul( + context.template device_context(), x_matrix, false, + y_matrix, false, 1, z, 0); if (z_dim.size() != 2) { z->Resize(z_dim); } } }; -template +template class MulGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -77,6 +78,7 @@ class MulGradKernel : public framework::OpKernel { Tensor* dx = ctx.Output(framework::GradVarName("X")); Tensor* dy = ctx.Output(framework::GradVarName("Y")); + auto& dev_ctx = ctx.template device_context(); if (dx) { dx->mutable_data(ctx.GetPlace()); Tensor dx_matrix = dx->dims().size() > 2 @@ -84,8 +86,8 @@ class MulGradKernel : public framework::OpKernel { : *dx; // dx = dout * y'. dx: M x K, dout : M x N, y : K x N - math::matmul(ctx.device_context(), dout_mat, false, y_matrix, - true, 1, &dx_matrix, 0); + math::matmul(dev_ctx, dout_mat, false, y_matrix, true, + 1, &dx_matrix, 0); } if (dy) { dy->mutable_data(ctx.GetPlace()); @@ -93,8 +95,8 @@ class MulGradKernel : public framework::OpKernel { ? framework::ReshapeToMatrix(*dy, y_num_col_dims) : *dy; // dy = x' * dout. dy K x N, dout : M x N, x : M x K - math::matmul(ctx.device_context(), x_matrix, true, dout_mat, - false, 1, &dy_matrix, 0); + math::matmul(dev_ctx, x_matrix, true, dout_mat, false, + 1, &dy_matrix, 0); } } }; diff --git a/paddle/operators/multiplex_op.cc b/paddle/operators/multiplex_op.cc index 8e7f544e0d..b1ee8051c4 100644 --- a/paddle/operators/multiplex_op.cc +++ b/paddle/operators/multiplex_op.cc @@ -119,7 +119,8 @@ REGISTER_OPERATOR(multiplex, ops::MultiplexOp, ops::MultiplexOpMaker, paddle::framework::DefaultGradOpDescMaker); REGISTER_OPERATOR(multiplex_grad, ops::MultiplexGradOp); REGISTER_OP_CPU_KERNEL( - multiplex, ops::MultiplexCPUKernel); + multiplex, + ops::MultiplexCPUKernel); REGISTER_OP_CPU_KERNEL( multiplex_grad, - ops::MultiplexGradCPUKernel); + ops::MultiplexGradCPUKernel); diff --git a/paddle/operators/multiplex_op.cu b/paddle/operators/multiplex_op.cu index 10dff8d021..47986e9ff8 100644 --- a/paddle/operators/multiplex_op.cu +++ b/paddle/operators/multiplex_op.cu @@ -36,7 +36,7 @@ class MultiplexGPUKernel : public framework::OpKernel { CopyFrom(*ids, platform::CPUPlace(), ctx.device_context(), &index_t_cpu); auto* index = index_t_cpu.data(); auto stream = ctx.cuda_device_context().stream(); - Place place = boost::get(ctx.GetPlace()); + platform::GPUPlace place = boost::get(ctx.GetPlace()); for (auto i = 0; i < rows; i++) { int32_t k = index[i]; PADDLE_ENFORCE_GE(k, 0, "index must be nonnegative."); @@ -60,7 +60,8 @@ class MultiplexGradGPUKernel : public framework::OpKernel { if (d_ins[i]) { d_ins[i]->mutable_data(ctx.GetPlace()); auto t = framework::EigenVector::Flatten(*d_ins[i]); - t.device(ctx.GetEigenDevice()) = t.constant(static_cast(0)); + t.device(*ctx.template device_context().eigen_device()) = + t.constant(static_cast(0)); } } @@ -72,7 +73,7 @@ class MultiplexGradGPUKernel : public framework::OpKernel { auto* index = index_t_cpu.data(); auto stream = ctx.cuda_device_context().stream(); - Place place = boost::get(ctx.GetPlace()); + platform::GPUPlace place = boost::get(ctx.GetPlace()); for (auto i = 0; i < rows; i++) { size_t k = static_cast(index[i]); if (d_ins[k]) { @@ -87,8 +88,9 @@ class MultiplexGradGPUKernel : public framework::OpKernel { namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( - multiplex, ops::MultiplexGPUKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( + multiplex, + ops::MultiplexGPUKernel); +REGISTER_OP_CUDA_KERNEL( multiplex_grad, - ops::MultiplexGradGPUKernel); + ops::MultiplexGradGPUKernel); diff --git a/paddle/operators/multiplex_op.h b/paddle/operators/multiplex_op.h index ab3cafaa32..3443151161 100644 --- a/paddle/operators/multiplex_op.h +++ b/paddle/operators/multiplex_op.h @@ -22,7 +22,7 @@ namespace paddle { namespace operators { -template +template class MultiplexCPUKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { @@ -35,7 +35,7 @@ class MultiplexCPUKernel : public framework::OpKernel { auto rows = ins[0]->dims()[0]; auto cols = ins[0]->numel() / rows; auto index = ids->data(); - Place place = boost::get(ctx.GetPlace()); + platform::CPUPlace place = boost::get(ctx.GetPlace()); for (auto i = 0; i < rows; i++) { int32_t k = index[i]; PADDLE_ENFORCE_GE(k, 0, "index must be nonnegative."); @@ -47,7 +47,7 @@ class MultiplexCPUKernel : public framework::OpKernel { } }; -template +template class MultiplexGradCPUKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { @@ -60,14 +60,15 @@ class MultiplexGradCPUKernel : public framework::OpKernel { if (d_ins[i]) { d_ins[i]->mutable_data(ctx.GetPlace()); auto t = framework::EigenVector::Flatten(*d_ins[i]); - t.device(ctx.GetEigenDevice()) = t.constant(static_cast(0)); + t.device(*ctx.template device_context().eigen_device()) = + t.constant(static_cast(0)); } } auto rows = ins[0]->dims()[0]; auto cols = ins[0]->numel() / rows; auto* index = ids->data(); - Place place = boost::get(ctx.GetPlace()); + platform::CPUPlace place = boost::get(ctx.GetPlace()); for (auto i = 0; i < rows; i++) { size_t k = static_cast(index[i]); if (d_ins[k]) { diff --git a/paddle/operators/nccl_op.cu.cc b/paddle/operators/nccl_op.cu.cc index 4f0a2a79ed..6ca6db7253 100644 --- a/paddle/operators/nccl_op.cu.cc +++ b/paddle/operators/nccl_op.cu.cc @@ -204,6 +204,6 @@ class NCCLBcastKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(ncclAllReduce, ops::NCCLAllReduceKernel); -REGISTER_OP_GPU_KERNEL(ncclBcast, ops::NCCLBcastKernel); -REGISTER_OP_GPU_KERNEL(ncclReduce, ops::NCCLReduceKernel); +REGISTER_OP_CUDA_KERNEL(ncclAllReduce, ops::NCCLAllReduceKernel); +REGISTER_OP_CUDA_KERNEL(ncclBcast, ops::NCCLBcastKernel); +REGISTER_OP_CUDA_KERNEL(ncclReduce, ops::NCCLReduceKernel); diff --git a/paddle/operators/nccl_op_test.cu.cc b/paddle/operators/nccl_op_test.cu.cc index bb7ae20286..d747cc0cf5 100644 --- a/paddle/operators/nccl_op_test.cu.cc +++ b/paddle/operators/nccl_op_test.cu.cc @@ -33,9 +33,9 @@ #include "paddle/platform/place.h" USE_NO_KERNEL_OP(ncclInit); -USE_GPU_ONLY_OP(ncclAllReduce); -USE_GPU_ONLY_OP(ncclReduce); -USE_GPU_ONLY_OP(ncclBcast); +USE_CUDA_ONLY_OP(ncclAllReduce); +USE_CUDA_ONLY_OP(ncclReduce); +USE_CUDA_ONLY_OP(ncclBcast); namespace f = paddle::framework; namespace p = paddle::platform; diff --git a/paddle/operators/nce_op.cc b/paddle/operators/nce_op.cc index 952da10434..5ad1610fde 100644 --- a/paddle/operators/nce_op.cc +++ b/paddle/operators/nce_op.cc @@ -67,7 +67,7 @@ class NCEOp : public framework::OperatorWithKernel { const framework::ExecutionContext& ctx) const override { return framework::OpKernelType( framework::ToDataType(ctx.Input("Input")->type()), - ctx.device_context()); + ctx.GetPlace()); } }; @@ -170,7 +170,7 @@ class NCEOpGrad : public framework::OperatorWithKernel { const framework::ExecutionContext& ctx) const override { return framework::OpKernelType( framework::ToDataType(ctx.Input("Input")->type()), - ctx.device_context()); + ctx.GetPlace()); } }; diff --git a/paddle/operators/nce_op.h b/paddle/operators/nce_op.h index 0a8a95de5f..6636dad060 100644 --- a/paddle/operators/nce_op.h +++ b/paddle/operators/nce_op.h @@ -28,7 +28,7 @@ template using EigenMatrix = framework::EigenMatrix; -template +template void PrepareSamples(const framework::ExecutionContext& context) { auto label = context.Input("Label"); const int64_t* label_data = label->data(); @@ -67,11 +67,11 @@ void PrepareSamples(const framework::ExecutionContext& context) { } } -template +template class NCEKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - PrepareSamples(context); + PrepareSamples(context); auto sample_labels = context.Output("SampleLabels"); const int64_t* sample_labels_data = sample_labels->data(); auto sample_out = context.Output("SampleLogits"); @@ -135,7 +135,7 @@ class NCEKernel : public framework::OpKernel { } }; -template +template class NCEGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc index adb75df6ef..936dde22c3 100644 --- a/paddle/operators/pad_op.cc +++ b/paddle/operators/pad_op.cc @@ -134,6 +134,7 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(pad, ops::PadOp, ops::PadOpMaker, ops::PadOpGradMaker); REGISTER_OPERATOR(pad_grad, ops::PadOpGrad); -REGISTER_OP_CPU_KERNEL(pad, ops::PadKernel); -REGISTER_OP_CPU_KERNEL(pad_grad, - ops::PadGradKernel); +REGISTER_OP_CPU_KERNEL( + pad, ops::PadKernel); +REGISTER_OP_CPU_KERNEL( + pad_grad, ops::PadGradKernel); diff --git a/paddle/operators/pad_op.cu b/paddle/operators/pad_op.cu index 555a7dba23..c309fb625c 100644 --- a/paddle/operators/pad_op.cu +++ b/paddle/operators/pad_op.cu @@ -16,6 +16,7 @@ #include "paddle/operators/pad_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(pad, ops::PadKernel); -REGISTER_OP_GPU_KERNEL(pad_grad, - ops::PadGradKernel); +REGISTER_OP_CUDA_KERNEL( + pad, ops::PadKernel); +REGISTER_OP_CUDA_KERNEL( + pad_grad, ops::PadGradKernel); diff --git a/paddle/operators/pad_op.h b/paddle/operators/pad_op.h index 9534dbf545..1b95942af3 100644 --- a/paddle/operators/pad_op.h +++ b/paddle/operators/pad_op.h @@ -26,7 +26,7 @@ template using EigenTensor = framework::EigenTensor; -template +template void PadFunction(const framework::ExecutionContext& context) { auto pads = context.Attr>("paddings"); Eigen::array, D> paddings; @@ -42,33 +42,34 @@ void PadFunction(const framework::ExecutionContext& context) { auto x_tensor = EigenTensor::From(*x); auto out_tensor = EigenTensor::From(*out); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); out_tensor.device(place) = x_tensor.pad(paddings, pad_value); } -template +template class PadKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { int rank = context.Input("X")->dims().size(); switch (rank) { case 1: - PadFunction(context); + PadFunction(context); break; case 2: - PadFunction(context); + PadFunction(context); break; case 3: - PadFunction(context); + PadFunction(context); break; case 4: - PadFunction(context); + PadFunction(context); break; case 5: - PadFunction(context); + PadFunction(context); break; case 6: - PadFunction(context); + PadFunction(context); break; default: PADDLE_THROW( @@ -77,7 +78,7 @@ class PadKernel : public framework::OpKernel { } }; -template +template void PadGradFunction(const framework::ExecutionContext& context) { auto pads = context.Attr>("paddings"); Eigen::array, D> paddings; @@ -91,12 +92,13 @@ void PadGradFunction(const framework::ExecutionContext& context) { d_x->mutable_data(context.GetPlace()); auto d_x_tensor = EigenTensor::From(*d_x); auto d_out_tensor = EigenTensor::From(*d_out); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); d_x_tensor.device(place) = d_out_tensor.pad(paddings, 0); } } -template +template class PadGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -104,22 +106,22 @@ class PadGradKernel : public framework::OpKernel { context.Input(framework::GradVarName("Out"))->dims().size(); switch (rank) { case 1: - PadGradFunction(context); + PadGradFunction(context); break; case 2: - PadGradFunction(context); + PadGradFunction(context); break; case 3: - PadGradFunction(context); + PadGradFunction(context); break; case 4: - PadGradFunction(context); + PadGradFunction(context); break; case 5: - PadGradFunction(context); + PadGradFunction(context); break; case 6: - PadGradFunction(context); + PadGradFunction(context); break; default: PADDLE_THROW( diff --git a/paddle/operators/pool_cudnn_op.cc b/paddle/operators/pool_cudnn_op.cc index be9fcc5661..77407f5cdf 100644 --- a/paddle/operators/pool_cudnn_op.cc +++ b/paddle/operators/pool_cudnn_op.cc @@ -19,19 +19,21 @@ namespace ops = paddle::operators; REGISTER_OP(pool2d_cudnn, ops::PoolOp, ops::Pool2dOpMaker, pool2d_cudnn_grad, ops::PoolOpGrad); -REGISTER_OP_CPU_KERNEL(pool2d_cudnn, - ops::PoolKernel, - ops::PoolKernel); -REGISTER_OP_CPU_KERNEL(pool2d_cudnn_grad, - ops::PoolGradKernel, - ops::PoolGradKernel) +REGISTER_OP_CPU_KERNEL( + pool2d_cudnn, ops::PoolKernel, + ops::PoolKernel); +REGISTER_OP_CPU_KERNEL( + pool2d_cudnn_grad, + ops::PoolGradKernel, + ops::PoolGradKernel) REGISTER_OP(pool3d_cudnn, ops::PoolOp, ops::Pool3dOpMaker, pool3d_cudnn_grad, ops::PoolOpGrad); -REGISTER_OP_CPU_KERNEL(pool3d_cudnn, - ops::PoolKernel, - ops::PoolKernel); -REGISTER_OP_CPU_KERNEL(pool3d_cudnn_grad, - ops::PoolGradKernel, - ops::PoolGradKernel) +REGISTER_OP_CPU_KERNEL( + pool3d_cudnn, ops::PoolKernel, + ops::PoolKernel); +REGISTER_OP_CPU_KERNEL( + pool3d_cudnn_grad, + ops::PoolGradKernel, + ops::PoolGradKernel) diff --git a/paddle/operators/pool_cudnn_op.cu.cc b/paddle/operators/pool_cudnn_op.cu.cc index 66dd194ccd..fc2b37bd0f 100644 --- a/paddle/operators/pool_cudnn_op.cu.cc +++ b/paddle/operators/pool_cudnn_op.cu.cc @@ -162,12 +162,12 @@ class PoolCudnnGradOpKernel : public framework::OpKernel { namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(pool2d_cudnn, ops::PoolCudnnOpKernel, - ops::PoolCudnnOpKernel); -REGISTER_OP_GPU_KERNEL(pool2d_cudnn_grad, ops::PoolCudnnGradOpKernel, - ops::PoolCudnnGradOpKernel); - -REGISTER_OP_GPU_KERNEL(pool3d_cudnn, ops::PoolCudnnOpKernel, - ops::PoolCudnnOpKernel); -REGISTER_OP_GPU_KERNEL(pool3d_cudnn_grad, ops::PoolCudnnGradOpKernel, - ops::PoolCudnnGradOpKernel); +REGISTER_OP_CUDA_KERNEL(pool2d_cudnn, ops::PoolCudnnOpKernel, + ops::PoolCudnnOpKernel); +REGISTER_OP_CUDA_KERNEL(pool2d_cudnn_grad, ops::PoolCudnnGradOpKernel, + ops::PoolCudnnGradOpKernel); + +REGISTER_OP_CUDA_KERNEL(pool3d_cudnn, ops::PoolCudnnOpKernel, + ops::PoolCudnnOpKernel); +REGISTER_OP_CUDA_KERNEL(pool3d_cudnn_grad, ops::PoolCudnnGradOpKernel, + ops::PoolCudnnGradOpKernel); diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index e26ffd86e5..45fa20280c 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -216,19 +216,19 @@ namespace ops = paddle::operators; REGISTER_OP(pool2d, ops::PoolOp, ops::Pool2dOpMaker, pool2d_grad, ops::PoolOpGrad); -REGISTER_OP_CPU_KERNEL(pool2d, - ops::PoolKernel, - ops::PoolKernel); -REGISTER_OP_CPU_KERNEL(pool2d_grad, - ops::PoolGradKernel, - ops::PoolGradKernel) +REGISTER_OP_CPU_KERNEL( + pool2d, ops::PoolKernel, + ops::PoolKernel); +REGISTER_OP_CPU_KERNEL( + pool2d_grad, ops::PoolGradKernel, + ops::PoolGradKernel) REGISTER_OP(pool3d, ops::PoolOp, ops::Pool3dOpMaker, pool3d_grad, ops::PoolOpGrad); -REGISTER_OP_CPU_KERNEL(pool3d, - ops::PoolKernel, - ops::PoolKernel); -REGISTER_OP_CPU_KERNEL(pool3d_grad, - ops::PoolGradKernel, - ops::PoolGradKernel); +REGISTER_OP_CPU_KERNEL( + pool3d, ops::PoolKernel, + ops::PoolKernel); +REGISTER_OP_CPU_KERNEL( + pool3d_grad, ops::PoolGradKernel, + ops::PoolGradKernel); diff --git a/paddle/operators/pool_op.cu.cc b/paddle/operators/pool_op.cu.cc index 1010cb7622..39a9dfbf79 100644 --- a/paddle/operators/pool_op.cu.cc +++ b/paddle/operators/pool_op.cu.cc @@ -16,16 +16,18 @@ limitations under the License. */ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(pool2d, - ops::PoolKernel, - ops::PoolKernel); -REGISTER_OP_GPU_KERNEL(pool2d_grad, - ops::PoolGradKernel, - ops::PoolGradKernel); +REGISTER_OP_CUDA_KERNEL( + pool2d, ops::PoolKernel, + ops::PoolKernel); +REGISTER_OP_CUDA_KERNEL( + pool2d_grad, + ops::PoolGradKernel, + ops::PoolGradKernel); -REGISTER_OP_GPU_KERNEL(pool3d, - ops::PoolKernel, - ops::PoolKernel); -REGISTER_OP_GPU_KERNEL(pool3d_grad, - ops::PoolGradKernel, - ops::PoolGradKernel); +REGISTER_OP_CUDA_KERNEL( + pool3d, ops::PoolKernel, + ops::PoolKernel); +REGISTER_OP_CUDA_KERNEL( + pool3d_grad, + ops::PoolGradKernel, + ops::PoolGradKernel); diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index 63492a89e8..ab85d587a3 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -50,7 +50,7 @@ class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker* op_checker); }; -template +template class PoolKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -67,41 +67,41 @@ class PoolKernel : public framework::OpKernel { ksize[i] = static_cast(in_x->dims()[i + 2]); } } - + auto& dev_ctx = context.template device_context(); switch (ksize.size()) { case 2: { if (pooling_type == "max") { paddle::operators::math::Pool2dFunctor< - Place, paddle::operators::math::MaxPool, T> + DeviceContext, paddle::operators::math::MaxPool, T> pool2d_forward; paddle::operators::math::MaxPool pool_process; - pool2d_forward(context.device_context(), *in_x, ksize, strides, - paddings, pool_process, out); + pool2d_forward(dev_ctx, *in_x, ksize, strides, paddings, pool_process, + out); } else if (pooling_type == "avg") { paddle::operators::math::Pool2dFunctor< - Place, paddle::operators::math::AvgPool, T> + DeviceContext, paddle::operators::math::AvgPool, T> pool2d_forward; paddle::operators::math::AvgPool pool_process; - pool2d_forward(context.device_context(), *in_x, ksize, strides, - paddings, pool_process, out); + pool2d_forward(dev_ctx, *in_x, ksize, strides, paddings, pool_process, + out); } } break; case 3: { if (pooling_type == "max") { paddle::operators::math::Pool3dFunctor< - Place, paddle::operators::math::MaxPool, T> + DeviceContext, paddle::operators::math::MaxPool, T> pool3d_forward; paddle::operators::math::MaxPool pool_process; - pool3d_forward(context.device_context(), *in_x, ksize, strides, - paddings, pool_process, out); + pool3d_forward(dev_ctx, *in_x, ksize, strides, paddings, pool_process, + out); } else if (pooling_type == "avg") { paddle::operators::math::Pool3dFunctor< - Place, paddle::operators::math::AvgPool, T> + DeviceContext, paddle::operators::math::AvgPool, T> pool3d_forward; paddle::operators::math::AvgPool pool_process; - pool3d_forward(context.device_context(), *in_x, ksize, strides, - paddings, pool_process, out); + pool3d_forward(dev_ctx, *in_x, ksize, strides, paddings, pool_process, + out); } } break; default: { PADDLE_THROW("Pool op only supports 2D and 3D input."); } @@ -109,7 +109,7 @@ class PoolKernel : public framework::OpKernel { } }; -template +template class PoolGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -130,42 +130,43 @@ class PoolGradKernel : public framework::OpKernel { ksize[i] = static_cast(in_x->dims()[i + 2]); } } - + auto& dev_ctx = context.template device_context(); if (in_x_grad) { in_x_grad->mutable_data(context.GetPlace()); auto temp = framework::EigenVector::Flatten(*in_x_grad); - temp.device(context.GetEigenDevice()) = + temp.device( + *context.template device_context().eigen_device()) = temp.constant(static_cast(0)); switch (ksize.size()) { case 2: { if (pooling_type == "max") { - paddle::operators::math::MaxPool2dGradFunctor + paddle::operators::math::MaxPool2dGradFunctor pool2d_backward; - pool2d_backward(context.device_context(), *in_x, *out, *out_grad, - ksize, strides, paddings, in_x_grad); + pool2d_backward(dev_ctx, *in_x, *out, *out_grad, ksize, strides, + paddings, in_x_grad); } else if (pooling_type == "avg") { paddle::operators::math::Pool2dGradFunctor< - Place, paddle::operators::math::AvgPoolGrad, T> + DeviceContext, paddle::operators::math::AvgPoolGrad, T> pool2d_backward; paddle::operators::math::AvgPoolGrad pool_process; - pool2d_backward(context.device_context(), *in_x, *out, *out_grad, - ksize, strides, paddings, pool_process, in_x_grad); + pool2d_backward(dev_ctx, *in_x, *out, *out_grad, ksize, strides, + paddings, pool_process, in_x_grad); } } break; case 3: { if (pooling_type == "max") { - paddle::operators::math::MaxPool3dGradFunctor + paddle::operators::math::MaxPool3dGradFunctor pool3d_backward; - pool3d_backward(context.device_context(), *in_x, *out, *out_grad, - ksize, strides, paddings, in_x_grad); + pool3d_backward(dev_ctx, *in_x, *out, *out_grad, ksize, strides, + paddings, in_x_grad); } else if (pooling_type == "avg") { paddle::operators::math::Pool3dGradFunctor< - Place, paddle::operators::math::AvgPoolGrad, T> + DeviceContext, paddle::operators::math::AvgPoolGrad, T> pool3d_backward; paddle::operators::math::AvgPoolGrad pool_process; - pool3d_backward(context.device_context(), *in_x, *out, *out_grad, - ksize, strides, paddings, pool_process, in_x_grad); + pool3d_backward(dev_ctx, *in_x, *out, *out_grad, ksize, strides, + paddings, pool_process, in_x_grad); } } break; default: { PADDLE_THROW("Pool op only supports 2D and 3D input."); } diff --git a/paddle/operators/pool_with_index_op.cc b/paddle/operators/pool_with_index_op.cc index b9c42a6912..1a2383f8b8 100644 --- a/paddle/operators/pool_with_index_op.cc +++ b/paddle/operators/pool_with_index_op.cc @@ -266,12 +266,15 @@ REGISTER_OP(max_pool2d_with_index, ops::MaxPoolWithIndexOp, REGISTER_OP_CPU_KERNEL( max_pool2d_with_index, - ops::MaxPoolWithIndexKernel, - ops::MaxPoolWithIndexKernel); + ops::MaxPoolWithIndexKernel, + ops::MaxPoolWithIndexKernel); REGISTER_OP_CPU_KERNEL( max_pool2d_with_index_grad, - ops::MaxPoolWithIndexGradKernel, - ops::MaxPoolWithIndexGradKernel) + ops::MaxPoolWithIndexGradKernel, + ops::MaxPoolWithIndexGradKernel) REGISTER_OP(max_pool3d_with_index, ops::MaxPoolWithIndexOp, ops::MaxPool3dWithIndexOpMaker, max_pool3d_with_index_grad, @@ -279,9 +282,12 @@ REGISTER_OP(max_pool3d_with_index, ops::MaxPoolWithIndexOp, REGISTER_OP_CPU_KERNEL( max_pool3d_with_index, - ops::MaxPoolWithIndexKernel, - ops::MaxPoolWithIndexKernel); + ops::MaxPoolWithIndexKernel, + ops::MaxPoolWithIndexKernel); REGISTER_OP_CPU_KERNEL( max_pool3d_with_index_grad, - ops::MaxPoolWithIndexGradKernel, - ops::MaxPoolWithIndexGradKernel) + ops::MaxPoolWithIndexGradKernel, + ops::MaxPoolWithIndexGradKernel) diff --git a/paddle/operators/pool_with_index_op.cu.cc b/paddle/operators/pool_with_index_op.cu.cc index 335064a7ee..4c9804da63 100644 --- a/paddle/operators/pool_with_index_op.cu.cc +++ b/paddle/operators/pool_with_index_op.cu.cc @@ -16,20 +16,28 @@ limitations under the License. */ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( max_pool2d_with_index, - ops::MaxPoolWithIndexKernel, - ops::MaxPoolWithIndexKernel); -REGISTER_OP_GPU_KERNEL( + ops::MaxPoolWithIndexKernel, + ops::MaxPoolWithIndexKernel); +REGISTER_OP_CUDA_KERNEL( max_pool2d_with_index_grad, - ops::MaxPoolWithIndexGradKernel, - ops::MaxPoolWithIndexGradKernel) + ops::MaxPoolWithIndexGradKernel, + ops::MaxPoolWithIndexGradKernel) -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( max_pool3d_with_index, - ops::MaxPoolWithIndexKernel, - ops::MaxPoolWithIndexKernel); -REGISTER_OP_GPU_KERNEL( + ops::MaxPoolWithIndexKernel, + ops::MaxPoolWithIndexKernel); +REGISTER_OP_CUDA_KERNEL( max_pool3d_with_index_grad, - ops::MaxPoolWithIndexGradKernel, - ops::MaxPoolWithIndexGradKernel) + ops::MaxPoolWithIndexGradKernel, + ops::MaxPoolWithIndexGradKernel) diff --git a/paddle/operators/pool_with_index_op.h b/paddle/operators/pool_with_index_op.h index 40766c7e82..4f4087d1dd 100644 --- a/paddle/operators/pool_with_index_op.h +++ b/paddle/operators/pool_with_index_op.h @@ -24,7 +24,7 @@ namespace operators { using Tensor = framework::Tensor; -template +template class MaxPoolWithIndexKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -35,6 +35,8 @@ class MaxPoolWithIndexKernel : public framework::OpKernel { std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); + + auto& dev_ctx = context.template device_context(); if (context.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; @@ -44,23 +46,23 @@ class MaxPoolWithIndexKernel : public framework::OpKernel { switch (ksize.size()) { case 2: { - paddle::operators::math::MaxPool2dWithIndexFunctor + paddle::operators::math::MaxPool2dWithIndexFunctor pool2d_forward; - pool2d_forward(context.device_context(), *in_x, ksize, strides, - paddings, out, mask); + pool2d_forward(dev_ctx, *in_x, ksize, strides, paddings, out, mask); } break; case 3: { - paddle::operators::math::MaxPool3dWithIndexFunctor + paddle::operators::math::MaxPool3dWithIndexFunctor pool3d_forward; - pool3d_forward(context.device_context(), *in_x, ksize, strides, - paddings, out, mask); + pool3d_forward(dev_ctx, *in_x, ksize, strides, paddings, out, mask); } break; default: { PADDLE_THROW("Pool op only supports 2D and 3D input."); } } } }; -template +template class MaxPoolWithIndexGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -81,18 +83,20 @@ class MaxPoolWithIndexGradKernel : public framework::OpKernel { if (in_x_grad) { in_x_grad->mutable_data(context.GetPlace()); - auto& device_ctx = context.device_context(); + auto& device_ctx = context.template device_context(); math::set_constant(device_ctx, in_x_grad, 0); switch (ksize.size()) { case 2: { - paddle::operators::math::MaxPool2dWithIndexGradFunctor + paddle::operators::math::MaxPool2dWithIndexGradFunctor pool2d_backward; pool2d_backward(device_ctx, *out_grad, *mask, ksize, strides, paddings, in_x_grad); } break; case 3: { - paddle::operators::math::MaxPool3dWithIndexGradFunctor + paddle::operators::math::MaxPool3dWithIndexGradFunctor pool3d_backward; pool3d_backward(device_ctx, *out_grad, *mask, ksize, strides, paddings, in_x_grad); diff --git a/paddle/operators/positive_negative_pair_op.h b/paddle/operators/positive_negative_pair_op.h index 2efd3777e0..977e59b7d2 100644 --- a/paddle/operators/positive_negative_pair_op.h +++ b/paddle/operators/positive_negative_pair_op.h @@ -22,7 +22,7 @@ namespace operators { using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; -template +template class PositiveNegativePairKernel : public framework::OpKernel { public: struct PredictionResult { diff --git a/paddle/operators/precision_recall_op.h b/paddle/operators/precision_recall_op.h index 4a871ce674..c0d55405a3 100644 --- a/paddle/operators/precision_recall_op.h +++ b/paddle/operators/precision_recall_op.h @@ -26,7 +26,7 @@ using EigenMatrix = framework::EigenMatrix; enum StateVariable { TP = 0, FP, TN, FN }; -template +template class PrecisionRecallKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { diff --git a/paddle/operators/prelu_op.cc b/paddle/operators/prelu_op.cc index 055c471b45..317a2a4015 100644 --- a/paddle/operators/prelu_op.cc +++ b/paddle/operators/prelu_op.cc @@ -85,7 +85,8 @@ namespace ops = paddle::operators; REGISTER_OP(prelu, ops::PReluOp, ops::PReluOpMaker, prelu_grad, ops::PReluGradOp); -REGISTER_OP_CPU_KERNEL(prelu, - ops::PReluKernel); -REGISTER_OP_CPU_KERNEL(prelu_grad, - ops::PReluGradKernel); +REGISTER_OP_CPU_KERNEL( + prelu, ops::PReluKernel); +REGISTER_OP_CPU_KERNEL( + prelu_grad, + ops::PReluGradKernel); diff --git a/paddle/operators/prelu_op.cu b/paddle/operators/prelu_op.cu index 9e391dabae..12033dee0e 100644 --- a/paddle/operators/prelu_op.cu +++ b/paddle/operators/prelu_op.cu @@ -14,8 +14,9 @@ #include "paddle/operators/prelu_op.h" -REGISTER_OP_GPU_KERNEL( - prelu, paddle::operators::PReluKernel); -REGISTER_OP_GPU_KERNEL( - prelu_grad, - paddle::operators::PReluGradKernel); +REGISTER_OP_CUDA_KERNEL( + prelu, + paddle::operators::PReluKernel); +REGISTER_OP_CUDA_KERNEL(prelu_grad, + paddle::operators::PReluGradKernel< + paddle::platform::CUDADeviceContext, float>); diff --git a/paddle/operators/prelu_op.h b/paddle/operators/prelu_op.h index 5ad31c2203..56f9a553ec 100644 --- a/paddle/operators/prelu_op.h +++ b/paddle/operators/prelu_op.h @@ -39,7 +39,7 @@ class PReluFunctor { const T* alpha_; }; -template +template class PReluKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -54,9 +54,9 @@ class PReluKernel : public framework::OpKernel { int numel = x->numel(); - Transform trans; - trans(context.device_context(), x_ptr, x_ptr + numel, o_ptr, - PReluFunctor(alpha_ptr)); + Transform trans; + trans(context.template device_context(), x_ptr, + x_ptr + numel, o_ptr, PReluFunctor(alpha_ptr)); } }; @@ -76,7 +76,7 @@ class PReluGradFunctor { const T* alpha_; }; -template +template class PReluGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -92,9 +92,9 @@ class PReluGradKernel : public framework::OpKernel { const T* out_ptr = out->data(); int numel = dx->numel(); - Transform trans; - trans(context.device_context(), out_ptr, out_ptr + numel, dout_ptr, dx_ptr, - PReluGradFunctor(alpha_ptr)); + Transform trans; + trans(context.template device_context(), out_ptr, + out_ptr + numel, dout_ptr, dx_ptr, PReluGradFunctor(alpha_ptr)); // TODO(Zhuoyuan): add dalpha upgrade when GPU kernels ready } diff --git a/paddle/operators/proximal_adagrad_op.cc b/paddle/operators/proximal_adagrad_op.cc index 36e460103a..cc350f6d26 100644 --- a/paddle/operators/proximal_adagrad_op.cc +++ b/paddle/operators/proximal_adagrad_op.cc @@ -114,4 +114,4 @@ REGISTER_OP_WITHOUT_GRADIENT(proximal_adagrad, ops::ProximalAdagradOp, ops::ProximalAdagradOpMaker); REGISTER_OP_CPU_KERNEL( proximal_adagrad, - ops::ProximalAdagradOpKernel); + ops::ProximalAdagradOpKernel); diff --git a/paddle/operators/proximal_adagrad_op.cu b/paddle/operators/proximal_adagrad_op.cu index d0ae039518..42a178f94b 100644 --- a/paddle/operators/proximal_adagrad_op.cu +++ b/paddle/operators/proximal_adagrad_op.cu @@ -15,6 +15,6 @@ specific language governing permissions and limitations under the License. */ #include "paddle/operators/proximal_adagrad_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( proximal_adagrad, - ops::ProximalAdagradOpKernel); + ops::ProximalAdagradOpKernel); diff --git a/paddle/operators/proximal_adagrad_op.h b/paddle/operators/proximal_adagrad_op.h index 7a1560e8cb..523924d80e 100644 --- a/paddle/operators/proximal_adagrad_op.h +++ b/paddle/operators/proximal_adagrad_op.h @@ -24,7 +24,7 @@ template using EigenVector = framework::EigenVector; -template +template class ProximalAdagradOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -45,20 +45,20 @@ class ProximalAdagradOpKernel : public framework::OpKernel { auto p_out = EigenVector::Flatten(*param_out); auto m_out = EigenVector::Flatten(*moment_out); - auto place = ctx.GetEigenDevice(); + auto* place = ctx.template device_context().eigen_device(); Eigen::DSizes grad_dsize(grad->numel()); - m_out.device(place) = m + g * g; + m_out.device(*place) = m + g * g; auto prox_param = p - lr.broadcast(grad_dsize) * g / m_out.sqrt(); if (l1 > static_cast(0)) { - p_out.device(place) = + p_out.device(*place) = prox_param.sign() * (((prox_param.abs() - (lr * l1).broadcast(grad_dsize)) .cwiseMax(static_cast(0.0))) / (static_cast(1.0) + (lr * l2).broadcast(grad_dsize))); } else { - p_out.device(place) = + p_out.device(*place) = prox_param / (static_cast(1.0) + (lr * l2).broadcast(grad_dsize)); } } diff --git a/paddle/operators/proximal_gd_op.cc b/paddle/operators/proximal_gd_op.cc index 5693d0ec9e..0b26beb3ac 100644 --- a/paddle/operators/proximal_gd_op.cc +++ b/paddle/operators/proximal_gd_op.cc @@ -94,4 +94,5 @@ namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(proximal_gd, ops::ProximalGDOp, ops::ProximalGDOpMaker); REGISTER_OP_CPU_KERNEL( - proximal_gd, ops::ProximalGDOpKernel); + proximal_gd, + ops::ProximalGDOpKernel); diff --git a/paddle/operators/proximal_gd_op.cu b/paddle/operators/proximal_gd_op.cu index 26f4ebaa0f..b7dd840d19 100644 --- a/paddle/operators/proximal_gd_op.cu +++ b/paddle/operators/proximal_gd_op.cu @@ -15,5 +15,6 @@ specific language governing permissions and limitations under the License. */ #include "paddle/operators/proximal_gd_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( - proximal_gd, ops::ProximalGDOpKernel); +REGISTER_OP_CUDA_KERNEL( + proximal_gd, + ops::ProximalGDOpKernel); diff --git a/paddle/operators/proximal_gd_op.h b/paddle/operators/proximal_gd_op.h index bebda02041..64648b3cca 100644 --- a/paddle/operators/proximal_gd_op.h +++ b/paddle/operators/proximal_gd_op.h @@ -24,7 +24,7 @@ template using EigenVector = framework::EigenVector; -template +template class ProximalGDOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -42,7 +42,7 @@ class ProximalGDOpKernel : public framework::OpKernel { auto lr = EigenVector::Flatten(*ctx.Input("LearningRate")); auto p_out = EigenVector::Flatten(*param_out); - auto place = ctx.GetEigenDevice(); + auto& place = *ctx.template device_context().eigen_device(); Eigen::DSizes grad_dsize(grad->numel()); diff --git a/paddle/operators/rank_loss_op.cc b/paddle/operators/rank_loss_op.cc index 912f88f455..b80b175792 100644 --- a/paddle/operators/rank_loss_op.cc +++ b/paddle/operators/rank_loss_op.cc @@ -123,7 +123,8 @@ namespace ops = paddle::operators; REGISTER_OP(rank_loss, ops::RankLossOp, ops::RankLossOpMaker, rank_loss_grad, ops::RankLossGradOp); -REGISTER_OP_CPU_KERNEL(rank_loss, - ops::RankLossKernel); REGISTER_OP_CPU_KERNEL( - rank_loss_grad, ops::RankLossGradKernel); + rank_loss, ops::RankLossKernel); +REGISTER_OP_CPU_KERNEL( + rank_loss_grad, + ops::RankLossGradKernel); diff --git a/paddle/operators/rank_loss_op.cu b/paddle/operators/rank_loss_op.cu index 5382e3a629..5aee66443d 100644 --- a/paddle/operators/rank_loss_op.cu +++ b/paddle/operators/rank_loss_op.cu @@ -14,9 +14,9 @@ #include "paddle/operators/rank_loss_op.h" -REGISTER_OP_GPU_KERNEL( - rank_loss, - paddle::operators::RankLossKernel); -REGISTER_OP_GPU_KERNEL( - rank_loss_grad, - paddle::operators::RankLossGradKernel); +REGISTER_OP_CUDA_KERNEL(rank_loss, + paddle::operators::RankLossKernel< + paddle::platform::CUDADeviceContext, float>); +REGISTER_OP_CUDA_KERNEL(rank_loss_grad, + paddle::operators::RankLossGradKernel< + paddle::platform::CUDADeviceContext, float>); diff --git a/paddle/operators/rank_loss_op.h b/paddle/operators/rank_loss_op.h index 703c77a0b2..ea24b61fd9 100644 --- a/paddle/operators/rank_loss_op.h +++ b/paddle/operators/rank_loss_op.h @@ -20,7 +20,7 @@ namespace paddle { namespace operators { -template +template class RankLossKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { @@ -35,13 +35,13 @@ class RankLossKernel : public framework::OpKernel { auto left = framework::EigenVector::Flatten(*left_t); auto right = framework::EigenVector::Flatten(*right_t); - auto& dev = ctx.GetEigenDevice(); + auto& dev = *ctx.template device_context().eigen_device(); out.device(dev) = (1. + (left - right).exp()).log() - label * (left - right); } }; -template +template class RankLossGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { @@ -55,7 +55,7 @@ class RankLossGradKernel : public framework::OpKernel { auto* left_t = ctx.Input("Left"); auto* right_t = ctx.Input("Right"); - auto& dev = ctx.GetEigenDevice(); + auto& dev = *ctx.template device_context().eigen_device(); auto d_out = framework::EigenVector::Flatten(*d_out_t); auto label = framework::EigenVector::Flatten(*label_t); auto left = framework::EigenVector::Flatten(*left_t); diff --git a/paddle/operators/reduce_op.cc b/paddle/operators/reduce_op.cc index 2589a54cfc..b754637bf2 100644 --- a/paddle/operators/reduce_op.cc +++ b/paddle/operators/reduce_op.cc @@ -180,12 +180,13 @@ REGISTER_OP(reduce_max, ops::ReduceOp, ops::ReduceMaxOpMaker, reduce_max_grad, REGISTER_OP(reduce_min, ops::ReduceOp, ops::ReduceMinOpMaker, reduce_min_grad, ops::ReduceGradOp); -#define REGISTER_REDUCE_CPU_KERNEL(reduce_type, functor, grad_functor) \ - REGISTER_OP_CPU_KERNEL( \ - reduce_type, \ - ops::ReduceKernel); \ - REGISTER_OP_CPU_KERNEL(reduce_type##_grad, \ - ops::ReduceGradKernel); +#define REGISTER_REDUCE_CPU_KERNEL(reduce_type, functor, grad_functor) \ + REGISTER_OP_CPU_KERNEL(reduce_type, \ + ops::ReduceKernel); \ + REGISTER_OP_CPU_KERNEL( \ + reduce_type##_grad, \ + ops::ReduceGradKernel); FOR_EACH_KERNEL_FUNCTOR(REGISTER_REDUCE_CPU_KERNEL); diff --git a/paddle/operators/reduce_op.cu b/paddle/operators/reduce_op.cu index d306e1a240..a10ace5253 100644 --- a/paddle/operators/reduce_op.cu +++ b/paddle/operators/reduce_op.cu @@ -17,12 +17,13 @@ namespace ops = paddle::operators; -#define REGISTER_REDUCE_GPU_KERNEL(reduce_type, functor, grad_functor) \ - REGISTER_OP_GPU_KERNEL( \ - reduce_type, \ - ops::ReduceKernel); \ - REGISTER_OP_GPU_KERNEL(reduce_type##_grad, \ - ops::ReduceGradKernel); +#define REGISTER_REDUCE_GPU_KERNEL(reduce_type, functor, grad_functor) \ + REGISTER_OP_CUDA_KERNEL( \ + reduce_type, ops::ReduceKernel); \ + REGISTER_OP_CUDA_KERNEL( \ + reduce_type##_grad, \ + ops::ReduceGradKernel); FOR_EACH_KERNEL_FUNCTOR(REGISTER_REDUCE_GPU_KERNEL); diff --git a/paddle/operators/reduce_op.h b/paddle/operators/reduce_op.h index dd6547542d..47ce910f28 100644 --- a/paddle/operators/reduce_op.h +++ b/paddle/operators/reduce_op.h @@ -32,55 +32,55 @@ template ; struct SumFunctor { - template - void operator()(const Place& place, X& x, Y& y, const Dim& dim) { + template + void operator()(const DeviceContext& place, X& x, Y& y, const Dim& dim) { y.device(place) = x.sum(dim); } }; struct SumGradFunctor { - template - void operator()(const Place& place, X& x, Y& y, DX& dx, DY& dy, + template + void operator()(const DeviceContext& place, X& x, Y& y, DX& dx, DY& dy, const Dim& dim, int size) { dx.device(place) = dy.broadcast(dim); } }; struct MeanFunctor { - template - void operator()(const Place& place, X& x, Y& y, const Dim& dim) { + template + void operator()(const DeviceContext& place, X& x, Y& y, const Dim& dim) { y.device(place) = x.mean(dim); } }; struct MeanGradFunctor { - template - void operator()(const Place& place, X& x, Y& y, DX& dx, DY& dy, + template + void operator()(const DeviceContext& place, X& x, Y& y, DX& dx, DY& dy, const Dim& dim, int size) { dx.device(place) = dy.broadcast(dim) / dx.constant(size); } }; struct MaxFunctor { - template - void operator()(const Place& place, X& x, Y& y, const Dim& dim) { + template + void operator()(const DeviceContext& place, X& x, Y& y, const Dim& dim) { y.device(place) = x.maximum(dim); } }; struct MinFunctor { - template - void operator()(const Place& place, X& x, Y& y, const Dim& dim) { + template + void operator()(const DeviceContext& place, X& x, Y& y, const Dim& dim) { y.device(place) = x.minimum(dim); } }; struct MaxOrMinGradFunctor { - template - void operator()(const Place& place, X& x, Y& y, DX& dx, DY& dy, + template + void operator()(const DeviceContext& place, X& x, Y& y, DX& dx, DY& dy, const Dim& dim, int size) { auto equals = x == y.broadcast(dim); auto ones = dx.constant(1); @@ -91,7 +91,7 @@ struct MaxOrMinGradFunctor { } }; -template +template class ReduceKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -139,7 +139,8 @@ class ReduceKernel : public framework::OpKernel { dims = framework::make_ddim(dims_vector); } - auto& place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); Functor functor; if (D == 1) { @@ -152,7 +153,7 @@ class ReduceKernel : public framework::OpKernel { } }; -template +template class ReduceGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -201,7 +202,8 @@ class ReduceGradKernel : public framework::OpKernel { Eigen::array broadcast_dim; for (size_t i = 0; i < D; ++i) broadcast_dim[i] = 1; broadcast_dim[dim] = input0->dims()[dim]; - auto& place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); Functor functor; functor(place, x, x_reduce, x_grad, x_reduce_grad, broadcast_dim, broadcast_dim[dim]); diff --git a/paddle/operators/reshape_op.cu b/paddle/operators/reshape_op.cu index dca6c15007..b7329238c0 100644 --- a/paddle/operators/reshape_op.cu +++ b/paddle/operators/reshape_op.cu @@ -14,9 +14,9 @@ #include "paddle/operators/reshape_op.h" -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( reshape, paddle::operators::ReshapeKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( reshape_grad, paddle::operators::ReshapeGradKernel); diff --git a/paddle/operators/reshape_op.h b/paddle/operators/reshape_op.h index 73fd1da642..92d8cbbb56 100644 --- a/paddle/operators/reshape_op.h +++ b/paddle/operators/reshape_op.h @@ -20,7 +20,7 @@ namespace paddle { namespace operators { -template +template class ReshapeKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { @@ -33,7 +33,7 @@ class ReshapeKernel : public framework::OpKernel { } }; -template +template class ReshapeGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { diff --git a/paddle/operators/rmsprop_op.cc b/paddle/operators/rmsprop_op.cc index a9c45f639c..fc3f9b8988 100644 --- a/paddle/operators/rmsprop_op.cc +++ b/paddle/operators/rmsprop_op.cc @@ -116,5 +116,5 @@ http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf) namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(rmsprop, ops::RmspropOp, ops::RmspropOpMaker); -REGISTER_OP_CPU_KERNEL(rmsprop, - ops::RmspropOpKernel); +REGISTER_OP_CPU_KERNEL( + rmsprop, ops::RmspropOpKernel); diff --git a/paddle/operators/rmsprop_op.cu b/paddle/operators/rmsprop_op.cu index 52634a5481..2a9fd6e104 100644 --- a/paddle/operators/rmsprop_op.cu +++ b/paddle/operators/rmsprop_op.cu @@ -16,5 +16,5 @@ #include "paddle/operators/rmsprop_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(rmsprop, - ops::RmspropOpKernel); +REGISTER_OP_CUDA_KERNEL( + rmsprop, ops::RmspropOpKernel); diff --git a/paddle/operators/rmsprop_op.h b/paddle/operators/rmsprop_op.h index 7bf2129010..16a561835d 100644 --- a/paddle/operators/rmsprop_op.h +++ b/paddle/operators/rmsprop_op.h @@ -24,7 +24,7 @@ template using EigenVector = framework::EigenVector; -template +template class RmspropOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -51,7 +51,7 @@ class RmspropOpKernel : public framework::OpKernel { auto p_out = EigenVector::Flatten(*param_out); auto mom_out = EigenVector::Flatten(*moment_out); auto ms_out = EigenVector::Flatten(*mean_square_out); - auto place = ctx.GetEigenDevice(); + auto& place = *ctx.template device_context().eigen_device(); Eigen::DSizes grad_dsize(grad->numel()); diff --git a/paddle/operators/roi_pool_op.cc b/paddle/operators/roi_pool_op.cc index 2b5e66c96b..75fcea8401 100644 --- a/paddle/operators/roi_pool_op.cc +++ b/paddle/operators/roi_pool_op.cc @@ -157,9 +157,10 @@ namespace ops = paddle::operators; REGISTER_OP(roi_pool, ops::ROIPoolOp, ops::ROIPoolOpMaker, roi_pool_grad, ops::ROIPoolGradOp); REGISTER_OP_CPU_KERNEL( - roi_pool, ops::CPUROIPoolOpKernel, - ops::CPUROIPoolOpKernel); + roi_pool, + ops::CPUROIPoolOpKernel, + ops::CPUROIPoolOpKernel); REGISTER_OP_CPU_KERNEL( roi_pool_grad, - ops::CPUROIPoolGradOpKernel, - ops::CPUROIPoolOpKernel); + ops::CPUROIPoolGradOpKernel, + ops::CPUROIPoolOpKernel); diff --git a/paddle/operators/roi_pool_op.cu b/paddle/operators/roi_pool_op.cu index 9a4c8ca752..a874befe4d 100644 --- a/paddle/operators/roi_pool_op.cu +++ b/paddle/operators/roi_pool_op.cu @@ -177,7 +177,7 @@ class GPUROIPoolGradOpKernel : public framework::OpKernel { if (x_grad) { x_grad->mutable_data(ctx.GetPlace()); math::SetConstant set_zero; - set_zero(ctx.device_context(), x_grad, static_cast(0)); + set_zero(ctx.cuda_device_context(), x_grad, static_cast(0)); int output_grad_size = out_grad->numel(); int blocks = NumBlocks(output_grad_size); @@ -199,10 +199,11 @@ class GPUROIPoolGradOpKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( - roi_pool, ops::GPUROIPoolOpKernel, - ops::GPUROIPoolOpKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( + roi_pool, + ops::GPUROIPoolOpKernel, + ops::GPUROIPoolOpKernel); +REGISTER_OP_CUDA_KERNEL( roi_pool_grad, - ops::GPUROIPoolGradOpKernel, - ops::GPUROIPoolOpKernel); + ops::GPUROIPoolGradOpKernel, + ops::GPUROIPoolOpKernel); diff --git a/paddle/operators/roi_pool_op.h b/paddle/operators/roi_pool_op.h index 3812c66c65..09a9d3d870 100644 --- a/paddle/operators/roi_pool_op.h +++ b/paddle/operators/roi_pool_op.h @@ -19,7 +19,7 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class CPUROIPoolOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -126,7 +126,7 @@ class CPUROIPoolOpKernel : public framework::OpKernel { } }; -template +template class CPUROIPoolGradOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -145,8 +145,9 @@ class CPUROIPoolGradOpKernel : public framework::OpKernel { const T* out_grad_data = out_grad->data(); const int64_t* argmax_data = argmax->data(); T* in_grad_data = in_grad->mutable_data(ctx.GetPlace()); - math::SetConstant set_zero; - set_zero(ctx.device_context(), in_grad, static_cast(0)); + math::SetConstant set_zero; + set_zero(ctx.template device_context(), in_grad, + static_cast(0)); auto in_stride = framework::stride(in->dims()); auto argmax_stride = framework::stride(argmax->dims()); diff --git a/paddle/operators/row_conv_op.cc b/paddle/operators/row_conv_op.cc index ea0bb99f8d..5203a5079c 100644 --- a/paddle/operators/row_conv_op.cc +++ b/paddle/operators/row_conv_op.cc @@ -124,7 +124,8 @@ $$ }; template -class RowConvKernel : public framework::OpKernel { +class RowConvKernel + : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { auto *x = context.Input("X"); @@ -169,7 +170,8 @@ class RowConvKernel : public framework::OpKernel { }; template -class RowConvGradKernel : public framework::OpKernel { +class RowConvGradKernel + : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { auto *x = context.Input("X"); @@ -251,7 +253,8 @@ class RowConvGradKernel : public framework::OpKernel { namespace ops = paddle::operators; REGISTER_OP(row_conv, ops::RowConvOp, ops::RowConvOpMaker, row_conv_grad, ops::RowConvGradOp); -REGISTER_OP_CPU_KERNEL(row_conv, - ops::RowConvKernel); REGISTER_OP_CPU_KERNEL( - row_conv_grad, ops::RowConvGradKernel); + row_conv, ops::RowConvKernel); +REGISTER_OP_CPU_KERNEL( + row_conv_grad, + ops::RowConvGradKernel); diff --git a/paddle/operators/row_conv_op.cu b/paddle/operators/row_conv_op.cu index e0d7ebda7e..3fc5eabcf5 100644 --- a/paddle/operators/row_conv_op.cu +++ b/paddle/operators/row_conv_op.cu @@ -292,7 +292,8 @@ __global__ void RowConvGradFilter(const T *in, const T *dout, int num_sequence, } // namespace template -class RowConvKernel : public framework::OpKernel { +class RowConvKernel + : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { auto *X = context.Input("X"); @@ -327,7 +328,8 @@ class RowConvKernel : public framework::OpKernel { }; template -class RowConvGradKernel : public framework::OpKernel { +class RowConvGradKernel + : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { auto *X = context.Input("X"); @@ -347,7 +349,7 @@ class RowConvGradKernel : public framework::OpKernel { size_t *idx = batch_indices.data(); auto &device_ctx = context.cuda_device_context(); - math::SetConstant zero; + math::SetConstant zero; if (dFilter) { T *dfilter = dFilter->mutable_data(context.GetPlace()); @@ -402,7 +404,8 @@ class RowConvGradKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(row_conv, - ops::RowConvKernel); -REGISTER_OP_GPU_KERNEL( - row_conv_grad, ops::RowConvGradKernel); +REGISTER_OP_CUDA_KERNEL( + row_conv, ops::RowConvKernel); +REGISTER_OP_CUDA_KERNEL( + row_conv_grad, + ops::RowConvGradKernel); diff --git a/paddle/operators/row_conv_op.h b/paddle/operators/row_conv_op.h index 525e83908d..80912ad8f7 100644 --- a/paddle/operators/row_conv_op.h +++ b/paddle/operators/row_conv_op.h @@ -18,13 +18,13 @@ namespace paddle { namespace operators { -template +template class RowConvKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override; }; -template +template class RowConvGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override; diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc index e5c10fec4d..d848be823e 100644 --- a/paddle/operators/scale_op.cc +++ b/paddle/operators/scale_op.cc @@ -75,8 +75,8 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(scale, ops::ScaleOp, ops::ScaleOpMaker, ops::ScaleGradMaker); -REGISTER_OP_CPU_KERNEL(scale, - ops::ScaleKernel, - ops::ScaleKernel, - ops::ScaleKernel, - ops::ScaleKernel); +REGISTER_OP_CPU_KERNEL( + scale, ops::ScaleKernel, + ops::ScaleKernel, + ops::ScaleKernel, + ops::ScaleKernel); diff --git a/paddle/operators/scale_op.cu b/paddle/operators/scale_op.cu index 0d70775159..0c7980430f 100644 --- a/paddle/operators/scale_op.cu +++ b/paddle/operators/scale_op.cu @@ -14,8 +14,10 @@ #include "paddle/operators/scale_op.h" -REGISTER_OP_GPU_KERNEL( - scale, paddle::operators::ScaleKernel, - paddle::operators::ScaleKernel, - paddle::operators::ScaleKernel, - paddle::operators::ScaleKernel); +REGISTER_OP_CUDA_KERNEL( + scale, + paddle::operators::ScaleKernel, + paddle::operators::ScaleKernel, + paddle::operators::ScaleKernel, + paddle::operators::ScaleKernel); diff --git a/paddle/operators/scale_op.h b/paddle/operators/scale_op.h index 4931294c9d..02a8c97a83 100644 --- a/paddle/operators/scale_op.h +++ b/paddle/operators/scale_op.h @@ -19,7 +19,7 @@ namespace paddle { namespace operators { -template +template class ScaleKernel : public framework::OpKernel { public: virtual void Compute(const framework::ExecutionContext& context) const { @@ -31,7 +31,8 @@ class ScaleKernel : public framework::OpKernel { auto eigen_out = framework::EigenVector::Flatten(*tensor); auto eigen_in = framework::EigenVector::Flatten(*in); - auto& dev = context.GetEigenDevice(); + auto& dev = + *context.template device_context().eigen_device(); eigen_out.device(dev) = scale * eigen_in; } }; diff --git a/paddle/operators/scatter_op.cu b/paddle/operators/scatter_op.cu index 3b32ae2fb7..6b43a1389f 100644 --- a/paddle/operators/scatter_op.cu +++ b/paddle/operators/scatter_op.cu @@ -59,5 +59,5 @@ class ScatterGradOpCUDAKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(scatter, ops::ScatterOpCUDAKernel); -REGISTER_OP_GPU_KERNEL(scatter_grad, ops::ScatterGradOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(scatter, ops::ScatterOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(scatter_grad, ops::ScatterGradOpCUDAKernel); diff --git a/paddle/operators/seq_expand_op.cc b/paddle/operators/seq_expand_op.cc index b862056ad4..ede9754697 100644 --- a/paddle/operators/seq_expand_op.cc +++ b/paddle/operators/seq_expand_op.cc @@ -148,8 +148,9 @@ class SeqExpandOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(seq_expand, ops::SeqExpandOp, ops::SeqExpandOpMaker, seq_expand_grad, ops::SeqExpandOpGrad); -REGISTER_OP_CPU_KERNEL(seq_expand, - ops::SeqExpandKernel); +REGISTER_OP_CPU_KERNEL( + seq_expand, + ops::SeqExpandKernel); REGISTER_OP_CPU_KERNEL( seq_expand_grad, - ops::SeqExpandGradKernel); + ops::SeqExpandGradKernel); diff --git a/paddle/operators/seq_expand_op.cu b/paddle/operators/seq_expand_op.cu index f1e4b82a76..8e67ce9ccb 100644 --- a/paddle/operators/seq_expand_op.cu +++ b/paddle/operators/seq_expand_op.cu @@ -16,8 +16,9 @@ #include "paddle/operators/seq_expand_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(seq_expand, - ops::SeqExpandKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( + seq_expand, + ops::SeqExpandKernel); +REGISTER_OP_CUDA_KERNEL( seq_expand_grad, - ops::SeqExpandGradKernel); + ops::SeqExpandGradKernel); diff --git a/paddle/operators/seq_expand_op.h b/paddle/operators/seq_expand_op.h index 4ef0d02cf8..fbee0db454 100644 --- a/paddle/operators/seq_expand_op.h +++ b/paddle/operators/seq_expand_op.h @@ -23,7 +23,7 @@ namespace operators { using LoDTensor = framework::LoDTensor; -template +template class SeqExpandKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -37,7 +37,8 @@ class SeqExpandKernel : public framework::OpKernel { "The size of last lod level in Input(Y)" "must be equal to dims[0] of Input(X)."); out->set_lod(y->lod()); - auto place = context.GetEigenDevice(); + auto* place = + context.template device_context().eigen_device(); size_t element_len = framework::product(x_dims) / x_dims[0]; T* out_data = out->mutable_data(context.GetPlace()); auto out_starts = out->lod().back(); @@ -50,7 +51,7 @@ class SeqExpandKernel : public framework::OpKernel { Eigen::TensorMap> out_t(out_data, scale, element_len); Eigen::array cast({{scale, 1}}); - out_t.device(place) = x_t.broadcast(cast); + out_t.device(*place) = x_t.broadcast(cast); x_data += element_len; out_data += element_len * scale; } @@ -69,7 +70,7 @@ class SeqExpandKernel : public framework::OpKernel { * Grad(X).lod = Input(X).lod * * */ -template +template class SeqExpandGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -89,8 +90,9 @@ class SeqExpandGradKernel : public framework::OpKernel { d_out_t(d_out_data, static_cast(repeat), element_len); Eigen::TensorMap> d_x_t(d_x_data, static_cast(element_len)); - auto place = context.GetEigenDevice(); - d_x_t.device(place) = d_out_t.sum(Eigen::array({{0}})); + auto place = + context.template device_context().eigen_device(); + d_x_t.device(*place) = d_out_t.sum(Eigen::array({{0}})); d_out_data += (repeat * element_len); d_x_data += element_len; } diff --git a/paddle/operators/sequence_concat_op.cc b/paddle/operators/sequence_concat_op.cc index d1de0b4447..9c7e5456e8 100644 --- a/paddle/operators/sequence_concat_op.cc +++ b/paddle/operators/sequence_concat_op.cc @@ -129,7 +129,7 @@ REGISTER_OP(sequence_concat, ops::SequenceConcatOp, ops::SequenceConcatOpMaker, sequence_concat_grad, ops::SequenceConcatGradOp); REGISTER_OP_CPU_KERNEL( sequence_concat, - ops::SequenceConcatOpKernel); + ops::SequenceConcatOpKernel); REGISTER_OP_CPU_KERNEL( sequence_concat_grad, - ops::SequenceConcatGradOpKernel); + ops::SequenceConcatGradOpKernel); diff --git a/paddle/operators/sequence_concat_op.cu.cc b/paddle/operators/sequence_concat_op.cu.cc index 9ca99c2258..144bdb5af6 100644 --- a/paddle/operators/sequence_concat_op.cu.cc +++ b/paddle/operators/sequence_concat_op.cu.cc @@ -15,9 +15,9 @@ limitations under the License. */ #include "paddle/operators/sequence_concat_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( sequence_concat, - ops::SequenceConcatOpKernel); -REGISTER_OP_GPU_KERNEL( - sequence_concat_grad, - ops::SequenceConcatGradOpKernel); + ops::SequenceConcatOpKernel); +REGISTER_OP_CUDA_KERNEL(sequence_concat_grad, + ops::SequenceConcatGradOpKernel< + paddle::platform::CUDADeviceContext, float>); diff --git a/paddle/operators/sequence_concat_op.h b/paddle/operators/sequence_concat_op.h index 09212070aa..8445224f46 100644 --- a/paddle/operators/sequence_concat_op.h +++ b/paddle/operators/sequence_concat_op.h @@ -59,7 +59,7 @@ LoD ConcatLoD(const std::vector ins, const size_t level) { return out_lod; } -template +template class SequenceConcatOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -119,7 +119,7 @@ class SequenceConcatOpKernel : public framework::OpKernel { } }; -template +template class SequenceConcatGradOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { diff --git a/paddle/operators/sequence_conv_op.cc b/paddle/operators/sequence_conv_op.cc index c5533732d4..f5c4f1c133 100644 --- a/paddle/operators/sequence_conv_op.cc +++ b/paddle/operators/sequence_conv_op.cc @@ -179,9 +179,10 @@ REGISTER_OP(sequence_conv, ops::SequenceConvOp, ops::SequenceConvOpMaker, sequence_conv_grad, ops::SequenceConvGradOp); REGISTER_OP_CPU_KERNEL( - sequence_conv, ops::SequenceConvKernel, - ops::SequenceConvKernel); + sequence_conv, + ops::SequenceConvKernel, + ops::SequenceConvKernel); REGISTER_OP_CPU_KERNEL( sequence_conv_grad, - ops::SequenceConvGradKernel, - ops::SequenceConvGradKernel); + ops::SequenceConvGradKernel, + ops::SequenceConvGradKernel); diff --git a/paddle/operators/sequence_conv_op.cu.cc b/paddle/operators/sequence_conv_op.cu.cc index c8136dbcb3..eacba79ace 100644 --- a/paddle/operators/sequence_conv_op.cu.cc +++ b/paddle/operators/sequence_conv_op.cu.cc @@ -15,10 +15,11 @@ #include "paddle/operators/sequence_conv_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( - sequence_conv, ops::SequenceConvKernel, - ops::SequenceConvKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( + sequence_conv, + ops::SequenceConvKernel, + ops::SequenceConvKernel); +REGISTER_OP_CUDA_KERNEL( sequence_conv_grad, - ops::SequenceConvGradKernel, - ops::SequenceConvGradKernel); + ops::SequenceConvGradKernel, + ops::SequenceConvGradKernel); diff --git a/paddle/operators/sequence_conv_op.h b/paddle/operators/sequence_conv_op.h index b8fbe2647c..bb584b7bfa 100644 --- a/paddle/operators/sequence_conv_op.h +++ b/paddle/operators/sequence_conv_op.h @@ -23,7 +23,7 @@ namespace operators { using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; -template +template class SequenceConvKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -56,21 +56,23 @@ class SequenceConvKernel : public framework::OpKernel { Tensor col; col.mutable_data(col_shape, context.GetPlace()); // Because if padding_trainable is false, padding data should be zeros. - math::SetConstant set_zero; - set_zero(context.device_context(), &col, static_cast(0)); + math::SetConstant set_zero; + auto& dev_ctx = context.template device_context(); + set_zero(dev_ctx, &col, static_cast(0)); - math::ContextProjectFunctor seq_project_functor; + math::ContextProjectFunctor seq_project_functor; - seq_project_functor(context.device_context(), *in, *padding_data, - padding_trainable, context_start, context_length, - context_stride, up_pad, down_pad, &col); + seq_project_functor(dev_ctx, *in, *padding_data, padding_trainable, + context_start, context_length, context_stride, up_pad, + down_pad, &col); - math::matmul(context.device_context(), col, false, filter, false, - static_cast(1.0), out, static_cast(0.0)); + math::matmul(dev_ctx, col, false, filter, false, + static_cast(1.0), out, + static_cast(0.0)); } }; -template +template class SequenceConvGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -95,7 +97,8 @@ class SequenceConvGradKernel : public framework::OpKernel { int down_pad = std::max(0, context_start + context_length - 1); int sequence_width = static_cast(in->dims()[1]); - math::SetConstant set_zero; + math::SetConstant set_zero; + auto& dev_ctx = context.template device_context(); // use col_shape in the im2col calculation framework::DDim col_shape = {in->dims()[0], sequence_width * context_length}; @@ -104,38 +107,36 @@ class SequenceConvGradKernel : public framework::OpKernel { if (in_g || filter_g || (padding_trainable && padding_data_g)) { col.mutable_data(col_shape, context.GetPlace()); // Because if padding_trainable is false, padding data should be zeros. - set_zero(context.device_context(), &col, static_cast(0)); - math::matmul(context.device_context(), *out_g, false, *filter, - true, T(1.0), &col, T(1.0)); + set_zero(dev_ctx, &col, static_cast(0)); + math::matmul(dev_ctx, *out_g, false, *filter, true, + T(1.0), &col, T(1.0)); } - math::ContextProjectFunctor seq_project_functor; - math::ContextProjectGradFunctor seq_project_grad_functor; + math::ContextProjectFunctor seq_project_functor; + math::ContextProjectGradFunctor seq_project_grad_functor; if (in_g) { in_g->mutable_data(context.GetPlace()); in_g->set_lod(in->lod()); - set_zero(context.device_context(), in_g, static_cast(0)); + set_zero(dev_ctx, in_g, static_cast(0)); - seq_project_grad_functor(context.device_context(), *in_g, - padding_trainable, context_start, context_length, - context_stride, up_pad, down_pad, false, true, - padding_data_g, &col); + seq_project_grad_functor(dev_ctx, *in_g, padding_trainable, context_start, + context_length, context_stride, up_pad, down_pad, + false, true, padding_data_g, &col); } if (padding_trainable && padding_data_g) { padding_data_g->mutable_data(context.GetPlace()); - set_zero(context.device_context(), padding_data_g, static_cast(0)); + set_zero(dev_ctx, padding_data_g, static_cast(0)); LoDTensor* input = const_cast(in); - seq_project_grad_functor(context.device_context(), *input, - padding_trainable, context_start, context_length, - context_stride, up_pad, down_pad, true, false, - padding_data_g, &col); + seq_project_grad_functor( + dev_ctx, *input, padding_trainable, context_start, context_length, + context_stride, up_pad, down_pad, true, false, padding_data_g, &col); } if (filter_g) { filter_g->mutable_data(context.GetPlace()); - set_zero(context.device_context(), filter_g, static_cast(0)); + set_zero(dev_ctx, filter_g, static_cast(0)); Tensor filter_grad = *filter_g; LoDTensor out_grad = *out_g; @@ -145,12 +146,12 @@ class SequenceConvGradKernel : public framework::OpKernel { padding_data = context.Input("PaddingData"); } - seq_project_functor(context.device_context(), *in, *padding_data, - padding_trainable, context_start, context_length, - context_stride, up_pad, down_pad, &col); + seq_project_functor(dev_ctx, *in, *padding_data, padding_trainable, + context_start, context_length, context_stride, up_pad, + down_pad, &col); - math::matmul(context.device_context(), col, true, out_grad, - false, T(1.0), &filter_grad, T(1.0)); + math::matmul(dev_ctx, col, true, out_grad, false, + T(1.0), &filter_grad, T(1.0)); } } }; diff --git a/paddle/operators/sequence_pool_op.cc b/paddle/operators/sequence_pool_op.cc index bfda8649cd..3526e45a1b 100644 --- a/paddle/operators/sequence_pool_op.cc +++ b/paddle/operators/sequence_pool_op.cc @@ -123,7 +123,8 @@ namespace ops = paddle::operators; REGISTER_OP(sequence_pool, ops::SequencePoolOp, ops::SequencePoolOpMaker, sequence_pool_grad, ops::SequencePoolGradOp); REGISTER_OP_CPU_KERNEL( - sequence_pool, ops::SequencePoolKernel); + sequence_pool, + ops::SequencePoolKernel); REGISTER_OP_CPU_KERNEL( sequence_pool_grad, - ops::SequencePoolGradKernel); + ops::SequencePoolGradKernel); diff --git a/paddle/operators/sequence_pool_op.cu b/paddle/operators/sequence_pool_op.cu index 66850772d5..fcd6508435 100644 --- a/paddle/operators/sequence_pool_op.cu +++ b/paddle/operators/sequence_pool_op.cu @@ -17,8 +17,9 @@ #include "paddle/operators/sequence_pool_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( - sequence_pool, ops::SequencePoolKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( + sequence_pool, + ops::SequencePoolKernel); +REGISTER_OP_CUDA_KERNEL( sequence_pool_grad, - ops::SequencePoolGradKernel); + ops::SequencePoolGradKernel); diff --git a/paddle/operators/sequence_pool_op.h b/paddle/operators/sequence_pool_op.h index 7f136d8cf0..7519aa1d72 100644 --- a/paddle/operators/sequence_pool_op.h +++ b/paddle/operators/sequence_pool_op.h @@ -30,7 +30,7 @@ template using EigenMatrix = framework::EigenMatrix; -template +template class SequencePoolKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -54,17 +54,18 @@ class SequencePoolKernel : public framework::OpKernel { auto lod_level_0 = lod[0]; out->mutable_data(context.GetPlace()); - + auto& dev_ctx = context.template device_context(); if (pooltype == "MAX") { - math::MaxSeqPoolFunctor max_pool; + math::MaxSeqPoolFunctor max_pool; auto* index = context.Output("MaxIndex"); index->Resize({dims}); index->mutable_data(context.GetPlace()); - max_pool(context.device_context(), *in, out, index); + max_pool(dev_ctx, *in, out, index); return; } - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); for (int i = 0; i < static_cast(lod_level_0.size()) - 1; ++i) { Tensor in_t = in->Slice(static_cast(lod_level_0[i]), static_cast(lod_level_0[i + 1])); @@ -91,7 +92,7 @@ class SequencePoolKernel : public framework::OpKernel { } }; -template +template class SequencePoolGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -105,20 +106,23 @@ class SequencePoolGradKernel : public framework::OpKernel { int64_t w = in->numel() / dims[0]; in_g->mutable_data(context.GetPlace()); + auto& dev_ctx = context.template device_context(); if (pooltype == "MAX") { - math::MaxSeqPoolGradFunctor max_pool_grad; + math::MaxSeqPoolGradFunctor max_pool_grad; auto* index = context.Input("MaxIndex"); - max_pool_grad(context.device_context(), *out_g, *index, in_g); + max_pool_grad(dev_ctx, *out_g, *index, in_g); return; } if (pooltype == "LAST" || pooltype == "FIRST") { // set X@Grad be zero at first when pooltype is LAST/FIRST - math::SetConstant functor; - functor(context.device_context(), in_g, 0); + math::SetConstant functor; + functor(dev_ctx, in_g, 0); } - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); + for (int i = 0; i < static_cast(lod.size()) - 1; ++i) { auto in_g_t = in_g->Slice(static_cast(lod[i]), static_cast(lod[i + 1])); diff --git a/paddle/operators/sequence_slice_op.cc b/paddle/operators/sequence_slice_op.cc index 255683a572..481db8f9e5 100644 --- a/paddle/operators/sequence_slice_op.cc +++ b/paddle/operators/sequence_slice_op.cc @@ -125,7 +125,7 @@ REGISTER_OP(sequence_slice, ops::SequenceSliceOp, ops::SequenceSliceOpMaker, sequence_slice_grad, ops::SequenceSliceGradOp); REGISTER_OP_CPU_KERNEL( sequence_slice, - ops::SequenceSliceOpKernel); + ops::SequenceSliceOpKernel); REGISTER_OP_CPU_KERNEL( sequence_slice_grad, - ops::SequenceSliceGradOpKernel); + ops::SequenceSliceGradOpKernel); diff --git a/paddle/operators/sequence_slice_op.cu b/paddle/operators/sequence_slice_op.cu index a9f59dadba..43a21d619f 100755 --- a/paddle/operators/sequence_slice_op.cu +++ b/paddle/operators/sequence_slice_op.cu @@ -15,9 +15,9 @@ limitations under the License. */ #include "paddle/operators/sequence_slice_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( sequence_slice, - ops::SequenceSliceOpKernel); -REGISTER_OP_GPU_KERNEL( + ops::SequenceSliceOpKernel); +REGISTER_OP_CUDA_KERNEL( sequence_slice_grad, - ops::SequenceSliceGradOpKernel); + ops::SequenceSliceGradOpKernel); diff --git a/paddle/operators/sequence_slice_op.h b/paddle/operators/sequence_slice_op.h index 428ef556da..14bcaebbb4 100644 --- a/paddle/operators/sequence_slice_op.h +++ b/paddle/operators/sequence_slice_op.h @@ -39,7 +39,7 @@ inline LoD SequenceSliceLoD(const T& in, const int64_t* offset_data, return out_lod; } -template +template class SequenceSliceOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -108,7 +108,7 @@ class SequenceSliceOpKernel : public framework::OpKernel { } }; -template +template class SequenceSliceGradOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -143,8 +143,9 @@ class SequenceSliceGradOpKernel : public framework::OpKernel { if (x_grad) { x_grad->mutable_data(ctx.GetPlace()); x_grad->set_lod(in->lod()); - math::SetConstant set_zero; - set_zero(ctx.device_context(), x_grad, static_cast(0)); + math::SetConstant set_zero; + set_zero(ctx.template device_context(), x_grad, + static_cast(0)); auto out_grad_stride = framework::stride(out_grad->dims()); diff --git a/paddle/operators/sequence_softmax_op.cc b/paddle/operators/sequence_softmax_op.cc index 32c1502566..37d5452e6b 100644 --- a/paddle/operators/sequence_softmax_op.cc +++ b/paddle/operators/sequence_softmax_op.cc @@ -99,7 +99,7 @@ REGISTER_OP(sequence_softmax, ops::SequenceSoftmaxOp, ops::SequenceSoftmaxGradOp); REGISTER_OP_CPU_KERNEL( sequence_softmax, - ops::SequenceSoftmaxKernel); + ops::SequenceSoftmaxKernel); REGISTER_OP_CPU_KERNEL( sequence_softmax_grad, - ops::SequenceSoftmaxGradKernel); + ops::SequenceSoftmaxGradKernel); diff --git a/paddle/operators/sequence_softmax_op.cu.cc b/paddle/operators/sequence_softmax_op.cu.cc index 7023795a3b..5f65b4daf9 100644 --- a/paddle/operators/sequence_softmax_op.cu.cc +++ b/paddle/operators/sequence_softmax_op.cu.cc @@ -15,9 +15,9 @@ limitations under the License. */ #include "paddle/operators/sequence_softmax_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( sequence_softmax, - ops::SequenceSoftmaxKernel) -REGISTER_OP_GPU_KERNEL( + ops::SequenceSoftmaxKernel) +REGISTER_OP_CUDA_KERNEL( sequence_softmax_grad, - ops::SequenceSoftmaxGradKernel); + ops::SequenceSoftmaxGradKernel); diff --git a/paddle/operators/sequence_softmax_op.h b/paddle/operators/sequence_softmax_op.h index 1b68dd0662..e889e88cb3 100644 --- a/paddle/operators/sequence_softmax_op.h +++ b/paddle/operators/sequence_softmax_op.h @@ -23,7 +23,7 @@ namespace operators { using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; -template +template class SequenceSoftmaxKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -52,12 +52,13 @@ class SequenceSoftmaxKernel : public framework::OpKernel { framework::DDim dims_i = framework::make_ddim({1UL, end_pos - start_pos}); x_i.Resize(dims_i); out_i.Resize(dims_i); - math::SoftmaxFunctor()(ctx.device_context(), &x_i, &out_i); + math::SoftmaxFunctor()( + ctx.template device_context(), &x_i, &out_i); } } }; -template +template class SequenceSoftmaxGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -83,8 +84,9 @@ class SequenceSoftmaxGradKernel : public framework::OpKernel { out_i.Resize(dims_i); out_grad_i.Resize(dims_i); x_grad_i.Resize(dims_i); - math::SoftmaxGradFunctor()(ctx.device_context(), &out_i, - &out_grad_i, &x_grad_i); + math::SoftmaxGradFunctor()( + ctx.template device_context(), &out_i, &out_grad_i, + &x_grad_i); } } }; diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc index 5576d7b8be..121bf60b27 100644 --- a/paddle/operators/sgd_op.cc +++ b/paddle/operators/sgd_op.cc @@ -62,8 +62,8 @@ $$param\_out = param - learning\_rate * grad$$ }; template -struct SparseSGDFunctor { - void operator()(const platform::DeviceContext& context, +struct SparseSGDFunctor { + void operator()(const platform::CPUDeviceContext& context, const framework::SelectedRows& input, const framework::Tensor& learning_rate, framework::Tensor* output) { @@ -90,13 +90,14 @@ struct SparseSGDFunctor { } }; -template struct SparseSGDFunctor; -template struct SparseSGDFunctor; +template struct SparseSGDFunctor; +template struct SparseSGDFunctor; } // namespace operators } // namespace paddle namespace ops = paddle::operators; REGISTER_OP_WITHOUT_GRADIENT(sgd, ops::SGDOp, ops::SGDOpMaker); -REGISTER_OP_CPU_KERNEL(sgd, ops::SGDOpKernel, - ops::SGDOpKernel); +REGISTER_OP_CPU_KERNEL( + sgd, ops::SGDOpKernel, + ops::SGDOpKernel); diff --git a/paddle/operators/sgd_op.cu b/paddle/operators/sgd_op.cu index 7b6c5ec306..a3c0db7e50 100644 --- a/paddle/operators/sgd_op.cu +++ b/paddle/operators/sgd_op.cu @@ -41,8 +41,8 @@ __global__ void SparseSGDFunctorKernel(const T* selected_rows, } // namespace template -struct SparseSGDFunctor { - void operator()(const platform::DeviceContext& context, +struct SparseSGDFunctor { + void operator()(const platform::CUDADeviceContext& context, const framework::SelectedRows& input, const framework::Tensor& learning_rate, framework::Tensor* output) { @@ -62,21 +62,19 @@ struct SparseSGDFunctor { const int block_size = 256; dim3 threads(block_size, 1); dim3 grid(1, in_rows.size()); - SparseSGDFunctorKernel< - T, 256><<(context) - .stream()>>>(in_data, in_rows.data(), - learning_rate.data(), out_data, - in_row_numel); + SparseSGDFunctorKernel<<>>( + in_data, in_rows.data(), learning_rate.data(), out_data, + in_row_numel); } }; -template struct SparseSGDFunctor; -template struct SparseSGDFunctor; +template struct SparseSGDFunctor; +template struct SparseSGDFunctor; } // namespace operators } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(sgd, ops::SGDOpKernel, - ops::SGDOpKernel); +REGISTER_OP_CUDA_KERNEL( + sgd, ops::SGDOpKernel, + ops::SGDOpKernel); diff --git a/paddle/operators/sgd_op.h b/paddle/operators/sgd_op.h index 78b595fc6c..c920025a91 100644 --- a/paddle/operators/sgd_op.h +++ b/paddle/operators/sgd_op.h @@ -20,15 +20,15 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template struct SparseSGDFunctor { - void operator()(const platform::DeviceContext& context, + void operator()(const DeviceContext& context, const framework::SelectedRows& input, const framework::Tensor& learning_rate, framework::Tensor* output); }; -template +template class SGDOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -46,7 +46,8 @@ class SGDOpKernel : public framework::OpKernel { auto g = framework::EigenVector::Flatten(*grad); auto o = framework::EigenVector::Flatten(*param_out); auto lr = framework::EigenVector::Flatten(*learning_rate); - auto place = ctx.GetEigenDevice(); + auto& place = + *ctx.template device_context().eigen_device(); Eigen::DSizes grad_dsize(grad->numel()); o.device(place) = p - lr.broadcast(grad_dsize) * g; @@ -56,8 +57,9 @@ class SGDOpKernel : public framework::OpKernel { // It's better to find a more elegant solution. PADDLE_ENFORCE_EQ(param, param_out); auto* grad = ctx.Input("Grad"); - SparseSGDFunctor functor; - functor(ctx.device_context(), *grad, *learning_rate, param_out); + SparseSGDFunctor functor; + functor(ctx.template device_context(), *grad, + *learning_rate, param_out); } else { PADDLE_THROW("Unsupported Variable Type of Grad"); } diff --git a/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc b/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc index 782f4c7936..b8a1bf122a 100644 --- a/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc +++ b/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc @@ -142,7 +142,7 @@ REGISTER_OP(sigmoid_cross_entropy_with_logits, ops::SigmoidCrossEntropyWithLogitsGradOp); REGISTER_OP_CPU_KERNEL(sigmoid_cross_entropy_with_logits, ops::SigmoidCrossEntropyWithLogitsKernel< - paddle::platform::CPUPlace, float>); + paddle::platform::CPUDeviceContext, float>); REGISTER_OP_CPU_KERNEL(sigmoid_cross_entropy_with_logits_grad, ops::SigmoidCrossEntropyWithLogitsGradKernel< - paddle::platform::CPUPlace, float>); + paddle::platform::CPUDeviceContext, float>); diff --git a/paddle/operators/sigmoid_cross_entropy_with_logits_op.cu b/paddle/operators/sigmoid_cross_entropy_with_logits_op.cu index 32a39956a1..1b569c93ed 100644 --- a/paddle/operators/sigmoid_cross_entropy_with_logits_op.cu +++ b/paddle/operators/sigmoid_cross_entropy_with_logits_op.cu @@ -16,9 +16,9 @@ #include "paddle/operators/sigmoid_cross_entropy_with_logits_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(sigmoid_cross_entropy_with_logits, - ops::SigmoidCrossEntropyWithLogitsKernel< - paddle::platform::GPUPlace, float>); -REGISTER_OP_GPU_KERNEL(sigmoid_cross_entropy_with_logits_grad, - ops::SigmoidCrossEntropyWithLogitsGradKernel< - paddle::platform::GPUPlace, float>); +REGISTER_OP_CUDA_KERNEL(sigmoid_cross_entropy_with_logits, + ops::SigmoidCrossEntropyWithLogitsKernel< + paddle::platform::CUDADeviceContext, float>); +REGISTER_OP_CUDA_KERNEL(sigmoid_cross_entropy_with_logits_grad, + ops::SigmoidCrossEntropyWithLogitsGradKernel< + paddle::platform::CUDADeviceContext, float>); diff --git a/paddle/operators/sigmoid_cross_entropy_with_logits_op.h b/paddle/operators/sigmoid_cross_entropy_with_logits_op.h index 2a9d9bbc77..8fe7c5ba82 100644 --- a/paddle/operators/sigmoid_cross_entropy_with_logits_op.h +++ b/paddle/operators/sigmoid_cross_entropy_with_logits_op.h @@ -20,7 +20,7 @@ namespace paddle { namespace operators { // Out = max(X, 0) - X * Labels + log(1 + exp(-abs(X))) -template +template class SigmoidCrossEntropyWithLogitsKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { @@ -32,7 +32,7 @@ class SigmoidCrossEntropyWithLogitsKernel : public framework::OpKernel { auto x = framework::EigenVector::Flatten(*X); auto labels = framework::EigenVector::Flatten(*Labels); auto out = framework::EigenVector::Flatten(*Out); - auto place = context.GetEigenDevice(); + auto &place = *context.device_context().eigen_device(); // term1 = max(x, 0) auto term1 = x.cwiseMax(static_cast(0)); @@ -46,7 +46,7 @@ class SigmoidCrossEntropyWithLogitsKernel : public framework::OpKernel { }; // dX = sigmoid(X) - labels -template +template class SigmoidCrossEntropyWithLogitsGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { @@ -62,7 +62,8 @@ class SigmoidCrossEntropyWithLogitsGradKernel : public framework::OpKernel { auto labels = framework::EigenVector::Flatten(*Labels); auto dout = framework::EigenVector::Flatten(*dOut); auto dx = framework::EigenVector::Flatten(*dX); - auto place = context.GetEigenDevice(); + auto &place = + *context.template device_context().eigen_device(); auto sigmoid_x = static_cast(1) / (static_cast(1) + (-x).exp()); dx.device(place) = dout * (sigmoid_x - labels); diff --git a/paddle/operators/sign_op.cc b/paddle/operators/sign_op.cc index 08bf2e4e7c..d5a7ccb77e 100644 --- a/paddle/operators/sign_op.cc +++ b/paddle/operators/sign_op.cc @@ -67,5 +67,5 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(sign, ops::SignOp, ops::SignOpMaker, ops::SignGradMaker); -REGISTER_OP_CPU_KERNEL(sign, - ops::SignKernel); +REGISTER_OP_CPU_KERNEL( + sign, ops::SignKernel); diff --git a/paddle/operators/sign_op.cu b/paddle/operators/sign_op.cu index 4d0638cb97..9bc1c65d21 100644 --- a/paddle/operators/sign_op.cu +++ b/paddle/operators/sign_op.cu @@ -14,5 +14,6 @@ #include "paddle/operators/sign_op.h" -REGISTER_OP_GPU_KERNEL( - sign, paddle::operators::SignKernel); +REGISTER_OP_CUDA_KERNEL( + sign, + paddle::operators::SignKernel); diff --git a/paddle/operators/sign_op.h b/paddle/operators/sign_op.h index ab5cd4bac0..2e476ed665 100644 --- a/paddle/operators/sign_op.h +++ b/paddle/operators/sign_op.h @@ -19,7 +19,7 @@ namespace paddle { namespace operators { -template +template class SignKernel : public framework::OpKernel { public: virtual void Compute(const framework::ExecutionContext& context) const { @@ -29,7 +29,8 @@ class SignKernel : public framework::OpKernel { auto eigen_out = framework::EigenVector::Flatten(*out); auto eigen_in = framework::EigenVector::Flatten(*in); - auto& place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); eigen_out.device(place) = eigen_in.sign(); } }; diff --git a/paddle/operators/smooth_l1_loss_op.cc b/paddle/operators/smooth_l1_loss_op.cc index 50543fcc14..56e8d9058f 100644 --- a/paddle/operators/smooth_l1_loss_op.cc +++ b/paddle/operators/smooth_l1_loss_op.cc @@ -138,7 +138,8 @@ REGISTER_OP(smooth_l1_loss, ops::SmoothL1LossOp, ops::SmoothL1LossOpMaker, smooth_l1_loss_grad, ops::SmoothL1LossGradOp); REGISTER_OP_CPU_KERNEL( - smooth_l1_loss, ops::SmoothL1LossKernel); + smooth_l1_loss, + ops::SmoothL1LossKernel); REGISTER_OP_CPU_KERNEL( smooth_l1_loss_grad, - ops::SmoothL1LossGradKernel); + ops::SmoothL1LossGradKernel); diff --git a/paddle/operators/smooth_l1_loss_op.cu b/paddle/operators/smooth_l1_loss_op.cu index 1c3172f438..8e94ebac64 100644 --- a/paddle/operators/smooth_l1_loss_op.cu +++ b/paddle/operators/smooth_l1_loss_op.cu @@ -17,8 +17,9 @@ #include "paddle/operators/smooth_l1_loss_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( - smooth_l1_loss, ops::SmoothL1LossKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( + smooth_l1_loss, + ops::SmoothL1LossKernel); +REGISTER_OP_CUDA_KERNEL( smooth_l1_loss_grad, - ops::SmoothL1LossGradKernel); + ops::SmoothL1LossGradKernel); diff --git a/paddle/operators/smooth_l1_loss_op.h b/paddle/operators/smooth_l1_loss_op.h index 39d0070b6c..1a70c9c63c 100644 --- a/paddle/operators/smooth_l1_loss_op.h +++ b/paddle/operators/smooth_l1_loss_op.h @@ -44,7 +44,7 @@ struct SmoothL1LossForward { T sigma2; }; -template +template class SmoothL1LossKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -57,7 +57,8 @@ class SmoothL1LossKernel : public framework::OpKernel { out0->mutable_data(context.GetPlace()); out1->mutable_data(context.GetPlace()); - auto place = context.GetEigenDevice(); + auto* place = + context.template device_context().eigen_device(); auto sigma = static_cast(context.Attr("sigma")); T sigma2 = sigma * sigma; @@ -67,12 +68,12 @@ class SmoothL1LossKernel : public framework::OpKernel { auto y = EigenVector::Flatten(*in1); auto diff = EigenVector::Flatten(*out0); - diff.device(place) = x - y; + diff.device(*place) = x - y; // multiply inside weight if (has_weight) { auto inside_weight = EigenVector::Flatten(*in2); // cache diff, reused in bp - diff.device(place) = diff * inside_weight; + diff.device(*place) = diff * inside_weight; } auto in_counts = in0->numel(); @@ -81,12 +82,12 @@ class SmoothL1LossKernel : public framework::OpKernel { context.GetPlace()); auto errors = EigenVector::Flatten(ptensor_errors); // apply smooth l1 forward - errors.device(place) = diff.unaryExpr(SmoothL1LossForward(sigma2)); + errors.device(*place) = diff.unaryExpr(SmoothL1LossForward(sigma2)); // multiply outside weight if (has_weight) { auto outside_weight = EigenVector::Flatten(*in3); - errors.device(place) = errors * outside_weight; + errors.device(*place) = errors * outside_weight; } auto loss = EigenVector::Flatten(*out1); // first dimension of 'X' is the number of samples @@ -94,7 +95,7 @@ class SmoothL1LossKernel : public framework::OpKernel { framework::make_ddim({static_cast(in0->dims()[0]), static_cast(in_counts / in0->dims()[0])}); auto errors_mat_view = EigenMatrix::From(ptensor_errors, mat_dims); - loss.device(place) = errors_mat_view.sum(Eigen::array({{1}})); + loss.device(*place) = errors_mat_view.sum(Eigen::array({{1}})); } }; @@ -114,7 +115,7 @@ struct SmoothL1LossBackward { T sigma2; }; -template +template class SmoothL1LossGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -126,7 +127,8 @@ class SmoothL1LossGradKernel : public framework::OpKernel { T sigma2 = sigma * sigma; bool has_weight = (in0 != nullptr) && (in1 != nullptr); - auto place = context.GetEigenDevice(); + auto* place = + context.template device_context().eigen_device(); auto in_dims = in2->dims(); auto counts = in2->numel(); @@ -139,7 +141,7 @@ class SmoothL1LossGradKernel : public framework::OpKernel { context.GetPlace()); auto diff = EigenVector::Flatten(ptensor_diff); // apply smooth l1 backwoard - diff.device(place) = EigenVector::Flatten(*in2).unaryExpr( + diff.device(*place) = EigenVector::Flatten(*in2).unaryExpr( SmoothL1LossBackward(sigma2)); // compute weights @@ -147,11 +149,11 @@ class SmoothL1LossGradKernel : public framework::OpKernel { ptensor_weights.mutable_data(mat_dims, context.GetPlace()); auto weights = EigenMatrix::From(ptensor_weights); // initialize to 1.0 - weights.device(place) = weights.constant(static_cast(1.0)); + weights.device(*place) = weights.constant(static_cast(1.0)); if (has_weight) { auto inside_weight = EigenMatrix::From(*in0, mat_dims); auto outside_weight = EigenMatrix::From(*in1, mat_dims); - weights.device(place) = inside_weight * outside_weight; + weights.device(*place) = inside_weight * outside_weight; } // compute gradients @@ -167,13 +169,13 @@ class SmoothL1LossGradKernel : public framework::OpKernel { if (out0) { out0->mutable_data(context.GetPlace()); auto x_grad = EigenMatrix::From(*out0, mat_dims); - x_grad.device(place) = gradients; + x_grad.device(*place) = gradients; } if (out1) { out1->mutable_data(context.GetPlace()); auto y_grad = EigenMatrix::From(*out1, mat_dims); - y_grad.device(place) = -1 * gradients; + y_grad.device(*place) = -1 * gradients; } } }; diff --git a/paddle/operators/softmax_op.cc b/paddle/operators/softmax_op.cc index 93e0525bad..0988c83d43 100644 --- a/paddle/operators/softmax_op.cc +++ b/paddle/operators/softmax_op.cc @@ -89,7 +89,8 @@ namespace ops = paddle::operators; REGISTER_OP(softmax, ops::SoftmaxOp, ops::SoftmaxOpMaker, softmax_grad, ops::SoftmaxOpGrad); -REGISTER_OP_CPU_KERNEL(softmax, - ops::SoftmaxKernel); REGISTER_OP_CPU_KERNEL( - softmax_grad, ops::SoftmaxGradKernel); + softmax, ops::SoftmaxKernel); +REGISTER_OP_CPU_KERNEL( + softmax_grad, + ops::SoftmaxGradKernel); diff --git a/paddle/operators/softmax_op.cu.cc b/paddle/operators/softmax_op.cu.cc index 013ace19ae..7b9882cbcf 100644 --- a/paddle/operators/softmax_op.cu.cc +++ b/paddle/operators/softmax_op.cu.cc @@ -16,7 +16,8 @@ namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(softmax, - ops::SoftmaxKernel); -REGISTER_OP_GPU_KERNEL( - softmax_grad, ops::SoftmaxGradKernel); +REGISTER_OP_CUDA_KERNEL( + softmax, ops::SoftmaxKernel); +REGISTER_OP_CUDA_KERNEL( + softmax_grad, + ops::SoftmaxGradKernel); diff --git a/paddle/operators/softmax_op.h b/paddle/operators/softmax_op.h index 44d1e63f1b..0f8998b99e 100644 --- a/paddle/operators/softmax_op.h +++ b/paddle/operators/softmax_op.h @@ -21,7 +21,7 @@ namespace operators { using Tensor = framework::Tensor; -template +template class SoftmaxKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -31,11 +31,12 @@ class SoftmaxKernel : public framework::OpKernel { // allocate memory on device. Y->mutable_data(context.GetPlace()); - math::SoftmaxFunctor()(context.device_context(), X, Y); + math::SoftmaxFunctor()( + context.template device_context(), X, Y); } }; -template +template class SoftmaxGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -46,7 +47,8 @@ class SoftmaxGradKernel : public framework::OpKernel { // allocate memory on device. dX->mutable_data(context.GetPlace()); - math::SoftmaxGradFunctor()(context.device_context(), Y, dY, dX); + math::SoftmaxGradFunctor()( + context.template device_context(), Y, dY, dX); } }; diff --git a/paddle/operators/softmax_with_cross_entropy_op.cu b/paddle/operators/softmax_with_cross_entropy_op.cu index b1faddac3f..6100c63f9a 100644 --- a/paddle/operators/softmax_with_cross_entropy_op.cu +++ b/paddle/operators/softmax_with_cross_entropy_op.cu @@ -69,10 +69,10 @@ class SoftmaxWithCrossEntropyCUDAKernel : public framework::OpKernel { softmax->mutable_data(context.GetPlace()); loss->mutable_data(context.GetPlace()); - math::SoftmaxFunctor()(context.device_context(), - logits, softmax); - math::CrossEntropyFunctor()( - context.device_context(), loss, softmax, labels, + math::SoftmaxFunctor()( + context.cuda_device_context(), logits, softmax); + math::CrossEntropyFunctor()( + context.cuda_device_context(), loss, softmax, labels, context.Attr("soft_label")); } }; @@ -98,18 +98,18 @@ class SoftmaxWithCrossEntropyGradCUDAKernel : public framework::OpKernel { if (context.Attr("soft_label")) { const T* label_data = labels->data(); - SoftCrossEntropyGradientKernel<<< - grid, block, 0, reinterpret_cast( - context.device_context()) - .stream()>>>(logit_grad_data, loss_grad_data, - label_data, batch_size, class_num); + SoftCrossEntropyGradientKernel< + T><<() + .stream()>>>(logit_grad_data, loss_grad_data, label_data, + batch_size, class_num); } else { const int64_t* label_data = labels->data(); - CrossEntropyGrad<<< - grid, block, 0, reinterpret_cast( - context.device_context()) - .stream()>>>(logit_grad_data, loss_grad_data, - label_data, batch_size, class_num); + CrossEntropyGrad< + T><<() + .stream()>>>(logit_grad_data, loss_grad_data, label_data, + batch_size, class_num); } } }; @@ -118,9 +118,9 @@ class SoftmaxWithCrossEntropyGradCUDAKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(softmax_with_cross_entropy, - ops::SoftmaxWithCrossEntropyCUDAKernel, - ops::SoftmaxWithCrossEntropyCUDAKernel); -REGISTER_OP_GPU_KERNEL(softmax_with_cross_entropy_grad, - ops::SoftmaxWithCrossEntropyGradCUDAKernel, - ops::SoftmaxWithCrossEntropyGradCUDAKernel); +REGISTER_OP_CUDA_KERNEL(softmax_with_cross_entropy, + ops::SoftmaxWithCrossEntropyCUDAKernel, + ops::SoftmaxWithCrossEntropyCUDAKernel); +REGISTER_OP_CUDA_KERNEL(softmax_with_cross_entropy_grad, + ops::SoftmaxWithCrossEntropyGradCUDAKernel, + ops::SoftmaxWithCrossEntropyGradCUDAKernel); diff --git a/paddle/operators/softmax_with_cross_entropy_op.h b/paddle/operators/softmax_with_cross_entropy_op.h index c4ab3f74b4..9c3431605b 100644 --- a/paddle/operators/softmax_with_cross_entropy_op.h +++ b/paddle/operators/softmax_with_cross_entropy_op.h @@ -40,11 +40,12 @@ class SoftmaxWithCrossEntropyKernel : public framework::OpKernel { softmax->mutable_data(context.GetPlace()); loss->mutable_data(context.GetPlace()); - math::SoftmaxFunctor()(context.device_context(), - logits, softmax); - math::CrossEntropyFunctor()( - context.device_context(), loss, softmax, labels, - context.Attr("soft_label")); + auto& dev_ctx = + context.template device_context(); + math::SoftmaxFunctor()(dev_ctx, logits, + softmax); + math::CrossEntropyFunctor()( + dev_ctx, loss, softmax, labels, context.Attr("soft_label")); } }; @@ -62,14 +63,15 @@ class SoftmaxWithCrossEntropyGradKernel : public framework::OpKernel { const int class_num = logit_grad->dims()[1]; auto out_grad_mat = EigenMatrix::From(*out_grad); auto logit_grad_mat = EigenMatrix::From(*logit_grad); - + auto& place = *context.template device_context() + .eigen_device(); if (context.Attr("soft_label")) { auto lbl_mat = EigenMatrix::From(*labels); - logit_grad_mat.device(context.GetEigenDevice()) = + logit_grad_mat.device(place) = out_grad_mat.broadcast(Eigen::DSizes(1, class_num)) * (logit_grad_mat - lbl_mat); } else { - logit_grad_mat.device(context.GetEigenDevice()) = + logit_grad_mat.device(place) = logit_grad_mat * out_grad_mat.broadcast(Eigen::DSizes(1, class_num)); diff --git a/paddle/operators/split_op.cu.cc b/paddle/operators/split_op.cu.cc index 93d1fc3c44..dbad0bbf68 100644 --- a/paddle/operators/split_op.cu.cc +++ b/paddle/operators/split_op.cu.cc @@ -14,5 +14,5 @@ limitations under the License. */ #include "paddle/operators/split_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(split, - ops::SplitOpKernel); +REGISTER_OP_CUDA_KERNEL( + split, ops::SplitOpKernel); diff --git a/paddle/operators/split_op.h b/paddle/operators/split_op.h index fa26e5f677..a38c435d53 100644 --- a/paddle/operators/split_op.h +++ b/paddle/operators/split_op.h @@ -21,7 +21,7 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class SplitOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { diff --git a/paddle/operators/squared_l2_distance_op.cc b/paddle/operators/squared_l2_distance_op.cc index bec2a2c18a..50bc6da196 100644 --- a/paddle/operators/squared_l2_distance_op.cc +++ b/paddle/operators/squared_l2_distance_op.cc @@ -115,7 +115,7 @@ REGISTER_OP(squared_l2_distance, ops::SquaredL2DistanceOp, ops::SquaredL2DistanceGradOp); REGISTER_OP_CPU_KERNEL( squared_l2_distance, - ops::SquaredL2DistanceKernel); -REGISTER_OP_CPU_KERNEL( - squared_l2_distance_grad, - ops::SquaredL2DistanceGradKernel); + ops::SquaredL2DistanceKernel); +REGISTER_OP_CPU_KERNEL(squared_l2_distance_grad, + ops::SquaredL2DistanceGradKernel< + paddle::platform::CPUDeviceContext, float>); diff --git a/paddle/operators/squared_l2_distance_op.cu b/paddle/operators/squared_l2_distance_op.cu index 3fe62f1a9c..ecc82ed1e4 100644 --- a/paddle/operators/squared_l2_distance_op.cu +++ b/paddle/operators/squared_l2_distance_op.cu @@ -17,9 +17,9 @@ #include "paddle/operators/squared_l2_distance_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( squared_l2_distance, - ops::SquaredL2DistanceKernel); -REGISTER_OP_GPU_KERNEL( - squared_l2_distance_grad, - ops::SquaredL2DistanceGradKernel); + ops::SquaredL2DistanceKernel); +REGISTER_OP_CUDA_KERNEL(squared_l2_distance_grad, + ops::SquaredL2DistanceGradKernel< + paddle::platform::CUDADeviceContext, float>); diff --git a/paddle/operators/squared_l2_distance_op.h b/paddle/operators/squared_l2_distance_op.h index 259ef40296..5bd5f4819a 100644 --- a/paddle/operators/squared_l2_distance_op.h +++ b/paddle/operators/squared_l2_distance_op.h @@ -27,7 +27,7 @@ template using EigenMatrix = framework::EigenMatrix; -template +template class SquaredL2DistanceKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -51,7 +51,8 @@ class SquaredL2DistanceKernel : public framework::OpKernel { auto sub_result = EigenMatrix::From(*out0); auto z = EigenVector::Flatten(*out1); - auto place = context.GetEigenDevice(); + auto& place = + *context.template device_context().eigen_device(); auto x_dims = x.dimensions(); auto y_dims = y.dimensions(); // buffer the substraction result @@ -67,7 +68,7 @@ class SquaredL2DistanceKernel : public framework::OpKernel { } }; -template +template class SquaredL2DistanceGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -89,7 +90,8 @@ class SquaredL2DistanceGradKernel : public framework::OpKernel { sub_result; // propagate back to input - auto eigen_place = context.GetEigenDevice(); + auto& eigen_place = + *context.template device_context().eigen_device(); if (x_g) { x_g->mutable_data(context.GetPlace()); // eigen matrix diff --git a/paddle/operators/squared_l2_norm_op.cc b/paddle/operators/squared_l2_norm_op.cc index 3c10e6159f..3cff61a02f 100644 --- a/paddle/operators/squared_l2_norm_op.cc +++ b/paddle/operators/squared_l2_norm_op.cc @@ -72,7 +72,7 @@ REGISTER_OP(squared_l2_norm, ops::SquaredL2NormOp, ops::SquaredL2NormOpMaker, squared_l2_norm_grad, ops::SquaredL2NormGradOp); REGISTER_OP_CPU_KERNEL( squared_l2_norm, - ops::SquaredL2NormKernel); + ops::SquaredL2NormKernel); REGISTER_OP_CPU_KERNEL( squared_l2_norm_grad, - ops::SquaredL2NormGradKernel); + ops::SquaredL2NormGradKernel); diff --git a/paddle/operators/squared_l2_norm_op.cu b/paddle/operators/squared_l2_norm_op.cu index d384e9c28c..2d6567d090 100644 --- a/paddle/operators/squared_l2_norm_op.cu +++ b/paddle/operators/squared_l2_norm_op.cu @@ -16,9 +16,9 @@ #include "paddle/operators/squared_l2_norm_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( squared_l2_norm, - ops::SquaredL2NormKernel); -REGISTER_OP_GPU_KERNEL( + ops::SquaredL2NormKernel); +REGISTER_OP_CUDA_KERNEL( squared_l2_norm_grad, - ops::SquaredL2NormGradKernel); + ops::SquaredL2NormGradKernel); diff --git a/paddle/operators/squared_l2_norm_op.h b/paddle/operators/squared_l2_norm_op.h index 48d7b1c2d5..0ced7e7d70 100644 --- a/paddle/operators/squared_l2_norm_op.h +++ b/paddle/operators/squared_l2_norm_op.h @@ -20,7 +20,7 @@ namespace paddle { namespace operators { // Out = sum(square(X)) -template +template class SquaredL2NormKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { @@ -30,14 +30,15 @@ class SquaredL2NormKernel : public framework::OpKernel { auto x = framework::EigenVector::Flatten(*X); auto out = framework::EigenScalar::From(*Out); - auto place = context.GetEigenDevice(); + auto *place = + context.template device_context().eigen_device(); - out.device(place) = x.square().sum(); + out.device(*place) = x.square().sum(); } }; // dX = X -template +template class SquaredL2NormGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { @@ -53,10 +54,11 @@ class SquaredL2NormGradKernel : public framework::OpKernel { auto x = framework::EigenVector::Flatten(*X); auto dout = framework::EigenVector::Flatten(*dOut); auto dx = framework::EigenVector::Flatten(*dX); - auto place = context.GetEigenDevice(); + auto *place = + context.template device_context().eigen_device(); Eigen::DSizes x_dsize(X->numel()); - dx.device(place) = (dout.broadcast(x_dsize) * x) * static_cast(2.0); + dx.device(*place) = (dout.broadcast(x_dsize) * x) * static_cast(2.0); } }; diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index 744b2fe3f2..cd52672f78 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -195,7 +195,8 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(sum, ops::SumOp, ops::SumOpMaker, ops::SumGradMaker, ops::SumOpVarTypeInference); -REGISTER_OP_CPU_KERNEL(sum, ops::SumKernel, - ops::SumKernel, - ops::SumKernel, - ops::SumKernel); +REGISTER_OP_CPU_KERNEL( + sum, ops::SumKernel, + ops::SumKernel, + ops::SumKernel, + ops::SumKernel); diff --git a/paddle/operators/sum_op.cu b/paddle/operators/sum_op.cu index 5c30dd4d47..873155076c 100644 --- a/paddle/operators/sum_op.cu +++ b/paddle/operators/sum_op.cu @@ -13,7 +13,8 @@ limitations under the License. */ #include "paddle/operators/sum_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(sum, ops::SumKernel, - ops::SumKernel, - ops::SumKernel, - ops::SumKernel); +REGISTER_OP_CUDA_KERNEL( + sum, ops::SumKernel, + ops::SumKernel, + ops::SumKernel, + ops::SumKernel); diff --git a/paddle/operators/sum_op.h b/paddle/operators/sum_op.h index ed6c80ce60..eaa36aa1ae 100644 --- a/paddle/operators/sum_op.h +++ b/paddle/operators/sum_op.h @@ -26,7 +26,7 @@ template using EigenVector = framework::EigenVector; -template +template class SumKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &context) const override { @@ -43,12 +43,14 @@ class SumKernel : public framework::OpKernel { auto result = EigenVector::Flatten(*out); if (!in_place) { - math::SetConstant constant_functor; - constant_functor(context.device_context(), out, 0.0); + math::SetConstant constant_functor; + constant_functor(context.template device_context(), out, + 0.0); } - math::SelectedRowsAddToTensor functor; - auto place = context.GetEigenDevice(); + math::SelectedRowsAddToTensor functor; + auto &place = + *context.template device_context().eigen_device(); // If in_place, just skip the first tensor for (int i = in_place ? 1 : 0; i < N; i++) { if (in_vars[i]->IsType()) { @@ -60,7 +62,7 @@ class SumKernel : public framework::OpKernel { result.device(place) = result + in; } else if (in_vars[i]->IsType()) { auto &in_t = in_vars[i]->Get(); - functor(context.device_context(), in_t, out); + functor(context.template device_context(), in_t, out); } else { PADDLE_THROW("Variable type must be LoDTensor/SelectedRows."); } @@ -82,14 +84,14 @@ class SumKernel : public framework::OpKernel { out_value->Resize(framework::make_ddim(in_dim_vec)); out_value->mutable_data(context.GetPlace()); - math::SelectedRowsAddTo functor; + math::SelectedRowsAddTo functor; int64_t offset = 0; for (int i = 0; i < N; i++) { PADDLE_ENFORCE_EQ(out->height(), in_vars[i]->Get().height()); - functor(context.device_context(), in_vars[i]->Get(), - offset, out); + functor(context.template device_context(), + in_vars[i]->Get(), offset, out); offset += in_vars[i]->Get().value().numel(); } } else if (out_var->IsType()) { @@ -112,7 +114,8 @@ class SumKernel : public framework::OpKernel { PADDLE_ENFORCE(out_array[i].lod() == in_array[i].lod()); auto in = EigenVector::Flatten(in_array[i]); auto result = EigenVector::Flatten(out_array[i]); - result.device(context.GetEigenDevice()) = result + in; + result.device(*context.template device_context() + .eigen_device()) = result + in; } } } diff --git a/paddle/operators/top_k_op.cu b/paddle/operators/top_k_op.cu index 7851c71bbe..453bd07267 100644 --- a/paddle/operators/top_k_op.cu +++ b/paddle/operators/top_k_op.cu @@ -317,4 +317,4 @@ class TopkOpCUDAKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -REGISTER_OP_GPU_KERNEL(top_k, paddle::operators::TopkOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(top_k, paddle::operators::TopkOpCUDAKernel); diff --git a/paddle/operators/top_k_op.h b/paddle/operators/top_k_op.h index bc8563717a..e9cd9bbd4d 100644 --- a/paddle/operators/top_k_op.h +++ b/paddle/operators/top_k_op.h @@ -27,7 +27,7 @@ template using EigenMatrix = framework::EigenMatrix; -template +template class TopkKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { diff --git a/paddle/operators/transpose_op.cc b/paddle/operators/transpose_op.cc index 94de3d5069..de5ff561ad 100644 --- a/paddle/operators/transpose_op.cc +++ b/paddle/operators/transpose_op.cc @@ -112,8 +112,8 @@ class TransposeOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(transpose, ops::TransposeOp, ops::TransposeOpMaker, transpose_grad, ops::TransposeOpGrad); -REGISTER_OP_CPU_KERNEL(transpose, - ops::TransposeKernel); +REGISTER_OP_CPU_KERNEL( + transpose, ops::TransposeKernel); REGISTER_OP_CPU_KERNEL( transpose_grad, - ops::TransposeGradKernel); + ops::TransposeGradKernel); diff --git a/paddle/operators/transpose_op.cu.cc b/paddle/operators/transpose_op.cu.cc index af3f581462..7d23f1493e 100644 --- a/paddle/operators/transpose_op.cu.cc +++ b/paddle/operators/transpose_op.cu.cc @@ -15,8 +15,9 @@ #include "paddle/operators/transpose_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(transpose, - ops::TransposeKernel); -REGISTER_OP_GPU_KERNEL( +REGISTER_OP_CUDA_KERNEL( + transpose, + ops::TransposeKernel); +REGISTER_OP_CUDA_KERNEL( transpose_grad, - ops::TransposeGradKernel); + ops::TransposeGradKernel); diff --git a/paddle/operators/transpose_op.h b/paddle/operators/transpose_op.h index e296032f41..d995271a6b 100644 --- a/paddle/operators/transpose_op.h +++ b/paddle/operators/transpose_op.h @@ -20,33 +20,33 @@ namespace paddle { namespace operators { -template -inline void TransCompute(const int dim, const platform::DeviceContext& dev_ctx, +template +inline void TransCompute(const int dim, const DeviceContext& dev_ctx, const framework::Tensor& in, framework::Tensor* out, const std::vector& axis) { switch (dim) { case 1: - math::Transpose trans1; + math::Transpose trans1; trans1(dev_ctx, in, out, axis); break; case 2: - math::Transpose trans2; + math::Transpose trans2; trans2(dev_ctx, in, out, axis); break; case 3: - math::Transpose trans3; + math::Transpose trans3; trans3(dev_ctx, in, out, axis); break; case 4: - math::Transpose trans4; + math::Transpose trans4; trans4(dev_ctx, in, out, axis); break; case 5: - math::Transpose trans5; + math::Transpose trans5; trans5(dev_ctx, in, out, axis); break; case 6: - math::Transpose trans6; + math::Transpose trans6; trans6(dev_ctx, in, out, axis); break; default: @@ -54,7 +54,7 @@ inline void TransCompute(const int dim, const platform::DeviceContext& dev_ctx, } } -template +template class TransposeKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -64,12 +64,12 @@ class TransposeKernel : public framework::OpKernel { std::vector axis = context.Attr>("axis"); int ndims = axis.size(); - auto& dev_ctx = context.device_context(); - TransCompute(ndims, dev_ctx, *x, out, axis); + auto& dev_ctx = context.template device_context(); + TransCompute(ndims, dev_ctx, *x, out, axis); } }; -template +template class TransposeGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -88,8 +88,9 @@ class TransposeGradKernel : public framework::OpKernel { } int ndims = axis.size(); - auto& dev_ctx = context.device_context(); - TransCompute(ndims, dev_ctx, *out_grad, x_grad, reversed_axis); + auto& dev_ctx = context.template device_context(); + TransCompute(ndims, dev_ctx, *out_grad, x_grad, + reversed_axis); } }; diff --git a/paddle/operators/uniform_random_op.cc b/paddle/operators/uniform_random_op.cc index fff1dc7ccd..2a49ee471f 100644 --- a/paddle/operators/uniform_random_op.cc +++ b/paddle/operators/uniform_random_op.cc @@ -67,7 +67,7 @@ class UniformRandomOp : public framework::OperatorWithKernel { const framework::ExecutionContext& ctx) const override { return framework::OpKernelType( static_cast(ctx.Attr("dtype")), - ctx.device_context()); + ctx.GetPlace()); } }; diff --git a/paddle/operators/uniform_random_op.cu b/paddle/operators/uniform_random_op.cu index 8b20bb8287..cfe9d293cf 100644 --- a/paddle/operators/uniform_random_op.cu +++ b/paddle/operators/uniform_random_op.cu @@ -63,6 +63,6 @@ class GPUUniformRandomKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -REGISTER_OP_GPU_KERNEL(uniform_random, - paddle::operators::GPUUniformRandomKernel, - paddle::operators::GPUUniformRandomKernel); +REGISTER_OP_CUDA_KERNEL(uniform_random, + paddle::operators::GPUUniformRandomKernel, + paddle::operators::GPUUniformRandomKernel); diff --git a/paddle/operators/unpool_op.cc b/paddle/operators/unpool_op.cc index 89c48e071c..49df2a530c 100644 --- a/paddle/operators/unpool_op.cc +++ b/paddle/operators/unpool_op.cc @@ -135,9 +135,10 @@ class UnpoolOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(unpool, ops::UnpoolOp, ops::Unpool2dOpMaker, unpool_grad, ops::UnpoolOpGrad); -REGISTER_OP_CPU_KERNEL(unpool, - ops::UnpoolKernel, - ops::UnpoolKernel); REGISTER_OP_CPU_KERNEL( - unpool_grad, ops::UnpoolGradKernel, - ops::UnpoolGradKernel); + unpool, ops::UnpoolKernel, + ops::UnpoolKernel); +REGISTER_OP_CPU_KERNEL( + unpool_grad, + ops::UnpoolGradKernel, + ops::UnpoolGradKernel); diff --git a/paddle/operators/unpool_op.cu.cc b/paddle/operators/unpool_op.cu.cc index 18aafb7dc7..9b002e35c4 100644 --- a/paddle/operators/unpool_op.cu.cc +++ b/paddle/operators/unpool_op.cu.cc @@ -15,9 +15,10 @@ limitations under the License. */ #include "paddle/operators/unpool_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(unpool, - ops::UnpoolKernel, - ops::UnpoolKernel); -REGISTER_OP_GPU_KERNEL( - unpool_grad, ops::UnpoolGradKernel, - ops::UnpoolGradKernel); +REGISTER_OP_CUDA_KERNEL( + unpool, ops::UnpoolKernel, + ops::UnpoolKernel); +REGISTER_OP_CUDA_KERNEL( + unpool_grad, + ops::UnpoolGradKernel, + ops::UnpoolGradKernel); diff --git a/paddle/operators/unpool_op.h b/paddle/operators/unpool_op.h index 243eb7e532..ee18b118c9 100644 --- a/paddle/operators/unpool_op.h +++ b/paddle/operators/unpool_op.h @@ -20,7 +20,7 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class UnpoolKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -32,15 +32,16 @@ class UnpoolKernel : public framework::OpKernel { std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); T* output_data = out->mutable_data(context.GetPlace()); + auto& dev_ctx = context.template device_context(); if (output_data) { - math::SetConstant set_zero; - set_zero(context.device_context(), out, static_cast(0)); + math::SetConstant set_zero; + set_zero(dev_ctx, out, static_cast(0)); } - math::Unpool2dMaxFunctor unpool2d_max_forward; - unpool2d_max_forward(context.device_context(), *in_x, *in_y, out); + math::Unpool2dMaxFunctor unpool2d_max_forward; + unpool2d_max_forward(dev_ctx, *in_x, *in_y, out); } }; -template +template class UnpoolGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -56,15 +57,14 @@ class UnpoolGradKernel : public framework::OpKernel { std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - auto& device_ctx = context.device_context(); - math::SetConstant zero; + auto& device_ctx = context.template device_context(); + math::SetConstant zero; if (in_x_grad) { in_x_grad->mutable_data(context.GetPlace()); zero(device_ctx, in_x_grad, static_cast(0)); } - math::Unpool2dMaxGradFunctor unpool2d_max_backward; - unpool2d_max_backward(context.device_context(), *in_x, *in_y, *out, - *out_grad, in_x_grad); + math::Unpool2dMaxGradFunctor unpool2d_max_backward; + unpool2d_max_backward(device_ctx, *in_x, *in_y, *out, *out_grad, in_x_grad); } }; } // namespace operators diff --git a/paddle/platform/device_context.cc b/paddle/platform/device_context.cc index ae4f0bf896..2c7f964216 100644 --- a/paddle/platform/device_context.cc +++ b/paddle/platform/device_context.cc @@ -15,12 +15,6 @@ limitations under the License. */ namespace paddle { namespace platform { -template <> -Eigen::DefaultDevice* DeviceContext::GetEigenDevice< - platform::CPUPlace, Eigen::DefaultDevice>() const { - return reinterpret_cast(this)->eigen_device(); -} - CPUDeviceContext::CPUDeviceContext() { eigen_device_.reset(new Eigen::DefaultDevice()); } @@ -37,12 +31,6 @@ Place CPUDeviceContext::GetPlace() const { return CPUPlace(); } #ifdef PADDLE_WITH_CUDA -template <> -Eigen::GpuDevice* -DeviceContext::GetEigenDevice() const { - return reinterpret_cast(this)->eigen_device(); -} - class EigenCudaStreamDevice : public Eigen::StreamInterface { public: EigenCudaStreamDevice() : scratch_(nullptr), semaphore_(nullptr) { diff --git a/paddle/platform/device_context.h b/paddle/platform/device_context.h index ef5f19214d..596d9d0bba 100644 --- a/paddle/platform/device_context.h +++ b/paddle/platform/device_context.h @@ -27,24 +27,11 @@ limitations under the License. */ namespace paddle { namespace platform { -template -struct EigenDeviceConverter; - -template <> -struct EigenDeviceConverter { - using EigenDeviceType = Eigen::DefaultDevice; -}; - class DeviceContext { public: virtual ~DeviceContext() {} virtual Place GetPlace() const = 0; - template ::EigenDeviceType> - DeviceType* GetEigenDevice() const; - virtual void Wait() const {} }; @@ -62,10 +49,6 @@ class CPUDeviceContext : public DeviceContext { }; #ifdef PADDLE_WITH_CUDA -template <> -struct EigenDeviceConverter { - using EigenDeviceType = Eigen::GpuDevice; -}; class EigenCudaStreamDevice; diff --git a/paddle/platform/device_context_test.cc b/paddle/platform/device_context_test.cc index 8bf5174c4a..4893cd92f6 100644 --- a/paddle/platform/device_context_test.cc +++ b/paddle/platform/device_context_test.cc @@ -22,9 +22,8 @@ TEST(Device, Init) { int count = paddle::platform::GetCUDADeviceCount(); for (int i = 0; i < count; i++) { - DeviceContext* device_context = new CUDADeviceContext(GPUPlace(i)); - Eigen::GpuDevice* gpu_device = - device_context->template GetEigenDevice(); + CUDADeviceContext* device_context = new CUDADeviceContext(GPUPlace(i)); + Eigen::GpuDevice* gpu_device = device_context->eigen_device(); ASSERT_NE(nullptr, gpu_device); delete device_context; } diff --git a/paddle/platform/transform.h b/paddle/platform/transform.h index bb9d59ec0a..148ebaed3d 100644 --- a/paddle/platform/transform.h +++ b/paddle/platform/transform.h @@ -31,7 +31,7 @@ namespace paddle { namespace platform { // Transform on host or device. It provides the same API in std library. -template +template struct Transform { template void operator()(const DeviceContext& context, InputIter first, InputIter last, @@ -45,16 +45,16 @@ struct Transform { }; template <> -struct Transform { +struct Transform { template - void operator()(const DeviceContext& context, InputIter first, InputIter last, - OutputIter result, UnaryOperation op) { + void operator()(const platform::CPUDeviceContext& context, InputIter first, + InputIter last, OutputIter result, UnaryOperation op) { std::transform(first, last, result, op); } template - void operator()(const DeviceContext& context, InputIter1 first1, + void operator()(const platform::CPUDeviceContext& context, InputIter1 first1, InputIter1 last1, InputIter2 first2, OutputIter result, BinaryOperation op) { std::transform(first1, last1, first2, result, op); @@ -63,27 +63,25 @@ struct Transform { #ifdef __NVCC__ template <> -struct Transform { +struct Transform { template - void operator()(const DeviceContext& context, InputIter first, InputIter last, - OutputIter result, UnaryOperation op) { + void operator()(const platform::CUDADeviceContext& context, InputIter first, + InputIter last, OutputIter result, UnaryOperation op) { auto place = context.GetPlace(); PADDLE_ENFORCE(is_gpu_place(place), "It must use GPU place."); - auto& ctx = reinterpret_cast(context); - thrust::transform(thrust::cuda::par.on(ctx.stream()), + thrust::transform(thrust::cuda::par.on(context.stream()), details::DevPtrCast(first), details::DevPtrCast(last), details::DevPtrCast(result), op); } template - void operator()(const DeviceContext& context, InputIter1 first1, + void operator()(const platform::CUDADeviceContext& context, InputIter1 first1, InputIter1 last1, InputIter2 first2, OutputIter result, BinaryOperation op) { auto place = context.GetPlace(); PADDLE_ENFORCE(is_gpu_place(place), "It must use GPU place."); - auto& ctx = reinterpret_cast(context); - thrust::transform(thrust::cuda::par.on(ctx.stream()), + thrust::transform(thrust::cuda::par.on(context.stream()), details::DevPtrCast(first1), details::DevPtrCast(last1), details::DevPtrCast(first2), details::DevPtrCast(result), op); diff --git a/paddle/platform/transform_test.cu b/paddle/platform/transform_test.cu index c76cab80e4..d36eac8379 100644 --- a/paddle/platform/transform_test.cu +++ b/paddle/platform/transform_test.cu @@ -39,7 +39,7 @@ TEST(Transform, CPUUnary) { using namespace paddle::platform; CPUDeviceContext ctx; float buf[4] = {0.1, 0.2, 0.3, 0.4}; - Transform trans; + Transform trans; trans(ctx, buf, buf + 4, buf, Scale(10)); for (int i = 0; i < 4; ++i) { ASSERT_NEAR(buf[i], static_cast(i + 1), 1e-5); @@ -54,7 +54,7 @@ TEST(Transform, GPUUnary) { float cpu_buf[4] = {0.1, 0.2, 0.3, 0.4}; float* gpu_buf = static_cast(Alloc(gpu0, sizeof(float) * 4)); Copy(gpu0, gpu_buf, CPUPlace(), cpu_buf, sizeof(cpu_buf)); - Transform trans; + Transform trans; trans(ctx, gpu_buf, gpu_buf + 4, gpu_buf, Scale(10)); ctx.Wait(); Copy(CPUPlace(), cpu_buf, gpu0, gpu_buf, sizeof(cpu_buf)); @@ -68,7 +68,7 @@ TEST(Transform, CPUBinary) { using namespace paddle::platform; using namespace paddle::memory; int buf[4] = {1, 2, 3, 4}; - Transform trans; + Transform trans; CPUDeviceContext ctx; trans(ctx, buf, buf + 4, buf, buf, Multiply()); for (int i = 0; i < 4; ++i) { @@ -84,7 +84,7 @@ TEST(Transform, GPUBinary) { CUDADeviceContext ctx(gpu0); int* gpu_buf = static_cast(Alloc(gpu0, sizeof(buf))); Copy(gpu0, gpu_buf, CPUPlace(), buf, sizeof(buf)); - Transform trans; + Transform trans; trans(ctx, gpu_buf, gpu_buf + 4, gpu_buf, gpu_buf, Multiply()); ctx.Wait(); Copy(CPUPlace(), buf, gpu0, gpu_buf, sizeof(buf)); From de8c4627776b2b9a60dbcc64c48e858c80a2715f Mon Sep 17 00:00:00 2001 From: QI JUN Date: Tue, 12 Dec 2017 14:42:33 +0800 Subject: [PATCH 32/94] Update new op docs (#6505) * update docs about how to add a new operator --- doc/howto/dev/new_op_cn.md | 98 ++++++++++++++++---------------------- doc/howto/dev/new_op_en.md | 93 ++++++++++++++++-------------------- 2 files changed, 80 insertions(+), 111 deletions(-) diff --git a/doc/howto/dev/new_op_cn.md b/doc/howto/dev/new_op_cn.md index 6cfc9536f2..44dbeecbbd 100644 --- a/doc/howto/dev/new_op_cn.md +++ b/doc/howto/dev/new_op_cn.md @@ -30,8 +30,8 @@ -------------- | :---------------------- OpProtoMake定义 | `.cc`文件,Backward Op不需要定义OpProtoMake Op定义 | `.cc`文件 -Kernel实现 | CPU、GPU共享Kernel实现在`.h`文件中,否则,CPU 实现在`.cc`文件中,GPU 实现在`.cu`文件中。 -注册Op | Op注册实现在`.cc`文件;Kernel注册CPU实现在`.cc`文件中,GPU实现在`.cu`文件中 +Kernel实现 | CPU、CUDA共享Kernel实现在`.h`文件中,否则,CPU 实现在`.cc`文件中,CUDA 实现在`.cu`文件中。 +注册Op | Op注册实现在`.cc`文件;Kernel注册CPU实现在`.cc`文件中,CUDA实现在`.cu`文件中 实现新的op都添加至目录[paddle/operators](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators)下,文件命名以`*_op.h`(如有) 、 `*_op.cc` 、`*_op.cu`(如有)结尾。**系统会根据文件名自动构建op和其对应的Python扩展。** @@ -153,7 +153,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, `MulKernel`继承自`framework::OpKernel`,带有下面两个模板参数: -- `typename Place`: 表示设备类型,不同设备(CPU、GPU)共享同一个Kernel时,需加该模板参数,不共享则不加,一个不共享的例子是[`OnehotCrossEntropyOpKernel`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43)。 +- `typename DeviceContext`: 表示设备类型,不同设备(CPU、CUDA)共享同一个Kernel时,需加该模板参数,不共享则不加,一个不共享的例子是[`OnehotCrossEntropyOpKernel`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43)。 - `typename T` : 表示数据类型,如`float`, `double`等。 @@ -165,7 +165,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, 下面是 `MulKernel` `Compute`的实现: ```cpp - template + template class MulKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -173,18 +173,16 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, auto* Y = context.Input("Y"); auto* Z = context.Output("Out"); Z->mutable_data(context.GetPlace()); - auto* device_context = - const_cast(context.device_context_); - math::matmul(*X, false, *Y, false, 1, Z, 0, device_context); + auto& device_context = context.template device_context(); + math::matmul(*X, false, *Y, false, 1, Z, 0, device_context); } }; - ``` -需要注意:**不同设备(CPU、GPU)共享一个Op定义,是否则共享同一个`OpKernel`,取决于`Compute`调用的函数是否支持不同设备。** +需要注意:**不同设备(CPU、CUDA)共享一个Op定义,是否则共享同一个`OpKernel`,取决于`Compute`调用的函数是否支持不同设备。** -`MulOp`的CPU、GPU实现共享同一个`Kernel`。`OpKernel`不共享的例子可以参考:[`OnehotCrossEntropyOpKernel`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43)。 +`MulOp`的CPU、CUDA实现共享同一个`Kernel`。`OpKernel`不共享的例子可以参考:[`OnehotCrossEntropyOpKernel`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43)。 -为了使`OpKernel`的计算过程书写更加简单,并且CPU、GPU的代码可以复用,我们通常借助 Eigen unsupported Tensor模块来实现`Compute`接口。关于在PaddlePaddle中如何使用Eigen库,请参考[使用文档](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/dev/use_eigen_cn.md)。 +为了使`OpKernel`的计算过程书写更加简单,并且CPU、CUDA的代码可以复用,我们通常借助 Eigen unsupported Tensor模块来实现`Compute`接口。关于在PaddlePaddle中如何使用Eigen库,请参考[使用文档](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/dev/use_eigen_cn.md)。 到此,前向Op实现完成。接下来,需要在`.cc`文件中注册该op和kernel。 @@ -197,9 +195,9 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, ```cpp namespace ops = paddle::operators; REGISTER_OP(mul, ops::MulOp, ops::MulOpMaker, mul_grad, ops::MulOpGrad); - REGISTER_OP_CPU_KERNEL(mul, ops::MulKernel); + REGISTER_OP_CPU_KERNEL(mul, ops::MulKernel); REGISTER_OP_CPU_KERNEL(mul_grad, - ops::MulGradKernel); + ops::MulGradKernel); ``` 在上面的代码中: @@ -209,17 +207,17 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, - `REGISTER_OP_CPU_KERNEL` :注册`ops::MulKernel`类,并特化模板参数为`paddle::platform::CPUPlace`和`float`类型,同理,注册`ops::MulGradKernel`类。 -- 在 `.cu`文件中注册GPU Kernel。 - - 请注意,如果GPU Kernel的实现基于Eigen unsupported模块,那么在 `.cu`的开始请加上宏定义 `#define EIGEN_USE_GPU`,代码示例如下: +- 在 `.cu`文件中注册CUDA Kernel。 + - 请注意,如果CUDA Kernel的实现基于Eigen unsupported模块,那么在 `.cu`的开始请加上宏定义 `#define EIGEN_USE_GPU`,代码示例如下: ```cpp // if use Eigen unsupported module before include head files - // #define EIGEN_USE_GPU + #define EIGEN_USE_GPU namespace ops = paddle::operators; - REGISTER_OP_GPU_KERNEL(mul, ops::MulKernel); - REGISTER_OP_GPU_KERNEL(mul_grad, - ops::MulGradKernel); + REGISTER_OP_CUDA_KERNEL(mul, ops::MulKernel); + REGISTER_OP_CUDA_KERNEL(mul_grad, + ops::MulGradKernel); ``` ### 5. 编译 @@ -236,71 +234,55 @@ make mul_op ## 实现单元测试 -单测包括对比前向Op不同设备(CPU、GPU)的实现、对比反向OP不同设备(CPU、GPU)的实现、反向Op的梯度测试。下面介绍介绍[`MulOp`的单元测试](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/framework/tests/test_mul_op.py)。 +单测包括对比前向Op不同设备(CPU、CUDA)的实现、对比反向OP不同设备(CPU、CUDA)的实现、反向Op的梯度测试。下面介绍介绍[`MulOp`的单元测试](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/framework/tests/test_mul_op.py)。 -### 前向Operator单元测试 -前向Op单元测试继承自`unittest.TestCase`,并定义元类`__metaclass__ = OpTestMeta`。各项更加具体的单元测试在`OpTestMeta`里完成。测试前向Operator,需要: +Op单元测试继承自`OpTest`。各项更加具体的单元测试在`TestMulOp`里完成。测试Operator,需要: 1. 在`setUp`函数定义输入、输出,以及相关的属性参数。 2. 生成随机的输入数据。 3. 在Python脚本中实现与前向operator相同的计算逻辑,得到输出值,与operator前向计算的输出进行对比。 +4. 反向计算已经自动集成进测试框架,直接调用相应接口即可。 ```python import unittest import numpy as np - from gradient_checker import GradientChecker, create_op - from op_test_util import OpTestMeta + from op_test import OpTest - class TestMulOp(unittest.TestCase): - __metaclass__ = OpTestMeta + class TestMulOp(OpTest): def setUp(self): - self.type = "mul" + self.op_type = "mul" self.inputs = { 'X': np.random.random((32, 84)).astype("float32"), 'Y': np.random.random((84, 100)).astype("float32") } self.outputs = {'Out': np.dot(self.inputs['X'], self.inputs['Y'])} - ``` -上面的代码首先导入依赖的包,下面是对`setUp`函数中操作的重要变量的详细解释: - -- `self.type = "mul" ` : 定义类型,与operator注册时注册的类型一致。 -- `self.inputs` : 定义输入,类型为`numpy.array`,并初始化。 -- `self.outputs` : 定义输出,并在Python脚本中完成与operator同样的计算逻辑,返回Python端的计算结果。 - - -### 反向Operator单元测试 + def test_check_output(self): + self.check_output() -反向Op单元测试继承自`GradientChecker`,而`GradientChecker`继承自`unittest.TestCase`,因此,**反向单元测试函数需要以`test_`开头**。 + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.5) -```python -class TestMulGradOp(GradientChecker): - def setUp(self): - self.op = create_op("mul") - self.inputs = { - 'X': np.random.random((32, 84)).astype("float32"), - 'Y': np.random.random((84, 100)).astype("float32") - } + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], 'Out', max_relative_error=0.5, no_grad_set=set("X")) - def test_check_grad_normal(self): - # mul op will enlarge the relative error - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.5) + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y')) - def test_check_grad_ingore_x(self): - self.check_grad( - ['Y'], 'Out', max_relative_error=0.5, no_grad_set=set("X")) + ``` - def test_check_grad_ingore_y(self): - self.check_grad( - ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y')) -``` +上面的代码首先导入依赖的包,下面是对`setUp`函数中操作的重要变量的详细解释: -下面解释代码中一些关键的地方: +- `self.op_type = "mul" ` : 定义类型,与operator注册时注册的类型一致。 +- `self.inputs` : 定义输入,类型为`numpy.array`,并初始化。 +- `self.outputs` : 定义输出,并在Python脚本中完成与operator同样的计算逻辑,返回Python端的计算结果。 -- 调用`create_op("mul")`创建反向Op对应的前向Op。 +而反向测试中: - `test_check_grad_normal`中调用`check_grad`使用数值法检测梯度正确性和稳定性。 - 第一个参数`["X", "Y"]` : 指定对输入变量`X`、`Y`做梯度检测。 - 第二个参数`"Out"` : 指定前向网络最终的输出目标变量`Out`。 @@ -328,5 +310,5 @@ ctest -R test_mul_op - 为每个Op创建单独的`*_op.h`(如有)、`*_op.cc`和`*_op.cu`(如有)。不允许一个文件中包含多个Op,这将会导致编译出错。 - 注册Op时的类型名,需要和该Op的名字一样。即不允许在`A_op.cc`里面,注册`REGISTER_OP(B, ...)`等,这将会导致单元测试出错。 -- 如果Op没有实现GPU Kernel,请不要创建空的`*_op.cu`,这将会导致单元测试出错。 +- 如果Op没有实现CUDA Kernel,请不要创建空的`*_op.cu`,这将会导致单元测试出错。 - 如果多个Op依赖一些共用的函数,可以创建非`*_op.*`格式的文件来存放,如`gather.h`文件。 diff --git a/doc/howto/dev/new_op_en.md b/doc/howto/dev/new_op_en.md index 1e88e1f5b4..510233306c 100644 --- a/doc/howto/dev/new_op_en.md +++ b/doc/howto/dev/new_op_en.md @@ -28,8 +28,8 @@ An operator can be differentiated by whether in has kernel methods. An operator -------------- | :---------------------- OpProtoMake definition | `.cc`files, Backward Op does not need an OpProtoMake interface. Op definition | `.cc` files -Kernel implementation | The kernel methods shared between CPU and GPU are defined in `.h` files. CPU-specific kernels live in `.cc` files, while GPU-specific kernels are implemented in `.cu`files. -Registering the Op | Ops are registered in `.cc` files; For Kernel registration, `.cc` files contain the CPU implementation, while `.cu` files contain the GPU implementation. +Kernel implementation | The kernel methods shared between CPU and CUDA are defined in `.h` files. CPU-specific kernels live in `.cc` files, while CUDA-specific kernels are implemented in `.cu`files. +Registering the Op | Ops are registered in `.cc` files; For Kernel registration, `.cc` files contain the CPU implementation, while `.cu` files contain the CUDA implementation. New Operator implementations are added to the list [paddle/operators](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators), with file names in the format `*_op.h` (if applicable), `*_op.cc`, `*_op.cu` (if applicable).** The system will use the naming scheme to automatically build operators and their corresponding Python extensions. ** @@ -151,7 +151,7 @@ Usually `OpProtoMaker` and `Op`'s type definitions are written in `.cc` files, w `MulKernel` inherits `framework::OpKernel`, which includes the following templates: -- `typename Place` denotes device type. When different devices, namely the CPU and the GPU, share the same kernel, this template needs to be added. If they don't share kernels, this must not be added. An example of a non-sharing kernel is [`OnehotCrossEntropyOpKernel`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43). +- `typename DeviceContext` denotes device context type. When different devices, namely the CPUDeviceContext and the CUDADeviceContext, share the same kernel, this template needs to be added. If they don't share kernels, this must not be added. An example of a non-sharing kernel is [`OnehotCrossEntropyOpKernel`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43). - `typename T` denotes data type, such as `float` or `double`. @@ -163,7 +163,7 @@ Usually `OpProtoMaker` and `Op`'s type definitions are written in `.cc` files, w `MulKernel`'s implementation of `Compute` is as follows: ```cpp - template + template class MulKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -171,16 +171,15 @@ Usually `OpProtoMaker` and `Op`'s type definitions are written in `.cc` files, w auto* Y = context.Input("Y"); auto* Z = context.Output("Out"); Z->mutable_data(context.GetPlace()); - auto* device_context = - const_cast(context.device_context_); - math::matmul(*X, false, *Y, false, 1, Z, 0, device_context); + auto& device_context = context.template device_context(); + math::matmul(*X, false, *Y, false, 1, Z, 0, device_context); } }; ``` -Note that **different devices (CPU, GPU)share an Op definition; whether or not they share the same `OpKernel` depends on whether `Compute` calls functions that support both devices.** +Note that **different devices (CPU, CUDA)share an Op definition; whether or not they share the same `OpKernel` depends on whether `Compute` calls functions that support both devices.** -`MulOp`'s CPU and GPU share the same `Kernel`. A non-sharing `OpKernel` example can be seen in [`OnehotCrossEntropyOpKernel`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43). +`MulOp`'s CPU and CUDA share the same `Kernel`. A non-sharing `OpKernel` example can be seen in [`OnehotCrossEntropyOpKernel`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43). To ease the writing of `OpKernel` compute, and for reusing code cross-device, [`Eigen-unsupported Tensor`](https://bitbucket.org/eigen/eigen/src/default/unsupported/Eigen/CXX11/src/Tensor/README.md?fileviewer=file-view-default) module is used to implement `Compute` interface. To learn about how the Eigen library is used in PaddlePaddle, please see [usage document](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/dev/use_eigen_cn.md). @@ -196,9 +195,9 @@ The definition of its corresponding backward operator, if applicable, is similar ```cpp namespace ops = paddle::operators; REGISTER_OP(mul, ops::MulOp, ops::MulOpMaker, mul_grad, ops::MulOpGrad); - REGISTER_OP_CPU_KERNEL(mul, ops::MulKernel); + REGISTER_OP_CPU_KERNEL(mul, ops::MulKernel); REGISTER_OP_CPU_KERNEL(mul_grad, - ops::MulGradKernel); + ops::MulGradKernel); ``` In that code block, @@ -208,17 +207,17 @@ The definition of its corresponding backward operator, if applicable, is similar - `REGISTER_OP_CPU_KERNEL` registers `ops::MulKernel` class and specialized template types `paddle::platform::CPUPlace` and `float`, which also registers `ops::MulGradKernel`. -- Registering GPU Kernel in `.cu` files - - Note that if GPU Kernel is implemented using the `Eigen unsupported` module, then on top of `.cu`, a macro definition `#define EIGEN_USE_GPU` is needed, such as +- Registering CUDA Kernel in `.cu` files + - Note that if CUDA Kernel is implemented using the `Eigen unsupported` module, then on top of `.cu`, a macro definition `#define EIGEN_USE_GPU` is needed, such as ```cpp // if use Eigen unsupported module before include head files #define EIGEN_USE_GPU namespace ops = paddle::operators; - REGISTER_OP_GPU_KERNEL(mul, ops::MulKernel); - REGISTER_OP_GPU_KERNEL(mul_grad, - ops::MulGradKernel); + REGISTER_OP_CUDA_KERNEL(mul, ops::MulKernel); + REGISTER_OP_CUDA_KERNEL(mul_grad, + ops::MulGradKernel); ``` ### 5. Compilation @@ -253,62 +252,50 @@ A forward operator unit test inherits `unittest.TestCase` and defines metaclass 2. Generating random input data. -3. Implementing the same computation logic in a Python script: +3. Implementing the same computation logic in a Python script. + +4. Call check gradient function to check the backward operator. ```python import unittest import numpy as np - from gradient_checker import GradientChecker, create_op - from op_test_util import OpTestMeta + from op_test import OpTest - class TestMulOp(unittest.TestCase): - __metaclass__ = OpTestMeta + class TestMulOp(OpTest): def setUp(self): - self.type = "mul" + self.op_type = "mul" self.inputs = { 'X': np.random.random((32, 84)).astype("float32"), 'Y': np.random.random((84, 100)).astype("float32") } self.outputs = {'Out': np.dot(self.inputs['X'], self.inputs['Y'])} - ``` -Get its output, and compare it with the forward operator's own output. - -The code above first loads required packages. In addition, we have - -- `self.type = "mul" ` defines the type that is identical to what the operator's registered type. -- `self.inputs` defines input, with type `numpy.array` and initializes it. -- `self.outputs` defines output and completes the same operator computation in the Python script, and returns its result from the Python script. -### Testing Backward Operators + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.5) -A backward operator unit test inherits `GradientChecker`, which inherits `unittest.TestCase`. As a result, **a backward operator unit test needs to be have the prefix `test_`**. + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], 'Out', max_relative_error=0.5, no_grad_set=set("X")) -```python -class TestMulGradOp(GradientChecker): - def setUp(self): - self.op = create_op("mul") - self.inputs = { - 'X': np.random.random((32, 84)).astype("float32"), - 'Y': np.random.random((84, 100)).astype("float32") - } + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y')) - def test_check_grad_normal(self): - # mul op will enlarge the relative error - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.5) + ``` +Get its output, and compare it with the forward operator's own output. - def test_check_grad_ingore_x(self): - self.check_grad( - ['Y'], 'Out', max_relative_error=0.5, no_grad_set=set("X")) +The code above first loads required packages. In addition, we have - def test_check_grad_ingore_y(self): - self.check_grad( - ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y')) -``` +- `self.op_type = "mul" ` defines the type that is identical to what the operator's registered type. +- `self.inputs` defines input, with type `numpy.array` and initializes it. +- `self.outputs` defines output and completes the same operator computation in the Python script, and returns its result from the Python script. -Some key points in the code above include: +Some key points in checking gradient above include: -- `create_op("mul")` creates the backward operator's corresponding forward operator. - `test_normal` calls `check_grad` to validate scaling tests' correctness and stability through numeric methods. - The first variable `["X", "Y"]` appoints `X` and `Y` to be scale tested. - The second variable `"Out"` points to the network's final output target `Out`. @@ -338,5 +325,5 @@ ctest -R test_mul_op - Every `*_op.h` (if applicable), `*_op.cc`, and `*_op.cu` (if applicable) must be created for a unique Op. Compiling will fail if multiple operators are included per file. - The type with which an operator is registered needs to be identical to the Op's name. Registering `REGISTER_OP(B, ...)` in `A_op.cc` will cause unit testing failures. -- If the operator does not implement a GPU kernel, please refrain from creating an empty `*_op.cu` file, or else unit tests will fail. +- If the operator does not implement a CUDA kernel, please refrain from creating an empty `*_op.cu` file, or else unit tests will fail. - If multiple operators rely on some shared methods, a file NOT named `*_op.*` can be created to store them, such as `gather.h`. From d918ccded35d972eba42b07767972bc4c2dd0921 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 12 Dec 2017 15:22:52 +0800 Subject: [PATCH 33/94] Add fill_op (#6477) * Add fill_op * Fix bug --- paddle/operators/fill_op.cc | 111 +++++++++++++++++++ python/paddle/v2/fluid/tests/test_fill_op.py | 24 ++++ 2 files changed, 135 insertions(+) create mode 100644 paddle/operators/fill_op.cc create mode 100644 python/paddle/v2/fluid/tests/test_fill_op.py diff --git a/paddle/operators/fill_op.cc b/paddle/operators/fill_op.cc new file mode 100644 index 0000000000..382e161c5d --- /dev/null +++ b/paddle/operators/fill_op.cc @@ -0,0 +1,111 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/data_type.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/detail/safe_ref.h" + +namespace paddle { +namespace operators { + +struct FillOpVisitor { + FillOpVisitor(framework::LoDTensor *tensor, const std::vector &value) + : tensor_(tensor), value_(value) {} + + template + void operator()() const { + platform::CPUPlace cpu; + auto *data = tensor_->mutable_data(cpu); + std::transform(value_.data(), value_.data() + tensor_->numel(), data, + [](float dat) { return static_cast(dat); }); + } + + framework::LoDTensor *tensor_; + const std::vector &value_; +}; + +class FillOp : public framework::OperatorBase { + public: + FillOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &out = + detail::Ref(detail::Ref(scope.FindVar(Output("Out")), + "Cannot find variable %s", Output("Out")) + .GetMutable()); + out.Resize(framework::make_ddim(Attr>("shape"))); + auto dtype = static_cast(Attr("dtype")); + platform::CPUPlace cpu; + auto force_cpu = Attr("force_cpu"); + out.mutable_data(force_cpu ? cpu : dev_ctx.GetPlace(), + framework::ToTypeIndex(dtype)); + + framework::LoDTensor tensor; + + if (force_cpu || platform::is_cpu_place(dev_ctx.GetPlace())) { + tensor.ShareDataWith(out); + } else { + // Always make tensor in CPU memory. + tensor.Resize(out.dims()); + tensor.mutable_data(cpu, framework::ToTypeIndex(dtype)); + } + + framework::VisitDataType( + dtype, FillOpVisitor(&tensor, Attr>("value"))); + + if (!force_cpu && platform::is_gpu_place(dev_ctx.GetPlace())) { + // Copy tensor to out + framework::CopyFrom(tensor, dev_ctx.GetPlace(), dev_ctx, &out); + } + } +}; + +class FillOpMaker : public framework::OpProtoAndCheckerMaker { + public: + FillOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddComment(R"DOC(Fill operator + +Fill an tensor with `value` and `shape`. The type of the tensor is specify by +`dtype`. +)DOC"); + AddOutput("Out", "(LoDTensor) The output tensor."); + AddAttr>( + "value", "The float values of tensor, which are flatten in row major"); + AddAttr>("shape", "The shape of output tensor"); + AddAttr("dtype", "The data type of output tensor, Default is float") + .SetDefault(framework::DataType::FP32); + AddAttr("force_cpu", + "Whether the output tensor must be at CPU memory or not. " + "Default is false.") + .SetDefault(false); + } +}; + +class FillOpInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + context->SetOutputDim( + "Out", + framework::make_ddim(context->Attrs().Get>("shape"))); + } +}; + +} // namespace operators +} // namespace paddle +namespace ops = paddle::operators; +REGISTER_OPERATOR(fill, ops::FillOp, ops::FillOpInferShape, ops::FillOpMaker); diff --git a/python/paddle/v2/fluid/tests/test_fill_op.py b/python/paddle/v2/fluid/tests/test_fill_op.py new file mode 100644 index 0000000000..88337598c8 --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_fill_op.py @@ -0,0 +1,24 @@ +import unittest +import numpy as np +from op_test import OpTest +import paddle.v2.fluid.core as core + + +class TestFillOp(OpTest): + def setUp(self): + self.op_type = "fill" + val = np.random.random(size=[100, 200]) + self.inputs = {} + self.attrs = { + 'value': val.flatten().tolist(), + 'shape': [100, 200], + 'dtype': int(core.DataType.FP64) + } + self.outputs = {'Out': val.astype('float64')} + + def test_check_output(self): + self.check_output() + + +if __name__ == '__main__': + unittest.main() From 9a89b041ba12ab8bcf9e36e218486b224eae1d33 Mon Sep 17 00:00:00 2001 From: guosheng Date: Mon, 11 Dec 2017 23:23:46 +0800 Subject: [PATCH 34/94] Add ChunkEvaluator for multi-batches --- paddle/operators/chunk_eval_op.cc | 18 +++++ paddle/operators/chunk_eval_op.h | 42 +++++++---- python/paddle/v2/fluid/evaluator.py | 73 ++++++++++++++++++- python/paddle/v2/fluid/layers.py | 14 +++- .../tests/book/test_label_semantic_roles.py | 12 +-- .../v2/fluid/tests/test_chunk_eval_op.py | 8 +- 6 files changed, 143 insertions(+), 24 deletions(-) diff --git a/paddle/operators/chunk_eval_op.cc b/paddle/operators/chunk_eval_op.cc index 94127ab33e..ff2a08ccac 100644 --- a/paddle/operators/chunk_eval_op.cc +++ b/paddle/operators/chunk_eval_op.cc @@ -32,6 +32,13 @@ class ChunkEvalOp : public framework::OperatorWithKernel { "Output(Recall) of ChunkEvalOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("F1-Score"), "Output(F1-Score) of ChunkEvalOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("NumInferChunks"), + "Output(NumInferChunks) of ChunkEvalOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("NumLabelChunks"), + "Output(NumLabelChunks) of ChunkEvalOp should not be null."); + PADDLE_ENFORCE( + ctx->HasOutput("NumCorrectChunks"), + "Output(NumCorrectChunks) of ChunkEvalOp should not be null."); auto inference_dim = ctx->GetInputDim("Inference"); auto label_dim = ctx->GetInputDim("Label"); @@ -42,6 +49,9 @@ class ChunkEvalOp : public framework::OperatorWithKernel { ctx->SetOutputDim("Precision", {1}); ctx->SetOutputDim("Recall", {1}); ctx->SetOutputDim("F1-Score", {1}); + ctx->SetOutputDim("NumInferChunks", {1}); + ctx->SetOutputDim("NumLabelChunks", {1}); + ctx->SetOutputDim("NumCorrectChunks", {1}); } protected: @@ -70,6 +80,14 @@ class ChunkEvalOpMaker : public framework::OpProtoAndCheckerMaker { "sensitivity) of chunks on the given mini-batch."); AddOutput("F1-Score", "(float). The evaluated F1-Score on the given mini-batch."); + AddOutput( + "NumInferChunks", + "(int). The number of chunks in Inference on the given mini-batch."); + AddOutput("NumLabelChunks", + "(int). The number of chunks in Label on the given mini-batch."); + AddOutput("NumCorrectChunks", + "(int). The number of chunks both in Inference and Label on the " + "given mini-batch."); AddAttr("num_chunk_types", "(int). The number of chunk type. See below for details."); AddAttr( diff --git a/paddle/operators/chunk_eval_op.h b/paddle/operators/chunk_eval_op.h index dd88f2553b..5d703333c7 100644 --- a/paddle/operators/chunk_eval_op.h +++ b/paddle/operators/chunk_eval_op.h @@ -111,9 +111,7 @@ class ChunkEvalKernel : public framework::OpKernel { std::vector label_segments; std::vector output_segments; std::set excluded_chunk_types; - int64_t num_output_segments = 0; - int64_t num_label_segments = 0; - int64_t num_correct = 0; + if (context.Attr("chunk_scheme") == "IOB") { num_tag_types = 2; tag_begin = 0; @@ -151,12 +149,24 @@ class ChunkEvalKernel : public framework::OpKernel { auto* precision = context.Output("Precision"); auto* recall = context.Output("Recall"); auto* f1 = context.Output("F1-Score"); + auto* num_infer_chunks = context.Output("NumInferChunks"); + auto* num_label_chunks = context.Output("NumLabelChunks"); + auto* num_correct_chunks = context.Output("NumCorrectChunks"); const int64_t* inference_data = inference->data(); const int64_t* label_data = label->data(); T* precision_data = precision->mutable_data(context.GetPlace()); T* racall_data = recall->mutable_data(context.GetPlace()); T* f1_data = f1->mutable_data(context.GetPlace()); + int64_t* num_infer_chunks_data = + num_infer_chunks->mutable_data(context.GetPlace()); + int64_t* num_label_chunks_data = + num_label_chunks->mutable_data(context.GetPlace()); + int64_t* num_correct_chunks_data = + num_correct_chunks->mutable_data(context.GetPlace()); + *num_infer_chunks_data = 0; + *num_label_chunks_data = 0; + *num_correct_chunks_data = 0; auto lod = label->lod(); PADDLE_ENFORCE_EQ(lod.size(), 1UL, "Only support one level sequence now."); @@ -166,17 +176,23 @@ class ChunkEvalKernel : public framework::OpKernel { for (int i = 0; i < num_sequences; ++i) { int seq_length = lod[0][i + 1] - lod[0][i]; EvalOneSeq(inference_data + lod[0][i], label_data + lod[0][i], seq_length, - output_segments, label_segments, num_output_segments, - num_label_segments, num_correct, num_chunk_types, - num_tag_types, other_chunk_type, tag_begin, tag_inside, - tag_end, tag_single, excluded_chunk_types); + output_segments, label_segments, *num_infer_chunks_data, + *num_label_chunks_data, *num_correct_chunks_data, + num_chunk_types, num_tag_types, other_chunk_type, tag_begin, + tag_inside, tag_end, tag_single, excluded_chunk_types); } - *precision_data = !num_output_segments ? 0 : static_cast(num_correct) / - num_output_segments; - *racall_data = !num_label_segments ? 0 : static_cast(num_correct) / - num_label_segments; - *f1_data = !num_correct ? 0 : 2 * (*precision_data) * (*racall_data) / - ((*precision_data) + (*racall_data)); + *precision_data = !(*num_infer_chunks_data) + ? 0 + : static_cast(*num_correct_chunks_data) / + (*num_infer_chunks_data); + *racall_data = !(*num_label_chunks_data) + ? 0 + : static_cast(*num_correct_chunks_data) / + (*num_label_chunks_data); + *f1_data = !(*num_correct_chunks_data) + ? 0 + : 2 * (*precision_data) * (*racall_data) / + ((*precision_data) + (*racall_data)); } void EvalOneSeq(const int64_t* output, const int64_t* label, int length, diff --git a/python/paddle/v2/fluid/evaluator.py b/python/paddle/v2/fluid/evaluator.py index 137c573622..2d23ff0a16 100644 --- a/python/paddle/v2/fluid/evaluator.py +++ b/python/paddle/v2/fluid/evaluator.py @@ -4,7 +4,7 @@ import layers from framework import Program, unique_name, Variable from layer_helper import LayerHelper -__all__ = ['Accuracy'] +__all__ = ['Accuracy', 'ChunkEvaluator'] def _clone_var_(block, var): @@ -132,3 +132,74 @@ class Accuracy(Evaluator): correct = layers.cast(correct, dtype='float32', **kwargs) out = layers.elementwise_div(x=correct, y=total, **kwargs) return np.array(executor.run(eval_program, fetch_list=[out])[0]) + + +class ChunkEvaluator(Evaluator): + """ + Accumulate counter numbers output by chunk_eval from mini-batches and + compute the precision recall and F1-score using the accumulated counter + numbers. + """ + + def __init__(self, + input, + label, + chunk_scheme, + num_chunk_types, + excluded_chunk_types=None, + **kwargs): + super(ChunkEvaluator, self).__init__("chunk_eval", **kwargs) + main_program = self.helper.main_program + if main_program.current_block().idx != 0: + raise ValueError("You can only invoke Evaluator in root block") + + self.num_infer_chunks = self.create_state( + dtype='int64', shape=[1], suffix='num_infer_chunks') + self.num_label_chunks = self.create_state( + dtype='int64', shape=[1], suffix='num_label_chunks') + self.num_correct_chunks = self.create_state( + dtype='int64', shape=[1], suffix='num_correct_chunks') + kwargs = {'main_program': main_program} + precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks = layers.chunk_eval( + input=input, + label=label, + chunk_scheme=chunk_scheme, + num_chunk_types=num_chunk_types, + excluded_chunk_types=excluded_chunk_types, + **kwargs) + layers.sums( + input=[self.num_infer_chunks, num_infer_chunks], + out=self.num_infer_chunks, + **kwargs) + layers.sums( + input=[self.num_label_chunks, num_label_chunks], + out=self.num_label_chunks, + **kwargs) + layers.sums( + input=[self.num_correct_chunks, num_correct_chunks], + out=self.num_correct_chunks, + **kwargs) + + self.metrics.extend([precision, recall, f1_score]) + + def eval(self, executor, eval_program=None): + if eval_program is None: + eval_program = Program() + block = eval_program.current_block() + kwargs = {'main_program': eval_program} + num_infer_chunks, num_label_chunks, num_correct_chunks = executor.run( + eval_program, + fetch_list=[_clone_var_(block, state) for state in self.states]) + num_infer_chunks = num_infer_chunks[0] + num_label_chunks = num_label_chunks[0] + num_correct_chunks = num_correct_chunks[0] + precision = float( + num_correct_chunks) / num_infer_chunks if num_infer_chunks else 0 + recall = float( + num_correct_chunks) / num_label_chunks if num_label_chunks else 0 + f1_score = float(2 * precision * recall) / ( + precision + recall) if num_correct_chunks else 0 + return np.array( + [precision], dtype='float32'), np.array( + [recall], dtype='float32'), np.array( + [f1_score], dtype='float32') diff --git a/python/paddle/v2/fluid/layers.py b/python/paddle/v2/fluid/layers.py index 98a04ea9c2..809534884b 100644 --- a/python/paddle/v2/fluid/layers.py +++ b/python/paddle/v2/fluid/layers.py @@ -640,8 +640,8 @@ def chunk_eval(input, excluded_chunk_types=None, **kwargs): """ - This function computes the accuracy using the input and label. - The output is the top_k inputs and their indices. + This function computes and outputs the precision, recall and + F1-score of chunk detection. """ helper = LayerHelper("chunk_eval", **kwargs) @@ -649,6 +649,9 @@ def chunk_eval(input, precision = helper.create_tmp_variable(dtype="float32") recall = helper.create_tmp_variable(dtype="float32") f1_score = helper.create_tmp_variable(dtype="float32") + num_infer_chunks = helper.create_tmp_variable(dtype="int64") + num_label_chunks = helper.create_tmp_variable(dtype="int64") + num_correct_chunks = helper.create_tmp_variable(dtype="int64") helper.append_op( type="chunk_eval", @@ -657,14 +660,17 @@ def chunk_eval(input, outputs={ "Precision": [precision], "Recall": [recall], - "F1-Score": [f1_score] + "F1-Score": [f1_score], + "NumInferChunks": [num_infer_chunks], + "NumLabelChunks": [num_label_chunks], + "NumCorrectChunks": [num_correct_chunks] }, attrs={ "num_chunk_types": num_chunk_types, 'chunk_scheme': chunk_scheme, 'excluded_chunk_types': excluded_chunk_types or [] }) - return precision, recall, f1_score + return precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks def sequence_conv(input, diff --git a/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py index d2693b602e..caa51b5df4 100644 --- a/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py @@ -150,7 +150,7 @@ def main(): crf_decode = fluid.layers.crf_decoding( input=feature_out, param_attr=fluid.ParamAttr(name='crfw')) - precision, recall, f1_score = fluid.layers.chunk_eval( + chunk_evaluator = fluid.evaluator.ChunkEvaluator( input=crf_decode, label=target, chunk_scheme="IOB", @@ -176,14 +176,16 @@ def main(): batch_id = 0 for pass_id in xrange(PASS_NUM): + chunk_evaluator.reset(exe) for data in train_data(): outs = exe.run(fluid.default_main_program(), feed=feeder.feed(data), - fetch_list=[avg_cost, precision, recall, f1_score]) + fetch_list=[avg_cost] + chunk_evaluator.metrics) + precision, recall, f1_score = chunk_evaluator.eval(exe) avg_cost_val = np.array(outs[0]) - precision_val = np.array(outs[1]) - recall_val = np.array(outs[2]) - f1_score_val = np.array(outs[3]) + precision_val = np.array(precision) + recall_val = np.array(recall) + f1_score_val = np.array(f1_score) if batch_id % 10 == 0: print("avg_cost=" + str(avg_cost_val)) diff --git a/python/paddle/v2/fluid/tests/test_chunk_eval_op.py b/python/paddle/v2/fluid/tests/test_chunk_eval_op.py index 819e65a653..53bf6f815b 100644 --- a/python/paddle/v2/fluid/tests/test_chunk_eval_op.py +++ b/python/paddle/v2/fluid/tests/test_chunk_eval_op.py @@ -147,7 +147,13 @@ class TestChunkEvalOp(OpTest): 'Recall': np.asarray( [recall], dtype='float32'), 'F1-Score': np.asarray( - [f1], dtype='float32') + [f1], dtype='float32'), + 'NumInferChunks': np.asarray( + [self.num_infer_chunks], dtype='int64'), + 'NumLabelChunks': np.asarray( + [self.num_label_chunks], dtype='int64'), + 'NumCorrectChunks': np.asarray( + [self.num_correct_chunks], dtype='int64') } def setUp(self): From 8f7d0b18145c8310fb36ad017aa0f1d4bce98c65 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Tue, 12 Dec 2017 15:54:38 +0800 Subject: [PATCH 35/94] add param_attr for nets (#6509) --- python/paddle/v2/fluid/layers.py | 6 ++++-- python/paddle/v2/fluid/nets.py | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/fluid/layers.py b/python/paddle/v2/fluid/layers.py index fd8a2ed18c..1f45487902 100644 --- a/python/paddle/v2/fluid/layers.py +++ b/python/paddle/v2/fluid/layers.py @@ -1732,8 +1732,10 @@ def conv2d_transpose(input, h_in = input.shape[2] w_in = input.shape[3] - filter_size_h = output_size[0] - (h_in - 1) * stride[0] + 2 * padding[0] - filter_size_w = output_size[1] - (w_in - 1) * stride[1] + 2 * padding[1] + filter_size_h = output_size[0] - \ + (h_in - 1) * stride[0] + 2 * padding[0] + filter_size_w = output_size[1] - \ + (w_in - 1) * stride[1] + 2 * padding[1] filter_size = [filter_size_h, filter_size_w] elif isinstance(filter_size, int): filter_size = [filter_size, filter_size] diff --git a/python/paddle/v2/fluid/nets.py b/python/paddle/v2/fluid/nets.py index 05728ad75a..7ef524318e 100644 --- a/python/paddle/v2/fluid/nets.py +++ b/python/paddle/v2/fluid/nets.py @@ -9,6 +9,7 @@ def simple_img_conv_pool(input, pool_size, pool_stride, act, + param_attr=None, pool_type='max', main_program=None, startup_program=None): @@ -16,6 +17,7 @@ def simple_img_conv_pool(input, input=input, num_filters=num_filters, filter_size=filter_size, + param_attr=param_attr, act=act, main_program=main_program, startup_program=startup_program) @@ -36,6 +38,7 @@ def img_conv_group(input, conv_padding=1, conv_filter_size=3, conv_act=None, + param_attr=None, conv_with_batchnorm=False, conv_batchnorm_drop_rate=None, pool_stride=1, @@ -57,6 +60,7 @@ def img_conv_group(input, conv_padding = __extend_list__(conv_padding) conv_filter_size = __extend_list__(conv_filter_size) + param_attr = __extend_list__(param_attr) conv_with_batchnorm = __extend_list__(conv_with_batchnorm) conv_batchnorm_drop_rate = __extend_list__(conv_batchnorm_drop_rate) @@ -70,6 +74,7 @@ def img_conv_group(input, num_filters=conv_num_filter[i], filter_size=conv_filter_size[i], padding=conv_padding[i], + param_attr=param_attr[i], act=local_conv_act, main_program=main_program, startup_program=startup_program) @@ -101,6 +106,7 @@ def img_conv_group(input, def sequence_conv_pool(input, num_filters, filter_size, + param_attr=None, act="sigmoid", pool_type="max", main_program=None, @@ -109,6 +115,7 @@ def sequence_conv_pool(input, input=input, num_filters=num_filters, filter_size=filter_size, + param_attr=param_attr, act=act, main_program=main_program, startup_program=startup_program) From e5dcefc4d3bf666627dac6047f71b99415c23a5a Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Tue, 12 Dec 2017 18:28:56 +0800 Subject: [PATCH 36/94] remove ATLAS library --- README.md | 2 +- cmake/cblas.cmake | 38 +----------------------- paddle/math/tests/test_matrixCompare.cpp | 2 +- 3 files changed, 3 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index db0fbd88b2..bbb2d49858 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Please refer to our [release announcement](https://github.com/PaddlePaddle/Paddl examples: - Optimized math operations through SSE/AVX intrinsics, BLAS libraries - (e.g. MKL, ATLAS, cuBLAS) or customized CPU/GPU kernels. + (e.g. MKL, OpenBLAS, cuBLAS) or customized CPU/GPU kernels. - Highly optimized recurrent networks which can handle **variable-length** sequence without padding. - Optimized local and distributed training for models with high dimensional diff --git a/cmake/cblas.cmake b/cmake/cblas.cmake index 13294c0548..6320b17520 100644 --- a/cmake/cblas.cmake +++ b/cmake/cblas.cmake @@ -3,7 +3,7 @@ # It will search MKLML, atlas, OpenBlas, reference-cblas in order. # # If any cblas implementation found, the following variable will be set. -# CBLAS_PROVIDER # one of MKLML, ATLAS, OPENBLAS, REFERENCE +# CBLAS_PROVIDER # one of MKLML, OPENBLAS, REFERENCE # CBLAS_INC_DIR # the include directory for cblas. # CBLAS_LIBS # a list of libraries should be linked by paddle. # # Each library should be full path to object file. @@ -25,42 +25,6 @@ if(WITH_MKLML AND MKLML_INC_DIR AND MKLML_LIB) return() endif() -## Then find atlas. -set(ATLAS_ROOT $ENV{ATLAS_ROOT} CACHE PATH "Folder contains Atlas") -set(ATLAS_INCLUDE_SEARCH_PATHS - ${ATLAS_ROOT}/include - /usr/include - /usr/include/atlas) -set(ATLAS_LIB_SEARCH_PATHS - ${ATLAS_ROOT}/lib - /usr/lib - /usr/lib/blas/atlas - /usr/lib/atlas - /usr/lib/atlas-base # special for ubuntu 14.04. - ) -find_path(ATLAS_INC_DIR NAMES cblas.h - PATHS ${ATLAS_INCLUDE_SEARCH_PATHS}) -find_path(ATLAS_CLAPACK_INC_DIR NAMES clapack.h - PATHS ${ATLAS_INCLUDE_SEARCH_PATHS}) -find_library(ATLAS_CBLAS_LIB NAMES cblas libcblas.so.3 - PATHS ${ATLAS_LIB_SEARCH_PATHS}) -find_library(ATLAS_CLAPACK_LIB NAMES lapack_atlas liblapack_atlas.so.3 - PATHS ${ATLAS_LIB_SEARCH_PATHS}) - -if(ATLAS_CLAPACK_INC_DIR AND ATLAS_INC_DIR AND ATLAS_CBLAS_LIB AND ATLAS_CLAPACK_LIB) - set(CBLAS_FOUND ON) - set(CBLAS_PROVIDER ATLAS) - set(CBLAS_INC_DIR ${ATLAS_INC_DIR} ${ATLAS_CLAPACK_INC_DIR}) - set(CBLAS_LIBRARIES ${ATLAS_CLAPACK_LIB} ${ATLAS_CBLAS_LIB}) - - add_definitions(-DPADDLE_USE_ATLAS) - add_definitions(-DLAPACK_FOUND) - - message(STATUS "Found ATLAS (include: ${ATLAS_INC_DIR}, library: ${CBLAS_LIBRARIES})") - message(STATUS "Found lapack in ATLAS (include: ${ATLAS_CLAPACK_INC_DIR})") - return() -endif() - ## Then find openblas. set(OPENBLAS_ROOT $ENV{OPENBLAS_ROOT} CACHE PATH "Folder contains Openblas") set(OPENBLAS_INCLUDE_SEARCH_PATHS diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index 7e5a1db44a..afb8d9d599 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -244,7 +244,7 @@ TEST(Matrix, unary) { LOG(WARNING) << "This version of PaddlePaddle was not built with LAPACK" << "support so we cannot test matrix inverse. To test " << "matrix inverse, please install LAPACKE " - << "and MKL/Openblas/ATLAS, and re-build PaddlePaddle."; + << "and MKL/Openblas, and re-build PaddlePaddle."; #endif } } From 6ecf08b17395944febedd9867c48000a9d7a9f9f Mon Sep 17 00:00:00 2001 From: guosheng Date: Tue, 12 Dec 2017 19:39:07 +0800 Subject: [PATCH 37/94] Enhance init_from_tar to support indicating parameters excluded from the initialized model --- python/paddle/v2/parameters.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/parameters.py b/python/paddle/v2/parameters.py index bd97dc1199..0dd7d38d8e 100644 --- a/python/paddle/v2/parameters.py +++ b/python/paddle/v2/parameters.py @@ -383,19 +383,21 @@ class Parameters(object): params.deserialize(param_name, f) return params - def init_from_tar(self, f): + def init_from_tar(self, f, exclude_params=[]): """ Different from `from_tar`, this interface can be used to init partial network parameters from another saved model. :param f: the initialized model file. :type f: tar file + :param exclude_params: the names of parameters that shouldn't be initialized from the model file. + :type exclude_params: list of strings :return: Nothing. """ tar_param = Parameters.from_tar(f) for pname in tar_param.names(): - if pname in self.names(): + if pname in self.names() and pname not in exclude_params: self.set(pname, tar_param.get(pname)) From 63ce906b088c641c3bf33a2b8aa6324a39310ffe Mon Sep 17 00:00:00 2001 From: guosheng Date: Tue, 12 Dec 2017 20:33:44 +0800 Subject: [PATCH 38/94] Refine ChunkEvalutor by following comments --- paddle/operators/chunk_eval_op.cc | 16 +++++++------ .../tests/book/test_label_semantic_roles.py | 23 +++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/paddle/operators/chunk_eval_op.cc b/paddle/operators/chunk_eval_op.cc index ff2a08ccac..894f355deb 100644 --- a/paddle/operators/chunk_eval_op.cc +++ b/paddle/operators/chunk_eval_op.cc @@ -80,14 +80,16 @@ class ChunkEvalOpMaker : public framework::OpProtoAndCheckerMaker { "sensitivity) of chunks on the given mini-batch."); AddOutput("F1-Score", "(float). The evaluated F1-Score on the given mini-batch."); + AddOutput("NumInferChunks", + "(int64_t). The number of chunks in Inference on the given " + "mini-batch."); AddOutput( - "NumInferChunks", - "(int). The number of chunks in Inference on the given mini-batch."); - AddOutput("NumLabelChunks", - "(int). The number of chunks in Label on the given mini-batch."); - AddOutput("NumCorrectChunks", - "(int). The number of chunks both in Inference and Label on the " - "given mini-batch."); + "NumLabelChunks", + "(int64_t). The number of chunks in Label on the given mini-batch."); + AddOutput( + "NumCorrectChunks", + "(int64_t). The number of chunks both in Inference and Label on the " + "given mini-batch."); AddAttr("num_chunk_types", "(int). The number of chunk type. See below for details."); AddAttr( diff --git a/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py index caa51b5df4..c3591a613a 100644 --- a/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py @@ -178,20 +178,19 @@ def main(): for pass_id in xrange(PASS_NUM): chunk_evaluator.reset(exe) for data in train_data(): - outs = exe.run(fluid.default_main_program(), - feed=feeder.feed(data), - fetch_list=[avg_cost] + chunk_evaluator.metrics) - precision, recall, f1_score = chunk_evaluator.eval(exe) - avg_cost_val = np.array(outs[0]) - precision_val = np.array(precision) - recall_val = np.array(recall) - f1_score_val = np.array(f1_score) + cost, precision, recall, f1_score = exe.run( + fluid.default_main_program(), + feed=feeder.feed(data), + fetch_list=[avg_cost] + chunk_evaluator.metrics) + pass_precision, pass_recall, pass_f1_score = chunk_evaluator.eval( + exe) if batch_id % 10 == 0: - print("avg_cost=" + str(avg_cost_val)) - print("precision_val=" + str(precision_val)) - print("recall_val:" + str(recall_val)) - print("f1_score_val:" + str(f1_score_val)) + print("avg_cost:" + str(cost) + " precision:" + str( + precision) + " recall:" + str(recall) + " f1_score:" + str( + f1_score) + " pass_precision:" + str( + pass_precision) + " pass_recall:" + str(pass_recall) + + " pass_f1_score:" + str(pass_f1_score)) # exit early for CI exit(0) From 3ef8ec37bb80427ab8abe23384c42a9e9056c87d Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Wed, 13 Dec 2017 10:35:57 +0800 Subject: [PATCH 39/94] fix the new_op doc (#6540) * fix the ending symbol * fix invalid content link --- doc/howto/dev/new_op_cn.md | 36 ++++++++++++++++++++---------------- doc/howto/dev/new_op_en.md | 19 ++++++++++--------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/doc/howto/dev/new_op_cn.md b/doc/howto/dev/new_op_cn.md index 44dbeecbbd..757a5840bc 100644 --- a/doc/howto/dev/new_op_cn.md +++ b/doc/howto/dev/new_op_cn.md @@ -1,17 +1,18 @@ # 如何写新的Operator - [概念简介](#概念简介) - - [实现C++类](#实现C++类) - - [定义ProtoMaker类](#定义ProtoMaker类) - - [定义Operator类](#定义Operator类) - - [定义OpKernel类](#定义OpKernel类) - - [注册Operator](#注册Operator) + - [实现C++类](#实现c类) + - [定义ProtoMaker类](#定义protomaker类) + - [定义Operator类](#定义operator类) + - [定义OpKernel类](#定义opkernel类) + - [注册Operator](#注册operator) - [编译](#编译) - - [绑定Python](#绑定Python) + - [绑定Python](#绑定python) - [实现单元测试](#实现单元测试) - - [前向Operator单测](#前向Operator单测) - - [反向Operator单测](#反向Operator单测) + - [前向Operator单测](#前向operator单测) + - [反向Operator单测](#反向operator单测) - [编译和执行](#编译和执行) + - [注意事项](#注意事项) ## 概念简介 @@ -43,7 +44,7 @@ Kernel实现 | CPU、CUDA共享Kernel实现在`.h`文件中,否则,CPU ## 实现C++类 -### 1. 定义ProtoMaker类 +### 定义ProtoMaker类 矩阵乘法的公式:$Out = X * Y$, 可见该计算由两个输入,一个输出组成。 @@ -100,7 +101,7 @@ The equation is: Out = scale*X - `AddAttr("scale", "...").SetDefault(1.0);` : 增加`scale`系数,作为参数属性,并且设置默认值为1.0。 -### 2. 定义Operator类 +### 定义Operator类 下面的点实现了MulOp的定义: @@ -149,7 +150,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, 通常`OpProtoMaker`和`Op`类的定义写在`.cc`文件中,和下面将要介绍的注册函数一起放在`.cc`中 -### 3. 定义OpKernel类 +### 定义OpKernel类 `MulKernel`继承自`framework::OpKernel`,带有下面两个模板参数: @@ -177,6 +178,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, math::matmul(*X, false, *Y, false, 1, Z, 0, device_context); } }; + ``` 需要注意:**不同设备(CPU、CUDA)共享一个Op定义,是否则共享同一个`OpKernel`,取决于`Compute`调用的函数是否支持不同设备。** @@ -188,7 +190,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, 到此,前向Op实现完成。接下来,需要在`.cc`文件中注册该op和kernel。 反向Op类的定义,反向OpKernel的定义与前向Op类似,这里不再赘述。**但需注意反向Op没有`ProtoMaker`**。 -### 4. 注册Operator +### 注册Operator - 在`.cc`文件中注册前向、反向Op类,注册CPU Kernel。 @@ -220,7 +222,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, ops::MulGradKernel); ``` -### 5. 编译 +### 编译 运行下面命令可以进行编译: @@ -236,6 +238,7 @@ make mul_op 单测包括对比前向Op不同设备(CPU、CUDA)的实现、对比反向OP不同设备(CPU、CUDA)的实现、反向Op的梯度测试。下面介绍介绍[`MulOp`的单元测试](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/framework/tests/test_mul_op.py)。 +### 前向Operator单测 Op单元测试继承自`OpTest`。各项更加具体的单元测试在`TestMulOp`里完成。测试Operator,需要: @@ -273,8 +276,7 @@ Op单元测试继承自`OpTest`。各项更加具体的单元测试在`TestMulOp def test_check_grad_ingore_y(self): self.check_grad( ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y')) - - ``` + ``` 上面的代码首先导入依赖的包,下面是对`setUp`函数中操作的重要变量的详细解释: @@ -282,6 +284,8 @@ Op单元测试继承自`OpTest`。各项更加具体的单元测试在`TestMulOp - `self.inputs` : 定义输入,类型为`numpy.array`,并初始化。 - `self.outputs` : 定义输出,并在Python脚本中完成与operator同样的计算逻辑,返回Python端的计算结果。 +### 反向operator单测 + 而反向测试中: - `test_check_grad_normal`中调用`check_grad`使用数值法检测梯度正确性和稳定性。 - 第一个参数`["X", "Y"]` : 指定对输入变量`X`、`Y`做梯度检测。 @@ -290,7 +294,7 @@ Op单元测试继承自`OpTest`。各项更加具体的单元测试在`TestMulOp - `test_check_grad_ingore_x`和`test_check_grad_ingore_y`分支用来测试只需要计算一个输入梯度的情况。 -### 编译和执行单元测试 +### 编译和执行 `python/paddle/v2/framework/tests` 目录下新增的 `test_*.py` 单元测试会被自动加入工程进行编译。 diff --git a/doc/howto/dev/new_op_en.md b/doc/howto/dev/new_op_en.md index 510233306c..fe86936bc1 100644 --- a/doc/howto/dev/new_op_en.md +++ b/doc/howto/dev/new_op_en.md @@ -1,8 +1,8 @@ # How to write a new operator - [Background](#background) - - [Implementing C++ Types](#implementing-c++-types) - - [Defining ProtoMaker](#defining-protoMaker) + - [Implementing C++ Types](#implementing-c-types) + - [Defining ProtoMaker](#defining-protomaker) - [Defining Operator](#defining-operator) - [Registering Operator](#registering-operator) - [Compilation](#compilation) @@ -41,7 +41,7 @@ Let's take matrix multiplication operator, [MulOp](https://github.com/PaddlePadd ## Implementing C++ Types -### 1. Defining Class ProtoMaker +### Defining ProtoMaker Matrix Multiplication can be written as $Out = X * Y$, meaning that the operation consists of two inputs and pne output. @@ -98,7 +98,7 @@ There are two changes in this example: - `AddAttr("scale", "...").SetDefault(1.0);` adds `scale`constant as an attribute, and sets the default value to 1.0. -### 2. Defining Operator +### Defining Operator The following code defines the interface for MulOp: @@ -147,7 +147,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, Usually `OpProtoMaker` and `Op`'s type definitions are written in `.cc` files, which also include the registration methods introduced later. -### 3. Defining OpKernel +### Defining OpKernel `MulKernel` inherits `framework::OpKernel`, which includes the following templates: @@ -188,7 +188,7 @@ This concludes the forward implementation of an operator. Next its operation and The definition of its corresponding backward operator, if applicable, is similar to that of an forward operator. **Note that a backward operator does not include a `ProtoMaker`**. -### 4. Registering Operator +### Registering Operator - In `.cc` files, register forward and backward operator classes and the CPU kernel. @@ -220,7 +220,7 @@ The definition of its corresponding backward operator, if applicable, is similar ops::MulGradKernel); ``` -### 5. Compilation +### Compilation Run the following commands to compile. @@ -284,8 +284,7 @@ A forward operator unit test inherits `unittest.TestCase` and defines metaclass def test_check_grad_ingore_y(self): self.check_grad( ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y')) - - ``` + ``` Get its output, and compare it with the forward operator's own output. The code above first loads required packages. In addition, we have @@ -294,6 +293,8 @@ The code above first loads required packages. In addition, we have - `self.inputs` defines input, with type `numpy.array` and initializes it. - `self.outputs` defines output and completes the same operator computation in the Python script, and returns its result from the Python script. +### Testing Backward Operators + Some key points in checking gradient above include: - `test_normal` calls `check_grad` to validate scaling tests' correctness and stability through numeric methods. From 8ad36cdb5d2c3a79c82c36cd7c2e79bc2d4cc4bf Mon Sep 17 00:00:00 2001 From: "Yang Yang(Tony)" Date: Wed, 13 Dec 2017 10:41:31 +0800 Subject: [PATCH 40/94] PaddlePaddle Fluid Source Overview (#6485) * first commit * Update read_source.md --- doc/howto/read_source.md | 67 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 doc/howto/read_source.md diff --git a/doc/howto/read_source.md b/doc/howto/read_source.md new file mode 100644 index 0000000000..383acb0c82 --- /dev/null +++ b/doc/howto/read_source.md @@ -0,0 +1,67 @@ +# PaddlePaddle Fluid Source Code Overview + +Examples: https://github.com/PaddlePaddle/Paddle/tree/develop/python/paddle/v2/fluid/tests/book + +Core: https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/framework + +Operator: https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators + +Optimizer: https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/optimizer + +Memory: https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/memory + +# Compile Time + +The following **defines** the NN. The definition goes into this [protocol buffer](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/framework.proto). + +```python +x = fluid.layers.data(name='x', shape=[13], dtype='float32') +y = fluid.layers.data(name='y', shape=[1], dtype='float32') + +y_predict = fluid.layers.fc(input=x, size=1, act=None) +cost = fluid.layers.square_error_cost(input=y_predict, label=y) +avg_cost = fluid.layers.mean(x=cost) + +sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) +sgd_optimizer.minimize(avg_cost) +``` + +- Variables: `x`, `y`, `y_predict`, `cost` and `avg_cost`. [Python](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/fluid/framework.py#L93) +- Layers: `fluid.layers.data`, `fluid.layers.fc` and `fluid.layers.mean` are layers. [Python](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/fluid/layers.py) + - Every Layer has one or more operators and variables/parameters + - All the operators are defined at [`paddle/operators/`](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators). Other worth-looking files: + - Base class: [`paddle/framework/operator.h`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h) + - Operator Registration: [`paddle/framework/op_registry.h`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/op_registry.h) + - Operator Lookup: [`paddle/framework/op_info.h`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/op_info.h) +- Optimizer: `fluid.optimizer.SGD`. It does the following + - Add backward operators. [[Python](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/fluid/backward.py), [C++](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/backward.cc)] + - Add optimizer operators. [[Python](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/fluid/optimizer.py), [C++](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/optimizer)] + +# Run Time + +The following **evaluates** the NN. Instantiates all the variables, operators. + +```python +place = fluid.CPUPlace() +feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) +exe = fluid.Executor(place) + +# Allocate memory. Initialize Parameter. +exe.run(fluid.default_startup_program()) + +# Allocate memory. Do computation. +exe.run(fluid.default_main_program(), + feed=feeder.feed(data), + fetch_list=[avg_cost]) +``` + +- Place: `place`. one of CPU, GPU or FPGA. [C++](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/place.h) + - The device handle are at [paddle/platform/device_context.h](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/device_context.h) +- Executor: `fluid.Executor(place)`. [[Python](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/fluid/executor.py), [C++](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/executor.cc)] + - Feeds the data: `feed=feeder.feed(data)` + - Evaluates all the operators + - Fetches the result: `fetch_list=[avg_cost]` +- Other worth looking files: + - Scope: [paddle/framework/scope.h](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/scope.h). Where all the variables live + - Variable: [paddle/framework/variable.h](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/variable.h). Where all the data (most likely tensors) live + - Tensor: [paddle/framework/tensor.h](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/tensor.h). Where we allocate memory through [`paddle/memory/`](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/memory) From 697facc92f58e6b65c76db2f1b6efc282b3c57a0 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Wed, 13 Dec 2017 10:43:20 +0800 Subject: [PATCH 41/94] "add registry interface" (#6449) * "add registry interface" * "move function to registry" * "rename with meaningful name" * "add exposed layers" * "fixed based on comments" * "remove unsed comments" --- python/paddle/v2/fluid/layers.py | 185 ++--------------- python/paddle/v2/fluid/registry.py | 186 ++++++++++++++++++ python/paddle/v2/fluid/tests/test_registry.py | 22 +++ 3 files changed, 221 insertions(+), 172 deletions(-) create mode 100644 python/paddle/v2/fluid/registry.py create mode 100644 python/paddle/v2/fluid/tests/test_registry.py diff --git a/python/paddle/v2/fluid/layers.py b/python/paddle/v2/fluid/layers.py index 1f45487902..9f5a219b20 100644 --- a/python/paddle/v2/fluid/layers.py +++ b/python/paddle/v2/fluid/layers.py @@ -1,12 +1,12 @@ -import core +import contextlib + import proto.framework_pb2 as framework_pb2 +import core from framework import OpProtoHolder, Variable, Program, Operator from initializer import Constant, Normal, Xavier, Initializer from paddle.v2.fluid.layer_helper import LayerHelper, unique_name -import re -import cStringIO +from registry import register_layer from param_attr import ParamAttr -import contextlib __all__ = [ 'fc', 'data', 'cross_entropy', 'conv2d', 'pool2d', 'embedding', 'concat', @@ -14,6 +14,15 @@ __all__ = [ 'batch_norm', 'accuracy', 'split_lod_tensor', 'While' ] +_REGISTER_LAYER_FROM_OPS = [ + 'mean', 'mul', 'elementwise_add', 'elementwise_div', 'dropout', 'reshape', + 'sigmoid', 'scale', 'transpose', 'sigmoid_cross_entropy_with_logits' +] + +for _OP in set(_REGISTER_LAYER_FROM_OPS): + globals()[_OP] = register_layer(_OP) + __all__.append(_OP) + def fc(input, size, @@ -309,174 +318,6 @@ def create_tensor(dtype, name=None, main_program=None, startup_program=None): return helper.create_variable(name=helper.name, dtype=dtype) -def _convert_(name): - """ - Formatting. - - Args: - name: The name/alias - - This function takes in a name and converts it to a standard format of - group1_group2. Where as per the regular expression, group1 can have - alphabets and numbers and group2 has capital alphabets. - - """ - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) - return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() - - -def _generate_doc_string_(op_proto): - """ - Generate docstring by OpProto - - Args: - op_proto (framework_pb2.OpProto): a protobuf message typed OpProto - - Returns: - str: the document string - """ - - def _type_to_str_(tp): - return framework_pb2.AttrType.Name(tp) - - if not isinstance(op_proto, framework_pb2.OpProto): - raise TypeError("OpProto should be `framework_pb2.OpProto`") - - buf = cStringIO.StringIO() - buf.write(op_proto.comment) - buf.write('\nArgs:\n') - for each_input in op_proto.inputs: - line_begin = ' {0}: '.format(_convert_(each_input.name)) - buf.write(line_begin) - buf.write(each_input.comment) - buf.write('\n') - buf.write(' ' * len(line_begin)) - buf.write('Duplicable: ') - buf.write(str(each_input.duplicable)) - buf.write(' Optional: ') - buf.write(str(each_input.dispensable)) - buf.write('\n') - - for each_attr in op_proto.attrs: - buf.write(' ') - buf.write(each_attr.name) - buf.write(' (') - buf.write(_type_to_str_(each_attr.type)) - buf.write('): ') - buf.write(each_attr.comment) - buf.write('\n') - - if len(op_proto.outputs) != 0: - buf.write('\nReturns:\n') - buf.write(' ') - for each_opt in op_proto.outputs: - if not each_opt.intermediate: - break - buf.write(each_opt.comment) - - return buf.getvalue() - - -def _create_op_func_(op_type): - """ - Create an Operator for a Function. - - Args: - op_type: The name of the operator to be created - - This function takes in the operator type (sigmoid, mean , average etc) and - creates the operator functionality. - - """ - op_proto = OpProtoHolder.instance().get_op_proto(op_type) - not_intermediate_outputs = \ - filter(lambda output: not output.intermediate, op_proto.outputs) - intermediate_outputs = \ - filter(lambda output: output.intermediate, op_proto.outputs) - - if len(not_intermediate_outputs) != 1: - raise ValueError("Only one non intermediate output operator can be", - "automatically generated") - - if not_intermediate_outputs[0].duplicable: - raise ValueError( - "Only non duplicable op can be automatically generated") - - for output in intermediate_outputs: - if output.duplicable: - raise ValueError("The op can be automatically generated only when ", - "all intermediate ops are not duplicable") - - o_name = not_intermediate_outputs[0].name - intermediate_output_names = [output.name for output in intermediate_outputs] - - def infer_and_check_dtype(op_proto, **kwargs): - """ - This function performs the sanity check for dtype and - instance type. - """ - dtype = None - for ipt in op_proto.inputs: - name = _convert_(ipt.name) - val = kwargs.pop(name, []) - if not isinstance(val, list) and not isinstance(val, tuple): - val = [val] - for each in val: - if not isinstance(each, Variable): - raise ValueError("input of {0} must be variable".format( - op_type)) - - if dtype is None: - dtype = each.dtype - elif dtype != each.dtype: - raise ValueError( - "operator {0} must input same dtype. {1} vs {2}".format( - op_type, dtype, each.dtype)) - - return dtype - - def func(**kwargs): - helper = LayerHelper(op_type, **kwargs) - - dtype = infer_and_check_dtype(op_proto, **kwargs) - - inputs = dict() - for ipt in op_proto.inputs: - name = _convert_(ipt.name) - val = kwargs.pop(name, []) - if not isinstance(val, list) and not isinstance(val, tuple): - val = [val] - inputs[ipt.name] = val - - outputs = dict() - out = helper.create_tmp_variable(dtype=dtype) - outputs[o_name] = [out] - for name in intermediate_output_names: - outputs[name] = [helper.create_tmp_variable(dtype=dtype)] - helper.append_op( - type=op_type, inputs=inputs, outputs=outputs, attrs=kwargs) - return helper.append_activation(out) - - func.__name__ = op_type - globals()[op_type] = func - func.__doc__ = _generate_doc_string_(op_proto) - global __all__ - __all__.append(op_type) - - -_create_op_func_('mean') -_create_op_func_('mul') -_create_op_func_('elementwise_add') -_create_op_func_('elementwise_div') -_create_op_func_('dropout') -_create_op_func_('reshape') -_create_op_func_('sigmoid') -_create_op_func_('scale') -_create_op_func_('reshape') -_create_op_func_('transpose') -_create_op_func_('sigmoid_cross_entropy_with_logits') - - def cast(x, dtype, main_program=None): """ This function takes in the input with input_dtype diff --git a/python/paddle/v2/fluid/registry.py b/python/paddle/v2/fluid/registry.py new file mode 100644 index 0000000000..6f5dd365de --- /dev/null +++ b/python/paddle/v2/fluid/registry.py @@ -0,0 +1,186 @@ +import re +import cStringIO +import warnings +import functools +import inspect + +import proto.framework_pb2 as framework_pb2 +from framework import OpProtoHolder, Variable, Program, Operator +from paddle.v2.fluid.layer_helper import LayerHelper, unique_name + +__all__ = ['deprecated', 'register_layer'] + + +def _convert_(name): + """ + Formatting. + + Args: + name: The name/alias + + This function takes in a name and converts it to a standard format of + group1_group2. Where as per the regular expression, group1 can have + alphabets and numbers and group2 has capital alphabets. + + """ + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + + +def _generate_doc_string_(op_proto): + """ + Generate docstring by OpProto + + Args: + op_proto (framework_pb2.OpProto): a protobuf message typed OpProto + + Returns: + str: the document string + """ + + def _type_to_str_(tp): + return framework_pb2.AttrType.Name(tp) + + if not isinstance(op_proto, framework_pb2.OpProto): + raise TypeError("OpProto should be `framework_pb2.OpProto`") + + buf = cStringIO.StringIO() + buf.write(op_proto.comment) + buf.write('\nArgs:\n') + for each_input in op_proto.inputs: + line_begin = ' {0}: '.format(_convert_(each_input.name)) + buf.write(line_begin) + buf.write(each_input.comment) + buf.write('\n') + buf.write(' ' * len(line_begin)) + buf.write('Duplicable: ') + buf.write(str(each_input.duplicable)) + buf.write(' Optional: ') + buf.write(str(each_input.dispensable)) + buf.write('\n') + + for each_attr in op_proto.attrs: + buf.write(' ') + buf.write(each_attr.name) + buf.write(' (') + buf.write(_type_to_str_(each_attr.type)) + buf.write('): ') + buf.write(each_attr.comment) + buf.write('\n') + + if len(op_proto.outputs) != 0: + buf.write('\nReturns:\n') + buf.write(' ') + for each_opt in op_proto.outputs: + if not each_opt.intermediate: + break + buf.write(each_opt.comment) + + return buf.getvalue() + + +def register_layer(op_type): + """ + Register an Python layer for an Operator + + Args: + op_type: The name of the operator to be created + + This function takes in the operator type (sigmoid, mean , average etc) and + creates the operator functionality. + + """ + op_proto = OpProtoHolder.instance().get_op_proto(op_type) + not_intermediate_outputs = \ + filter(lambda output: not output.intermediate, op_proto.outputs) + intermediate_outputs = \ + filter(lambda output: output.intermediate, op_proto.outputs) + + if len(not_intermediate_outputs) != 1: + raise ValueError("Only one non intermediate output operator can be", + "automatically generated") + + if not_intermediate_outputs[0].duplicable: + raise ValueError( + "Only non duplicable op can be automatically generated") + + for output in intermediate_outputs: + if output.duplicable: + raise ValueError("The op can be automatically generated only when ", + "all intermediate ops are not duplicable") + + o_name = not_intermediate_outputs[0].name + intermediate_output_names = [output.name for output in intermediate_outputs] + + def infer_and_check_dtype(op_proto, **kwargs): + """ + This function performs the sanity check for dtype and + instance type. + """ + dtype = None + for ipt in op_proto.inputs: + name = _convert_(ipt.name) + val = kwargs.pop(name, []) + if not isinstance(val, list) and not isinstance(val, tuple): + val = [val] + for each in val: + if not isinstance(each, Variable): + raise ValueError("input of {0} must be variable".format( + op_type)) + + if dtype is None: + dtype = each.dtype + elif dtype != each.dtype: + raise ValueError( + "operator {0} must input same dtype. {1} vs {2}".format( + op_type, dtype, each.dtype)) + + return dtype + + def func(**kwargs): + helper = LayerHelper(op_type, **kwargs) + + dtype = infer_and_check_dtype(op_proto, **kwargs) + + inputs = dict() + for ipt in op_proto.inputs: + name = _convert_(ipt.name) + val = kwargs.pop(name, []) + if not isinstance(val, list) and not isinstance(val, tuple): + val = [val] + inputs[ipt.name] = val + + outputs = dict() + out = helper.create_tmp_variable(dtype=dtype) + outputs[o_name] = [out] + for name in intermediate_output_names: + outputs[name] = [helper.create_tmp_variable(dtype=dtype)] + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=kwargs) + return helper.append_activation(out) + + func.__name__ = op_type + func.__doc__ = _generate_doc_string_(op_proto) + return func + + +def deprecated(func_or_class): + """ + Deprecated warning decorator. It will result a warning message. + Should be used before class or function, member function + """ + + @functools.wraps(func) + def func_wrapper(*args, **kwargs): + """ + Wrap func with deprecated warning + """ + warnings.simplefilter('always', DeprecationWarning) #turn off filter + warnings.warn( + "Call to deprecated function {}.".format(func.__name__), + category=DeprecationWarning, + stacklevel=2) + warnings.simplefilter('default', DeprecationWarning) #reset filter + return func(*args, **kwargs) + + return func_wrapper diff --git a/python/paddle/v2/fluid/tests/test_registry.py b/python/paddle/v2/fluid/tests/test_registry.py new file mode 100644 index 0000000000..f8328f31cf --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_registry.py @@ -0,0 +1,22 @@ +import unittest +import warnings + +import paddle.v2.fluid as fluid +import paddle.v2.fluid.framework as framework +import paddle.v2.fluid.layers as layers +import paddle.v2.fluid.registry as registry + + +class TestRegistry(unittest.TestCase): + def test_registry_layer(self): + self.layer_type = "mean" + program = framework.Program() + + x = fluid.layers.data(name='X', shape=[10, 10], dtype='float32') + output = layers.mean(x) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + + X = np.random.random((10, 10)).astype("float32") + mean_out = exe.run(program, feed={"X": X}, fetch_list=[output]) + self.assertAlmostEqual(np.mean(X), mean_out) From 2cc34532437cb73c00cfe77facb63cb3578ae651 Mon Sep 17 00:00:00 2001 From: guosheng Date: Wed, 13 Dec 2017 10:58:50 +0800 Subject: [PATCH 42/94] Refine the doc of parameters.init_from_tar --- python/paddle/v2/parameters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/v2/parameters.py b/python/paddle/v2/parameters.py index 0dd7d38d8e..7b7d1a1d16 100644 --- a/python/paddle/v2/parameters.py +++ b/python/paddle/v2/parameters.py @@ -390,7 +390,8 @@ class Parameters(object): :param f: the initialized model file. :type f: tar file - :param exclude_params: the names of parameters that shouldn't be initialized from the model file. + :param exclude_params: the names of parameters that should + not be initialized from the model file. :type exclude_params: list of strings :return: Nothing. """ From ea093283e6cdc89bed0ffa0f2e534f9583765465 Mon Sep 17 00:00:00 2001 From: sweetsky0901 Date: Wed, 13 Dec 2017 12:39:29 +0800 Subject: [PATCH 43/94] for code review by zhaolong --- paddle/operators/spp_op.cc | 24 +++++++---- paddle/operators/spp_op.cu.cc | 11 ++--- paddle/operators/spp_op.h | 75 +++++++++++++++-------------------- 3 files changed, 55 insertions(+), 55 deletions(-) diff --git a/paddle/operators/spp_op.cc b/paddle/operators/spp_op.cc index 5e51e73ecc..c4bd4f5ab3 100644 --- a/paddle/operators/spp_op.cc +++ b/paddle/operators/spp_op.cc @@ -31,9 +31,15 @@ class SppOpMaker : public framework::OpProtoAndCheckerMaker { "M = C * H * W"); AddAttr("pyramid_height", "(int), multi level pooling"); AddComment(R"DOC( - "Does spatial pyramid pooling on the input image by taking the max, - etc. within regions so that the result vector of different sized - images are of the same size + "With spatial pyramid pooling, the input image can + be of any sizes. This not only allows arbitrary aspect + ratios, but also allows arbitrary scales. We can resize + the input image to any scale (e.g., min(w, h)=180, 224, + ...) and apply the same deep network. When the + input image is at different scales, the network (with + the same filter sizes) will extract features at different + scales. The scales play important roles in traditional + methods. Input shape: $(N, C_{in}, H_{in}, W_{in})$ Output shape: $(H_{out}, W_{out})$ Where @@ -41,6 +47,7 @@ class SppOpMaker : public framework::OpProtoAndCheckerMaker { H_{out} = N \\ W_{out} = (((4^pyramid_height) - 1) / (4 - 1))$ * C_{in} $$ + paper https://arxiv.org/pdf/1406.4729v4.pdf )DOC"); } }; @@ -79,8 +86,9 @@ class SppOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(spp, ops::SppOp, ops::SppOpMaker, spp_grad, ops::SppOpGrad); -REGISTER_OP_CPU_KERNEL(spp, ops::SppKernel, - ops::SppKernel); -REGISTER_OP_CPU_KERNEL(spp_grad, - ops::SppGradKernel, - ops::SppGradKernel); +REGISTER_OP_CPU_KERNEL( + spp, ops::SppKernel, + ops::SppKernel); +REGISTER_OP_CPU_KERNEL( + spp_grad, ops::SppGradKernel, + ops::SppGradKernel); diff --git a/paddle/operators/spp_op.cu.cc b/paddle/operators/spp_op.cu.cc index a7057907ce..761e4d6c4a 100644 --- a/paddle/operators/spp_op.cu.cc +++ b/paddle/operators/spp_op.cu.cc @@ -15,8 +15,9 @@ limitations under the License. */ #include "paddle/operators/spp_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(spp, ops::SppKernel, - ops::SppKernel); -REGISTER_OP_GPU_KERNEL(spp_grad, - ops::SppGradKernel, - ops::SppGradKernel); +REGISTER_OP_CUDA_KERNEL( + spp, ops::SppKernel, + ops::SppKernel); +REGISTER_OP_CUDA_KERNEL( + spp_grad, ops::SppGradKernel, + ops::SppGradKernel); diff --git a/paddle/operators/spp_op.h b/paddle/operators/spp_op.h index 0f2c43ee65..16510cb826 100644 --- a/paddle/operators/spp_op.h +++ b/paddle/operators/spp_op.h @@ -20,7 +20,7 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template class SppKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -43,39 +43,32 @@ class SppKernel : public framework::OpKernel { std::vector paddings({padding_h, padding_w}); // pooling output shape framework::Tensor out_level; - std::vector output_shape_vec({in_x->dims()[0], in_x->dims()[1]}); - output_shape_vec.push_back( - (input_h - kernel_size_h + 2 * padding_h) / kernel_size_h + 1); - output_shape_vec.push_back( - (input_w - kernel_size_w + 2 * padding_w) / kernel_size_w + 1); + std::vector output_shape_vec( + {in_x->dims()[0], in_x->dims()[1], bins, bins}); framework::DDim output_shape(framework::make_ddim(output_shape_vec)); out_level.mutable_data(output_shape, context.GetPlace()); // pooling - math::Pool2dFunctor, T> pool_forward; + math::Pool2dFunctor, T> pool_forward; math::MaxPool max_process; - pool_forward(context.device_context(), *in_x, kernel_size, strides, - paddings, max_process, &out_level); + pool_forward(context.template device_context(), *in_x, + kernel_size, strides, paddings, max_process, &out_level); // flatten pooling output shape - framework::Tensor out_flatten_level; int output_flatten_w = in_x->dims()[1] * bins * bins; std::vector output_flatten_shape_vec( {in_x->dims()[0], output_flatten_w}); framework::DDim output_flatten_shape( framework::make_ddim(output_flatten_shape_vec)); - out_flatten_level.ShareDataWith(out_level); - out_flatten_level.Resize(output_flatten_shape); + out_level.Resize(output_flatten_shape); // concat - auto out_flatten_level_stride = - framework::stride(out_flatten_level.dims()); - StridedMemcpy(context.device_context(), out_flatten_level.data(), - out_flatten_level_stride, out_flatten_level.dims(), + auto out_level_stride = framework::stride(out_level.dims()); + StridedMemcpy(context.template device_context(), + out_level.data(), out_level_stride, out_level.dims(), out_stride, out->data() + output_offset); - output_offset += - out_flatten_level.dims()[1] * out_flatten_level_stride[1]; + output_offset += out_level.dims()[1] * out_level_stride[1]; } } }; -template +template class SppGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { @@ -86,8 +79,8 @@ class SppGradKernel : public framework::OpKernel { framework::Tensor* in_x_grad = context.Output(framework::GradVarName("X")); int pyramid_height = context.template Attr("pyramid_height"); - auto& device_ctx = context.device_context(); - math::SetConstant zero; + auto& device_ctx = context.template device_context(); + math::SetConstant zero; in_x_grad->mutable_data(context.GetPlace()); zero(device_ctx, in_x_grad, static_cast(0)); auto out_stride = framework::stride(out->dims()); @@ -104,45 +97,43 @@ class SppGradKernel : public framework::OpKernel { std::vector strides({kernel_size_h, kernel_size_w}); std::vector paddings({padding_h, padding_w}); // split out and outgrad ... to flatten - framework::Tensor out_flatten_level; - framework::Tensor outgrad_flatten_level; + framework::Tensor out_level; + framework::Tensor outgrad_level; int out_flatten_w = in_x->dims()[1] * bins * bins; std::vector out_flatten_shape_vec( {in_x->dims()[0], out_flatten_w}); framework::DDim out_flatten_shape( framework::make_ddim(out_flatten_shape_vec)); - out_flatten_level.mutable_data(out_flatten_shape, context.GetPlace()); - outgrad_flatten_level.mutable_data(out_flatten_shape, - context.GetPlace()); - auto flatten_stride = framework::stride(out_flatten_level.dims()); + out_level.mutable_data(out_flatten_shape, context.GetPlace()); + outgrad_level.mutable_data(out_flatten_shape, context.GetPlace()); + auto flatten_stride = framework::stride(out_level.dims()); // memcpy - StridedMemcpy(context.device_context(), out->data() + out_offset, - out_stride, out_flatten_level.dims(), flatten_stride, - out_flatten_level.data()); + StridedMemcpy(context.template device_context(), + out->data() + out_offset, out_stride, + out_level.dims(), flatten_stride, out_level.data()); - StridedMemcpy(context.device_context(), + StridedMemcpy(context.template device_context(), out_grad->data() + out_offset, out_stride, - outgrad_flatten_level.dims(), flatten_stride, - outgrad_flatten_level.data()); - out_offset += out_flatten_level.dims()[1] * out_stride[1]; + outgrad_level.dims(), flatten_stride, + outgrad_level.data()); + out_offset += out_level.dims()[1] * out_stride[1]; // flatten backward to nchw - framework::Tensor out_level; - framework::Tensor outgrad_level; + std::vector out_shape_vec({in_x->dims()[0], in_x->dims()[1]}); out_shape_vec.push_back( (input_h - kernel_size_h + 2 * padding_h) / kernel_size_h + 1); out_shape_vec.push_back( (input_w - kernel_size_w + 2 * padding_w) / kernel_size_w + 1); framework::DDim out_shape(framework::make_ddim(out_shape_vec)); - out_level.ShareDataWith(out_flatten_level); + out_level.ShareDataWith(out_level); out_level.Resize(out_shape); - outgrad_level.ShareDataWith(outgrad_flatten_level); + outgrad_level.ShareDataWith(outgrad_level); outgrad_level.Resize(out_shape); // pooling backward - math::MaxPool2dGradFunctor pool2d_backward; - pool2d_backward(context.device_context(), *in_x, *&out_level, - *&outgrad_level, kernel_size, strides, paddings, - in_x_grad); + math::MaxPool2dGradFunctor pool2d_backward; + pool2d_backward(context.template device_context(), *in_x, + *&out_level, *&outgrad_level, kernel_size, strides, + paddings, in_x_grad); } } }; From 316ab4e07d0dd4fce3efa8f3b32380a829ffbd48 Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 13 Dec 2017 12:40:11 +0800 Subject: [PATCH 44/94] christmas gift: fix a typo --- paddle/scripts/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index 1e1fcc50dc..f0620498cf 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -20,7 +20,7 @@ binaries. ## Run The Build -### Build Evironments +### Build Environments The pre-built build environment images are: From 1ba8f7fe71fc8df40669a24fc5af88d0904240f1 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 13 Dec 2017 14:18:19 +0800 Subject: [PATCH 45/94] The comments in reshape_op is wrong (#6565) --- paddle/operators/reshape_op.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc index 39bf2118d6..7fd33bf662 100644 --- a/paddle/operators/reshape_op.cc +++ b/paddle/operators/reshape_op.cc @@ -84,9 +84,9 @@ Given a 2-D tensor X with 2 rows and 2 columns [[1, 2], [3, 4]] and target shape = [1, 4], the reshape operator will transform -the tensor X into a 1-D tensor: +the tensor X into a 2-D tensor: - [1, 2, 3, 4] + [[1, 2, 3, 4]] )DOC"); } From d069f2ca0a92dd07eb06c9a02528c41b0702f131 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 13 Dec 2017 14:37:05 +0800 Subject: [PATCH 46/94] Make fluid.layers.fc support multiple param_attr (#6532) Fix #6531 --- python/paddle/v2/fluid/param_attr.py | 2 ++ python/paddle/v2/fluid/tests/test_layers.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/param_attr.py b/python/paddle/v2/fluid/param_attr.py index 86088fdd7c..7952a5ea51 100644 --- a/python/paddle/v2/fluid/param_attr.py +++ b/python/paddle/v2/fluid/param_attr.py @@ -36,6 +36,8 @@ class ParamAttr(object): def to_attr(arg): if arg is None: return ParamAttr() + elif isinstance(arg, list) or isinstance(arg, tuple): + return [ParamAttr.to_attr(a) for a in arg] elif isinstance(arg, ParamAttr): return arg elif isinstance(arg, str) or isinstance(arg, unicode): diff --git a/python/paddle/v2/fluid/tests/test_layers.py b/python/paddle/v2/fluid/tests/test_layers.py index 57f6a362de..9b88080158 100644 --- a/python/paddle/v2/fluid/tests/test_layers.py +++ b/python/paddle/v2/fluid/tests/test_layers.py @@ -29,7 +29,10 @@ class TestBook(unittest.TestCase): label = layers.data(name='label', shape=[1], dtype='int32') hidden1 = layers.fc(input=images, size=128, act='relu') hidden2 = layers.fc(input=hidden1, size=64, act='relu') - predict = layers.fc(input=hidden2, size=10, act='softmax') + predict = layers.fc(input=[hidden2, hidden1], + size=10, + act='softmax', + param_attr=["sftmax.w1", "sftmax.w2"]) cost = layers.cross_entropy(input=predict, label=label) avg_cost = layers.mean(x=cost) self.assertIsNotNone(avg_cost) From 0e18bc88366e3987e1cf0dfef382725bf7391bf3 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Wed, 13 Dec 2017 14:45:15 +0800 Subject: [PATCH 47/94] update paddledev to paddlepaddle --- doc/faq/build_and_install/index_cn.rst | 2 +- doc/getstarted/build_and_install/docker_install_cn.rst | 4 ++-- doc/getstarted/build_and_install/docker_install_en.rst | 4 ++-- .../cluster_train_v2/openmpi/docker_cluster/Dockerfile | 2 +- paddle/scripts/tools/build_docs/build_docs.sh | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/faq/build_and_install/index_cn.rst b/doc/faq/build_and_install/index_cn.rst index f1677e216f..a2bdeead78 100644 --- a/doc/faq/build_and_install/index_cn.rst +++ b/doc/faq/build_and_install/index_cn.rst @@ -14,7 +14,7 @@ $ export CUDA_SO="$(\ls usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') - $ docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddlepaddle:latest-gpu + $ docker run ${CUDA_SO} ${DEVICES} -it paddlepaddle/paddle:latest-gpu 更多关于Docker的安装与使用, 请参考 `PaddlePaddle Docker 文档 `_ 。 diff --git a/doc/getstarted/build_and_install/docker_install_cn.rst b/doc/getstarted/build_and_install/docker_install_cn.rst index f78b1fb0e1..1eb06e4182 100644 --- a/doc/getstarted/build_and_install/docker_install_cn.rst +++ b/doc/getstarted/build_and_install/docker_install_cn.rst @@ -114,7 +114,7 @@ PaddlePaddle Book是为用户和开发者制作的一个交互式的Jupyter Note .. code-block:: bash - nvidia-docker run -it -v $PWD:/work paddledev/paddle:latest-gpu /bin/bash + nvidia-docker run -it -v $PWD:/work paddlepaddle/paddle:latest-gpu /bin/bash **注: 如果没有安装nvidia-docker,可以尝试以下的方法,将CUDA库和Linux设备挂载到Docker容器内:** @@ -122,7 +122,7 @@ PaddlePaddle Book是为用户和开发者制作的一个交互式的Jupyter Note export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') - docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:latest-gpu + docker run ${CUDA_SO} ${DEVICES} -it paddlepaddle/paddle:latest-gpu **关于AVX:** diff --git a/doc/getstarted/build_and_install/docker_install_en.rst b/doc/getstarted/build_and_install/docker_install_en.rst index d7acc7aeb7..5a46c598f2 100644 --- a/doc/getstarted/build_and_install/docker_install_en.rst +++ b/doc/getstarted/build_and_install/docker_install_en.rst @@ -122,7 +122,7 @@ GPU driver installed before move on. .. code-block:: bash - nvidia-docker run -it -v $PWD:/work paddledev/paddle:latest-gpu /bin/bash + nvidia-docker run -it -v $PWD:/work paddlepaddle/paddle:latest-gpu /bin/bash **NOTE: If you don't have nvidia-docker installed, try the following method to mount CUDA libs and devices into the container.** @@ -130,7 +130,7 @@ GPU driver installed before move on. export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') - docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:latest-gpu + docker run ${CUDA_SO} ${DEVICES} -it paddlepaddle/paddle:latest-gpu **About AVX:** diff --git a/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/Dockerfile b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/Dockerfile index 1a2d19e823..c2f631bdf4 100644 --- a/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/Dockerfile +++ b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/Dockerfile @@ -1,7 +1,7 @@ # Build this image: docker build -t mpi . # -FROM paddledev/paddle:0.10.0rc3 +FROM paddlepaddle/paddle:0.10.0rc3 ENV DEBIAN_FRONTEND noninteractive diff --git a/paddle/scripts/tools/build_docs/build_docs.sh b/paddle/scripts/tools/build_docs/build_docs.sh index c6cbbc4eef..f9bc8bf63a 100755 --- a/paddle/scripts/tools/build_docs/build_docs.sh +++ b/paddle/scripts/tools/build_docs/build_docs.sh @@ -5,4 +5,4 @@ docker run --rm \ -e "WITH_AVX=ON" \ -e "WITH_DOC=ON" \ -e "WOBOQ=ON" \ - ${1:-"paddledev/paddle:dev"} + ${1:-"paddlepaddle/paddle:latest-dev"} From 0a8addf802e2198084b9cc66e49ca4ae2fdd8125 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 13 Dec 2017 15:24:53 +0800 Subject: [PATCH 48/94] Make cast op support bool (#6562) Also add `elemwise_sub/mul/abs/clip` layers --- paddle/operators/cast_op.cc | 3 ++- paddle/operators/cast_op.cu | 3 ++- python/paddle/v2/fluid/layers.py | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/paddle/operators/cast_op.cc b/paddle/operators/cast_op.cc index 42bff69a1e..d641b8fc9f 100644 --- a/paddle/operators/cast_op.cc +++ b/paddle/operators/cast_op.cc @@ -74,4 +74,5 @@ REGISTER_OP_WITH_KERNEL(cast, ops::CastOpGradMaker, ops::CastOpInferShape, REGISTER_OP_CPU_KERNEL(cast, ops::CastOpKernel, ops::CastOpKernel, ops::CastOpKernel, - ops::CastOpKernel); + ops::CastOpKernel, + ops::CastOpKernel); diff --git a/paddle/operators/cast_op.cu b/paddle/operators/cast_op.cu index 4681deaa62..91e6fb391c 100644 --- a/paddle/operators/cast_op.cu +++ b/paddle/operators/cast_op.cu @@ -19,4 +19,5 @@ using CastOpKernel = paddle::operators::CastOpKernel; REGISTER_OP_CUDA_KERNEL(cast, CastOpKernel, CastOpKernel, - CastOpKernel, CastOpKernel); + CastOpKernel, CastOpKernel, + CastOpKernel); diff --git a/python/paddle/v2/fluid/layers.py b/python/paddle/v2/fluid/layers.py index 9f5a219b20..f67d6d08c7 100644 --- a/python/paddle/v2/fluid/layers.py +++ b/python/paddle/v2/fluid/layers.py @@ -15,8 +15,9 @@ __all__ = [ ] _REGISTER_LAYER_FROM_OPS = [ - 'mean', 'mul', 'elementwise_add', 'elementwise_div', 'dropout', 'reshape', - 'sigmoid', 'scale', 'transpose', 'sigmoid_cross_entropy_with_logits' + 'mean', 'mul', 'dropout', 'reshape', 'sigmoid', 'scale', 'transpose', + 'sigmoid_cross_entropy_with_logits', 'elementwise_add', 'elementwise_div', + 'elementwise_sub', 'elementwise_mul', 'clip', 'abs' ] for _OP in set(_REGISTER_LAYER_FROM_OPS): From 842b485f6a15e0ddba7ff345cb17d440cc950374 Mon Sep 17 00:00:00 2001 From: guosheng Date: Wed, 13 Dec 2017 17:09:15 +0800 Subject: [PATCH 49/94] Enhance ReduceOp to support reducing over all elements --- paddle/operators/reduce_op.cc | 32 +++-- paddle/operators/reduce_op.h | 119 ++++++++++++------ .../paddle/v2/fluid/tests/test_reduce_op.py | 14 +++ 3 files changed, 113 insertions(+), 52 deletions(-) diff --git a/paddle/operators/reduce_op.cc b/paddle/operators/reduce_op.cc index b754637bf2..fedc2a5c37 100644 --- a/paddle/operators/reduce_op.cc +++ b/paddle/operators/reduce_op.cc @@ -37,18 +37,23 @@ class ReduceOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_LT( dim, x_rank, "The dim should be in the range [-rank(input), rank(input))."); - bool keep_dim = ctx->Attrs().Get("keep_dim"); - auto dims_vector = vectorize(x_dims); - if (keep_dim || x_rank == 1) { - dims_vector[dim] = 1; + bool reduce_all = ctx->Attrs().Get("reduce_all"); + if (reduce_all) { + ctx->SetOutputDim("Out", {1}); } else { - dims_vector.erase(dims_vector.begin() + dim); - } - auto out_dims = framework::make_ddim(dims_vector); - ctx->SetOutputDim("Out", out_dims); - if (dim != 0) { - // Only pass LoD when not reducing on the first dim. - ctx->ShareLoD("X", /*->*/ "Out"); + bool keep_dim = ctx->Attrs().Get("keep_dim"); + auto dims_vector = vectorize(x_dims); + if (keep_dim || x_rank == 1) { + dims_vector[dim] = 1; + } else { + dims_vector.erase(dims_vector.begin() + dim); + } + auto out_dims = framework::make_ddim(dims_vector); + ctx->SetOutputDim("Out", out_dims); + if (dim != 0) { + // Only pass LoD when not reducing on the first dim. + ctx->ShareLoD("X", /*->*/ "Out"); + } } } }; @@ -95,11 +100,16 @@ class ReduceOpMaker : public framework::OpProtoAndCheckerMaker { "(bool, default false) " "If true, retain the reduced dimension with length 1.") .SetDefault(false); + AddAttr("reduce_all", + "(bool, default false) " + "If true, output a scalar reduced along all dimensions.") + .SetDefault(false); comment_ = R"DOC( {ReduceOp} Operator. This operator computes the {reduce} of input tensor along the given dimension. The result tensor has 1 fewer dimension than the input unless keep_dim is true. +If reduce_all is true, just reduce along all dimensions and output a scalar. )DOC"; AddComment(comment_); diff --git a/paddle/operators/reduce_op.h b/paddle/operators/reduce_op.h index 47ce910f28..7bd99cb1e6 100644 --- a/paddle/operators/reduce_op.h +++ b/paddle/operators/reduce_op.h @@ -26,10 +26,12 @@ using DDim = framework::DDim; template using EigenTensor = framework::EigenTensor; - template using EigenScalar = framework::EigenScalar; +template +using EigenVector = framework::EigenVector; struct SumFunctor { template @@ -95,26 +97,41 @@ template class ReduceKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - int rank = context.Input("X")->dims().size(); - switch (rank) { - case 1: - ReduceCompute<1>(context); - break; - case 2: - ReduceCompute<2>(context); - break; - case 3: - ReduceCompute<3>(context); - break; - case 4: - ReduceCompute<4>(context); - break; - case 5: - ReduceCompute<5>(context); - break; - case 6: - ReduceCompute<6>(context); - break; + bool reduce_all = context.Attr("reduce_all"); + if (reduce_all) { + // Flatten and reduce 1-D tensor + auto* input = context.Input("X"); + auto* output = context.Output("Out"); + output->mutable_data(context.GetPlace()); + auto x = EigenVector::Flatten(*input); + auto out = EigenScalar::From(*output); + auto& place = + *context.template device_context().eigen_device(); + auto reduce_dim = Eigen::array({{0}}); + Functor functor; + functor(place, x, out, reduce_dim); + } else { + int rank = context.Input("X")->dims().size(); + switch (rank) { + case 1: + ReduceCompute<1>(context); + break; + case 2: + ReduceCompute<2>(context); + break; + case 3: + ReduceCompute<3>(context); + break; + case 4: + ReduceCompute<4>(context); + break; + case 5: + ReduceCompute<5>(context); + break; + case 6: + ReduceCompute<6>(context); + break; + } } } @@ -157,26 +174,46 @@ template class ReduceGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - int rank = context.Input("X")->dims().size(); - switch (rank) { - case 1: - ReduceGradCompute<1>(context); - break; - case 2: - ReduceGradCompute<2>(context); - break; - case 3: - ReduceGradCompute<3>(context); - break; - case 4: - ReduceGradCompute<4>(context); - break; - case 5: - ReduceGradCompute<5>(context); - break; - case 6: - ReduceGradCompute<6>(context); - break; + bool reduce_all = context.Attr("reduce_all"); + if (reduce_all) { + auto* input0 = context.Input("X"); + auto* input1 = context.Input("Out"); + auto* input2 = context.Input(framework::GradVarName("Out")); + auto* output = context.Output(framework::GradVarName("X")); + output->mutable_data(context.GetPlace()); + auto x = EigenVector::Flatten(*input0); + auto x_reduce = EigenVector::From(*input1); + auto x_reduce_grad = EigenVector::From(*input2); + auto x_grad = EigenVector::Flatten(*output); + auto& place = + *context.template device_context().eigen_device(); + auto broadcast_dim = + Eigen::array({{static_cast(input0->numel())}}); + Functor functor; + functor(place, x, x_reduce, x_grad, x_reduce_grad, broadcast_dim, + broadcast_dim[0]); + } else { + int rank = context.Input("X")->dims().size(); + switch (rank) { + case 1: + ReduceGradCompute<1>(context); + break; + case 2: + ReduceGradCompute<2>(context); + break; + case 3: + ReduceGradCompute<3>(context); + break; + case 4: + ReduceGradCompute<4>(context); + break; + case 5: + ReduceGradCompute<5>(context); + break; + case 6: + ReduceGradCompute<6>(context); + break; + } } } diff --git a/python/paddle/v2/fluid/tests/test_reduce_op.py b/python/paddle/v2/fluid/tests/test_reduce_op.py index 70359d60cb..a021d4dd91 100644 --- a/python/paddle/v2/fluid/tests/test_reduce_op.py +++ b/python/paddle/v2/fluid/tests/test_reduce_op.py @@ -85,5 +85,19 @@ class Test1DReduce(OpTest): self.check_grad(['X'], 'Out') +class TestReduceAll(OpTest): + def setUp(self): + self.op_type = "reduce_sum" + self.inputs = {'X': np.random.random((5, 6, 2, 10)).astype("float32")} + self.attrs = {'reduce_all': True} + self.outputs = {'Out': self.inputs['X'].sum()} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + if __name__ == '__main__': unittest.main() From fd12776c570207697b790bdbd44cf466fb1141d7 Mon Sep 17 00:00:00 2001 From: QI JUN Date: Wed, 13 Dec 2017 21:16:50 +0800 Subject: [PATCH 50/94] Design doc: how to support a new device/library doc (#6577) * add support new device doc * add more info --- doc/design/support_new_device.md | 248 +++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 doc/design/support_new_device.md diff --git a/doc/design/support_new_device.md b/doc/design/support_new_device.md new file mode 100644 index 0000000000..92443e4392 --- /dev/null +++ b/doc/design/support_new_device.md @@ -0,0 +1,248 @@ +# Design Doc: Support new Device/Library + +## Background + +Deep learning has a high demand for computing resources. New high-performance device and computing library are coming constantly. The deep learning framework has to integrate these high-performance device and computing library flexibly. + +On the one hand, hardware and computing library are not usually one-to-one coresponding relations. For example, in Intel CPU, there are Eigen and MKL computing library. And in Nvidia GPU, there are Eigen and cuDNN computing library. We have to implement specific kernels for an operator for each computing library. + +On the other hand, users usually do not want to care about the low-level hardware and computing library when writing a neural network configuration. In Fluid, `Layer` is exposed in `Python`, and `Operator` is exposed in `C++`. Both `Layer` and `Operator` are independent on hardwares. + +So, how to support a new Device/Library in Fluid becomes a challenge. + + +## Basic: Integrate A New Device/Library + +For a general overview of fluid, please refer to [overview doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/read_source.md). + +There are mainly there parts we have to consider in integrating a new device/library: + +- Place and DeviceContext: indicates the device id and manages hardware resources + +- Memory and Tensor: malloc/free data on certain device + +- Math Functor and OpKernel: implement computing unit on certain device/library + +### Place and DeviceContext + + +#### Place +Fluid use class [Place](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/place.h#L55) to represent specific device and computing library. There are inheritance relationships between different kinds of `Place`. + +``` + | CPUPlace --> MKLDNNPlace +Place --| CUDAPlace --> CUDNNPlace + | FPGAPlace +``` + +And `Place` is defined as follows: + +``` +typedef boost::variant Place; +``` + +#### DeviceContext + +Fluid use class [DeviceContext](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/device_context.h#L30) to manage the resources in certain hardware, such as CUDA stream in `CDUADeviceContext`. There are also inheritance relationships between different kinds of `DeviceContext`. + + +``` + /-> CPUDeviceContext --> MKLDeviceContext +DeviceContext ----> CUDADeviceContext --> CUDNNDeviceContext + \-> FPGADeviceContext +``` + +A example of Nvidia GPU is as follows: + +- DeviceContext + + +``` +class DeviceContext { + virtual Place GetPlace() const = 0; +}; +``` + + +- CUDADeviceContext + + +``` +class CUDADeviceContext : public DeviceContext { + Place GetPlace() const override { return place_; } +private: + CUDAPlace place_; + cudaStream_t stream_; + cublasHandle_t cublas_handle_; + std::unique_ptr eigen_device_; // binds with stream_ +}; +``` + +- CUDNNDeviceContext + +``` +class CUDNNDeviceContext : public CUDADeviceContext { + private: + cudnnHandle_t cudnn_handle_; +}; +``` + + +### Memory and Tensor + + +#### memory module + +Fluid provide following [memory interfaces](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/memory/memory.h#L36): + +``` +template +void* Alloc(Place place, size_t size); + +template +void Free(Place place, void* ptr); + +template +size_t Used(Place place); +``` + +To implementing these interfaces, we have to implement MemoryAllocator for specific Device + + +#### Tensor + +[Tensor](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/tensor.h#L36) holds data with some shape in certain Place. + +```cpp +class Tensor { + public: + /*! Return a pointer to mutable memory block. */ + template + inline T* data(); + + /** + * @brief Return a pointer to mutable memory block. + * @note If not exist, then allocation. + */ + template + inline T* mutable_data(platform::Place place); + + /** + * @brief Return a pointer to mutable memory block. + * + * @param[in] dims The dimensions of the memory block. + * @param[in] place The place of the memory block. + * + * @note If not exist, then allocation. + */ + template + inline T* mutable_data(DDim dims, platform::Place place); + + /*! Resize the dimensions of the memory block. */ + inline Tensor& Resize(const DDim& dims); + + /*! Return the dimensions of the memory block. */ + inline const DDim& dims() const; + + private: + /*! holds the memory block if allocated. */ + std::shared_ptr holder_; + + /*! points to dimensions of memory block. */ + DDim dim_; +}; +``` + +`Placeholder` is used to delay memory allocation; that is, we can first define a tensor, using `Resize` to configure its shape, and then call `mutuable_data` to allocate the actual memory. + +```cpp +paddle::framework::Tensor t; +paddle::platform::CPUPlace place; +// set size first +t.Resize({2, 3}); +// allocate memory on CPU later +t.mutable_data(place); +``` + + + +### Math Functor and OpKernel + +Fluid implements computing unit based on different DeviceContext. Some computing unit is shared between operators. These common part will be put in operators/math directory as basic Functors. + +Let's take [MaxOutFunctor](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/math/maxouting.h#L27) as an example: + +The interface is defined in header file. + +``` +template +class MaxOutFunctor { + public: + void operator()(const DeviceContext& context, const framework::Tensor& input, + framework::Tensor* output, int groups); +}; +``` + +CPU implement in .cc file + +``` +template +class MaxOutFunctor { + public: + void operator()(const platform::CPUDeviceContext& context, + const framework::Tensor& input, framework::Tensor* output, + int groups) { + ... + } +}; +``` + +CUDA implement in .cu file + +``` +template +class MaxOutFunctor { + public: + void operator()(const platform::CUDADeviceContext& context, + const framework::Tensor& input, framework::Tensor* output, + int groups) { + ... + } +}; +``` + + +We get computing handle from concrete DeviceContext, and make compution on tensors. + +The implement of `OpKernel` is similar to math functors, the extra thing we need to do is registering the OpKernel to global map. + +Fluid provides different register interface in op_registry.h + + +Let's take [Crop](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/crop_op.cc#L134) operator as an example: + +In .cc file: + +``` +REGISTER_OP_CPU_KERNEL(crop, ops::CropKernel); +REGISTER_OP_CPU_KERNEL( + crop_grad, ops::CropGradKernel); +``` + +In .cu file: + +``` +REGISTER_OP_CUDA_KERNEL(crop, ops::CropKernel); +REGISTER_OP_CUDA_KERNEL( + crop_grad, ops::CropGradKernel); +``` + + +## Advanced topics: How to switch between different Device/Library + +Generally, we will impelement OpKernel for all Device/Library of an Operator. We can easily train a Convolutional Neural Network in GPU. However, some OpKernel is not sutibale in a specific Device. For example, crf operator can be only run at CPU, whereas most other operators can be run at GPU. To achieve high performance in such circumstance, we have to switch between different Device/Library. + + +We will discuss how to implement an efficient OpKernel switch policy. + +- TBD From 16a61004da780fb1025ea7d537e425403d6ce465 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Wed, 13 Dec 2017 16:47:24 -0800 Subject: [PATCH 51/94] Add fluid.md (#6547) * Add fluid.md * in response to comments from Tony and Abhinav --- doc/design/fluid-compiler.graffle | Bin 0 -> 3405 bytes doc/design/fluid-compiler.png | Bin 0 -> 124118 bytes doc/design/fluid.md | 122 ++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 doc/design/fluid-compiler.graffle create mode 100644 doc/design/fluid-compiler.png create mode 100644 doc/design/fluid.md diff --git a/doc/design/fluid-compiler.graffle b/doc/design/fluid-compiler.graffle new file mode 100644 index 0000000000000000000000000000000000000000..c933df2cb855462c52b2d25f7f9a99b95652961d GIT binary patch literal 3405 zcmV-T4YKkdiwFP!000030PS5{bK*D_e(wAVZ(eqHCILyl_hd3fZk_a`lT65EyNjwV zVFY+%Y}z)tbXWiPm27;s38BLfKzWF5bR-?i=lgUdAIUp^JPs`3p17Xr*uQVW8rT$w zZ8#m%?)|=beRZ_0ZT_*lvGb3E^Zl#O7l*>oGCg0oxZXQG*%vmq>-ENPXpwsT;OaoQ zI6Y}z3ABuQ{qSs4*zEiM@O{1h@bFN>OsZx$119LzFI;CxT>t5m7Pd_@)H;4=lU5U! zo>7PX+A$4(cjMj89eLVq7`}N=PVp0QPwWnP+^Mr)X`*TSq(|Icuv3ptqb#&;)4)D0 z@+K@f&V*eTvy*o_p6}9I-lZDWoPlljTxT?_ozt@qF79?MqU_YiA}ML8=rtr#^^vM5 z3QCFW)T5$96-K_(==^Ww`Ek|0J0kTceGeOVtV+8>s~h%BBNiLq`&f)gcu`uc#`o`F zO8`H%gzr$6^eq8GQT*{EqcF>{Z(*;Px#?JDQfn^D$w49?wJJ(Ca46RFpDdC{MmmRy zv`FVJNlhh(=?5-;NYqLcJ3CLSLt8X%is&pk3ER*kUmRyJg+=UR`yPAeuGupaTZkGn zXv_4YOmZ~MZ=@ifYV$%a#~KanhGq8bf>yAoRUq9QVuKbD%ew?yyCOBe;3N{I7tEdy zuwPIJ0jO?6v<*c8!1oe-FKfcDfc_Pcha}?s%7UW=5@+WpR$a(rBCa{u#p_L^5g_j^SXIBr7ny@J4NR4Zz0tR@bq8ziqX;pw+cadNlgj^@tCIV`AMC zpLSTgxRKB+IRyONh-*+SZ8BV<7cQi#!f7?J+yCJpH}ZAzdH0 zRRlI#L+o~ff6-5^uLisJ&0gOpb{89zEWpM_pp+*fp=ES|@afM}!FTDM4Jpvw9~G1x zHqkAGM$x9TPas9})8iH~+)5CJxK)=Ow<5?N|%!=2xbckWiK4h z^ixAP_Iogh1vOdKL?BXE4Jn!mbXAUrw^8Ox#}7uh>9J$FN{~}McfUVLy7>p4ktli8 z@r`M+q*GWZx+EfciyBPdSlNSrQBHS;oEhroUBsK|B3J?hC`@1{EP);?`;M03^@vSe z({Ow3mLahzo$P^iFF~K;KgtTA{~<)dLDAR&At^X$fE`3J5E1o2M3Mp#83ZCUH#;aW z5K&mq#L6{Y8~Y;{TT}E?NO&Olo^%9?QfH2RR7E$!Sgf)W*{2WEb*u}A>XyzMPZ^~w z1xh0A)l}-FDnD{YSv6t!E;ai*b=SfqN-Q)2dV=?;kEqzRI=f&Kv zPG;KDo<#yBJttrxYP#~eo7T0yXN~qBE&-4U~Wxzf1%_P~5CVDUMF8$CNuA5OZ z*5QbDWF$&1`VFU~hmEI9#xAzJYy}J}xo);%nmXZCY+98%f$7Pc7V}bjL}HaaE{-Ea zY0ME|O*ulDfg?oy=Q%=D^lFX(RYII2R-Gfz+v5lnOj09BTu+LC;w)|ioRuP^@Le9P z_90XVx%;qw)BxWiHOOo3KFG7U585lZ4^XaZaXRGJVkyK)V!iwa_+pZ%o-5ZCP?N*K zYWRj)UG^8$g((64nv1{YdhOR-AI`3Yo?$HH^_DKH5%-6|V9K(;m@bUaIb=CxUmLO~ z)W9skTo`Ec;UoYoJ8ET)T8`Q|qxS6PWbG?z|K~9!cCfGP>p7ua6<}fe_fwA-^Qon*KWQIzPCU=D%9AN z+%||WR+lUES*T+6xp`4$?XRKsYOr-iCslULoV zow;WU7=r%Ex&kDH;(h9#X_}nw3^~&+#9j6>w@Q6`LwpskZNNU7CDINlvoc?K9Nkm_ z{d_RL>Vx?=7^^QyYa3@*9Tq)_8RVa0h6xe+6TolyBxdggoWs5C1b5I~sNK_T}k<`6~cUx0+m4sXEW&Efr);2nG~a-sNUOYX!l`VZ+#j&F^_n>*ffcD$V- zaWPF042B7^7}O%5#;AqMK3?V<)^pU(8?^ygg~LZ;jA-&gFss7oQxmi?gU>IsC{J~X zfy=k8=fIseaQ!|JhPczQh|t;?x=zoWj3OiNMD`Ixt zc40?zm_y_Hmvb6ghdssvE^Pf%N1pJ(^!uZ>;F94p8;-N*bzxzH3`ja$G{IvAj&bfJ zC^9JJgKKuuG!QrJ|Z3X7Yj#@?z+eyEq}M4XEHg?JfKy46{7;F%bWgkl~fnG;m6bIG{2Nro>Og zO=^yVQF9#>yYc6Gw@~s74@gOTGpj~`Mez%IIjz&!v`m9|)2VS|l80xDtui>Otuykg zO|D^^1MHK60p)somTH);;e0af1Jg@xa~%4d+M+v=Z4!z^ocWeVMBX4LZv|=|+P(B= zL?T@}j$cs5CfIja`0OdMdw#!A0UGZij&uzm5wUF%GZhB;U!b7>;0(>F8MZypt=pwHWUQ_-^<#2fP`OJ z%N6Ph3QA!$GDK6j#da?*e&+aO`WrcSO-ieorY&dINSYeT)P*xf+M`eB&YMD(fw&qj zXU#pw_npBf>{1O9D@aAYkIU;?4)(KP9-m7v3*3aL!Wxbp*Zk(#*rIxycFx?26&X&4 z@pwr^dkz0v=lG-b`4$55pFQi#$+6Rr{@VNNq4BNLKEu*(juRG`0@Jqja zd}B3e`u?GOb%j2D{c?K-8Yj1hjpkP>bM$CE?0=Z7{wW=n{7myOE}2yR1sw;tnFSan z#TN(Pk$h?;a(Be^q0dIi_saB(M6{3Xd+cRA zg)ql}`lo>!%#{rNtA^p5_tUsnL1xwiW7{Av%zJH{Uq>VvO-{J`YmXe-zUfYC_M37~ zuSnOorcH-i)ug&Va$VXIK}#m4R317QWH=m;K>iyxamtZOI=}I>fSdN=QZED)jV(XX z*<=I)ibyYzs%57mVwXHKuaK+~iqkJgR%#_tUVuKLHhwPo*6gI6m}J+flDns{Yg)K{ zOc=g%j)*Ac{FZSic}aR1_KF)KSV0%8!I9XM3>3wZM)=}5gS^L=tA~l)U^-6pO39#P j!C47B-j$SI_EVaQ3bc_NhZe=j?#BNCVJ-kgRI300VN{3* literal 0 HcmV?d00001 diff --git a/doc/design/fluid-compiler.png b/doc/design/fluid-compiler.png new file mode 100644 index 0000000000000000000000000000000000000000..1b0ffed2039c91a3a00bbb719da08c91c3acf7bb GIT binary patch literal 124118 zcmeFZcU)6h8#RiI3>rijC8Bh@fPw-M=}iR-f{;Tfp^1nD2qkm~2ntvMbr7W^(h?wa z2vuMdq$7kN9hKf|Xo0)qJ38vjd*}XfzrVhBevZP)$tnBn{XFYgYdzZ(i`jeMivm+cN31~?|BUCxaExAfi+ zbZ25>X0t}>JL_v+mbI|A6EnYVf6Y?N-Od5*&BP?{E(?CNvvf8Wa<{X^;$+um0R0*gENpM!io&RI*Gg_E^| zv$Z`|h;dx=YxXYA3I`4_PW0`+|6HfD^^L#IgvI@NTi^!88J~!s5<4mW?bzU}@{Fyr zI+i$lTNlRRA*{8tqJ;dHoqyi^*V+E_Ej4>P2PaD$4tz}!@#j~*-}dwVT29uM;Qkn= zQ9SkM{@-u=d4KJj_RjWTtevbaE?}K4oxsWd{Px!i{O^1Gc`SKx#&CT%eE)eiU$%mW zqX?51|Ms92VS(>64lyw)GhH~Vf^^?9(X+jz9ulmcjg7wf7?J(^U2VhPpCCj;j3V{I zmKvU&hQqBBOOc(w2|pRRr}{G2Nax8#Vax6N(eLLsV`Oo|CX%$q`qFN)o8s_dgq-B& z;(A}sf_&M$T=m01CgyEIxBmCn|Hj~d^WgusWe`eW4$xVwBxAR7Y6$xo-6l>(=%DJ~ z9QZm8?@r8IY%Z01TKr&P=a#LU%1nR#<=3%w8`=VQO>`?~O&_xnQHi^w4()N{{;yZ{=ixa+ln2~T-!S^|x_&-B<5NH9EzDGBHi4gy@Y4aoua4U} z3*WwZ^Y-f<{MWFkT*TyOQ^XST!e;yfwqo zG3(U7F4#~p<^Bk4#le3+w-^|q!orTd|8xsKY}4Qta?mX3|LtGT%^40x2#vOV_+Q8H z>j$scI17`EX14usp6}<{*AMPq0wW|QcKZBx^YH6uKkomYWgEKng!`SZr}(eI{g1=S z{XE>$yMf5^{L-;K8Hce!Qf=&y@d7%fFAwkB?L6f79~c zX7K-sY3cWJl*8l8+6nF3`u_d(73CGjBx8nV24Gh@6b|-FW8|Suc9& z_haS(Z1YAh>&4{Nnfz*)SA$#J9vQxSayVT(UiGo`b>V-E6$aZl3vc*UiGbK7^nt~5 zam;e}(JQ$b#>JfqwC<-9J&%aKnh{c22%Gjl@|tW&r8m_19M7WHGC|aPmGSz?;Y$;R zpMr5rxS{&2+Az^%v0qWa|1egenM{_-+gq~^{^uGz#G8v;Y;4+69)n{GPs^!sz;caY?UHi3xL6O`Y)2t{iS3tA>7j^T6m44`Lz;*O-;lC|ie|+-IX(GxwI)zT=X3>GcPDPQX~iWQVe< z1#EhX-P~v|LZrQxY<=kjUGb+Hk)}N}r-us4YXUR+@+iBMc z{k75LSoCvLp@TV$s%BxNUaTP4!wwFKPzw?v+@2+Xr? zeVb9bWau_wgeAX8Z$C-GbPj#6iggJAE5@VCYr3?@VskjXpf}s`)7uXyQ@myQ#wu~L zY;8)f+y_r3iDndD3!=3bPnX$nC@h3_-I(dGm~sgl?6NPnD-tMG(usdFoeb5o=!|NESavjBu4&YVMFkYLma>NOvP!Z z&l+x1U_tjz6;CV5eR?R_J-XoIHANooc546&NzFLLq_il{zOUDFX~H`3Vwh9i;zpC! zbe_ACZ~TLh8KIWx=CkD-yH}>5f<^w5*K0V}K0np!cEjJ>b$H_L!Qjk<5XH_$xoLTt zL6f(~y`@w0j_&`oFlR_iexlP`8EawtR<3p7S65$ujAZ73!bSM@oiw`z=1r}M*JR$^ zP4_9sskwryF<-O z<(u@TfO`WfEfENtG#hHhQiF6~A)RaFnXv9#h`cdV#UVC>RBH1T{CMX|=57Hb0r~EV z{OsA{`9?Xrscy1shk4Kk@XVqIF2nS#8X_z$>z?H$$9RfV$wGH~eAXt#(&j|37hHe0 z%gt*r423RiA>*vdWt~am6BJ@NTU?%Yd`=|?#3@qPI@#+rohR8OFPR%s>A*4pt9_Z5 zcqzQd=IX7qB!Y^uk-Kj5tH=lw$-a%%adBzZDI%$CjQd4~diC;1LP)nJDP8sBFFVf& z34!scfujD}2%@K(o=4c$ZaFD!;*ys#-RA^Q97YS`q$hR58yqsAI()6Aty zE2E{)YKFygLVROJ+-|{ zSxAa^+n58|cOSXzFfQOdqe4WJv%MxZk8wiF@wZf=1j6OEy!t4aR%AM{*^qRbIcFxR zA2|#SKKH*0iQ1S3*uz{x;=n=~pqRt-uTUo2hz_@HOFz`8YQ>Ei?O zA|L$9VE7c>;e%y3SuI89eYyI4o!5y1CEt7AK``;h(^Y57z2Mm<4izk7wiH^li5q@% z44>qa5@AS6V0LAxDLedLy#K(!8!^@4t-A3O`xvvS;te!kSPH)a8h)_ZcU_lhuE<0&AEZw3!BCziAE0C(+ z8Rp)6JlVjJOvH^CBG%J0RNz6dGA=$5cPKO|$=ul}-*!}BO^IixUezqdm;G_tejm7D zs&yg6-j!;mgB!;b_RJ(X=+!MoSl_f{?@GIUIGm`qx;jgxy%!28=@MB=i@VAflNxn5ERYulR$~v!1|Hdv#>^H0)>nb4S z*-)<#VYYSb(gP{}h9nn|GmUSN1O-QOUSX~U&@eD#`1`eIPib*Vfr@NFd`@?X*fyJt z>05YO4;9M5oUwy4M7FjM*P;|f3d>i9BGqEO=o+dg%1fwDYxChXZYd?ZRunHU6uULY zsZ6bcgIC)8k+!q2ia zSbUr$PWM<+zhOmT-~==>Y4mbhrM$cx#q?sO8@9f&{;#O)pfPw<4xu`lNl!+*e0vg91Mm zEMJO4q3qY?mKu+#S&T)aY{tDU<)x-36iN_!)P@w6b{=WgARZ8F%_O~u99mol*rX@Q z_i^AU$OSpsnb+eT?be4cASO|olX>nV)jkvK7sshh-ugCuWuS54JZ5GxgmOQq~`tkFfK5H!08xDTIl(#kJbz8T`6@ zF3%19s~S-^8h>(#iR(>#YP>cb`vqM5U%`lHlEWAXoRGv zkgrz}BlAK6<24hX8DU|B1VlhT;WSmgXhTl9>&U9W4y6zid;Bv=>1eHjdd;eH<0TQX zXS|mu>$@x$tCoJWJ9VNM`On8yTq8u3b;ifOOxaS7vB5uu4a4N_hKpX>?&ej8TX^A|D80 z!n#MK&JvA6t9^Zz2Uw}MXYPO2Fov0uv=wpv1m$C4hFG#?#>ocT6v6Gs92SRLSI_p{ z9;j}dT6}M%QuW*ypf)UaWM<3(FTEW9rYht`kA*M$z;vR<+8t6;I<2#D^(zuD=_^%y zMJ*}eSrIDEx9d7hyA7jnJJOIkIvutL4Y|td5HD zEq0BVRJ|a;^flES34}AnL&I*1IESGjo98{`93v7<&(JWUtDa*wuU+iS7>ec0fF{lx zBi4zuCJ!DV-BhxjdA1S623u-#niE*4Zo`L(K$Wq90;0~X9GvS>EuQ5BRii#On04f7 zyRo4&?NKkm@|F0ye0_pyPk@Z)UhYMA3kytaFL|An3+kc3Ksl{fAz4wk?fyu|Hw2#PZ%G__LDAHtE{qXL5h4Zc zX$BS%R@zlBFzz3XDO_hn%sR6zAMkrkWQob^l}iO#4(*R{>B+-TvIE$-i##|G+%{`g7{vevGB?prue;+dtr%qHj;8%w)^U6 z$m7^)T%G2xjN*b#GQs1fz!k*i#=Cc3C)fE}OeFE#0E995R`a_RHW6Z*WV|=s1&5y* z8h&Uv8Lzs))Q6GSYr2(IpU=WLE)%+n^*SD%ioeyyN4@Gqy0 z8@PGXtuw6gC~Fggyt8Y2Czp?_{{x4(jJowY%#9fYz0hiAm?TAtI<~$te{g9C)VZ!z zJ(#=iZ!|=l>wYV~KzYbBSD+dI^KlGZJ7w|qPK@XwWTd23uraOy;abHmeW%^hrT6c*t5c;E@E96Q4n^Ac z?oXMakR#UGo6^Tm?_5iWe?}^qyU2aSOOAbXbU1+jw5@(e`zQyZZ+xUBp~t^H-6%-! zX0TyiL7$IY?(ucMN@M8a4XU?kWx;(ARnpm@UMSM?XMvn|JVJ3Y0@ z6*8V?QHpY3=8L*l#+u`GVPQjiIVtJll!j(Ste}tF`?Yt|yRD6}FII4W;9?mAJAdfg zz`_bSWIl6J@$}3iI<4AGgc&M_={oJwOZ-D;?`QwS@_$D$*X;JoX;PF!?*EW(B)taE zv&4d0)n+iC%=;6%LP;Jwb1-Fd7v^~gPb5o(7sf#~rx~lLklPWY$m63w32N{>Q2tMH za?#i4n}%+89?^^_v_m*&kUUH2OI>c;9>VL7-9kcP5q(Vx_hv&x3b}mVI?dzEF;ktr zrC!C7x%PdEYUf7#D{dRVA!Zb-hX`PwoA$w3YR3|W)R8Q4VDWkyKVWt=J?! zD=1;*N^ZdBLu7{b7A!7OO$9ya>rk!9}*;@TyRv;hiBtEP%)^AWlB>SA!H3mO3o3idWN7XHxM~MdM=Jp<`z( zYk$uIgd_Qu8>lvu9IlvO1^?Os{2w2$1s8Fd>i(dYZkX89?ZkeGNWKgLD4?x~?WJCB z#gqA6-JkR~ZVDJQ(0R0U&k;rR#mknj>FOI~pvlenWyjC%ixgDxIR&pJ=l!{+q?OAE zU!ve!6jgNX+=D#<8Rx*G8zUO#k>^&Bf#pYGN()8zV2C`-t_U2l* zHlJ)FT+T2pFOw_pM$pnkmSa4}gr&JA0OXzy{Vb#&EcYO7+f3MrB;$9`660gmC+Yb^*dfM$n^Ij)fnqo8P42JeRK?-`~aivUc zH(l}a=VDZJyY^_&=;~tCj)uqSUUVk4y0(R?U7NpfDQM+}bcP2iW4Dj6s;Hc!SN6Lv z;>MrzvHk?H{YtowJqbTL+zk@VYq4F0BHMaJq5v&gJmNeXn*@dfdj`dlMhMhhj`!t4 z^{0%b1Gcd(a3?6^E>w2Cy6_Zj`D(~*QEiamuT!;CH1{e*2u_SY3(dZ|SadYj`#pO) z@&3i0Tobq+PmIoh$;#ZIF=w)e1%*2x7q;Qty^~+I(5$hEno@R5s7)z~aNx>7fIli) zi;jAeI&>U@(gXQ}lT%|`0xcf7@-0pSIt&(YRzf!=jQmc8nlYdYKQdOd$de)*iiBRO zavPVcI>?L}O1;Xd)ZwSvPlu$#DtBKwQxT8@ix?!(!^KUf{OwB5hbXS5biJKU@M6gU zDexs=cAM;oFmTr4_p`Ynfu4{O{4T%m`s_e)@wid;^?P1;yh|Fh1S+)6$FTv8d9fg_ve9?Xab>kRdgwBg1D* z?qJCn6q|S*PddE1(mK5GhTGKn@&@kd({IdSGW3{q^;fCs+%m7l(sWuxL|HOq;5!h& zT{OI*^%B3Zh6_=~ZX3aIdv7d==2YW8x%2(JjxCzOuEkuqPPbK92lg2-s%sYnIYnxxopX z$l)i^BJA)3gRZ)FNpS{Y-kzFnZ)Kc{o8CE;==W5LqT#k0rNw-QShIJ{O`ZkY=VMp zWfSok_IFVI--rznKu_^8+`b|B3qSk-wmz|l6wBnM4 zOx1bUBJAn+Jw^4`k>sFTuU(SooE-if4voWp5|*_cE$V{N7sKoG{_)(QbId!0qvq_l zF#YR2Tm=fw*})EuA8+mN2~EcY=+uLK%uN4PcK-7N<(FXm?$u@q{=L@yQICFA0>t#% zau(nR`+q;V-&w%d?GCe0`f)=1{Yx4i;AVdJWn=pBME`Kb_h6ibM?|W9{u<_g{s{~u zsLnU{F#XFVZrQqvQEnlF>Hj+5gWzU#H%>79^V9zBG7gD=syBsi?VqsU_qU{6%kUbk z8!<8c>-ZqSGrTIZr1md&#t{i-Vi_ z|9L{j-K_oYQNOzNI!QA!&3*Qx-YbpAXmaOdS3wu8&(|K%NI5`p)d%uR_atN0fjZ-y zLeOfwUhd7SqIzliZIkP;XsfzsC#I}{;FIPsu+Ybb?aFV+$M%d#H#o$4bnOte@P0Wz zwvGYX(S1i!O(Y$F$9#|Z8wJ_d-#szlIj5*vVG!}`*$sxK<0U?(aN!?@c&-?Xm5(pW zsM6P!5&GygpsUh+=&PrlN3_R*Gv| zm1^V+#;CE_xs12FtSyYRRC&^-N~<~3S-!P>B#&JtzPm%1V9OiO77tsAH1SEeWSvwv#KRvk5=(z8N{jBw)FuX9Z2yfSpB z#ZYhj<)kFG`;6S+@46GWITRhtiG^uACLN7YJoel7!co%B+_1S zo3(3y5Z}i@;U(6UfgnMK?CM5&(3-4OP=;SCA{dtYSoIRGNt9raRWwY^C;M`GdmDGa zAE3m%uNfo%p@#r?W=ps8Ap(=_#M;HDT8dc=Etx$U4RGagn#JL-Q{lU1qdc}&%PKh2jE@~6-nmAQZ8c4y@ z7pXlC#V>~Y+L}ulC~IZjx)=b3H{LN(w)z=p86xm7jDWF3NUzTv4o6?}88K&Q1SLd|LrswY5A(LqXl2|f)>!JtikZ)AzZR6<7Igj{HY5|Q zxPoicdiUGUVXq?r$E}ncm}_Dbyoy$kK#o3SROk>7KVjH*`$8+9ed1fF(Rj>Npc+z;YO{&oWJKz+^Uz;knA`=_0zS8(j zSA$=x)h33rb}+_5m=Bd=3bZ(PSBoxhAqSwyryQW)GRycJOEHG}`n=~GitKj!Q#C7I zYF4=2E2iQ$_3LgA^WGkgKzkU^xTjCuW^zjxuvq~GZ4Vy6YU;1^XUF%lWV0yUSxHtl z*~>k+71Ned!NKV+{#^hKZQ8nP4He<-`q-KKYZ}vl{LHHY$(hjn^PN0WHSF_5<*Vj~ z$0tEu=op_0gfiRX=AGE$$)O=fI?%>XJSJbW7)2fEiUJdA>5tu7&zaSs((^Dh#Z<7+ z6^I|vNx0!gF$c=b9}F^zAE>|gq7>`!dL$D&{OUrM(#CQX*RwvbzV>WAACGU(Ft*`o zH>lzjH_WZM`1`Q|!tY?_+-!BR+^LOlOAoxoDBO5A44`73DFa;V*s`F{a=cuAW2%Q6 zP%J&&M8&%r3u%#UTuvs8@1P88ZPAXVg)PV31A9)FMC5|5Wbhs(Q3GacZe*cOE+B96+r+ehG^Xt) zWuH~lZ}_?KnWu9Fkvt8m>2BG8Az%7pI8fZ&GDP&@?|t;G%X3y8K+)4x z^Y|eB0C(`1Y)P2vHJ0`)DgL9|YE&Z$FFk?VM$Dap*qHX+#NqF9{T}4{QKgKt+3VCc z3hGzfL0WJ@l?n@fHcrB|sbBq|7fbwTjLO}ysoTxYBq8}8N&S43!{wg!PYR3j9T`pz zeJ2aU7Pi>UoT&;?Ww7m&Dq)j?6{mbqXL!@}POjNFbjRMt`@jgbzB?P&aGM+SJVlE! z;%xxhzb1z-Kk4WJSYm1FDaQBbT7VTPf5^79>n?QMvqN)4462Ba8CPr{x_yCPRDSD2 zm(A;4S8&CI6Cj=kJUe{pS$*?42t&xj_H?qxPoP9neER)ZCxAC1SIG8W>U-*RB!!-B zS%>Z*z}f%})5Y-VtWD%--Q*5P<3>C9u_-ZBbm<9#hz8tMgs%+nA?Qp`U>+%=p5As_ zb#2Vg|1gv#(vRsb%p%I$0mU-VZs9X*Fsd zLFA1Bd^U-LL(&&0MMM)GXR>*&nz~YB->0vEs#?Ndhozd@=C?u<`c(HF0K-O*u97ppVH{!ER?sD_wTYqlu z*jI+%f`p~Y(R%IyIE@u`KkZktA z|KSoJ8<-6R*tqN7bq7|lVv$Si7ou8ULw$_C7d zrld87FwI_(wqk~`b#!;JU3T^46Bx0n8LwB?!b(lq+rnSw$DkgEyIoT z5odbeZ(@Jr;IA)0%Rcua8*-);Y3tD{0@qk#OFJtp=pP65@jG2~Tjd*6_7=;#kN9_( zk=ZP758UTwfsZ}#S3dKJ`)St$M(O^2t?-WF=AIyN(!T-bIc?>zK#+JJt*ofcXNr=Y zFb&~jUBF2;E?C6S1`5{zijw&lnn!S=Wad}Mha{1;IOJzu?^vr|Jv;Epg=%a?@YgK{ zs>uml2Vqb)C_>~MG)R~~k5Swa3#@ExF8-lMt8Q?cdOHRVKaM%ebECr^1wawa;nUlN zm-3}Y+eAkc?KoEnTEPni9A0n1YFIW%69*na)@fuz1_@2nO%7&JD=)&B z@IMVyme{_IN+)pqQ7(A@ax@!rTdBX`(YE#z`iWgzHUqNQ{p0UPt~D(F5^5HiEA)`! zxMwR63z_xZ1d&Ce3yD9~c!#n;sj#d3jPtj(S!1(>U6?4;gerR584JLFfT|&X!r6Pf zvZji33tFR6F@PLB-JmZssGhH;I3-y z>3({ABh92#_Cele#{2a=*h@&0xMJa=+Xb?a@eJW&4}d|%GmCu+%MX<2*tYiVUbTF4 zbhAgo`HX%+0f)tH!HaEds_lVAEWFnBb2+zNuxoLZ#`6_o+wxzK*7{ku6(#>X5RaAL z`cf{u3~qscTU>r6+}iqtd>^_ecM$-w6osa#5dXq>wIIW+Yj3(8vO7&ep!8Ixs}*L) z?Ld(e-@sKen9IP#Q=BhIYE@g;@^3Wow4NZI2Gxjkz5M}|?8O za4EItST*fP?k<87UTlmfd0e)zCaAJ=pb>YnD+ebGhX@^M`5esqrE^L7Eun7*DqM2u z*E}Yr%BSkY%O{3zGP1JqNIJ%>l(s=D-y}zhI3k>#mJ#RWj@JFA;R=E|{ou%_s7-AA z)US1QTVpqWt+3LRX31ptZ%V1`NXUE4aZOEb=7%f^n64&39n7yEj{D{7B2u1%-QoCU z4ahb|~Ki*!(Yx3VVB7*;&&#wW09Ye1>=J9Jldj*YS?5 zgwol!yh}9)Lki~uS&3y5d7$8qC!;?pgcpRkm8a!X$5W`Pdm@yUo?H@phnX*IAMF5z zx07_hd)P2gk(o?s5gdBcDlBbMTVVuc&cdQ8iI6o553rm8Bb6gv*V6oWSSkachmu052W3E01JLi=fYtf5^Vs&?M+(if%=_Rdv^lWG+O~BCaGUHUjO-xY zXXig2$jY;aS^)I19w~J~;4gZwM}W-H%bGzjTvXe?Mz5TP9}3}@Yd>-&YHyi)vf>hc z9!EYe_rl+Hv4MliXIBjJQg=XY5r;kDE*4a|kM^Fb43-@`R&cRGl-&=>wT+ny2evb6 zaYcxtz~wjN!x29y(0*ynj}KulJE@Vb} zaRdPG>z;@awy!mC=pAMrA1bRY;SA+uA;Sq9&_p6&o1*2YR}@#S&qJxOT>wBo8Mx2& znt;)pd|?&iXx>eLh!VC5M@eO?1Zy8%M8jt4YboLqScQfCS_83IrS+NH`1RT9rR7f{ zMu6h!>5l8%1Vn!hu%ya59}9>ge-e%C@!#g_eRu)L$>V{xJIk-wZGgg}>-i~*?ij_~5Qn{czCGQqs~VVG4!q{e z@ThFO@O47?tsiFCk{KC?_ZiA>$YVI#ipReI&=^SrJ`6Dr*vX@dQ$RmMVeMPM?o2v7 z23CvH{Ntcf1ZQ4rV3%H79VZz9Gj%$kO|UiB*pM{`TmkPVzEiuw8sz>;@Tsl)N{pDz64NO#Im`-cZ?SfR6oo&)vJSF0q8HNQ5zm_ zU$&Bz`b2l&8bG%b6|91}!?$Ie5xE&Atm(S1Cq>o07DfgFRT{_m$fa0d{l;#zPnWGV zUQt*?67{oQ^8V`4^7351=OV^U_5>TL^HDqN3lrB5mZxnyPI`@{CSR^XQHiJ~$`w{^1_6D)&DETIFx!Twhy@_QB@g=7MajJ_fcIC&jmAr zQd8pXNX1h{DJY#%PZx&8J=GIODqWIbu_lfxkS!pcupV{V&hX~8z7*DnnW`$$ z1cN5|`OD)}QY?eqX=vgRcu2VKw7xuxwTFhDi9jNij}M5Cbpt=pEZ{VX70>sIEG5Er zK^26g8Cmab1AzYMOu3qn2ITfeG&~cSjT61n%!2?tX?8N-dn z`Q9r|(mQ5A)s%Mmg+zvDE~wBQ1&RH_)9&XYLiFH?!0aoJg_s7jDw@tNd*Q#-sA19qXf(F02u+Z9J0hbqFf@{N$3vokp^w zckqv5j|T9`&=@_VKWDuQ%JI2@r|kjsx9;!;EZlcHaB708IbuiM*AVgbqL1BT2DBX; z`=Eegy~CE0Ha_F`pc6z>TIrBu@Xoi5-ycFd4gdTp-K-TwkmP)FfG6%tj}~C~8O=5NfJOLJbfY0a7I25sXGx7zFr%A2=ze9J8(wR` z8)OT(ZR`{>+9DRe~{>5pZ~Gfn`7d4Ng~vrS!`u4KX7*+K=(s8L&*rr*U?WMB#1#BvcM(LnjbNo z=P*zyb{xu-0gXfBfS_VCRg<%t?FVdJwsfC&E5nn=(^v>fg?*KJ4Uc7Q+K+QjtyUeA zqAV3Lz%Vuza3O1yDUSqcO)%C}{-IEhJO|BJra@%~rpTlvlx1S8M@$Z78|2vR)H< zh2gnEZE}q8;8j3{5;7r55@F|LbNp?#1CsVc@>N9IED&UdMKF>i>O8|cHlsUF97{?H z)zdT|@naPj5wO}wkU=t*fQMs-JhdbX<_Dj`0HeMpC7YpR5kY+7w%+}N$xV67Z=l15 zc}~@7-`DW+HKl)krVNPog90Mn8pc4jz%V5Y0(N0W>2;8?c zUwcC_CI(E6sigY`4f}~gx_%jk4Q6)-G%7NU_N}J^Mw-{8L#Zj<%Xb3^Wa_1-qc720 z`Es+H(fPm_WrTI{`}AGRy`{Ws``Ha&`3r3-Mc*LCx6g~OD65S=1Nm=YcNqZBOMne7 z7SajJ1Zt7n{L59)Qou2B`w~<%_RP2|*ZOJK+4*9E@ZAPNDzmGVR6scfG^|-Y3+e-v ziRw6jm6!T?-uHjMs8~R&c%E5l44VDBL{N;51BN@Xg~EmRmozCRQeR$OU;sUQ>tqQt zLLAB>TtBK6V@q&nc>8+L$uDTsd<^Wo#dZ%wH6d`0qRD8fno{hHooH9@5+@GI7EHUj zR&$UhpykWoC>aK{@)rhV)=5)T2JCC10UM2v#9btT9tsBI+#~;p!7TtO&ni4RcK7~U z%1+y*z+&Y&Z95_h0R@xMSjdJ(XxV^;A>j=4uWYeUz^3{!I{5arTP~grO9XD8AV7pF z5*xCOV+R|r>WPFd4b#FhuAW9fd8>O&Cda}Zfi>o@8Ns3!=TK3&BZK*`i`uev?qWaa z%P9cvB&@mTapI`N9qJ273?IGLx3SjUHj??tZyvatS ziyaezN{w^MY2Tzz#<;y=6metWLEpdXAyjE=FyN1PPhGpC`TS%>u+oNmD_<}}m&7&7 zRNrWdiB=NxH#u-iIfL!@V^=FdyWddoc+j4mO7}>Cn;=QmH`Vc!nS_Aq3!|HKISYh9 zajD-OK>glxLll*f8|$+wJTEr20(4zFfUAF_4mz7JuTW?jlOQ?3=rd=jFIwXx4o2U$ z{A(Zqn&AUhhj+8G&1~OA0@$6`K>oZ2np|?()Ilw9jUoF)yR;j$XIZEh)HfCD3yD2@ z0B-a7R^;p$(F74r^hfncP2)gK(H)h-C<`nz_EPde$E#E1G^kJG?{|PVYUl$Uz1xZY z`V&COv><014_03-dX20nnf7q@-p~WcDC=5)WsEO0Oft+P7%2h0ir?RO(TJMkGOh;{ z`{$9Z?RND)(5=Tpnd=7(g-rFtzMr2ZaBT#&IbJQ@ZMt^}#Ntj+^vt2wG{g zb+5?0k^18Zh=@9B%4%$zb9LhRmUVlib7vTGZ~)up8t)7Uw;hZ&Rh#xS$Nkp8CTN^V zXm$pL^Uj-~YdVJkabIJ=*xytN$RH#EQKE#`)gACoQ_B&&t`+_)`<5Oo-W!Uf@TJcW zH@Qhq0=aJZ#cFbr4{=Tp%I6U?P&(PN0;P@g@)qqSdn!j152NkU* zt!TqvgC^lVmZv;0{eiSfy?|_x93-LowKz#{r4vdsuYc*0&w}CdVgY6d_xyP8sA0Nc z-ujSZpN$CyA0#aGs0~<8IT&z*ktX$$F7N9q@yM}m)CaXvFa)0f#_p-9gOg1X-_0RL zgJb}#G11{Lm~P5QKN5zTfo}$&BLVvw_UC>b2xh{bFAckPpkav7?^_Jw*9>+kTV#0#@2-`164uAuq>qW03d6GjFl;w5mczcI?bH8q*r4ABAfhnBNrd25)<Q1`s&L5KPTr5bo8(s|~-@oN##YEZf{*W_8yNbuD@MgdWHL`2PiHDymVj96QSS z-7Na^n4Exs=gFw}_9_>lShizPoh2SPRa873OzzKEYinyq{!f69**QtHSfAH=JIYdE z*R{tm*2DPKZa@erb5DG2M#T-1)=fPNk$$#z&b0R8PCmfu-w z_hBH1g2rgMYL|zgPvXuJ*`V|`bZQYs;i_10)r=;#Vu5SjE3{AbrJMDHKvflvI_o#V3 zO1`7eF#%kD1BmpKI}_q9N#vo3D8487`P5rO?xi2^i`oM8&S<;=QrDx!sh%!+oLS|a z?MWgxe=Gt2Q>yJ?#O)c8X4bE3PKcGcpxfy|v}>m|wgX~$#sgadhMP=4J{W?qwOioY zG6iM&C>5ofd?gQD#+zI_J>JwOvj89z$l!+b<5dG}0Y?M8;j}bYb7R;xplW_;{^u@P z0)```2a$8*&h9IjKw_9Y>Fojwb-4GpaS6Q+F6zRPe>4BLi!$*8#(2+N@B+;{s_GN(6CY>-gOeEf^Da?PCEWBl&H;LK&EcYY&hwd9K+8m( zG1Bgg#oHYP4It3P4Cc4!5oGmn`Jhf5W|+wUWUszVZz5?zZ|~*j=H@;Zc>>q2?gI5Y zpYAPzI>!QFC<|O}cwmmFb=rs*4+Qd};RQpUbVk1<2ODxQoyPeO7v4buV_9q%HUDiP z$f=ltD4Dc!B~r@z8l!Q!6sTQbt>)foL;`<{9Pl*tl$CiRI{g3y7#@0W3zTb0nt^2J zjjQ2nnV=VdtoQQlb}uldn(znj+Rq2`Su^?wr4<^xj{va95XBjO5!YrFRs-M|m`sYq zI>nm}*)5_~3WKHXd*urZtC0>S0MoUuzrw_N&)w@fPl>`rQ1wDAdqAPXfciuj2-aCf zf}RaA&;Hq~|n4&1A#EN4^xDi`s`X*L!_8@p7ZDo<^B3mwBd<9S}{U zY~m$mvZkOKHeO6yA_KXtd+qG--NaF}Rg`0(0?5qy&Z9aE@G}Xz^spOk0EtyBl!3<+ z2_C%gv#P5*ZDxaY-C7H^Ro`pIu&-Fou5cXAIxJkT1aWxf`n^4!fhRSvySUB zAfhQ{t7D=C_X5vi!Fh8V=51HhI25k`!n)IW7=;)F{MYk%ae!1wY9RWgcMMDgSODI~8kFNz~FnyQ$vNVqy$*IGtI)!Nm(? z$mfm40I2cGI^SFtTLSQmeBY=?5EvGd=Dh*Akzt>$r;*BJ5Eh?RKTE_r_k!2&v<7kr zP6aOl<9^|%)F#6Hc3gTLF@JMoEz-w5?v_#5TZZ=r_(bbJe7^pCxTHv+@xk|%#+RtE zq7FuEvqW#09X>;_-qsJi+Go3u zyjK;>7kNED2|AMIIi3z}fE&!;uNGK%@~Syuh>hJIQ1s~vfOcX44b(Y;%vpF80V0|7 zOcKe~n=8YDx(>Xl46lT1&HkW5LOsKl2n4z7K+Uob)ReaE=m+Z5o3udDb{pe-TLaWp z!hUB8;B3RpfxBn}v`S&0UyJ?(ysr*)y{@$`}9$S7EW&HSeDR|Ek_u#NGC z0;&~D0fuMk3@Nfwuha^p&3#rYpy9ZvA5v0o=Gb=gYw-MQXq2&R>uchN$#`V&UoK!k zjb4!%OHam3C|#SHjF3T za#lWBPpU*Vh!R(>z$I>AVG8qss9SplXRE8P(Jfq!hPdX`$s~MxNctP*> zh?RV@KaAi=3*6B@9=~D;&m|HlO}`N1?;k|4R1U}bmJzo>p}eE3l{DSK+)4wMVxA;M zx8VRW9q{ac0q2%(a|nccEWZMj8}=@Sep0z&1-#5=OvC|HOsj>W+4D0JXAsQ4vG@zN zaosQU54a%8x$c&D(a^9np~BZ<;nE94MY{dT;nkdq5Z8?homW~cuOVidPKF!U ziT~1o2Vj_b=C7|L#0Mj^GKbOu5lI3CL2U-;pk!#lk)KW*tX@#!RF3Os+v+z?QhD8Z zxbS_WO3fc7c@3-~&nlq;Ee^xPS~69E)- zX*K>j-Jvof8L2k*wv|gcc<-qLT)BYst;&$qfg%_Ua>PcY{&Q~4dCISFjf`vvmw=gS z1$cyP{#24y9aI%@8!DKH2-IopmTI$i)f`gfYrq<%Q?;uxa}%I;@rt6vf+tO$L2vQO zb$x%~=!+qiGSKSCAk?rvX6!A*q*^PMYQl4fDfXde)luLHpkQOrrF~N67@0hw+Qjb3Lbz|srWyWivuD!XRA<>MwiK*#)A!$|g%UijqOjjO7Z}(%i6dZJZ zX{yJFLzD6XM7kiaC~jY_e9V*)G=srHd2y>X8$jgT(R1}&N-PJmY97-yf%WmB&in!a zTiLqxAJwzJc9xj}ZJ;h+tE{O^_VhK@4WpES7=-#bD3LYQ_KoA?lP>l#+QlNI?Xd~x zAFbMte=gB=;NQx#1cVNJD~1$6IUl8!xsN7> z+k)gTRwxgq*weCW$o(q^L*%c8n|P+O^POpy3P~S2#CWBMF?iR^1q8Ppw$%0iVe7l& zsc!%OOAnPD8L^?JZrGbnxvx#G`?93z@2${zSrDGp^@04-uWUo-fF|!H3*Sq_^ zOW)t;kNffH;V$oUuJ?7l#`E=jJ}*ST_ex$DxPXj+^iO5N5KWtLaVG2wA9s=iLq1Rm zNI$OdZwVW^{IHKY^}BF`?JiCVOK>(W^N%wmlUAi_Rto){(AVyws#N^Z8tH)SFGJ8( zp9^T535>j@yx=`E=>XE%!c67y4z8RoO{MhID&+uY8bGVQXDgg)PYgKl)t*v$D`h`lMJ{wXy)g9F z+D1y!W92C$%11H*L=@LmCD#OKK?C0G)hdsD)Cb)gxkXk_m0gKobkVyry*_(M!{G*t zz`pk=KUJ@x#^4>ufFjgHF}T&Nn9$ubDDP#3#J3(`s8D^4M;*u1gv5 z`^*ZN~_A2I@fCt6wt<+oVLc6@eDUtgX! zqepLmxWF#70jd1lOAfC?hF?=+O3K$cNt%)F6?C3+|6^5IgDv$t(?Xo&+U#H3Z@=mL zFGL}2cL=oI(q;Xc7ytX#ff0`~3_CJYO`#sofmDCaA@kz0!ZTva5+INzdTwVe@Y1)b zjG_iCVA2%i5bzK?cC`%PXc-Ci)Z!JO-B3;U0Y$>>Q^NBDZ+I)9t8k}RPqelNqOWE7 z9S)^$X+BJY${ee=oE_Py0{SaFJOFR8{r>qs0>W5x3^6QuRxJ+%AH8>PF?{1If;cu{ z?`@S~M&pkUqe9n^;m#0826J_5pAoe zRTpo^-mO|OOZ?)lJr^^`$EdwuYlkl>(>v0LYTkF+mYhfbd%==2UqTe+vV>;}s{zm* zIrv4Q?ZeBkCGGu%qFbx_2gY)3CCZ>N`3w+dm|mi!#81QSFBL42whqA24^sJ9DpTD| zCLvlZsA;cFRpkQXw35wz)ThiNE0jY6BZ)x!e1UuM1yLEN=;ciTnCrjhT9g%ktwp=c zxkxV(@t)(qg!hqDd6X8{q1tEKtk4u5rPXdW1kPouo5^NfyvRroxBn}I6(PHHa0cO5 z?j9RQc(zbFCLule-mwHsecXgoKG{Ihw1lV>1HE&prYQ8|85>-5ae9IbL!0 z*XGo%zGaaxGmb7j0zbAapG<3nP#;(J?Hw#AK z4q&lS>*jaQ@q$)yt_Z}Mc->T}Kf>CjJMOhGSO_mwwfLvRd#k3af8$dwOxm?vy3m~S z<6A=?G6Xz{qj$cioRxBl0c6S3fTXq^ENc%-ZGkYIO-4mu_-F~R&1|GJ7g4ED{?-dz zwut78BK2B7)F_+SkzO`kmGj>bD(L%8$nG$X^6C$4zs$hJXr7>#dnA|5wS%i3(jPaK zNObI2Qq#UhJKEi(*)-%n$KB5-m0KM)|8bY5c$v!m8ILfzK6TywV>?Im2)p1g;q8Kr z`me~b-xcuwX)l~yOfh>)ebKL2pm0arlL!u%Y2Z4D3(Pi0wag>QRsON_ua~`FCl1P zrqSl=iXk z6(0%;{BxF*PtQTGra@!Vji{+Rh|niYjkzf+^Km!;ZqS|HeaO?=Q}>B?SYLsahhUOJ zI{igPTBQnY)RQa^AFJx$_!$!W>Q3zTIoy20vH6$9=0D9TL>iemAm3SP54XL&RdO1~ z=<)M`8Bm~2EOii6Co^X+&^rV~j*tBO0;Q=xb+f%3SH}9WD(mrEGgAvTt~(3LI&CPR zru@5M?D3-l(90ciRY^9_I6`AnkPB?QZg|saApAM{|D{k{!t?AnU-dz<+%UefF(7JD#@x1w3Kzx6rRb|22xpp4SbhILxWp z-7GkMGD_5~I0kYlpY528SzmrS6@$=cgtO4FUiBPUi@bZus89X8ZvD;QtB=3R!Wx3> zd2-Fv1RR;F|Bg0d5LAp~n`YwFJDl|Jx2`6XV1P29RSWB+{wjjL=v zOgzmESm6LzzbkD$&-E`2~2?E2v4-|)7GyR<9 zbtX(Y+Srtmbg$CfNAi=r=-RpX(JVT62#v4!oYq1^{L@pt%*XGq#_Mi`o!~QH{R*52 zOPYpqbFV)yW*3Pd9g}qvmeJ*JXC~Uc2sVbypAca2j)h@jj6>woM0+hLj2Irc97PsR z@Q47Iac&)C({LfHEfN6hI!i_}i5SPmi3dG@v23zoO4ii-H1F?s^Pc3qbVw!=>%ZhZ zTo^+vQjrtm556Xk*N)POa{l6=s8LX!y5i(C$IvcRxOt?JD8w1jwKSHNZ^ZD=$^Pee zewBfjV&p!xW6QyJG+@62Y62Ii0H>f@DuDQ;PUC(Lj7H62aMm-^KnDL95ea-3Vqa~$ zH41~{qN**g$HU&Ys8Hwxr8Bkn|C&P8sl!9e7(4%_#uPDILyrNbw(V0O=>*y6m>Qsz z(=Zy(gw3icyl)>uY$~mMw0n;`Gz?DdOm++>lu8`5V4TpX+f(i?@|TNYpfKQpMWc0h zcODdE0w*2LX;CofoQlj8L5ZP{z-xIMVlGA`Rvn+W=9Ssl^l4w5%kZN(f|WXR=o0H` zT8!T(rZbCHuKqy=lJ7o=W7*~nag_1bFG!saAv@Q!=_b^4et`$=kh|@P5=JIiS|*U*qbzuc99-BtxHA}10OC^LmaWpVK58!_eNu< zeR{YgW3KluQl?$I$;Ee(Ab<5fU8S{u)6=)epa1)@I66v5TbVb|qS?wr7?h^!QejOjmq9p13D$cY~uu&qk2$ z*p8=4ov9Q{2_}7I`QH~szB8Ob;4PG4!5@m*su-N&&Z^kH#GpKZhftEGUn|rgZVOBz z8^lwyO8jn{t;liIEfQ@nsIVya_Ce(|wKw@5?zsO7Uxf ze?5ySm_oPE@W%kga>vJEz3~NcF_CzB&>k2tc8F9DCNqZ8{sC2oFYV?QbM2Xcebj)= zd1pdZwI9I9#u3ct6y^{5rNS>q-HMZs6|D$oRUDJqC|mHu;{5@C&IsUW&oix&4WYQY zk(i(H_`TgSIbI~TLX1qqQe8jn!UkiA1dykWwOXwmrEMQprA+xqVy{8PC za89&f5&8lCC@`r#f$W{V$QE;MNU_q#nYj=XYaqvof^TriL3YX5=mR^R;%R>V(QstC z=7D~8L$$z`@4&MSw$?# z_U$H5IJTQxYa?pH8SC$CwXylb9Rczax+srx{MkdhNMFS6@-dqq+5;8ye|wxM(3=#o z;N^Fk(*?4wx>d0|pp$Bno~+|HbU`BE2$HHeWMqr5U~~L<9AVR!zzpWAJzMQ*_%$D7 z<3L)afAMYk^O!r$GF=6*#P0np?KFR0Fs!xpal9+}&Yg8fh7@5;uK;byxqT_SfzuY5 zLS6QM(ts1%?WVG|F|=K5hA*mV0(Zg~n7WpKor9dD;P%~*nuJD@jsymu(dJmZ5q3(o z0VfqUu|CU#5~_ak=bSzg&w|s-v8++KYsYi5v5)+6ncBh2b*d*P2er>O;A$MOJn$|n zGc$0o91x}zl$f^1wCsYS_^q`4as)_WshHeJ!X|_|)ME|RO|bY&_i)6H`B>T8zt<4f z!Qj$v$}T7b8c+WD;!HQlqro6^2JsR@X7gw*P`39SMxm-DZ;${8U<20cb|93Ffvt^0 zCKCM91?D}kUQ}abAYeq&X7XxPWdC^2qJbk`Yp`Q*g)x(ZOb#oT=FSHn7K8_%xz)TE zc^m%*E~ZQo&vrTCCPX@J3FC7-oU*slsoEkV|GP|mIy{tGDy?(FyFe&h% zzX17vD0XN?#j@21!^W-XHhu(-aHB#Yn3Ew`)CUX zHCDC?8gI=%sYxV$<1GM^zuSrSB^Y}KByNP5&^2r}?|=_|b4f7Y6z8(~y-r9@ zT77CGbLHfpt%JQ0XEjtlla|vg-J1{+^zeK9Q)eybiFXf|g>Amp^)FXF9M0f1{cH`o z*JU%sgN&m6^KPv-IOW3sJ`Nhsv(+&;U zG;nN1!_H;ke-5u)auuzd){UkSp@zJMd}!WX7wwuU#FmOI2wW&Xmdw_2*Jr~2Gmm>Sl9LS4b& z$o}k$U7oJAmDWwi_~ek+{LhA&kPz>QZadSNWjy|5hLPJ)W~OhVJ+TCaH>X$JuJRc? zLS~0E(0ba8N87$c#zp!LD*)KnYCTp}!y!&}jawrkt1z1_0kOu%zAG~hMz0^@__emd zSsSuOj4=OR;QE}mSraK>0kg&b!}dkGP-R&d55 zCVI_*nv&7ub^F_^Pl7T3?QG;TGf6MRdnSgUE zGQawvb$=ClQCm=xzXj@1G{TYn`3-hFgT?W_=~ynk$H7u3@Bk^V%ZQ-T_hbFT^l}E) zhJ{CX!*DX#SBah{1|8E7(yU84s%EkPc=0n(xLPATF#gt87{l6P=GsrOmEgTTwZ23| z%w5Mk_=N2zzp58lPKR-7MgE)rvuL@z?A@?v3@2~<66=QJho0rRPa&Y{n!0PgXXO3w zZvtmb@-h?vtPT>#5B~R#Xe9Y|V~VC><|eA4bpF;cC@~uR36+Y^`+gc0rCjp%Ym1AX zD||3IqTZl9t_++_WQ^0740L2Utg}jXD>7&|odgxo_mF86>p^hwtJr{e_fp+mV6n6^ zf~xs|NLZZ%jLTx&)}}7D*NT7?_<1zFqeiIgL0+wKAV01Y@ibbJs8g@PCeKhTnntp3 zeWeH3uH~HApYs$l{`E?kt|q82?3Y(`bco+cH(b^#@|CJA_fCcc?~F}>r%Aaqxxd~- zQ~f@MH2&h5%-@0L7S{LmjW4GSfy)9&`0P66t}|Os{JG^xGIzY!5pgp+lspLY>XHe) zmDs~P2&!t9!N7jwa}>6dz?lPQ>!(l}%(=h258`vx5PnnZroos~1=AdV}>b|Dhh z?dX~T3M%CLKcW2&oDPCqzkZ5EMfG52+Pc>J$ho6`(#0+kwObHUK`B7z^Y7_oIzf?i z(E{uw;J%2ZXiQ=uFd0jr5?~p+5W>nLA#*d4a7Yyfa{MQFgR?VVj@HrmKE$3EVMC>A6yPCvs(+7nwHUIlrka7jxZu`H@3Pxjvcq`2k4AwSt*tT}?~|ZiU8`eR+Jq6>Pw_OsUkcrT0S{TO$jK@d?v&V$VJMiY3RA(EJ^$cWj&?A-;ugfkRCuvfVsjakw&S(o;;IjLk;!C?v`H+K5?fa( zFmgyDeh_o}EV2cI$nOabpHk5-3}+A?lGXnghX^n{z4^sMSyL}6rF*4K1E%1@d3D)I+wHuKCiRV^D|w<4 z(Aiu6>LOrV{5JB(^F>(=2=l@e-Fq}0u@lY}$|?dqY??Aokc$b1EEI-RJ5t*2aW2iZ z!@u)`=S8MCzZ&RHieVXtT2xT}05fUg)gsABuzH@C-`TvsdcjQYX1sLor21Ze*zFzA zCx37}>(yNT*}wN2D8r_>k6_N3aCAwYmbiO!L{516Yg3-2#g80$wFrWY(uD)|!1SlE zKdZ0E9F2C$J30j~XqxI4stf%p6w@0ou}>#O4X3MHOwgpHG{A9n#mJ2a#tF%S{%A&cZKdbW<&x3OWd6t^7F?n_Zh44*OnDtI$2MP z90>Aa(K}lxPdgT9Ze|<<=YKaho&w|-g5_&;Y1lekDtGTAwW4!o04j%720Do?@x&89 z8B8;qLE)J4OOR!q|6EKDuZ-`|x%K@leG?=?kF1L%xOxZ^-fm}v6RCxe& zhG%)}SIQAx_a5pitwsmITIPI9(Y|A^1cmLs-?3t5&&h$p#1i&6r?+aJsj^YeHPN$b z%=t36y=9Q$j7ZmYpy!A3-|NthcVXY7vJfi8L8oZubn^wANFfZwJrz`-X@)`x+FUtB zZ5}ZG8$!G@ZfJX6>LnJ5BEZG^V$6uUi(yn$Yf?50F8Q6CM4jy)`avdXje{^Je7Hvq zXO^JQt>ZfH&Cj_#yS>#iv6Why(|mvm8^botZmW^8j=eLMySlfhXV|u>Ws{=Ja5Z0a zfxx74A@v~`F*IAexjZ4ts;_BQY>M|j0*53+F|63nDl{={E!FDshGWqp#$7$zZ*l{rTNn{hSY z)_M3X)V+YReXcdnZrh7*xYlchaQL9K?!Jx1lr(v7Q5N-Ee9wLk#(w!Z*Tqz=xp5YA ze-^2pQtx4cM%NmjuEf^4JZPo=KwpapLkv{dz3yQvNg=R%UZ?2x;^yy}4Xs+gf*phrmF@^l}YRyNZkduQ#XP;ylx{Xt6J6~ud|i<)YgWnl@x zj|SeKxp*S5wHBsCx#U)Qj2J6?QV6G>{<`9UV{ba8u6HN9a)7gFEq_=KIbFD%%uFxyN0b`Lvbjoc2S_;2MCOi>zop{ zlWAMOUE_`^mxp^o#QNjR7ohZAe^p)oJxUaoaq6@#3a??VwUEEO9^F)25BklUa0-&A zPFKqOUb9`K*)hAeSDTjQcP;!`UJ4p=8KuzI{vqi}c!N4N0~@e}?++(k7Jk2V2YU~T z54xrnz;+<@qFQK_^4bX8LdC1lZn2!4mf6Xm-1$2DyfU8(Z3tah)w`@?Di==J6GB+! zQ02WZL;O+B)0F}p>UmYpdGWQ?1s@gT(V6=(i7Bp`+1{Z|DGw22QpR@1c(qq(ONrav zlTNEH)3Hcaij*%6tBUQ%inyz-dWDpZTU{sXmD#e>nCR!j}L)Kpk(*0osg z%3S{r&c%D~v+~A_!+q6HsaMWtV=STKk&0LNQ%*G29`KGJK&z7kkHBA~uz%ik$fFsYFc`H^}I*;4Q5EFq? z^PW41HoXNWeFHQW1fwuEUrlTwnkX?Ip;}QdfcJxYdPIAn=C;^c2hy-@05r#ZL@}08 zp~c;R$8dU{Y+sVzaEbHVJXQ8N<#1X-SL&4GTv%faFZZ9gNAu{BEnrjof1m7Kq$6jz zz^iRY4mU(6e}$G?QHgjpMJA=WK43WIK)ty_y(Wgx{8~g9f7vu$5tC)=%Eo@N+8zYf z<)jt>ZZa8eG-(azJh0tE1<#J-11p;My-=djaP0N?|z zvRJHrf3hOWwykD}T0#BmW19el-na#=n}5EK*#UO@?KjwT)!x~hq3Ow(7nY=&6g&Hq z8}e}mZ91>mul)gn5>9wR+x>8l;;1N)17pxqd7MT($))M{!B1@l`tF6kwY5>Cy+{N% zmy0v)UKdp`)cAW@XB_+4_%)T{71E$2PrT&D9(-Ay)KEWD#W>diV6nxX_nw~g5sC5U zm|{`zI+5!s8?hpe7a@g;WAaoRl<$|+SEPn?K#FLUpE3&b?$wK`7G|duj7cd&^40;R zfWeWWhS_H}ef9Hiec2voop<(l<39^@(K%eUwuA(>&RIbBYhBO`Fp^fD8k8PYt)uCT zEJSP(g@MZI*69E@<+&zLjnLC|0igZK_mPNsUR4SgPr(oiW8_5dJks$8YIE!6{?#P- z0m6E})c8+XKr0Hwab>|ev z;-otWt07X!ly#uayOgUIZ;1#4<pX_Rpa0i7+ zS4=f;g>w9ssltVgA1)qmsKy{m%#SUAQu$Q6l;zD_nI9uBp_}VJ(GQ$D8<+=Y2?Gd} z&)w+j;-|?7NQD*vqs5&m_j$5t-Ss^@M2vA}U+FiJh4)Pv{Z~GFW=Ch<2R0N?1GxYSYKfH3Ej_lK*3jS+J}jgh~#)hd(t*&e|(F4z$p>e zqEUr+(hnE5UkcB3kDYq=UI4X+lqoQhbZX_Jw9BsB9w<&es+^~mlZj)Js3U)>3II`V z?yAny{V8zb_o*|uvO6?MIqcn;*HfLlBm)^PC`zHC*C;SA$S%VjoVwgCs@Rt*1g-)t z-O90<3vZ}U%Idy+DlqS=Wo35+s`l%Q&Pe68PEb+{hf=7FnE^zSuhF7NG&==lf*dY| zQU$Yw7+DN{@62Kf8A#>($bc}6wOUk}U5!n2vA#gNpCgh`!@#`dw$IYuW13&am%v+R zb$AF=N9BI@cbm^c%>H!TKHxF0e=PXrdjy0&N96S$8B2kW(b@JDyj@B!02(Vn_)-f{ zB#&>F-L1F&)kkQPnu{#lSgx!Q?3`gty*Kwt#4aZkuie==qbnb${{g}bg znt`jY=m`RfYNPk!UlpSnk^9TK+(N5F>?f6U98@unLfh6rU{?}XYSXF;vw#7Zg@46^ zd8CIqNtq4Je_jAEyq&bS$H(}(>Si~4aKVB6goD#PIf*SW{FtrS zq_1_3yV}#}EGbUSW}M!uyV)J6=AR_mcw&LCcn@221%0g~UYW8`c$y@2*RormSX*v~y zo8W7T%o$#L*1Tp58rq-}b;lBrj&%cYVmGqvMwtPA%(+AfSI~3J00O*hCTu3Fhq0{{ zb_o5D-!f?)HBLLhrCI|`%DDh>NL-xxLmv9+wM8nE+vnI>E3P~&Yp2_JPF@5cgRHD^ z0`#J7fM|0Fb^e>F6apVf3$c6g#K?3V{!kbAp4?bO8^cfm;k5abr7gns2P}!)EN@eT zvTGWfxsRj=aLdl9MmvT(Z>~QNNPqDW8#cWz&({=Nv?I$IFz=S$Z#3qHQTZLgAO+T) zhZ`sJ4o<>MK02E=kMrVg&&P+7?!-Sv=7z-Hyi^?*6*HCq6gMtzz(y1zz%VIzA#XYX z&gv31p>74`f_}|Qh7t)MdF;fFk)*+H80lOrte2tv9R{E%jKI5+?e}_wNh(6<)reD} z9GN*1-uUteSLu+?^#(y0RrxnIFMoc?pTAVR?}i!32!A6ZFeN+yzu2xw(T{HDFW!5O zseK6mdFJ-KM<)f{Tu1=)bO>gVcVIjA6Kr;hK%r9t7^}mE&y*+hK!FmK1d4*o?MwCz zc4eIeN=Guvov-E-7G90ekVwtJ-{mN5H&O~sr(y31JO%pgLswFS#*YjfyWnz<0w8>| zczHl3o{y1XhmA}m@9s@MF}-rrmRqYLzka0*ibHxM22y6qKff^WGJ+wGsi0|he!UO3 z#BpYsI3mcXM?ly{GxR+7vG_?~B8-6zSHFNNaO#ZI%w`a+5TKtgMr9&lZF+8g0dTtf%I0zwOLuYRac zLR!~&e8eyQ8t?)9X@hu+A=h4lUIrfpevoO30+1UGSUrK1(p+${%hzIE7>q>cJK&}o z)kcNGGJk@wUN~f z0;5Cg`Lg%*9ylY*hhS%PZ5S308k42~Q)y`Q*L0}owios{I0M_`EV4aH&HPY1zoXO{ zYlzt;$hEvs!ZjKdH^+*JNr|*mk>(#V2HfNXu>CGwHvc9u0O!Cij?@*YrJo$iZ0t0e z+B^#(Rmk4e12C4>IdhleRxdO104I{#leRJ}4R&tTGF|y_l1lbo+_^@Q#{jLJut_}5vU79p z4|*h{iYZ1KN~2>BqQ=;0fDyL1zi%;Lz;WP;?CrJ!@r& zBd)+i*!??l$?gI4Uys^{Jjy!!_}QJ0dhw_dps8@PV#o^pdc{X~FJ3IUjtG)~-p)(B zA*dK@oUsyd;d1nzC&$PQggxO3?k$rF_GbQr(oY!?_sz~a2j)hovHsg_5aD4jY>C`| z^7}{d0WY9Q&|3Q_kALCT!W1`0@n_WGo4m7MR1Wd=_2N)C7g@CS3KuQ%FA%hGTs@3< z-sk=}&`QgI{xYMB9B5>uh7|PTYzV0Ft)#m9_m7a{ew_rFUH_{8+*kD)wGlXV1(vP^ zM2Ce4SqzE(m-}pKNY>8H1;;qPnov_b8!S!9sDB3eJ%zBXD=^15ZaAz^zDk5Y)rMGKe1bjUvUyAPabRwuJYH?u`Df`eqkrU3wM@Mxhcl>HrhGVFVfeGUOnZ^<`9)Swa>s|ExgbNr zp`L3E0zG@63EDu#XQUZ!+MTIqxq)mKz2X{g|C?)PGkfHSmO?`b+c|RR0li=A%G?S* zy(>-T)8_BIsq-pId<-w|$L+`?fsY~VD}wLmxt(GEFtaCozmk^>!p=F$*wpzNHKUks zvRxpH@|Edwif_X+kuvwz+sow0l*#JLe3Jo~FYp~{qn+CaoG3x2%Qr38I{f0t%G@2L zl0qOvr~QM6eGv8sayF1-;`yE#NPf$5|`ag5`AU6EzprEOUJ`c?YL2>u=C&J-#b+4PMnQ-#S3xy*Z{ShCiLe0Fn5WuW{gIPi&Des0RBw_dSmBLO1;3ykEn)17kW7$ym-IkAI$P%ST)OO6f%X%)d|ZKS=&$9# zkOKT@8zsBO4WA*`ss0p`^T)y2qmSe7*6~ICb!d1XiFo^jz1;bF0#A|VhNX7eZMJx9 z1!wGr7mz#yR?GIVF!nGvD%yZ+A%1jFvs`7`P2zE{Z$XASGNhhwK&kK75?rs8Cd&Zn zuK~-3uz4^RubyM?%esm;%dyZeG;AC9O#c1SJx9*b{5+d0P+OU|^YObp@1L`Vky|kc ze_Mg$>@ou}qq$2GVQ%q**Q1fy#KU%ba2i{id>YIYxXV+vGe0=_oyn#4eYl2SU;jMm z#ndG<`{^rfjh(twU58F9UNYIB?ux{p-st|rxPf>aPXjA|IL+>_eX)4Py3eAZ2zu!v zxXNP8;6PB7u%X}_MOC^GY&-%qbG2z$08GuL0UPBH8_kE zBexzvA}VIb7ru9cV(8xoL$t=5MVe0DkJRlQK$aYJv$C;s|FUjCr-elfciz1r3$+}M zaf9O|xE+Hr2x9j)-}3Bz+7S4A=lv62&a~pSgjGTNRS}jNYTUSj0iGpPjWdFYq9B%> zz{!UnyD1*mO7YWBRiK)uJHansUgl{ek|eyyBZiCLcUqKn;BuLa)YS`k)kCdiFSUk`k z)jZcn4ah%8>6Yb{bbfAD8d%Iqjz`i2`k6SC7Kz~HJxm_;=`9eVyYIhP3TbVILn_4T zg=^|G1a1->9|Bi0$_&V>*Il zs}U_dVAuUwj|sY&8o&pr5 z`qPuxu_&_0Qg_g-loUlxir7Wye=%x?@bb z+F}?V|4#p}4ac$@^A>?hXFx?_3ki^O?%Yd}<$L@#UnNz+g65k>*)iLbO*0~+|GCNO)-KaGLyMM>~ ze}SZ%tP}68|N3nGq`(nk>4N!n0a%!>d{&e{j}d?tBbQ&RA%t~QwhOr0T&$a)C`d<` zGT-+`W9aV-wIopMDAOTm`*BYw^I9@1j9wO)4CwXh5}F+H&}$(_nlm?=?0!E-oDF&O zyOOGVzkdWTBJd0;;%A`>28Jsz1uTK9&RH-@2tN{W%@2<612Aa?u!?irHt=If6H{+(-Y zK)QS5O)qlwWccBkp$d;rV@eRsHQSdW&F~Lq%MztCNJ(G#iHnseD7NEqjQUvr0EhUP(>>=0MGy=VHYvmCffRWwvi zTt5G289fg(!32tg-}Cf;p8!dyOv)6W(>1zy&AVF>?~fS(-X$lwsD-Dv_>rq{ps7cu zKFx?%x72iOGgII`&yNm254We{!f*vLF2BN>1jRk6VGK%$7q|D{6b;+uP9xMTzH4OG z5C41w2-`wP2PKp~VT!sY#W;!Fh$P#E&?N`J(Z=#qf|OJR8N)Ruvfb&#RjYO{T(E3x z#6jL)jnU4%xXbhZgQ}qzsFNc>rZ;9lWI>cfQCA+?d^&v+OyLIY(xGE4SsRNQctSz< z%AHE+i8v)m)bu?TMUD0?oC%JM`7gZS*BhZZ20O8ktBw3mAq_BuY#fl?;W|?rxW>la zUAD`hJuq1q~{VH+VJ%L?MX-jSNOi1~pyI?Q45p#OIO%ZWuY z^B)=40%)E|eQ^}uarhr;^M3sL{@1)RBA?K@vn;-25&QZl${TNlesbpF?8_t}LB z%CD;Wr8s~iIz|_`wt}E|JZ$SX3-~q6n9N5;&Kb8eC^5l^Ute@(>lx>Z6Q)d zAgukof8%P|2ZV7!21O%2AA!by}fEUlcE@0?OY8@k#g9v&6xr}JV4KZ1V-s~8`{yUF7`FC!_ zIjj)es!9-RawzCCTFDZ#->bJH+9^C3gv(LOe_x^9?Rn;0T%sqQpMz%!?PhLj5NYv0 zHsR}CD7-7p@69QGi2MROGVDj7c>=G7BV@D0r3(nB0>&F==cO^lh{7C(l^nyV?tp<= zes7I0A~Sl=U`;~$>iQSt+A@H!7f?xrubhqDz~cF7e^BT<6M|8KOhH{P_>p)J-FFLB za}xz0;;nzWt^ac}D(RT2L#avr$+ghAYr4*Ve1J(N2wY4{0{H~ufZhkR%2&?M0V)&n zcLAw^)HJ_>5&bqOV<)@vKF=e77Yy87O9Yb}=MZjt162#S3kAqyi{j%c+n6?#u}|Vp zD>}lM-4fQ4a-fNi^o92$v>a?Z*DkQefx0~K8bdYkUZ-=2h~+a(wnTL}OA+;+v#iil zaZ-{Lv&~c~lTEA05X-W>DK86}op}PuRe@!sb=FPbTHE+ zD^mZi!P-rFBRy??l_3IYXX(O;to4^2i8G1#7}+-1+!dA?l&z0{p8VKg4%pZtG+GNq zpdhdWvh4h8X<_nn!H?W0y13o+i-#5?o^NZMApJi%K|KFZT!MA};KQ+h$HtOpc83y_ zeWDsa9N;Wuy5Hh~+}XGwtE_7`6K&f(ua?)V5|-G>n+`YkDdw9wU9eU9<_MF3Nn~K@ zagQuTdRe+{1O(+aAV~0tkv_nFmXjQ7f|krXH3_{f+_bEey%~{>2|W!GXco(GH;urJL^E%nnQy@C{8r1(t=`j4KzF+4kr|!I z(yby{0sjGEtU26JpJMd3r2#yxHeKO5?HBzsN}X#6V;>nHjiD&TEc6LP~!hEnu z^@m(i3>4_vS;a71b3;p^z5F8T9$W_5ViVR@0+&gyU1o&QraNVyra<-JRK@ld9+0RR zV0_PG^4@O#WU=jCUsJ^qf%7^BiVb%wn*z8T9vdEaiTs1a7bCL-8P!d6d5_XZyCM_f zCDT5p-w_5*#;;A~f~XH#06+cb+<0C)%S5N>W4s#qHvSmUI)km=y2bZAMT{ecGV(O5 z^?@!r1Ko2&lPT4D_+8|V#$pJs7VWCv*U(=KTw@;8po+*$SJhJ|u?yA8 zwudtiO^+6gS_WOIn%V0%53IMQ`n7&>uYYt@0*mjxWO5MBlkI$1Qs?G=QPul%2x)KY zT1K)3ss?nj^&Bl}#1}zh>=V6gZ6{5O*){}q*=&WxM#G9Ow)0&$kpA#_!70i>Fhd?k zx-+VIr}2}b1$ItD(6_Dt=h4jt6oMSl)s zEA*+FW$n`G;P1x9e`rn$kHlSv&fzM>QU8WqD5GwOGe>s%D3M~Z&Y=(9p{o$C{lkFMzq{%_!IF zf^j4;g*-@Fx}Vo<1~zSO+n5HjAv24z=$dn*h47b(deT6&(qAOHM~Qh1hVE|5=AXzS z+Jk1o(dx=*HDU({Kzfa$*!C!mz4g<@K7h%Z-1kptKa4q{RJlSym#1MWn#bhTxGk_@ zog^HA!Zf#{WLW6;5S5CIjJDxsp0klGbt9<=?{@8sjq1 z;Vp2bq1AOZeKlNa&6~>~PoS%gBu3l3xI}%@`L&-ciU5IOpyqm~NQYb4yeM=fR>g!z zRAsYQ#nww04GBbLRjqIEmtp}5 zT8w0P1kdk(TP}Q&WJh`G`4wrDPu+I@7tl{&*p2eZI>SY$pU3K_PoSZcsRHWE4W1MO zp;N&5r$FJ+B%#wujn%qtZLRZNa93@{N=jOn)0vRiY`L@EWAO@wS=Y>TeT-6SOk=w) zhh|YVcDClL-%|ETMLUO1L_V89^~&d3q-8!w}s)fvq#*Z{ZD zgZfK2{hg3*R zW*BegKI>3cdJZKARct-sR9O6L<2onmfVf!QrcPbi&N+uFkq&2k(3!ReW+FXI_uWJ> zq-+uz*!uW!(s9Gy)K0RnVL{-Y>x`+2Y1}xn?h1?|TRaTiw$|5OS0+2yN4)mk;AO@K z=aFjXW7$iVlmRrpEt`R$3x9WQb-E-U7dTAG_Fzo^%{F#t?iATzDoz=v_^Z8hJA&qs zkZqD2mHA+;{9G;l65v)!u_npp@>0NUWxJhfUTfTwGTar8J$jvM?2Fr6F<8kIhH>PT z4&}$Ai`WFCrtUac>T)rc%$BbEh7Hh1C;rGlYrf6e-+RNkLnwZjHX?#xd{WHab#-`p z5bjjMbo+SV4ly`#xu-X!@S5p+*KkzxE}njTZ@PUgBHKB%r}xc`1HbA}>o1jOVqTuQ za2&>}$j#83z4&!a5P)}Ov}-2p8NN@`9Xa3}=^0_JFwxG3JB<``7t~4KP1~?t{ZYa4 zZlt9Ab$MUJ+tuc070Q9}C7fG|uD6?RY#;w5_9{=s;!Kt=Z>Q^z4LjYdEdhKg15p)k z84SdJgItpQB=a2&w+qxRYyY>RpCZNY5CmRR_Bl0dlZhV$9K!MA5mf=q3R}))uEwxS zIx}!T=uItmfU~f$0O#3@ zK{Q6#s9{=g=See3wYOM!McFk+UUQ^e%!%k)o1 za=pUA;%>_=>J#7!(T${k_vt~HUajXj_&T4|q$atk>oak#{Tj5!oXE0B`iV1`ZAra+ zoY?RDWJ>Y*u;hcF>1b1RGX*vOAz9~LBV3BPceVqtvN&Pof zf@L~y$y^=s=vhw0ZQ)+vjN|7;_0~Jic{!Lg6R)^T8@*WitjZxQ+P@eh#XK#B%aGxZ z8j&i$@O{kz`z+ptyDj&_x#sm-D@^U!9p!ccucy;iP-rKe!Q6knLjsv6qLmW~WQw{= zOd}}yV$3%;9Y?5Z8Y)x4buygn-JxorRm327nnc&E!MtqaM?pL)t&J?h=7V~a(kG#J zXqD0D#$*;(#*f~La+-;q@y@>Y_J(yvNNBVqDGswLhzO^K&bM#A@m<{1+JdvUFIN8Y z^N`CYgkCV^@CvXj-osh4H!MMS@j%3R(Hc>v28Azmw5$L{{F`OrFj?%{#heYm+Mchu zoKu|LyTIl^g1%u;EXP-WGTC6#y!So=D3?-vXQZt4<`u3Ry!~Q2w#EkkcG{zoaUhaH z{e3>B@wLTzzg_mk0Y&3zQ>&Yi%MCq6z1j_tb}tvL6eWWF8k|OH%#yN*!v|1)p7BZL zO93B?1x@c~OzKQulh9XS4Pmf#u*~oJhTD&u#Bt*2WT<3@<%5%R3Bk-wS7}dXnDhl; zdBX1v?K`^YN^~!2y=Nd)&?NX~$P2AW-PD%1+)GH<5dR$`Jaf;9`} z-)4CHIKDXcK0QZn%`A+POIc4P%f1;sxao@6Ed&)WZWz?C*{ahzzZ?px@-WuwgAS_* zaScSA>+Tt*WVay`VnkQ$0_r0zuw>HAH$WXbV9iPI`^a4?+Hk${+*#vitYK|>PoYR; zdRl4cA|iT=-FDfYy62d-_H{CIt9D`BTUj*U3NLFa7?;XaxX8xe{Ww@h6WX&LIl$cwejSf8cNTp2eR#Jp4%K4BAFqtvt*#=7{N*MH$iaH9%CdvPmT z-Cp}f`zG^8R%TQ|eEicZuep<{Q4Q;xZ_9aO>11akoB1dDCGWN_oLx$GE0eL1jf8ts z24v}j8%On+vpNbd{X@9;r(57ah7R8>^jrUp-=ELW7}yh%G&6Op9K71M;8r-(lFc>; zV-e_jKRHS-%eEcUaoWS9(1+qS{Gi3Dg!>Ng+&J(J5MzUU|L|gP;` zLJtHoLD$mkT87*=EciXPeoD%gUXG|Mln{$WJfA+dB-SJd5_1y#Ce2@fiC@Qr(OUo| zMa@bD#_+M<5QAi1%l__g0hJoF)#{@kFLUCtp`->}(H5_;?S#cwQikM^y+1I|g)^v3 zXqiaJwLhd?{&v!aV7S@;xssJ+~}7psB0k56MmjBK!stPERK$;ZV2lTxXg zucpB=UAD5%eQFZS`i<9X63Q`$#jIWxB_HBKD+)(EzB86WS#Qsb3bcyY;$dCfNbohu z8oc+pZ~fKA`1Ut*H;W9Th(Z1d`bo1$sS4lEu{B3n_dB~Mhz4ZFOv?~HSs2h8ekxP` z&S<2HCBrTc?}mBnCaCQW0UDx=kH z-fydLbg?XqV=mX=xeH*cwoO9Q!Gg=+_E4Nr(dj5P?@g(SN_hd7l-D8rfWDXuIXqkj zaCK|xz`U_ATrn`M-HDb<24v-@WBRj>ZZvkMG@B z5e&m6t>VwIxFwgfH{5H|^Q4au`a7N9rK%E3y3^xjte|cYXMcI+ha^i)&tY^m$mPD| zM24EEYsTArhL4?80wlu$xL0rj)c-=;!pVg-)%H1&mGn`YH*oQ|8fHvG{hiYgGrt}x z>?#hN^p;XPXy`zX&~H-IZzh@%9wZ)JA{VXU2aC>zay2_sa7c(QqMyn0AKA@r+j zc)(^0Hsue>#ZY?aI;C~{CVIDuZ~X`ec8B7g{{$km_89^QxUA|z@ZF8j$UtZPV!2e0Va2auj38*rwSP?TBKW*XTcVEAdTizrNmIK772--phnPm_UCCE!z-OW=XiuV7wOz)o+d7*DfSMjW3?uPNx zKZP;S=z#;YP|n5nPh}_GODixIq3=4Ogjueqq>{OV)gfF&7$N*#dnE3~)U{p#Z-{7_#_JNemwxhCXY`gq zalNq;pBattY-!FV4t5nbM>a(@BBAC4dePt>TaGF&OMr#_)q`z0p^0-Xcd+jX^r45@ z)R3##wgsORDA}G__%wdKh6tq6(g&Q&SCi7x(SiD*mz!q`mjXTUCk$lv5%a>}{+tub z=$WO5gwEC$6JY!FgAwNR?E_*k3IFJGjqE4LTEBSAEQ;6hc4>}+Q{5DomhW_!!GbsD zvmGA??`G{{P;xx$Rt06+W!UxC&t+CEZ_P69rj0Vs;aReC?tP?Aftm? zv;I&fcB6lzA_gt$71(p3n)jK$`RJiwf@$5!!S$~(|Btcnj%xzh+Fl~K2#SR!Dn*eR z0R;hRf~X+969hsOjM97WQ32^9AkskyJs`aoQA9(pp-7kBdxvl0y}KKB@9Td5w_5=eOOi%jyvA@g0XoqJ7dF`MWU;|syIH&-mwm7 zB`iTGnFZ@Rb#90H)?=6ZdHE)PREXE;(MG9% zDGHXEY`R`Ysen;QO3Q>=4IB6jKVl%a?nkFK8^zc4yaxB zv`&XkO1*sQJ9XzRhEPe|G;*h@z>fY@|EqJ9W{0IIt;^JFuE?*(u73OSQ4t3pv-7?p z6*yZi1z*n28eSlD$*ECGsUz*iODF$8lt6VdGcl`6MT9jfKm7sn0kH+#;cjdO@|*!@ zd}`;auYbw99cp#>yp*>4yuo^Ai;S|fkY@RVdNN+~Sg1{8Pn-mRofyCqyE_AB)CNO~ zq4!f8Q@7XWdOMzdYcMtI%|l-lFny=CAw-1IXS&{*U0Qqd%UbsI^B9;5-X>gOI`i(u z?3{KzZTtF@g)e@s>lDvTaf9~H8@u^a>{CWk)>A4Zy|W|_AHDsUzM6DDxt)(hOuW4? ztn09H+OT6R#Rs3;|L>ds>jMZ2pS7ku*`F~dybt`~1Voz|W>S*z-SG-b4Y|z{9=sn9 z?}%X5PBBqFD7&5`Gj!&aEV%Q5uQsz$6Pe^) z9}`2rEhb)3J?vIf`t$=cHZ#wsu2d>)Z0-;I4p7FX0Ez(67N=hj_8dy=P)-4C@{S8w zP`9!?9;fzLN*l59<)X!!i7<$JjW2*~j=b078{TtQmrx43CquF3Ri7(>!MEx;r<}i5s`T}@Puwf51~)O>yRnO_PaMc(ypzM0|VbG z)T=|_8T}4CqqP#HfOCY0U?eAjjKh~o65kc9&5)L5+9AeTN2B(rU}m$%ypQf2ZQq8Y+9O$SMw}xxFS3wf zj=?#UQUoAHcQkzrBFzCZB^j|`*T0>e9xzSc;hVyAGq{g67v}mz^@Kjb;^^^iUpexp z_WEz2KOup*V9*%Y?uvpq9+fR(JfKg8CT8Y1sI~_20=+HTfK@U)D+!*i5pc2FH$@9N zR3CCz`vdT?Gs?Y9YV!W_cjX1~*7Fe*m18aO-8jcYo|kpAsTcM-Vsq|?i#)Gbf8>)n z_s2Uu$<=2T_&J>4rEFn7AH43}iZFUD<2LJ@uTUNJMAZEHx8J&2JkFqUU9#p|7mvYV zlU)3jPj;GawkN+1$#lCTUv#QT+Vw)@+sMy~QYqOUsnf^QUY{_D?!R}ZWhf>8wcv%+ z2}e&EgYCMtw=yd@!<@Prqb4Kux#HRt3&p_~yBp~0i!Kotojr!hWN_MrB zfO1gqjOVoe4pggFPlR0BA)fdblyAY!L8jv;juhs=xIuprChuM+-cq3&@VW(NX_iaS<6a5LcC%iMyv<-MnR9VZhL=A@f zjkaaeC)5l@^(~$R%uFZ4P^Ot0&ebc`($`ab=hWpX~_4yyXt`P)n zQA;CgDL*DrKZtYg0c@)O#N_nPl>l~Jyq`D#kbOpsJJZhQo2f8f0$Okukbks%rWbQA z$OP4jDyJ_)vvGiBM$lnF7wJeJLQUB`bQLLCVU&TKF)I66o#~|{fwVxrS}<)tcBVRN zg)($5Sqi8;dK=+1!yKRJCt;kkwv`;3U*-2BWU5b3MAbGuh|HCadK!5kaNjSlNUd5L z#A7REj+5^jU%#q5{jgB*jl$T=qFP{#A(H5qL;TYt z7aKJ}#oT`*oUadnBiC`rzvD6AQ3Dri3;%P$Q zpcZc+Se)~t@6Ss%-FY08Wfe!o2`qa!{x$Z0&BgRPZ0wu0t<(q#WsKdClF9Wu%v=_gTW@C$|wOaWS{VaLQQd)44@@S zRc@w?J^p8c^Z^&7yw7rjk4unC29s#U7qzls?$JThAsNa`;rsmTd9&0*#K*F)*!*>m zEuxf*3XGA>eEWF!iXidvzdq)@_H1O1l)5ON9yjk86=af=EXd8P~Uv`6Lm&;S1C_%)5S9JFhluY>0IOwXlX<>Two!FJ97iajQ}fSZzm@J={SZN`emW9|cBdR2Y ztlaRT)eYWs1^4$^vFf~EeD$@tzgSTabc6nVApmHmHKK@I0V+@Jztu-l`8}@tV+1SR zmrbeyzGfc@q7%BtlNQ&x1iIKCAz~;{wpW%uoZ6u5z_3Q5g5Rx`J5?IRPOGVTs1;Nj z=;av*R|m=bTO9d>bSY^bZVE7F$sU-*4#s2L$j$49Z(eho!DMUi`T~CqV*E}9^B7eD-cPA=LaJY!i zAyp@NCKAV;w&9pM_0TFuclMUfA#saOiv8<3_Ls)_h2p7K?V-3_B{m1IOeOZ!`~1mAJ>LYu*wK>)LrEWA-)@@~J~$AeVJM zf*bfht#fUL-X;aP9IF~`&`*V~$<1c*^7@6-=j}zmT51g0)do$^52QSrJ1qrZVjecZ z%*=x=W8iMaJhNzKlIiX}#4tICW|I zOY5=6{ay2w4A#o!I(LIkHS`KHKS6Az@ahTvDDX#?cn2%&$2=-)p)xTQ@39J$o0nw1 zu8X9R_G2yLd*@PcZk%zVU|T7fqKB|q_U|U{zLEr*ESxLXdYN54A(y#GUQ0 zBSn}up}^DdmtZdcAdh27dGJ)hD$svck|~lH@@d^mR9LJj==bBJ2bawzC+BdB3rnpp z$Uh3*;TT>L__$kBuW3Ke%Fg-705IlXAwq=SzY)9YMd6oqf;_`@dxg@7k< zS8J0&Up#>wN>C;>&C|2EF>mOcrkt&OGnVt>6mN%R2QXX}8Q-tt_wM>2cVBx!j!eqz z0kDaYQp292tm-U391V+1KDgL@IJxxcBI3^wQ-#dyDqQW& zuiZkvqGFGp2k^N?i0q?NaGQ^6--|p#{sIQ93d1!ql z-EXrP->=`ite|)TM2Ek3*V zz0`UkZQzr`muc@z&^YBPZ=?({q;&E4+>H=IvcZ&SbL*5lry!Ld4 zo3}4&MMf`1t~tEoJY^W4r(|*6tKPLgajZ>Vp>Xj@`?hg$+s#kO2e8NjrFQLsa1qFgjo3z^yS6)Tr;{t;kdMszT3(1)uuY$AHGnp9eSAsUr)C{u2(%FgM8v=`Uw5= zQV|Hs&yT2+Y?j_lN(idzs5`3_BF_{N{j;~}Zt|qOaVHMzO^MG@QqX?!##2K09O+4z z#@SonZ}w6>iJ4igY))?3@=bYgW5_-Oe(e(=KI3+*Ga!qygokpOp4Xek#%mjNJ-oW7QI3_qh`>4S9bUQlmxC} z(UIAsvGphEkZGMalbI-}mO{xT_*0IJHKC{k=O z3L8mEnoNm$0AL;h5D#-5qOyZ;BzW}ZGBLwg@p%eDjdt`G*r^~Nh3&-1(3_*PYqI49 zvU1!4^xCeUhOuWXPhUJ1nSIEQY=3d;dtW2lYb>Z>GhMeq`Ur9oAM;$v4Y6? zlef%Blk-NjYuT%wwYPL#2zCDw71hkov1<30hVtB`5gjDS7Zu51Bx znU?DbmoWI`?RTo5#FXg)^*ui)+=b>ZpKkGOHpkvx9cama$CtXW8mZ&GJ>W+12Puv6 zIaIVQm4}vC!BuJK{2kFUqoC2aKX^DKm3Y~ckg11|K1Z1s6>gh6jU}#2@;1hpWl~0=OO~+XEGF|^ zn7CD|kafkn`!HHWX-@K)|H10|*QgR0(mh?aQOVZrLTgEm$q_lS-S*n5&mVaB|1;B= z4?NJh#ll%?Kr(xTY>RL*KWC|Y48mZx>9-P?r+6wxsZ6a$nU4*+8ou9-sSzcXW%`g6 zbVJ^%Z9U`MrQo|v|L3~!e+YXp{JU^-F_w}=SWk#zOn)U6D>*)FZR-h8F^Gl`D>v%y zu36BXL3%m7xvxm_BBhbK$n!bzLmTS z{z$l<-%z}y%9dQmgnbBkrcm$6bWLn(Jtkzp%Cm7%ub-#WdJ~0= zD!O1oAULfhzmed+1Zo{4{DBO)n$53!rK5+7x+-vZOKGuMVcnx~k~(IW!<5#a^Jpi} zO6aXtc4eF!Z_laKC77HwZNhL=Ih_vSAxeysS7dPS9HdmuEqn|g8mMr-bysIGIY3oW zJzHg~I_{?sN?iR=$zY-3iu7y^?FVa93Cl?vJ-^i+fD8nO%E}} z%nELr0_pjJVwZ4uSbx~3u=`|~=fte??}pFz+$6@bwgO5#=k05T$jgiwmoQmpkHu$A zS7E>Rh_e1cMhe#dRnD{8x!FHBkl;Isgy$rpE2~eCEaSCg3(peEHVW+viml^U)W#Lj zg+C8w=6+4Q&33Tjg34{pz5JoNF2~)_IAzm7|LcG8O}#FbsNB;q7*w7+a*`RE!&d9A z(sdEE?kUDC@2BYzQkjGAFQuJ?ZKzFf6p)>H^wNZ+g4Ax%>jM2qpB&HZS!{@`QXTU* zHSyIQ&DGl~-RcuiLQyAU9j@L>A~;_l=7rZRV-y!IjvJ#Hq+Gc>ot$6Jp299*ucIq} zPTyFjM!aQHgF*8`*bp?6(>%RSvt{+!qe)IG9n77-1dlxp2+DKoJU)W6K9eSaI^D^J zkhF%6GCvS_|ki=sxFr>AJ<2Jn7ry<5!R-<1%Rfp9{!< zKMGMCK7-5{jkgQr_c&PBsq1ee92qa-&qMR>9-TQ6LUKh59cZmVvXS!+E({kDm!?Y# zP}Nk$Yeb1ca>vgz=6;JR^JlXnqarF3zlR%`@_{UQ>UO8GE4l0;~)T9m^L$+6#hFkp>n!U znJ}z)As?hA!t< zypnX;TfCo+>@W8R`L^I1A_D0KQiY`|Q63Oh-o}icRaw8)IHwZ3nT>tnm%6DErwoMo z%n946*2@3_!DU#*oyf!6&dWN0S995)S4GBbm48gyk0)C3uX`?$a1|0@$#sJ7cfxy! zBBNUJS*hnV3(w^I;DMfLR}*eMQ^)X$D%cs_g{lD<)KPYCS0`JPZlR5v=2MHI^;0U0 zs{jI?i%f}Qz~36uG&H@^*U792H(h7+F-=N_78%>;HMLhS-tQ14Fcr$SL{cR*vG0x% zeCH`aLN1D_8c4g8=vXjY9p-GxGn4!~M=-}GyANkEwBQ^$sG;}q1)w!zPU^uKc;@1W znU9T;e3VMJr=b#T=Rh~SxjvrI%(ygO6m)3&f+7IXHh>558305@XfNz~fu5>zUk z!^h*k3}VjLdTl{2>YE*C4Grl5@amKe*Fq1Hd~yxySqDKS+&$1ieGYKtj=h-|Ym0Gr zE(dh2ywjV2YV<1T{vCPf%MCBPika(*5pda<`wUtnn$?^eCnH z*2mN5K;x11DBxz3!J|IR=DM0jV9ScCe*>nSJS62Yn0!yme;qnc5&8dDTC!5`AGkBQmOnU0u0OM1eI_o#$ z03g#7@o2u4&Plq)v3{3o?u6z@J~pG9R->at5gW@>+T}qEV!{vE2aHl4k+qj>1b_z4O%lE7p0@3$%7Kn605T!V7|H_PZYAOw5~LhKDm}QwJ?73@-)2 zUP&+ZPXJ*vThSGh?8*t8{}Rj?e{+c)w%I@@Y-_mk0idiZnHTpyjb_o*F)RF0aoxFF zX|zr=xET*8L3yyVlq)*ZMPeH~MK55gHJb2(Lbht+TWn{ucvb3}csLE#9b0+CHT0v} zTJS(p_lgO75o&(YuzSNa^mScP24NUk>(5xRuf%pu_i=*Z)tuEZLl&x~akqin+do1V z>ASq{(~7$ZJq&)oM%v(L6Vlxjv2`kf+1_fkNpMJBue&=U%MQhNO$SsDg)-}aV%ane z1dZTeZJGk@NuYD~e~deQvZmD#9I);9p+*ZEGXu0einr}T_y)wZ;MZ+e#{=N^Y#UPJ zxi6HLo}X(t?^0dWnT`400w8(ZWK{S{^@$FbNnycD^w%HNylEhE30njx3=yPtmcjD& z`Q_3#L>@;XUUWKfCp&-6j^-hscOzk=ZQ9*jjPC)rI5a0ccyZJ`un;lrx}fqD`hv<2 zBbo4*|EHb}qzvn3Hbr+t$~1>>(VJI|x)2v}&nH3`3`0AU=BZ)1%__Q%=$vp4jMMM!xvu4e(Wy-(6b#usiC4p-Jk!8K4}v z=utZUR*vS2SWAu$chlyw9Z>~<>-e@?kka3sg#0AtI^_5XMx5i@E&I$0JsPsU$nM_e z8f-*p`l-2q*YO^~AnZdZ0pN@+^TeUE9M9e^m4rL9BNpvGtQ1)1SqIi0UiMO+>jvmX zFL@efX~@#zC?Y5~rZm}Zw{%n$r!JnJX_(sZh12vTu?+MTTg2-w*s1aXa(NbJ_Obf! zPeK*mkia%MZqi!7f4I(Kir+-coLU_VX7Hlh8Ezn}k0!5Q2$fFw!Xx!?i6D(X_C(Odi8z?2bY&o`e1v!1F$ZI` zh;CPtI_3pD)A1k6Qm>GhaG0kavJ-yw$5%i9NTpk|X}O=fa&dgfPg<+XW_4WDLB&<5 zB`KT&dl@U~rFoAi*c$M`$mb204LToO^t@LaL218A3Zvb2b<%EGWmgRzw!PC`r%Y>b zH?d{bXEav&wQyh|=yP{Y*P!9nedh>J9YS|Er(nSq>sH@ukya6XOwOYXWaT|GkZkFi zBJ)beVaM|WGcHx$au;1fCpP#3ARASqyK*=-THS#|NrB)eT(;zsRAh^h8`ZUB&}$EW z)Y{zG$@}qJUF;NQKs>=TK{vHz3fn?=`9JT=pAJsOUC-wxb`^?i=Z;#M4-!5?s!n># zJ_l71)1%lotTP3rJ4MASJmV7R%F*|0f@GKnki_8&erepM&V5PrZYOkt_3cKwB4*e3 z)!HZqpVIVJ%SRp&XBaiykC*jiDJB92|R=0`C zY1{J)w}i^C>!AU=`>r`U+S=PXk?M<${j;hVr$J@b z>ll37=?X(G11j2ir~`k9E4GgX12zh#sW*yCjtp(Qih=N?E;REi=tsCf?%h7mac#M6b22y zj5WZHUJNvTuIY%nuG+;6`!xO%zIBr8dwth^C0bs$^zX`Pr)IZqP>t4#sg5;_c2w;x z0`*s%{B=^&qGWRl;Hhm@>Sq?OSKiI%M`-rU1Gs&e1!#ad9H^zYR6(CMn&zMF<(1!7 zOqtL(sgQ^?s+4`l*!&I9=Fn7ei=HbokRAfK!cjRq^YJW)%B-9^>{L2l5`VeHX^KQ- z*y~^}l!b#o=wH5Yqd7KSHPCEvNN@d7EqxRS{8JfQniv09fd=+-(>A zLT^i@xV*r7r*ah3MLN89DM3Y~jO}bUm(U78%A_E{DZdKL>HqHD_tEQVG4SW?WDdu6 zH`V~*Owo_Q{PYIS`EB9ZtkAhPi;=T9F8Jyzy_8RWmKMXYb7h05kS~LYT<5a6I+NZM>)RNjR z4vLM)EtX9QSdD)Ev9EgfahtRys)#{x*ey!8F63E+iC*RQ@*f#O1 z?@IubBHDFgx7Ir(rRsA|v~;fX50i@RxNPp`GDISm*02tfN43}J3}Ar|-HbkUzo(Nw zx?gGW8L-ew3(cGP>te`HUv*p7oT|ar`r-*4N)^$2lnddr&z)xWVSs}+*41nP&-2Lw zAh(Ovi}$0p3k#e8KQ(2v*+djlqu^9mLj=`O@w$bdN0lP%h@V#Y+omOECITcD|J1`E zJ`h5p*mmHcrhDE|J@S1k4f)=~)A?a?;fI_eoc=x!ji0}*_9V~XP5>YX{U>N|!CwOi z-RbJ+jaRVX3+sl`v=8ov++3fkI-Fq+7Jc&$uq=#%LzXK5co`41*4-p4Ed+!0>3KiH z{Pn);Hm)O$-AeUoVf6uYp{@+oHy6RPC3F?ux#qqv8qj!ZcX|9pyvM=zlaZcPR|HNT zBecI(zmratdGm}yXug^n*7YM&;2kY*qrO&_Lf z9By0U+0y?qN)ner)NF=U7FOqVzpotK{EVH=Ss`8FTSCm^@e^x&bG5)&=a@lM$_s z8v_GL`QgDX8)JtV;?Ame8jpwig6dPw7h%yqqc4A~o1b}Du##|h@FAZ$$fnWqAeI5U zG?&>ut{SYno638)_aR+UYdh(bUizUNU-^%G(JS4Xh5$&&toDAhCPsAXx{VF0n;Bji zC*8)K-`GyuXl>Q8s(ba?lZL|hksgM&nTCb3>(g>q%cr#GnAD z@x4ul`fzUz{c=-pu4i98x^}i27B7SAy4-x9!Dtq~>XT0pbnbn^`nH0Bx!``x4F ztW(Llwr*-xi94J49W$1$@pv#2f}syT7RDEDaU_yx#{y0yThCnaNZx|%>QdCkDlIZ$ zB@;5Auj#-i!&X$el4l(>S_=gOX6V$AbbCM#+$lCqD)Y;fd^y z*N3b!htd@#q&M$_(j(I2=?T2Q733ef<3HN-2gsKT96k_2Oh*r8&*;x?3pX%Ai`iW` zK80=IkA_q25x?E6(YPb>WuEU!452sq8|s1Qb0~uBs2X;M*c5DsV6CyI$RcQHB$>@Z z-ZG}>rk_+{P}9W629+N|MAAiaIIEO057U15U}*`j3=y$DkZ^h<`vWeFH)7_gyJtCd$s4G<&R>&#nL$!u3gpPXB-CxCkWlbBlE*=rZN=~ zf(92sb09md6UN-Rpa}nJMPwf{+7JPgmDar<6uSr1 zCwmi0dYoUhYZBh?Ba8m|11!`fy3#heqqHny#%7o1m}0*`ka>A3W-;fUsh}X@s`0wt zJ>b*IsC_6bBB&cR$-zM6s$MGm(TY$EkA%(e%95KaW(Zdk*Y}+yo(|i+J@1D(`<30& z`|Z|g<^`DcQ{BwlGzA?-gK0fu%t*@rB=Y__wf}5Dn8u^mK?`SXnW?$CuD!g}&x8?| zi%go!Y6Ux4)6|#lA3#fOFsYH(axgP_(U0DnBlgJc{^u^9h3IPEs*ZTfXLeg-TdJ3* z8lKA!F(TtHPxU-4~ID-)Z&S(G@(;|3roQ+@Q>xeo2M z>SCmcn_%SRrihIh%I>*%FgvN=vgh6Ygkv&8nMPFU0AGvffdjrVBka1WB+E3)<$dQ+J@Gi{y2ya}i zo3t%#SNye9Yj9Y4InTfUJ%Xlraoi-SX%V2-Pfdd1SX#A3E~Ia{6h`rs&#smzFEcvJ zgTy4EclOnbEb3^xX59iJc#|OJc-_1Wf86s_njH7k(K@y zqcWuv!A*&TF2moi2S{q;2VCA^$gYY2$#EWb?l)~|$kKHuy zB&0*CTm26DE@T&&&PcK7SsbHZq_5j|eeh7_*9XJT^#R>%p1AU+mPMR=Rh2P)V2d{+ zo^zJwX~Y}}Dp}&Zkh439^QD~I+rfep(1TCM5XRBJS~_W`S53U#TZRyOMR<&vk)pK+ z{lrCN2JeTPK0ny)m|CSKw<<;#ag4C9+x9SV(V1?ym9OSigM`muLbvhfeMIe?GsJYz zCv7o=M0erRY1d61+6E`t^TY0tokB^HZXMLba~$;beK;f1Xu!B@q;sf(^2Z2r(Ep3x ziGa*bXH9Zc9qN~tQL#76SYQU4m0i=j#QbbXH?(~qI#V(G`+W(v&`;Yr2?m21C9OMN zdBML{7c+Xp>L*G(t&IlcsAwKB&zt9sp8fngwLNYQ6GRf!TDrSy_fi$R%d?m^rVtS1 zR?PtWnF=N}_H4VKewdBgJLySYLbw}(Q;%0P*d1F1&4<6Jq5AD~V zKiD2_c*2-Y5Zd}*60b_EiL;?q)4enomKTt^ePXSBVvLIRYjdLo%rsC!wT4+on)4t^ zSe`@kVjBIf_1NuZHCe`A+XJT8cWRh%x>}_ypRAt>B5EQK^hrINwG)fpTxYee47-#B z!CaMaP2rLPw6-2OTh={;heon;|1xIltC|l^(Vgy*FUk;qhvLZ#M)8`q2HD-c=F1%*h4Q}K`Ylw#H zrS@!b;Hw-YY)JOjZ%QuXIz9?5`~Wsrtgkzw%shTu`ft)h%Jf>QX7gffTBGU? zdqFMv*12pC4c)6Q_wjn6RhP(Pf6JvCu=7^JFPgv$YvF#?2e)DNzgEA%y{GC?+1=4x zCT)C6@|~w#TYK14%&yHOrrl)vzUW;hHdIr=hUhpmTJ9~^W_w->mB+yLN;fx9(f_#G zgqz{VwB#_~aX(Vc?}LT?c5IDRly0(X*4m-DfEwwKYnEX}OPh1dvNMdbj#P8i@5ejV zw$%Pbij?o6?E5y${iK^tv5#8y*j=&mJTn1RqYhS228&0T6eRj*d-%=QY#PU&YpR9? zXMTQr!}k|=5jc|fh-eRK*mF1-v&)t`Gs$kHuwYIG?cm>HZ0-;{8}sCY2i$(8i~pPX zs%&lfze0uoYa`j;NS%-(4_lLyfLu+|dxYv4er9wHlHw|M2@* z(eZ0YI&|)a8fG{rzCQeIAg0lp@QwP9V@8dV?WvSIEKfWiW$>=7C)PIHv94R-;e=Cb zyPcRz>)2Ob#?|Cwg=M+tWifMR`}=bLA~Q*z(w&tFj$s2#UYf!v@hN5Qs7lP*wSmV% zs&pi}y48Bivj^cTa%iC>bvL1zh%k%rfnU3~_?_ID?#T}i4sS-zNsBS;5XXYHlQHTA zH1y>KGlnkLFkTI-+;tUn@EfIBOkj1u=#fO*Klc<2NYRlCCoRhia7)A8-HnSO590|_ zn4wPeZgIIA+i&AnU42{QK#4=WUqi(D<6sh8Y$p7dJ_?hv6cMcsoJu(B=ZK4ml<}F3 z>P0!Qmls49gM{Pd!n{;O=bgOFN7_4K)Xw#DrwR4cKeQN=AQ3{bCxUuo+{-QUv9=LO zQP;>*b_CQVe53UerUt@LH`A7NaO`-n`8qpJpMU7shE-ktis8R6`}lN^e;92Tk_wC| z&_2Qnt@^xl87+2G8tUn}`01iwd=(;dv43W_n5G<)cFgs?D>_SU?nSQ0uSIAMLTi{S z-LQXS*Z3OOo0_OTix;CVHgW371|Rxs$qG6a_V!l-M{)DQ=uA~?Q6R;HIj<$Wpgjow9 zs=k$ILzMT>dtI7G8Lp{X9U%{RE zwpdP%zA`$_Ez43Iht_g@{wtdhCOvFU-+dpQ)L`Qn#EC+Nms3?!1hY4A6s(Y%=6tF` zZxx#)v{~9dJxY@J|d4Dhm0$r;qVx%@wgw`Gg1DA z;6=h&9L?XsN2{Tv_LS3}F(oTo{jUE|%lfeK)O9z)n%{p2zwtw+fVbr-ZijsU6rFjbAMUP2BvD9ctf(1M(=*} zUln+Ne>Fk7-0hr^vDW+hDhZ`Xf}iwSlH%=D$K+Y=uGSIZsj(>G`r~yu-#F|ntG44tS^GY! z`H^F)?iDp@hR4bQY11OTKv(w2!uwM$mITG&-HSX_QaS>e5+!uB| zsm;QZ�=oxvv$e8v5STxg3LT)HGdvmlvlR+_b6ZerNdsY$!DHUw0M?cJ@Ovw*lj= zwRXt-n(ez$+_-yP-4|!`!_-j5oSrc_@-%v|e+III4ikrIW;mQ|X8eb&6KH$(R8N1W z9B7s3tibpjX}z9o&E{mqL#vQwBXp>$g;|2)t$>Z#l<82a;Dux^8q3Rl2*zM`E}E9n zU&{Y)C~MNJq!;dJdG)feq+^%(FrNN~z)4qkQJBu|ygQp2S^-~Xv7u#;;DA&2ZIWbn zUGs2@+1S6|NLW1i&v~<>YaR=~BNyBZUrD%>DFaD0;i?v3RHW zy_1LCr`CRo8ZVeKt9u%a)C*+!m2~|EL7yp6_}waTHUfR?eEnCMVN%R|!)+y10jyC2 z7E$`5%rS^pXBjuNinS-h7(r%D3Uk6wQ|bJR>yN$Ks#5QUh)gxrO}TB^(+F`f2_ZM( z3!mr9(d3tN71yNQWP@@SR3+^_NF3*0A~)0jbkSxgjEmrK{^2SSfmqcCU;ByYnbHk* zoIiqkVRwi{S8TjTYiDnh^cqY%jJk9%ByXMj#(mEH{UvmPs$moD`D}&x?Xii)=Ul(G zA!L^=v+PG}n<~9}=yuslYP|c#CjEGB%t<2o3uC)_VbmUO3NS@jA4kC+iR)=ygmqtb z=6JQp#xHdh)9c`x`M#d}xrp(6$|9~P3B-JF((C;Tu@4K9sLbo5jbKWg3pvu0ddapR z7(m^lQJ%ldjz<5RCm$;DJGiu+F*&9W@q0T5ftoCzuOlh)r$%HHxAz^NY4xDmY-6_0 zfjM&)jGowomlrwkQ;qtk*2Mj46HsJl&L`Y%RAJ?WLleYpM|L5xH%-DuA^jLEuPjT! z!OUkkwpeAMJ881bkEd1B=X~Z?FpFs3zqRm4q3(iPyUL)+X+?~rGvb_R<^JHMlKhFk zQkTx9C?&`(I`6?UuIU(7iVOonu(Ot{VVhwac%SIkj_w2M5=}l`Q4tY4q86xBLb$Va z>?m$p2;I3U4TslP+*wAx81Rv%l7f*X2Fw%F_vd(3bN@03`5OdDwM2@=QPFnTvm1!t zvK@m=no!gB4yrrfta2VS$x0F1ySqGY;`W^_97C^ZH61KX0@st*hE z%A_oQbsazgwnr&-ZnTfRF5&}PBfI9~Odq7Kt_VdkmqL!Ah1yVia8)GiDJOzI>CHqT zpooco)38rba(1q?7R^u_@msEAka8pNdNM_(eu1@=N%3%1>TYW<&mQ{a`AB9`pe#Lx z;$hMb(=5O6e(P9D%ZDt*X8|DQomHe5nEy+QkfD?iUM4DQ zg)$fS(%gnLr2BuOs(yN!vr2v`sm&(Xc)vI=L^2bOTDLV`4RZ*$FLa5DIa1O|=w{$& zu1pWfOp6W;i^OVowT+i8qIrnKB(2)Imff^1)Le%&wcSW#vO3mGJe=HOxS#zpu%cd6 zLopNQZ*KvBNXs+r2Fi^k?1`VaImQo8BXNQ8^D|eZK`uP#gm%_%Cw%H7k2flfd^s$1e zM{?MI7bo>B&&9*(=W#qRS!lI{h24niIJ7QD?!}5+-KJGw@r$+3N$_g*q5z&pWfLS6 zIJzB^v56>gHN`W(5}Py#iehH09}7Y5a!y%b?5&t%d&H@$DY#QAGcF|*sPckBGy5W5 z>0Yr8*~T0h)MH_&N9>$N=(Bp6UuFw5SU(i)h{yr`D6_P@ifytPYvAYqtT_pL z2Ayt}pt&b$mAG-;OOp|jIdwCYBOVH$9SajWk|HM%@1M?aLLP~G^1^miLJPD((;k*t z*G;A zP;Pf>o3->{m*7!@9rq9PRN7|bb2%NWv-3Eyf>Sq_-;8cg`b3Hz&3gY`*^n-bq zI&Rh5YOawFwdYS1;5gtJTXsRT+MjUAjDr7`4q>6u(yVv6V|BLv{=GhZqM`nc3*Z{o zWldv;!tBCiO?NZYR>SD98{s{7b~;T_u?6FfbqJqGKaAc)t;nxl$!8=IPC~@&!KINz}9-E-AUhMQAu*Os3f3w;i!dlguX?t2~pwj{D}k zmH{d`-;JwafH_gg?&=m$_YsHbCWvJB2`hP2vn_c<@yUG?$m6GaGPFO~-S~Hz|MStB zc7bSU=z7Pi=<7oy0XO^>X?x*`)Ci{k$JcvDv*G{yS6i6yFwYj`p2!H-YMqBw#)WGRE|X9dSLKw-%Za zo(O$_NTAE|t7V@a#CrvUwZh?b%e+>@W1gPJ*>mpEojtutg1MFl)^NKqaVfhLYjE2* z&AW#{&s!watzVqdpQEDJp53rRS8RE_d{`J37^^Vbp{mTQO5M})0Dt-jD65LVXAuF~_{xIVcx z3t}9~Kug!#I!vxjJSLN-S)FjvPFET8>EiSM^W2*R_pMjyp}LN6;|*Gq@?_}c$9Rvs zrZ)CmDD;YA;=cQI7Eb~A3Ne#Zh>=V@4sv~ky-=r#ZDL)|fw7BJxLww9%8v|?kdU9R zo3h)Mga#yYGt_U5gQzT96=kvW*IrOcXj_6X(8LkLnnP-U4$XEjH6M%|kyqQzJG8^$DoJUcPLiZog z5^sgJcSelFIqj`>y}@W`W(bMea-)pW^P>4a>$o3X6|?cN_#hxGCAzTxja93~jJ5q< zF^uIZ*@_x7;H)SOBk#vZ;=&Ua+tkR(j(z9E>cJJ4?U_Ta5pt~?O9mFGO^v=DJqhlX z&;CiHdC(@E!jSP9FFp5uO2FDzk*RK4*!2LH5wdIu>e=Oj#!D(T!H?7)`s`@f`7LaP zJEty&Y2@txd??pH8D8|3PqMQ+6(ZGc#%l8x&rVGv(%N;pFb+P6Q>a!%-e7`B&hswJ zR7?h=q9)^k&F%SIPT8TbBF{$XMW#j58#p35H*vwE9iEaaFsJk1Zu_tPyl(O*rVA%1 zc^I`I>}1U|g6a%GDQ55jL}Ma7V1s;VKe+oBA8Vpvc2+vNn81-|LT9>>Adi{H4}n3~ zAg+p=gffYICD$%;4P~$(UNml!+?+t$H}}ku^Jp&ShqMsiV*(s0y?Tv%rL@cKX201( z+S=rGcaCK1etc-Q53&@)`G(n{HtS=lz!4*pnZOW6_2O~@dhjuEJG8FIt}eczfO0cj z6to^pSMuI*VIG_w@z>G9RYZryEcN8J=P#$nqDH)+5^p*vGcj|=N-uN=3(SwTSUHgd zqK@DN8YEHuyx_+LipIo_A0U;I^DVKsRYXZ$mol!M{14b3gz5 zTi;Pm@X~ST_FDC^lIsjvGh-}K)a{UO`B7oRVqnyw^U#DIsHftCZybMYrtDdy=^|KP zi?e5fpxD|pYVC$-JV$%aADfMXH%EFP+p_b)sBEr$aS5VS{```Ykuho(5Zi0B?Y)EP z(Gj}bjg(b?_cE;^;6zY)P5dGoq~G<(Pb=h*OAocY+Z??)G#X7^zV0?#6_XS5VC`gv zz0S_z;cka)NZ2>1hLGKyiV79@nQ1y!7V>&rvw6#^2YV1;!F2D}GrkD>d4%i^% zS~4E;SyQ+UU`r~o%rh$)>p+pAe|Kgjz=$Hj{pR<}gg&#L$NWO15Gj_8To*L5ts{zD ze3U3V+*Gv~F!X%1J|g4tq?(2^yAWsVroJ9F**Sbhhm-v=q9K;wI_j1tnr4VmK~U=! z0~o8~ZZpp_2+P_}P;Vp4(hh4{WMBTUfob{Vv!4IFKKK?r9vJ(Yp? zFI~&nZp-57dZvXF9TXPL6w2Kw6VIP-b*Xc()Or`z10q(<>1BBPbPzoSUa$My=wSz; zsbX)(>f7Cl(r8LMn2QySY;*N6=JPIF=cGkZsZHGRG5q5qu;>iP6ybXGO3eNz#?)GX zeG(HAZg8yi8_1tO@$L;g_4SdOov_PKtN-22{8vq?By~=1BdpXf>l1$>j{7k`u=#Uf z7C(e1WYL+i-3|}EuNYP<+8X-hQ>DX|iqxxzD-_KT>ALWYGwdk9+jKWk#IPoW_)4m= zh?p4^km_6c$?dSy`r^Kb+_x>!Owt#V=KXK4L$Dg7GUKRgBff~G^Tqi+ptOmufhVHW z(cNLRK|z4sZyxewe@!dbOM7$=SS9fZ5B6AHjX+~IHa0iay!0F~$k-2DN`aoRGD@qf z%xHexWX^ePTl4q(FK0Z9T907U_$b*_A=X~zi6JwbSr=wRCu1L=bl~wJwf&yl<|4SK zO-N!V*C0f%%p~H%w>rGP@W1I%N<1QeJ4fT)gdW?Ds z=U$L=sJtRGr7|-~g9wAvoI?_)Le|c*ws%r%GR2H+Jo?!65;rJMYaICH+@l#sqb)m! zA+70t#tKZUeQ3cJ7rPS7 zP>9tT4Sr=;^x)e?F`a9$V|{wG#d9{GIfOn#x7@)5<=Yr-Za?Y)NiQVcEOI?M+XEu7 z=nnoPvbpB;W?Z@8Qow}QzcSwL9iB>+#OJHY|Bps;kbnEkuU~G+tZ%^rf{Co>;&8vJ zya!zwjokg+8`Q37Q4FhYwMblP0+GM0`SXc3rByhu+f4oF89JFFA_&4w-SFs!9lgyM zntb3RiB(VEAGwZwq9#iesQ}s}idJkWJM28H`OMlFaT;V7PORFMa1!!>n7xy*(HVKT zyZiIT9yG?cVUf%?=;amPD*6Glh&MSqTND*+wkz4UtXHJ3?eRRxMy_HF7&`@;W|2Y2 zS9?IhOfyk)`LIoo^D;&eZZkN1?!NXTE2|yi0_M%XLgeGo_YS;^ZW+Ey`hQubgewB$ zp9&!O6s;jCu0NGWv}-RsYwg2~goL{a8N~EjR6DQTS}J%rCS!zYsKPlIG&)qjRP281 z2(qSzEe9-@tu~2DHNWk@V)PfqciMI4hUM{*a$&%*o@`_I7rOEEv+0M^y3=NBUk`SN zhRR~OwC(W-%5H4tplgb_cXM-Hj&Ix|VpBTB7P~(4r=XRV^$Zxve|DSCL!HsOvCP6F zr$S16Q+(^>^`;GWR$u;KL;U1>^H7oQv;zN(5XA7fm(n9FAf@@w5Qaq^eUx>7o=>Iw(uRHDkGF=0hT5Ocax>L$$egPRz? zs!s0?aS3>F?SJu(BEAW^T2Wj&IR8h;GQJeyGvbAGdliIz(r3c#S{&hM26#X4YT zcbbd!@K5??-RuozyRWLoZpJ1ynDNUO_GgvhCUp3@#OwS1uB1l-B9tkOfoJ5^$j$t| z!~$oV)se@Zz#o}9V?3ssyOdJb&c9bj_+r2{uU&UpTgovnx{^w2Xus}S20U3*p+2C#qXz^5EJiL~>UnG10GD)}_W7l@EZx|A=QxWs zcet+S-f3DpioBme{1PN#7DM*Aw)dx4ef=H^=+7Edno{LBxPQa3tqSBcx^fj$hW&;;UzB1E^4ay5o@z7OcEDcJ^jf^FCQ&*n}Ff!<`A#=@T*@m-@U{I9(9Kf)TT2n)FBT{A?v-Sf8X+2%zV z@4KC8x-6+62wR&bS+vi&4;I@lbTx52++s(p(C%r#j;1NrjB7RdU4YB|wl>3BVh={UoCJk@-0H=KO^W?W7ZFJ+f{C907Z`R=2che&y#UpJ-ud|Uy2@n*;GF9&|3b&9|MTxnoNB#yc-F~(kbbX- z!1a5`+L)$9iVt128izRwI!{wG}*B}*F^Jhzj-HjY{vPWBE zL8M%%pnnk@`!Bp8LY;q`BcAbxylI#7XwI-Y)AB*uT!D7*_j?s}8vfS<+mIH9fBomy zVIJG5NfUtW0P1mi$#?u1s+5o>O_P+CT6Wdm(urkGxzqkp^Gkiy0q@u?U6f`&s2T5Z z&#qhQ?9=UYjQwzEm;7VK*SgbfZYp+UH+AF)_k$ITfm7VH55RZ8)Hnn-%Z((y(V&K{ zH6Wj&>oj(1F*usiS5!z6p8s)Yj<+73qg%=W=!smqrK_Ti1N%(qL=8^ zk8VIwhjXcg+AF>TKN6OfOXI1$rPnq869|Z?IiqOd9S?V1bHif?AXR5aa3hAH^supw zxJZvH$%?hS7){{>!`kv3mjx3vLYu(i0N39hwij=RTBH^5nifgzP^BPihZp)7eX)9l zbtl$;KI=K&w3N*L0$>+fEz}ATtZ%<1XF`u~xt0O^N{`>oSp>d|gAc-H;CaxL#ie48 zwm!Y7m05r3ujwih$iX4W{$rfNh8)GGCRM9o_UnQ6!oo>w1Le}glVfrzRrqTLGqEXW zuT>ciwCZ@qq_6Z^IR1yALf&6&wRI4teO7_ua@g+>);Ye`?*Yq+(Y>^8sTJ?Q=^E1{ zQ0rX#mkH|o92oB62x{9q)YD4U0=fsg06%^3A26kuO@Cn4_$HL}w^Wzke`Z>iX5vA| z??8$J3-S9#IGJ)I*+&mr;9pb3HA0VwqXrviWN(VfkTYwn-57ZrKFgNun)yq0a>7R^ zRcC_O8be(=1^=<@LAJ1oV9=L|$1I*7C>4kT?7*T4DIYE*I@sx!{ziUKCopK;MHi%+ z3if2=P9!P^bdG?EnP#>d1I34Y9HQo3m{tJLV#_keq6-#`&MoG)FtL&3s@w@@X+pHX zlV|d8qxr^b4z#y^)_9o;9m0kT*BC!*OKBxB6!{Al3WEp^fuZv(w<$RlVeCyI^`19a zHExb9!*e||MJ4uTRPiDB;=6=62K zaIi`-W&BC51TIkr$*@?<|DM^Nz-!(`{h;&3QxJ52tVgO5R~gpJbf|5~fqbExH5sSp zHkz00Q- z*)5q~n#~`zze+cTBzatbwBK{AP>R<_(ryO!Tbzj(vONy}lOdX)m_sVhDnFpc z(H5SW(rM!L&Kid6%{?re3E|H9@5z$^zpH!aga(--Yr)_n(D+w_Gi}=`XmhrAcQpk{ z&O{CETvmHV>#huy-JG1|>-Swi|kZ0)iyRWurX zNCrquTZ3e{fgF>gdoHmJTDrfAooPTFZCXVdSh8`lUwSb3`Wu#td~{@)oa6@ngF>>X zm3n6BeDlM)R;tgd)XmnPxoZcBjVLd=7Eyad6ODiHz5&`9P=64qIL4UULeo&Wj&7+g z8it{im#sm60ixVk&h}^7Jq;OyR-XVQfq_oX{2=Jq8hZBPuk6Rn-z`Q=apgV9Hc-*| zdwFYYO@g-6Nw4GAKX-n9!b>vFEngGEqThxUP@b%g0P3K^y?~++5a1D73z4~)YV9rZ zEf_7ysQQ6z#3xD|r+9){`~w!w3vIdF_jLKHnB8-T$6yd}%WO8#c*~@VL)$>BvcUih zALS;|%BLb%L`mzeAfAxD;KL|Y5?1Y@F4#_SN!~f`C3QOX80FQ>_U*V;4$wOYc^PJ< z3FuArN|UfBJnDh`U=~4{9uzQ>~@0b#mryCt1&IK&FHn7-`P`r z`ZoJY`vUbe3N133t7|?z1o(rBoSTIDM;#z4;|i5l%+Oy^*1mhMESlQ7Lrs`Q(z7of z5&L>=l#q)xBcXB{D#}3Y?>}X@@dX;i<*U?g`@pb1y1&tJjBlHI0tRmPYhMQrkDG*9 zBzg%+PL(2n6^yh%8l3H8Tj-i@ zXy*hI$T^5O7LSEDXkUH(8ak_BjV2!Xl?~Tic#w$Ev&UP>c>cY-GvI26AfdMwWw1Vr z#0$jMhxAK3YqoHBZ^K|fbFH%LN8%G}e7nwZD~o3s!Ot1Zy!saKCSL1Z`^#SxgFsV5 zaeSCpSyVKG{4P&&u{^4S%HD;JE(&$vhm*%`T4r=P@liV}4y8K5H?2uew0IN10?2q^ zB|jJitHY^;Xo?}@bJ2-fxn*kx$>Wpu3SP#GG+u~Fwero$xFi|b3w+OP$D1s7 zNO9X&4o83ucoVPZh9-24Z?=eRUOOZ*GGT*yWq+#Bvvt0s>CAza?gv}tzbar=MaPVZ zx4oXEbf?UlOw~@j0&6RyRk5(5C}k^=pIM;D1gV7rj$pNt;J+=-Vt<=@Q>w!OSd4Pc z$D;aC)b_RQ>XBUo%B7>K#~I+7!7C+i3kq%z7f{LfJ?&KtS`J$qF57}Ne-XoEoplU5YNc}-p)($*QF&Eg5Nxs4f=dPgJ<~-R3(g2PJuDnG{55WYVS|(2;SZH5 z+M&F{|J1e;U*6!yX_=L!P4esdrnpyxnaxn@nhM{7CO!8l3*`DOdoKKzE&gXw_ziO8 zBB`k<7VoVFkjxPZw{Z0>UEx!>C}H1=uVo|>R-PBFjgaBOnFJL{uNG;0*5Bw09cP>$ z_5fCnPDU=Jti@u2@9Bb7T;HXsSxPguzgK)%WDdg%q0_y3bOU|Wt!IO|1;0aV!Jn!N zX?)N85-}j1>ZP}_Xs4HZv*aU*e9G{5&mQ8`FJRDd^Wn+-{J!GYw0H~pm6*ViTbfL( zo_@{ax?S-eY?IbkXdCDT(7u%07zJA|+E0(*AVX#;5F5T94KOiz)jT*Fh9Uvaf9?o&EDZT9bLzTm7N~ylJZo zm~=~8yA6(M{i0bg9^z3dVEb?fjl)YcNJAtTVIua44!g}Ob0r5+F%`+v^(l+C=(x_#6l^+9<1M-Qy7@PLW!PcBkn&E^f14w+UKn#n|5ZBs#m7l%&w zAb&N?uTu*a4s>3rhC_`ua&Kfi5f7*_4Xv`u2-dq<1)E(yrU1+LfxfPFIe5ZRK(?S3 z^++Ij7&#UV+~cP0(2SV&WT(%BuJ(c%S2i_Wd{9cmCDz08A(Q_!Ii3g~lr*n|X_V#1=_1u1}qmG;C z!_HZR8$GD8y=cn<74~$Uw53uvfX9{&@{T1}Wy7iOs8=0v=SzimSPH&p#qT_B*;UDe zpDCY>j)wg9gw)h$zOJ{QS>KUZ3%{?390S)sIOsgjOf+!9REU>_X_Xg^Y9l6^> z1sYsD5A!qj!#7vywz}YjlOl+q=j&aohs`0}ki8*YtK$3U?z>MsnuGdwPlUAH+u3P& zYIQNkzL1mD{Wt+CU967+JiGCx<7I>=kVL*aV=GK^d1JcHr#(x$ucyj*o%7sjupNUj zqqnRpF27!OcCO?f!X`4_9N$sfw_9Gge`MM9e$bYQzC7@Tp=6(NN%2s?T?Jkb4J%=5%*ccsOLEoiobfp(T0qyZQF#7{|fBr~~ z3uTpBU{7-Ec}vNUwb5{HUUW8+*kAqHSdn)SR+Xk^Bb|TfQcO5}=)r_LM=vsqZx{HR z(`Vit9}!$ob|vDt$uKG=r@_Yqk(GC#Rg;!m_#OK&Wbu&Ew9zZg()$6Ogs&_zgWfuY zm#%q>W;|m!7B%>UK%b1)gkAKnp4>UY5%T+XW3lUf2C->}#4v~$?#kZx{;qum#q6V) zv>i&B&(Vc|8NzC45&ijE1xt?NZO5M?H$#sL+jJ-djYOT1?oX*(gR?Xk_@wrhgI@)sZ-s%ut#@s88qCr4W}RjXnPZbu}_QgW)Net z3pI~CdWo~2<(ZYPZ6e{t2TTVqhDd0UJZLA7%5rneOuI{~XL(J*B2~X%QaMy_N4_2M z^w=uT>Yr>8-66R79q)=EG}f{_#BbKT(*kb++ig;17~Nz6N$0jR@g~S^g-*EA<{GqU zE%*Q(a$pYwkrx4h{un>z;=XUYq(1nDQg=W7^tF0}Np3kZoNkKU%2;q7;xiR`f-kBQYMvOtdr(vyfHT5xV6dSzw`D4vyYxKWahW!zNXAAjuy;>OA$zJXTtY~PBL128O)5C88@S^*Tg>RHS~S^`<(pNb=_a~={)oY*KT}zH+s(P z&vc$I*JL(*@oo}4Vku?}CV{+V+mxl9i>8vZsX-`QVlNqHN3bR|+q>8iP(VWHLb10< zGTCEfYWtR|2=ANhPGKthHD!Nx+#~b))^lH$hXbx|X2?0RQ>y3bJrPMbVT=wBOF z$g^++8<7DR_q%b?CEYKhrm`ZlZs8P8I&FV{tv27yJbOXi)^0gBbAN{K@gO;7emRt^ zCm>~$0$roYnjblQX;1?DH=j0u?9L|CXw-qZ_&1`c`f~}-(ebGxoS^a?0yE*en*Ngu z;VF%jt+LU$9F8Jb`{*ydRuEFYpFZ0>w4*U?eKYXr{;LKT?YU7*kw`;kvzCy$RRch0 zD6cB4uF}cw@~zCn`IcFIjUjn^MD-TI@8RvUaj9$%(+dJa?C)A1XRdCFcK@MS)I-iA z;8eOE+_|!%AcQf$T3Oj03|))DWX?=F=Kt1047o1my6*yRZYTz+ef!m3A_?MXy_yeG zq}Wnp0}4fVR4fB>o~ID&ML)!^%sQ4}APBNb)Dl%k8Jw9L4Uaq=l8MzydfH@Z5Y@TCf;FEgFeb{j0$DB+EP zNn0l!0dfAdNg3P$NCa4TBLqv~Xwe)c`hBXHs5&NCCX|Q*fHtJM8^fC8i zs1c#usub_BoAapgL5#0gv9!il1oo?%Vz8O`BnS(t!V8)l2v zNo?WUz@N^H#OPzULv~K6vdO+d7QFd7^{1d}Mon%sPlrblm8W{A_#RDFj_UGmZ2nFo zPa7Zhegmv5{WqAcJtpyu-z!qDyG@gIV?nIZ@6|a20@g_>Sba6LnJb*FUJX5FRmwB5 zaOAp)gU=<1!a0H@R)T#&P=LP%J_VNGg~@3xC3cS^MKCv^=yqtzT4TLTXG|X$8xu`< zLs&EnlzX4MmVAHag7>2&s%V$5jK`-VUYvn$LnhUX5Lmjby;Z$*XrMlAcoR1)=@eO9 zYA`Z+tgjGymwwJG65Zu~N5fOwVm(|%bb#E&Z|r~NS+3?v>`+vx3cs@!y39#jIH8SF z8*d1vCRr%Dfm)7Au|}7B3EJA`2I3xf1E8F$xin#@J^rG)9qFm)U9u+QLs32CoUGC_}{nOr*iQ<|l;7Pyf!LLM8?!whnW8)-cm&0}Ti zb6y9UL9C`KLwzptLWXbZ+!rau0$pI9URi;&hBX4f`%MaMX!p(Q<`woIk0R==`h$52 zv>JkWh-#O!$WU8#pxVj(cZYdM9xI9CkiQz25oM$6YTUnSJ@0JVbP_GpfY9U18>q?w z$Z3tlx!eFXWr9H^-oyMd@g7Drc1T&{laf#?>NV&zFdE&Bb+`f@$~pPc3DL1oKDz0* zJ%v#B*mxM>mMqz8gs7UeInLh2M-JwX-Bkjvl5?%3B%s@|o_VCIV>lu>TH zLD{-02C*4<(UVlA?(+uPglD+N_~LG0bEw7Ho^Ao0gG!-(+-CPKRL&6C2!j$=pqS@Xv9|r$?IhUIpeLs3VL^D)K6LRGF zB&_X}JzD!@Ze(H?$AZ?p_U`~Map=j7KSXWZkFG;cc%Bu733oJaD{coT9vxiu;CyjP zGmsD&Q9f|~s*AX&+o+ZclozTraI(OyZ1f=C`#fEEBdxcW>Ja9NJ%2NcV+?zFVz3D6k^R=f&6iTLB4)^uu;eF$9qaXQrP<%Wda`qYbI}Ri_o7<>omZ z_36c%A-qA%;4@oq#};nPSKm!}7C-(izPI*sxH!}f%f0euVG3B{nF(-JsyW-wJVjpj zD8clNHW1E}UJ3o}nr>>HF6S|-C7T9WZfX=sM&@Gs@CAJ~)hQUTPs6ta!t~45NvHI( zJgqGgjJK*E2SUL)PJc4qVyD>5+2Y$MhV>~MjSrctnL{@lTTNdCeH%S`KY1K#0=!^y z>ugEK>_^90=_&~=f8Wi=s7$qm>bR9b`)aSX6*+eM4Pp%0{t@bYm<)7rj_jrg4MyfoM9EF)|;+u=7c1*Y7 z^UWSGPA+`R+l)uVX;MXJU{JLDyO0Ms2tLQ?I5%o((zadTgt!sTIP{G2yNWzdOigz7 zqs>3;Z`w9szkle-;7J9Z4nMKJxO-gn2lN`_F7*C^WN zadmTsQ3Sr~=MB_nFW5CJHwyNfb)u|Px(xh>BWM^0Ps6imVOXc)G%aK2Ay7{|Z(wW1 zdX)XucuESnqIaPuZ2u>D2wzqcBtTUqIrCBaZX&dMgJnIi@tdboTdzqH8h`@r1rO5$ zN3}MiW&529_0uNA)y~E>vWZ$c+uyPF6UHo@OwBIC(f;lcZgJy5+zgemWzXiO4biaD+cu z6SU{#{v|j()TTQ=TBp+`Zve;xVh2Ud&EMV*Cs}yoK%UF|YYKbx-?qKMABiG-E5ZJ2 zMK=^|L}ucai|zj()=cOwDb_z+;C4$J99^^>n}G z@Rgc?Ia2NXi+1iUeV8H?*T!uagXjk3B@J#QzbFhAk=Pf!{&3oI()|ilD4+Y4GoYlO z=WDPX^7{m{-q}(4oflUFSjFOM9t79mo}++_6zbaaK6UjhCHPH`BUZW2Tk-Z}-QJme zYQI;S$Q7I(R8`Oqh(Bj2%PG<|p^H5uOZpol^UH~V*^VoEgT^pU6v23Dik@6HUB||r z_pgKAX6v&pv<`b{TgURc+(v)>Xx`DZd4Jdq4v1}2RO*i)`LvaBeWs;2S=V-wJN#8o zAKj(+!2~m7o?6NJnNg(N3TN*vkR=@eS;el@2(+>N?6%$mG@jaFb|;Z$xUyqA-~?`X zC(M}$ZcO(6>UoQ2lO1$|@UgdZdN^?!kO*nWN6*ZW6cK(i>E5*;pCeN;_wCxzMYQVb z42{NY4%l&9q`FW?ts_U{sXFnl{?|RCT0$x#RuP2f6 zhHpk$h*PIaep`4ua$je7@=b1}fUumQy+igh_4l^IywMdM444UBN)Y?`XQG4GffkBG zHZr)AtFK`Jy6W6|v=Y54xc!JJeZZE*mEiaMKqB;jzNtlP>zAzhL>6;kF+%}YqPHlz));n)S2~PGw!cKd9_b&-Pq#7xx(TGylVOO zd&7?6w?({1_iP(QHdya**x6 z_O2?-3(YrCP8^&}W+91XJ@ZN13d8G}nEX1Xt2FaiBdeO(kivKgD4Jqyn@kCBxi6k7 z*e#I2ig22eYeN)&qnzqKCm{>H+VRWHw(lDrVp#-d>#byq^$bhbVxBtNb&9|wSPdc< z5QqK)pxgv+J@fiOHUwo?Y-62mzdr6L0!dRs43Ut}s&$=)t^u~XZhS54G#+R|=X>9T zdX*0b;!mhVkX=dH_H08DTc=5A&4zssvdOjy^g0y1?K3PINpn20eIEuyJ~jUd>sk2h z0-xm2kVLsgQW}(#Sf_5kORTaCZ$Npi70%V5$Ldd1B-_P)B5^d3U>$y) zPp8xseWjq4Oo6HW;Q8?F>)xOc?W=5Qo$Plq_OD$Dii5|jZJR8QkH(s;+0D0wYC`GEEAI+OE+y^KvpfkH7qGUt?FE)e`3XXCYV4=w(TO*- zbnJ4J;cQ%;tQ~bi8>@&K^lkAx1&kp?V2}|5e$#r!D*$^ZeUy!BdgtOHR=u@L9Sv2{ zMC#*WzjM_1+S+)rdt?9V&7)C#?(naWKVc$fU|QQ#>%DC{`6oNny8Gu}d=|3jow=KM zx_}YX`kF_D`=tS-qEA-WHe_1(ml>^dNQ41GJc0cVi$S?WkxwJ6X1E;BADp!1?3u+I zU^Is~oIWVM<*RWxSspzdh^s5I{9oPU#!{> zzRraC=)S7BIomH1j>-{!jgPiMRB$X&ptpE%u!;z;M7 zfK+IEzv2G6`VWo53Fr0DCFds7DanfRf^nIJZUYJRcKWzjd)ouQJtBQd4@)6ppx|1!url=zp{rs(?GS>vXe8z@|KTin)Jhluva|0 z$`Ks%r!JKn6hsofdSMUW8bpZ-!MzrbPwg|)=hX_*g0;vibgH)jBGLf&Vr^8%(&Xkp z`8Q~E@hKXN;>^MA`2)IIGaEbYwq*`(9L%#Wd27#Y+i~PDFNNGq`K~`x&duPiKQ?KP z?!F9jE5L8h6y^B|4(`|!adfFfV-JG3>s z`{+P9>%__WFJIlPN<(eHTZ;{e=R_v)Yi;V%Z7 zDBc4w^@^~{P<}U@r8_9f^mUIqbI01}kJ~4393p%-+$do_EpzBkRVP&~aRR7zAK}Ee z!C;zK6w!t!Ffyp)llPfMwC#!d3R^kM0R)`4)`*WhF7?C-GQ*WN0E8tH%eefDrX0li`S zM}gZ|#t-<@g97)*TYR91;x~kNnT{nR;nKs!f(o8ZQcG!(MGoIO%%k_NU>K$-hagCe zO6zJe_B{PDyMsLBwXM4vkZ=>0+9Xn6tbIju3x&POCw0J9C=@x?K{TzVy;t4;uwXKE z2Kg=|v*tim+FE9}F>B|eG5SIl!2jhPEt%e?&8)LAU1*N6jz65i%!>KI?WvvcLE`wm zKrZ&7Q;7<>rB}J+*bsw!UGMMAtk^E~4C)Q7`8n^Clnkb)wSrbBoe2`4Q@st^wx z(MQ&%3k0OGfssC6Kk^1%bOT0!HGWev_jt}uCVe$Q|DyFSQ0Cv$W#-@4C>weAc6J6_ z-!XTj-2y|pu$fR1sC*bu^}H2ZU}RKBlE-DV%X=o*jB-+Rfde)0N&RdEbAfTB&$2wW+NB?4L^-v$O60)dXT*yi1- z#D{}UeY%uKNW&}k-EW3CAfdB5xYJ;=s>W%Sevf#E16WOJgB!nY>~nuK5ImY)l}6T& z`I4xwo*{_t)`%p?AUMI;chMx3_Z~gx?FQ{}e?4E3M zjA~KaE@(D{UpcD(=nhwPA~hfcTC=df4r|G(BYG&)si7cVY^kkAg=tgZj6IWKW_p5+S;E7XKr)0+$%oR`c?2!&cj;k({`1t z$97xz*MUvb$+h75T=Q=DEr#xaQq~FkTrC}z3mP~%if&&W{bfIO^II#Jmz5jliYG-_ zqVd0ip0#tJolSej+mlM0um&wx62O<%1FN#_ky7b6VGoY3x!RXZ8sXwaJWuAs*7jpa z=|`K4`=&U!7ia$)bg?@&=50ZzpH=n847t?(cUQI^^$36K>mIQjB)k(?nEf`Wz8;MB z%RwvuPP~(f)RZRMR)igl%=^ISg81I?bz$dqhWj;7<4$?pOCMvEDGi>jo}qW3o5)a+ z$sNOBLBKF0k>4xR-eQGALDFC1DrZBN>y)|Tx)0fE+kSxT2V6Pq_4ujZLEiJYN1pLR zmrt!8r$-e5d2;xe6W=*c*S)_J)R^?VWMRMRB%Ra##1^dgKDJWKF<`1L`?-vq&a58+ z<&cTzAiB99XsTl0@&=xBgLPb}3XGxOG`P!-+W!7TCn9%bBki>7(dda?9ckEL(Lxeu zYhz$7>0>pxVL*Gd0zc?=eJSV;Pjhyh`kG~0J#%h=^Zk+6nZg7WX?yRwTH%qvM@^ zKXZy+SU0Q2i90NO#yXPbZ1uT=d(eck{sFXa`uw>~?08qRyYmbD=PK3L`!bVtA}BtK zH{=E1)z^mev(y|O+$+^zS2+D5nI2PYZ9MM2rv%9_{65v*X~c)K-TGFVkGx%e+OQ*Q zN{Dvuqk^Ll^bcEEbveUOqp34>-J(42_Nc^hU!WNTVqNDvA5$d{)r6lt?UX|tigxop zy>TWdO$k$fBD9Bn<7#7w#l{G5>2caO{r68I?mnM~X-&${HVv-T*$n}*E&TZ3L*71i zRBL2Es+Q-w;zjEgfdRv^SDg)=|KJy9p3O243i^YfIP_8Z9rl_sO>kSgM46!M7fE=G z25LW(Aq#$aC{G&69=*^dNvKOY`MURyFSSh3{HrcMN2BBMbx-M=Lf=_62$wnhv6Hf*BX&;eqjd}cxueu)k z$Iv^Z8VRPKbdVJII`tov_cFU?mDUJA`R7X3RU4yJ#Jbnm*I?7(!K_Qk&C=#?o9lx? zrVK6fX~PkNOkd=tZZW{m*Z8Pr{*Mb;p4LchM+3w<%x_HD=*?tPL-vT{G}mWmwCAp- zT63sJ?X1l3U3WDrq?BT-9%;Pc+iE%2wu`Q+K-NI&K6ke2rLB`aN7F;=$zdhSpH6pOmQqx9`}1tOc~T&d?s-WhZR3CpW*vfQ)oC{Oq(V& zXWQ8Je4D>nSVvTab7Cyqa09JZ-G#S8h{&0pCEY5ZVe4Tey5h8D}wF&Q|IQP`Ek)VrwK_q2J!?M61#^c{R=e9%4Y>%ScsvYT=4 zho#@a{l30j^s`ufx(_RanP z+?fhT#E>hI+TunE`8B=+TQ3SzR=sBcTT?MG-tZey#+eZ-Ue4v51sx6-pu!hsd&Bv( zxjzhQvu7nG`&HzV2j(2kUvjfaKOI&=3h}?4ObJhDJT5_iw}fc-n_^m$4+QeTLu38N z8Gdy)9M%G_r3OXHBU&<{>6rpy176~(9UOt`dj2 z)(|tzB1u1JqI`A@PVWnP(6fEMX^nr|wt;_KIqUGtCb!FNg%SvWuPQ&4M+YxoB+rQG zin1nd)lYdOG+(&T-@LAz02$@0;O|!$GHJ{8uKvb&6(3$R@~t8UlgSr}-hVR6xbIZi zPx#U5875EkOAJ`|^DAx^|LEG}D??VZDh_^BN(?~6(VE1nIjJSP%Gd1C=TVlk+^F!u z=+PgfzvOtq#JU_7F2hYYviotD?_+z!(rrR6CV9X@Ri)oD${7+xxCgeWGTx8qUgutf zM1Cp6q=#VDbhMDa+7q|k&V693)vh>qxhR#*Pkz?-;e5Q69IVQ6<&WT;I8o8OxFjub z=Rczxn2Nso#hDcz#7g|mJ&PZf>&aDd?KZ~|St)ZzuVFdgv(J&{*;X#0d{7rFqd}dP zogW-LH!gV7{Ntx>%B_ig`SaT z-w3r)+NG&wigUV{00Tgxdz}2(1|)gizZ}HuE@_#A^R_~F?vWUI=M<}uY$@)N6dO-Q z>r1j8LlW+zcSK{HOsC@Is3#O%bD((iIDkFDglV_O?d-0AG^L2I(^?_wR;{bUgzSEC zyG#sjCQ8)MgO>FmeVu)n-EX(*ytHRr4Z#v>y7?r5x%>tJMoMQEcB@~jo==9-51*W% z3xpFfRh7UgKj|EAbv*0w;$056z>lW0O3hMDPYdqW+-~1FlRD5wM0zQ3cCqqosFMV} z=om^>1xf5)p20Km!KPMak8iEIpy{|U5$2FtcutcxB~fr;g32WFQ5u_hg9#~_bBzM8{GO)3_%9(>&x@GT5ely~%$p7qjTsDP$^&PG-NJ}OMy z9XopQb)=q?6b@xc20T}g?QjblTG$)Rk|~=jYKcAQQnhdx=9arTUO!V;EbGVh;5quC zPLo&^8Oi$8EJ`WLZn+!UlCQrC+1c#kjUDqw=#Hn=t!!E;Oz-(Z==q_E`(C+mbRVubSvTuI#9S`3 zj8YJTG4o_AyAhiu0Y!E!1y0#B3EJOf0}zr@4B_o>?>4J1DOVu&ZK;|z8ZLd#U;7s49g_SGo?J791@P?HVqYjr zg%`QRk(H#)Vul1sw%>h99iN)}#0*iqp(8(?*?>mWUHL^Vs*F&|h|&hbo+72NOxn?hi(-T}ZSxLxQfP;NIYEgs zig#9YqWiOZi3`_qm0Dsv&uzsh|jcu;7TVTKgcbl1Rb`PN;)7Q>rWTLUj%A)p+XggkEuFXX%n3c zw*k4h+-d&zdU#aXF2qAt6}ER~2R>N=Wm*DG;JfIZ%7C7ffoRz*%?fuWOVLgT`SXWl zcWBD}dNE+CdeZV%b=WoEp(Y~I)-(Fp#_O^iTWt$cxX9ZOf<2$6@^P(PVuzZjMKXVy z{ju&<`FOh1*5TM*>2v_yBY7McR4e6zOXB$5zGEkczM_}q=)&&$8uQ6- zyJ#losp%R$OJ>RMcsd63Nx34YJ07RKM044>z3$Q7upa*ou`tgcT#nH#i)hWRTif0R z*&il5IGik{trG9}w+!0{@k4>fVVNy>Bbr3q5%&%=4|h=bH4Pa`wttm zmg$>iE_?P-GwzY|$-+*)X-{yx zLLX~A=8b1O2{7^zq2hCV1y&ps((-(zn3Y&cHWjZiC?a}(DO326*@Svz83h$wmjkTA zwYz)vz8@Ehx|d<2T_pyvenis~-C_FduW7P!6%#ADC4a!YE(uM+qANcx#d>1=(dfvlNn8?cmU_JR9&=Sxe41R##MrxkJKDmm^<`{cVJc@I{xI`G zUE)xqVyQ{9D8xuHC#-=_<2>cEb^^8T9@QEUZPG6HznDlsN<4Mh7`x`grTz<>p|d5C zS=grhZRs*@`p~SP9k3m|r8l+7UI50QOykx!Eds|KYY^l&Z3*&7V5eSC#$iIw#%3Fd zW|-5IzB_1iq|q{88{?Z&ai?>j&WSmu_}07fET=o80}DST+KHw)uo$^^_35I6NAeIX zCqbd?(&oX~t4yU*ejSGk!}#zQKvg~d3i+_!DvgIlVJl`1ZM$;0!!ED&UGUL8mufR8 zRzp~=y2?to!t{IRmbTmLkJ9ie##~)~z5A5REXyJ;O&x%pocDF?Upf7%ZfVe3JuGMY zItBS9lzHAY4Z^@422~Z2KCZ0imZkx)DZWS%vGA6-j0hmTZU$GsxdgOpsK`5Vt9H}T zOS?W+`zF&p_@p3IcCbz}@?%qlM-l~okTNQj?8Z5p)6j9`^?Hdl^;xy%BhAs}o#qCv zkJ@wKsc~nTaP@5J6R4HD*@W^5;c9~I~GL;`0m3k z1dGpC6unX=oNan87|WkYvzpa!TI0;XlT{+nqXO^fmi;1D(SMkMF~d}ieKGl@|7P=m zLS5c<54qDs6Wue08Ic$-yH2f6z57(boLjlJtsYx31qBKq$t4}>=;^WV zp@x#&{$`JikVH4+Q={d$@9RQcFEnn1`@gE2%{9`(Z+}8&YWdy_ihP{w#Jt3VbZ2|Kg@$iL)%crukbZ`*gj&(SKCwy|ix zSm(_hwF$^c6k|%r_o1s|w2u&>O0JcL-6{=lG&{l@5tElh>*Wm+CO+Zs4R@w$gu-Ux zi-p+TUhHv_?u73#Qf$65&BfGvwU2Yp@vQ{M)(`*!HjBs{sRc@I?A}N@*$C(i_Dq-uFo1Yfi z*EN!#%p~{wahbAgVD1%bAba|9($_+gGf7`NEkFvY51| zBTf_d%i9o-g0a-`+E%nA%m|2{VcJ#PmaGIG&iFV!7c8c2I^x*sw+1-ZxH*Ag_Ihp75w#tP}>W3Qf~$^6SmSdxJgDOrMWyza5n9 zH;nZ@5!foaa_m5wt@xCp;U|;p1ZJMaKjHUy9T;@`;Mqvnq$QMQDfma>1;eBtXezlewi}Q47GFHMra4Pdv#a@YcUF%tWg_V&gs{C+q9*{v%a&& zR9RoW%AOq5k{wFpoh~kc+<5q|4l#(1-!I!cD@S^WWVQTglsLvHFyM=TxQ zo=#oku&;3*@;kf6E3?;@=U$mTo9b#ks2CKOANx}7OYgcfZWe?^Mjzjws$hkopK<*T zwK*n^xxo}0715WmwA-HGa;dPn0vmy(c6ABuOVi2{>=l@T&()fCx@>U>9WQXA9;Y z+r|}=h8I7};P$6Xpf!XpPwJH1WD%4mVI^WobBXGD3cVQvh_8roG>~vX`UD_U-F6&; z@oiwduJFE=Hh`Io{dUr}!e-yRzk9)IxaLDp^NxtXwx>+FX=j@UgkvpI6gX^{;sjlr zwoC1KTSB%=7ht1uY6eZt`YpBI4b@prz_OQ&7{-TJZ%-6?`|HNz2&Zm1YVIzNQ&xC+&@?6TGd=^l2CE<_CFAb`G>rp{3gww*6NJX{xoXt38+}M=kqdIwN>C2 zbnA3`9muLq*CO`8(WApD8Ieyf)x@IN{-K#n)be(enKKi&5Pf?`dl6OwnT5zgvV^kDgq+#2Hnpp-Y}hLe_n09%-~F}p%xfVZ^Lf5Cv#CotrKf&;7Cue*22M z9!BJWS0{T)0fZh;wXriQ%{!!FZ3q}knD%16rjTS zx2`#KJo@)f{6^^S3-d=eL{IZRLVpy#&DN$g1>R(fGNYl)dJv*O+V4j*Uw}!m4m^Er zqiHT&6tz$n8_h+ZiGiLaFncdr-=0!E%c(&@}4~}WGw*I`u{kt8+7P++D zi@G)BUUTE4r%gMU`vfxR?S1%%Cm5@i=@(+Sdx^NmvreK~&D#v%{pXb|H|;A^i;bni zn<;`3aBchmQGs#t=l+(RN)ymJ=}S0V-GVYv0QIlb{D*azY4+jw>?yg*5;6{F_KWp@ zRf0w`_5)xUrIc~70p_ui!?}@TE*MH(M4fW~4;Msv9b#sZVZ0gZQiISc-qhmX^eXRP z7=RF3r%T)xbbf7Q9xXt+)4X=^qr=O|Iw*jf#U*U6@#QH*0#m*s%7qTK5y$3sVqxoq z0Ks@qm(=pzCIC#+nH~PuoBYR9w`K-KcDkMe^b*~IafWp6nxAY)CcvKdCW8w}I8Zk8hwNv`T^uk=v%)1ZnUHF&2EqQA0 zt&5Uaq7R6Cq8t&aR|E@@4JThXtp5RaOrr)vt)?~LY75b6Iy=f1;0a3|9r)Gr9*Ob$ z;R|c~pD9U6nAqZfd9dFktm7K}q0MmxA=nJxB{Xlr}btly=5h*;2~t?0!EoIdwSw!;upnAw|JhDSVO;Kwk-Ba-b~DJ;91-$~=Xj zWQwA1&fl(Qh)X>hR{`3_&}@`UfPz|xiSMV`PXBjAqVO@Hjj8ry!$I+UY21k`Gf@A_ z%a0%)JRwW%&OA)r>S1~63*R%29mgm>14ot#2n`&X&1`(>|72ZQ5TCj)C4^PpV-Now zZc6kp#P&tf^`|&mCqEEO({k!tA;xH*IqAv6l1%u{nRIXv1c=|Z9lQa z7Zb3@(6^Q@+2mS@4%*LAJbDnb#@0~lYzwKU+1?8vv2sB)yt(Co)gu|!!{oY}vk!fK zXT>{J`dV9O>eb*Zf;OpoQKpXMIc4_}JDy@L|ARmLw8h6x?-(56aRSg%M-RRu=00bQ z!ieV-vT%C;{{4UB-yh^=7t(;wt}=kmJC8(XAex?rWzE2*26ed$!`|qbZlihuS-Ci#!bzhvYbLLOS*Hal0MZlD!)O{-U)g!=YQoyyZz! zBAX5_S%kp!@?~5O45v`+%Ua&KJRMxCy=pPFQxUfu<6>04EX=2`k>mL*)n2YLvX$P|vVMO)!f^1-haBleOIJ8xfz}Fmfpa zFoZ*~@x-(0@LeRq?zMkqUQ_iZw%YeD&K-)F^BTofnMk((j;x}$&g#cHy8sUwzi1YI zNhND+>42&R4q`&-ok=3;&@e|3aI9=zVrC z;PmFDNi*;}&s<7;;=nI#`>`SO;a0uPb|}(B`K5PFg}6TNjWFT{LSZ<{8Q3n#UnCtT ztS75%u=HHyYRh`*-7sy{m{~;Ltc&4H+VH2Nh#r&+PLls5KYVK?XsY8HxP5lg#dX*j z`|0YxZcS@ZP#WI^lL8e!q&>sfu5D}Hqb~qpT4M$@CD#>x+t)F7ezQQ(?WR4*i`I_O zhv(cvLV^#P6k@!RIsat~{gsekQG(AHj_?D;AKmLnCkiQ$&qE%&gH8vtdBn|)uw|(` zg%0bsIZa!yvVjUh#4XI8vYJdwUr|%V(14VkY@$n+Gu`)+>g%r9OEpSTMiDHKOZHWs zo&xTwQMG{YG3>uzkC-Ra^+Ico!)I%+k$DeG?^X?+uRfIB1uxMAoNl7@|A23Zf?Dl@ zN4NGl7*ibhVWon_w`swkF;FB`;O@D9f%Jb6);#(;p$KaAZ?*YGXEpWo$0@t=!MwWF z4`e`(U=7V~Ytx!F_xbNz=-vJJsLp6 z^VzcjXu+6U)URP<(BJFUp0B?H!QM9btZq^&tr}Oz#ui1RMF?akCCCB&9u_*Clbq@Twu?s#U+t$Z)iyrl@r9`6j+Yh_0mFJ9GcnEflMWIpf zt_k&Rnll>IpmpEKs-p0N#1|Xjky^eLxNKy==3u4#ZRLZNkmFGb*h z0eOrEgeJw&^-Mhz2izxbVZnUw{zXY2ruiqHfnb0Foye=t=2yMwmsnTIRvh{#+o9(w zU5pv=VNs2$=3hZ7Iq!z-Ei^)Tu0+F#3rpASWb~R{LZ3IwI4PT({I92bozU@4-7a9rM ze*PKzw&k`Bpy-Tjga&_GeNw4QQJcpbg&M)RYM3TV^K3flL9xhua<8kUYzT8W(Y)6^ zJ0YNknRXb}ipmE>AWa^>^o|Rs#CUoP;7$*>rF%=NX+K)&%%uSYH>?BoIh>v1Kzsu- z2J6BH+Rt!@eB3lJnUrXoJ&P%qpwT$<$uj~ra>@%GexDv~$dytJY)hV7RKT~EP75sS zlO;3}fPT;&L91uv!va&cHbK| z58Vf05NG9f-tkMTE7x2uhIxg3Re3|B^MDew-HOQr^953+<{W;A*xX&5pX`tvek>UXg z#B#ghldrmIq4lXk#h7H7GM2@F)NSJ6G zysMY~lM~5P*dC*sT$XcY=P-X3`~ek)Xz&4>41y((*`Y@A93EjxlXszz7d2=7#Bu(V z8h;q+2~<|&LI0Q8Tf~56V7{dmVe28`)})`~+aIU@PR9y}R;p6{&c2sJ zf0?Eo996e!K>CR)Xr%PMA+i@ItB33vdQ+mg$JzGXCGKTYeDf`nK4&6^>mlwA`6^fo zbD1wZ=hQq}Fk!t4){;xMcF`7ynTR2~+wIwP+|?tOt=V%HTZ}^72jb9ZRBZfriVPj+ zDY{nr)b1i}ah#uGHbCab^66Q{POJP4hX;!MjwKc0a&)N&)hV=xEMH(S-Lw6z&qh(I zcW)d%k0G~TKl-eew56L%X&JV^3=G~w#NBHwLzEwVEeBVt#uTxAL9*%>3wu9J(fVFL zM2S^w*g3V7npdmi)R7m$!C}H#{W6lb*r7K`kG+^H(^6?|WUGb0`?nHz?}+laKT$uI zv)}yg``Y5O?cJ#RaQlY@p!1ymqQ#( zw}Y-dnaZgql5WGp!LE30nH4)lh0a&vi7{nG3LZ%tqYBzw)M{f26sLaWIk1!aliBJh zbakQj>^Tlm9tDmd;y#~}62^U1jd9--3cU%qx&P5+1iU z*58V}TvA@5_>^Fzwh#PT?y;%DH~+B$-BO_2bA;iRvIh9s#eVg(cC_T7MWQb)qzxLu$^ls_ZV5svh zOYRpNzo2oX-Ustkn{!TfwlWmlZ($bz=yh9cxsvs%FWUvT#I%_3;}X8Ts2u`e@lp%u z*14*5F6?c}82sEihDW}hY~%}$%x~|A)FgLkcZ2e4ndJf(NEhC9Dow)7Ju6Q>LR2Um zzy7vKrqq=%iV1r@d^-PKEorWOPEpE4Vff>LltrEMIja~&_*2ec?DeR;{z(_jZ3(5K zY4qNa@G$F`#%yr7dDj)*(8OYzZ<#?GmFv?ybNyX2f9tcoreQONt1i?7#r<>5aGE@Y zIlVtp%n@Lr?w2Aqv7ZMNJFTg4;uET>431?oy3{uIVU-FYrFxRE z{5dbj4H2~@tKH{pQuWJF`J%RdEiK%fL2=|5E!ZW14t&|~qC>SyV{V`m$h=`I$wdaw zr-Sp&DEAa|-Fk*QP$}*RS$-#^ObJ&(nApSex>B4e9?OBdAW5#T{g#2OvHXOp4SB71L(d)E)4I{mt#NoZ-*7ry1zh=+F^H0NQ@ULFKp(5`2qUgdKaSJ zMHZXR`cN!nV?6d#ENqkQ=8(hHvP-M>`5UpKTHHZpcQP*(^iev%2_>_zQe$kp!qoBn zEht?rn`G#If|cvoFk&IRCBJ2$Z#r9V;m6Vw^GmwNI?zB#cbWfT$ivSL$Dd)?>&ZS@S1`yCZlM;yNx?`_oQkfS=v2(o~?2j&Mvb$_21c-6|EEe4+O@RL!<_BSa=v?-!{)8KudQ)cf*$cm zpf7lYIeV~h#_9c*Kp#JQUN|hT#b3s`h(Ev-vVJ`37a;8IW z`3gWr1?GT(ue4-EQ099sn(Ht?%87#};-rn{x}y<-?NNM;xPvtY0^lF)NCXHWE0Dwt$cSrbHL80x0hr?`=aC!^51g!nR zJTQB4+F5Q>*grv@siX5bkbdVagimp-%JPe)mg z5YKu;7D&DX4b4rN_)2vWysMWzoK=T%G(hmCG^d6V8{d1n22)_ylZ;ejc(!T-#Pi27$N$|*&?8i-zgBnm}%(;bPTER z>Q`c~#|Tb|obB6wZJ9bio!1e&@gluA3stEBbPa))a-_bpPsf~heZInkPxGNg{0@e1Zh`Qe*jMt`FDJ}`b zaNP6Vb(|U9u+9BCB8BE{*;kH6WddC#Z(D$EF55N}^f-&nLkRzB%Ykisl7$ky&{CwI zpgX2oUlnEjUr?k8hs~F{D5gsOO0Vze5~Qq&iX*w6B|j;KU=U5#0@Pn{nP}~O3S1}u zXx)m4N+xp*bgL~EyX`&gX`Q`X@^Pd>kYsU}Xgnn3$@D1ED5)cg&2X+zJp^&P=yco% z8N}`2<@8UoUC4#xBZZG*6-|Ss0oR7*cw$mIV6kE|5o78z%{oagb!gVEFfnITOO2x4 zulhgIjk89LVFX{wEEp%NT(%+9$0$08a_y}SV3q}CtSjvJ{DXbs<*Z(Oxr`)}W5e^q zz94M4)Vt2>WpRJfWPe(G&?OLW&o|X~KB%57m%_0NbX`F?{fe8w&kD4>1J=|~E~r=G zSdJ>(IgK5;%V|EIXkKLr=1Zf;2Jh8}SVAsYjz$#UQE|LW++#@Fo5E!#L5cSTr9oi7{B27=_HGWOopCwp-YftGdB_x#J;7zJE@DunFzny1Jn6)!`_ zYgJGkdg%c~jk%vWUk?jjYuxr_;PEg9m+?&3D~_8{IX4Y2>6mNj7O)M0F79RpL4t9) zgs9$|Ax#!QlG=<9+1mBttOta}ILd|4MPDM2PIG)>U4#%dv;FeFClP*~1%^%s6E^DE zD8rVIUvRdg_q)=8`r`cZKA1!0IpyXH(?7sXepT9J66i`cq9m!D^ZJ~xp=sESB7PKT zF6xtS@6r!!p?|z^O0&xts0yo$oB}m~6#1jQ3j@`9+~qohv|FT(jJszDb~izkPT8)z z?Fof3$luX^j^qHgbjYsn|(?y8iH`%@}5Bf)q>Sz7mO z>1?86PFv|IE)bJ6$mevWiTbNV&20yzo#nTanqIPwR7&2egT&1pj3jKnc`x!ri2;*P z3;uq-+|ZEL!B&czK#%~kNYGm0*`7?fl}Tro(s7r4U$88hQcz^mUFX)KzE0S^e7hZ z8yaRd8J;nz0Gcl1{}AYZM^Z8sRpI<)a;)y|i}~C?{0Pn?!WLKbsTst0isF24I#Lgr zQTdFNS}Oo0U+sa}2NFLu701hec+A64k<$6Hibt^;(G&xi=}PwbGC3dRB4bLZ1_ zLG_P|LHSH2Yk3$kcQwYn36fPM>sUa@!E5q4G@T1op!DS0uw+z;S@4#7V@sx@7G^_f zZ*w!`Tb=FyN|yX5jY4tb%Q*X$El|8xM!~h~-h@t{kBBw{a5IEj{LUOAjpvwtiE3}1 z?($j>h^s{KvDf2UCCWmrd$-*`R?}2M0CKdISnZzJZ3<~3EEg;6lI5**G^oj8+Ytny zzJlCVXxeFtHThey(jS}rRT>ixndA~Lt+1p~1`HXQ>-@kaycWHSlwPKE;fGCqJP=-6 z5QTiBK#uCIS2r-rzL)(#ws%J+Lcumi^+je3c&}|T-><@ycJU5X%n^Z0l;a=WE6P)a zrem%R$edYWOcC*H!kWv!R&%63TzlyfdT?#p2$3PARw=y1oD=LKgH_j$SCb0l>T%^l zBr^#$76$;F1Idh}m;b|V`V+n9Gifw3CwK|uBOyTL#_Ls@J#yM>(L0?|oJEWf!NcsE zR-{f{JalWL54~oXylMj}K6WJ-^>Q`&okDOkbu@`aQlYl!i&-|s$brzmPp|xmrRG%h zl9$Pvl^pKh4ru{!yE~3kg2iV{QcnV#94sy(5HwsE=XCKECX%qh4*j0gAvGG{5}MP% zeF{;@w4{a|M~a)P&0A|+TFZIwG_I2J>EHEAeb{FcD=OQ5Z&4bw` z2LjbD>lm8aSCzk`m;~O)11hSnSN2n`i_}HA!JmS)kBT{O;`X9`sZ# zuP%IeNsFM9k93SzpH#NhK$Q;tBf1_t(;sp#U^5Pj^zc6N?1!M#lC{r4>e%|1>Rl!7 z)s$%)?_Y6XOeu^bGgJhx>c(lj1U6=i_PI0!gi4~qh6B%f(P}{ijRVhAPh3OKEFzZQ z{*PSbpM`~;y(fHIR~q;xYxeHc@oG!$)|(U4--bSczAx;zx%Tnblz`_qH(~|@gTm66 zl)1nwIuBG9O=%Csh>H_hJaFQo8L~8JR+b*RK?XTkTr#*jLh@NAT zi;3Oc8x9PLMEFjg@;dj;ww`8saKZ4#v|U~rFrayx{aHP|?-hG~j4we(RAkdiRV&xy z+C=lZ*9*BF@z4)KWq^or+KOxAQ|P08Dj~-I5fc9s7j9`{P^O-6=5%RSKi@T{;#1#j z(`YoT22nvSlIkD3iG_E}@GRgB5%_+@V;;C&(=ka*{|tDR5Zzs)R;k;a45xlzD%PQN!oqj5FX!idnZ2U|5bwW zr+xmqFqae{APQ<4K@^(1uAs$tIY)+vuWQPhm`Nb>c-JaARo)UCagzd-fHnnNFa9z5 znSvq8xD1uFuP!y08-#or0_#~kgR0$M)b7j7{ZoK&)Wwh&_Jw!(=ZE)Xx#=A4I4-f1 zoW2`jq&UpPJ@R;BB!rs`O;0jns(M4X@M-j;A-}qa`0@R z*p+`s=ky);{rPXaP(k)Ntid-k;Kp6Y&9glVKH~HGzD@H*&d7WVw;{vrB6#DJzFa8y zZFk&6WKrx*RZj($fb6PgJ8#zpzi(dqYd+;qm*+nVMS4ki)+BXLiJL4X6cjc*e?7ff zDyXX&^={>>ttt^@Rq)+FI)S9SSZ9gc^ztiox2lE-9MU4W(@Y*0{nZlqE5H6FE`E8W zBdEkY_XeA=*wy}j&jQGX&_aN+Qos76W9_$nG*>;}bzQeL%7fT+&U6k^YDHRvQ`Uie z{x@ICe+?qLc?ZA#`$+qrEEG}n*@WN~Cu)97{T=0mH1s&5?yKP+H;U!d&MEKsK0xn{ z^i3t_4@!x_vOV0YCsZe{LdE<}Z@+S~aO$Z#{|`lg67n;I+sq#+TU?v>+9F4!(ijB; zDu#aulB|>V*>X3O9~6#^4E6cs@7<|n@YC5x5XgMv)RM2R{*OrapQt!RpZ$sktOqNl z#2rjvOkGqdC14w?D_NN6+is_GS0_Y2IPro(IfwTqUM~3l1EBilDdq|g>_T?vA>#Q~ zJ>7tVy-E91TxgoNz27Kf{SJWjRpozqg8#u^O8J?&Gq^(jkoypl0zOehMF!5rT%-(j zJ(sa4lb&sM!AJZH=ig>V{<;t5o~Ah~*!$wxN$|^W#ZPxd|HbY%3lVmrMrN;X?5 zNe_6AqWY&6FNGlqmQ1xlYpGOZhLkX=Mt*aH*IT;g>ZSE37q^XiKJfhG>#JqXbe&|~ zeCWE%Jg?|?0u2OPTh*Q2l^c1!t*rmmv-elD=&x-0KZo?Y0V0}smK4)i6x83K!Ag8s zd*+bNb%Jp3)y?f&eQ$MdxC_So{HMP}x%BU8^2CQkFg)qvYiZCEauTBO-`Q>Uo+9+1_hm=w5`MYK`>&y(Lt0bO z5V|#|dHM^pJU=VvzAlb7a~JGyNbMaV559e^+gDp^K=v;_IH#;gjmp4P)(O{qj>25; zzef@LX+?^*Xxb-4q2xFmW{geJa@4x_6wfxt4`LT&TJ*5s?Pl+Li81AK>^wJahmY*| ze2J<3dwytXVHllp7!AVdo;{zdOD7fdQ`(RxgUPao%%&3eUkL!bc7mB1WoC=CCVWC{ z47Xfd)&UNmJq`Wn!_hX5M%acnP0B#fkKmKay6|cHE+@BVc_YpX1(pWCYBpX9ib@?F zG04%)aMvq~hvFseeWlv3{@6KaVFdc?fcIl=RYQSmD?RS~RSQN}nII=TSn$pvNo((69Z#=$= z3u@S|S@v&Q*t){kwOzj9PhyR|VJ==t{Q&1-OAmJI=2|}c`RLctP(o^iqfJk#o;9Bv zhrGupDUkuzu4eCci@niS-VjGw*+D6+1;Qm zZE0px=4B0r=-XeG#U7we=5}5pxh6b9>ZAqxbIzPwEpsfglzqP^4~5j6mkg_K)F1ZC zw^q@26w;M!z)rfZsrFpr<>I10$BP)W7S>A7q7iY@K7tRGlH_*PV?3xc=4k8RpW;fU z35YY`-?kQ=qcT-Gq8?g88fV$=ocpo=Ur+3}G%Ifo=E0|3&B5-)ea>nlisu0*J}Ldo zxfGJlUC*})@< z%kE$MHAZCxtjUE7InXxuCj>9;K~hg-l?rzxjd-r#<=yGffXLR#{2Q<=MI*R_3TZgN zPByFMMHi#|H~Ug|V>|<;om*}68AH~dibcz$Lh1AgfO%)O8g8p4=$B30oI&H=1mkx8 z=q~tQOq=GA9nk5{_vYNZ?7E-*xq)moD1%q@7J$b09H>h9-A-Rg+`Xpz2o3&gXQAU zQQy!btJT%9T#F?V13u&XA2$D9S_QL%AeI3#hct)eA~XzodiSIys7=kIe_@RlI+oM) zQZzBYcCs20e5S>Fii&nqd>~GiYfS2$YQ0P4%DoZgB=B=^Pr&Uymr7pBSh-+MH4yK! zIJFIBhVSZ^Ek{Gu8Ve~YT&wXa74aVG8aDlH1p%MKx4TgmH-62a)t`_9O$D6m3JGjd zBINk)&_vb$R#^lOR5VC)MWuK<86oJf{NPsPtA2uR@o^`?%ZxQ;sh`hqC##@i+5mZ+ zQLtHBDgB}7KoI50R94Jw?Nm>B%v9n+;3-O}(DJLksbYZ_s*Y$k)8`cN`ovdye1Yl$~rTr4_MO z+eT`mH(ymmZ0@By`<8x7Yf&7i(2QJWRM|K@5=-EE(B@M&=|gwJwZ@r#W#`B^td=*< z|C?L&bY~$r1WT;_LzSO_EPySl!@SBpD z&kX`Lx~I+$ufc~?NZ$JN*UA<)`>1~=bcd`_tom`zI1>%0mI{bm9c3yA3VR;LU^pi% zsvnjIoPRrQ%9z&G26z0#Q)pLq_W&P0HD9J7o+M^;ig)0fYD$X1ke||yvSuHvyuz`; z=Y@pzOFm8<#pWYYaP??y^2BxL(X}2V>VxL(@^>v3Cf$s?;E&I7uR}^BrOt?SvA0j? z%^1v5L5(X$fE(_}!6D&svF@q*vVgsz5Bwlhs@^?X)xaQ)yiLB{;!P zD`8tYl~H2gb?%lqj0d?-2!Ui`N8I^#C?c8R(y-S&Tw`I=qd*hadwS!hK6Bfx;#;mM zuV%?K&u;*$VPC(@$YP{|^J@f?S%zX*(-V>)blD)<_B$n8uajwK7ka*?jM&s9&Pi5s zj(LFPV1)duk|F)lg--#(-oIukAGP5;Tlok_ndXyC zX++&#n4CB2(g?JZFLd}|JDzuj_vCh~(+<=5AKYbgOj@+W6}%8aw`+t_aR%7(!(8W2 zq44ml=ZNXmfJ^qijc1geG)|Y5Z;g@h6DdIkFNS7l;!oYZC}hFUcj*+ec=e@)+1XR~ zK3q*_k!`s6Ld5IN&qtaf4D5IAv0oO=XP~?H!CyyGi~W7EQi9^}of8fh`MmH_UmB-= zCYyF`c5HSW9(Ql3KlR@ld=X#*^GV&`ckf=ZN?cW{LpMA82m@m_yRIvbLBybv%x);Xp zFM#IP1^Iap(t%k;NXA-B%=(4>_ocu=BV~S*6{QL9t$6#TeGb5lRWTvdvbGXVIuY=3 zJ1N)bKqBWV&rc&&wVkrKV~6F9V)vygzZVS*xX`USr0lNeaB=_-%aX#ec~`ZhyuiXf zKtbKXuV)-qFKTkD16vxbXF|p)8S#5Q93A40?ijk--=k3u^JjE!S!^P?*7)#B7A8EF zy;v<_U#}BG8&h&lzQc6jxX*F;sMcewp`)DEWSTIP!cRn6qj7_^dlA<1TQA*@fAB`T zUM%I!`>bn~Hc{iBX6O-g4Uw8<8XLDl&T8GPJ}+x}O&qT}nv=!=B7ze+R}YRKO?q!_ z-u9?LhaUGdJg-=i@7Y20E)3H%RyBO0yztF`<>OxJ^fpW7_HbvdgO33|&o0nEZ9c$` zvA@qb|Kv};=PDW7`p3T;wj<&EJ4eH?aG+yx?yb&|LW=ekCRa5BD>D9SC zN_0TGi2?lVJO-;BeBQMr)jfziQn<~#d7}xvDlhpsyWJF_%yO_Kq$ zac0sGWwS8ef8nO{+c6S1Q@_MEJuHcEkSmEgx{R8z!D46^pjJ(%Hx$j;rv zzenOV!0Dnz&D2TE);4Bj$0OAX)lnmh!AmG$M3}m+Goqv0_s7v-{%U$}M6|}^eRVro zD)H5dO=?fZL~xJih&tr5U)N9#4u`%^(+piYQA2HrvO&k+R1b`2<DKaB*}T}YPONS& z?%ffWzqmI-FUx6$7qLF7iYBKn?35?YR0ug6sOF4+(qO#R;%nC{h|TvP$GZ2taCl)^ z`eD$)DPQlMxLbB$3ewhaT4K}F-gt8Y?^eI+p7nG#zK*fkZQ8t@wc5^(sV6jv8-I7<$yKRUpm`Pt!M( zf3B>F3^4Qkf%(7A6!DNSE)$=f*5@<-9@4cL-ugB=V5%go4QG-Eyc>9P!CVdHX^Z4EC-aY^P z{WJd)xZt|puh;YWejT2#>jFPBzs~A>MrivAvNll8s~Qz(U6=3P(yhv zXrjlV@L7btUc1TrWdM4r_GysgANr6xy^#*`*p>zzVPP2XPevGBBf$(+`A_Kjj)6ow z_!DpVw<7E`B;Ea3h{%1q|NQQ^wo=VUufFxx42lsPA(`+|#Fh(k=wZr@#ZY^&DlDnt zJ|UruvTIQ%CPFhUHC}{VStT~yyT#;>EW}I>rd7=+gZ*0^oIm$NcH9%G(b%~>#cYUHRhDbr0&9^=+Yvp6!Hnfi;F8Wsp5MS z-`sN3v&{52y8_Jq^WR|OnDh~vawtIGOADTHkGoSHU&K2+A${I$&SRRC*&VC5hFZJU zwo6C3&SxprwwvB{NglYGkuT~^6ZIm!E3a?KFF&D+I3o%s>O+IL@`0w26WL*MC>6!5RZkvv=~A!z#SA&)v?CT$x_Ng>Vv;@hBh|Wm z&s(O91 z0$+?0v$2p9X`+(eM!6R>y0#V{t-pOG2FD+|nV1c*DQve4eK#>Or(lgUKl{GTzbkcg^EObdn*i z%Yo#6rfRRwntFk{r?v?+@uLA=el5B_{4jFBnBdPVx0h zqE3%Kb|+Jz4M#_AZpoKbs%Hz&v%i7_zEX<6K!6Up+-td^Zj{L~+Q1fhk)ZR*N$TU2 z&Sy6-UG?r(*{n{AGa_^~9eJ`p%F)tTZroyBe7Hf~HkTDX-0=t)t5|5im?tr*aRYzK z#)J#~&ER?0jE)$>;`djz<(2TX>b(o&hl-Bh%GjOkfg%^i_w>6~+|^8e-uBd-IeIVW zD2`S%fbK0WRLAx*v1MXyM0b&rg37{EBj{3@hVf_lackO!EBg+4@uZ4ErMAWRK8> zSVDB^LC`JKuJX;gz%VW_<3x<$I=Tv$b7FKYoS!L>;bB7uN7gDC=MFratE7N)y&I$q zaH&Q9m42Lzg#bl1vrPPXF56Z|!wJK=3y0+mpJ#(4ajI$fJzPrfB#9Jlod>ih%mH})n-r9X!J@&y)UAD*vUa37RgsG< zzlNs@kVA7r;}5MtwO(wW?yC{?!SlMl#sY9@G8>63Rm_fQ8%(9h9!v#);=?vap+aU{ z#jUH_5QOU-3FwSzc)7CTKQg2Li#)1qRuXJM@(aBZgat$WP_dm%n2Slij}spvURU{^ z8Hrg(QN7w_Xmp+&O#gvLBCN>|u&SkhXi_PhJV>71XZ&S>K#D4-oe2+Y$d-t*Hdorq zX)cArU=X=|I6JSB*f+S#p`v$?s)rS}*e)F9q8l=D*f}bfDQ$i7^?JIyDT6|iH?vFm zd{+|R(paoI{(ZFZG#A7V^#a}3a_+?IMgn3z7?f(c$ex%N=6~5d-0qyB9xsou7RG|^ zj8IyfHVTFfzlZq;)b!2wA~$y{K28V!#%DFPcYWx3zG3f1z9yfbxh|nAFVuDWfc#m% zToDj#m9d^MSfuXkgz3=%e*j!U%{bY36bJ-yodG1I*Dxz@fko>c7~}4W;YTW75d5n* zFj+bdKVF{-3T)`mJaLZEw})y*h-mKenQHm!Jh-JFQq8|b+%Y^IoFk9 z-+#VW-yt1scM!z%YHrU=uP5WPI`*ASxN@eld%Lyn`H#!_XA~7@8VXv{YX2L2D;=}; zgYbr}3+K3*Df(wm9mufX1(4|t&YW0F$a&mLYxfs7-wDJ$fxN2Ja}XB$dj`H*)0o7j zdADcQBXhUh%lI@@=U|bMy2+y~?{?j}=PZ-GNaQ@Q$2@JF z{&hZH7Z6@*5=e!P#kmT(>!p#M%quC?Is__ z{!~6iw}%wUlSIgC9=5sV)NCL`)N8ba1a7*hxGnd%gUBK<7n~hF3#B6`%x|g+JE94N z3x->VK>@mhz%^kk0xr$d19_d=VtB`^%i>OQ;#89g*E`EmrPVZ+4o@xTo2#-psUxP0 ztHZGL=5XaK<2mqRTdl&$Qcv_&Tr`hx8O?uGYY6;36)?mlJtlbb?$P1Ivpe7H*h5Fu zc5iLfK!lm@WgRzC*qdEqdQP{t(sUkG@Z5GthP_!hiupQm_l!7 zQ#;v@!{#E7$sc0!RQl00PWTQ5s9?6NW()}X0t^0y|FQ2N>3`ZS-&9IkNh_SXlk|I& z^dFxB1;a^hmP6&H*@@@iiL)haLhYn(E^OpNVE+1ai1n`6R%|Au>)Lz&Sz-A<9{E@N z?Z|FD<@^Zl&C0`URZ2Nzl6vud21XRwe_&u8saVRk%<=pe_x_e_2i&>2iV~BeU07$@f+4Re%aHKK@oYl ztW8ardo2DQsBv6ALybkGmJ_{gZ+!Mv@%2woT=}kSV1-%t5FECUzLo%_Co4L*NI%CBwXkrYtP&{UDY z;(rp#o3Bo~+1iva>S`V9;qr30p^p6-y&PC8JXu&UTUsNnW800}=Bu zY^W9A`{Tbq`}uw8m&NP#F>~W3CMCw3iw6=oVL0i-p@*hqLl9pVqv(l3{{0Gee^EvB zmTlueU$a=Y>i7d+&w47saWha4fl+SCa+b2+jTJiaclu*Q1to-A!@XTjYUjn*fU0TwbEuV&?@Ja~{`?kH-(mq=OKZRK{RV3) z6ms5u_X0gk6vo8nM8%y;_-bxY9Jmzl%0UAL5^wO zE1?E2<$GVGCyw_UPS4GhWKbil?K~? z)0vuYv90qBWD6^i*G$1uPD5Y8e_}(7O!x{QY=TyOjdKhe=2J|nGbWM^6(^TkI-k*( zEpri2y{-~+RKFj177mQl!Z^7pIHyjLNh({?Yv`?K=BAB%7Es)W&$6t5MblZ=_6P4? z94w(+(|adSmNdLsaAWmue8Od`yVil|v@S$I;md zuNDw|HGi$hdWkJ|>#h$6p-RnBZ%>y-EmM4-bC*2u4FbLj7e04{!&(BT%hG?^ZzM=* zmKH}{UZ(obT@jCpUy7Lsfvddc@3+=v>CP*Xy#+cFxXQ3K#V27w+$ioq5cMzhOQ3W4 ziq$J;uFq?LWXj#$87859VhH5c`&yZK_&flGjW762hd`F#<*;sN~yB|DMZ#8%~GNHtZv!^kw8&>#@=4b8MgUGQK2$tY4B( zeMSRQ#)(CkwODtPN;&aMuKYgO#8{<(bCz6Gw&Xg0@&-f~J zOK!`l{$iK^h?T0^RxR!kgP}>w-VVoF^H3(IKA=h_Iwm z_BPZLfg5N{7Gd0X0ypUOb8Ozd0?$ffo$@_Cc38eS@dR$${t(~y13v??zvOC$vMiYz z9YHWYq*$jX^eI+L@ut1j=g98{elZzO7TY7(cEqH|KjqD%+25*l2iX?oYMnua>oBbm zWc3zR&hQk2`;4k<#y6)`{6737P`sJ?&g8GcRD<^S%U5Yg0p_4$ZdzD z?Nti)+f5zO#!yt|`r%^dp?ckvNhp1G#gVX_G5*vjXU2?AfCfjS-lOGklqdFN5$jnb+(YBf*+@s zP2J$sl1!~Z);9g>nz?)%Y;L&}>VGYNhn<72I!Z+s$8<(TZ3aj}mcAnE3EIU42y$}B z;L~`gZ*aw}(MpoRy$R1n+G^H4usDh;YJp3jo_5)8+ZN&y9);a?A^L2>1S1~1tY~h! zu$(VkTs@_Ft1y%f`67-d(|>`mxTiUQmL7;e4yY7ro|K!IaB?`NsO5{uAor9WzeF2j zBKxMVF-%_Mko=UgW2X~OuDf3Z9(nL+1%Aqqy$!vv_vvL#*cza%8#D@TdynPF%ODy0 zx>jGR*PI#Fd~XO}LC39jv!og?0ecYzIrB@PrNk7bPv9!#AZV7C7M-dyIRWHoHkrN; z0?v$Q`2Md=eg05-Mann&@6zl%38A^hbERXVuRCSV)46q+f{DJwa`%WfX(mffe392b zM*G;%*GHw?8%{3mZFvNK2SXf7DpHudP(SnKHVK3{_stUBx^%)v41$+Mm(7ZmNyK#I zxRt7y@X7`C4qE8upyZ-ftHde8_3zj{76>rH=K=UPO=k2kn_Ydno>g?IQcC-H zM)fT0uHrqy-M9-MrwhMng$5>f(DDUm!4u5ay450oOl(F6;!X#yQq9KZIA0$3jK|?WB&6Z{Z5=MlkYee zIFP^!rQDoR%4dbexG1-Vr{XKsr!uH`JbK}F{u)@h&}*gke!mF`{OgFL1RCi3z|i;1 zBNMELxBW^JIn-3btNJa<^PP4TOe8)#&nyakWP5I~7xz6KVX0|z_h0`L+`uCvE!hQK zx_~C62QF(N00BBlDNftE;LSxs^#OAc2vBTZFB-;fp3O zdz!)as=GJayCT!l0y&*>;@70Bt>cMbbQ1^t#N{M?MuieMwb z9b+eJm?bo2Bto?A8X`4VqQz z=Os}V`;(eWGuD*#-{IcIv$G4)rAkB_W6Yj8y-W0vo?=3bgq6YaiOp<~CH|gkapzro zw!hE?(r!?eg4VbW+N*F!O5JI&FmoNzQ5h<9bATUh$6}>Y874{!oA+pE91myAgmR+E zyi4P(8F1k>0UoZbHLaJ+g0^&YYBN?vFtFWAONegk^kwsp23Ajh}C5~z$4-Pz}R6oCIYVRKNv0w+{i$n4)>AHzscl3K5r>>7N zRa2*kv`yEZD!j}niC4^%-7>dF48S96V&;vj9fF88uaIqy`jLj>X$=QlcWsg@QR&53 z{bo#|bf%~`9Q_8y;aRbHe^Kd4_u3jKFKl~tz1mdJF@11x_JyQ6bgWC2yCX|r?ou0* zG<$sy=0c4=7}jjx30)(-wt0Y=eA3W;DkW*IwoOZTP+{VdyfM4kNdKS_^_iZLHL)4d zpkucs2(&h-0Rf~+TrzigdQbAG*pwWM@TY6Tx!70)0k#nY26rtUCsm?jNLTtZE24Ei zT>fl*NpKb(NZJRIpDW;D1b<}DmkshJk+a0H!i03h5`K(fi09OyaxT^z&*#Eycx6!h zEg|R+jdy!!(pq#HB+nha_JP0=)Ntw^*t;98n1(|Ju#^DOYO9J)XaG5e|FAmvSSfwy z>L8~W;<*Xq7Yh~Qs<7C5n(@O)w`%x2So5n@$sr}dRDzecrr;h`-u8yLhh>7sOVfH7N@T}bJYy{gbX78aaV_8}FJkMq%kaH} z!|CKUK`!Gg7x_(cL5v{%;Rrue__;&0@x|b-h+%#|tosa9+VDZNJMFMyn*b1JY?uDq zV^|X`{uE0rtnakTMksd5EkVxuu- z7!VlL@TOZ235==3`|>Orq|08uc8nOl+Zo1bdm_CnixTLT#2q<6iWh)yU7_F8PfPmJ zs~N)cGB*bZ`&w=AWVD1IiWRU1+DOa_7|F~nm1wI(9xXLVapjGD*VdG_hi6#(FbaF& zSN_>vO|P{Ljl2M^tJmKKSWpH~jNU&ujdA+nBIJ@m7(u)qT5p?o$$?bKmV}Md9YmpZ z`oS=iaO>4jN(a*Aie?&+_9n5rTV8WQ!=!mZOG{HN#T#9rMY#nij#y=A)|T$_2T(P+Y;@qlr5?P{qM+L+~PA&kZV9P zqCHdrw9;pqGu}~7PjX-V9yYT7$sln{%UO1{_GOCOq#*0618+MNZ|mZh{97MuCohGj z`6E(6<+_aSt%z@>o3`7z1?_e)mhaF!_x9@9Tkd-LEL!LPP9ViWMSN7HZ$b3l&ZrSa zSfpk3^ZV<;Uk72Mo_Cg7^hzrpPVMpzrqqK!!0ZZ(cs8DReb&vq@wep`zhLc-833Ks zKv%MWOiqD+eSM-9)~)}|vstbdR2}o8wHa$~#nN;rV@_^4@xTPmmg1o`ueoP%u^FZ* zc86a5m%S*;k%RRe$+(&7S^CcWYYM+3y^^Zp<$RW}itYpn2{@r&etmu+^Dt!o<-86| zW50tJN$cRhD@i4y$#%_)u9%~G+&ni_r4Rdtx!q|W?A!=^EIv(mtsUQ8sNSY6?*D`1~G1sTxueFKD7Qt)XYzfhYoVsjaLX2sD zzk8g7w6ycqs#Aa_;IA~G5E8@OT_MVEo{+WYoZ%u*2JLgUbhh$~y=pQZ{P+@N(f@>? z=8t0>2+1##Rhngm3-zKz%s4E=Xq6m0OQs|0I_5$ZzxCP!!XGLB(h0j~h5P$jY%tjJ zSiN0k#$%zNf&qUEWJi{@I}Hj=MxcsS%v+QJY5c^R;wzI}^fZX`>vYq6u^i_K=u2{= zPPj++Tmvk&eWu!(T6f!v|H=Hk-qZBx2Bw)RHAX!Q^W~;DGw-+KT4*VSB0HtVoU2Ig z^{BU|c+@wz>>Hjoos>~$vF-o#7|eqDv2 zbjl)R^~e8Rr-@jN`SgeB$z{tvU5hw$;KW*bKS-c(L(dc%F@3lD4_qbqN2Pr>QXeVY zv-g9Cr z$?3o)CBe#){I6=%stde&un%lE&|w|`en|h~0xSE>1Cv_~$5OA_JqYIftnBeyn5%<0 zQ}N&MIDWIP90yXSHIdx1edq)1!V61S^AnEBU8Z%wXET#?gfSB@H@qivNk42nFocZ| zKYrWD=|RxA^I&0SMMWYs2z6h(HqC>(oTnZzt9keh!r~-qYg6o8X&5~<(FIdAW{-gD zU*!V@^(2IJRRl8W37b0cnJ6f(1eTnNNXR;c*XfS{d5g&o{651Zr<;`0e}H>tx@l;t z9}zv1`~1`U{q#$VG*9Ao$^Pe2*juOW=7Uc_s=n=5CGEL6j{+oKmgtRN7dz!9Kk>tz z6v5p!4rDM8iw)UHZXKY8rV^ST0G_Ydy7+4(AMy*Jxb^f&dO22#0PR8LZ=tlnhkj%{ zt|A}i8-EQ|El!L474;FZ^jD(Zh<-VFvo$+c<$gP3oBl;cusN46T0R7Rt`2NZzjG5N z%#VrCf1&gam~_R}Y%|)Xn)(eS-RpN|@?g|5vjV#|YI!ucIz4U25 z;sfVCaZe>Wx@*D%%@so1Eaezx*s(`btk6n;Cx>FkS~2011LxVv!%_x6jdq`)NI}(% z==9r%O8arD1VEP`wKG)&J6}JxDO+c;P{VfePSbDCX&&`j)C3P+m z(atD}p4fM+-{O@{vJ?A^LruN8fDRf9r_kQRAVyLWJ86Wfr}b1k(PvL)Yl(tB**y&>dK(x^@>exGxjbFBa2Eb9{Z>%Hjv zs8FM$SOd(NOhp4xb%RG`j+>417m_9wHgIQCLd~~cd}6}gYYtLd$6B}M#wmZ)8^aDM zpHGd?9%sV4uEo|Ja8>Y?bBOK^T_f+Xk_iHCs|!QpYR?iz7_$aholZ9TvM$<1q&w5% zh8A%~JcR>bRz-+Ws@e}?1U=YOqHPl;b}{2xq|344|EJY{Og+(Yp(&N{N0 z)#d3i@_Jv+BHKILtQ!@7vU)(nEaugV0hr=OstPFifiu+X9@NSCk)WhzXX>q=-rgI5 zB29YG=W#Pz7L^E#dKhjEbnfx(Se*#NWt*l72Fk~m%M_+~cD^J^y*x~l_`K`q?U&|Z z<%(seEq<#PnUErdR;$OF0?#o{uTchXtY>NpSpk=@YX1|EucxYA8*A^uNbBXi%&s@V zyc8X90>4WL#PXTmps-=&`+M<&a9T`WdwjDbL&aYs4}M%{Vsd#p-tVcr*VE;bflcfU z(G*zW>q#>eS~Ll@>8FK{1L8Jj2O!-9vVAOM>|xynR=T~pkAJ))cmR|dvTKX#8aS^ zC~r=`=Ql)fr?PzOW5{(C8QnT7!KwN8Qez>iFg}^DKPsgheI+<)e@9Q#LSLI-Lj_a)<&9I)#3fA!QA$2WbT(q%I zoWL0)KaIRAIp5pR+f2^1hMW&7MP0wjprE8YbkbeCJ8rO4&`s#o%~w^LG+v%guK86k zk79^+iN?2+BeB=%@@`{Z<;h*dZopo~_U~cJ{9uPD&FvJM%z6n6xE;9v8>7!QmcC8P zucO;SM@7Mc;)CeqP7Dfr>#R(99g)dDLwGK#!*jk099AxJpeYWhb z!|F2leT2;0#9+}lR8OtFb3ii$l7&WhdJ4?aVn28Xn;Y&j%0*tr#)WFtkN>zd zkW$;wYvrCvJXtmAQKHu!q)|)MO3KMlDv7o4SzRO6N@DWg+c#3;G|0<@}eIPO5zT>%8 zddC!X9ok`UwC<#Ovu?3_urbHWBAf4ug4gn9bw>mZR!Gmv+^AT{FOVyI98J_&QPXC) z^s_|QYY%8v4QfLPZJd26axwx0h+BU-h3}20YZ{%_;B~MBdRPUPkCDw5j38_K6|=;G zEBI1idht#52^W+7R!Q+qjxzXY1rew|y?6oT$D5Ds7iwy{FJ6*g<6zR>ZOUHG$9Za# zLqh#+)-;+IT?53g_!pSAQwCcytX5ad(^K=O`}8pPAtQC59CngB19>HxH{R$1f1*AW z1H!i}jkR4_d%=#C$*pt&+ff43?boJq>a-VLRdddF*&uzc1$R_RGbDiP7DCFB^v_Q4 zH4nAlk*iBHym^_T%frn=0eMC$4%6;uP7W?_oRu0XyO8LB4GdvM=~dj&;=Z-tW=bx2 zuqQgK_=6!?#2!N`(W%yC1G~7s_y})#CioLTZk~~29mJ2O0o9KKK(R7Nb5r~kr!J=G z;w%=H%s72e z)-EuZ>7djf_^}$?)zwKJ=@?6?h-jtj{015XEO$vf(t98=>#pu+`VEuHhQ~L)Du@VU z75__%->zO_BMLZ*PU$YLMcYGG1bc2K8tz|Zb7W_Y@e3K_?_a>C%nRsA5=7>k;*FO#b8Bx5bQ z7o{X8=vdp!7!8$EH%yfmT{Ag~fBal2hqY%4s1b%!oVsg|e>-mrLi+4D7n=i~PhyWH z1$Qk5kR{H9ZJqeDu!?BEDM!c$g8^k3Id=j;fd&^~p0YCR$g_P@eW5+ziTT_B@@!fv zgpE7e8y!X5KM_%8@_F#Fb{M5|aWgp!Ih=ZR%k^@4G3nZvA$2j?l7=w2J8zdTP-vzj z6}zxQ1=`*o?b`ZGr#?=uj}2!jptb78JwRDf%MzO)cBYzY+@k~GOy7nN`g=f@~Z!rV|)11p}H9_{IUaD=%^!8Y4hZhfwJoB@gVlb z0ZL({#fcst5HMcQ^i1~TyKK{5ZE>#?Y4+jFfi_2A(w5-6X-6bqgkS;LWM3p`52d)! zsmXgzUm}M@MF~Hb%$YV<-P*9Gv0nEbpfC{7AyopkX z8BJycff8KKx++pVmAx{u+ZP)ZV=)5FWm9OfcTdq3UBOIfXCG@Aca)*;s$q2kTl>f{ z`csoi_ko>clnoVn=R2wea;8Yut^Tp)l|N>y_6Em_T;ylg`>;6Yis&*mn1WKqdc=_G zv`ng$(tidQHJ9RSFaFUOnp%j34G}7Cid1U!TYM^bL+{l z&y0X}tCf4;F4_`m4f%NZp37EOmmSWZ0ezgz-r5dto}Zerh|3fs$zXH3Z8qkmx=wr$ z0Xmp;BFM#cW|`cV9F54jKe0N{vRhv_9$t=0WM;dLzT%Xa+(%b*-VIeYiEDvYP)-F* z=?;`GB)39AYNWTz{<`c;;1$F>lsZGNCp<-~ zEx~J(Q5wX@fQTnV#6&Ioi{j0Q27j@F_V~ZD$&n=3h5yYJ-#BSv`2qGRB z(;>P_TFd{k(Z4L683;5|HFK+5J6|h&(kUNJiHHOQAPhGtt6exW9P?Vt{7$-IIrc9U ziXpKZ8hucUhsnoD|&&~v42(SoaGj#p1Xn#)4i97~+Jl-%wAF5_>m@SnHd zQUd>_TmJKzf0pGRr2K<|e^Brb3jRUCKPdPI1^=Mn9~At9f`3r(4+{Q4!GEA2r1srK aV$2ip-0sZfdzJxzh{N88D!&Z*{{I2&Ho|WJ literal 0 HcmV?d00001 diff --git a/doc/design/fluid.md b/doc/design/fluid.md new file mode 100644 index 0000000000..b3d039d6db --- /dev/null +++ b/doc/design/fluid.md @@ -0,0 +1,122 @@ +# Design Doc: PaddlePaddle Fluid + +## Why Fluid + +When Baidu developed PaddlePaddle in 2013, the only well-known open source deep learning system was Caffe. However, when it open-sourced PaddlePaddle in 2016, there had been many other choices over there. We were facing a challenge -- why would we open source yet another one? + +Fluid is the answer. Fluid is similar to PyTorch and TensorFlow Eager Execution, which describes the "process" of training or inference a model, but not the model itself. Indeed, in PyTorch, Eager Execution, and Fluid, there is no such a concept of the model at all. I will explain in this article, Fluid is currently more extreme in this idea than PyTorch and Eager Execution, and we are pushing Fluid towards a compiler and even a new programming language for deep learning + +## The Evolution of Deep Learning Systems + +Deep learning infrastructure is one of the fastest involving technology. Within only four years, there have been three generations of technologies invented. + +| Since around | model = sequence of layers | model = graph of operators | No model | +|--|--|--|--| +| 2013 | Caffe, Theano, Torch, PaddlePaddle | | | +| 2015 | | TensorFlow, MxNet, Caffe2, ONNX, n-graph | | +| 2016 | | | PyTorch, TensorFlow Eager Execution, PaddlePaddle Fluid | + +From the above table, we see that the technology is evolving towards the removal of the concept of the model. To better understand the reason, let us compare the *programming paradigms*, or, the ways we program deep learning applications using these systems. + +## Deep Learning Programming Paradigms + +With any system listed as the first or second generation, e.g., Caffe or TensorFlow, an AI application training program looks like the following: + +```python +x = layer.data("image") +l = layer.data("label") +f = layer.fc(x, W) +s = layer.softmax(f) +c = layer.mse(l, s) + +for i in xrange(1000): # train for 1000 iterations + m = read_minibatch() + forward({input=x, data=m}, minimize=c) + backward(...) + +print W # print the trained model parameters. +``` + +The above program includes two parts: + +1. the first part describes the model, and +2. the second part describes the training process (or inference process). + +This paradigm has a well-known problem that limits programmers' productivity. Suppose that we made some mistakes at configuring the model in the first part of the program, when we run the program, it wouldn't prompt error messages until the execution enters the second part, when the invocation to `forward` or `backward` raise errors. It is difficult for the programmer to realize and locate that there is a mistake many lines away from where the error appears. + +This problem of hard to debug a program is the primary reason that programmers prefer PyTorch than elder systems. Using PyTorch, we would write the above program like the following + +```python +W = tensor(...) + +for i in xrange(1000): # train for 1000 iterations + m = read_minibatch() + x = m["image"] + l = m["label"] + f = layer.fc(x, W) + s = layer.softmax(f) + c = layer.mse(l, s) + backward() + +print W # print the trained model parameters. +``` + +We can see that the main difference is the moving of the model configuration, the first part, into the train loop. This change would allow that mistakes in model configuration reported where they appear. This change also represents the model, or its forward pass, by the process in the training loop. + +## Describe Arbitrary Models for the Future + +Describing the process instead of the model also brings Fluid the flexibility to define models not yet invented. + +As we can program the process, we can write an RNN as a loop, instead of an RNN layer or operator. A PyTorch example could look like + +```python +for i in xrange(1000): + m = read_minibatch() + x = m["sentence"] + for t in xrange x.len(): + h[t] = the_step(x[t]) +``` + +With Fluid, the training loop and the RNN in the above program are not Python loop, but a "loop structure" provided by Fluid and implemented in C++: + +```python +train_loop = layers.While(cond) +with train_loop.block(): + m = read_minibatch() + x = m["sentence"] + rnn = layers.While(...) + with rnn.block(): + h[t] = the_step(input[t]) +``` + +A real Fluid example is [here](https://github.com/PaddlePaddle/Paddle/blob/a91efdde6910ce92a78e3aa7157412c4c88d9ee8/python/paddle/v2/fluid/tests/test_while_op.py#L36-L44). + +From these examples, you can see that Fluid programs look similar to their PyTorch equivalent, except that Fluid's loop structure, wrapped with Python's `with` statement, could run much faster than Python's loop. + +We have more examples of the [`if-then-else`](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/if_else_op.md) structure of Fluid. + +## Turing Completeness + +In computability theory, a system of data-manipulation rules, such as a programming language, is said to be Turing complete if it can be used to simulate any Turing machine. For a programming language, if it provides if-then-else and loop, it is Turing complete. From above examples, Fluid seems Turing complete; however, I would like to point out is a slight difference between the if-then-else of Fluid and that in a programming language is that the former runs both of its branches. It splits the input minibatch into two -- one for the true condition and one for the false. I am not sure if this is equivalent to the if-then-else that makes programming languages Turing-complete. I talked with [Yuang Yu](https://research.google.com/pubs/104812.html), but I need to figure out more. + +## The Execution of a Fluid Program + +There are two ways to run a Fluid program. When we run an example program, it creates a protobuf message [`ProgramDesc`](https://github.com/PaddlePaddle/Paddle/blob/a91efdde6910ce92a78e3aa7157412c4c88d9ee8/paddle/framework/framework.proto#L145) that describes the process and conceptually likes an [abstract syntax tree](https://en.wikipedia.org/wiki/Abstract_syntax_tree). + +We have a C++ class [`Executor`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/executor.h), which runs a `ProgramDesc` like that an interpreter runs a Python program. + +We are moving towards a compiler, which we will explain in more details later in this article. + +## Backward Compatibility + +Given all advantages from the removal of the concept *model*, hardware manufacturers might still prefer the existence of the concept model, so they could build their hardware reads and runs a trained model for inference. For example, Nervana, a startup company acquired by Intel, has been working on an XPU that reads models in the format known as [n-graph](https://github.com/NervanaSystems/ngraph). Similarly, [Movidius](https://www.movidius.com/) is producing a mobile deep learning chip that reads and runs graphs of operators too. The well-known [ONNX](https://github.com/onnx/onnx) is also a file format of graphs of operators. + +For Fluid, we can write a converter that extracts parts in the `ProgramDesc` protobuf message, converts them into a graph of operators, and exports into the ONNX or n-graph format. + +## Towards a Deep Learning Language and the Compiler + +We can change the if-then-else and loop structure a little bit in the above Fluid example programs so to make it a new programming language, different from Python. + +Even if we don't invent a new language, as long as we get the `ProgramDesc` message filled in, we can write a transpiler, which translates each invocation to an operator into a C++ call to a kernel function of that operator. For example, a transpiler that weaves the CUDA kernels outputs an NVIDIA-friendly C++ program, which can be built using `nvcc`. Another transpiler could generate MKL-friendly code that should be built using `icc` from Intel. More interestingly, we can translate a Fluid program into its distributed version of two `ProgramDesc` messages, one for running on the trainer process, and the other one for the parameter server. For more details of the last example, let us check the [concurrent programming design](concurrent_programming.md). The following figure explains this two-stage process: + +![](fluid-compiler.png) From 4d93f0fc1a5708c9d10cb17d161a7cc7f0391297 Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Thu, 14 Dec 2017 07:01:05 +0530 Subject: [PATCH 52/94] Polish the support new device doc (#6594) --- doc/design/support_new_device.md | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/doc/design/support_new_device.md b/doc/design/support_new_device.md index 92443e4392..fd23dc211a 100644 --- a/doc/design/support_new_device.md +++ b/doc/design/support_new_device.md @@ -1,33 +1,33 @@ -# Design Doc: Support new Device/Library +# Design Doc: Supporting new Device/Library ## Background -Deep learning has a high demand for computing resources. New high-performance device and computing library are coming constantly. The deep learning framework has to integrate these high-performance device and computing library flexibly. +Deep learning has a high demand for computing resources. New high-performance devices and computing libraries are appearing very frequently. Deep learning frameworks have to integrate these high-performance devices and computing libraries flexibly and efficiently. -On the one hand, hardware and computing library are not usually one-to-one coresponding relations. For example, in Intel CPU, there are Eigen and MKL computing library. And in Nvidia GPU, there are Eigen and cuDNN computing library. We have to implement specific kernels for an operator for each computing library. +On one hand, hardware and computing libraries usually do not have a one-to-one correspondence. For example,Intel CPUs support Eigen and MKL computing libraries while Nvidia GPUs support Eigen and cuDNN computing libraries. We have to implement operator specific kernels for each computing library. -On the other hand, users usually do not want to care about the low-level hardware and computing library when writing a neural network configuration. In Fluid, `Layer` is exposed in `Python`, and `Operator` is exposed in `C++`. Both `Layer` and `Operator` are independent on hardwares. +On the other hand, users usually do not want to care about the low-level hardware and computing libraries when writing a neural network configuration. In Fluid, `Layer` is exposed in `Python`, and `Operator` is exposed in `C++`. Both `Layer` and `Operator` are hardware independent. So, how to support a new Device/Library in Fluid becomes a challenge. ## Basic: Integrate A New Device/Library -For a general overview of fluid, please refer to [overview doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/read_source.md). +For a general overview of fluid, please refer to the [overview doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/read_source.md). -There are mainly there parts we have to consider in integrating a new device/library: +There are mainly three parts that we have to consider while integrating a new device/library: - Place and DeviceContext: indicates the device id and manages hardware resources - Memory and Tensor: malloc/free data on certain device -- Math Functor and OpKernel: implement computing unit on certain device/library +- Math Functor and OpKernel: implement computing unit on certain devices/libraries ### Place and DeviceContext #### Place -Fluid use class [Place](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/place.h#L55) to represent specific device and computing library. There are inheritance relationships between different kinds of `Place`. +Fluid uses class [Place](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/place.h#L55) to represent different devices and computing libraries. There are inheritance relationships between different kinds of `Place`. ``` | CPUPlace --> MKLDNNPlace @@ -43,7 +43,7 @@ typedef boost::variant Place; #### DeviceContext -Fluid use class [DeviceContext](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/device_context.h#L30) to manage the resources in certain hardware, such as CUDA stream in `CDUADeviceContext`. There are also inheritance relationships between different kinds of `DeviceContext`. +Fluid uses class [DeviceContext](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/device_context.h#L30) to manage the resources in different hardwares, such as CUDA stream in `CDUADeviceContext`. There are also inheritance relationships between different kinds of `DeviceContext`. ``` @@ -52,7 +52,7 @@ DeviceContext ----> CUDADeviceContext --> CUDNNDeviceContext \-> FPGADeviceContext ``` -A example of Nvidia GPU is as follows: +An example of Nvidia GPU is as follows: - DeviceContext @@ -93,7 +93,7 @@ class CUDNNDeviceContext : public CUDADeviceContext { #### memory module -Fluid provide following [memory interfaces](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/memory/memory.h#L36): +Fluid provides the following [memory interfaces](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/memory/memory.h#L36): ``` template @@ -106,12 +106,12 @@ template size_t Used(Place place); ``` -To implementing these interfaces, we have to implement MemoryAllocator for specific Device +To implementing these interfaces, we have to implement MemoryAllocator for different Devices #### Tensor -[Tensor](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/tensor.h#L36) holds data with some shape in certain Place. +[Tensor](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/tensor.h#L36) holds data with some shape in a specific Place. ```cpp class Tensor { @@ -168,7 +168,7 @@ t.mutable_data(place); ### Math Functor and OpKernel -Fluid implements computing unit based on different DeviceContext. Some computing unit is shared between operators. These common part will be put in operators/math directory as basic Functors. +Fluid implements computing units based on different DeviceContexts. Some computing units are shared between operators. This common part will be put in operators/math directory as basic Functors. Let's take [MaxOutFunctor](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/math/maxouting.h#L27) as an example: @@ -183,7 +183,7 @@ class MaxOutFunctor { }; ``` -CPU implement in .cc file +CPU implemention is in .cc file ``` template @@ -197,7 +197,7 @@ class MaxOutFunctor { }; ``` -CUDA implement in .cu file +CUDA implemention is in .cu file ``` template @@ -212,11 +212,11 @@ class MaxOutFunctor { ``` -We get computing handle from concrete DeviceContext, and make compution on tensors. +We get computing handle from a concrete DeviceContext, and make compution on tensors. -The implement of `OpKernel` is similar to math functors, the extra thing we need to do is registering the OpKernel to global map. +The implemention of `OpKernel` is similar to math functors, the extra thing we need to do is to register the OpKernel in a global map. -Fluid provides different register interface in op_registry.h +Fluid provides different register interfaces in op_registry.h Let's take [Crop](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/crop_op.cc#L134) operator as an example: @@ -240,7 +240,7 @@ REGISTER_OP_CUDA_KERNEL( ## Advanced topics: How to switch between different Device/Library -Generally, we will impelement OpKernel for all Device/Library of an Operator. We can easily train a Convolutional Neural Network in GPU. However, some OpKernel is not sutibale in a specific Device. For example, crf operator can be only run at CPU, whereas most other operators can be run at GPU. To achieve high performance in such circumstance, we have to switch between different Device/Library. +Generally, we will impelement OpKernel for all Device/Library of an Operator. We can easily train a Convolutional Neural Network in GPU. However, some OpKernel is not sutibale on a specific Device. For example, crf operator can only run on CPU, whereas most other operators can run at GPU. To achieve high performance in such circumstance, we have to switch between different Device/Library. We will discuss how to implement an efficient OpKernel switch policy. From c5afc3fe7cf1f1f93399cb9d6c92ed6f86a5fc74 Mon Sep 17 00:00:00 2001 From: kavyasrinet Date: Wed, 13 Dec 2017 18:40:30 -0800 Subject: [PATCH 53/94] Updating the design doc of Fluid (#6597) --- doc/design/fluid.md | 54 ++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/doc/design/fluid.md b/doc/design/fluid.md index b3d039d6db..585dc8ef39 100644 --- a/doc/design/fluid.md +++ b/doc/design/fluid.md @@ -2,25 +2,25 @@ ## Why Fluid -When Baidu developed PaddlePaddle in 2013, the only well-known open source deep learning system was Caffe. However, when it open-sourced PaddlePaddle in 2016, there had been many other choices over there. We were facing a challenge -- why would we open source yet another one? +When Baidu developed PaddlePaddle in 2013, the only well-known open source deep learning system at the time was Caffe. However, when PaddlePaddle was open-sourced in 2016, many other choices were available. There was a challenge -- what is the need for open sourcing yet another deep learning framework? -Fluid is the answer. Fluid is similar to PyTorch and TensorFlow Eager Execution, which describes the "process" of training or inference a model, but not the model itself. Indeed, in PyTorch, Eager Execution, and Fluid, there is no such a concept of the model at all. I will explain in this article, Fluid is currently more extreme in this idea than PyTorch and Eager Execution, and we are pushing Fluid towards a compiler and even a new programming language for deep learning +Fluid is the answer. Fluid is similar to PyTorch and TensorFlow Eager Execution, which describes the "process" of training or inference using the concept of a model. In fact in PyTorch, TensorFlow Eager Execution and Fluid, there is no concept of a model at all. The details are covered in the sections below. Fluid is currently more extreme in the above mentioned idea than PyTorch and Eager Execution, and we are trying to push Fluid towards the directions of a compiler and a new programming language for deep learning. ## The Evolution of Deep Learning Systems -Deep learning infrastructure is one of the fastest involving technology. Within only four years, there have been three generations of technologies invented. +Deep learning infrastructure is one of the fastest evolving technologies. Within four years, there have already been three generations of technologies invented. -| Since around | model = sequence of layers | model = graph of operators | No model | +| Existed since | model as sequence of layers | model as graph of operators | No model | |--|--|--|--| | 2013 | Caffe, Theano, Torch, PaddlePaddle | | | | 2015 | | TensorFlow, MxNet, Caffe2, ONNX, n-graph | | | 2016 | | | PyTorch, TensorFlow Eager Execution, PaddlePaddle Fluid | -From the above table, we see that the technology is evolving towards the removal of the concept of the model. To better understand the reason, let us compare the *programming paradigms*, or, the ways we program deep learning applications using these systems. +From the above table, we see that the deep learning technology is evolving towards getting rid of the concept of a model. To understand the reasons behind this direction, a comparison of the *programming paradigms* or the ways to program deep learning applications using these systems, would be helpful. The following section goes over these. ## Deep Learning Programming Paradigms -With any system listed as the first or second generation, e.g., Caffe or TensorFlow, an AI application training program looks like the following: +With the systems listed as the first or second generation, e.g., Caffe or TensorFlow, an AI application training program looks like the following: ```python x = layer.data("image") @@ -33,18 +33,18 @@ for i in xrange(1000): # train for 1000 iterations m = read_minibatch() forward({input=x, data=m}, minimize=c) backward(...) - + print W # print the trained model parameters. ``` The above program includes two parts: -1. the first part describes the model, and -2. the second part describes the training process (or inference process). +1. The first part describes the model, and +2. The second part describes the training process (or inference process) for the model. -This paradigm has a well-known problem that limits programmers' productivity. Suppose that we made some mistakes at configuring the model in the first part of the program, when we run the program, it wouldn't prompt error messages until the execution enters the second part, when the invocation to `forward` or `backward` raise errors. It is difficult for the programmer to realize and locate that there is a mistake many lines away from where the error appears. +This paradigm has a well-known problem that limits the productivity of programmers. If the programmer made a mistake in configuring the model, the error messages wouldn't show up until the second part is executed and `forward` and `backward` propagations are performed. This makes it difficult for the programmer to debug and locate a mistake that is located blocks away from the actual error prompt. -This problem of hard to debug a program is the primary reason that programmers prefer PyTorch than elder systems. Using PyTorch, we would write the above program like the following +This problem of being hard to debug and re-iterate fast on a program is the primary reason that programmers, in general, prefer PyTorch over the older systems. Using PyTorch, we would write the above program as following: ```python W = tensor(...) @@ -57,17 +57,17 @@ for i in xrange(1000): # train for 1000 iterations s = layer.softmax(f) c = layer.mse(l, s) backward() - + print W # print the trained model parameters. ``` -We can see that the main difference is the moving of the model configuration, the first part, into the train loop. This change would allow that mistakes in model configuration reported where they appear. This change also represents the model, or its forward pass, by the process in the training loop. +We can see that the main difference is the moving the model configuration part (the first step) into the training loop. This change would allow the mistakes in model configuration to be reported where they actually appear in the programming block. This change also represents the model better, or its forward pass, by keeping the configuration process in the training loop. ## Describe Arbitrary Models for the Future -Describing the process instead of the model also brings Fluid the flexibility to define models not yet invented. +Describing the process instead of the model also brings Fluid, the flexibility to define different non-standard models that haven't been invented yet. -As we can program the process, we can write an RNN as a loop, instead of an RNN layer or operator. A PyTorch example could look like +As we write out the program for the process, we can write an RNN as a loop, instead of an RNN as a layer or as an operator. A PyTorch example would look like the following: ```python for i in xrange(1000): @@ -77,7 +77,7 @@ for i in xrange(1000): h[t] = the_step(x[t]) ``` -With Fluid, the training loop and the RNN in the above program are not Python loop, but a "loop structure" provided by Fluid and implemented in C++: +With Fluid, the training loop and the RNN in the above program are not really Python loops, but just a "loop structure" provided by Fluid and implemented in C++ as the following: ```python train_loop = layers.While(cond) @@ -89,34 +89,34 @@ with train_loop.block(): h[t] = the_step(input[t]) ``` -A real Fluid example is [here](https://github.com/PaddlePaddle/Paddle/blob/a91efdde6910ce92a78e3aa7157412c4c88d9ee8/python/paddle/v2/fluid/tests/test_while_op.py#L36-L44). +An actual Fluid example is described [here](https://github.com/PaddlePaddle/Paddle/blob/a91efdde6910ce92a78e3aa7157412c4c88d9ee8/python/paddle/v2/fluid/tests/test_while_op.py#L36-L44). -From these examples, you can see that Fluid programs look similar to their PyTorch equivalent, except that Fluid's loop structure, wrapped with Python's `with` statement, could run much faster than Python's loop. +From the example, the Fluid programs look very similar to their PyTorch equivalent programs, except that Fluid's loop structure, wrapped with Python's `with` statement, could run much faster than just a Python loop. We have more examples of the [`if-then-else`](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/if_else_op.md) structure of Fluid. ## Turing Completeness -In computability theory, a system of data-manipulation rules, such as a programming language, is said to be Turing complete if it can be used to simulate any Turing machine. For a programming language, if it provides if-then-else and loop, it is Turing complete. From above examples, Fluid seems Turing complete; however, I would like to point out is a slight difference between the if-then-else of Fluid and that in a programming language is that the former runs both of its branches. It splits the input minibatch into two -- one for the true condition and one for the false. I am not sure if this is equivalent to the if-then-else that makes programming languages Turing-complete. I talked with [Yuang Yu](https://research.google.com/pubs/104812.html), but I need to figure out more. +In computability theory, a system of data-manipulation rules, such as a programming language, is said to be Turing complete if it can be used to simulate any Turing machine. For a programming language, if it provides if-then-else and loop, it is Turing complete. From the above examples, Fluid seems to be Turing complete; however, it is noteworthy to notice that there is a slight difference between the `if-then-else` of Fluid and that of a programming language. The difference being that the former runs both of its branches and splits the input mini-batch into two -- one for the True condition and another for the False condition. This hasn't been researched in depth if this is equivalent to the `if-then-else` in programming languages that makes them Turing-complete. Based on a conversation with [Yuang Yu](https://research.google.com/pubs/104812.html), it seems to be the case but this needs to be looked into in-depth. ## The Execution of a Fluid Program -There are two ways to run a Fluid program. When we run an example program, it creates a protobuf message [`ProgramDesc`](https://github.com/PaddlePaddle/Paddle/blob/a91efdde6910ce92a78e3aa7157412c4c88d9ee8/paddle/framework/framework.proto#L145) that describes the process and conceptually likes an [abstract syntax tree](https://en.wikipedia.org/wiki/Abstract_syntax_tree). +There are two ways to execute a Fluid program. When a program is executed, it creates a protobuf message [`ProgramDesc`](https://github.com/PaddlePaddle/Paddle/blob/a91efdde6910ce92a78e3aa7157412c4c88d9ee8/paddle/framework/framework.proto#L145) that describes the process and is conceptually like an [abstract syntax tree](https://en.wikipedia.org/wiki/Abstract_syntax_tree). -We have a C++ class [`Executor`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/executor.h), which runs a `ProgramDesc` like that an interpreter runs a Python program. +There is a C++ class [`Executor`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/executor.h), which runs a `ProgramDesc`, similar to how an interpreter runs a Python program. -We are moving towards a compiler, which we will explain in more details later in this article. +Fluid is moving towards the direction of a compiler, which is explain in more detail later in this article. -## Backward Compatibility +## Backward Compatibility of Fluid -Given all advantages from the removal of the concept *model*, hardware manufacturers might still prefer the existence of the concept model, so they could build their hardware reads and runs a trained model for inference. For example, Nervana, a startup company acquired by Intel, has been working on an XPU that reads models in the format known as [n-graph](https://github.com/NervanaSystems/ngraph). Similarly, [Movidius](https://www.movidius.com/) is producing a mobile deep learning chip that reads and runs graphs of operators too. The well-known [ONNX](https://github.com/onnx/onnx) is also a file format of graphs of operators. +Given all the advantages from the removal of the concept of a *model*, hardware manufacturers might still prefer the existence of the concept of a model, so it would be easier for them to support multiple frameworks all at once and could run a trained model during inference. For example, Nervana, a startup company acquired by Intel, has been working on an XPU that reads the models in the format known as [n-graph](https://github.com/NervanaSystems/ngraph). Similarly, [Movidius](https://www.movidius.com/) is producing a mobile deep learning chip that reads and runs graphs of operators. The well-known [ONNX](https://github.com/onnx/onnx) is also a file format of graphs of operators. -For Fluid, we can write a converter that extracts parts in the `ProgramDesc` protobuf message, converts them into a graph of operators, and exports into the ONNX or n-graph format. +For Fluid, we can write a converter that extracts the parts in the `ProgramDesc` protobuf message, converts them into a graph of operators, and exports the graph into the ONNX or n-graph format. ## Towards a Deep Learning Language and the Compiler -We can change the if-then-else and loop structure a little bit in the above Fluid example programs so to make it a new programming language, different from Python. +We can change the `if-then-else` and loop structure a little bit in the above Fluid example programs, to make it into a new programming language, different than Python. -Even if we don't invent a new language, as long as we get the `ProgramDesc` message filled in, we can write a transpiler, which translates each invocation to an operator into a C++ call to a kernel function of that operator. For example, a transpiler that weaves the CUDA kernels outputs an NVIDIA-friendly C++ program, which can be built using `nvcc`. Another transpiler could generate MKL-friendly code that should be built using `icc` from Intel. More interestingly, we can translate a Fluid program into its distributed version of two `ProgramDesc` messages, one for running on the trainer process, and the other one for the parameter server. For more details of the last example, let us check the [concurrent programming design](concurrent_programming.md). The following figure explains this two-stage process: +Even if we do not invent a new language, as long as we get the `ProgramDesc` message filled in, we can write a transpiler, which translates each invocation to an operator, into a C++ call to a kernel function of that operator. For example, a transpiler that weaves the CUDA kernels outputs an NVIDIA-friendly C++ program, which can be built using `nvcc`. Another transpiler could generate MKL-friendly code that should be built using `icc` from Intel. More interestingly, we can translate a Fluid program into its distributed version of two `ProgramDesc` messages, one for running on the trainer process, and the other one for the parameter server. For more details of the last example, the [concurrent programming design](concurrent_programming.md) document would be a good pointer. The following figure explains the proposed two-stage process: ![](fluid-compiler.png) From 97c3de0cfbf150670317e289e7e8e58651d907cb Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Thu, 14 Dec 2017 11:21:41 +0800 Subject: [PATCH 54/94] follow comments --- paddle/operators/conv_transpose_op.h | 1 - 1 file changed, 1 deletion(-) diff --git a/paddle/operators/conv_transpose_op.h b/paddle/operators/conv_transpose_op.h index 0aae000dc4..1171b0435f 100644 --- a/paddle/operators/conv_transpose_op.h +++ b/paddle/operators/conv_transpose_op.h @@ -225,7 +225,6 @@ class GemmConvTransposeGradKernel : public framework::OpKernel { if (input_grad) { input_grad->mutable_data(context.GetPlace()); - // set_zero is unnecessary, math::matmul will reset input_grad. } if (filter_grad) { // filter size (m, c, k_h, k_w) filter_grad->mutable_data(context.GetPlace()); From 685d1e3b330ff6605935ed4a33825bdeb97ced07 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 14 Dec 2017 04:27:05 +0000 Subject: [PATCH 55/94] Enable reshape_op to support dimension inference --- paddle/operators/reshape_op.cc | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc index 39bf2118d6..306dfa8069 100644 --- a/paddle/operators/reshape_op.cc +++ b/paddle/operators/reshape_op.cc @@ -34,21 +34,27 @@ class ReshapeOp : public framework::OperatorWithKernel { auto shape = ctx->Attrs().Get>("shape"); PADDLE_ENFORCE(shape.size() > 0, "Attr(shape) shouldn't be empty."); auto x_dims = ctx->GetInputDim("X"); - // TODO(qiao) change batch_size - for (size_t i = 1; i < shape.size(); ++i) { - PADDLE_ENFORCE(shape[i] > 0, - "Each dimension of Attr(shape) " - "must be positive except the first one."); - } - if (shape[0] < 0) { - shape[0] = x_dims[0]; + + std::vector neg_dims_idx; + for (size_t i = 0; i < shape.size(); ++i) { + PADDLE_ENFORCE(shape[i] > 0 || shape[i] == -1, + "Each dimension of Attr(shape) must be positive or -1."); + if (shape[i] == -1) { + neg_dims_idx.push_back(i); + PADDLE_ENFORCE(neg_dims_idx.size() <= 1, + "Only one dimension of Attr(shape) can be -1."); + } } + // capacity check int64_t capacity = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); int64_t in_size = framework::product(x_dims); - PADDLE_ENFORCE_EQ(capacity, in_size, + if (neg_dims_idx.size() == 1) { + shape[neg_dims_idx[0]] = in_size / (-capacity); + PADDLE_ENFORCE(shape[neg_dims_idx[0]] > 0, "The size of Input(X) mismatches with Attr(shape)."); + } // resize output std::vector shape_int64(shape.size(), 0); std::transform(shape.begin(), shape.end(), shape_int64.begin(), @@ -88,6 +94,9 @@ the tensor X into a 1-D tensor: [1, 2, 3, 4] +One dimension in the target shape can be set -1, and the real dimension +will be infered from the original shape of Input(X) and other +dimensions in the target shape. )DOC"); } }; From 0a75ed6f5bcad53b3622ee89d48a5613cc4ef396 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 14 Dec 2017 05:06:19 +0000 Subject: [PATCH 56/94] Add unit test for dimension inference in reshape_op --- paddle/operators/reshape_op.cc | 14 +++++++++----- python/paddle/v2/fluid/tests/test_reshape_op.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc index 306dfa8069..164f3104eb 100644 --- a/paddle/operators/reshape_op.cc +++ b/paddle/operators/reshape_op.cc @@ -42,19 +42,23 @@ class ReshapeOp : public framework::OperatorWithKernel { if (shape[i] == -1) { neg_dims_idx.push_back(i); PADDLE_ENFORCE(neg_dims_idx.size() <= 1, - "Only one dimension of Attr(shape) can be -1."); + "Only one dimension of Attr(shape) can be unknown."); } } - // capacity check int64_t capacity = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); int64_t in_size = framework::product(x_dims); if (neg_dims_idx.size() == 1) { - shape[neg_dims_idx[0]] = in_size / (-capacity); - PADDLE_ENFORCE(shape[neg_dims_idx[0]] > 0, - "The size of Input(X) mismatches with Attr(shape)."); + // dim infer + shape[neg_dims_idx[0]] = in_size / (-capacity); + // recalculate capacity + capacity = std::accumulate(shape.begin(), shape.end(), 1, + std::multiplies()); } + // capacity check + PADDLE_ENFORCE(capacity == in_size, + "The size of Input(X) mismatches with Attr(shape)."); // resize output std::vector shape_int64(shape.size(), 0); std::transform(shape.begin(), shape.end(), shape_int64.begin(), diff --git a/python/paddle/v2/fluid/tests/test_reshape_op.py b/python/paddle/v2/fluid/tests/test_reshape_op.py index 16bb6bb2af..18ee3aece6 100644 --- a/python/paddle/v2/fluid/tests/test_reshape_op.py +++ b/python/paddle/v2/fluid/tests/test_reshape_op.py @@ -17,5 +17,19 @@ class TestReshapeOp(OpTest): self.check_grad(["X"], "Out") +class TestReshapeOpDimInfer(OpTest): + def setUp(self): + self.op_type = "reshape" + self.inputs = {'X': np.random.random((10, 20)).astype("float32")} + self.attrs = {'shape': [4, -1, 5]} + self.outputs = {'Out': self.inputs['X'].reshape(self.attrs['shape'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["X"], "Out") + + if __name__ == '__main__': unittest.main() From 9b1675048561e2708af40948b192f24a199bf70a Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Wed, 13 Dec 2017 22:44:33 -0800 Subject: [PATCH 57/94] "remove AllReduce2 comments" --- doc/design/paddle_nccl.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/doc/design/paddle_nccl.md b/doc/design/paddle_nccl.md index dba451b9b3..5e28144b95 100644 --- a/doc/design/paddle_nccl.md +++ b/doc/design/paddle_nccl.md @@ -53,8 +53,13 @@ Need to notice that AllReduce operator force GPUs synchronized at that point. Th As it shown in the picture, when each GPU compute the gradient of `W`, followed with a `AllReduce` operator, accumulate the `dW` to full batch of data, then run the optimize process individually and apply the gradient to its `W`. -- **AllReduce2** -If we use the NCCL2 AllReduce primitive, every GPU optimized full batch of data, wasted (n-1) GPU compute resources. In addition, AllReduce will only utilize the communicate resource during synchronization, then update the gradient will be a seperated phase. In fact, we can amortize the update gradient time cost into the communicating phase. -- Every parameter has its root card. That card will call **Reduce** operator and collect the gradients from GPUs. -- The whole model's parameter will be hashed to different root card, ensure the load balance between GPUs. -Then we have another version AllReduce operator. Other part keep the same with before. +- **AllReduce** + Need to note that our AllReduce operator is a ring-base AllReduce implementation. If we use the NCCL2 AllReduce primitive, every GPU optimized full batch of data, wasted (n-1) GPU compute resources. In addition, NCCL2 built-in AllReduce will only utilize the communicating resource during synchronization, then update the gradient will be a subsequent phase. In fact, we can amortize the update gradient time cost into the communicating phase. The process is +1. Every parameter has its root card. That card will responsible for aggregating the gradients from GPUs. +2. The whole model's parameter will be hashed to different root card, ensure the load balance between GPUs. +3. Logically neighberhood card will start send parameter to the next one. After one round, the parameter main card will aggregate the full gradients. +4. Then the root card will optimize the parameter. +5. This parameter card will send its optimized result to its neighberhood, then the neighberhood will send parameter to its next one. +6. Finish the sychronization round. + +The total time cost will be 2 * (n-1) * per-parameter-send-time, we reach the goal of amortize the upgrade time into communicating phase. From dafd449c68c23c642ba117a55135c823c6594772 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 14 Dec 2017 15:43:14 +0800 Subject: [PATCH 58/94] Unify `step_block` and `block` to `sub_block` --- paddle/framework/backward.cc | 4 ++-- paddle/operators/conditional_block_op.cc | 8 ++++---- paddle/operators/recurrent_op.cc | 2 +- paddle/operators/while_op.cc | 2 +- python/paddle/v2/fluid/layers.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index a17036c652..faf6e60cbd 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -430,14 +430,14 @@ std::vector> MakeBlockBackward( std::vector> op_grads; if ((*it)->Type() == "recurrent" || (*it)->Type() == "while") { - int step_block_idx = (*it)->GetBlockAttr("step_block"); + int step_block_idx = (*it)->GetBlockAttr("sub_block"); BlockDescBind* backward_block = CreateStepBlock( program_desc, no_grad_vars, grad_to_var, step_block_idx); op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var, {backward_block}); } else if ((*it)->Type() == "conditional_block") { BlockDescBind* backward_block = CreateStepBlock(program_desc, no_grad_vars, grad_to_var, - (*it)->GetBlockAttr("block")); + (*it)->GetBlockAttr("sub_block")); op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var, {backward_block}); } else { op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var); diff --git a/paddle/operators/conditional_block_op.cc b/paddle/operators/conditional_block_op.cc index 03c58a7eab..6f2ef9174e 100644 --- a/paddle/operators/conditional_block_op.cc +++ b/paddle/operators/conditional_block_op.cc @@ -65,7 +65,7 @@ class ConditionalBlockOp : public ConditionalOp { scopes->front() = &scope.NewScope(); auto &cur_scope = *scopes->front(); - auto *block = Attr("block"); + auto *block = Attr("sub_block"); framework::Executor exec(dev_ctx); exec.Run(*block->Program(), &cur_scope, block->ID(), false); } @@ -88,7 +88,7 @@ class ConditionalBlockOpProtoMaker : public framework::OpProtoAndCheckerMaker { "unify the conditional block, rnn and while op, the type of " "scope is std::vector"); AddAttr( - "block", "The step block of conditional block operator"); + "sub_block", "The step block of conditional block operator"); AddComment(R"DOC(Conditional block operator Run the sub-block if X is not empty. Params is the other inputs and Out is the @@ -117,7 +117,7 @@ class ConditionalBlockGradOp : public ConditionalOp { auto &scopes = scope_var->Get>(); framework::Scope &cur_scope = *scopes[0]; - auto *block = Attr("block"); + auto *block = Attr("sub_block"); framework::Executor exec(dev_ctx); exec.Run(*block->Program(), &cur_scope, block->ID(), false); @@ -181,7 +181,7 @@ class ConditionalBlockGradMaker : public framework::SingleGradOpDescMaker { grad_op->SetInput("Scope", Output("Scope")); grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); grad_op->SetOutput(framework::GradVarName("Params"), InputGrad("Params")); - grad_op->SetBlockAttr("block", *this->grad_block_[0]); + grad_op->SetBlockAttr("sub_block", *this->grad_block_[0]); return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index 29f9163643..232d926f7b 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -25,7 +25,7 @@ constexpr char kOutputs[] = "outputs"; constexpr char kStepScopes[] = "step_scopes"; constexpr char kExStates[] = "ex_states"; constexpr char kStates[] = "states"; -constexpr char kStepBlock[] = "step_block"; +constexpr char kStepBlock[] = "sub_block"; constexpr char kReverse[] = "reverse"; constexpr char kIsTrain[] = "is_train"; #define GRAD_SUFFIX "@GRAD" diff --git a/paddle/operators/while_op.cc b/paddle/operators/while_op.cc index b8e44bcc5a..9a092a570f 100644 --- a/paddle/operators/while_op.cc +++ b/paddle/operators/while_op.cc @@ -25,7 +25,7 @@ namespace operators { using StepScopeVar = std::vector; using LoDTensor = framework::LoDTensor; -constexpr char kStepBlock[] = "step_block"; +constexpr char kStepBlock[] = "sub_block"; constexpr char kCondition[] = "Condition"; constexpr char kStepScopes[] = "StepScopes"; constexpr char kParameters[] = "X"; diff --git a/python/paddle/v2/fluid/layers.py b/python/paddle/v2/fluid/layers.py index f67d6d08c7..2781017ec4 100644 --- a/python/paddle/v2/fluid/layers.py +++ b/python/paddle/v2/fluid/layers.py @@ -1130,7 +1130,7 @@ class StaticRNN(object): attrs={ 'ex_states': pre_memories, 'states': memories, - 'step_block': rnn_block + 'sub_block': rnn_block }) @@ -1207,7 +1207,7 @@ class While(object): }, outputs={'Out': out_vars, 'StepScopes': [step_scope]}, - attrs={'step_block': while_block}) + attrs={'sub_block': while_block}) def lstm(x, @@ -1671,7 +1671,7 @@ class ConditionalBlock(object): }, outputs={'Out': out_list, 'Scope': [step_scope]}, - attrs={'block': inside_block}) + attrs={'sub_block': inside_block}) class IfElseBlockGuard(object): From 0e9b393b340990cf581ec9f6e5f33af74912c0b6 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 14 Dec 2017 16:21:58 +0800 Subject: [PATCH 59/94] "derived cudnnDevice context" (#6585) * "derived cudnnDevice context" * "leave remove cudnn handle from CUDADeviceContext" * "fix math function error" --- paddle/operators/math/math_function.cu | 7 +++++++ paddle/platform/device_context.cc | 16 ++++++++++++++++ paddle/platform/device_context.h | 16 ++++++++++++++++ paddle/platform/device_context_test.cc | 16 ++++++++++++++++ paddle/platform/place.h | 7 ++++++- 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/paddle/operators/math/math_function.cu b/paddle/operators/math/math_function.cu index 1b560a7e2d..e33070c40f 100644 --- a/paddle/operators/math/math_function.cu +++ b/paddle/operators/math/math_function.cu @@ -273,6 +273,13 @@ void set_constant_with_place( TensorSetConstantGPU(context, tensor, value)); } +template <> +void set_constant_with_place( + const platform::DeviceContext& context, framework::Tensor* tensor, + float value) { + set_constant_with_place(context, tensor, value); +} + template struct RowwiseAdd; template struct RowwiseAdd; template struct ColwiseSum; diff --git a/paddle/platform/device_context.cc b/paddle/platform/device_context.cc index 2c7f964216..1c72b50559 100644 --- a/paddle/platform/device_context.cc +++ b/paddle/platform/device_context.cc @@ -125,6 +125,22 @@ cudnnHandle_t CUDADeviceContext::cudnn_handle() const { return cudnn_handle_; } cudaStream_t CUDADeviceContext::stream() const { return stream_; } +CudnnDeviceContext::CudnnDeviceContext(CudnnPlace place) + : CUDADeviceContext(place), place_(place) { + PADDLE_ENFORCE(dynload::cudnnCreate(&cudnn_handle_)); + PADDLE_ENFORCE(dynload::cudnnSetStream(cudnn_handle_, stream())); +} + +CudnnDeviceContext::~CudnnDeviceContext() { + SetDeviceId(place_.device); + Wait(); + PADDLE_ENFORCE(dynload::cudnnDestroy(cudnn_handle_)); +} + +Place CudnnDeviceContext::GetPlace() const { return CudnnPlace(); } + +cudnnHandle_t CudnnDeviceContext::cudnn_handle() const { return cudnn_handle_; } + #endif } // namespace platform diff --git a/paddle/platform/device_context.h b/paddle/platform/device_context.h index 596d9d0bba..f67194993d 100644 --- a/paddle/platform/device_context.h +++ b/paddle/platform/device_context.h @@ -86,6 +86,22 @@ class CUDADeviceContext : public DeviceContext { cublasHandle_t cublas_handle_; }; +class CudnnDeviceContext : public CUDADeviceContext { + public: + explicit CudnnDeviceContext(CudnnPlace place); + virtual ~CudnnDeviceContext(); + + /*! \brief Return place in the device context. */ + Place GetPlace() const final; + + /*! \brief Return cudnn handle in the device context. */ + cudnnHandle_t cudnn_handle() const; + + private: + cudnnHandle_t cudnn_handle_; + CudnnPlace place_; +}; + #endif } // namespace platform diff --git a/paddle/platform/device_context_test.cc b/paddle/platform/device_context_test.cc index 4893cd92f6..be3b2af5af 100644 --- a/paddle/platform/device_context_test.cc +++ b/paddle/platform/device_context_test.cc @@ -46,3 +46,19 @@ TEST(Device, CUDADeviceContext) { delete device_context; } } + +TEST(Device, CudnnDeviceContext) { + using paddle::platform::CudnnDeviceContext; + using paddle::platform::CudnnPlace; + if (paddle::platform::dynload::HasCUDNN()) { + int count = paddle::platform::GetCUDADeviceCount(); + for (int i = 0; i < count; ++i) { + CudnnDeviceContext* device_context = + new CudnnDeviceContext(CudnnPlace(i)); + cudnnHandle_t cudnn_handle = device_context->cudnn_handle(); + ASSERT_NE(nullptr, cudnn_handle); + ASSERT_NE(nullptr, device_context->stream()); + delete device_context; + } + } +} diff --git a/paddle/platform/place.h b/paddle/platform/place.h index 5370360a7d..f0dcec8f52 100644 --- a/paddle/platform/place.h +++ b/paddle/platform/place.h @@ -43,6 +43,11 @@ struct GPUPlace { int device; }; +struct CudnnPlace : public GPUPlace { + CudnnPlace() : GPUPlace() {} + explicit CudnnPlace(int d) : GPUPlace(d) {} +}; + struct IsGPUPlace : public boost::static_visitor { bool operator()(const CPUPlace &) const { return false; } bool operator()(const GPUPlace &gpu) const { return true; } @@ -52,7 +57,7 @@ struct IsGPUPlace : public boost::static_visitor { // should be less equal than 2^(NUM_PLACE_TYPE_LIMIT_IN_BIT) #define NUM_PLACE_TYPE_LIMIT_IN_BIT 4 -typedef boost::variant Place; +typedef boost::variant Place; // static check number of place types is less equal than // 2^(NUM_PLACE_TYPE_LIMIT_IN_BIT) From e11a561c128a894ce03bd07bd3985f55f74220f8 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 14 Dec 2017 17:06:25 +0800 Subject: [PATCH 60/94] update --- paddle/pybind/pybind.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index c16d3e0cbe..9ea4e70a26 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -282,6 +282,16 @@ All parameter, weight, gradient are variables in Paddle. } return ret_values; }); + m.def("get_grad_op_desc", + [](const OpDescBind &op_desc, + const std::unordered_set &no_grad_set, + std::unordered_map &grad_to_var, + const std::vector &grad_sub_block) { + return framework::OpInfoMap::Instance() + .Get(op_desc.Type()) + .GradOpMaker()(op_desc, no_grad_set, &grad_to_var, + grad_sub_block); + }); m.def("prune", [](const ProgramDescBind &origin, const std::vector> &targets) { ProgramDescBind prog_with_targets(origin); From 1059796cbfa48273b8c056fee23f8e405dc24bb8 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 14 Dec 2017 17:16:47 +0800 Subject: [PATCH 61/94] update the link of docs icon --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bbb2d49858..ceeb6d9e51 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ [![Build Status](https://travis-ci.org/PaddlePaddle/Paddle.svg?branch=develop)](https://travis-ci.org/PaddlePaddle/Paddle) -[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://doc.paddlepaddle.org/develop/doc/) -[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](http://doc.paddlepaddle.org/develop/doc_cn/) +[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://www.paddlepaddle.org/docs/develop/documentation/en/getstarted/index_en.html) +[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](http://www.paddlepaddle.org/docs/develop/documentation/zh/getstarted/index_cn.html) [![Coverage Status](https://coveralls.io/repos/github/PaddlePaddle/Paddle/badge.svg?branch=develop)](https://coveralls.io/github/PaddlePaddle/Paddle?branch=develop) [![Release](https://img.shields.io/github/release/PaddlePaddle/Paddle.svg)](https://github.com/PaddlePaddle/Paddle/releases) [![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE) From 991826317a4192cd4fb3e8505a4a7c60c0fe830c Mon Sep 17 00:00:00 2001 From: Leding Li Date: Thu, 14 Dec 2017 17:23:39 +0800 Subject: [PATCH 62/94] Add extern "C" to paddle_error_string (#6611) --- paddle/capi/error.cpp | 2 +- paddle/capi/error.h | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/paddle/capi/error.cpp b/paddle/capi/error.cpp index 169b65f921..96ce31b45f 100644 --- a/paddle/capi/error.cpp +++ b/paddle/capi/error.cpp @@ -14,7 +14,7 @@ limitations under the License. */ #include "error.h" -const char* paddle_error_string(paddle_error err) { +extern "C" const char* paddle_error_string(paddle_error err) { switch (err) { case kPD_NULLPTR: return "nullptr error"; diff --git a/paddle/capi/error.h b/paddle/capi/error.h index 9d9d0ed63a..2da9e0a3ef 100644 --- a/paddle/capi/error.h +++ b/paddle/capi/error.h @@ -29,9 +29,17 @@ typedef enum { kPD_UNDEFINED_ERROR = -1, } paddle_error; +#ifdef __cplusplus +extern "C" { +#endif + /** * Error string for Paddle API. */ PD_API const char* paddle_error_string(paddle_error err); +#ifdef __cplusplus +} +#endif + #endif From 044a13d02262f1cf84ee685a0575cf2ec28e5623 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 14 Dec 2017 17:50:56 +0800 Subject: [PATCH 63/94] expose GradOpMaker to Python --- paddle/pybind/pybind.cc | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 9ea4e70a26..1faf24bcb8 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -282,15 +282,22 @@ All parameter, weight, gradient are variables in Paddle. } return ret_values; }); - m.def("get_grad_op_desc", + m.def("get_grad_op_descs", [](const OpDescBind &op_desc, const std::unordered_set &no_grad_set, std::unordered_map &grad_to_var, const std::vector &grad_sub_block) { - return framework::OpInfoMap::Instance() - .Get(op_desc.Type()) - .GradOpMaker()(op_desc, no_grad_set, &grad_to_var, - grad_sub_block); + std::vector> grad_op_descs = + framework::OpInfoMap::Instance() + .Get(op_desc.Type()) + .GradOpMaker()(op_desc, no_grad_set, &grad_to_var, + grad_sub_block); + std::vector grad_op_desc_ptrs(grad_op_descs.size()); + std::transform( + grad_op_descs.begin(), grad_op_descs.end(), + grad_op_desc_ptrs.begin(), + [](std::unique_ptr &p) { return p.release(); }); + return grad_op_desc_ptrs; }); m.def("prune", [](const ProgramDescBind &origin, const std::vector> &targets) { From e0698e33a869f72282ae9f7d78a33c5a4ac2ef00 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 14 Dec 2017 17:57:49 +0800 Subject: [PATCH 64/94] Make layers as a python module (#6564) * Make cast op support bool Also add `elemwise_sub/mul/abs/clip` layers * Make fuild.layers as a module * Move layers as a module * Split layers.py into layers module * Fix CI * Fix CI --- python/paddle/v2/fluid/layers/__init__.py | 17 + .../{layers.py => layers/control_flow.py} | 1023 +---------------- python/paddle/v2/fluid/layers/io.py | 57 + python/paddle/v2/fluid/layers/nn.py | 785 +++++++++++++ python/paddle/v2/fluid/layers/ops.py | 9 + python/paddle/v2/fluid/layers/tensor.py | 130 +++ .../book/test_image_classification_train.py | 4 +- .../book/test_understand_sentiment_lstm.py | 48 +- python/setup.py.in | 1 + 9 files changed, 1057 insertions(+), 1017 deletions(-) create mode 100644 python/paddle/v2/fluid/layers/__init__.py rename python/paddle/v2/fluid/{layers.py => layers/control_flow.py} (51%) create mode 100644 python/paddle/v2/fluid/layers/io.py create mode 100644 python/paddle/v2/fluid/layers/nn.py create mode 100644 python/paddle/v2/fluid/layers/ops.py create mode 100644 python/paddle/v2/fluid/layers/tensor.py diff --git a/python/paddle/v2/fluid/layers/__init__.py b/python/paddle/v2/fluid/layers/__init__.py new file mode 100644 index 0000000000..249f570e13 --- /dev/null +++ b/python/paddle/v2/fluid/layers/__init__.py @@ -0,0 +1,17 @@ +import ops +from ops import * +import nn +from nn import * +import io +from io import * +import tensor +from tensor import * +import control_flow +from control_flow import * + +__all__ = [] +__all__ += nn.__all__ +__all__ += io.__all__ +__all__ += tensor.__all__ +__all__ += control_flow.__all__ +__all__ += ops.__all__ diff --git a/python/paddle/v2/fluid/layers.py b/python/paddle/v2/fluid/layers/control_flow.py similarity index 51% rename from python/paddle/v2/fluid/layers.py rename to python/paddle/v2/fluid/layers/control_flow.py index 2781017ec4..5af6c78977 100644 --- a/python/paddle/v2/fluid/layers.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -1,424 +1,18 @@ +from ..layer_helper import LayerHelper, unique_name +from ..framework import Program, Variable, Operator +from .. import core +from tensor import assign, fill_constant import contextlib -import proto.framework_pb2 as framework_pb2 -import core -from framework import OpProtoHolder, Variable, Program, Operator -from initializer import Constant, Normal, Xavier, Initializer -from paddle.v2.fluid.layer_helper import LayerHelper, unique_name -from registry import register_layer -from param_attr import ParamAttr - __all__ = [ - 'fc', 'data', 'cross_entropy', 'conv2d', 'pool2d', 'embedding', 'concat', - 'StaticRNN', 'cast', 'sequence_conv', 'sequence_pool', 'sums', 'cos_sim', - 'batch_norm', 'accuracy', 'split_lod_tensor', 'While' -] - -_REGISTER_LAYER_FROM_OPS = [ - 'mean', 'mul', 'dropout', 'reshape', 'sigmoid', 'scale', 'transpose', - 'sigmoid_cross_entropy_with_logits', 'elementwise_add', 'elementwise_div', - 'elementwise_sub', 'elementwise_mul', 'clip', 'abs' + 'split_lod_tensor', 'merge_lod_tensor', 'BlockGuard', 'StaticRNNGuard', + 'StaticRNNMemoryLink', 'WhileGuard', 'While', 'lod_rank_table', + 'max_sequence_len', 'topk', 'lod_tensor_to_array', 'array_to_lod_tensor', + 'increment', 'array_write', 'create_array', 'less_than', 'array_read', + 'shrink_memory', 'array_length', 'IfElse', 'DynamicRNN', 'ConditionalBlock', + 'StaticRNN' ] -for _OP in set(_REGISTER_LAYER_FROM_OPS): - globals()[_OP] = register_layer(_OP) - __all__.append(_OP) - - -def fc(input, - size, - num_flatten_dims=1, - param_attr=None, - bias_attr=None, - act=None, - name=None, - main_program=None, - startup_program=None): - """ - Fully Connected Layer. - - Args: - input: The input tensor to the function - size: The size of the layer - num_flatten_dims: Number of columns in input - param_attr: The parameters/weights to the FC Layer - param_initializer: Initializer used for the weight/parameter. If None, XavierInitializer() is used - bias_attr: The bias parameter for the FC layer - bias_initializer: Initializer used for the bias. If None, then ConstantInitializer() is used - act: Activation to be applied to the output of FC layer - name: Name/alias of the function - main_program: Name of the main program that calls this - startup_program: Name of the startup program - - This function can take in multiple inputs and performs the Fully Connected - function (linear transformation) on top of each of them. - So for input x, the output will be : Wx + b. Where W is the parameter, - b the bias and x is the input. - - The function also applies an activation (non-linearity) on top of the - output, if activation is passed in the input. - - All the input variables of this function are passed in as local variables - to the LayerHelper constructor. - - """ - helper = LayerHelper('fc', **locals()) - - dtype = helper.input_dtype() - - mul_results = [] - for input_var, param_attr in helper.iter_inputs_and_params(): - input_shape = input_var.shape - param_shape = [ - reduce(lambda a, b: a * b, input_shape[num_flatten_dims:], 1) - ] + [size] - w = helper.create_parameter( - attr=param_attr, shape=param_shape, dtype=dtype, is_bias=False) - tmp = helper.create_tmp_variable(dtype) - helper.append_op( - type="mul", - inputs={ - "X": input_var, - "Y": w, - }, - outputs={"Out": tmp}, - attrs={'x_num_col_dims': num_flatten_dims, - 'y_num_col_dims': 1}) - mul_results.append(tmp) - - # sum - if len(mul_results) == 1: - pre_bias = mul_results[0] - else: - pre_bias = helper.create_tmp_variable(dtype) - helper.append_op( - type="sum", inputs={"X": mul_results}, outputs={"Out": pre_bias}) - # add bias - pre_activation = helper.append_bias_op(pre_bias) - # add activation - return helper.append_activation(pre_activation) - - -def embedding(input, - size, - is_sparse=False, - param_attr=None, - dtype='float32', - main_program=None, - startup_program=None): - """ - Embedding Layer. - - Args: - param_initializer: - input: The input to the function - size: The size of the layer - is_sparse: A flag that decleares whether the input is sparse - param_attr: Parameters for this layer - dtype: The type of data : float32, float_16, int etc - main_program: Name of the main program that calls this - startup_program: Name of the startup program - - This function can take in the input (which is a vector of IDs) and - performs a lookup in the lookup_table using these IDs, to result into - the embedding of each ID in the input. - - All the input variables of this function are passed in as local variables - to the LayerHelper constructor. - - """ - - helper = LayerHelper('embedding', **locals()) - w = helper.create_parameter( - attr=helper.param_attr, shape=size, dtype=dtype, is_bias=False) - tmp = helper.create_tmp_variable(dtype) - helper.append_op( - type='lookup_table', - inputs={'Ids': input, - 'W': w}, - outputs={'Out': tmp}, - attrs={'is_sparse': is_sparse}) - return tmp - - -# TODO(qijun): expose H0 and C0 -def dynamic_lstm(input, - size, - param_attr=None, - bias_attr=None, - use_peepholes=True, - is_reverse=False, - gate_activation='sigmoid', - cell_activation='tanh', - candidate_activation='tanh', - dtype='float32', - main_program=None, - startup_program=None): - helper = LayerHelper('lstm', **locals()) - size = size / 4 - weight = helper.create_parameter( - attr=helper.param_attr, shape=[size, 4 * size], dtype=dtype) - bias_size = [1, 7 * size] - if not use_peepholes: - bias_size[1] = 4 * size - bias = helper.create_parameter( - attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True) - - hidden = helper.create_tmp_variable(dtype) - cell = helper.create_tmp_variable(dtype) - batch_gate = helper.create_tmp_variable(dtype) - batch_cell_pre_act = helper.create_tmp_variable(dtype) - - helper.append_op( - type='lstm', - inputs={'Input': input, - 'Weight': weight, - 'Bias': bias}, - outputs={ - 'Hidden': hidden, - 'Cell': cell, - 'BatchGate': batch_gate, - 'BatchCellPreAct': batch_cell_pre_act - }, - attrs={ - 'use_peepholes': use_peepholes, - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'cell_activation': cell_activation, - 'candidate_activation': candidate_activation - }) - return hidden, cell - - -def gru_unit(input, - hidden, - size, - weight=None, - bias=None, - activation='tanh', - gate_activation='sigmoid', - main_program=None, - startup_program=None): - """ - GRUUnit Operator implements partial calculations of the GRU unit as following: - - $$ - update \ gate: u_t = actGate(xu_t + W_u * h_{t-1} + b_u) \\ - reset \ gate: r_t = actGate(xr_t + W_r * h_{t-1} + b_r) \\ - output \ candidate: {h}_t = actNode(xc_t + W_c * dot(r_t, h_{t-1}) + b_c) \\ - output: h_t = dot((1 - u_t), h_{t-1}) + dot(u_t, {h}_t) - $$ - - which is same as one time step of GRU Operator. - - @note To implement the complete GRU unit, fully-connected operator must be - used before to feed xu, xr and xc as the Input of GRUUnit operator. - - TODO(ChunweiYan) add more document here - """ - activation_dict = dict( - identity=0, - sigmoid=1, - tanh=2, - relu=3, ) - activation = activation_dict[activation] - gate_activation = activation_dict[gate_activation] - - helper = LayerHelper('gru_unit', **locals()) - dtype = helper.input_dtype() - size = size / 3 - - # create weight - if weight is None: - weight = helper.create_parameter( - attr=helper.param_attr, shape=[size, 3 * size], dtype=dtype) - - # create bias - if bias is None: - bias_size = [1, 3 * size] - bias = helper.create_parameter( - attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True) - - gate = helper.create_tmp_variable(dtype) - reset_hidden_pre = helper.create_tmp_variable(dtype) - updated_hidden = helper.create_tmp_variable(dtype) - - helper.append_op( - type='gru_unit', - inputs={'Input': input, - 'HiddenPrev': hidden, - 'Weight': weight}, - outputs={ - 'Gate': gate, - 'ResetHiddenPrev': reset_hidden_pre, - 'Hidden': updated_hidden, - }, - attrs={ - 'activation': 0, - 'gate_activation': 1, - }) - - return updated_hidden, reset_hidden_pre, gate - - -def data(name, - shape, - append_batch_size=True, - dtype='float32', - lod_level=0, - type=core.VarDesc.VarType.LOD_TENSOR, - main_program=None, - startup_program=None, - stop_gradient=True): - """ - Data Layer. - - Args: - name: The name/alias of the function - shape: Tuple declaring the shape. - append_batch_size: Whether or not to append the data as a batch. - dtype: The type of data : float32, float_16, int etc - type: The output type. By default it is LOD_TENSOR. - lod_level(int): The LoD Level. 0 means the input data is not a sequence. - main_program: Name of the main program that calls this - startup_program: Name of the startup program - stop_gradient: A boolean that mentions whether gradient should flow. - - This function takes in input and based on whether data has - to be returned back as a minibatch, it creates the global variable using - the helper functions. The global variables can be accessed by all the - following operations and layers in the graph. - - All the input variables of this function are passed in as local variables - to the LayerHelper constructor. - - """ - helper = LayerHelper('data', **locals()) - shape = list(shape) - for i in xrange(len(shape)): - if shape[i] is None: - shape[i] = -1 - append_batch_size = False - elif shape[i] < 0: - append_batch_size = False - - if append_batch_size: - shape = [-1] + shape # append batch size as -1 - - return helper.create_global_variable( - name=name, - shape=shape, - dtype=dtype, - type=type, - stop_gradient=stop_gradient, - lod_level=lod_level) - - -def create_tensor(dtype, name=None, main_program=None, startup_program=None): - helper = LayerHelper("create_tensor", **locals()) - return helper.create_variable(name=helper.name, dtype=dtype) - - -def cast(x, dtype, main_program=None): - """ - This function takes in the input with input_dtype - and casts it to the output_dtype as the output. - """ - helper = LayerHelper('cast', **locals()) - out = helper.create_tmp_variable(dtype=dtype) - helper.append_op( - type='cast', - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={'in_dtype': x.dtype, - 'out_dtype': out.dtype}) - return out - - -def concat(input, axis, main_program=None, startup_program=None): - """ - This function concats the input along the axis mentioned - and returns that as the output. - """ - helper = LayerHelper('concat', **locals()) - out = helper.create_tmp_variable(dtype=helper.input_dtype()) - helper.append_op( - type='concat', - inputs={'X': input}, - outputs={'Out': [out]}, - attrs={'axis': axis}) - return out - - -def sums(input, out=None, main_program=None, startup_program=None): - """ - This function takes in the input and performs the sum operation on it - and returns that as the output. - """ - helper = LayerHelper('sum', **locals()) - if out is None: - out = helper.create_tmp_variable(dtype=helper.input_dtype()) - helper.append_op(type='sum', inputs={'X': input}, outputs={'Out': out}) - return out - - -def linear_chain_crf(input, - label, - param_attr=None, - main_program=None, - startup_program=None): - helper = LayerHelper('linear_chain_crf', **locals()) - size = input.shape[1] - transition = helper.create_parameter( - attr=helper.param_attr, - shape=[size + 2, size], - dtype=helper.input_dtype()) - alpha = helper.create_tmp_variable(dtype=helper.input_dtype()) - emission_exps = helper.create_tmp_variable(dtype=helper.input_dtype()) - transition_exps = helper.create_tmp_variable(dtype=helper.input_dtype()) - log_likelihood = helper.create_tmp_variable(dtype=helper.input_dtype()) - helper.append_op( - type='linear_chain_crf', - inputs={"Emission": [input], - "Transition": transition, - "Label": label}, - outputs={ - "Alpha": [alpha], - "EmissionExps": [emission_exps], - "TransitionExps": transition_exps, - "LogLikelihood": log_likelihood - }) - - return log_likelihood - - -def crf_decoding(input, - param_attr, - label=None, - main_program=None, - startup_program=None): - helper = LayerHelper('crf_decoding', **locals()) - transition = helper.get_parameter(param_attr.name) - viterbi_path = helper.create_tmp_variable(dtype=helper.input_dtype()) - helper.append_op( - type='crf_decoding', - inputs={"Emission": [input], - "Transition": transition, - "Label": label}, - outputs={"ViterbiPath": [viterbi_path]}) - - return viterbi_path - - -def assign(input, output, main_program=None, startup_program=None): - helper = LayerHelper('assign', **locals()) - helper.append_op( - type='scale', - inputs={'X': [input]}, - outputs={'Out': [output]}, - attrs={'scale': 1.0}) - return output - def split_lod_tensor(input, mask, @@ -460,404 +54,6 @@ def merge_lod_tensor(in_true, return out -def cos_sim(X, Y, **kwargs): - """ - This function performs the cosine similarity between two tensors - X and Y and returns that as the output. - """ - helper = LayerHelper('cos_sim', **kwargs) - out = helper.create_tmp_variable(dtype=X.dtype) - xnorm = helper.create_tmp_variable(dtype=X.dtype) - ynorm = helper.create_tmp_variable(dtype=X.dtype) - helper.append_op( - type='cos_sim', - inputs={'X': [X], - 'Y': [Y]}, - outputs={'Out': [out], - 'XNorm': [xnorm], - 'YNorm': [ynorm]}) - return out - - -def cross_entropy(input, label, **kwargs): - """ - This function computes cross_entropy using the input and label. - """ - helper = LayerHelper('cross_entropy', **kwargs) - out = helper.create_tmp_variable(dtype=input.dtype) - helper.append_op( - type='cross_entropy', - inputs={'X': [input], - 'Label': [label]}, - outputs={'Y': [out]}, - attrs=kwargs) - return out - - -def square_error_cost(input, label, **kwargs): - """ - This functions returns the squared error cost using the input and label. - The output is appending the op to do the above. - """ - helper = LayerHelper('square_error_cost', **kwargs) - minus_out = helper.create_tmp_variable(dtype=input.dtype) - helper.append_op( - type='elementwise_sub', - inputs={'X': [input], - 'Y': [label]}, - outputs={'Out': [minus_out]}) - - square_out = helper.create_tmp_variable(dtype=input.dtype) - helper.append_op( - type='square', inputs={'X': [minus_out]}, outputs={'Y': [square_out]}) - return square_out - - -def accuracy(input, label, k=1, correct=None, total=None, **kwargs): - """ - This function computes the accuracy using the input and label. - The output is the top_k inputs and their indices. - """ - helper = LayerHelper("accuracy", **kwargs) - topk_out = helper.create_tmp_variable(dtype=input.dtype) - topk_indices = helper.create_tmp_variable(dtype="int64") - helper.append_op( - type="top_k", - inputs={"X": [input]}, - outputs={"Out": [topk_out], - "Indices": [topk_indices]}, - attrs={"k": k}) - acc_out = helper.create_tmp_variable(dtype="float32") - if correct is None: - correct = helper.create_tmp_variable(dtype="int64") - if total is None: - total = helper.create_tmp_variable(dtype="int64") - helper.append_op( - type="accuracy", - inputs={ - "Out": [topk_out], - "Indices": [topk_indices], - "Label": [label] - }, - outputs={ - "Accuracy": [acc_out], - "Correct": [correct], - "Total": [total], - }) - return acc_out - - -def chunk_eval(input, - label, - chunk_scheme, - num_chunk_types, - excluded_chunk_types=None, - **kwargs): - """ - This function computes the accuracy using the input and label. - The output is the top_k inputs and their indices. - """ - helper = LayerHelper("chunk_eval", **kwargs) - - # prepare output - precision = helper.create_tmp_variable(dtype="float32") - recall = helper.create_tmp_variable(dtype="float32") - f1_score = helper.create_tmp_variable(dtype="float32") - - helper.append_op( - type="chunk_eval", - inputs={"Inference": [input], - "Label": [label]}, - outputs={ - "Precision": [precision], - "Recall": [recall], - "F1-Score": [f1_score] - }, - attrs={ - "num_chunk_types": num_chunk_types, - 'chunk_scheme': chunk_scheme, - 'excluded_chunk_types': excluded_chunk_types or [] - }) - return precision, recall, f1_score - - -def sequence_conv(input, - num_filters, - filter_size=3, - filter_stride=1, - padding=None, - bias_attr=None, - param_attr=None, - act=None, - main_program=None, - startup_program=None): - """ - This function creates the op for sequence_conv, using the inputs and - other convolutional configurations for the filters and stride as given - in the input parameters to the function. - """ - - # FIXME(dzh) : want to unify the argument of python layer - # function. So we ignore some unecessary attributes. - # such as, padding_trainable, context_start. - - helper = LayerHelper('sequence_conv', **locals()) - dtype = helper.input_dtype() - filter_shape = [filter_size * input.shape[1], num_filters] - filter_param = helper.create_parameter( - attr=helper.param_attr, shape=filter_shape, dtype=dtype) - pre_bias = helper.create_tmp_variable(dtype) - - helper.append_op( - type='sequence_conv', - inputs={ - 'X': [input], - 'Filter': [filter_param], - }, - outputs={"Out": pre_bias}, - attrs={ - 'contextStride': filter_stride, - 'contextStart': -int(filter_size / 2), - 'contextLength': filter_size - }) - pre_act = helper.append_bias_op(pre_bias) - return helper.append_activation(pre_act) - - -def conv2d(input, - num_filters, - filter_size, - stride=None, - padding=None, - groups=None, - param_attr=None, - bias_attr=None, - act=None, - name=None, - main_program=None, - startup_program=None): - """ - This function creates the op for a 2-dimensional Convolution. - This is performed using the parameters of filters(size, dimensionality etc) - , stride and other configurations for a Convolution operation. - This funciton can also append an activation on top of the - conv-2d output, if mentioned in the input parameters. - """ - - if stride is None: - stride = [1, 1] - helper = LayerHelper('conv2d', **locals()) - dtype = helper.input_dtype() - - num_channels = input.shape[1] - if groups is None: - num_filter_channels = num_channels - else: - if num_channels % groups != 0: - raise ValueError("num_channels must be divisible by groups.") - num_filter_channels = num_channels / groups - - if isinstance(filter_size, int): - filter_size = [filter_size, filter_size] - if isinstance(stride, int): - stride = [stride, stride] - if isinstance(padding, int): - padding = [padding, padding] - - input_shape = input.shape - filter_shape = [num_filters, num_filter_channels] + filter_size - - def _get_default_param_initializer(): - std = (2.0 / (filter_size[0]**2 * num_channels))**0.5 - return Normal(0.0, std, 0) - - filter_param = helper.create_parameter( - attr=helper.param_attr, - shape=filter_shape, - dtype=dtype, - default_initializer=_get_default_param_initializer()) - - pre_bias = helper.create_tmp_variable(dtype) - - helper.append_op( - type='conv2d_cudnn', - inputs={ - 'Input': input, - 'Filter': filter_param, - }, - outputs={"Output": pre_bias}, - attrs={'strides': stride, - 'paddings': padding, - 'groups': groups}) - - pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) - - return helper.append_activation(pre_act) - - -def sequence_pool(input, pool_type, **kwargs): - """ - This function add the operator for sequence pooling. - This is applied on top of the input using pool_type mentioned - in the parameters. - """ - helper = LayerHelper('sequence_pool', input=input, **kwargs) - dtype = helper.input_dtype() - pool_out = helper.create_tmp_variable(dtype) - max_index = helper.create_tmp_variable(dtype) - - helper.append_op( - type="sequence_pool", - inputs={"X": input}, - outputs={"Out": pool_out, - "MaxIndex": max_index}, - attrs={"pooltype": pool_type.upper()}) - - return pool_out - - -def pool2d(input, - pool_size, - pool_type, - pool_stride=None, - pool_padding=None, - global_pooling=False, - main_program=None, - startup_program=None): - """ - This function adds the operator for pooling in 2 dimensions, using the - pooling configurations mentioned in input parameters. - """ - if pool_padding is None: - pool_padding = [0, 0] - if pool_stride is None: - pool_stride = [1, 1] - if pool_type not in ["max", "avg"]: - raise ValueError( - "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", - str(pool_type)) - if isinstance(pool_size, int): - pool_size = [pool_size, pool_size] - if isinstance(pool_stride, int): - pool_stride = [pool_stride, pool_stride] - if isinstance(pool_padding, int): - pool_padding = [pool_padding, pool_padding] - - helper = LayerHelper('pool2d', **locals()) - dtype = helper.input_dtype() - pool_out = helper.create_tmp_variable(dtype) - - helper.append_op( - type="pool2d", - inputs={"X": input}, - outputs={"Out": pool_out}, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "global_pooling": global_pooling, - "strides": pool_stride, - "paddings": pool_padding - }) - - return pool_out - - -def batch_norm(input, - act=None, - is_test=False, - momentum=0.9, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - data_layout='NCHW', - main_program=None, - startup_program=None): - """ - This function helps create an operator to implement - the BatchNorm layer using the configurations from the input parameters. - """ - helper = LayerHelper('batch_norm', **locals()) - dtype = helper.input_dtype() - - input_shape = input.shape - if data_layout == 'NCHW': - channel_num = input_shape[1] - else: - if data_layout == 'NHWC': - channel_num = input_shape[-1] - else: - raise ValueError("unsupported data layout:" + data_layout) - - param_shape = [channel_num] - - # create parameter - scale = helper.create_parameter( - attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - default_initializer=Constant(1.0)) - - bias = helper.create_parameter( - attr=helper.param_attr, shape=param_shape, dtype=dtype, is_bias=True) - - mean = helper.create_global_variable( - dtype=input.dtype, shape=param_shape, persistable=True) - helper.set_variable_initializer(var=mean, initializer=Constant(0.0)) - - variance = helper.create_global_variable( - dtype=input.dtype, shape=param_shape, persistable=True) - helper.set_variable_initializer(var=variance, initializer=Constant(1.0)) - - # create output - # mean and mean_out share the same memory - mean_out = mean - # variance and variance out share the same memory - variance_out = variance - saved_mean = helper.create_tmp_variable(dtype) - saved_variance = helper.create_tmp_variable(dtype) - - batch_norm_out = helper.create_tmp_variable(dtype) - - helper.append_op( - type="batch_norm", - inputs={ - "X": input, - "Scale": scale, - "Bias": bias, - "Mean": mean, - "Variance": variance - }, - outputs={ - "Y": batch_norm_out, - "MeanOut": mean_out, - "VarianceOut": variance_out, - "SavedMean": saved_mean, - "SavedVariance": saved_variance - }, - attrs={"momentum": momentum, - "epsilon": epsilon, - "is_test": is_test}) - - return helper.append_activation(batch_norm_out) - - -def beam_search_decode(ids, scores, main_program=None, startup_program=None): - helper = LayerHelper('beam_search_decode', **locals()) - sentence_ids = helper.create_tmp_variable(dtype=ids.dtype) - sentence_scores = helper.create_tmp_variable(dtype=ids.dtype) - - helper.append_op( - type="beam_search_decode", - inputs={"Ids": ids, - "Scores": scores}, - outputs={ - "SentenceIds": sentence_ids, - "SentenceScores": sentence_scores - }) - - return sentence_ids, sentence_scores - - class BlockGuard(object): """ BlockGuard class. @@ -1210,50 +406,6 @@ class While(object): attrs={'sub_block': while_block}) -def lstm(x, - c_pre_init, - hidden_dim, - forget_bias=None, - main_program=None, - startup_program=None): - """ - This function helps create an operator for the LSTM (Long Short Term - Memory) cell that can be used inside an RNN. - """ - helper = LayerHelper('lstm_unit', **locals()) - rnn = StaticRNN() - with rnn.step(): - c_pre = rnn.memory(init=c_pre_init) - x_t = rnn.step_input(x) - - before_fc = concat( - input=[x_t, c_pre], - axis=1, - main_program=main_program, - startup_program=startup_program) - after_fc = fc(input=before_fc, - size=hidden_dim * 4, - main_program=main_program, - startup_program=startup_program) - - dtype = x.dtype - c = helper.create_tmp_variable(dtype) - h = helper.create_tmp_variable(dtype) - - helper.append_op( - type='lstm_unit', - inputs={"X": after_fc, - "C_prev": c_pre}, - outputs={"C": c, - "H": h}, - attrs={"forget_bias": forget_bias}) - - rnn.update_memory(c_pre, c) - rnn.output(h) - - return rnn() - - def lod_rank_table(x, level=0, main_program=None): """ This function creates an operator for creating a LOD_RANK_TABLE @@ -1331,72 +483,6 @@ def array_to_lod_tensor(x, table, main_program=None, startup_program=None): return tmp -def fill_constant(shape, - dtype, - value, - out=None, - main_program=None, - startup_program=None): - """ - This function creates a tensor , with shape as mentioned in the input and - specified dtype and fills this up with a constant value that - comes in the input. It also sets the stop_gradient to be True. - """ - helper = LayerHelper("fill_constant", **locals()) - if out is None: - out = helper.create_tmp_variable(dtype=dtype) - helper.append_op( - type='fill_constant', - inputs={}, - outputs={'Out': [out]}, - attrs={'shape': shape, - 'dtype': out.dtype, - 'value': float(value)}) - out.stop_gradient = True - return out - - -def fill_constant_batch_size_like(input, - shape, - dtype, - value, - input_dim_idx=0, - output_dim_idx=0, - main_program=None, - startup_program=None): - helper = LayerHelper("fill_constant_batch_size_like", **locals()) - out = helper.create_tmp_variable(dtype=dtype) - helper.append_op( - type='fill_constant_batch_size_like', - inputs={'Input': input}, - outputs={'Out': [out]}, - attrs={ - 'shape': shape, - 'dtype': out.dtype, - 'value': float(value), - 'input_dim_idx': input_dim_idx, - 'output_dim_idx': output_dim_idx - }) - out.stop_gradient = True - return out - - -def ones(shape, dtype, main_program=None): - """ - This function performs the same function as fill_constant() declared above - with the constant value being 1.0. - """ - return fill_constant(value=1.0, **locals()) - - -def zeros(shape, dtype, main_program=None): - """ - This function performs the same function as fill_constant() declared above - with the constant value being 0.0. - """ - return fill_constant(value=0.0, **locals()) - - def increment(x, value=1.0, in_place=True, @@ -1508,95 +594,6 @@ def array_length(array, main_program=None): return tmp -def conv2d_transpose(input, - num_filters, - output_size=None, - filter_size=None, - padding=None, - stride=None, - param_attr=None, - main_program=None, - startup_program=None): - """ - The transpose of conv2d layer. - - This layer is also known as deconvolution layer. - - Args: - input(Variable): The input image with [N, C, H, W] format. - num_filters(int): The number of filter. It is as same as the output - image channel. - output_size(int|tuple|None): The output image size. If output size is a - tuple, it must contain two integers, (image_H, image_W). This - parameter only works when filter_size is None. - filter_size(int|tuple|None): The filter size. If filter_size is a tuple, - it must contain two integers, (filter_size_H, filter_size_W). - Otherwise, the filter will be a square. None if use output size to - calculate filter_size - padding(int|tuple): The padding size. If padding is a tuple, it must - contain two integers, (padding_H, padding_W). Otherwise, the - padding_H = padding_W = padding. - stride(int|tuple): The stride size. If stride is a tuple, it must - contain two integers, (stride_H, stride_W). Otherwise, the - stride_H = stride_W = stride. - param_attr: Parameter Attribute. - main_program(Program): the main program - startup_program(Program): the startup program - - Returns: - Variable: Output image. - """ - helper = LayerHelper("conv2d_transpose", **locals()) - if not isinstance(input, Variable): - raise TypeError("Input of conv2d_transpose must be Variable") - input_channel = input.shape[1] - - op_attr = dict() - - if isinstance(padding, int): - op_attr['paddings'] = [padding, padding] - elif padding is not None: - op_attr['paddings'] = padding - - if isinstance(stride, int): - op_attr['strides'] = stride - elif stride is not None: - op_attr['strides'] = stride - - if filter_size is None: - if output_size is None: - raise ValueError("output_size must be set when filter_size is None") - if isinstance(output_size, int): - output_size = [output_size, output_size] - - padding = op_attr.get('paddings', [0, 0]) - stride = op_attr.get('strides', [1, 1]) - - h_in = input.shape[2] - w_in = input.shape[3] - filter_size_h = output_size[0] - \ - (h_in - 1) * stride[0] + 2 * padding[0] - filter_size_w = output_size[1] - \ - (w_in - 1) * stride[1] + 2 * padding[1] - filter_size = [filter_size_h, filter_size_w] - elif isinstance(filter_size, int): - filter_size = [filter_size, filter_size] - - filter_shape = [input_channel, num_filters] + filter_size - img_filter = helper.create_parameter( - dtype=input.dtype, shape=filter_shape, attr=helper.param_attr) - - out = helper.create_tmp_variable(dtype=input.dtype) - helper.append_op( - type='conv2d_transpose', - inputs={'Input': [input], - 'Filter': [img_filter]}, - outputs={'Output': out}, - attrs=op_attr) - - return out - - class ConditionalBlockGuard(BlockGuard): def __init__(self, block): if not isinstance(block, ConditionalBlock): diff --git a/python/paddle/v2/fluid/layers/io.py b/python/paddle/v2/fluid/layers/io.py new file mode 100644 index 0000000000..f03d8e3c3e --- /dev/null +++ b/python/paddle/v2/fluid/layers/io.py @@ -0,0 +1,57 @@ +from .. import core +from ..layer_helper import LayerHelper + +__all__ = ['data'] + + +def data(name, + shape, + append_batch_size=True, + dtype='float32', + lod_level=0, + type=core.VarDesc.VarType.LOD_TENSOR, + main_program=None, + startup_program=None, + stop_gradient=True): + """ + Data Layer. + + Args: + name: The name/alias of the function + shape: Tuple declaring the shape. + append_batch_size: Whether or not to append the data as a batch. + dtype: The type of data : float32, float_16, int etc + type: The output type. By default it is LOD_TENSOR. + lod_level(int): The LoD Level. 0 means the input data is not a sequence. + main_program: Name of the main program that calls this + startup_program: Name of the startup program + stop_gradient: A boolean that mentions whether gradient should flow. + + This function takes in input and based on whether data has + to be returned back as a minibatch, it creates the global variable using + the helper functions. The global variables can be accessed by all the + following operations and layers in the graph. + + All the input variables of this function are passed in as local variables + to the LayerHelper constructor. + + """ + helper = LayerHelper('data', **locals()) + shape = list(shape) + for i in xrange(len(shape)): + if shape[i] is None: + shape[i] = -1 + append_batch_size = False + elif shape[i] < 0: + append_batch_size = False + + if append_batch_size: + shape = [-1] + shape # append batch size as -1 + + return helper.create_global_variable( + name=name, + shape=shape, + dtype=dtype, + type=type, + stop_gradient=stop_gradient, + lod_level=lod_level) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py new file mode 100644 index 0000000000..f231f38b3e --- /dev/null +++ b/python/paddle/v2/fluid/layers/nn.py @@ -0,0 +1,785 @@ +""" +All layers just related to the neural network. +""" + +from ..layer_helper import LayerHelper +from ..initializer import Normal, Constant +from ..framework import Variable + +__all__ = [ + 'fc', 'embedding', 'dynamic_lstm', 'gru_unit', 'linear_chain_crf', + 'crf_decoding', 'cos_sim', 'cross_entropy', 'square_error_cost', 'accuracy', + 'chunk_eval', 'sequence_conv', 'conv2d', 'sequence_pool', 'pool2d', + 'batch_norm', 'beam_search_decode', 'conv2d_transpose' +] + + +def fc(input, + size, + num_flatten_dims=1, + param_attr=None, + bias_attr=None, + act=None, + name=None, + main_program=None, + startup_program=None): + """ + Fully Connected Layer. + + Args: + input: The input tensor to the function + size: The size of the layer + num_flatten_dims: Number of columns in input + param_attr: The parameters/weights to the FC Layer + param_initializer: Initializer used for the weight/parameter. If None, XavierInitializer() is used + bias_attr: The bias parameter for the FC layer + bias_initializer: Initializer used for the bias. If None, then ConstantInitializer() is used + act: Activation to be applied to the output of FC layer + name: Name/alias of the function + main_program: Name of the main program that calls this + startup_program: Name of the startup program + + This function can take in multiple inputs and performs the Fully Connected + function (linear transformation) on top of each of them. + So for input x, the output will be : Wx + b. Where W is the parameter, + b the bias and x is the input. + + The function also applies an activation (non-linearity) on top of the + output, if activation is passed in the input. + + All the input variables of this function are passed in as local variables + to the LayerHelper constructor. + + """ + helper = LayerHelper('fc', **locals()) + + dtype = helper.input_dtype() + + mul_results = [] + for input_var, param_attr in helper.iter_inputs_and_params(): + input_shape = input_var.shape + param_shape = [ + reduce(lambda a, b: a * b, input_shape[num_flatten_dims:], 1) + ] + [size] + w = helper.create_parameter( + attr=param_attr, shape=param_shape, dtype=dtype, is_bias=False) + tmp = helper.create_tmp_variable(dtype) + helper.append_op( + type="mul", + inputs={ + "X": input_var, + "Y": w, + }, + outputs={"Out": tmp}, + attrs={'x_num_col_dims': num_flatten_dims, + 'y_num_col_dims': 1}) + mul_results.append(tmp) + + # sum + if len(mul_results) == 1: + pre_bias = mul_results[0] + else: + pre_bias = helper.create_tmp_variable(dtype) + helper.append_op( + type="sum", inputs={"X": mul_results}, outputs={"Out": pre_bias}) + # add bias + pre_activation = helper.append_bias_op(pre_bias) + # add activation + return helper.append_activation(pre_activation) + + +def embedding(input, + size, + is_sparse=False, + param_attr=None, + dtype='float32', + main_program=None, + startup_program=None): + """ + Embedding Layer. + + Args: + param_initializer: + input: The input to the function + size: The size of the layer + is_sparse: A flag that decleares whether the input is sparse + param_attr: Parameters for this layer + dtype: The type of data : float32, float_16, int etc + main_program: Name of the main program that calls this + startup_program: Name of the startup program + + This function can take in the input (which is a vector of IDs) and + performs a lookup in the lookup_table using these IDs, to result into + the embedding of each ID in the input. + + All the input variables of this function are passed in as local variables + to the LayerHelper constructor. + + """ + + helper = LayerHelper('embedding', **locals()) + w = helper.create_parameter( + attr=helper.param_attr, shape=size, dtype=dtype, is_bias=False) + tmp = helper.create_tmp_variable(dtype) + helper.append_op( + type='lookup_table', + inputs={'Ids': input, + 'W': w}, + outputs={'Out': tmp}, + attrs={'is_sparse': is_sparse}) + return tmp + + +# TODO(qijun): expose H0 and C0 +def dynamic_lstm(input, + size, + param_attr=None, + bias_attr=None, + use_peepholes=True, + is_reverse=False, + gate_activation='sigmoid', + cell_activation='tanh', + candidate_activation='tanh', + dtype='float32', + main_program=None, + startup_program=None): + helper = LayerHelper('lstm', **locals()) + size = size / 4 + weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, 4 * size], dtype=dtype) + bias_size = [1, 7 * size] + if not use_peepholes: + bias_size[1] = 4 * size + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True) + + hidden = helper.create_tmp_variable(dtype) + cell = helper.create_tmp_variable(dtype) + batch_gate = helper.create_tmp_variable(dtype) + batch_cell_pre_act = helper.create_tmp_variable(dtype) + + helper.append_op( + type='lstm', + inputs={'Input': input, + 'Weight': weight, + 'Bias': bias}, + outputs={ + 'Hidden': hidden, + 'Cell': cell, + 'BatchGate': batch_gate, + 'BatchCellPreAct': batch_cell_pre_act + }, + attrs={ + 'use_peepholes': use_peepholes, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'cell_activation': cell_activation, + 'candidate_activation': candidate_activation + }) + return hidden, cell + + +def gru_unit(input, + hidden, + size, + weight=None, + bias=None, + activation='tanh', + gate_activation='sigmoid', + main_program=None, + startup_program=None): + """ + GRUUnit Operator implements partial calculations of the GRU unit as following: + + $$ + update \ gate: u_t = actGate(xu_t + W_u * h_{t-1} + b_u) \\ + reset \ gate: r_t = actGate(xr_t + W_r * h_{t-1} + b_r) \\ + output \ candidate: {h}_t = actNode(xc_t + W_c * dot(r_t, h_{t-1}) + b_c) \\ + output: h_t = dot((1 - u_t), h_{t-1}) + dot(u_t, {h}_t) + $$ + + which is same as one time step of GRU Operator. + + @note To implement the complete GRU unit, fully-connected operator must be + used before to feed xu, xr and xc as the Input of GRUUnit operator. + + TODO(ChunweiYan) add more document here + """ + activation_dict = dict( + identity=0, + sigmoid=1, + tanh=2, + relu=3, ) + activation = activation_dict[activation] + gate_activation = activation_dict[gate_activation] + + helper = LayerHelper('gru_unit', **locals()) + dtype = helper.input_dtype() + size = size / 3 + + # create weight + if weight is None: + weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, 3 * size], dtype=dtype) + + # create bias + if bias is None: + bias_size = [1, 3 * size] + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True) + + gate = helper.create_tmp_variable(dtype) + reset_hidden_pre = helper.create_tmp_variable(dtype) + updated_hidden = helper.create_tmp_variable(dtype) + + helper.append_op( + type='gru_unit', + inputs={'Input': input, + 'HiddenPrev': hidden, + 'Weight': weight}, + outputs={ + 'Gate': gate, + 'ResetHiddenPrev': reset_hidden_pre, + 'Hidden': updated_hidden, + }, + attrs={ + 'activation': 0, + 'gate_activation': 1, + }) + + return updated_hidden, reset_hidden_pre, gate + + +def linear_chain_crf(input, + label, + param_attr=None, + main_program=None, + startup_program=None): + helper = LayerHelper('linear_chain_crf', **locals()) + size = input.shape[1] + transition = helper.create_parameter( + attr=helper.param_attr, + shape=[size + 2, size], + dtype=helper.input_dtype()) + alpha = helper.create_tmp_variable(dtype=helper.input_dtype()) + emission_exps = helper.create_tmp_variable(dtype=helper.input_dtype()) + transition_exps = helper.create_tmp_variable(dtype=helper.input_dtype()) + log_likelihood = helper.create_tmp_variable(dtype=helper.input_dtype()) + helper.append_op( + type='linear_chain_crf', + inputs={"Emission": [input], + "Transition": transition, + "Label": label}, + outputs={ + "Alpha": [alpha], + "EmissionExps": [emission_exps], + "TransitionExps": transition_exps, + "LogLikelihood": log_likelihood + }) + + return log_likelihood + + +def crf_decoding(input, + param_attr, + label=None, + main_program=None, + startup_program=None): + helper = LayerHelper('crf_decoding', **locals()) + transition = helper.get_parameter(param_attr.name) + viterbi_path = helper.create_tmp_variable(dtype=helper.input_dtype()) + helper.append_op( + type='crf_decoding', + inputs={"Emission": [input], + "Transition": transition, + "Label": label}, + outputs={"ViterbiPath": [viterbi_path]}) + + return viterbi_path + + +def cos_sim(X, Y, **kwargs): + """ + This function performs the cosine similarity between two tensors + X and Y and returns that as the output. + """ + helper = LayerHelper('cos_sim', **kwargs) + out = helper.create_tmp_variable(dtype=X.dtype) + xnorm = helper.create_tmp_variable(dtype=X.dtype) + ynorm = helper.create_tmp_variable(dtype=X.dtype) + helper.append_op( + type='cos_sim', + inputs={'X': [X], + 'Y': [Y]}, + outputs={'Out': [out], + 'XNorm': [xnorm], + 'YNorm': [ynorm]}) + return out + + +def cross_entropy(input, label, **kwargs): + """ + This function computes cross_entropy using the input and label. + """ + helper = LayerHelper('cross_entropy', **kwargs) + out = helper.create_tmp_variable(dtype=input.dtype) + helper.append_op( + type='cross_entropy', + inputs={'X': [input], + 'Label': [label]}, + outputs={'Y': [out]}, + attrs=kwargs) + return out + + +def square_error_cost(input, label, **kwargs): + """ + This functions returns the squared error cost using the input and label. + The output is appending the op to do the above. + """ + helper = LayerHelper('square_error_cost', **kwargs) + minus_out = helper.create_tmp_variable(dtype=input.dtype) + helper.append_op( + type='elementwise_sub', + inputs={'X': [input], + 'Y': [label]}, + outputs={'Out': [minus_out]}) + + square_out = helper.create_tmp_variable(dtype=input.dtype) + helper.append_op( + type='square', inputs={'X': [minus_out]}, outputs={'Y': [square_out]}) + return square_out + + +def accuracy(input, label, k=1, correct=None, total=None, **kwargs): + """ + This function computes the accuracy using the input and label. + The output is the top_k inputs and their indices. + """ + helper = LayerHelper("accuracy", **kwargs) + topk_out = helper.create_tmp_variable(dtype=input.dtype) + topk_indices = helper.create_tmp_variable(dtype="int64") + helper.append_op( + type="top_k", + inputs={"X": [input]}, + outputs={"Out": [topk_out], + "Indices": [topk_indices]}, + attrs={"k": k}) + acc_out = helper.create_tmp_variable(dtype="float32") + if correct is None: + correct = helper.create_tmp_variable(dtype="int64") + if total is None: + total = helper.create_tmp_variable(dtype="int64") + helper.append_op( + type="accuracy", + inputs={ + "Out": [topk_out], + "Indices": [topk_indices], + "Label": [label] + }, + outputs={ + "Accuracy": [acc_out], + "Correct": [correct], + "Total": [total], + }) + return acc_out + + +def chunk_eval(input, + label, + chunk_scheme, + num_chunk_types, + excluded_chunk_types=None, + **kwargs): + """ + This function computes the accuracy using the input and label. + The output is the top_k inputs and their indices. + """ + helper = LayerHelper("chunk_eval", **kwargs) + + # prepare output + precision = helper.create_tmp_variable(dtype="float32") + recall = helper.create_tmp_variable(dtype="float32") + f1_score = helper.create_tmp_variable(dtype="float32") + + helper.append_op( + type="chunk_eval", + inputs={"Inference": [input], + "Label": [label]}, + outputs={ + "Precision": [precision], + "Recall": [recall], + "F1-Score": [f1_score] + }, + attrs={ + "num_chunk_types": num_chunk_types, + 'chunk_scheme': chunk_scheme, + 'excluded_chunk_types': excluded_chunk_types or [] + }) + return precision, recall, f1_score + + +def sequence_conv(input, + num_filters, + filter_size=3, + filter_stride=1, + padding=None, + bias_attr=None, + param_attr=None, + act=None, + main_program=None, + startup_program=None): + """ + This function creates the op for sequence_conv, using the inputs and + other convolutional configurations for the filters and stride as given + in the input parameters to the function. + """ + + # FIXME(dzh) : want to unify the argument of python layer + # function. So we ignore some unecessary attributes. + # such as, padding_trainable, context_start. + + helper = LayerHelper('sequence_conv', **locals()) + dtype = helper.input_dtype() + filter_shape = [filter_size * input.shape[1], num_filters] + filter_param = helper.create_parameter( + attr=helper.param_attr, shape=filter_shape, dtype=dtype) + pre_bias = helper.create_tmp_variable(dtype) + + helper.append_op( + type='sequence_conv', + inputs={ + 'X': [input], + 'Filter': [filter_param], + }, + outputs={"Out": pre_bias}, + attrs={ + 'contextStride': filter_stride, + 'contextStart': -int(filter_size / 2), + 'contextLength': filter_size + }) + pre_act = helper.append_bias_op(pre_bias) + return helper.append_activation(pre_act) + + +def conv2d(input, + num_filters, + filter_size, + stride=None, + padding=None, + groups=None, + param_attr=None, + bias_attr=None, + act=None, + name=None, + main_program=None, + startup_program=None): + """ + This function creates the op for a 2-dimensional Convolution. + This is performed using the parameters of filters(size, dimensionality etc) + , stride and other configurations for a Convolution operation. + This funciton can also append an activation on top of the + conv-2d output, if mentioned in the input parameters. + """ + + if stride is None: + stride = [1, 1] + helper = LayerHelper('conv2d', **locals()) + dtype = helper.input_dtype() + + num_channels = input.shape[1] + if groups is None: + num_filter_channels = num_channels + else: + if num_channels % groups != 0: + raise ValueError("num_channels must be divisible by groups.") + num_filter_channels = num_channels / groups + + if isinstance(filter_size, int): + filter_size = [filter_size, filter_size] + if isinstance(stride, int): + stride = [stride, stride] + if isinstance(padding, int): + padding = [padding, padding] + + input_shape = input.shape + filter_shape = [num_filters, num_filter_channels] + filter_size + + def _get_default_param_initializer(): + std = (2.0 / (filter_size[0]**2 * num_channels))**0.5 + return Normal(0.0, std, 0) + + filter_param = helper.create_parameter( + attr=helper.param_attr, + shape=filter_shape, + dtype=dtype, + default_initializer=_get_default_param_initializer()) + + pre_bias = helper.create_tmp_variable(dtype) + + helper.append_op( + type='conv2d_cudnn', + inputs={ + 'Input': input, + 'Filter': filter_param, + }, + outputs={"Output": pre_bias}, + attrs={'strides': stride, + 'paddings': padding, + 'groups': groups}) + + pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) + + return helper.append_activation(pre_act) + + +def sequence_pool(input, pool_type, **kwargs): + """ + This function add the operator for sequence pooling. + This is applied on top of the input using pool_type mentioned + in the parameters. + """ + helper = LayerHelper('sequence_pool', input=input, **kwargs) + dtype = helper.input_dtype() + pool_out = helper.create_tmp_variable(dtype) + max_index = helper.create_tmp_variable(dtype) + + helper.append_op( + type="sequence_pool", + inputs={"X": input}, + outputs={"Out": pool_out, + "MaxIndex": max_index}, + attrs={"pooltype": pool_type.upper()}) + + return pool_out + + +def pool2d(input, + pool_size, + pool_type, + pool_stride=None, + pool_padding=None, + global_pooling=False, + main_program=None, + startup_program=None): + """ + This function adds the operator for pooling in 2 dimensions, using the + pooling configurations mentioned in input parameters. + """ + if pool_padding is None: + pool_padding = [0, 0] + if pool_stride is None: + pool_stride = [1, 1] + if pool_type not in ["max", "avg"]: + raise ValueError( + "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", + str(pool_type)) + if isinstance(pool_size, int): + pool_size = [pool_size, pool_size] + if isinstance(pool_stride, int): + pool_stride = [pool_stride, pool_stride] + if isinstance(pool_padding, int): + pool_padding = [pool_padding, pool_padding] + + helper = LayerHelper('pool2d', **locals()) + dtype = helper.input_dtype() + pool_out = helper.create_tmp_variable(dtype) + + helper.append_op( + type="pool2d", + inputs={"X": input}, + outputs={"Out": pool_out}, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "global_pooling": global_pooling, + "strides": pool_stride, + "paddings": pool_padding + }) + + return pool_out + + +def batch_norm(input, + act=None, + is_test=False, + momentum=0.9, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + data_layout='NCHW', + main_program=None, + startup_program=None): + """ + This function helps create an operator to implement + the BatchNorm layer using the configurations from the input parameters. + """ + helper = LayerHelper('batch_norm', **locals()) + dtype = helper.input_dtype() + + input_shape = input.shape + if data_layout == 'NCHW': + channel_num = input_shape[1] + else: + if data_layout == 'NHWC': + channel_num = input_shape[-1] + else: + raise ValueError("unsupported data layout:" + data_layout) + + param_shape = [channel_num] + + # create parameter + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + default_initializer=Constant(1.0)) + + bias = helper.create_parameter( + attr=helper.param_attr, shape=param_shape, dtype=dtype, is_bias=True) + + mean = helper.create_global_variable( + dtype=input.dtype, shape=param_shape, persistable=True) + helper.set_variable_initializer(var=mean, initializer=Constant(0.0)) + + variance = helper.create_global_variable( + dtype=input.dtype, shape=param_shape, persistable=True) + helper.set_variable_initializer(var=variance, initializer=Constant(1.0)) + + # create output + # mean and mean_out share the same memory + mean_out = mean + # variance and variance out share the same memory + variance_out = variance + saved_mean = helper.create_tmp_variable(dtype) + saved_variance = helper.create_tmp_variable(dtype) + + batch_norm_out = helper.create_tmp_variable(dtype) + + helper.append_op( + type="batch_norm", + inputs={ + "X": input, + "Scale": scale, + "Bias": bias, + "Mean": mean, + "Variance": variance + }, + outputs={ + "Y": batch_norm_out, + "MeanOut": mean_out, + "VarianceOut": variance_out, + "SavedMean": saved_mean, + "SavedVariance": saved_variance + }, + attrs={"momentum": momentum, + "epsilon": epsilon, + "is_test": is_test}) + + return helper.append_activation(batch_norm_out) + + +def beam_search_decode(ids, scores, main_program=None, startup_program=None): + helper = LayerHelper('beam_search_decode', **locals()) + sentence_ids = helper.create_tmp_variable(dtype=ids.dtype) + sentence_scores = helper.create_tmp_variable(dtype=ids.dtype) + + helper.append_op( + type="beam_search_decode", + inputs={"Ids": ids, + "Scores": scores}, + outputs={ + "SentenceIds": sentence_ids, + "SentenceScores": sentence_scores + }) + + return sentence_ids, sentence_scores + + +def conv2d_transpose(input, + num_filters, + output_size=None, + filter_size=None, + padding=None, + stride=None, + param_attr=None, + main_program=None, + startup_program=None): + """ + The transpose of conv2d layer. + + This layer is also known as deconvolution layer. + + Args: + input(Variable): The input image with [N, C, H, W] format. + num_filters(int): The number of filter. It is as same as the output + image channel. + output_size(int|tuple|None): The output image size. If output size is a + tuple, it must contain two integers, (image_H, image_W). This + parameter only works when filter_size is None. + filter_size(int|tuple|None): The filter size. If filter_size is a tuple, + it must contain two integers, (filter_size_H, filter_size_W). + Otherwise, the filter will be a square. None if use output size to + calculate filter_size + padding(int|tuple): The padding size. If padding is a tuple, it must + contain two integers, (padding_H, padding_W). Otherwise, the + padding_H = padding_W = padding. + stride(int|tuple): The stride size. If stride is a tuple, it must + contain two integers, (stride_H, stride_W). Otherwise, the + stride_H = stride_W = stride. + param_attr: Parameter Attribute. + main_program(Program): the main program + startup_program(Program): the startup program + + Returns: + Variable: Output image. + """ + helper = LayerHelper("conv2d_transpose", **locals()) + if not isinstance(input, Variable): + raise TypeError("Input of conv2d_transpose must be Variable") + input_channel = input.shape[1] + + op_attr = dict() + + if isinstance(padding, int): + op_attr['paddings'] = [padding, padding] + elif padding is not None: + op_attr['paddings'] = padding + + if isinstance(stride, int): + op_attr['strides'] = stride + elif stride is not None: + op_attr['strides'] = stride + + if filter_size is None: + if output_size is None: + raise ValueError("output_size must be set when filter_size is None") + if isinstance(output_size, int): + output_size = [output_size, output_size] + + padding = op_attr.get('paddings', [0, 0]) + stride = op_attr.get('strides', [1, 1]) + + h_in = input.shape[2] + w_in = input.shape[3] + filter_size_h = output_size[0] - \ + (h_in - 1) * stride[0] + 2 * padding[0] + filter_size_w = output_size[1] - \ + (w_in - 1) * stride[1] + 2 * padding[1] + filter_size = [filter_size_h, filter_size_w] + elif isinstance(filter_size, int): + filter_size = [filter_size, filter_size] + + filter_shape = [input_channel, num_filters] + filter_size + img_filter = helper.create_parameter( + dtype=input.dtype, shape=filter_shape, attr=helper.param_attr) + + out = helper.create_tmp_variable(dtype=input.dtype) + helper.append_op( + type='conv2d_transpose', + inputs={'Input': [input], + 'Filter': [img_filter]}, + outputs={'Output': out}, + attrs=op_attr) + + return out diff --git a/python/paddle/v2/fluid/layers/ops.py b/python/paddle/v2/fluid/layers/ops.py new file mode 100644 index 0000000000..fa312ace60 --- /dev/null +++ b/python/paddle/v2/fluid/layers/ops.py @@ -0,0 +1,9 @@ +from ..registry import register_layer +__all__ = [ + 'mean', 'mul', 'dropout', 'reshape', 'sigmoid', 'scale', 'transpose', + 'sigmoid_cross_entropy_with_logits', 'elementwise_add', 'elementwise_div', + 'elementwise_sub', 'elementwise_mul', 'clip', 'abs' +] + +for _OP in set(__all__): + globals()[_OP] = register_layer(_OP) diff --git a/python/paddle/v2/fluid/layers/tensor.py b/python/paddle/v2/fluid/layers/tensor.py new file mode 100644 index 0000000000..a839ed897d --- /dev/null +++ b/python/paddle/v2/fluid/layers/tensor.py @@ -0,0 +1,130 @@ +from ..layer_helper import LayerHelper + +__all__ = [ + 'create_tensor', 'cast', 'concat', 'sums', 'assign', + 'fill_constant_batch_size_like', 'fill_constant', 'ones', 'zeros' +] + + +def create_tensor(dtype, name=None, main_program=None, startup_program=None): + helper = LayerHelper("create_tensor", **locals()) + return helper.create_variable(name=helper.name, dtype=dtype) + + +def cast(x, dtype, main_program=None): + """ + This function takes in the input with input_dtype + and casts it to the output_dtype as the output. + """ + helper = LayerHelper('cast', **locals()) + out = helper.create_tmp_variable(dtype=dtype) + helper.append_op( + type='cast', + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={'in_dtype': x.dtype, + 'out_dtype': out.dtype}) + return out + + +def concat(input, axis, main_program=None, startup_program=None): + """ + This function concats the input along the axis mentioned + and returns that as the output. + """ + helper = LayerHelper('concat', **locals()) + out = helper.create_tmp_variable(dtype=helper.input_dtype()) + helper.append_op( + type='concat', + inputs={'X': input}, + outputs={'Out': [out]}, + attrs={'axis': axis}) + return out + + +def sums(input, out=None, main_program=None, startup_program=None): + """ + This function takes in the input and performs the sum operation on it + and returns that as the output. + """ + helper = LayerHelper('sum', **locals()) + if out is None: + out = helper.create_tmp_variable(dtype=helper.input_dtype()) + helper.append_op(type='sum', inputs={'X': input}, outputs={'Out': out}) + return out + + +def assign(input, output, main_program=None, startup_program=None): + helper = LayerHelper('assign', **locals()) + helper.append_op( + type='scale', + inputs={'X': [input]}, + outputs={'Out': [output]}, + attrs={'scale': 1.0}) + return output + + +def fill_constant(shape, + dtype, + value, + out=None, + main_program=None, + startup_program=None): + """ + This function creates a tensor , with shape as mentioned in the input and + specified dtype and fills this up with a constant value that + comes in the input. It also sets the stop_gradient to be True. + """ + helper = LayerHelper("fill_constant", **locals()) + if out is None: + out = helper.create_tmp_variable(dtype=dtype) + helper.append_op( + type='fill_constant', + inputs={}, + outputs={'Out': [out]}, + attrs={'shape': shape, + 'dtype': out.dtype, + 'value': float(value)}) + out.stop_gradient = True + return out + + +def fill_constant_batch_size_like(input, + shape, + dtype, + value, + input_dim_idx=0, + output_dim_idx=0, + main_program=None, + startup_program=None): + helper = LayerHelper("fill_constant_batch_size_like", **locals()) + out = helper.create_tmp_variable(dtype=dtype) + helper.append_op( + type='fill_constant_batch_size_like', + inputs={'Input': input}, + outputs={'Out': [out]}, + attrs={ + 'shape': shape, + 'dtype': out.dtype, + 'value': float(value), + 'input_dim_idx': input_dim_idx, + 'output_dim_idx': output_dim_idx + }) + out.stop_gradient = True + return out + + +def ones(shape, dtype, main_program=None): + """ + This function performs the same function as fill_constant() declared above + with the constant value being 1.0. + """ + return fill_constant(value=1.0, **locals()) + + +def zeros(shape, dtype, main_program=None): + """ + This function performs the same function as fill_constant() declared above + with the constant value being 0.0. + """ + return fill_constant(value=0.0, **locals()) diff --git a/python/paddle/v2/fluid/tests/book/test_image_classification_train.py b/python/paddle/v2/fluid/tests/book/test_image_classification_train.py index 4e71b6f345..3d336ffe95 100644 --- a/python/paddle/v2/fluid/tests/book/test_image_classification_train.py +++ b/python/paddle/v2/fluid/tests/book/test_image_classification_train.py @@ -1,9 +1,9 @@ from __future__ import print_function -import numpy as np +import sys + import paddle.v2 as paddle import paddle.v2.fluid as fluid -import sys def resnet_cifar10(input, depth=32): diff --git a/python/paddle/v2/fluid/tests/book/test_understand_sentiment_lstm.py b/python/paddle/v2/fluid/tests/book/test_understand_sentiment_lstm.py index 80f8599679..c0b051f862 100644 --- a/python/paddle/v2/fluid/tests/book/test_understand_sentiment_lstm.py +++ b/python/paddle/v2/fluid/tests/book/test_understand_sentiment_lstm.py @@ -1,6 +1,51 @@ import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid +from paddle.v2.fluid.layer_helper import LayerHelper + + +def lstm(x, + c_pre_init, + hidden_dim, + forget_bias=None, + main_program=None, + startup_program=None): + """ + This function helps create an operator for the LSTM (Long Short Term + Memory) cell that can be used inside an RNN. + """ + helper = LayerHelper('lstm_unit', **locals()) + rnn = fluid.layers.StaticRNN() + with rnn.step(): + c_pre = rnn.memory(init=c_pre_init) + x_t = rnn.step_input(x) + + before_fc = fluid.layers.concat( + input=[x_t, c_pre], + axis=1, + main_program=main_program, + startup_program=startup_program) + after_fc = fluid.layers.fc(input=before_fc, + size=hidden_dim * 4, + main_program=main_program, + startup_program=startup_program) + + dtype = x.dtype + c = helper.create_tmp_variable(dtype) + h = helper.create_tmp_variable(dtype) + + helper.append_op( + type='lstm_unit', + inputs={"X": after_fc, + "C_prev": c_pre}, + outputs={"C": c, + "H": h}, + attrs={"forget_bias": forget_bias}) + + rnn.update_memory(c_pre, c) + rnn.output(h) + + return rnn() def lstm_net(dict_dim, class_dim=2, emb_dim=32, seq_len=80, batch_size=50): @@ -23,8 +68,7 @@ def lstm_net(dict_dim, class_dim=2, emb_dim=32, seq_len=80, batch_size=50): c_pre_init = fluid.layers.fill_constant( dtype=emb.dtype, shape=[batch_size, emb_dim], value=0.0) c_pre_init.stop_gradient = False - layer_1_out = fluid.layers.lstm( - emb, c_pre_init=c_pre_init, hidden_dim=emb_dim) + layer_1_out = lstm(emb, c_pre_init=c_pre_init, hidden_dim=emb_dim) layer_1_out = fluid.layers.transpose(x=layer_1_out, axis=[1, 0, 2]) prediction = fluid.layers.fc(input=layer_1_out, diff --git a/python/setup.py.in b/python/setup.py.in index 9ccb4dc176..8396fb44cf 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -68,6 +68,7 @@ packages=['paddle', 'paddle.v2.plot', 'paddle.v2.fluid', 'paddle.v2.fluid.proto', + 'paddle.v2.fluid.layers', 'py_paddle'] with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f: From a2dfabb46a5dc2ac11d57b91c3fe8ed749c17c82 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 14 Dec 2017 03:07:21 -0800 Subject: [PATCH 65/94] "fix based on comments" --- doc/design/paddle_nccl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/paddle_nccl.md b/doc/design/paddle_nccl.md index 5e28144b95..7f0d8e14e4 100644 --- a/doc/design/paddle_nccl.md +++ b/doc/design/paddle_nccl.md @@ -7,7 +7,7 @@ This Design Doc refers to the NCCL feature in paddle. We propose an approach t ## Motivation -NCCL is a NVIDIA library support Multi-GPU communicating and optimized for NVIDIA GPUs, it provides routines such as all-gather, all-reduce, broadcast, reduce, reduce-scatter, that can achieve high bandwidth over PCIe and NVLink high-speed interconnect. [NCCL](https://developer.nvidia.com/nccl). With NCCL library, we can easily accelerate the training in parallel. +[NCCL](https://developer.nvidia.com/nccl) is a NVIDIA library support Multi-GPU communicating and optimized for NVIDIA GPUs, it provides routines such as all-gather, all-reduce, broadcast, reduce, reduce-scatter, that can achieve high bandwidth over PCIe and NVLink high-speed interconnect. With NCCL library, we can easily accelerate the training in parallel. - Pros 1. easily plug-in with [NCCL2](https://developer.nvidia.com/nccl) library. From a02a68dc6daecbd3d5b17b57db03e0b3f916646e Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 14 Dec 2017 06:09:54 -0800 Subject: [PATCH 66/94] "fixed based on comment" --- doc/design/paddle_nccl.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/design/paddle_nccl.md b/doc/design/paddle_nccl.md index 7f0d8e14e4..c7dac70998 100644 --- a/doc/design/paddle_nccl.md +++ b/doc/design/paddle_nccl.md @@ -28,9 +28,9 @@ Besides, it needs interfaces to synchronize model update with each different GPU As mentioned above, we wrap the NCCL routines as several kinds of operators. Need to note that NCCL need to create Communicator between gpu at the beginning, so there is a NCCLInit operator created. -### Graph Converter +### Transpiler -To be compatible with [parameter server design doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/ops/dist_train.md), the graph converter converts the user defined operation graph into sub-graphs to be executed on different devices. +To be compatible with [parameter server design doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/ops/dist_train.md), the transpiler compiles the user defined operation graph into sub-graphs to be executed on different devices. 1. The user-defined model will be a single device program @@ -40,7 +40,7 @@ To be compatible with [parameter server design doc](https://github.com/PaddlePad -After convert, the graph as shows +After compiling, the graph as shows From e0c33176460ffe322c8ff6da024628741423d020 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 14 Dec 2017 23:14:18 +0800 Subject: [PATCH 67/94] add MKLDNNPlace --- paddle/platform/place.cc | 6 ++++++ paddle/platform/place.h | 19 ++++++++++++++++++- paddle/platform/place_test.cc | 6 ++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/paddle/platform/place.cc b/paddle/platform/place.cc index 856e54df89..16f1774285 100644 --- a/paddle/platform/place.cc +++ b/paddle/platform/place.cc @@ -23,6 +23,7 @@ class PlacePrinter : public boost::static_visitor<> { public: explicit PlacePrinter(std::ostream &os) : os_(os) {} void operator()(const CPUPlace &) { os_ << "CPUPlace"; } + void operator()(const MKLDNNPlace &) { os_ << "MKLDNNPlace"; } void operator()(const GPUPlace &p) { os_ << "GPUPlace(" << p.device << ")"; } private: @@ -38,6 +39,7 @@ const Place &get_place() { return the_default_place; } const GPUPlace default_gpu() { return GPUPlace(0); } const CPUPlace default_cpu() { return CPUPlace(); } +const MKLDNNPlace default_mkldnn() { return MKLDNNPlace(); } bool is_gpu_place(const Place &p) { return boost::apply_visitor(IsGPUPlace(), p); @@ -46,6 +48,10 @@ bool is_cpu_place(const Place &p) { return !boost::apply_visitor(IsGPUPlace(), p); } +bool is_mkldnn_place(const Place &p) { + return boost::apply_visitor(IsMKLDNNPlace(), p); +} + bool places_are_same_class(const Place &p1, const Place &p2) { return p1.which() == p2.which(); } diff --git a/paddle/platform/place.h b/paddle/platform/place.h index 5370360a7d..e745b2e839 100644 --- a/paddle/platform/place.h +++ b/paddle/platform/place.h @@ -31,6 +31,14 @@ struct CPUPlace { inline bool operator!=(const CPUPlace &) const { return false; } }; +struct MKLDNNPlace : public CPUPlace { + MKLDNNPlace() {} + + // needed for variant equality comparison + inline bool operator==(const MKLDNNPlace &) const { return true; } + inline bool operator!=(const MKLDNNPlace &) const { return false; } +}; + struct GPUPlace { GPUPlace() : GPUPlace(0) {} explicit GPUPlace(int d) : device(d) {} @@ -45,14 +53,21 @@ struct GPUPlace { struct IsGPUPlace : public boost::static_visitor { bool operator()(const CPUPlace &) const { return false; } + bool operator()(const MKLDNNPlace &) const { return false; } bool operator()(const GPUPlace &gpu) const { return true; } }; +struct IsMKLDNNPlace : public boost::static_visitor { + bool operator()(const MKLDNNPlace &) const { return true; } + bool operator()(const CPUPlace &) const { return false; } + bool operator()(const GPUPlace &) const { return false; } +}; + // Define the max number of Place in bit length. i.e., the max number of places // should be less equal than 2^(NUM_PLACE_TYPE_LIMIT_IN_BIT) #define NUM_PLACE_TYPE_LIMIT_IN_BIT 4 -typedef boost::variant Place; +typedef boost::variant Place; // static check number of place types is less equal than // 2^(NUM_PLACE_TYPE_LIMIT_IN_BIT) @@ -65,9 +80,11 @@ const Place &get_place(); const GPUPlace default_gpu(); const CPUPlace default_cpu(); +const MKLDNNPlace default_mkldnn(); bool is_gpu_place(const Place &); bool is_cpu_place(const Place &); +bool is_mkldnn_place(const Place &); bool places_are_same_class(const Place &, const Place &); std::ostream &operator<<(std::ostream &, const Place &); diff --git a/paddle/platform/place_test.cc b/paddle/platform/place_test.cc index 33e2e5a439..184af12c23 100644 --- a/paddle/platform/place_test.cc +++ b/paddle/platform/place_test.cc @@ -21,9 +21,15 @@ TEST(Place, Default) { EXPECT_TRUE(paddle::platform::is_gpu_place(paddle::platform::get_place())); EXPECT_TRUE(paddle::platform::is_gpu_place(paddle::platform::default_gpu())); EXPECT_TRUE(paddle::platform::is_cpu_place(paddle::platform::default_cpu())); + EXPECT_TRUE( + paddle::platform::is_mkldnn_place(paddle::platform::default_mkldnn())); paddle::platform::set_place(paddle::platform::CPUPlace()); EXPECT_TRUE(paddle::platform::is_cpu_place(paddle::platform::get_place())); + + paddle::platform::set_place(paddle::platform::MKLDNNPlace()); + EXPECT_FALSE(paddle::platform::is_cpu_place(paddle::platform::get_place())); + EXPECT_TRUE(paddle::platform::is_mkldnn_place(paddle::platform::get_place())); } TEST(Place, Print) { From f271210595953fee610dd049909d7e3a40f58285 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 15 Dec 2017 15:15:18 +0800 Subject: [PATCH 68/94] fix undefined issue when with_gpu --- paddle/operators/math/math_function.cc | 8 ++++++++ paddle/platform/place.h | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/paddle/operators/math/math_function.cc b/paddle/operators/math/math_function.cc index 2b35e4532a..a05810d778 100644 --- a/paddle/operators/math/math_function.cc +++ b/paddle/operators/math/math_function.cc @@ -277,6 +277,14 @@ void set_constant_with_place( TensorSetConstantCPU(tensor, value)); } +template <> +void set_constant_with_place( + const platform::DeviceContext& context, framework::Tensor* tensor, + float value) { + framework::VisitDataType(framework::ToDataType(tensor->type()), + TensorSetConstantCPU(tensor, value)); +} + struct TensorSetConstantWithPlace : public boost::static_visitor { TensorSetConstantWithPlace(const platform::DeviceContext& context, framework::Tensor* tensor, float value) diff --git a/paddle/platform/place.h b/paddle/platform/place.h index e745b2e839..70acc5b2d4 100644 --- a/paddle/platform/place.h +++ b/paddle/platform/place.h @@ -31,7 +31,7 @@ struct CPUPlace { inline bool operator!=(const CPUPlace &) const { return false; } }; -struct MKLDNNPlace : public CPUPlace { +struct MKLDNNPlace { MKLDNNPlace() {} // needed for variant equality comparison From a92f057ed1d52f8eed341212e11a76153512a1cf Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 15 Dec 2017 16:46:27 +0800 Subject: [PATCH 69/94] fix conflict of Place --- paddle/platform/place.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/platform/place.h b/paddle/platform/place.h index 5a1ce52800..4526945792 100644 --- a/paddle/platform/place.h +++ b/paddle/platform/place.h @@ -72,7 +72,7 @@ struct IsMKLDNNPlace : public boost::static_visitor { // should be less equal than 2^(NUM_PLACE_TYPE_LIMIT_IN_BIT) #define NUM_PLACE_TYPE_LIMIT_IN_BIT 4 -typedef boost::variant Place; +typedef boost::variant Place; // static check number of place types is less equal than // 2^(NUM_PLACE_TYPE_LIMIT_IN_BIT) From bf269d67b3d1e8d88797022e4ea542f34c06df47 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 15 Dec 2017 16:48:00 +0800 Subject: [PATCH 70/94] fix place_test on MKLDNNPlace --- paddle/platform/place.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/platform/place.cc b/paddle/platform/place.cc index 16f1774285..25fe8d21b4 100644 --- a/paddle/platform/place.cc +++ b/paddle/platform/place.cc @@ -45,7 +45,7 @@ bool is_gpu_place(const Place &p) { return boost::apply_visitor(IsGPUPlace(), p); } bool is_cpu_place(const Place &p) { - return !boost::apply_visitor(IsGPUPlace(), p); + return !is_gpu_place(p) && !is_mkldnn_place(p); } bool is_mkldnn_place(const Place &p) { From a243edf44bbfad597cc16de1e8c21943c909898a Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 14 Dec 2017 14:31:49 -0800 Subject: [PATCH 71/94] add cross compiling doc in en --- doc/mobile/cross_compiling_for_ios_en.md | 120 +++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 doc/mobile/cross_compiling_for_ios_en.md diff --git a/doc/mobile/cross_compiling_for_ios_en.md b/doc/mobile/cross_compiling_for_ios_en.md new file mode 100644 index 0000000000..5428f501e8 --- /dev/null +++ b/doc/mobile/cross_compiling_for_ios_en.md @@ -0,0 +1,120 @@ +# iOS Compiling Guide + +This tutorial will walk you through cross compiling the PaddlePaddle library for iOS from the source in MacOS. + +## Preparation + +Apple provides Xcode for cross-compiling and IDE for iOS development. Download from App store or [here](https://developer.apple.com/cn/xcode/). To verify your installation, run command as follows + +```bash +$ xcodebuild -version +Xcode 9.0 +Build version 9A235 +``` + +## Cross-compiling configurations + +PaddlePaddle provides cross-compiling toolchain configuration documentation [cmake/cross_compiling/ios.cmake](https://github.com/PaddlePaddle/Paddle/blob/develop/cmake/cross_compiling/ios.cmake), which has some default settings for frequently used compilers. + +There are some mandatory environment variables need to be set before cross compiling PaddlePaddle for iOS: + +- `CMAKE_SYSTEM_NAME`, CMake compiling target platform name, has to be `iOS`. PaddlePaddle CMake will compile all the third party dependencies and enforce some parameters (`WITH_C_API=ON`、`WITH_GPU=OFF`、`WITH_AVX=OFF`、`WITH_PYTHON=OFF`、`WITH_RDMA=OFF`) when this variable is set with value `iOS`. + +- `WITH_C_API`, Whether to compile inference C-API library, has to be `ON`, since C-API is the only supported interface for inferencing in iOS. +- `WITH_SWIG_PY`, has to be `ON`. It's not supported to inference or train via swig in iOS. + +Optional environment variables for iOS are: + +- `IOS_PLATFORM`, either `OS` (default) or `SIMULATOR`. + - `OS`, build targets ARM-based physical devices like iPhone or iPad. + - `SIMULATOR`, build targets x86 architecture simulators. +- `IOS_ARCH`, target architecture. By default, all architecture types will be compiled. If you need to specify the architecture to compile for, please find valid values for different `IOS_PLATFORM` settings from the table below: + + + + + + + + + + + + + + + + + + + + + + +
IOS_PLATFORMIOS_ARCH
OSarmv7, armv7s, arm64
SIMULATORi386, x86_64
+ +- `IOS_DEPLOYMENT_TARGET`, minimum iOS version to deployment, `7.0` by default. +- `IOS_ENABLE_BITCODE`, whether to enable [Bitcode](https://developer.apple.com/library/content/documentation/IDEs/Conceptual/AppDistributionGuide/AppThinning/AppThinning.html#//apple_ref/doc/uid/TP40012582-CH35-SW3), values can be `ON/OFF`, `ON` by default. +- `IOS_USE_VECLIB_FOR_BLAS`, whether to use [vecLib](https://developer.apple.com/documentation/accelerate/veclib) framework for BLAS computing. values can be `ON/OFF`, `OFF` by default. +- `IOS_DEVELOPMENT_ROOT`, the path to `Developer` directory, can be explicitly set with your `/path/to/platform/Developer`. If left blank, PaddlePaddle will automatically pick the Xcode corresponding `platform`'s `Developer` directory based on your `IOS_PLATFORM` value. +- `IOS_SDK_ROOT`, the path to `SDK` root, can be explicitly set with your `/path/to/platform/Developer/SDKs/SDK`. if left black, PaddlePaddle will pick the latest SDK in the directory of `IOS_DEVELOPMENT_ROOT`. + +other settings: + +- `USE_EIGEN_FOR_BLAS`, whether to use Eigen for matrix computing. effective when `IOS_USE_VECLIB_FOR_BLAS=OFF`. Values can be `ON/OFF`, `OFF` by default. +- `HOST_C/CXX_COMPILER`, host C/C++ compiler. Uses value from environment variable `CC/CXX` by default or `cc/c++` if `CC/CXX` doesn't exist. + +some typical cmake configurations: + +```bash +cmake -DCMAKE_SYSTEM_NAME=iOS \ + -DIOS_PLATFORM=OS \ + -DIOS_ARCH="armv7;arm64" \ + -DIOS_ENABLE_BITCODE=ON \ + -DIOS_USE_VECLIB_FOR_BLAS=ON \ + -DCMAKE_INSTALL_PREFIX=your/path/to/install \ + -DWITH_C_API=ON \ + -DWITH_TESTING=OFF \ + -DWITH_SWIG_PY=OFF \ + .. +``` + +```bash +cmake -DCMAKE_SYSTEM_NAME=iOS \ + -DIOS_PLATFORM=SIMULATOR \ + -DIOS_ARCH="x86_64" \ + -DIOS_USE_VECLIB_FOR_BLAS=ON \ + -DCMAKE_INSTALL_PREFIX=your/path/to/install \ + -DWITH_C_API=ON \ + -DWITH_TESTING=OFF \ + -DWITH_SWIG_PY=OFF \ + .. +``` + +You can set other compiling parameters for your own need. I.E. if you are trying to minimize the library size, set `CMAKE_BUILD_TYPE` with `MinSizeRel`; or if the performance is your concern, set `CMAKE_BUILD_TYPE` with `Release`. You can even manipulate the PaddlePaddle compiling procedure by manually set `CMAKE_C/CXX_FLAGS` values. + +**TIPS for a better performance**: + +- set `CMAKE_BUILD_TYPE` with `Release` +- set `IOS_USE_VECLIB_FOR_BLAS` with `ON` + +## Compile and install + +After CMake, run following commands, PaddlePaddle will download the compile 3rd party dependencies, compile and install PaddlePaddle inference library. + +``` +$ make +$ make install +``` + +Please Note: if you compiled PaddlePaddle in the source directory for other platforms, do remove `third_party` and `build` directory within the source with `rm -rf` to ensure that all the 3rd party libraries dependencies and PaddlePaddle is newly compiled with current CMake configuration. + +`your/path/to/install` directory will have following directories after `compile` and `install`: + +- `include`, contains all the C-API header files. +- `lib`, contains PaddlePaddle C-API static library. +- `third_party` contains all the 3rd party libraries. + +Please note: if PaddlePaddle library need to support both physical devices and simulators, you will need to compile correspondingly, then merge fat library with `lipo` + +Now you will have PaddlePaddle library compiled and installed, the fat library can be used in deep learning related iOS APPs. Please refer to C-API documentation for usage guides. From d2b6d2ef8a42feed15ac5ce4cd43877436f8abe1 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 14 Dec 2017 14:35:44 -0800 Subject: [PATCH 72/94] minor tweaks. --- doc/mobile/cross_compiling_for_ios_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/mobile/cross_compiling_for_ios_en.md b/doc/mobile/cross_compiling_for_ios_en.md index 5428f501e8..e2a1f6fc5f 100644 --- a/doc/mobile/cross_compiling_for_ios_en.md +++ b/doc/mobile/cross_compiling_for_ios_en.md @@ -1,4 +1,4 @@ -# iOS Compiling Guide +# PaddlePaddle Compiling Guide for iOS This tutorial will walk you through cross compiling the PaddlePaddle library for iOS from the source in MacOS. From f5dc9eadaafc74e606aef849145693ece2e743e9 Mon Sep 17 00:00:00 2001 From: zhushuang02 Date: Fri, 15 Dec 2017 10:22:59 +0800 Subject: [PATCH 73/94] Add refer code to get started --- doc/getstarted/concepts/src/infer.py | 14 ++++++++++++++ doc/getstarted/concepts/src/train.py | 5 +++++ doc/getstarted/concepts/use_concepts_cn.rst | 5 +++++ 3 files changed, 24 insertions(+) create mode 100644 doc/getstarted/concepts/src/infer.py diff --git a/doc/getstarted/concepts/src/infer.py b/doc/getstarted/concepts/src/infer.py new file mode 100644 index 0000000000..780d831da8 --- /dev/null +++ b/doc/getstarted/concepts/src/infer.py @@ -0,0 +1,14 @@ +import paddle.v2 as paddle +import numpy as np + +paddle.init(use_gpu=False) +x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(2)) +y_predict = paddle.layer.fc(input=x, size=1, act=paddle.activation.Linear()) + +# loading the model which generated by training +with open('hello_params_pass_90.tar', 'r') as f: + parameters = paddle.parameters.Parameters.from_tar(f) + +i = [[[1, 2]]] + +print paddle.infer(output_layer=y_predict, parameters=parameters, input=i) diff --git a/doc/getstarted/concepts/src/train.py b/doc/getstarted/concepts/src/train.py index 8aceb23406..4bccbfca3c 100644 --- a/doc/getstarted/concepts/src/train.py +++ b/doc/getstarted/concepts/src/train.py @@ -26,6 +26,11 @@ def event_handler(event): if event.batch_id % 1 == 0: print "Pass %d, Batch %d, Cost %f" % (event.pass_id, event.batch_id, event.cost) + # product model every 10 pass + if isinstance(event, paddle.event.EndPass): + if event.pass_id % 10 == 0: + with open('params_pass_%d.tar' % event.pass_id, 'w') as f: + trainer.save_parameter_to_tar(f) # define training dataset reader diff --git a/doc/getstarted/concepts/use_concepts_cn.rst b/doc/getstarted/concepts/use_concepts_cn.rst index c243083794..7ff0b29ac3 100644 --- a/doc/getstarted/concepts/use_concepts_cn.rst +++ b/doc/getstarted/concepts/use_concepts_cn.rst @@ -147,4 +147,9 @@ PaddlePaddle支持不同类型的输入数据,主要包括四种类型,和 .. literalinclude:: src/train.py :linenos: +使用以上训练好的模型进行预测的例子: + +.. literalinclude:: src/infer.py + :linenos: + 有关线性回归的实际应用,可以参考PaddlePaddle book的 `第一章节 `_。 From 7fe3c5789d4d65cd082550d2144108aabdfbda4b Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Fri, 15 Dec 2017 02:40:27 +0000 Subject: [PATCH 74/94] Fix typo in cross-compiling documentation of iOS and add it to the index. --- doc/mobile/cross_compiling_for_ios_cn.md | 4 ++-- doc/mobile/cross_compiling_for_ios_en.md | 8 ++++---- doc/mobile/index_en.rst | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/mobile/cross_compiling_for_ios_cn.md b/doc/mobile/cross_compiling_for_ios_cn.md index 9da48e7f21..d5196d9a4c 100644 --- a/doc/mobile/cross_compiling_for_ios_cn.md +++ b/doc/mobile/cross_compiling_for_ios_cn.md @@ -18,11 +18,11 @@ PaddlePaddle为交叉编译提供了工具链配置文档[cmake/cross_compiling/ - `CMAKE_SYSTEM_NAME`,CMake编译的目标平台,必须设置为`iOS`。在设置`CMAKE_SYSTEM_NAME=iOS`后,PaddlePaddle的CMake系统会自动编译所有的第三方依赖库,并且强制设置一些PaddlePaddle参数的值(`WITH_C_API=ON`、`WITH_GPU=OFF`、`WITH_AVX=OFF`、`WITH_PYTHON=OFF`、`WITH_RDMA=OFF`)。 - `WITH_C_API`,是否编译C-API预测库,必须设置为ON。在iOS平台上只支持使用C-API来预测。 -- `WITH_SWIG_PY`,必须设置为ON。在iOS平台上不支持通过swig调用来训练或者预测。 +- `WITH_SWIG_PY`,必须设置为`OFF`。在iOS平台上不支持通过swig调用来训练或者预测。 iOS平台可选配置参数: -- `IOS_PLATFORM`,可设置为`OS/SIMULATOR`,默认值为`OS`。 +- `IOS_PLATFORM`,可设置为`OS`(默认值)或`SIMULATOR`。 - `OS`,构建目标为`arm`架构的iPhone或者iPad等物理设备。 - `SIMULATOR`,构建目标为`x86`架构的模拟器平台。 - `IOS_ARCH`,目标架构。针对不同的`IOS_PLATFORM`,可设置的目标架构如下表所示,默认编译所有架构: diff --git a/doc/mobile/cross_compiling_for_ios_en.md b/doc/mobile/cross_compiling_for_ios_en.md index e2a1f6fc5f..aa390cd61f 100644 --- a/doc/mobile/cross_compiling_for_ios_en.md +++ b/doc/mobile/cross_compiling_for_ios_en.md @@ -18,12 +18,12 @@ PaddlePaddle provides cross-compiling toolchain configuration documentation [cma There are some mandatory environment variables need to be set before cross compiling PaddlePaddle for iOS: -- `CMAKE_SYSTEM_NAME`, CMake compiling target platform name, has to be `iOS`. PaddlePaddle CMake will compile all the third party dependencies and enforce some parameters (`WITH_C_API=ON`、`WITH_GPU=OFF`、`WITH_AVX=OFF`、`WITH_PYTHON=OFF`、`WITH_RDMA=OFF`) when this variable is set with value `iOS`. +- `CMAKE_SYSTEM_NAME`, CMake compiling target platform name, has to be `iOS`. PaddlePaddle CMake will compile all the third party dependencies and enforce some parameters (`WITH_C_API=ON`, `WITH_GPU=OFF`, `WITH_AVX=OFF`, `WITH_PYTHON=OFF`,`WITH_RDMA=OFF`) when this variable is set with value `iOS`. - `WITH_C_API`, Whether to compile inference C-API library, has to be `ON`, since C-API is the only supported interface for inferencing in iOS. -- `WITH_SWIG_PY`, has to be `ON`. It's not supported to inference or train via swig in iOS. +- `WITH_SWIG_PY`, has to be `OFF`. It's not supported to inference or train via swig in iOS. -Optional environment variables for iOS are: +Optional environment variables for iOS are: - `IOS_PLATFORM`, either `OS` (default) or `SIMULATOR`. - `OS`, build targets ARM-based physical devices like iPhone or iPad. @@ -115,6 +115,6 @@ Please Note: if you compiled PaddlePaddle in the source directory for other plat - `lib`, contains PaddlePaddle C-API static library. - `third_party` contains all the 3rd party libraries. -Please note: if PaddlePaddle library need to support both physical devices and simulators, you will need to compile correspondingly, then merge fat library with `lipo` +Please note: if PaddlePaddle library need to support both physical devices and simulators, you will need to compile correspondingly, then merge fat library with `lipo`. Now you will have PaddlePaddle library compiled and installed, the fat library can be used in deep learning related iOS APPs. Please refer to C-API documentation for usage guides. diff --git a/doc/mobile/index_en.rst b/doc/mobile/index_en.rst index 3c08d73671..ef421dacad 100644 --- a/doc/mobile/index_en.rst +++ b/doc/mobile/index_en.rst @@ -5,4 +5,5 @@ MOBILE :maxdepth: 1 cross_compiling_for_android_en.md + cross_compiling_for_ios_en.md cross_compiling_for_raspberry_en.md From d7b67f2b7460bd68d4db41c63dbc3c92d4b95497 Mon Sep 17 00:00:00 2001 From: Yancey Date: Fri, 15 Dec 2017 13:16:44 +0800 Subject: [PATCH 75/94] fix pipe_reader on multi passes (#6627) fix pipe reader on multi passes --- python/paddle/v2/reader/decorator.py | 4 +-- .../paddle/v2/reader/tests/decorator_test.py | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/reader/decorator.py b/python/paddle/v2/reader/decorator.py index 7e457f987d..27c82c95f7 100644 --- a/python/paddle/v2/reader/decorator.py +++ b/python/paddle/v2/reader/decorator.py @@ -390,8 +390,6 @@ def pipe_reader(left_cmd, if not callable(parser): raise TypeError("parser must be a callable object") - process = subprocess.Popen( - left_cmd.split(" "), bufsize=bufsize, stdout=subprocess.PIPE) # TODO(typhoonzero): add a thread to read stderr # Always init a decompress object is better than @@ -400,6 +398,8 @@ def pipe_reader(left_cmd, 32 + zlib.MAX_WBITS) # offset 32 to skip the header def reader(): + process = subprocess.Popen( + left_cmd.split(" "), bufsize=bufsize, stdout=subprocess.PIPE) remained = "" while True: buff = process.stdout.read(bufsize) diff --git a/python/paddle/v2/reader/tests/decorator_test.py b/python/paddle/v2/reader/tests/decorator_test.py index 5a92951b10..06e14796da 100644 --- a/python/paddle/v2/reader/tests/decorator_test.py +++ b/python/paddle/v2/reader/tests/decorator_test.py @@ -145,5 +145,35 @@ class TestXmap(unittest.TestCase): self.assertEqual(e, mapper(idx)) +class TestPipeReader(unittest.TestCase): + def test_pipe_reader(self): + def simple_parser(lines): + return lines + + import tempfile + + records = [str(i) for i in xrange(5)] + temp = tempfile.NamedTemporaryFile() + try: + with open(temp.name, 'w') as f: + for r in records: + f.write('%s\n' % r) + + cmd = "cat %s" % temp.name + reader = paddle.v2.reader.pipe_reader( + cmd, simple_parser, bufsize=128) + for i in xrange(4): + result = [] + for r in reader(): + result.append(r) + + for idx, e in enumerate(records): + print e, result[idx] + self.assertEqual(e, result[idx]) + finally: + # delete the temporary file + temp.close() + + if __name__ == '__main__': unittest.main() From 537a0d4c0e22dae88b7c917928bda2f976cb6700 Mon Sep 17 00:00:00 2001 From: Coke Date: Fri, 15 Dec 2017 13:47:11 +0800 Subject: [PATCH 76/94] Update infer.py --- doc/getstarted/concepts/src/infer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/getstarted/concepts/src/infer.py b/doc/getstarted/concepts/src/infer.py index 780d831da8..4cc58dfee0 100644 --- a/doc/getstarted/concepts/src/infer.py +++ b/doc/getstarted/concepts/src/infer.py @@ -6,9 +6,13 @@ x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(2)) y_predict = paddle.layer.fc(input=x, size=1, act=paddle.activation.Linear()) # loading the model which generated by training -with open('hello_params_pass_90.tar', 'r') as f: +with open('params_pass_90.tar', 'r') as f: parameters = paddle.parameters.Parameters.from_tar(f) -i = [[[1, 2]]] - +# Input multiple sets of data,Output the infer result in a array. +i = [[[1, 2]], [[3, 4]], [[5, 6]]] print paddle.infer(output_layer=y_predict, parameters=parameters, input=i) +# Will print: +# [[ -3.24491572] +# [ -6.94668722] +# [-10.64845848]] From d62552dc99583396ddd639fee58242ff9aae368f Mon Sep 17 00:00:00 2001 From: Coke Date: Fri, 15 Dec 2017 13:52:12 +0800 Subject: [PATCH 77/94] Update use_concepts_cn.rst --- doc/getstarted/concepts/use_concepts_cn.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/getstarted/concepts/use_concepts_cn.rst b/doc/getstarted/concepts/use_concepts_cn.rst index 7ff0b29ac3..e695ff283e 100644 --- a/doc/getstarted/concepts/use_concepts_cn.rst +++ b/doc/getstarted/concepts/use_concepts_cn.rst @@ -147,7 +147,7 @@ PaddlePaddle支持不同类型的输入数据,主要包括四种类型,和 .. literalinclude:: src/train.py :linenos: -使用以上训练好的模型进行预测的例子: +使用以上训练好的模型进行预测,取其中一个模型params_pass_90.tar,输入需要预测的向量组,然后打印输出: .. literalinclude:: src/infer.py :linenos: From d5cab4f07c3e63970642cd428a9a33b284ddf4f1 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 15 Dec 2017 14:22:30 +0800 Subject: [PATCH 78/94] Fix compile on CUDA9.1 & MacOS (#6642) --- paddle/math/float16.h | 4 ++-- paddle/platform/dynload/nccl.cc | 5 +++++ paddle/platform/dynload/nccl.h | 24 ++++++++++++------------ paddle/platform/nccl_test.cu | 4 ++-- paddle/platform/variant.h | 13 +++++++++++++ python/.gitignore | 1 + 6 files changed, 35 insertions(+), 16 deletions(-) diff --git a/paddle/math/float16.h b/paddle/math/float16.h index 76ad3a0123..efebbce504 100644 --- a/paddle/math/float16.h +++ b/paddle/math/float16.h @@ -79,7 +79,7 @@ public: #ifdef PADDLE_CUDA_FP16 HOSTDEVICE inline explicit float16(const half& h) { #if CUDA_VERSION >= 9000 - x = reinterpret_cast<__half_raw*>(&h)->x; + x = reinterpret_cast<__half_raw*>(const_cast(&h))->x; #else x = h.x; #endif // CUDA_VERSION >= 9000 @@ -145,7 +145,7 @@ public: #ifdef PADDLE_CUDA_FP16 HOSTDEVICE inline float16& operator=(const half& rhs) { #if CUDA_VERSION >= 9000 - x = reinterpret_cast<__half_raw*>(&rhs)->x; + x = reinterpret_cast<__half_raw*>(const_cast(&rhs))->x; #else x = rhs.x; #endif diff --git a/paddle/platform/dynload/nccl.cc b/paddle/platform/dynload/nccl.cc index 8f92b8d94d..91168f37ef 100644 --- a/paddle/platform/dynload/nccl.cc +++ b/paddle/platform/dynload/nccl.cc @@ -25,6 +25,11 @@ void *nccl_dso_handle; NCCL_RAND_ROUTINE_EACH(DEFINE_WRAP); +void LoadNCCLDSO() { + platform::call_once(nccl_dso_flag, + [] { GetNCCLDsoHandle(&nccl_dso_handle); }); +} + } // namespace dynload } // namespace platform } // namespace paddle diff --git a/paddle/platform/dynload/nccl.h b/paddle/platform/dynload/nccl.h index 981b2ab258..11007c1031 100644 --- a/paddle/platform/dynload/nccl.h +++ b/paddle/platform/dynload/nccl.h @@ -28,18 +28,18 @@ extern std::once_flag nccl_dso_flag; extern void* nccl_dso_handle; #ifdef PADDLE_USE_DSO -#define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - auto operator()(Args... args) -> decltype(__name(args...)) { \ - using nccl_func = decltype(__name(args...)) (*)(Args...); \ - platform::call_once(nccl_dso_flag, \ - paddle::platform::dynload::GetNCCLDsoHandle, \ - &nccl_dso_handle); \ - void* p_##__name = dlsym(nccl_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ - } \ - }; \ +extern void LoadNCCLDSO(); + +#define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + using nccl_func = decltype(__name(args...)) (*)(Args...); \ + paddle::platform::dynload::LoadNCCLDSO(); \ + void* p_##__name = dlsym(nccl_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + }; \ extern DynLoad__##__name __name #else #define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ diff --git a/paddle/platform/nccl_test.cu b/paddle/platform/nccl_test.cu index c99dae68be..94ab360a19 100644 --- a/paddle/platform/nccl_test.cu +++ b/paddle/platform/nccl_test.cu @@ -31,7 +31,7 @@ namespace platform { TEST(NCCL, init) { std::vector comms; comms.resize(dev_count); - PADDLE_ENFORCE(dynload::ncclCommInitAll(comms.data(), dev_count, nullptr)); + dynload::ncclCommInitAll(comms.data(), dev_count, nullptr); for (int i = 0; i < dev_count; ++i) { dynload::ncclCommDestroy(comms[i]); } @@ -62,7 +62,7 @@ TEST(NCCL, all_reduce) { std::vector comms; comms.resize(dev_count); VLOG(1) << "Initializing ncclComm"; - PADDLE_ENFORCE(dynload::ncclCommInitAll(comms.data(), dev_count, nullptr)); + dynload::ncclCommInitAll(comms.data(), dev_count, nullptr); VLOG(1) << "ncclComm initialized"; VLOG(1) << "Creating thread data"; std::vector>> data; diff --git a/paddle/platform/variant.h b/paddle/platform/variant.h index 619897ca19..284b4c42ac 100644 --- a/paddle/platform/variant.h +++ b/paddle/platform/variant.h @@ -14,6 +14,19 @@ #pragma once +#ifdef __CUDACC__ +#ifdef __CUDACC_VER_MAJOR__ +// CUDA 9 define `__CUDACC_VER__` as a warning message, manually define +// __CUDACC_VER__ instead. +#undef __CUDACC_VER__ + +#define __CUDACC_VER__ \ + (__CUDACC_VER_MAJOR__ * 10000 + __CUDACC_VER_MINOR__ * 100 + \ + __CUDACC_VER_BUILD__) +#endif + +#endif + #include #ifdef PADDLE_WITH_CUDA diff --git a/python/.gitignore b/python/.gitignore index cc7d0ece4a..1ba1d4c9b0 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -2,6 +2,7 @@ build dist paddle.egg-info +paddlepaddle_gpu.egg-info .idea paddle/proto/*.py paddle/proto/*.pyc From d37ed6cb13fc63aa2c81ece8de005bfbea4916be Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 15 Dec 2017 07:41:54 +0000 Subject: [PATCH 79/94] polish code in reshape_op --- paddle/operators/reshape_op.cc | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc index 5baf459536..d82d828747 100644 --- a/paddle/operators/reshape_op.cc +++ b/paddle/operators/reshape_op.cc @@ -36,10 +36,13 @@ class ReshapeOp : public framework::OperatorWithKernel { auto x_dims = ctx->GetInputDim("X"); std::vector neg_dims_idx; + // set some dimension to -1 if it is unknown + const int unknown_size = -1; for (size_t i = 0; i < shape.size(); ++i) { - PADDLE_ENFORCE(shape[i] > 0 || shape[i] == -1, - "Each dimension of Attr(shape) must be positive or -1."); - if (shape[i] == -1) { + PADDLE_ENFORCE(shape[i] > 0 || shape[i] == unknown_size, + "Each dimension of Attr(shape) must be positive or %d.", + unknown_size); + if (shape[i] == unknown_size) { neg_dims_idx.push_back(i); PADDLE_ENFORCE(neg_dims_idx.size() <= 1, "Only one dimension of Attr(shape) can be unknown."); @@ -53,8 +56,7 @@ class ReshapeOp : public framework::OperatorWithKernel { // dim infer shape[neg_dims_idx[0]] = in_size / (-capacity); // recalculate capacity - capacity = std::accumulate(shape.begin(), shape.end(), 1, - std::multiplies()); + capacity = shape[neg_dims_idx[0]] * (-capacity); } // capacity check PADDLE_ENFORCE(capacity == in_size, @@ -98,9 +100,9 @@ the tensor X into a 2-D tensor: [[1, 2, 3, 4]] -One dimension in the target shape can be set -1, and the real dimension -will be infered from the original shape of Input(X) and other -dimensions in the target shape. +One dimension in the target shape can be set -1, representing that its +size is unknown. In this case, the real dimension will be infered from +the original shape of Input(X) and other dimensions in the target shape. )DOC"); } }; From acaef9a1302a7907f9d42dffb7a75616b483b658 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 14 Dec 2017 20:58:10 +0800 Subject: [PATCH 80/94] rename mkldnn doc --- doc/design/{mkldnn => mkl}/image/engine.png | Bin doc/design/{mkldnn => mkl}/image/gradients.png | Bin doc/design/{mkldnn => mkl}/image/layers.png | Bin doc/design/{mkldnn => mkl}/image/matrix.png | Bin doc/design/{mkldnn => mkl}/image/overview.png | Bin doc/design/{mkldnn/README.MD => mkl/mkldnn.md} | 1 - 6 files changed, 1 deletion(-) rename doc/design/{mkldnn => mkl}/image/engine.png (100%) rename doc/design/{mkldnn => mkl}/image/gradients.png (100%) rename doc/design/{mkldnn => mkl}/image/layers.png (100%) rename doc/design/{mkldnn => mkl}/image/matrix.png (100%) rename doc/design/{mkldnn => mkl}/image/overview.png (100%) rename doc/design/{mkldnn/README.MD => mkl/mkldnn.md} (99%) diff --git a/doc/design/mkldnn/image/engine.png b/doc/design/mkl/image/engine.png similarity index 100% rename from doc/design/mkldnn/image/engine.png rename to doc/design/mkl/image/engine.png diff --git a/doc/design/mkldnn/image/gradients.png b/doc/design/mkl/image/gradients.png similarity index 100% rename from doc/design/mkldnn/image/gradients.png rename to doc/design/mkl/image/gradients.png diff --git a/doc/design/mkldnn/image/layers.png b/doc/design/mkl/image/layers.png similarity index 100% rename from doc/design/mkldnn/image/layers.png rename to doc/design/mkl/image/layers.png diff --git a/doc/design/mkldnn/image/matrix.png b/doc/design/mkl/image/matrix.png similarity index 100% rename from doc/design/mkldnn/image/matrix.png rename to doc/design/mkl/image/matrix.png diff --git a/doc/design/mkldnn/image/overview.png b/doc/design/mkl/image/overview.png similarity index 100% rename from doc/design/mkldnn/image/overview.png rename to doc/design/mkl/image/overview.png diff --git a/doc/design/mkldnn/README.MD b/doc/design/mkl/mkldnn.md similarity index 99% rename from doc/design/mkldnn/README.MD rename to doc/design/mkl/mkldnn.md index 61d453de24..e2fe1e6b26 100644 --- a/doc/design/mkldnn/README.MD +++ b/doc/design/mkl/mkldnn.md @@ -208,4 +208,3 @@ if use_mkldnn 但是在PaddlePaddle中,无论是重构前的layer还是重构后的op,都不会想要知道next layer/op的信息。 4. MKL-DNN的高性能格式与PaddlePaddle原有的`NCHW`不同(PaddlePaddle中的cuDNN部分使用的也是`NCHW`,所以不存在这个问题)。 所以需要引入一个转换方法,并且只需要在必要的时候转换这种格式,才能更好的发挥MKL-DNN的性能。 - From 1670ec081bd35c36e63298de2b31f0b88007888b Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 14 Dec 2017 21:03:13 +0800 Subject: [PATCH 81/94] add mkl packed desgin doc --- doc/design/mkl/mkl_packed.md | 91 ++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 doc/design/mkl/mkl_packed.md diff --git a/doc/design/mkl/mkl_packed.md b/doc/design/mkl/mkl_packed.md new file mode 100644 index 0000000000..a9fb56d709 --- /dev/null +++ b/doc/design/mkl/mkl_packed.md @@ -0,0 +1,91 @@ +# Intel® MKL Packed Optimization on PaddlePaddle: Design Doc + + +## Contents + +- [Overview](#overview) +- [Key Points](#key-points) + - [Background](#background) + - [Solution](#solution) +- [Actions](#actions) + - [CMake](#cmake) + - [Layers](#layers) + - [Unit Tests](#unit-tests) + - [Python API](#python-api) + - [Benchmarking](#benchmarking) + + +## Overview +我们计划将 Intel® MKL 中引入的 GEMM Packed APIs\[[1](#references)\] 集成到 PaddlePaddle 中,充分发挥英特尔平台的优势,有效提升PaddlePaddle在英特尔架构上的性能。 +现阶段的优化主要针对 Recurrent Neural Network(以下简称RNN)相关层(包括`RecurrentLayer`, `GatedRecurrentLayer`和`LstmLayer`), 以及 PaddlePaddle V1 API。 + +## Key Points + +### Background +为了达到最佳性能, Intel® MKL 中的 cblas_?gemm 会在计算前将原数据转换为更适合英特尔平台的Packed格式, 这一数据格式的转换操作 (Packing),在问题本身的计算量比较小的时候显得相对来说较为耗时。 +在现有的某些情况下(例如RNN),多次调用 cblas_?gemm 时会使用相同的原数据,每次调用时对原数据的重复Packing便成为了冗余。 + +为了最大程度减少多次调用 cblas_?gemm 在Packing上的耗时,Intel® MKL 引入了以下四个API: + * cblas_?gemm_alloc + * cblas_?gemm_pack + * cblas_?gemm_compute + * cblas_?gemm_free + +通过使用这些API,我们可以先完成对原数据的Packing操作,再把已转换为Packed格式的数据传递给那些复用同一数据的gemm_compute函数,从而避免了Packing冗余。 + +### Solution +在RNN的case下,同一次 forward/backward 过程中所有time state共享同一个weight矩阵。当只做 inference 时,各次 forward 之间也都使用相同的weight矩阵,没有必要在每次forward中每个time state的计算时对weight进行重复的packing操作。 + +我们通过使用新引入的GEMM Packed APIs,在layer init时先完成对weight的packing操作,然后在 forward/backward 时复用已pack过后的weight,并在每次weight更新后重新Packing。 + +* 优化前,对于sequence length = `T` 的model, `N` 次iteration执行的Packing次数为: + - `inference`: `N * T` + - `training`: `2 * N * T` +* 优化后,对于sequence length = `T` 的model, `N` 次iteration执行的Packing次数减少至: + - `inference`: `1` + - `training`: `2 * N` + +## Actions + +添加的相关文件和目录结构如下: + +```txt +PaddlePaddle/Paddle +├── ... +└── paddle/ + ├── ... + └── gserver/ + ├── ... + ├── layers/ + │ ├── ... + │ ├── MKLPackedRecurrentLayer.* + | ├── MKLPackedGatedRecurrentLayer.* + | ├── MKLPackedLstmLayer.* + | └── MKLPackedGemm.h + └── tests/ + ├── ... + └── test_MKLPacked.cpp +``` + +### CMake +在对应的`CMakeLists.txt`中根据`WITH_MKL`是否打开,来决定是否开启MKL Packed相关功能。 + +### Layers +所有的`MKLPacked*Layer`都继承于PaddlePaddle的基类`Layer`, 并添加头文件 `MKLPackedGemm.h`,该文件中实现的对相关GEMM Packed APIs做了封装。 + +### Unit Tests +我们会添加`test_MKLPacked.cpp`用于MKL Packed优化后layer的测试。 +对于每一个新加的RNN layer,我们会对比如下2个方面: +1. 对比优化后layer自身,sequence mode(`rnn_use_batch=false`)与batch mode(`rnn_use_batch=true`)的结果。 +2. 对比优化后layer与相对应的PaddlePaddle原有layer, 在batch mode下的结果。 + +### Python API +TBD + +### Benchmarking +会添加相应的脚本用于测试和对比在使用MKL Packed recurrent layers 前后的网络性能。 + +## References +1. [Introducing the new Packed APIs for GEMM](https://software.intel.com/en-us/articles/introducing-the-new-packed-apis-for-gemm) + + From 3f2fa0a1f65f6ca7d6881a4bac93d190b2032b78 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Sat, 16 Dec 2017 13:50:37 +0800 Subject: [PATCH 82/94] follow comments and refine doc --- doc/design/mkl/mkl_packed.md | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/doc/design/mkl/mkl_packed.md b/doc/design/mkl/mkl_packed.md index a9fb56d709..55d13d777c 100644 --- a/doc/design/mkl/mkl_packed.md +++ b/doc/design/mkl/mkl_packed.md @@ -1,4 +1,4 @@ -# Intel® MKL Packed Optimization on PaddlePaddle: Design Doc +# Intel® MKL Packed on PaddlePaddle: Design Doc ## Contents @@ -17,13 +17,17 @@ ## Overview 我们计划将 Intel® MKL 中引入的 GEMM Packed APIs\[[1](#references)\] 集成到 PaddlePaddle 中,充分发挥英特尔平台的优势,有效提升PaddlePaddle在英特尔架构上的性能。 -现阶段的优化主要针对 Recurrent Neural Network(以下简称RNN)相关层(包括`RecurrentLayer`, `GatedRecurrentLayer`和`LstmLayer`), 以及 PaddlePaddle V1 API。 +现阶段的优化主要针对 Recurrent Neural Network(以下简称RNN)相关层(包括`RecurrentLayer`, `GatedRecurrentLayer`和`LstmLayer`), 以及 PaddlePaddle V1 API。 ## Key Points ### Background -为了达到最佳性能, Intel® MKL 中的 cblas_?gemm 会在计算前将原数据转换为更适合英特尔平台的Packed格式, 这一数据格式的转换操作 (Packing),在问题本身的计算量比较小的时候显得相对来说较为耗时。 -在现有的某些情况下(例如RNN),多次调用 cblas_?gemm 时会使用相同的原数据,每次调用时对原数据的重复Packing便成为了冗余。 +目前PaddlePaddle采用了 Intel® MKL库的cblas_?gemm函数,这个函数本身会在计算前将原数据转换为更适合英特尔平台的内部格式。 + +1. 转换耗时 \ +这一数据格式的转换操作(Packing),在问题本身的计算量比较小的时候,显得相对来说较为耗时。例如在DeepSpeech2 \[[2](#references)\] 的Vanilla RNN部分中,矩阵大小是`batch_size * 2048`。 +2. 转换冗余 \ +由于在现有的某些情况下(例如RNN),多次调用 cblas_?gemm 会使用相同的原数据,因此,每次调用时对原数据的重复Packing便成为了冗余。 为了最大程度减少多次调用 cblas_?gemm 在Packing上的耗时,Intel® MKL 引入了以下四个API: * cblas_?gemm_alloc @@ -34,16 +38,16 @@ 通过使用这些API,我们可以先完成对原数据的Packing操作,再把已转换为Packed格式的数据传递给那些复用同一数据的gemm_compute函数,从而避免了Packing冗余。 ### Solution -在RNN的case下,同一次 forward/backward 过程中所有time state共享同一个weight矩阵。当只做 inference 时,各次 forward 之间也都使用相同的weight矩阵,没有必要在每次forward中每个time state的计算时对weight进行重复的packing操作。 +在RNN的情况下,同一次**前向/后向**(forward/backward)过程中所有**时间步**(time step)共享同一个**权重**(weight)。当只做**预测**(inference)时,各次**前向**之间也都使用了相同的**权重**,没有必要在每次**前向**中每个**时间步**的计算时对**权重**进行重复的Packing操作。 -我们通过使用新引入的GEMM Packed APIs,在layer init时先完成对weight的packing操作,然后在 forward/backward 时复用已pack过后的weight,并在每次weight更新后重新Packing。 +我们通过使用新引入的GEMM Packed APIs,在层**初始化**的时时候,先完成对**权重**的Packing操作,然后在**前向/后向**时复用已经转换过的**权重**,并在每次**权重**更新后,对新的**权重**进行转换用于下次迭代。 -* 优化前,对于sequence length = `T` 的model, `N` 次iteration执行的Packing次数为: - - `inference`: `N * T` - - `training`: `2 * N * T` -* 优化后,对于sequence length = `T` 的model, `N` 次iteration执行的Packing次数减少至: - - `inference`: `1` - - `training`: `2 * N` +* 优化前,对于序列长度(sequence length)为`T`的网络模型(model), `N`次迭代执行的转换次数为: + - `inference`: `N * T` + - `training`: `2 * N * T` +* 优化后,对于同样设置的网络模型,其转换次数减少至: + - `inference`: `1` + - `training`: `2 * N` ## Actions @@ -71,7 +75,7 @@ PaddlePaddle/Paddle 在对应的`CMakeLists.txt`中根据`WITH_MKL`是否打开,来决定是否开启MKL Packed相关功能。 ### Layers -所有的`MKLPacked*Layer`都继承于PaddlePaddle的基类`Layer`, 并添加头文件 `MKLPackedGemm.h`,该文件中实现的对相关GEMM Packed APIs做了封装。 +所有的`MKLPacked*Layer`都继承于PaddlePaddle的基类`Layer`, 并添加头文件 `MKLPackedGemm.h`,该文件对相关GEMM Packed APIs做了封装。 ### Unit Tests 我们会添加`test_MKLPacked.cpp`用于MKL Packed优化后layer的测试。 @@ -87,5 +91,5 @@ TBD ## References 1. [Introducing the new Packed APIs for GEMM](https://software.intel.com/en-us/articles/introducing-the-new-packed-apis-for-gemm) - +2. [DeepSpeech2 on PaddlePaddle](https://github.com/PaddlePaddle/DeepSpeech#deepspeech2-on-paddlepaddle) From c1a6870706bf593e6f8e394d5cbcbf20f9fd1f23 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Sat, 16 Dec 2017 15:06:17 +0800 Subject: [PATCH 83/94] follow comments and refine doc --- doc/design/mkl/mkl_packed.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/design/mkl/mkl_packed.md b/doc/design/mkl/mkl_packed.md index 55d13d777c..c07f7d0cbe 100644 --- a/doc/design/mkl/mkl_packed.md +++ b/doc/design/mkl/mkl_packed.md @@ -22,7 +22,7 @@ ## Key Points ### Background -目前PaddlePaddle采用了 Intel® MKL库的cblas_?gemm函数,这个函数本身会在计算前将原数据转换为更适合英特尔平台的内部格式。 +目前PaddlePaddle采用了 Intel® MKL库的[cblas_?gemm](https://software.intel.com/en-us/mkl-developer-reference-c-cblas-gemm)函数,这个函数本身会在计算前将原数据转换为更适合英特尔平台的内部格式。 1. 转换耗时 \ 这一数据格式的转换操作(Packing),在问题本身的计算量比较小的时候,显得相对来说较为耗时。例如在DeepSpeech2 \[[2](#references)\] 的Vanilla RNN部分中,矩阵大小是`batch_size * 2048`。 @@ -38,9 +38,9 @@ 通过使用这些API,我们可以先完成对原数据的Packing操作,再把已转换为Packed格式的数据传递给那些复用同一数据的gemm_compute函数,从而避免了Packing冗余。 ### Solution -在RNN的情况下,同一次**前向/后向**(forward/backward)过程中所有**时间步**(time step)共享同一个**权重**(weight)。当只做**预测**(inference)时,各次**前向**之间也都使用了相同的**权重**,没有必要在每次**前向**中每个**时间步**的计算时对**权重**进行重复的Packing操作。 +在RNN的情况下,同一次前向、后向(forward/backward)过程中所有时间步(time step)共享同一个权重(weight)。当只做推断(inference)时,各次前向之间也都使用了相同的权重,没有必要在每次前向中每个时间步的计算时对权重进行重复的Packing操作。 -我们通过使用新引入的GEMM Packed APIs,在层**初始化**的时时候,先完成对**权重**的Packing操作,然后在**前向/后向**时复用已经转换过的**权重**,并在每次**权重**更新后,对新的**权重**进行转换用于下次迭代。 +我们通过使用新引入的GEMM Packed APIs,在层初始化的时候,先完成对权重的Packing操作,然后在前向,后向时复用已经转换过的权重,并在每次权重更新后,对新的权重进行转换用于下次迭代。 * 优化前,对于序列长度(sequence length)为`T`的网络模型(model), `N`次迭代执行的转换次数为: - `inference`: `N * T` From 0fce0fe6983f4f167b873465fc90cff08fc31bd9 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Fri, 15 Dec 2017 15:48:58 +0800 Subject: [PATCH 84/94] Reduce memory usage in conv layer and RoI layer for mobile inference. --- paddle/function/GemmConvOp.cpp | 5 +++++ paddle/gserver/layers/ROIPoolLayer.cpp | 27 +++++++++++++++++--------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/paddle/function/GemmConvOp.cpp b/paddle/function/GemmConvOp.cpp index 8d34eee886..ffbf366fa9 100644 --- a/paddle/function/GemmConvOp.cpp +++ b/paddle/function/GemmConvOp.cpp @@ -233,6 +233,11 @@ public: inputGrad += inputChannels * inputHeight * inputWidth; outputGrad += outputChannels * outputHeight * outputWidth; } +#ifdef PADDLE_MOBILE_INFERENCE + if (Device == DEVICE_TYPE_CPU) { + delete memory_; + } +#endif } }; diff --git a/paddle/gserver/layers/ROIPoolLayer.cpp b/paddle/gserver/layers/ROIPoolLayer.cpp index 2c8256b91c..7d7c30b4d8 100644 --- a/paddle/gserver/layers/ROIPoolLayer.cpp +++ b/paddle/gserver/layers/ROIPoolLayer.cpp @@ -84,12 +84,15 @@ void ROIPoolLayer::forward(PassType passType) { size_t poolChannelOffset = pooledHeight_ * pooledWidth_; real* outputData = outputValue->getData(); - Matrix::resizeOrCreate(maxIdxs_, - numROIs, - channels_ * pooledHeight_ * pooledWidth_, - false, - false); - real* argmaxData = maxIdxs_->getData(); + real* argmaxData = nullptr; + if (passType != PASS_TEST) { + Matrix::resizeOrCreate(maxIdxs_, + numROIs, + channels_ * pooledHeight_ * pooledWidth_, + false, + false); + argmaxData = maxIdxs_->getData(); + } for (size_t n = 0; n < numROIs; ++n) { // the first five elememts of each RoI should be: @@ -128,14 +131,18 @@ void ROIPoolLayer::forward(PassType passType) { bool isEmpty = (hend <= hstart) || (wend <= wstart); size_t poolIndex = ph * pooledWidth_ + pw; outputData[poolIndex] = isEmpty ? 0 : -FLT_MAX; - argmaxData[poolIndex] = -1; + if (argmaxData) { + argmaxData[poolIndex] = -1; + } for (size_t h = hstart; h < hend; ++h) { for (size_t w = wstart; w < wend; ++w) { size_t index = h * width_ + w; if (batchData[index] > outputData[poolIndex]) { outputData[poolIndex] = batchData[index]; - argmaxData[poolIndex] = index; + if (argmaxData) { + argmaxData[poolIndex] = index; + } } } } @@ -143,7 +150,9 @@ void ROIPoolLayer::forward(PassType passType) { } batchData += channelOffset; outputData += poolChannelOffset; - argmaxData += poolChannelOffset; + if (argmaxData) { + argmaxData += poolChannelOffset; + } } bottomROIs += roiOffset; } From 349609207e0d6b6a2ff1548b8d0773c7857989f0 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Fri, 15 Dec 2017 16:22:19 +0800 Subject: [PATCH 85/94] Fix the error function/GemmConvOp. --- paddle/function/GemmConvOp.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/paddle/function/GemmConvOp.cpp b/paddle/function/GemmConvOp.cpp index ffbf366fa9..3387cae396 100644 --- a/paddle/function/GemmConvOp.cpp +++ b/paddle/function/GemmConvOp.cpp @@ -126,6 +126,11 @@ public: inputData += inputChannels * inputHeight * inputWidth; outputData += outputChannels * outputHeight * outputWidth; } +#ifdef PADDLE_MOBILE_INFERENCE + if (Device == DEVICE_TYPE_CPU) { + delete memory_; + } +#endif } }; @@ -233,11 +238,6 @@ public: inputGrad += inputChannels * inputHeight * inputWidth; outputGrad += outputChannels * outputHeight * outputWidth; } -#ifdef PADDLE_MOBILE_INFERENCE - if (Device == DEVICE_TYPE_CPU) { - delete memory_; - } -#endif } }; From 1b0c7d7c7afd4c5f09eb43358dd4851ba1735c3f Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 15 Dec 2017 16:24:27 +0800 Subject: [PATCH 86/94] Simplize system_allocator and fix GPU_INFO (#6653) --- paddle/memory/detail/system_allocator.cc | 50 ++++++------------------ paddle/platform/gpu_info.cc | 19 ++++----- 2 files changed, 23 insertions(+), 46 deletions(-) diff --git a/paddle/memory/detail/system_allocator.cc b/paddle/memory/detail/system_allocator.cc index 6a815a1b57..509250debc 100644 --- a/paddle/memory/detail/system_allocator.cc +++ b/paddle/memory/detail/system_allocator.cc @@ -19,6 +19,7 @@ limitations under the License. */ #include // for malloc and free #include // for mlock and munlock +#include // for std::max #include "gflags/gflags.h" @@ -28,7 +29,7 @@ limitations under the License. */ // of memory available to the system for paging. So, by default, we // should set false to use_pinned_memory. DEFINE_bool(use_pinned_memory, true, "If set, allocate cpu pinned memory."); - +DECLARE_double(fraction_of_gpu_memory_to_use); namespace paddle { namespace memory { namespace detail { @@ -77,45 +78,20 @@ void* GPUAllocator::Alloc(size_t& index, size_t size) { // CUDA documentation doesn't explain if cudaMalloc returns nullptr // if size is 0. We just make sure it does. if (size <= 0) return nullptr; - - size_t available = 0; - size_t capacity = 0; - paddle::platform::GpuMemoryUsage(available, capacity); - - // Reserve memory for page tables, etc. - size_t reserving = 0.05 * capacity + paddle::platform::GpuMinChunkSize(); - size_t usable = available > reserving ? available - reserving : 0; - - // If remaining size no less than expected size, using general - // cudaMalloc to allocate GPU memory. - void* p = 0; - if (size <= usable) { - cudaError_t result = cudaMalloc(&p, size); - if (result == cudaSuccess) { - index = 0; - gpu_alloc_size_ += size; - return p; - } - } - - // If remaining size less than expected size or cudaMalloc failed, - // cudaMallocHost will be considered as a fallback allocator. - // - // NOTE: here, we use GpuMaxAllocSize() as the maximum memory size - // of host fallback allocation. Allocates too much would reduce - // the amount of memory available to the underlying system for paging. - usable = paddle::platform::GpuMaxAllocSize() - fallback_alloc_size_; - - if (size > usable) return nullptr; - - cudaError_t result = cudaMallocHost(&p, size); + void* p; + cudaError_t result = cudaMalloc(&p, size); if (result == cudaSuccess) { - index = 1; - fallback_alloc_size_ += size; + index = 0; + gpu_alloc_size_ += size; return p; + } else { + LOG(WARNING) + << "Cannot malloc " << size / 1024.0 / 1024.0 + << " MB GPU memory. Please shrink FLAGS_fraction_of_gpu_memory_to_use " + "environment variable to a lower value. Current value is " + << FLAGS_fraction_of_gpu_memory_to_use; + return nullptr; } - - return nullptr; } void GPUAllocator::Free(void* p, size_t size, size_t index) { diff --git a/paddle/platform/gpu_info.cc b/paddle/platform/gpu_info.cc index 4fa2eaed31..541eca5f39 100644 --- a/paddle/platform/gpu_info.cc +++ b/paddle/platform/gpu_info.cc @@ -73,19 +73,20 @@ size_t GpuMaxChunkSize() { size_t available = 0; GpuMemoryUsage(available, total); - - // Reserving the rest memory for page tables, etc. - size_t reserving = 0.05 * total; - + VLOG(10) << "GPU Usage " << available / 1024 / 1024 << "M/" + << total / 1024 / 1024 << "M"; + size_t reserving = static_cast(0.05 * total); // If available less than minimum chunk size, no usable memory exists. available = - std::max(std::max(available, GpuMinChunkSize()) - GpuMinChunkSize(), - reserving) - - reserving; + std::min(std::max(available, GpuMinChunkSize()) - GpuMinChunkSize(), + total - reserving); + + // Reserving the rest memory for page tables, etc. - size_t allocating = FLAGS_fraction_of_gpu_memory_to_use * total; + size_t allocating = static_cast(FLAGS_fraction_of_gpu_memory_to_use * + (total - reserving)); - PADDLE_ENFORCE_LT(allocating, available); + PADDLE_ENFORCE_LE(allocating, available); return allocating; } From e1049c478697329b504ac682125210112fc13295 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 15 Dec 2017 17:18:28 +0800 Subject: [PATCH 87/94] add fluid python api in chinese website --- doc/api/index_cn.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/api/index_cn.rst b/doc/api/index_cn.rst index 9be0b370ee..84f9097a6c 100644 --- a/doc/api/index_cn.rst +++ b/doc/api/index_cn.rst @@ -7,3 +7,4 @@ API 模型配置 数据访问 训练与应用 + v2/fluid.rst From 84cb542c13905f5113a6e91d86129ec7a1a90e06 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 15 Dec 2017 17:23:03 +0800 Subject: [PATCH 88/94] use intel openmp to speedup seq2batch when WITH_MKL --- paddle/gserver/layers/SequenceToBatch.cpp | 25 +++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/paddle/gserver/layers/SequenceToBatch.cpp b/paddle/gserver/layers/SequenceToBatch.cpp index 5fa7b6f488..7892d475a6 100644 --- a/paddle/gserver/layers/SequenceToBatch.cpp +++ b/paddle/gserver/layers/SequenceToBatch.cpp @@ -171,12 +171,25 @@ void SequenceToBatch::sequence2BatchCopy(Matrix &batch, hl_sequence2batch_copy( batchData, seqData, idxData, seqWidth, batchCount, seq2batch); } else { - for (int i = 0; i < batchCount; ++i) { - if (seq2batch) { - memcpy(batch.rowBuf(i), - sequence.rowBuf(idxData[i]), - seqWidth * sizeof(real)); - } else { + if (seq2batch) { + const int blockMemSize = 8 * 1024; + const int blockSize = blockMemSize / sizeof(real); +#ifdef PADDLE_USE_MKLML +#pragma omp parallel for collapse(2) +#endif + for (int i = 0; i < batchCount; ++i) { + for (int j = 0; j < seqWidth; j += blockSize) { + memcpy(batch.rowBuf(i) + j, + sequence.rowBuf(idxData[i]) + j, + (j + blockSize > seqWidth) ? (seqWidth - j) * sizeof(real) + : blockMemSize); + } + } + } else { +#ifdef PADDLE_USE_MKLML +#pragma omp parallel for +#endif + for (int i = 0; i < batchCount; ++i) { memcpy(sequence.rowBuf(idxData[i]), batch.rowBuf(i), seqWidth * sizeof(real)); From 9c27c13e469f4c2f11f00c6e2570927e1d640e3c Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Sat, 16 Dec 2017 21:16:01 +0800 Subject: [PATCH 89/94] follow comments using macro to separate the original implements --- paddle/gserver/layers/SequenceToBatch.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/paddle/gserver/layers/SequenceToBatch.cpp b/paddle/gserver/layers/SequenceToBatch.cpp index 7892d475a6..6b769378d2 100644 --- a/paddle/gserver/layers/SequenceToBatch.cpp +++ b/paddle/gserver/layers/SequenceToBatch.cpp @@ -172,11 +172,10 @@ void SequenceToBatch::sequence2BatchCopy(Matrix &batch, batchData, seqData, idxData, seqWidth, batchCount, seq2batch); } else { if (seq2batch) { +#ifdef PADDLE_USE_MKLML const int blockMemSize = 8 * 1024; const int blockSize = blockMemSize / sizeof(real); -#ifdef PADDLE_USE_MKLML #pragma omp parallel for collapse(2) -#endif for (int i = 0; i < batchCount; ++i) { for (int j = 0; j < seqWidth; j += blockSize) { memcpy(batch.rowBuf(i) + j, @@ -185,6 +184,13 @@ void SequenceToBatch::sequence2BatchCopy(Matrix &batch, : blockMemSize); } } +#else + for (int i = 0; i < batchCount; ++i) { + memcpy(batch.rowBuf(i), + sequence.rowBuf(idxData[i]), + seqWidth * sizeof(real)); + } +#endif } else { #ifdef PADDLE_USE_MKLML #pragma omp parallel for From 77cf7d4f9e42d4c153aad648bbe5d7049ab2ff29 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sat, 16 Dec 2017 18:49:50 +0800 Subject: [PATCH 90/94] Fix read_source.md (#6673) Optimizer is not a part of fluid. The `paddle::platform` was missing. --- doc/howto/read_source.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/howto/read_source.md b/doc/howto/read_source.md index 383acb0c82..e4211abb3b 100644 --- a/doc/howto/read_source.md +++ b/doc/howto/read_source.md @@ -6,10 +6,10 @@ Core: https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/framework Operator: https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators -Optimizer: https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/optimizer - Memory: https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/memory +Platform: https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/platform + # Compile Time The following **defines** the NN. The definition goes into this [protocol buffer](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/framework.proto). From 1eac27630047e59cb461846022bada5bf0f490c1 Mon Sep 17 00:00:00 2001 From: sweetsky0901 Date: Sun, 17 Dec 2017 22:09:29 +0800 Subject: [PATCH 91/94] add spp avg --- paddle/operators/spp_op.cc | 5 +++ paddle/operators/spp_op.h | 34 ++++++++++++++++----- python/paddle/v2/fluid/tests/test_spp_op.py | 23 +++++++++++--- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/paddle/operators/spp_op.cc b/paddle/operators/spp_op.cc index c4bd4f5ab3..b1807b6261 100644 --- a/paddle/operators/spp_op.cc +++ b/paddle/operators/spp_op.cc @@ -30,6 +30,11 @@ class SppOpMaker : public framework::OpProtoAndCheckerMaker { "N * M." "M = C * H * W"); AddAttr("pyramid_height", "(int), multi level pooling"); + AddAttr( + "pooling_type", + "(string), pooling type, can be \"max\" for max-pooling " + "and \"avg\" for average-pooling.") + .InEnum({"max", "avg"}); AddComment(R"DOC( "With spatial pyramid pooling, the input image can be of any sizes. This not only allows arbitrary aspect diff --git a/paddle/operators/spp_op.h b/paddle/operators/spp_op.h index 16510cb826..f35b305d02 100644 --- a/paddle/operators/spp_op.h +++ b/paddle/operators/spp_op.h @@ -27,6 +27,8 @@ class SppKernel : public framework::OpKernel { const framework::Tensor* in_x = context.Input("X"); auto* out = context.Output("Out"); int pyramid_height = context.template Attr("pyramid_height"); + std::string pooling_type = + context.template Attr("pooling_type"); out->mutable_data(context.GetPlace()); auto out_stride = framework::stride(out->dims()); int input_h = in_x->dims()[2]; @@ -48,10 +50,17 @@ class SppKernel : public framework::OpKernel { framework::DDim output_shape(framework::make_ddim(output_shape_vec)); out_level.mutable_data(output_shape, context.GetPlace()); // pooling - math::Pool2dFunctor, T> pool_forward; - math::MaxPool max_process; - pool_forward(context.template device_context(), *in_x, - kernel_size, strides, paddings, max_process, &out_level); + if (pooling_type == "max") { + math::Pool2dFunctor, T> pool_forward; + math::MaxPool max_process; + pool_forward(context.template device_context(), *in_x, + kernel_size, strides, paddings, max_process, &out_level); + } else if (pooling_type == "avg") { + math::Pool2dFunctor, T> pool_forward; + math::AvgPool avg_process; + pool_forward(context.template device_context(), *in_x, + kernel_size, strides, paddings, avg_process, &out_level); + } // flatten pooling output shape int output_flatten_w = in_x->dims()[1] * bins * bins; std::vector output_flatten_shape_vec( @@ -79,6 +88,8 @@ class SppGradKernel : public framework::OpKernel { framework::Tensor* in_x_grad = context.Output(framework::GradVarName("X")); int pyramid_height = context.template Attr("pyramid_height"); + std::string pooling_type = + context.template Attr("pooling_type"); auto& device_ctx = context.template device_context(); math::SetConstant zero; in_x_grad->mutable_data(context.GetPlace()); @@ -130,10 +141,19 @@ class SppGradKernel : public framework::OpKernel { outgrad_level.ShareDataWith(outgrad_level); outgrad_level.Resize(out_shape); // pooling backward - math::MaxPool2dGradFunctor pool2d_backward; - pool2d_backward(context.template device_context(), *in_x, + if (pooling_type == "max") { + math::MaxPool2dGradFunctor pool2d_backward; + pool2d_backward(context.template device_context(), *in_x, + *&out_level, *&outgrad_level, kernel_size, strides, + paddings, in_x_grad); + } else if (pooling_type == "avg") { + math::Pool2dGradFunctor, T> + pool_backward; + math::AvgPoolGrad avg_process; + pool_backward(context.template device_context(), *in_x, *&out_level, *&outgrad_level, kernel_size, strides, - paddings, in_x_grad); + paddings, avg_process, in_x_grad); + } } } }; diff --git a/python/paddle/v2/fluid/tests/test_spp_op.py b/python/paddle/v2/fluid/tests/test_spp_op.py index b57f4a795d..007723f0e3 100644 --- a/python/paddle/v2/fluid/tests/test_spp_op.py +++ b/python/paddle/v2/fluid/tests/test_spp_op.py @@ -2,6 +2,7 @@ import unittest import numpy as np from op_test import OpTest from test_pool2d_op import max_pool2D_forward_naive +from test_pool2d_op import avg_pool2D_forward_naive class TestSppOp(OpTest): @@ -24,8 +25,8 @@ class TestSppOp(OpTest): bins.astype("double")).astype("int32") padding[1] = ( (kernel_size[1] * bins - wsize + 1) / 2).astype("int32") - out_level = max_pool2D_forward_naive(input, kernel_size, - kernel_size, padding) + out_level = self.pool2D_forward_naive(input, kernel_size, + kernel_size, padding) out_level_flatten.append( out_level.reshape(nsize, bins * bins * csize)) if i == 0: @@ -34,7 +35,10 @@ class TestSppOp(OpTest): output = np.concatenate((output, out_level_flatten[i]), 1) # output = np.concatenate(out_level_flatten.tolist(), 0); self.inputs = {'X': input.astype('float32'), } - self.attrs = {'pyramid_height': self.pyramid_height} + self.attrs = { + 'pyramid_height': self.pyramid_height, + 'pooling_type': self.pool_type + } self.outputs = {'Out': output.astype('float32')} @@ -42,11 +46,22 @@ class TestSppOp(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['X'], 'Out', max_relative_error=0.05) + if self.pool_type != "avg": + self.check_grad(['X'], 'Out', max_relative_error=0.05) def init_test_case(self): self.shape = [3, 2, 4, 4] self.pyramid_height = 3 + self.pool2D_forward_naive = max_pool2D_forward_naive + self.pool_type = "max" + + +class TestCase2(TestSppOp): + def init_test_case(self): + self.shape = [3, 2, 4, 4] + self.pyramid_height = 3 + self.pool2D_forward_naive = avg_pool2D_forward_naive + self.pool_type = "avg" if __name__ == '__main__': From 93a2d9c59d46589d3fefaaee09a3003f45fda9e5 Mon Sep 17 00:00:00 2001 From: QI JUN Date: Mon, 18 Dec 2017 11:53:22 +0800 Subject: [PATCH 92/94] add more place test and rename Cudnn to CUDNN (#6621) * add more place_test and rename Cudnn to CUDNN * fix ci --- paddle/operators/math/math_function.cu | 2 +- paddle/platform/device_context.cc | 8 ++++---- paddle/platform/device_context.h | 8 ++++---- paddle/platform/device_context_test.cc | 10 +++++----- paddle/platform/place.h | 8 ++++---- paddle/platform/place_test.cc | 6 ++++++ 6 files changed, 24 insertions(+), 18 deletions(-) diff --git a/paddle/operators/math/math_function.cu b/paddle/operators/math/math_function.cu index e33070c40f..7852bb53a9 100644 --- a/paddle/operators/math/math_function.cu +++ b/paddle/operators/math/math_function.cu @@ -274,7 +274,7 @@ void set_constant_with_place( } template <> -void set_constant_with_place( +void set_constant_with_place( const platform::DeviceContext& context, framework::Tensor* tensor, float value) { set_constant_with_place(context, tensor, value); diff --git a/paddle/platform/device_context.cc b/paddle/platform/device_context.cc index 1c72b50559..8cdc5f4340 100644 --- a/paddle/platform/device_context.cc +++ b/paddle/platform/device_context.cc @@ -125,21 +125,21 @@ cudnnHandle_t CUDADeviceContext::cudnn_handle() const { return cudnn_handle_; } cudaStream_t CUDADeviceContext::stream() const { return stream_; } -CudnnDeviceContext::CudnnDeviceContext(CudnnPlace place) +CUDNNDeviceContext::CUDNNDeviceContext(CUDNNPlace place) : CUDADeviceContext(place), place_(place) { PADDLE_ENFORCE(dynload::cudnnCreate(&cudnn_handle_)); PADDLE_ENFORCE(dynload::cudnnSetStream(cudnn_handle_, stream())); } -CudnnDeviceContext::~CudnnDeviceContext() { +CUDNNDeviceContext::~CUDNNDeviceContext() { SetDeviceId(place_.device); Wait(); PADDLE_ENFORCE(dynload::cudnnDestroy(cudnn_handle_)); } -Place CudnnDeviceContext::GetPlace() const { return CudnnPlace(); } +Place CUDNNDeviceContext::GetPlace() const { return CUDNNPlace(); } -cudnnHandle_t CudnnDeviceContext::cudnn_handle() const { return cudnn_handle_; } +cudnnHandle_t CUDNNDeviceContext::cudnn_handle() const { return cudnn_handle_; } #endif diff --git a/paddle/platform/device_context.h b/paddle/platform/device_context.h index f67194993d..56813a1d5b 100644 --- a/paddle/platform/device_context.h +++ b/paddle/platform/device_context.h @@ -86,10 +86,10 @@ class CUDADeviceContext : public DeviceContext { cublasHandle_t cublas_handle_; }; -class CudnnDeviceContext : public CUDADeviceContext { +class CUDNNDeviceContext : public CUDADeviceContext { public: - explicit CudnnDeviceContext(CudnnPlace place); - virtual ~CudnnDeviceContext(); + explicit CUDNNDeviceContext(CUDNNPlace place); + virtual ~CUDNNDeviceContext(); /*! \brief Return place in the device context. */ Place GetPlace() const final; @@ -99,7 +99,7 @@ class CudnnDeviceContext : public CUDADeviceContext { private: cudnnHandle_t cudnn_handle_; - CudnnPlace place_; + CUDNNPlace place_; }; #endif diff --git a/paddle/platform/device_context_test.cc b/paddle/platform/device_context_test.cc index be3b2af5af..109c13a881 100644 --- a/paddle/platform/device_context_test.cc +++ b/paddle/platform/device_context_test.cc @@ -47,14 +47,14 @@ TEST(Device, CUDADeviceContext) { } } -TEST(Device, CudnnDeviceContext) { - using paddle::platform::CudnnDeviceContext; - using paddle::platform::CudnnPlace; +TEST(Device, CUDNNDeviceContext) { + using paddle::platform::CUDNNDeviceContext; + using paddle::platform::CUDNNPlace; if (paddle::platform::dynload::HasCUDNN()) { int count = paddle::platform::GetCUDADeviceCount(); for (int i = 0; i < count; ++i) { - CudnnDeviceContext* device_context = - new CudnnDeviceContext(CudnnPlace(i)); + CUDNNDeviceContext* device_context = + new CUDNNDeviceContext(CUDNNPlace(i)); cudnnHandle_t cudnn_handle = device_context->cudnn_handle(); ASSERT_NE(nullptr, cudnn_handle); ASSERT_NE(nullptr, device_context->stream()); diff --git a/paddle/platform/place.h b/paddle/platform/place.h index 4526945792..ca98920d41 100644 --- a/paddle/platform/place.h +++ b/paddle/platform/place.h @@ -51,9 +51,9 @@ struct GPUPlace { int device; }; -struct CudnnPlace : public GPUPlace { - CudnnPlace() : GPUPlace() {} - explicit CudnnPlace(int d) : GPUPlace(d) {} +struct CUDNNPlace : public GPUPlace { + CUDNNPlace() : GPUPlace() {} + explicit CUDNNPlace(int d) : GPUPlace(d) {} }; struct IsGPUPlace : public boost::static_visitor { @@ -72,7 +72,7 @@ struct IsMKLDNNPlace : public boost::static_visitor { // should be less equal than 2^(NUM_PLACE_TYPE_LIMIT_IN_BIT) #define NUM_PLACE_TYPE_LIMIT_IN_BIT 4 -typedef boost::variant Place; +typedef boost::variant Place; // static check number of place types is less equal than // 2^(NUM_PLACE_TYPE_LIMIT_IN_BIT) diff --git a/paddle/platform/place_test.cc b/paddle/platform/place_test.cc index 184af12c23..c536b59ed8 100644 --- a/paddle/platform/place_test.cc +++ b/paddle/platform/place_test.cc @@ -5,16 +5,22 @@ TEST(Place, Equality) { paddle::platform::CPUPlace cpu; paddle::platform::GPUPlace g0(0), g1(1), gg0(0); + paddle::platform::CUDNNPlace d0(0), d1(1), dd0(0); EXPECT_EQ(cpu, cpu); EXPECT_EQ(g0, g0); EXPECT_EQ(g1, g1); EXPECT_EQ(g0, gg0); + EXPECT_EQ(d0, dd0); EXPECT_NE(g0, g1); + EXPECT_NE(d0, d1); EXPECT_TRUE(paddle::platform::places_are_same_class(g0, gg0)); EXPECT_FALSE(paddle::platform::places_are_same_class(g0, cpu)); + + EXPECT_TRUE(paddle::platform::is_gpu_place(d0)); + EXPECT_FALSE(paddle::platform::places_are_same_class(g0, d0)); } TEST(Place, Default) { From 964f01e3e880d4c835489ce5fad8215a964c188d Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 18 Dec 2017 11:57:20 +0800 Subject: [PATCH 93/94] fix simple_gru2 doc --- python/paddle/trainer_config_helpers/networks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/paddle/trainer_config_helpers/networks.py b/python/paddle/trainer_config_helpers/networks.py index 9776ae1805..8bfe56d795 100644 --- a/python/paddle/trainer_config_helpers/networks.py +++ b/python/paddle/trainer_config_helpers/networks.py @@ -1119,8 +1119,9 @@ def simple_gru2(input, :param gru_bias_attr: bias parameter attribute of gru layer, False means no bias, None means default bias. :type gru_bias_attr: ParameterAttribute|False|None - :param gru_layer_attr: Extra attribute of the gru layer. - :type gru_layer_attr: ExtraLayerAttribute + :param gru_param_attr: param parameter attribute of gru layer, + None means default param. + :type gru_param_attr: ParameterAttribute|None :return: the gru group. :rtype: LayerOutput """ From 24fda392207e1fd608e66ea376a79f9f9716c3ca Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 18 Dec 2017 13:01:41 +0800 Subject: [PATCH 94/94] Feature/global context (#6537) * "add DeviceContextPool" * "add devicecontextpool in pybind" * "add comments in python side " * "fix static link error" * "fix CI error" * "add executor.py" * "fix CI error" * "add with gpu macro" * "remove comment out codes" * "add TODO items" * "update init devices" --- paddle/framework/CMakeLists.txt | 3 ++ paddle/framework/ddim_test.cc | 13 +++++ paddle/framework/executor.cc | 33 ++---------- paddle/framework/executor.h | 84 ++++++++++++++++++++++++++++-- paddle/framework/init.cc | 80 ++++++++++++++++++++++++++++ paddle/framework/init.h | 28 ++++++++++ paddle/framework/init_test.cc | 27 ++++++++++ paddle/pybind/CMakeLists.txt | 2 +- paddle/pybind/pybind.cc | 23 ++------ python/paddle/v2/fluid/executor.py | 7 +++ 10 files changed, 248 insertions(+), 52 deletions(-) create mode 100644 paddle/framework/init.cc create mode 100644 paddle/framework/init.h create mode 100644 paddle/framework/init_test.cc diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 4b0eff3adb..206e298eb2 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -58,3 +58,6 @@ cc_test(var_type_inference_test SRCS var_type_inference_test.cc DEPS op_registry proto_desc) cc_library(selected_rows SRCS selected_rows.cc DEPS tensor) cc_test(selected_rows_test SRCS selected_rows_test.cc DEPS selected_rows) + +cc_library(init SRCS init.cc DEPS gflags executor place stringpiece) +cc_test(init_test SRCS init_test.cc DEPS init) diff --git a/paddle/framework/ddim_test.cc b/paddle/framework/ddim_test.cc index 756232b1b5..bd5ea09d7d 100644 --- a/paddle/framework/ddim_test.cc +++ b/paddle/framework/ddim_test.cc @@ -1,3 +1,16 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ #include #include diff --git a/paddle/framework/executor.cc b/paddle/framework/executor.cc index 83aa927c29..a8b8a6f8e8 100644 --- a/paddle/framework/executor.cc +++ b/paddle/framework/executor.cc @@ -33,32 +33,12 @@ namespace framework { const std::string kFeedOpType = "feed"; const std::string kFetchOpType = "fetch"; -Executor::Executor(const std::vector& places) : own_(true) { - PADDLE_ENFORCE_GT(places.size(), 0); - device_contexts_.resize(places.size()); - for (size_t i = 0; i < places.size(); i++) { - if (platform::is_cpu_place(places[i])) { - device_contexts_[i] = new platform::CPUDeviceContext( - boost::get(places[i])); - } else if (platform::is_gpu_place(places[i])) { -#ifdef PADDLE_WITH_CUDA - device_contexts_[i] = new platform::CUDADeviceContext( - boost::get(places[i])); -#else - PADDLE_THROW( - "'GPUPlace' is not supported, Please re-compile with WITH_GPU " - "option"); -#endif - } - } -} +DeviceContextPool* DeviceContextPool::pool = nullptr; -Executor::~Executor() { - if (own_) { - for (auto& device_context : device_contexts_) { - delete device_context; - } - } +Executor::Executor(const std::vector& places) { + DeviceContextPool& pool = DeviceContextPool::Get(); + auto borrowed_contexts = pool.Borrow(places); + device_contexts_.swap(borrowed_contexts); } static void CreateTensor(Variable* var, VarDesc::VarType var_type) { @@ -132,8 +112,5 @@ void Executor::Run(const ProgramDescBind& pdesc, Scope* scope, int block_id, } } -Executor::Executor(const platform::DeviceContext& device) - : device_contexts_({&device}), own_(false) {} - } // namespace framework } // namespace paddle diff --git a/paddle/framework/executor.h b/paddle/framework/executor.h index b745f4f647..073e04729b 100644 --- a/paddle/framework/executor.h +++ b/paddle/framework/executor.h @@ -14,19 +14,98 @@ limitations under the License. */ #pragma once +#include +#include + #include "paddle/framework/op_info.h" #include "paddle/framework/program_desc.h" #include "paddle/framework/scope.h" #include "paddle/framework/tensor.h" +#include "paddle/platform/device_context.h" namespace paddle { namespace framework { +class DeviceContextPool { + public: + static DeviceContextPool& Get() { + PADDLE_ENFORCE_NOT_NULL(pool, "Need to Create DeviceContextPool first!"); + return *pool; + } + + static DeviceContextPool& Create(const std::vector& places) { + if (pool == nullptr) { + pool = new DeviceContextPool(places); + } + return *pool; + } + + std::vector Borrow( + const std::vector& places) { + PADDLE_ENFORCE_GT(places.size(), 0); + PADDLE_ENFORCE_LE(places.size(), device_contexts_.size()); + std::vector borrowed_contexts; + for (auto& place : places) { + auto range = device_contexts_.equal_range(place); + if (range.first == range.second) { + PADDLE_THROW( + "'Place' is not supported, Please re-compile with WITH_GPU " + "option"); + } + // TODO(dzhwinter) : assign the first found device. Will enhanced later. + // device load balancer maybe useful here. + borrowed_contexts.emplace_back(range.first->second); + } + return borrowed_contexts; + } + + explicit DeviceContextPool(const std::vector& places) { + PADDLE_ENFORCE_GT(places.size(), 0); + for (size_t i = 0; i < places.size(); i++) { + if (platform::is_cpu_place(places[i])) { + device_contexts_.emplace( + places[i], new platform::CPUDeviceContext( + boost::get(places[i]))); + } else if (platform::is_gpu_place(places[i])) { +#ifdef PADDLE_WITH_CUDA + device_contexts_.emplace( + places[i], new platform::CUDADeviceContext( + boost::get(places[i]))); +#else + PADDLE_THROW( + "'GPUPlace' is not supported, Please re-compile with WITH_GPU " + "option"); +#endif + } + } + } + + ~DeviceContextPool() {} + + private: + static DeviceContextPool* pool; + struct Hash { + std::hash hash_; + size_t operator()(const platform::Place& place) const { + return hash_(place.which()); + } + }; + std::unordered_multimap + device_contexts_; + DISABLE_COPY_AND_ASSIGN(DeviceContextPool); +}; + class Executor { public: + // TODO(dzhwinter) : Do not rely on this function, it will be removed + explicit Executor(const platform::DeviceContext& device) + : Executor(std::vector({device.GetPlace()})) {} + + explicit Executor(const platform::Place& place) + : Executor(std::vector({place})) {} + explicit Executor(const std::vector& places); - explicit Executor(const platform::DeviceContext& devices); - ~Executor(); /* @Brief * Runtime evaluation of the given ProgramDesc under certain Scope @@ -39,7 +118,6 @@ class Executor { private: std::vector device_contexts_; - bool own_; }; } // namespace framework diff --git a/paddle/framework/init.cc b/paddle/framework/init.cc new file mode 100644 index 0000000000..1c4476f4b3 --- /dev/null +++ b/paddle/framework/init.cc @@ -0,0 +1,80 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include +#include + +#include "paddle/framework/executor.h" +#include "paddle/framework/init.h" +#include "paddle/platform/place.h" +#include "paddle/string/piece.h" + +namespace paddle { +namespace framework { + +std::once_flag gflags_init_flag; + +// TODO(qijun) move init gflags to init.cc +void InitGflags(std::vector &argv) { + std::call_once(gflags_init_flag, [&]() { + int argc = argv.size(); + char **arr = new char *[argv.size()]; + std::string line; + for (size_t i = 0; i < argv.size(); i++) { + arr[i] = &argv[i][0]; + line += argv[i]; + line += ' '; + } + google::ParseCommandLineFlags(&argc, &arr, true); + VLOG(1) << "Init commandline: " << line; + }); +} + +bool InitDevices(const std::vector &devices) { + // device format + // CPU + // GPU:1 + // TODO(dzhwinter) : add device format annotation for users. + std::vector places; + for (auto &device : devices) { + auto p = string::Piece(device); + if (string::Find(p, ':', 0) == string::Piece::npos) { + places.emplace_back(platform::CPUPlace()); + } else if (string::HasPrefix(p, "GPU")) { +#ifdef PADDLE_WITH_CUDA + auto pos = string::RFind(p, ':', string::Piece::npos); + auto number = device.substr(pos + 1); + places.emplace_back(platform::GPUPlace(std::stoi(number))); +#else + LOG(WARNING) + << "'GPU' is not supported, Please re-compile with WITH_GPU option"; +#endif + } else { + return false; + } + } + + if (std::find_if(places.begin(), places.end(), + [&](const platform::Place &place) { + return platform::is_cpu_place(place); + }) == places.end()) { + places.emplace_back(platform::CPUPlace()); + LOG(WARNING) << "Not specified any device, use CPU by Default."; + } + DeviceContextPool::Create(places); + return true; + return true; +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/init.h b/paddle/framework/init.h new file mode 100644 index 0000000000..1715cd81e6 --- /dev/null +++ b/paddle/framework/init.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#pragma once +#include + +#include "gflags/gflags.h" +#include "glog/logging.h" + +namespace paddle { +namespace framework { + +void InitGflags(std::vector &argv); + +bool InitDevices(const std::vector &devices); + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/init_test.cc b/paddle/framework/init_test.cc new file mode 100644 index 0000000000..f65e881a76 --- /dev/null +++ b/paddle/framework/init_test.cc @@ -0,0 +1,27 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include "gtest/gtest.h" + +#include "paddle/framework/init.h" + +TEST(Init, InitDevices) { + using paddle::framework::InitDevices; + std::vector ds1 = {"CPU"}; + ASSERT_EQ(InitDevices(ds1), true); + +#ifdef PADDLE_WITH_CUDA + std::vector ds2 = {"CPU", "GPU:0", "GPU:1"}; + ASSERT_EQ(InitDevices(ds2), true); +#endif +} diff --git a/paddle/pybind/CMakeLists.txt b/paddle/pybind/CMakeLists.txt index fd55f410d3..1fb69de90d 100644 --- a/paddle/pybind/CMakeLists.txt +++ b/paddle/pybind/CMakeLists.txt @@ -1,7 +1,7 @@ if(WITH_PYTHON) cc_library(paddle_pybind SHARED SRCS pybind.cc exception.cc protobuf.cc - DEPS pybind python backward proto_desc paddle_memory executor prune + DEPS pybind python backward proto_desc paddle_memory executor prune init ${GLOB_OP_LIB}) endif(WITH_PYTHON) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 1faf24bcb8..4248db34c6 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -16,11 +16,11 @@ limitations under the License. */ #include // for call_once #include -#include "gflags/gflags.h" #include "paddle/framework/backward.h" #include "paddle/framework/executor.h" #include "paddle/framework/feed_fetch_method.h" #include "paddle/framework/framework.pb.h" +#include "paddle/framework/init.h" #include "paddle/framework/lod_rank_table.h" #include "paddle/framework/lod_tensor.h" #include "paddle/framework/lod_tensor_array.h" @@ -51,24 +51,6 @@ static size_t UniqueIntegerGenerator(const std::string &prefix) { return generators[prefix].fetch_add(1); } -std::once_flag gflags_init_flag; - -// TODO(qijun) move init gflags to init.cc -void InitGflags(std::vector &argv) { - std::call_once(gflags_init_flag, [&]() { - int argc = argv.size(); - char **arr = new char *[argv.size()]; - std::string line; - for (size_t i = 0; i < argv.size(); i++) { - arr[i] = &argv[i][0]; - line += argv[i]; - line += ' '; - } - google::ParseCommandLineFlags(&argc, &arr, true); - VLOG(1) << "Init commandline: " << line; - }); -} - bool IsCompileGPU() { #ifndef PADDLE_WITH_CUDA return false; @@ -438,7 +420,8 @@ All parameter, weight, gradient are variables in Paddle. .def("run", &Executor::Run); m.def("unique_integer", UniqueIntegerGenerator); - m.def("init_gflags", InitGflags); + m.def("init_gflags", framework::InitGflags); + m.def("init_devices", &framework::InitDevices); m.def("is_compile_gpu", IsCompileGPU); m.def("set_feed_variable", framework::SetFeedVariable); diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index bdc82eede9..9a99b045dc 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -46,6 +46,13 @@ class Executor(object): p.set_place(each) act_places.append(p) + # TODO(dzhwinter) : consider that our fluid tests all written in + # GPUPlace(gpu_id), this will be changed in next PR. + if core.is_compile_gpu(): + core.init_devices(["CPU", "GPU:0"]) + else: + core.init_devices(["CPU"]) + self.executor = core.Executor(act_places) self.places = places