From 04eb4f89c76dabe1ff0838dbd9046462a869bb80 Mon Sep 17 00:00:00 2001 From: Cathy Wong Date: Wed, 15 Jul 2020 20:25:03 -0400 Subject: [PATCH] Cleanup dataset UT: Replace save_and_check --- .../data/dataset/golden/batch_01_result.npz | Bin 1509 -> 1961 bytes .../data/dataset/golden/batch_02_result.npz | Bin 1325 -> 1597 bytes .../data/dataset/golden/batch_03_result.npz | Bin 1509 -> 1871 bytes .../data/dataset/golden/batch_04_result.npz | Bin 1597 -> 1781 bytes .../data/dataset/golden/batch_05_result.npz | Bin 1509 -> 2123 bytes .../data/dataset/golden/batch_06_result.npz | Bin 1509 -> 1727 bytes .../data/dataset/golden/batch_07_result.npz | Bin 1509 -> 1826 bytes .../data/dataset/golden/batch_08_result.npz | Bin 1509 -> 1781 bytes .../data/dataset/golden/batch_09_result.npz | Bin 1509 -> 1727 bytes .../data/dataset/golden/batch_10_result.npz | Bin 264 -> 264 bytes .../data/dataset/golden/batch_11_result.npz | Bin 582 -> 767 bytes .../data/dataset/golden/batch_12_result.npz | Bin 1509 -> 2123 bytes .../data/dataset/golden/shuffle_01_result.npz | Bin 1507 -> 1691 bytes .../data/dataset/golden/shuffle_02_result.npz | Bin 1507 -> 1691 bytes .../data/dataset/golden/shuffle_03_result.npz | Bin 1507 -> 1691 bytes .../data/dataset/golden/shuffle_04_result.npz | Bin 635 -> 819 bytes .../data/dataset/golden/shuffle_05_result.npz | Bin 1507 -> 1691 bytes .../dataset/golden/test_2ops_batch_repeat.npz | Bin 3137 -> 3602 bytes .../golden/test_2ops_batch_shuffle.npz | Bin 1914 -> 2360 bytes .../dataset/golden/test_2ops_repeat_batch.npz | Bin 3149 -> 3614 bytes .../golden/test_2ops_repeat_shuffle.npz | Bin 3831 -> 4042 bytes .../golden/test_2ops_shuffle_batch.npz | Bin 1914 -> 2360 bytes .../golden/test_2ops_shuffle_repeat.npz | Bin 3831 -> 4042 bytes .../dataDistributionAll.json | 8 -- .../dataDistributionRandom.json | 8 -- .../dataDistributionUnique.json | 8 -- .../ut/data/dataset/testPK/distribution.json | 7 -- tests/ut/python/dataset/test_2ops.py | 34 ++--- tests/ut/python/dataset/test_batch.py | 48 ++------ tests/ut/python/dataset/test_datasets_clue.py | 4 + ...enerator.py => test_datasets_generator.py} | 116 +++++++++--------- ...reader_op.py => test_datasets_tfrecord.py} | 0 tests/ut/python/dataset/test_iterator.py | 8 +- tests/ut/python/dataset/test_shuffle.py | 18 +-- tests/ut/python/dataset/util.py | 82 ++++--------- 35 files changed, 117 insertions(+), 224 deletions(-) delete mode 100644 tests/ut/data/dataset/testImageNetData2/dataDistributionAll.json delete mode 100644 tests/ut/data/dataset/testImageNetData2/dataDistributionRandom.json delete mode 100644 tests/ut/data/dataset/testImageNetData2/dataDistributionUnique.json delete mode 100644 tests/ut/data/dataset/testPK/distribution.json rename tests/ut/python/dataset/{test_generator.py => test_datasets_generator.py} (92%) rename tests/ut/python/dataset/{test_tfreader_op.py => test_datasets_tfrecord.py} (100%) diff --git a/tests/ut/data/dataset/golden/batch_01_result.npz b/tests/ut/data/dataset/golden/batch_01_result.npz index 7da040cd589b3a21e3e0db623d803c47f2e20c0a..b2dd3bd71e63a03ef2075782ab57671e1947ea37 100644 GIT binary patch literal 1961 zcmbW2*;`Xb7{*TsYg}*vcO~GGQ4v`b0T-gEM|lMzilQ>ckQ@}ngbBfxsukO!l`gim zi*~WLc3EVp@BL;@2;S@>%N?cGf$P_< zmgZTOo&{Es6-g$;l@;;+A&)h~Kpm(>J+8HOw1f_MtTWcxK+GA4CIb!rz`3?Sxj)d8 zNT!mJcsP-aIU4Vb^bI)V11BQ=4%ec-++V%2+<)Hx|A%kb8;uX1?jNd%CX!CY>A}8K zFY}EI`NByjnurgil7rEdPg!o1FCKGM<$*P#vYI^I%6nO5x2qfh4+@ZVN#&+PW2vEj z$EWf-v-mUH>Qp`~crL4gcI6ZBc6vz9sVS|kt<7T`<^_!@HMRSSRMSMKk9suG7ar)1 zr>bhzbg2r3R5LhdHe17EV-cxl!4q+$noU-OEP`TqHJC#%7uf{!kfXtTf(6JWScp6g z77;8)KEV(6 z-5V)yvV%EkBGqO)Sop`txV(jOqa7^H$U!@};Kq$x>D^l?Z?l6X=?zjf*}<}#caNpz z?UZ-e!R49VJMCcLuiN+1vQ2rH9bA?EJgIiu_IRF?N?sjP%~FBEdpKL}E4it*RIR4a z<27z$Sjg1alvk>~rn2KjZKu4?6xGyIs{N*vZWX2KpnO1J9_C{K7P3SK1r}j3mY_t} zzLWAH0Y6GnhNT3D1(snsRv@6|Bb2)Y1Xf}dRuf2pa;!lG)@q9;w{h({byTV_Hyq=P zOmq@W>UMiMN+D)?*>QWhhv2yBCv3qXz4+9<|s&Z?6f|p&kv`h)vjx zEojs&@I1p`5C~!`wxJ2zu>(6{Q+QEe7j~l=z#g=q6>U2FC5B%Y2w^YUu@C#vfde>5 z;f6pb4&gA4pbHX55uSX=^+i~xUY6<=?thi@wPs#}`X0RLn)W)`Eo0gnu4!))+%~4Y z<(l?3!8^vZcU{xoBY5AK_JM2KhXfxP(>}(1J=5JAk0giG9jQKH$fum2A;Rze9Nqd> z`GVj}L=1vSP_(g+GkZYMa#N12U9 z>2@~b#-i7n-*7*Rna!-@>65MoYic%gQ~COwb8r97ce}c83jUUe(j2|Yhwk)Z zCZH(JKcqwy)3&oC@nX5ADji&O7CMv8_%*pSpIT9s7s~5s!K&o#=#&Q1xroGkv=LpyR`Qp}QxfaitY%9LGwNYIs zzh*6*wXJ-qSgG1u`D$1y%~j!Ip*bs6DDS0jM$Jk8wrokufPjh+TH)K4t?uGNwN|#m zvTfN%8(o-?K`2nSWhgDf0{&%{s&C|;WHOm(&|txIRqoAw$jE&nBlnAsLFXCKs6(|f zvR%LjKLs5EEeKF>Kv2sz3OWUX2vKlQAdEc}L^Oz zC(%PmpMZv5^x+gGrv&;jfI&nl=@$?f!Z1!#G9VDc8N_jxlEHs@be@K5FY4KIw(mL< zo-=2^uBjo<)Gj~6o}XP-PJ32%xrhmj;2cJA9v5&CV;CoEX9OlNi78ydWn95kOj9u~ zpyL{@V+J>H6SFX=cvj#R=8ynz8}mqFfr=vnDcnICcd>{i+{1k;o)cKc3LfAg9wCFr z$Wn0>rkRoFkt4l;Jn2OgNXKB2j$@T{0#8XNu|_(Db<#_CMtT{~Nv~jo^s4d4Uya6h zdc`Q2Hm3hEzG=-wH}rs)Tr>3cKi|7E*A2bPo0&26?oXe-H1D}#=sn)Dn}*)^`S&Ny znOQ?0@a{1TJ^J_8@6DN8hCb{8<_u%UP9mYy8x8UWsHC^yBb|qzG>I0{1q4V_XeGUa zHqtbLq<0Y_T|}64X}xp)eyFXI2NlbIffJL=9+v~y0U5NLG@D*eXk`Wv6s FKL8Y9nW6vy diff --git a/tests/ut/data/dataset/golden/batch_02_result.npz b/tests/ut/data/dataset/golden/batch_02_result.npz index 1d126c043eb4014502e43d36bcd266859afbf932..671e5161c7daed71c0bb395684c11938bc4e698e 100644 GIT binary patch literal 1597 zcmbW2&sP&y6vt;00wn$bHK^bZ0*ZNPj1Z7t{ve7nny-YE);6}C#z`_rY)En^!8R3Z z`U5GpyRO{y^qlUy>pA@oy6XSnzKgo)vM0UsW=;sFr<*>`y}Wntecw0v6Jmd&`#Rp`{ezl)B{y{qC5@f&1%jH&xOL9 z%i%~Uyjr$umQl)-t-NW67mZ@osF=JKvyo72G!l9i`u`))@a9Sz>y>&mSGLUP z`bM!<;O8~!fsAG5%B5<}+Q`)cq&QW9Qr^ju3T2Bt3u>0U+tjf_J^>YecwSJaTbi%c zD`tSY(jNYp%QMss1?o2WS12IhO{*N;qiN4e@4d>lBAPWJuiVn2Xx_9x&s zOf%i-%8%#iq)w-J!)dlb@v@r}_Gmz5rI3IJUY-mIbil`xVS!F`@#Ks^H~c2g&hpL? zfdKaL;+(*K9NlM-hCmQ~IErKZ*f#}^qaOn}!HWfflQ@Ob805t*fe?l;j5EBrEpQeiIES!} zvAsv8CTUS8jdky^-QE2j8Dv5>3S^_)lZ~?EG)j_7mt~`*WTU(#8)Zc{%G*w(w4X5% zqjz+=&zs(5doRJT8Z4id@(-l^`%?Y`NB%>u(xv=IQa&T)A4+*+Zy!$NDXUYCH{{vO ze}T5U&t<&YVr+Msi}z6rZyVIL7SuMLg%;0UlsY};Q$H3E7{z&9Km->N#U&iRJeG zw_olVs^~;~yv0^c@T+aV+cn8_Lo$6LnSSCh{gkWflIdrX=~KycQ!;(F_bba5N`_UZ z&vkmvn;L9iz+mk!k!AlCa_qlGp8Xa~``&oLCEHl#z&Cireg|vpU!uVNTRi3$!>rMF m*=P!?C+PjJ8=t$v9S!@E+r9r&@Y&anqVnG~|9>NY1O5T_zp|kK literal 1325 zcmbVMOHUI~6rSk|T0}sLAo3W%+Kbl8Q=Y;g!azSM6<_$M*vIfNTDa39+6V2TZExeZ7ZdkX<+<`AoHg?qD?Fx!_m;vDExfj66|7Wlb<4_{ zc6u)LG;dNqzmeKDDHfv zr4d;T1?s-^FUx>{dqt(}o~((-$X&tJxKGOAErR$M~VF7_2oW})3a1l{lqA7$x zFZ$4r0SsaYmoZGq9)S^zVhrP$z!gklijwCAbWGzaW^fI&n1eyd3j*_4Kn%cjEFz92 zN=A@MCFMn=38TmmUP6|z7bamJ)(HEtPB?%K!a-~j4&e#mW#jAjQt6QX02B-xQ$LN* zS~(aobZ;dXHT1e~ubj%5p*L14xQEZ9V^Yjb7ZV;Y~0>fz~#^S@j zqffI8Ls_m-V1(lFf$FO0_>{}Y*PwD#q-qbkdOL&tF5{MQ+aIwf!f}6#&p+Mmuk`tc zV(~;g6dj1gBesIuL!%Qm@rmKkxXrm}uJqNGSNiVw{(tk%c*4=ivGJ*@a4c?DjZKaw zMtI)Ply@L*hhx!+M0_%w@Jhq+@M7maOA~dfGwr=^sl=&q(t-8qC(e%qKo)(r5-UlM`;$ zMBipJkoWs+l5Q~rg(=@dW}s+c;jvms-^27hVg`y+zDLbK>7(DSDcX{)W}s|k-vE7& znF0Twi_aYTxEZKOZe__f(_B7XT6$|E@`NQ%a%DTsT$SErW>4wNCi8h(=W{ug9b5yQ z%48PL=qx64*Qs;oq|K6D^z9bdift&uc79q=U&_myI0(($~ zz3>w_FBqmA71+l~uRtaCqY4L<9O>oRP%kf7@*-FF(fU_vg}J(Dbb`kTPw3zY9emOWev9yPI{5QC_>{V!tFB00M806j+w_~J-T9Y> z>S842dod-Ss{f>DcT=LNT3^;{hI_q~(ko4>ms3)u3G_-zpcQQ_`6~CA6{toHYEg%J z97F>eDd9bVCN!f3hj182a1^Zs=L7;chT~|%37kYbOjQf7agX}~r*Ik_0M4KjUFaq; zFA&68^xz!MqZb!&k>KkBedxy}T*ehxxQc=FE32NcCi#XX-{hum(cbRh1EyZEceFt7 zIs&~%=7ARIeJ#*KN1zV~f2aleNDH*!2=p=GPqaXvYJon(s;U?qiH71+@{uJ!r{iPV z7YH%pml$Nfh_JfmUlINq5dz=9X8$dQ6#O0G?{S^L4;W_uBSsYb6XBn6gTNEqWd93B s`DEG&`Ria+5H5Ft=fD1eEXQPo}tu#~Jso{SW*%+`G_?>w0pU2{?`$Z>L|*IrrvwzMDyI3jU^u(imOJ+rGLl9Z(eK zA5tQUY1^6pc%f8Pl{PLq3!O=4yh+YXrsh=Tq4G4Ex5_y?I;usVO+{l`biQa;Y_pIl z+Ih<@WmWpd)vWZtlugqzAfO_ICQ#b!F3wk~B`YjjW_`5L zsbLv}0`;W~P0O%=e^#aHGr1v|OeX3ySTI(R8?&#{a+65Q&Ej>?c}6s9P_49V74X4N zL7SjX0SdMVG^2%rtpY)WDA*y`TLmfqB|MwQZ<2{*o6-4rWRcSd$1S#&`C+R zfQBw~V?QPP1rDGG2N9*@fPg?R`f!Mn9)TDRBaS1K9Q@0p^E6z0QO};Ueb<@roH_e- zP4#-F*7@o4{H(Kb$g{G}MNFU{M{x`T7{qa$zz|tGEHI1_jN&9t;WW-*jEZpq9cOV4 z<2a8Cn1DgWBLWw32?+p~F^MFmsMs%%!WB&8Dz0G$*Kvc2M+Igvhnu*C+eqUMGE_VU z(@e_&WJw2+BR!5h=?Pe*LzpKW#y!#zERc?3k@O_)lb*r@($iQXJ!5?RQLlfaSB#P| zW9*mlS!*P^p$EL=tf9AlUv+2B8G5@nGj8Y|@85rH+;iU0JH2HW488lqm%ELb2}AGk z?lBBK`t#Gb#>_=S@ACkc3}eksBB9jkb@BzMq?h3%orIq>i6+u11V~e8CcT0d(rE-q zuOdWx4Pnxm$@&YrTu)H98U;6)HOs6yX5D1gEoR+jR+?FNn3ZAHU1piiH|o%`Yab?M zIY#CgX)$u1k@pz6z{o{L-e=?kMlLaO`E}6w_-Q!YLV`Bz?6}Ct5+fxeZAO+ESz+V~ zBOfyI5hEWnvWgd`Rgq7!@f1{F#Q(38nI4EcYR*x#c21`dXm$!J{f6lJj?e1f+?Se; diff --git a/tests/ut/data/dataset/golden/batch_04_result.npz b/tests/ut/data/dataset/golden/batch_04_result.npz index 39198c56927049e0cfb3edddef7458706a56208a..aed34bf1e7ae924755dcdbe785d96ccc06092b41 100644 GIT binary patch literal 1781 zcmbVNSyxk66uviMN*q894u}#^%mFbn$>hLQl&kq9L=*|?HQpo_K|{j13AU+N(*dbi z?O-Q5xVlzf`qbzCgZ>TQ`_R7hIo;>vHip&Jm!7ruUVAwE?34X{IX41#l}{50g!syt9029j1bVFyP1ftATXmp?F< zv2(VWj%DnmCBoxoDr-?ZJ8v#oV0WtF?e)Hrvq@v13a&7i^q*ArgjY)$6!OO}_{ zMqTtXCx>_)6sT+5GsV3O?x;%96}}}L4u^`gq{3*9Z;d}S_%;^xQa+JM#j*?OT+aaC zZt!|$@CK3{A*E0(ng-tqmGmySNc-R>-H0mEyHQPg4{A)y;CrcJA8IMkggVmu;UV1& zFX;ml#mA;)MFke3S5J~3G>{I!!G}gNjxp@U9*gedp_bx4 zbE%A(<3WQTrz5&adi3A^C>CE+nr`2#kN&0q;x9RUx>j9w`gN`T_f;ozLf0D0nE_pE z+SvH9R5Pe+EzUM-KSbvb>ss5ZU+r*-|GJ5`Lb zH_6W!e3Z`ANX~8=r8C8Ia*ECX$K(K=*`1fOb0&5{PRtqBxEz+8lfiX{z1W8)>=$Fc zNPR9bG~)mcqJ;t>20vQShC}3l;V{~91Oaj`Gjco7iKFC9Fm&Mve+nv?K*9 z`}Xzn+XjzO@f{K~L_;r7%eZWrkS&w4r6pU=$(DCZE$69XLAJanTi%x~Q?liv_>*lS zd|#5M4W6NzB@+HGlmyKkfb23rg4Yf<*)leQ!wxLDGAtEfESJHkz~Mm|hcW~Pe~Y0T zJ?KRr`f&mS7!>HfP4~+)3}F}}IEhm@jWZaf#5)Wc&f*-#a2^*h4xNGz87|@yLI5sf z0%1&2aD^d)E11GnT*EZ3V}^o{7^0ZP4cx>n7`TmCdU#<~RKnFQlw3C445!acr_HtjrrVen6==u?u< zU{asY5hwi;3DRF6N%~7zq`$%(>928Dym8;q_6^Ka;9D$^{tov@e~}je5AAdM?VZzTr_-12OunqW_S(Pooju7;65gheQ6F8# zAAR<(Kluzpy#q$b$dpRy{#d?PF^x8zRE26$jc*gv6UkZAcxF70#KjEzOlH zMJFg*W;}e%$sy^70dqqJresjSJ7Y4tCbuLKiFl1e3r5Rw>)h*<+$K_TJKwEh)2~j6 zqYBeW$u2%(+9eu3TCqjntNj)VW`?lwN88@oL_bx*flShsVVbm=B8zgzP)IwcPa z?8QEGV87~fgnfDh4&WdTp_7TD0%3Ha8;3cG2pmBVjv~T|5D@4^AC7U-D-gwT#Bf4U z`batUFSqK1T%*y3QFSD)$Kwr;>hWEBv4*`5Or2<$`oK?r!%xFXN}d$x$4Ly}6i(v| z&SFsc8=${a0z(+a2+rX=F5n_Yxo}#*!X;eB7_Q(d#$hvgM&KH*BM#sOCXm1+lV=5z zxQQv;!fj0B4(>8JC@_Or+{1l5Knf3$R^5i!Z5WwMN{(QTdJb9YdE}@U;7~7Oo;r%h z8J1ftzk~%QE@P28h9}f3cuKvBB~>!c65Iae_gd{OzlzLXvq%55e+<_r*KNz!n8anBJX~9TVI*7 zt-c23mTmv-iIm%SCSxggc&U>XN1|YieFo3+jiixkXmV z_djpKOB%kS;bje9)$nH;{!tzNF}GDT{1Xj-uHkDM{$kUw?h+2&;Zv5rjPp3udGP+A zex~7{Yxox${-uVm*Ws_YV?)Eg((u88xy)*B9=iPJ9%qvEGLYl*j0gl4K`wt&X zb~yAm)sg1#g+gVcGpcJFoQ{r>q+X~e_1Iasu%LK})3M&M!4nA9`9q!wZqLRdPrBPv zQ4

eAQ(&p+HclXZfn?f_A=cwXZg4$7+1KJ8M|Fdz1VB4_9Nnzq)>1Z9|5?CKSwA zS6>ybwE6lPTxFr4zoxn_9IE$+U38dLuIhl9r4x=VjGOK(XZ$uMlrT}jiB!aGWm5Cf zK)9hc=wfndoc)+ZxlDlr&TULBp-UmY)M;lovQ1%OVSbZ6B0tCrvu*iyVLMgovgiFZ zRb_RR)#1z>wik9#!j8gD`Hse>CZA8(S=a?mjnaf&5vNf%VRyu9)I-=42^#eh_C}&c zeT03Hq)|U%emWloN+AJ>Iv6d?R7gUy4zh&V3Moj{!5HCK1sB@rAV)Y( zp)K0!AXhkEp*=e2V1jU>LPvDc!6e~ih0f@rgDJv1g*0^4fmb+Hp&PmfZPuwaGhN4u zsv2LI(}dG);xmLZtwYC}nvU4B`Y~q-Ei3PYwa?w0m@S-Rc@txad?74vhof7fE9MFd zEN|D?ib7$L;!y72tBA*iW}YSay;uQ8iWyeGxUHB9sgL5fP%b4+w*i z+DA)XAzT$Hd9>22g_V&?n*s~h2-hm~Ku`2SZ(aT>;W~vr=!<^nuM^e68U;57U?2wR zM6HktgE0g{;n9hZuuef?7=~knPK1T^3h5Y$42;q;wzfmg7_JwdVn6Ar!iHG?X(3M& zZZMwMDBNT`ak_A`@kFEW4C9G2g=ZO0oGsj9Jh4@{&3NJ*;dbMR9l~>sC(aX|Z#;2< z@WLG_+O_tGSfe$&>?~d+yx3-UiEwAWtz)AmcB$|(L+o23 zw!=Q*-7W2)9T@F`dxZD4bb+>6RQ&sd_qT}OTJr~l54LFDTI&76hgzhLDJ(o7d{|*L zGLeOBjKNssV4UuSM}&_mEOHVu@d{SXH z<{%%yToj-XMS9{X;nNDmn1>R~#{w+GA}rPu&j=4Hlwt{%Vi}f0umWYRw-3Dy=ki(M zb9N!07aq#DH{r-G{DSaBBkW7UmyNKm2wyeAz9xL#2>XWcO(X1E!ncjEhlTGLVc!+L zXM}xU_<<4jL*Yk8*pG#u;G|9~udMck8u+R3Gn>#6;pgz_Hv2;OCCWAWO87PW8hs=D z76FaE6Mm1NMn4FTqC%q|g+F1HMn4OWVYNobg}XlWiY*Q}AfDe8OIs{q}pkSXsE7~a7FAzkCf&&6!?4lqd$ju%KItBKkoq~e`9oR>~A%Xok zK*3>w2s$Y^B5)9gDCqi6Pq*DsZs&G7#+}?I$GM5iKf!z#-OXs18WlKm0JbP;U zt~22|Q~PyI4S1$@_!;#4?67j$v$DfQTwn;p7{NK5#|2!(C|NrrFotnV;1Vul5?3%q z#e{%?tGI?~T*nM%VN&s|zzy6)62L9YA%%G=4hf`j8w3QTyFQ7nr5jN>4R!GP2lyn@cq!U;py@Y3^m+_o*66>T_%+G%{ z8e8-iqh!jQ`p5hjZ6=0k1ia*`X|(_O+MT&(8lB$Ev}ts`fB#4Gnd_#}?Jb)zjo#ls zJ!#I&nnu6(jAXKt9rpa-~Vn%j1gNv+;!kS{pyw2;mtK$=D? z>20);E+9yH2O-kC2$L?(HD1x}UXr@iDY(z9C1x!%>jASKGV2kuGR%6+tSqyhFw0Wk zsG{YzKTOKnT4TxaCQyP*2`X((E-|vqNXbZt zkrhT(8M(>G7mWOYkv}rBhF6wdl|SVYY3RO)|9dAhy%2ZQ)lsy4PNxxQRRx`Xqjdj` H&+2~xalf0d diff --git a/tests/ut/data/dataset/golden/batch_06_result.npz b/tests/ut/data/dataset/golden/batch_06_result.npz index 6e8e923eb90df1d9468cb19b9fd5b64a5ecd0d57..5b1f3e7971ae91038996a92ba5bcec83e116c127 100644 GIT binary patch literal 1727 zcmbVNOIK4@7`->)mG}S^eBgtCVh)Irm%MxsMY)`z+RgzX>m`B$g? zJwE?@#?IMhI+n4MmeZazQ(24q*#&di;=LH{@%4B0_@4Ow|MC{xiS)|Saz2pA*j8X^ zC6!y`^O|{Y%(fDlbT((NBywKTN>6#y$lxLRA5G@+ z%a)hwqb~k8r$?v(8uWGY%#c^W9o4zJN?XF=aHz;j0mgH*HU3=EHWBsmJdsJovWw|l z-waBaB)mvH}+V35)ZRECz(%W%pCPd zIxZ=|hdE*V{ClzZmZyB|USs?(<40d<(`N*0E1P~J*!bI8c`{%GTPl-5BiQ=p&G)6x zoHT-MmB)sRV8?eq-78IojbLZxGb2XO|Hlt+OOsI}*j@R8F~j)hI92lXC+U==(~{2c ziO+75udLj-Qm!moP>EL7>zvZ7EY*1>RavD8rIHdz(glIN*oRi^XW1?a9Kb;wLK|-; z1$<~n2M+Vb5IBNP9EG1Zmjnd5(2Zlfxh&9w;|Sn{BX@PYh6X7l36iex>8IX#&w__l z#->%qA}V87RmNsi#;&Q1T~`^KRT;aXGImpCEV|L)B%7CtS~cT9Kg~(HCFwRZ-U+eJ z1tybz-}wP;iu6E|79=f7dcYhH|K*7jQ~(80ThIAc%7~j|p7BMNGorW=`M|E+Yis3Z@XoG&ffSBDjhfT*GzD;s$PV^N~Ol zbGU`uxC4p1h*kf@oGWF7-jnpcq&$oBL5MFD=SF#~3|LhLJW&RGs0?_j3@9iAK2ipJ ztPJ=>8StqxU=5pGjxVN7J5SFftxNihNuI-Gh7H6SKSzS`1(J*}VKIJzdB!hspYba! zFuuYfmEE`WC2x`@PN7r<1m-q%x^PQu#|QtG2q(K^3>N%<7T zz5+@}F)SJW!rQa%L%OMMlDrr_+&WII>W%%9fE& z7p$CVmnV(ol1cT_qER#{7sJtTKYa&!qTy%Z{~N)oH=AEwDOO_Hf@Q{5R+r05D`tpbf`qF|eV9{~!s3k0!=f{;KnHdD|humvp?>=0c6+#;(`heva+>VpCJui;^Br^tqaA8kU>A0y9eb!nr@&t9!+vy7 z(j^c^C%SNek^=$<(TzihP;yW}pa;D;Oi8yu6h{!lQA!T|<;!Z1c~3a4=fXE92}n1F_J zIFB)mV*-=Vsd!Z20xlvB;1Z^gz%&*61d_On8C=0t%;FlZQ?XxQ4)eHyo4AD(ZX->_ z0T@O~9z%xoII^TCkRv?_lXMUZq(it%I*diq5iF6O!adT{xKDZp%cN)Z&p&Fluk;tA zWK#b6Ywd5}zOUaiu4^6cvI$-5diUv0eP&YE zy4`zpU5otu@U=d3LDza+z(rkOw-b*m)mn{w0V?Swcu1$&%*C);zOrFzY6>ZZRvxtlP{=GwTks4Er6mX_@sG zld>!$bBr_@xxmP~j9g^o5+m<1@;)P%8M*SxZ@>IBY;HbI8@6{`U}TYzl93i8ON=Zt za+Q$}82ONqj~H3ObHgml$C+3Xswd?A*U3x|#2HolC|Wff5d Bn`r<5 diff --git a/tests/ut/data/dataset/golden/batch_07_result.npz b/tests/ut/data/dataset/golden/batch_07_result.npz index b25854e17998a5c418de8c2b8604d9902cc9d10e..c5fca2c73afbf1f367b71e12f8b87c64ae390822 100644 GIT binary patch literal 1826 zcmbVNNp}-R5T22|Nh~&CvzW!Q!KiJF@P;=Q4;Vb456i+}kbyyrq%k(gl3J2U6iiSS z<6yFr%_JdC$a2ejx#b_^7v#=8hvbs8Qr#nj{PJ>1zgP9T>#OSOsjs>x2xmpTVvlC! zm(@q9Csm3f#HG|LW;z}2^d(X`RjHFk;V3{c9tS4Jf>Ww;N4e{bS(#|sJLK^$jC(sg z-sxmIn>G{SWIARE^QalmSY*%4m??|aY_P-A-PYl`=lTD`op(kP^K+@3FPcnSzPb5$ zc9za-=G@`56-_2G+4Oug>*k8>ik z+`KyEpwAp1;5AU7E^*fccQZIcDp?o!x+}J+}jg zb*;U$i>B)@4pvsw?ifE|@RPL72+656MJ?9zw5+ErrNKvK6^jMbWdUE*?hI{kR@Saq zHoq*JteC;UupK+lh@Euoa}2w%8+*_siathx0R|76(Sp5X7-!grR_uqD7C}a#ZD_{< zTAXL-z(M$MNRUw8ihaF&!r&KZtBWL)FPnpCxFq}Wvh2r@-H%h`bVc^#RoRc%WIq~q zKVBy=Ec@|>>_=1fV`RBTTnHMnEz6Vlbn~deV-#kQO#7+)dCGfJ=ADswXYIVV2)`}! z#%10)nKxnQO%j-rdAZD+mU%Pc3a+^cJu#j&_&j;vA$jFrq>277^7U$oFVX9gDud@r zEQ!7@67*V$pmJ{RQi6pNH)XWkE74M7#Ne+pbfOF0=s_O~QQ@l?De9_#5>e7$Ato;E*91O= zMTT!MO?nkKNq>tO0e(l|Gt83Vd)y-Z18$T45pjA7tt|g3;tN7`)I0y{w??g39C`7Q Vzx>GsQ7M0Is-i0W1pF>n{{T8P3*!I) literal 1509 zcmbW1%Tp6s6vn&rfFwRZ4L^=D?B zT@>sUh@y*veFD3&hl1|^^>o|q=XP$V1Ki1Na*&(2{6oxl(LIcIshYrE>_a#9Q;S}K z12~97=%J)fKtnJ3aF~+A0!Pq~qli&*L_lBwgE&S>zd#(vk-!N`j(+FSc^a<0m}k$~ zzUxeQ&Yb%`a09criCdV%ZQP;aNr8DR;4bdr zKC*a#92HN&G_!ITdD0OSNKd0kdIlEhC>BY_@Q8FAOQaK6COwPCr04L2^gLEbFBpIS z+iZTJSB#QLWAdBvNoys#p$EO>qM>*E^U9sMWawSq%#@*bzkmOsbYI@B@zZd)r6g_G*>RbX6-G)% z+Kj9+vc||YMm}ZaGe-Wx$U0t_R!#nzPh_C_qW&M9%=AFq(QuBUjdMDKV4G7=={HK( IcYId=2ZOzvZU6uP diff --git a/tests/ut/data/dataset/golden/batch_08_result.npz b/tests/ut/data/dataset/golden/batch_08_result.npz index b02f0eb324d9bac2f965d03ae35b37741938e6b7..27fa114d57c8de10023d7e0a1a92bdd7d8daa36d 100644 GIT binary patch literal 1781 zcmbVNNp}-R5T22|Nh~&CvzW!Q!KiJF@P;=Q51R+{VObaqGBAjdG{y#5a*bpX1rrn! zh73B=Dbncil3V9~UE8vc(oL!<#p-?DT;2|AGv$Q$(+@LKY;^lfgos4GYQrVsX z+GZ-QrT^?J27S6#T@Lznt^W7rQsua=HI*v^ zy4Jk5_JgyB5eNCmAzf=JuN~I4wpYL0b;<^f=vsSu;|X2!|M}zVV&$Z+b(T&s=#;Mi zbEk?{_9p1GL8AZtJo)lbUyl` zz;5h8GxqWWT@u)b{WyRY4uS$cw4x0MIe@?+wBs=R99$M0cAyhSI2ae`!chcp%xQh4 z><0QNWY7eIgxRjFtAU#{DYbW1YVVrV-jvkd^`iD}@P>%g-n7)-O{u+GQhT?h_Kej! zNB2CW1S@g-dg+cqQ9j}>n;GP$=NU34A>$GY}Z3%6n54x*JWp!4w=qPGot!gtEE zy%Xe5kMr$$D#>_PlJTA-V_A~%tVqTRuYF&V@m!LzD#`djlJTJ=<0EX~JvNsz?Hqk< z&?mh0Q?}1wGR_Oc*uO-a{pU!q{{j~KFEQi1!C&#huW^qPYnWyK4d&Q?i~H=q!vpr; sBgvmQD@(6pfiP58qx--9R4lX7$U7hB_5W5F)$(tpDylNbg`Z^fH&CAVtpET3 literal 1509 zcmbW1%Tp6s6vn#~5|Su_8hqe`2E|-3MnvW1gI1K*d=esxf*NBdX#|BNw-XrSqs+#l zbn9l^Oj+fA*7G0uZ@9NC-MEgYPr4ecsoBg;ENQX(3y0``_$ZYdO=mzl^3ysRmt12aV_?8CKlIXizT~i zo5gI&E?7=^%3P^fRIe2rCof(xOC{QO)cZhc(=NZwcL$xxp zT~Gr*1swt{2vD$7Ac$59b_s+KreL=~1luTx3T#Ik1)Tyr&`!Y~fe!4XV6VU~?51F! zKop%6>=)RBy%co)*3)fwfZMs94ss{A$sumy@((lLMRzmWrD_8EupeDGKrMO%4&o3F zqnnam0S!Iq#Su!52pmNpjv+?LQ2~K|4B$8=eFAZuKmsQzIrcA)&eL%1#XNh?_FZSf zbLQ;VHP!E#+Tv%x^RvauanH&Y7jc0>oWc-J;|$JX7$aougup1qFphILj|;en2`VN8 zbX>w^OyUZzVhRQoPYPVabtD1Yz%){rq2i!G8aFYETeyum+`(Nco)VbH0`B2H9w38< z$Wn0#rkRnakt02WJn2~!NQYsOj$n~=6pu;AutYkJWzutaLV6xgNiSf9^rG?kd!zA% zUNK50jESGd-&!-#4L#r`mkhoA>l=6GvY~f+Gn0ni_2I+E<~>&oz1v%M)zEwY`sYz| zX3Eg}yn75okNx=cr8#rW&<8xgb;H=SlT0f0MuU6-D(MaQNT=Z^O`(N!1_9DEf}}Um zN;-=W=`DmwZzDoFH{Ezmmpe)7R;S=Dv*ww#z^r@By3ec!%*rt9A+xf~dc-W#`9>XD zZu7&WEYHXSBP~WQGV(Daml(Os$R~_^%E%Q)uD%O7A3qI;TTIf1ogJ4LS!SeUq|L|* zBdd&DW8^v`|6t@ZM%M7!w5sxXE|G@ni~9fTWTpq=j=FOcZJyI<1cFXMrQZl$|Kzj! E7hs^8WB>pF diff --git a/tests/ut/data/dataset/golden/batch_09_result.npz b/tests/ut/data/dataset/golden/batch_09_result.npz index 6e8e923eb90df1d9468cb19b9fd5b64a5ecd0d57..5b1f3e7971ae91038996a92ba5bcec83e116c127 100644 GIT binary patch literal 1727 zcmbVNOIK4@7`->)mG}S^eBgtCVh)Irm%MxsMY)`z+RgzX>m`B$g? zJwE?@#?IMhI+n4MmeZazQ(24q*#&di;=LH{@%4B0_@4Ow|MC{xiS)|Saz2pA*j8X^ zC6!y`^O|{Y%(fDlbT((NBywKTN>6#y$lxLRA5G@+ z%a)hwqb~k8r$?v(8uWGY%#c^W9o4zJN?XF=aHz;j0mgH*HU3=EHWBsmJdsJovWw|l z-waBaB)mvH}+V35)ZRECz(%W%pCPd zIxZ=|hdE*V{ClzZmZyB|USs?(<40d<(`N*0E1P~J*!bI8c`{%GTPl-5BiQ=p&G)6x zoHT-MmB)sRV8?eq-78IojbLZxGb2XO|Hlt+OOsI}*j@R8F~j)hI92lXC+U==(~{2c ziO+75udLj-Qm!moP>EL7>zvZ7EY*1>RavD8rIHdz(glIN*oRi^XW1?a9Kb;wLK|-; z1$<~n2M+Vb5IBNP9EG1Zmjnd5(2Zlfxh&9w;|Sn{BX@PYh6X7l36iex>8IX#&w__l z#->%qA}V87RmNsi#;&Q1T~`^KRT;aXGImpCEV|L)B%7CtS~cT9Kg~(HCFwRZ-U+eJ z1tybz-}wP;iu6E|79=f7dcYhH|K*7jQ~(80ThIAc%7~j|p7BMNGorW=`M|E+Yis3Z@XoG&ffSBDjhfT*GzD;s$PV^N~Ol zbGU`uxC4p1h*kf@oGWF7-jnpcq&$oBL5MFD=SF#~3|LhLJW&RGs0?_j3@9iAK2ipJ ztPJ=>8StqxU=5pGjxVN7J5SFftxNihNuI-Gh7H6SKSzS`1(J*}VKIJzdB!hspYba! zFuuYfmEE`WC2x`@PN7r<1m-q%x^PQu#|QtG2q(K^3>N%<7T zz5+@}F)SJW!rQa%L%OMMlDrr_+&WII>W%%9fE& z7p$CVmnV(ol1cT_qER#{7sJtTKYa&!qTy%Z{~N)oH=AEwDOO_Hf@Q{5R+r05D`tpbf`qF|eV9{~!s3k0!=f{;KnHdD|humvp?>=0c6+#;(`heva+>VpCJui;^Br^tqaA8kU>A0y9eb!nr@&t9!+vy7 z(j^c^C%SNek^=$<(TzihP;yW}pa;D;Oi8yu6h{!lQA!T|<;!Z1c~3a4=fXE92}n1F_J zIFB)mV*-=Vsd!Z20xlvB;1Z^gz%&*61d_On8C=0t%;FlZQ?XxQ4)eHyo4AD(ZX->_ z0T@O~9z%xoII^TCkRv?_lXMUZq(it%I*diq5iF6O!adT{xKDZp%cN)Z&p&Fluk;tA zWK#b6Ywd5}zOUaiu4^6cvI$-5diUv0eP&YE zy4`zpU5otu@U=d3LDza+z(rkOw-b*m)mn{w0V?Swcu1$&%*C);zOrFzY6>ZZRvxtlP{=GwTks4Er6mX_@sG zld>!$bBr_@xxmP~j9g^o5+m<1@;)P%8M*SxZ@>IBY;HbI8@6{`U}TYzl93i8ON=Zt za+Q$}82ONqj~H3ObHgml$C+3Xswd?A*U3x|#2HolC|Wff5d Bn`r<5 diff --git a/tests/ut/data/dataset/golden/batch_10_result.npz b/tests/ut/data/dataset/golden/batch_10_result.npz index 5568b7e0cc208d6550f2d69b667437074e1246a7..426e1567800aad0b501708d1340554f496d24886 100644 GIT binary patch delta 47 vcmeBR>R=KI@MdNaVSoTd1~ZT7;)z0OYz8`-ItsND8!abp7hwSlp0@-5&<_hY delta 49 xcmeBR>R=KI@MdNaVSoTdh6j7nnI{URaaiam7-;G!)J|-)oVZhn1uT5t5&-k%Z6)m4)4gH?gs{ zk=O7IEPQ}!6d9$&+~ews(~<^8JAtdDj`N#BdlA5jo(hfrX{%WIr>kS^{3(aEVc;SgdM`} zbh;*(Wz5*vBg6^&goC2apJmL{a7Z{J91~7dx}`etm*8|lKsf6w{oE3M5pW0>mI#VD zXM#2+tJa&c-9wUaNk|dWLdD5)NCoSy-8EXW1DlW`WC^*#=+pB)l&DNj2l;BUBn)o) Q{yWt$#uLF8bv4Gm0pa_2!vFvP delta 288 zcmey*dW^+1z?+#xgaHB+8SZuLIJJ+FfdPa$8AKQoi;Ch6^zsTS85sn?f*^(a7$=&# zav16;7;5S$)J|-)n7Au|h0&X_eKHTDdwpynpGFr4)WimF#?;tCevQ~d0dzq@4R0oI z=2W1x5Kvkeh(*wqh{BbK0j0$?yji?if$AkRyxF|jQ$boKy%Q5-3#HtTzyJUL0npCG zSo=b0FNTKy|AE3XUOKka88xy)*B9=iPJ9%qvEGLYl*j0gl4K`wt&X zb~yAm)sg1#g+gVcGpcJFoQ{r>q+X~e_1Iasu%LK})3M&M!4nA9`9q!wZqLRdPrBPv zQ4

eAQ(&p+HclXZfn?f_A=cwXZg4$7+1KJ8M|Fdz1VB4_9Nnzq)>1Z9|5?CKSwA zS6>ybwE6lPTxFr4zoxn_9IE$+U38dLuIhl9r4x=VjGOK(XZ$uMlrT}jiB!aGWm5Cf zK)9hc=wfndoc)+ZxlDlr&TULBp-UmY)M;lovQ1%OVSbZ6B0tCrvu*iyVLMgovgiFZ zRb_RR)#1z>wik9#!j8gD`Hse>CZA8(S=a?mjnaf&5vNf%VRyu9)I-=42^#eh_C}&c zeT03Hq)|U%emWloN+AJ>Iv6d?R7gUy4zh&V3Moj{!5HCK1sB@rAV)Y( zp)K0!AXhkEp*=e2V1jU>LPvDc!6e~ih0f@rgDJv1g*0^4fmb+Hp&PmfZPuwaGhN4u zsv2LI(}dG);xmLZtwYC}nvU4B`Y~q-Ei3PYwa?w0m@S-Rc@txad?74vhof7fE9MFd zEN|D?ib7$L;!y72tBA*iW}YSay;uQ8iWyeGxUHB9sgL5fP%b4+w*i z+DA)XAzT$Hd9>22g_V&?n*s~h2-hm~Ku`2SZ(aT>;W~vr=!<^nuM^e68U;57U?2wR zM6HktgE0g{;n9hZuuef?7=~knPK1T^3h5Y$42;q;wzfmg7_JwdVn6Ar!iHG?X(3M& zZZMwMDBNT`ak_A`@kFEW4C9G2g=ZO0oGsj9Jh4@{&3NJ*;dbMR9l~>sC(aX|Z#;2< z@WLG_+O_tGSfe$&>?~d+yx3-UiEwAWtz)AmcB$|(L+o23 zw!=Q*-7W2)9T@F`dxZD4bb+>6RQ&sd_qT}OTJr~l54LFDTI&76hgzhLDJ(o7d{|*L zGLeOBjKNssV4UuSM}&_mEOHVu@d{SXH z<{%%yToj-XMS9{X;nNDmn1>R~#{w+GA}rPu&j=4Hlwt{%Vi}f0umWYRw-3Dy=ki(M zb9N!07aq#DH{r-G{DSaBBkW7UmyNKm2wyeAz9xL#2>XWcO(X1E!ncjEhlTGLVc!+L zXM}xU_<<4jL*Yk8*pG#u;G|9~udMck8u+R3Gn>#6;pgz_Hv2;OCCWAWO87PW8hs=D z76FaE6Mm1NMn4FTqC%q|g+F1HMn4OWVYNobg}XlWiY*Q}AfDe8OIs{q}pkSXsE7~a7FAzkCf&&6!?4lqd$ju%KItBKkoq~e`9oR>~A%Xok zK*3>w2s$Y^B5)9gDCqi6Pq*DsZs&G7#+}?I$GM5iKf!z#-OXs18WlKm0JbP;U zt~22|Q~PyI4S1$@_!;#4?67j$v$DfQTwn;p7{NK5#|2!(C|NrrFotnV;1Vul5?3%q z#e{%?tGI?~T*nM%VN&s|zzy6)62L9YA%%G=4hf`j8w3QTyFQ7nr5jN>4R!GP2lyn@cq!U;py@Y3^m+_o*66>T_%+G%{ z8e8-iqh!jQ`p5hjZ6=0k1ia*`X|(_O+MT&(8lB$Ev}ts`fB#4Gnd_#}?Jb)zjo#ls zJ!#I&nnu6(jAXKt9rpa-~Vn%j1gNv+;!kS{pyw2;mtK$=D? z>20);E+9yH2O-kC2$L?(HD1x}UXr@iDY(z9C1x!%>jASKGV2kuGR%6+tSqyhFw0Wk zsG{YzKTOKnT4TxaCQyP*2`X((E-|vqNXbZt zkrhT(8M(>G7mWOYkv}rBhF6wdl|SVYY3RO)|9dAhy%2ZQ)lsy4PNxxQRRx`Xqjdj` H&+2~xalf0d diff --git a/tests/ut/data/dataset/golden/shuffle_01_result.npz b/tests/ut/data/dataset/golden/shuffle_01_result.npz index 467ea74a4c48094f9294ee43a993c9cca451d65e..589afc1271adf03c2eefd2083a2864be85a35ab3 100644 GIT binary patch literal 1691 zcmbVNTUQfT7@bK7B=G`D@PZctiaB765RjV}Mo~tyB}5br>KG@t1mrkWzTmxIo~<^+jAxbZ(B%ej((}W z>HoF6U6SlyKnh8^X=X-ZrAkefxZMHC%o}!iUN2S+idWb4ia|b`j)vny(eT^w{~y7+H&@zNuhe3>vT4NDH;Pt) z_N&)|8Pmv>OI6F<$XP)yHD?7&`DT{OkhZvIPR?@gHg8+vJ_Z>9c;54NXKLQ6Rg56- zNPFm*i&NYW3Gy}%Eb$hA7JSq<#Xj?rdlXjqZ9m~ z#ygqDyWrI|eh41I!wIS0XpqqjnJ@%DVGja?N6v9aU30ly$BMXfKNBl zb_eVVqj`Q(}(M`r# z20sF13^MG;0W!`p9KlgCA`A!7Nd{vGp@)nih9LGCMmj)C+q+mTmUW8{Yy3QIB&sT% ze{6**MZ3MADn9jR$BC)G{n=>z-ppK7Xa9CHBdRj!I-{x*`RgZVT3l6n+{~D&^tnrq zt7>btOR6&P!!OU9OHZiE@W+ooHl0aT>2;hIZSg#x()cv(c1E1nIpJl&f;gPUuk4}E zVeYCR*J1h^Lm2%Sz!@TCmfaNh#g3we_XlhYt zDz)RcMN7k$);%GuB_XZ*LR!1UI~L~;G`>vL9*Sy@M74B+sIAkVS45%5qRx*grs%vTH}7)2ao7{?_{V3KAQ8PZt6 zV?4oAXn2MU1=ks-FpU{p#uZ$}HOx}5#E`;0Ea5&LU>Ohbh=OIe*{&Q~uakXkO!11w zIT2$D;;VvqD?yireQRt8;$I8mKM}-#Du}NM;y;7U`WwOereOW;p3Y_qCEcv?&o%yz z3e=IO*_$740uaG7D8U?~Fyd-=NgK!%!2){v*@B`KeckqhvTdeAaVe#*>u@q!a f$opU46OxTH>h=}d`nsgx6W8MM$2r} z8n;tr6&uSc_p_dF;MZ_Ji}lDEn^J2DZDyqt|h!;#gZ zU9mHTbkWXRDmd89v&HphF^vM_X^g$xx&_FsT#`_Z7a69wNY88 zelyiz+O~4VLb+mZ2l zfe<>$I3^IpJ~Fzv*hwE}gp)b(kG-93PjY8Yo2LZA=s_<|6R2CD5B)fU2svTy&S|Vi z;3$rv3&$zbD-gvw#BiRR(*h@O5~t8jPM?6l00wcEoPKW9Y5$B1*i#;Hw@@QFmEvDL ztAljf9dM^p=i`h#C}7|QZej|za2wMwY4@`NcW@VR0QWG11ZF81b*J4k=sAHQ3}XZr za1obq8KYDg6IjAB9$^KKk-`(CDR^FB4C9!<6NXPMvbOIZsSFlEU6)#AyVU_f{X|(J#8K48{EW-Y~Ovl<1Lne~WSE6jS#tQ51Js4r5{(oB2Gv<%a- zOv^DXzxzp2t;NVyMm}TY8Y9;k`TV_Ked@Fn_XTq|n7hf`0&|PZEyZbIHL+x*&B!t% zD~#M?Qz5zNxBql*UFm9*B>U%=f|72U>EURxT$QCxv1upTLEFAfEX^mEW$BglI-D~qSu;Ex3U4li zBcbqW$+S$pm@b()!``0L3l)R*D{Fe$pf#I{gkpn{(3{Zzw?NICEpDurtI=%9G@|Pp z1uIYI)vJNDX=F>qie+wOtpJxAS%G4%5#=(ZE$*3>Gu*q)TNZgMgA6}B?|7S&nzO29 zBf#5J9(v4$N$!ILd7Jwed4R#2l4*C7?@J^S@j96R%vgMX=Do%buvCEJ*-{~0$rr7W zaeh$a9Zch$@aY;q1ex$~T&mUU~{kD-O|I0A${@DiRt zt8S$13D_M*a{Q#mPf_*L44pV+XuOxf11~utw&SBT+iH+$qmSV*y2$BgID(_(oMAYK z4sy;iw4jxo0fzlJK+ZV^AN=Hm8G`60hcUFFot#020QMP1%1@>3Q>+$By2Xbyex42z zQI!Gprx1ncv=`KwzuiSt{pF8({kKNsqN;@d{J~iat7DL${xPT}w(qe|;Bu?QpdTF5u@&rp7zs_(B$I*im6uBYB z+d@uvgq-5dNfYT1n!3B=v&Bhf zOKV<8DOWfyS4p+e6Xqk?59+6VEj&x-1$!w)JoK zcp`c{6+JZ3V@32xiyqHJ4}Fh(M{_)*@hsKPF(|l>8<@pS+`=4GqQr1%v_O z*wS@|_MG81?jR1}F6NQI0+m@~NMRX|@dQtyVFhX0%ri`43e&iZE4Yeln4!%V?sRO; zJ1@yghGC2#hEa^+62>t>nFUujy9BY$kiT*^OR_XUfrME71C%+`J^%m! literal 1507 zcmbW1OH&hB6vw+00!c(c4LUmWtAJtD)+OVZ!lkD_Ooy{uH(5UT@6!HZoH}d``mL*e)pVH>6?PTC89M~ul9TG z?dO$%rm1g8i)dNL$qdDd<(jT_vQs6hpjvN}i*u=EU3;$m94*+Dyc3-;qA%v7F(bNK za;i?Ym?=30Te)Ym8x@=Um9=cyro9-C8N(w3G2<8GyH~jG&lk5g%e8pEZTT|wXz+S zZA(6CbAC()p+SEwLklu2;9t_o`$FzYCXf0b zj85?;sAfj14xJo(1@>V-8T$kd;2;_M1@@qWjEF!B0%UXvv|~3J2Lyr$k#SHUf-W)+ z3ACb(jKcz9>>{IEz(6nhaDp~R7~!TI{mSkI>AIpj#sBTIS#Pf0IggLD*Yq+`gFj^hdG1Z>hvct&~|1=1^6CB15y z{nj4_{h??xX-$6dHm3F0=SJi2X5^Y>M*scfZmwI_c4W#j`#${nxS2L>nE?+lW0^hw zyfm7TS(izJLF*8M$GZ9bVc^%k1_d2?VJqKqtMG(CUo_8FL7cCg&Qz(qkTN zq$&7FZ=;oT0WG9=5GK6~Kj|U@r1zM0pIJ)@I=N2HWoA84Z=Ry9Fzq4J(oB2Av<%Z8 zGcC)s9MkelD{Q}##IPB;`X;DeXyDERm#Z%CI5&yUDWg3FJ>gp=mzNJ$Lv?_;ApHX_g H`G6I{m diff --git a/tests/ut/data/dataset/golden/shuffle_03_result.npz b/tests/ut/data/dataset/golden/shuffle_03_result.npz index 6a6e62f3ffa75ce7a915a0f92fccaef73dbe3933..297b54d9cac8ebe73a18a777f816f1689f22e7a5 100644 GIT binary patch literal 1691 zcmbW2+gB4;6vk%~0wi8Q3109*Krsi55dw1a!YImUK1m{q26c>+WRPe`awdaqD%RK* zskCo>@};Y5^}Wyi5BhKT-iP+3Z(S?hb21&$)zz1tweqcf&OVua^4nj|q~L7}O3g7K zC3^o%_Owfq{qajdNjJ^RXtY$R$x^piv^UyGdwiEzUQDgX(o5-8IB!&QW_UIf-b{ug zq3~+iv`oE}DVur2US80PRfE>6YkI|?Ih%=uV#ATp>(Kunfx0(W+E}mDqPenZMAtWp zR)Nl|*8&;S$dyY~%iPFW0WLLn1xop5mdlW~xMyC@a_=^8TjD+j8Gd-)@pflx-l|oM z0Pje9=w~LUc_$>u+uXmz0}S4@OskuGPa=_sH)sgKoW=KM-)npyO9v>QD;G1>LdhDN zm+Fm%uJHqqNe{w9+5<0X5N)KrXeT`cAL(Iqk{&?^=~4Je`w$>Kw%B;1 z8)>@(c7?G#Kd$i;^z@T#r;`@#QHN}3yr01XF9jinHuxwQU}#4N1%nKo@KbP#VK4Sk zaGD{2JroQvbmIU8XBhUQi-Iu2LG)0-7=q}fV3^?$4jV?=PsQw0tQO0<#YZ%LmJSk8 zfBCJ^_(&-_?m2bt5A~-|a}re*pF26PDqX*BIhj#a>2)(>s?zUfVye>j;lmHj2aK!A zkh^U{RR+KR`DrtAK~=)P|M;;vnN*b#_d8Rn`uA~mvBmOyTH`Zx+Sz|#bBJ9O#2i?c z1S|*7WdY0~b49Rn=*%%3!BO<#7!gtgDnUu(R~bSWz#vXh<{HCsoWM!+(`26EG=^{n zVVYcLU>L>-&eG%tLj>m##d*TqYNyEwzMyfHPJVN@-vs)W@KjuQ3c^#jg{KyUrxLjcM4{YOOXN`2ivqx3ktfzx!VI5zs6rMD7cDin8$V8zyeg-S!9^T3}$f=mv9+Z zFh`5)3^#ELaR9fmhy;?fSYk-w4wi5i_pprnctDF~hBQ|25RdQ}8lE82@^`YYjcH!d zI43Y>oGuOf)~E{bmH@vYz`qpWKM~+R72s;@@VYDaf9n f_rKmJWFcqN?JIQm?UI5|yi;UJmZm6?5VOAkB}>u} literal 1507 zcmbW1%TE(g6vk&-3T;I}Ek5u;L9rLC74X5s2csyX{iIYB1+ms?I|vGG@3g4(5xY^N zTN4vE#>BmA{{#Iq+_-X~8xwc+%;{vn7&qRg-<)&q?eBc|rgM|kRc|)GvRtghvIWbGEw3z<7s;N)^ge=^0Zq()(OCOi7=BiU69Q$wp^su3RaaLD@9p zp>0kMNk0^*&t+gr1_it`DpeoJb%{hGUZcT+(XwpLyhzFQA|*G7mwx+%XjGw^DcK_6 zftP|-fd=>}*eKA5CJHtQ_z|FBvp_S}QxFmeVjTr-0_V4FY) zZ4_)5*n+JTwExl5X}5#hIh}TLCx^X@yE)k1+{i(92yDZ4v||TR!`#Pd(J2r{C%UkQ zX1WA+Vi$I!gOWW0d(n-3h)}XuK%fV`*iT8fKokcM!$C^+{pHa<4aZ)@wP&A$<4m~D z?EN}EdR&KVEcCkO)>zpu(1(5u;1CYu2##WqTtr=eYpfj*&~OT;F@`fZi*e|*GA1yD zVT|Ayj^hMQVw8#p1h0i*Qh6IdcWsek%jt9_xr z7$u|n=uiDaxSnXb=5v!%y4LdTsWWq0*V^2fFVUJl*6l7E*R`&< zA8*!YbX|-5c>krIoYS>l_c!Ntebr7pu2gF^@&%}*7vLeCfR{9Z2GU71k|yCJy$C<) z6q-mcAwYT=LDK1o+7o(QiBq>K1y`9h!>n0mU1Qc9v#v8M#jG35N;B&wvkdzkwP~5v z7n8CqBXf*289C3$TZ~*_@~G;D4@P8+s&Twr98k&=-XBTI}d zGjfHIcNuw)k@p!{!4tzQ%LkcQ5~?TU{nx!r2jYyXeHE?V(sZDWc0r}jAU$96P5m3w C8l2|< diff --git a/tests/ut/data/dataset/golden/shuffle_04_result.npz b/tests/ut/data/dataset/golden/shuffle_04_result.npz index a3b9469f9cde16a39cd991b923f5817771b1c992..704cc82389786b0f5ec3de32057361a52b5d5c6b 100644 GIT binary patch delta 491 zcmY+B$xZ@65QcjeheZX{!Ce;DaRFI`c=L#%E*!aZNRNyO2>}YZaDuZ}9)rZh_z=E; z58%nGCleD7V>J?CD(R%V`paMSSN-TOMxg?lZnD2fGW}b98>@`T6JRD2-EJ+DZgu(! zi%gOd9IMZTm@=pa>3-L4)*c$IUbcuKj$w;qq|An{D>$xzBAlu*g|c;A<+w(GX+8R- z48;%>hhx+N6*R}G1}G&3@0)GW!*z}ugm0E!KVA2ozDRScbn{huO2|iark$(fHpd-; zcSjw?9PC-3!vYxKBhmRTi&q%q7$-0>>w#pGlUC=+d=B?H9uRq0rfNfijy!(ybv)*H zLV)G*Lskabe|e6l7Wg3mL0E((ScVXU=Zwn9UE>+YbJDpe)7<3XQl4(?MoV=2nC5s% jPzD4MS+IrU^e|_qE1;;R=GmZcy28<|nle6UlZ^cWI5~W5 delta 296 zcmdnY_M62tz?+#xgaHB+864j-EdR{NzyQLW3?dAPMMd!jdU*wvj0^%`L6E}Fj2lhm z80}*V`82vXpjsQe8B=2m`8B+myqSRv0TdaiprD2~i#Mw`n>Ra9h0x?ICYgF+4Q~!_ zPH!%6Zf_oMUT;2cexQ_ycVc2}p{REfkQ8%2{{H{}2S8hZ9C3I1ukPyBql*UFm9*B>U%=f|72U>EURxT$QCxv1upTLEFAfEX^mEW$BglI-D~qSu;Ex3U4li zBcbqW$+S$pm@b()!``0L3l)R*D{Fe$pf#I{gkpn{(3{Zzw?NICEpDurtI=%9G@|Pp z1uIYI)vJNDX=F>qie+wOtpJxAS%G4%5#=(ZE$*3>Gu*q)TNZgMgA6}B?|7S&nzO29 zBf#5J9(v4$N$!ILd7Jwed4R#2l4*C7?@J^S@j96R%vgMX=Do%buvCEJ*-{~0$rr7W zaeh$a9Zch$@aY;q1ex$~T&mUU~{kD-O|I0A${@DiRt zt8S$13D_M*a{Q#mPf_*L44pV+XuOxf11~utw&SBT+iH+$qmSV*y2$BgID(_(oMAYK z4sy;iw4jxo0fzlJK+ZV^AN=Hm8G`60hcUFFot#020QMP1%1@>3Q>+$By2Xbyex42z zQI!Gprx1ncv=`KwzuiSt{pF8({kKNsqN;@d{J~iat7DL${xPT}w(qe|;Bu?QpdTF5u@&rp7zs_(B$I*im6uBYB z+d@uvgq-5dNfYT1n!3B=v&Bhf zOKV<8DOWfyS4p+e6Xqk?59+6VEj&x-1$!w)JoK zcp`c{6+JZ3V@32xiyqHJ4}Fh(M{_)*@hsKPF(|l>8<@pS+`=4GqQr1%v_O z*wS@|_MG81?jR1}F6NQI0+m@~NMRX|@dQtyVFhX0%ri`43e&iZE4Yeln4!%V?sRO; zJ1@yghGC2#hEa^+62>t>nFUujy9BY$kiT*^OR_XUfrME71C%+`J^%m! literal 1507 zcmbW1OH&hB6vw+00!c(c4LUmWtAJtD)+OVZ!lkD_Ooy{uH(5UT@6!HZoH}d``mL*e)pVH>6?PTC89M~ul9TG z?dO$%rm1g8i)dNL$qdDd<(jT_vQs6hpjvN}i*u=EU3;$m94*+Dyc3-;qA%v7F(bNK za;i?Ym?=30Te)Ym8x@=Um9=cyro9-C8N(w3G2<8GyH~jG&lk5g%e8pEZTT|wXz+S zZA(6CbAC()p+SEwLklu2;9t_o`$FzYCXf0b zj85?;sAfj14xJo(1@>V-8T$kd;2;_M1@@qWjEF!B0%UXvv|~3J2Lyr$k#SHUf-W)+ z3ACb(jKcz9>>{IEz(6nhaDp~R7~!TI{mSkI>AIpj#sBTIS#Pf0IggLD*Yq+`gFj^hdG1Z>hvct&~|1=1^6CB15y z{nj4_{h??xX-$6dHm3F0=SJi2X5^Y>M*scfZmwI_c4W#j`#${nxS2L>nE?+lW0^hw zyfm7TS(izJLF*8M$GZ9bVc^%k1_d2?VJqKqtMG(CUo_8FL7cCg&Qz(qkTN zq$&7FZ=;oT0WG9=5GK6~Kj|U@r1zM0pIJ)@I=N2HWoA84Z=Ry9Fzq4J(oB2Av<%Z8 zGcC)s9MkelD{Q}##IPB;`X;DeXyDERm#Z%CI5&yUDWg3FJ>gp=mzNJ$Lv?_;ApHX_g H`G6I{m diff --git a/tests/ut/data/dataset/golden/test_2ops_batch_repeat.npz b/tests/ut/data/dataset/golden/test_2ops_batch_repeat.npz index b4346fd796775cc864759d2729741b31615ea0d4..cba3a7fa01d24e67810ad5b4c89526c333049b92 100644 GIT binary patch literal 3602 zcmbVPd3;pW6`sjPGVBocZCK|5A&>+@SOWwIT_~#kF+>wbrH8_?>g-z5#we|8e-8cfa$U<(?DHy>F(LGPCjn z<}*6bFYC#Q13d!)mb^fIAgXo!w8BJdM=;PwNOoi+e-2i!Sy{V27}y@z6^XlTF&&v5 zj_j_96oeyAiTV~DcbQ%qZEkbHw{4ELy3kcwLAZEgL3mI2%YQ5SWBWUsoP`8kQXpsp+wwRy&whlc^MVK1}}5Jm$k~vRurUM z%DC9eNx8<89j$K2>sFV6HfzefTnbQdzn8bl3n|L13xeJ4b+4|juIj=^9#o$6dNf?> zcs*5J2=-V@bA4M=B3V@G^>V!4%JKR@`c?%xySk!|*N=jD_NNRy^C=V00hASW9d97K z22nPM!IXpN5bB2KP|C$~80E3caJY=HiAGAIQIaSuiAI}5W8gJb5{;8Y<0a7qNfhBk zZe32Q&Z4*n=ZTO>kjd&mE)4wXq#*VyP?SNL08w%u!vzt>zu&a->X zw|kY_y%yNL7N+(}cUm0x7CBx8;uk}fK$f0aL8Ykx1ulbmxhYjBpz9TdX*86LipJ1brlX)66pf?tP-2QEP{d_E4&H?{p4s;% zzwga3H`%_o_M=Y#IBeed-9 z-UWKM?R$^k_g>H!*uFdczP|$ctG4e8ZQt%`XR>sxEwq!+t(%AwUX^R$#%WTb^}}wNwym$ z+f9<~8{&O{ah0LLK6MbB>EM+el3Z9BZ+=1iGC-Eet-HcQWE!Wb-ddU za69A<$eob8&YZKm{W-e_=6mg&{UJ4H7|}n1zYlW19k!$Xuss0!m>sqUQ^SUkhrmA! zdBjfKqy9DXC(wVk!}gcdup#8H;Qt2syPdel{E2%U^gryx{WCRj2zdhhlaQzE*gfr! z-7}z{wPW|3KX#7yJPa>DjzeBlq-Y`%y`*RoP3GhHGU!(n70?tYuPQ2}sd7wT1OG3` zzZFfRBG_J6R7@ot@&@QP6-}oZP~K8hN;4(oZSe0vPADp)S+Jc{G@Isd$h)B5Q#6<6 zL3v-%d@7fa|A7Ah@?S*@Xd!GLDq2Jp9P$z9j}?=l@yiuX z-aa}{Fu#B}ZOAxuH%kw(@jJ=UJq7a{2}&Wi6$72h*^|bCW?;=NfSn4T) zQx|t9rWN)VS&y#-Y?KGsPxztIGtN(bcnzeYS8i zzZ&Ln~6qPRx{WHu!g~AfVB*o0M;?s!a@8c6+!rVZgx6{0q=A!1K#O82E3EQKz7;y zgrIsRc&CjFc&8`>-l>5B?-XNzo!q)Syo~tUZew#xG^yj@9qriLS2W za7FB9*2_PrJQ6l~6YC4IeSM4Qz2DrIvTqfAK+2Bhnn57_HsOaJI&`zilMsEBAJ-!K zn8UYjH1<}}$NT$uqDM~Na-8kRt3e=d8}p~sn`HfzdfO!~UGH|$)AjBU{giso2Z6jD z!l&!qDSEozU81M!-7R{$-aVrG^}23S?*-LRSWCf-{LC-<8HeUlpH6tAQ~RT?ma_RL PKp=>JGqE0(A9kMuq9Djg literal 3137 zcmbVPX;@WN6n&4GhYBh<;J^dLcThw`5X1o#1!c>SWobqDfajC~_W>4ZMT3zxXOj(P z**w@>+C1Al&!Y`CS*fMXMQfeMxwrVf{&e~FVeNJ9-s|l179UV8;Sq_h(39%gm6Nb< zdz{O~h;=2p{J~&tc2;9^yW5qZNxsNUJ{v0MS5z-_yH>eY`x*i*^+8{u*SDt1m+AE_ zZVI*r{f)Iv!G-|oQvZsU0Qi=r{^kI3Rgme;&GhDFWO|SH{?EtL5nkWewz9cBtG+21 z$XeO9qIDV0>u>ke1_Sj?jV-Ogw)$3&17XT5fTzwZcl=byj$%6*13>RN|Hx ziIB)nE2{hDhSv7xfX9kn5Qagk3al7$iM!K^oo9KJgfDP|U1P;nR#ujG;n9l}wOYOE z*40?ORgKk0t&iay;pq@}pvH<<5+>mQ35rSpeU(H>G(bNE^;m%ZN<0z=kf@}W^ae;$ z(nsO}1}I68z5oN2^ppMogL*0qHVVl`VTe%}Y7~YUh2cga#VB}1647X8Z?b1^25p(ygWa>Eqy>B{qRm{u zD*FPC0e!5wfcEYSK*@37S0mP#TR7gng%d!ZXl|jS`xa1g68Mu5rzm_#86ZwoGFrya z4{Jf6rX*9wBI#6;CF26*PY1sav0h2Gj0bUsk{p@9tusNNr6gDKkesb#qU7t6bHJaA zI8RA|Ov2vzN(yB%OE!SMK*}=Rv<< zT=1gff|tO*jCjSk;8oiNuYrEuxZn-P1#g0X3-Pvb!8^7K-Ua=hal!kJ3qAnt$B zf{$$%>;(OZalxmK3qAw?IbxS_!56j*z6AZ1alzM)3%(J*zsA}vby&ZZdaU0`1J>^) zfOU^7#=2LQVBIH6vHl>-u>L5^vF_Kk2XyUEy7p&X`-`sqRoDKeYk$|Zf9Tpjb?ske zJN9&S?ZmH0H0mlVI#9ONtCOG$zhBFWBkkO+^Z4CsyZ6I%o@DQb>pXem#?7Jn2%V?c z??>u9ZPT_zq5UYGr`z?>I`{3}vNNKq5gHKoUVXzyN{>fPn;&0D}mk z00t9810)l~01P3B1sDoonhzt%=EDitJcWSGy##EYO2Fo61Z+NnfXzn|uz5NGn~wr0 z$0c_F`$)36B4G0j0yZB_z~*BJ*gTVf&Bqe3c@_bik0W67Yyi`IJV`dsAzK($0hk#OTHAK&-u{{#L8p15JS diff --git a/tests/ut/data/dataset/golden/test_2ops_batch_shuffle.npz b/tests/ut/data/dataset/golden/test_2ops_batch_shuffle.npz index e3273425d396efc44d078aa58f9f0a6534b98528..54ff4435e016a073b000e27384c7f2786a15f41c 100644 GIT binary patch literal 2360 zcmbW3cUTlx6vbzEscTp4ZL#fB5d{>n1*|AXirBD>xa00{XgW=sN znx>ShrD*}vtQ?UUu5WT_?JcMZRZ?ABOXe1rEpTb8wKe{b(GWEKV}1U$rTz?`zq-zh zn1OI*of$F|o))NWFvuI02kH&RA}_<2JvhU+&iDT>Z?h*DZd_I0lo_lujm%YzwUHY3 z8)))YnntiL+z>GvgAuRP>?&_KWM`!d+6L*K?5dKUO){ZeCJMNa0{2Fl6e|ryn(7U& zOs;VAn5B6#1sYtNWNNwe3V149q-$lXl9H05C@<>=@*}c!)n;9`5fxtMgLSo)4K?9N zR<3NT%XUJS?Wqn(3FxvTQUiuAI}xNIfuJ)I73f0H6>frV@F>uophuC`9F3CogiEnr zWWC`f=u;eBuSj1KACgGYksL6Lilo^7Ss~d^m;IR@Kn)aVj}ELKB+wC^4F2UO7J_F_ zWiVL4i&hE`As#A_fJ6ov0!c_#co=b}*b;mCS~FaP!v);%sPqV?vjo!6nL)Nd7j#uP zhj^qwYqVi7N}w&;8AtS(9g?}a9L=IUY78}2z=w1ej}z#J{^~Qv6HgErfPoB-6Bq

uzdB414D0^QJ^ zxg`QU&{Hj`Bwi}e3%wZx1p1&aml#L+EqA1>(q)j9At6J026ut&J~!1$?CSAM|{4(6ij8XRbUzlttW8-`GwR)0@E>r zxr+sg0JY>2;%x#mQOw{{ffAIeC!x#Bn7Ev}g4*sF`O3JFcMxCY82RcJBkv@?hPu}A z$#rp`Tu*$15$He{A1Jr}ZPQgR*DR`LaN1Q2mv}Fn&BY&KF!kM`z<1_aZ@zc)C?R933 zr+bEpXQ}6?=LO0zi|_@3a?Dou`9&7VZL?$Um<^$dQG4L3z&OdU?CQ% zC2tVFDXa>r z@1s5u$io=sJ{1^?acaqD#Gea{#{>pn2porewN=ZcR@H<9W|RCl@ogH``cozwLXLTKgJC#B57cm01RtJI(W7Uv^j=yP8?e Y&YZ74Wk^&nRNe-cHk!ZpTQ9(W0sU#&2><{9 literal 1914 zcmbW2*;5o(6vlfP)=>c!TySAPaSjNGsDQXIiqgs_izo;x;xN+)vi2~T5HVsBLL{4s z*~nr{%)XdCG5a2~{|&z9A*od5nS6bFdZ@B0FS%9c%sIbv&)0Y9>S=^KK0~udj@FWT zq4)1ZO|vXT%g_Sha7Rh;z|g2mOIOgEXcet-vA(&kvCXBO(oXwA#&9s~tMvND8hnLb zU)Nwb5)KS>42DC7g=+$R!v^_qcVNh1E)|8|vO;fZexdh__kRx0xH~v7(mym>92^WA z#r-3Fkshu$FzV?D8^OVW;YfHS81a~zJ8HtFS zG!vtfLy^%T!(%43#&M$!6=pItxF*e%CetHux4KBj%sKV-^|dn`%AqP^&h0$kZqAc- zGgXj|`9`~$CRl)l4AOwdEXO!p1et<3xEU-G zWFV8lV!`N$;xa)YikOrOim}eZ6~vnbOR$tlr63E-EWCwyt6(`+Fxe(p ziEImRC$6IT1%B)x*df@7YOA!9xLU9ayP50~)Ic{ZzMH&;rVI99FTox`Ent=Q64wg$ zp^ga`T#p9TvX8uuR`1+GL+lnBiT69V&=kD|b{`-=NNaXB-V)pRA>vkN<89H6v*j@P z5!zAb_S$2&cZ|5hxxM4j+ha?ByptAm4lEQqFoU?uIj|Ga17k}!c@OR6&G*AcM6W7L$%yRK0&)%P=fWWO$th} z!D_jO_`F~vHZhqJl%ZU;+)I8R?S4T8HnaACpb}fGmIsM12)1GylZOP`QKecQMo*yK zyoi(ZkHDaR6kYU>AxQr?LiA4{KtGL6`X|v#{}lS@pT-ILXV6W5N%!aJSH1igFnLz@ z`~JFe!%i;i{>)hAIo+Q%J$>1(Jg@r`VwEepKkbhRuU&aTzy9aU%ndtvQLp+ZHti+d zUl0>`S=VQ$y`uZGV*;=0{@hE~j@tsSA(@MM9Vzs0AfEnBB+y?)9Q|8x)4yG-jnB+5 zcn2=}cj2LbuWsfnf8IwT{Rc>*|4?atq_nOnt&f$~Cray6rS(}YpF7UUpDV2|l-8Hl zch^$;N~wLV)V@(_-zv55l-l=7?FXfHU8(&z`xR4huJ^RMkJ`eOqb@;%2_B%vdaGgyqG^G diff --git a/tests/ut/data/dataset/golden/test_2ops_repeat_batch.npz b/tests/ut/data/dataset/golden/test_2ops_repeat_batch.npz index 7a9a70861ca0a3cebc41e2adb5e0ed0dbc220eb7..40b0489a598a133146812ce35d4251497ae6eb05 100644 GIT binary patch literal 3614 zcmbVPXLyvy5x!GHIy#8nj(Q6Sfg}*3N+3X>K?Tu7`4FeOPe>q6+SLhL25iMK#wI;Z zut{ue$LYms?sj@{OR-a3Qb_C+_g)e^k>8oUJ;^+e|Mxek1>xi2_y2@?Gh>PF!<{{a zu@2)G9`0^Wwqd=|o=}tVV;zaEr0I?&Lpor+LW#Jw>L3M9=#0u>v(7xJvl?`^q9El` z#^pLEEX)LBOoKI1HFBHQBRMeAl{=X1Mhsw#Cr^7MSV|?h1WRB z1~Hy;@SZ?}@SaGycu%4{c9{&9DHH-RwWjY94AWfHutc4XsBoJBx0w=p7D9V^wnUvH zQ6q@z`;9rNg%-s%oaaL3LFTJdxiIk4OF={`P?SNL01FglQ8qxKq8u6ouu$RT@_e90 zm|3JKM1!FeD;h#W0ZJ4Nqu~IH6^)>g0812&qR~8~6f>48%BL|<$`p;IaRAHC&%WGc zUtzPuXC-E?a!Xikm*DAgm%YMeUt_Zm*lBTGuk~~#=C6aShio`MUzN>=G#g>wx7_m8LPh4|J2FFinTDU(pPj$#fKSv!Yov8%j*k9E$kN$HDuM z7DbBYf;gaP9?fS{E9f>w1+)OlK}ChMkm*C9+s{9x!|78>z}(?ZsWW{_e4~szrLNQ| zAzBiAH{^&rsteLbbs^{;cT^Xp@5xcn$J{9$PoL7opf7Q!)SEt~4}iYZoze$Wr-U0Y zuqaCx>&rZS0-l#cPWE4Sl*tuRLsv>=o|5wUkgd!Q!}TLlLm!nI`j}Ma$E7kqVJq{K zaJfn<^HZmC`KtjB7wD(omEp8xxY}0fHJEZn(tJj$^s|!TTFG#otl0|=BqJ6TdWW3&Q zy13txxZk!*{SF+zE5UD(rQRxWZ)_Nxw+nq+f#mmAgs5 zPTeGAc?kT&kl(oL^xO1x`W@)syX*7^cbz=_NAQn89(9%XSi15a2mL2kd4G15hcthI z;jfT$kiWU|`+K_lo&f!%E5CoF$`8~23I1P@e=AxD_AH4#tgQ=SQ+Pb%6>HNIzpf@`S` z6QO3X>X}SCWZ5BGeClZn9C8FV&{p;yWbL`aZKLg=@&xaoof0l&9R}NBh}arw7yO0_ z-c5Tr&M<2qF5F)7K#dT*kD4UTNb4}l4x@SKuUu}0CSUxy8$ual{l*F>jnRw~%)OBv z##@I8c9oU2M`#%?!b|p_-Z2K1(=h zt7f)f?yM4Mj`fJxLD^xh_(-=g^90+5W9D0XfpF4x%mTsEd9bcR>#)!ci^L{<$P@{- zt;iHxdx>z;kj!Gish(t(Scg(OEEStHCsQWa_9wH<+LsF_UCOKwER725S!o?s*~tP2 zdgR5M&aoYF4G6^T5`JDfZj$w?{`GgWK43pbSRb&T3uM0A58_@30&#nU@88!&qPu;0 z=BViX`#C0h|9*~(?)HP?PMV8BH5b-XFe5+nz5dc+bE&5n-q@-8gI7=4{BIx-#AhkS IhviS+JN$Op+5i9m literal 3149 zcmbW4cT`kI6vlVyEGVd8!NP*#4v2^ff>=ONP_7iQVL@DAtq8njF(H~riXobqW|}F+ z#Pni%VtO^E_g+meCeakr6Y_m;ci!f3PX5W`+&RDd&6|7Y&Rov!R!L}Bg3bS=*ml?o zx9^X!*=W(W1e@FIt;^18Ztbw!;!Tk+a+1%EiUs9Wi|w`zwvDbvPg{f6Rp4}Os&r*K zT`OC>KCipEuEpEvpJ}?M=Ql*sr_8QRnqEv^2N*yzLD>hhg)-Twv6H6bXpGT(?2o7lUI=xW2EB(%m3b(0ZOQBhIWjmI3M&}YQf zZ>=?Ysam7A+7`t#!qX{sPpuKBBt$|1@rs!MeUwBbjYJ1XX+HA{V5=4K^~)P%l6j>c`W2{y1?=N z0wL!>&Q&r{2Em!2BuNH)KzWe!l?;)ga3(4lCdrfwAWu@_loU9Vm88mW%2OauRWd@- z;1ntuDK5%Ikf$k8k`8COk_;I|xft>cC7CiB&P*j)GKO*qo$ zXsnt8xm=%3g>^QSkgN3B%(c#@8uC1SHuJ5sSpa#VKAT0>+0;N@tj}hNbv8>OFVkmJ zYn{z<$aSz4J!^1VHPmCeLD$e2r~yTIps$3j(uJ(H3RwertuAC;pb#u+g1#QstgC6U zs%eF6=xV%yYOtgYx)0W_OWI(SbPVKUbx9q8lCb1B=o?|1^qn7X-T4WSH|sm^4BR=E zoCy6S*vb0hPq8k33*=Mv#difR9!pMxz7@7jNw$oIaJrIlGM?9a2ITEZawHednMx)| zo`?Eb(9edQqa#N@mDRIF~9Zky3NXWzcuQE>|*3W@B!*k}?r4xdQT)O6Ev8oIOe^q|#h+74)lN z*C?ryxtP0FNwv)5lItK}uVlU~fOCVAg|f(8awGJcU^gqNk;RzXt7M5R<&s+<->PJp z)WW$<$#SVPm)s8h4%nSNd+jc(*Y3vjJ-XNS1$*sY==Z_y*S+?D)oTwzen|J)!@*vA z1p1?}$8@hfZuQy|koW6edotK-2cSO%ds_F}GghxX3;8+SYtILJ?FHyB!d}w7_OjJ$ zuRwlP_u6a0UV9z-8?ZNZuf1jU+S`!d(Y^L=u-D#${yyvj-D@9Oy><}tN4nQO4))q7 z&_9KJrhDyktJl7O{H5--uY$eywYc52#v!T4_>DAR{8kz$ePUHew} z=Oau%%~~I6`mQ5;4*KV#Oh3bl7frv^+N{?>Ke%~s4E=xYKh|6y+P#FY<8bx5JOaeoVCNNYMfm~~Fy2%G;}sGZuZX~S(+G?=oxlo; z39MiSffdXIY>VQTOFS&2#1v2mAD&(*UuEuD1m>PiVD2&ka|?mF=Mb2?oDk{u_`DUM l`m8Fkha`ml*DnBnHZvpM#y_h5Jy0d#5ftqB%Ok{c_y^B@Z;${0 diff --git a/tests/ut/data/dataset/golden/test_2ops_repeat_shuffle.npz b/tests/ut/data/dataset/golden/test_2ops_repeat_shuffle.npz index b0ab3fb798b1bbce1bb8e99589a3ea397913c7e1..2ae358d16bf263f32c8c3a2d9d72a21174201851 100644 GIT binary patch literal 4042 zcmbVPcYIXG5x!GHI*KTwH&L~K5J&+&KdaHtHWa2@$~f=m zq)g+}uku0t7M@9X76ZJ<^RxIj|p$xpoQYPNxCx;IP!0_PSZrtnjRYt$ltGyQOAL*oe1N5f#?m-| zQbQAHB0!m;0vZocZZ72_3P1f62C>XggoXnwcaAGus1-wPE1m617ipE7-)c9BHEvpK zUF>ykHdSsC>kW;jF^IWAL3-t7aergf--MNKM%&`-wyFz^s0O{w1>LUm0bK){T;v@( zJJ7YD>kLhz$q?!dO`)kCPy^_lZk3I?ZqU0x?>01zrbF0cXa>z>>%E}&8JbJ;AT$}8 zPen}c2YtX$F)fC0&`=32Vfqm0sKHzoKsan@AuVFM8T65%`HX4Opl*S>)ukt{nFH;C zZga~z8Z4_FbcajAu^ibU8X3DT(>CM0vHG1h0{k z;M#Mmyv%=X)PEgT@p|$028sDbiT@_?`DXF|7K!+bczi3LguH;aiR-u5aJ@r(hs^Dr zGWK2K=iN^A9#OqloIERm-Y0I~FCjkQ9DY#ze@NVZSZ4f@L1$Na-2Z6Q{}^WbakNjM zebRkN&jsJzr$9gNHo&J-8vx(wXTX0J?Q?G9eLmQDUjY3@w|l>o+PxU^W$<4?`>NZ0 zUki5M*Fk^7?d)%+b~c853;egyzGG-9m4f)Lp)xAxP4zv{-#4_3mP7b~p$b~zG5zE zB~gA0{Sf5E!2cQT6+=z5AG*I7IzR_` z$X`MK&Cnr=LioF(!_+K8{sI1d%lVwWUF0{aB_3mVS@ScVuM^Y$kQ$&Iyp4$aKUMs#xC+p&W6$hYHEUm%>E@OHdlIppDIf*MTJZj$KalCqNp z^MwktDXO0;oLs4PnqaX!;98`PExPVG@!rUvEOEfbv_ z&33t9J)LcZ>Q@LSSEsEMEVn1}v{DULX}4N*@>9pI5zIeIV76BE>x7fvJGM$N|I%TD z^=hy|yN#lgs^6L?^{;cM8^ewvDRaC7e{X-7Pp(*mjQ^?A30c=%idN|LAoUYw@F>%kqG5 zTDJ!UyShE322t$}i*7w_K<1hSZ=_9}#v`ha3AdTHfNBxEm8vDrt!fb0&J&&bX%n3C zb5!;1!l|DQ!RjY{Keu!wVz$p7gEDH5>!?n2YR}<~FYY|L>h~9zS?5ocnQGa)cT=eqOFQ|Mc%hfLp45Z|{M8ELL zl_ykxH_Okb_4kN=N}#`2v>r8>GfV= z`I`AZBKq|D9u+ygp2tL<7v$&hL9*v1?Nu->B@GnLD9HR@f4`BN)Wcslz9{!+yn(X$ QPg^LA&kFQsAhE{7bme(oSw+%yxIE(@Avti4Buma_uMz{mbnAF4U%6_8VRn+k;PS) z@9r2BiEx!fN+T)XZ<<%$-q{n446!0N(#h+tx*fF*d!mt(kyDA*bXSX?SRPNDu1}Q3 z6NftdjGt<6>hN3BlsBf@y3)|Qj-)!%uuVl-e8HlbW$`od|NDyd=C`zWpXlr+0%i`f%C5q%z|TXug>A z2AZTd$l$z^bkZAaq(BM*LyQzjKfq8U17sjzn2|im2MjkdNCpE+jSQ3FfDuMU$|%4{ zBc(C|Fv?siX3KH)N)*CqBQfa@7^4S|)llPd_m0=S6ExC9&2N$>5!bXPYwRhS%~VZd znvo$g6fvhO%s{OPHB4{={4*iX(h#$Q=#a}G&(S#LK`xNzLY`-2w2Xl<-^f@Qmj*3> zyijXe6jTX$G2|si#>)g4ON~sFN!(old6|)EG9AWpBQqpHc?IN^M#^Lkj8#U;WiI8_ zkSmQCnF(W!ky$dE@>r!RlO7PF75wr*Z)0`_iF3*xz_E6oYY1iaE)$)d{BFo zay@E>+@c+AbsbGZKBTQX>{@pO@=+sknT(DcGcrY{vZKc#x2b%>$?cFkusYS{IhPN) zOT%WeVR2G7^pjZEXt~$Aa(f_Or#(BB?HO)44gCz(^;+{;S934q8?>J{X8Va-Zi0R@ z);S~dWIlxRMi$6ImU02|Ek+i}Vi*^VERm&Y>X)Ej#=2FX^=SO|lujHyNoB;VpMV zzQ@QGsfBT`kvgfjx7-K)eyj(K?3O+7J!oXF?BgvDL4LE5{gQ<779$6w$=>o-=x@V% z*hqtHh3^p~jk1lmJPP?SBim&MjK_`alwJ0gw?lsd>m9i>{Z2R2?}Gc?I*#wjjw4Qa zFZB0eyelmQdpMw5rtk3A+ zJmm)GX~@s$;Cwb5oX zVNKtG{#~r^=`?)bO~Vf$|4^slN8vR582V4Jp4Io~r|$jv8RVbq`}2$N{rM&IUt#@P z$MiRDOn(dccRHreg=6}A=zqZaqrME!yO-ew$bZt8;m_gA@E7QR#d=X^?r(19{to#c zI&=RFXYOCn|Bdw@o$UX*$^IYYzU*ZC5&W;~hTD(QO8R-hN51vhk@X8K2UF`82L2+e z+1d5`SG2|i-hw?@{clYW28>!b?P?+*z41Fg1RHb5F=d81VGHSh-q8-@fQ zL#;13XPD*eIl}{gsnvpWMp)K!M#7SGD(R0Rv;amES^;AShXG>=&46))BY^RQV}J>S zHo!!}alj-(3J@n80!$|Gz$pZVn5uiG>7MBXo;-uVd=dl(H3X(HlfYQB2#h|Pz@TM> zqkuUI<$&rnBdVW><0}jtbFE$I?0ME6{M@;$f4;RRIeP)^$`{@Kh1Ngi-X{+R=Pj~! z;)REv4eZ6VtDo=d%i2qN_Xqx^w4Vw671lr2`IlKc%xAf^!+cld+`lsC{#Div z^IL80u>MNgSJkt|`ontH(!MJHb=Du&S7q(6p7qwA?)tMK$KD8A3efm!LLOifAs?`r zPz0zUL;*tR2iQU=0n`F&`_6*uD8&HvfSO2eUmv)EVj*BFfrmB{7-rj5+qUbr9Rvp0 zNnl#L2#n1y*ESpcu*inoONm+RBQW-ULNOqzaDc#rng|Shkib+@1lHF~V2l<5!?zN6 zavH#|@vQI=MMgPH5#!PeK0=Wd9wqSDV*~~|e$}=%-FAY&5bXpe)@E|S%&i30 veOmzOjDI_*xx7K5d8PTU^jkGZ-WGt}_%}v>xi&~)5rruJEXC)e_Sfr69xWP< diff --git a/tests/ut/data/dataset/golden/test_2ops_shuffle_batch.npz b/tests/ut/data/dataset/golden/test_2ops_shuffle_batch.npz index 579431fcc905586d5c5abb38ca9a17a71e1f0b61..2939b29786bf0cb7acfc381d7afe1c27b473d762 100644 GIT binary patch literal 2360 zcmbW3cXSk06vk(>sY{2@+fw%dA&>+@XeQJ^K2iuRlp$<(ClCmmyvYWQfW!g_cCleW z6zq!Fv3IfeUJ!dh1q*gvzc*z8&+#AUoNssU@80{~o%imW*)n(&(>43*qb2X$H(__G zrYWUq=~~bYF@TTMMc{EIhMwwVHlLS0SMZyM|?1YA+P4$LP zrc@;GHcRtlDl~XD%CvIn6Yy4eNY}^~B_$~$8cmIHlg8R}F?}5Ic(?rH?77q&`Q!!E1iN%y zPK*~jiFmR=7j$LZ6oGE&&S`ZymAsIeW-o#+r`z{WR}nD;dY~tb83Mh~+e#@WFQG~W z`eOinWdZ{cP@~NxE*HqaP#UuYGBM0bnN2>2nrknIF6YIUtAcpG;5>tvvOr)khFB>J z$rn+&fFFJ6TP)BQ{nRX##7mA{L0tyz4Px#w{gyjIR_QXtj4)+b{Z?)F+fssMR=?HQ z{kEK7h1GAhcE7D8h*ed{HZQB~>gkYH+nILTH4sFtwrf1Hv#R%w?){u>xk{JE zGxP+i$*ScW$oPE`lYD&sV=^;Q`h>@qeIoNkq| zsfbPAe02Bcgym*kp23JSsk5lFsdL=rJvY9*=MkUp?(G(5Z@KIjkY7k$~~h9isPTq}@`95v!~ z#McXqz(^W52;^cE_ng0Pq~j*)X6hFAaO`pphc0g=-tC@?+njU3wA;z=pzajN!)SuL z1jb;j8u4!8dj!T|JdHg9$06UI>RvkTqwc335SWJP40uqW2vB(s5kD+21I08R5hy{a zRpF!Lk5P{cRA4@RPY5i)LY49)@lyhepwrkZuo#v0R8Q0K4D~GaoO?{3caDiJUm$+b zJ#a4_IdCtNze2q#P==Wt^EH8T%u-XmPW*0IsCNYlFo6N@ z2~5N!mG?gJ2Lh8Zg~mRCsVKDW>O=DV)JOk5ksmuJQkS35{i*wa`OJC1FgboYs%j!Z zvq^rg%P$!ECG`~yKEuC8NIldK5PX9$!M9kdz;^@(QAKbF%M|#Y;0G)x_z|@V{6ugV z)dW9dg#y12{E8Za-w;&iBF%X#RM*x8qw;rM{=tYpslN&fIu`Eo)6L+&3k#AQgM-9` zZfbnp6&AEPu-=X^J(k_uu?^i!AYhzV@%+Q?NM@oH*LT~VC2C~GCE0Pw77rZSv0riB zOi^~>!C1_)8JB9urCD*^wr<^S$N4OKuwyf>g&o(@;sfIIwo>-q_;`j9HCq!^WtPE{ jkna7jKRcYnxtbZx%G|#`Wk^ylR4#)@8^!Mf)|>1fXa(1+ literal 1914 zcmbW2*;5o(6vlfP)^Pz9TySAPaSjNGsDQXIiqgs_Frp}^h{MbvC|eJM2@xYEAw;rk zHezB-%)XdCF=k)Q{x{@352jL;XY$=U(?gY2dC9FhXU_SZKHt54Z}+qh&csY3I&zJR z*U#VlJJ~Rlr5TxqKOAl^E$JH=b{H8NstZ+7jSCIU^}bezand;D35Es(VNaFYGt%fO za(g=a!;!GRuf0DU3@Kde?->k{4|e$nLd>PI$X#CKE-NT-7)2+QXqhf8SsvJQRqyEF*fAt1lR>S`HXvRzi)V!*Y&WiA`3Lz=2dG zBVpW1j$Is#3=f1{R!U0(Z?v(}N`(Q(xRusqxdhG@2kD44qoJXpZi+)YR7b3t9p~Dt zS<+_B7AmHN+N^XzB9a(n2xmxPFh?*Gvlz@3B*4jFo?tf88Dt9PVjhDm!F()WFkj%p z3_K+3>L{msyYUaQ8);)Iqzb-x+E$^yp(vEy_}q=9!No~yq>sRAShr`At=Nub#*2227w#7OsWKVSg!C!;!U^i z$mZw{v%H1nYMR&X+#2uPMqFbrc6)5GY}r9xOEc{)-5KA~UBq?vin&5{R%#Ku$?Iti z_69b_H_%7C$KI)?*iNx!FZn)Nv!E1f2=)uguvV!ZAZ`(?!+IvIf^t;oSq_pPq8+vm zwJmThRCfExJ7@txB{mQQ1y$InW(g5@3N~RglVgG{sMfP|k$2OM3wB}` zYbOMCfNJR>?iK7tJ(E5`0~&QpKluR7611R|wXonI4yl$w;)vid+L#Opj-Xw)oFu=E zcDsGh!|{W@gZPyFw2Z`_7IvQ|zms-G;KLq*Q9%>-s#(qw-zC_GW+rzF_Tzw_ zl&xoBromtn$3+&50*3nAQKpFS}xT^DbRG8Wnia^m_if@<%jz$uw_HO~sOz zO>g$(`Ej+^%p21^uOpF*dIJgcZ^B9c7Lw_&!a@HwQt0178vVQV zQ)l?|9+K$ahl~CLq|$#_XN*owG5APpeXO;vX{}GR)^)A*sn+^TYkjW%XqDO*TJ1}% z_LWxqx{m7~W$7EO_N`X?POE*d)qc=wH?-Q1TJ5KaRQ2c0p(KCSlD}xlU$x|K)Bmsj qP{jJZqr?YCLZcY!!}Q0~ha~;UbQlh!f`#G~RsI)MT|yK9 diff --git a/tests/ut/data/dataset/golden/test_2ops_shuffle_repeat.npz b/tests/ut/data/dataset/golden/test_2ops_shuffle_repeat.npz index 5c39c0e64b87b77844a10c529ff628368029b4bb..6e2486024fde4b75c934940e56abeff6d9cd7955 100644 GIT binary patch literal 4042 zcmbVPXLwY{72Q=sT8b#5H&J&0A&>+@RM7>xs34jsi&*WhKp-T&(F$7zj9dVNyK%#X z*e1O=J#l)E(|dEO(~FbXDNaw6GiTnsHTu5%$=h$@_1a z@#=Gx>kC66(+WaGp}5v9i%UDZ`@*4-B#og_n(I>iuDZs(;n0cD$!IdwlhDx>#nDp@ z(URh5TbEAjcxOwOPNodKIo{Ee0^f5a-kpN1s!EE>=am$nF8=?oNPl*sv-fy+UumLC zr%I3acBI>pZ@e$kqEm^k&YraHO{61U$XZ1@lUD16C3M!y+7NE_vd?=tOkPkU#&tkN41^9ohW8!0*Qm^VtYa33w%xQ~%M+>0cP`&b!*`#34UeSF=( z8FUjQ7x#%0!F`hC<371I)ITr)Foj;GI&m5;r_)6-!^~iSnG7(?x|>bkbLeg^-9-^O z)tu*NTbA^YQDig^o^V-2(cMLW=Vs#FfmK1;)A+bV5n7%z%(o zGE-(5It99I@aiA-7c1r+fx6u-!qH$6j)CqNJnDEbsuSuimsxj^nb~UE&AKO;bsBW9 zTmKWm`dF?CJFS1$4Hh*w`D$2dr#=O^I7O$tbH!$WK>GMtW`DV`IEsXdyJ-*ev3C#rF#$?{k0PkRc zch;Iphw5GQ^KK`;hnDZ9lV=#{eGKt_hWLPU_(A&r5FLJ)4nK0uyQ?DUeKh8M4B37h z?GtF9bkFWn!L$1`=x5yq_>8{+u)Is>8l6q<2kUxR{Guq2anq@C^e^Ihe z_M0Jp1^qWA2P6jJ?@A6z3y1sz{6EqDij zu2Hr_lSAAjI}lXoT74e5ow5tm5aQjk$3)lp)}X+45$e{;I%tLxua^zRY?###C$~{H zff_-)S!(E~&>D=i-6+##s?eNcdNl3L(FkFTwJRdWlT?o-HixM(7-tQ}+in7NJRtQ% zVsoIvY?9SaCdZRhPa!tvtTC8s4W`*{y6FbxRZM$7uNl^ECOMneEMk|}Y-=#bc5|uY zDWapq_8`&9>gSQ;nWX0v^H@S&3#>tj?G{qUeW^={?bg(btbQ>$ZctrD?C()sZVf7I zw}d(#UwSFAJ;8LP)h{E*qf=KA^YlbM%dNo*+pVOI=d@l$Y>#Ta+UnPkOd!{cuXHRAOX;6KoX$KfWrXY2DAcr1{?v< z1{?+GF`xq=Z9p49uK~vZP8iS*aGe2hfa?t~A^IG1g8^pLNdt`MQwErPP8(p-xX}QU z#u)=ltbPMb^qUMYv2HfN#Ja_RR(#hN;KVimLfSgI;%R*==$Jl>J09FQbTw zi~32IFF#@RcT=7p$oEj56ZpT^$j@fR-$#8W-%Eq^_fwzo{{ZDoeh*U4%>N`9DTElmFvJ4%RD`O6x0NT1y)xoK=+lzy5w>ZT!nTZagUW VXS`8z&7ZbV7{6ucPxBA?e*qr9PAC8X literal 3831 zcmbW4cU)9g7RP58Kp4AXZ)2T{A}SUT3wD$vMHKsp0}M6<-eWK!Y9z5mlbB+f>50h} zcav;M+uqmhy(C+b>?T`NOxbL*mHdA9&ApHC`Q)EGKUcozckaFC-1E+bnJNjT=LEcG zd?3``ySSxaAi$9w$O*(BgIDTik{OdfW`&y@Jg*a!z;P#Sd9cn2opjQxoeU*G z$&%qGoXphJhGa);BJA|qoQ7jo7CBiGkl;xtyV?mW32hESJ>>MRsHiCG!lVC_lBCn8 z?o`a_t71++h5fPrA zu_u^pCYsojOzg=f^(i=O)Tqi^fK+lJPcpC55#;$wCdp(N#Y(0~l=1?|3zg)_bQp`2 zr7_VHna7RH<(k`*i&zWyvfvmv#tLY$XiXJx7k8( zha5A7-eC(}3wfuhQQX$34syMzXoIb20&=6N(Jot~-H`W~8tt_;+6TGGyu|(XB{oBD z!Duxr9J|7W+-AZiQ(;lgcIXE%E;Gly+#a_B@)f3L2U9h}l0(oBV_a#@e8ir)6Y^E2 zo>!;pi6uv&AHz7VWTwo5a6(Ce%;xJp3Hcf&b7U@zQ%dGZVS@T;=+|OgXWsSo_FbQW ze1mxpH>TbLmfQsWW{k5+iex^7Ta*;b0`7Jz5#QYGskd`d~RtY?%@L;j4C4N?Q+ zvr0C~CLQH*=ucogsbq;Ph3_dP%Vas1JPrACN><297@t?NN=o&TFF^ky#+Q2DoG;s( z^A)(iYFh1UsaC@-Ux)q;jBlF8`Ic>*Z$rLl8s|HH<9rwT_b|TSGujVqv>(F#BNOe% zezc!J|0%}LOa*^#EBFh@zcdy6m0!VMLw^S2S6}1^TNPubInm$zF!nA-`cR z!<+tPcnkVJFy1yj_fOk%??8Un^xVJvp8GfS|6u&rboT#jXa65^SE{q!0RHOQ=5~X$ zVs4uEBcy%0|J-!VUJJSzoURfe=AZFI%pL6cbF`*E2e?Bt^Ai9k4230$c+4F}s0R!u zGyp~rb^%5b>Hwn%y8)vKdjVqzO@Oh4M!-12K0t)92QZ!x2TUL^#6+V^B5>7Y0(YK5 zU_Ma-(@+GakxO8#sRTx!MqsQw0%J`lFx7lO>5?I(k4Er;1@;VWXIOitw)?+)JmsII z?QzyFpk4B|oj+UqC*1q=PH(?C+K#?{{sqsTOS|+^S69lOr~M<(ox9)j7ixQ|wTrYp z+3r7|_CdFBYkgTPZP%Z3KqgPGHCw zfhq4GaHCoR1Meho$2fuKt0Qo7J%QmH2;4aV;Kz8Lu#qCO+(lrx-2{f)!)a)sPVioe z479JC*hGC*#i(gpq@Ss=H!(@*T7_*bWm{;k!I4K{V_G)?<@@P-bF?zVm zaRMWrATZC91g3Wlf$5zByf25-^f0??3C!U-0&}>Y5R501?io;Bew75%azY>UUDZof ar;FM69b>+^R!K%Cg&_VF;^zhZ?e#qtcNoe5 diff --git a/tests/ut/data/dataset/testImageNetData2/dataDistributionAll.json b/tests/ut/data/dataset/testImageNetData2/dataDistributionAll.json deleted file mode 100644 index 3ebc4c989c..0000000000 --- a/tests/ut/data/dataset/testImageNetData2/dataDistributionAll.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "deviceNum":4, - "deviceId": 2, - "shardConfig":"ALL", - "shuffle":"ON", - "seed": 0, - "epoch": 2 -} diff --git a/tests/ut/data/dataset/testImageNetData2/dataDistributionRandom.json b/tests/ut/data/dataset/testImageNetData2/dataDistributionRandom.json deleted file mode 100644 index a0f468f91d..0000000000 --- a/tests/ut/data/dataset/testImageNetData2/dataDistributionRandom.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "deviceNum":4, - "deviceId": 2, - "shardConfig":"RANDOM", - "shuffle":"ON", - "seed": 0, - "epoch": 1 -} diff --git a/tests/ut/data/dataset/testImageNetData2/dataDistributionUnique.json b/tests/ut/data/dataset/testImageNetData2/dataDistributionUnique.json deleted file mode 100644 index a4eeddd9ae..0000000000 --- a/tests/ut/data/dataset/testImageNetData2/dataDistributionUnique.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "deviceNum":4, - "deviceId": 2, - "shardConfig":"UNIQUE", - "shuffle":"ON", - "seed": 0, - "epoch": 3 -} diff --git a/tests/ut/data/dataset/testPK/distribution.json b/tests/ut/data/dataset/testPK/distribution.json deleted file mode 100644 index 33f869f653..0000000000 --- a/tests/ut/data/dataset/testPK/distribution.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "deviceNum":1, - "deviceId": 0, - "shardConfig":"RANDOM", - "shuffle":"OFF", - "seed": 0 -} diff --git a/tests/ut/python/dataset/test_2ops.py b/tests/ut/python/dataset/test_2ops.py index ef60a42e27..cf781d6dfd 100644 --- a/tests/ut/python/dataset/test_2ops.py +++ b/tests/ut/python/dataset/test_2ops.py @@ -12,15 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -from util import save_and_check - import mindspore.dataset as ds from mindspore import log as logger +from util import save_and_check_dict DATA_DIR = ["../data/dataset/testTFTestAllTypes/test.data"] SCHEMA_DIR = "../data/dataset/testTFTestAllTypes/datasetSchema.json" -COLUMNS = ["col_1d", "col_2d", "col_3d", "col_binary", "col_float", - "col_sint16", "col_sint32", "col_sint64"] GENERATE_GOLDEN = False @@ -33,9 +30,6 @@ def test_2ops_repeat_shuffle(): repeat_count = 2 buffer_size = 5 seed = 0 - parameters = {"params": {'repeat_count': repeat_count, - 'buffer_size': buffer_size, - 'seed': seed}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) @@ -44,7 +38,7 @@ def test_2ops_repeat_shuffle(): data1 = data1.shuffle(buffer_size=buffer_size) filename = "test_2ops_repeat_shuffle.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_2ops_shuffle_repeat(): @@ -56,10 +50,6 @@ def test_2ops_shuffle_repeat(): repeat_count = 2 buffer_size = 5 seed = 0 - parameters = {"params": {'repeat_count': repeat_count, - 'buffer_size': buffer_size, - 'reshuffle_each_iteration': False, - 'seed': seed}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) @@ -68,7 +58,7 @@ def test_2ops_shuffle_repeat(): data1 = data1.repeat(repeat_count) filename = "test_2ops_shuffle_repeat.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_2ops_repeat_batch(): @@ -79,8 +69,6 @@ def test_2ops_repeat_batch(): # define parameters repeat_count = 2 batch_size = 5 - parameters = {"params": {'repeat_count': repeat_count, - 'batch_size': batch_size}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) @@ -88,7 +76,7 @@ def test_2ops_repeat_batch(): data1 = data1.batch(batch_size, drop_remainder=True) filename = "test_2ops_repeat_batch.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_2ops_batch_repeat(): @@ -99,8 +87,6 @@ def test_2ops_batch_repeat(): # define parameters repeat_count = 2 batch_size = 5 - parameters = {"params": {'repeat_count': repeat_count, - 'batch_size': batch_size}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) @@ -108,7 +94,7 @@ def test_2ops_batch_repeat(): data1 = data1.repeat(repeat_count) filename = "test_2ops_batch_repeat.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_2ops_batch_shuffle(): @@ -120,9 +106,6 @@ def test_2ops_batch_shuffle(): buffer_size = 5 seed = 0 batch_size = 2 - parameters = {"params": {'buffer_size': buffer_size, - 'seed': seed, - 'batch_size': batch_size}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) @@ -131,7 +114,7 @@ def test_2ops_batch_shuffle(): data1 = data1.shuffle(buffer_size=buffer_size) filename = "test_2ops_batch_shuffle.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_2ops_shuffle_batch(): @@ -143,9 +126,6 @@ def test_2ops_shuffle_batch(): buffer_size = 5 seed = 0 batch_size = 2 - parameters = {"params": {'buffer_size': buffer_size, - 'seed': seed, - 'batch_size': batch_size}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) @@ -154,7 +134,7 @@ def test_2ops_shuffle_batch(): data1 = data1.batch(batch_size, drop_remainder=True) filename = "test_2ops_shuffle_batch.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) if __name__ == '__main__': diff --git a/tests/ut/python/dataset/test_batch.py b/tests/ut/python/dataset/test_batch.py index 9b9baeec33..1220d98344 100644 --- a/tests/ut/python/dataset/test_batch.py +++ b/tests/ut/python/dataset/test_batch.py @@ -14,7 +14,7 @@ # ============================================================================== import mindspore.dataset as ds from mindspore import log as logger -from util import save_and_check +from util import save_and_check_dict # Note: Number of rows in test.data dataset: 12 DATA_DIR = ["../data/dataset/testTFTestAllTypes/test.data"] @@ -29,8 +29,6 @@ def test_batch_01(): # define parameters batch_size = 2 drop_remainder = True - parameters = {"params": {'batch_size': batch_size, - 'drop_remainder': drop_remainder}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -38,7 +36,7 @@ def test_batch_01(): assert sum([1 for _ in data1]) == 6 filename = "batch_01_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_02(): @@ -49,8 +47,6 @@ def test_batch_02(): # define parameters batch_size = 5 drop_remainder = True - parameters = {"params": {'batch_size': batch_size, - 'drop_remainder': drop_remainder}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -58,7 +54,7 @@ def test_batch_02(): assert sum([1 for _ in data1]) == 2 filename = "batch_02_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_03(): @@ -69,8 +65,6 @@ def test_batch_03(): # define parameters batch_size = 3 drop_remainder = False - parameters = {"params": {'batch_size': batch_size, - 'drop_remainder': drop_remainder}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -78,7 +72,7 @@ def test_batch_03(): assert sum([1 for _ in data1]) == 4 filename = "batch_03_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_04(): @@ -89,8 +83,6 @@ def test_batch_04(): # define parameters batch_size = 7 drop_remainder = False - parameters = {"params": {'batch_size': batch_size, - 'drop_remainder': drop_remainder}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -98,7 +90,7 @@ def test_batch_04(): assert sum([1 for _ in data1]) == 2 filename = "batch_04_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_05(): @@ -108,7 +100,6 @@ def test_batch_05(): logger.info("test_batch_05") # define parameters batch_size = 1 - parameters = {"params": {'batch_size': batch_size}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -116,7 +107,7 @@ def test_batch_05(): assert sum([1 for _ in data1]) == 12 filename = "batch_05_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_06(): @@ -127,8 +118,6 @@ def test_batch_06(): # define parameters batch_size = 12 drop_remainder = False - parameters = {"params": {'batch_size': batch_size, - 'drop_remainder': drop_remainder}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -136,7 +125,7 @@ def test_batch_06(): assert sum([1 for _ in data1]) == 1 filename = "batch_06_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_07(): @@ -148,9 +137,6 @@ def test_batch_07(): batch_size = 4 drop_remainder = False num_parallel_workers = 2 - parameters = {"params": {'batch_size': batch_size, - 'drop_remainder': drop_remainder, - 'num_parallel_workers': num_parallel_workers}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -159,7 +145,7 @@ def test_batch_07(): assert sum([1 for _ in data1]) == 3 filename = "batch_07_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_08(): @@ -170,8 +156,6 @@ def test_batch_08(): # define parameters batch_size = 6 num_parallel_workers = 1 - parameters = {"params": {'batch_size': batch_size, - 'num_parallel_workers': num_parallel_workers}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -179,7 +163,7 @@ def test_batch_08(): assert sum([1 for _ in data1]) == 2 filename = "batch_08_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_09(): @@ -190,8 +174,6 @@ def test_batch_09(): # define parameters batch_size = 13 drop_remainder = False - parameters = {"params": {'batch_size': batch_size, - 'drop_remainder': drop_remainder}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -199,7 +181,7 @@ def test_batch_09(): assert sum([1 for _ in data1]) == 1 filename = "batch_09_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_10(): @@ -210,8 +192,6 @@ def test_batch_10(): # define parameters batch_size = 99 drop_remainder = True - parameters = {"params": {'batch_size': batch_size, - 'drop_remainder': drop_remainder}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -219,7 +199,7 @@ def test_batch_10(): assert sum([1 for _ in data1]) == 0 filename = "batch_10_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_11(): @@ -229,7 +209,6 @@ def test_batch_11(): logger.info("test_batch_11") # define parameters batch_size = 1 - parameters = {"params": {'batch_size': batch_size}} # apply dataset operations # Use schema file with 1 row @@ -239,7 +218,7 @@ def test_batch_11(): assert sum([1 for _ in data1]) == 1 filename = "batch_11_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_12(): @@ -249,7 +228,6 @@ def test_batch_12(): logger.info("test_batch_12") # define parameters batch_size = True - parameters = {"params": {'batch_size': batch_size}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -257,7 +235,7 @@ def test_batch_12(): assert sum([1 for _ in data1]) == 12 filename = "batch_12_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_batch_exception_01(): diff --git a/tests/ut/python/dataset/test_datasets_clue.py b/tests/ut/python/dataset/test_datasets_clue.py index e1959acb42..0d8a60f5d1 100644 --- a/tests/ut/python/dataset/test_datasets_clue.py +++ b/tests/ut/python/dataset/test_datasets_clue.py @@ -356,9 +356,13 @@ def test_clue_to_device(): if __name__ == "__main__": test_clue() + test_clue_num_shards() + test_clue_num_samples() + test_textline_dataset_get_datasetsize() test_clue_afqmc() test_clue_cmnli() test_clue_csl() test_clue_iflytek() test_clue_tnews() test_clue_wsc() + test_clue_to_device() diff --git a/tests/ut/python/dataset/test_generator.py b/tests/ut/python/dataset/test_datasets_generator.py similarity index 92% rename from tests/ut/python/dataset/test_generator.py rename to tests/ut/python/dataset/test_datasets_generator.py index 926b84a7f4..b7061cf0f3 100644 --- a/tests/ut/python/dataset/test_generator.py +++ b/tests/ut/python/dataset/test_datasets_generator.py @@ -26,7 +26,7 @@ def generator_1d(): yield (np.array([i]),) -def test_case_0(): +def test_generator_0(): """ Test 1D Generator """ @@ -48,7 +48,7 @@ def generator_md(): yield (np.array([[i, i + 1], [i + 2, i + 3]]),) -def test_case_1(): +def test_generator_1(): """ Test MD Generator """ @@ -70,7 +70,7 @@ def generator_mc(maxid=64): yield (np.array([i]), np.array([[i, i + 1], [i + 2, i + 3]])) -def test_case_2(): +def test_generator_2(): """ Test multi column generator """ @@ -88,7 +88,7 @@ def test_case_2(): i = i + 1 -def test_case_3(): +def test_generator_3(): """ Test 1D Generator + repeat(4) """ @@ -108,7 +108,7 @@ def test_case_3(): i = 0 -def test_case_4(): +def test_generator_4(): """ Test fixed size 1D Generator + batch """ @@ -146,7 +146,7 @@ def type_tester(t): i = i + 4 -def test_case_5(): +def test_generator_5(): """ Test 1D Generator on different data type """ @@ -173,7 +173,7 @@ def type_tester_with_type_check(t, c): i = i + 4 -def test_case_6(): +def test_generator_6(): """ Test 1D Generator on different data type with type check """ @@ -208,7 +208,7 @@ def type_tester_with_type_check_2c(t, c): i = i + 4 -def test_case_7(): +def test_generator_7(): """ Test 2 column Generator on different data type with type check """ @@ -223,7 +223,7 @@ def test_case_7(): type_tester_with_type_check_2c(np_types[i], [None, de_types[i]]) -def test_case_8(): +def test_generator_8(): """ Test multi column generator with few mapops """ @@ -249,7 +249,7 @@ def test_case_8(): i = i + 1 -def test_case_9(): +def test_generator_9(): """ Test map column order when len(input_columns) == len(output_columns). """ @@ -280,7 +280,7 @@ def test_case_9(): i = i + 1 -def test_case_10(): +def test_generator_10(): """ Test map column order when len(input_columns) != len(output_columns). """ @@ -303,7 +303,7 @@ def test_case_10(): i = i + 1 -def test_case_11(): +def test_generator_11(): """ Test map column order when len(input_columns) != len(output_columns). """ @@ -327,7 +327,7 @@ def test_case_11(): i = i + 1 -def test_case_12(): +def test_generator_12(): """ Test map column order when input_columns and output_columns are None. """ @@ -361,7 +361,7 @@ def test_case_12(): i = i + 1 -def test_case_13(): +def test_generator_13(): """ Test map column order when input_columns is None. """ @@ -391,7 +391,7 @@ def test_case_13(): i = i + 1 -def test_case_14(): +def test_generator_14(): """ Test 1D Generator MP + CPP sampler """ @@ -408,7 +408,7 @@ def test_case_14(): i = 0 -def test_case_15(): +def test_generator_15(): """ Test 1D Generator MP + Python sampler """ @@ -426,7 +426,7 @@ def test_case_15(): i = 0 -def test_case_16(): +def test_generator_16(): """ Test multi column generator Mp + CPP sampler """ @@ -445,7 +445,7 @@ def test_case_16(): i = i + 1 -def test_case_17(): +def test_generator_17(): """ Test multi column generator Mp + Python sampler """ @@ -465,7 +465,7 @@ def test_case_17(): i = i + 1 -def test_case_error_1(): +def test_generator_error_1(): def generator_np(): for i in range(64): yield (np.array([{i}]),) @@ -477,7 +477,7 @@ def test_case_error_1(): assert "Invalid data type" in str(info.value) -def test_case_error_2(): +def test_generator_error_2(): def generator_np(): for i in range(64): yield ({i},) @@ -489,7 +489,7 @@ def test_case_error_2(): assert "Generator should return a tuple of numpy arrays" in str(info.value) -def test_case_error_3(): +def test_generator_error_3(): with pytest.raises(ValueError) as info: # apply dataset operations data1 = ds.GeneratorDataset(generator_mc(2048), ["label", "image"]) @@ -501,7 +501,7 @@ def test_case_error_3(): assert "When (len(input_columns) != len(output_columns)), columns_order must be specified." in str(info.value) -def test_case_error_4(): +def test_generator_error_4(): with pytest.raises(RuntimeError) as info: # apply dataset operations data1 = ds.GeneratorDataset(generator_mc(2048), ["label", "image"]) @@ -513,7 +513,7 @@ def test_case_error_4(): assert "Unexpected error. Result of a tensorOp doesn't match output column names" in str(info.value) -def test_sequential_sampler(): +def test_generator_sequential_sampler(): source = [(np.array([x]),) for x in range(64)] ds1 = ds.GeneratorDataset(source, ["data"], sampler=ds.SequentialSampler()) i = 0 @@ -523,14 +523,14 @@ def test_sequential_sampler(): i = i + 1 -def test_random_sampler(): +def test_generator_random_sampler(): source = [(np.array([x]),) for x in range(64)] ds1 = ds.GeneratorDataset(source, ["data"], shuffle=True) for _ in ds1.create_dict_iterator(): # each data is a dictionary pass -def test_distributed_sampler(): +def test_generator_distributed_sampler(): source = [(np.array([x]),) for x in range(64)] for sid in range(8): ds1 = ds.GeneratorDataset(source, ["data"], shuffle=False, num_shards=8, shard_id=sid) @@ -541,7 +541,7 @@ def test_distributed_sampler(): i = i + 8 -def test_num_samples(): +def test_generator_num_samples(): source = [(np.array([x]),) for x in range(64)] num_samples = 32 ds1 = ds.GeneratorDataset(source, ["data"], sampler=ds.SequentialSampler(num_samples=num_samples)) @@ -564,7 +564,7 @@ def test_num_samples(): assert count == num_samples -def test_num_samples_underflow(): +def test_generator_num_samples_underflow(): source = [(np.array([x]),) for x in range(64)] num_samples = 256 ds2 = ds.GeneratorDataset(source, ["data"], sampler=[i for i in range(64)], num_samples=num_samples) @@ -600,7 +600,7 @@ def type_tester_with_type_check_2c_schema(t, c): i = i + 4 -def test_schema(): +def test_generator_schema(): """ Test 2 column Generator on different data type with type check with schema input """ @@ -615,9 +615,9 @@ def test_schema(): type_tester_with_type_check_2c_schema(np_types[i], [de_types[i], de_types[i]]) -def manual_test_keyborad_interrupt(): +def manual_test_generator_keyboard_interrupt(): """ - Test keyborad_interrupt + Test keyboard_interrupt """ logger.info("Test 1D Generator MP : 0 - 63") @@ -635,31 +635,31 @@ def manual_test_keyborad_interrupt(): if __name__ == "__main__": - test_case_0() - test_case_1() - test_case_2() - test_case_3() - test_case_4() - test_case_5() - test_case_6() - test_case_7() - test_case_8() - test_case_9() - test_case_10() - test_case_11() - test_case_12() - test_case_13() - test_case_14() - test_case_15() - test_case_16() - test_case_17() - test_case_error_1() - test_case_error_2() - test_case_error_3() - test_case_error_4() - test_sequential_sampler() - test_distributed_sampler() - test_random_sampler() - test_num_samples() - test_num_samples_underflow() - test_schema() + test_generator_0() + test_generator_1() + test_generator_2() + test_generator_3() + test_generator_4() + test_generator_5() + test_generator_6() + test_generator_7() + test_generator_8() + test_generator_9() + test_generator_10() + test_generator_11() + test_generator_12() + test_generator_13() + test_generator_14() + test_generator_15() + test_generator_16() + test_generator_17() + test_generator_error_1() + test_generator_error_2() + test_generator_error_3() + test_generator_error_4() + test_generator_sequential_sampler() + test_generator_distributed_sampler() + test_generator_random_sampler() + test_generator_num_samples() + test_generator_num_samples_underflow() + test_generator_schema() diff --git a/tests/ut/python/dataset/test_tfreader_op.py b/tests/ut/python/dataset/test_datasets_tfrecord.py similarity index 100% rename from tests/ut/python/dataset/test_tfreader_op.py rename to tests/ut/python/dataset/test_datasets_tfrecord.py diff --git a/tests/ut/python/dataset/test_iterator.py b/tests/ut/python/dataset/test_iterator.py index af5a66e89e..6413e79612 100644 --- a/tests/ut/python/dataset/test_iterator.py +++ b/tests/ut/python/dataset/test_iterator.py @@ -33,7 +33,7 @@ def check(project_columns): assert all([np.array_equal(d1, d2) for d1, d2 in zip(data_actual, data_expected)]) -def test_case_iterator(): +def test_iterator_create_tuple(): """ Test creating tuple iterator """ @@ -95,7 +95,9 @@ class MyDict(dict): def test_tree_copy(): - # Testing copying the tree with a pyfunc that cannot be pickled + """ + Testing copying the tree with a pyfunc that cannot be pickled + """ data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=COLUMNS) data1 = data.map(operations=[MyDict()]) @@ -110,4 +112,6 @@ def test_tree_copy(): if __name__ == '__main__': + test_iterator_create_tuple() + test_iterator_weak_ref() test_tree_copy() diff --git a/tests/ut/python/dataset/test_shuffle.py b/tests/ut/python/dataset/test_shuffle.py index 460c491ca1..6da7a1c885 100644 --- a/tests/ut/python/dataset/test_shuffle.py +++ b/tests/ut/python/dataset/test_shuffle.py @@ -13,10 +13,9 @@ # limitations under the License. # ============================================================================== import numpy as np -from util import save_and_check - import mindspore.dataset as ds from mindspore import log as logger +from util import save_and_check_dict # Note: Number of rows in test.data dataset: 12 DATA_DIR = ["../data/dataset/testTFTestAllTypes/test.data"] @@ -31,7 +30,6 @@ def test_shuffle_01(): # define parameters buffer_size = 5 seed = 1 - parameters = {"params": {'buffer_size': buffer_size, "seed": seed}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -39,7 +37,7 @@ def test_shuffle_01(): data1 = data1.shuffle(buffer_size=buffer_size) filename = "shuffle_01_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_shuffle_02(): @@ -50,7 +48,6 @@ def test_shuffle_02(): # define parameters buffer_size = 12 seed = 1 - parameters = {"params": {'buffer_size': buffer_size, "seed": seed}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -58,7 +55,7 @@ def test_shuffle_02(): data1 = data1.shuffle(buffer_size=buffer_size) filename = "shuffle_02_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_shuffle_03(): @@ -69,7 +66,6 @@ def test_shuffle_03(): # define parameters buffer_size = 2 seed = 1 - parameters = {"params": {'buffer_size': buffer_size, "seed": seed}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -77,7 +73,7 @@ def test_shuffle_03(): data1 = data1.shuffle(buffer_size) filename = "shuffle_03_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_shuffle_04(): @@ -88,7 +84,6 @@ def test_shuffle_04(): # define parameters buffer_size = 2 seed = 1 - parameters = {"params": {'buffer_size': buffer_size, "seed": seed}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, num_samples=2) @@ -96,7 +91,7 @@ def test_shuffle_04(): data1 = data1.shuffle(buffer_size=buffer_size) filename = "shuffle_04_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_shuffle_05(): @@ -107,7 +102,6 @@ def test_shuffle_05(): # define parameters buffer_size = 13 seed = 1 - parameters = {"params": {'buffer_size': buffer_size, "seed": seed}} # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) @@ -115,7 +109,7 @@ def test_shuffle_05(): data1 = data1.shuffle(buffer_size=buffer_size) filename = "shuffle_05_result.npz" - save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + save_and_check_dict(data1, filename, generate_golden=GENERATE_GOLDEN) def test_shuffle_06(): diff --git a/tests/ut/python/dataset/util.py b/tests/ut/python/dataset/util.py index 11c5735406..649f638b49 100644 --- a/tests/ut/python/dataset/util.py +++ b/tests/ut/python/dataset/util.py @@ -24,9 +24,6 @@ import numpy as np import mindspore.dataset as ds from mindspore import log as logger -# These are the column names defined in the testTFTestAllTypes dataset -COLUMNS = ["col_1d", "col_2d", "col_3d", "col_binary", "col_float", - "col_sint16", "col_sint32", "col_sint64"] # These are list of plot title in different visualize modes PLOT_TITLE_DICT = { 1: ["Original image", "Transformed image"], @@ -82,39 +79,6 @@ def _save_json(filename, parameters, result_dict): fout.write(jsbeautifier.beautify(json.dumps(out_dict), options)) -def save_and_check(data, parameters, filename, generate_golden=False): - """ - Save the dataset dictionary and compare (as numpy array) with golden file. - Use create_dict_iterator to access the dataset. - Note: save_and_check() is deprecated; use save_and_check_dict(). - """ - num_iter = 0 - result_dict = {} - for column_name in COLUMNS: - result_dict[column_name] = [] - - for item in data.create_dict_iterator(): # each data is a dictionary - for data_key in list(item.keys()): - if data_key not in result_dict: - result_dict[data_key] = [] - result_dict[data_key].append(item[data_key].tolist()) - num_iter += 1 - - logger.info("Number of data in data1: {}".format(num_iter)) - - cur_dir = os.path.dirname(os.path.realpath(__file__)) - golden_ref_dir = os.path.join(cur_dir, "../../data/dataset", 'golden', filename) - if generate_golden: - # Save as the golden result - _save_golden(cur_dir, golden_ref_dir, result_dict) - - _compare_to_golden(golden_ref_dir, result_dict) - - if SAVE_JSON: - # Save result to a json file for inspection - _save_json(filename, parameters, result_dict) - - def save_and_check_dict(data, filename, generate_golden=False): """ Save the dataset dictionary and compare (as dictionary) with golden file. @@ -203,6 +167,29 @@ def save_and_check_tuple(data, parameters, filename, generate_golden=False): _save_json(filename, parameters, result_dict) +def config_get_set_seed(seed_new): + """ + Get and return the original configuration seed value. + Set the new configuration seed value. + """ + seed_original = ds.config.get_seed() + ds.config.set_seed(seed_new) + logger.info("seed: original = {} new = {} ".format(seed_original, seed_new)) + return seed_original + + +def config_get_set_num_parallel_workers(num_parallel_workers_new): + """ + Get and return the original configuration num_parallel_workers value. + Set the new configuration num_parallel_workers value. + """ + num_parallel_workers_original = ds.config.get_num_parallel_workers() + ds.config.set_num_parallel_workers(num_parallel_workers_new) + logger.info("num_parallel_workers: original = {} new = {} ".format(num_parallel_workers_original, + num_parallel_workers_new)) + return num_parallel_workers_original + + def diff_mse(in1, in2): mse = (np.square(in1.astype(float) / 255 - in2.astype(float) / 255)).mean() return mse * 100 @@ -265,29 +252,6 @@ def visualize_image(image_original, image_de, mse=None, image_lib=None): plt.show() -def config_get_set_seed(seed_new): - """ - Get and return the original configuration seed value. - Set the new configuration seed value. - """ - seed_original = ds.config.get_seed() - ds.config.set_seed(seed_new) - logger.info("seed: original = {} new = {} ".format(seed_original, seed_new)) - return seed_original - - -def config_get_set_num_parallel_workers(num_parallel_workers_new): - """ - Get and return the original configuration num_parallel_workers value. - Set the new configuration num_parallel_workers value. - """ - num_parallel_workers_original = ds.config.get_num_parallel_workers() - ds.config.set_num_parallel_workers(num_parallel_workers_new) - logger.info("num_parallel_workers: original = {} new = {} ".format(num_parallel_workers_original, - num_parallel_workers_new)) - return num_parallel_workers_original - - def visualize_with_bounding_boxes(orig, aug, annot_name="annotation", plot_rows=3): """ Take a list of un-augmented and augmented images with "annotation" bounding boxes