From 0b3957bfcb2dd2887a20eb9e3eb8400729eedbd0 Mon Sep 17 00:00:00 2001 From: Ben Epstein Date: Tue, 30 Aug 2022 15:02:56 -0400 Subject: [PATCH 1/3] test that should fail but is not --- tests/data/unittest.parquet | Bin 0 -> 42011 bytes tests/extract_test.py | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/data/unittest.parquet diff --git a/tests/data/unittest.parquet b/tests/data/unittest.parquet new file mode 100644 index 0000000000000000000000000000000000000000..508abbb337cec7ba7f9cffad6a1a8e59cee81fd4 GIT binary patch literal 42011 zcmdU&3wRV&mdC3q8E6<(4Bc(ViEDJ_ETd>12m(6T9Ug*!gNPzCyM}3)l1ysC`1rne$WBqI{-&B`O4 zBAv2gAzs3_l&YxG%fpKMb;;K7Z@Q|cX;7&$e@bXxQ@5?%`+lt+hUKC3&&5ER(*R?ilI%$hDm%z?T zkizZ4$8qazq3TzI5;xzg+!TC|Scvb#PUf~cC{~4SiuCET7cL+iOL{J&gNdR&et3JIO1M%SyrT*S^Md;i)KyH z<1SaDTUA&eg+q}nPbVpiaBrp^VOJJmVbT2ga~6>}kHuh3EO4Aku#ZtUBSTtJJnVb7Jx9nP>W*(S zxV^<)x(8WaVu`%B#FKFk4vMA$Hic7d9qAA2`wJZ0iH~bfj%y<~aJ3Z~(+N+XYl$sG zM-@6@`k~k|i(;LOy&X{>zQtBDw$|FHMk`ckz8SXOtZNvjkCcVC_EtMQr_eHj+&Ksu ze-t#x;5utr!bfNAjJH8&_21NUb0PMpzbgOg@i(yMo$Z6_%G@Is;`^{3;kG)OZQo$K zvw7*PT?Npbzz*kjJvj;J^&N?_0Rt}XjkQpRO+}$d?#2Uzm zM4>7P_oF)N^az_o&{@Za!=~XD1ogn0!+o~k5+!aN9P)(-*`h9PpNv`^5XGYj1l3th zBVp@E3xay^@f);H@o%-R_Jz3Gg6K{lsMP^cGKN4GSg5|!f=FUZ@PB(Gu{)LBLm;TmD!m(O#$6x~S7hfLi`v<8|3|Ia+#s7oMMOS)Lurw<@dvfliL!om;Pe^MqpoDVMSTrb7`<_ zs-OOPz|ddzVdwasOR4x^#$ojP&X-q?o{nd5cS2r*GrhSl>+x=@qZ#*2w>zPy{+b5O zQ?Vnu9r?KTY+Wz?6~{>MmBbPH=uCTX^Uz-p!paAeJHQzPvAFTx%XilAWzG6AGV4<- zdY}W`={YuusJ~{zrr8z*^}w2g$HCPeMi#U=xO55oLWC^{DqX160a3hwKnVJ49&DXw zK~N7i1hoc+$c-B{Fu)gLfCZ6DT7p^~5G4;22tj`>gk1|Q2bNMjFUH#iynj~Q_=F0_id6)0d-3tz`Vhr~<1Uj1^3Ib4#Hqn{EK%X~ z&(0rZU}O+iVOPxUidA_dS6+qR$4`B&PhLvw`=mJ5)jrbEBDG5q#QjrawHn>8!ttk7 zSoXBa^o_4O&qhc^Ovs{vnh24XO7Yv6h5#<+3tAYX+r$WjP55`{pm=z(grn$-kG z5NnnHqC(3mn}a&BIjA)-2iKfgz8qN=2i2U^>foqSiAV2(47>+OLDyT>kG!rLRuZVz&@thl2+N?)QJRJgUgs6yq7UL3xz zJqNAsk?^R9rr#nN$}*xn^SCqqe(ZZ3qo1Uys-{5Y%OozY#im8)Ld!-KYBu6@qXnoo z_UjdUu^WZh%X!*Zh?m4dd|%c^ZYv&4yYbmyVR+se>*OXCDmJltb9?de?OAWaV9$f& zE|UgQ`YHi&YOI!5unW9${wM{J0eUUavY(-%VSaK4VAn+w&O4;}Z;`r;XRXefw{+ri$>?V!244~0);7+g?}Y?I?^Uk>{>w`{^etqzWw zw_AOgShU%{?xda za!|ch|1NU8Yje<^W3_`qt={e6F<42sE?V9v99*9nH{UkyLjk<+{6T~cWey99d-C8V zu@K*49g1(;R!4^_{r~}-3eMh}!5?ZwA;>pco5K=w-h?e{AW5)h|?Tl}9Y z>^ZS$h1x&p>Zet=o(+>~np(Dii_>gd-}=w50)M?K&e2O1BKYvK$;QNWJ_OE||hHiuxv z!zoAeamqneUBhQ8Z2rvR5S&;Xf;BM5ggB#6;`jM+UpoJf3otG*Awz>Y7jB zIC;Y25S&;Xf;BLQg!WkbKa_)hn^tmCg?*n}974@o9D=nYIW{*@4yx|f{;vvkjTZ{X zt}h9PsO~m@p+du-lMKRscVjR1iyM2nClg*03-Ntf>$|OZFug$bpGFX;$}?o@?ypoh zdy0X$z4-WUeH!8S+jE!a0^$3LC*xKJNd4CY#HsRX&fq`c&WI@7kB@gxlzSrmxyrk) z_oWy?o^dmOVMGmQu#b5IBZ}hnLuf{xdGEd{=kP6dDL>k>IhdaM3o>z8WZ$KBR*m9h za^@QqHh!ZEqJYl73rH2knvLXCdInh8w>YA;oF`aty?0UQ)o_dd$#6o;u*7w|2M=x0UBnHSYQ7^r>c!>skex~7t>h|Sx2G6dL8fme0 zDdPC+z4cFPaO5clgVlhq0MaEOOz}_Sy2s*)e)1A*z*ZOf_39r6hHiuw!aMYDj4yyO|E!UuAxy2zkv8qV02G$po z436w6ejHfBIVRa0g4My%@GRw^darJU25VPX9D);zL$C(skkB5@t0)K6dkxQNu=zQQ zL#TO+L$G!vN68w>LG@nqY7KU+zEC*UzCbucy;rhUgMH5@8HD`?$X@K%KlXCVBwi8= z@qJnAyRCRI9k43O5QI-A;^KpKztG_DItJqQ;^VvZ5sL@MT_z2r@bXx51&@;KzX_TsQT$IRvYNqizS~5cJ>csQ+HK zI0PpahhPoLQ9RAy$e!uPfhD|prr8{V)xpv52IUa+-%iwjJ1q{uiNzsUgK{Vm+M{_l zSiAM{<-@Q4T@>RpLlrd7*Hu{SU(7(0_YS|Gk}L5cUfqd$HdR z*~=}Pcu6e8_hqf`w&KBbz^Zr$L45Sz?@<3$GZ42IAK$G{BRn|nGHD={?-3A}{@aWC z@7?o9c_1>7f8W=qH-59HgXOj{z&@1mm=JBNQ2{lVEy0i%jXQ9 zeZxZ$N|@{p)gq3+BCP%h`};>p9$>LXv0-M!^><~qp8~-WQ-R>VtmgRswiPXOfNlH} zfe{p8eVqnnbv8!;*FhU4*W}3a<;de43SBu$tqzWsddfi+VdKXdZ28#cpib;6qSn9) zWUj%{eZC(DmT+Y|*KQBN>fk6nN;#+^Y&oJqnr_?#!v6-Yxz`znoq6xgSZabpZGe8ApI%NZU@C~^t`ymVWem- zJ3%?95-j~ML0?!G(uxdyvGXZ7Xw;^W)1zJ$S@yJA0TvX>i# zm)NIejpMfB!E`(e%gssYAmvJYyP2X|=9{3Ug-CC_ei=M?hWxmkh(4Tb$(~c8;=UB`CkFYv5 zn?bzzv<4MlC6PF&Zo@^FUt@Hhx^2bRZP9HPuln{U?_(<~b+3KajBoC1ftTPvzRZ2~ zfy&(9wmQ1XrnBT)@X~FoztP~sW_>7h`SLk~XCDzRZAQYTB>fG?Q4buw$R2kCN`&ff z5yxM*ZEw-w)L*>DbzlEMVdfby*$ez&u!PUE^X&Oaup%=Zr|K7{*)W1`t9b%5@IR4; zgOpxPf)l+_m|3s})(QDm{J8#4cmzVe6@R`Rze256{F=TVu>MIJKhtYk)V%+HNg9+b zvEmn;Sn&(ipz+5S`o)hWW_96~brQkqjDIPOpXoF!{--hir>*z}CqAw{`;jnVvo^zo zQhxCZ)nT1Pu;SrH|1ss1a@Q- zVz|{DhdA&ujzKRcc@WhZCO+gh3uB3?B5;e<6yM*rqJ_@F4X+RgUB&VE>&nw$bGa3N z5Z4}!pk(GzSyI?^+`zegB<-#H92pSA!ppMn4~pH2WW zpBeyCv-yM6Ec+li`#mHxKRqNrgM2&l!b1x3wS)9L>L4?}d30$H9GSL`jGpWfPS-~! ziFwFDW`1yxnYSBc=F%!r#Y+#d|!wn?oRs+d7&p>i6GLW4k3}ol- z0{J<$KzfQRs7Ou5DQIsVDM(FzCrCx!C6JsCDY_<%gB+%xjP4AFyRkt|a^fP8oKOfP zC;9=&33xzuVjPg2um&IDLT<|Ew=MD?){~M*22zsK1yYz^1=^c51yYmD1X7V?1SBW_ zz%dR7JIuWn-5U;HtPgf_vvh#u>=__As|85THUYA;K!EJ*3?M&i0;DW=0i-yC0JJwR z0HkKUKBQtDJxE^P9<9{}z1Owq;c&P+8}uYELSb@xF)Ps8y{-&Wv+fI0u`UZFubYaV&4^<1qa|s0lL#ix z&CH5*ON<@+FI8^If4Bd!v3%w z6N&4_^vTItgd2B`*5kn-*a!WuwK;qaZVs83^FU4xZVXNg9&%ky4)Z*8i0%Y~XCK%f z#s!E${!<1GP?Rx= Hv3dUwW{2mD literal 0 HcmV?d00001 diff --git a/tests/extract_test.py b/tests/extract_test.py index c4fb2290dc..b3f45b36e4 100644 --- a/tests/extract_test.py +++ b/tests/extract_test.py @@ -1,3 +1,4 @@ +import threading from concurrent.futures import ThreadPoolExecutor from threading import Barrier import time @@ -33,3 +34,25 @@ def run(_ignore): dff.extract() pool = ThreadPoolExecutor(max_workers=100) _values = list(pool.map(run, range(100))) + + +def test_extract_pre_filtered_and_not_pre_filtered(ds_local) -> None: + with vaex.cache.off(): + @vaex.delayed + def add(a, b): + return a + b + + df = vaex.example()#ds_local.copy() + print("orig is filtered", df.filtered) + df2 = df[df.y >= 41] + print("new is filtered", df2.filtered) + total_promise = add( + df.sum(df.x, delay=True), + df2.sum(df2.y + 45, delay=True) + ) + print("promise is fulfilled", total_promise.isFulfilled) + print("task 0 is pre_filtered", df.executor.tasks[0].pre_filter) + print("task 1 is pre_filtered", df.executor.tasks[1].pre_filter) + df = df.extract() # has 1 pre_filter task and 1 non_pre_filter task + df.execute() + print(total_promise.get()) From 94bcb9d619b769b2166a63eb2e7b79c886f134ed Mon Sep 17 00:00:00 2001 From: Ben Epstein Date: Tue, 30 Aug 2022 15:03:24 -0400 Subject: [PATCH 2/3] fix test --- tests/extract_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/extract_test.py b/tests/extract_test.py index b3f45b36e4..f4987fd3f6 100644 --- a/tests/extract_test.py +++ b/tests/extract_test.py @@ -42,7 +42,7 @@ def test_extract_pre_filtered_and_not_pre_filtered(ds_local) -> None: def add(a, b): return a + b - df = vaex.example()#ds_local.copy() + df = ds_local.copy() print("orig is filtered", df.filtered) df2 = df[df.y >= 41] print("new is filtered", df2.filtered) From b5abaf49a9ff2bbb647f399e7b7ec59fb8160e73 Mon Sep 17 00:00:00 2001 From: "Maarten A. Breddels" Date: Wed, 31 Aug 2022 11:06:16 +0200 Subject: [PATCH 3/3] make test fail --- tests/execution_test.py | 21 +++++++++++++++++++++ tests/extract_test.py | 21 --------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/execution_test.py b/tests/execution_test.py index f02d7d6447..14af86f5a1 100644 --- a/tests/execution_test.py +++ b/tests/execution_test.py @@ -373,6 +373,27 @@ async def means(): + +def test_mixed_pre_filtered() -> None: + df = vaex.from_arrays(x=np.arange(10)) + df['y'] = df.x**2 + with vaex.cache.off(): + @vaex.delayed + def add(a, b): + return a + b + + df2 = df[df.x > 2] + p1 = df2.sum(df.x, delay=True) + p2 = df2.nop(df.y, delay=True) + passes = df.executor.passes + tasks = df.executor.tasks.copy() + assert len(df.executor.tasks) == 2 + df.execute() + assert df.executor.passes == passes + 1 + # make sure we don't leave exceptions + p1.get() + p2.get() + # def test_add_and_cancel_tasks(df_executor): # df = df_executor diff --git a/tests/extract_test.py b/tests/extract_test.py index f4987fd3f6..c8b4b8a65a 100644 --- a/tests/extract_test.py +++ b/tests/extract_test.py @@ -35,24 +35,3 @@ def run(_ignore): pool = ThreadPoolExecutor(max_workers=100) _values = list(pool.map(run, range(100))) - -def test_extract_pre_filtered_and_not_pre_filtered(ds_local) -> None: - with vaex.cache.off(): - @vaex.delayed - def add(a, b): - return a + b - - df = ds_local.copy() - print("orig is filtered", df.filtered) - df2 = df[df.y >= 41] - print("new is filtered", df2.filtered) - total_promise = add( - df.sum(df.x, delay=True), - df2.sum(df2.y + 45, delay=True) - ) - print("promise is fulfilled", total_promise.isFulfilled) - print("task 0 is pre_filtered", df.executor.tasks[0].pre_filter) - print("task 1 is pre_filtered", df.executor.tasks[1].pre_filter) - df = df.extract() # has 1 pre_filter task and 1 non_pre_filter task - df.execute() - print(total_promise.get())