From 37763b2a63eb98647c712a5f02d5ac9f03096e41 Mon Sep 17 00:00:00 2001
From: fixminer <fixminer@fixminer.com>
Date: Thu, 9 Apr 2020 13:08:05 +0200
Subject: [PATCH] java dataset collector fix

---
 .../__pycache__/abstractPatch.cpython-37.pyc  | Bin 7749 -> 0 bytes
 python/app.log                                |  13 --
 .../common/__pycache__/commons.cpython-37.pyc | Bin 19663 -> 0 bytes
 python/common/commons.py                      |  24 ++-
 python/data/redis/stopServer.sh               |   3 +-
 python/javaDS.py                              | 178 ++++++++++++------
 python/main.py                                |  52 ++---
 python/otherDatasets.py                       |   7 +-
 8 files changed, 178 insertions(+), 99 deletions(-)
 delete mode 100644 python/__pycache__/abstractPatch.cpython-37.pyc
 delete mode 100644 python/app.log
 delete mode 100644 python/common/__pycache__/commons.cpython-37.pyc

diff --git a/python/__pycache__/abstractPatch.cpython-37.pyc b/python/__pycache__/abstractPatch.cpython-37.pyc
deleted file mode 100644
index 61ebda9a3461c9d654ff77dce1143c78255ab6d2..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 7749
zcmcIpd3YSjRqyWUp6Qv<9Fpa8?W{4`U^%<;K@yh57~7Jacw@_2OFp()uiTo~Gis}+
zdtBWkOG?jXv+`y)8%!JsBm|PkkN_dvXTp7k`@WV20^i49{)fN#%OBqtey>_{$TrD8
zFm_c}y{dZk>eV|-CrhP*g5QBxKK$avcPYw$aHspLqj4N1oYfSC6jD9K;;-hZvx=i#
z&roKkV(GG_qh-jJfmW8Xlp~Y!RG=aaQHjb_p(+j22yLNJ+DhAKJMEyIw2L02-L!|^
zLXXp4`UcuZ-$-wzC+KbTO|+lJ=m5Q)o}{N}oDR|<I!s6Co9P|&PI?!83q4KWO3%=@
z(NX$#I!50?$LR!}qzRg&Q*@f%P4A&I^enxX&eA!WqVqIO&(Q^%p;@{}m*_HGq36k>
z@1z&#D!oW^bd7A9ry3DCjcZnp>eQeGa_KrPl1EMQDWDeJAVDET)TSl6Nz1fCtMopK
zsY5T(`{`x+0KGzMbc<f457KM&I(>-Vpbyh`(QW!}`Uu^jkJ9(h_tMAc`{?`W<Mavo
z0s18UApH=1iat%Bp>?`TKTJPDpQX>ykJ69PkJIPrC+G|Glk`*c)ATd+v-ETH^Yjb!
zi}XwM%k(SsMfwu`D*YP$I{gOyCjA!uHvJC$F8v<;KK%iGnf{Rei2j)Vg#NTq$9evY
z{(RkZa(C733jGC2zxS8h6)R7F^#G(me?0(sctw%^W?=kpfn$jNj{g3ZVwLD0=pXSd
z)0^~9_*UrO=%49dZmCw4{*}H0$S|oh%Gkf(V_BOTQxo+-lAk;=dtz?-#O!;L9NCc_
zI#HrOJNe$Ht(_<Qz+Y(w?Qp`g!*J56dA6`4H}I|E2_fv26SbNXhE_>-CWMoYRb}^#
z??$fexvP#y@)9z;(sHcIi7<3TG!xm8({%jEDgfX%d~PSkNiV1^TIG{L;5oK`){Put
zd*}o8Vt+|_!VaTT%PoP;B{36rR9gUz?a;BR6AK`+hLzprTP5{p{4lcpn$sh*G7&Ue
zt|w`}2x6_R6M;`4I`D0806T-jNVIEFAkO$pK`lMyFu;-Px1By+#mS)GDW7V#qLp4h
z+0sYZ16sw?uH(@L8>_N2$mOBaff$&4rrz%ipJ}!N5e?3+obl`Ka(^4EIB*A6?pb@u
zrl4k3&)V~j=g<JIb567n&>*gA3K}?XajNZkU6z)n+RgbshexLE+M?ZX1~F`(wgv8V
z!3iM(G(gqxbPz_H39U{GXEUTUE!-yVV#$$GQ+loqQL4KRP1#Kd(FG@J3%`GY!i?K&
zd5#?1Xv=jwXI#JGIhSnV+VkEf`bsmAz+m@8yJp%gM?Ctft8#4cs<ukVLp&<X+;mwI
zp+i<BZTD&1I<w-}7DV7PKl<SE?1CE(um^n+-0Y7J&5FJhSrtCkAe(nQ0O&6)@<{i1
zdmh1pflVKK#PDfhH>8H{e#BsZoATua*BelR(XOHe^??nFXI<Zc!byZfQq8-)!mQmm
z<)y};IO{A&-A?|T!$Nk!sat5WgmQDF*MxvU7@+9Z=qZ1Jb(=QnOXZZ`ZVvKlI|Jku
z+x{laDKV@1=!SbuzA7HG`T{7sLzi8TYPLuRt5bn!+TID@4`9~VB2~}XV$l%;kZq|X
zrbilpS5n9&+iQaYSFq9TqT#Iu>3@9lP`Yw(f_W|=))HamVbKC#(wd)i!F6dUq592v
zvIp*^dd3F>L#M`M%jv_cL_Kv&L0y{({yR|Wnq%$k<LKST#?a`1ZuA#yVN4HBEu;Y6
zF)`0W|7_S;D>LQXv<iJOO7u3&bCRp2;*;n;&(|X$O!Bp^%}$JYb{>i0`s6HICc#_D
zDJIvv#L5DF)2$_0I5YSlYq;U~j&N&pA?%^#Kw_p%P*KICpytZ^ob%i;vWf!`EAPPn
z%!N@C&XSw(<JfQmz|F)Ew%>3-7CW!R1fkMhBzhx=g2eFH3MV-xjv&?<J8S|I1k}pm
zMtobWSXsvl9k-qsH(A9JJv|=`r9<Gxf>T>`$jY~2Rb(^Cw*AnqJBdbtl?|mSO7t3x
z5gfZyx7%KnWa>Ct4wYlq780W&f_5v>U3Pz{Ya%i0z$az6)NRel<77Q4J61kyIW;L{
zNmfWR%zJoVl4V05)U9%>tCx~(i6Q$|bx~{{uyPv^t7MBtn=M1S!0dK$NV^3??*O-S
zz*aH+rW6?v3+P%wel|deP!Nz5<VS+6Y^y!*xiu@-67CX$jwIg_L4@lD6Z0$<VatX<
z8|@I@Ajw2xB{AUQ!JS;)MI_*%AF)ZcvhovJrXM)VHK!FNCSsTtA_yQQSFi{z;hO4v
zSy|j9j8~EaSG_ht_;3ZVf!GN0S;u9flW1<;GTb@~K-R6N1AOU_1h_jy_6SJomJToK
zC)pMoWIHh&A#jOC)2<1~(GpvZBn$tCaKNfU%(}r!dj0}Pj8;;?%sL@B>LdlbXJf1!
z9BaeJy-BH9g6+t{215c@jdSNO?cIN0jaN%!$B*wHKQMND_Wbd^`&WzI?upsstH$K1
zvw#_Are^qaae5M;E!`PdAQxh0EyF=4emN8ah5O-tRZ${kU431-tE$L#I+=(w94$tc
zL;CGZN00UEIvKAkltt`}>;^C+GOlN1gYr8SDl}9o-qt$VI7@|{%6g9To?F1i{i8}3
z`kGQ#kqwn@nR2AUBiBvLuJ&dR%h?r)W26V&G5{UzLAO4Dd0P+jc8qAWV^I_T+1(rY
zR+pk3a&*_Cc1IO&?TtUSX}o)m-96|o-VJg4o+<aMI}|0!!xGru0^+E9ND-B>Ormoq
z(OCwd{__G{RaUj94p{lahc<p=S^}A2!8U3^+mG(?94p^~(+TGmofTvQFtJ{{>4!BQ
z;vau`hZ&U6L=`LR%3XBFRL&kC)3tyc&LfsTuZUSpxIcaHB8+Qz(Dq$#F<5~$X&;2f
zU59pr2L%i*?9su<38RBBWKAdmtZ2{{HRoV!1wP(CXwO4{5x`F~t>dkg#5|gso@384
z6)4K;<6dAByUTMhYVP<^$&c_D8q!D|+jIbyvn{-IH!3~=%NnV;XXMn}BIQ+WCm$*^
zC8jawzIqTxGQyVUMq{er!=tZ@Ac*7{#YI5GB~)ioBt>lHz{gGdZem=7o!2<`l8iH=
zHp0~n+$m0GULWzf9^$N=`ZX}e&>Bg-HYKlJqWc=C&-YM+=4j#Y|4#EDX6DP+!~EQ4
z%ttn1?%l#Z%zDHv;7tSfp{F+kJ{jp<h|9b&%i(?XDV*vGTD?0F&-37B&hc$+TnRg*
zFv&rd;3qb7kg1U~p-z$-Qn53n05oLK!n;aJLvMRA$9u5|Zl>Z<x5x64(vXdUMFd|D
zapM1PZzfpo{eQ9NJ?7KbKQCX3+;c3S<h-EKU{{KGq7MHg9CAaeID>4f7G2;@qQeXM
ziPnMxp2UL#5)H(2hTUp;E5hOpbMj)7B0OgZR|HD{qvbhEjwe{-EK?$#qxnRK;ldS5
zyXJ&l0hMCGmynb;8H#r@oh*M!rl9IZRUIvis@t?(>ZrC2|8}XSx=k&hY=g36R*hAd
ze_0e}#RN)t2vsZ%$O~FW>txV|;J{I-{4m!sVm*cnfXO!S%^bzk9Yjf=l1CF)u~@vw
z6^rdyHYu_ap2Ok6GihopX-W>KpCe$?VU!~eh*?r-IMK$cYd{pr=R|b~C*XgAZ=<H8
z;$H=&gd(Yb1G;oaU73xP>nbZaX}2|1iL_W<Q#(34-CIhmPr_Jr^gHVEes14Uuf7FN
zHzQA^t!1RM#e?IzepgvH?kb(!@_)y<N#*K)b<CI@F~(wZk-3tGGt9ynelgBdc3?$e
z`O9$uEB1GaaXuZ3i|L9e%lrdW#eQa7T+6A-jW;eUH?9CzPP)#gAHudQu~`pe!_3^k
zzC+9Jh=;K6lbw>>7h`eh(fh`wHB-geZ)n*2P2PVqWn@aqu3+TQW#zKskLya5TP(n}
z-%^!MSyfick-46a%N*KV<|kZ0Un*Zwo>zSBN-7umsag0I(^*D_F&8ZMl$IsWK#_)G
z7a9$isjDL$HCADw;J`x2+eft<I=hra9Y<9NBU?lq_b<TLYvYF!)qOy0?iY@7DuYa_
zb?gR^hP+KyS$%9n(*`|hcb>S0Il|^@9#x`@6%s876SHX}|A5nj%405CQfD{|ac^))
z8U+mRY4HS-jwF9dM$WLuf*2!<<XA;4GU7B8VPzK{_f2-Vvz+A_+zL}`@AI;2Jg$af
ziaSgiayn_{QxXknbra25e&GCwa8i@P8-_<vDVna8p)uu58QN2Y_E$AC{i?gTCreuI
z)AkleRPiCK604|c91-zv8wx9+RIrm!uvohe^;*}U8YagVUIvPgU)N(qpPfQO=@cP`
z`E?^M+*KirZ|e+2S;V4RWX415`MXfQyHLuwxL!#6EI>ELF9QNqEv_jLUKksw)l1P(
zoPohHpmh3nWlist{c(&L82dzbOpatQGTMbeNplivbjnfbdO0rh-x(;|&`3o*ox81f
zDsg4K0{Tm!wz@nYRiTJX>m?Yy;m(Mp*GzYghwmDyviwXug7G7rEpdreJsOU;tRXV#
zjK-sw2V=uj#<_U7Zg#fDRh(_BJlB0FP;yy`Mxrfobq(j11~Z>VGpm(}^v3qGH{bgN
z$?IOtj>|M3iQxhgnox#q2!pc=j!m3(;UyTcY?$Qbv^lg__aIG2cnmYUcS8vFJ`r~R
z)7YqZ0XLXrWj+;(_jAYeo#<vD7MCsK-NJ4ySbLCO`7(C|N@QF<C#_+7zQLnmD$)+G
z%y`Jy3?IR92+?fe0fG{p-~ki;df@tEL83<%PII+z3C{q*oE**b3jn`^I7!CEi#WUk
zUcf!ayU_NN61AJH)5tDE)}^s+Qf;KJq>AtgkfuuVDz`q!)oZ9MypniK@VqNNf_|bl
zq!3$XmvqsR(<HNa1LP2O%jBb<Wfji2v~20jwM4BaW|xda-H3<#5!lPX^KlgEBo*n8
z;Ax>nqv{C0S!miQ%7`|q?odrl2XsX09L6=lt9ur37@eL{eFdGBPshp&W=Cb8+tE8l
zC)>%v->^MI@Wu)z{cwTQbro%RG7jr8QjApiw*nauVZAfN9%(y-5$Pm<@n@M%Das(p
zgfD<Ps1Q~yMv57mYbxefFqVxg{AV@kDRb*kQp~ErV`{9XROnP-{hxvL&o36lo3Q;=
z*#6y7VZ9h*f2b_Cs@#gJuc#ZIp!!mElP4%nCF)!n!E>;E@2mJF#YwkbKjkBMf{@Et
zl$`=xhKTT$R%mQU2F2oI03<p7W6sQ}S@AyEbNnS2FV0B@DV2yB1chlwu7@|9ptT}C
zz)1Rrcv!46;4@sw7>ez1DrK7`N_>h3IPBwh6`9=O<p_c!-`QkK25lLHXOMVWeA=pP
zDBX;`gojSp_}3YQl$^f!d=juPp@e%;DVhdhDX_SxWe7|mgReT0HJ@j5Tu=qi1}E4Q
zkBtcavuRT1N5a4#Pd}k#gZLaqM24%+a`gqCn8BZrB+INqGgtya{N!O3$nkn@*AIK`
zb_?mIoFko_jIUBh@G_%hN(Ro-I}mc$W<YJvImV7Q<m|!JOpU28l!tH{6Dl$!Pm|Ur
f+G&gDRJGmeKK$eQh_)AeK+(*%n?>`<!oL3jTzuMp

diff --git a/python/app.log b/python/app.log
deleted file mode 100644
index acfbe1e..0000000
--- a/python/app.log
+++ /dev/null
@@ -1,13 +0,0 @@
-2020-04-07 17:31:35,222 - 31200 - INFO - commons.py:setEnv - ROOT_DIR : /Users/anilkoyuncu/projects/release/test/fixminer_source/python
-2020-04-07 17:31:35,224 - 31200 - INFO - commons.py:setEnv - REPO_PATH : /Users/anilkoyuncu/projects/test/richedit-data/datasets
-2020-04-07 17:31:35,224 - 31200 - INFO - commons.py:setEnv - CODE_PATH : /Users/anilkoyuncu/projects/release/test/fixminer_source/python/code/
-2020-04-07 17:31:35,224 - 31200 - INFO - commons.py:setEnv - COMMIT_DFS : /Users/anilkoyuncu/projects/test/richedit-data/commitsDF/
-2020-04-07 17:31:35,224 - 31200 - INFO - commons.py:setEnv - BUG_POINT : /Users/anilkoyuncu/projects/test/richedit-data/bugPoints/
-2020-04-07 17:31:35,224 - 31200 - INFO - commons.py:setEnv - COMMIT_FOLDER : /Users/anilkoyuncu/projects/test/richedit-data/commits/
-2020-04-07 17:31:35,225 - 31200 - INFO - commons.py:setEnv - FEATURE_DIR : /Users/anilkoyuncu/projects/test/richedit-data/features/
-2020-04-07 17:31:35,225 - 31200 - INFO - commons.py:setEnv - CLASSIFIER_DIR : /Users/anilkoyuncu/projects/test/richedit-data/classifiers/
-2020-04-07 17:31:35,225 - 31200 - INFO - commons.py:setEnv - PREDICTION_DIR : /Users/anilkoyuncu/projects/test/richedit-data/predictions/
-2020-04-07 17:31:35,225 - 31200 - INFO - commons.py:setEnv - DATASET_DIR : /Users/anilkoyuncu/projects/test/richedit-data/datasets/
-2020-04-07 17:31:41,346 - 31200 - INFO - commons.py:shellCallTemplate - JAVA_HOME='/Library/Java/JavaVirtualMachines/jdk1.8.0_181.jdk/Contents/Home' java  -jar /Users/anilkoyuncu/projects/release/test/fixminer_source/target/FixPatternMiner-1.0.0-jar-with-dependencies.jar /Users/anilkoyuncu/projects/release/test/fixminer_source/src/main/resources/config.yml RICHEDITSCRIPT 
-2020-04-07 17:31:41,551 - 31200 - ERROR - commons.py:shellCallTemplate - Error: Could not find or load main class edu.lu.uni.serval.richedit.Launcher
-
diff --git a/python/common/__pycache__/commons.cpython-37.pyc b/python/common/__pycache__/commons.cpython-37.pyc
deleted file mode 100644
index a5d87b21d236f1f6902d6c2edf73630019af1a36..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 19663
zcmdUXdvqMvdEdNtclH4g1R=g9tz=RnB$ALQS(as55+9&MnF3(|Hsy)r?PBi$SYThg
zGa!KVLe4Q|%Wk8{ZBwUpT^CH-v`*4Cb$oJs+H?9xUun<D>1mTb(y5!Kj+>TlnmCW9
zt<om_{k|Ctc0nph-JFwCkaOqGcki9KkMDiIyXS|73I_frzVG`!-l`hLFY;mguY}C=
zxPlig!%&7Y8-^F(P0y5P%d_Nedk*eaBh$=y88fbv^|Dfy^K!V`4Y!&1@=002D@a+<
zE8*@mhML3PaC48hr#a${Bz5+BqbSQX_BF@6vF1JAJp#|-dB3+`@;T%Wcn1dRn+?}<
z<=qFpLvlat9hUnM?+EU0<7o3<@7|=%ecpXiw$Iy#d%kghv+R{oR!~J%QbTH3?NK9Y
zuNqbR)R?+wvE)6V_NxPL7~X^GpgM&6d(>fd1ova=sJa*T<LXs)zbe0BdJm}w)PsOL
ztlp!J;eJ9LR}bMnt{zq=aDPONt4DBuRK2E7s#9ojQhi*VRujmbQWf>MI)mJ4^@Ms7
zwI<Yi)l;}XrrxKX#{F^ie)SCQXVeGOS=^scA5_oc{-k<NJ&*f)eRu1rTZTIKB~zVO
z7v8YE_w_+8s!2ee?t@%XF97m>^`g3r*3YPy)Q52YfV!ey#{I0CQde>Rp!#EKT74M(
zdRA4{ocaiI&#8I!QFRTu=hZ9fW2kjb=JdRG{slvM>U-WW)c5!oZra{OWzHJowcEt6
zsc|!M!t1Jud#+IrLRHt}c9g9L3-yK{Im_*OE6UUZf~_bs-R}6U$eF%8J&6XR7pjeh
zucmdo<_E!~)@=>w@Lav=w^zf-^-f*;iZ8y<UTuXw3!Uj|xa1W()!IsR(Vwa|eJ@{Y
zRD(dW#ik!FwH5P$w`ZZPo7Kj2RpSw@y^*-|a_dIBRt@Xz7Vy+>&ii`CU+}fxs`=iK
zYS&hqek+X2F(NdqHoUy9uFbYrHL7HmR$D89SE#j{&3bsHy%>0<v<H*xVU5$W{bwU{
z0aq}LBs8{6W!^H0w3t9^>b40ero{;{;wTd|e82M*LyrI|$vYpO_(<UEV4~WpH&)sk
ztF7AVL`S!m{aP4IXusiC1Aij)gK%P@zTT|2d|e5omlK^0OhjvfolI}R>CQ%Ex2m0o
z*dy=)!!Y$I(zMGC9(YNYd&scMw~S4rXKY!av23&RfH`4iIeW|8G<&ANa|zt#@HQ<#
z^F50&v@7&v$Y@<uma>)embGd3?49=^-(Oq4Arq9D>dqcNQ4MOGp7Ef3s(kzeMo(NB
zSMt>GZ}^QCmNzLS&c%<<96zyu=^`d1Wk87lC^H^(bJb4gbfdi(nQNGQce_#KG~3D_
zF9DC+i;MNvqL*LvxxkCS9XCLhU-L2-C(nQ61#f6J)PA-3Vzs3jzK-l*BZ#s<e;}@x
zyLN77>hjbJUj7nMFeLJo@PeJELRG1xsaL#&UMCOZPUPrzJM^sDl9v|*me$tNnH{vy
zd$BT6jy*!>#_<sBM`Bp6S+YuI$t;)!t56t5zF_LZD5(+A_{YV~<B|pbEucWpST;g)
z*;3|NBea*DtqdW8qYw#=?4}v!0CAOl(|UVkXK6R>Yrx_U8#Bf$Mh}a62rT|k{HE+#
zLUMq(2_z?wd><q)kOCmZq)s7)loH60K!#CgPXZZ9A$t?ZXbRbvK*lhRLOde0yyvDR
z`TZ$$1keKs^xy#WPy#(X06mgGj}AcZO`!J;K<`hW<pJme3G~4M=z9|Au>t7u1p3eb
z^x*`0VgNdxKpz=^KAJ#J4nR*O(9;9Zi3IxC0QB($dS(FnL;`*CtfAg3bNUp@i(4i2
zzU|q*Wq~;m^0YvP4jC!r{qZXFL7oxF!9K_b1oB`X<g7qW_dz}=ka8d7S%Ey(2YF5)
zXZj$|3*^ZdBCO@yROHTFy*gL9czH&XI3iO+SXFOSz0A`~ZE$1frHdbW+RFu<dJDoK
zatI{Tuhqb2A+?|lRYS1f(973boz;+X3AF2X+EIRHa{6jz`rO=$o~!G%C12G;C^M+u
zVWs@Vb93jUI4UO0d~T!TN2TeRt1nGnn5)cvY<dzRF6Q%BAXB}x=*s2Uxo)NgJ~a{L
zFI>GiDevhf!YH_SX(B3Ixcc(Tmoc(Sv)xQkZ`LO~clPqjmpRm~qrxVhFV4LjH!QNk
zEO=eRpYV!OZ5E8{%49dc;8(*{?FSQ4@zUhExsS|D${PyvtBcbR>JUIt9*nRuef9Fx
zTz6POhC(Vmxy*4+c=?NymoS#uCtvCo(P9SkqS3@FunFg9X0AJ$7F^o?mbY)Gypnu&
zwwv!%bpSnKdBV$1pPQMTyx1L$L1dl5cK-`;nRL^Q#WOlNl+4Gat5+^g&U9m0U0<kE
zu6n~4uAG~ly?p8N<cu^cb)a$7A#mC)mQK%1Uc7u^?()?sDa|KK&N<;C&Q1yfbq`K_
zwB3N(5KPQgo55;pu`>7G3D)(BGm|f0ol90}XdC0o&y<e`-Qg5XlIL_i<L5m)3z<|%
zr#pTg9T-^*8|hoC7_x70hzW3TWJ%k;F$rXRNFO3?SnVLn`K=pu-EQdvggAo9SqM_U
zr6Ome+H6EltlI%^JCW09SC!77s-|d)98gi4!mX+o13dwdm)mGID0)4+wy?NcxMe|D
zldT&frlwKb$dt@s+jWP{vD~OR@Ne&ad%v0dvvB9-q!5%8vPpew)Ec$pynZj<nW{ZD
zKY&X<*F{FbENY0_vusjnc}r9sNJ&R!R95BQvZ2~gYPu>9BV*?|HaC*zEgPzn)w7mi
zRIRwdGIE(T2cv4m)i<Cj<!%|^o9?H~o&~LNcq-;c#<=Mfl^@h}y+d^DW=>HtkOk=$
znJq808wyIX1)~irAXt>&=yaV^%kBBdM$&bWl&KD(EmuW9j-hpjnDuqIQReKA=OY(u
zBGlv!pVNz?{X%Dk-szQ~rBqNVFs~q$_#8+t^wW5qyc0zl9HGJvM8oI8P}k>ILqAsN
zZT)`K^Rm)f>nGVL)o^1~CJQ}lX0`Q0C<?}q7$!BD5flGx*SZf+j;TKY_&b@^aN*R`
zHPK!1oWn)CR~GIkk=gL5vGzoP39TMfn$T9zQkC^q=ItlB7+cOQ<5g&&n@-QsFNK+{
ztg?H~Eo(E=%LFe$*>RL})52n4iFZ_-Yews%juGZo3i>mqv6(fE?l8XOq9wFk)>DAK
zo_)pmn9(|knt9ax@qwDqm+z=)QF+3mXQt-H9eof35FOIj8rA@2vk|#4OWFzyy89T+
zj#hiERo4EEdH{?lw{@9hSZ)Cq>aB9P<d@gldIgos&`tfWb?Wi)g3zCS0<YGbE$#9n
zMRLd0Rvl)LA6eSRSN*E4Eoth8Q3l^{>nK0DUh@U_#&eNfYpTdbXCku`WufFk<V9Jj
z7(`|>GX33)x>p833vbSEb{hB+7brN2#4s&rrej!R2imHG{3zD?Fs`GfzJl^}fvK;i
zl<DCCW&~^~WpQzDnu|7#640kCE49$AjLL`!0$RmW78D3$LuFTNJwsaS#U_l~vwP-E
z=qoL!2V+)rg4~ps=`2EmW8$XAO<4}1#owTUP|!~y1-TT0)p?K*tq_?}sQ#grY1Pl8
z1{SUZt1-$dpJG~{CaeeqzpzwqsF(dFxS-vsE6=SF0JAf)X)|Qv$3e=_Gid!&xWs-m
zKoSQrVJ;@DWJy`7*pnhs%k3B*77eKGQ19bmhGuAmGUS^k?wGm*G(pWNpdG(~L98|-
zyIJi-w29k#qq0hCsnQ9?Ej`N`d$#*KoeX}ktI-g~c&^b%hWRsS5@eAWSmT1NKZab|
zS*|F{cJo^hYRqqyldLaR2kV<j*4LEvrQr!eR`PTUYph^?VwJ&-Q4VgZ@>uOw=q8ZN
zEfCou3|SS97@JPZ4PmC<G|2!k1;v=(dR7N`d;?40*<SjW(ZhhEVP<w#58*BtQ0Fv{
z=k*i;LQz+lBU#D%>uKcR@n|W3U4K~e3lN6kct)l%ZZ`+Gm%kBoD^kz1G3UGJM_)so
ze~T;NJd-1iV)2DeVCP9`jh*8k@4PPH5>pUt&Jd{rR!9W+6y#aF+?JQERpIGqX!6Z*
zTTkK^<!E683nRC=QDI*1>P-X(NImf9Q1nau2vLoF(iPm(bsa!LB7RJ*$y$m70gO1N
zRjdeT73`Bvgv29~nTnj1dZV$u0;D_5Wf;%?-t(Q{sOG%x?s<l9oc=aSVth1*F^{^6
z=i9HK!|5b)1&g?362FOz0#e~}NJXF@FdZP24LS0+H!~Yg_cF`ybHJNn#?mCTx11h~
z-~&Lm10sJQU#7wg<Vki%p0v7<4H?LW&nB{Ae_uN%ZnsOCyoeTAwD_lJk%QcMq?d_R
z2HY)KWE*lwz8vOzIZ}3?q{+RW6PW}_vm?#G{G0_U8@VaL>22YoKZs%LXPG>Q1k3Om
zZts}+M3kfgB*t{n>@v!S8ZUxFP%6s9+W}V;B};Cts#_38nj|vr3a-g3^(B7eMJAV-
zyujo`NFp=by?!GiM_#Cdf%>%-AnA8dFDN51AUYg-48mjtsvX6M3lZ`lF4uIeGp1fb
zopgQaflJr70o2-FUr6rE?lu0r`S!D$*^MuS*0LpPTrPBa+2st_hPyu1bCm-Y^O>|X
z-^0uwG&T#p!g97}t~h#$Zj+ewc3Prc24&CelwE)3nt}Jaz@%)7NqP9p43!(eq_2lX
zV3OO1NnZqB6oD7N4!kG=FZPE!?FKQa_<B)rU<oat!F=A-ub@o{ZC-?w7z#@qS#OBq
zkm~FkMsMgM*d<il?A8!540l&BEx$D!_blAgEA7Ix((9$)1Jl4}DL>x@(-u)AfAxO`
z982&{*8n2s-N9cdk3>;i1Zloj5<v%!{wJ;L;OK%scm*jO?mg@DW06_kg;ehp&eB&6
zEvWPCqd*05hGvcK{WYj7v$4s4`fLMk;zsZ+GTU{>NBWc~TJ%1drDsDg6A33Q9!>*@
z{+0@a9(G7YEA%KP6pCE!Gg)9F<T3p2<neJ<B8@<~2&)>-1jq}K&s512Y_mqS0|Qw|
zNOLmvD@-Uk;zT@##lF#jlNiBSlnW7oc=n4<2EU<@5z~U@TEo^LX^lG6*oGm2gupsz
zfBSdwmr@x8L+W6B1P`K*LuI>Vy*;*RhoC0V1%&l2yI;Q+@nlU!*4jL+#KmsCqaQ=-
zc+`$sZFZ<Pwda>L4Fu1H1{St!?M7rS>>lQ5P`%+-@P>}BDxG?5rQtup(Z(`s*wkxy
zLyA+Bn`wt1!b5C8NQXy&S?qA^)CR+h{hdi?JximrIzXG9zUni0gji)S^$jH65KUq9
z1bv5t<c`j=bg;AUV`sByYCu%MfxC`e+Fgp+R3R?o0Uma6vRjZ&;6D)e++u)5W8Sjq
zwFi%PrXusS=r8Y>6WoKTaWf^#LD6kgYd*IhXb9?0;AOjemuJZt8X0H%JB2xX9ko*}
zopZ?R0<PdbB<lxz=0)Q*`ir5buVi(WTSnKj5J40>_SD=Z`4lE<6V<VWL*7nTm(1Dj
zZj|c5C3NI+i#5)%CluC(U@CNwPa>aA5Z98E$NmIgN1{XxzG?2m^w5@ngTfqBa}ykc
zY;KcQJ)zRqo0h;}Hr|A;xPBv~@t8Si(CWWwV@&g(Gu|{7Oj!Q*8?Yjzc19riK1eon
zz)~~c50Q;rwwGHld)Y5rd{%|6oYXH$efb`=Ah!q8v*(^Qu1`TEP%|x6KZ(^Yl=1gM
z{f6HvSIcBX<@Q1u-jU!W3_&>P2yXdn14ZSvI?O__GO-Ce?I5VnH+*?hb2VH=!A99%
zuQgT!ytmxG;cM^@aJ|JcIO05IH~nAy5H^tlUp{xbTs{XwbII4`wd#hvD{P~@-dc$(
z)`K$s(+IuGH^33ocUH?7OslNw)kW;g$(!U$>31&kyXcp$V;82{h=;aR4a!r=z*!m(
z-dAzy8gx8)(-LUEeB#U_k4%l9dOYrUK?>l~h_?X)c|4W=$s;EeF0a1%ja$vg6c03!
z`UCT{(=J@?B=gKw3#*HGCvaMt5W>In7?a*0m=g{vgpU;)j(k{Kg11;YU9M^$Ha(UI
ztfM>;uNJfQ*1*d+6McO-ofcw1{I(ya5Po|O)mg%gCA@5YBcA9!1Q1_#GH+~gn+zu<
zE<TWr-XVY$be(-hEPx?<zZZ340=IM>`N(YTqG`PX@W)BYd?!TjahQ)4B1UB!SMDZ&
z2YNyhm!440i7MdInCMdj2K&&xTOTSreTa=bJRABtlJr~r4oAHYm-sK_TVv4s!8|^T
zkw0%}r)S;qT{|ATv(TtwuQA~6^?W@H;$Bc9%Ot3s4*cJIF?jbc>YIG`8%VGepJrYZ
zf%J?1&KE@~OFtVgMKaJb8p=QqQojn#c%blFI^>+`Qggaseo=*qY&Zdt=ZZ@o?*93u
zozIWEk&PYOD5I;bML)73v?GW9claaxFe>4FaMQ=UBPyu+Lb6XUp5^hp=b%>LjkH#q
zl|Cv6ygcT)!f6k@VYyMl0<ln;TP5@DIFI)Wg(XkI5|tl6K}@1vWP^xfEJ~(QG^xFJ
zAnS`96dicRJtgQ;V|ErPea!`!BL}e)tKUXhI@3fcUKel$4<gxiTgFI5Sp**i6mmLk
zde%)!O{;qhd*IKOPXv#h3?3gpiOl0CgENvjbFv&ff!yiSr{U6x@(f!D+wDf+*`bfk
ztd_qX>d)ajh|O>%rU3#WGOLj}zl$g$w^Oa_H5mI%4oUP7OZ1RGgXXE8$Ig=`WB}B8
zXzgHY)m*XkL0HO0&pKor0eQv)W7#cJKSI?>!Q&|^lrtq1mni?=c}|>dRM|VqLjXx@
z6`lY)GS@w;+J@VLTP(ZTNR*wKsl4*S`G9?;N5g=63I)}4Ed6Kkl&UB!V;mIB3nTbV
zBy>$I!zV^p5S$X`+dua?YvYS=8gH6!S_^O~DhJ{{RuF7ha5id*>tDHFa-bMALfDMU
z=2n&tMF@B2`>F8E!HaD7un`X1aReR$Moi_gd62tiT%UvW<-%V3Vc1?SY_Ef~*HodG
z?YRrqezDm<we3lEUw3IsU7vde*EK*}PbICth1Q6^id*;cwDdn~Z53&qrLFU?=kK({
zWX6kAx2ZXS6BV1)^$NYkpsy4w@1XxWmi0Bzf6x4^wQPJA_XRLK>M?)1-`0Zrcy%7Z
zFbVBPCe}?H{ZX#g*8Zi{7W~-jer+{uCypgj5sWaBI_RwP?CU!`q5A-K3ZSC+4QvTa
zwfm$%T06>6FAe4{0o5~MJFH^!3Eu25Dln}CuyYVjgXowWK3EWa7~Ol1>++(H02$b9
zs$6YRqQ=f`?!_VK1AGC|H5&d{=*A1sk|36&eTlDg8l*J2J%G>Pefmu%oUf=<4aj^T
z`0?f_i)O2hP+w#@9VD6*61l<BDq=&{T5!n!1py#<hjB(hydm<#gk@LYFIhwwN&NZ9
zUey>F_Yq;{t=D4fj;uVHU3ruKG63;5%Fw{XeS%YZ421?f?UuP89Ld5z*FFsQ2wW!i
z2)NTxxRmKO(SM7tHT4&96PiI|@s^}C^UDClG(*+|difWimoKFBLdANUUK}tj2nm&J
zmVsU}>q{a0XrLGP!-QTkLNC*sStz|(&`WLuUI=m5yP%b6U_19(zMNvayOmc|k@I)Z
zN`mdzr$H;zDXp;eH_*Buw1U>XLP9Hr0a_`%UU(N;$s@Eao)69i|Gr!D(0?9{x8>rO
zn2)KWZ~b-z^j}2z7yGE;{=P-M8!hN9G$1ADTgd1iV)B=m{ADD+AKDl7Utv{ZpZ=>%
zQnc28jgMbpGKk9hM*tfy#(n=$mJ?@ouvq^XAOAX&zro~hGWl^P^zX%@^p4?+P(@H*
zO7}m=8Hka0<Sxitf=~tWA{FHQD&Cx8EipF5+IR2(AD7@B{IxOW{yZ>u>#mra0p`NT
z*N3^;b)8}^HkN3{3Fdm6xn8D+Ag4`tV}CCvn49l8*kBT#+{f>KEX6|dd$;HI3U}ae
zf`!}sp5|Y|@4t=K#gyL{@5=AD{d)sGUg7tDggE@QJ{&&Qx3Vw$deQ$wN958U1d+KT
zaaU|E-3^<)3_0(Wm^i*`q#oHlbZiP60Ucv>ya{ew31hZm#_YnqM(jQar(A;HKY`jQ
zesd=(#cv01`~MHW`}CRb8o$HR)(~aMW~o<#a{Nb#>HoYB(;wcw+*AJAA3CxR|Gyx+
z{sQoa+H?z>{|NA*HlhdlxMxF&`2*m+XIC$#e!M679(mc0LbC5Jv+>iMt&^zCjp4iT
zcHlqYJF3Zj;PLuvsGsWcc|6cAp$!tyI|>Q_XaAhBzPxQlK^K4ZRhv;Bo3V2))=wO|
z>#&ak`{UErS?Kz3<k9)(_A=tg%dQ{mWzqI1+;#?cexS|g5F3K6WbE#_<i6A~u8+Y*
zN0ST@%FgM<dN`m#JLr+P3TO_B1r0wkG-zWb2RFgHyC)>lJ+VP|S?u5GL$L>Fds6?U
zTQ=xJE;_G7nI7m34Ib%DdAm1>J-;Qci(A%Bb62lmb2;-+c=b87(C3kC>n_X8n^7Ld
zq~Br)%C2r=D{9-HgYE7|Q9E+hs*M$Wif^LZTEZb<xaV;m0A@Yn4D{Cl_6i(OMMfmC
zMF@RLD%63BTWonR6S~#oh;7!5wFyiAB#Zc-L=PUAA#fNY7)2ow&Ovn8VQdx_VA76S
z`tJZbFk3_^K`V0T*h>uW2Ud?;tv7*U;<<&vNM)BBgG3u3f<2YKhRuncGU+Ff5hJRc
z81~{wK@0|sdh-bI@FsSVV`B>H=J%pHUGy3*dZl0GQ{qIjYmGo$^q7CQSzU&6G~DQ$
zeyWYXvorm%J57!yRM!0Z;!@a;DXAd^t+9xP4E$J+vNx)YRTw=P_y?K@`-sd95B8g!
zyohJo)PEPX<4G15$iN4Fo1f>%C49@&e;>KjU$9ND7XS?CFF;h5aRA!^xUgTYuw&W7
z&NX(3usd{<d+MOv5Yrw<Scx_RYRk-5gQc?W;sxE|AZ&MLeGTvBX#n3b%b2Vsl<b5&
zjc22bn!i{_FcafXFSlR-A~I*5#{%NW2eZASm+nj_qp;eYi|0+K{K=5Shx1SH_H;;O
zL!@;X67_BjiNnE#zyWM?#lw=20OqsthOXQ>R0bBnOu|qzo7i38@RbWVSP!@HV>qWk
z<$D=2q4g(wZtw#P7*Jx&gc-!Xh)VcwEw0aBgCRwgHi@t(O?9)}ObbqdacMO_Frj;7
zb8%-z^*_L%x<ibJ4Nlb>_427sxtl3-qrh{fuTH&iyTA%#*Q#}KU&E_hmW>>2(qp$@
zmf{?02oKTRf7g4Mh*|sua#2R&1`$p34+;NAOy0xfr<hDK`NvFXId~4gt6$=?SWBtN
zgp&~56uo#p&($K+g%ij+CMO}_41#*a-AtMc`lnHriV>nD*RO^vXdnp<?D@b*SY|X4
zw$Tuud3Jq9b^sB>iJ(FYqG1$qLt@@yTip35+%DITL7h3d{zwnJ;I#))3IT&%CG7t6
zEO7kubO_02dR+VbHQ<SO^Iqg`Rv8|P>#Kfw+Z}MSEc*fa2KNKnngL|x8F&hKOi6Zh
zwkwu#7)8TT@3-Lb%jiFW{zo?AHTCC_^NQf7Z~-o0{~Cei6*wqxq^aHtDiws%{3)rs
z;%`LopZYDHZSh+bwDqu&EMAGoT-iOzd*Y5(_*vFJ!pY`J8cx>C!i6w~tAy=iYz^a|
z{uz{9nyZoN^N$Npf)Qf50#1>H*kYQnV<~5a1Y$L9MpwcY;M@W!z_Q9Hc-S`GFbhcy
zj{;5)<&o#V9K&zlau9XlE+g6k<OhEmPyF@rYUr)pW^w(gUQyuy2^QuN>WPeE;9(Mw
z*3jl~ue6-!slc~D94IR=7zxYSf^4_-SMkmwkU&lhGTgT{+;g`|2sayA+>Y+~7U~SE
zd(iLwZ^2cIr#)LE>VP`9DC23}hZcL)A-)y4QFS=U?Ndi2m)#nRqbrvA4pSYKw`8~W
z$0J`ppzg)F#ge*D!i{sw2VuY7ugdBHdCqPfVr$`y4<<pwnYop7i;K70m}tmwaC30+
zJ<Te;e7;&lR6+xBoe5w8o4+w2Gmnkn6}^F0u>^ATzheGh^EKJHc~pq+LA~oQ0*=35
zc<$2NcqS?=G!RRV{dfe>W!LHoN=cLtmfCAgzuF2S2lx_3c|OE-a@cvQqeOhaw{5(L
zDn$dQDx~pk>I#s6lv(J#i+z6Cj*P6O>f#sw9RGgpITYAn&{1~29fs{@(jjVF^7ozR
zJR4`Yy8A9*ixXr}emSO}S#&<iTte71#anhQz%fF^zAnlAb}sHp4r~m5ICWb|N5Z5d
zw(8WQ;{1Gj9Wi_0a>4jv&Pmv5w8KUn5!b7rw7ok3oV8<7l-<5B)ED7X^ooqBW(X+?
z^&jUaa(JgyEl_V+-=NtU({mm>79d?&G-lUBMl)3B{RXC3Dp%L*K{U*=z9ODgt86st
z%_y@W734jvfb9<{i#?>0UzhhGs#V(X-Pp!yx1v2s9|JIkWz1q^FV+?SMK$6;G6~?l
zaI#`(U?o`c@jdt?*;<rMx@j%6;1lH<(j$w*CZYay4%+TC@JcqctLu+5=oI)N$4UfC
zG$S7)8#9BW1U%RHgLu||kI8Q4&#61#UO!2G^b9Ic`vALlm^uhkVZh2ku_>@Gkaf&a
zXBaT5e-h+^8pCjkGSpn}17E6%27}k!gNyP_IM#l=UHB6```}HxIToU6dO3voW7mKj
z3jD?0%zxfopGW*(9^3!WZDM}42#ew-Y>m&GuU&jKvsu_IZkEW4!W<&65PJpgwg>s*
z_PJ$nw-LmYx!_}FZzMj&+}oo@!QJ)(4nC*$^@i0LIRd!dUUg5x?M8c}Tkv!;2pYZF
zw>j1u#raDcoQm9N6o<6LPj&rA!0jCUD|pvXuSDLZ4kg^Kur&;Bw<qCt8AOdO_lA_a
zVk37H^+zyPQynJH0&nE=k?m(Z?FGL(ik|IJ_bxiZ?Fie08utO_q_BO$(S~rwdLQ5q
zz6IXj8tWCd#?*V_R$KR|V@YnmI=&6-4Ka893WVuH>S08zfq!!35WJ8A@C+a#2IJ(Y
z%$-rmdrwG7ZtH;XUT{_P2`9LP_vXk`VePrght;F_w^yAMUh6I&0q;G<Q{D2M+d7)^
z-p69ztA7@_qW=REdT4MeBh%9DwE#CzaNuQUm#Q6{O3s~&BO$!(Tzdspl>V1edxzud
zpW{33%8d2DVNrx+fCUS&=x?y>-!b`lCjXwvQ6x}`zlj^;e!xg#8u|soO@ZrQL|%W3
z089Tz=6@;9|0m}EGm@0+CLc||jvc$FSZNN)ZIX?=d_22t-I0xCvAw&9LvI;j?w&(_
z<P{cmUB!$o$yXblrRrVzwZ4rPqaF5+QWFcrC??h{?tL5bi0`4z22?lzL^!Y=j$n1I
z3#Z~mG2>9XWQ%&4T^?j7vLpWmvXb~4-J=P6IdeBd+jA2E6C`u=6-XxiZNB!uF!^O9
znCAb={I4+iRRS#i9p-<H`Fn*M>fa#XH<|o5Bv8m=`4GsKt;Ya@SXSu&&f;8keMy+&
z|6mJhe)@khd56hwGx=Xk{x_4~Ve-33ydnhvOl*X~1R9Eip+O<Ayvfn4L5HzvAZ0^?
zlOUXd34Q^>&>%>}xg#0B$Fnv9jm<|xV^_uSG9utLywiq8FsTs_Wr(?9mSe49yYJyM
z_B##zCG;Y)8fQF*4fP0NWb=A2bE8b&mCL)^%kXio%`{pWwy=47)H-RWY#z~ybRMOQ
z9(!Kce+O)gr%fOf;<!dIf5L5va5P;<nExSkAZ7kER$N@d{28zSCczV7zYp^l=9)v~
z7qJlA#R;>>#!O`^$CC=84B`Yr&xzGUsD}B0C$Md>?EEG+3pR^XL$^xd(AF^3+g|Y&
zlwrOBs^%R~)EYq`a)E)u2m-e^Sz{0~gqC}IrQQ%!Wu9CszJ)~wY*gXY_0~QmY*n72
zHV-^QZ60{WIqIF~t+C{|zq@r$e8L9ea6jhEgnCWYIJbNtJh*&l>o67Sts}5!&8?&S
zDUMs#Eek?S6lhUp5A?CqaVudt&hu#o$KrBO8>oZC?9J3;Sf5Km;Q9x!W{XeA-_L>;
zafE(b%k?tCV>H>?obWqgZl2Td7r1eqQY};(ct)W$cC(+@Jl#<XVg|osp2C+4q77E4
z1>T)>^>5+z`Y<}=L0_tR4mq9tctB+x+K<qr=ZflvM=!HZekk-AS*?zj1$812MyS?v
zLF1yCdL>wMjmo?quF+h}@=P)$Ns*MxqhJW-*dgeceuZiuWnWU)8@83@#|U%};PK(e
z3LeLG#<OlfMv1oOWko@Q&K}}C@(wv8{$gD>`LY%gj3)FuGXL*Nu0@jTY&?xWN{a1M
z%s_=BS<o)1au_tp$m2LtW-LDO2YO}f@h~EdxiJD&&C(B$Hu3HB&qnTfToOCu8~hm?
z{Dh6^*^J9X=!S_ik)S{Ph)G7}xIGXJpNEaJD2}zcYOvD%G=kyq0}eb|hgg6r_wUQ%
z&XRyR2FmHx)`{_Q{F@<&s6;g7Jfi35YLEy5#Nvf97!|kA-_55Vk=MX?R*!=)5ntKh
zaXZLFd=61yaj2ePZH-<C{?S-wUbxPa$iR1CRKQD+nvA5aAYg@_2%IBj&<dPnJiJ)J
zCVG4%L}BA_Jk#~d7ZAT0>Q#DJr6XnhJ;PN+M}gY#^gYAePMOGszCt-{kv^Y94!#C>
z5mX!t$@huqq#cDR0Y=Lf@Z1zay4>w=L>c~~MniBA=R&sU8>)|FBO5b^Ul*$4z*$QW
z4CW+U5{(6Y91D)$dchHTeT>AEjT`Th){$F*9fF9D$~TE*CCr$Sp*JE6RLYv&1A}XR
zdb@V2NZ{2w6MmQzjvv2R#&lz^m^nlTk$uHM%iY7UBl+_(2%4;hM2~=hl;Vd0Zx~_}
znuJzyk$JF=gMfTxH2Tgwc*FBJth-YY`oRg*+ch%%FkrvRK?!bvBD$ly27~W2LP3<x
zEN6lzeJhoyP~q2B;ZCp6^K%{HzDY^0Qo+yLR4V!revn`2`>N<c+=tf>qTn0+G*vp+
zaje3S!0@rNfPh!dv)UbbQ=%q$rqF_8fY)T~>N~*d0TSrK@I7wmqx>|!fOTbDOuZpa
zE7>s6q-@!F;ECauss{0fRj!R+uc6DI^AT?)uzRQ^0sip`qA8Z;m{4}_oJsOVx!n~9
z_nb~*d*VGb%6aG`c(I2wu?RYfm2>fmYx)Ge9KQ+!UlY5$z46cMTmV{Mka)B}en3Zl
zWJS)D%Q0m+!z$-0<5MBm*qaKIDw8E99VTl`)|tG{<daO^X2MuI31E}JoG4%GtU_M~
zJJDZd$xkr(8WZkdh^JlCH!TiMalMHxtEtln^VAO^hjY8}83eBI-IhLb&&H_pEEgxZ
zfh6|k!LvuNo-L_(`lw{ZPc9FG!1E_5tZqPU{5$Mc+_A!5x9H|^Ic~Oapm4I_xEcJ*
hxyRkC>$*or4~-rjz0ch%Posssg)@bh-N#(>{{Zpj5<dU{

diff --git a/python/common/commons.py b/python/common/commons.py
index 2845156..3a727ef 100644
--- a/python/common/commons.py
+++ b/python/common/commons.py
@@ -97,6 +97,7 @@ def setEnv(args):
     os.environ["DATA_PATH"] = cfg['fixminer']['datapath']
     os.environ["PROJECT_TYPE"] = cfg['fixminer']['projectType']
     os.environ["PROJECT_LIST"] = cfg['fixminer']['projectList']
+    os.environ["REDIS_PORT"] = str(cfg['fixminer']['portDumps'])
 
     # import yaml
     #
@@ -177,7 +178,22 @@ def getRun():
 
 
 
-
+# def shellCallTemplate(cmd,enc='utf-8'):
+#     process = subprocess.Popen(cmd,
+#                                stdout=subprocess.PIPE,stderr=PIPE, shell=True,encoding=enc,
+#                                universal_newlines=True)
+#
+#     while True:
+#         output = process.stdout.readline()
+#         print(output.strip())
+#         # Do something else
+#         return_code = process.poll()
+#         if return_code is not None:
+#             print('RETURN CODE', return_code)
+#             # Process has finished, read rest of the output
+#             for output in process.stdout.readlines():
+#                 print(output.strip())
+#             break
 
 def shellCallTemplate(cmd,enc='utf-8'):
     try:
@@ -510,9 +526,9 @@ def get_class_weights(y):
     return  {cls: round(float(majority)/float(count), 2) for cls, count in counter.items()}
 
 
-def stopDB(dbDir,portInner,dbName):
-    cmd = "bash " + dbDir + "/" + "stopServer.sh " + " " + portInner;
-
+def stopDB(dbDir,portInner):
+    # cmd = "bash " + dbDir + "/" + "stopServer.sh " + " " + portInner;
+    cmd = "redis-cli -p " + portInner + " shutdown save"
     o, e = shellGitCheckout(cmd)
     logging.info(o)
 
diff --git a/python/data/redis/stopServer.sh b/python/data/redis/stopServer.sh
index dd3bcfb..433c9ea 100644
--- a/python/data/redis/stopServer.sh
+++ b/python/data/redis/stopServer.sh
@@ -1,3 +1,4 @@
 #!/bin/bash
-source activate redisEnv
+
+#source activate redisEnv
 redis-cli -p $1 shutdown save
diff --git a/python/javaDS.py b/python/javaDS.py
index 89175d9..444e93b 100644
--- a/python/javaDS.py
+++ b/python/javaDS.py
@@ -1,64 +1,132 @@
-def createDS(subject):
-    # # if job == 'clone':
-    from commitCollector import *
+from common.commons import *
+from commitCollector import *
+DATA_PATH = os.environ["DATA_PATH"]
+COMMIT_DFS = os.environ["COMMIT_DFS"]
+# DATASET_PATH = '/Users/anilkoyuncu/projects/datasets'
+REPO_PATH = os.environ["REPO_PATH"]
+DATASET_PATH = os.environ["REPO_PATH"]
+DATASET = os.environ["dataset"]
+ROOT = os.environ["ROOT_DIR"]
+PROJECT_LIST = os.environ["PROJECT_LIST"]
 
-    caseClone(subject)
-    # # elif job == 'collect':
-    from commitCollector import *
-
-    caseCollect(subject)
-    # # elif job == 'fix':
-    from filterBugFixingCommits import caseFix
-
-    caseFix(subject)
-    #
-    # # elif job =='brDownload':
-    from bugReportDownloader import caseBRDownload
-
-    caseBRDownload(subject)
-    # # elif job =='brParser':
-    from bugReportParser import step1
-
-    step1(subject)
-
-    # elif job =='dataset':
-
-    if not isfile(join(DATA_PATH, 'singleBR.pickle')):
-
-        brs = load_zipped_pickle(join(DATA_PATH, subject + "bugReportsComplete.pickle"))
-
-        subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+from otherDatasets import markBugFixingPatches
 
 
-        def getCommit(x):
-            bid, project = x
+def createDS():
 
-            subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
-            repo = subjects.query("Subject == '{0}'".format(project)).Repo.tolist()[0]
-            commits = load_zipped_pickle(join(DATA_PATH, COMMIT_DFS, repo + '.pickle'))
-            correspondingCommit = commits.query("fix =='{0}'".format(bid)).commit.tolist()
-            if len(correspondingCommit) == 1:
-                return [bid, correspondingCommit[0], project]
-            else:
-                return None
-                print('error')
+    pjList = PROJECT_LIST.split(',')
+    if not os.path.exists(DATASET_PATH):
+        os.mkdir(DATASET_PATH)
+
+    subjects = pd.read_csv(join(ROOT,'data', 'dataset.csv'))
 
 
-        wl = brs[['bid', 'project']].values.tolist()
-        dataL = parallelRunMerge(getCommit, wl)
-
-        commits = pd.DataFrame(
-            columns=['bid', 'commit', 'project'],
-            data=list(filter(None.__ne__, dataL)))
-
-        save_zipped_pickle(commits, join(DATA_PATH, 'singleBR.pickle'))
+    if pjList == ['ALL']:
+        tuples = subjects[['Repo','GitRepo','Branch']].values.tolist()
     else:
-        commits = load_zipped_pickle(join(DATA_PATH, 'singleBR.pickle'))
-        subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
-    logging.info('done matching commits')
-    commits['repo'] = commits.project.apply(lambda x: subjects.query("Subject == '{0}'".format(x)).Repo.tolist()[0])
+        # repos = subjects.query("Subject == '{0}'".format(subject)).Repo.tolist()
+        tuples = subjects[subjects.Repo.isin(pjList)][['Repo', 'GitRepo','Branch']].values.tolist()
 
-    workList = commits[['commit', 'repo']].values.tolist()
-    from dataset import prepareFiles
+    for t in tuples:
+        repo,src,branch = t
+        logging.info(repo)
+        if isfile(join(COMMIT_DFS,repo+'Fix.pickle')):
+            commits = load_zipped_pickle(join(COMMIT_DFS,repo+'Fix.pickle'))
+        else:
+            cmd = 'git config --global http.postBuffer 157286400'
+            shellCallTemplate(cmd)
+            cmd = 'git -C ' + DATASET_PATH + ' clone ' + src
+            shellCallTemplate(cmd)
+            logging.info(repo)
+            getCommitFromRepo(join(REPO_PATH, repo), join(COMMIT_DFS, repo),branch)
+            rDF = makeDF(join(COMMIT_DFS,repo + '.commits'))
+            save_zipped_pickle(rDF, join(COMMIT_DFS, repo + ".pickle"))
+            # return rDF
+            commits = rDF
+            commits = markBugFixingPatches(commits,repo)
+        commits = commits[commits.files.apply(lambda x: np.any([i == 'M' for i in x.values()]))]
+        # keep only commits that are changing c files (.c)
+        commits = commits[commits.files.apply(lambda x: np.all([i.endswith('.java') for i in x.keys()]))]
+        #not a revert commit
+        # commits = commits[~commits.log.apply(lambda x: x.startswith('Revert'))]
+        # commits = commits[commits.files.apply(lambda x: len(x) == 1)]
+        # commits['cocci'] = commits.log.apply(lambda x: True if re.search('cocci|coccinelle', x) else False)
+        # coccis = commits[commits.cocci].commit.values.tolist()
+        fixes = commits[commits.fixes.str.len()!=0].commit.values.tolist()
+        # links = commits[commits.links.str.len()!=0].commit.values.tolist()
 
-    parallelRun(prepareFiles, workList)
\ No newline at end of file
+        # bugs = set(fixes).union(links).union(coccis)
+        # bugs = set(fixes)#.union(coccis)
+        commits = commits[commits.commit.isin(fixes)]
+        print(len(commits))
+        # for s in a.commit.values.tolist():
+        from otherDatasets import prepareFiles
+        parallelRun(prepareFiles,commits[['commit','files']].values.tolist(),repo)
+
+    # # if job == 'clone':
+    # for repo,src in subjects[['Repo','GitRepo']].values.tolist():
+    #     if(pjList != ['ALL']):
+    #         if repo in pjList:
+    #             print(repo)
+    #             cmd = 'git -C ' + DATASET_PATH + ' clone ' + src
+    #             shellCallTemplate(cmd)
+    #             logging.info(repo)
+
+    # caseClone(subject)
+
+    # caseCollect(subject)
+    # # elif job == 'fix':
+    # from filterBugFixingCommits import caseFix
+    #
+    # caseFix(subject)
+    # #
+    # # # elif job =='brDownload':
+    # from bugReportDownloader import caseBRDownload
+    #
+    # caseBRDownload(subject)
+    # # # elif job =='brParser':
+    # from bugReportParser import step1
+    #
+    # step1(subject)
+    #
+    # # elif job =='dataset':
+    #
+    # if not isfile(join(DATA_PATH, 'singleBR.pickle')):
+    #
+    #     brs = load_zipped_pickle(join(DATA_PATH, subject + "bugReportsComplete.pickle"))
+    #
+    #     subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+    #
+    #
+    #     def getCommit(x):
+    #         bid, project = x
+    #
+    #         subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+    #         repo = subjects.query("Subject == '{0}'".format(project)).Repo.tolist()[0]
+    #         commits = load_zipped_pickle(join(DATA_PATH, COMMIT_DFS, repo + '.pickle'))
+    #         correspondingCommit = commits.query("fix =='{0}'".format(bid)).commit.tolist()
+    #         if len(correspondingCommit) == 1:
+    #             return [bid, correspondingCommit[0], project]
+    #         else:
+    #             return None
+    #             print('error')
+    #
+    #
+    #     wl = brs[['bid', 'project']].values.tolist()
+    #     dataL = parallelRunMerge(getCommit, wl)
+    #
+    #     commits = pd.DataFrame(
+    #         columns=['bid', 'commit', 'project'],
+    #         data=list(filter(None.__ne__, dataL)))
+    #
+    #     save_zipped_pickle(commits, join(DATA_PATH, 'singleBR.pickle'))
+    # else:
+    #     commits = load_zipped_pickle(join(DATA_PATH, 'singleBR.pickle'))
+    #     subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+    # logging.info('done matching commits')
+    # commits['repo'] = commits.project.apply(lambda x: subjects.query("Subject == '{0}'".format(x)).Repo.tolist()[0])
+    #
+    # workList = commits[['commit', 'repo']].values.tolist()
+    # from dataset import prepareFiles
+    #
+    # parallelRun(prepareFiles, workList)
\ No newline at end of file
diff --git a/python/main.py b/python/main.py
index 881e57a..20d55e7 100644
--- a/python/main.py
+++ b/python/main.py
@@ -15,6 +15,7 @@ if __name__ == '__main__':
         setEnv(args)
 
         job = args.job
+        job = "cluster"
         ROOT_DIR = os.environ["ROOT_DIR"]
         REPO_PATH = os.environ["REPO_PATH"]
         CODE_PATH = os.environ["CODE_PATH"]
@@ -25,11 +26,12 @@ if __name__ == '__main__':
         FEATURE_DIR = os.environ["FEATURE_DIR"]
         DATASET_DIR = os.environ["DATASET_DIR"]
         PROJECT_TYPE = os.environ["PROJECT_TYPE"]
+        REDIS_PORT = os.environ["REDIS_PORT"]
         jdk8 = os.environ["JDK8"]
         pd.options.mode.chained_assignment = None
 
 
-        subject = 'ALL'
+        # subject = 'ALL'
         rootType = 'if'
 
         print(job)
@@ -37,7 +39,7 @@ if __name__ == '__main__':
 
         if job == 'dataset4j':
             from javaDS import createDS
-            createDS(subject)
+            createDS()
         # elif job == 'linuxDS':
         #     from linuxDataset import collectBugFixPatches
         #     collectBugFixPatches()
@@ -45,6 +47,8 @@ if __name__ == '__main__':
             from otherDatasets import core
             core()
         elif job =='richEditScript':
+            dbDir = join(DATA_PATH, 'redis')
+            stopDB(dbDir, REDIS_PORT)
             cmd = "JAVA_HOME='" + jdk8 + "' java  -jar " + join(Path(ROOT_DIR).parent, 'target','FixPatternMiner-1.0.0-jar-with-dependencies.jar') + " " + args.prop + " RICHEDITSCRIPT "
             output = shellCallTemplate(cmd)
             logging.info(output)
@@ -64,7 +68,7 @@ if __name__ == '__main__':
             from pairs import importShape
             importShape()
 
-        elif job =='compareShapes':
+        elif job =='compare':
              # cmd = "mvn exec:java -f '/data/fixminer_source/' -Dexec.mainClass='edu.lu.uni.serval.richedit.akka.compare.CompareTrees' -Dexec.args='"+ " shape " + join(DATA_PATH,"redis") +" ALLdumps-gumInput.rdb " + "clusterl0-gumInputALL.rdb /data/richedit-core/python/data/richEditScript'"
             cmd = "JAVA_HOME='" + jdk8 + "' java  -jar " + join(Path(ROOT_DIR).parent, 'target','FixPatternMiner-1.0.0-jar-with-dependencies.jar') + " " + args.prop + " COMPARE "
             output = shellCallTemplate(cmd)
@@ -81,27 +85,27 @@ if __name__ == '__main__':
             startDB(dbDir, "6399", PROJECT_TYPE)
             cluster(join(DATA_PATH,'shapes'),join(DATA_PATH, 'pairs'),'shapes',rootType)
 
-        elif job =='actionSI':
-            from pairs import actionPairs
-            actionPairs(rootType)
-
-        # elif job =='importActionPairs':
-            from pairs import importAction
-            importAction(rootType)
-
-        elif job =='compareActions':
-            # cmd = "JAVA_HOME='"+jdk8+"' java -Xmx8096m -Djava.util.concurrent.ForkJoinPool.common.parallelism=64 -jar "+  join(DATA_PATH,'CompareTrees.jar') + " action " + join(DATA_PATH,"redis") +" ALLdumps-gumInput.rdb " + "clusterl1-gumInputALL.rdb"
-
-            cmd = "JAVA_HOME='" + jdk8 + "' java -jar " + join(DATA_PATH, 'FixPatternMiner-1.0.1.jar') + " " + join(DATA_PATH, 'app.properties') + " COMPARE " + 'L2'
-            output = shellCallTemplate(cmd)
-            logging.info(output)
-
-        elif job == 'clusterActions':
-            from abstractPatch import cluster
-
-            dbDir = join(DATA_PATH, 'redis')
-            startDB(dbDir, "6399", PROJECT_TYPE)
-            cluster( join(DATA_PATH, 'actions'),join(DATA_PATH, 'pairsAction'),'actions',rootType)
+        # elif job =='actionSI':
+        #     from pairs import actionPairs
+        #     actionPairs(rootType)
+        #
+        # # elif job =='importActionPairs':
+        #     from pairs import importAction
+        #     importAction(rootType)
+        #
+        # elif job =='compareActions':
+        #     # cmd = "JAVA_HOME='"+jdk8+"' java -Xmx8096m -Djava.util.concurrent.ForkJoinPool.common.parallelism=64 -jar "+  join(DATA_PATH,'CompareTrees.jar') + " action " + join(DATA_PATH,"redis") +" ALLdumps-gumInput.rdb " + "clusterl1-gumInputALL.rdb"
+        #
+        #     cmd = "JAVA_HOME='" + jdk8 + "' java -jar " + join(DATA_PATH, 'FixPatternMiner-1.0.1.jar') + " " + join(DATA_PATH, 'app.properties') + " COMPARE " + 'L2'
+        #     output = shellCallTemplate(cmd)
+        #     logging.info(output)
+        #
+        # elif job == 'clusterActions':
+        #     from abstractPatch import cluster
+        #
+        #     dbDir = join(DATA_PATH, 'redis')
+        #     startDB(dbDir, "6399", PROJECT_TYPE)
+        #     cluster( join(DATA_PATH, 'actions'),join(DATA_PATH, 'pairsAction'),'actions',rootType)
 
         elif job == 'tokenSI':
             from pairs import tokenPairs
diff --git a/python/otherDatasets.py b/python/otherDatasets.py
index b9998db..d5b7589 100644
--- a/python/otherDatasets.py
+++ b/python/otherDatasets.py
@@ -120,8 +120,9 @@ def prepareFiles(t,dsName):
         nonTest = []
         for k,v in files.items():
             if v == 'M':
-                if k.endswith('.c') or k.endswith(u'.h'):
-                   nonTest.append(k)
+                nonTest.append(k)
+                # if k.endswith('.c') or k.endswith(u'.h'):
+                #    nonTest.append(k)
         # nonTest = [f for f in files.keys() if f.endswith('.c') or f.endswith(u'.h')]
 
         cmd = 'git -C ' + join(DATASET_PATH,dsName) + ' rev-parse --short=6 ' + shaOld
@@ -276,6 +277,8 @@ def core():
         if(pjList != ['ALL']):
             if repo in pjList:
                  print(repo)
+                 cmd = 'git config --global http.postBuffer 157286400'
+                 shellCallTemplate(cmd)
                  cmd = 'git -C ' + DATASET_PATH + ' clone ' + src
                  shellCallTemplate(cmd)
                  logging.info(repo)