diff --git a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java index 35fd469..6438486 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java @@ -34,6 +34,7 @@ public class Launcher { String eDiffTimeout = appProps.getProperty("eDiffTimeout","900"); String parallelism = appProps.getProperty("parallelism","FORKJOIN"); String hostname = appProps.getProperty("hostname","localhost"); + String hunkLimit = appProps.getProperty("hunkLimit","10"); String input = appProps.getProperty("inputPath","FORKJOIN"); String redisPath = appProps.getProperty("redisPath","FORKJOIN"); @@ -57,12 +58,12 @@ public class Launcher { // // log.info(parameters); - mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter, srcMLPath,hostname); + mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter, srcMLPath,hostname,hunkLimit); } - public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter,String srcMLPath,String hostname){ + public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter,String srcMLPath,String hostname,String hunkLimit){ String dbDir; @@ -78,7 +79,7 @@ public class Launcher { try { switch (jobType) { case "RICHEDITSCRIPT": - EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter); + EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter,hunkLimit); break; case "LOAD": diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java index fc1787a..bf77187 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java @@ -26,12 +26,17 @@ public class EDiffHunkParser extends EDiffParser { private static Logger logger = LoggerFactory.getLogger(EDiffHunkParser.class); @Override - public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath,String rootType) { + public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath,String hunkLimit) { List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile,srcMLPath); - if (actionSets.size() != 0) { + if (actionSets != null && actionSets.size() != 0) { boolean processActionSet = true; + if (actionSets.size() > Integer.valueOf(hunkLimit)){ + processActionSet = false; + logger.debug("Skipping {} set size {}",diffentryFile.getName(),hunkLimit); + } + int hunkSet = 0; if(processActionSet){ for (HierarchicalActionSet actionSet : actionSets) { @@ -79,7 +84,7 @@ public class EDiffHunkParser extends EDiffParser { } catch (Exception e) { logger.error("error",e); - e.printStackTrace(); +// e.printStackTrace(); } hunkSet++; } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java index b1ee053..54360a7 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java @@ -65,7 +65,7 @@ public class EDiffWorker extends UntypedActor { future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); counter ++; - if (counter % 10 == 0) { + if (counter % 1000 == 0) { log.info("Worker #" + id +" finalized parsing " + counter + " files... remaing "+ (files.size() - counter)); } } catch (TimeoutException e) { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouperForC.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouperForC.java index 8912196..68362a0 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouperForC.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouperForC.java @@ -11,6 +11,7 @@ import edu.lu.uni.serval.utils.ListSorter; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; /** * Regroup GumTree results to a hierarchical construction. @@ -147,19 +148,19 @@ public class HierarchicalRegrouperForC { } private boolean isPossibileSubAction(Action parent, Action child) { - if ((parent instanceof Update && !(child instanceof Addition)) - || (parent instanceof Delete && child instanceof Delete) - || (parent instanceof Insert && (child instanceof Insert))) { - int startPosition = child.getPosition(); - int length = child.getLength(); - int startPosition2 = parent.getPosition(); - int length2 = parent.getLength(); - - if (!(startPosition2 <= startPosition && startPosition + length <= startPosition2 + length2)) { - // when act is not the sub-set of action. - return false; - } - } +// if ((parent instanceof Update && !(child instanceof Addition)) +// || (parent instanceof Delete && child instanceof Delete) +// || (parent instanceof Insert && (child instanceof Insert))) { +// int startPosition = child.getPosition(); +// int length = child.getLength(); +// int startPosition2 = parent.getPosition(); +// int length2 = parent.getLength(); +// +// if (!(startPosition2 <= startPosition && startPosition + length <= startPosition2 + length2)) { +// // when act is not the sub-set of action. +// return false; +// } +// } return true; } @@ -282,10 +283,18 @@ public class HierarchicalRegrouperForC { int nodeType = tree.getType(); - +// List collect = tree.getChildren().stream().filter(m -> m.getType() == 6).collect(Collectors.toList()); +// if (collect.size() > 0){ +// return true; +// } if (NodeMap_new.StatementMap.containsKey(nodeType)){ return true; } +// else{ +// if((nodeType ==6) && tree.getParent().getType() == 1){ +// return true; +// } +// } // if (nodeType == 11 || nodeType == 16 || nodeType == 18 || nodeType == 21 // || nodeType == 22 || nodeType == 23 || nodeType == 24 || nodeType == 84 diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java index 2684122..4f9f5ae 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java @@ -5,6 +5,7 @@ import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; import edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser; import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser; +import edu.lu.uni.serval.utils.ClusterToPattern; import edu.lu.uni.serval.utils.EDiffHelper; import edu.lu.uni.serval.utils.PoolBuilder; import org.apache.commons.io.FileUtils; @@ -16,6 +17,8 @@ import redis.clients.jedis.JedisPool; import java.io.File; import java.io.IOException; import java.lang.instrument.Instrumentation; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.*; @@ -28,35 +31,75 @@ public class HunkParserTest { public void testSimple() throws IOException { // String input = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/revFiles/7f52f3_3845d29_drivers#pci#host#pcie-altera.c"; - String root = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/linux/"; +// String root = "/Users/anil.koyuncu/projects/fixminer/gumInputLinux/linux/"; + String root = "/Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/gumInputLinux/"; + String filename =""; +// filename ="freebsd_ceca9b8_b864ac4_sys#kern#sched_ule.c"; //too long +// filename ="openbsd_e592ec_39c81a_sys#arch#i386#pci#pci_machdep.c"; //not parseable 56 "parameter_list" "" () ( (57 "parameter" "" () ( (22 "decl" "" () ()))) +// filename ="openbsd_cbb6d2_4cab495_sys#lib#libsa#printf.c"; +// filename ="freebsd_bb1ce4_10d4c2_sbin#gvinum#gvinum.c"; //too long +// filename ="freebsd_253913_35ea52_sys#netinet#ip_carp.c"; //ok +// filename ="FFmpeg_4c9d1c_3da860_libavutil#file_open.c"; //not sure ifdef +// filename ="gstreamer_0af74c_e8bae0_libs#gst#net#gstptpclock.c"; //not sure ifder +// filename ="freebsd_abdbcc6_030063_sys#netinet#ip_carp.c"; //ok +// filename ="linux_80d348_5b394b_fs#overlayfs#inode.c"; //ok +// filename ="openbsd_43b185_e7495b_usr.bin#cvs#rcs.c"; //okish +// filename ="openbsd_e592ec_39c81a_sys#arch#i386#pci#pci_machdep.c"; //(56 "parameter_list" "" () ( (57 "parameter" "" () ( (22 "decl" "" () ()))) +// filename ="openbsd_cbb6d2_4cab495_sys#lib#libsa#printf.c"; //not parseable 56 "parameter_list" "" () ( (57 "parameter" "" () ( (22 "decl" "" () ()))) +// filename ="FFmpeg_9219ec_647696_libavfilter#trim.c"; //partial +// filename ="vlc_92b7fd_f745f6_modules#control#dbus#dbus.c"; //okish +// filename ="vlc_eeb662_966879_modules#video_chroma#copy.c"; //ok +// filename ="omp_19fae3_1e4dcd_src#mca#mpool#sm#mpool_sm_mmap.c"; // cannot find +// filename ="FFmpeg_a8343bf_2b2039_libavformat#riff.c"; // ok +// filename ="freebsd_32766e4_200ff4_sbin#routed#parms.c"; // ok + filename ="openbsd_150ddd_cf0e20_usr.sbin#user#user.c"; //notok + filename ="openbsd_6fac1e_c3b383_usr.bin#tmux#window-copy.c"; //notok + filename ="freebsd_0cb6f2_b4c742_sys#dev#ipw#if_ipw.c"; //notok + filename ="php-src_7defd5_da06f7_ext#mbstring#mbstring.c"; //notok (19 "expr_stmt" "" () ())))) + filename ="libtiff_177169_71715f_tools#tiff2ps.c"; //notok (19 "expr_stmt" "" () ())))) + filename ="linux_955c1dd_0aaee4_drivers#gpu#drm#i915#gvt#handlers.c"; //notok (19 "expr_stmt" "" () ())))) -// String filename = "8dd302_c4ef85_net#core#dev_ioctl.c"; //ok -// String filename = "1e793f6_77f18a_drivers#scsi#megaraid#megaraid_sas_base.c"; //OK -// String filename = "6a28fd_93ad867_drivers#tty#goldfish.c"; //m,issing -// String filename = "b90f7c_ff51ff_kernel#sched#fair.c"; //wrong and wired -// String filename = "ed8f68_b1c8047_fs#ext3#dir.c";//ok +// filename ="FFmpeg_0726b2_66d2ff_libav#jpeg.c"; + String pj = filename.split("_")[0]; + filename = filename.replace(pj+"_",""); + root = root + pj + "/"; +// root = root + "codeflaws/"; -// String filename = "bc3d12_9a26653_drivers#scsi#libfc#fc_disc.c"; //okish - String filename = "118154_0c5f81_arch#x86#kvm#svm.c"; -// String filename = "bcbd94f_43e43c9_drivers#md#dm-crypt.c"; //emin degilim -// String filename = "f1727b4_6c1e7e_arch#x86#kvm#vmx#nested.c"; //komplex not sure -// String filename = "5924f17_5925a05_net#ipv4#tcp.c"; -// String filename = "bd0b9ac_b237721_drivers#irqchip#irq-dw-apb-ictl.c"; //missing -// String filename = "052831_3985e8_include#net#ip_tunnels.h"; -// String filename = "e76019_28647b_drivers#gpu#drm#i915#i915_drv.h"; -// String filename = "4cbe4d_b124f4_include#linux#mlx4#device.h"; //enum case stops at block -// String filename = "7bf7eac_c01daf_include#linux#dax.h"; + + +// filename = "474-A-15925943-15925951.c"; //mot ok +// filename = "6-C-11536006-11536039.c"; //okish +// filename = "500-A-18298071-18298124.c"; //ok +// filename = "106-B-4027414-4027447.c"; //ok +// filename = "572-B-12669194-12669278.c"; //ok +// filename = "514-A-16254510-16254521.c"; //ok +// filename = "405-B-12287356-12287584.c"; //ok +// filename = "630-R-17825199-17825235.c"; //notok File revFile = new File(root + "revFiles/"+ filename); File prevFile =new File(root + "prevFiles/prev_"+filename); +// File diffFile =new File(); + String path = root + "DiffEntries/"+filename + ".txt"; + System.out.println(path); +// String data = ""; +// data = new String(Files.readAllBytes(Paths.get(root + "DiffEntries/"+filename + ".txt"))); + EDiffHunkParser parser = new EDiffHunkParser(); - String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML/src2srcml"; + String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml"; +// String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml"; parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath); // ITree t = new SrcmlCppTreeGenerator().generateFromFile(input).getRoot(); // Assert.assertEquals(148, t.getSize()); } + @Test + public void dumpFnction() throws Exception { + String pattern = "function/20/gstreamer_0af74c_e8bae0_libs#gst#net#gstptpclock.c.txt_0"; +// String pattern = "function/20/FFmpeg_4c9d1c_3da860_libavutil#file_open.c.txt_0"; + ClusterToPattern.main("6399","/Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis","ALLdumps-gumInput.rdb ",pattern); + } + @Test public void newCTest(){ String root = "/Users/anilkoyuncu/projects/gumInputLinux/linux/"; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java index e19232f..ff399d8 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java @@ -26,7 +26,7 @@ public class EnhancedASTDiff { private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class); - public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType) throws Exception { + public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType,String hunkLimit) throws Exception { String parameters = String.format("\nInput path %s",inputPath); @@ -62,6 +62,8 @@ public class EnhancedASTDiff { Stream stream = Arrays.stream(listOfFiles); List folders = stream .filter(x -> !x.getName().startsWith(".")) + .filter(x -> !x.getName().startsWith("cocci")) + .filter(x -> !x.getName().endsWith(".index")) .collect(Collectors.toList()); @@ -83,7 +85,7 @@ public class EnhancedASTDiff { ActorSystem system = null; ActorRef parsingActor = null; - final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath,rootType); + final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath,hunkLimit); try { log.info("Akka begins..."); log.info("{} files to process ...", allMessageFiles.size()); @@ -92,10 +94,11 @@ public class EnhancedASTDiff { parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor"); parsingActor.tell(msg, ActorRef.noSender()); } catch (Exception e) { - system.terminate(); + system.shutdown(); e.printStackTrace(); }finally { - system.terminate(); + system.awaitTermination(); +// system.shutdown(); } break; case "FORKJOIN": diff --git a/src/main/resource/app.properties b/src/main/resource/app.properties index cc66c6a..e3d6a1d 100755 --- a/src/main/resource/app.properties +++ b/src/main/resource/app.properties @@ -5,11 +5,13 @@ portDumps = 6399 parallelism = AKKA numOfWorkers = 30 hostname = localhost +hunkLimit = 10 #inputPath = /Users/anilkoyuncu/projects/gumInputLinux inputPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/gumInputLinux redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis -srcMLPath= /Users/anil.koyuncu/Downloads/srcML/src2srcml +#srcMLPath= /Users/anil.koyuncu/Downloads/srcML/src2srcml +srcMLPath= /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml actionType = ALL eDiffTimeout = 900