azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

limit added and removed position from groupping

Browse Source
This commit is contained in:
ANIL KOYUNCU
2020-01-25 20:36:01 +01:00
parent 6a9d0d3fec
commit 32b638f482
7 changed files with 106 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+4 -3
View File
@@ -34,6 +34,7 @@ public class Launcher {
String eDiffTimeout = appProps.getProperty("eDiffTimeout","900");
String parallelism = appProps.getProperty("parallelism","FORKJOIN");
String hostname = appProps.getProperty("hostname","localhost");
String hunkLimit = appProps.getProperty("hunkLimit","10");
String input = appProps.getProperty("inputPath","FORKJOIN");
String redisPath = appProps.getProperty("redisPath","FORKJOIN");
@@ -57,12 +58,12 @@ public class Launcher {
//
// log.info(parameters);
mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter, srcMLPath,hostname);
mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter, srcMLPath,hostname,hunkLimit);
}
public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter,String srcMLPath,String hostname){
public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter,String srcMLPath,String hostname,String hunkLimit){
String dbDir;
@@ -78,7 +79,7 @@ public class Launcher {
try {
switch (jobType) {
case "RICHEDITSCRIPT":
EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter);
EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter,hunkLimit);
break;
case "LOAD":
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java
+8 -3
View File
@@ -26,12 +26,17 @@ public class EDiffHunkParser extends EDiffParser {
private static Logger logger = LoggerFactory.getLogger(EDiffHunkParser.class);
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath,String rootType) {
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath,String hunkLimit) {
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile,srcMLPath);
if (actionSets.size() != 0) {
if (actionSets != null && actionSets.size() != 0) {
boolean processActionSet = true;
if (actionSets.size() > Integer.valueOf(hunkLimit)){
processActionSet = false;
logger.debug("Skipping {} set size {}",diffentryFile.getName(),hunkLimit);
}
int hunkSet = 0;
if(processActionSet){
for (HierarchicalActionSet actionSet : actionSets) {
@@ -79,7 +84,7 @@ public class EDiffHunkParser extends EDiffParser {
} catch (Exception e) {
logger.error("error",e);
e.printStackTrace();
// e.printStackTrace();
}
hunkSet++;
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java
+1 -1
View File
@@ -65,7 +65,7 @@ public class EDiffWorker extends UntypedActor {
future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS);
counter ++;
if (counter % 10 == 0) {
if (counter % 1000 == 0) {
log.info("Worker #" + id +" finalized parsing " + counter + " files... remaing "+ (files.size() - counter));
}
} catch (TimeoutException e) {
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouperForC.java
+23 -14
View File
@@ -11,6 +11,7 @@ import edu.lu.uni.serval.utils.ListSorter;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Regroup GumTree results to a hierarchical construction.
@@ -147,19 +148,19 @@ public class HierarchicalRegrouperForC {
}
private boolean isPossibileSubAction(Action parent, Action child) {
if ((parent instanceof Update && !(child instanceof Addition))
|| (parent instanceof Delete && child instanceof Delete)
|| (parent instanceof Insert && (child instanceof Insert))) {
int startPosition = child.getPosition();
int length = child.getLength();
int startPosition2 = parent.getPosition();
int length2 = parent.getLength();
if (!(startPosition2 <= startPosition && startPosition + length <= startPosition2 + length2)) {
// when act is not the sub-set of action.
return false;
}
}
// if ((parent instanceof Update && !(child instanceof Addition))
// || (parent instanceof Delete && child instanceof Delete)
// || (parent instanceof Insert && (child instanceof Insert))) {
// int startPosition = child.getPosition();
// int length = child.getLength();
// int startPosition2 = parent.getPosition();
// int length2 = parent.getLength();
//
// if (!(startPosition2 <= startPosition && startPosition + length <= startPosition2 + length2)) {
// // when act is not the sub-set of action.
// return false;
// }
// }
return true;
}
@@ -282,10 +283,18 @@ public class HierarchicalRegrouperForC {
int nodeType = tree.getType();
// List<ITree> collect = tree.getChildren().stream().filter(m -> m.getType() == 6).collect(Collectors.toList());
// if (collect.size() > 0){
// return true;
// }
if (NodeMap_new.StatementMap.containsKey(nodeType)){
return true;
}
// else{
// if((nodeType ==6) && tree.getParent().getType() == 1){
// return true;
// }
// }
// if (nodeType == 11 || nodeType == 16 || nodeType == 18 || nodeType == 21
// || nodeType == 22 || nodeType == 23 || nodeType == 24 || nodeType == 84
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java
+60 -17
View File
@@ -5,6 +5,7 @@ import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser;
import edu.lu.uni.serval.utils.ClusterToPattern;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.io.FileUtils;
@@ -16,6 +17,8 @@ import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.IOException;
import java.lang.instrument.Instrumentation;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
@@ -28,35 +31,75 @@ public class HunkParserTest {
public void testSimple() throws IOException {
// String input = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/revFiles/7f52f3_3845d29_drivers#pci#host#pcie-altera.c";
String root = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/linux/";
// String root = "/Users/anil.koyuncu/projects/fixminer/gumInputLinux/linux/";
String root = "/Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/gumInputLinux/";
String filename ="";
// filename ="freebsd_ceca9b8_b864ac4_sys#kern#sched_ule.c"; //too long
// filename ="openbsd_e592ec_39c81a_sys#arch#i386#pci#pci_machdep.c"; //not parseable 56 "parameter_list" "" () ( (57 "parameter" "" () ( (22 "decl" "" () ())))
// filename ="openbsd_cbb6d2_4cab495_sys#lib#libsa#printf.c";
// filename ="freebsd_bb1ce4_10d4c2_sbin#gvinum#gvinum.c"; //too long
// filename ="freebsd_253913_35ea52_sys#netinet#ip_carp.c"; //ok
// filename ="FFmpeg_4c9d1c_3da860_libavutil#file_open.c"; //not sure ifdef
// filename ="gstreamer_0af74c_e8bae0_libs#gst#net#gstptpclock.c"; //not sure ifder
// filename ="freebsd_abdbcc6_030063_sys#netinet#ip_carp.c"; //ok
// filename ="linux_80d348_5b394b_fs#overlayfs#inode.c"; //ok
// filename ="openbsd_43b185_e7495b_usr.bin#cvs#rcs.c"; //okish
// filename ="openbsd_e592ec_39c81a_sys#arch#i386#pci#pci_machdep.c"; //(56 "parameter_list" "" () ( (57 "parameter" "" () ( (22 "decl" "" () ())))
// filename ="openbsd_cbb6d2_4cab495_sys#lib#libsa#printf.c"; //not parseable 56 "parameter_list" "" () ( (57 "parameter" "" () ( (22 "decl" "" () ())))
// filename ="FFmpeg_9219ec_647696_libavfilter#trim.c"; //partial
// filename ="vlc_92b7fd_f745f6_modules#control#dbus#dbus.c"; //okish
// filename ="vlc_eeb662_966879_modules#video_chroma#copy.c"; //ok
// filename ="omp_19fae3_1e4dcd_src#mca#mpool#sm#mpool_sm_mmap.c"; // cannot find
// filename ="FFmpeg_a8343bf_2b2039_libavformat#riff.c"; // ok
// filename ="freebsd_32766e4_200ff4_sbin#routed#parms.c"; // ok
filename ="openbsd_150ddd_cf0e20_usr.sbin#user#user.c"; //notok
filename ="openbsd_6fac1e_c3b383_usr.bin#tmux#window-copy.c"; //notok
filename ="freebsd_0cb6f2_b4c742_sys#dev#ipw#if_ipw.c"; //notok
filename ="php-src_7defd5_da06f7_ext#mbstring#mbstring.c"; //notok (19 "expr_stmt" "" () ()))))
filename ="libtiff_177169_71715f_tools#tiff2ps.c"; //notok (19 "expr_stmt" "" () ()))))
filename ="linux_955c1dd_0aaee4_drivers#gpu#drm#i915#gvt#handlers.c"; //notok (19 "expr_stmt" "" () ()))))
// String filename = "8dd302_c4ef85_net#core#dev_ioctl.c"; //ok
// String filename = "1e793f6_77f18a_drivers#scsi#megaraid#megaraid_sas_base.c"; //OK
// String filename = "6a28fd_93ad867_drivers#tty#goldfish.c"; //m,issing
// String filename = "b90f7c_ff51ff_kernel#sched#fair.c"; //wrong and wired
// String filename = "ed8f68_b1c8047_fs#ext3#dir.c";//ok
// filename ="FFmpeg_0726b2_66d2ff_libav#jpeg.c";
String pj = filename.split("_")[0];
filename = filename.replace(pj+"_","");
root = root + pj + "/";
// root = root + "codeflaws/";
// String filename = "bc3d12_9a26653_drivers#scsi#libfc#fc_disc.c"; //okish
String filename = "118154_0c5f81_arch#x86#kvm#svm.c";
// String filename = "bcbd94f_43e43c9_drivers#md#dm-crypt.c"; //emin degilim
// String filename = "f1727b4_6c1e7e_arch#x86#kvm#vmx#nested.c"; //komplex not sure
// String filename = "5924f17_5925a05_net#ipv4#tcp.c";
// String filename = "bd0b9ac_b237721_drivers#irqchip#irq-dw-apb-ictl.c"; //missing
// String filename = "052831_3985e8_include#net#ip_tunnels.h";
// String filename = "e76019_28647b_drivers#gpu#drm#i915#i915_drv.h";
// String filename = "4cbe4d_b124f4_include#linux#mlx4#device.h"; //enum case stops at block
// String filename = "7bf7eac_c01daf_include#linux#dax.h";
// filename = "474-A-15925943-15925951.c"; //mot ok
// filename = "6-C-11536006-11536039.c"; //okish
// filename = "500-A-18298071-18298124.c"; //ok
// filename = "106-B-4027414-4027447.c"; //ok
// filename = "572-B-12669194-12669278.c"; //ok
// filename = "514-A-16254510-16254521.c"; //ok
// filename = "405-B-12287356-12287584.c"; //ok
// filename = "630-R-17825199-17825235.c"; //notok
File revFile = new File(root + "revFiles/"+ filename);
File prevFile =new File(root + "prevFiles/prev_"+filename);
// File diffFile =new File();
String path = root + "DiffEntries/"+filename + ".txt";
System.out.println(path);
// String data = "";
// data = new String(Files.readAllBytes(Paths.get(root + "DiffEntries/"+filename + ".txt")));
EDiffHunkParser parser = new EDiffHunkParser();
String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML/src2srcml";
String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml";
// String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml";
parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath);
// ITree t = new SrcmlCppTreeGenerator().generateFromFile(input).getRoot();
// Assert.assertEquals(148, t.getSize());
}
@Test
public void dumpFnction() throws Exception {
String pattern = "function/20/gstreamer_0af74c_e8bae0_libs#gst#net#gstptpclock.c.txt_0";
// String pattern = "function/20/FFmpeg_4c9d1c_3da860_libavutil#file_open.c.txt_0";
ClusterToPattern.main("6399","/Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis","ALLdumps-gumInput.rdb ",pattern);
}
@Test
public void newCTest(){
String root = "/Users/anilkoyuncu/projects/gumInputLinux/linux/";
src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+7 -4
View File
@@ -26,7 +26,7 @@ public class EnhancedASTDiff {
private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class);
public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType) throws Exception {
public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType,String hunkLimit) throws Exception {
String parameters = String.format("\nInput path %s",inputPath);
@@ -62,6 +62,8 @@ public class EnhancedASTDiff {
Stream<File> stream = Arrays.stream(listOfFiles);
List<File> folders = stream
.filter(x -> !x.getName().startsWith("."))
.filter(x -> !x.getName().startsWith("cocci"))
.filter(x -> !x.getName().endsWith(".index"))
.collect(Collectors.toList());
@@ -83,7 +85,7 @@ public class EnhancedASTDiff {
ActorSystem system = null;
ActorRef parsingActor = null;
final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath,rootType);
final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath,hunkLimit);
try {
log.info("Akka begins...");
log.info("{} files to process ...", allMessageFiles.size());
@@ -92,10 +94,11 @@ public class EnhancedASTDiff {
parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor");
parsingActor.tell(msg, ActorRef.noSender());
} catch (Exception e) {
system.terminate();
system.shutdown();
e.printStackTrace();
}finally {
system.terminate();
system.awaitTermination();
// system.shutdown();
}
break;
case "FORKJOIN":
src/main/resource/app.properties
+3 -1
View File
@@ -5,11 +5,13 @@ portDumps = 6399
parallelism = AKKA
numOfWorkers = 30
hostname = localhost
hunkLimit = 10
#inputPath = /Users/anilkoyuncu/projects/gumInputLinux
inputPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/gumInputLinux
redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
srcMLPath= /Users/anil.koyuncu/Downloads/srcML/src2srcml
#srcMLPath= /Users/anil.koyuncu/Downloads/srcML/src2srcml
srcMLPath= /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml
actionType = ALL
eDiffTimeout = 900