diff --git a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java index 66c909e..a0d0456 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java @@ -22,8 +22,8 @@ public class Launcher { Properties appProps = new Properties(); - String appConfigPath = "/Users/haoyetian/Documents/Lu_code/FixMiner/fixminer_source/src/main/resource/app.properties"; -// String appConfigPath = args[0]; +// String appConfigPath = "/Users/haoyetian/Documents/Lu_code/FixMiner/fixminer_source/src/main/resource/app.properties"; + String appConfigPath = args[0]; appProps.load(new FileInputStream(appConfigPath)); String portInner = appProps.getProperty("portInner","6380"); @@ -33,13 +33,15 @@ public class Launcher { String actionType = appProps.getProperty("actionType","ALL"); String eDiffTimeout = appProps.getProperty("eDiffTimeout","900"); String parallelism = appProps.getProperty("parallelism","FORKJOIN"); + String input = appProps.getProperty("inputPath","FORKJOIN"); String redisPath = appProps.getProperty("redisPath","FORKJOIN"); + String srcMLPath = appProps.getProperty("srcMLPath","FORKJOIN"); -// String parameter = args[2]; - String parameter = null; -// String jobType = args[1]; - String jobType = "RICHEDITSCRIPT"; + String parameter = args[2]; +// String parameter = null; + String jobType = args[1]; +// String jobType = "RICHEDITSCRIPT"; // String parameters = String.format("\nportInner %s " + // "\nnumOfWorkers %s " + @@ -50,12 +52,12 @@ public class Launcher { // // log.info(parameters); - mainLaunch(portInner, numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter); + mainLaunch(portInner, numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter, srcMLPath); } - public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter){ + public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter,String srcMLPath){ String dbDir; @@ -71,7 +73,7 @@ public class Launcher { try { switch (jobType) { case "RICHEDITSCRIPT": - EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName); + EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath); break; case "COMPARE": diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java index 9e714ea..5316137 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java @@ -54,7 +54,7 @@ public class EDiffActor extends UntypedActor { int toIndex = (i + 1) * average + counter; List filesOfWorkers = files.subList(fromIndex, toIndex); - final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getInnerPool()); + final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getInnerPool(),((EDiffMessage) message).getSrcMLPath()); mineRouter.tell(workMsg, getSelf()); logger.info("Assign {} task to worker #" + (i + 1) ,filesOfWorkers.size()); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java index cd30b79..d6bcdd6 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java @@ -8,6 +8,8 @@ import redis.clients.jedis.JedisPool; import java.io.File; import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.ObjectOutputStream; import java.util.List; @@ -22,8 +24,8 @@ public class EDiffHunkParser extends EDiffParser { private static Logger logger = LoggerFactory.getLogger(EDiffHunkParser.class); @Override - public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool) { - List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); + public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath) { + List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile,srcMLPath); if (actionSets.size() != 0) { boolean processActionSet = true; @@ -31,7 +33,7 @@ public class EDiffHunkParser extends EDiffParser { int hunkSet = 0; if(processActionSet){ for (HierarchicalActionSet actionSet : actionSets) { - FileOutputStream f = null; +// FileOutputStream f = null; try { @@ -51,6 +53,16 @@ public class EDiffHunkParser extends EDiffParser { inner.hset("dump".getBytes(),key.getBytes(),EDiffHelper.kryoSerialize(actionSet)); } +// File f = new File(root+"dumps/"+astNodeType+"/"+String.valueOf(size)+"/"); +// f.mkdirs(); +// f = new File(root+"dumps/"+key); +// +// +// FileOutputStream fos = new FileOutputStream(f); +// ObjectOutputStream oos = new ObjectOutputStream(fos); +// oos.writeObject(EDiffHelper.kryoSerialize(actionSet)); +// oos.flush(); +// oos.close(); } catch (Exception e) { logger.error("error",e); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java index a2c8697..b70960c 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java @@ -10,6 +10,7 @@ public class EDiffMessage extends BaseMessage{ private List msgFiles; + public JedisPool getInnerPool() { return innerPool; } @@ -20,14 +21,21 @@ public class EDiffMessage extends BaseMessage{ private JedisPool innerPool; + public String getSrcMLPath() { + return srcMLPath; + } + + private String srcMLPath; + - public EDiffMessage(int id, List msgFiles,String eDiffTimeout,JedisPool pool) { + public EDiffMessage(int id, List msgFiles,String eDiffTimeout,JedisPool pool,String srcMLPath) { super(id,new Long(eDiffTimeout)); this.msgFiles = msgFiles; this.innerPool = pool; + this.srcMLPath = srcMLPath; } public EDiffMessage(int id, List msgFiles,Long eDiffTimeout,JedisPool pool) { super(id,eDiffTimeout); @@ -36,6 +44,14 @@ public class EDiffMessage extends BaseMessage{ } + public EDiffMessage(int id, List filesOfWorkers, long seconds_to_wait, JedisPool innerPool, String srcMLPath) { + + super(id,seconds_to_wait); + this.msgFiles = filesOfWorkers; + this.innerPool = innerPool; + this.srcMLPath = srcMLPath; + } + public List getMsgFiles() { return msgFiles; } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java index 93e011c..2d2cf35 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java @@ -38,7 +38,7 @@ public class EDiffParser extends Parser { * @param revFile * @return */ - protected List parseChangedSourceCodeWithGumTree2(File prevFile, File revFile) { + protected List parseChangedSourceCodeWithGumTree2(File prevFile, File revFile,String srcMLPath) { List actionSets = new ArrayList<>(); // GumTree results boolean isJava =false; @@ -46,7 +46,9 @@ public class EDiffParser extends Parser { if (revFile.getName().endsWith(".c") & prevFile.getName().endsWith(".c") || revFile.getName().endsWith(".h") & prevFile.getName().endsWith(".h")){ // gumTreeResults = new GumTreeComparer().compareCFilesWithGumTree(prevFile, revFile); - gumTreeResults = new GumTreeCComparer().compareCFilesWithGumTree(prevFile, revFile); + + + gumTreeResults = new GumTreeCComparer().compareCFilesWithGumTree(prevFile, revFile,srcMLPath); }else{ gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); isJava = true; @@ -79,10 +81,15 @@ public class EDiffParser extends Parser { } @Override - public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool) { + public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath) { } +// @Override +// public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool) { +// +// } + } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java index 2b18154..960a770 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java @@ -43,6 +43,7 @@ public class EDiffWorker extends UntypedActor { int id = msg.getId(); int counter = 0; JedisPool innerPool = msg.getInnerPool(); + String srcMLPath = msg.getSrcMLPath(); for (MessageFile msgFile : files) { File revFile = msgFile.getRevFile(); @@ -55,7 +56,9 @@ public class EDiffWorker extends UntypedActor { final ExecutorService executor = Executors.newSingleThreadExecutor(); // schedule the work - final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getInnerPool())); + + + final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getInnerPool(),srcMLPath)); try { // wait for task to complete future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java index d24bf1e..ffdb4c6 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java @@ -12,24 +12,39 @@ public class HunkParserTest { public void testSimple() throws IOException { // String input = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/revFiles/7f52f3_3845d29_drivers#pci#host#pcie-altera.c"; - String root = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/linux/"; + String root = "/Users/anilkoyuncu/projects/gumInputLinux/linux/"; // String filename = "8dd302_c4ef85_net#core#dev_ioctl.c"; + String filename = "6a28fd_93ad867_drivers#tty#goldfish.c"; // String filename = "5924f17_5925a05_net#ipv4#tcp.c"; // String filename = "bd0b9ac_b237721_drivers#irqchip#irq-dw-apb-ictl.c"; // String filename = "052831_3985e8_include#net#ip_tunnels.h"; // String filename = "e76019_28647b_drivers#gpu#drm#i915#i915_drv.h"; - String filename = "4cbe4d_b124f4_include#linux#mlx4#device.h"; //enum case stops at block +// String filename = "4cbe4d_b124f4_include#linux#mlx4#device.h"; //enum case stops at block // String filename = "7bf7eac_c01daf_include#linux#dax.h"; File revFile = new File(root + "revFiles/"+ filename); File prevFile =new File(root + "prevFiles/prev_"+filename); EDiffHunkParser parser = new EDiffHunkParser(); - parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile); + String srcMLPath = "/Users/anilkoyuncu/Downloads/srcML2/src2srcml"; + parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath); // ITree t = new SrcmlCppTreeGenerator().generateFromFile(input).getRoot(); // Assert.assertEquals(148, t.getSize()); } + @Test + public void newCTest(){ + String root = "/Users/anilkoyuncu/projects/gumInputLinux/linux/"; + String filename = "6a28fd_93ad867_drivers#tty#goldfish.c"; + + String srcMLPath = "/Users/anilkoyuncu/Downloads/srcML2/src2srcml"; + File revFile = new File(root + "revFiles/"+ filename); + File prevFile =new File(root + "prevFiles/prev_"+filename); + File diffFile = new File(root + "DiffEntries/"+filename+".txt"); + EDiffHunkParser parser = new EDiffHunkParser(); + parser.parseFixPatterns(prevFile,revFile,diffFile,"gumInputLinux",null,srcMLPath); + } + @Test public void testSimpleJava() throws IOException { @@ -44,7 +59,7 @@ public class HunkParserTest { EDiffHunkParser parser = new EDiffHunkParser(); - parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile); + parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile,""); // ITree t = new SrcmlCppTreeGenerator().generateFromFile(input).getRoot(); // Assert.assertEquals(148, t.getSize()); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java index 349410b..a738898 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java @@ -20,7 +20,7 @@ public abstract class Parser implements ParserInterface { protected String originalTree = ""; // Guide of generating patches. protected String actionSets = ""; // Guide of generating patches. - public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool); + public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath); @Override diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java index b4b48a4..4c23d50 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java @@ -12,6 +12,7 @@ public class RunnableParser implements Runnable { private Parser parser; private String project; private JedisPool pool; + private String srcMLPath; public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) { this.prevFile = prevFile; @@ -29,8 +30,18 @@ public class RunnableParser implements Runnable { this.pool = innerPool; } + public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath) { + this.prevFile = prevFile; + this.revFile = revFile; + this.diffentryFile = diffentryFile; + this.parser = parser; + this.project = project; + this.pool = innerPool; + this.srcMLPath = srcMLPath; + } + @Override public void run() { - parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool); + parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool,srcMLPath); } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java index 862bcc5..7ed5b90 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java @@ -24,7 +24,7 @@ public class EnhancedASTDiff { private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class); - public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName) throws Exception { + public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath) throws Exception { String parameters = String.format("\nInput path %s",inputPath); @@ -51,7 +51,7 @@ public class EnhancedASTDiff { List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" - msgFiles = msgFiles.subList(0,10000); +// msgFiles = msgFiles.subList(0,10000); if (msgFiles == null) continue; allMessageFiles.addAll(msgFiles); @@ -63,7 +63,8 @@ public class EnhancedASTDiff { case "AKKA": ActorSystem system = null; ActorRef parsingActor = null; - final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool); + + final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath); try { log.info("Akka begins..."); log.info("{} files to process ...", allMessageFiles.size()); @@ -87,7 +88,7 @@ public class EnhancedASTDiff { forEach(m -> { EDiffHunkParser parser = new EDiffHunkParser(); - parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool); + parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,"/Users/anilkoyuncu/Downloads/srcML2/src2srcml"); if (counter % 10 == 0) { log.info("Finalized parsing " + counter + " files... remaining " + (allMessageFiles.size() - counter)); } diff --git a/src/main/resource/app.properties b/src/main/resource/app.properties index 8e57561..62363bb 100755 --- a/src/main/resource/app.properties +++ b/src/main/resource/app.properties @@ -1,12 +1,13 @@ -jobType = RICHEDITSCRIPT + pjName = gumInput portInner = 6380 portDumps = 6399 parallelism = AKKA numOfWorkers = 1 -#inputPath = /Users/haoyetian/Documents/Lu_code/FixMiner/fixminer-core/python/data/gumInput -inputPath = /Users/haoyetian/Documents/Lu_code/test -redisPath = /Users/haoyetian/Documents/Lu_code/FixMiner/fixminer-core/python/data/redis + +inputPath = /Users/anilkoyuncu/projects/gumInputLinux/ +redisPath = /Users/anilkoyuncu/projects/fixminer/fixminer-core/python/data/redis +srcMLPath= /Users/anilkoyuncu/Downloads/srcML2/src2srcml actionType =ALL eDiffTimeout = 900