azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changes for srcml

Browse Source
This commit is contained in:
fixminer
2020-01-06 11:14:46 +01:00
parent 913d2615d0
commit 422a3546b6
11 changed files with 100 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+11 -9
View File
@@ -22,8 +22,8 @@ public class Launcher {
Properties appProps = new Properties();
String appConfigPath = "/Users/haoyetian/Documents/Lu_code/FixMiner/fixminer_source/src/main/resource/app.properties";
// String appConfigPath = args[0];
// String appConfigPath = "/Users/haoyetian/Documents/Lu_code/FixMiner/fixminer_source/src/main/resource/app.properties";
String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath));
String portInner = appProps.getProperty("portInner","6380");
@@ -33,13 +33,15 @@ public class Launcher {
String actionType = appProps.getProperty("actionType","ALL");
String eDiffTimeout = appProps.getProperty("eDiffTimeout","900");
String parallelism = appProps.getProperty("parallelism","FORKJOIN");
String input = appProps.getProperty("inputPath","FORKJOIN");
String redisPath = appProps.getProperty("redisPath","FORKJOIN");
String srcMLPath = appProps.getProperty("srcMLPath","FORKJOIN");
// String parameter = args[2];
String parameter = null;
// String jobType = args[1];
String jobType = "RICHEDITSCRIPT";
String parameter = args[2];
// String parameter = null;
String jobType = args[1];
// String jobType = "RICHEDITSCRIPT";
// String parameters = String.format("\nportInner %s " +
// "\nnumOfWorkers %s " +
@@ -50,12 +52,12 @@ public class Launcher {
//
// log.info(parameters);
mainLaunch(portInner, numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter);
mainLaunch(portInner, numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter, srcMLPath);
}
public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter){
public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter,String srcMLPath){
String dbDir;
@@ -71,7 +73,7 @@ public class Launcher {
try {
switch (jobType) {
case "RICHEDITSCRIPT":
EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName);
EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath);
break;
case "COMPARE":
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java
+1 -1
View File
@@ -54,7 +54,7 @@ public class EDiffActor extends UntypedActor {
int toIndex = (i + 1) * average + counter;
List<MessageFile> filesOfWorkers = files.subList(fromIndex, toIndex);
final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getInnerPool());
final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getInnerPool(),((EDiffMessage) message).getSrcMLPath());
mineRouter.tell(workMsg, getSelf());
logger.info("Assign {} task to worker #" + (i + 1) ,filesOfWorkers.size());
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java
+15 -3
View File
@@ -8,6 +8,8 @@ import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.ObjectOutputStream;
import java.util.List;
@@ -22,8 +24,8 @@ public class EDiffHunkParser extends EDiffParser {
private static Logger logger = LoggerFactory.getLogger(EDiffHunkParser.class);
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool) {
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile);
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath) {
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile,srcMLPath);
if (actionSets.size() != 0) {
boolean processActionSet = true;
@@ -31,7 +33,7 @@ public class EDiffHunkParser extends EDiffParser {
int hunkSet = 0;
if(processActionSet){
for (HierarchicalActionSet actionSet : actionSets) {
FileOutputStream f = null;
// FileOutputStream f = null;
try {
@@ -51,6 +53,16 @@ public class EDiffHunkParser extends EDiffParser {
inner.hset("dump".getBytes(),key.getBytes(),EDiffHelper.kryoSerialize(actionSet));
}
// File f = new File(root+"dumps/"+astNodeType+"/"+String.valueOf(size)+"/");
// f.mkdirs();
// f = new File(root+"dumps/"+key);
//
//
// FileOutputStream fos = new FileOutputStream(f);
// ObjectOutputStream oos = new ObjectOutputStream(fos);
// oos.writeObject(EDiffHelper.kryoSerialize(actionSet));
// oos.flush();
// oos.close();
} catch (Exception e) {
logger.error("error",e);
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java
+17 -1
View File
@@ -10,6 +10,7 @@ public class EDiffMessage extends BaseMessage{
private List<MessageFile> msgFiles;
public JedisPool getInnerPool() {
return innerPool;
}
@@ -20,14 +21,21 @@ public class EDiffMessage extends BaseMessage{
private JedisPool innerPool;
public String getSrcMLPath() {
return srcMLPath;
}
private String srcMLPath;
public EDiffMessage(int id, List<MessageFile> msgFiles,String eDiffTimeout,JedisPool pool) {
public EDiffMessage(int id, List<MessageFile> msgFiles,String eDiffTimeout,JedisPool pool,String srcMLPath) {
super(id,new Long(eDiffTimeout));
this.msgFiles = msgFiles;
this.innerPool = pool;
this.srcMLPath = srcMLPath;
}
public EDiffMessage(int id, List<MessageFile> msgFiles,Long eDiffTimeout,JedisPool pool) {
super(id,eDiffTimeout);
@@ -36,6 +44,14 @@ public class EDiffMessage extends BaseMessage{
}
public EDiffMessage(int id, List<MessageFile> filesOfWorkers, long seconds_to_wait, JedisPool innerPool, String srcMLPath) {
super(id,seconds_to_wait);
this.msgFiles = filesOfWorkers;
this.innerPool = innerPool;
this.srcMLPath = srcMLPath;
}
public List<MessageFile> getMsgFiles() {
return msgFiles;
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java
+10 -3
View File
@@ -38,7 +38,7 @@ public class EDiffParser extends Parser {
* @param revFile
* @return
*/
protected List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree2(File prevFile, File revFile) {
protected List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree2(File prevFile, File revFile,String srcMLPath) {
List<HierarchicalActionSet> actionSets = new ArrayList<>();
// GumTree results
boolean isJava =false;
@@ -46,7 +46,9 @@ public class EDiffParser extends Parser {
if (revFile.getName().endsWith(".c") & prevFile.getName().endsWith(".c") || revFile.getName().endsWith(".h") & prevFile.getName().endsWith(".h")){
// gumTreeResults = new GumTreeComparer().compareCFilesWithGumTree(prevFile, revFile);
gumTreeResults = new GumTreeCComparer().compareCFilesWithGumTree(prevFile, revFile);
gumTreeResults = new GumTreeCComparer().compareCFilesWithGumTree(prevFile, revFile,srcMLPath);
}else{
gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
isJava = true;
@@ -79,10 +81,15 @@ public class EDiffParser extends Parser {
}
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool) {
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath) {
}
// @Override
// public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool) {
//
// }
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java
+4 -1
View File
@@ -43,6 +43,7 @@ public class EDiffWorker extends UntypedActor {
int id = msg.getId();
int counter = 0;
JedisPool innerPool = msg.getInnerPool();
String srcMLPath = msg.getSrcMLPath();
for (MessageFile msgFile : files) {
File revFile = msgFile.getRevFile();
@@ -55,7 +56,9 @@ public class EDiffWorker extends UntypedActor {
final ExecutorService executor = Executors.newSingleThreadExecutor();
// schedule the work
final Future<?> future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getInnerPool()));
final Future<?> future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getInnerPool(),srcMLPath));
try {
// wait for task to complete
future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS);
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java
+19 -4
View File
@@ -12,24 +12,39 @@ public class HunkParserTest {
public void testSimple() throws IOException {
// String input = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/revFiles/7f52f3_3845d29_drivers#pci#host#pcie-altera.c";
String root = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/linux/";
String root = "/Users/anilkoyuncu/projects/gumInputLinux/linux/";
// String filename = "8dd302_c4ef85_net#core#dev_ioctl.c";
String filename = "6a28fd_93ad867_drivers#tty#goldfish.c";
// String filename = "5924f17_5925a05_net#ipv4#tcp.c";
// String filename = "bd0b9ac_b237721_drivers#irqchip#irq-dw-apb-ictl.c";
// String filename = "052831_3985e8_include#net#ip_tunnels.h";
// String filename = "e76019_28647b_drivers#gpu#drm#i915#i915_drv.h";
String filename = "4cbe4d_b124f4_include#linux#mlx4#device.h"; //enum case stops at block
// String filename = "4cbe4d_b124f4_include#linux#mlx4#device.h"; //enum case stops at block
// String filename = "7bf7eac_c01daf_include#linux#dax.h";
File revFile = new File(root + "revFiles/"+ filename);
File prevFile =new File(root + "prevFiles/prev_"+filename);
EDiffHunkParser parser = new EDiffHunkParser();
parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile);
String srcMLPath = "/Users/anilkoyuncu/Downloads/srcML2/src2srcml";
parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath);
// ITree t = new SrcmlCppTreeGenerator().generateFromFile(input).getRoot();
// Assert.assertEquals(148, t.getSize());
}
@Test
public void newCTest(){
String root = "/Users/anilkoyuncu/projects/gumInputLinux/linux/";
String filename = "6a28fd_93ad867_drivers#tty#goldfish.c";
String srcMLPath = "/Users/anilkoyuncu/Downloads/srcML2/src2srcml";
File revFile = new File(root + "revFiles/"+ filename);
File prevFile =new File(root + "prevFiles/prev_"+filename);
File diffFile = new File(root + "DiffEntries/"+filename+".txt");
EDiffHunkParser parser = new EDiffHunkParser();
parser.parseFixPatterns(prevFile,revFile,diffFile,"gumInputLinux",null,srcMLPath);
}
@Test
public void testSimpleJava() throws IOException {
@@ -44,7 +59,7 @@ public class HunkParserTest {
EDiffHunkParser parser = new EDiffHunkParser();
parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile);
parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile,"");
// ITree t = new SrcmlCppTreeGenerator().generateFromFile(input).getRoot();
// Assert.assertEquals(148, t.getSize());
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java
+1 -1
View File
@@ -20,7 +20,7 @@ public abstract class Parser implements ParserInterface {
protected String originalTree = ""; // Guide of generating patches.
protected String actionSets = ""; // Guide of generating patches.
public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool);
public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath);
@Override
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java
+12 -1
View File
@@ -12,6 +12,7 @@ public class RunnableParser implements Runnable {
private Parser parser;
private String project;
private JedisPool pool;
private String srcMLPath;
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) {
this.prevFile = prevFile;
@@ -29,8 +30,18 @@ public class RunnableParser implements Runnable {
this.pool = innerPool;
}
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath) {
this.prevFile = prevFile;
this.revFile = revFile;
this.diffentryFile = diffentryFile;
this.parser = parser;
this.project = project;
this.pool = innerPool;
this.srcMLPath = srcMLPath;
}
@Override
public void run() {
parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool);
parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool,srcMLPath);
}
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+5 -4
View File
@@ -24,7 +24,7 @@ public class EnhancedASTDiff {
private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class);
public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName) throws Exception {
public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath) throws Exception {
String parameters = String.format("\nInput path %s",inputPath);
@@ -51,7 +51,7 @@ public class EnhancedASTDiff {
List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
msgFiles = msgFiles.subList(0,10000);
// msgFiles = msgFiles.subList(0,10000);
if (msgFiles == null)
continue;
allMessageFiles.addAll(msgFiles);
@@ -63,7 +63,8 @@ public class EnhancedASTDiff {
case "AKKA":
ActorSystem system = null;
ActorRef parsingActor = null;
final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool);
final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath);
try {
log.info("Akka begins...");
log.info("{} files to process ...", allMessageFiles.size());
@@ -87,7 +88,7 @@ public class EnhancedASTDiff {
forEach(m ->
{
EDiffHunkParser parser = new EDiffHunkParser();
parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool);
parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,"/Users/anilkoyuncu/Downloads/srcML2/src2srcml");
if (counter % 10 == 0) {
log.info("Finalized parsing " + counter + " files... remaining " + (allMessageFiles.size() - counter));
}
src/main/resource/app.properties
+5 -4
View File
@@ -1,12 +1,13 @@
jobType = RICHEDITSCRIPT
pjName = gumInput
portInner = 6380
portDumps = 6399
parallelism = AKKA
numOfWorkers = 1
#inputPath = /Users/haoyetian/Documents/Lu_code/FixMiner/fixminer-core/python/data/gumInput
inputPath = /Users/haoyetian/Documents/Lu_code/test
redisPath = /Users/haoyetian/Documents/Lu_code/FixMiner/fixminer-core/python/data/redis
inputPath = /Users/anilkoyuncu/projects/gumInputLinux/
redisPath = /Users/anilkoyuncu/projects/fixminer/fixminer-core/python/data/redis
srcMLPath= /Users/anilkoyuncu/Downloads/srcML2/src2srcml
actionType =ALL
eDiffTimeout = 900