azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change in richedit

Browse Source
This commit is contained in:
mimic
2020-03-05 14:02:32 +01:00
parent 3b45f0e95e
commit 131087a3f1
7 changed files with 101 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pom.xml
+6
View File
@@ -29,6 +29,12 @@
<artifactId>javatuples</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>me.tongfei</groupId>
<artifactId>progressbar</artifactId>
<version>0.8.1</version>
</dependency>
<!-- akka -->
<dependency>
src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+7 -11
View File
@@ -26,16 +26,16 @@ public class Launcher {
// String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath));
// String portInner = appProps.getProperty("portInner","6380");
String numOfWorkers = appProps.getProperty("numOfWorkers", "10");
String portDumps = appProps.getProperty("portDumps","6399");
String pjName = appProps.getProperty("pjName","allDataset");
String actionType = appProps.getProperty("actionType","ALL");
String eDiffTimeout = appProps.getProperty("eDiffTimeout","900");
String parallelism = appProps.getProperty("parallelism","FORKJOIN");
String hostname = appProps.getProperty("hostname","localhost");
String hunkLimit = appProps.getProperty("hunkLimit","10");
String patchSize = appProps.getProperty("patchSize","50");
String projectL = appProps.getProperty("projectList","");
String[] projectList = projectL.split(",");
String input = appProps.getProperty("inputPath","FORKJOIN");
String redisPath = appProps.getProperty("redisPath","FORKJOIN");
String srcMLPath = appProps.getProperty("srcMLPath","FORKJOIN");
@@ -58,12 +58,12 @@ public class Launcher {
//
// log.info(parameters);
mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter, srcMLPath,hostname,hunkLimit);
mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,input,redisPath,parameter, srcMLPath,hostname,hunkLimit,projectList,patchSize);
}
public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter,String srcMLPath,String hostname,String hunkLimit){
public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String input, String redisPath,String parameter,String srcMLPath,String hostname,String hunkLimit,String[] projectList,String patchSize){
String dbDir;
@@ -79,11 +79,7 @@ public class Launcher {
try {
switch (jobType) {
case "RICHEDITSCRIPT":
EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter,hunkLimit);
break;
case "LOAD":
EnhancedASTDiff.load(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter);
EnhancedASTDiff.main(gumInput, pjName, portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter,hunkLimit,projectList,patchSize);
break;
case "COMPARE":
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java
+7 -5
View File
@@ -5,6 +5,7 @@ import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import me.tongfei.progressbar.ProgressBar;
import org.apache.commons.text.similarity.JaroWinklerDistance;
import org.javatuples.Pair;
import org.slf4j.Logger;
@@ -74,8 +75,8 @@ public class CompareTrees {
final Future<?> future = executor.submit(new RunnableCompare(job, errorPairs, filenames, outerPool, i));
results.add(future);
}
for (Future<?> future:results){
for (Future<?> future : ProgressBar.wrap(results, "Comparing")){
// for (Future<?> future:results){
try {
// wait for task to complete
future.get();
@@ -86,11 +87,12 @@ public class CompareTrees {
} catch (ExecutionException e) {
e.printStackTrace();
} finally {
executor.shutdownNow();
}
// finally {
// executor.shutdownNow();
// }
}
executor.shutdownNow();
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java
+24 -16
View File
@@ -18,7 +18,7 @@ import java.util.List;
/**
* Parse fix violations with GumTree in terms of multiple statements.
*
*
* @author kui.liu
*
*/
@@ -27,18 +27,27 @@ public class EDiffHunkParser extends EDiffParser {
private static Logger logger = LoggerFactory.getLogger(EDiffHunkParser.class);
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath,String hunkLimit) {
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile,srcMLPath);
String datasetName = project;
String[] split1 = diffentryFile.getParent().split(datasetName);
String root = split1[0];
String pj = split1[1].split("/")[1];
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath);
if (actionSets != null && actionSets.size() != 0) {
boolean processActionSet = true;
if (actionSets.size() > Integer.valueOf(hunkLimit)){
if (actionSets.size() > Integer.valueOf(hunkLimit)) {
processActionSet = false;
logger.debug("Skipping {} set size {}",diffentryFile.getName(),hunkLimit);
logger.debug("Skipping {} set size {}", diffentryFile.getName(), hunkLimit);
}
int hunkSet = 0;
if(processActionSet){
if (processActionSet) {
for (HierarchicalActionSet actionSet : actionSets) {
// FileOutputStream f = null;
@@ -51,23 +60,18 @@ public class EDiffHunkParser extends EDiffParser {
actionSet.toString();
int size = actionSet.getActionSize();
String datasetName = project;
String[] split1 = diffentryFile.getParent().split(datasetName);
String root = split1[0];
String pj = split1[1].split("/")[1];
String key = astNodeType+"/"+String.valueOf(size)+"/" + pj +"_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
String key = astNodeType + "/" + String.valueOf(size) + "/" + pj + "_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
ITree targetTree = EDiffHelper.getTargets(actionSet);
ITree actionTree = EDiffHelper.getActionTrees(actionSet);
ITree shapeTree = EDiffHelper.getShapeTree(actionSet);
try (Jedis inner = innerPool.getResource()) {
inner.hset("dump",key,actionSet.toString());
inner.hset(key,"actionTree",actionTree.toStaticHashString());
inner.hset(key,"targetTree",targetTree.toStaticHashString());
inner.hset(key,"shapeTree",shapeTree.toStaticHashString());
inner.hset("dump", key, actionSet.toString());
inner.hset(key, "actionTree", actionTree.toStaticHashString());
inner.hset(key, "targetTree", targetTree.toStaticHashString());
inner.hset(key, "shapeTree", shapeTree.toStaticHashString());
}
// File f = new File(root+"dumps/"+astNodeType+"/"+String.valueOf(size)+"/");
// f.mkdirs();
@@ -83,15 +87,19 @@ public class EDiffHunkParser extends EDiffParser {
// oos.close();
} catch (Exception e) {
logger.error("error",e);
logger.error("error", e);
// e.printStackTrace();
}
hunkSet++;
}
try (Jedis inner = innerPool.getResource()) {
inner.hset("diffEntry", pj + "_" + diffentryFile.getName(), "1");
}
}
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java
+9 -1
View File
@@ -8,14 +8,22 @@ public class MessageFile {
private File prevFile;
private File diffEntryFile;
private File positionFile;
private String project;
public MessageFile(File revFile, File prevFile, File diffEntryFile) {
public MessageFile(File revFile, File prevFile, File diffEntryFile,String project) {
super();
this.revFile = revFile;
this.prevFile = prevFile;
this.diffEntryFile = diffEntryFile;
this.project = project;
}
public String getProject() { return project;}
public void setProject(String project) { this.project = project;}
public File getRevFile() {
return revFile;
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+38 -212
View File
@@ -1,26 +1,21 @@
package edu.lu.uni.serval.fixminer.jobs;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import edu.lu.uni.serval.fixminer.akka.compare.CompareTrees;
import edu.lu.uni.serval.fixminer.akka.ediff.*;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser;
import edu.lu.uni.serval.fixminer.akka.ediff.FileHelper;
import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.PoolBuilder;
import me.tongfei.progressbar.ProgressBar;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.*;
import java.util.Map;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -31,7 +26,7 @@ public class EnhancedASTDiff {
private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class);
public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType,String hunkLimit) throws Exception {
public static void main(String inputPath, String project, String portInner, String dbDir, String chunkName,String srcMLPath,String parameter,String hunkLimit,String[] projectList,String patchSize) throws Exception {
String parameters = String.format("\nInput path %s",inputPath);
@@ -39,39 +34,19 @@ public class EnhancedASTDiff {
CallShell cs = new CallShell();
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
// if (rootType == null){
cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
// }else{
// cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner));
// }
cs.runShell(cmd, portInner);
JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
if (rootType == "add"){
try (Jedis inner = innerPool.getResource()) {
inner.select(2);
inner.flushDB();
inner.select(1);
inner.flushDB();
inner.select(0);
inner.del("compare");
}
List<Predicate<File>> allPredicates = new ArrayList<Predicate<File>>();
for (String s : projectList) {
Predicate<File> predicate = x->x.getName().endsWith(s);
allPredicates.add(predicate);
}
Predicate<File> predicate1 = x->x.getName().endsWith("libtiff");
Predicate<File> predicate2 = x->x.getName().endsWith("php-src");
Predicate<File> predicate3 = x->x.getName().endsWith("cpython");
Predicate<File> predicate4 = x->x.getName().endsWith("wireshark");
Predicate<File> predicate5 = x->x.getName().endsWith("gzip");
Predicate<File> predicate6 = x->x.getName().endsWith("gmp");
Predicate<File> predicate7 = x->x.getName().endsWith("lighttpd1.4");
Predicate<File> predicate8 = x->x.getName().endsWith("lighttpd2");
File folder = new File(inputPath);
File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles);
@@ -79,15 +54,14 @@ public class EnhancedASTDiff {
.filter(x -> !x.getName().startsWith("."))
.filter(x -> !x.getName().startsWith("cocci"))
.filter(x -> !x.getName().endsWith(".index"))
.filter(predicate1.or(predicate2).or(predicate3).or(predicate4).or(predicate5).or(predicate6).or(predicate7).or(predicate8))
.filter(allPredicates.stream().reduce(x->false, Predicate::or))
.collect(Collectors.toList());
List<MessageFile> allMessageFiles = new ArrayList<>();
for (File target : folders) {
List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/",project,patchSize); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
// msgFiles = msgFiles.subList(0,3000);
if (msgFiles == null)
@@ -97,122 +71,31 @@ public class EnhancedASTDiff {
}
switch (parallelism){
case "AKKA":
ActorSystem system = null;
ActorRef parsingActor = null;
Map<String, String> diffEntry;
try (Jedis inner = innerPool.getResource()) {
diffEntry = inner.hgetAll("diffEntry");
final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath,hunkLimit);
try {
log.info("Akka begins...");
log.info("{} files to process ...", allMessageFiles.size());
system = ActorSystem.create("Mining-FixPattern-System");
parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor");
parsingActor.tell(msg, ActorRef.noSender());
} catch (Exception e) {
system.shutdown();
e.printStackTrace();
}finally {
system.awaitTermination();
// system.shutdown();
}
break;
case "FORKJOIN":
int counter = new Object() {
int counter = 0;
{
allMessageFiles.stream().
parallel().
peek(x -> counter++).
forEach(m ->
{
EDiffHunkParser parser = new EDiffHunkParser();
parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,srcMLPath,hunkLimit);
if (counter % 10 == 0) {
log.info("Finalized parsing " + counter + " files... remaining " + (allMessageFiles.size() - counter));
}
}
);
}
}.counter;
log.info("Finished parsing {} files",counter);
break;
default:
// ProgressBar.wrap(allMessageFiles.stream().
// parallel(),"Task").
// forEach(m ->
// {
// EDiffHunkParser parser = new EDiffHunkParser();
// parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,srcMLPath,hunkLimit);
//// if (counter % 10 == 0) {
//// log.info("Finalized parsing " + counter + " files... remaining " + (allMessageFiles.size() - counter));
//// }
// }
// );
Integer numberOfWorkers = Integer.valueOf(numOfWorkers);
final ExecutorService executor = Executors.newWorkStealingPool(numberOfWorkers);
ArrayList<Future<?>> results = new ArrayList<Future<?>>();
for (MessageFile msgFile : allMessageFiles) {
File revFile = msgFile.getRevFile();
File prevFile = msgFile.getPrevFile();
File diffentryFile = msgFile.getDiffEntryFile();
// schedule the work
// log.info("Starting job {}",i);
// final Future<?> future = executor.submit(new CompareTrees.RunnableCompare(job, errorPairs, filenames, outerPool, i));
EDiffHunkParser parser = new EDiffHunkParser();
final Future<?> future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,innerPool,srcMLPath,hunkLimit));
results.add(future);
}
try(ProgressBar compare = new ProgressBar("Compare", allMessageFiles.size())){
// for (Future<?> future : ProgressBar.wrap(results, "Comparing")){
for (Future<?> future:results){
try {
// wait for task to complete
// future.get();
future.get(new Long(eDiffTimeout), TimeUnit.SECONDS);
compare.step();
} catch (TimeoutException e) {
future.cancel(true);
compare.step();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
// finally {
// executor.shutdownNow();
// }
}
}
executor.shutdownNow();
log.error("Unknown parallelism {}", parallelism);
break;
}
log.info("{} files to process ...", allMessageFiles.size());
if (diffEntry != null) {
log.info("{} files already process ...", diffEntry.size());
allMessageFiles = allMessageFiles.stream().filter(f -> !diffEntry.containsKey(f.getProject() + "_" + f.getDiffEntryFile().getName())).collect(Collectors.toList());
log.info("{} files to process ...", allMessageFiles.size());
}
ProgressBar.wrap(allMessageFiles.stream().
parallel(),"Task").
forEach(m ->
{
EDiffHunkParser parser = new EDiffHunkParser();
parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,srcMLPath,hunkLimit);
}
);
}
private static List<MessageFile> getMessageFiles(String gumTreeInput) {
private static List<MessageFile> getMessageFiles(String gumTreeInput,String datasetName,String patchSize) {
String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
File revFilesPath = new File(inputPath + "revFiles/");
log.info(revFilesPath.getPath());
@@ -234,12 +117,18 @@ public class EnhancedASTDiff {
int count = 0;
while (matcher.find())
count++;
if(count>51)
if(count>= Integer.valueOf(patchSize))
// if(count>201)
continue;
// if(FileHelper.readFile(diffentryFile).split("@@\\s\\-\\d+,*\\d*\\s\\+\\d+,*\\d*\\s@@").length > 2)
// continue;
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
// String datasetName = project;
String[] split1 = diffentryFile.getParent().split(datasetName);
String root = split1[0];
String pj = split1[1].split("/")[1];
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile,pj);
msgFiles.add(msgFile);
@@ -251,69 +140,6 @@ public class EnhancedASTDiff {
}
}
public static void load(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType) throws Exception {
String parameters = String.format("\nInput path %s",inputPath);
log.info(parameters);
CallShell cs = new CallShell();
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
// if (rootType == null){
// cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
// }else{
// cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner));
// }
cs.runShell(cmd, portInner);
JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
try (Jedis inner = innerPool.getResource()) {
inner.flushAll();
}
File folder = new File(new File(inputPath).getParent() + "/dumps/" + rootType);
File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles);
List<File> folders = stream
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
List<File> allMessageFiles = new ArrayList<>();
for (File target : folders) {
if(target.getName().startsWith("."))
continue;
List<File> files = Arrays.asList(target.listFiles());
if (files.size() > 1){
allMessageFiles.addAll(files);
}
}
log.info("Message size: "+allMessageFiles.size());
allMessageFiles.stream().
parallel().
forEach(x-> loadCore(x,innerPool));
}
public static void loadCore(File file2load, JedisPool innerPool){
try (Jedis inner = innerPool.getResource()) {
// byte[] dump = Files.readAllBytes(Paths.get(file2load.getPath()));
byte[] dump = FileUtils.readFileToByteArray(file2load);
// HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(dump);
String key = file2load.getPath().split("/dumps/")[1];
inner.hset("dump".getBytes(),key.getBytes(),dump);
// actionSet.toString();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}
src/main/resource/app.properties
+10 -7
View File
@@ -1,20 +1,23 @@
pjName = gumInput
portInner = 6380
pjName = patches
portDumps = 6399
parallelism = AKKA
numOfWorkers = 14
hostname = localhost
hunkLimit = 10
patchSize = 50
projectList = libtiff,php-src,cpython,wireshark,gzip,gmp,lighttpd1.4,lighttpd2
#inputPath = /Users/anilkoyuncu/projects/gumInputLinux
inputPath = /Users/anil.koyuncu/projects/fixminer/fixminer-data/gumInputLinux
#inputPath = /Users/anil.koyuncu/projects/fixminer/fixminer-data/gumInputLinux
inputPath = /Users/anil.koyuncu/projects/test/fixminer-data/patches
#redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
srcMLPath= /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml
#redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
redisPath = /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis
#srcMLPath= /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml
srcMLPath= /Users/anil.koyuncu/projects/test/srcML/bin/srcml
#srcMLPath= /usr/local/bin/srcml
actionType = ALL
eDiffTimeout = 900
#ENHANCEDASTDIFF,CACHE,LEVEL1,LEVEL2,LEVEL3