azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new dataset

Browse Source
This commit is contained in:
Anil Koyuncu
2018-04-18 18:54:49 +02:00
parent ec0b90a6b0
commit 30f33270c1
5 changed files with 100 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java
+8 -6
View File
@@ -21,13 +21,9 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis;
/**
* Created by anilkoyuncu on 19/03/2018.
*/
@@ -39,7 +35,7 @@ public class TreeLoaderClusterL1 {
private static Logger log = LoggerFactory.getLogger(TreeLoaderClusterL1.class);
// public static void main(String[] args){
public static void main(String portInner,String serverWait,String port,String inputPath,String level1DB,String level1Path) throws Exception {
public static void main(String portInner,String serverWait,String port,String inputPath,String level1DB,String level1Path,String innerTypePrefix) throws Exception {
// String inputPath;
// String outputPath;
@@ -81,6 +77,7 @@ public class TreeLoaderClusterL1 {
Stream<File> stream = Arrays.stream(listFiles);
List<File> dbs = stream
.filter(x -> x.getName().endsWith(".rdb"))
.filter(x-> x.getName().startsWith(innerTypePrefix))
.collect(Collectors.toList());
for (File db : dbs) {
String cmdInner = "bash "+inputPath + "/" + "startServer.sh" +" %s %s %s";
@@ -137,12 +134,17 @@ public class TreeLoaderClusterL1 {
innerPool.close();
}
String stopServer = "bash "+level1Path + "/" + "stopServer.sh" +" %s";
String stopServer = "bash "+inputPath + "/" + "stopServer.sh" +" %s";
stopServer = String.format(stopServer,Integer.valueOf(portInner));
// loadRedis(stopServer,serverWait);
cs.runShell(stopServer,serverWait);
}
String stopServer1 = "bash "+inputPath + "/" + "stopServer.sh" +" %s";
stopServer1 = String.format(stopServer1,Integer.valueOf(port));
// loadRedis(stopServer,serverWait);
cs.runShell(stopServer1,serverWait);
// calculatePairsOfClusters(inputPath, outputPath);
// mainCompare(inputPath,port,pairsCSVPath,importScript);
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java
+2
View File
@@ -95,6 +95,8 @@ public class FixedViolationHunkParser extends FixedViolationParser {
}
}
// public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) {
// // GumTree results
// List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile);
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster.java
+16 -12
View File
@@ -119,15 +119,18 @@ public class MultiThreadTreeLoaderCluster {
log.info("Load done");
}
public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner) {
public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String serverWait,String type) throws Exception {
CallShell cs = new CallShell();
String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner));
edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
cs.runShell(cmd1,serverWait);
// String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
// cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port));
String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port));
cs.runShell(cmd2,serverWait);
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000");
@@ -164,7 +167,8 @@ public class MultiThreadTreeLoaderCluster {
if (size == 0) {
String comd = String.format(cmd3,f.getPath(),portInner);
loadRedis(comd);
// loadRedis(comd);
cs.runShell(comd);
scan = jedis.scan("0", sc);
size = scan.getResult().size();
@@ -179,7 +183,7 @@ public class MultiThreadTreeLoaderCluster {
//76
scan.getResult().parallelStream()
.forEach(m -> coreCompare(m, jedisPool, clusterName,outerPool));
.forEach(m -> coreCompare(m, jedisPool, clusterName,outerPool,type));
jedis.save();
@@ -207,7 +211,7 @@ public class MultiThreadTreeLoaderCluster {
}
public static Pair<ITree,String> getTree(String firstValue, JedisPool outerPool){
public static Pair<ITree,String> getTree(String firstValue, JedisPool outerPool,String type){
// HierarchicalActionSet actionSet = null;
@@ -245,7 +249,7 @@ public class MultiThreadTreeLoaderCluster {
try {
inner = outerPool.getResource();
String filename = project + "/ActionSetDumps/" + pureFileName + ".txt_" + actionSetPosition;
String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition;
String si= inner.get(filename);
HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(si);
@@ -405,7 +409,7 @@ public class MultiThreadTreeLoaderCluster {
private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool) {
private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool,String type) {
try (Jedis jedis = jedisPool.getResource()) {
@@ -459,8 +463,8 @@ public class MultiThreadTreeLoaderCluster {
// }
try {
Pair<ITree, String> oldPair = getTree(firstValue, outerPool);
Pair<ITree, String> newPair = getTree(secondValue, outerPool);
Pair<ITree, String> oldPair = getTree(firstValue, outerPool,type);
Pair<ITree, String> newPair = getTree(secondValue, outerPool,type);
ITree oldTree = oldPair.getValue0();
ITree newTree = newPair.getValue0();
@@ -631,7 +635,7 @@ orginal calculate pairs, from all dumps of the projects
String line = null;
try {
FileOutputStream fos = new FileOutputStream(outputPath + "pairs/" +filename+".txt");
FileOutputStream fos = new FileOutputStream(outputPath + "/pairs/" +filename+".txt");
DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos));
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java
+62 -30
View File
@@ -161,17 +161,18 @@ public class MultiThreadTreeLoaderCluster3 {
}
// public static void mainCompare(String inputPath,String port,String pairsCSVPath,String importScript) {
public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner) {
public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String serverWait, String type) throws Exception {
CallShell cs = new CallShell();
String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner));
edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
cs.runShell(cmd1,serverWait);
String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port));
edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000");
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000");
cs.runShell(cmd2,serverWait);
String cmd3;
cmd3 = "bash " + importScript +" %s %s";
@@ -207,8 +208,9 @@ public class MultiThreadTreeLoaderCluster3 {
if (size == 0) {
String comd = String.format(cmd3, f.getPath(), portInner);
edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster.
loadRedis(comd);
cs.runShell(comd);
// edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster.
// loadRedis(comd);
scan = jedis.scan("0", sc);
size = scan.getResult().size();
@@ -221,7 +223,7 @@ public class MultiThreadTreeLoaderCluster3 {
scan.getResult().parallelStream()
.forEach(m -> coreCompare(m, jedisPool, clusterName, outerPool));
.forEach(m -> coreCompare(m, jedisPool, clusterName, outerPool,type));
jedis.save();
@@ -242,7 +244,7 @@ public class MultiThreadTreeLoaderCluster3 {
public static ITree getTree(String firstValue, JedisPool outerPool){
public static ITree getTree(String firstValue, JedisPool outerPool,String type){
// String gumTreeInput = "/Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/";
String[] split2 = firstValue.split("/");
String cluster = split2[1];
@@ -268,7 +270,7 @@ public class MultiThreadTreeLoaderCluster3 {
try {
inner = outerPool.getResource();
String fn = project + "/ActionSetDumps/" + s + ".txt_" + actionSetPosition;
String fn = project + "/"+type+"/" + s + ".txt_" + actionSetPosition;
String si= inner.get(fn);
HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(si);
@@ -443,6 +445,36 @@ public class MultiThreadTreeLoaderCluster3 {
addToken = true;
}
}else if(sType.equals("7")){//assignment
m.add(hours.trim().split("=")[0]);
m.add(to.trim().split("=")[0]);
addToken = true;
}else if(sType.equals("43") && oldDescendant.getParent() != null && oldDescendant.getParent().getParent() != null){// catch clause
if(oldDescendant.getParent().getType() == 44 && oldDescendant.getParent().getParent().getType() == 12){
m.add(hours.trim());
m.add(to.trim());
addToken = true;
}
}else if(sType.equals("59") && oldDescendant.getChildren().size()==1 && oldDescendant.getHeight()==1 ){
if( oldDescendant.getChild(0).getType() == 34 || oldDescendant.getChild(0).getType() == 45) {
m.add(hours.trim().split("=")[0]);
m.add(to.trim().split("=")[0]);
addToken = true;
}
}else if(sType.equals("42") && oldDescendant.getParent() != null && oldDescendant.getParent().getType() == 27 ){//infix varuable change
m.add(hours.trim());
m.add(to.trim());
addToken = true;
}else if(sType.equals("-1") && oldDescendant.getChildren().size()==0 && oldDescendant.getHeight()==0 ){//operator
m.add(hours.trim());
m.add(to.trim());
addToken = true;
}else if(sType.equals("83") && oldDescendant.getChildren().size()==0 && oldDescendant.getHeight()==0 ){//modifier
m.add(hours.trim());
m.add(to.trim());
addToken = true;
}else{
m.add(hours.trim());
m.add(to.trim());
@@ -504,23 +536,23 @@ public class MultiThreadTreeLoaderCluster3 {
oldTokens.add(methodName);
}else if (s.startsWith("MethodName:")){
String methodName = s.split(":")[1];
// ITree parent = oldDescendant.getParent();
//
// if(parent!= null && parent.getType() == 32 && !alreadyAddParentMethodName){ //parent is method invocation statement
// String parentLabel = parent.getLabel();
// String[] pns = parentLabel.split("\\." + methodName);
// if(pns.length > 1) {
// String parentName = pns[0];
// String[] parentNameSplit = parentName.split("@@");
// if (parentNameSplit.length > 1) {
// String parentMethodName = parentNameSplit[1];
// String s1 = parentMethodName.split("@TO@")[0];
// oldTokens.add(s1.trim());
// alreadyAddParentMethodName = true;
// }
// }
//
// }
ITree parent = oldDescendant.getParent();
if(parent!= null && parent.getType() == 32 && !alreadyAddParentMethodName){ //parent is method invocation statement
String parentLabel = parent.getLabel();
String[] pns = parentLabel.split("\\." + methodName);
if(pns.length > 1) {
String parentName = pns[0];
String[] parentNameSplit = parentName.split("@@");
if (parentNameSplit.length > 1) {
String parentMethodName = parentNameSplit[1];
String s1 = parentMethodName.split("@TO@")[0];
oldTokens.add(s1.trim());
alreadyAddParentMethodName = true;
}
}
}
oldTokens.add(methodName);
}else if( sType.equals("59") || sType.equals("43")|| sType.equals("53") || sType.equals("7") || sType.equals("27") || sType.equals("83") || sType.equals("44") ||sType.equals("78") || sType.equals("41") || addToken){
// if(sType.equals("27") || sType.equals("83") || sType.equals("44")){
@@ -570,7 +602,7 @@ public class MultiThreadTreeLoaderCluster3 {
return oldTokens;
}
private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool) {
private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool,String type) {
try (Jedis jedis = jedisPool.getResource()) {
@@ -621,8 +653,8 @@ public class MultiThreadTreeLoaderCluster3 {
// }
try {
ITree oldTree = getTree(firstValue,outerPool);
ITree newTree = getTree(secondValue,outerPool);
ITree oldTree = getTree(firstValue,outerPool,type);
ITree newTree = getTree(secondValue,outerPool,type);
if(oldTree == null || newTree == null) {
jedis.select(0);
src/main/java/edu/lu/uni/serval/Launcher.java
+12 -7
View File
@@ -46,11 +46,11 @@ public class Launcher {
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs";
// gumInput = "/Users/anilkoyuncu/bugStudy/dataset/Defects4J/";
portInner = "6380";
serverWait = "50000";
serverWait = "5000";
chunkName = "Bug13April.txt.csv.rdb";
// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
numOfWorkers = "10";
jobType = "AKKA";
jobType = "LEVEL1DB";
port = "6399";
// pairsPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImportDefects4J";
// gumOutput = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputDefects4J";
@@ -97,23 +97,28 @@ public class Launcher {
AkkaTreeLoader.main(portInner, serverWait, dbDir, chunk +"MOV"+".txt.csv.rdb", port, "MOV"+dumpsName);
break;
case "LEVEL1DB":
TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-BugsDotJar.rdb", dbDir + "/level1-BugsDotJar/");
TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "UPD"+".rdb", dbDir + "/level1-Bug13April/",pjName + "UPD");
TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "INS"+".rdb", dbDir + "/level1-Bug13April/",pjName + "INS");
TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "DEL"+".rdb", dbDir + "/level1-Bug13April/",pjName + "DEL");
TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "MOV"+".rdb", dbDir + "/level1-Bug13April/",pjName + "MOV");
break;
//CALC python abstractPatch.py to from cluster folder
case "L2CALCPAIRS":
// MultiThreadTreeLoaderCluster.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterDefect4J","/Users/anilkoyuncu/bugStudy/dataset/");
MultiThreadTreeLoaderCluster.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/cluster", datasetPath);
MultiThreadTreeLoaderCluster.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterBug13April", datasetPath);
break;
case "L2PAIRDB":
// MultiThreadTreeLoaderCluster.mainCompare("6300","/Users/anilkoyuncu/bugStudy/dataset/pairs-csv","/Users/anilkoyuncu/bugStudy/dataset/redisSingleImport.sh",dbDir,"clusterl1-d4j.rdb",dumpsName,"6301");
MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath + "/pairs-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-13april.rdb", dumpsName, "6301");
MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath + "/pairs-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-13april.rdb", "UPD"+dumpsName, "6301",serverWait,"UPD");
break;
//CALC via python
case "L3CALCPAIRS":
// MultiThreadTreeLoaderCluster3.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterDefect4J-2l",datasetPath);
MultiThreadTreeLoaderCluster3.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/cluster-2l", datasetPath);
MultiThreadTreeLoaderCluster3.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterBug13April-2l", datasetPath);
break;
case "L3PAIRDB":
MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath + "/pairs-2l-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-13april.rdb", dumpsName, "6301");
MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath + "/pairs-2l-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-13april.rdb", "UPD"+dumpsName, "6301",serverWait,"UPD");
break;
}
} catch (Exception e) {