azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new version

Browse Source
This commit is contained in:
mimic
2019-03-03 11:42:44 +01:00
parent 711f8776c6
commit ea93442150
96 changed files with 3169 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+6 -6
View File
@@ -68,7 +68,7 @@ public class Launcher {
gumInput = datasetPath +"/"+pjName+"/";
gumOutput = datasetPath + "/EnhancedASTDiff" + pjName;
dbDir = datasetPath + "/redis";
pairsPath = datasetPath + "/pairsImport"+pjName;
pairsPath = datasetPath + "/pairs";
dumpsName = "dumps-"+pjName+".rdb";
int iCursor = Integer.valueOf(cursor);
@@ -81,11 +81,11 @@ public class Launcher {
StoreEDiffInCache.main(gumOutput, portDumps, dbDir, actionType+dumpsName,actionType);
break;
case "SI":
CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor);
ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath,chunk);
// CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor);
ImportPairs2DB.main(pairsPath, portInner, dbDir,datasetPath,chunk);
break;
case "SIMI":
AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,iCursor,chunk,eDiffTimeout,parallelism);
AkkaTreeLoader.main(portInner, dbDir, "pairs.rdb" , portDumps, actionType+dumpsName,pairsPath,numOfWorkers,iCursor,chunk,eDiffTimeout,parallelism);
break;
case "LEVEL1":
@@ -136,9 +136,9 @@ public class Launcher {
private static void level2(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput,int cursor) throws Exception {
String stopServer;
MultiThreadTreeLoaderCluster.calculatePairsOfClusters(datasetPath + "/cluster"+pjName+ actionType, datasetPath,actionType);
// MultiThreadTreeLoaderCluster.calculatePairsOfClusters(datasetPath + "/cluster"+pjName+ actionType, datasetPath,actionType);
MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath+"/pairs"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType,cursor);
MultiThreadTreeLoaderCluster.mainCompare(port, datasetPath+"/pairsAction", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6380",actionType,cursor);
CallShell cs3 =new CallShell();
String db22 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java
+2 -1
View File
@@ -87,7 +87,8 @@ public class AkkaTreeParser {
log.info("Scanning ");
sc.count(cursor);
sc.match("pair_[0-9]*");
sc.match("*");
// sc.match("pair_[0-9]*");
scan = inner.scan("0", sc);
int size = scan.getResult().size();
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java
+5 -4
View File
@@ -38,11 +38,12 @@ public class Compare {
String i = split[1];
String j = split[2];
String keyName = split[0];
oldTree = EDiffHelper.getSimpliedTree(i,outerPool);
oldTree = EDiffHelper.getSimpliedTree(keyName,i,outerPool);
newTree = EDiffHelper.getSimpliedTree(j,outerPool);
newTree = EDiffHelper.getSimpliedTree(keyName,j,outerPool);
Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
m.match();
@@ -66,7 +67,7 @@ public class Compare {
if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
|| ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) {
String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j);
String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j);
log.info("{} tagged to be similar" ,matchKey);
jedis.select(1);
@@ -76,7 +77,7 @@ public class Compare {
jedis.select(0);
jedis.del("pair_" + (String.valueOf(i)) + "_" + String.valueOf(j));
jedis.del(keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j));
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java
+197
View File
@@ -0,0 +1,197 @@
package edu.lu.uni.serval.fixminer.akka.compare;
import com.github.gumtreediff.actions.ActionGenerator;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.matchers.Matcher;
import com.github.gumtreediff.matchers.Matchers;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.text.similarity.JaroWinklerDistance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static edu.lu.uni.serval.fixminer.jobs.MultiThreadTreeLoaderCluster3.getNames;
/**
* Created by anilkoyuncu on 03/04/2018.
*/
public class CompareTrees {
private static Logger log = LoggerFactory.getLogger(CompareTrees.class);
public static void main(String[] args) throws IOException {
String portInner = "6380";
String port = "6399";
//
JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000);
List<String> listOfPairs = AkkaTreeParser.getMessages(innerPool,100000000);
// listOfPairs.stream().parallel().
int counter = new Object() {
int counter = 0;
{
listOfPairs.parallelStream().
peek(x -> counter++).
forEach(m ->
{
// Compare compare = new Compare();
coreCompare(m, args[0],innerPool, outerPool);
if (counter % 1000 == 0) {
log.info("Finalized parsing " + counter + " files... remaing " + (listOfPairs.size() - counter));
}
}
);
}
}.counter;
// coreCompare(args[0],args[1],args[2],args[3]);
}
public static void coreCompare(String pairName, String treeType,JedisPool innerPool,JedisPool outerPool ) {
// String portInner = "6380";
// String port = "6399";
// JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
//
// JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000);
Map<String, String> resultMap;
Jedis jedis = null;
ITree oldTree = null;
ITree newTree = null;
try {
jedis = innerPool.getResource();
String[] split = pairName.split("_");
String i = split[1];
String j = split[2];
String keyName = split[0];
switch (treeType) {
case "shape":
oldTree = EDiffHelper.getShapes(keyName, i, outerPool);
newTree = EDiffHelper.getShapes(keyName, j, outerPool);
break;
case "action":
oldTree = EDiffHelper.getActions(keyName, i, outerPool,innerPool);
newTree = EDiffHelper.getActions(keyName, j, outerPool,innerPool);
break;
case "token":
oldTree = EDiffHelper.getTokens(keyName, i, outerPool,innerPool);
newTree = EDiffHelper.getTokens(keyName, j, outerPool,innerPool);
List<String> oldTokens = new ArrayList<>();
List<String> newTokens = new ArrayList<>();
oldTokens = getNames(oldTree,oldTokens);
newTokens = getNames(newTree,newTokens);
CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]);
CharSequence[] newSequences = newTokens.toArray(new CharSequence[newTokens.size()]);
JaroWinklerDistance jwd = new JaroWinklerDistance();
Double overallSimi = Double.valueOf(0);
if(oldSequences.length > 0 && (oldSequences.length == newSequences.length)){
for (int idx = 0; idx < newSequences.length; idx++) {
Double simi = jwd.apply(newSequences[idx], oldSequences[idx]);
overallSimi = simi + overallSimi;
}
overallSimi = overallSimi / oldSequences.length;
}else{
overallSimi = Double.valueOf(0);
// if(oldSequences.length != 0) {
// log.info("ERROR");
// }
}
int retval = Double.compare(overallSimi, Double.valueOf(0.8));
String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j);
if(retval >= 0){
String result = i + "," + j + ","+String.join(",", oldTokens);
jedis.select(2);
jedis.set(matchKey, result);
}
jedis.select(0);
jedis.del(matchKey);
return;
default:
break;
}
Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
m.match();
ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
ag.generate();
List<Action> actions = ag.getActions();
double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree);
String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1);
double diceSimilarity1 = m.diceSimilarity(oldTree, newTree);
String diceSimilarity = String.format("%1.2f", diceSimilarity1);
double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree);
String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1);
String editDistance = String.valueOf(actions.size());
String result = i + "," + j + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j);
if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
|| ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) {
// log.info("{} tagged to be similar" ,matchKey);
jedis.select(2);
jedis.set(matchKey, result);
}
jedis.select(0);
jedis.del(matchKey);
} catch (Exception e) {
log.debug("{} not comparable", pairName);
// e.printStackTrace();
}finally {
if (jedis != null) {
jedis.close();
}
}
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java
+44 -34
View File
@@ -4,6 +4,7 @@ import com.github.gumtreediff.actions.model.Delete;
import com.github.gumtreediff.actions.model.Insert;
import com.github.gumtreediff.actions.model.Move;
import com.github.gumtreediff.actions.model.Update;
import edu.lu.uni.serval.utils.FileHelper;
import java.io.*;
import java.util.List;
@@ -23,41 +24,41 @@ public class EDiffHunkParser extends EDiffParser {
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project,String actionType) {
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile);
if (actionSets.size() != 0) {
String folder= null;
boolean processActionSet = false;
// String folder= null;
boolean processActionSet = true;
// switch (actionType){
// case "ALL":
if(actionType.equals("ALL")){
folder = "/ALL/";
processActionSet = true;
}else if(actionType.equals("UPD") || actionType.equals("INS") || actionType.equals("DEL") || actionType.equals("MOV")|| actionType.equals("MIX")){
boolean isUPD = actionSets.stream().allMatch(p -> p.getAction() instanceof Update);
boolean isINS = actionSets.stream().allMatch(p -> p.getAction() instanceof Insert);
boolean isDEL = actionSets.stream().allMatch(p -> p.getAction() instanceof Delete);
boolean isMOV = actionSets.stream().allMatch(p -> p.getAction() instanceof Move);
if(isUPD){
folder = "/UPD/";
processActionSet = true;
}else if(isINS){
folder = "/INS/";
processActionSet = true;
}else if(isDEL){
folder = "/DEL/";
processActionSet = true;
}else if(isMOV){
folder = "/MOV/";
processActionSet = true;
}else{
folder = "/MIX/";
processActionSet = true;
}
}else{
processActionSet = false;
System.err.print(actionType + "not known");
}
// if(actionType.equals("ALL")){
// folder = "/ALL/";
// processActionSet = true;
// }else if(actionType.equals("UPD") || actionType.equals("INS") || actionType.equals("DEL") || actionType.equals("MOV")|| actionType.equals("MIX")){
// boolean isUPD = actionSets.stream().allMatch(p -> p.getAction() instanceof Update);
// boolean isINS = actionSets.stream().allMatch(p -> p.getAction() instanceof Insert);
// boolean isDEL = actionSets.stream().allMatch(p -> p.getAction() instanceof Delete);
// boolean isMOV = actionSets.stream().allMatch(p -> p.getAction() instanceof Move);
// if(isUPD){
// folder = "/UPD/";
// processActionSet = true;
// }else if(isINS){
// folder = "/INS/";
// processActionSet = true;
// }else if(isDEL){
// folder = "/DEL/";
// processActionSet = true;
// }else if(isMOV){
// folder = "/MOV/";
// processActionSet = true;
// }else{
// folder = "/MIX/";
// processActionSet = true;
// }
// }else{
//
//
// processActionSet = false;
// System.err.print(actionType + "not known");
//
// }
int hunkSet = 0;
@@ -70,12 +71,21 @@ public class EDiffHunkParser extends EDiffParser {
FileOutputStream f = null;
try {
String astNodeType = actionSet.getAstNodeType();
int size = actionSet.toString().split("\\n").length;
// int size = actionSet.strList.size();
String datasetName = project;
String[] split1 = diffentryFile.getParent().split(datasetName);
String root = split1[0];
String pj = split1[1].split("/")[1];
String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/" + pj + folder + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
File file = new File(root + "EnhancedASTDiff" + datasetName + "/"+astNodeType+"/"+String.valueOf(size)+"/");
file.mkdirs();
String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/"+astNodeType+"/"+String.valueOf(size)+"/" + pj +"_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
// String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/" + pj + folder + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
f = new FileOutputStream(new File(hunkTreeFileName));
ObjectOutputStream o = new ObjectOutputStream(f);
o.writeObject(actionSet);
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java
+1 -1
View File
@@ -168,7 +168,7 @@ public class HierarchicalActionSet implements Comparable<HierarchicalActionSet>,
return this.startPosition.compareTo(o.startPosition);//this.action.compareTo(o.action);
}
private List<String> strList = new ArrayList<>();
@Override
src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java
+53 -29
View File
@@ -8,11 +8,16 @@ import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser.akkaCompare;
@@ -47,44 +52,63 @@ public class AkkaTreeLoader {
String pairsIndexFile = pairsPath + "/"+ chunkName;
// String pairsIndexFile = pairsPath + "/"+ chunkName;
//0.txt.rdb
pairsIndexFile = pairsIndexFile.replace(chunk+".rdb",".index");
// pairsIndexFile = pairsIndexFile.replace(chunk+".rdb",".index");
File folder = new File(pairsPath);
File[] rootsFolder = folder.listFiles();
Pattern pattern = Pattern.compile(",");
String csvFile = pairsIndexFile;
try {
try (BufferedReader in = new BufferedReader(new FileReader(csvFile));){
Map<String,String> namefreq = in
.lines()
.map(x -> pattern.split(x))
.collect(HashMap::new, (map, x) ->
map.put(x[0], x[1]),
Map::putAll);
Stream<File> stream = Arrays.stream(rootsFolder);
List<File> roots = stream
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
Jedis inner = null;
for(File root:roots) {
File[] files = root.listFiles();
Stream<File> fileStream = Arrays.stream(files);
List<File> pairsIndexFiles = fileStream
.filter(x -> !x.getName().startsWith("."))
.filter(x -> x.getName().endsWith(".index"))
.collect(Collectors.toList());
for (File pairsIndexFile : pairsIndexFiles) {
Pattern pattern = Pattern.compile(",");
String csvFile = pairsIndexFile.getPath();
try {
inner = outerPool.getResource();
try (BufferedReader in = new BufferedReader(new FileReader(csvFile));) {
Map<String, String> namefreq = in
.lines()
.map(x -> pattern.split(x))
.collect(HashMap::new, (map, x) ->
map.put(root.getName()+"-"+pairsIndexFile.getName().split("\\.")[0]+"-"+x[0], x[1]),
Map::putAll);
Jedis inner = null;
try {
inner = outerPool.getResource();
for (Map.Entry<String, String> entry : namefreq.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
inner.select(1);
inner.set(key, value);
}
} finally {
if (inner != null) {
inner.close();
}
}
for (Map.Entry<String, String> entry : namefreq.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
inner.select(1);
inner.set(key,value);
}
}finally {
if (inner != null) {
inner.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
} catch (IOException e) {
e.printStackTrace();
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+9 -9
View File
@@ -50,15 +50,15 @@ public class EnhancedASTDiff {
// a
if (actionType.equals("ALL")) {
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/" + actionType);
} else {
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD");
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS");
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL");
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV");
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MIX");
}
// if (actionType.equals("ALL")) {
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/" + actionType);
// } else {
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD");
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS");
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL");
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV");
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MIX");
// }
}
switch (parallelism){
src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java
+29 -16
View File
@@ -30,36 +30,49 @@ public class ImportPairs2DB {
File folder = new File(csvInputPath);
File[] subFolders = folder.listFiles();
Stream<File> stream = Arrays.stream(subFolders);
List<File> pjs = stream
.filter(x -> x.getName().endsWith(chunkType))
List<File> roots = stream
// .filter(x -> x.getName().endsWith(chunkType))
.collect(Collectors.toList());
Integer portInt = Integer.valueOf(portInner);
for (File pj : pjs) {
String cmd = "bash " + dbDir + "/" + "startServer.sh" + " %s %s %s";
cmd = String.format(cmd, dbDir,"pairs.rdb", portInt);
log.info(cmd);
CallShell cs = new CallShell();
cs.runShell(cmd, portInner);
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd = String.format(cmd, dbDir,pj.getName() +".rdb", portInt);
log.info(cmd);
CallShell cs = new CallShell();
cs.runShell(cmd, portInner);
for (File root : roots) {
cmd = "bash "+datasetPath + "/redisSingleImport.sh" +" %s %s";
File[] files = root.listFiles();
Stream<File> fileStream = Arrays.stream(files);
List<File> pairs = fileStream
.filter(x -> x.getName().endsWith(chunkType))
.collect(Collectors.toList());
cmd = String.format(cmd, pj.getPath(), portInt);
log.info(cmd);
cs.runShell(cmd,portInner);
for(File pj:pairs) {
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
String stopServer2 = String.format(stopServer,portInt);
cs.runShell(stopServer2, portInner);
portInt++;
cmd = "bash " + datasetPath + "/redisSingleImport.sh" + " %s %s %s";
cmd = String.format(cmd, pj.getPath(), portInt,root.getName()+"-"+pj.getName().split("\\.")[0]);
log.info(cmd);
cs.runShell(cmd,portInner);
}
}
String stopServer = "bash " + dbDir + "/" + "stopServer.sh" + " %s";
String stopServer2 = String.format(stopServer, portInt);
cs.runShell(stopServer2, portInner);
// portInt++;
log.info(parameters);
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java
+89 -38
View File
@@ -20,10 +20,8 @@ import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -49,8 +47,8 @@ public class MultiThreadTreeLoaderCluster {
String cmd3;
cmd3 = "bash " + importScript +" %s %s";
// String cmd3;
// cmd3 = "bash " + importScript +" %s %s";
JedisPool jedisPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
@@ -65,41 +63,74 @@ public class MultiThreadTreeLoaderCluster {
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
for (File f:folders){
for (File f:folders) {
// if(f.getName().startsWith("cluster0")) {
File[] files = f.listFiles();
Stream<File> fileStream = Arrays.stream(files);
List<File> pairs = fileStream
.filter(x -> !x.getName().startsWith("."))
.filter(x -> !x.getName().endsWith(".index"))
.collect(Collectors.toList());
for (File pair : pairs) {
try (Jedis jedis = jedisPool.getResource()) {
// do operations with jedis resource
ScanParams sc = new ScanParams();
sc.count(cursor);
sc.match("pair_[0-9]*");
sc.match(f.getName()+"*");
log.info("Scanning");
ScanResult<String> scan = jedis.scan("0", sc);
int size = scan.getResult().size();
if (size == 0) {
String comd = String.format(cmd3,f.getPath(),portInner);
String cmd3 = "bash " + importScript + " %s %s %s";
cmd3 = String.format(cmd3, pair.getPath(), portInner,f.getName()+"-"+pair.getName().split("\\.")[0]);
// String comd = String.format(cmd3, f.getPath(), portInner);
// loadRedis(comd);
log.info("Importing {} pairs for cluster {}",size,f.getName());
cs.runShell(comd,portInner);
log.info("Importing {} pairs for cluster {}", size, f.getName());
cs.runShell(cmd3, portInner);
scan = jedis.scan("0", sc);
size = scan.getResult().size();
}
log.info("Scanned {} for cluster {}",String.valueOf(size),f.getName());
log.info("Scanned {} for cluster {}", String.valueOf(size), f.getName());
Pattern pattern = Pattern.compile(",");
String csvFile = pair.getPath();
csvFile = csvFile.replace("txt","index");
try (BufferedReader in = new BufferedReader(new FileReader(csvFile));) {
Map<String, String> namefreq = in
.lines()
.map(x -> pattern.split(x))
.collect(HashMap::new, (map, x) ->
map.put(f.getName()+"-"+pair.getName().split("\\.")[0]+"-"+x[0], x[1]),
Map::putAll);
for (Map.Entry<String, String> entry : namefreq.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
jedis.select(1);
jedis.set(key, value);
}
String clusterName = f.getName().replaceAll("[^0-9]+", "");
}
// String clusterName = f.getName().replaceAll("[^0-9]+", "");
//76
scan.getResult().parallelStream()
.forEach(m -> coreCompare(m, jedisPool, clusterName,outerPool,type));
.forEach(m -> coreCompare(m, jedisPool, f.getName(), outerPool, type));
jedis.save();
@@ -108,6 +139,7 @@ public class MultiThreadTreeLoaderCluster {
// }
}
}
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
String stopServer1 = String.format(stopServer,Integer.valueOf(portInner));
@@ -122,26 +154,40 @@ public class MultiThreadTreeLoaderCluster {
public static Pair<ITree,String> getTree(String firstValue, JedisPool outerPool,String type){
public static Pair<ITree,String> getTree(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool){
ITree tree = null;
Jedis inner = null;
String[] split2 = firstValue.split("/");
String fullFileName = split2[split2.length-1];
String[] split = fullFileName.split(".txt_");
String pureFileName = split[0];
String[] splitPJ = split[1].split("_");
String project = splitPJ[1];
String actionSetPosition = splitPJ[0];
Jedis outer = null;
// String[] split2 = firstValue.split("/");
//
// String fullFileName = split2[split2.length-1];
// String[] split = fullFileName.split(".txt_");
// String pureFileName = split[0];
// String[] splitPJ = split[1].split("_");
// String project = splitPJ[1];
// String actionSetPosition = splitPJ[0];
try {
inner = outerPool.getResource();
String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition;
String si= inner.get(filename);
inner = innerPool.getResource();
inner.select(1);
String dist2load = inner.get(prefix+"-"+firstValue);
outer = outerPool.getResource();
outer.select(0);
String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load);
Set<String> keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load);
if(keys.size() == 1) {
s = (String) keys.toArray()[0];
}else{
throw new Error("cok key");
}
// String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition;
String si= outer.get(s);
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si);
@@ -161,8 +207,11 @@ public class MultiThreadTreeLoaderCluster {
if (inner != null) {
inner.close();
}
if (outer != null) {
outer.close();
}
}
Pair<ITree, String> pair = new Pair<>(tree,project);
Pair<ITree, String> pair = new Pair<>(tree,null);
return pair;
@@ -197,19 +246,20 @@ public class MultiThreadTreeLoaderCluster {
catch (Exception e){
e.printStackTrace();
}
String firstValue = resultMap.get("0");
String secondValue = resultMap.get("1");
// String firstValue = resultMap.get("0");
// String secondValue = resultMap.get("1");
try {
Pair<ITree, String> oldPair = getTree(firstValue, outerPool,type);
Pair<ITree, String> newPair = getTree(secondValue, outerPool,type);
String keyName = split[0];
Pair<ITree, String> oldPair = getTree(keyName,i, outerPool,jedisPool);
Pair<ITree, String> newPair = getTree(keyName,j, outerPool,jedisPool);
ITree oldTree = oldPair.getValue0();
ITree newTree = newPair.getValue0();
String oldProject = oldPair.getValue1();
String newProject = newPair.getValue1();
// String oldProject = oldPair.getValue1();
// String newProject = newPair.getValue1();
@@ -231,23 +281,24 @@ public class MultiThreadTreeLoaderCluster {
String editDistance = String.valueOf(actions.size());
// jedis.select(1);
String result = resultMap.get("0") + "," + oldProject +"," + resultMap.get("1") + "," +newProject+ "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
String result = i+ "," + j + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
// jedis.set(resultKey, result);
String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j);
if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
|| ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) {
String matchKey = "match-"+clusterName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j);
jedis.select(1);
jedis.select(2);
jedis.set(matchKey, result);
}
jedis.select(0);
String pairKey = "pair_" + (String.valueOf(i)) + "_" + String.valueOf(j);
jedis.del(pairKey);
jedis.del(matchKey);
// log.info("Completed " + resultKey);
}catch (Exception e){
log.debug(e.toString() + " {}",(name));
e.printStackTrace();
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java
+42 -29
View File
@@ -38,42 +38,55 @@ public class StoreEDiffInCache {
File folder = new File(inputPath);
File[] subFolders = folder.listFiles();
Stream<File> stream = Arrays.stream(subFolders);
List<File> pjs = stream
List<File> roots = stream
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
List<String> workList = new ArrayList<String>();
File[] dumps;
for (File pj : pjs) {
String pjName = pj.getName();
File[] files = pj.listFiles();
Stream<File> fileStream = Arrays.stream(files);
List<File> fs;
// if (operation.equals("ALLOP")){
// fs= fileStream
// .filter(x -> x.getName().startsWith("UPD") ||
// x.getName().startsWith("INS") ||
// x.getName().startsWith("DEL") ||
// x.getName().startsWith("MOV"))
// .collect(Collectors.toList());
// File[] files1 = fs.get(0).listFiles();
// File[] files2 = fs.get(1).listFiles();
// File[] files3 = fs.get(2).listFiles();
// File[] files4 = fs.get(3).listFiles();
// dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new);
// }else{
fs = fileStream
.filter(x -> x.getName().startsWith(operation))
.collect(Collectors.toList());
dumps = fs.get(0).listFiles();
// }
for (File root : roots) {
String pjName = root.getName();
File[] sizes = root.listFiles();
Stream<File> sizesStream = Arrays.stream(sizes);
List<File> sizeFolders = sizesStream
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
for (File sf:sizeFolders){
File[] files = sf.listFiles();
int length = files.length;
if (length == 1)
continue;
Stream<File> fileStream = Arrays.stream(files);
List<File> fs;
// if (operation.equals("ALLOP")){
// fs= fileStream
// .filter(x -> x.getName().startsWith("UPD") ||
// x.getName().startsWith("INS") ||
// x.getName().startsWith("DEL") ||
// x.getName().startsWith("MOV"))
// .collect(Collectors.toList());
// File[] files1 = fs.get(0).listFiles();
// File[] files2 = fs.get(1).listFiles();
// File[] files3 = fs.get(2).listFiles();
// File[] files4 = fs.get(3).listFiles();
// dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new);
// }else{
// fs = fileStream
// .filter(x -> x.getName().startsWith(operation))
// .collect(Collectors.toList());
// dumps = fs.get(0).listFiles();
// }
for (File f : dumps) {
String name = f.getName();
for (File f : files) {
String name = f.getName();
String key = pjName + "/"+ sf.getName()+"/" + name;
String result = key +","+f.getPath();
workList.add(result);
}
String key = pjName + "/"+ operation+"/" + name;
String result = key +","+f.getPath();
workList.add(result);
}
}
src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java
+71
View File
@@ -0,0 +1,71 @@
package edu.lu.uni.serval.utils;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class ClusterToPattern {
public static void main(String[] args) throws IOException, ClassNotFoundException {
String shapesFolder = "/Users/anil.koyuncu/projects/fixminer-all/enhancedASTDiff/python/data/shapes";
String port = "6399";
JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000);
File folder = new File(shapesFolder);
File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles);
List<File> shapes = stream
.filter(x -> !x.getName().startsWith("."))
.filter(x -> !x.getName().endsWith(".pickle"))
.collect(Collectors.toList());
for(File shape:shapes){
File[] clusters = shape.listFiles();
Stream<File> fileStream = Arrays.stream(clusters);
List<File> fileList = fileStream.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
for(File cluster:fileList){
File[] files = cluster.listFiles();
File file = files[0];
String key = shape.getName() + "/*/"+file.getName();
Jedis outer = outerPool.getResource();
String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load);
Set<String> keys = outer.keys(key);
if (keys.size() == 1) {
s = (String) keys.toArray()[0];
} else {
throw new Error("cok key");
}
String si = outer.get(s);
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si);
String s1 = actionSet.toString();
File saveFile = new File(folder.getParent()+"/shapePatterns/" +shape.getName()+"/"+cluster.getName()+".pattern");
saveFile.getParentFile().mkdirs();
BufferedWriter writer = new BufferedWriter(new FileWriter(saveFile));
writer.write(s1);
writer.close();
}
}
}
}
src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java
+150 -7
View File
@@ -10,10 +10,7 @@ import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.*;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import java.util.stream.Collectors;
/**
@@ -130,7 +127,7 @@ public class EDiffHelper {
.collect(Collectors.toList());
}
public static ITree getSimpliedTree(String fn, JedisPool outerPool) {
public static ITree getSimpliedTree(String prefix, String fn, JedisPool outerPool) {
ITree tree = null;
Jedis inner = null;
@@ -140,9 +137,9 @@ public class EDiffHelper {
log.info("wait");
}
inner.select(1);
String dist2load = inner.get(fn);
String dist2load = inner.get(prefix+"-"+fn);
inner.select(0);
String s = inner.get(dist2load);
String s = inner.get(prefix.replace("-","/") +"/"+dist2load);
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s);
ITree parent = null;
@@ -164,6 +161,152 @@ public class EDiffHelper {
return tree;
}
public static ITree getShapes(String prefix, String fn, JedisPool outerPool) {
ITree tree = null;
Jedis inner = null;
try {
inner = outerPool.getResource();
while (!inner.ping().equals("PONG")){
log.info("wait");
}
inner.select(1);
String dist2load = inner.get(prefix+"-"+fn);
inner.select(0);
String s = inner.get(prefix.replace("-","/") +"/"+dist2load);
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s);
ITree parent = null;
ITree children =null;
TreeContext tc = new TreeContext();
tree = EDiffHelper.getASTTree(actionSet, parent, children,tc);
//tree.setParent(null);
tc.validate();
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}finally {
if (inner != null) {
inner.close();
}
}
return tree;
}
public static ITree getActions(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool) {
ITree tree = null;
Jedis inner = null;
Jedis outer = null;
try {
inner = innerPool.getResource();
inner.select(1);
String dist2load = inner.get(prefix + "-" + firstValue);
outer = outerPool.getResource();
outer.select(0);
String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load);
Set<String> keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load);
if (keys.size() == 1) {
s = (String) keys.toArray()[0];
} else {
throw new Error("cok key");
}
// String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition;
String si = outer.get(s);
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si);
ITree parent = null;
ITree children = null;
TreeContext tc = new TreeContext();
tree = EDiffHelper.getActionTree(actionSet, parent, children, tc);
//tree.setParent(null);
tc.validate();
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
} finally {
if (inner != null) {
inner.close();
}
if (outer != null) {
outer.close();
}
}
return tree;
}
public static ITree getTokens(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool) {
ITree tree = null;
Jedis inner = null;
Jedis outer = null;
try {
inner = innerPool.getResource();
inner.select(1);
String dist2load = inner.get(prefix + "-" + firstValue);
outer = outerPool.getResource();
outer.select(0);
String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load);
Set<String> keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load);
if (keys.size() == 1) {
s = (String) keys.toArray()[0];
} else {
throw new Error("cok key");
}
// String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition;
String si = outer.get(s);
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si);
ITree parent = null;
ITree children = null;
TreeContext tc = new TreeContext();
tree = EDiffHelper.getTokenTree(actionSet, parent, children, tc);
tree.setParent(null);
tc.validate();
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
} finally {
if (inner != null) {
inner.close();
}
if (outer != null) {
outer.close();
}
}
return tree;
}
public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){
int newType = 0;