azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changes

Browse Source
This commit is contained in:
ANIL KOYUNCU
2020-01-12 09:47:56 +01:00
parent 49a0aeba0b
commit 5be6024507
9 changed files with 583 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+10 -9
View File
@@ -22,8 +22,8 @@ public class Launcher {
Properties appProps = new Properties();
// String appConfigPath = "/Users/anil.koyuncu/projects/test/fixminerC/src/main/resource/app.properties";
String appConfigPath = args[0];
String appConfigPath = "/Users/anil.koyuncu/projects/fixminer/fixminer_source/src/main/resource/app.properties";
// String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath));
String portInner = appProps.getProperty("portInner","6380");
@@ -38,12 +38,12 @@ public class Launcher {
String redisPath = appProps.getProperty("redisPath","FORKJOIN");
String srcMLPath = appProps.getProperty("srcMLPath","FORKJOIN");
String parameter = args[2];
// String parameter = null;
// String parameter = "decl";
// String parameter = "L2";
String jobType = args[1];
// String jobType = "RICHEDITSCRIPT";
// String parameter = args[2];
String parameter = null;
// String parameter = "if";
// String parameter = "L1";
// String jobType = args[1];
String jobType = "RICHEDITSCRIPT";
// String jobType = "LOAD";
// String jobType = "COMPARE";
@@ -89,7 +89,8 @@ public class Launcher {
String compareDBName;
switch (parameter){
case "L1":
job = "shape";
// job = "shape";
job = "single";
compareDBName = "clusterl0-gumInputALL.rdb";
break;
case "L2":
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java
+52 -18
View File
@@ -42,7 +42,7 @@ public class CompareTrees {
// log.info(cmdInner);
// cs.runShell(cmdInner, portInner);
String numOfWorkers = "100000000";//args[4];
// String numOfWorkers = "100000000";//args[4];
String host = "localhost";//args[5];
// -Djava.util.concurrent.ForkJoinPool.common.parallelism=256
@@ -63,7 +63,11 @@ public class CompareTrees {
listOfPairs.stream().parallel().forEach(m->coreCompare(m, job,null, samePairs,errorPairs,filenames,outerPool));
try (Jedis jedis = outerPool.getResource()) {
jedis.select(2);
for (String samePair : samePairs) {
// jedis.hset("compare", errorPair, "1");
jedis.set(samePair, "1");
}
jedis.select(0);
// jedis.flushDB();
jedis.del("compare");
@@ -108,6 +112,32 @@ public class CompareTrees {
// jedis.srem("pairs",matchKey);
// JedisPool outerPool = null;
switch (treeType) {
case "single":
oldPair = EDiffHelper.getActions(keyName, i, outerPool, filenames);
newPair = EDiffHelper.getActions(keyName, j, outerPool, filenames);
ITree oldActionTree = oldPair.getValue0();
ITree newActionTree = newPair.getValue0();
HierarchicalActionSet oldProject = oldPair.getValue1();
HierarchicalActionSet newProject = newPair.getValue1();
ITree oldShapeTree = EDiffHelper.getShapeTree(oldProject);
ITree newShapeTree = EDiffHelper.getShapeTree(newProject);
ITree oldTargetTree = EDiffHelper.getTargets(oldProject);
ITree newTargetTree = EDiffHelper.getTargets(newProject);
if(oldShapeTree.toStaticHashString().equals(newShapeTree.toStaticHashString())){
if(oldActionTree.toStaticHashString().equals(newActionTree.toStaticHashString())){
if(oldTargetTree.toStaticHashString().equals(newTargetTree.toStaticHashString())){
samePairs.add(matchKey);
}
}
}
return;
// break;
case "shape":
oldTree = EDiffHelper.getShapes(keyName, i, outerPool,filenames);
newTree = EDiffHelper.getShapes(keyName, j, outerPool,filenames);
@@ -145,12 +175,12 @@ public class CompareTrees {
String result = i + "," + j + "," + String.join(",", oldTokens);
// jedis.select(2);
// jedis.set(matchKey, result);
try (Jedis jedis = innerPool.getResource()) {
// jedis.del(matchKey);
jedis.select(2);
jedis.set(matchKey, result);
}
// samePairs.add(matchKey);
// try (Jedis jedis = innerPool.getResource()) {
//// jedis.del(matchKey);
// jedis.select(2);
// jedis.set(matchKey, result);
// }
samePairs.add(matchKey);
// try (Jedis jedis = innerPool.getResource()) {
//// jedis.del(matchKey);
// jedis.select(2);
@@ -180,18 +210,22 @@ public class CompareTrees {
oldTree = EDiffHelper.getTargets(oldProject);
newTree = EDiffHelper.getTargets(newProject);
if (oldTree.toStaticHashString().equals(newTree.toStaticHashString())) {
try (Jedis jedis = innerPool.getResource()) {
// jedis.del(matchKey);
jedis.select(2);
jedis.set(matchKey, result);
}
samePairs.add(matchKey);
// try (Jedis jedis = innerPool.getResource()) {
//// jedis.del(matchKey);
// jedis.select(2);
// jedis.set(matchKey, result);
//
// }
}
} else {
try (Jedis jedis = innerPool.getResource()) {
// jedis.del(matchKey);
jedis.select(2);
jedis.set(matchKey, result);
}
samePairs.add(matchKey);
// try (Jedis jedis = innerPool.getResource()) {
//// jedis.del(matchKey);
// jedis.select(2);
// jedis.set(matchKey, result);
//
// }
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/DefaultKryoContext.java
+3 -6
View File
@@ -3,10 +3,7 @@ package edu.lu.uni.serval.fixminer.akka.ediff;
import java.io.ByteArrayOutputStream;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.kryo.io.UnsafeInput;
import com.esotericsoftware.kryo.io.UnsafeOutput;
import com.esotericsoftware.kryo.io.*;
import com.esotericsoftware.kryo.pool.KryoFactory;
import com.esotericsoftware.kryo.pool.KryoPool;
import org.objenesis.strategy.SerializingInstantiatorStrategy;
@@ -51,7 +48,7 @@ public class DefaultKryoContext implements KryoContext{
public byte[] serialze(Object obj, int bufferSize)
{
ByteArrayOutputStream base = new ByteArrayOutputStream();
Output output = new Output(base, bufferSize);
UnsafeMemoryOutput output = new UnsafeMemoryOutput(base, bufferSize);
Kryo kryo = pool.borrow();
@@ -75,7 +72,7 @@ public class DefaultKryoContext implements KryoContext{
Kryo kryo = pool.borrow();
Input input = new Input(serialized);
UnsafeMemoryInput input = new UnsafeMemoryInput(serialized);
obj = kryo.readObject(input, clazz);
pool.release(kryo);
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java
+2
View File
@@ -62,6 +62,8 @@ public class EDiffHunkParser extends EDiffParser {
f = new File(root+"dumps/"+key);
FileUtils.writeByteArrayToFile(f,EDiffHelper.kryoSerialize(actionSet));
// FileUtils.writeByteArrayToFile(f,EDiffHelper.commonsSerialize(actionSet));
// FileUtils.writeByteArrayToFile(f,actionSet.toString().getBytes());
// FileOutputStream fos = new FileOutputStream(f);
// ObjectOutputStream oos = new ObjectOutputStream(fos);
// oos.writeObject(EDiffHelper.kryoSerialize(actionSet));
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/FileHelper.java
+383
View File
@@ -0,0 +1,383 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
import java.io.BufferedInputStream;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class FileHelper {
/**
* @param filePath
*/
public static void createDirectory(String filePath) {
File file = new File(filePath);
if (file.exists()) {
deleteDirectory(filePath);
}
file.mkdirs();
}
public static void createFile(File file, String content) {
FileWriter writer = null;
BufferedWriter bw = null;
try {
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
if (!file.exists()) file.createNewFile();
writer = new FileWriter(file);
bw = new BufferedWriter(writer);
bw.write(content);
bw.flush();
} catch (IOException e) {
e.printStackTrace();
} finally {
close(bw);
close(writer);
}
}
public static void deleteDirectory(String dir) {
File file = new File(dir);
if (file.exists()) {
if (file.isDirectory()) {
File[] files = file.listFiles();
if (files.length > 0) {
for (File f : files) {
if (f.isFile()) {
deleteFile(f.getAbsolutePath());
} else {
deleteDirectory(f.getAbsolutePath());
}
}
}
file.delete();
} else {
deleteFile(dir);
}
}
}
public static void deleteFiles(String dir) {
File file = new File(dir);
if (file.exists()) {
if (file.isDirectory()) {
File[] files = file.listFiles();
if (files.length > 0) {
for (File f : files) {
if (f.isFile()) {
deleteFile(f.getAbsolutePath());
} else {
deleteFiles(f.getAbsolutePath());
}
}
}
} else {
deleteFile(dir);
}
}
}
public static void deleteFile(String fileName) {
File file = new File(fileName);
if (file.exists()) {
if (file.isFile()) {
file.delete();
} else {
deleteDirectory(fileName);
}
}
}
public static List<File> getAllDirectories(String filePath) {
return listAllDirectories(new File(filePath));
}
/**
* List all files in the directory.
*
* @param filePath
* @param type
* @return
*/
public static List<File> getAllFiles(String filePath, String type) {
return listAllFiles(new File(filePath), type);
}
public static List<File> getAllFilesInCurrentDiectory(String filePath, String type) {
return getAllFilesInCurrentDiectory(new File(filePath), type);
}
public static List<File> getAllFilesInCurrentDiectory(File directory, String type) {
List<File> fileList = new ArrayList<>();
if (!directory.exists()) {
return null;
}
File[] files = directory.listFiles();
for (File file : files) {
if (file.isFile()) {
if (file.toString().endsWith(type)) {
fileList.add(file);
}
}
}
return fileList;
}
public static String getFileName(String filePath) {
File file = new File(filePath);
if (file.exists()) {
return file.getName();
} else {
return null;
}
}
public static String getFileNameWithoutExtension(File file) {
if (file.exists()) {
String fileName = file.getName();
fileName = fileName.substring(0, fileName.lastIndexOf("."));
return fileName;
} else {
return null;
}
}
public static String getFileExtension(File file) {
String fileName = file.getName();
String extension = fileName.substring(fileName.lastIndexOf("."));
return extension;
}
public static String getFileParentPath(String filePath) {
File file = new File(filePath);
if (file.exists()) {
return file.getParent() + "/";
}
return "";
}
/**
* Check whether a file path is valid or not.
*
* @param path, file path.
* @return true, the file path is valid.
* false, the file path is invalid.
*/
public static boolean isValidPath(String path) {
File file = new File(path);
if (file.exists()) {
return true;
}
return false;
}
/**
* Recursively list all files in file.
*
* @param file
* @return
*/
private static List<File> listAllFiles(File file, String type) {
List<File> fileList = new ArrayList<>();
if (!file.exists()) {
return null;
}
File[] files = file.listFiles();
for (File f : files) {
if (f.isFile()) {
if (f.toString().endsWith(type)) {
fileList.add(f);
}
} else {
List<File> fl = listAllFiles(f, type);
if (fl != null && fl.size() > 0) {
fileList.addAll(fl);
}
}
}
return fileList;
}
/**
* Recursively list all directories in file.
*
* @param file
* @return
*/
private static List<File> listAllDirectories(File file) {
List<File> fileList = new ArrayList<>();
File[] files = file.listFiles();
for (File f : files) {
if (f.isDirectory()) {
fileList.add(f);
fileList.addAll(listAllDirectories(f));
}
}
return fileList;
}
public static void makeDirectory(String fileName) {
deleteFile(fileName);
File file = new File(fileName).getParentFile();
if (!file.exists()) {
file.mkdirs();
}
}
/**
* Read the content of a file.
*
* @param fileName
* @return String, the content of a file.
*/
public static String readFile(String fileName) {
return readFile(new File(fileName));
}
/**
* Read the content of a file.
*
* @param file
* @return String, the content of a file.
*/
public static String readFile(File file) {
byte[] input = null;
BufferedInputStream bis = null;
try {
bis = new BufferedInputStream(new FileInputStream(file));
input = new byte[bis.available()];
bis.read(input);
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
close(bis);
}
String sourceCode = null;
if (input != null) {
sourceCode = new String(input);
}
return sourceCode;
}
/**
* Output output into a file.
*
* @param fileName, output file name.
* @param data, output data.
* @param append, the output data will be appended previous data in the file or not.
*/
public static void outputToFile(String fileName, StringBuilder data, boolean append) {
outputToFile(fileName, data.toString(), append);
}
public static void outputToFile(File file, StringBuilder data, boolean append) {
outputToFile(file, data.toString(), append);
}
public static void outputToFile(String fileName, String data, boolean append) {
File file = new File(fileName);
outputToFile(file, data, append);
}
public static void outputToFile(File file, String data, boolean append) {
FileWriter writer = null;
BufferedWriter bw = null;
try {
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
if (!file.exists()) {
file.createNewFile();
}
writer = new FileWriter(file, append);
bw = new BufferedWriter(writer);
bw.write(data);
bw.flush();
} catch (IOException e) {
e.printStackTrace();
} finally {
close(bw);
close(writer);
}
}
private static void close(FileWriter writer) {
try {
if (writer != null) {
writer.close();
writer = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
private static void close(BufferedWriter bw) {
try {
if (bw != null) {
bw.close();
bw = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
private static void close(BufferedInputStream bis) {
try {
if (bis != null) {
bis.close();
bis = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static List<File> getAllSubDirectories(String fileName) {
File file = new File(fileName);
List<File> subDirectories = new ArrayList<>();
if (file.exists()) {
File[] files = file.listFiles();
for (File f : files) {
if (f.isDirectory()) {
subDirectories.add(f);
}
}
}
return subDirectories;
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java
+108 -3
View File
@@ -1,9 +1,14 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
import com.github.gumtreediff.gen.srcml.NodeMap_new;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.io.FileUtils;
import org.javatuples.Pair;
import org.junit.Test;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
@@ -13,6 +18,9 @@ import java.io.IOException;
import java.lang.instrument.Instrumentation;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class HunkParserTest {
@@ -89,18 +97,77 @@ public class HunkParserTest {
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "localhost",Integer.valueOf("6399"),20000000);
EDiffHunkParser parser = new EDiffHunkParser();
String root = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/linux/";
String filename = "bb67dd_0922c7_sound#soc#sof#intel#hda.c";
String root = "/Users/anil.koyuncu/projects/fixminer/gumInputLinux/linux/";
String filename = "43f8987_f596c8_drivers#acpi#nfit#core.c";
File revFile = new File(root + "revFiles/"+ filename);
File prevFile =new File(root + "prevFiles/prev_"+filename);
File diffFile = new File(root + "DiffEntries/"+filename+".txt");
String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML/src2srcml";
parser.parseFixPatterns(prevFile,revFile,diffFile,"gumInputLinux",outerPool,srcMLPath,"if");
String key = "if/3/linux_bb67dd_0922c7_sound#soc#sof#intel#hda.c.txt_0";
File file2load = new File("/Users/anil.koyuncu/projects/test/fixminer-core/python/data/dumps/"+ key);
File file2load = new File("/Users/anil.koyuncu/projects/fixminer/dumps/"+ key);
byte[] dump = FileUtils.readFileToByteArray(file2load);
// String line = FileHelper.readFile(file2load);
// ITree parent = null;
// ITree children = null;
//// if(line.isEmpty())
//// continue;
// TreeContext tc = new TreeContext();
// line = line.replace("\"", "");
// String[] split1 = line.split("\n");
// int length = split1.length;
// List<String> strings = new LinkedList<String>(Arrays.asList(split1));
// ITree treeFromString = null;
// int prevLev = 0;
// int childPosition = 0;
// for (String l:strings) {
// int level = 0;
// Pattern pattern = Pattern.compile("---");
// Matcher matcher = pattern.matcher(l);
// while (matcher.find())
// level++;
//
// l = l.replace("---","");
// l = l.trim();
//
// String[] split2;
// List<Integer> keysByValue;
// split2 = l.split(" ");
// keysByValue = NodeMap_new.getKeysByValue(NodeMap_new.map, split2[1]);
//
//
//
// if(level == 0){
// parent = tc.createTree(keysByValue.get(0), split2[0], null);
// tc.setRoot(parent);
//
// }else if (level > prevLev) {
// if (children == null) {
// children = tc.createTree(keysByValue.get(0), split2[0], null);
// children.setParentAndUpdateChildren(parent);
// } else {
//
// ITree tree = tc.createTree(keysByValue.get(0), split2[0], null);
// tree.setParentAndUpdateChildren(children);
// children = tree;
// }
// }else if (level == prevLev){
// ITree innerParent = children.getParent();
// children = tc.createTree(keysByValue.get(0), split2[0], null);
// children.setParentAndUpdateChildren(innerParent);
// }else {
// ITree innerParent = children.getParent();
// children = tc.createTree(keysByValue.get(0), split2[0], null);
// children.setParentAndUpdateChildren(innerParent.getParent());
// }
// prevLev = level;
// }
// tc.validate();
// parent.getLength();
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(dump);
// HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.commonsDeserialize(dump);
actionSet.toString();
@@ -122,5 +189,43 @@ public class HunkParserTest {
// }
// }
}
@Test
public void testCompare(){
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "localhost",Integer.valueOf("6399"),20000000);
Pair<ITree, HierarchicalActionSet> oldPair = null;
Pair<ITree, HierarchicalActionSet> newPair = null;
String matchKey = null;
ArrayList<String> samePairs = new ArrayList<>();
String keyName = "if-3";
String i = "2";
String j = "21";
HashMap<String, String> filenames = AkkaTreeParser.filenames(outerPool);
oldPair = EDiffHelper.getActions(keyName, i, outerPool, filenames);
newPair = EDiffHelper.getActions(keyName, j, outerPool, filenames);
ITree oldActionTree = oldPair.getValue0();
ITree newActionTree = newPair.getValue0();
HierarchicalActionSet oldProject = oldPair.getValue1();
HierarchicalActionSet newProject = newPair.getValue1();
ITree oldShapeTree = EDiffHelper.getShapeTree(oldProject);
ITree newShapeTree = EDiffHelper.getShapeTree(newProject);
ITree oldTargetTree = EDiffHelper.getTargets(oldProject);
ITree newTargetTree = EDiffHelper.getTargets(newProject);
String oldShape = oldShapeTree.toStaticHashString();
String newShape = newShapeTree.toStaticHashString();
if(oldShape.equals(newShape)){
if(oldActionTree.toStaticHashString().equals(newActionTree.toStaticHashString())){
if(oldTargetTree.toStaticHashString().equals(newTargetTree.toStaticHashString())){
samePairs.add(matchKey);
}
}
}
}
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+1 -1
View File
@@ -57,7 +57,7 @@ public class EnhancedASTDiff {
List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
// msgFiles = msgFiles.subList(0,10000);
// msgFiles = msgFiles.subList(0,1000);
if (msgFiles == null)
continue;
allMessageFiles.addAll(msgFiles);
src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java
+21 -6
View File
@@ -8,6 +8,7 @@ import com.github.gumtreediff.tree.TreeUtils;
import edu.lu.uni.serval.fixminer.akka.ediff.DefaultKryoContext;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.fixminer.akka.ediff.KryoContext;
import org.apache.commons.lang3.SerializationUtils;
import org.javatuples.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,6 +65,14 @@ public class EDiffHelper {
return baos.toByteArray();
}
public static byte[] commonsSerialize(Serializable o){
return SerializationUtils.serialize(o);
}
public static Object commonsDeserialize(byte[] data){
return SerializationUtils.deserialize(data);
}
public static byte[] kryoSerialize(Serializable o ){
KryoContext kryoContext = DefaultKryoContext.newKryoContextFactory();
@@ -279,12 +288,7 @@ public class EDiffHelper {
byte[] s = outer.hget("dump".getBytes(), key.getBytes());
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(s);
ITree parent = null;
ITree children = null;
TreeContext tc = new TreeContext();
tree = EDiffHelper.getASTTree(actionSet, parent, children, tc);
//tree.setParent(null);
tc.validate();
tree = getShapeTree(actionSet);
} catch (Exception e) {
e.printStackTrace();
@@ -294,6 +298,17 @@ public class EDiffHelper {
}
public static ITree getShapeTree(HierarchicalActionSet actionSet) {
ITree tree = null;
ITree parent = null;
ITree children = null;
TreeContext tc = new TreeContext();
tree = EDiffHelper.getASTTree(actionSet, parent, children, tc);
//tree.setParent(null);
tc.validate();
return tree;
}
public static ITree getTargets(HierarchicalActionSet actionSet) {
src/main/resource/app.properties
+3 -3
View File
@@ -5,10 +5,10 @@ portDumps = 6399
parallelism = AKKA
numOfWorkers = 14
inputPath = /Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux
redisPath = /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis
inputPath = /Users/anil.koyuncu/projects/fixminer/gumInputLinux
redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
srcMLPath= /Users/anil.koyuncu/Downloads/srcML/src2srcml
actionType =ALL
actionType = ALL
eDiffTimeout = 900
#ENHANCEDASTDIFF,CACHE,LEVEL1,LEVEL2,LEVEL3