azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

enabled back token iteration

Browse Source
This commit is contained in:
mimic
2020-06-09 14:51:19 +02:00
parent aa2a17bc4a
commit a26275cf4a
12 changed files with 256 additions and 464 deletions
Download Patch File Download Diff File Expand all files Collapse all files
richedit/src/main/java/edu/lu/uni/serval/Launcher.java
+28 -28
View File
@@ -73,18 +73,18 @@ public class Launcher {
String srcMLPath = (String) fixminer.get("srcMLPath");
// String parameter = args[2];
String parameter = "L1";
// String parameter = "L1";
String jobType = args[1];
// String jobType = "RICHEDITSCRIPT";
// String jobType = "COMPARE";
// jobType = "COMPARE";
mainLaunch( numOfWorkers, jobType, portDumps,projectType,input,redisPath,parameter, srcMLPath,hunkLimit,projectList,patchSize);
mainLaunch( numOfWorkers, jobType, portDumps,projectType,input,redisPath, srcMLPath,hunkLimit,projectList,patchSize);
}
public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String projectType, String input, String redisPath,String parameter,String srcMLPath,String hunkLimit,String[] projectList,String patchSize){
public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String projectType, String input, String redisPath,String srcMLPath,String hunkLimit,String[] projectList,String patchSize){
String dbDir;
@@ -100,36 +100,36 @@ public class Launcher {
try {
switch (jobType) {
case "RICHEDITSCRIPT":
EnhancedASTDiff.main(gumInput, portDumps, dbDir, dumpsName, srcMLPath,parameter,hunkLimit,projectList,patchSize,projectType);
EnhancedASTDiff.main(gumInput, portDumps, dbDir, dumpsName, srcMLPath,hunkLimit,projectList,patchSize,projectType);
break;
case "COMPARE":
String job;
String compareDBName;
switch (parameter){
case "L1":
// job = "shape";
job = "single";
compareDBName = "clusterl0-gumInputALL.rdb";
break;
case "L2":
job = "action";
compareDBName = "clusterl1-gumInputALL.rdb";
break;
case "L3":
job = "token";
compareDBName = "clusterl2-gumInputALL.rdb";
break;
default:
throw new Error("unknown level please specify L1,L2,L3");
}
// String job;
// String compareDBName;
// switch (parameter){
// case "L1":
//// job = "shape";
// job = "single";
// compareDBName = "clusterl0-gumInputALL.rdb";
// break;
// case "L2":
// job = "action";
// compareDBName = "clusterl1-gumInputALL.rdb";
// break;
// case "L3":
// job = "token";
// compareDBName = "clusterl2-gumInputALL.rdb";
// break;
// default:
// throw new Error("unknown level please specify L1,L2,L3");
// }
CompareTrees.main(redisPath, portDumps,dumpsName, job,numOfWorkers);
break;
case "PATTERN":
ClusterToPattern.main(portDumps,redisPath, dumpsName, parameter);
CompareTrees.main(redisPath, portDumps,dumpsName, numOfWorkers);
break;
// case "PATTERN":
// ClusterToPattern.main(portDumps,redisPath, dumpsName, parameter);
// break;
default:
throw new Error("unknown Job");
richedit/src/main/java/edu/lu/uni/serval/richedit/Launcher2.java
-134
View File
@@ -1,134 +0,0 @@
package edu.lu.uni.serval.richedit;
import edu.lu.uni.serval.richedit.jobs.CompareTrees;
import edu.lu.uni.serval.richedit.jobs.EnhancedASTDiff;
import edu.lu.uni.serval.utils.ClusterToPattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Properties;
/**
* Created by anilkoyuncu on 14/04/2018.
*/
public class Launcher2 {
private static Logger log = LoggerFactory.getLogger(Launcher2.class);
public static void main(String[] args) throws IOException {
Properties appProps = new Properties();
String hostname = "Unknown";
try
{
InetAddress addr;
addr = InetAddress.getLocalHost();
hostname = addr.getHostName();
}
catch (UnknownHostException ex)
{
System.out.println("Hostname can not be resolved");
}
String appConfigPath;
if (hostname.equals("Unknown")){
appConfigPath = "src/main/resource/app.properties";
}
else{
appConfigPath = "src/main/resource/"+hostname+".app.properties";
}
// String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath));
String numOfWorkers = appProps.getProperty("numOfWorkers", "10");
String portDumps = appProps.getProperty("portDumps","6399");
String projectType = appProps.getProperty("projectType","java");
String hunkLimit = appProps.getProperty("hunkLimit","10");
String patchSize = appProps.getProperty("patchSize","50");
String projectL = appProps.getProperty("projectList","");
String[] projectList = projectL.split(",");
String input = appProps.getProperty("inputPath","FORKJOIN");
String redisPath = appProps.getProperty("redisPath","FORKJOIN");
String srcMLPath = appProps.getProperty("srcMLPath","FORKJOIN");
// String parameter = args[2];
String parameter = "L1";
// String jobType = args[1];
// String jobType = "RICHEDITSCRIPT";
String jobType = "COMPARE";
mainLaunch( numOfWorkers, jobType, portDumps,projectType,input,redisPath,parameter, srcMLPath,hunkLimit,projectList,patchSize);
}
public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String projectType, String input, String redisPath,String parameter,String srcMLPath,String hunkLimit,String[] projectList,String patchSize){
String dbDir;
String dumpsName;
String gumInput;
dumpsName = "dumps-"+projectType+".rdb";
gumInput = input;
dbDir = redisPath;
try {
switch (jobType) {
case "RICHEDITSCRIPT":
EnhancedASTDiff.main(gumInput, portDumps, dbDir, dumpsName, srcMLPath,parameter,hunkLimit,projectList,patchSize,projectType);
break;
case "COMPARE":
String job;
String compareDBName;
switch (parameter){
case "L1":
// job = "shape";
job = "single";
compareDBName = "clusterl0-gumInputALL.rdb";
break;
case "L2":
job = "action";
compareDBName = "clusterl1-gumInputALL.rdb";
break;
case "L3":
job = "token";
compareDBName = "clusterl2-gumInputALL.rdb";
break;
default:
throw new Error("unknown level please specify L1,L2,L3");
}
CompareTrees.main(redisPath, portDumps,dumpsName, job,numOfWorkers);
break;
case "PATTERN":
ClusterToPattern.main(portDumps,redisPath, dumpsName, parameter);
break;
default:
throw new Error("unknown Job");
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/EDiffHunkParser.java
+4
View File
@@ -62,12 +62,16 @@ public class EDiffHunkParser extends EDiffParser {
ITree targetTree = EDiffHelper.getTargets(actionSet, isJava);
ITree actionTree = EDiffHelper.getActionTrees(actionSet);
ITree shapeTree = EDiffHelper.getShapeTree(actionSet, isJava);
ITree tokenTree = EDiffHelper.getTokenTree(actionSet,isJava);
String tokens = EDiffHelper.getNames2(tokenTree);
// EDiffHelper.getTokenTree(actionSet, parent, children, tc);
try (Jedis inner = innerPool.getResource()) {
inner.hset("dump", key, actionSet.toString());
inner.hset(key, "actionTree", actionTree.toStaticHashString());
inner.hset(key, "targetTree", targetTree.toStaticHashString());
inner.hset(key, "shapeTree", shapeTree.toStaticHashString());
inner.hset(key, "tokens", tokens);
}
// File f = new File(root+"dumps/"+astNodeType+"/"+String.valueOf(size)+"/");
// f.mkdirs();
richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/CompareTrees.java
+73 -7
View File
@@ -4,6 +4,7 @@ import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import me.tongfei.progressbar.ProgressBar;
import org.apache.commons.text.similarity.JaroWinklerDistance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
@@ -28,7 +29,7 @@ public class CompareTrees {
private static Logger log = LoggerFactory.getLogger(CompareTrees.class);
public static void main(String redisPath, String portDumps, String dumpsName, String job,String numOfWorkers) throws Exception {
public static void main(String redisPath, String portDumps, String dumpsName, String numOfWorkers) throws Exception {
// shape /Users/anil.koyuncu/projects/test/richedit-core/python/data/redis ALLdumps-gumInput.rdb clusterl0-gumInputALL.rdb /Users/anil.koyuncu/projects/test/richedit-core/python/data/richEditScript
@@ -55,6 +56,7 @@ public class CompareTrees {
// List<String> listOfPairs = AkkaTreeParser.getMessages(innerPool,Integer.valueOf(numOfWorkers));
HashMap<String, String> filenames = getFilenames(outerPool);
String job= getLevel(outerPool);
// List<String> listOfPairs = AkkaTreeParser.files2compare(outerPool);
@@ -69,12 +71,13 @@ public class CompareTrees {
}
IntStream stream = IntStream.range(0, compare.intValue());
String finalJob = job;
ProgressBar.wrap(stream.
parallel(),"Task").
forEach(m ->
{
newCoreCompare(job, errorPairs, filenames, outerPool);
newCoreCompare(finalJob, errorPairs, filenames, outerPool);
}
);
@@ -164,14 +167,16 @@ public class CompareTrees {
String keyName = split[0];
matchKey = keyName + "/" + (String.valueOf(i)) + "/" + String.valueOf(j);
if (matchKey == null){
return false;
}
Map<String, String> oldTreeString = EDiffHelper.getTreeString(keyName, i, outerPool, filenames);
Map<String, String> newTreeString = EDiffHelper.getTreeString(keyName, j, outerPool, filenames);
switch (treeType) {
case "single":
if (matchKey == null){
return false;
}
Map<String, String> oldTreeString = EDiffHelper.getTreeString(keyName, i, outerPool, filenames);
Map<String, String> newTreeString = EDiffHelper.getTreeString(keyName, j, outerPool, filenames);
String oldShapeTree =oldTreeString.get("shapeTree");
String newShapeTree =newTreeString.get("shapeTree");
@@ -196,6 +201,35 @@ public class CompareTrees {
}
}
return true;
case "token":
String oldTokens = oldTreeString.get("tokens");
String newTokens = newTreeString.get("tokens");
// EDiffHelper.getTokens(keyName, i, outerPool, filenames);
// newTree = EDiffHelper.getTokens(keyName, j, outerPool, innerPool);
// String oldTokens = EDiffHelper.getNames2(oldTree);
// String newTokens = EDiffHelper.getNames2(newTree);
//
JaroWinklerDistance jwd = new JaroWinklerDistance();
//
//
Double overallSimi = Double.valueOf(0);
//
if (!(oldTokens.trim().isEmpty() || newTokens.trim().isEmpty())) {
overallSimi = jwd.apply(oldTokens, newTokens);
}
int retval = Double.compare(overallSimi, Double.valueOf(1));
if (retval >= 0) {
try (Jedis jedis = outerPool.getResource()) {
jedis.select(3);
jedis.set(matchKey, "1");
}
}
return true;
default:
return true;
// break;
@@ -212,6 +246,35 @@ public class CompareTrees {
}
public static String getLevel(JedisPool innerPool){
HashMap<String, String> fileMap =new HashMap<String, String>();
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
inner.select(1);
String level = inner.get("level");
switch (level){
case "l1":
return "single";
case "l2":
return "token";
default:
return "";
}
}
}
public static HashMap<String, String> getFilenames(JedisPool innerPool){
@@ -244,4 +307,7 @@ public class CompareTrees {
}
richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/EnhancedASTDiff.java
+1 -1
View File
@@ -26,7 +26,7 @@ public class EnhancedASTDiff {
private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class);
public static void main(String inputPath, String portInner, String dbDir, String chunkName,String srcMLPath,String parameter,String hunkLimit,String[] projectList,String patchSize,String projectType) throws Exception {
public static void main(String inputPath, String portInner, String dbDir, String chunkName,String srcMLPath,String hunkLimit,String[] projectList,String patchSize,String projectType) throws Exception {
String parameters = String.format("\nInput path %s", inputPath);
richedit/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java
+26 -6
View File
@@ -71,15 +71,22 @@ public class EDiffHelper {
}
public static ITree getTokenTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){
public static ITree getTokenTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc,boolean isJava){
int newType = 0;
String astNodeType = actionSet.getAstNodeType();
String label = actionSet.getAction().toString();
Map<Integer, String> nodeMap;
if(isJava){
nodeMap = ASTNodeMap.map;
}else{
nodeMap = NodeMap_new.map;
}
// List<Integer> keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType);
List<Integer> keysByValue = getKeysByValue(NodeMap_new.map, astNodeType);
List<Integer> keysByValue = getKeysByValue(nodeMap, astNodeType);
if(keysByValue.size() != 1){
log.error("More than 1");
@@ -102,7 +109,7 @@ public class EDiffHelper {
if(actionSet.getParent() == null){
children = parent;
}
getTokenTree(subAction,children,null,tc);
getTokenTree(subAction,children,null,tc,isJava);
}
@@ -307,6 +314,17 @@ public class EDiffHelper {
return tree;
}
public static ITree getTokenTree(HierarchicalActionSet actionSet,boolean isJava) {
ITree tree = null;
ITree parent = null;
ITree children = null;
TreeContext tc = new TreeContext();
tree = EDiffHelper.getTokenTree(actionSet, parent, children, tc,isJava);
//tree.setParent(null);
tc.validate();
return tree;
}
public static ITree getTargets(HierarchicalActionSet actionSet,boolean isJava) {
@@ -420,13 +438,15 @@ public class EDiffHelper {
// String[] split = prefix.split("-");
// String key = split[0] + "/"+split[1]+"/" + dist2load;
//
// byte[] s = outer.hget("dump".getBytes(), key.getBytes());
// actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(s);
// String s = outer.hget("dump", key);
// actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s);
//
//// actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(s);
//
// ITree parent = null;
// ITree children = null;
// TreeContext tc = new TreeContext();
// tree = EDiffHelper.getTokenTree(actionSet, parent, children, tc);
// tree = EDiffHelper.getTokenTree(actionSet, parent, children, tc,is);
// tree.setParent(null);
// tc.validate();
//// getLeaves(tree);