azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[+] Multithreaded timeout

Browse Source
This commit is contained in:
Azalea (on HyDEV-Daisy)
2022-06-07 10:23:45 -04:00
parent 1a6ff6ca63
commit 6b924c5825
7 changed files with 236 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/EDiffHunkParser.java
+51 -11
View File
@@ -2,13 +2,19 @@ package edu.lu.uni.serval.richedit.ediff;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.Timer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.concurrent.TimeoutException;
/**
@@ -21,8 +27,33 @@ public class EDiffHunkParser extends EDiffParser
private static final Logger logger = LoggerFactory.getLogger(EDiffHunkParser.class);
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool, String srcMLPath, String hunkLimit, boolean isJava)
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project,
JedisPool innerPool, String srcMLPath, String hunkLimit, boolean isJava)
{
int hunkSize = Integer.parseInt(hunkLimit);
Path prevpath = Paths.get(prevFile.getPath());
Path revpath = Paths.get(prevFile.getPath());
Path diffpath = Paths.get(diffentryFile.getPath());
// try
// {
// if (((Files.size(prevpath) / 1024.0) > 7) || ((Files.size(revpath) / 1024.0) > 7) || ((Files.size(diffpath) / 1024.0) > 1)){
// // logger.info("bigfile {}, skipping...", prevFile);
// return;
// }
// }
// catch (IOException e)
// {
// throw new RuntimeException(e);
// }
// if (!prevFile.getPath().equals("/workspace/EECS-Research/data/0/patches/camel/prevFiles/prev_0da32d_a27076_components#camel-jclouds#src#main#java#org#apache#camel#component#jclouds#JcloudsBlobStoreProducer.java"))
// return;
// Timer timer = new Timer();
//
// logger.info("prevfile {}", prevFile);
// logger.info("revfile {}", revFile);
// logger.info("diffentryFile {}", diffentryFile);
try
{
String[] split1 = diffentryFile.getParent().split(project);
@@ -30,32 +61,41 @@ public class EDiffHunkParser extends EDiffParser
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath, isJava);
if (actionSets != null && actionSets.size() != 0)
{
if (actionSets == null || actionSets.size() == 0 || actionSets.size() > hunkSize) return;
int hunkSet = 0;
if (actionSets.size() <= Integer.parseInt(hunkLimit))
{
for (HierarchicalActionSet actionSet : actionSets)
{
String astNodeType = actionSet.getAstNodeType();
actionSet.toString();
int size = actionSet.getActionSize();
// actionSet.toString();
// Size limit (due to large trees generated for unknown reason)
int maxSize = 100;
int size = actionSet.getActionSizeRec(maxSize);
if (size > maxSize) continue;
// System.out.println(size);
// timer.log("getActionSizeRec, size = " + size);
// int size = actionSet.strList.size();
String key = astNodeType + "/" + size + "/" + pj + "_" + diffentryFile.getName() + "_" + hunkSet;
ITree targetTree = EDiffHelper.getTargets(actionSet, isJava);
// timer.log("getTargets");
ITree actionTree = EDiffHelper.getActionTrees(actionSet);
// timer.log("getActionTrees");
ITree shapeTree = EDiffHelper.getShapeTree(actionSet, isJava);
// timer.log("getShapeTree");
ITree tokenTree = EDiffHelper.getTokenTree(actionSet, isJava);
// timer.log("getTokenTree");
String tokens = EDiffHelper.getNames2(tokenTree);
// timer.log("getNames2");
try (Jedis inner = innerPool.getResource())
{
inner.hset("dump", key, actionSet.toString());
// inner.hset("dump", key, "");
inner.hset(key, "actionTree", actionTree.toStaticHashString());
inner.hset(key, "targetTree", targetTree.toStaticHashString());
inner.hset(key, "shapeTree", shapeTree.toStaticHashString());
inner.hset(key, "tokens", tokens);
}
// timer.log("dump");
hunkSet++;
}
@@ -63,9 +103,9 @@ public class EDiffHunkParser extends EDiffParser
{
inner.hset("diffEntry", pj + "_" + diffentryFile.getName(), "1");
}
// logger.info("Finished processing {}", prevFile);
}
}
}
catch (Exception e)
{
logger.error("error", e);
richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/EDiffParser.java
-10
View File
@@ -41,22 +41,14 @@ public class EDiffParser extends Parser
public List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree2(File prevFile, File revFile, String srcMLPath, boolean isJava)
{
List<HierarchicalActionSet> actionSets = new ArrayList<>();
// GumTree results
// boolean isJava =false;
List<Action> gumTreeResults = null;
if (isJava)
{
// if (revFile.getName().endsWith(".c") & prevFile.getName().endsWith(".c") || revFile.getName().endsWith(".h") & prevFile.getName().endsWith(".h")){
// gumTreeResults = new GumTreeComparer().compareCFilesWithGumTree(prevFile, revFile);
gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
}
else
{
gumTreeResults = new GumTreeCComparer().compareCFilesWithGumTree(prevFile, revFile, srcMLPath);
}
if (gumTreeResults == null)
{
@@ -110,6 +102,4 @@ public class EDiffParser extends Parser
// public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool) {
//
// }
}
richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/HierarchicalActionSet.java
+31 -24
View File
@@ -2,9 +2,12 @@ package edu.lu.uni.serval.richedit.ediff;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.tree.ITree;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
/**
@@ -14,6 +17,7 @@ import java.util.List;
*/
public class HierarchicalActionSet implements Comparable<HierarchicalActionSet>, Serializable
{
private static final Logger log = LoggerFactory.getLogger(EDiffHunkParser.class);
private String astNodeType;
@@ -212,46 +216,51 @@ public class HierarchicalActionSet implements Comparable<HierarchicalActionSet>,
return this.startPosition.compareTo(o.startPosition);//this.action.compareTo(o.action);
}
private final List<String> strList = new ArrayList<>();
final List<String> strList = new ArrayList<>();
public int getActionSize()
// public int getActionSize()
// {
// return strList.size();
// }
private Integer size = null;
public int getActionSizeRec(int maxSize)
{
return strList.size();
if (this.size != null) return this.size;
int size = 1;
for (HierarchicalActionSet s : subActions)
{
size += s.getActionSizeRec(maxSize);
if (size > maxSize) return maxSize + 1;
}
return this.size = size;
}
@Override
public String toString()
{
String str = actionString;
// log.info("Calling toString on {}", actionString);
if (strList.size() == 0)
{
strList.add(str);
strList.add(actionString);
// TODO: Can we use unique subActions instead of full subActions with many duplicates?
// List<HierarchicalActionSet> usedSubActions = subActions.size() > 20 ? new ArrayList<>(new HashSet<>(subActions)) : subActions;
for (HierarchicalActionSet actionSet : subActions)
{
actionSet.toString();
List<String> strList1 = actionSet.strList;
for (String str1 : strList1)
{
strList.add("---" + str1);
}
}
}
else
{
strList.clear();
strList.add(str);
for (HierarchicalActionSet actionSet : subActions)
{
actionSet.toString();
List<String> strList1 = actionSet.strList;
for (String str1 : strList1)
for (String str1 : actionSet.strList)
{
strList.add("---" + str1);
}
}
}
str = "";
String str = "";
for (String str1 : strList)
{
str += str1 + "\n";
@@ -259,6 +268,4 @@ public class HierarchicalActionSet implements Comparable<HierarchicalActionSet>,
return str;
}
}
richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/CompareTrees.java
+49 -5
View File
@@ -1,5 +1,6 @@
package edu.lu.uni.serval.richedit.jobs;
import edu.lu.uni.serval.richedit.ediff.EDiffHunkParser;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
@@ -13,6 +14,10 @@ import redis.clients.jedis.JedisPool;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -38,12 +43,51 @@ public class CompareTrees
compare = inner.scard("compare");
}
IntStream stream = IntStream.range(0, compare.intValue());
ProgressBar.wrap(stream.parallel(), "Task").forEach(m ->
{
newCoreCompare(job, errorPairs, filenames, outerPool);
ExecutorService executor = Executors.newScheduledThreadPool(36);
stream.forEach(m ->
{
String id = UUID.randomUUID().toString();
Future<?> future = executor.submit(() ->
{
Thread cur = Thread.currentThread();
// Monitor thread
Thread monitor = new Thread(() ->
{
try
{
Thread.sleep(60 * 1000);
cur.stop();
log.info("Cancelled {} because of timeout", cur.getName());
// Update progress bar
// pb.step();
}
);
catch (InterruptedException ignored) {}
});
monitor.start();
// Parse
newCoreCompare(job, errorPairs, filenames, outerPool);
// Update progress bar
// pb.step();
System.out.print(".");
// Stop monitoring thread
monitor.interrupt();
});
});
executor.shutdown();
}
// ProgressBar.wrap(stream.parallel(), "Task").forEach(m ->
// {
// newCoreCompare(job, errorPairs, filenames, outerPool);
// }
// );
log.info("End process");
}
@@ -74,7 +118,7 @@ public class CompareTrees
}
Map<String, String> oldTreeString = EDiffHelper.getTreeString(keyName, i, outerPool, filenames);
Map<String, String> newTreeString = EDiffHelper.getTreeString(keyName, j, outerPool, filenames);
//log.info("start switching");
switch (treeType)
{
case "single":
richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/EnhancedASTDiff.java
+60 -13
View File
@@ -2,7 +2,6 @@ package edu.lu.uni.serval.richedit.jobs;
import edu.lu.uni.serval.richedit.ediff.EDiffHunkParser;
import edu.lu.uni.serval.richedit.ediff.MessageFile;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import me.tongfei.progressbar.ProgressBar;
@@ -12,10 +11,8 @@ import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.*;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -39,7 +36,10 @@ public class EnhancedASTDiff
// Find patches
File folder = new File(inputPath);
File[] listOfFiles = folder.listFiles();
if (listOfFiles == null) throw new Exception("No projects found, please verify the projects in the input path");
if (listOfFiles == null)
{
throw new Exception("No projects found, please verify the projects in the input path");
}
Stream<File> stream = Arrays.stream(listOfFiles);
List<File> folders;
if (projectList.length == 1 && projectList[0].equals("ALL"))
@@ -92,22 +92,63 @@ public class EnhancedASTDiff
{
log.info("{} files already process ...", diffEntry.size());
allMessageFiles = allMessageFiles.stream().filter(f -> !diffEntry.containsKey(f.getProject() + "_" + f.getDiffEntryFile().getName())).collect(Collectors.toList());
log.info("{} files to process ...",allMessageFiles.size());
log.info("{} files to process ...", allMessageFiles.size());
}
ProgressBar.wrap(allMessageFiles.stream().parallel(), "Task").forEach(m ->
// try (ProgressBar pb = new ProgressBar("Task", allMessageFiles.size()))
{
ExecutorService executor = Executors.newScheduledThreadPool(36);
allMessageFiles.forEach(m ->
{
String id = UUID.randomUUID().toString();
Future<?> future = executor.submit(() ->
{
Thread cur = Thread.currentThread();
// Monitor thread
Thread monitor = new Thread(() ->
{
try
{
Thread.sleep(60 * 1000);
cur.stop();
log.info("Cancelled {} because of timeout", cur.getName());
// Update progress bar
// pb.step();
}
catch (InterruptedException ignored) {}
});
monitor.start();
// Parse
EDiffHunkParser parser = new EDiffHunkParser();
parser.parseFixPatterns(m.getPrevFile(), m.getRevFile(), m.getDiffEntryFile(), project, innerPool, srcMLPath, hunkLimit, isJava);
// Update progress bar
// pb.step();
System.out.print(".");
// Stop monitoring thread
monitor.interrupt();
});
});
executor.shutdown();
}
);
// ProgressBar.wrap(allMessageFiles.stream().parallel(), "Task").forEach(m ->
// {
// EDiffHunkParser parser = new EDiffHunkParser();
// parser.parseFixPatterns(m.getPrevFile(), m.getRevFile(), m.getDiffEntryFile(), project, innerPool, srcMLPath, hunkLimit, isJava);
// }
// );
}
private static List<MessageFile> getMessageFiles(String gumTreeInput, String datasetName, String patchSize, boolean isJava)
{
String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
File revFilesPath = new File(inputPath + "revFiles/");
File revFilesPath = new File(gumTreeInput + "revFiles/");
log.info(revFilesPath.getPath());
File[] revFiles = revFilesPath.listFiles();
if (revFiles != null)
@@ -128,7 +169,10 @@ public class EnhancedASTDiff
{
count++;
}
if (count >= Integer.valueOf(patchSize)) continue;
if (count >= Integer.parseInt(patchSize))
{
continue;
}
String[] split1 = diffentryFile.getParent().split(datasetName);
String root = split1[0];
String pj = split1[1].split("/")[1];
@@ -140,6 +184,9 @@ public class EnhancedASTDiff
return msgFiles;
}
else return null;
else
{
return null;
}
}
}
richedit/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java
+3 -20
View File
@@ -12,40 +12,23 @@ public class ClusterToPattern
public static void main(String port, String redisPath, String dumpsName, String parameter) throws Exception
{
CallShell cs = new CallShell();
String cmd = "bash " + redisPath + "/" + "startServer.sh" + " %s %s %s";
cmd = String.format(cmd, redisPath, dumpsName, Integer.valueOf(port));
log.trace(cmd);
CallShell.runShell(cmd, port);
String host = "localhost";//args[5];
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host, Integer.valueOf(port), 20000000);
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host, Integer.parseInt(port), 20000000);
String export = export(parameter, outerPool);
System.out.println(export);
}
private static String export(String filename, JedisPool outerPool)
{
try (Jedis outer = outerPool.getResource())
{
while (!outer.ping().equals("PONG"))
{
log.info("wait");
while (!outer.ping().equals("PONG")) log.info("wait");
return outer.hget("dump", filename);
}
// byte[] s = outer.hget("dump".getBytes(), filename.getBytes());
// HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(s);
// if (actionSet == null){
// throw new Error(filename +" not found");
// }
String s1 = outer.hget("dump", filename);
// outer.close();
return s1;
}
}
}
richedit/src/main/java/edu/lu/uni/serval/utils/Timer.java
+18
View File
@@ -0,0 +1,18 @@
package edu.lu.uni.serval.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Timer
{
private static final Logger logger = LoggerFactory.getLogger(Timer.class);
long start = System.nanoTime();
public void log(String msg)
{
long elapse = (long) ((System.nanoTime() - start) / 1e6);
logger.info("{}ms: {}", elapse, msg);
start = System.nanoTime();
}
}