From 6b924c582529a7492b3250cebf1fe2464585359b Mon Sep 17 00:00:00 2001 From: "Azalea (on HyDEV-Daisy)" Date: Tue, 7 Jun 2022 10:23:45 -0400 Subject: [PATCH] [+] Multithreaded timeout --- .../richedit/ediff/EDiffHunkParser.java | 104 ++++++++++++------ .../serval/richedit/ediff/EDiffParser.java | 10 -- .../richedit/ediff/HierarchicalActionSet.java | 55 +++++---- .../serval/richedit/jobs/CompareTrees.java | 54 ++++++++- .../serval/richedit/jobs/EnhancedASTDiff.java | 79 ++++++++++--- .../lu/uni/serval/utils/ClusterToPattern.java | 23 +--- .../java/edu/lu/uni/serval/utils/Timer.java | 18 +++ 7 files changed, 236 insertions(+), 107 deletions(-) create mode 100644 richedit/src/main/java/edu/lu/uni/serval/utils/Timer.java diff --git a/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/EDiffHunkParser.java b/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/EDiffHunkParser.java index c03c486..a43fa31 100755 --- a/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/EDiffHunkParser.java +++ b/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/EDiffHunkParser.java @@ -2,13 +2,19 @@ package edu.lu.uni.serval.richedit.ediff; import com.github.gumtreediff.tree.ITree; import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.Timer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; +import java.util.concurrent.TimeoutException; /** @@ -21,8 +27,33 @@ public class EDiffHunkParser extends EDiffParser private static final Logger logger = LoggerFactory.getLogger(EDiffHunkParser.class); @Override - public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool, String srcMLPath, String hunkLimit, boolean isJava) + public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, + JedisPool innerPool, String srcMLPath, String hunkLimit, boolean isJava) { + int hunkSize = Integer.parseInt(hunkLimit); + Path prevpath = Paths.get(prevFile.getPath()); + Path revpath = Paths.get(prevFile.getPath()); + Path diffpath = Paths.get(diffentryFile.getPath()); +// try +// { +// if (((Files.size(prevpath) / 1024.0) > 7) || ((Files.size(revpath) / 1024.0) > 7) || ((Files.size(diffpath) / 1024.0) > 1)){ +// // logger.info("bigfile {}, skipping...", prevFile); +// return; +// } +// } +// catch (IOException e) +// { +// throw new RuntimeException(e); +// } + // if (!prevFile.getPath().equals("/workspace/EECS-Research/data/0/patches/camel/prevFiles/prev_0da32d_a27076_components#camel-jclouds#src#main#java#org#apache#camel#component#jclouds#JcloudsBlobStoreProducer.java")) +// return; + +// Timer timer = new Timer(); +// +// logger.info("prevfile {}", prevFile); +// logger.info("revfile {}", revFile); +// logger.info("diffentryFile {}", diffentryFile); + try { String[] split1 = diffentryFile.getParent().split(project); @@ -30,42 +61,51 @@ public class EDiffHunkParser extends EDiffParser List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath, isJava); - if (actionSets != null && actionSets.size() != 0) + if (actionSets == null || actionSets.size() == 0 || actionSets.size() > hunkSize) return; + + int hunkSet = 0; + for (HierarchicalActionSet actionSet : actionSets) { - int hunkSet = 0; - if (actionSets.size() <= Integer.parseInt(hunkLimit)) + String astNodeType = actionSet.getAstNodeType(); +// actionSet.toString(); + // Size limit (due to large trees generated for unknown reason) + int maxSize = 100; + int size = actionSet.getActionSizeRec(maxSize); + if (size > maxSize) continue; +// System.out.println(size); + // timer.log("getActionSizeRec, size = " + size); +// int size = actionSet.strList.size(); + String key = astNodeType + "/" + size + "/" + pj + "_" + diffentryFile.getName() + "_" + hunkSet; + ITree targetTree = EDiffHelper.getTargets(actionSet, isJava); + // timer.log("getTargets"); + ITree actionTree = EDiffHelper.getActionTrees(actionSet); + // timer.log("getActionTrees"); + ITree shapeTree = EDiffHelper.getShapeTree(actionSet, isJava); + // timer.log("getShapeTree"); + ITree tokenTree = EDiffHelper.getTokenTree(actionSet, isJava); + // timer.log("getTokenTree"); + String tokens = EDiffHelper.getNames2(tokenTree); + // timer.log("getNames2"); + try (Jedis inner = innerPool.getResource()) { - for (HierarchicalActionSet actionSet : actionSets) - { - String astNodeType = actionSet.getAstNodeType(); - actionSet.toString(); - int size = actionSet.getActionSize(); - - String key = astNodeType + "/" + size + "/" + pj + "_" + diffentryFile.getName() + "_" + hunkSet; - - ITree targetTree = EDiffHelper.getTargets(actionSet, isJava); - ITree actionTree = EDiffHelper.getActionTrees(actionSet); - ITree shapeTree = EDiffHelper.getShapeTree(actionSet, isJava); - ITree tokenTree = EDiffHelper.getTokenTree(actionSet, isJava); - String tokens = EDiffHelper.getNames2(tokenTree); - try (Jedis inner = innerPool.getResource()) - { - inner.hset("dump", key, actionSet.toString()); - inner.hset(key, "actionTree", actionTree.toStaticHashString()); - inner.hset(key, "targetTree", targetTree.toStaticHashString()); - inner.hset(key, "shapeTree", shapeTree.toStaticHashString()); - inner.hset(key, "tokens", tokens); - } - - hunkSet++; - } - try (Jedis inner = innerPool.getResource()) - { - inner.hset("diffEntry", pj + "_" + diffentryFile.getName(), "1"); - } + inner.hset("dump", key, actionSet.toString()); +// inner.hset("dump", key, ""); + inner.hset(key, "actionTree", actionTree.toStaticHashString()); + inner.hset(key, "targetTree", targetTree.toStaticHashString()); + inner.hset(key, "shapeTree", shapeTree.toStaticHashString()); + inner.hset(key, "tokens", tokens); } + // timer.log("dump"); + + hunkSet++; } + try (Jedis inner = innerPool.getResource()) + { + inner.hset("diffEntry", pj + "_" + diffentryFile.getName(), "1"); + } + // logger.info("Finished processing {}", prevFile); } + catch (Exception e) { logger.error("error", e); diff --git a/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/EDiffParser.java b/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/EDiffParser.java index b04da3b..ee54790 100755 --- a/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/EDiffParser.java +++ b/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/EDiffParser.java @@ -41,22 +41,14 @@ public class EDiffParser extends Parser public List parseChangedSourceCodeWithGumTree2(File prevFile, File revFile, String srcMLPath, boolean isJava) { List actionSets = new ArrayList<>(); - // GumTree results - // boolean isJava =false; List gumTreeResults = null; if (isJava) { - // if (revFile.getName().endsWith(".c") & prevFile.getName().endsWith(".c") || revFile.getName().endsWith(".h") & prevFile.getName().endsWith(".h")){ - // gumTreeResults = new GumTreeComparer().compareCFilesWithGumTree(prevFile, revFile); - gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); - - } else { gumTreeResults = new GumTreeCComparer().compareCFilesWithGumTree(prevFile, revFile, srcMLPath); - } if (gumTreeResults == null) { @@ -110,6 +102,4 @@ public class EDiffParser extends Parser // public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool) { // // } - - } diff --git a/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/HierarchicalActionSet.java b/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/HierarchicalActionSet.java index 578c859..75f0cc5 100755 --- a/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/HierarchicalActionSet.java +++ b/richedit/src/main/java/edu/lu/uni/serval/richedit/ediff/HierarchicalActionSet.java @@ -2,9 +2,12 @@ package edu.lu.uni.serval.richedit.ediff; import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.tree.ITree; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; /** @@ -14,6 +17,7 @@ import java.util.List; */ public class HierarchicalActionSet implements Comparable, Serializable { + private static final Logger log = LoggerFactory.getLogger(EDiffHunkParser.class); private String astNodeType; @@ -212,46 +216,51 @@ public class HierarchicalActionSet implements Comparable, return this.startPosition.compareTo(o.startPosition);//this.action.compareTo(o.action); } - private final List strList = new ArrayList<>(); + final List strList = new ArrayList<>(); - public int getActionSize() +// public int getActionSize() +// { +// return strList.size(); +// } + + private Integer size = null; + + public int getActionSizeRec(int maxSize) { - return strList.size(); + if (this.size != null) return this.size; + + int size = 1; + for (HierarchicalActionSet s : subActions) + { + size += s.getActionSizeRec(maxSize); + if (size > maxSize) return maxSize + 1; + } + + return this.size = size; } @Override public String toString() { - String str = actionString; +// log.info("Calling toString on {}", actionString); if (strList.size() == 0) { - strList.add(str); + strList.add(actionString); + + // TODO: Can we use unique subActions instead of full subActions with many duplicates? +// List usedSubActions = subActions.size() > 20 ? new ArrayList<>(new HashSet<>(subActions)) : subActions; + for (HierarchicalActionSet actionSet : subActions) { actionSet.toString(); - List strList1 = actionSet.strList; - for (String str1 : strList1) - { - strList.add("---" + str1); - } - } - } - else - { - strList.clear(); - strList.add(str); - for (HierarchicalActionSet actionSet : subActions) - { - actionSet.toString(); - List strList1 = actionSet.strList; - for (String str1 : strList1) + for (String str1 : actionSet.strList) { strList.add("---" + str1); } } } - str = ""; + String str = ""; for (String str1 : strList) { str += str1 + "\n"; @@ -259,6 +268,4 @@ public class HierarchicalActionSet implements Comparable, return str; } - - } diff --git a/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/CompareTrees.java b/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/CompareTrees.java index 7d15b88..3e1843d 100755 --- a/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/CompareTrees.java +++ b/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/CompareTrees.java @@ -1,5 +1,6 @@ package edu.lu.uni.serval.richedit.jobs; +import edu.lu.uni.serval.richedit.ediff.EDiffHunkParser; import edu.lu.uni.serval.utils.CallShell; import edu.lu.uni.serval.utils.EDiffHelper; import edu.lu.uni.serval.utils.PoolBuilder; @@ -13,6 +14,10 @@ import redis.clients.jedis.JedisPool; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -38,12 +43,51 @@ public class CompareTrees compare = inner.scard("compare"); } IntStream stream = IntStream.range(0, compare.intValue()); + { + ExecutorService executor = Executors.newScheduledThreadPool(36); - ProgressBar.wrap(stream.parallel(), "Task").forEach(m -> + stream.forEach(m -> { - newCoreCompare(job, errorPairs, filenames, outerPool); - } - ); + String id = UUID.randomUUID().toString(); + Future future = executor.submit(() -> + { + Thread cur = Thread.currentThread(); + + // Monitor thread + Thread monitor = new Thread(() -> + { + try + { + Thread.sleep(60 * 1000); + cur.stop(); + log.info("Cancelled {} because of timeout", cur.getName()); + + // Update progress bar + // pb.step(); + } + catch (InterruptedException ignored) {} + }); + monitor.start(); + + // Parse + newCoreCompare(job, errorPairs, filenames, outerPool); + + // Update progress bar + // pb.step(); + System.out.print("."); + + // Stop monitoring thread + monitor.interrupt(); + }); + }); + + executor.shutdown(); + } +// ProgressBar.wrap(stream.parallel(), "Task").forEach(m -> +// { +// newCoreCompare(job, errorPairs, filenames, outerPool); +// } +// ); log.info("End process"); } @@ -74,7 +118,7 @@ public class CompareTrees } Map oldTreeString = EDiffHelper.getTreeString(keyName, i, outerPool, filenames); Map newTreeString = EDiffHelper.getTreeString(keyName, j, outerPool, filenames); - + //log.info("start switching"); switch (treeType) { case "single": diff --git a/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/EnhancedASTDiff.java b/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/EnhancedASTDiff.java index a365a15..7334507 100755 --- a/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/EnhancedASTDiff.java +++ b/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/EnhancedASTDiff.java @@ -2,7 +2,6 @@ package edu.lu.uni.serval.richedit.jobs; import edu.lu.uni.serval.richedit.ediff.EDiffHunkParser; import edu.lu.uni.serval.richedit.ediff.MessageFile; -import edu.lu.uni.serval.utils.CallShell; import edu.lu.uni.serval.utils.FileHelper; import edu.lu.uni.serval.utils.PoolBuilder; import me.tongfei.progressbar.ProgressBar; @@ -12,10 +11,8 @@ import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.*; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -39,7 +36,10 @@ public class EnhancedASTDiff // Find patches File folder = new File(inputPath); File[] listOfFiles = folder.listFiles(); - if (listOfFiles == null) throw new Exception("No projects found, please verify the projects in the input path"); + if (listOfFiles == null) + { + throw new Exception("No projects found, please verify the projects in the input path"); + } Stream stream = Arrays.stream(listOfFiles); List folders; if (projectList.length == 1 && projectList[0].equals("ALL")) @@ -92,22 +92,63 @@ public class EnhancedASTDiff { log.info("{} files already process ...", diffEntry.size()); allMessageFiles = allMessageFiles.stream().filter(f -> !diffEntry.containsKey(f.getProject() + "_" + f.getDiffEntryFile().getName())).collect(Collectors.toList()); - log.info("{} files to process ...",allMessageFiles.size()); + log.info("{} files to process ...", allMessageFiles.size()); } - ProgressBar.wrap(allMessageFiles.stream().parallel(), "Task").forEach(m -> +// try (ProgressBar pb = new ProgressBar("Task", allMessageFiles.size())) + { + ExecutorService executor = Executors.newScheduledThreadPool(36); + + allMessageFiles.forEach(m -> { - EDiffHunkParser parser = new EDiffHunkParser(); - parser.parseFixPatterns(m.getPrevFile(), m.getRevFile(), m.getDiffEntryFile(), project, innerPool, srcMLPath, hunkLimit, isJava); - } - ); + String id = UUID.randomUUID().toString(); + Future future = executor.submit(() -> + { + Thread cur = Thread.currentThread(); + + // Monitor thread + Thread monitor = new Thread(() -> + { + try + { + Thread.sleep(60 * 1000); + cur.stop(); + log.info("Cancelled {} because of timeout", cur.getName()); + + // Update progress bar +// pb.step(); + } + catch (InterruptedException ignored) {} + }); + monitor.start(); + + // Parse + EDiffHunkParser parser = new EDiffHunkParser(); + parser.parseFixPatterns(m.getPrevFile(), m.getRevFile(), m.getDiffEntryFile(), project, innerPool, srcMLPath, hunkLimit, isJava); + + // Update progress bar +// pb.step(); + System.out.print("."); + + // Stop monitoring thread + monitor.interrupt(); + }); + }); + + executor.shutdown(); + } +// ProgressBar.wrap(allMessageFiles.stream().parallel(), "Task").forEach(m -> +// { +// EDiffHunkParser parser = new EDiffHunkParser(); +// parser.parseFixPatterns(m.getPrevFile(), m.getRevFile(), m.getDiffEntryFile(), project, innerPool, srcMLPath, hunkLimit, isJava); +// } +// ); } private static List getMessageFiles(String gumTreeInput, String datasetName, String patchSize, boolean isJava) { - String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile - File revFilesPath = new File(inputPath + "revFiles/"); + File revFilesPath = new File(gumTreeInput + "revFiles/"); log.info(revFilesPath.getPath()); File[] revFiles = revFilesPath.listFiles(); if (revFiles != null) @@ -128,7 +169,10 @@ public class EnhancedASTDiff { count++; } - if (count >= Integer.valueOf(patchSize)) continue; + if (count >= Integer.parseInt(patchSize)) + { + continue; + } String[] split1 = diffentryFile.getParent().split(datasetName); String root = split1[0]; String pj = split1[1].split("/")[1]; @@ -140,6 +184,9 @@ public class EnhancedASTDiff return msgFiles; } - else return null; + else + { + return null; + } } } diff --git a/richedit/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java b/richedit/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java index c4c74f9..43f77f5 100755 --- a/richedit/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java +++ b/richedit/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java @@ -12,40 +12,23 @@ public class ClusterToPattern public static void main(String port, String redisPath, String dumpsName, String parameter) throws Exception { - CallShell cs = new CallShell(); String cmd = "bash " + redisPath + "/" + "startServer.sh" + " %s %s %s"; cmd = String.format(cmd, redisPath, dumpsName, Integer.valueOf(port)); log.trace(cmd); CallShell.runShell(cmd, port); String host = "localhost";//args[5]; - final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host, Integer.valueOf(port), 20000000); + final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host, Integer.parseInt(port), 20000000); String export = export(parameter, outerPool); System.out.println(export); - } private static String export(String filename, JedisPool outerPool) { - try (Jedis outer = outerPool.getResource()) { - while (!outer.ping().equals("PONG")) - { - log.info("wait"); - } - // byte[] s = outer.hget("dump".getBytes(), filename.getBytes()); - // HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(s); - // if (actionSet == null){ - // throw new Error(filename +" not found"); - // } - String s1 = outer.hget("dump", filename); - // outer.close(); - return s1; + while (!outer.ping().equals("PONG")) log.info("wait"); + return outer.hget("dump", filename); } - - } - - } diff --git a/richedit/src/main/java/edu/lu/uni/serval/utils/Timer.java b/richedit/src/main/java/edu/lu/uni/serval/utils/Timer.java new file mode 100644 index 0000000..5c3904c --- /dev/null +++ b/richedit/src/main/java/edu/lu/uni/serval/utils/Timer.java @@ -0,0 +1,18 @@ +package edu.lu.uni.serval.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Timer +{ + private static final Logger logger = LoggerFactory.getLogger(Timer.class); + + long start = System.nanoTime(); + + public void log(String msg) + { + long elapse = (long) ((System.nanoTime() - start) / 1e6); + logger.info("{}ms: {}", elapse, msg); + start = System.nanoTime(); + } +}