diff --git a/src/main/java/edu/lu/uni/serval/bugLocalization/ParseAlarms.java b/src/main/java/edu/lu/uni/serval/bugLocalization/ParseAlarms.java deleted file mode 100644 index 16b3f57..0000000 --- a/src/main/java/edu/lu/uni/serval/bugLocalization/ParseAlarms.java +++ /dev/null @@ -1,104 +0,0 @@ -package edu.lu.uni.serval.bugLocalization; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; - -import com.github.gumtreediff.tree.ITree; - -import edu.lu.uni.serval.FixPatternParser.Tokenizer; -import edu.lu.uni.serval.FixPatternParser.violations.Violation; -import edu.lu.uni.serval.config.Configuration; -import edu.lu.uni.serval.gumtree.regroup.SimpleTree; -import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; -import edu.lu.uni.serval.utils.FileHelper; - -public class ParseAlarms { - - public static void main(String[] args) { - /* - * Alarm java files. - * Position files. - */ - String fixedAlarmFilesPath = Configuration.GUM_TREE_INPUT + "prevFiles/"; - String positionsFilePath = Configuration.GUM_TREE_INPUT + "positions/"; - - String unfixedAlarmFilesPath = Configuration.GUM_TREE_INPUT + "unfixAlarms/"; - String un_positionsFilePath = Configuration.GUM_TREE_INPUT + "un_positions/"; - - StringBuilder tokensBuilder = new StringBuilder(); - List javaFiles = FileHelper.getAllFilesInCurrentDiectory(fixedAlarmFilesPath, ".java"); - int counter = 0; - for (File javaFile : javaFiles) { - String fileName = javaFile.getName().replace(".java", ".txt"); - fileName = fileName.substring(5); - - List violations = readViolations(positionsFilePath + fileName); - - - for (Violation violation : violations) { - int startLine = violation.getStartLineNum(); - int endLine = violation.getEndLineNum(); - String alarmType = violation.getAlarmType(); - - if (endLine > startLine + 5) continue; - - AlarmTree alarmTree = new AlarmTree(javaFile, startLine, endLine); - alarmTree.extract(); - List matchedTrees = alarmTree.getAlarmTrees(); - SimpleTree simpleTree = new SimpleTree(); - simpleTree.setLabel("Block"); - simpleTree.setNodeType("Block"); - List children = new ArrayList<>(); - - for (ITree matchedTree : matchedTrees) { - SimpleTree simpleT = new SimplifyTree().canonicalizeSourceCodeTree(matchedTree, null); - children.add(simpleT); - } - simpleTree.setChildren(children); - - String tokens = Tokenizer.getTokensDeepFirst(simpleTree); - tokensBuilder.append(fileName + "\n" + alarmType + ": " + tokens + "\n"); - counter ++; - if (counter % 100 == 0) { - FileHelper.outputToFile(Configuration.ROOT_PATH + "Alarms_tokens/fixedAlarms.list", tokensBuilder, true); - tokensBuilder.setLength(0); - } - } - } - - FileHelper.outputToFile(Configuration.ROOT_PATH + "Alarms_tokens/fixedAlarms.list", tokensBuilder, true); - tokensBuilder.setLength(0); - } - - private static List readViolations(String file) { - List violations = new ArrayList<>(); - - String fileContent = FileHelper.readFile(file); - BufferedReader reader = null; - reader = new BufferedReader(new StringReader(fileContent)); - String line = null; - try { - while ((line = reader.readLine()) != null) { - String[] positionStr = line.split(":"); - int startLine = Integer.parseInt(positionStr[0]); - int endLine = Integer.parseInt(positionStr[1]); - String alarmType = positionStr[2]; - Violation violation = new Violation(startLine, endLine, alarmType); - violations.add(violation); - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - reader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - return violations; - } -} diff --git a/src/main/java/edu/lu/uni/serval/bugLocalization/AlarmTree.java b/src/main/java/edu/lu/uni/serval/violation/code/parser/AlarmTree.java similarity index 66% rename from src/main/java/edu/lu/uni/serval/bugLocalization/AlarmTree.java rename to src/main/java/edu/lu/uni/serval/violation/code/parser/AlarmTree.java index ee71e1a..b714eea 100644 --- a/src/main/java/edu/lu/uni/serval/bugLocalization/AlarmTree.java +++ b/src/main/java/edu/lu/uni/serval/violation/code/parser/AlarmTree.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.bugLocalization; +package edu.lu.uni.serval.violation.code.parser; import java.io.File; import java.util.ArrayList; @@ -49,7 +49,7 @@ public class AlarmTree { public int getAlarmFinalEndLine() { return alarmFinalEndLine; } - + public void extract() { ITree rootTree = new GumTreeGenerator().generateITreeForJavaFile(file, GumTreeType.EXP_JDT); @@ -62,7 +62,7 @@ public class AlarmTree { } int endPosition = startPosition + tree.getLength(); - int endLine = cUnit.getLineNumber(endPosition); + int endLine = cUnit.getLineNumber(endPosition - 1); if (endLine < alarmStartLine) continue; matchTrees(tree.getChildren()); @@ -73,6 +73,38 @@ public class AlarmTree { this.alarmFinalStartLine = cUnit.getLineNumber(this.matchedTrees.get(0).getPos()); ITree lastTree = matchedTrees.get(size - 1); this.alarmFinalEndLine = cUnit.getLineNumber(lastTree.getPos() + lastTree.getLength()); + } else { + System.err.println(this.file.getName() + "===" + this.alarmStartLine + ":" + this.alarmEndLine); + } + + } + + public void extract(String type) { + ITree rootTree = new GumTreeGenerator().generateITreeForJavaFile(file, GumTreeType.EXP_JDT); + + List trees = rootTree.getChildren(); + for (ITree tree : trees) { + int startPosition = tree.getPos(); + int startLine = cUnit.getLineNumber(startPosition); + if (startLine > alarmEndLine) { + break; + } + + int endPosition = startPosition + tree.getLength(); + int endLine = cUnit.getLineNumber(endPosition - 1); + if (endLine < alarmStartLine) continue; + + matchTrees(tree.getChildren()); + } + + int size = matchedTrees.size(); + if (size > 0) { + this.alarmFinalStartLine = cUnit.getLineNumber(this.matchedTrees.get(0).getPos()); + ITree lastTree = matchedTrees.get(size - 1); + this.alarmFinalEndLine = cUnit.getLineNumber(lastTree.getPos() + lastTree.getLength()); + } else { + System.err.println(type); + System.err.println(this.file.getName() + "===" + this.alarmStartLine + ":" + this.alarmEndLine); } } @@ -89,32 +121,69 @@ public class AlarmTree { int endLine = cUnit.getLineNumber(endPosition); if (endLine < alarmStartLine) continue; - if (startLine >= alarmStartLine) { - if (!isStatement(tree)) { + if (endLine == alarmEndLine) { + if (tree.getType() == 31) { // MethodDeclaration + matchTrees(tree.getChildren()); + } else if (isStatement(tree)) { + addToMatchedTrees(tree); + } else { ITree parent = getParentStatement(tree); if (parent == null) { - if (tree.getLabel().equals("MethodBody") || tree.getType() == 8) { + if (tree.getType() == 8) { // 8: Block matchTrees(tree.getChildren()); } continue; } - tree = parent; + addToMatchedTrees(parent); } - if (!matchedTrees.contains(tree)) { - matchedTrees.add(tree); - } - if (containsBlockStatement(tree)) { - endLine = cUnit.getLineNumber(tree.getPos() + tree.getLength()); - if (endLine > alarmEndLine) { - tree = removeBlock(tree); + continue; + } + + if (startLine >= alarmStartLine) { + if (isStatement(tree)) { + addToMatchedTrees(tree); + } else { + ITree parent = getParentStatement(tree); + if (parent == null) { + if (tree.getType() == 8) { + matchTrees(tree.getChildren()); + } + continue; } + addToMatchedTrees(parent); } } else { +// if (tree.getType() == 14) { +// ITree parent = getParentStatement(tree); +// if (parent == null) { +// matchTrees(tree.getChildren()); +// } else { +// addToMatchedTrees(parent); +// } +// } else { +// matchTrees(tree.getChildren()); +// } matchTrees(tree.getChildren()); } } } + private void addToMatchedTrees(ITree tree) { + if (!matchedTrees.contains(tree)) { + /* + * TODO with the same parent, or the sub trees. + * In the same method body. + */ + matchedTrees.add(tree); + } + if (containsBlockStatement(tree)) { + int endLine = cUnit.getLineNumber(tree.getPos() + tree.getLength()); + if (endLine > alarmEndLine) { + tree = removeBlock(tree); + } + } + } + private ITree removeBlock(ITree tree) { List oldChildren = tree.getChildren(); List newChildren = new ArrayList<>(); @@ -127,7 +196,7 @@ public class AlarmTree { int endPosition = startPosition + child.getLength(); int endLine = cUnit.getLineNumber(endPosition); if (endLine > this.alarmEndLine) { - if (child.getType() == 8 || containsBlockStatement(child)) { + if (child.getType() == 8 || containsBlockStatement(child)) { // 8: Block child = removeBlock(child); if (child.getChildren().size() == 0) { continue; @@ -144,8 +213,12 @@ public class AlarmTree { ITree parent = tree; do { parent = parent.getParent(); + if (parent == null) { + return null; + } + int type = parent.getType(); - if (type == 1 || type == 31 || type == 55) { + if (type == 1 || type == 31 || type == 55 || type == 71) { // AnonymousClassDeclaration // MethodDeclaration Initializer (type == 28) // TypeDeclaration diff --git a/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationCodeParser.java b/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationCodeParser.java new file mode 100644 index 0000000..27cbd58 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationCodeParser.java @@ -0,0 +1,147 @@ +package edu.lu.uni.serval.violation.code.parser; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.github.gumtreediff.tree.ITree; + +import edu.lu.uni.serval.FixPatternParser.Tokenizer; +import edu.lu.uni.serval.FixPatternParser.violations.Violation; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.gumtree.regroup.SimpleTree; +import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; +import edu.lu.uni.serval.utils.FileHelper; + +public class ViolationCodeParser { + + private static Logger log = LoggerFactory.getLogger(ViolationCodeParser.class); + + public static void main(String[] args) { + /* + * Fixed violations: + * Violation code files. + * Position files. + */ + String fixedViolationFilesPath = Configuration.GUM_TREE_INPUT + "prevFiles/"; + String positionsFilePath = Configuration.GUM_TREE_INPUT + "positions/"; + int subIndex = 5; + String outputPath = Configuration.ROOT_PATH + "Alarms_tokens/fixedAlarms.list"; + FileHelper.deleteFile(outputPath); + new ViolationCodeParser().parse(fixedViolationFilesPath, positionsFilePath, subIndex, outputPath); + + /* + * UnFixed violations: + * Violation code files. + * Position files. + */ + String unfixedViolationFilesPath = Configuration.GUM_TREE_INPUT + "unfixAlarms/"; + String un_positionsFilePath = Configuration.GUM_TREE_INPUT + "un_positions/"; + subIndex = 8; + outputPath = Configuration.ROOT_PATH + "Alarms_tokens/unfixedAlarms.list"; + FileHelper.deleteFile(outputPath); + new ViolationCodeParser().parse(unfixedViolationFilesPath, un_positionsFilePath, subIndex, outputPath); + } + + public void parse(String alarmFilesPath, String positionFilesPath, int subIndex, String outputPath) { + StringBuilder tokensBuilder = new StringBuilder(); + List javaFiles = FileHelper.getAllFilesInCurrentDiectory(alarmFilesPath, ".java"); + int counter = 0; + int a = 0; + int maxLength = 0; + for (File javaFile : javaFiles) { + String fileName = javaFile.getName().replace(".java", ".txt"); + fileName = fileName.substring(subIndex); + + List violations = readViolationInfo(positionFilesPath + fileName); + + for (Violation violation : violations) { + int startLine = violation.getStartLineNum(); + int endLine = violation.getEndLineNum(); + String alarmType = violation.getAlarmType(); + +// if (endLine > startLine + 5) { +// log.warn("#Large_Violation_Hunk: " + fileName.replace("#", "/").replace(".txt", ".java") + ":" + startLine + ":" + endLine + ":" + alarmType); +// continue; +// } + + AlarmTree alarmTree = new AlarmTree(javaFile, startLine, endLine); + alarmTree.extract(); + List matchedTrees = alarmTree.getAlarmTrees(); + if (matchedTrees.size() == 0) { + System.out.println(fileName + " == " + startLine + " : " + endLine); + a ++; + log.warn("#Null_Violation_Hunk: " + fileName.replace("#", "/").replace(".txt", ".java") + ":" + startLine + ":" + endLine + ":" + alarmType); + continue; + } + SimpleTree simpleTree = new SimpleTree(); + simpleTree.setLabel("Block"); + simpleTree.setNodeType("Block"); + List children = new ArrayList<>(); + + for (ITree matchedTree : matchedTrees) { + SimpleTree simpleT = new SimplifyTree().canonicalizeSourceCodeTree(matchedTree, null); + children.add(simpleT); + } + simpleTree.setChildren(children); + + String tokens = Tokenizer.getTokensDeepFirst(simpleTree); + String[] tokensArray = tokens.split(" "); + int length = tokensArray.length; + if (length > maxLength) maxLength = length; + tokensBuilder.append(alarmType + ":" + fileName + ":" + alarmTree.getAlarmFinalStartLine() + ":" + alarmTree.getAlarmFinalEndLine() + ":" + tokens + "\n"); + counter ++; + if (counter % 5000 == 0) { + FileHelper.outputToFile(outputPath, tokensBuilder, true); + tokensBuilder.setLength(0); + } + } + } + + System.out.println(counter); + System.out.println(a); + System.out.println("MaxLength: " + maxLength); + FileHelper.outputToFile(outputPath, tokensBuilder, true); + tokensBuilder.setLength(0); + } + + private List readViolationInfo(String file) { + List violations = new ArrayList<>(); + + String fileContent = FileHelper.readFile(file); + BufferedReader reader = null; + reader = new BufferedReader(new StringReader(fileContent)); + String line = null; + try { + while ((line = reader.readLine()) != null) { + String[] positionStr = line.split(":"); + int startLine = Integer.parseInt(positionStr[1]); + int endLine = Integer.parseInt(positionStr[2]); + String alarmType = positionStr[0]; + + if (startLine == -1 || endLine == -1) { + log.warn("#Illegal_Line_Position: " + FileHelper.getFileName(file).replace("#", "/").replace(".txt", ".java") + ":" + startLine + ":" + endLine + ":" + alarmType); + continue; + } + + Violation violation = new Violation(startLine, endLine, alarmType); + violations.add(violation); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return violations; + } +}