Update

src/main/java/edu/lu/uni/serval/bugLocalization/AlarmTree.java

@@ -0,0 +1,191 @@
+package edu.lu.uni.serval.bugLocalization;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.eclipse.jdt.core.dom.CompilationUnit;
+
+import com.github.gumtreediff.tree.ITree;
+
+import edu.lu.uni.serval.FixPatternParser.CUCreator;
+import edu.lu.uni.serval.gumtree.GumTreeGenerator;
+import edu.lu.uni.serval.gumtree.GumTreeGenerator.GumTreeType;
+
+public class AlarmTree {
+
+	private File file;
+	private int alarmStartLine;
+	private int alarmEndLine;
+	private CompilationUnit cUnit;
+	
+	private int alarmFinalStartLine;
+	private int alarmFinalEndLine;
+	
+	private List<ITree> matchedTrees = new ArrayList<>();
+	
+	public AlarmTree(File file, int startLine, int endLine) {
+		super();
+		this.file = file;
+		this.alarmStartLine = startLine;
+		this.alarmEndLine = endLine;
+		
+		CUCreator cuCreator = new CUCreator();
+		this.cUnit = cuCreator.createCompilationUnit(this.file);
+	}
+	
+	public AlarmTree(String fileName, int startLine, int endLine) {
+		this(new File(fileName), startLine, endLine);
+	}
+	
+	public List<ITree> getAlarmTrees() {
+		return this.matchedTrees;
+	}
+	
+	public int getAlarmFinalStartLine() {
+		return alarmFinalStartLine;
+	}
+
+	public int getAlarmFinalEndLine() {
+		return alarmFinalEndLine;
+	}
+
+	public void extract() {
+		ITree rootTree = new GumTreeGenerator().generateITreeForJavaFile(file, GumTreeType.EXP_JDT);
+		
+		List<ITree> trees = rootTree.getChildren();
+		for (ITree tree : trees) {
+			int startPosition = tree.getPos();
+			int startLine = cUnit.getLineNumber(startPosition);
+			if (startLine > alarmEndLine) {
+				break;
+			}
+			
+			int endPosition = startPosition + tree.getLength();
+			int endLine = cUnit.getLineNumber(endPosition);
+			if (endLine < alarmStartLine) continue;
+			
+			matchTrees(tree.getChildren());
+		}
+		
+		int size = matchedTrees.size();
+		if (size > 0) {
+			this.alarmFinalStartLine = cUnit.getLineNumber(this.matchedTrees.get(0).getPos());
+			ITree lastTree = matchedTrees.get(size - 1);
+			this.alarmFinalEndLine = cUnit.getLineNumber(lastTree.getPos() + lastTree.getLength());
+		}
+		
+	}
+
+	private void matchTrees(List<ITree> trees) {
+		for (ITree tree : trees) {
+			int startPosition = tree.getPos();
+			int startLine = cUnit.getLineNumber(startPosition);
+			if (startLine > alarmEndLine) {
+				break;
+			} 
+			
+			int endPosition = startPosition + tree.getLength();
+			int endLine = cUnit.getLineNumber(endPosition);
+			if (endLine < alarmStartLine) continue;
+			
+			if (startLine >= alarmStartLine) {
+				if (!isStatement(tree)) {
+					ITree parent = getParentStatement(tree);
+					if (parent == null) {
+						if (tree.getLabel().equals("MethodBody") || tree.getType() == 8) {
+							matchTrees(tree.getChildren());
+						}
+						continue;
+					}
+					tree = parent;
+				}
+				if (!matchedTrees.contains(tree)) {
+					matchedTrees.add(tree);
+				}
+				if (containsBlockStatement(tree)) {
+					endLine = cUnit.getLineNumber(tree.getPos() + tree.getLength());
+					if (endLine > alarmEndLine) {
+						tree = removeBlock(tree);
+					}
+				}
+			} else {
+				matchTrees(tree.getChildren());
+			}
+		}
+	}
+
+	private ITree removeBlock(ITree tree) {
+		List<ITree> oldChildren = tree.getChildren();
+		List<ITree> newChildren = new ArrayList<>();
+		for (ITree child : oldChildren) {
+			int startPosition = child.getPos();
+			int startLine = cUnit.getLineNumber(startPosition);
+			if (startLine > alarmEndLine) {
+				break;
+			} 
+			int endPosition = startPosition + child.getLength();
+			int endLine = cUnit.getLineNumber(endPosition);
+			if (endLine > this.alarmEndLine) {
+				if (child.getType() == 8 || containsBlockStatement(child)) {
+					child = removeBlock(child);
+					if (child.getChildren().size() == 0) {
+						continue;
+					}
+				}
+			}
+			newChildren.add(child);
+		}
+		tree.setChildren(newChildren);
+		return tree;
+	}
+
+	private ITree getParentStatement(ITree tree) {
+		ITree parent = tree;
+		do {
+			parent = parent.getParent();
+			int type = parent.getType();
+			if (type == 1 || type == 31 || type == 55) {
+				// AnonymousClassDeclaration
+				// MethodDeclaration  Initializer (type == 28)
+				// TypeDeclaration
+				return null;
+			}
+		} while (!isStatement(parent));
+
+		return parent;
+	}
+
+	private boolean isStatement(ITree tree) {
+		int type = tree.getType(); // 8 Block
+		if (type == 6) return true;  // AssertStatement
+		if (type == 10) return true; // BreakStatement
+		if (type == 17) return true; // ConstructorInvocation
+		if (type == 18) return true; // ContinueStatement
+		if (type == 21) return true; // ExpressionStatement
+		if (type == 23) return true; // FieldDeclaration
+		if (type == 41) return true; // ReturnStatement
+		if (type == 46) return true; // SuperConstructorInvocation
+		if (type == 49) return true; // SwitchCase
+		if (type == 53) return true; // ThrowStatement
+		if (type == 56) return true; // TypeDeclarationStatement
+		if (type == 60) return true; // VariableDeclarationStatement
+		if (containsBlockStatement(tree)) return true;
+		return false;
+	}
+
+	private boolean containsBlockStatement(ITree tree) {
+		int type = tree.getType();
+		if (type == 12) return true; // catchClause
+		if (type == 19) return true; // DoStatement
+		if (type == 24) return true; // ForStatement
+		if (type == 25) return true; // IfStatement
+		if (type == 30) return true; // LabeledStatement
+		if (type == 50) return true; // SwitchStatement
+		if (type == 51) return true; // SynchronizedStatement
+		if (type == 54) return true; // TryStatement
+		if (type == 61) return true; // WhileStatement
+		if (type == 70) return true; // EnhancedForStatement
+		return false;
+	}
+}

src/main/java/edu/lu/uni/serval/bugLocalization/BuggyCodeParser.java

@@ -1,13 +0,0 @@
-package edu.lu.uni.serval.bugLocalization;
-
-import com.github.gumtreediff.tree.ITree;
-
-import edu.lu.uni.serval.gumtree.GumTreeGenerator;
-import edu.lu.uni.serval.gumtree.GumTreeGenerator.GumTreeType;
-
-public class BuggyCodeParser {
-
-	public void parseBuggyCode(String buggyCode) {
-		ITree tree = new GumTreeGenerator().generateITreeForCodeBlock(buggyCode, GumTreeType.EXP_JDT);
-	}
-}

src/main/java/edu/lu/uni/serval/bugLocalization/BuggyTreeParser.java

@@ -0,0 +1,106 @@
+package edu.lu.uni.serval.bugLocalization;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.github.gumtreediff.tree.ITree;
+
+import edu.lu.uni.serval.FixPatternParser.Tokenizer;
+import edu.lu.uni.serval.config.Configuration;
+import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
+import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
+import edu.lu.uni.serval.utils.FileHelper;
+
+/**
+ * 
+ * Testing data
+ * localized bugs by FindBugs.
+ * Use their java file, use GumTree to parse it, simplify the buggy code.
+ * get all buggy code tokens.
+ * 
+ * training data
+ * 
+ * 
+ * Word2Vec embeds data.
+ * 
+ * 
+ * @author kui.liu
+ *
+ */
+public class BuggyTreeParser {
+
+	public static void main(String[] args) {
+		List<String> data = readInfo("Dataset/localized-violations1.list");
+		StringBuilder bugTokens = new StringBuilder();
+		int i = 0;
+        for (String bug : data) {
+        	i ++;
+        	String[] info = bug.split(" : ");
+        	String bugId = info[0].trim();
+        	String fileName = info[2].trim();
+        	int startLine = Integer.parseInt(info[3].trim());
+        	int endLine = Integer.parseInt(info[4].trim());
+        	
+        	bugId = bugId.substring(0, bugId.lastIndexOf("."));
+        	String[] bugIdInfo = bugId.split("_");
+        	String projectName = bugIdInfo[0];
+        	String bugIdSt = bugIdInfo[1];
+        	String path = "../FPM_Violations/Testing/CheckedOut/" + projectName + "/" + bugIdSt + "/buggy/";
+        	if ("Chart".equals(projectName)) {
+        		path += "source/";
+        	} else {
+        		path += "src/";
+        	}
+        	List<File> javaFiles = FileHelper.getAllFiles(path, ".java");
+        	for (File javaFile : javaFiles) {
+        		if (javaFile.getPath().endsWith(fileName)) {
+        			AlarmTree alarmTree = new AlarmTree(javaFile, startLine, endLine);
+        			alarmTree.extract();
+        			List<ITree> matchedTrees = alarmTree.getAlarmTrees();
+        			SimpleTree simpleTree = new SimpleTree();
+        			simpleTree.setLabel("Block");
+    				simpleTree.setNodeType("Block");
+    				List<SimpleTree> children = new ArrayList<>();
+    				
+    				for (ITree matchedTree : matchedTrees) {
+    					SimpleTree simpleT = new SimplifyTree().canonicalizeSourceCodeTree(matchedTree, null);
+    					children.add(simpleT);
+    				}
+    				simpleTree.setChildren(children);
+    				
+    				String tokens = Tokenizer.getTokensDeepFirst(simpleTree);
+    				bugTokens.append(tokens + "\n");
+    				System.out.println(i + "### " + tokens);
+        		}
+        	}
+        }
+        
+        FileHelper.outputToFile(Configuration.EMBEDDING_DATA_TOKENS2, bugTokens, false);
+	}
+
+	private static List<String> readInfo(String file) {
+		List<String> info = new ArrayList<>();
+		String content = FileHelper.readFile(file);
+		BufferedReader reader = new BufferedReader(new StringReader(content));
+		String line = null;
+		try {
+			while ((line = reader.readLine()) != null) {
+				info.add(line);
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		} finally {
+			try {
+				reader.close();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+		return info;
+	}
+
+}

src/main/java/edu/lu/uni/serval/bugLocalization/ParseAlarms.java

@@ -0,0 +1,104 @@
+package edu.lu.uni.serval.bugLocalization;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.github.gumtreediff.tree.ITree;
+
+import edu.lu.uni.serval.FixPatternParser.Tokenizer;
+import edu.lu.uni.serval.FixPatternParser.violations.Violation;
+import edu.lu.uni.serval.config.Configuration;
+import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
+import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
+import edu.lu.uni.serval.utils.FileHelper;
+
+public class ParseAlarms {
+
+	public static void main(String[] args) {
+		/*
+		 * Alarm java files.
+		 * Position files.
+		 */
+		String fixedAlarmFilesPath = Configuration.GUM_TREE_INPUT + "prevFiles/";
+		String positionsFilePath = Configuration.GUM_TREE_INPUT + "positions/";
+		
+		String unfixedAlarmFilesPath = Configuration.GUM_TREE_INPUT + "unfixAlarms/";
+		String un_positionsFilePath = Configuration.GUM_TREE_INPUT + "un_positions/";
+		
+		StringBuilder tokensBuilder = new StringBuilder();
+		List<File> javaFiles = FileHelper.getAllFilesInCurrentDiectory(fixedAlarmFilesPath, ".java");
+		int counter = 0;
+		for (File javaFile : javaFiles) {
+			String fileName = javaFile.getName().replace(".java", ".txt");
+			fileName = fileName.substring(5);
+			
+			List<Violation> violations = readViolations(positionsFilePath + fileName);
+			
+			
+			for (Violation violation : violations) {
+				int startLine = violation.getStartLineNum();
+				int endLine = violation.getEndLineNum();
+				String alarmType = violation.getAlarmType();
+				
+				if (endLine > startLine + 5) continue;
+				
+				AlarmTree alarmTree = new AlarmTree(javaFile, startLine, endLine);
+				alarmTree.extract();
+				List<ITree> matchedTrees = alarmTree.getAlarmTrees();
+				SimpleTree simpleTree = new SimpleTree();
+				simpleTree.setLabel("Block");
+				simpleTree.setNodeType("Block");
+				List<SimpleTree> children = new ArrayList<>();
+				
+				for (ITree matchedTree : matchedTrees) {
+					SimpleTree simpleT = new SimplifyTree().canonicalizeSourceCodeTree(matchedTree, null);
+					children.add(simpleT);
+				}
+				simpleTree.setChildren(children);
+				
+				String tokens = Tokenizer.getTokensDeepFirst(simpleTree);
+				tokensBuilder.append(fileName + "\n" + alarmType + ": " + tokens + "\n");
+				counter ++;
+				if (counter % 100 == 0) {
+					FileHelper.outputToFile(Configuration.ROOT_PATH + "Alarms_tokens/fixedAlarms.list", tokensBuilder, true);
+					tokensBuilder.setLength(0);
+				}
+			}
+		}
+		
+		FileHelper.outputToFile(Configuration.ROOT_PATH + "Alarms_tokens/fixedAlarms.list", tokensBuilder, true);
+		tokensBuilder.setLength(0);
+	}
+
+	private static List<Violation> readViolations(String file) {
+		List<Violation> violations = new ArrayList<>();
+		
+		String fileContent = FileHelper.readFile(file);
+		BufferedReader reader = null;
+		reader = new BufferedReader(new StringReader(fileContent));
+		String line = null;
+		try {
+			while ((line = reader.readLine()) != null) {
+				String[] positionStr = line.split(":");
+				int startLine = Integer.parseInt(positionStr[0]);
+				int endLine = Integer.parseInt(positionStr[1]);
+				String alarmType = positionStr[2];
+				Violation violation = new Violation(startLine, endLine, alarmType);
+				violations.add(violation);
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		} finally {
+			try {
+				reader.close();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+		return violations;
+	}
+}

src/main/java/edu/lu/uni/serval/config/Configuration.java

@@ -2,9 +2,9 @@ package edu.lu.uni.serval.config;
 
 public class Configuration {
 	
-	public static final long SECONDS_TO_WAIT = 60L;
+	public static final long SECONDS_TO_WAIT = 10L;
 	
-	private static final String ROOT_PATH = "../FPM_Violations/"; // The root path of all output data.
+	public static final String ROOT_PATH = "../FPM_Violations/"; // The root path of all output data.
 
 	public static final int HUNK_SIZE = 10; // The limitation of source code lines of each DiffEntry, which will be selected as training data.
 	public static final String BUGGY_TREE_SIGNAL = "BUGGY_TREE###"; // The starting signal of the tree of buggy source code .
@@ -15,7 +15,7 @@ public class Configuration {
 	
 
 	// the output path of GumTree results.
-	private static final String GUM_TREE_OUTPUT = ROOT_PATH + "GumTreeResults/";
+	private static final String GUM_TREE_OUTPUT = ROOT_PATH + "GumTreeResults_2/";
 	public static final String EDITSCRIPTS_FILE_PATH = GUM_TREE_OUTPUT + "editScripts/";
 	public static final String PATCH_SOURCECODE_FILE_PATH = GUM_TREE_OUTPUT + "sourceCode/";
 	public static final String BUGGYTREE_FILE_PATH = GUM_TREE_OUTPUT + "buggyTrees/";

src/main/java/edu/lu/uni/serval/gumtree/regroup/SimplifyTree.java

@@ -181,7 +181,10 @@ public class SimplifyTree {
 				if ((astNode.equals("SimpleName") || astNode.equals("MethodInvocation")) && label.startsWith("MethodName:")) {
 					simpleTree.setNodeType("MethodName");
 					label = label.substring(11);
-					label = label.substring(0, label.indexOf(":["));
+					int argusIndex = label.indexOf(":[");
+					if (argusIndex > 0) {
+						label = label.substring(0, argusIndex);
+					}
 					simpleTree.setLabel(label);
 				} else {
 					simpleTree.setLabel(astNode);
@@ -198,7 +201,10 @@ public class SimplifyTree {
 				if (label.startsWith("MethodName:")) { // <MethodName, name>
 					simpleTree.setNodeType("MethodName");
 					label = label.substring(11);
-					label = label.substring(0, label.indexOf(":["));
+					int argusIndex = label.indexOf(":[");
+					if (argusIndex > 0) {
+						label = label.substring(0, argusIndex);
+					}
 					simpleTree.setLabel(label);
 				} else if (label.startsWith("Name:")) {
 					label = label.substring(5);
@@ -223,7 +229,10 @@ public class SimplifyTree {
 				} else if ((astNode.equals("SimpleName") || astNode.equals("MethodInvocation")) && label.startsWith("MethodName:")) {
 					simpleTree.setNodeType("MethodName");
 					label = label.substring(11);
-					label = label.substring(0, label.indexOf(":["));
+					int argusIndex = label.indexOf(":[");
+					if (argusIndex > 0) {
+						label = label.substring(0, argusIndex);
+					}
 					simpleTree.setLabel(label);
 				} else {
 					simpleTree.setLabel(label.replaceAll(" ", ""));
@@ -250,7 +259,10 @@ public class SimplifyTree {
 				if ((astNode.equals("SimpleName") || astNode.equals("MethodInvocation")) && label.startsWith("MethodName:")) {
 					simpleTree.setNodeType("MethodName");
 					label = label.substring(11);
-					label = label.substring(0, label.indexOf(":["));
+					int argusIndex = label.indexOf(":[");
+					if (argusIndex > 0) {
+						label = label.substring(0, argusIndex);
+					}
 					simpleTree.setLabel(label);
 				} else {
 					simpleTree.setLabel(astNode);
@@ -268,7 +280,10 @@ public class SimplifyTree {
 				if (label.startsWith("MethodName:")) { // <MethodName, name>
 					simpleTree.setNodeType("MethodName");
 					label = label.substring(11);
-					label = label.substring(0, label.indexOf(":["));
+					int argusIndex = label.indexOf(":[");
+					if (argusIndex > 0) {
+						label = label.substring(0, argusIndex);
+					}
 					simpleTree.setLabel(label);
 				} else if (label.startsWith("Name:")) {
 					label = label.substring(5);
@@ -293,7 +308,10 @@ public class SimplifyTree {
 				} else if ((astNode.equals("SimpleName") || astNode.equals("MethodInvocation")) && label.startsWith("MethodName:")) {
 					simpleTree.setNodeType("MethodName");
 					label = label.substring(11);
-					label = label.substring(0, label.indexOf(":["));
+					int argusIndex = label.indexOf(":[");
+					if (argusIndex > 0) {
+						label = label.substring(0, argusIndex);
+					}
 					simpleTree.setLabel(label);
 				} else {
 					simpleTree.setLabel(label.replaceAll(" ", ""));

src/main/java/edu/lu/uni/serval/violation/parse/AlarmsReader.java

@@ -91,4 +91,77 @@ public class AlarmsReader {
 		return violationsMap;
 	}
 	
+	public Map<String, Violation> readAlarmsList2(String fileName) {
+		Map<String, Violation> violationsMap = new HashMap<>();
+		FileInputStream fis = null;
+		Scanner scanner = null;
+		try {
+			fis = new FileInputStream(fileName);
+			scanner = new Scanner(fis);
+			
+			while (scanner.hasNextLine()) {
+				String line = scanner.nextLine();
+				int arrowIndex = line.indexOf("=>");
+				String alarmInfo = arrowIndex > 0 ? line.substring(0, arrowIndex) : line;
+
+				String[] buggyElements = alarmInfo.split(",");
+
+				String alarmType = buggyElements[0];
+				String projectName = buggyElements[1];
+				String buggyCommitId = buggyElements[2];
+				String buggyFile = buggyElements[3];
+				int startLine = Integer.parseInt(buggyElements[4]);
+				int endLine = Integer.parseInt(buggyElements[5]);
+				
+				Alarm alarm = new Alarm(buggyCommitId, buggyFile, "", "");
+				
+				Violation violation;
+				if (violationsMap.containsKey(projectName)) {
+					violation = violationsMap.get(projectName);
+				} else {
+					violation = new Violation(projectName);
+					violationsMap.put(projectName, violation);
+				}
+				List<Alarm> alarms = violation.getAlarms();
+				int index = alarms.indexOf(alarm);
+				if (index >= 0) {
+					Alarm tempAlarm = alarms.get(index);
+					Map<Integer, Integer> positions = tempAlarm.getPositions();
+					if (positions.containsKey(startLine)) {
+						int end = positions.get(startLine);
+						if (endLine < end) {
+							positions.put(startLine, endLine);
+							tempAlarm.getAlarmTypes().put(startLine, alarmType);
+						}
+					} else {
+						positions.put(startLine, endLine);
+						tempAlarm.getAlarmTypes().put(startLine, alarmType);
+						counter ++;
+					}
+				} else {
+					Map<Integer, String> alarmTypes = new HashMap<>();
+					alarmTypes.put(startLine, alarmType);
+					alarm.setAlarmTypes(alarmTypes);
+					Map<Integer, Integer> positions = new HashMap<>();
+					positions.put(startLine, endLine);
+					alarm.setPositions(positions);
+					alarms.add(alarm);
+					counter ++;
+				}
+			}
+		} catch (FileNotFoundException e) {
+			e.printStackTrace();
+		} finally {
+			try {
+				scanner.close();
+				fis.close();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+		
+		System.out.println(counter);
+		return violationsMap;
+	}
+	
 }

src/main/java/edu/lu/uni/serval/violation/parse/Statistic.java

@@ -12,11 +12,15 @@ public class Statistic {
 	public static void main(String[] args) {
 		String statistic = "../FPM_Violations/OUTPUT";
 		List<File> files = FileHelper.getAllFiles(statistic, ".list");
+		
 		int testAlarms = 0;
 		int nullGumTreeResults = 0;
 		int nullMappingGumTreeResults = 0;
 		int pureDeletion = 0;
 		int timeout = 0;
+		int noSourceCodeChagnes = 0;
+		int largeHunk = 0;
+		int nullSourceCode = 0;
 		for (File file : files) {
 			String content = FileHelper.readFile(file);
 			BufferedReader reader = new BufferedReader(new StringReader(content));
@@ -33,6 +37,12 @@ public class Statistic {
 						pureDeletion += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim());
 					} else if (line.startsWith("Time")) {
 						timeout += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim());
+					} else if (line.startsWith("noSourceCodeChagnes")) {
+						noSourceCodeChagnes += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim());
+					} else if (line.startsWith("largeHunk")) {
+						largeHunk += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim());
+					} else if (line.startsWith("nullSourceCode")) {
+						nullSourceCode += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim());
 					} 
 				}
 			} catch (IOException e) {
@@ -57,5 +67,8 @@ public class Statistic {
 		System.out.println("nullMappingGumTreeResults: " + nullMappingGumTreeResults);
 		System.out.println("pureDeletion: " + pureDeletion);
 		System.out.println("Timeout: " + timeout);
+		System.out.println("noSourceCodeChagnes: " + noSourceCodeChagnes);
+		System.out.println("largeHunk: " + largeHunk);
+		System.out.println("nullSourceCode: " + nullSourceCode);
 	}
 }

src/main/java/edu/lu/uni/serval/violation/parse/TestViolationParser.java

@@ -25,31 +25,88 @@ public class TestViolationParser {
 				}
 			}
 		}
-		String fixedAlarmFile = "Dataset/fixed-alarms-v0.3.list.txt";
+		
+		String unfixedAlarmFile = "Dataset/Unfixed-Alarms/";
+		final String previousFilesPath = Configuration.GUM_TREE_INPUT + "unfixAlarms/";
+		final String positionsFilePath = Configuration.GUM_TREE_INPUT + "un_positions/";
+		FileHelper.createDirectory(previousFilesPath);
+		FileHelper.createDirectory(positionsFilePath);
+		
+		List<File> unfixedAlarmFiles = FileHelper.getAllFilesInCurrentDiectory(unfixedAlarmFile, ".csv");
+		
+		for (File file : unfixedAlarmFiles) {
+			ViolationParser parser = new ViolationParser();
+			parser.parseViolations(file, repositoriesList, previousFilesPath, positionsFilePath);
+		}
+	
+		
+//		String fixedAlarmFile = "Dataset/fixed-alarms-v0.3.list.txt";
 
 		/**
 		 * 544 projects.
+		 * 42322 files
 		 * 84103 alarms.
+		 * 19928 parsed instances.  1984
+
+		 * 22 bugs
 		 * 
-		 * cd ../../7/buggy/
-		 * mvn compile
-		 * mvn package -Dmaven.test.skip=true
-		 * 
-		 * Lang 54, 56-65
-		 * Math 105 and 106
+NP_ALWAYS_NULL : org/jfree/chart/renderer/category/AbstractCategoryItemRenderer.java : 1800 : 1800
+Chart_1.xml
+UC_USELESS_CONDITION : org/jfree/data/KeyedObjects2D.java : 231 : 231
+Chart_22.xml
+DLS_DEAD_LOCAL_STORE : org/jfree/chart/renderer/GrayPaintScale.java : 125 : 125
+Chart_24.xml
+NP_NULL_ON_SOME_PATH : org/jfree/chart/plot/XYPlot.java : 4493 : 4493
+Chart_4.xml
+SF_SWITCH_NO_DEFAULT : com/google/javascript/jscomp/UnreachableCodeElimination.java : 151 : 171
+Closure_127.xml
+DLS_DEAD_LOCAL_STORE : com/google/javascript/jscomp/ScopedAliases.java : 276 : 276
+Closure_24.xml
+SF_SWITCH_NO_DEFAULT : com/google/javascript/jscomp/TypedScopeCreator.java : 550 : 580
+Closure_43.xml
+UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR : com/google/debugging/sourcemap/SourceMapConsumerV3.java : 488 : 488
+Closure_47.xml
+SF_SWITCH_NO_DEFAULT : com/google/javascript/jscomp/MakeDeclaredNamesUnique.java : 116 : 147
+Closure_49.xml
+DLS_DEAD_LOCAL_STORE : com/google/javascript/jscomp/CollapseProperties.java : 482 : 482
+Closure_89.xml
+SF_SWITCH_NO_DEFAULT : org/apache/commons/lang3/time/FastDateParser.java : 315 : 338
+Lang_10.xml
+DM_CONVERT_CASE : org/apache/commons/lang/StringUtils.java : 1048 : 1048
+Lang_40.xml
+SF_SWITCH_NO_DEFAULT : org/apache/commons/lang/math/NumberUtils.java : 449 : 491
+Lang_58.xml
+SF_SWITCH_NO_DEFAULT : org/apache/commons/lang/Entities.java : 916 : 919
+Lang_62.xml
+FE_FLOATING_POINT_EQUALITY : org/apache/commons/math3/util/FastMath.java : 1545 : 1545
+Math_15.xml
+DLS_DEAD_LOCAL_STORE : org/apache/commons/math/optimization/direct/BOBYQAOptimizer.java : 1658 : 1658
+Math_38.xml
+FE_FLOATING_POINT_EQUALITY : org/apache/commons/math/analysis/solvers/BaseSecantSolver.java : 187 : 187
+Math_50.xml
+CO_COMPARETO_INCORRECT_FLOATING : org/apache/commons/math/fraction/Fraction.java : 261 : 261
+Math_91.xml
+DLS_DEAD_LOCAL_STORE : org/mockito/internal/invocation/InvocationMatcher.java : 122 : 122
+Mockito_1.xml
+EQ_CHECK_FOR_OPERAND_NOT_COMPATIBLE_WITH_THIS : org/mockito/internal/creation/DelegatingMethod.java : 55 : 55
+Mockito_11.xml
+REC_CATCH_EXCEPTION : org/mockito/internal/creation/instance/ConstructorInstantiator.java : 26 : 26
+Mockito_21.xml
+DM_NUMBER_CTOR : org/joda/time/chrono/ZonedChronology.java : 469 : 469
+Time_26.xml
 		 */
 
-		final String previousFilesPath = Configuration.GUM_TREE_INPUT + "prevFiles/";
-		final String revisedFilesPath = Configuration.GUM_TREE_INPUT + "revFiles/";
-		final String positionsFilePath = Configuration.GUM_TREE_INPUT + "positions/";
-		final String diffentryFilePath = Configuration.GUM_TREE_INPUT + "diffentries/";
-		FileHelper.createDirectory(previousFilesPath);
-		FileHelper.createDirectory(revisedFilesPath);
-		FileHelper.createDirectory(positionsFilePath);
-		FileHelper.createDirectory(diffentryFilePath);
-		
-		ViolationParser parser = new ViolationParser();
-		parser.parseViolations(fixedAlarmFile, repositoriesList, previousFilesPath, revisedFilesPath, positionsFilePath, diffentryFilePath);
+//		final String previousFilesPath = Configuration.GUM_TREE_INPUT + "prevFiles/";
+//		final String revisedFilesPath = Configuration.GUM_TREE_INPUT + "revFiles/";
+//		final String positionsFilePath = Configuration.GUM_TREE_INPUT + "positions/";
+//		final String diffentryFilePath = Configuration.GUM_TREE_INPUT + "diffentries/";
+//		FileHelper.createDirectory(previousFilesPath);
+//		FileHelper.createDirectory(revisedFilesPath);
+//		FileHelper.createDirectory(positionsFilePath);
+//		FileHelper.createDirectory(diffentryFilePath);
+//		
+//		ViolationParser parser = new ViolationParser();
+//		parser.parseViolations(fixedAlarmFile, repositoriesList, previousFilesPath, revisedFilesPath, positionsFilePath, diffentryFilePath);
 	}
 
 }

src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java

@@ -156,16 +156,16 @@ public class ViolationParser {
 	/**
 	 * Output data in terms of alarm types.
 	 * 
-	 * @param fixedAlarmFile
+	 * @param alarmsInfo
 	 * @param repos
 	 * @param previousFilesPath
 	 * @param revisedFilesPath
 	 * @param positionsFilePath
 	 * @param diffentryFilePath
 	 */
-	public void parseViolations2(String fixedAlarmFile, List<File> repos, String previousFilesPath, String revisedFilesPath, String positionsFilePath, String diffentryFilePath) {
+	public void parseViolations(File alarmsInfo, List<File> repos, String violationFilesPath, String positionsFilePath) {
 		AlarmsReader reader = new AlarmsReader();
-		Map<String, Violation> violations = reader.readAlarmsList(fixedAlarmFile);
+		Map<String, Violation> violations = reader.readAlarmsList2(alarmsInfo.getPath());
 		int a = 0;
 		for (Map.Entry<String , Violation> entry : violations.entrySet()) {
 			String projectName = entry.getKey();
@@ -185,7 +185,7 @@ public class ViolationParser {
 			List<Alarm> alarms = violation.getAlarms();
 			
 			String repoPath = repoName + "/.git";
-			GitRepository gitRepo = new GitRepository(repoPath, revisedFilesPath, previousFilesPath);
+			GitRepository gitRepo = new GitRepository(repoPath, "", "");
 			try {
 				gitRepo.open();
 				for (Alarm alarm : alarms) {
@@ -197,36 +197,18 @@ public class ViolationParser {
 						continue;
 					}
 					
-					String fixedCommitId = alarm.getFixedCommitId();
-					String fixedFileName = alarm.getFixedFileName();
-					String fixedFileContent = gitRepo.getFileContentByCommitIdAndFileName(fixedCommitId, fixedFileName);
-					if (fixedFileContent == null || "".equals(fixedFileContent)) {
-						System.out.println(projectName);
-						continue;
-					}
-					
-					String diffentry = gitRepo.getDiffentryByTwoCommitIds(buggyCommitId, fixedCommitId, fixedFileName);
-					if (diffentry == null) {
-						System.out.println(projectName);
-						continue;
-					}
-					
-					String commitId = buggyCommitId.substring(0, 6) + "_" + fixedCommitId.substring(0, 6);
-					String fileName = fixedFileName.replaceAll("/", "#");
+					String commitId = buggyCommitId.substring(0, 6) + "_";
+					String fileName = buggyFileName.replaceAll("/", "#");
 					fileName = projectName + "_" + commitId + fileName;
 					if (fileName.length() > 240) {
-						List<File> files = FileHelper.getAllFilesInCurrentDiectory(revisedFilesPath, ".java");
+						List<File> files = FileHelper.getAllFilesInCurrentDiectory(violationFilesPath, ".java");
 						fileName = files.size() + "TooLongFileName.java";
 					}
-					String buggyFile = previousFilesPath + "prev_" + fileName;
-					String fixedFile = revisedFilesPath + fileName;
+					String buggyFile = violationFilesPath + "unfixed_" + fileName;
 					fileName = fileName.replace(".java", ".txt");
 					String positionFile = positionsFilePath + fileName;
-					String diffentryFile = diffentryFilePath + fileName;
 					FileHelper.outputToFile(buggyFile, buggyFileContent, false);
-					FileHelper.outputToFile(fixedFile, fixedFileContent, false);
 					FileHelper.outputToFile(positionFile, readPosition(alarm.getPositions(), alarm.getAlarmTypes()), false);
-					FileHelper.outputToFile(diffentryFile, diffentry, false);
 				}
 			} catch (GitRepositoryNotFoundException e) {
 				System.out.println("Exception: " + projectName);