Improve the algorithm of parsing violation information.

src/main/java/edu/lu/uni/serval/violation/Alarm.java

@@ -1,5 +1,6 @@
 package edu.lu.uni.serval.violation;
 
+import java.util.List;
 import java.util.Map;
 
 public class Alarm {
@@ -8,8 +9,11 @@ public class Alarm {
 	private String buggyFileName;
 	private String fixedCommitId;
 	private String fixedFileName;
+	@Deprecated
 	private Map<Integer, String> alarmTypes;
+	@Deprecated
 	private Map<Integer, Integer> positions;
+	private List<String> alarmTypesAndPositions;
 	
 	public Alarm(String buggyCommitId, String buggyFileName, String fixedCommitId, String fixedFileName) {
 		super();
@@ -35,22 +39,34 @@ public class Alarm {
 		return fixedFileName;
 	}
 
+	@Deprecated
 	public Map<Integer, String> getAlarmTypes() {
 		return alarmTypes;
 	}
 
+	@Deprecated
 	public void setAlarmTypes(Map<Integer, String> alarmTypes) {
 		this.alarmTypes = alarmTypes;
 	}
 
+	@Deprecated
 	public Map<Integer, Integer> getPositions() {
 		return positions;
 	}
 
+	@Deprecated
 	public void setPositions(Map<Integer, Integer> positions) {
 		this.positions = positions;
 	}
 
+	public List<String> getAlarmTypesAndPositions() {
+		return alarmTypesAndPositions;
+	}
+
+	public void setAlarmTypesAndPositions(List<String> alarmTypesAndPositions) {
+		this.alarmTypesAndPositions = alarmTypesAndPositions;
+	}
+
 	@Override
 	public boolean equals(Object obj) {
 		if (obj instanceof Alarm) {

src/main/java/edu/lu/uni/serval/violation/parse/AlarmsReader.java

@@ -3,19 +3,27 @@ package edu.lu.uni.serval.violation.parse;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Scanner;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import edu.lu.uni.serval.violation.Alarm;
 import edu.lu.uni.serval.violation.Violation;
 
 public class AlarmsReader {
+	
+	@SuppressWarnings("unused")
+	private static Logger log = LoggerFactory.getLogger(AlarmsReader.class);
 
 	int counter = 0;
 	
 	public Map<String, Violation> readAlarmsList(String fileName) {
+//		StringBuilder builder = new StringBuilder();
 		Map<String, Violation> violationsMap = new HashMap<>();
 		FileInputStream fis = null;
 		Scanner scanner = null;
@@ -31,16 +39,24 @@ public class AlarmsReader {
 				String[] buggyElements = buggyInfo.split(":");
 				String[] fixedElements = fixedInfo.split(":");
 
-				String alarmType = buggyElements[0];
+//				String alarmType = buggyElements[0];
 				String projectName = buggyElements[1];
 				String buggyCommitId = buggyElements[2];
 				String buggyFile = buggyElements[3];
-				int startLine = Integer.parseInt(buggyElements[4]);
+//				int startLine = Integer.parseInt(buggyElements[4]);
-				int endLine = Integer.parseInt(buggyElements[5]);
+//				int endLine = Integer.parseInt(buggyElements[5]);
 				String fixCommitId = fixedElements[1];
 				String fixedFile = fixedElements[2];
 				
+//				if (startLine == -1 || endLine == -1 || endLine == 1) {
+////					log.error("FIXED ALARM WRONG_POSITION: " + line);
+//					builder.append(line + "\n");
+//					continue;
+//				}
+				
 				Alarm alarm = new Alarm(buggyCommitId, buggyFile, fixCommitId, fixedFile);
+//				String alarmTypeAndPosition = buggyElements[0] + ":" + startLine + ":" + endLine;
+				String alarmTypeAndPosition = buggyElements[0] + ":" + buggyElements[4] + ":" + buggyElements[5];
 				
 				Violation violation;
 				if (violationsMap.containsKey(projectName)) {
@@ -53,25 +69,29 @@ public class AlarmsReader {
 				int index = alarms.indexOf(alarm);
 				if (index >= 0) {
 					Alarm tempAlarm = alarms.get(index);
-					Map<Integer, Integer> positions = tempAlarm.getPositions();
+//					Map<Integer, Integer> positions = tempAlarm.getPositions();
-					if (positions.containsKey(startLine)) {
+//					if (positions.containsKey(startLine)) {
-						int end = positions.get(startLine);
+//						int end = positions.get(startLine);
-						if (endLine < end) {
+//						if (endLine < end) {
-							positions.put(startLine, endLine);
+//							positions.put(startLine, endLine);
-							tempAlarm.getAlarmTypes().put(startLine, alarmType);
+//							tempAlarm.getAlarmTypes().put(startLine, alarmType);
-						}
+//						}
-					} else {
+//					} else {
-						positions.put(startLine, endLine);
+//						positions.put(startLine, endLine);
-						tempAlarm.getAlarmTypes().put(startLine, alarmType);
+//						tempAlarm.getAlarmTypes().put(startLine, alarmType);
 						counter ++;
-					}
+//					}
+					tempAlarm.getAlarmTypesAndPositions().add(alarmTypeAndPosition);
 				} else {
-					Map<Integer, String> alarmTypes = new HashMap<>();
+//					Map<Integer, String> alarmTypes = new HashMap<>();
-					alarmTypes.put(startLine, alarmType);
+//					alarmTypes.put(startLine, alarmType);
-					alarm.setAlarmTypes(alarmTypes);
+//					alarm.setAlarmTypes(alarmTypes);
-					Map<Integer, Integer> positions = new HashMap<>();
+//					Map<Integer, Integer> positions = new HashMap<>();
-					positions.put(startLine, endLine);
+//					positions.put(startLine, endLine);
-					alarm.setPositions(positions);
+//					alarm.setPositions(positions);
+					List<String> alarmTypesAndPositions = new ArrayList<>();
+					alarmTypesAndPositions.add(alarmTypeAndPosition);
+					alarm.setAlarmTypesAndPositions(alarmTypesAndPositions);
 					alarms.add(alarm);
 					counter ++;
 				}
@@ -88,10 +108,12 @@ public class AlarmsReader {
 		}
 		
 		System.out.println(counter);
+//		FileHelper.outputToFile("../FPM_Violations/TuneParameters/WrongPosition/FixedAlarm.list", builder, false);
 		return violationsMap;
 	}
 	
 	public Map<String, Violation> readAlarmsList2(String fileName) {
+//		StringBuilder builder = new StringBuilder();
 		Map<String, Violation> violationsMap = new HashMap<>();
 		FileInputStream fis = null;
 		Scanner scanner = null;
@@ -106,12 +128,19 @@ public class AlarmsReader {
 
 				String[] buggyElements = alarmInfo.split(",");
 
-				String alarmType = buggyElements[0];
+//				String alarmType = buggyElements[0];
 				String projectName = buggyElements[1];
 				String buggyCommitId = buggyElements[2];
 				String buggyFile = buggyElements[3];
-				int startLine = Integer.parseInt(buggyElements[4]);
+//				int startLine = Integer.parseInt(buggyElements[4]);
-				int endLine = Integer.parseInt(buggyElements[5]);
+//				int endLine = Integer.parseInt(buggyElements[5]);
+//				if (startLine == -1 || endLine == -1 || endLine == 1) {
+////					log.error("UNFIXED ALARM WRONG_POSITION: " + line);
+//					builder.append(line + "\n");
+//					continue;
+//				}
+//				String alarmTypeAndPosition = buggyElements[0] + ":" + startLine + ":" + endLine; // Alarm type : start line : end line.
+				String alarmTypeAndPosition = buggyElements[0] + ":" + buggyElements[4] + ":" + buggyElements[5]; // Alarm type : start line : end line.
 				
 				Alarm alarm = new Alarm(buggyCommitId, buggyFile, "", "");
 				
@@ -126,25 +155,29 @@ public class AlarmsReader {
 				int index = alarms.indexOf(alarm);
 				if (index >= 0) {
 					Alarm tempAlarm = alarms.get(index);
-					Map<Integer, Integer> positions = tempAlarm.getPositions();
+//					Map<Integer, Integer> positions = tempAlarm.getPositions();
-					if (positions.containsKey(startLine)) {
+//					if (positions.containsKey(startLine)) {
-						int end = positions.get(startLine);
+//						int end = positions.get(startLine);
-						if (endLine < end) {
+//						if (endLine < end) {
-							positions.put(startLine, endLine);
+//							positions.put(startLine, endLine);
-							tempAlarm.getAlarmTypes().put(startLine, alarmType);
+//							tempAlarm.getAlarmTypes().put(startLine, alarmType);
-						}
+//						}
-					} else {
+//					} else {
-						positions.put(startLine, endLine);
+//						positions.put(startLine, endLine);
-						tempAlarm.getAlarmTypes().put(startLine, alarmType);
+//						tempAlarm.getAlarmTypes().put(startLine, alarmType);
 						counter ++;
-					}
+//					}
+					tempAlarm.getAlarmTypesAndPositions().add(alarmTypeAndPosition);
 				} else {
-					Map<Integer, String> alarmTypes = new HashMap<>();
+//					Map<Integer, String> alarmTypes = new HashMap<>();
-					alarmTypes.put(startLine, alarmType);
+//					alarmTypes.put(startLine, alarmType);
-					alarm.setAlarmTypes(alarmTypes);
+//					alarm.setAlarmTypes(alarmTypes);
-					Map<Integer, Integer> positions = new HashMap<>();
+//					Map<Integer, Integer> positions = new HashMap<>();
-					positions.put(startLine, endLine);
+//					positions.put(startLine, endLine);
-					alarm.setPositions(positions);
+//					alarm.setPositions(positions);
+					List<String> alarmTypesAndPositions = new ArrayList<>();
+					alarmTypesAndPositions.add(alarmTypeAndPosition);
+					alarm.setAlarmTypesAndPositions(alarmTypesAndPositions);
 					alarms.add(alarm);
 					counter ++;
 				}
@@ -161,6 +194,7 @@ public class AlarmsReader {
 		}
 		
 		System.out.println(counter);
+//		FileHelper.outputToFile("../FPM_Violations/TuneParameters/WrongPosition/UnFixedAlarm.list", builder, true);
 		return violationsMap;
 	}
 	

src/main/java/edu/lu/uni/serval/violation/parse/TestViolationParser.java

@@ -1,6 +1,7 @@
 package edu.lu.uni.serval.violation.parse;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -11,7 +12,7 @@ public class TestViolationParser {
 
 	private static final String REPO_PATH = "/Volumes/MacBook/repos/";
 	
-	public static void main(String[] args) {
+	public static void main(String[] args) throws IOException {
 		List<File> repositoriesList = new ArrayList<>();
 		File repositories = new File(REPO_PATH);
 		File[] subFiles = repositories.listFiles();
@@ -27,86 +28,31 @@ public class TestViolationParser {
 		}
 		
 		String unfixedAlarmFile = "Dataset/Unfixed-Alarms/";
-		final String previousFilesPath = Configuration.GUM_TREE_INPUT + "unfixAlarms/";
+		final String unfixedFilesPath = Configuration.GUM_TREE_INPUT + "unfixAlarms/";
-		final String positionsFilePath = Configuration.GUM_TREE_INPUT + "un_positions/";
+		final String unfixedOositionsFilePath = Configuration.GUM_TREE_INPUT + "un_positions/";
-		FileHelper.createDirectory(previousFilesPath);
+		FileHelper.createDirectory(unfixedFilesPath);
-		FileHelper.createDirectory(positionsFilePath);
+		FileHelper.createDirectory(unfixedOositionsFilePath);
 		
 		List<File> unfixedAlarmFiles = FileHelper.getAllFilesInCurrentDiectory(unfixedAlarmFile, ".csv");
 		
 		for (File file : unfixedAlarmFiles) {
 			ViolationParser parser = new ViolationParser();
-			parser.parseViolations(file, repositoriesList, previousFilesPath, positionsFilePath);
+			parser.parseViolations(file, repositoriesList, unfixedFilesPath, unfixedOositionsFilePath);
 		}
-	
 		
-//		String fixedAlarmFile = "Dataset/fixed-alarms-v0.3.list.txt";
+		String fixedAlarmFile = "Dataset/fixed-alarms-v1.0.list";
-
+		final String previousFilesPath = Configuration.GUM_TREE_INPUT + "prevFiles/";
-		/**
+		final String revisedFilesPath = Configuration.GUM_TREE_INPUT + "revFiles/";
-		 * 544 projects.
+		final String positionsFilePath = Configuration.GUM_TREE_INPUT + "positions/";
-		 * 42322 files
+		final String diffentryFilePath = Configuration.GUM_TREE_INPUT + "diffentries/";
-		 * 84103 alarms.
+		FileHelper.createDirectory(previousFilesPath);
-		 * 19928 parsed instances.  1984
+		FileHelper.createDirectory(revisedFilesPath);
-
+		FileHelper.createDirectory(positionsFilePath);
-		 * 22 bugs
+		FileHelper.createDirectory(diffentryFilePath);
-		 * 
+		
-NP_ALWAYS_NULL : org/jfree/chart/renderer/category/AbstractCategoryItemRenderer.java : 1800 : 1800
+		ViolationParser parser = new ViolationParser();
-Chart_1.xml
+		parser.parseViolations(fixedAlarmFile, repositoriesList, previousFilesPath, revisedFilesPath, positionsFilePath, diffentryFilePath);
-UC_USELESS_CONDITION : org/jfree/data/KeyedObjects2D.java : 231 : 231
+	
-Chart_22.xml
-DLS_DEAD_LOCAL_STORE : org/jfree/chart/renderer/GrayPaintScale.java : 125 : 125
-Chart_24.xml
-NP_NULL_ON_SOME_PATH : org/jfree/chart/plot/XYPlot.java : 4493 : 4493
-Chart_4.xml
-SF_SWITCH_NO_DEFAULT : com/google/javascript/jscomp/UnreachableCodeElimination.java : 151 : 171
-Closure_127.xml
-DLS_DEAD_LOCAL_STORE : com/google/javascript/jscomp/ScopedAliases.java : 276 : 276
-Closure_24.xml
-SF_SWITCH_NO_DEFAULT : com/google/javascript/jscomp/TypedScopeCreator.java : 550 : 580
-Closure_43.xml
-UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR : com/google/debugging/sourcemap/SourceMapConsumerV3.java : 488 : 488
-Closure_47.xml
-SF_SWITCH_NO_DEFAULT : com/google/javascript/jscomp/MakeDeclaredNamesUnique.java : 116 : 147
-Closure_49.xml
-DLS_DEAD_LOCAL_STORE : com/google/javascript/jscomp/CollapseProperties.java : 482 : 482
-Closure_89.xml
-SF_SWITCH_NO_DEFAULT : org/apache/commons/lang3/time/FastDateParser.java : 315 : 338
-Lang_10.xml
-DM_CONVERT_CASE : org/apache/commons/lang/StringUtils.java : 1048 : 1048
-Lang_40.xml
-SF_SWITCH_NO_DEFAULT : org/apache/commons/lang/math/NumberUtils.java : 449 : 491
-Lang_58.xml
-SF_SWITCH_NO_DEFAULT : org/apache/commons/lang/Entities.java : 916 : 919
-Lang_62.xml
-FE_FLOATING_POINT_EQUALITY : org/apache/commons/math3/util/FastMath.java : 1545 : 1545
-Math_15.xml
-DLS_DEAD_LOCAL_STORE : org/apache/commons/math/optimization/direct/BOBYQAOptimizer.java : 1658 : 1658
-Math_38.xml
-FE_FLOATING_POINT_EQUALITY : org/apache/commons/math/analysis/solvers/BaseSecantSolver.java : 187 : 187
-Math_50.xml
-CO_COMPARETO_INCORRECT_FLOATING : org/apache/commons/math/fraction/Fraction.java : 261 : 261
-Math_91.xml
-DLS_DEAD_LOCAL_STORE : org/mockito/internal/invocation/InvocationMatcher.java : 122 : 122
-Mockito_1.xml
-EQ_CHECK_FOR_OPERAND_NOT_COMPATIBLE_WITH_THIS : org/mockito/internal/creation/DelegatingMethod.java : 55 : 55
-Mockito_11.xml
-REC_CATCH_EXCEPTION : org/mockito/internal/creation/instance/ConstructorInstantiator.java : 26 : 26
-Mockito_21.xml
-DM_NUMBER_CTOR : org/joda/time/chrono/ZonedChronology.java : 469 : 469
-Time_26.xml
-		 */
-
-//		final String previousFilesPath = Configuration.GUM_TREE_INPUT + "prevFiles/";
-//		final String revisedFilesPath = Configuration.GUM_TREE_INPUT + "revFiles/";
-//		final String positionsFilePath = Configuration.GUM_TREE_INPUT + "positions/";
-//		final String diffentryFilePath = Configuration.GUM_TREE_INPUT + "diffentries/";
-//		FileHelper.createDirectory(previousFilesPath);
-//		FileHelper.createDirectory(revisedFilesPath);
-//		FileHelper.createDirectory(positionsFilePath);
-//		FileHelper.createDirectory(diffentryFilePath);
-//		
-//		ViolationParser parser = new ViolationParser();
-//		parser.parseViolations(fixedAlarmFile, repositoriesList, previousFilesPath, revisedFilesPath, positionsFilePath, diffentryFilePath);
 	}
-
+	
 }

src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java

@@ -28,6 +28,16 @@ public class ViolationParser {
 	
 	Map<String, Integer> alarmTypesCounter = new HashMap<>();
 	
+	/**
+	 * Travel Git repositories to get the violation-localized java files, violation-removed java files, and violation types and positions.
+	 * 
+	 * @param fixedAlarmFile
+	 * @param repos
+	 * @param previousFilesPath
+	 * @param revisedFilesPath
+	 * @param positionsFilePath
+	 * @param diffentryFilePath
+	 */
 	public void parseViolations(String fixedAlarmFile, List<File> repos, String previousFilesPath, String revisedFilesPath, String positionsFilePath, String diffentryFilePath) {
 		AlarmsReader reader = new AlarmsReader();
 		Map<String, Violation> violations = reader.readAlarmsList(fixedAlarmFile);
@@ -35,7 +45,7 @@ public class ViolationParser {
 		int a = 0;
 		int exceptionsCounter = 0;
 		int violationsAmount = 0;
-		System.out.println(violations.size());
+//		System.out.println(violations.size());
 		for (Map.Entry<String , Violation> entry : violations.entrySet()) {
 			String projectName = entry.getKey();
 			String repoName = "";
@@ -100,7 +110,8 @@ public class ViolationParser {
 					String diffentryFile = diffentryFilePath + fileName;
 					FileHelper.outputToFile(buggyFile, buggyFileContent, false);
 					FileHelper.outputToFile(fixedFile, fixedFileContent, false);
-					FileHelper.outputToFile(positionFile, readPosition(alarm.getPositions(), alarm.getAlarmTypes()), false);
+//					FileHelper.outputToFile(positionFile, readPosition(alarm.getPositions(), alarm.getAlarmTypes()), false);
+					FileHelper.outputToFile(positionFile, readAlarmTypeAndPosition(alarm.getAlarmTypesAndPositions()), false);
 					FileHelper.outputToFile(diffentryFile, diffentry, false);
 
 					violationsAmount += counter(alarm);
@@ -135,21 +146,33 @@ public class ViolationParser {
 		MapSorter<String, Integer> sorter = new MapSorter<String, Integer>();
 		alarmTypesCounter = sorter.sortByKeyAscending(alarmTypesCounter);
 		String[] columns = { "Alarm Type", "amount" };
-		Exporter.exportOutliers(alarmTypesCounter, new File(Configuration.GUM_TREE_INPUT + "AlarmTypes.xls"), 1, columns);
+		Exporter.exportOutliers(alarmTypesCounter, new File(Configuration.GUM_TREE_INPUT + "FixedAlarmTypes.xls"), 1, columns);
 	}
 	
 	private int counter(Alarm alarm) {
 		int counter = 0;
-		Map<Integer, String> alarmTypes = alarm.getAlarmTypes();
+//		Map<Integer, String> alarmTypes = alarm.getAlarmTypes();
-		counter += alarmTypes.size();
+//		counter += alarmTypes.size();
-		for (Map.Entry<Integer, String> entry : alarmTypes.entrySet()) {
+//		for (Map.Entry<Integer, String> entry : alarmTypes.entrySet()) {
-			String type = entry.getValue();
+//			String type = entry.getValue();
-			if (this.alarmTypesCounter.containsKey(entry.getValue())) {
+//			if (this.alarmTypesCounter.containsKey(entry.getValue())) {
-				this.alarmTypesCounter.put(type, this.alarmTypesCounter.get(type) + 1);
+//				this.alarmTypesCounter.put(type, this.alarmTypesCounter.get(type) + 1);
+//			} else {
+//				this.alarmTypesCounter.put(type, 1);
+//			}
+//		}
+		
+		List<String> alarmTypesAndPositions = alarm.getAlarmTypesAndPositions();
+		for (String alarmTypeAndPosition : alarmTypesAndPositions) {
+			String[] elements = alarmTypeAndPosition.split(":");
+			String alarmType = elements[0];
+			if (this.alarmTypesCounter.containsKey(alarmType)) {
+				this.alarmTypesCounter.put(alarmType, this.alarmTypesCounter.get(alarmType) + 1);
 			} else {
-				this.alarmTypesCounter.put(type, 1);
+				this.alarmTypesCounter.put(alarmType, 1);
 			}
 		}
+		counter += alarmTypesAndPositions.size();
 		return counter;
 	}
 
@@ -208,7 +231,7 @@ public class ViolationParser {
 					fileName = fileName.replace(".java", ".txt");
 					String positionFile = positionsFilePath + fileName;
 					FileHelper.outputToFile(buggyFile, buggyFileContent, false);
-					FileHelper.outputToFile(positionFile, readPosition(alarm.getPositions(), alarm.getAlarmTypes()), false);
+					FileHelper.outputToFile(positionFile, readAlarmTypeAndPosition(alarm.getAlarmTypesAndPositions()), false);
 				}
 			} catch (GitRepositoryNotFoundException e) {
 				System.out.println("Exception: " + projectName);
@@ -229,7 +252,17 @@ public class ViolationParser {
 		System.out.println(a);
 	}
 
-	private String readPosition(Map<Integer, Integer> positions, Map<Integer, String> alarmTypes) {
+	private String readAlarmTypeAndPosition(List<String> alarmTypesAndPositions) {
+		String positionsStr = "";
+		for (String element : alarmTypesAndPositions) {
+			positionsStr += element + "\n";
+		}
+		return positionsStr;
+	}
+	
+	@SuppressWarnings("unused")
+	@Deprecated
+	private String readAlarmTypeAndPosition(Map<Integer, Integer> positions, Map<Integer, String> alarmTypes) {
 		String positionsStr = "";
 		for (Map.Entry<Integer, String> entry : alarmTypes.entrySet()) {
 			int key = entry.getKey();
@@ -237,4 +270,5 @@ public class ViolationParser {
 		}
 		return positionsStr;
 	}
+	
 }