Add more information.

src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java

@@ -19,7 +19,6 @@ import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
 import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter;
 import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
 import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
-import edu.lu.uni.serval.utils.ListSorter;
 
 /**
  * Parse fix violations with GumTree in terms of multiple statements.
@@ -33,10 +32,13 @@ public class FixedViolationHunkParser extends FixedViolationParser {
 	 * ResultType:
 	 * 0: normal GumTree results.
 	 * 1: null GumTree result.
+	 * 2: No source code changes.
 	 */
 	public int resultType = 0;
 	public int nullMappingGumTreeResult = 0;
 	public int pureDeletions = 0;
+	public int largeHunk = 0;
+	public int nullSourceCode = 0;
 	
 	public FixedViolationHunkParser() {
 	}
@@ -51,33 +53,36 @@ public class FixedViolationHunkParser extends FixedViolationParser {
 		// TODO remove the modification of variable names or not?
 		List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); // only remove non-statement source code, eg. method declaration 
 		
-		if (actionSets.size() > 0) {
+		if (actionSets == null) {
+			this.resultType = 1;
+		} else if (actionSets.size() == 0) {
+			this.resultType = 2;
+		} else {
 			List<Violation> violations = readPositionsAndAlarmTypes();
-			if (violations.size() > 1) {
-				ListSorter<Violation> sorter = new ListSorter<>(violations);
-				violations = sorter.sortAscending();
-			}
 			
 			List<DiffEntryHunk> diffentryHunks1 = new DiffEntryReader().readHunks2(diffentryFile);
 			// Select hunks by positions of violations.
 			for (Violation violation : violations) {
-				int startLineNum = violation.getStartLineNum();
+				int violationStartLineNum = violation.getStartLineNum();
-				int endLineNum = violation.getEndLineNum();
+				int violationEndLineNum = violation.getEndLineNum();
 				for (int index = 0, hunkListSize = diffentryHunks1.size(); index < hunkListSize; index ++) {
 					DiffEntryHunk hunk = diffentryHunks1.get(index);
 					int startLine = hunk.getBugLineStartNum();
 					int range = hunk.getBugRange();
-					if (startLineNum > startLine + range) continue;
+					if (violationStartLineNum > startLine + range - 1) continue;
-					if (endLineNum < startLine) break;
+					if (violationEndLineNum < startLine) break;
 					
 					if (violation.getBugStartLineNum() == 0) {
 						violation.setBugStartLineNum(startLine);
 						violation.setFixStartLineNum(hunk.getFixLineStartNum());
 					}
-					violation.setBugEndLineNum(startLine + range);
+					violation.setBugEndLineNum(startLine + range - 1);
-					violation.setFixEndLineNum(hunk.getFixLineStartNum() + hunk.getFixRange());
+					violation.setFixEndLineNum(hunk.getFixLineStartNum() + hunk.getFixRange() - 1);
 					violation.getHunks().add(hunk);
 				}
+				if (violation.getBugStartLineNum() == 0 && violation.getBugEndLineNum() == 0) {
+					System.err.println("WRONG");
+				}
 			}
 			
 			//Filter out the modify actions, which are not in the DiffEntry hunks.
@@ -195,7 +200,10 @@ public class FixedViolationHunkParser extends FixedViolationParser {
 					isInsert = true;
 //					continue;
 				}
-				if (bugEndLine - bugStartLine >= Configuration.HUNK_SIZE || fixEndLine - fixStartLine >= Configuration.HUNK_SIZE) continue;
+				if (bugEndLine - bugStartLine > Configuration.HUNK_SIZE || fixEndLine - fixStartLine > Configuration.HUNK_SIZE) {
+					this.largeHunk ++;
+					continue;
+				}
 
 				simpleTree.setChildren(children);
 				simpleTree.setParent(null);
@@ -203,7 +211,7 @@ public class FixedViolationHunkParser extends FixedViolationParser {
 				// Source Code of patches.
 				String patchSourceCode = getPatchSourceCode(prevFile, revFile, bugStartLine, bugEndLine, fixStartLine, fixEndLine, isInsert);
 				if ("".equals(patchSourceCode)) {
-					this.nullMappingGumTreeResult ++;
+					this.nullSourceCode ++;
 					continue;
 				}
 				
@@ -218,22 +226,21 @@ public class FixedViolationHunkParser extends FixedViolationParser {
 				// 4. semi-source code: 
 				String[] editScriptTokens = astEditScripts.split(" ");
 				int size = editScriptTokens.length;
-				if (size == 1) {
+//				if (size == 1) {
-					this.nullMappingGumTreeResult ++;
+//					this.nullMappingGumTreeResult ++;
-					continue;
+//					continue;
-				}
+//				}
 				String alarmType = violation.getAlarmType();
 				
 //				String patchPosition = "\n" + revFile.getName() + "Position: " + violation.getStartLineNum() + " --> "  + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n";
 				String patchPosition = "\n" + "Position: " + violation.getStartLineNum() + " --> "  + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n";
 				String info = Configuration.PATCH_SIGNAL + "\nAlarm Type :" + violation.getAlarmType() + "\n" + patchPosition + patchSourceCode + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets, bugEndPosition, fixEndPosition) + "\n";
 				if (noUpdate(editScriptTokens)) {
-//					System.err.println(info);
-					this.testingInfo += info;
 					
 					if (!"SE_NO_SERIALVERSIONID".equals(alarmType)) {
 						if (containsFiledDeclaration(hunkActionSets)) {
 							this.nullMappingGumTreeResult ++;
+							this.testingInfo += info + revFile.getName() + "\n";
 							continue;
 						}
 					}
@@ -252,6 +259,7 @@ public class FixedViolationHunkParser extends FixedViolationParser {
 						
 					} else {
 						this.nullMappingGumTreeResult ++;
+						this.testingInfo += info + revFile.getName() + "\n";
 						continue;
 					}
 				}
@@ -270,8 +278,6 @@ public class FixedViolationHunkParser extends FixedViolationParser {
 //				this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n";
 				
 			}
-		} else {
-			this.resultType = 1;
 		}
 	}
 

src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java

@@ -45,7 +45,11 @@ public class FixedViolationParser extends Parser {
 		List<HierarchicalActionSet> actionSets = new ArrayList<>();
 		// GumTree results
 		List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
-		if (gumTreeResults != null && gumTreeResults.size() > 0) {
+		if (gumTreeResults == null) {
+			return null;
+		} else if (gumTreeResults.size() == 0){
+			return actionSets;
+		} else {
 			// Regroup GumTre results.
 			List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
 			for (HierarchicalActionSet actionSet : allActionSets) {
@@ -54,9 +58,9 @@ public class FixedViolationParser extends Parser {
 					actionSets.add(actionSet);
 				}
 			}
+			
+			return actionSets;
 		}
-		
-		return actionSets;
 	}
 
 	protected List<Violation> readPositionsAndAlarmTypes() {

src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java

@@ -70,6 +70,7 @@ public class ParseFixPatternWorker extends UntypedActor {
 			StringBuilder sizes = new StringBuilder();
 			StringBuilder tokens = new StringBuilder();
 			StringBuilder alarmTypes = new StringBuilder();
+			StringBuilder testingInfo = new StringBuilder();
 
 			int id = msg.getId();
 			int counter = 0;
@@ -77,8 +78,11 @@ public class ParseFixPatternWorker extends UntypedActor {
 			int testAlarms = 0;
 			int nullGumTreeResults = 0;
 			int nullMappingGumTreeResults = 0;
+			int noSourceCodeChagnes = 0;
 			int pureDeletion = 0;
 			int expNums = 0;
+			int largeHunk = 0;
+			int nullSourceCode = 0;
 			
 			for (MessageFile msgFile : files) {
 				File revFile = msgFile.getRevFile();
@@ -107,11 +111,16 @@ public class ParseFixPatternWorker extends UntypedActor {
 					
 					nullMappingGumTreeResults += parser.nullMappingGumTreeResult;
 					pureDeletion += parser.pureDeletions;
+					largeHunk += parser.largeHunk;
+					nullSourceCode += parser.nullSourceCode;
+					testingInfo.append(parser.testingInfo);
 					
 					String editScript = parser.getAstEditScripts();
 					if ("".equals(editScript)) {
 						if (parser.resultType == 1) {
 							nullGumTreeResults += countAlarms(positionFile);
+						} else if (parser.resultType == 2) {
+							noSourceCodeChagnes += countAlarms(positionFile);
 						}
 					} else {
 						editScripts.append(editScript);
@@ -134,17 +143,19 @@ public class ParseFixPatternWorker extends UntypedActor {
 							FileHelper.outputToFile(alarmTypesFilePath + "alarmTypes_" + id + ".list", alarmTypes, true);
 							alarmTypes.setLength(0);
 							log.info("Worker #" + id +"Finish of parsing " + counter + " files...");
+							FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true);
+							testingInfo.setLength(0);
 						}
 					}
 				} catch (TimeoutException e) {
 					err.println("task timed out");
 					future.cancel(true);
-					expNums ++;
+					expNums += countAlarms(positionFile);
 				} catch (InterruptedException e) {
-					expNums ++;
+					expNums += countAlarms(positionFile);
 					err.println("task interrupted");
 				} catch (ExecutionException e) {
-					expNums ++;
+					expNums += countAlarms(positionFile);
 					err.println("task aborted");
 				} finally {
 					executor.shutdownNow();
@@ -163,8 +174,12 @@ public class ParseFixPatternWorker extends UntypedActor {
 				
 				FileHelper.outputToFile(alarmTypesFilePath + "alarmTypes_" + id + ".list", alarmTypes, true);
 				alarmTypes.setLength(0);
+				FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true);
+				testingInfo.setLength(0);
 			}
-			String statistic = "testAlarms: " + testAlarms + "\nnullGumTreeResults: " + nullGumTreeResults + "\nnullMappingGumTreeResults: " + nullMappingGumTreeResults + "\npureDeletion: " + pureDeletion + "\nTimeout: " + expNums;
+			String statistic = "testAlarms: " + testAlarms + "\nnullGumTreeResults: " + nullGumTreeResults + "\nnullMappingGumTreeResults: " + nullMappingGumTreeResults +
+					"\nnoSourceCodeChagnes: " + noSourceCodeChagnes + "\npureDeletion: " + pureDeletion + "\nTimeout: " + expNums +
+					"\nlargeHunk: " + largeHunk + "\nnullSourceCode: " + nullSourceCode;
 			FileHelper.outputToFile("OUTPUT/statistic_" + id + ".list", statistic, false);
 
 			log.info("Worker #" + id +"Finish of parsing " + counter + " files...");

src/main/java/edu/lu/uni/serval/config/Configuration.java

@@ -6,7 +6,7 @@ public class Configuration {
 	
 	private static final String ROOT_PATH = "../FPM_Violations/"; // The root path of all output data.
 
-	public static final int HUNK_SIZE = 7; // The limitation of source code lines of each DiffEntry, which will be selected as training data.
+	public static final int HUNK_SIZE = 10; // The limitation of source code lines of each DiffEntry, which will be selected as training data.
 	public static final String BUGGY_TREE_SIGNAL = "BUGGY_TREE###"; // The starting signal of the tree of buggy source code .
 	public static final String PATCH_SIGNAL = "PATCH###"; // The starting signal of each patch.
 	

src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java

@@ -393,11 +393,11 @@ public class HunkActionFilter {
 								violation.getActionSets().add(actionSet);
 							}
 						} else {
-							if (isRanged(actionSet, violation)) {
+//							if (isRanged(actionSet, violation)) {
-								if (Math.abs(violationStartLine - actionFixStartLine) <= 5 
+//							}
-										&& Math.abs(violationEndLine - actionFixEndLine) <= 5) {
+							if (Math.abs(violationStartLine - actionFixStartLine) <= 5 
-									violation.getActionSets().add(actionSet);
+									&& Math.abs(violationEndLine - actionFixEndLine) <= 5) {
-								}
+								violation.getActionSets().add(actionSet);
 							}
 						}
 					}
@@ -432,7 +432,7 @@ public class HunkActionFilter {
 			int bugEndLine = bugStartLine + hunk.getBugRange();
 			int fixStartLine = hunk.getFixLineStartNum();
 			int fixEndLine = fixStartLine + hunk.getFixRange();
-			if (fixStartLine > actionEndLine || bugStartLine > violationEndLine) break;
+//			if (fixStartLine > actionEndLine || bugStartLine > violationEndLine) break;
 			if (fixEndLine < actionStartLine || bugEndLine < violationStartLine) continue;
 			
 			String hunkContent = hunk.getHunk();

src/main/java/edu/lu/uni/serval/violation/parse/Statistic.java

@@ -0,0 +1,61 @@
+package edu.lu.uni.serval.violation.parse;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.List;
+
+import edu.lu.uni.serval.utils.FileHelper;
+
+public class Statistic {
+	public static void main(String[] args) {
+		String statistic = "../FPM_Violations/OUTPUT";
+		List<File> files = FileHelper.getAllFiles(statistic, ".list");
+		int testAlarms = 0;
+		int nullGumTreeResults = 0;
+		int nullMappingGumTreeResults = 0;
+		int pureDeletion = 0;
+		int timeout = 0;
+		for (File file : files) {
+			String content = FileHelper.readFile(file);
+			BufferedReader reader = new BufferedReader(new StringReader(content));
+			String line = null;
+			try {
+				while ((line = reader.readLine()) !=  null) {
+					if (line.startsWith("test")) {
+						testAlarms += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim());
+					} else if (line.startsWith("nullGum")) {
+						nullGumTreeResults += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim());
+					} else if (line.startsWith("nullMap")) {
+						nullMappingGumTreeResults += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim());
+					} else if (line.startsWith("pure")) {
+						pureDeletion += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim());
+					} else if (line.startsWith("Time")) {
+						timeout += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim());
+					} 
+				}
+			} catch (IOException e) {
+				e.printStackTrace();
+			} finally {
+				try {
+					reader.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+		
+//		TestAlarms: 5175
+//		nullGumTreeResults: 13449
+//		nullMappingGumTreeResults: 33010
+//		pureDeletion: 7598
+//		Timeout: 263
+		
+		System.out.println("TestAlarms: " + testAlarms);
+		System.out.println("nullGumTreeResults: " + nullGumTreeResults);
+		System.out.println("nullMappingGumTreeResults: " + nullMappingGumTreeResults);
+		System.out.println("pureDeletion: " + pureDeletion);
+		System.out.println("Timeout: " + timeout);
+	}
+}