azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update.

Browse Source
This commit is contained in:
Kui LIU
2017-08-09 18:11:01 +02:00
parent 5b8ca570cd
commit 32ee706995
47 changed files with 1242 additions and 2510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pom.xml
-11
View File
@@ -16,17 +16,6 @@
</properties>
<dependencies>
<!-- <dependency>
<groupId>edu.lu.uni.serval</groupId>
<artifactId>MyCluster</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>edu.lu.uni.serval</groupId>
<artifactId>MyFeatureLearner</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency> -->
<dependency>
<groupId>edu.lu.uni</groupId>
src/main/java/edu/lu/uni/serval/FixPattern/info/FixPattern.java
-24
View File
@@ -1,24 +0,0 @@
package edu.lu.uni.serval.FixPattern.info;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
public class FixPattern {
private SimpleTree buggyCodeTree; // it will be used to compute the similarity.
private HierarchicalActionSet editScripts; // it will be used to generate new patches.
public SimpleTree getBuggyCodeTree() {
return buggyCodeTree;
}
public HierarchicalActionSet getEditScripts() {
return editScripts;
}
public FixPattern(SimpleTree buggyCodeTree, HierarchicalActionSet editScripts) {
super();
this.buggyCodeTree = buggyCodeTree;
this.editScripts = editScripts;
}
}
src/main/java/edu/lu/uni/serval/FixPattern/info/GumTreeAnalysis.java
-163
View File
@@ -1,163 +0,0 @@
package edu.lu.uni.serval.FixPattern.info;
//import java.io.File;
//import java.io.IOException;
//import java.util.ArrayList;
//import java.util.HashMap;
//import java.util.List;
//import java.util.Map;
import org.eclipse.jdt.core.dom.ASTParser;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
//import com.github.gumtreediff.actions.ActionGenerator;
//import com.github.gumtreediff.actions.model.Action;
//import com.github.gumtreediff.gen.jdt.JdtTreeGenerator;
//import com.github.gumtreediff.gen.jdt.cd.CdJdtTreeGenerator;
//import com.github.gumtreediff.matchers.Matcher;
//import com.github.gumtreediff.matchers.Matchers;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap;
import edu.lu.uni.serval.gen.jdt.exp.ExpJdtTreeGenerator;
@Deprecated
public class GumTreeAnalysis {
// private static void analyzeBugFixes(String gitRepoPath, String outputPath) {
// log.info("Repo: " + gitRepoPath);
//
// GitTraveller gitTraveller = new GitTraveller(gitRepoPath, outputPath);
// gitTraveller.travelGitRepo();
//// Map<String, List<CommitFile>> commitFiles = gitTraveller.getCommitFiles();
//
// List<MyDiffEntry> allDiffEntries = gitTraveller.getAllDiffEntries();
// String previousFilesPath = gitTraveller.getPreviousFilesPath();
// String revisedFilesPath = gitTraveller.getRevisedFilesPath();
// for (MyDiffEntry diff : allDiffEntries) {
// String fileA = previousFilesPath + diff.getPrevFile();
// String fileB = revisedFilesPath + diff.getRevFile();
// List<String> gumTreeResults = GumTreeAnalysis.compareTwoFilesWithGumTree(fileA, fileB);
// if (gumTreeResults.size() == 0) {
// continue;
// }
// StringBuilder builder = new StringBuilder();
// builder.append("Previous File: " + fileA + "\n");
// builder.append("Revised File: " + fileB + "\n");
// String diffs = "";
// for (ModifiedDetails md : diff.getModifiedDetails()) {
// diffs += md.getLineNumber() + "\n";
// diffs += md.getFragment() + "\n";
// }
// builder.append("DiffEntry: " + diffs);
// for (String gumTreeResult : gumTreeResults) {
// builder.append(gumTreeResult.toString() + "\n");
// }
// FileHelper.outputToFile("OUTPUT/GumTreeResults/" + FileHelper.getRepositoryName(gitRepoPath) + "/" + diff.getRevFile().replace(".java", ".txt"), builder, false);
// }
//
//// DiffEntryParser diffEntryParser = new DiffEntryParser(allDiffEntries);
//// diffEntryParser.parseDiffEntries();
////
//// // <String, List>: String ==> revisedFileName.
//// Map<String, List<ModifiedFragment>> parsedDiffEntries = diffEntryParser.getParsedDiffEntries();
//// diffEntryParser = null;
//// allDiffEntries = null;
////
//// gitTraveller = null;
////
//// for (Map.Entry<String, List<ModifiedFragment>> entry : parsedDiffEntries.entrySet()) {
//// String revisedFileName = entry.getKey();
//// String fileA = previousFilesPath + "prev_" + revisedFileName;
//// String fileB = revisedFilesPath + revisedFileName;
//// System.err.println("FileName" + fileA);
//// List<String> gumTreeResults = GumTreeAnalysis.compareTwoFilesWithGumTree(fileA, fileB);
//// StringBuilder builder = new StringBuilder();
//// builder.append("Previous File: " + fileA + "\n");
//// builder.append("Revised File: " + fileB + "\n");
//// builder.append("DiffEntry: ");
//// for (String gumTreeResult : gumTreeResults) {
//// builder.append(gumTreeResult + "\n");
//// }
//// FileHelper.outputToFile("OUTPUT/GumTreeResults/" + FileHelper.getRepositoryName(gitRepoPath) + "/" + revisedFileName.replace(".java", ".txt"), builder, false);
//// }
//
// }
//
// public static List<String> compareTwoFilesWithGumTree(String prevFile, String revFile) {
// List<String> gumTreeResults = new ArrayList<String>();
//
// try {
//// TreeContext tc1 = new ExpJdtTreeGenerator().generateFromFile(prevFile);
//// TreeContext tc2 = new ExpJdtTreeGenerator().generateFromFile(revFile);
//// TreeContext tc1 = new JdtTreeGenerator().generateFromFile(prevFile);
//// TreeContext tc2 = new JdtTreeGenerator().generateFromFile(revFile);
// TreeContext tc1 = new RowTokenJdtTreeGenerator().generateFromFile(prevFile);
// TreeContext tc2 = new RowTokenJdtTreeGenerator().generateFromFile(revFile);
//// TreeContext tc1 = new CdJdtTreeGenerator().generateFromFile(prevFile);
//// TreeContext tc2 = new CdJdtTreeGenerator().generateFromFile(revFile);
// ITree t1 = tc1.getRoot();
// ITree t2 = tc2.getRoot();
//
// Matcher m = Matchers.getInstance().getMatcher(t1, t2);
// m.match();
//
// ActionGenerator ag = new ActionGenerator(t1, t2, m.getMappings());
// ag.generate();
//
// List<Action> actions = ag.getActions();
// for(Action ac : actions){
// String actionStr = parseAction(ac.toString());
// gumTreeResults.add(actionStr);
// }
//
// } catch (IOException e) {
// e.printStackTrace();
// }
// return gumTreeResults;
// }
// private static String parseAction(String actStr) {
// // UPD 25@@!a from !a to isTrue(a) at 69
// String[] actStrArrays = actStr.split("@@");
// actStr = "";
// int length = actStrArrays.length;
// for (int i = 0; i < length - 1; i++) {
// String actStrFrag = actStrArrays[i];
// int index = actStrFrag.lastIndexOf(" ") + 1;
// String nodeType = actStrFrag.substring(index);
// String backup = nodeType;
// try {
// nodeType = ASTNodeMap.map.get(Integer.parseInt(nodeType));
// } catch (NumberFormatException e) {
// nodeType = backup;
// log.info(actStr);
// }
// actStrFrag = actStrFrag.substring(0, index) + nodeType + "@@";
// actStr += actStrFrag;
// }
// actStr += actStrArrays[length - 1];
// return actStr;
// }
private static String parseAction(String actStr) {
// UPD 25@@!a from !a to isTrue(a) at 69
String[] actStrArrays = actStr.split("@@");
actStr = "";
int length = actStrArrays.length;
for (int i =0; i < length - 1; i ++) {
String actStrFrag = actStrArrays[i];
int index = actStrFrag.lastIndexOf(" ") + 1;
String nodeType = actStrFrag.substring(index);
nodeType = ASTNodeMap.map.get(Integer.parseInt(nodeType));
actStrFrag = actStrFrag.substring(0, index) + nodeType + "@@";
actStr += actStrFrag;
}
actStr += actStrArrays[length - 1];
return actStr;
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Specifications.txt
-47
View File
@@ -1,47 +0,0 @@
Step 1:
Prepare data for tokens embedding of edit scripts.
Input data: parsed results of patches with GumTree.
Select token vectors of edit scripts by the value of upper whisker.
Step 2:
Embed tokens of all selected edit scripts.
Step 3:
Prepare data for features learning of selected edit scripts.
Vectorize edit scripts with embedded tokens of edit scripts.
Step 4:
Learn features of all selected edit scripts with CNN algorithm.
Input data: vectorized edit scripts.
Step 5:
Prepare data for clustering of edit scripts.
Input data: learned features of edit scripts by CNN.
Step 6:
Clustering of edit scripts with extracted features of edit scripts.
Step 7:
Analyze cluster results to obtain common fix patterns.
Step 8:
Prepare testing data for evaluation.
Parse java projects to get the token vectors of all statements.
Step 9:
Prepare data for evaluation.
Merge token vectors of source code of training data and testing data.
Step 10:
Prepare data for evaluation.
Embed tokens of source code vectors of training data and testing data.
Step 11:
Prepare data for evaluation.
Vectorize data (token vectors of source code) for deep learning.
Step 12:
Evaluation: extract features of testing data and predict their labels.
Step 13:
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step1.java
-35
View File
@@ -1,35 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Prepare data for tokens embedding of edit scripts.
*
* Input data: parsed results of patches with GumTree.
*
* @author kui.liu
*
*/
public class Step1 {
public static void main(String[] args) {
String editScriptsFile = Configuration.EDITSCRIPTS_FILE;
String patchesSourceCodeFile = Configuration.PATCH_SOURCECODE_FILE;
String buggyTokensFile = Configuration.BUGGY_CODE_TOKENS_FILE;
String editScriptSizesFile = Configuration.EDITSCRIPT_SIZES_FILE;
FileHelper.deleteFile(editScriptsFile);
FileHelper.deleteFile(patchesSourceCodeFile);
FileHelper.deleteFile(buggyTokensFile);
FileHelper.deleteFile(editScriptSizesFile);
String selectedEditScripts = Configuration.SELECTED_EDITSCRIPTES_FILE;
String selectedPatches = Configuration.SELECTED_PATCHES_SOURE_CODE_FILE;
String selectedBuggyTokens = Configuration.SELECTED_BUGGY_TOKEN_FILE;
FileHelper.deleteFile(selectedEditScripts);
FileHelper.deleteFile(selectedPatches);
FileHelper.deleteFile(selectedBuggyTokens);
DataPreparation.prepareDataForTokenEmbedding();
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step10.java
-33
View File
@@ -1,33 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import edu.lu.uni.serval.FixPatternMining.TokenEmbedder;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Prepare data for evaluation.
*
* Embed tokens of source code vectors of training data and testing data.
*
* @author kui.liu
*
*/
public class Step10 {
public static void main(String[] args) {
boolean isSupervisedLearning = true;
if (isSupervisedLearning) {// supervised learning
String outputFileName = Configuration.EMBEDDED_ALL_TOKENS2;
FileHelper.deleteFile(outputFileName);
// Data pre-processing
TokenEmbedder embedder2 = new TokenEmbedder();
embedder2.embedTokensOfSourceCodeForSupervisedTesting();
} else { // un-supervised learning
String outputFileName = Configuration.EMBEDDED_ALL_TOKENS1;
FileHelper.deleteFile(outputFileName);
// Data pre-processing
TokenEmbedder embedder2 = new TokenEmbedder();
embedder2.embedTokensOfSourceCodeForUnsupervisedTesting();
}
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step11.java
-37
View File
@@ -1,37 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import java.util.Map;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Prepare data for evaluation.
*
* Vectorize data for deep learning.
*
* @author kui.liu
*
*/
public class Step11 {
public static void main(String[] args) {
boolean isSupervisedLearning = true;
if (isSupervisedLearning) {// supervised learning
String trainingDataPath = Configuration.TRAINING_DATA;
FileHelper.deleteFile(trainingDataPath);
String testingDataPath = Configuration.TESTING_DATA;
FileHelper.deleteDirectory(testingDataPath);
Map<Integer, Integer> commonClustersMappingLabel = DataPreparation.readCommonCLusters();
DataPreparation.prepareDataForFeatureLearningOfEvaluation2(commonClustersMappingLabel);
} else { // un-supervised learning
String outputData = Configuration.VECTORIED_ALL_SOURCE_CODE1;
FileHelper.deleteFile(outputData);
// Before embedding tokens.
// List<File> files = FileHelper.getAllFilesInCurrentDiectory(Configuration.TEST_DATA_FILE, ".list");
DataPreparation.prepareDataForFeatureLearningOfEvaluation1();
}
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step12.java
-40
View File
@@ -1,40 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import java.io.File;
import java.util.List;
import edu.lu.uni.serval.FixPatternMining.FeatureLearner;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Evaluation: extract features of testing data and predict their labels.
*
* @author kui.liu
*
*/
public class Step12 {
public static void main(String[] args) {
boolean isSupervisedLearning = true;
if (isSupervisedLearning) {// supervised learning
List<File> testingDataFiles = FileHelper.getAllFilesInCurrentDiectory(Configuration.TESTING_DATA, ".csv");
for (int i = 0, size = testingDataFiles.size(); i < size; i ++) {
if (i == 0) {
// TODO: we can test this model by our clustered resutls.
FeatureLearner learner2 = new FeatureLearner();
learner2.learnFeaturesOfSourceCode2(testingDataFiles.get(i));
} else {
FeatureLearner learner2 = new FeatureLearner();
learner2.learnFeaturesOfSourceCode3(testingDataFiles.get(i));
}
}
} else { // un-supervised learning
FeatureLearner learner2 = new FeatureLearner();
learner2.learnFeaturesOfSourceCode();
// Extracted Features: Configuration.EXTRACTED_FEATURES_TESTING;
// Compute the similarity: cosin similarity
}
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step13.java
-165
View File
@@ -1,165 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Separate features of training data cluster by cluster.
*
* Classify testing data by possibilities: 90%, 80%, 70%, and 60%, ignore others.
*
* @author kui.liu
*
*/
public class Step13 {
public static void main(String[] args) {
boolean isSupervisedLearning = true;
if (isSupervisedLearning) {// supervised learning
// features of training data
DataPreparation.separateTrainingDataFeatures();
String positionFiles = Configuration.TEST_POSITION_FILE;
String featureFiles = Configuration.FEATURES_OF_TESTING_DATA;
String labelFiles = Configuration.PREDICTED_RESULTS_OF_TESTING_DATA;
String possibilitiesFilePath = Configuration.POSSIBILITIES_OF_TESTING_DATA;
List<File> possibilitiesFiles = FileHelper.getAllFilesInCurrentDiectory(possibilitiesFilePath, ".csv");
String bugs90 = Configuration.TESTING_DATA_BUGS90;
String bugs80 = Configuration.TESTING_DATA_BUGS80;
String bugs70 = Configuration.TESTING_DATA_BUGS70;
String bugs60 = Configuration.TESTING_DATA_BUGS60;
StringBuilder builder90 = new StringBuilder();
int counter90 = 0;
StringBuilder builder80 = new StringBuilder();
int counter80 = 0;
StringBuilder builder70 = new StringBuilder();
int counter70 = 0;
StringBuilder builder60 = new StringBuilder();
int counter60 = 0;
for (File possibilitiesFile : possibilitiesFiles) {
String fileName = possibilitiesFile.getName();
String positionFile = positionFiles + "Positions" + fileName.substring(fileName.lastIndexOf("_"), fileName.lastIndexOf(".")) + ".list";
String featureFile = featureFiles + fileName;
String labelFile = labelFiles + fileName;
List<String> possibilities = readData(fileName);
List<String> positions = readData(positionFile);
List<String> features = readData(featureFile);
List<Integer> labels = readLabel(labelFile);
for (int index = 0, size = possibilities.size(); index < size; index ++) {
String possibilityStr = possibilities.get(index);
String[] array = possibilityStr.split(", ");
int label = labels.get(index);
double possibility = Double.parseDouble(array[label]);
String position = positions.get(index);
String feature = features.get(index);
// And Label
if (possibility >= 0.9) {
builder90.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n");
counter90 ++;
if (counter90 % 1000 == 0) {
FileHelper.outputToFile(bugs90, builder90, true);
builder90.setLength(0);
}
} else if (possibility >= 0.8) {
builder80.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n");
counter80 ++;
if (counter80 % 1000 == 0) {
FileHelper.outputToFile(bugs80, builder80, true);
builder80.setLength(0);
}
} else if (possibility >= 0.7) {
builder70.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n");
counter70 ++;
if (counter70 % 1000 == 0) {
FileHelper.outputToFile(bugs70, builder70, true);
builder70.setLength(0);
}
} else if (possibility >= 0.6) {
builder60.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n");
counter60 ++;
if (counter60 % 1000 == 0) {
FileHelper.outputToFile(bugs60, builder60, true);
builder60.setLength(0);
}
}
}
}
FileHelper.outputToFile(bugs90, builder90, true);
FileHelper.outputToFile(bugs80, builder80, true);
FileHelper.outputToFile(bugs70, builder70, true);
FileHelper.outputToFile(bugs60, builder60, true);
// label: clusterNum, re-compute similarity with each element. 90, 80, 70, 60.
// similarity: patches --> fixing bug.
} else { // un-supervised learning
// Extracted Features: Configuration.EXTRACTED_FEATURES_TESTING;
// Compute the similarity: cosin similarity
}
}
private static List<Integer> readLabel(String labelFile) {
List<Integer> labels = new ArrayList<>();
String fileContent = FileHelper.readFile(labelFile);
BufferedReader reader = null;
try {
reader = new BufferedReader(new StringReader(fileContent));
String line = null;
while ((line = reader.readLine()) != null) {
String[] labelsStr = line.split(", ");
for (int i = 0, length = labelsStr.length; i < length; i ++) {
Double d = Double.parseDouble(labelsStr[i]);
labels.add(d.intValue());
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return labels;
}
private static List<String> readData(String positionFile) {
List<String> positions = new ArrayList<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(positionFile);
scanner = new Scanner(fis);
while (scanner.hasNextLine()) {
positions.add(scanner.nextLine());
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step14.java
-225
View File
@@ -1,225 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.ListSorter;
import edu.lu.uni.serval.utils.MapSorter;
/**
* Compute similarities for each potential bug instance by computing the similarities with all instances in related cluster.
*
* List top 5 most similar instances?
*
* @author kui.liu
*
*/
public class Step14 {
public static void main(String[] args) {
String featuresOfTrainingDataPath = Configuration.FEATURES_OF_COMMON_CLUSTERS;
List<File> featuresOfTrainingDataFiles = FileHelper.getAllFiles(featuresOfTrainingDataPath, ".csv");// TODO: type
Map<Integer, List<String>> features = readFeaturesOfTrainingData(featuresOfTrainingDataFiles);
Map<Integer, List<String>> patches = readPatchesOfTraingData(Configuration.CLUSTERED_PATCHES_FILE);
Map<Integer, Integer> labelMapClusterNum = DataPreparation.readLabelMapClusterNum();
// potential bugs' information
String bugs90 = Configuration.TESTING_DATA_BUGS90;
String bugs80 = Configuration.TESTING_DATA_BUGS80;
String bugs70 = Configuration.TESTING_DATA_BUGS70;
String bugs60 = Configuration.TESTING_DATA_BUGS60;
List<String> bugsList = new ArrayList<>();
bugsList.add(bugs90);
bugsList.add(bugs80);
bugsList.add(bugs70);
bugsList.add(bugs60);
for (String bugs : bugsList) {
List<String> bugsInfo = readData(bugs);
String filePath = bugs.substring(0, bugs.lastIndexOf(".")) + "/";
StringBuilder builder = new StringBuilder();
for (String singleBugInfo : bugsInfo) {
String[] infoArray = singleBugInfo.split(":");
String label = infoArray[0];// TODO
String feature = infoArray[1]; // TODO
String position = infoArray[2]; // TODO
int labelInt = Integer.parseInt(label);
int clusterNum = labelMapClusterNum.get(labelInt);
Map<Double, Integer> mostSimilarIndex = computeSimilarities(feature, features.get(clusterNum));
List<Integer> patchesIndex = new ArrayList<>();
if (mostSimilarIndex.size() > 0) {
for (Map.Entry<Double, Integer> entry : mostSimilarIndex.entrySet()) {
patchesIndex.add(entry.getValue());
}
String bug = "BUG####" + position + "\n";
String patchesStr = readPatches(patchesIndex, patches.get(Integer.parseInt(label)));
// output: bug + patchesStr;
builder.append(bug).append(patchesStr);
}
}
FileHelper.outputToFile(filePath + "patches.list", builder, false);
}
}
private static Map<Integer, List<String>> readPatchesOfTraingData(String clusteredPatchesFile) {
Map<Integer, List<String>> map = new HashMap<>();
List<File> files = FileHelper.getAllFiles(clusteredPatchesFile, ".list");
for (File file : files) {
String fileName = file.getName();
int clusterNum = Integer.parseInt(fileName.substring(fileName.lastIndexOf("_") + 1, fileName.lastIndexOf(".")));
List<String> patches = readPatches(file);
map.put(clusterNum, patches);
}
return map;
}
private static List<String> readPatches(File file) {
List<String> patches = new ArrayList<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(file);
scanner = new Scanner(fis);
String singlePatch = "";
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if (Configuration.PATCH_SIGNAL.equals(line)) {
if (!"".equals(singlePatch)) {
patches.add(singlePatch);
singlePatch = "";
}
}
singlePatch += line + "\n";
}
patches.add(singlePatch);
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return patches;
}
private static String readPatches(List<Integer> patchesIndex, List<String> patchesTrainingData) {
String patches = "";
for(Integer index : patchesIndex) {
patches += patchesTrainingData.get(index) + "\n";
}
return patches;
}
private static Map<Double, Integer> computeSimilarities(String feature, List<String> trainingFeatures) {
Map<Double, Integer> mostSimilarIndex = new HashMap<>();
List<Double> similarities = new ArrayList<>();
for (int i = 0; i < 10; i ++) similarities.add(0.0);
for (int index = 0, size = trainingFeatures.size(); index < size; index ++) {
String trainingFeature = trainingFeatures.get(index);
double similarity = computeSimilarity(feature, trainingFeature);
double aborted = addToSimilarityies(similarity, similarities);
if (aborted > 0.0) {
mostSimilarIndex.put(similarity, index);
if (aborted == 0.1) mostSimilarIndex.remove(aborted);
}
}
if (mostSimilarIndex.size() > 0) {
MapSorter<Double, Integer> sorter = new MapSorter<>();
mostSimilarIndex = sorter.sortByKeyDescending(mostSimilarIndex);
}
return mostSimilarIndex;
}
private static double addToSimilarityies(double similarity, List<Double> similarities) {
double lastSimilarity = similarities.get(9);
if (similarity >= 0.8 && similarity > lastSimilarity) { // TODO : 9 ?
similarities.set(9, similarity);
ListSorter<Double> sorter = new ListSorter<Double>(similarities);
similarities = sorter.sortDescending();
return lastSimilarity == 0.0 ? 0.1 : lastSimilarity;
}
return 0.0;
}
private static double computeSimilarity(String feature, String trainingFeature) {
// TODO Auto-generated method stub
return 0;
}
private static Map<Integer, List<String>> readFeaturesOfTrainingData(List<File> featureFiles) {
Map<Integer, List<String>> features = new HashMap<>();
for (File file : featureFiles) {
String fileName = file.getName();
String label = fileName.substring(fileName.lastIndexOf("_") + 1, fileName.lastIndexOf("."));
int clusterNum = Integer.parseInt(label);
List<String> featuresList = readData(file.getPath());
features.put(clusterNum, featuresList);
}
return features;
}
private static List<String> readData(String positionFile) {
List<String> positions = new ArrayList<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(positionFile);
scanner = new Scanner(fis);
while (scanner.hasNextLine()) {
positions.add(scanner.nextLine());
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
public static void quickSort(List<Double> arr){
qsort(arr, 0, arr.size() - 1);
}
private static void qsort(List<Double> arr, int low, int high){
if (low < high){
int pivot=partition(arr, low, high); //将数组分为两部分
qsort(arr, low, pivot-1); //递归排序左子数组
qsort(arr, pivot+1, high); //递归排序右子数组
}
}
private static int partition(List<Double> arr, int low, int high){
double pivot = arr.get(low); //枢轴记录
while (low<high){
while (low<high && arr.get(high)>=pivot) --high;
arr.set(low, arr.get(high)); //交换比枢轴小的记录到左端
while (low<high && arr.get(low)<=pivot) ++low;
arr.set(high, arr.get(low)); //交换比枢轴小的记录到右端
}
//扫描完成，枢轴到位
arr.set(low, pivot);
//返回的是枢轴的位置
return low;
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step2.java
-25
View File
@@ -1,25 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import edu.lu.uni.serval.FixPatternMining.TokenEmbedder;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Embed tokens of all selected edit scripts.
*
* Input data: all tokens of selected edit scripts.
*
* @author kui.liu
*
*/
public class Step2 {
public static void main(String[] args) {
String outputFileName = Configuration.EMBEDDED_EDIT_SCRIPT_TOKENS;
FileHelper.deleteFile(outputFileName);
TokenEmbedder embedder = new TokenEmbedder();
embedder.embedTokensOfEditScripts();
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step3.java
-24
View File
@@ -1,24 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Prepare data for features learning of selected edit scripts.
*
* Vectorize edit scripts with embedded tokens of edit scripts.
*
* @author kui.liu
*
*/
public class Step3 {
public static void main(String[] args) {
String vectorizedEditScripts = Configuration.VECTORIED_EDIT_SCRIPTS;
FileHelper.deleteFile(vectorizedEditScripts);
DataPreparation.prepareDataForFeatureLearning();
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step4.java
-25
View File
@@ -1,25 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import edu.lu.uni.serval.FixPatternMining.FeatureLearner;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Learn features of all selected edit scripts with CNN algorithm.
*
* Input data: vectorized edit scripts.
*
* @author kui.liu
*
*/
public class Step4 {
public static void main(String[] args) {
String extractedFeatures = Configuration.EXTRACTED_FEATURES;
FileHelper.deleteDirectory(extractedFeatures);
FeatureLearner learner = new FeatureLearner();
learner.learnFeatures();
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step5.java
-24
View File
@@ -1,24 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Prepare data for clustering of edit scripts.
*
* Input data: learned features of edit scripts by CNN.
*
* @author kui.liu
*
*/
public class Step5 {
public static void main(String[] args) {
String clusterInput = Configuration.CLUSTER_INPUT;
FileHelper.deleteFile(clusterInput);
DataPreparation.prepareDataForClustering();
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step6.java
-23
View File
@@ -1,23 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import edu.lu.uni.serval.FixPatternMining.Cluster;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Clustering of edit scripts with extracted features of edit scripts.
*
* @author kui.liu
*
*/
public class Step6 {
public static void main(String[] args) {
String clusterOutput = Configuration.CLUSTER_OUTPUT;
FileHelper.deleteFile(clusterOutput);
Cluster cluster = new Cluster();
cluster.cluster();
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step7.java
-47
View File
@@ -1,47 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import java.util.List;
import java.util.Map;
import edu.lu.uni.serval.FixPatternMining.ClusterAnalyser;
import edu.lu.uni.serval.FixPatternMining.CommonPatterns;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Analyze cluster results to obtain common fix patterns.
*
* @author kui.liu
*
*/
public class Step7 {
public static void main(String[] args) {
String clusteredPatches = Configuration.CLUSTERED_PATCHES_FILE;
String clusteredBuggyTokens = Configuration.CLUSTERED_TOKENSS_FILE;
FileHelper.deleteDirectory(clusteredPatches);
FileHelper.deleteDirectory(clusteredBuggyTokens);
// analyze cluster results.
ClusterAnalyser analyser = new ClusterAnalyser();
analyser.readClusterResutls();
analyser.clusterPatchSourceCode();
analyser.clusterBuggyCodeTokens(); // the results will be used to compute similarity with target java code to localize bugs.
List<Integer> clusterResults = analyser.getClusterResults();
// Common patterns.
CommonPatterns commonPatterns = new CommonPatterns(); // Metrics TODO
// <Integer, Integer>: <ClusterNum, Label for supervised learning>
Map<Integer, Integer> commonClustersMappingLabel = commonPatterns.identifyCommonPatterns(clusterResults);
String clusterMappingLabel = "Label : ClusterNum\n";
for (Map.Entry<Integer, Integer> entry : commonClustersMappingLabel.entrySet()) {
clusterMappingLabel += entry.getValue() + " : " + entry.getKey() + "\n";
}
FileHelper.outputToFile(Configuration.CLUSTERNUMBER_LABEL_MAP, clusterMappingLabel, false);
int totalNumberOfTrainingData = commonPatterns.getTotalNumberofTrainingData();
FileHelper.outputToFile(Configuration.NUMBER_OF_TRAINING_DATA, "" + totalNumberOfTrainingData, false);
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step9.java
-33
View File
@@ -1,33 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.App;
import java.util.Map;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Prepare data for evaluation.
*
* Merge token vectors of source code of training data and testing data.
*
* @author kui.liu
*
*/
public class Step9 {
public static void main(String[] args) {
boolean isSupervisedLearning = true;
if (isSupervisedLearning) {// supervised learning
Map<Integer, Integer> commonClustersMappingLabel = DataPreparation.readCommonCLusters();
String outputFile = Configuration.EMBEDDING_DATA_TOKENS2;
FileHelper.deleteFile(outputFile);
// Data merge
DataPreparation.prepareTokensForEvaluation2(commonClustersMappingLabel);
} else { // un-supervised learning
// Data merge
DataPreparation.prepareTokensForEvaluation1();
}
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/Cluster.java
-45
View File
@@ -1,45 +0,0 @@
package edu.lu.uni.serval.FixPatternMining;
import edu.lu.uni.serval.Clusters.XMeansCluster;
import edu.lu.uni.serval.config.Configuration;
import weka.core.EuclideanDistance;
/**
* Cluster features with X-means clustering algorithm.
*
* @author kui.liu
*
*/
public class Cluster {
public void cluster() {
String arffFile = Configuration.CLUSTER_INPUT;
String clusterResults = Configuration.CLUSTER_OUTPUT;
XMeansCluster cluster = new XMeansCluster();
try {
/*
* The below 5 parameters have default values.
*/
cluster.setDistanceF(new EuclideanDistance());
cluster.setUseKDTree(true);
cluster.setMaxNumberOfIterations(1000);
// The below 2 parameters are recommended to be the same.
cluster.setMaxKMeans(200);
cluster.setMaxKMeansForChildren(200);
/*
* The values of the below 3 parameters should be set by developers.
*/
cluster.setSeed(200);
cluster.setMaxNumClusters(100);
cluster.setMinNumClusters(1);
// X-means clustering is beginning.
cluster.cluster(arffFile, clusterResults);
// X-means clustering is finished.
} catch (Exception e) {
e.printStackTrace();
}
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/ClusterAnalyser.java
-146
View File
@@ -1,146 +0,0 @@
package edu.lu.uni.serval.FixPatternMining;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
public class ClusterAnalyser {
private List<Integer> clusterResults; // each element is a cluster number.
public void readClusterResutls() {
clusterResults = DataPreparation.readClusterResults();
}
public void clusterBuggyCodeTokens() {
String selectedTokens = Configuration.SELECTED_BUGGY_TOKEN_FILE;
String clusteredTokens = Configuration.CLUSTERED_TOKENSS_FILE;
FileInputStream fis = null;
Scanner scanner = null;
Map<Integer, StringBuilder> builderMap = new HashMap<>();
Map<Integer, Integer> countersMap = new HashMap<>();
try {
fis = new FileInputStream(selectedTokens);
scanner = new Scanner(fis);
int index = 0;
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
int clusterNum = clusterResults.get(index);
StringBuilder builder = getBuilder(builderMap, clusterNum);
builder.append(line).append("\n");
int counter = getCounter(countersMap, clusterNum);
if (counter % 1000 == 0) {
FileHelper.outputToFile(clusteredTokens + "Tokens_" + clusterNum + ".list", builder, true);
builder.setLength(0);
builderMap.put(clusterNum, builder);
}
index ++;
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
for (Map.Entry<Integer, StringBuilder> entry : builderMap.entrySet()) {
int clusterNum = entry.getKey();
StringBuilder builder = entry.getValue();
FileHelper.outputToFile(clusteredTokens + "Tokens_" + clusterNum + ".list", builder, true);
builder.setLength(0);
}
}
public void clusterPatchSourceCode() {
String selectedPatches = Configuration.SELECTED_PATCHES_SOURE_CODE_FILE;
String clusteredPatches = Configuration.CLUSTERED_PATCHES_FILE;
FileInputStream fis = null;
Scanner scanner = null;
Map<Integer, StringBuilder> builderMap = new HashMap<>();
Map<Integer, Integer> countersMap = new HashMap<>();
try {
fis = new FileInputStream(selectedPatches);
scanner = new Scanner(fis);
String singlePatch = "";
int index = -1;
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if ("".equals(line)) continue;
if ("PATCH###".equals(line)) {
if (!"".equals(singlePatch)) {
int clusterNum = clusterResults.get(index);
StringBuilder builder = getBuilder(builderMap, clusterNum);
builder.append(singlePatch);
int counter = getCounter(countersMap, clusterNum);
if (counter % 1000 == 0) {
FileHelper.outputToFile(clusteredPatches + "PatchesCluster_" + clusterNum + ".list", builder, true);
builder.setLength(0);
builderMap.put(clusterNum, builder);
}
}
singlePatch = "";
index ++;
}
singlePatch += line + "\n";
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
for (Map.Entry<Integer, StringBuilder> entry : builderMap.entrySet()) {
int clusterNum = entry.getKey();
StringBuilder builder = entry.getValue();
FileHelper.outputToFile(clusteredPatches + "PatchesCluster_" + clusterNum + ".list", builder, true);
builder.setLength(0);
}
}
private int getCounter(Map<Integer, Integer> countersMap, int clusterNum) {
int counter = 1;
if (countersMap.containsKey(clusterNum)) {
counter += countersMap.get(clusterNum);
}
countersMap.put(clusterNum, counter);
return counter;
}
private StringBuilder getBuilder(Map<Integer, StringBuilder> builderMap, int clusterNum) {
if (builderMap.containsKey(clusterNum)) {
return builderMap.get(clusterNum);
} else {
StringBuilder builder = new StringBuilder();
builderMap.put(clusterNum, builder);
return builder;
}
}
public List<Integer> getClusterResults() {
return clusterResults;
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/ClusterResults.java
-35
View File
@@ -1,35 +0,0 @@
package edu.lu.uni.serval.FixPatternMining;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import edu.lu.uni.serval.utils.FileHelper;
public class ClusterResults {
/**
* Read the cluster results from the file of cluster results.
*
* @param clusterResultsFile, the file of cluster results.
* @return List<Integer>, each integer is a cluster number.
* @throws IOException
*/
public static List<Integer> readClusterResults(File clusterResultsFile) throws IOException {
List<Integer> clusterResultsList = new ArrayList<>();
String clusterResults = FileHelper.readFile(clusterResultsFile);
BufferedReader reader = new BufferedReader(new StringReader(clusterResults));
String line = null;
while ((line = reader.readLine()) != null) {
int cluster = Integer.parseInt(line);
clusterResultsList.add(cluster);
}
reader.close();
return clusterResultsList;
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/CommonPatterns.java
-86
View File
@@ -1,86 +0,0 @@
package edu.lu.uni.serval.FixPatternMining;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.MapSorter;
public class CommonPatterns {
private static final int LEAST_NUMBER = 100;
private int totalNumberofTrainingData = 0;
public Map<Integer, Integer> identifyCommonPatterns(List<Integer> clusterResults) {
Map<Integer, List<Integer>> clusterMap = DataPreparation.readClusterResult(clusterResults);
// TODO how to select the common patterns, number or ratio?
List<Integer> commonClusterNum = getCommonClustersByNumber(clusterMap); // Integer: clusterNum.
Map<Integer, Integer> clusterNumMapLabel = new HashMap<>(); // <ClusterNum, Label for supervised learning>
for (int i = 0, size = commonClusterNum.size(); i < size; i ++) {
clusterNumMapLabel.put(commonClusterNum.get(i), i);
}
return clusterNumMapLabel;
}
private List<Integer> getCommonClustersByNumber(Map<Integer, List<Integer>> clusterMap) {
List<Integer> commonClusterNum = new ArrayList<>();
String numbersMapStr = "";// numbers of instances in each common cluster.
for (Map.Entry<Integer, List<Integer>> entry : clusterMap.entrySet()) {
List<Integer> elements = entry.getValue();
int size = elements.size();
if (size >= LEAST_NUMBER) { // TODO how to set this threshold?
int key = entry.getKey();
commonClusterNum.add(key);
totalNumberofTrainingData += size;
numbersMapStr += key + ":" + size + "\n";
}
}
FileHelper.outputToFile(Configuration.COMMON_CLUSTERS_SIZES, numbersMapStr, false);
return commonClusterNum;
}
private List<Integer> getCommonClustersByRatio(Map<Integer, List<Integer>> clusterMap, List<Integer> clusterResults) {
List<Integer> commonClusterNum = new ArrayList<>();
double sizes = (double) clusterResults.size();
Map<Integer, Double> ratios = new HashMap<>();
for (Map.Entry<Integer, List<Integer>> entry : clusterMap.entrySet()) {
List<Integer> elements = entry.getValue();
ratios.put(entry.getKey(), (double) elements.size() / sizes);
}
String numbersMapStr = "";// numbers of instances in each common cluster.
MapSorter<Integer, Double> sorter = new MapSorter<Integer, Double>();
ratios = sorter.sortByValueDescending(ratios);
double counterRatio = 0.0;
for (Map.Entry<Integer, Double> entry : ratios.entrySet()) {
counterRatio += entry.getValue();
int key = entry.getKey();
commonClusterNum.add(key);
numbersMapStr += key + ":" + clusterMap.get(key).size() + "\n";
totalNumberofTrainingData += clusterMap.get(entry.getKey()).size();
if (counterRatio >= 0.8) { // TODO: how to set the value of this threshold?
break;
}
}
FileHelper.outputToFile(Configuration.COMMON_CLUSTERS_SIZES, numbersMapStr, false);
return commonClusterNum;
}
public int getTotalNumberofTrainingData() {
return totalNumberofTrainingData;
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/DataPrepare/DataPreparation.java
-661
View File
@@ -1,661 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.DataPrepare;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.MaxSizeSelector.MaxSizeType;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.data.DataPreparer;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Prepare data for fix patterns mining and evaluation.
*
* @author kui.liu
*
*/
public class DataPreparation {
/**
* Prepare data for token embedding in the process of fix patterns mining.
*/
public static void prepareDataForTokenEmbedding() {
// Collect all data into one file.
String editScriptsFilePath = Configuration.EDITSCRIPTS_FILE_PATH;
String patchesSourceCodeFilePath = Configuration.PATCH_SOURCECODE_FILE_PATH;
String buggyTokensFilePath = Configuration.BUGGYTREE_FILE_PATH;
String editScriptSizesFilePath = Configuration.EDITSCRIPT_SIZES_FILE_PATH;
String editScriptsFile = Configuration.EDITSCRIPT_SIZES_FILE;
String patchesSourceCodeFile = Configuration.PATCH_SOURCECODE_FILE;
String buggyTokensFile = Configuration.BUGGY_CODE_TOKENS_FILE;
String editScriptSizesFile = Configuration.EDITSCRIPT_SIZES_FILE;
File file = new File(editScriptsFilePath);
File[] subFiles = file.listFiles();
// Merge results of parsed patches.
for (File subFile : subFiles) {
String fileName = subFile.getName(); // edistScripts file
String id = fileName.substring(fileName.lastIndexOf("_"));
FileHelper.outputToFile(editScriptsFile, FileHelper.readFile(subFile), true);
String patchesSourceCode = patchesSourceCodeFilePath + "patches" + id;
FileHelper.outputToFile(patchesSourceCodeFile, FileHelper.readFile(patchesSourceCode), true);
String sizes = editScriptSizesFile + "sizes" + id;
FileHelper.outputToFile(editScriptSizesFilePath, FileHelper.readFile(sizes), true);
String buggyTokens = buggyTokensFilePath + "tokens" + id;
FileHelper.outputToFile(buggyTokensFile, FileHelper.readFile(buggyTokens), true);
}
// Select data by the size of edit script vectors.
List<Integer> sizesList;
try {
sizesList = MaxSizeSelector.readSizes(editScriptSizesFile);
int maxSize = MaxSizeSelector.selectMaxSize(MaxSizeType.ThirdQuartile, sizesList);
List<Integer> outlierIndexes = new ArrayList<>();
for (int i = 0, size = sizesList.size(); i < size; i ++) {
if (sizesList.get(i) > maxSize) {
outlierIndexes.add(i);
}
}
FileHelper.outputToFile(Configuration.MAX_TOKEN_VECTORS_SIZE_OF_EDIT_SCRIPTS, "" + maxSize, false);
selectData(editScriptsFile, outlierIndexes, Configuration.SELECTED_EDITSCRIPTES_FILE);
selectData(patchesSourceCodeFile, outlierIndexes, Configuration.PATCH_SIGNAL, Configuration.SELECTED_PATCHES_SOURE_CODE_FILE);
int maxTokenVectorSize = selectDataOfSourceCodeTokens(buggyTokensFile, outlierIndexes, Configuration.SELECTED_BUGGY_TOKEN_FILE);
FileHelper.outputToFile(Configuration.MAX_TOKEN_VECTORS_SIZE_OF_SOURCE_CODE, "" + maxTokenVectorSize, false);
} catch (IOException e) {
e.printStackTrace();
}
}
private static void selectData(String intputFile, List<Integer> outlierIndexList, String outputFile) {
List<Integer> outlierIndexes = new ArrayList<>();
outlierIndexes.addAll(outlierIndexList);
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(intputFile);
scanner = new Scanner(fis);
int index = 0;
StringBuilder builder = new StringBuilder();
int counter = 0;
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if (outlierIndexes.contains(index)) {
outlierIndexes.remove(new Integer(index));
} else {
builder.append(line + "\n");
if (++ counter % 100000 == 0) {
FileHelper.outputToFile(outputFile, builder, true);
builder.setLength(0);
}
}
index ++;
}
FileHelper.outputToFile(outputFile, builder, true);
builder.setLength(0);
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
if (scanner != null) {
scanner.close();
scanner = null;
}
if (fis != null) {
fis.close();
fis = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
private static void selectData(String inputFile, List<Integer> outlierIndexes, String startingSignal, String outputFile) {
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(inputFile);
scanner = new Scanner(fis);
int index = -1;
StringBuilder builder = new StringBuilder();
int counter = 0;
String singleEntity = "";
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if (line.equals(startingSignal)) {
if (!"".equals(singleEntity)) {
if (outlierIndexes.contains(index)) {
outlierIndexes.remove(new Integer(index));
} else {
builder.append(singleEntity + "\n");
if (++ counter % 100000 == 0) {
FileHelper.outputToFile(outputFile, builder, true);
builder.setLength(0);
}
}
singleEntity = "";
}
index ++;
}
singleEntity += line + "\n";
}
FileHelper.outputToFile(outputFile, builder, true);
builder.setLength(0);
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
if (scanner != null) {
scanner.close();
scanner = null;
}
if (fis != null) {
fis.close();
fis = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
private static int selectDataOfSourceCodeTokens(String inputFile, List<Integer> outlierIndexList, String outputFile) {
List<Integer> outlierIndexes = new ArrayList<>();
outlierIndexes.addAll(outlierIndexList);
FileInputStream fis = null;
Scanner scanner = null;
int size = 0;
try {
fis = new FileInputStream(inputFile);
scanner = new Scanner(fis);
int index = 0;
StringBuilder builder = new StringBuilder();
int counter = 0;
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if (outlierIndexes.contains(index)) {
outlierIndexes.remove(new Integer(index));
} else {
builder.append(line + "\n");
if (++ counter % 100000 == 0) {
FileHelper.outputToFile(outputFile, builder, true);
builder.setLength(0);
}
String[] tokens = line.split(" ");
if (tokens.length > size) size = tokens.length;
}
index ++;
}
FileHelper.outputToFile(outputFile, builder, true);
builder.setLength(0);
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
if (scanner != null) {
scanner.close();
scanner = null;
}
if (fis != null) {
fis.close();
fis = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
return size;
}
/**
* Prepare data for feature learning.
*/
public static void prepareDataForFeatureLearning() {
String zeroVector = "";
for (int i =0, length = Configuration.VECTOR_SIZE_OF_EMBEDED_TOKEN1 - 1; i < length; i ++) {
zeroVector += "0, ";
}
zeroVector += "0";
int maxSize = Integer.parseInt(FileHelper.readFile(Configuration.MAX_TOKEN_VECTORS_SIZE_OF_EDIT_SCRIPTS).trim());
String embeddedTokensFile = Configuration.EMBEDDED_EDIT_SCRIPT_TOKENS;
Map<String, String> embeddedTokens = readEmbeddedTokens(embeddedTokensFile);
String editScriptsFile = Configuration.SELECTED_EDITSCRIPTES_FILE;
String outputFile = Configuration.VECTORIED_EDIT_SCRIPTS;
dataPrepare(editScriptsFile, maxSize, outputFile, embeddedTokens, zeroVector);
}
private static Map<String, String> readEmbeddedTokens(String embeddedTokensFile) {
Map<String, String> embeddedTokens = new HashMap<>();
File file = new File(embeddedTokensFile);
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(file);
scanner = new Scanner(fis);
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
int firstBlankIndex = line.indexOf(" ");
String token = line.substring(0, firstBlankIndex);
String value = line.substring(firstBlankIndex + 1).replaceAll(" ", ", ");
embeddedTokens.put(token, value);
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return embeddedTokens;
}
private static void dataPrepare(String inputFile, int maxSize, String outputFile, Map<String, String> embeddedTokens, String zeroVector) {
File file = new File(inputFile);
FileInputStream fis = null;
Scanner scanner = null;
StringBuilder builder = new StringBuilder();
int counter = 0;
try {
fis = new FileInputStream(file);
scanner = new Scanner(fis);
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
StringBuilder vectorStr = convertToVector(embeddedTokens, line, maxSize, zeroVector);
builder.append(vectorStr);
if (++ counter % 10000 == 0) {
FileHelper.outputToFile(outputFile, builder, true);
builder.setLength(0);
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
FileHelper.outputToFile(outputFile, builder, true);
builder.setLength(0);
}
private static StringBuilder convertToVector(Map<String, String> embeddedTokens, String line, int maxSize, String zeroVector) {
String[] tokens = line.split(" ");
StringBuilder vectorStr = new StringBuilder();
int length = tokens.length;
if (length == maxSize) {
for (int i = 0; i < length - 1; i ++) {
String token = tokens[i];
vectorStr.append(embeddedTokens.get(token) + ", ");
}
vectorStr.append(embeddedTokens.get(tokens[length - 1]) + "\n");
} else {
for (int i = 0; i < length; i ++) {
String token = tokens[i];
vectorStr.append(embeddedTokens.get(token) + ", ");
}
for (int i = length; i < maxSize - 1; i ++) {
vectorStr.append(zeroVector + ", ");
}
vectorStr.append(zeroVector + "\n");
}
return vectorStr;
}
/**
* Prepare data for clustering.
*/
public static void prepareDataForClustering() {
String featureFile = Configuration.EXTRACTED_FEATURES + "vectorizedEditScripts.csv";
String arffFile = Configuration.CLUSTER_INPUT;
DataPreparer.prepareData(featureFile, arffFile);
}
/**
* Read cluster results.
*/
public static List<Integer> readClusterResults() {
List<Integer> clusterResults = new ArrayList<>();
String clusterResultsFile = Configuration.CLUSTER_OUTPUT;
String results = FileHelper.readFile(clusterResultsFile);
BufferedReader reader = null;
try {
reader = new BufferedReader(new StringReader(results));
String line = null;
while ((line = reader.readLine()) != null) {
clusterResults.add(Integer.parseInt(line));
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return clusterResults;
}
public static Map<Integer, List<Integer>> readClusterResult(List<Integer> clusterResults) {
Map<Integer, List<Integer>> clusters = new HashMap<>();
for (int i = 0, size = clusterResults.size(); i < size; i ++) {
int clusterNo = clusterResults.get(i);
if (clusters.containsKey(clusterNo)) {
clusters.get(clusterNo).add(i + 1);
} else {
List<Integer> newCLuster = new ArrayList<>();
newCLuster.add(i + 1);
clusters.put(clusterNo, newCLuster);
}
}
return clusters;
}
/**
* Data for un-supervised learning.
*/
public static void prepareTokensForEvaluation1() {
String outputFile = Configuration.EMBEDDING_DATA_TOKENS1;
FileHelper.outputToFile(outputFile, FileHelper.readFile(Configuration.SELECTED_BUGGY_TOKEN_FILE), false);
List<File> files = FileHelper.getAllFilesInCurrentDiectory(Configuration.TEST_DATA_FILE, ".list");
for (File file : files) {
FileHelper.outputToFile(outputFile, FileHelper.readFile(file), true);
}
}
public static void prepareDataForFeatureLearningOfEvaluation1() {
String zeroVector = "";
for (int i =0, length = Configuration.VECTOR_SIZE_OF_EMBEDED_TOKEN2 - 1; i < length; i ++) {
zeroVector += "0, ";
}
zeroVector += "0";
int maxSize = Integer.parseInt(FileHelper.readFile(Configuration.MAX_TOKEN_VECTORS_SIZE_OF_SOURCE_CODE));
String allEmbeddedTokens = Configuration.EMBEDDED_ALL_TOKENS1;
Map<String, String> embeddedTokens = readEmbeddedTokens(allEmbeddedTokens);
// Testing data
String clusteredTokens = Configuration.TEST_DATA_FILE;
List<File> files = FileHelper.getAllFilesInCurrentDiectory(clusteredTokens, ".list");
for (File file : files) {
}
String allTokensOfSourceCode = Configuration.EMBEDDING_DATA_TOKENS1; // TODO testing data should be separated.
dataPrepare(allTokensOfSourceCode, maxSize, Configuration.VECTORIED_ALL_SOURCE_CODE1, embeddedTokens, zeroVector);
}
/**
* Data for supervised learning.
*/
public static void prepareTokensForEvaluation2(Map<Integer, Integer> commonClustersMappingLabel) {
String clusteredTokens = Configuration.CLUSTERED_TOKENSS_FILE;
String outputFile = Configuration.EMBEDDING_DATA_TOKENS2;
List<File> files = FileHelper.getAllFilesInCurrentDiectory(clusteredTokens, ".list");
for (File file : files) {
String fileName = file.getName();
String clusterNumStr = fileName.substring(fileName.lastIndexOf("_") + 1, fileName.lastIndexOf(".list"));
int clusterNum = Integer.parseInt(clusterNumStr);
if (commonClustersMappingLabel.containsKey(clusterNum)) {
String content = FileHelper.readFile(file);
FileHelper.outputToFile(outputFile, content, true);
}
}
files.clear();
files = FileHelper.getAllFilesInCurrentDiectory(Configuration.TEST_DATA_FILE, ".list");
for (File file : files) {
FileHelper.outputToFile(outputFile, FileHelper.readFile(file), true);
}
}
public static void prepareDataForFeatureLearningOfEvaluation2(Map<Integer, Integer> commonClustersMappingLabel) {
String zeroVector = "";
for (int i =0, length = Configuration.VECTOR_SIZE_OF_EMBEDED_TOKEN2 - 1; i < length; i ++) {
zeroVector += "0, ";
}
zeroVector += "0";
String allEmbeddedTokensOfEvaluation = Configuration.EMBEDDED_ALL_TOKENS2;
Map<String, String> embeddedTokens = readEmbeddedTokens(allEmbeddedTokensOfEvaluation);
int maxSize = Integer.parseInt(FileHelper.readFile(Configuration.MAX_TOKEN_VECTORS_SIZE_OF_SOURCE_CODE));
// Training data
String clusteredTokens = Configuration.CLUSTERED_TOKENSS_FILE;
List<File> files = FileHelper.getAllFilesInCurrentDiectory(clusteredTokens, ".list");
for (File file : files) {
String fileName = file.getName();
String clusterNumStr = fileName.substring(fileName.lastIndexOf("_") + 1, fileName.lastIndexOf(".list"));
int clusterNum = Integer.parseInt(clusterNumStr);
if (commonClustersMappingLabel.containsKey(clusterNum)) {
dataPrepare(file.getPath(), maxSize, Configuration.TRAINING_DATA, embeddedTokens, zeroVector, clusterNum);
}
}
// Testing data
files.clear();
String testingData = Configuration.TEST_DATA_FILE;
files = FileHelper.getAllFilesInCurrentDiectory(testingData, ".list");
String testingDataPath = Configuration.TESTING_DATA;
for (File file : files) {
String fileName = file.getName();
fileName.replace(".list", ".csv");
dataPrepare(file.getPath(), maxSize, testingDataPath + fileName, embeddedTokens, zeroVector, 0);
}
}
private static void dataPrepare(String inputFile, int maxSize, String outputFile, Map<String, String> embeddedTokens,
String zeroVector, int clusterNum) {
FileInputStream fis = null;
Scanner scanner = null;
StringBuilder builder = new StringBuilder();
int counter = 0;
try {
fis = new FileInputStream(inputFile);
scanner = new Scanner(fis);
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
StringBuilder vectorStr = convertToVector(embeddedTokens, line, maxSize, zeroVector, clusterNum);
builder.append(vectorStr);
if (++ counter % 10000 == 0) {
FileHelper.outputToFile(outputFile, builder, true);
builder.setLength(0);
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
FileHelper.outputToFile(outputFile, builder, true);
builder.setLength(0);
}
private static StringBuilder convertToVector(Map<String, String> embeddedTokens, String line, int maxSize, String zeroVector, int clusterNum) {
String[] tokens = line.split(" ");
StringBuilder vectorStr = new StringBuilder();
int length = tokens.length;
if (length == maxSize) {
for (int i = 0; i < length; i ++) {
String token = tokens[i];
vectorStr.append(embeddedTokens.get(token) + ", ");
}
} else {
for (int i = 0; i < length; i ++) {
String token = tokens[i];
vectorStr.append(embeddedTokens.get(token) + ", ");
}
for (int i = length; i < maxSize; i ++) {
vectorStr.append(zeroVector + ", ");
}
}
vectorStr.append(clusterNum + "\n");
return vectorStr;
}
public static Map<Integer, Integer> readCommonCLusters() {
Map<Integer, Integer> commonClustersMappingLabel = new HashMap<>();
String commonClusters = FileHelper.readFile(Configuration.CLUSTERNUMBER_LABEL_MAP);
BufferedReader reader = null;
try {
reader = new BufferedReader(new StringReader(commonClusters));
String line = reader.readLine();
while ((line = reader.readLine()) != null) {
String[] strArray = line.split(" : ");
int key = Integer.parseInt(strArray[1]);
int value = Integer.parseInt(strArray[0]);
commonClustersMappingLabel.put(key, value);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return commonClustersMappingLabel;
}
public static void separateTrainingDataFeatures() {
String trainingDataFeatures = Configuration.FEATURES_OF_TRAINING_DATA;
List<File> featureFiles = FileHelper.getAllFilesInCurrentDiectory(trainingDataFeatures, ".csv");
File featureFile = featureFiles.get(0);
// File featureFile = new File(Configuration.FEATURES_OF_TRAINING_DATA + "");
Map<Integer, Integer> numbersMap = readNumberOfInstances(); // <Integer, Integer>: <clusterNum, numOfInstances>
Map<Integer, Integer> orders = new HashMap<>(); //<Integer, Integer> : <order, clusterNum>
Map<Integer, String> fileNames = new HashMap<>();
String clusteredTokens = Configuration.CLUSTERED_TOKENSS_FILE;
List<File> files = FileHelper.getAllFilesInCurrentDiectory(clusteredTokens, ".list");
int order = 1;
for (File file : files) {
String fileName = file.getName();
String clusterNumStr = fileName.substring(fileName.lastIndexOf("_") + 1, fileName.lastIndexOf(".list"));
int clusterNum = Integer.parseInt(clusterNumStr);
if (numbersMap.containsKey(clusterNum)) {
orders.put(order, clusterNum);
fileNames.put(order, fileName);
order ++;
}
}
String featuresOfClusterPath = Configuration.FEATURES_OF_COMMON_CLUSTERS;
order = 1;
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(featureFile);
scanner = new Scanner(fis);
int counter = 0;
StringBuilder features = new StringBuilder();
while (scanner.hasNextLine()) {
features.append(scanner.nextLine() + "\n");
counter ++;
if (counter == numbersMap.get(orders.get(order))) {
FileHelper.outputToFile(featuresOfClusterPath + fileNames.get(order), features, false);
features.setLength(0);
counter = 0;
order ++;
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
private static Map<Integer, Integer> readNumberOfInstances() {
Map<Integer, Integer> numbersMap = new HashMap<>();
String fileContent = FileHelper.readFile(Configuration.COMMON_CLUSTERS_SIZES);
BufferedReader reader = null;
try {
reader = new BufferedReader(new StringReader(fileContent));
String line = null;
while ((line = reader.readLine()) != null) {
String[] numbers = line.split(":");
numbersMap.put(Integer.parseInt(numbers[0]), Integer.parseInt(numbers[1]));
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return numbersMap;
}
public static Map<Integer, Integer> readLabelMapClusterNum() {
Map<Integer, Integer> labelMapClusterNumMap = new HashMap<>();
String fileContent = FileHelper.readFile(Configuration.CLUSTERNUMBER_LABEL_MAP);
BufferedReader reader = null;
reader = new BufferedReader(new StringReader(fileContent));
String line = null;
try {
while ((line = reader.readLine()) != null) {
String[] labelMapClusterNum = line.split(":");
labelMapClusterNumMap.put(Integer.parseInt(labelMapClusterNum[0]), Integer.parseInt(labelMapClusterNum[1]));
}
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/DataPrepare/MaxSizeSelector.java
-66
View File
@@ -1,66 +0,0 @@
package edu.lu.uni.serval.FixPatternMining.DataPrepare;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.ListSorter;
public class MaxSizeSelector {
public enum MaxSizeType {
UpperWhisker, ThirdQuartile
}
public static List<Integer> readSizes(String sizeFilePath) throws IOException {
List<Integer> sizes = new ArrayList<>();
String sizesStr = FileHelper.readFile(sizeFilePath);
BufferedReader br = new BufferedReader(new StringReader(sizesStr));
String line = null;
while ((line = br.readLine()) != null) {
sizes.add(Integer.parseInt(line.trim()));
}
return sizes;
}
public static int selectMaxSize(MaxSizeType maxSizeType, List<Integer> sizesDistribution) {
int maxSize = 0;
switch (maxSizeType) {
case UpperWhisker:
maxSize = upperWhisker(sizesDistribution);
break;
case ThirdQuartile:
maxSize = thirdQuarter(sizesDistribution);
break;
}
return maxSize;
}
private static int upperWhisker(List<Integer> sizesDistribution) {
List<Integer> sizes = new ArrayList<>();
sizes.addAll(sizesDistribution);
ListSorter<Integer> sorter = new ListSorter<Integer>(sizes);
sizesDistribution = sorter.sortAscending();
int firstQuarterIndex = sizesDistribution.size() * 25 / 100;
int firstQuarter = sizesDistribution.get(firstQuarterIndex);
int thirdQuarterIndex = sizesDistribution.size() * 75 / 100;
int thirdQuarter = sizesDistribution.get(thirdQuarterIndex);
int upperWhisker = thirdQuarter + (int) (1.5 * (thirdQuarter - firstQuarter));
return upperWhisker;
}
private static int thirdQuarter(List<Integer> sizesDistribution) {
List<Integer> sizes = new ArrayList<>();
sizes.addAll(sizesDistribution);
ListSorter<Integer> sorter = new ListSorter<Integer>(sizes);
sizesDistribution = sorter.sortAscending();
int thirdQuarterIndex = sizesDistribution.size() * 75 / 100;
int thirdQuarter = sizesDistribution.get(thirdQuarterIndex);
return thirdQuarter;
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/FeatureLearner.java
-121
View File
@@ -1,121 +0,0 @@
package edu.lu.uni.serval.FixPatternMining;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.deeplearner.CNNFeatureExtractor2;
import edu.lu.uni.serval.deeplearner.CNNSupervisedLearning;
import edu.lu.uni.serval.utils.FileHelper;
public class FeatureLearner {
/**
* Learn features of edit scripts for fix patterns mining.
*/
public void learnFeatures() {
String editScriptsVectorFile = Configuration.VECTORIED_EDIT_SCRIPTS; // input
int sizeOfVector = Integer.parseInt(FileHelper.readFile(Configuration.MAX_TOKEN_VECTORS_SIZE_OF_EDIT_SCRIPTS).trim());
int sizeOfTokenVec = Configuration.VECTOR_SIZE_OF_EMBEDED_TOKEN1;
int batchSize = 1000;
int sizeOfFeatureVector = 200;
try {
CNNFeatureExtractor2 learner = new CNNFeatureExtractor2(new File(editScriptsVectorFile), sizeOfVector, sizeOfTokenVec, batchSize, sizeOfFeatureVector);
learner.setNumberOfEpochs(20);
learner.setSeed(123);
learner.setNumOfOutOfLayer1(20);
learner.setNumOfOutOfLayer2(50);
learner.setOutputPath(Configuration.EXTRACTED_FEATURES);
learner.extracteFeaturesWithCNN();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public void learnFeaturesOfSourceCode() {
int sizeOfVector = Integer.parseInt(FileHelper.readFile(Configuration.MAX_TOKEN_VECTORS_SIZE_OF_SOURCE_CODE));
int sizeOfTokenVec = Configuration.VECTOR_SIZE_OF_EMBEDED_TOKEN2;
int batchSize = 1000;
int sizeOfExtractedFeatureVector = 200;
try {
CNNFeatureExtractor2 learner = new CNNFeatureExtractor2(new File(Configuration.VECTORIED_ALL_SOURCE_CODE1), sizeOfVector, sizeOfTokenVec, batchSize, sizeOfExtractedFeatureVector);
learner.setNumberOfEpochs(20);
learner.setSeed(123);
learner.setNumOfOutOfLayer1(20);
learner.setNumOfOutOfLayer2(50);
learner.setOutputPath(Configuration.EXTRACTED_FEATURES_EVALUATION);
learner.extracteFeaturesWithCNN();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
/**
* Supervised learning.
*/
public void learnFeaturesOfSourceCode2(File testingData) {
int sizeOfVector = Integer.parseInt(FileHelper.readFile(Configuration.MAX_TOKEN_VECTORS_SIZE_OF_SOURCE_CODE));
int sizeOfTokenVec = Configuration.VECTOR_SIZE_OF_EMBEDED_TOKEN2;
int batchSize = 1000;
int sizeOfExtractedFeatureVector = 200;
try {
int clusterNum = DataPreparation.readCommonCLusters().size();
File trainingData = new File(Configuration.TRAINING_DATA);
CNNSupervisedLearning learner = new CNNSupervisedLearning(trainingData, sizeOfVector,
sizeOfTokenVec, batchSize, sizeOfExtractedFeatureVector, clusterNum, testingData);
learner.setNumberOfEpochs(20);
learner.setSeed(123);
learner.setNumOfOutOfLayer1(20);
learner.setNumOfOutOfLayer2(50);
learner.setOutputPath(Configuration.FEATURES_OF_TRAINING_DATA);
learner.setFeatresOfTestingData(Configuration.FEATURES_OF_TESTING_DATA);
learner.setPossibilitiesOfPrediction(Configuration.POSSIBILITIES_OF_TESTING_DATA);
learner.setPredictedResultsOfTestingData(Configuration.PREDICTED_RESULTS_OF_TESTING_DATA);
learner.setModelFile(Configuration.SUPERVISED_LEARNING_MODEL);
learner.extracteFeaturesWithCNN();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
/**
* Supervised learning by loading a model.
*/
public void learnFeaturesOfSourceCode3(File testingData) {
int batchSize = 1000;
try {
String modelFile = Configuration.SUPERVISED_LEARNING_MODEL;
CNNSupervisedLearning learner = new CNNSupervisedLearning(batchSize, testingData, modelFile);
learner.setFeatresOfTestingData(Configuration.FEATURES_OF_TESTING_DATA);
learner.setPossibilitiesOfPrediction(Configuration.POSSIBILITIES_OF_TESTING_DATA);
learner.setPredictedResultsOfTestingData(Configuration.PREDICTED_RESULTS_OF_TESTING_DATA);
learner.extracteFeaturesWithCNNByLoadingModel();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
src/main/java/edu/lu/uni/serval/FixPatternMining/TokenEmbedder.java
-65
View File
@@ -1,65 +0,0 @@
package edu.lu.uni.serval.FixPatternMining;
import java.io.File;
import java.io.IOException;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.deeplearner.Word2VecEncoder;
/**
* Encode tokens of edit scripts with Word2Vec.
*
* @author kui.liu
*
*/
public class TokenEmbedder {
/**
* Embed tokens for fix patterns mining.
*/
public void embedTokensOfEditScripts() {
Word2VecEncoder encoder = new Word2VecEncoder();
int windowSize = 2;
encoder.setWindowSize(windowSize);
try {
File inputFile = new File(Configuration.SELECTED_EDITSCRIPTES_FILE);
int minWordFrequency = 1;
int layerSize = Configuration.VECTOR_SIZE_OF_EMBEDED_TOKEN1;
String outputFileName = Configuration.EMBEDDED_EDIT_SCRIPT_TOKENS;
encoder.embedTokens(inputFile, minWordFrequency, layerSize, outputFileName);
} catch (IOException e) {
e.printStackTrace();
}
}
public void embedTokensOfSourceCodeForSupervisedTesting() {
Word2VecEncoder encoder = new Word2VecEncoder();
int windowSize = 2;
encoder.setWindowSize(windowSize);
try {
File inputFile = new File(Configuration.EMBEDDING_DATA_TOKENS2);
int minWordFrequency = 1;
int layerSize = Configuration.VECTOR_SIZE_OF_EMBEDED_TOKEN2;
String outputFileName = Configuration.EMBEDDED_ALL_TOKENS2;
encoder.embedTokens(inputFile, minWordFrequency, layerSize, outputFileName);
} catch (IOException e) {
e.printStackTrace();
}
}
public void embedTokensOfSourceCodeForUnsupervisedTesting() {
Word2VecEncoder encoder = new Word2VecEncoder();
int windowSize = 2;
encoder.setWindowSize(windowSize);
try {
File inputFile = new File(Configuration.EMBEDDING_DATA_TOKENS1);
int minWordFrequency = 1;
int layerSize = Configuration.VECTOR_SIZE_OF_EMBEDED_TOKEN2;
String outputFileName = Configuration.EMBEDDED_ALL_TOKENS1;
encoder.embedTokens(inputFile, minWordFrequency, layerSize, outputFileName);
} catch (IOException e) {
e.printStackTrace();
}
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/Tokenizer.java
+2
View File
@@ -26,6 +26,8 @@ public class Tokenizer {
tokens += astNodeType + " charLiteral ";
} else if ("ArrayInitializer".equals(astNodeType)) {
tokens += astNodeType + " arrayInitializer ";
} else if ("LambdaExpression".equals(astNodeType)) {
tokens += astNodeType + " lambda ";
} else {
tokens += astNodeType + " " + simpleTree.getLabel() + " ";
}
src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchSingleStatementParser.java
+2 -3
View File
@@ -61,8 +61,7 @@ public class CommitPatchSingleStatementParser extends CommitPatchParser {
startPosition = firstAndLastMov.get(0).getNode().getPos();
ITree lastTree = firstAndLastMov.get(1).getNode();
endPosition = lastTree.getPos() + lastTree.getLength();
} else { // Ignore the pure insert actions without any move
// actions.
} else { // Ignore the pure insert actions without any move actions.
continue;
}
} else if (actionStr.startsWith("UPD")) {
@@ -124,7 +123,7 @@ public class CommitPatchSingleStatementParser extends CommitPatchParser {
// 1. First level: AST node type.
String astEditScripts = getASTEditScripts(actionSet);
int size = astEditScripts.split(" ").length;
if (size == 1) {
if (size < 2) {
// System.out.println(actionSet);
continue;
}
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java
+86 -99
View File
@@ -1,12 +1,8 @@
package edu.lu.uni.serval.FixPatternParser.violations;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import edu.lu.uni.serval.FixPatternParser.Tokenizer;
import edu.lu.uni.serval.config.Configuration;
@@ -14,75 +10,106 @@ import edu.lu.uni.serval.diffentry.DiffEntryHunk;
import edu.lu.uni.serval.diffentry.DiffEntryReader;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter;
import edu.lu.uni.serval.gumtree.regroup.HunkFixPattern;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
import edu.lu.uni.serval.utils.MapSorter;
import edu.lu.uni.serval.utils.ListSorter;
/**
* Parse fix violations with GumTree.
* Parse fix violations with GumTree in terms of multiple statements.
*
* @author kui.liu
*
*/
public class FixedViolationHunkParser extends FixedViolationParser {
int counter;
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) {
// GumTree results
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree(prevFile, revFile);
// GumTree results
// TODO remove the modification of variable names or not?
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); // only remove non-statement source code, eg. method declaration
if (actionSets.size() > 0) {
Map<Integer, Integer> positions = readPositions();
if (positions.size() > 1) {
MapSorter<Integer, Integer> sorter = new MapSorter<>();
positions = sorter.sortByKeyAscending(positions);
List<Violation> violations = readPositionsAndAlarmTypes();
if (violations.size() > 1) {
ListSorter<Violation> sorter = new ListSorter<>(violations);
violations = sorter.sortAscending();
}
List<DiffEntryHunk> diffentryHunks1 = new DiffEntryReader().readHunks(diffentryFile);
int index = 0;
int hunkListSize = diffentryHunks1.size();
// Map<Integer, DiffEntryHunk> diffentryHunks = new HashMap<>();
List<DiffEntryHunk> diffentryHunks1 = new DiffEntryReader().readHunks2(diffentryFile);
// Select hunks by positions of violations.
List<DiffEntryHunk> diffentryHunks = new ArrayList<>();
for (Map.Entry<Integer, Integer> entry : positions.entrySet()) {
int startRange = entry.getKey();
int endRange = entry.getValue();
for (; index < hunkListSize; index ++) {
for (Violation violation : violations) {
int startLineNum = violation.getStartLineNum();
int endLineNum = violation.getEndLineNum();
for (int index = 0, hunkListSize = diffentryHunks1.size(); index < hunkListSize; index ++) {
DiffEntryHunk hunk = diffentryHunks1.get(index);
int startLine = hunk.getBugLineStartNum();
int range = hunk.getBugRange();
if (startRange > startLine + range) continue;
if (endRange < startLine) break;
// startRange and endRange
// diffentryHunks.put(startRange, hunk);
diffentryHunks.add(hunk);
if (startLineNum > startLine + range) continue;
if (endLineNum < startLine) break;
if (violation.getBugStartLineNum() == 0) {
violation.setBugStartLineNum(startLine);
violation.setFixStartLineNum(hunk.getFixLineStartNum());
}
violation.setBugEndLineNum(startLine + range);
violation.setFixEndLineNum(hunk.getFixLineStartNum() + hunk.getFixRange());
violation.getHunks().add(hunk);
}
}
//Filter out the modify actions, which are not in the DiffEntry hunks.
HunkActionFilter hunkFilter = new HunkActionFilter();
List<HunkFixPattern> allHunkFixPatterns = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, actionSets, revFile, prevFile);
List<Violation> selectedViolations = hunkFilter.filterActionsByModifiedRange2(violations, actionSets, revFile, prevFile);
for (HunkFixPattern hunkFixPattern : allHunkFixPatterns) {
for (Violation violation : selectedViolations) {
List<HierarchicalActionSet> hunkActionSets = violation.getActionSets();
// Remove overlapped UPD and INS
List<HierarchicalActionSet> addActions = new ArrayList<>();
List<HierarchicalActionSet> insertActions = new ArrayList<>();
for (HierarchicalActionSet hunkActionSet : hunkActionSets) {
if (hunkActionSet.getActionString().startsWith("INS")) insertActions.add(hunkActionSet);
if (hunkActionSet.getActionString().startsWith("UPD")) addActions.add(hunkActionSet);
}
List<HierarchicalActionSet> selectedActionSets = new ArrayList<>();
for (HierarchicalActionSet hunkActionSet : hunkActionSets) {
if (insertActions.contains(hunkActionSet)) {
boolean isIntersection = false;
int bugStartL1 = hunkActionSet.getBugStartLineNum();
int bugEndL1 = hunkActionSet.getBugEndLineNum();
for (HierarchicalActionSet addAction : addActions) {
int bugStartL = addAction.getBugStartLineNum();
int bugEndL = addAction.getBugEndLineNum();
if (bugEndL1 < bugStartL || bugStartL1 > bugEndL) {
continue;
}
isIntersection = true;
break;
}
if (!isIntersection) {
selectedActionSets.add(hunkActionSet);
}
continue;
}
selectedActionSets.add(hunkActionSet);
}
// Range of buggy source code
int startLine = 0;
int endLine = 0;
int bugStartLine = 0;
int bugEndLine = 0;
// Range of fixing source code
int startLine2 = 0;
int endLine2 = 0;
int fixStartLine = 0;
int fixEndLine = 0;
/*
* Convert the ITree of buggy code to a simple tree.
* It will be used to compute the similarity.
*/
List<HierarchicalActionSet> hunkActionSets = hunkFixPattern.getHunkActionSets();
SimpleTree simpleTree = new SimpleTree();
simpleTree.setLabel("Block");
simpleTree.setNodeType("Block");
List<SimpleTree> children = new ArrayList<>();
String astEditScripts = "";
for (HierarchicalActionSet hunkActionSet : hunkActionSets) {
for (HierarchicalActionSet hunkActionSet : selectedActionSets) {
SimplifyTree abstractIdentifier = new SimplifyTree();
abstractIdentifier.abstractTree(hunkActionSet);
SimpleTree simpleT = hunkActionSet.getSimpleTree();
@@ -101,33 +128,41 @@ public class FixedViolationHunkParser extends FixedViolationParser {
// 3. abstract identifiers:
// 4. semi-source code:
if (startLine == 0) {
startLine = hunkActionSet.getBugStartLineNum();
endLine = hunkActionSet.getBugEndLineNum();
startLine2 = hunkActionSet.getFixStartLineNum();
endLine2 = hunkActionSet.getFixEndLineNum();
} else {
if (startLine > hunkActionSet.getBugStartLineNum()) startLine = hunkActionSet.getBugStartLineNum();
if (startLine2 > hunkActionSet.getFixStartLineNum()) startLine2 = hunkActionSet.getFixStartLineNum();
if (endLine < hunkActionSet.getBugEndLineNum()) endLine = hunkActionSet.getBugEndLineNum();
if (endLine2 < hunkActionSet.getFixEndLineNum()) endLine2 = hunkActionSet.getFixEndLineNum();
int actionBugStart = hunkActionSet.getBugStartLineNum();
int actionBugEnd = hunkActionSet.getBugEndLineNum();
int actionFixStart = hunkActionSet.getFixStartLineNum();
int actionFixEnd = hunkActionSet.getFixEndLineNum();
if (bugStartLine == 0) {
bugStartLine = actionBugStart;
}
if (fixStartLine == 0) {
fixStartLine = actionFixStart;
}
if (bugEndLine < actionBugEnd) bugEndLine = actionBugEnd;
if (fixEndLine < actionFixEnd) fixEndLine = actionFixEnd;
}
if (children.size() == 0) continue;
if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue;
if (bugStartLine == 0) {
bugStartLine = violation.getStartLineNum();
if (bugEndLine < bugStartLine) bugEndLine = violation.getEndLineNum();
}
if (bugEndLine - bugStartLine >= Configuration.HUNK_SIZE || fixEndLine - fixStartLine >= Configuration.HUNK_SIZE) continue;
simpleTree.setChildren(children);
simpleTree.setParent(null);
// Source Code of patches.
String patchSourceCode = getPatchSourceCode(hunkFixPattern.getHunk(), startLine, endLine, startLine2, endLine2);
String patchSourceCode = getPatchSourceCode(prevFile, revFile, bugStartLine, bugEndLine, fixStartLine, fixEndLine);
if ("".equals(patchSourceCode)) continue;
this.patchesSourceCode += "PATCH###\n" + patchSourceCode + "\n";
counter ++;
String patchPosition = "";//"###:" + counter + "\n" + revFile.getName() + "\nPosition: " + violation.getStartLineNum() + " --> " + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n";
this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + patchPosition + patchSourceCode + "\n";
int size = astEditScripts.split(" ").length;
this.sizes += size + "\n";
this.astEditScripts += astEditScripts + "\n";
this.alarmTypes += violation.getAlarmType() + "\n";
// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n";
this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n";
// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n";
@@ -137,52 +172,4 @@ public class FixedViolationHunkParser extends FixedViolationParser {
}
}
private String getPatchSourceCode(DiffEntryHunk hunk, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) {
String sourceCode = hunk.getHunk();
int bugStartLine = hunk.getBugLineStartNum();
int fixStartLine = hunk.getFixLineStartNum();String buggyStatements = "";
String fixedStatements = "";
BufferedReader reader = null;
try {
reader = new BufferedReader(new StringReader(sourceCode));
String line = null;
int bugLines = 0;
int fixLines = 0;
int contextLines = 0; // counter of non-buggy code line.
while ((line = reader.readLine()) != null) {
int bugLineIndex = bugLines + contextLines;
int fixLineIndex = fixLines + contextLines;
if (line.startsWith("-")) {
if (bugStartLine + bugLineIndex >= startLineNum && bugStartLine + bugLineIndex <= endLineNum) {
buggyStatements += line + "\n";
}
bugLines ++;
} else if (line.startsWith("+")) {
if (fixStartLine + fixLineIndex >= startLineNum2 && fixStartLine + fixLineIndex <= endLineNum2) {
fixedStatements += line + "\n";
}
fixLines ++;
} else {
contextLines ++;
}
if (bugStartLine + bugLineIndex > endLineNum && fixStartLine + fixLineIndex > endLineNum2) {
break;
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (reader != null) {
reader.close();
reader = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
return buggyStatements + fixedStatements;
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java
+42 -14
View File
@@ -4,10 +4,15 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
import java.util.ArrayList;
import java.util.List;
import com.github.gumtreediff.actions.model.Action;
import edu.lu.uni.serval.FixPatternParser.Parser;
import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper;
import edu.lu.uni.serval.utils.FileHelper;
/**
@@ -18,7 +23,8 @@ import edu.lu.uni.serval.utils.FileHelper;
*/
public class FixedViolationParser extends Parser {
File positionFile = null;
private File positionFile = null;
protected String alarmTypes = "";
public void setPositionFile(File positionFile) {
this.positionFile = positionFile;
@@ -28,17 +34,33 @@ public class FixedViolationParser extends Parser {
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) {
}
protected boolean inPositions(int startLine, int endLine, Map<Integer, Integer> positions) {
for (Map.Entry<Integer, Integer> entry : positions.entrySet()) {
int startPosi = entry.getKey();
int endPosi = entry.getValue();
if (endLine >= startPosi && startLine <= endPosi) return true;
/**
* Regroup GumTree results without remove the modification of variable names.
*
* @param prevFile
* @param revFile
* @return
*/
protected List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree2(File prevFile, File revFile) {
List<HierarchicalActionSet> actionSets = new ArrayList<>();
// GumTree results
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
if (gumTreeResults != null && gumTreeResults.size() > 0) {
// Regroup GumTre results.
List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
for (HierarchicalActionSet actionSet : allActionSets) {
String astNodeType = actionSet.getAstNodeType();
if (astNodeType.endsWith("Statement") || "FieldDeclaration".equals(astNodeType)) {
actionSets.add(actionSet);
}
}
}
return false;
return actionSets;
}
protected Map<Integer, Integer> readPositions() {
Map<Integer, Integer> positions = new HashMap<>();
protected List<Violation> readPositionsAndAlarmTypes() {
List<Violation> violations = new ArrayList<>();
String fileContent = FileHelper.readFile(positionFile);
BufferedReader reader = null;
reader = new BufferedReader(new StringReader(fileContent));
@@ -48,7 +70,10 @@ public class FixedViolationParser extends Parser {
String[] positionStr = line.split(":");
int startLine = Integer.parseInt(positionStr[0]);
int endLine = Integer.parseInt(positionStr[1]);
positions.put(startLine, endLine);
String alarmType = positionStr[2];
Violation violation = new Violation(startLine, endLine, alarmType);
violations.add(violation);
}
} catch (IOException e) {
e.printStackTrace();
@@ -59,10 +84,9 @@ public class FixedViolationParser extends Parser {
e.printStackTrace();
}
}
return positions;
return violations;
}
protected String getPatchSourceCode(File prevFile, File revFile, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) {
String buggyStatements = readSourceCode(prevFile, startLineNum, endLineNum, "-");
String fixedStatements = readSourceCode(revFile, startLineNum2, endLineNum2, "+");
@@ -96,4 +120,8 @@ public class FixedViolationParser extends Parser {
return sourceCode;
}
public String getAlarmTypes() {
return alarmTypes;
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java
+14 -6
View File
@@ -2,7 +2,6 @@ package edu.lu.uni.serval.FixPatternParser.violations;
import java.io.File;
import java.util.List;
import java.util.Map;
import org.eclipse.jdt.core.dom.CompilationUnit;
@@ -19,7 +18,7 @@ import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
/**
* Parse fixed violations with GumTree.
* Parse fixed violations with GumTree in terms of single statement.
*
* @author kui.liu
*
@@ -40,7 +39,7 @@ public class FixedViolationSingleStatementParser extends FixedViolationParser {
}
// Read the positions of checked violations
Map<Integer, Integer> positions = readPositions();
List<Violation> violations = readPositionsAndAlarmTypes();
for (HierarchicalActionSet actionSet : actionSets) {
// position of buggy statements
int startPosition = 0;
@@ -96,9 +95,9 @@ public class FixedViolationSingleStatementParser extends FixedViolationParser {
int startLine2 = revUnit.getLineNumber(startPosition2);
int endLine2 = revUnit.getLineNumber(endPosition2);
if (!inPositions(startLine, endLine, positions)) {
continue;
}
Violation violation = findViolation(startLine, endLine, violations);
if (violation == null) continue;
if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue;
/*
@@ -128,6 +127,7 @@ public class FixedViolationSingleStatementParser extends FixedViolationParser {
this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + revFile.getName() + "\n" + patchSourceCode + "\n";
this.sizes += size + "\n";
this.astEditScripts += astEditScripts + "\n";
this.alarmTypes += violation.getAlarmType() + "\n";
// 2. source code: raw tokens
// String rawTokenEditScripts = getRawTokenEditScripts(actionSet);
// // 3. abstract identifiers:
@@ -146,4 +146,12 @@ public class FixedViolationSingleStatementParser extends FixedViolationParser {
}
}
protected Violation findViolation(int startLine, int endLine, List<Violation> violations) {
for (Violation violation : violations) {
int vStartLine = violation.getStartLineNum();
int vEndLine = violation.getEndLineNum();
if (!(startLine > vEndLine && endLine< vStartLine)) return violation;
}
return null;
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParser.java
+269
View File
@@ -0,0 +1,269 @@
package edu.lu.uni.serval.FixPatternParser.violations;
import static java.lang.System.err;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import edu.lu.uni.serval.FixPatternParser.RunnableParser;
import edu.lu.uni.serval.MultipleThreadsParser.MessageFile;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
public class TestHunkParser {
public static void main(String[] args) {
// input data
final List<MessageFile> msgFiles = getMessageFiles("GumTreeInput/");
System.out.println(msgFiles.size());
// output path
final String editScriptsFilePath = "GumTreeResults/editScripts.list";
final String patchesSourceCodeFilePath = "GumTreeResults/patchSourceCode.list";
final String buggyTokensFilePath = "GumTreeResults/tokens.list";
final String editScriptSizesFilePath = "GumTreeResults/editScriptSizes.list";
final String alarmTypesFilePath = "GumTreeResults/alarmTypes.list";
FileHelper.deleteDirectory(editScriptsFilePath);
FileHelper.deleteDirectory(patchesSourceCodeFilePath);
FileHelper.deleteDirectory(buggyTokensFilePath);
FileHelper.deleteDirectory(editScriptSizesFilePath);
FileHelper.deleteDirectory(alarmTypesFilePath);
StringBuilder astEditScripts = new StringBuilder();
StringBuilder tokens = new StringBuilder();
StringBuilder sizes = new StringBuilder();
StringBuilder patches = new StringBuilder();
StringBuilder alarmTypes = new StringBuilder();
int a = 0;
int counter = 0;
for (MessageFile msgFile : msgFiles) {
FixedViolationHunkParser parser = new FixedViolationHunkParser();
parser.counter = counter;
parser.setPositionFile(msgFile.getPositionFile());
final ExecutorService executor = Executors.newSingleThreadExecutor();
// schedule the work
final Future<?> future = executor.submit(new RunnableParser(msgFile.getPrevFile(),
msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser));
try {
// where we wait for task to complete
// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
future.get(20L, TimeUnit.SECONDS);
String editScripts = parser.getAstEditScripts();
if (!editScripts.equals("")) {
astEditScripts.append(editScripts);
tokens.append(parser.getTokensOfSourceCode());
sizes.append(parser.getSizes());
patches.append(parser.getPatchesSourceCode());
alarmTypes.append(parser.getAlarmTypes());
counter = parser.counter;
a ++;
if (a % 100 == 0) {
FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
astEditScripts.setLength(0);
tokens.setLength(0);
sizes.setLength(0);
patches.setLength(0);
alarmTypes.setLength(0);
System.out.println("Finish of parsing " + a + " files......");
}
}
} catch (TimeoutException e) {
err.println("task timed out");
future.cancel(true /* mayInterruptIfRunning */ );
} catch (InterruptedException e) {
err.println("task interrupted");
} catch (ExecutionException e) {
err.println("task aborted");
} finally {
executor.shutdownNow();
}
}
FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
astEditScripts.setLength(0);
tokens.setLength(0);
sizes.setLength(0);
patches.setLength(0);
alarmTypes.setLength(0);
System.out.println(a);
// classifyByAlarmTypes();
}
private static List<MessageFile> getMessageFiles(String gumTreeInput) {
String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
File revFilesPath = new File(inputPath + "revFiles/");
File[] revFiles = revFilesPath.listFiles(); // project folders
List<MessageFile> msgFiles = new ArrayList<>();
for (File revFile : revFiles) {
if (revFile.getName().endsWith(".java")) {
String fileName = revFile.getName();
File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
fileName = fileName.replace(".java", ".txt");
File diffentryFile = new File(gumTreeInput + "diffentries/" + fileName); // DiffEntry file
File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
msgFile.setPositionFile(positionFile);
msgFiles.add(msgFile);
}
}
return msgFiles;
}
public static void classifyByAlarmTypes() {
final String alarmTypesFilePath = Configuration.ALARM_TYPES_FILE;
List<String> alarmTypes = readStringList(alarmTypesFilePath);
//edit scripts, sizes of edit scripts, buggy tokens, patches.
classifyByAlarmTypes(alarmTypes, Configuration.EDITSCRIPT_SIZES_FILE);
classifyByAlarmTypes(alarmTypes, Configuration.EDITSCRIPTS_FILE);
classifyByAlarmTypes(alarmTypes, Configuration.BUGGY_CODE_TOKENS_FILE);
classifyByAlarmTypes2(alarmTypes, Configuration.PATCH_SOURCECODE_FILE);
}
private static void classifyByAlarmTypes(List<String> alarmTypes, String file) {
Map<String, StringBuilder> buildersMap = new HashMap<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(file);
scanner = new Scanner(fis);
int counter = 0;
while (scanner.hasNextLine()) {
String alarmType = alarmTypes.get(counter);
StringBuilder builder = getBuilder(buildersMap, alarmType);
builder.append(scanner.nextLine() + "\n");
counter ++;
if (counter % 1000 == 0) {
outputBuilders(buildersMap, file);
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
outputBuilders(buildersMap, file);
}
private static void classifyByAlarmTypes2(List<String> alarmTypes, String patchSourcecodeFile) {
Map<String, StringBuilder> buildersMap = new HashMap<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(patchSourcecodeFile);
scanner = new Scanner(fis);
int counter = 0;
String singlePatch = "";
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if (Configuration.PATCH_SIGNAL.equals(line)) {
if (!"".equals(singlePatch)) {
String alarmType = alarmTypes.get(counter);
StringBuilder builder = getBuilder(buildersMap, alarmType);
builder.append(scanner.nextLine() + "\n");
counter ++;
if (counter % 2000 == 0) {
outputBuilders(buildersMap, patchSourcecodeFile);
}
}
singlePatch = line + "\n";
}
singlePatch += line + "\n";
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
outputBuilders(buildersMap, patchSourcecodeFile);
}
private static void outputBuilders(Map<String, StringBuilder> map, String fileNameStr) {
File file = new File(fileNameStr);
String fileName = file.getName();
String parentPath = file.getParent();
for (Map.Entry<String, StringBuilder> entry : map.entrySet()) {
String alarmType = entry.getKey();
StringBuilder builder = entry.getValue();
FileHelper.outputToFile(parentPath + "/" + alarmType + "/" + fileName, builder, true);
builder.setLength(0);
entry.setValue(builder);
}
}
public static List<String> readStringList(String inputFile) {
List<String> list = new ArrayList<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(inputFile);
scanner = new Scanner(fis);
while(scanner.hasNextLine()) {
list.add(scanner.nextLine());
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return list;
}
private static StringBuilder getBuilder(Map<String, StringBuilder> buildersMap, String alarmType) {
if (buildersMap.containsKey(alarmType)) {
return buildersMap.get(alarmType);
} else {
StringBuilder builder = new StringBuilder();
buildersMap.put(alarmType, builder);
return builder;
}
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/Violation.java
+95
View File
@@ -0,0 +1,95 @@
package edu.lu.uni.serval.FixPatternParser.violations;
import java.util.ArrayList;
import java.util.List;
import edu.lu.uni.serval.diffentry.DiffEntryHunk;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
public class Violation implements Comparable<Violation> {
private Integer startLineNum;
private int endLineNum;
private int bugStartLineNum;
private int bugEndLineNum;
private int fixStartLineNum;
private int fixEndLineNum;
private String alarmType;
private List<DiffEntryHunk> hunks = new ArrayList<>();
private List<HierarchicalActionSet> actionSets;
public Violation(Integer startLineNum, int endLineNum, String alarmType) {
super();
this.startLineNum = startLineNum;
this.endLineNum = endLineNum;
this.alarmType = alarmType;
this.actionSets = new ArrayList<>();
}
public Integer getStartLineNum() {
return startLineNum;
}
public int getEndLineNum() {
return endLineNum;
}
public int getBugStartLineNum() {
return bugStartLineNum;
}
public void setBugStartLineNum(int bugStartLineNum) {
this.bugStartLineNum = bugStartLineNum;
}
public int getBugEndLineNum() {
return bugEndLineNum;
}
public void setBugEndLineNum(int bugEndLineNum) {
this.bugEndLineNum = bugEndLineNum;
}
public int getFixStartLineNum() {
return fixStartLineNum;
}
public void setFixStartLineNum(int fixStartLineNum) {
this.fixStartLineNum = fixStartLineNum;
}
public int getFixEndLineNum() {
return fixEndLineNum;
}
public void setFixEndLineNum(int fixEndLineNum) {
this.fixEndLineNum = fixEndLineNum;
}
public List<DiffEntryHunk> getHunks() {
return hunks;
}
public void setHunks(List<DiffEntryHunk> hunks) {
this.hunks = hunks;
}
public String getAlarmType() {
return alarmType;
}
public List<HierarchicalActionSet> getActionSets() {
return actionSets;
}
@Override
public int compareTo(Violation v) {
return this.startLineNum.compareTo(v.startLineNum);
}
@Override
public String toString() {
return this.startLineNum + " : " + this.endLineNum + " : " + this.alarmType;
}
}
src/main/java/edu/lu/uni/serval/MultipleThreadsParser/AkkaParser.java
+61 -5
View File
@@ -12,14 +12,22 @@ import akka.actor.ActorSystem;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Multi-thread parser of parsing the difference between buggy code file and fixed code file.
*
* @author kui.liu
*
*/
public class AkkaParser {
private static Logger log = LoggerFactory.getLogger(AkkaParser.class);
@SuppressWarnings("deprecation")
public static void main(String[] args) {
// input data
log.info("Get the input data...");
final List<MessageFile> msgFiles = getMessageFiles();
// final List<MessageFile> msgFiles = getMessageFiles();
final List<MessageFile> msgFiles = getMessageFiles("GumTreeInput/");
log.info("MessageFiles: " + msgFiles.size());
// output path
@@ -27,27 +35,37 @@ public class AkkaParser {
final String patchesSourceCodeFilePath = Configuration.PATCH_SOURCECODE_FILE_PATH;
final String buggyTokensFilePath = Configuration.BUGGY_CODE_TOKEN_FILE_PATH;
final String editScriptSizesFilePath = Configuration.EDITSCRIPT_SIZES_FILE_PATH;
final String alarmTypesFilePath = Configuration.ALARM_TYPES_FILE_PATH;
FileHelper.deleteDirectory(editScriptsFilePath);
FileHelper.deleteDirectory(patchesSourceCodeFilePath);
FileHelper.deleteDirectory(buggyTokensFilePath);
FileHelper.deleteDirectory(editScriptSizesFilePath);
FileHelper.deleteDirectory(alarmTypesFilePath);
ActorSystem system = null;
ActorRef parsingActor = null;
final int numberOfWorkers = 200;
int numberOfWorkers = 20;
final WorkMessage msg = new WorkMessage(0, msgFiles);
try {
log.info("Akka begins...");
system = ActorSystem.create("Mining-FixPattern-System");
parsingActor = system.actorOf(ParseFixPatternActor.props(numberOfWorkers, editScriptsFilePath, patchesSourceCodeFilePath, buggyTokensFilePath, editScriptSizesFilePath), "mine-fix-pattern-actor");
parsingActor = system.actorOf(ParseFixPatternActor.props(numberOfWorkers, editScriptsFilePath,
patchesSourceCodeFilePath, buggyTokensFilePath, editScriptSizesFilePath, alarmTypesFilePath), "mine-fix-pattern-actor");
parsingActor.tell(msg, ActorRef.noSender());
} catch (Exception e) {
system.shutdown();
e.printStackTrace();
}
}
private static List<MessageFile> getMessageFiles() {
/**
* Get bug commit-related files.
*
* @return
*/
public static List<MessageFile> getMessageFiles() {
String inputPath = Configuration.GUM_TREE_INPUT; //DiffEntries prevFiles revFiles
File inputFileDirector = new File(inputPath);
File[] files = inputFileDirector.listFiles(); // project folders
@@ -56,6 +74,16 @@ public class AkkaParser {
for (File file : files) {
if (!file.isDirectory()) continue;
// if (!(file.getName().startsWith("k") || file.getName().startsWith("l"))) continue;
if (file.getName().startsWith("a") || file.getName().startsWith("b")
|| file.getName().startsWith("c") || file.getName().startsWith("d")
|| file.getName().startsWith("e") || file.getName().startsWith("f")
|| file.getName().startsWith("g") || file.getName().startsWith("h")
||file.getName().startsWith("h") || file.getName().startsWith("i")
|| file.getName().startsWith("k") || file.getName().startsWith("l")
|| file.getName().startsWith("j") || file.getName().startsWith("t")) continue;
// if (!file.getName().startsWith("j")) continue;
log.info("Project name: " + file.getName());
String projectFolder = file.getPath();
File revFileFolder = new File(projectFolder + "/revFiles/");// revised file folder
File[] revFiles = revFileFolder.listFiles();
@@ -71,4 +99,32 @@ public class AkkaParser {
return msgFiles;
}
/**
* Get violation-related files.
*
* @param gumTreeInput
* @return
*/
public static List<MessageFile> getMessageFiles(String gumTreeInput) {
String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
File revFilesPath = new File(inputPath + "revFiles/");
File[] revFiles = revFilesPath.listFiles(); // project folders
List<MessageFile> msgFiles = new ArrayList<>();
for (File revFile : revFiles) {
if (revFile.getName().endsWith(".java")) {
String fileName = revFile.getName();
File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
fileName = fileName.replace(".java", ".txt");
File diffentryFile = new File(gumTreeInput + "diffentries/" + fileName); // DiffEntry file
File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
msgFile.setPositionFile(positionFile);
msgFiles.add(msgFile);
}
}
return msgFiles;
}
}
src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternActor.java
+13 -5
View File
@@ -20,13 +20,16 @@ public class ParseFixPatternActor extends UntypedActor {
private final int numberOfWorkers;
private int counter = 0;
public ParseFixPatternActor(int numberOfWorkers, String editScriptsFilePath, String patchesSourceCodeFilePath, String buggyTokensFilePath, String editScriptSizesFilePath) {
public ParseFixPatternActor(int numberOfWorkers, String editScriptsFilePath, String patchesSourceCodeFilePath,
String buggyTokensFilePath, String editScriptSizesFilePath, String alarmTypesFilePath) {
mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers)
.props(ParseFixPatternWorker.props(editScriptsFilePath, patchesSourceCodeFilePath, buggyTokensFilePath, editScriptSizesFilePath)), "mine-fix-pattern-router");
.props(ParseFixPatternWorker.props(editScriptsFilePath, patchesSourceCodeFilePath,
buggyTokensFilePath, editScriptSizesFilePath, alarmTypesFilePath)), "mine-fix-pattern-router");
this.numberOfWorkers = numberOfWorkers;
}
public static Props props(final int numberOfWorkers, final String editScriptsFilePath, final String patchesSourceCodeFilePath, final String buggyTokensFilePath, final String editScriptSizesFilePath) {
public static Props props(final int numberOfWorkers, final String editScriptsFilePath, final String patchesSourceCodeFilePath,
final String buggyTokensFilePath, final String editScriptSizesFilePath, final String alarmTypesFilePath) {
return Props.create(new Creator<ParseFixPatternActor>() {
@@ -34,12 +37,14 @@ public class ParseFixPatternActor extends UntypedActor {
@Override
public ParseFixPatternActor create() throws Exception {
return new ParseFixPatternActor(numberOfWorkers, editScriptsFilePath, patchesSourceCodeFilePath, buggyTokensFilePath, editScriptSizesFilePath);
return new ParseFixPatternActor(numberOfWorkers, editScriptsFilePath, patchesSourceCodeFilePath,
buggyTokensFilePath, editScriptSizesFilePath, alarmTypesFilePath);
}
});
}
@SuppressWarnings("deprecation")
@Override
public void onReceive(Object message) throws Exception {
if (message instanceof WorkMessage) {
@@ -61,7 +66,10 @@ public class ParseFixPatternActor extends UntypedActor {
logger.info("Assign a task to worker #" + (i + 1) + "...");
}
} else if ("STOP".equals(message.toString())) {
if (++ counter >= numberOfWorkers) {
counter ++;
logger.info(counter + " workers finished their work...");
if (counter >= numberOfWorkers) {
logger.info("All workers finished their work...");
this.getContext().stop(mineRouter);
this.getContext().stop(getSelf());
this.getContext().system().shutdown();
src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java
+53 -25
View File
@@ -17,8 +17,10 @@ import org.slf4j.LoggerFactory;
import akka.actor.Props;
import akka.actor.UntypedActor;
import akka.japi.Creator;
import edu.lu.uni.serval.FixPatternParser.Parser;
import edu.lu.uni.serval.FixPatternParser.RunnableParser;
import edu.lu.uni.serval.FixPatternParser.patch.CommitPatchSingleStatementParser;
import edu.lu.uni.serval.FixPatternParser.violations.FixedViolationHunkParser;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
@@ -29,22 +31,27 @@ public class ParseFixPatternWorker extends UntypedActor {
private String patchesSourceCodeFilePath;
private String editScriptSizesFilePath;
private String buggyTokensFilePath;
private String alarmTypesFilePath;
public ParseFixPatternWorker(String editScriptsFilePath, String patchesSourceCodeFilePath, String buggyTokensFilePath, String editScriptSizesFilePath) {
public ParseFixPatternWorker(String editScriptsFilePath, String patchesSourceCodeFilePath,
String buggyTokensFilePath, String editScriptSizesFilePath, String alarmTypesFilePath) {
this.editScriptsFilePath = editScriptsFilePath;
this.patchesSourceCodeFilePath = patchesSourceCodeFilePath;
this.editScriptSizesFilePath = editScriptSizesFilePath;
this.buggyTokensFilePath = buggyTokensFilePath;
this.alarmTypesFilePath = alarmTypesFilePath;
}
public static Props props(final String editScriptsFile, final String patchesSourceCodeFile, final String buggyTokensFilePath, final String editScriptSizesFilePath) {
public static Props props(final String editScriptsFile, final String patchesSourceCodeFile, final String buggyTokensFilePath,
final String editScriptSizesFilePath, final String alarmTypesFilePath) {
return Props.create(new Creator<ParseFixPatternWorker>() {
private static final long serialVersionUID = -7615153844097275009L;
@Override
public ParseFixPatternWorker create() throws Exception {
return new ParseFixPatternWorker(editScriptsFile, patchesSourceCodeFile, buggyTokensFilePath, editScriptSizesFilePath);
return new ParseFixPatternWorker(editScriptsFile, patchesSourceCodeFile,
buggyTokensFilePath, editScriptSizesFilePath, alarmTypesFilePath);
}
});
@@ -58,16 +65,25 @@ public class ParseFixPatternWorker extends UntypedActor {
StringBuilder editScripts = new StringBuilder();
StringBuilder patchesSourceCode = new StringBuilder();
StringBuilder sizes = new StringBuilder();
// StringBuilder buggyTrees = new StringBuilder();
StringBuilder tokens = new StringBuilder();
StringBuilder alarmTypes = new StringBuilder();
int id = msg.getId();
int counter = 0;
boolean containsAlarmTypes = false;
for (MessageFile msgFile : files) {
counter ++;
File revFile = msgFile.getRevFile();
File prevFile = msgFile.getPrevFile();
File diffentryFile = msgFile.getDiffEntryFile();
CommitPatchSingleStatementParser parser = new CommitPatchSingleStatementParser();
File positionFile = msgFile.getPositionFile();
Parser parser = null;
if (positionFile == null) {
parser = new CommitPatchSingleStatementParser();
} else {
parser = new FixedViolationHunkParser();
containsAlarmTypes = true;
}
final ExecutorService executor = Executors.newSingleThreadExecutor();
// schedule the work
@@ -76,28 +92,33 @@ public class ParseFixPatternWorker extends UntypedActor {
// wait for task to complete
future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
editScripts.append(parser.getAstEditScripts());
patchesSourceCode.append(parser.getPatchesSourceCode());
sizes.append(parser.getSizes());
// buggyTrees.append(parser.getBuggyTrees());
tokens.append(parser.getTokensOfSourceCode());
counter ++;
if (counter % 100 == 0) {
FileHelper.outputToFile(editScriptsFilePath + "edistScripts_" + id + ".list", editScripts, true);
FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true);
FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true);
// FileHelper.outputToFile(buggyTreesFilePath + "buggyTrees_" + id + ".list", buggyTrees, true);
FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true);
editScripts.setLength(0);
patchesSourceCode.setLength(0);
sizes.setLength(0);
// buggyTrees.setLength(0);
tokens.setLength(0);
log.info("Worker #" + id +"Finish of parsing " + counter + " files...");
String editScript = parser.getAstEditScripts();
if (!"".equals(editScript)) {
editScripts.append(editScript);
patchesSourceCode.append(parser.getPatchesSourceCode());
sizes.append(parser.getSizes());
tokens.append(parser.getTokensOfSourceCode());
if (positionFile == null) alarmTypes.append(((FixedViolationHunkParser) parser).getAlarmTypes());
if (counter % 100 == 0) {
FileHelper.outputToFile(editScriptsFilePath + "edistScripts_" + id + ".list", editScripts, true);
FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true);
FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true);
FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true);
editScripts.setLength(0);
patchesSourceCode.setLength(0);
sizes.setLength(0);
tokens.setLength(0);
if (containsAlarmTypes) {
FileHelper.outputToFile(alarmTypesFilePath + "alarmTypes_" + id + ".list", alarmTypes, true);
alarmTypes.setLength(0);
}
log.info("Worker #" + id +"Finish of parsing " + counter + " files...");
}
}
} catch (TimeoutException e) {
err.println("task timed out");
future.cancel(true /* mayInterruptIfRunning */ );
future.cancel(true);
} catch (InterruptedException e) {
err.println("task interrupted");
} catch (ExecutionException e) {
@@ -111,8 +132,15 @@ public class ParseFixPatternWorker extends UntypedActor {
FileHelper.outputToFile(editScriptsFilePath + "edistScripts_" + id + ".list", editScripts, true);
FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true);
FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true);
// FileHelper.outputToFile(buggyTreesFilePath + "buggyTrees_" + id + ".list", buggyTrees, true);
FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true);
editScripts.setLength(0);
patchesSourceCode.setLength(0);
sizes.setLength(0);
tokens.setLength(0);
if (containsAlarmTypes) {
FileHelper.outputToFile(alarmTypesFilePath + "alarmTypes_" + id + ".list", alarmTypes, true);
alarmTypes.setLength(0);
}
}
log.info("Worker #" + id +"Finish of parsing " + counter + " files...");
src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step8.java → src/main/java/edu/lu/uni/serval/bugLocalization/Step8.java
+1 -2
View File
@@ -1,8 +1,7 @@
package edu.lu.uni.serval.FixPatternMining.App;
package edu.lu.uni.serval.bugLocalization;
import java.io.File;
import edu.lu.uni.serval.bugLocalization.ProjectScanner;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
src/main/java/edu/lu/uni/serval/config/Configuration.java
+6 -2
View File
@@ -2,9 +2,9 @@ package edu.lu.uni.serval.config;
public class Configuration {
public static final long SECONDS_TO_WAIT = 60L;
public static final long SECONDS_TO_WAIT = 20L;
private static final String ROOT_PATH = "../"; // The root path of all output data.
private static final String ROOT_PATH = "../FPM_Violations/"; // The root path of all output data.
public static final int HUNK_SIZE = 7; // The limitation of source code lines of each DiffEntry, which will be selected as training data.
public static final String BUGGY_TREE_SIGNAL = "BUGGY_TREE###"; // The starting signal of the tree of buggy source code .
@@ -13,6 +13,7 @@ public class Configuration {
// input path of GumTree. (i.e., Fix patterns parser)
public static final String GUM_TREE_INPUT = ROOT_PATH + "GumTreeInput/";// Buggy version file VS. Fixing version file, (DiffEntry File)
// the output path of GumTree results.
private static final String GUM_TREE_OUTPUT = ROOT_PATH + "GumTreeResults/";
public static final String EDITSCRIPTS_FILE_PATH = GUM_TREE_OUTPUT + "editScripts/";
@@ -20,12 +21,14 @@ public class Configuration {
public static final String BUGGYTREE_FILE_PATH = GUM_TREE_OUTPUT + "buggyTrees/";
public static final String BUGGY_CODE_TOKEN_FILE_PATH = GUM_TREE_OUTPUT + "tokens/";
public static final String EDITSCRIPT_SIZES_FILE_PATH = GUM_TREE_OUTPUT + "editScriptSizes/";
public static final String ALARM_TYPES_FILE_PATH = GUM_TREE_OUTPUT + "alarmTypes/";
public static final String EDITSCRIPTS_FILE = GUM_TREE_OUTPUT + "editScripts.list";
public static final String PATCH_SOURCECODE_FILE = GUM_TREE_OUTPUT + "patchSourceCode.list";
public static final String BUGGYTREES_FILE = GUM_TREE_OUTPUT + "buggyTrees.list";
public static final String BUGGY_CODE_TOKENS_FILE = GUM_TREE_OUTPUT + "tokens.list";
public static final String EDITSCRIPT_SIZES_FILE = GUM_TREE_OUTPUT + "editScriptSizes.list";
public static final String ALARM_TYPES_FILE = GUM_TREE_OUTPUT + "alarmTypes.list";
public static final int VECTOR_SIZE_OF_EMBEDED_TOKEN1 = 100; // tokens of edit scripts.
public static final int VECTOR_SIZE_OF_EMBEDED_TOKEN2 = 200; // tokens of source code
@@ -40,6 +43,7 @@ public class Configuration {
public static final String SELECTED_BUGGY_TREE_FILE = EMBEDDING_INPUT + "buggyTrees.list";
public static final String SELECTED_BUGGY_TOKEN_FILE = EMBEDDING_INPUT + "tokens.list"; // Selected token vectors of buggy source code.
public static final String SELECTED_EDITSCRIPTES_FILE = EMBEDDING_INPUT + "editScripts.list"; // Selected edit script vectors.
public static final String SELECTED_ALARM_TYPES_FILE = EMBEDDING_INPUT + "alarmTypes.list"; // Selected edit script vectors.
// the input path of feature learning.
public static final String FEATURE_LEARNING_INPUT = MINING_INPUT + "FeatureLearning/";
public static final String EMBEDDED_EDIT_SCRIPT_TOKENS = FEATURE_LEARNING_INPUT + "embeddedEditScriptTokens.list"; // All embedded tokens of selected edit scripts.
src/main/java/edu/lu/uni/serval/diffentry/DiffEntryHunk.java
+5
View File
@@ -40,4 +40,9 @@ public class DiffEntryHunk {
this.hunk = hunk;
}
@Override
public String toString() {
return "@@ -" + this.bugLineStartNum + ", " + this.bugRange + " +" + this.fixLineStartNum + ", " + this.fixRange + "\n" + this.hunk;
}
}
src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalActionSet.java
+1 -1
View File
@@ -20,7 +20,7 @@ public class HierarchicalActionSet implements Comparable<HierarchicalActionSet>
private String actionString;
private int startPosition;
private int length;
private int bugStartLineNum;
private int bugStartLineNum = 0;
private int bugEndLineNum;
private int fixStartLineNum;
private int fixEndLineNum;
src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java
+4 -1
View File
@@ -114,7 +114,10 @@ public class HierarchicalRegrouper {
actionSet.setParent(actSet);
actSet.getSubActions().add(actionSet);
ListSorter<HierarchicalActionSet> sorter = new ListSorter<HierarchicalActionSet>(actSet.getSubActions());
actSet.setSubActions(sorter.sortAscending());
List<HierarchicalActionSet> subActions = sorter.sortAscending();
if (subActions != null) {
actSet.setSubActions(subActions);
}
break;
} else {
if ((!(action instanceof Insert) && !(act instanceof Insert))
src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java
+411 -89
View File
@@ -1,6 +1,9 @@
package edu.lu.uni.serval.gumtree.regroup;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
@@ -11,12 +14,13 @@ import com.github.gumtreediff.actions.model.Update;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.FixPatternParser.CUCreator;
import edu.lu.uni.serval.FixPatternParser.violations.Violation;
import edu.lu.uni.serval.diffentry.DiffEntryHunk;
public class HunkActionFilter {
/**
* Filter out the modify actions, which are not in the DiffEntry hunks.
* Filter out the modify actions, which are not in the DiffEntry hunks, without considering the same parent node.
*
* @param hunks
* @param actionSets
@@ -66,10 +70,10 @@ public class HunkActionFilter {
endPosition2 = startPosition2 + newNode.getLength();
}
}
startLine = prevUnit.getLineNumber(startPosition);
endLine = prevUnit.getLineNumber(endPosition);
startLine2 = revUnit.getLineNumber(startPosition2);
endLine2 = revUnit.getLineNumber(endPosition2);
startLine = startPosition == 0 ? 0 : prevUnit.getLineNumber(startPosition);
endLine = endPosition == 0 ? 0 : prevUnit.getLineNumber(endPosition);
startLine2 = startPosition2 == 0 ? 0 : revUnit.getLineNumber(startPosition2);
endLine2 = endPosition2 == 0 ? 0 : revUnit.getLineNumber(endPosition2);
for (DiffEntryHunk hunk : hunks) {
int bugStartLine = hunk.getBugLineStartNum();
@@ -84,7 +88,6 @@ public class HunkActionFilter {
if (endLine2 < fixStartLine ) {
uselessActions.add(actionSet);
}
break;
} else {
if (bugStartLine + bugRange < startLine) {
continue;
@@ -107,7 +110,7 @@ public class HunkActionFilter {
}
/**
* Filter out the modify actions, which are not in the DiffEntry hunks.
* Filter out the modify actions, which are not in the DiffEntry hunks, with considering the same parent node.
*
* @param hunks
* @param actionSets
@@ -127,107 +130,58 @@ public class HunkActionFilter {
int i = 0;
int size = actionSets.size();
for (DiffEntryHunk hunk : hunks) {
int bugStartLine = hunk.getBugLineStartNum();
int bugRange = hunk.getBugRange();
int fixStartLine = hunk.getFixLineStartNum();
int fixRange = hunk.getFixRange();
int hunkBugStartLine = hunk.getBugLineStartNum();
int hunkBugRange = hunk.getBugRange();
int hunkFixStartLine = hunk.getFixLineStartNum();
int hunkFixRange = hunk.getFixRange();
for (; i < size; i ++) {
// position of buggy statements
int startPosition = 0;
int endPosition = 0;
int startLine = 0;
int endLine = 0;
// position of fixed statements
int startPosition2 = 0;
int endPosition2 = 0;
int startLine2 = 0;
int endLine2 = 0;
HierarchicalActionSet actionSet = actionSets.get(i);
int actionBugStartLine = actionSet.getBugStartLineNum();
if (actionBugStartLine == 0) {
actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit);
}
int actionBugEndLine = actionSet.getBugEndLineNum();
int actionFixStartLine = actionSet.getFixStartLineNum();
int actionFixEndLine = actionSet.getFixEndLineNum();
String actionStr = actionSet.getActionString();
ITree parentITree = null;
ITree previousParent = null;
List<HierarchicalActionSet> hunkActionSets = new ArrayList<>();
if (actionStr.startsWith("INS")) {
startPosition2 = actionSet.getStartPosition();
endPosition2 = startPosition2 + actionSet.getLength();
List<Move> firstAndLastMov = getFirstAndLastMoveAction(actionSet);
if (firstAndLastMov != null) {
startPosition = firstAndLastMov.get(0).getNode().getPos();
ITree lastTree = firstAndLastMov.get(1).getNode();
endPosition = lastTree.getPos() + lastTree.getLength();
}
} else {
startPosition = actionSet.getStartPosition(); // range of actions
endPosition = startPosition + actionSet.getLength();
if (actionStr.startsWith("UPD")) {
Update update = (Update) actionSet.getAction();
ITree newNode = update.getNewNode();
startPosition2 = newNode.getPos();
endPosition2 = startPosition2 + newNode.getLength();
String astNodeType = actionSet.getAstNodeType();
if ("EnhancedForStatement".equals(astNodeType) || "ForStatement".equals(astNodeType)
|| "DoStatement".equals(astNodeType) || "WhileStatement".equals(astNodeType)
|| "LabeledStatement".equals(astNodeType) || "SynchronizedStatement".equals(astNodeType)
|| "IfStatement".equals(astNodeType) || "TryStatement".equals(astNodeType)) {
List<ITree> children = update.getNode().getChildren();
endPosition = getEndPosition(children);
List<ITree> newChildren = newNode.getChildren();
endPosition2 = getEndPosition(newChildren);
if (endPosition == 0) {
endPosition = startPosition + actionSet.getLength();
}
if (endPosition2 == 0) {
endPosition2 = startPosition2 + newNode.getLength();
}
}
}
}
startLine = prevUnit.getLineNumber(startPosition);
endLine = prevUnit.getLineNumber(endPosition);
startLine2 = revUnit.getLineNumber(startPosition2);
endLine2 = revUnit.getLineNumber(endPosition2);
actionSet.setBugStartLineNum(startLine);
actionSet.setBugEndLineNum(endLine);
actionSet.setFixStartLineNum(startLine2);
actionSet.setFixEndLineNum(endLine2);
if (actionStr.startsWith("INS")) {
if (fixStartLine + fixRange < startLine2) {
addHunkActionSets(hunkActionSets, allHunkFixPatterns, hunk);
if (hunkFixStartLine + hunkFixRange < actionFixStartLine) {
addHunkActionSets(hunkActionSets, allHunkFixPatterns, hunk);// save the previous non-null hunkFixPattern.
break;
}
if (endLine2 >= fixStartLine ) {
ITree parent = addToHunkActionSets(actionSet, hunkActionSets, allHunkFixPatterns, startLine, startLine2, endLine, endLine2, parentITree, hunk);
if (actionFixEndLine >= hunkFixStartLine ) {
ITree parent = addToHunkActionSets(actionSet, hunkActionSets, allHunkFixPatterns, previousParent, hunk);
if (parent != null) {
if (parent != parentITree) {
if (parent != previousParent) {
hunkActionSets = new ArrayList<>();
}
hunkActionSets.add(actionSet);
} else if (hunkActionSets.size() > 0) {
hunkActionSets = new ArrayList<>();
}
parentITree = parent;
previousParent = parent;
}
} else { // UPD, DEL, MOV
if (bugStartLine + bugRange < startLine) {
addHunkActionSets(hunkActionSets, allHunkFixPatterns, hunk);
if (hunkBugStartLine + hunkBugRange < actionBugStartLine) {
addHunkActionSets(hunkActionSets, allHunkFixPatterns, hunk);// save the previous non-null hunkFixPattern.
break;
}
if (endLine >= bugStartLine ) {
ITree parent = addToHunkActionSets(actionSet, hunkActionSets, allHunkFixPatterns, startLine, startLine2, endLine, endLine2, parentITree, hunk);
if (parent != null) {
if (parent != parentITree) {
if (actionBugEndLine >= hunkBugStartLine ) {
ITree parent = addToHunkActionSets(actionSet, hunkActionSets, allHunkFixPatterns, previousParent, hunk);
if (parent != null) { // same parent
if (parent != previousParent) {
hunkActionSets = new ArrayList<>();
}
hunkActionSets.add(actionSet);
} else if (hunkActionSets.size() > 0) {
hunkActionSets = new ArrayList<>();
}
parentITree = parent;
previousParent = parent;
}
}
addHunkActionSets(hunkActionSets, allHunkFixPatterns, hunk);
@@ -255,8 +209,8 @@ public class HunkActionFilter {
}
}
private ITree addToHunkActionSets(HierarchicalActionSet actionSet, List<HierarchicalActionSet> hunkActionSets, List<HunkFixPattern> allHunkFixPatterns,
int startLine, int startLine2, int endLine, int endLine2, ITree parentITree, DiffEntryHunk hunk) {
private ITree addToHunkActionSets(HierarchicalActionSet actionSet, List<HierarchicalActionSet> hunkActionSets,
List<HunkFixPattern> allHunkFixPatterns, ITree previousParent, DiffEntryHunk hunk) {
String astNodeType = actionSet.getAstNodeType();
if ("FieldDeclaration".equals(astNodeType)) {
addHunkActionSets(hunkActionSets, allHunkFixPatterns, hunk);
@@ -267,16 +221,16 @@ public class HunkActionFilter {
return null;
} else {
ITree currentParent = actionSet.getNode().getParent();
if (parentITree == null) {
parentITree = currentParent;
if (previousParent == null) {
previousParent = currentParent;
} else {
if (!parentITree.equals(currentParent)) {
if (!previousParent.equals(currentParent)) {
HunkFixPattern hunkFixPattern = new HunkFixPattern(hunk, hunkActionSets);
allHunkFixPatterns.add(hunkFixPattern);
parentITree = currentParent;
previousParent = currentParent;
}
}
return parentITree;
return previousParent;
}
}
@@ -323,4 +277,372 @@ public class HunkActionFilter {
return firstAndLastMoveActions;
}
/**
* Filter out the modify actions, which are not in the DiffEntry hunks, without considering the same parent node.
*
* @param violations
* @param actionSets
* @param revFile
* @param prevFile
* @return
*/
public List<Violation> filterActionsByModifiedRange(List<Violation> violations,
List<HierarchicalActionSet> actionSets, File revFile, File prevFile) {
List<Violation> selectedViolations = new ArrayList<>();
CUCreator cuCreator = new CUCreator();
CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile);
CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile);
if (prevUnit == null || revUnit == null) {
return selectedViolations;
}
for (Violation violation : violations) {
int startLine = violation.getStartLineNum();
int endLine = violation.getEndLineNum();
int bugStartLine = violation.getBugStartLineNum();
int bugEndLine = violation.getBugEndLineNum();
int fixStartLine = violation.getFixStartLineNum();
int fixEndLine = violation.getFixEndLineNum();
for (HierarchicalActionSet actionSet : actionSets) {
int actionBugStartLine = actionSet.getBugStartLineNum();
if (actionBugStartLine == 0) {
actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit);
}
int actionBugEndLine = actionSet.getBugEndLineNum();
int actionFixStartLine = actionSet.getFixStartLineNum();
int actionFixEndLine = actionSet.getFixEndLineNum();
String actionStr = actionSet.getActionString();
if (actionStr.startsWith("INS")) {
if (fixStartLine <= actionFixStartLine && actionFixEndLine <= fixEndLine) {
if (actionBugStartLine != 0) {
if (startLine <= actionBugEndLine && endLine >= actionBugStartLine) {
violation.getActionSets().add(actionSet);
}
} else {
violation.getActionSets().add(actionSet);
}
}
} else {
if (bugEndLine < actionBugStartLine) {
break;
}
if (bugStartLine <= actionBugStartLine && actionBugEndLine <= bugEndLine) {
if (startLine <= actionBugEndLine && endLine >= actionBugStartLine) {
violation.getActionSets().add(actionSet);
}
}
}
}
if (violation.getActionSets().size() > 0) {
selectedViolations.add(violation);
}
}
return selectedViolations;
}
/**
* Filter out the modify actions, which are not in the DiffEntry hunks, with considering the same parent node.
*
* @param violations
* @param actionSets
* @param revFile
* @param prevFile
* @return
*/
public List<Violation> filterActionsByModifiedRange2(List<Violation> violations,
List<HierarchicalActionSet> actionSets, File revFile, File prevFile) {
List<Violation> selectedViolations = new ArrayList<>();
CUCreator cuCreator = new CUCreator();
CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile);
CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile);
if (prevUnit == null || revUnit == null) {
return selectedViolations;
}
for (Violation violation : violations) {
int startLine = violation.getStartLineNum();
int endLine = violation.getEndLineNum();
int bugStartLine = violation.getBugStartLineNum();
int bugEndLine = violation.getBugEndLineNum();
int fixStartLine = violation.getFixStartLineNum();
int fixEndLine = violation.getFixEndLineNum();
for (HierarchicalActionSet actionSet : actionSets) {
int actionBugStartLine = actionSet.getBugStartLineNum();
if (actionBugStartLine == 0) {
actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit);
}
int actionBugEndLine = actionSet.getBugEndLineNum();
int actionFixStartLine = actionSet.getFixStartLineNum();
int actionFixEndLine = actionSet.getFixEndLineNum();
String actionStr = actionSet.getActionString();
if (actionStr.startsWith("INS")) {
if (fixStartLine <= actionFixStartLine && actionFixEndLine <= fixEndLine) {
if (actionBugStartLine != 0) {
if (startLine <= actionBugEndLine && endLine >= actionBugStartLine) {
violation.getActionSets().add(actionSet);
}
} else {
if (isRanged(actionSet, violation)) violation.getActionSets().add(actionSet);
}
}
} else {
if (bugEndLine < actionBugStartLine) {
break;
}
if (bugStartLine <= actionBugStartLine && actionBugEndLine <= bugEndLine) {
if (startLine <= actionBugEndLine && endLine >= actionBugStartLine) {
violation.getActionSets().add(actionSet);
}
}
}
}
if (violation.getActionSets().size() > 0) {
selectedViolations.add(violation);
}
}
return selectedViolations;
}
private boolean isRanged(HierarchicalActionSet actionSet, Violation violation) {
int actionStartLine = actionSet.getFixStartLineNum();
int actionEndLine = actionSet.getFixEndLineNum();
int violationStartLine = violation.getStartLineNum();
int violationEndLine = violation.getEndLineNum();
List<DiffEntryHunk> hunks = violation.getHunks();
for (DiffEntryHunk hunk : hunks) {
int bugStartLine = hunk.getBugLineStartNum();
int bugEndLine = bugStartLine + hunk.getBugRange();
int fixStartLine = hunk.getFixLineStartNum();
int fixEndLine = fixStartLine + hunk.getFixRange();
if (fixStartLine > actionEndLine || bugStartLine > violationEndLine) break;
if (fixEndLine < actionStartLine || bugEndLine < violationStartLine) continue;
String hunkContent = hunk.getHunk();
BufferedReader reader = null;
int counterOfContext = 0;
int counterOfDeletedLines = 0;
int counterOfAddedLines = 0;
int bugStarts = 0;
int bugEnds = 0;
int fixStarts = 0;
int fixEnds = 0;
int contextStarts = 0;
try {
reader = new BufferedReader(new StringReader(hunkContent));
String line = null;
while ((line = reader.readLine()) != null) {
if (line.startsWith("-")) {
counterOfDeletedLines ++;
if (bugStarts == 0) {
bugStarts = bugStartLine + counterOfContext + counterOfDeletedLines - 1;
}
contextStarts = 0;
if (fixStarts != 0) {
fixEnds = fixStartLine + counterOfContext + counterOfAddedLines - 1;
if (fixStarts > actionEndLine) break;
if (fixEnds < actionStartLine) {
fixStarts = 0;
continue;
}
return true;
}
}
else if (line.startsWith("+")) {
counterOfAddedLines ++;
if (bugStarts == 0) {
bugStarts = contextStarts;
}
bugEnds = bugStartLine + counterOfContext + counterOfDeletedLines - 1;
if (violationEndLine < bugStarts) break;
if (violationStartLine > bugEnds) {
bugStarts = 0;
}
if (bugStarts != 0 && fixStarts == 0) {
fixStarts = fixStartLine + counterOfContext + counterOfAddedLines - 1;
}
}
else {
counterOfContext ++;
bugStarts = 0;
if (contextStarts == 0) {
contextStarts = bugStartLine + counterOfContext + counterOfDeletedLines - 1;
}
if (fixStarts != 0) {
fixEnds = fixStartLine + counterOfContext + counterOfAddedLines - 1;
if (fixStarts > actionEndLine) break;
if (fixEnds < actionStartLine) {
fixStarts = 0;
continue;
}
return true;
}
}
}
if (fixStarts != 0) {
fixEnds = fixStartLine + counterOfContext + counterOfAddedLines - 1;
if (fixStarts > actionEndLine) break;
if (fixEnds < actionStartLine) {
fixStarts = 0;
continue;
}
return true;
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return false;
}
/**
* Filter out the modify actions, which are not in the DiffEntry hunks, without considering DiffEntry hunks.
*
* @param violations
* @param actionSets
* @param revFile
* @param prevFile
* @return
*/
public List<Violation> filterActionsByModifiedRange3(List<Violation> violations,
List<HierarchicalActionSet> actionSets, File revFile, File prevFile) {
List<Violation> selectedViolations = new ArrayList<>();
CUCreator cuCreator = new CUCreator();
CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile);
CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile);
if (prevUnit == null || revUnit == null) {
return selectedViolations;
}
for (Violation violation : violations) {
int startLine = violation.getStartLineNum();
int endLine = violation.getEndLineNum();
// ITree parent = null;
// List<HierarchicalActionSet> actionSetsWithSameParent = new ArrayList<>(); //TODO
for (HierarchicalActionSet actionSet : actionSets) {
int actionBugStartLine = actionSet.getBugStartLineNum();
if (actionBugStartLine == 0) {
actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit);
}
int actionBugEndLine = actionSet.getBugEndLineNum();
int actionFixStartLine = actionSet.getFixStartLineNum();
int actionFixEndLine = actionSet.getFixEndLineNum();
String actionStr = actionSet.getActionString();
if (actionStr.startsWith("INS")) { // FIXME It is impossible to locate the INS action by the buggy line range.
if (startLine <= actionFixStartLine && actionFixEndLine <= endLine) {
if (actionBugStartLine != 0) {
if (startLine <= actionBugEndLine && endLine >= actionBugStartLine) {
violation.getActionSets().add(actionSet);
}
} else {
violation.getActionSets().add(actionSet);
}
}
} else {
if (endLine < actionBugStartLine) {
break;
}
if (startLine <= actionBugStartLine && actionBugEndLine <= endLine) {
if (startLine <= actionBugEndLine && endLine >= actionBugStartLine) {
violation.getActionSets().add(actionSet);
}
}
}
}
if (violation.getActionSets().size() > 0) {
selectedViolations.add(violation);
}
}
return selectedViolations;
}
private int setLineNumbers(HierarchicalActionSet actionSet, CompilationUnit prevUnit, CompilationUnit revUnit) {
int actionBugStartLine = actionSet.getBugStartLineNum();
int actionBugEndLine;
int actionFixStartLine;
int actionFixEndLine;
// position of buggy statements
int bugStartPosition = 0;
int bugEndPosition = 0;
// position of fixed statements
int fixStartPosition = 0;
int fixEndPosition = 0;
String actionStr = actionSet.getActionString();
if (actionStr.startsWith("INS")) {
fixStartPosition = actionSet.getStartPosition();
fixEndPosition = fixStartPosition + actionSet.getLength();
List<Move> firstAndLastMov = getFirstAndLastMoveAction(actionSet);
if (firstAndLastMov != null) {
bugStartPosition = firstAndLastMov.get(0).getNode().getPos();
ITree lastTree = firstAndLastMov.get(1).getNode();
bugEndPosition = lastTree.getPos() + lastTree.getLength();
}
} else {
bugStartPosition = actionSet.getStartPosition(); // range of actions
bugEndPosition = bugStartPosition + actionSet.getLength();
if (actionStr.startsWith("UPD")) {
Update update = (Update) actionSet.getAction();
ITree newNode = update.getNewNode();
fixStartPosition = newNode.getPos();
fixEndPosition = fixStartPosition + newNode.getLength();
String astNodeType = actionSet.getAstNodeType();
if ("EnhancedForStatement".equals(astNodeType) || "ForStatement".equals(astNodeType)
|| "DoStatement".equals(astNodeType) || "WhileStatement".equals(astNodeType)
|| "LabeledStatement".equals(astNodeType) || "SynchronizedStatement".equals(astNodeType)
|| "IfStatement".equals(astNodeType) || "TryStatement".equals(astNodeType)) {
List<ITree> children = update.getNode().getChildren();
bugEndPosition = getEndPosition(children);
List<ITree> newChildren = newNode.getChildren();
fixEndPosition = getEndPosition(newChildren);
if (bugEndPosition == 0) {
bugEndPosition = bugStartPosition + actionSet.getLength();
}
if (fixEndPosition == 0) {
fixEndPosition = fixStartPosition + newNode.getLength();
}
}
}
}
actionBugStartLine = bugStartPosition == 0 ? 0 : prevUnit.getLineNumber(bugStartPosition);
actionBugEndLine = bugEndPosition == 0 ? 0 : prevUnit.getLineNumber(bugEndPosition);
actionFixStartLine = fixStartPosition == 0 ? 0 : revUnit.getLineNumber(fixStartPosition);
actionFixEndLine = fixEndPosition == 0 ? 0 : revUnit.getLineNumber(fixEndPosition);
actionSet.setBugStartLineNum(actionBugStartLine);
actionSet.setBugEndLineNum(actionBugEndLine);
actionSet.setFixStartLineNum(actionFixStartLine);
actionSet.setFixEndLineNum(actionFixEndLine);
return actionBugStartLine;
}
}
src/main/java/edu/lu/uni/serval/gumtree/regroup/SimplifyTree.java
+11 -5
View File
@@ -116,8 +116,9 @@ public class SimplifyTree {
simpleTree.setLabel(canonicalizeTypeStr(label).replaceAll(" ", ""));
} else {
if ((astNode.equals("SimpleName") || astNode.equals("MethodInvocation")) && label.startsWith("MethodName:")) {
label = label.substring(11);
simpleTree.setNodeType("MethodName");
label = label.substring(11);
label = label.substring(0, label.indexOf(":["));
simpleTree.setLabel(label);
} else {
simpleTree.setLabel(astNode);
@@ -132,8 +133,9 @@ public class SimplifyTree {
if (astNode.endsWith("Name")) {
// variableName, methodName, QualifiedName
if (label.startsWith("MethodName:")) { // <MethodName, name>
label = label.substring(11);
simpleTree.setNodeType("MethodName");
label = label.substring(11);
label = label.substring(0, label.indexOf(":["));
simpleTree.setLabel(label);
} else if (label.startsWith("Name:")) {
label = label.substring(5);
@@ -155,9 +157,10 @@ public class SimplifyTree {
simpleTree.setLabel(canonicalizeTypeStr(label).replaceAll(" ", ""));
} else if (astNode.startsWith("Type")) {
simpleTree.setLabel(canonicalizeTypeStr(label).replaceAll(" ", ""));
} else if (astNode.equals("MethodInvocation") && label.startsWith("MethodName:")) {
label = label.substring(11);
} else if ((astNode.equals("SimpleName") || astNode.equals("MethodInvocation")) && label.startsWith("MethodName:")) {
simpleTree.setNodeType("MethodName");
label = label.substring(11);
label = label.substring(0, label.indexOf(":["));
simpleTree.setLabel(label);
} else {
simpleTree.setLabel(label.replaceAll(" ", ""));
@@ -591,7 +594,10 @@ public class SimplifyTree {
List<Action> allMoveActions = getAllMoveActions2(actionSet);
if (allMoveActions != null && allMoveActions.size() > 0) {
ListSorter<Action> sorter = new ListSorter<Action>(allMoveActions);
allMoveActions = sorter.sortAscending();
List<Action> moveActions = sorter.sortAscending();
if (moveActions != null) {
allMoveActions = moveActions;
}
return allMoveActions;
} else {// FIXME: pure INS actions.
return null;
src/main/java/edu/lu/uni/serval/violation/Alarm.java
+10 -19
View File
@@ -1,24 +1,21 @@
package edu.lu.uni.serval.violation;
import java.util.HashMap;
import java.util.Map;
public class Alarm implements Comparable<Alarm> {
public class Alarm {
private String buggyCommitId;
private String buggyFileName;
private Map<Integer, Integer> positions; // <Integer, Integer>: <StartLine, EndLine>
private String fixedCommitId;
private String fixedFileName;
private Map<Integer, String> positions;
public Alarm(String buggyCommitId, String buggyFileName, String fixedCommitId,
String fixedFileName) {
public Alarm(String buggyCommitId, String buggyFileName, String fixedCommitId, String fixedFileName) {
super();
this.buggyCommitId = buggyCommitId;
this.buggyFileName = buggyFileName;
this.fixedCommitId = fixedCommitId;
this.fixedFileName = fixedFileName;
this.positions = new HashMap<>();
}
public String getBuggyCommitId() {
@@ -29,10 +26,6 @@ public class Alarm implements Comparable<Alarm> {
return buggyFileName;
}
public Map<Integer, Integer> getPositions() {
return positions;
}
public String getFixedCommitId() {
return fixedCommitId;
}
@@ -41,13 +34,12 @@ public class Alarm implements Comparable<Alarm> {
return fixedFileName;
}
@Override
public int compareTo(Alarm a) {
int compareResult = this.buggyCommitId.compareTo(a.buggyCommitId);
if (compareResult == 0) {
compareResult = this.buggyFileName.compareTo(a.buggyFileName);
}
return compareResult;
public Map<Integer, String> getPositions() {
return positions;
}
public void setPositions(Map<Integer, String> positions) {
this.positions = positions;
}
@Override
@@ -61,6 +53,5 @@ public class Alarm implements Comparable<Alarm> {
}
return false;
}
}
src/main/java/edu/lu/uni/serval/violation/Violation.java
+6
View File
@@ -3,6 +3,12 @@ package edu.lu.uni.serval.violation;
import java.util.ArrayList;
import java.util.List;
/**
* Violation: all alarms identified by FindBugs in one java project.
*
* @author kui.liu
*
*/
public class Violation {
private String project;
src/main/java/edu/lu/uni/serval/violation/parse/AlarmsReader.java
+19 -19
View File
@@ -14,7 +14,7 @@ import edu.lu.uni.serval.violation.Violation;
public class AlarmsReader {
public Map<String, Violation> readAlarmsList(String fileName) {
Map<String, Violation> violations = new HashMap<>();
Map<String, Violation> violationsMap = new HashMap<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
@@ -29,36 +29,35 @@ public class AlarmsReader {
String[] buggyElements = buggyInfo.split(":");
String[] fixedElements = fixedInfo.split(":");
String projectName = buggyElements[0];
if (!projectName.equals(fixedElements[0])) continue;
String commitId = buggyElements[1];
String buggyFile = buggyElements[2];
int startLine = Integer.parseInt(buggyElements[3]);
int endLine = Integer.parseInt(buggyElements[4]);
String projectName = buggyElements[1];
String buggyCommitId = buggyElements[2];
String buggyFile = buggyElements[3];
int startLine = Integer.parseInt(buggyElements[4]);
String endLineAndAlarmType = buggyElements[5] + ":" + buggyElements[0];
String fixCommitId = fixedElements[1];
String fixedFile = fixedElements[2];
Alarm alarm = new Alarm(commitId, buggyFile, fixCommitId, fixedFile);
Alarm alarm = new Alarm(buggyCommitId, buggyFile, fixCommitId, fixedFile);
Violation violation;
if (violations.containsKey(projectName)) {
violation = violations.get(projectName);
if (violationsMap.containsKey(projectName)) {
violation = violationsMap.get(projectName);
} else {
violation = new Violation(projectName);
violations.put(projectName, violation);
violationsMap.put(projectName, violation);
}
List<Alarm> alarms = violation.getAlarms();
int index = alarms.indexOf(alarm);
if (index != -1) {
Alarm tempA = alarms.get(index);
Map<Integer, Integer> positions = tempA.getPositions();
positions.put(startLine, endLine);
if (index >= 0) {
Alarm tempAlarm = alarms.get(index);
Map<Integer, String> positions = tempAlarm.getPositions();
positions.put(startLine, endLineAndAlarmType);
} else {
alarm.getPositions().put(startLine, endLine);
Map<Integer, String> positions = new HashMap<>();
positions.put(startLine, endLineAndAlarmType);
alarm.setPositions(positions);
alarms.add(alarm);
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
@@ -70,6 +69,7 @@ public class AlarmsReader {
e.printStackTrace();
}
}
return violations;
return violationsMap;
}
}
src/main/java/edu/lu/uni/serval/violation/parse/TestViolationParser.java
+1 -1
View File
@@ -8,7 +8,7 @@ import edu.lu.uni.serval.utils.FileHelper;
public class TestViolationParser {
private static final String REPO_PATH = "/Volumes/MacBook/repositories/";
private static final String REPO_PATH = "/Volumes/MacBook/repos/";
private static final String previousFilesPath = "GumTreeInput/prevFiles/";
private static final String revisedFilesPath = "GumTreeInput/revFiles/";
private static final String positionsFilePath = "GumTreeInput/positions/";
src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java
+130 -8
View File
@@ -2,10 +2,10 @@ package edu.lu.uni.serval.violation.parse;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.errors.RevisionSyntaxException;
import edu.lu.uni.serval.git.exception.GitRepositoryNotFoundException;
@@ -23,6 +23,116 @@ import edu.lu.uni.serval.violation.Violation;
public class ViolationParser {
public void parseViolations(String fixedAlarmFile, List<File> repos, String previousFilesPath, String revisedFilesPath, String positionsFilePath, String diffentryFilePath) {
AlarmsReader reader = new AlarmsReader();
Map<String, Violation> violations = reader.readAlarmsList(fixedAlarmFile);
List<String> throwExpProjs = new ArrayList<>();
int a = 0;
int exceptionsCounter = 0;
for (Map.Entry<String , Violation> entry : violations.entrySet()) {
String projectName = entry.getKey();
String repoName = "";
for (File repo : repos) {
if (repo.getName().equals(projectName)) {
repoName = repo.getPath() + "/";
break;
}
}
if ("".equals(repoName)) {
a ++;
System.out.println(projectName);
continue;
}
Violation violation = entry.getValue();
List<Alarm> alarms = violation.getAlarms();
String repoPath = repoName + "/.git";
GitRepository gitRepo = new GitRepository(repoPath, revisedFilesPath, previousFilesPath);
try {
gitRepo.open();
for (Alarm alarm : alarms) {
String buggyCommitId = alarm.getBuggyCommitId();
String buggyFileName = alarm.getBuggyFileName();
String buggyFileContent = gitRepo.getFileContentByCommitIdAndFileName(buggyCommitId, buggyFileName);
if (buggyFileContent == null || "".equals(buggyFileContent)) {
// System.out.println(projectName);
throwExpProjs.add(projectName);
exceptionsCounter ++;
continue;
}
String fixedCommitId = alarm.getFixedCommitId();
String fixedFileName = alarm.getFixedFileName();
String fixedFileContent = gitRepo.getFileContentByCommitIdAndFileName(fixedCommitId, fixedFileName);
if (fixedFileContent == null || "".equals(fixedFileContent)) {
// System.out.println(projectName);
throwExpProjs.add(projectName);
exceptionsCounter ++;
continue;
}
String diffentry = gitRepo.getDiffentryByTwoCommitIds(buggyCommitId, fixedCommitId, fixedFileName);
if (diffentry == null) {
// System.out.println(projectName);
throwExpProjs.add(projectName);
exceptionsCounter ++;
continue;
}
String commitId = buggyCommitId.substring(0, 6) + "_" + fixedCommitId.substring(0, 6);
String fileName = fixedFileName.replaceAll("/", "#");
fileName = projectName + "_" + commitId + fileName;
if (fileName.length() > 240) {
List<File> files = FileHelper.getAllFilesInCurrentDiectory(revisedFilesPath, ".java");
fileName = files.size() + "TooLongFileName.java";
}
String buggyFile = previousFilesPath + "prev_" + fileName;
String fixedFile = revisedFilesPath + fileName;
fileName = fileName.replace(".java", ".txt");
String positionFile = positionsFilePath + fileName;
String diffentryFile = diffentryFilePath + fileName;
FileHelper.outputToFile(buggyFile, buggyFileContent, false);
FileHelper.outputToFile(fixedFile, fixedFileContent, false);
FileHelper.outputToFile(positionFile, readPosition(alarm.getPositions()), false);
FileHelper.outputToFile(diffentryFile, diffentry, false);
}
} catch (GitRepositoryNotFoundException e) {
System.out.println("Exception: " + projectName);
exceptionsCounter ++;
e.printStackTrace();
} catch (NotValidGitRepositoryException e) {
System.out.println("Exception: " + projectName);
exceptionsCounter ++;
e.printStackTrace();
} catch (IOException e) {
System.out.println("Exception: " + projectName);
exceptionsCounter ++;
e.printStackTrace();
} catch (RevisionSyntaxException e) {
System.out.println("Exception: " + projectName);
exceptionsCounter ++;
e.printStackTrace();
} finally {
gitRepo.close();
}
}
System.out.println(a);
System.out.println(exceptionsCounter);
System.out.println(throwExpProjs.size());
System.out.println(throwExpProjs);
}
/**
* Output data in terms of alarm types.
*
* @param fixedAlarmFile
* @param repos
* @param previousFilesPath
* @param revisedFilesPath
* @param positionsFilePath
* @param diffentryFilePath
*/
public void parseViolations2(String fixedAlarmFile, List<File> repos, String previousFilesPath, String revisedFilesPath, String positionsFilePath, String diffentryFilePath) {
AlarmsReader reader = new AlarmsReader();
Map<String, Violation> violations = reader.readAlarmsList(fixedAlarmFile);
int a = 0;
@@ -51,15 +161,24 @@ public class ViolationParser {
String buggyCommitId = alarm.getBuggyCommitId();
String buggyFileName = alarm.getBuggyFileName();
String buggyFileContent = gitRepo.getFileContentByCommitIdAndFileName(buggyCommitId, buggyFileName);
if (buggyFileContent == null || "".equals(buggyFileContent)) continue;
if (buggyFileContent == null || "".equals(buggyFileContent)) {
System.out.println(projectName);
continue;
}
String fixedCommitId = alarm.getFixedCommitId();
String fixedFileName = alarm.getFixedFileName();
String fixedFileContent = gitRepo.getFileContentByCommitIdAndFileName(fixedCommitId, fixedFileName);
if (fixedFileContent == null || "".equals(fixedFileContent)) continue;
if (fixedFileContent == null || "".equals(fixedFileContent)) {
System.out.println(projectName);
continue;
}
String diffentry = gitRepo.getDiffentryByTwoCommitIds(buggyCommitId, fixedCommitId, fixedFileName);
if (diffentry == null) continue;
if (diffentry == null) {
System.out.println(projectName);
continue;
}
String commitId = buggyCommitId.substring(0, 6) + "_" + fixedCommitId.substring(0, 6);
String fileName = fixedFileName.replaceAll("/", "#");
@@ -79,14 +198,16 @@ public class ViolationParser {
FileHelper.outputToFile(diffentryFile, diffentry, false);
}
} catch (GitRepositoryNotFoundException e) {
System.out.println("Exception: " + projectName);
e.printStackTrace();
} catch (NotValidGitRepositoryException e) {
System.out.println("Exception: " + projectName);
e.printStackTrace();
} catch (IOException e) {
System.out.println("Exception: " + projectName);
e.printStackTrace();
} catch (RevisionSyntaxException e) {
e.printStackTrace();
} catch (GitAPIException e) {
System.out.println("Exception: " + projectName);
e.printStackTrace();
} finally {
gitRepo.close();
@@ -95,11 +216,12 @@ public class ViolationParser {
System.out.println(a);
}
private String readPosition(Map<Integer, Integer> positions) {
private String readPosition(Map<Integer, String> positions) {
String positionsStr = "";
for (Map.Entry<Integer, Integer> entry : positions.entrySet()) {
for (Map.Entry<Integer, String> entry : positions.entrySet()) {
positionsStr += entry.getKey() + ":" + entry.getValue() + "\n";
}
return positionsStr;
}
}