Implement the process of bug localization.

This commit is contained in:
Kui LIU
2017-08-04 13:45:42 +02:00
parent 690253867d
commit 501cc50e70
4 changed files with 511 additions and 6 deletions
@@ -0,0 +1,47 @@
Step 1:
Prepare data for tokens embedding of edit scripts.
Input data: parsed results of patches with GumTree.
Select token vectors of edit scripts by the value of upper whisker.
Step 2:
Embed tokens of all selected edit scripts.
Step 3:
Prepare data for features learning of selected edit scripts.
Vectorize edit scripts with embedded tokens of edit scripts.
Step 4:
Learn features of all selected edit scripts with CNN algorithm.
Input data: vectorized edit scripts.
Step 5:
Prepare data for clustering of edit scripts.
Input data: learned features of edit scripts by CNN.
Step 6:
Clustering of edit scripts with extracted features of edit scripts.
Step 7:
Analyze cluster results to obtain common fix patterns.
Step 8:
Prepare testing data for evaluation.
Parse java projects to get the token vectors of all statements.
Step 9:
Prepare data for evaluation.
Merge token vectors of source code of training data and testing data.
Step 10:
Prepare data for evaluation.
Embed tokens of source code vectors of training data and testing data.
Step 11:
Prepare data for evaluation.
Vectorize data (token vectors of source code) for deep learning.
Step 12:
Evaluation: extract features of testing data and predict their labels.
Step 13:
@@ -1,13 +1,23 @@
package edu.lu.uni.serval.FixPatternMining.App;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Evaluation: extract features of testing data and predict their labels.
* Separate features of training data cluster by cluster.
*
* Classify testing data by possibilities: 90%, 80%, 70%, and 60%, ignore others.
*
* @author kui.liu
*
@@ -17,17 +27,139 @@ public class Step13 {
public static void main(String[] args) {
boolean isSupervisedLearning = true;
if (isSupervisedLearning) {// supervised learning
// localization file, feature file,
// label --> possibility --> 90, 80, 70, 60 others ignored, level one localization
// features of training data
DataPreparation.separateTrainingDataFeatures();
String positionFiles = Configuration.TEST_POSITION_FILE;
String featureFiles = Configuration.FEATURES_OF_TESTING_DATA;
String labelFiles = Configuration.PREDICTED_RESULTS_OF_TESTING_DATA;
String possibilitiesFilePath = Configuration.POSSIBILITIES_OF_TESTING_DATA;
List<File> possibilitiesFiles = FileHelper.getAllFilesInCurrentDiectory(possibilitiesFilePath, ".csv");
String bugs90 = Configuration.TESTING_DATA_BUGS90;
String bugs80 = Configuration.TESTING_DATA_BUGS80;
String bugs70 = Configuration.TESTING_DATA_BUGS70;
String bugs60 = Configuration.TESTING_DATA_BUGS60;
StringBuilder builder90 = new StringBuilder();
int counter90 = 0;
StringBuilder builder80 = new StringBuilder();
int counter80 = 0;
StringBuilder builder70 = new StringBuilder();
int counter70 = 0;
StringBuilder builder60 = new StringBuilder();
int counter60 = 0;
for (File possibilitiesFile : possibilitiesFiles) {
String fileName = possibilitiesFile.getName();
String positionFile = positionFiles + "Positions" + fileName.substring(fileName.lastIndexOf("_"), fileName.lastIndexOf(".")) + ".list";
String featureFile = featureFiles + fileName;
String labelFile = labelFiles + fileName;
List<String> possibilities = readData(fileName);
List<String> positions = readData(positionFile);
List<String> features = readData(featureFile);
List<Integer> labels = readLabel(labelFile);
for (int index = 0, size = possibilities.size(); index < size; index ++) {
String possibilityStr = possibilities.get(index);
String[] array = possibilityStr.split(", ");
int label = labels.get(index);
double possibility = Double.parseDouble(array[label]);
String position = positions.get(index);
String feature = features.get(index);
// And Label
if (possibility >= 0.9) {
builder90.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n");
counter90 ++;
if (counter90 % 1000 == 0) {
FileHelper.outputToFile(bugs90, builder90, true);
builder90.setLength(0);
}
} else if (possibility >= 0.8) {
builder80.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n");
counter80 ++;
if (counter80 % 1000 == 0) {
FileHelper.outputToFile(bugs80, builder80, true);
builder80.setLength(0);
}
} else if (possibility >= 0.7) {
builder70.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n");
counter70 ++;
if (counter70 % 1000 == 0) {
FileHelper.outputToFile(bugs70, builder70, true);
builder70.setLength(0);
}
} else if (possibility >= 0.6) {
builder60.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n");
counter60 ++;
if (counter60 % 1000 == 0) {
FileHelper.outputToFile(bugs60, builder60, true);
builder60.setLength(0);
}
}
}
}
FileHelper.outputToFile(bugs90, builder90, true);
FileHelper.outputToFile(bugs80, builder80, true);
FileHelper.outputToFile(bugs70, builder70, true);
FileHelper.outputToFile(bugs60, builder60, true);
// label: clusterNum, re-compute similarity with each element. 90, 80, 70, 60.
// similarity: patches --> fixing bug.
List<File> testingDataFiles = FileHelper.getAllFilesInCurrentDiectory(Configuration.TESTING_DATA, ".csv");
for (int i = 0, size = testingDataFiles.size(); i < size; i ++) {
}
} else { // un-supervised learning
// Extracted Features: Configuration.EXTRACTED_FEATURES_TESTING;
// Compute the similarity: cosin similarity
}
}
private static List<Integer> readLabel(String labelFile) {
List<Integer> labels = new ArrayList<>();
String fileContent = FileHelper.readFile(labelFile);
BufferedReader reader = null;
try {
reader = new BufferedReader(new StringReader(fileContent));
String line = null;
while ((line = reader.readLine()) != null) {
String[] labelsStr = line.split(", ");
for (int i = 0, length = labelsStr.length; i < length; i ++) {
Double d = Double.parseDouble(labelsStr[i]);
labels.add(d.intValue());
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return labels;
}
private static List<String> readData(String positionFile) {
List<String> positions = new ArrayList<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(positionFile);
scanner = new Scanner(fis);
while (scanner.hasNextLine()) {
positions.add(scanner.nextLine());
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
}
@@ -0,0 +1,225 @@
package edu.lu.uni.serval.FixPatternMining.App;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.ListSorter;
import edu.lu.uni.serval.utils.MapSorter;
/**
* Compute similarities for each potential bug instance by computing the similarities with all instances in related cluster.
*
* List top 5 most similar instances?
*
* @author kui.liu
*
*/
public class Step14 {
public static void main(String[] args) {
String featuresOfTrainingDataPath = Configuration.FEATURES_OF_COMMON_CLUSTERS;
List<File> featuresOfTrainingDataFiles = FileHelper.getAllFiles(featuresOfTrainingDataPath, ".csv");// TODO: type
Map<Integer, List<String>> features = readFeaturesOfTrainingData(featuresOfTrainingDataFiles);
Map<Integer, List<String>> patches = readPatchesOfTraingData(Configuration.CLUSTERED_PATCHES_FILE);
Map<Integer, Integer> labelMapClusterNum = DataPreparation.readLabelMapClusterNum();
// potential bugs' information
String bugs90 = Configuration.TESTING_DATA_BUGS90;
String bugs80 = Configuration.TESTING_DATA_BUGS80;
String bugs70 = Configuration.TESTING_DATA_BUGS70;
String bugs60 = Configuration.TESTING_DATA_BUGS60;
List<String> bugsList = new ArrayList<>();
bugsList.add(bugs90);
bugsList.add(bugs80);
bugsList.add(bugs70);
bugsList.add(bugs60);
for (String bugs : bugsList) {
List<String> bugsInfo = readData(bugs);
String filePath = bugs.substring(0, bugs.lastIndexOf(".")) + "/";
StringBuilder builder = new StringBuilder();
for (String singleBugInfo : bugsInfo) {
String[] infoArray = singleBugInfo.split(":");
String label = infoArray[0];// TODO
String feature = infoArray[1]; // TODO
String position = infoArray[2]; // TODO
int labelInt = Integer.parseInt(label);
int clusterNum = labelMapClusterNum.get(labelInt);
Map<Double, Integer> mostSimilarIndex = computeSimilarities(feature, features.get(clusterNum));
List<Integer> patchesIndex = new ArrayList<>();
if (mostSimilarIndex.size() > 0) {
for (Map.Entry<Double, Integer> entry : mostSimilarIndex.entrySet()) {
patchesIndex.add(entry.getValue());
}
String bug = "BUG####" + position + "\n";
String patchesStr = readPatches(patchesIndex, patches.get(Integer.parseInt(label)));
// output: bug + patchesStr;
builder.append(bug).append(patchesStr);
}
}
FileHelper.outputToFile(filePath + "patches.list", builder, false);
}
}
private static Map<Integer, List<String>> readPatchesOfTraingData(String clusteredPatchesFile) {
Map<Integer, List<String>> map = new HashMap<>();
List<File> files = FileHelper.getAllFiles(clusteredPatchesFile, ".list");
for (File file : files) {
String fileName = file.getName();
int clusterNum = Integer.parseInt(fileName.substring(fileName.lastIndexOf("_") + 1, fileName.lastIndexOf(".")));
List<String> patches = readPatches(file);
map.put(clusterNum, patches);
}
return map;
}
private static List<String> readPatches(File file) {
List<String> patches = new ArrayList<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(file);
scanner = new Scanner(fis);
String singlePatch = "";
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if (Configuration.PATCH_SIGNAL.equals(line)) {
if (!"".equals(singlePatch)) {
patches.add(singlePatch);
singlePatch = "";
}
}
singlePatch += line + "\n";
}
patches.add(singlePatch);
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return patches;
}
private static String readPatches(List<Integer> patchesIndex, List<String> patchesTrainingData) {
String patches = "";
for(Integer index : patchesIndex) {
patches += patchesTrainingData.get(index) + "\n";
}
return patches;
}
private static Map<Double, Integer> computeSimilarities(String feature, List<String> trainingFeatures) {
Map<Double, Integer> mostSimilarIndex = new HashMap<>();
List<Double> similarities = new ArrayList<>();
for (int i = 0; i < 10; i ++) similarities.add(0.0);
for (int index = 0, size = trainingFeatures.size(); index < size; index ++) {
String trainingFeature = trainingFeatures.get(index);
double similarity = computeSimilarity(feature, trainingFeature);
double aborted = addToSimilarityies(similarity, similarities);
if (aborted > 0.0) {
mostSimilarIndex.put(similarity, index);
if (aborted == 0.1) mostSimilarIndex.remove(aborted);
}
}
if (mostSimilarIndex.size() > 0) {
MapSorter<Double, Integer> sorter = new MapSorter<>();
mostSimilarIndex = sorter.sortByKeyDescending(mostSimilarIndex);
}
return mostSimilarIndex;
}
private static double addToSimilarityies(double similarity, List<Double> similarities) {
double lastSimilarity = similarities.get(9);
if (similarity >= 0.8 && similarity > lastSimilarity) { // TODO : 9 ?
similarities.set(9, similarity);
ListSorter<Double> sorter = new ListSorter<Double>(similarities);
similarities = sorter.sortDescending();
return lastSimilarity == 0.0 ? 0.1 : lastSimilarity;
}
return 0.0;
}
private static double computeSimilarity(String feature, String trainingFeature) {
// TODO Auto-generated method stub
return 0;
}
private static Map<Integer, List<String>> readFeaturesOfTrainingData(List<File> featureFiles) {
Map<Integer, List<String>> features = new HashMap<>();
for (File file : featureFiles) {
String fileName = file.getName();
String label = fileName.substring(fileName.lastIndexOf("_") + 1, fileName.lastIndexOf("."));
int clusterNum = Integer.parseInt(label);
List<String> featuresList = readData(file.getPath());
features.put(clusterNum, featuresList);
}
return features;
}
private static List<String> readData(String positionFile) {
List<String> positions = new ArrayList<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(positionFile);
scanner = new Scanner(fis);
while (scanner.hasNextLine()) {
positions.add(scanner.nextLine());
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
public static void quickSort(List<Double> arr){
qsort(arr, 0, arr.size() - 1);
}
private static void qsort(List<Double> arr, int low, int high){
if (low < high){
int pivot=partition(arr, low, high); //将数组分为两部分
qsort(arr, low, pivot-1); //递归排序左子数组
qsort(arr, pivot+1, high); //递归排序右子数组
}
}
private static int partition(List<Double> arr, int low, int high){
double pivot = arr.get(low); //枢轴记录
while (low<high){
while (low<high && arr.get(high)>=pivot) --high;
arr.set(low, arr.get(high)); //交换比枢轴小的记录到左端
while (low<high && arr.get(low)<=pivot) ++low;
arr.set(high, arr.get(low)); //交换比枢轴小的记录到右端
}
//扫描完成,枢轴到位
arr.set(low, pivot);
//返回的是枢轴的位置
return low;
}
}
@@ -557,4 +557,105 @@ public class DataPreparation {
}
return commonClustersMappingLabel;
}
public static void separateTrainingDataFeatures() {
String trainingDataFeatures = Configuration.FEATURES_OF_TRAINING_DATA;
List<File> featureFiles = FileHelper.getAllFilesInCurrentDiectory(trainingDataFeatures, ".csv");
File featureFile = featureFiles.get(0);
// File featureFile = new File(Configuration.FEATURES_OF_TRAINING_DATA + "");
Map<Integer, Integer> numbersMap = readNumberOfInstances(); // <Integer, Integer>: <clusterNum, numOfInstances>
Map<Integer, Integer> orders = new HashMap<>(); //<Integer, Integer> : <order, clusterNum>
Map<Integer, String> fileNames = new HashMap<>();
String clusteredTokens = Configuration.CLUSTERED_TOKENSS_FILE;
List<File> files = FileHelper.getAllFilesInCurrentDiectory(clusteredTokens, ".list");
int order = 1;
for (File file : files) {
String fileName = file.getName();
String clusterNumStr = fileName.substring(fileName.lastIndexOf("_") + 1, fileName.lastIndexOf(".list"));
int clusterNum = Integer.parseInt(clusterNumStr);
if (numbersMap.containsKey(clusterNum)) {
orders.put(order, clusterNum);
fileNames.put(order, fileName);
order ++;
}
}
String featuresOfClusterPath = Configuration.FEATURES_OF_COMMON_CLUSTERS;
order = 1;
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(featureFile);
scanner = new Scanner(fis);
int counter = 0;
StringBuilder features = new StringBuilder();
while (scanner.hasNextLine()) {
features.append(scanner.nextLine() + "\n");
counter ++;
if (counter == numbersMap.get(orders.get(order))) {
FileHelper.outputToFile(featuresOfClusterPath + fileNames.get(order), features, false);
features.setLength(0);
counter = 0;
order ++;
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
private static Map<Integer, Integer> readNumberOfInstances() {
Map<Integer, Integer> numbersMap = new HashMap<>();
String fileContent = FileHelper.readFile(Configuration.COMMON_CLUSTERS_SIZES);
BufferedReader reader = null;
try {
reader = new BufferedReader(new StringReader(fileContent));
String line = null;
while ((line = reader.readLine()) != null) {
String[] numbers = line.split(":");
numbersMap.put(Integer.parseInt(numbers[0]), Integer.parseInt(numbers[1]));
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return numbersMap;
}
public static Map<Integer, Integer> readLabelMapClusterNum() {
Map<Integer, Integer> labelMapClusterNumMap = new HashMap<>();
String fileContent = FileHelper.readFile(Configuration.CLUSTERNUMBER_LABEL_MAP);
BufferedReader reader = null;
reader = new BufferedReader(new StringReader(fileContent));
String line = null;
try {
while ((line = reader.readLine()) != null) {
String[] labelMapClusterNum = line.split(":");
labelMapClusterNumMap.put(Integer.parseInt(labelMapClusterNum[0]), Integer.parseInt(labelMapClusterNum[1]));
}
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
}