From 501cc50e70950554caaf2537c9f16c80c6eb8163 Mon Sep 17 00:00:00 2001 From: Kui LIU Date: Fri, 4 Aug 2017 13:45:42 +0200 Subject: [PATCH] Implement the process of bug localization. --- .../FixPatternMining/App/Specifications.txt | 47 ++++ .../serval/FixPatternMining/App/Step13.java | 144 ++++++++++- .../serval/FixPatternMining/App/Step14.java | 225 ++++++++++++++++++ .../DataPrepare/DataPreparation.java | 101 ++++++++ 4 files changed, 511 insertions(+), 6 deletions(-) create mode 100644 src/main/java/edu/lu/uni/serval/FixPatternMining/App/Specifications.txt create mode 100644 src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step14.java diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Specifications.txt b/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Specifications.txt new file mode 100644 index 0000000..c434cb6 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Specifications.txt @@ -0,0 +1,47 @@ +Step 1: + Prepare data for tokens embedding of edit scripts. + Input data: parsed results of patches with GumTree. + Select token vectors of edit scripts by the value of upper whisker. + +Step 2: + Embed tokens of all selected edit scripts. + +Step 3: + Prepare data for features learning of selected edit scripts. + Vectorize edit scripts with embedded tokens of edit scripts. + +Step 4: + Learn features of all selected edit scripts with CNN algorithm. + Input data: vectorized edit scripts. + +Step 5: + Prepare data for clustering of edit scripts. + Input data: learned features of edit scripts by CNN. + +Step 6: + Clustering of edit scripts with extracted features of edit scripts. + +Step 7: + Analyze cluster results to obtain common fix patterns. + +Step 8: + Prepare testing data for evaluation. + Parse java projects to get the token vectors of all statements. + +Step 9: + Prepare data for evaluation. + Merge token vectors of source code of training data and testing data. + +Step 10: + Prepare data for evaluation. + Embed tokens of source code vectors of training data and testing data. + +Step 11: + Prepare data for evaluation. + Vectorize data (token vectors of source code) for deep learning. + +Step 12: + Evaluation: extract features of testing data and predict their labels. + +Step 13: + \ No newline at end of file diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step13.java b/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step13.java index c891ea4..61b997e 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step13.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step13.java @@ -1,13 +1,23 @@ package edu.lu.uni.serval.FixPatternMining.App; +import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; import java.util.List; +import java.util.Scanner; +import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation; import edu.lu.uni.serval.config.Configuration; import edu.lu.uni.serval.utils.FileHelper; /** - * Evaluation: extract features of testing data and predict their labels. + * Separate features of training data cluster by cluster. + * + * Classify testing data by possibilities: 90%, 80%, 70%, and 60%, ignore others. * * @author kui.liu * @@ -17,17 +27,139 @@ public class Step13 { public static void main(String[] args) { boolean isSupervisedLearning = true; if (isSupervisedLearning) {// supervised learning - // localization file, feature file, - // label --> possibility --> 90, 80, 70, 60 others ignored, level one localization + // features of training data + + DataPreparation.separateTrainingDataFeatures(); + + String positionFiles = Configuration.TEST_POSITION_FILE; + String featureFiles = Configuration.FEATURES_OF_TESTING_DATA; + String labelFiles = Configuration.PREDICTED_RESULTS_OF_TESTING_DATA; + String possibilitiesFilePath = Configuration.POSSIBILITIES_OF_TESTING_DATA; + List possibilitiesFiles = FileHelper.getAllFilesInCurrentDiectory(possibilitiesFilePath, ".csv"); + + String bugs90 = Configuration.TESTING_DATA_BUGS90; + String bugs80 = Configuration.TESTING_DATA_BUGS80; + String bugs70 = Configuration.TESTING_DATA_BUGS70; + String bugs60 = Configuration.TESTING_DATA_BUGS60; + + StringBuilder builder90 = new StringBuilder(); + int counter90 = 0; + StringBuilder builder80 = new StringBuilder(); + int counter80 = 0; + StringBuilder builder70 = new StringBuilder(); + int counter70 = 0; + StringBuilder builder60 = new StringBuilder(); + int counter60 = 0; + for (File possibilitiesFile : possibilitiesFiles) { + String fileName = possibilitiesFile.getName(); + String positionFile = positionFiles + "Positions" + fileName.substring(fileName.lastIndexOf("_"), fileName.lastIndexOf(".")) + ".list"; + String featureFile = featureFiles + fileName; + String labelFile = labelFiles + fileName; + + List possibilities = readData(fileName); + List positions = readData(positionFile); + List features = readData(featureFile); + List labels = readLabel(labelFile); + for (int index = 0, size = possibilities.size(); index < size; index ++) { + String possibilityStr = possibilities.get(index); + String[] array = possibilityStr.split(", "); + int label = labels.get(index); + double possibility = Double.parseDouble(array[label]); + + String position = positions.get(index); + String feature = features.get(index); + // And Label + if (possibility >= 0.9) { + builder90.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n"); + counter90 ++; + if (counter90 % 1000 == 0) { + FileHelper.outputToFile(bugs90, builder90, true); + builder90.setLength(0); + } + } else if (possibility >= 0.8) { + builder80.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n"); + counter80 ++; + if (counter80 % 1000 == 0) { + FileHelper.outputToFile(bugs80, builder80, true); + builder80.setLength(0); + } + } else if (possibility >= 0.7) { + builder70.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n"); + counter70 ++; + if (counter70 % 1000 == 0) { + FileHelper.outputToFile(bugs70, builder70, true); + builder70.setLength(0); + } + } else if (possibility >= 0.6) { + builder60.append("LABEL:" + label + "Feature:" + feature + "Position:" + position + "\n"); + counter60 ++; + if (counter60 % 1000 == 0) { + FileHelper.outputToFile(bugs60, builder60, true); + builder60.setLength(0); + } + } + } + } + FileHelper.outputToFile(bugs90, builder90, true); + FileHelper.outputToFile(bugs80, builder80, true); + FileHelper.outputToFile(bugs70, builder70, true); + FileHelper.outputToFile(bugs60, builder60, true); // label: clusterNum, re-compute similarity with each element. 90, 80, 70, 60. // similarity: patches --> fixing bug. - List testingDataFiles = FileHelper.getAllFilesInCurrentDiectory(Configuration.TESTING_DATA, ".csv"); - for (int i = 0, size = testingDataFiles.size(); i < size; i ++) { - } } else { // un-supervised learning // Extracted Features: Configuration.EXTRACTED_FEATURES_TESTING; // Compute the similarity: cosin similarity } } + + private static List readLabel(String labelFile) { + List labels = new ArrayList<>(); + String fileContent = FileHelper.readFile(labelFile); + BufferedReader reader = null; + try { + reader = new BufferedReader(new StringReader(fileContent)); + String line = null; + while ((line = reader.readLine()) != null) { + String[] labelsStr = line.split(", "); + for (int i = 0, length = labelsStr.length; i < length; i ++) { + Double d = Double.parseDouble(labelsStr[i]); + labels.add(d.intValue()); + } + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return labels; + } + + private static List readData(String positionFile) { + List positions = new ArrayList<>(); + FileInputStream fis = null; + Scanner scanner = null; + try { + fis = new FileInputStream(positionFile); + scanner = new Scanner(fis); + while (scanner.hasNextLine()) { + positions.add(scanner.nextLine()); + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + try { + scanner.close(); + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } + } diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step14.java b/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step14.java new file mode 100644 index 0000000..e2e6713 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step14.java @@ -0,0 +1,225 @@ +package edu.lu.uni.serval.FixPatternMining.App; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; + +import edu.lu.uni.serval.FixPatternMining.DataPrepare.DataPreparation; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.ListSorter; +import edu.lu.uni.serval.utils.MapSorter; + +/** + * Compute similarities for each potential bug instance by computing the similarities with all instances in related cluster. + * + * List top 5 most similar instances? + * + * @author kui.liu + * + */ +public class Step14 { + + public static void main(String[] args) { + String featuresOfTrainingDataPath = Configuration.FEATURES_OF_COMMON_CLUSTERS; + List featuresOfTrainingDataFiles = FileHelper.getAllFiles(featuresOfTrainingDataPath, ".csv");// TODO: type + Map> features = readFeaturesOfTrainingData(featuresOfTrainingDataFiles); + Map> patches = readPatchesOfTraingData(Configuration.CLUSTERED_PATCHES_FILE); + Map labelMapClusterNum = DataPreparation.readLabelMapClusterNum(); + + // potential bugs' information + String bugs90 = Configuration.TESTING_DATA_BUGS90; + String bugs80 = Configuration.TESTING_DATA_BUGS80; + String bugs70 = Configuration.TESTING_DATA_BUGS70; + String bugs60 = Configuration.TESTING_DATA_BUGS60; + List bugsList = new ArrayList<>(); + bugsList.add(bugs90); + bugsList.add(bugs80); + bugsList.add(bugs70); + bugsList.add(bugs60); + + for (String bugs : bugsList) { + List bugsInfo = readData(bugs); + String filePath = bugs.substring(0, bugs.lastIndexOf(".")) + "/"; + StringBuilder builder = new StringBuilder(); + for (String singleBugInfo : bugsInfo) { + String[] infoArray = singleBugInfo.split(":"); + String label = infoArray[0];// TODO + String feature = infoArray[1]; // TODO + String position = infoArray[2]; // TODO + + int labelInt = Integer.parseInt(label); + int clusterNum = labelMapClusterNum.get(labelInt); + Map mostSimilarIndex = computeSimilarities(feature, features.get(clusterNum)); + List patchesIndex = new ArrayList<>(); + if (mostSimilarIndex.size() > 0) { + for (Map.Entry entry : mostSimilarIndex.entrySet()) { + patchesIndex.add(entry.getValue()); + } + + String bug = "BUG####" + position + "\n"; + String patchesStr = readPatches(patchesIndex, patches.get(Integer.parseInt(label))); + // output: bug + patchesStr; + builder.append(bug).append(patchesStr); + } + } + FileHelper.outputToFile(filePath + "patches.list", builder, false); + } + + } + + private static Map> readPatchesOfTraingData(String clusteredPatchesFile) { + Map> map = new HashMap<>(); + List files = FileHelper.getAllFiles(clusteredPatchesFile, ".list"); + for (File file : files) { + String fileName = file.getName(); + int clusterNum = Integer.parseInt(fileName.substring(fileName.lastIndexOf("_") + 1, fileName.lastIndexOf("."))); + List patches = readPatches(file); + map.put(clusterNum, patches); + } + return map; + } + + private static List readPatches(File file) { + List patches = new ArrayList<>(); + FileInputStream fis = null; + Scanner scanner = null; + try { + fis = new FileInputStream(file); + scanner = new Scanner(fis); + String singlePatch = ""; + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + if (Configuration.PATCH_SIGNAL.equals(line)) { + if (!"".equals(singlePatch)) { + patches.add(singlePatch); + singlePatch = ""; + } + } + singlePatch += line + "\n"; + } + patches.add(singlePatch); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + try { + scanner.close(); + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return patches; + } + + private static String readPatches(List patchesIndex, List patchesTrainingData) { + String patches = ""; + for(Integer index : patchesIndex) { + patches += patchesTrainingData.get(index) + "\n"; + } + return patches; + } + + private static Map computeSimilarities(String feature, List trainingFeatures) { + Map mostSimilarIndex = new HashMap<>(); + List similarities = new ArrayList<>(); + for (int i = 0; i < 10; i ++) similarities.add(0.0); + + for (int index = 0, size = trainingFeatures.size(); index < size; index ++) { + String trainingFeature = trainingFeatures.get(index); + double similarity = computeSimilarity(feature, trainingFeature); + double aborted = addToSimilarityies(similarity, similarities); + if (aborted > 0.0) { + mostSimilarIndex.put(similarity, index); + if (aborted == 0.1) mostSimilarIndex.remove(aborted); + } + } + + if (mostSimilarIndex.size() > 0) { + MapSorter sorter = new MapSorter<>(); + mostSimilarIndex = sorter.sortByKeyDescending(mostSimilarIndex); + } + return mostSimilarIndex; + } + + private static double addToSimilarityies(double similarity, List similarities) { + double lastSimilarity = similarities.get(9); + if (similarity >= 0.8 && similarity > lastSimilarity) { // TODO : 9 ? + similarities.set(9, similarity); + ListSorter sorter = new ListSorter(similarities); + similarities = sorter.sortDescending(); + return lastSimilarity == 0.0 ? 0.1 : lastSimilarity; + } + return 0.0; + } + + private static double computeSimilarity(String feature, String trainingFeature) { + // TODO Auto-generated method stub + return 0; + } + + private static Map> readFeaturesOfTrainingData(List featureFiles) { + Map> features = new HashMap<>(); + for (File file : featureFiles) { + String fileName = file.getName(); + String label = fileName.substring(fileName.lastIndexOf("_") + 1, fileName.lastIndexOf(".")); + int clusterNum = Integer.parseInt(label); + List featuresList = readData(file.getPath()); + features.put(clusterNum, featuresList); + } + return features; + } + + private static List readData(String positionFile) { + List positions = new ArrayList<>(); + FileInputStream fis = null; + Scanner scanner = null; + try { + fis = new FileInputStream(positionFile); + scanner = new Scanner(fis); + while (scanner.hasNextLine()) { + positions.add(scanner.nextLine()); + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + try { + scanner.close(); + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } + + public static void quickSort(List arr){ + qsort(arr, 0, arr.size() - 1); + } + private static void qsort(List arr, int low, int high){ + if (low < high){ + int pivot=partition(arr, low, high); //将数组分为两部分 + qsort(arr, low, pivot-1); //递归排序左子数组 + qsort(arr, pivot+1, high); //递归排序右子数组 + } + } + private static int partition(List arr, int low, int high){ + double pivot = arr.get(low); //枢轴记录 + while (low=pivot) --high; + arr.set(low, arr.get(high)); //交换比枢轴小的记录到左端 + while (low featureFiles = FileHelper.getAllFilesInCurrentDiectory(trainingDataFeatures, ".csv"); + File featureFile = featureFiles.get(0); + // File featureFile = new File(Configuration.FEATURES_OF_TRAINING_DATA + ""); + + Map numbersMap = readNumberOfInstances(); // : + Map orders = new HashMap<>(); // : + Map fileNames = new HashMap<>(); + String clusteredTokens = Configuration.CLUSTERED_TOKENSS_FILE; + List files = FileHelper.getAllFilesInCurrentDiectory(clusteredTokens, ".list"); + int order = 1; + for (File file : files) { + String fileName = file.getName(); + String clusterNumStr = fileName.substring(fileName.lastIndexOf("_") + 1, fileName.lastIndexOf(".list")); + int clusterNum = Integer.parseInt(clusterNumStr); + if (numbersMap.containsKey(clusterNum)) { + orders.put(order, clusterNum); + fileNames.put(order, fileName); + order ++; + } + } + + String featuresOfClusterPath = Configuration.FEATURES_OF_COMMON_CLUSTERS; + order = 1; + FileInputStream fis = null; + Scanner scanner = null; + try { + fis = new FileInputStream(featureFile); + scanner = new Scanner(fis); + int counter = 0; + StringBuilder features = new StringBuilder(); + while (scanner.hasNextLine()) { + features.append(scanner.nextLine() + "\n"); + counter ++; + if (counter == numbersMap.get(orders.get(order))) { + FileHelper.outputToFile(featuresOfClusterPath + fileNames.get(order), features, false); + features.setLength(0); + counter = 0; + order ++; + } + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + try { + scanner.close(); + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + private static Map readNumberOfInstances() { + Map numbersMap = new HashMap<>(); + String fileContent = FileHelper.readFile(Configuration.COMMON_CLUSTERS_SIZES); + BufferedReader reader = null; + try { + reader = new BufferedReader(new StringReader(fileContent)); + String line = null; + while ((line = reader.readLine()) != null) { + String[] numbers = line.split(":"); + numbersMap.put(Integer.parseInt(numbers[0]), Integer.parseInt(numbers[1])); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return numbersMap; + } + + public static Map readLabelMapClusterNum() { + Map labelMapClusterNumMap = new HashMap<>(); + String fileContent = FileHelper.readFile(Configuration.CLUSTERNUMBER_LABEL_MAP); + BufferedReader reader = null; + reader = new BufferedReader(new StringReader(fileContent)); + String line = null; + try { + while ((line = reader.readLine()) != null) { + String[] labelMapClusterNum = line.split(":"); + labelMapClusterNumMap.put(Integer.parseInt(labelMapClusterNum[0]), Integer.parseInt(labelMapClusterNum[1])); + } + } catch (IOException e) { + e.printStackTrace(); + }finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } + }