From 14ad7b3c0fec73a4671e9c32737feef8b852a735 Mon Sep 17 00:00:00 2001 From: Kui LIU Date: Thu, 14 Sep 2017 15:00:44 +0200 Subject: [PATCH] Statistics. --- .../lu/uni/serval/statistics/.Rapp.history | 0 .../uni/serval/statistics/FixedProjects.java | 26 + .../statistics/ReadSpearmanResults.java | 104 +++ .../edu/lu/uni/serval/statistics/Spearman.R | 362 +++++++++ .../statistics/SpearmanCodeGenerator.java | 81 ++ .../lu/uni/serval/statistics/Statistic.java | 691 ++++++++++++++++++ .../statistics/TenFoldPossibilities.java | 243 ++++++ .../statistics/TenFoldPossibilities2.java | 256 +++++++ .../statistics/TenFoldPossibilities3.java | 256 +++++++ .../uni/serval/statistics/TenFoldProjs.java | 121 +++ 10 files changed, 2140 insertions(+) create mode 100644 src/main/java/edu/lu/uni/serval/statistics/.Rapp.history create mode 100644 src/main/java/edu/lu/uni/serval/statistics/FixedProjects.java create mode 100644 src/main/java/edu/lu/uni/serval/statistics/ReadSpearmanResults.java create mode 100644 src/main/java/edu/lu/uni/serval/statistics/Spearman.R create mode 100644 src/main/java/edu/lu/uni/serval/statistics/SpearmanCodeGenerator.java create mode 100644 src/main/java/edu/lu/uni/serval/statistics/Statistic.java create mode 100644 src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities.java create mode 100644 src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities2.java create mode 100644 src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities3.java create mode 100644 src/main/java/edu/lu/uni/serval/statistics/TenFoldProjs.java diff --git a/src/main/java/edu/lu/uni/serval/statistics/.Rapp.history b/src/main/java/edu/lu/uni/serval/statistics/.Rapp.history new file mode 100644 index 0000000..e69de29 diff --git a/src/main/java/edu/lu/uni/serval/statistics/FixedProjects.java b/src/main/java/edu/lu/uni/serval/statistics/FixedProjects.java new file mode 100644 index 0000000..122cfab --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/statistics/FixedProjects.java @@ -0,0 +1,26 @@ +package edu.lu.uni.serval.statistics; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; + +import edu.lu.uni.serval.utils.FileHelper; + +public class FixedProjects { + + public static void main(String[] args) throws IOException { + String content = FileHelper.readFile("../FPM_Violations/RQ1/Quantity-per-FixedProj.csv"); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = reader.readLine(); + StringBuilder builder = new StringBuilder(); + while ((line = reader.readLine()) != null) { + String[] elements = line.split(","); +// String projectName = elements[0]; + builder.append(elements[0] + "\n"); + } + reader.close(); + + FileHelper.outputToFile("../FPM_Violations/RQ1/fixedProjects.list", builder, false); + } + +} diff --git a/src/main/java/edu/lu/uni/serval/statistics/ReadSpearmanResults.java b/src/main/java/edu/lu/uni/serval/statistics/ReadSpearmanResults.java new file mode 100644 index 0000000..9763c70 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/statistics/ReadSpearmanResults.java @@ -0,0 +1,104 @@ +package edu.lu.uni.serval.statistics; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; + +import edu.lu.uni.serval.utils.FileHelper; + +public class ReadSpearmanResults { + + public static void main(String[] args) throws IOException { + String fileName = "../FPM_Violations/RQ1/SpearmanResults.list"; + String content = FileHelper.readFile(fileName); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + String str1 = "1"; + String str2 = "2"; + String str3 = "3"; + String str4 = "4"; + String str5 = "5"; + String str6 = "6"; + String str7 = "7"; + String str8 = "8"; + String str9 = "9"; + String str10 = "10"; + int counter = 0; + while ((line = reader.readLine()) != null) { +// String[] elements = line.split(""); + + counter ++; + + if ((counter / 10 == 2 || counter / 10 == 4) && counter % 10 == 1) { + str1 += "@@@@1"; + str2 += "@@@@2"; + str3 += "@@@@3"; + str4 += "@@@@4"; + str5 += "@@@@5"; + str6 += "@@@@6"; + str7 += "@@@@7"; + str8 += "@@@@8"; + str9 += "@@@@9"; + str10 += "@@@@10"; + } + + + int n = counter % 10; + switch (n) { + case 1: + str1 += "@" + line; + break; + case 2: + str2 += "@" + line; + break; + case 3: + str3 += "@" + line; + break; + case 4: + str4 += "@" + line; + break; + case 5: + str5 += "@" + line; + break; + case 6: + str6 += "@" + line; + break; + case 7: + str7 += "@" + line; + break; + case 8: + str8 += "@" + line; + break; + case 9: + str9 += "@" + line; + break; + case 0: + str10 += "@" + line; + break; + } + } + reader.close(); + + str1 = str1.replace(".", ","); + str2 = str2.replace(".", ","); + str3 = str3.replace(".", ","); + str4 = str4.replace(".", ","); + str5 = str5.replace(".", ","); + str6 = str6.replace(".", ","); + str7 = str7.replace(".", ","); + str8 = str8.replace(".", ","); + str9 = str9.replace(".", ","); + str10 = str10.replace(".", ","); + System.out.println(str1); + System.out.println(str2); + System.out.println(str3); + System.out.println(str4); + System.out.println(str5); + System.out.println(str6); + System.out.println(str7); + System.out.println(str8); + System.out.println(str9); + System.out.println(str10); + } + +} diff --git a/src/main/java/edu/lu/uni/serval/statistics/Spearman.R b/src/main/java/edu/lu/uni/serval/statistics/Spearman.R new file mode 100644 index 0000000..549ad42 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/statistics/Spearman.R @@ -0,0 +1,362 @@ +a <- read.csv("~/Public/git/FPM_Violations/RQ1/TenFolds-Q/Ten-fold-all-fixed-violation-type1-70.csv", header=T) +d<-cor.test( ~ X0 + X0.T, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = FALSE, sep = "@") + +d<-cor.test( ~ X1 + X1.T, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X2 + X2.T, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X3 + X3.T, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X4 + X4.T, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X5 + X5.T, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X6 + X6.T, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X7 + X7.T, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X8 + X8.T, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X9 + X9.T, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X0.R + X0.T.R, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X1.R + X1.T.R, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X2.R + X2.T.R, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X3.R + X3.T.R, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X4.R + X4.T.R, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X5.R + X5.T.R, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X6.R + X6.T.R, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X7.R + X7.T.R, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X8.R + X8.T.R, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X9.R + X9.T.R, + data=a, method = "spearman", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X0 + X0.T, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X1 + X1.T, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X2 + X2.T, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X3 + X3.T, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X4 + X4.T, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X5 + X5.T, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X6 + X6.T, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X7 + X7.T, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X8 + X8.T, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X9 + X9.T, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X0.R + X0.T.R, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X1.R + X1.T.R, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X2.R + X2.T.R, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X3.R + X3.T.R, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X4.R + X4.T.R, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X5.R + X5.T.R, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X6.R + X6.T.R, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X7.R + X7.T.R, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X8.R + X8.T.R, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X9.R + X9.T.R, + data=a, method = "pearson", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X0 + X0.T, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X1 + X1.T, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X2 + X2.T, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X3 + X3.T, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X4 + X4.T, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X5 + X5.T, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X6 + X6.T, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X7 + X7.T, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X8 + X8.T, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X9 + X9.T, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X0.R + X0.T.R, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X1.R + X1.T.R, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X2.R + X2.T.R, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X3.R + X3.T.R, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X4.R + X4.T.R, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X5.R + X5.T.R, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X6.R + X6.T.R, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X7.R + X7.T.R, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X8.R + X8.T.R, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +d<-cor.test( ~ X9.R + X9.T.R, + data=a, method = "kendall", + continuity = FALSE, + conf.level = 0.95) +write(c(d$estimate, d$p.value), file = "~/Public/git/FPM_Violations/RQ1/SpearmanResults.list",append = TRUE, sep = "@") + +a=1 diff --git a/src/main/java/edu/lu/uni/serval/statistics/SpearmanCodeGenerator.java b/src/main/java/edu/lu/uni/serval/statistics/SpearmanCodeGenerator.java new file mode 100644 index 0000000..055e522 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/statistics/SpearmanCodeGenerator.java @@ -0,0 +1,81 @@ +package edu.lu.uni.serval.statistics; + + +public class SpearmanCodeGenerator { + + public static void main(String[] args) { + generateSpearmanCode(); + } + + public static void generateSpearmanCode() { + for (int i = 0; i < 10; i ++) { + String s = "d<-cor.test( ~ X" + i + " + X" + i + ".T, \n"; + s += " data=a,"; + s += " method = \"spearman\",\n"; + s += " continuity = FALSE,\n"; + s += " conf.level = 0.95)\n"; + if (i == 0) { + s += "write(c(d$estimate, d$p.value), file = \"~/Public/git/FPM_Violations/RQ1/SpearmanResults.list\"," + + "append = FALSE, sep = \"@\")\n"; + } else { + s += "write(c(d$estimate, d$p.value), file = \"~/Public/git/FPM_Violations/RQ1/SpearmanResults.list\"," + + "append = TRUE, sep = \"@\")\n"; + } + + System.out.println(s); + } + for (int i = 0; i < 10; i ++) { + String s = "d<-cor.test( ~ X" + i + ".R + X" + i + ".T.R, \n"; + s += " data=a,"; + s += " method = \"spearman\",\n"; + s += " continuity = FALSE,\n"; + s += " conf.level = 0.95)\n"; + s += "write(c(d$estimate, d$p.value), file = \"~/Public/git/FPM_Violations/RQ1/SpearmanResults.list\"," + + "append = TRUE, sep = \"@\")\n"; + System.out.println(s); + } + + for (int i = 0; i < 10; i ++) { + String s = "d<-cor.test( ~ X" + i + " + X" + i + ".T, \n"; + s += " data=a,"; + s += " method = \"pearson\",\n"; + s += " continuity = FALSE,\n"; + s += " conf.level = 0.95)\n"; + s += "write(c(d$estimate, d$p.value), file = \"~/Public/git/FPM_Violations/RQ1/SpearmanResults.list\"," + + "append = TRUE, sep = \"@\")\n"; + System.out.println(s); + } + for (int i = 0; i < 10; i ++) { + String s = "d<-cor.test( ~ X" + i + ".R + X" + i + ".T.R, \n"; + s += " data=a,"; + s += " method = \"pearson\",\n"; + s += " continuity = FALSE,\n"; + s += " conf.level = 0.95)\n"; + s += "write(c(d$estimate, d$p.value), file = \"~/Public/git/FPM_Violations/RQ1/SpearmanResults.list\"," + + "append = TRUE, sep = \"@\")\n"; + System.out.println(s); + } + + for (int i = 0; i < 10; i ++) { + String s = "d<-cor.test( ~ X" + i + " + X" + i + ".T, \n"; + s += " data=a,"; + s += " method = \"kendall\",\n"; + s += " continuity = FALSE,\n"; + s += " conf.level = 0.95)\n"; + s += "write(c(d$estimate, d$p.value), file = \"~/Public/git/FPM_Violations/RQ1/SpearmanResults.list\"," + + "append = TRUE, sep = \"@\")\n"; + System.out.println(s); + } + for (int i = 0; i < 10; i ++) { + String s = "d<-cor.test( ~ X" + i + ".R + X" + i + ".T.R, \n"; + s += " data=a,"; + s += " method = \"kendall\",\n"; + s += " continuity = FALSE,\n"; + s += " conf.level = 0.95)\n"; + s += "write(c(d$estimate, d$p.value), file = \"~/Public/git/FPM_Violations/RQ1/SpearmanResults.list\"," + + "append = TRUE, sep = \"@\")\n"; + System.out.println(s); + } + } + +} diff --git a/src/main/java/edu/lu/uni/serval/statistics/Statistic.java b/src/main/java/edu/lu/uni/serval/statistics/Statistic.java new file mode 100644 index 0000000..d5e5ad5 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/statistics/Statistic.java @@ -0,0 +1,691 @@ +package edu.lu.uni.serval.statistics; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; + +import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.ListSorter; +import edu.lu.uni.serval.utils.MapSorter; + +public class Statistic { + public static void main(String[] args) throws IOException { +// /* +// * Quantities' distribution of all violation types. +// */ +// quantityOfEachViolationType(); +// /* +// * Widespread of each violation type. +// */ +// widespreadOfEachViolationType(); +// /* +// * Statistics of categories of all violation types. +// */ +// statisticWithCategories(); +// +// /* +// * Quantity of each violation type in each project. +// */ +// reloadData(); +// +// /* +// * Quantities' distribution of all fixed violation types. +// * Widespread of each fixed violation type. +// * Statistics of categories of all fixed violation types. +// * Quantity of each fixed violation type in each project. +// */ +// statisticOfFixedViolations(); +// fixedVSunfixed(); + + /** + * Do statistics from two files: + */ + statistics("../FPM_Violations/RQ1/all-leafnodes-per-project-vtype.csv", ""); + statistics("../FPM_Violations/RQ1/distinct-fixed-summary-per-project-vtype.csv", "Fixed"); +// fixedVSunfixed(); + +// String statistic = "../FPM_Violations/OUTPUT"; +// List files = FileHelper.getAllFiles(statistic, ".list"); +// +// int testAlarms = 0; +// int nullGumTreeResults = 0; +// int nullMappingGumTreeResults = 0; +// int pureDeletion = 0; +// int timeout = 0; +// int noSourceCodeChagnes = 0; +// int largeHunk = 0; +// int nullSourceCode = 0; +// for (File file : files) { +// String content = FileHelper.readFile(file); +// BufferedReader reader = new BufferedReader(new StringReader(content)); +// String line = null; +// try { +// while ((line = reader.readLine()) != null) { +// if (line.startsWith("test")) { +// testAlarms += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); +// } else if (line.startsWith("nullGum")) { +// nullGumTreeResults += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); +// } else if (line.startsWith("nullMap")) { +// nullMappingGumTreeResults += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); +// } else if (line.startsWith("pure")) { +// pureDeletion += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); +// } else if (line.startsWith("Time")) { +// timeout += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); +// } else if (line.startsWith("noSourceCodeChagnes")) { +// noSourceCodeChagnes += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); +// } else if (line.startsWith("largeHunk")) { +// largeHunk += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); +// } else if (line.startsWith("nullSourceCode")) { +// nullSourceCode += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); +// } +// } +// } catch (IOException e) { +// e.printStackTrace(); +// } finally { +// try { +// reader.close(); +// } catch (IOException e) { +// e.printStackTrace(); +// } +// } +// } +// +//// TestAlarms: 5175 +//// nullGumTreeResults: 13449 +//// nullMappingGumTreeResults: 33010 +//// pureDeletion: 7598 +//// Timeout: 263 +// +// System.out.println("TestAlarms: " + testAlarms); +// System.out.println("nullGumTreeResults: " + nullGumTreeResults); +// System.out.println("nullMappingGumTreeResults: " + nullMappingGumTreeResults); +// System.out.println("pureDeletion: " + pureDeletion); +// System.out.println("Timeout: " + timeout); +// System.out.println("noSourceCodeChagnes: " + noSourceCodeChagnes); +// System.out.println("largeHunk: " + largeHunk); +// System.out.println("nullSourceCode: " + nullSourceCode); + } + + public static void statistics(String fileName, String type) throws IOException { + FileInputStream fis = new FileInputStream(fileName); + Scanner scanner = new Scanner(fis); + + Map violationTypesMap = new HashMap<>(); + Map projectsMap = new HashMap<>(); + Map perVperProjMap = new HashMap<>(); + Map> widespreadViolationsMap = new HashMap<>(); + + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + String[] elements = line.split(","); + String projectName = elements[0]; + String violationType = elements[1]; + int quantity = Integer.parseInt(elements[2]); + + addToMap(projectsMap, projectName, quantity); + addToMap(violationTypesMap, violationType, quantity); + addToMap(perVperProjMap, projectName + "," + violationType, quantity); + + if (widespreadViolationsMap.containsKey(violationType)) { + List projectList = widespreadViolationsMap.get(violationType); + if (!projectList.contains(projectName)) { + projectList.add(projectName); + } + } else { + List projectList = new ArrayList<>(); + projectList.add(projectName); + widespreadViolationsMap.put(violationType, projectList); + } + } + + scanner.close(); + fis.close(); + + + // Category + MapSorter sorter = new MapSorter(); + violationTypesMap = sorter.sortByValueDescending(violationTypesMap); + projectsMap = sorter.sortByValueDescending(projectsMap); + + Map categories = new HashMap<>(); + Map> categoryVList = new HashMap<>(); + String content = FileHelper.readFile("../FPM_Violations/RQ1/ViolationCategory.list"); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + while ((line = reader.readLine()) != null) { + String[] elements = line.split("@@"); + categories.put(elements[1], elements[0]); // Violation type, category + } + reader.close(); + + // Sort Violation types by widespread. + Map widespreadOfAllViolations = new HashMap<>(); + for (Map.Entry> entry : widespreadViolationsMap.entrySet()) { + widespreadOfAllViolations.put(entry.getKey(), entry.getValue().size()); + } + widespreadOfAllViolations = sorter.sortByValueDescending(widespreadOfAllViolations); + StringBuilder wbuilder = new StringBuilder("Type,Identifier,Quantity,Category\n"); + int identifier1 = 0; + for (Map.Entry entry : widespreadOfAllViolations.entrySet()) { + identifier1 ++; + String violationType = entry.getKey(); + String category = categories.get(violationType); + if (category == null || category.equals("null")) { + category = "Other"; + } + wbuilder.append(violationType + "," + identifier1 + "," + entry.getValue() + "," + category + "\n"); + + if (categoryVList.containsKey(category)) { + categoryVList.get(category).add(violationType); + } else { + List list = new ArrayList<>(); + list.add(violationType); + categoryVList.put(category, list); + } + } + for (Map.Entry> entry : categoryVList.entrySet()) { + System.out.println(entry.getKey() + ":" + entry.getValue().size()); + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Widespread-per-" + type + "V-Type.csv", wbuilder, false); + + // output statistics + List sortedViolationTypes = new ArrayList<>(); + + Map quantityOfCategory = new HashMap<>(); + StringBuilder violationsBuilder = new StringBuilder("Type,Identifier,Quantity,Widespread,Category\n"); + int identifier = 0; + for (Map.Entry entry : violationTypesMap.entrySet()) { + String violationType = entry.getKey(); + int quantity = entry.getValue(); + identifier ++; + String category = categories.get(violationType); + if (category == null || category.equals("null")) { + category = "Other"; + } + violationsBuilder.append(violationType + "," + identifier + "," + quantity + "," + widespreadViolationsMap.get(violationType).size() + + "," + category + "\n"); + + if (quantityOfCategory.containsKey(category)) { + quantityOfCategory.put(category, quantityOfCategory.get(category) + quantity); + } else { + quantityOfCategory.put(category, quantity); + } + + sortedViolationTypes.add(violationType); + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Quantity-per-" + type + "V-Type.csv", violationsBuilder, false); + + StringBuilder pBuilder = new StringBuilder("Project,Quantity\n"); + List projectNames = new ArrayList<>(); + for (Map.Entry entry : projectsMap.entrySet()) { + String project = entry.getKey(); + pBuilder.append(project + "," + entry.getValue() + "\n"); + projectNames.add(project); + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Quantity-per-" + type + "Proj.csv", pBuilder, false); + + StringBuilder categoryBuilder = new StringBuilder("Category,Quantity\n"); + for (Map.Entry entry : quantityOfCategory.entrySet()) { + categoryBuilder.append(entry.getKey() + "," + entry.getValue() + "\n"); + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Quantity-per-" + type + "Category.csv", categoryBuilder, false); + + StringBuilder builder = new StringBuilder("Projects"); + String a = ""; + String b = ""; + for (int i = 0; i < sortedViolationTypes.size(); i ++) { + builder.append("," + sortedViolationTypes.get(i)); + if (i < 50) { + a += "a$" + sortedViolationTypes.get(i) + ","; + b += "'" + (i + 1) + "',"; + } + } + builder.append("\n"); + System.out.println(a); + System.out.println(b); +// for (int i = 0; i < sortedViolationTypes.size(); i ++) { +// builder.append("," + i); +// } +// builder.append("\n"); + for (int i = 0; i < projectNames.size(); i++) { + String projectName = projectNames.get(i); + builder.append(projectName); + for (int j = 0; j < sortedViolationTypes.size(); j ++) { + String violationType = sortedViolationTypes.get(j); + String key = projectName + "," + violationType; + Integer value = perVperProjMap.get(key); + if (value == null) { + value = 0; + } + builder.append("," + value); + } + builder.append("\n"); + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Distribution-per-project-per-" + type + "type.csv", builder, false); + + StringBuilder ssbuilder = new StringBuilder("Type, Identifier, Quantity, Category\n"); + Map perTypePerProj = new HashMap<>(); + for (int j = 0; j < sortedViolationTypes.size(); j ++) { + String violationType = sortedViolationTypes.get(j); + List projs = new ArrayList<>(); + + for (int i = 0; i < projectNames.size(); i++) { + String projectName = projectNames.get(i); + String key = projectName + "," + violationType; + Integer value = perVperProjMap.get(key); + + if (value != null) { + String category = categories.get(violationType); + if (category == null || category.equals("null")) { + category = "Other"; + } + ssbuilder.append(violationType + "," + (j + 1) + "," + value + "," + category + "\n"); + + projs.add(value); + } + } + + ListSorter sorter2 = new ListSorter(projs); + projs = sorter2.sortAscending(); + int index = projs.size() % 2 == 0 ? projs.size() / 2 - 1 : projs.size() / 2; + perTypePerProj.put(violationType, projs.get(index)); + } + + FileHelper.outputToFile("../FPM_Violations/RQ1/Distribution-per-project-per-" + type + "type2.csv", ssbuilder, false); + StringBuilder ssbuilder2 = new StringBuilder("Type, Identifier, Quantity, Category\n"); + perTypePerProj = sorter.sortByValueDescending(perTypePerProj); + int j = 0; + for (Map.Entry entry : perTypePerProj.entrySet()) { + String violationType = entry.getKey(); +// System.out.println(entry.getValue()); + j ++; + for (int i = 0; i < projectNames.size(); i++) { + String projectName = projectNames.get(i); + String key = projectName + "," + violationType; + Integer value = perVperProjMap.get(key); + + if (value != null) { + String category = categories.get(violationType); + if (category == null || category.equals("null")) { + category = "Other"; + } + ssbuilder2.append(violationType + "," + j + "," + value + "," + category + "\n"); + } + } + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Distribution-per-project-per-" + type + "type3.csv", ssbuilder2, false); +// StringBuilder pVpPBuilder = new StringBuilder(); +// for (Map.Entry entry : perVperProjMap.entrySet()) { +// pVpPBuilder.append(entry.getKey() + "," + entry.getValue() + "\n"); +// } +// FileHelper.outputToFile("../FPM_Violations/RQ1/Per-project-per-" + type + "type.csv", pVpPBuilder, false); + } + + private static void addToMap(Map map, String key, int value) { + if (map.containsKey(key)) { + map.put(key, map.get(key) + value); + } else { + map.put(key, value); + } + } + + public static void quantityOfEachViolationType() { + Map violationQuantities = readTypeQuantityMap("../FPM_Violations/RQ1/distinct-per-vtype.csv"); + + Map violationWidespread = readWidespread("../FPM_Violations/RQ1/distinct-per-project-vtype.csv"); + + StringBuilder buidler = new StringBuilder("Violation Type, Identifier, Quantity, Quantity of Projects, Ratio\n"); + int identifier = 0; + double totality = 15961605d; + int sum = 0; + for (Map.Entry entry : violationQuantities.entrySet()) { + String key = entry.getKey(); + identifier ++; + int quantity = entry.getValue(); + sum += quantity; + buidler.append(key + "," + identifier + "," + quantity + "," + violationWidespread.get(key) + "," + (sum / totality) + "\n"); +// if (identifier >= 50) break; + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Quantity-per-V-type.csv", buidler, false); + } + + public static void widespreadOfEachViolationType() { + Map violationWidespread = readWidespread("../FPM_Violations/RQ1/distinct-per-project-vtype.csv"); + System.out.println("Violation types: " + violationWidespread.size()); + + StringBuilder buidler = new StringBuilder("Violation Type, Identifier, Quantity of Projects\n"); + int identifier = 0; + for (Map.Entry entry : violationWidespread.entrySet()) { + identifier ++; + buidler.append(entry.getKey() + "," + identifier + "," + entry.getValue() + "\n"); +// if (identifier >= 50) break; + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Widespread-per-V-type.csv", buidler, false); + } + + public static void statisticWithCategories() throws IOException { + Map categories = new HashMap<>(); + String content = FileHelper.readFile("../FPM_Violations/RQ1/ViolationCategory.list"); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + while ((line = reader.readLine()) != null) { + String[] elements = line.split("@@"); + categories.put(elements[1], elements[0]); // Violation type, category + } + reader.close(); + + Map quantityOfCategory = new HashMap<>(); + + String content2 = FileHelper.readFile("../FPM_Violations/RQ1/distinct-per-vtype.csv"); + reader = new BufferedReader(new StringReader(content2)); + line = reader.readLine(); + while ((line = reader.readLine()) != null) { + String[] elements = line.split(","); + String violationType = elements[0]; + int quantity = Integer.parseInt(elements[1]); + String category = categories.get(violationType); + if (quantityOfCategory.containsKey(category)) { + quantityOfCategory.put(category, quantityOfCategory.get(category) + quantity); + } else { + quantityOfCategory.put(category, quantity); + } + } + reader.close(); + + String categoryQuantity = "Category Type, Quantity\n"; + for (Map.Entry entry : quantityOfCategory.entrySet()) { + categoryQuantity += entry.getKey() + "," + entry.getValue() + "\n"; + } + + FileHelper.outputToFile("../FPM_Violations/RQ1/Quantity-per-Category.csv", categoryQuantity, false); + } + + public static void reloadData() { + // ordered projects by quantity of violations + Map projectQuantities = readTypeQuantityMap("../FPM_Violations/RQ1/distinct-per-project.csv"); + List projectNames = new ArrayList<>(); + for (Map.Entry entry : projectQuantities.entrySet()) { + projectNames.add(entry.getKey()); + } + + // get ordered types by quantity of violations. + Map violations = readTypeQuantityMap("../FPM_Violations/RQ1/distinct-per-vtype.csv"); + List violationTypes = new ArrayList<>(); + for (Map.Entry entry : violations.entrySet()) { + violationTypes.add(entry.getKey()); + } + + Map perVperProjs = readData("../FPM_Violations/RQ1/distinct-per-project-vtype.csv"); + + StringBuilder builder = new StringBuilder("Projects"); + for (int i = 0; i < violationTypes.size(); i ++) { + builder.append("," + violationTypes.get(i)); + } + builder.append("\n"); + for (int i = 0; i < projectNames.size(); i++) { + String projectName = projectNames.get(i); + builder.append(projectName); + for (int j = 0; j < violationTypes.size(); j ++) { + String violationType = violationTypes.get(j); + String key = projectName + "," + violationType; + String value = perVperProjs.get(key); + if (value == null) { + value = "0"; + } + builder.append("," + value); + + } + builder.append("\n"); + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Distribution-per-project-per-type.csv", builder, false); + } + + public static void statisticOfFixedViolations() throws IOException { + String fileName = "../FPM_Violations/RQ1/fixed-alarms-v1.0.list"; + FileInputStream fis = new FileInputStream(fileName); + Scanner scanner = new Scanner(fis); + + Map violations = new HashMap<>(); + Map projects = new HashMap<>(); + Map perVperPro = new HashMap<>(); + Map> widespreadFixedV = new HashMap<>(); + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + String[] elements = line.split(":"); + String violationType = elements[0]; + String projectName = elements[1]; + + if (violations.containsKey(violationType)) { + violations.put(violationType, violations.get(violationType) + 1); + } else { + violations.put(violationType, 1); + } + + if (projects.containsKey(projectName)) { + projects.put(projectName, projects.get(projectName) + 1); + } else { + projects.put(projectName, 1); + } + + String perVperProj = projectName + "," + violationType; + if (perVperPro.containsKey(perVperProj)) { + perVperPro.put(perVperProj, perVperPro.get(perVperProj) + 1); + } else { + perVperPro.put(perVperProj, 1); + } + + if (widespreadFixedV.containsKey(violationType)) { + List projectList = widespreadFixedV.get(violationType); + if (!projectList.contains(projectName)) { + projectList.add(projectName); + } + } else { + List projectList = new ArrayList<>(); + projectList.add(projectName); + widespreadFixedV.put(violationType, projectList); + } + } + + scanner.close(); + fis.close(); + + MapSorter sorter = new MapSorter(); + violations = sorter.sortByValueDescending(violations); + projects = sorter.sortByValueDescending(projects); + + Map categories = new HashMap<>(); + String content = FileHelper.readFile("../FPM_Violations/RQ1/ViolationCategory.list"); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + while ((line = reader.readLine()) != null) { + String[] elements = line.split("@@"); + categories.put(elements[1], elements[0]); // Violation type, category + } + reader.close(); + + List violationTypes = new ArrayList<>(); + + Map quantityOfCategory = new HashMap<>(); + StringBuilder violationsBuilder = new StringBuilder(); + for (Map.Entry entry : violations.entrySet()) { + String violationType = entry.getKey(); + int quantity = entry.getValue(); + + violationsBuilder.append(violationType + "," + quantity + "," + widespreadFixedV.get(violationType).size() + "\n"); + + String category = categories.get(violationType); + if (quantityOfCategory.containsKey(category)) { + quantityOfCategory.put(category, quantityOfCategory.get(category) + quantity); + } else { + quantityOfCategory.put(category, quantity); + } + + violationTypes.add(violationType); + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Quantity-per-FixedV-Type.csv", violationsBuilder, false); + + StringBuilder pBuilder = new StringBuilder(); + List projectNames = new ArrayList<>(); + for (Map.Entry entry : projects.entrySet()) { + String project = entry.getKey(); + pBuilder.append(project + "," + entry.getValue() + "\n"); + projectNames.add(project); + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Quantity-per-FixedV-Proj.csv", pBuilder, false); + + StringBuilder categoryBuilder = new StringBuilder(); + for (Map.Entry entry : quantityOfCategory.entrySet()) { + categoryBuilder.append(entry.getKey() + "," + entry.getValue() + "\n"); + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Quantity-per-Fixed-Category.csv", categoryBuilder, false); + + StringBuilder builder = new StringBuilder("Projects"); + for (int i = 0; i < violationTypes.size(); i ++) { + builder.append("," + violationTypes.get(i)); + if (i < 500) { + } + } + builder.append("\n"); + for (int i = 0; i < projectNames.size(); i++) { + String projectName = projectNames.get(i); + builder.append(projectName); + for (int j = 0; j < violationTypes.size(); j ++) { + String violationType = violationTypes.get(j); + String key = projectName + "," + violationType; + Integer value = perVperPro.get(key); + if (value == null) { + value = 0; + } + builder.append("," + value); + } + builder.append("\n"); + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Distribution-per-project-per-Fixed-type.csv", builder, false); + + StringBuilder pVpPBuilder = new StringBuilder(); + for (Map.Entry entry : perVperPro.entrySet()) { + pVpPBuilder.append(entry.getKey() + "," + entry.getValue() + "\n"); + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Per-project-per-Fixed-type.csv", pVpPBuilder, false); + } + + public static void fixedVSunfixed() throws IOException { + String fileName = "../FPM_Violations/RQ1/Quantity-per-V-type.csv"; // all violations + Map allViolations = readTypeQuantityMap(fileName); + String file = "../FPM_Violations/RQ1/Quantity-per-FixedV-Type.csv"; // fixed violations + Map fixedViolations = readTypeQuantityMap(file); + + MapSorter sorter = new MapSorter<>(); + allViolations = sorter.sortByValueDescending(allViolations); + StringBuilder builder = new StringBuilder("Type,fixed,unfixed,all,fixed Ratio,unfixed Ratio\n"); + for (Map.Entry entry : allViolations.entrySet()) { + String violationType = entry.getKey(); + int quantity = entry.getValue(); + Integer fixedQuantity = fixedViolations.get(violationType); + if (fixedQuantity == null) { + builder.append(violationType + "," + 0 + "," + quantity + "," + quantity + ",0.0,1\n"); + } else { + builder.append(violationType + "," + fixedQuantity + "," + (quantity - fixedQuantity) + + "," + quantity + "," + ((double)fixedQuantity) / ((double) quantity) + + "," + ((double)(quantity - fixedQuantity)) / ((double) quantity) + "\n"); + } + } + FileHelper.outputToFile("../FPM_Violations/RQ1/Distribution-per-Fixed-type-VS-per-unFixed-type.csv", builder, false); + } + + private static Map readData(String fileName) { + Map perVperPros = new HashMap<>(); + FileInputStream fis = null; + Scanner scanner = null; + try { + fis = new FileInputStream(fileName); + scanner = new Scanner(fis); + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + String[] elements = line.split(","); + String key = elements[0] + "," + elements[1]; + String value = elements[2]; + perVperPros.put(key, value); + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + try { + scanner.close(); + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + return perVperPros; + } + + private static Map readTypeQuantityMap(String fileName) { + Map map = new HashMap<>(); + String fileContent = FileHelper.readFile(fileName); + BufferedReader reader = new BufferedReader(new StringReader(fileContent)); + + try { + String line= reader.readLine(); + while ((line = reader.readLine()) != null) { + String[] elements = line.split(","); + map.put(elements[0], Integer.valueOf(elements[1])); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + MapSorter sorter = new MapSorter<>(); + map = sorter.sortByValueDescending(map); + return map; + } + + private static Map readWidespread(String fileName) { + Map violationWidespread = new HashMap(); + + String fileContent = FileHelper.readFile(fileName); + BufferedReader reader = new BufferedReader(new StringReader(fileContent)); + + try { + String line = null; + while ((line = reader.readLine()) != null) { + String[] strArray = line.split(","); + String key = strArray[1]; + if (violationWidespread.containsKey(key)) { + violationWidespread.put(key, violationWidespread.get(key) + 1); + } else { + violationWidespread.put(key, 1); + } + +// String[] elements = line.split(","); +// violationWidespread.put(elements[0], Integer.valueOf(elements[1])); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + MapSorter sorter = new MapSorter<>(); + violationWidespread = sorter.sortByValueDescending(violationWidespread); + return violationWidespread; + } +} diff --git a/src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities.java b/src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities.java new file mode 100644 index 0000000..58b276a --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities.java @@ -0,0 +1,243 @@ +package edu.lu.uni.serval.statistics; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.MapSorter; + +public class TenFoldPossibilities { + + public static void main(String[] args) throws IOException { + rankingInTenFolds("../FPM_Violations/RQ1/TenFolds/", "../FPM_Violations/RQ1/Quantity-per-V-type.csv", "TenFolds"); // ten folds of all projects. + rankingInTenFolds("../FPM_Violations/RQ1/FixedTenFolds-1/", "../FPM_Violations/RQ1/Quantity-per-FixedV-Type.csv", "FixedTenFolds-1");// all fixed violation types are contained. + } + + public static void rankingInTenFolds(String tenFoldFiles, String typeFile, String outputPath) throws NumberFormatException, IOException { + List files = FileHelper.getAllFilesInCurrentDiectory(tenFoldFiles, ".list"); + + Map> tenFoldsOfAllViolationTypes = new HashMap<>(); + Map> tenFoldsOfAllFixedViolationTypes1 = new HashMap<>(); + Map> tenFoldsOfAllFixedViolationTypes2 = new HashMap<>(); + Map> testingTenFoldsOfAllViolationTypes = new HashMap<>(); + Map> testingTenFoldsOfAllFixedViolationTypes1 = new HashMap<>(); + Map> testingTenFoldsOfAllFixedViolationTypes2 = new HashMap<>(); + for (int i = 0; i < 10; i ++) { + List projects = selectPorjects(files, i); + + // the distribution of all violation types in these 9 sub-sets. + Map violations = new HashMap<>(); + int quantityOfAllViolations = readViolationsPerProject(violations, projects, "../FPM_Violations/RQ1/all-leafnodes-per-project-vtype.csv"); + // the distribution of all fixed violation types in these 9 sub-sets. + Map fixedViolations = new HashMap<>(); + int quantityOfAllFixedViolations = readViolationsPerProject(fixedViolations, projects, "../FPM_Violations/RQ1/distinct-fixed-summary-per-project-vtype.csv"); + + // Ratio of each violation type in all violations. + Map ratioOfEachViolationType = new HashMap<>(); // ratio of each violation type + Map ratioOfEachFixedViolationType1 = new HashMap<>(); // quantity of each fixed violation type / quantity of each violation type. + Map ratioOfEachFixedViolationType2 = new HashMap<>(); // quantity of each fixed violation type / quantityOfAllFixedViolations + + for (Map.Entry entry : violations.entrySet()) { + String violationType = entry.getKey(); + int quantity = entry.getValue(); + double ratio = (double) quantity / quantityOfAllViolations; + ratioOfEachViolationType.put(violationType, ratio); + + if (fixedViolations.containsKey(violationType)) { + int quantity1 = fixedViolations.get(violationType); + double ratio1 = (double)quantity1 / quantity; + double ratio2 = (double)quantity1 / quantityOfAllFixedViolations; + ratioOfEachFixedViolationType1.put(violationType, ratio1); + ratioOfEachFixedViolationType2.put(violationType, ratio2); + } else { + ratioOfEachFixedViolationType1.put(violationType, 0d); + } + } + + tenFoldsOfAllViolationTypes.put(i, ratioOfEachViolationType); + tenFoldsOfAllFixedViolationTypes1.put(i, ratioOfEachFixedViolationType1); + tenFoldsOfAllFixedViolationTypes2.put(i, ratioOfEachFixedViolationType2); + + + // Testing Data + List testingProjects = readList(files.get(i)); + Map testingViolations = new HashMap<>(); + int testingQuantitifOfAllViolations = readViolationsPerProject(testingViolations, testingProjects, "../FPM_Violations/RQ1/all-leafnodes-per-project-vtype.csv"); + Map testingFixedViolations = new HashMap<>(); + int testingQuantityOfAllFixedViolations = readViolationsPerProject(testingFixedViolations, testingProjects, "../FPM_Violations/RQ1/distinct-fixed-summary-per-project-vtype.csv"); + + Map testingRatioOfEachViolationType = new HashMap<>(); + Map testingRatioOfEachFixedViolationType1 = new HashMap<>(); + Map testingRatioOfEachFixedViolationType2 = new HashMap<>(); + + for (Map.Entry entry : testingViolations.entrySet()) { + String violationType = entry.getKey(); + int quantity = entry.getValue(); + double ratio = (double) quantity / testingQuantitifOfAllViolations; + + testingRatioOfEachViolationType.put(violationType, ratio); + if (testingFixedViolations.containsKey(violationType)) { + int quantity1 = testingFixedViolations.get(violationType); + testingRatioOfEachFixedViolationType1.put(violationType, (double)quantity1 / quantity); + testingRatioOfEachFixedViolationType2.put(violationType, (double)quantity1 / testingQuantityOfAllFixedViolations); + } else { + testingRatioOfEachFixedViolationType1.put(violationType, 0d); + } + } + testingTenFoldsOfAllViolationTypes.put(i, testingRatioOfEachViolationType); + testingTenFoldsOfAllFixedViolationTypes1.put(i, testingRatioOfEachFixedViolationType1); + testingTenFoldsOfAllFixedViolationTypes2.put(i, testingRatioOfEachFixedViolationType2); + } + + List violationTypes = readTypes(typeFile); + outputTenFolds(tenFoldsOfAllViolationTypes, testingTenFoldsOfAllViolationTypes, violationTypes, "../FPM_Violations/RQ1/" + outputPath + "/Ten-fold-all-violation-type.csv"); + outputTenFolds(tenFoldsOfAllFixedViolationTypes1, testingTenFoldsOfAllFixedViolationTypes1, violationTypes, "../FPM_Violations/RQ1/" + outputPath + "/Ten-fold-all-fixed-violation-type1.csv"); + outputTenFolds(tenFoldsOfAllFixedViolationTypes2, testingTenFoldsOfAllFixedViolationTypes2, violationTypes, "../FPM_Violations/RQ1/" + outputPath + "/Ten-fold-all-fixed-violation-type2.csv"); + } + + private static List readTypes(String fileName) throws IOException { + List violationTypes = new ArrayList<>(); + String content = FileHelper.readFile(fileName); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = reader.readLine(); + while ((line = reader.readLine()) != null) { + String[] elements = line.split(","); + violationTypes.add(elements[0]); + } + reader.close(); + return violationTypes; + } + + private static void outputTenFolds(Map> ratiosMap, Map> testingMap, List violationTypes, String fileName) { + StringBuilder builder = new StringBuilder("Violatype, 0, 0-R, 0-T, 0-T-R, 1, 1-R, 1-T, 1-T-R, 2, 2-R, 2-T, 2-T-R, 3, 3-R, 3-T, 3-T-R, 4, 4-R, 4-T, 4-T-R, 5, 5-R, 5-T, 5-T-R, 6, 6-R, 6-T, 6-T-R, 7, 7-R, 7-T, 7-T-R, 8, 8-R, 8-T, 8-T-R, 9, 9-R, 9-T, 9-T-R\n"); + + Map> ratiosRankingMap = new HashMap<>(); + Map> testingDataRankingMap = new HashMap<>(); + + for (String type : violationTypes) { + builder.append(type); + for (int i = 0; i < 10; i ++) { + Map ratios = ratiosMap.get(i); + Map testingData = testingMap.get(i); + + Map ratiosRanking; + Map testingDataRanking; + if (!ratiosRankingMap.containsKey(i)) { + ratiosRanking = rankAlarmTypes(ratios); + testingDataRanking = rankAlarmTypes(testingData); + } else { + ratiosRanking = ratiosRankingMap.get(i); + testingDataRanking = testingDataRankingMap.get(i); + } + + + if (ratios.containsKey(type)) { + builder.append(", " + ratios.get(type));//.toString().replace(".", ",") + } else { + builder.append(", 0.0"); + } + if (ratiosRanking.containsKey(type)) { + builder.append(", " + ratiosRanking.get(type)); + } else { + builder.append(", " + (ratiosRanking.size() + 1)); + } + if (testingData.containsKey(type)) { + builder.append(", " + testingData.get(type)); + } else { + builder.append(", 0.0"); + } + if (testingDataRanking.containsKey(type)) { + builder.append(", " + testingDataRanking.get(type)); + } else { + builder.append(", " + (testingDataRanking.size() + 1)); + } + } + builder.append("\n"); + } + FileHelper.outputToFile(fileName, builder, false); + } + + private static Map rankAlarmTypes(Map possibilityMap) { + MapSorter sorter = new MapSorter<>(); + possibilityMap = sorter.sortByValueDescending(possibilityMap); + Map ranking = new HashMap<>(); + int ranker = 0; + double possibility = 0; + for (Map.Entry entry : possibilityMap.entrySet()) { + String alarmType = entry.getKey(); + double value = entry.getValue(); + if (possibility != value) { + ranker = ranking.size() + 1; + } + ranking.put(alarmType, ranker); + } + return ranking; + } + + private static int readViolationsPerProject(Map violations, List projects, String fileName) throws NumberFormatException, IOException { + int quantityOfAllViolations = 0; + + String content = FileHelper.readFile(fileName); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + while ((line = reader.readLine()) != null) { + String[] elements = line.split(","); + String projectName = elements[0]; + if (projects.contains(projectName)) { + String alarmType = elements[1]; + int quantity = Integer.parseInt(elements[2]); + + quantityOfAllViolations += quantity; + if (violations.containsKey(alarmType)) { + violations.put(alarmType, violations.get(alarmType) + quantity); + } else { + violations.put(alarmType, quantity); + } + } + } + reader.close(); + return quantityOfAllViolations; + } + + private static List selectPorjects(List files, int i) { + List projects = new ArrayList<>(); + for (File file : files) { + if (file.getName().equals("Fold_" + i + ".list")) { + continue; + } + + projects.addAll(readList(file)); + } + return projects; + } + + private static List readList(File file) { + List projects = new ArrayList<>(); + + String content = FileHelper.readFile(file); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + try { + while ((line = reader.readLine()) != null) { + projects.add(line); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + return projects; + } + +} diff --git a/src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities2.java b/src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities2.java new file mode 100644 index 0000000..96ba1b6 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities2.java @@ -0,0 +1,256 @@ +package edu.lu.uni.serval.statistics; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.MapSorter; + +/** + * Fixed violation types are selected by their widespread. + * + * @author kui.liu + * + */ +public class TenFoldPossibilities2 { + + public static void main(String[] args) throws IOException { + rankingInTenFolds("../FPM_Violations/RQ1/FixedTenFolds-W/", "../FPM_Violations/RQ1/Widespread-per-FixedV-Type.csv", "FixedTenFolds-W"); + } + + public static void rankingInTenFolds(String tenFoldFiles, String typeFile, String outputPath) throws NumberFormatException, IOException { + int topNumber = 10; + List violationTypes = readTypes(typeFile, topNumber); + + List files = FileHelper.getAllFilesInCurrentDiectory(tenFoldFiles, ".list"); + + Map> tenFoldsOfAllViolationTypes = new HashMap<>(); + Map> tenFoldsOfAllFixedViolationTypes1 = new HashMap<>(); + Map> tenFoldsOfAllFixedViolationTypes2 = new HashMap<>(); + Map> testingTenFoldsOfAllViolationTypes = new HashMap<>(); + Map> testingTenFoldsOfAllFixedViolationTypes1 = new HashMap<>(); + Map> testingTenFoldsOfAllFixedViolationTypes2 = new HashMap<>(); + for (int i = 0; i < 10; i ++) { + List projects = selectPorjects(files, i); + + // all violations in these 9 sub-sets. + Map violations = new HashMap<>(); + int quantityOfAllViolations = readViolationsPerProject(violationTypes, violations, projects, "../FPM_Violations/RQ1/all-leafnodes-per-project-vtype.csv"); + Map fixedViolations = new HashMap<>(); + int quantityOfAllFixedViolations = readViolationsPerProject(violationTypes, fixedViolations, projects, "../FPM_Violations/RQ1/distinct-fixed-summary-per-project-vtype.csv"); + + // Ratio of each violation type in all violations. + Map ratioOfEachViolationType = new HashMap<>(); // ratio of each violation type + Map ratioOfEachFixedViolationType1 = new HashMap<>(); // quantity of each fixed violation type / quantity of each violation type. + Map ratioOfEachFixedViolationType2 = new HashMap<>(); // quantity of each fixed violation type / quantityOfAllFixedViolations + + for (Map.Entry entry : violations.entrySet()) { + String violationType = entry.getKey(); + int quantity = entry.getValue(); + double ratio = (double) quantity / quantityOfAllViolations; + ratioOfEachViolationType.put(violationType, ratio); + + if (fixedViolations.containsKey(violationType)) { + int quantity1 = fixedViolations.get(violationType); + double ratio1 = (double)quantity1 / quantity; + double ratio2 = (double)quantity1 / quantityOfAllFixedViolations; + ratioOfEachFixedViolationType1.put(violationType, ratio1); + ratioOfEachFixedViolationType2.put(violationType, ratio2); + } else { + ratioOfEachFixedViolationType1.put(violationType, 0d); + } + } + + tenFoldsOfAllViolationTypes.put(i, ratioOfEachViolationType); + tenFoldsOfAllFixedViolationTypes1.put(i, ratioOfEachFixedViolationType1); + tenFoldsOfAllFixedViolationTypes2.put(i, ratioOfEachFixedViolationType2); + + + // Testing Data + List testingProjects = readList(files.get(i)); + Map testingViolations = new HashMap<>(); + int testingQuantitifOfAllViolations = readViolationsPerProject(violationTypes, testingViolations, testingProjects, "../FPM_Violations/RQ1/all-leafnodes-per-project-vtype.csv"); + Map testingFixedViolations = new HashMap<>(); + int testingQuantityOfAllFixedViolations = readViolationsPerProject(violationTypes, testingFixedViolations, testingProjects, "../FPM_Violations/RQ1/distinct-fixed-summary-per-project-vtype.csv"); + + Map testingRatioOfEachViolationType = new HashMap<>(); + Map testingRatioOfEachFixedViolationType1 = new HashMap<>(); + Map testingRatioOfEachFixedViolationType2 = new HashMap<>(); + + for (Map.Entry entry : testingViolations.entrySet()) { + String violationType = entry.getKey(); + int quantity = entry.getValue(); + double ratio = (double) quantity / testingQuantitifOfAllViolations; + + testingRatioOfEachViolationType.put(violationType, ratio); + if (testingFixedViolations.containsKey(violationType)) { + int quantity1 = testingFixedViolations.get(violationType); + testingRatioOfEachFixedViolationType1.put(violationType, (double)quantity1 / quantity); + testingRatioOfEachFixedViolationType2.put(violationType, (double)quantity1 / testingQuantityOfAllFixedViolations); + } else { + testingRatioOfEachFixedViolationType1.put(violationType, 0d); + } + } + testingTenFoldsOfAllViolationTypes.put(i, testingRatioOfEachViolationType); + testingTenFoldsOfAllFixedViolationTypes1.put(i, testingRatioOfEachFixedViolationType1); + testingTenFoldsOfAllFixedViolationTypes2.put(i, testingRatioOfEachFixedViolationType2); + } + + outputTenFolds(tenFoldsOfAllViolationTypes, testingTenFoldsOfAllViolationTypes, violationTypes, "../FPM_Violations/RQ1/" + outputPath + "/Ten-fold-all-violation-type-" + topNumber + ".csv"); + outputTenFolds(tenFoldsOfAllFixedViolationTypes1, testingTenFoldsOfAllFixedViolationTypes1, violationTypes, "../FPM_Violations/RQ1/" + outputPath + "/Ten-fold-all-fixed-violation-type1-" + topNumber + ".csv"); + outputTenFolds(tenFoldsOfAllFixedViolationTypes2, testingTenFoldsOfAllFixedViolationTypes2, violationTypes, "../FPM_Violations/RQ1/" + outputPath + "/Ten-fold-all-fixed-violation-type2-" + topNumber + ".csv"); + } + + private static List readTypes(String fileName, int number) throws IOException { + List violationTypes = new ArrayList<>(); + String content = FileHelper.readFile(fileName); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = reader.readLine(); + int counter = 0; + while ((line = reader.readLine()) != null) { + String[] elements = line.split(","); + violationTypes.add(elements[0]); + + if (++counter == number) break; + } + reader.close(); + return violationTypes; + } + + private static void outputTenFolds(Map> ratiosMap, Map> testingMap, List violationTypes, String fileName) { + StringBuilder builder = new StringBuilder("Violatype, 0, 0-R, 0-T, 0-T-R, 1, 1-R, 1-T, 1-T-R, 2, 2-R, 2-T, 2-T-R, 3, 3-R, 3-T, 3-T-R, 4, 4-R, 4-T, 4-T-R, 5, 5-R, 5-T, 5-T-R, 6, 6-R, 6-T, 6-T-R, 7, 7-R, 7-T, 7-T-R, 8, 8-R, 8-T, 8-T-R, 9, 9-R, 9-T, 9-T-R\n"); + + Map> ratiosRankingMap = new HashMap<>(); + Map> testingDataRankingMap = new HashMap<>(); + + for (String type : violationTypes) { + builder.append(type); + for (int i = 0; i < 10; i ++) { + Map ratios = ratiosMap.get(i); + Map testingData = testingMap.get(i); + + Map ratiosRanking; + Map testingDataRanking; + if (!ratiosRankingMap.containsKey(i)) { + ratiosRanking = rankAlarmTypes(ratios); + testingDataRanking = rankAlarmTypes(testingData); + } else { + ratiosRanking = ratiosRankingMap.get(i); + testingDataRanking = testingDataRankingMap.get(i); + } + + + if (ratios.containsKey(type)) { + builder.append(", " + ratios.get(type));//.toString().replace(".", ",") + } else { + builder.append(", 0.0"); + } + if (ratiosRanking.containsKey(type)) { + builder.append(", " + ratiosRanking.get(type)); + } else { + builder.append(", " + (ratiosRanking.size() + 1)); + } + if (testingData.containsKey(type)) { + builder.append(", " + testingData.get(type)); + } else { + builder.append(", 0.0"); + } + if (testingDataRanking.containsKey(type)) { + builder.append(", " + testingDataRanking.get(type)); + } else { + builder.append(", " + (testingDataRanking.size() + 1)); + } + } + builder.append("\n"); + } + FileHelper.outputToFile(fileName, builder, false); + } + + private static Map rankAlarmTypes(Map possibilityMap) { + MapSorter sorter = new MapSorter<>(); + possibilityMap = sorter.sortByValueDescending(possibilityMap); + Map ranking = new HashMap<>(); + int ranker = 0; + double possibility = 0; + for (Map.Entry entry : possibilityMap.entrySet()) { + String alarmType = entry.getKey(); + double value = entry.getValue(); + if (possibility != value) { + ranker = ranking.size() + 1; + } + ranking.put(alarmType, ranker); + } + return ranking; + } + + private static int readViolationsPerProject(List violationTypes, Map violations, List projects, String fileName) throws NumberFormatException, IOException { + int quantityOfAllViolations = 0; + + String content = FileHelper.readFile(fileName); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + while ((line = reader.readLine()) != null) { + String[] elements = line.split(","); + String projectName = elements[0]; + if (projects.contains(projectName)) { + String alarmType = elements[1]; + + if (violationTypes.contains(alarmType)) { + int quantity = Integer.parseInt(elements[2]); + + quantityOfAllViolations += quantity; + if (violations.containsKey(alarmType)) { + violations.put(alarmType, violations.get(alarmType) + quantity); + } else { + violations.put(alarmType, quantity); + } + } + + } + } + reader.close(); + return quantityOfAllViolations; + } + + private static List selectPorjects(List files, int i) { + List projects = new ArrayList<>(); + for (File file : files) { + if (file.getName().equals("Fold_" + i + ".list")) { + continue; + } + + projects.addAll(readList(file)); + } + return projects; + } + + private static List readList(File file) { + List projects = new ArrayList<>(); + + String content = FileHelper.readFile(file); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + try { + while ((line = reader.readLine()) != null) { + projects.add(line); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + return projects; + } + +} diff --git a/src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities3.java b/src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities3.java new file mode 100644 index 0000000..3aabe88 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/statistics/TenFoldPossibilities3.java @@ -0,0 +1,256 @@ +package edu.lu.uni.serval.statistics; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.MapSorter; + +/** + * Fixed violation types are selected by their quantities. + * + * @author kui.liu + * + */ +public class TenFoldPossibilities3 { + + public static void main(String[] args) throws IOException { + rankingInTenFolds("../FPM_Violations/RQ1/TenFolds/", "../FPM_Violations/RQ1/Quantity-per-V-type.csv", "TenFolds-Q"); // ten folds of all projects. +// rankingInTenFolds("../FPM_Violations/RQ1/FixedTenFolds-Q/", "../FPM_Violations/RQ1/Quantity-per-FixedV-Type.csv", "FixedTenFolds-Q"); + } + + public static void rankingInTenFolds(String tenFoldFiles, String typeFile, String outputPath) throws NumberFormatException, IOException { + int topNumber = 70; + List violationTypes = readTypes(typeFile, topNumber); + + List files = FileHelper.getAllFilesInCurrentDiectory(tenFoldFiles, ".list"); + + Map> tenFoldsOfAllViolationTypes = new HashMap<>(); + Map> tenFoldsOfAllFixedViolationTypes1 = new HashMap<>(); + Map> tenFoldsOfAllFixedViolationTypes2 = new HashMap<>(); + Map> testingTenFoldsOfAllViolationTypes = new HashMap<>(); + Map> testingTenFoldsOfAllFixedViolationTypes1 = new HashMap<>(); + Map> testingTenFoldsOfAllFixedViolationTypes2 = new HashMap<>(); + for (int i = 0; i < 10; i ++) { + List projects = selectPorjects(files, i); + + // all violations in these 9 sub-sets. + Map violations = new HashMap<>(); + int quantityOfAllViolations = readViolationsPerProject(violationTypes, violations, projects, "../FPM_Violations/RQ1/all-leafnodes-per-project-vtype.csv"); + Map fixedViolations = new HashMap<>(); + int quantityOfAllFixedViolations = readViolationsPerProject(violationTypes, fixedViolations, projects, "../FPM_Violations/RQ1/distinct-fixed-summary-per-project-vtype.csv"); + + // Ratio of each violation type in all violations. + Map ratioOfEachViolationType = new HashMap<>(); // ratio of each violation type + Map ratioOfEachFixedViolationType1 = new HashMap<>(); // quantity of each fixed violation type / quantity of each violation type. + Map ratioOfEachFixedViolationType2 = new HashMap<>(); // quantity of each fixed violation type / quantityOfAllFixedViolations + + for (Map.Entry entry : violations.entrySet()) { + String violationType = entry.getKey(); + int quantity = entry.getValue(); + double ratio = (double) quantity / quantityOfAllViolations; + ratioOfEachViolationType.put(violationType, ratio); + + if (fixedViolations.containsKey(violationType)) { + int quantity1 = fixedViolations.get(violationType); + double ratio1 = (double)quantity1 / quantity; + double ratio2 = (double)quantity1 / quantityOfAllFixedViolations; + ratioOfEachFixedViolationType1.put(violationType, ratio1); + ratioOfEachFixedViolationType2.put(violationType, ratio2); + } else { + ratioOfEachFixedViolationType1.put(violationType, 0d); + } + } + + tenFoldsOfAllViolationTypes.put(i, ratioOfEachViolationType); + tenFoldsOfAllFixedViolationTypes1.put(i, ratioOfEachFixedViolationType1); + tenFoldsOfAllFixedViolationTypes2.put(i, ratioOfEachFixedViolationType2); + + + // Testing Data + List testingProjects = readList(files.get(i)); + Map testingViolations = new HashMap<>(); + int testingQuantitifOfAllViolations = readViolationsPerProject(violationTypes, testingViolations, testingProjects, "../FPM_Violations/RQ1/all-leafnodes-per-project-vtype.csv"); + Map testingFixedViolations = new HashMap<>(); + int testingQuantityOfAllFixedViolations = readViolationsPerProject(violationTypes, testingFixedViolations, testingProjects, "../FPM_Violations/RQ1/distinct-fixed-summary-per-project-vtype.csv"); + + Map testingRatioOfEachViolationType = new HashMap<>(); + Map testingRatioOfEachFixedViolationType1 = new HashMap<>(); + Map testingRatioOfEachFixedViolationType2 = new HashMap<>(); + + for (Map.Entry entry : testingViolations.entrySet()) { + String violationType = entry.getKey(); + int quantity = entry.getValue(); + double ratio = (double) quantity / testingQuantitifOfAllViolations; + + testingRatioOfEachViolationType.put(violationType, ratio); + if (testingFixedViolations.containsKey(violationType)) { + int quantity1 = testingFixedViolations.get(violationType); + testingRatioOfEachFixedViolationType1.put(violationType, (double)quantity1 / quantity); + testingRatioOfEachFixedViolationType2.put(violationType, (double)quantity1 / testingQuantityOfAllFixedViolations); + } else { + testingRatioOfEachFixedViolationType1.put(violationType, 0d); + } + } + testingTenFoldsOfAllViolationTypes.put(i, testingRatioOfEachViolationType); + testingTenFoldsOfAllFixedViolationTypes1.put(i, testingRatioOfEachFixedViolationType1); + testingTenFoldsOfAllFixedViolationTypes2.put(i, testingRatioOfEachFixedViolationType2); + } + + outputTenFolds(tenFoldsOfAllViolationTypes, testingTenFoldsOfAllViolationTypes, violationTypes, "../FPM_Violations/RQ1/" + outputPath + "/Ten-fold-all-violation-type" + topNumber + ".csv"); + outputTenFolds(tenFoldsOfAllFixedViolationTypes1, testingTenFoldsOfAllFixedViolationTypes1, violationTypes, "../FPM_Violations/RQ1/" + outputPath + "/Ten-fold-all-fixed-violation-type1-" + topNumber + ".csv"); + outputTenFolds(tenFoldsOfAllFixedViolationTypes2, testingTenFoldsOfAllFixedViolationTypes2, violationTypes, "../FPM_Violations/RQ1/" + outputPath + "/Ten-fold-all-fixed-violation-type2-" + topNumber + ".csv"); + } + + private static List readTypes(String fileName, int number) throws IOException { + List violationTypes = new ArrayList<>(); + String content = FileHelper.readFile(fileName); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = reader.readLine(); + int counter = 0; + while ((line = reader.readLine()) != null) { + String[] elements = line.split(","); + violationTypes.add(elements[0]); + + if(++ counter == number) break; + } + reader.close(); + return violationTypes; + } + + private static void outputTenFolds(Map> ratiosMap, Map> testingMap, List violationTypes, String fileName) { + StringBuilder builder = new StringBuilder("Violatype, 0, 0-R, 0-T, 0-T-R, 1, 1-R, 1-T, 1-T-R, 2, 2-R, 2-T, 2-T-R, 3, 3-R, 3-T, 3-T-R, 4, 4-R, 4-T, 4-T-R, 5, 5-R, 5-T, 5-T-R, 6, 6-R, 6-T, 6-T-R, 7, 7-R, 7-T, 7-T-R, 8, 8-R, 8-T, 8-T-R, 9, 9-R, 9-T, 9-T-R\n"); + + Map> ratiosRankingMap = new HashMap<>(); + Map> testingDataRankingMap = new HashMap<>(); + + for (String type : violationTypes) { + builder.append(type); + for (int i = 0; i < 10; i ++) { + Map ratios = ratiosMap.get(i); + Map testingData = testingMap.get(i); + + Map ratiosRanking; + Map testingDataRanking; + if (!ratiosRankingMap.containsKey(i)) { + ratiosRanking = rankAlarmTypes(ratios); + testingDataRanking = rankAlarmTypes(testingData); + } else { + ratiosRanking = ratiosRankingMap.get(i); + testingDataRanking = testingDataRankingMap.get(i); + } + + + if (ratios.containsKey(type)) { + builder.append(", " + ratios.get(type));//.toString().replace(".", ",") + } else { + builder.append(", 0.0"); + } + if (ratiosRanking.containsKey(type)) { + builder.append(", " + ratiosRanking.get(type)); + } else { + builder.append(", " + (ratiosRanking.size() + 1)); + } + if (testingData.containsKey(type)) { + builder.append(", " + testingData.get(type)); + } else { + builder.append(", 0.0"); + } + if (testingDataRanking.containsKey(type)) { + builder.append(", " + testingDataRanking.get(type)); + } else { + builder.append(", " + (testingDataRanking.size() + 1)); + } + } + builder.append("\n"); + } + FileHelper.outputToFile(fileName, builder, false); + } + + private static Map rankAlarmTypes(Map possibilityMap) { + MapSorter sorter = new MapSorter<>(); + possibilityMap = sorter.sortByValueDescending(possibilityMap); + Map ranking = new HashMap<>(); + int ranker = 0; + double possibility = 0; + for (Map.Entry entry : possibilityMap.entrySet()) { + String alarmType = entry.getKey(); + double value = entry.getValue(); + if (possibility != value) { + ranker = ranking.size() + 1; + } + ranking.put(alarmType, ranker); + } + return ranking; + } + + private static int readViolationsPerProject(List violationTypes, Map violations, List projects, String fileName) throws NumberFormatException, IOException { + int quantityOfAllViolations = 0; + + String content = FileHelper.readFile(fileName); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + while ((line = reader.readLine()) != null) { + String[] elements = line.split(","); + String projectName = elements[0]; + if (projects.contains(projectName)) { + String alarmType = elements[1]; + if (violationTypes.contains(alarmType)) { + int quantity = Integer.parseInt(elements[2]); + + quantityOfAllViolations += quantity; + if (violations.containsKey(alarmType)) { + violations.put(alarmType, violations.get(alarmType) + quantity); + } else { + violations.put(alarmType, quantity); + } + } + + } + } + reader.close(); + return quantityOfAllViolations; + } + + private static List selectPorjects(List files, int i) { + List projects = new ArrayList<>(); + for (File file : files) { + if (file.getName().equals("Fold_" + i + ".list")) { + continue; + } + + projects.addAll(readList(file)); + } + return projects; + } + + private static List readList(File file) { + List projects = new ArrayList<>(); + + String content = FileHelper.readFile(file); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + try { + while ((line = reader.readLine()) != null) { + projects.add(line); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + return projects; + } + +} diff --git a/src/main/java/edu/lu/uni/serval/statistics/TenFoldProjs.java b/src/main/java/edu/lu/uni/serval/statistics/TenFoldProjs.java new file mode 100644 index 0000000..f697a47 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/statistics/TenFoldProjs.java @@ -0,0 +1,121 @@ +package edu.lu.uni.serval.statistics; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import edu.lu.uni.serval.utils.FileHelper; + +public class TenFoldProjs { + + public static void main(String[] args) { + List allProjects = readList("../FPM_Violations/RQ1/Quantity-per-Proj.csv"); + randomSeparateProjects(allProjects, "../FPM_Violations/RQ1/TenFolds/"); + List allFixedProjects = readList("../FPM_Violations/RQ1/Quantity-per-FixedProj.csv"); + randomSeparateProjects(allFixedProjects, "../FPM_Violations/RQ1/FixedTenFolds-1/"); + + List selectedProjects = selectFixedProjects("../FPM_Violations/RQ1/Quantity-per-FixedProj.csv"); + randomSeparateProjects(selectedProjects, "../FPM_Violations/RQ1/FixedTenFolds-2/"); + } + + private static void randomSeparateProjects(List projectNames, String outputPath) { + List selectedIndexes = new ArrayList<>(); + + int number = projectNames.size(); + int number2 = (int) Math.round((double) number / 10); + + + Map> map = new HashMap<>(); + + for (int i = 0; i < 9; i ++) { + int counter = 0; + List value = new ArrayList<>(); + + while (counter < number2) { + Random random = new Random(); + int index = random.nextInt(number); + if (!selectedIndexes.contains(index)) { + counter ++; + value.add(projectNames.get(index)); + selectedIndexes.add(index); + + if (selectedIndexes.size() == number) break; + } + } + + map.put(i, value); + } + + List value = new ArrayList<>(); + for (int i = 0; i < number; i ++) { + if (!selectedIndexes.contains(i)) { + value.add(projectNames.get(i)); + } + } + map.put(9, value); + + for (Map.Entry> entry : map.entrySet()) { + int key = entry.getKey(); + List projects = entry.getValue(); + System.out.println(projects.size()); + String pojectsStr = ""; + for (String project : projects) { + pojectsStr += project + "\n"; + } + FileHelper.outputToFile(outputPath + "Fold_" + key + ".list", pojectsStr, false); + } + } + + private static List selectFixedProjects(String fileName) { + List list = new ArrayList<>(); + String fileContent = FileHelper.readFile(fileName); + BufferedReader reader = new BufferedReader(new StringReader(fileContent)); + String line= null; + try { + line = reader.readLine(); + while ((line = reader.readLine()) != null) { + String[] elements = line.split(","); + list.add(elements[0]); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + return list; + } + + private static List readList(String fileName) { + List list = new ArrayList<>(); + String fileContent = FileHelper.readFile(fileName); + BufferedReader reader = new BufferedReader(new StringReader(fileContent)); + String line= null; + try { + line = reader.readLine(); + while ((line = reader.readLine()) != null) { + list.add(line.split(",")[0]);; + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + return list; + } + +}