diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..8796191 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +FixPatternMiner \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..9d2856b --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..aad31c1 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_2.xml b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_2.xml new file mode 100644 index 0000000..a8b0d9c --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_7.xml b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_7.xml new file mode 100644 index 0000000..cdd7959 --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_7.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_2_3.xml b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_2_3.xml new file mode 100644 index 0000000..6fec8f4 --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_2_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_2.xml b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_2.xml new file mode 100644 index 0000000..1c76d0b --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_7.xml b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_7.xml new file mode 100644 index 0000000..6c2a760 --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_7.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_core_1_2_3.xml b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_2_3.xml new file mode 100644 index 0000000..9eb8596 --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_2_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_github_mpkorstanje_simmetrics_core_3_0_3.xml b/.idea/libraries/Maven__com_github_mpkorstanje_simmetrics_core_3_0_3.xml new file mode 100644 index 0000000..a009f87 --- /dev/null +++ b/.idea/libraries/Maven__com_github_mpkorstanje_simmetrics_core_3_0_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml b/.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml new file mode 100644 index 0000000..0e66824 --- /dev/null +++ b/.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_code_gson_gson_2_3.xml b/.idea/libraries/Maven__com_google_code_gson_gson_2_3.xml new file mode 100644 index 0000000..4dcaf15 --- /dev/null +++ b/.idea/libraries/Maven__com_google_code_gson_gson_2_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_guava_guava_11_0_2.xml b/.idea/libraries/Maven__com_google_guava_guava_11_0_2.xml new file mode 100644 index 0000000..01a573a --- /dev/null +++ b/.idea/libraries/Maven__com_google_guava_guava_11_0_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_guava_guava_18_0.xml b/.idea/libraries/Maven__com_google_guava_guava_18_0.xml new file mode 100644 index 0000000..bbd71d7 --- /dev/null +++ b/.idea/libraries/Maven__com_google_guava_guava_18_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_nanohttpd_nanohttpd_2_1_1.xml b/.idea/libraries/Maven__com_nanohttpd_nanohttpd_2_1_1.xml new file mode 100644 index 0000000..7c43448 --- /dev/null +++ b/.idea/libraries/Maven__com_nanohttpd_nanohttpd_2_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_nanohttpd_nanohttpd_webserver_2_1_1.xml b/.idea/libraries/Maven__com_nanohttpd_nanohttpd_webserver_2_1_1.xml new file mode 100644 index 0000000..c265679 --- /dev/null +++ b/.idea/libraries/Maven__com_nanohttpd_nanohttpd_webserver_2_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_rabbitmq_amqp_client_4_0_0.xml b/.idea/libraries/Maven__com_rabbitmq_amqp_client_4_0_0.xml new file mode 100644 index 0000000..0583b43 --- /dev/null +++ b/.idea/libraries/Maven__com_rabbitmq_amqp_client_4_0_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sparkjava_spark_core_2_6_0.xml b/.idea/libraries/Maven__com_sparkjava_spark_core_2_6_0.xml new file mode 100644 index 0000000..a1b274c --- /dev/null +++ b/.idea/libraries/Maven__com_sparkjava_spark_core_2_6_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_typesafe_akka_akka_actor_2_11_2_4_11.xml b/.idea/libraries/Maven__com_typesafe_akka_akka_actor_2_11_2_4_11.xml new file mode 100644 index 0000000..09604f8 --- /dev/null +++ b/.idea/libraries/Maven__com_typesafe_akka_akka_actor_2_11_2_4_11.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_typesafe_config_1_3_0.xml b/.idea/libraries/Maven__com_typesafe_config_1_3_0.xml new file mode 100644 index 0000000..f92ac86 --- /dev/null +++ b/.idea/libraries/Maven__com_typesafe_config_1_3_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_codec_commons_codec_1_10.xml b/.idea/libraries/Maven__commons_codec_commons_codec_1_10.xml new file mode 100644 index 0000000..27424a1 --- /dev/null +++ b/.idea/libraries/Maven__commons_codec_commons_codec_1_10.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml b/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml new file mode 100644 index 0000000..a66d039 --- /dev/null +++ b/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__dom4j_dom4j_1_6_1.xml b/.idea/libraries/Maven__dom4j_dom4j_1_6_1.xml new file mode 100644 index 0000000..14681ee --- /dev/null +++ b/.idea/libraries/Maven__dom4j_dom4j_1_6_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_servlet_javax_servlet_api_3_1_0.xml b/.idea/libraries/Maven__javax_servlet_javax_servlet_api_3_1_0.xml new file mode 100644 index 0000000..c24f7e3 --- /dev/null +++ b/.idea/libraries/Maven__javax_servlet_javax_servlet_api_3_1_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__junit_junit_3_8_1.xml b/.idea/libraries/Maven__junit_junit_3_8_1.xml new file mode 100644 index 0000000..71b2993 --- /dev/null +++ b/.idea/libraries/Maven__junit_junit_3_8_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__junit_junit_4_11.xml b/.idea/libraries/Maven__junit_junit_4_11.xml new file mode 100644 index 0000000..f33320d --- /dev/null +++ b/.idea/libraries/Maven__junit_junit_4_11.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__junit_junit_4_12.xml b/.idea/libraries/Maven__junit_junit_4_12.xml new file mode 100644 index 0000000..d411041 --- /dev/null +++ b/.idea/libraries/Maven__junit_junit_4_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__log4j_log4j_1_2_14.xml b/.idea/libraries/Maven__log4j_log4j_1_2_14.xml new file mode 100644 index 0000000..2825a67 --- /dev/null +++ b/.idea/libraries/Maven__log4j_log4j_1_2_14.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__net_sf_trove4j_trove4j_3_0_3.xml b/.idea/libraries/Maven__net_sf_trove4j_trove4j_3_0_3.xml new file mode 100644 index 0000000..72d4d7e --- /dev/null +++ b/.idea/libraries/Maven__net_sf_trove4j_trove4j_3_0_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__net_sourceforge_jexcelapi_jxl_2_6_12.xml b/.idea/libraries/Maven__net_sourceforge_jexcelapi_jxl_2_6_12.xml new file mode 100644 index 0000000..19e6af8 --- /dev/null +++ b/.idea/libraries/Maven__net_sourceforge_jexcelapi_jxl_2_6_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml new file mode 100644 index 0000000..32bfe3b --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_7.xml b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_7.xml new file mode 100644 index 0000000..28edf49 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_7.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_pool2_2_4_2.xml b/.idea/libraries/Maven__org_apache_commons_commons_pool2_2_4_2.xml new file mode 100644 index 0000000..2970b5e --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_pool2_2_4_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_text_1_3.xml b/.idea/libraries/Maven__org_apache_commons_commons_text_1_3.xml new file mode 100644 index 0000000..2930e62 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_text_1_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_poi_poi_3_12.xml b/.idea/libraries/Maven__org_apache_poi_poi_3_12.xml new file mode 100644 index 0000000..f3bfa63 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_poi_poi_3_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_poi_poi_ooxml_3_12.xml b/.idea/libraries/Maven__org_apache_poi_poi_ooxml_3_12.xml new file mode 100644 index 0000000..0f92377 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_poi_poi_ooxml_3_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_poi_poi_ooxml_schemas_3_12.xml b/.idea/libraries/Maven__org_apache_poi_poi_ooxml_schemas_3_12.xml new file mode 100644 index 0000000..fb871da --- /dev/null +++ b/.idea/libraries/Maven__org_apache_poi_poi_ooxml_schemas_3_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_xmlbeans_xmlbeans_2_6_0.xml b/.idea/libraries/Maven__org_apache_xmlbeans_xmlbeans_2_6_0.xml new file mode 100644 index 0000000..cb0f76d --- /dev/null +++ b/.idea/libraries/Maven__org_apache_xmlbeans_xmlbeans_2_6_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_birt_runtime_org_eclipse_core_resources_3_10_0_v20150423_0755.xml b/.idea/libraries/Maven__org_eclipse_birt_runtime_org_eclipse_core_resources_3_10_0_v20150423_0755.xml new file mode 100644 index 0000000..bec40e7 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_birt_runtime_org_eclipse_core_resources_3_10_0_v20150423_0755.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_core_contenttype_3_4_200_v20140207_1251.xml b/.idea/libraries/Maven__org_eclipse_core_contenttype_3_4_200_v20140207_1251.xml new file mode 100644 index 0000000..f23ea73 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_core_contenttype_3_4_200_v20140207_1251.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_core_jobs_3_6_0_v20140424_0053.xml b/.idea/libraries/Maven__org_eclipse_core_jobs_3_6_0_v20140424_0053.xml new file mode 100644 index 0000000..ecd0a0f --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_core_jobs_3_6_0_v20140424_0053.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_core_runtime_3_10_0_v20140318_2214.xml b/.idea/libraries/Maven__org_eclipse_core_runtime_3_10_0_v20140318_2214.xml new file mode 100644 index 0000000..b1c50eb --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_core_runtime_3_10_0_v20140318_2214.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_equinox_app_1_3_200_v20130910_1609.xml b/.idea/libraries/Maven__org_eclipse_equinox_app_1_3_200_v20130910_1609.xml new file mode 100644 index 0000000..c5372f9 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_equinox_app_1_3_200_v20130910_1609.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_equinox_common_3_6_200_v20130402_1505.xml b/.idea/libraries/Maven__org_eclipse_equinox_common_3_6_200_v20130402_1505.xml new file mode 100644 index 0000000..0e8e398 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_equinox_common_3_6_200_v20130402_1505.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_equinox_preferences_3_5_200_v20140224_1527.xml b/.idea/libraries/Maven__org_eclipse_equinox_preferences_3_5_200_v20140224_1527.xml new file mode 100644 index 0000000..cf2ff45 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_equinox_preferences_3_5_200_v20140224_1527.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_equinox_registry_3_5_400_v20140428_1507.xml b/.idea/libraries/Maven__org_eclipse_equinox_registry_3_5_400_v20140428_1507.xml new file mode 100644 index 0000000..a35d91c --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_equinox_registry_3_5_400_v20140428_1507.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_client_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_client_9_4_4_v20170414.xml new file mode 100644 index 0000000..753cd46 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_client_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_9_4_4_v20170414.xml new file mode 100644 index 0000000..24b9595 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_9_4_4_v20170414.xml new file mode 100644 index 0000000..be4af0a --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_security_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_security_9_4_4_v20170414.xml new file mode 100644 index 0000000..0a0a61a --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_security_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_server_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_server_9_4_4_v20170414.xml new file mode 100644 index 0000000..aa28eb9 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_server_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_servlet_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_servlet_9_4_4_v20170414.xml new file mode 100644 index 0000000..8923ab4 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_servlet_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_9_4_4_v20170414.xml new file mode 100644 index 0000000..b7d325c --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_webapp_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_webapp_9_4_4_v20170414.xml new file mode 100644 index 0000000..bd09afa --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_webapp_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_xml_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_xml_9_4_4_v20170414.xml new file mode 100644 index 0000000..9f8b202 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_xml_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_api_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_api_9_4_4_v20170414.xml new file mode 100644 index 0000000..597060a --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_api_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_client_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_client_9_4_4_v20170414.xml new file mode 100644 index 0000000..be2f9c9 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_client_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_common_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_common_9_4_4_v20170414.xml new file mode 100644 index 0000000..af4b520 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_common_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_server_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_server_9_4_4_v20170414.xml new file mode 100644 index 0000000..26b182a --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_server_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_servlet_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_servlet_9_4_4_v20170414.xml new file mode 100644 index 0000000..d11dc2b --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_servlet_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_osgi_3_10_0_v20140606_1445.xml b/.idea/libraries/Maven__org_eclipse_osgi_3_10_0_v20140606_1445.xml new file mode 100644 index 0000000..4f8664b --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_osgi_3_10_0_v20140606_1445.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_tycho_org_eclipse_jdt_core_3_12_2_v20161117_1814.xml b/.idea/libraries/Maven__org_eclipse_tycho_org_eclipse_jdt_core_3_12_2_v20161117_1814.xml new file mode 100644 index 0000000..152bde1 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_tycho_org_eclipse_jdt_core_3_12_2_v20161117_1814.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml new file mode 100644 index 0000000..f58bbc1 --- /dev/null +++ b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_javassist_javassist_3_16_1_GA.xml b/.idea/libraries/Maven__org_javassist_javassist_3_16_1_GA.xml new file mode 100644 index 0000000..7b4e548 --- /dev/null +++ b/.idea/libraries/Maven__org_javassist_javassist_3_16_1_GA.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_javatuples_javatuples_1_2.xml b/.idea/libraries/Maven__org_javatuples_javatuples_1_2.xml new file mode 100644 index 0000000..b524531 --- /dev/null +++ b/.idea/libraries/Maven__org_javatuples_javatuples_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_reflections_reflections_0_9_9_RC1.xml b/.idea/libraries/Maven__org_reflections_reflections_0_9_9_RC1.xml new file mode 100644 index 0000000..95dd64a --- /dev/null +++ b/.idea/libraries/Maven__org_reflections_reflections_0_9_9_RC1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_rendersnake_rendersnake_1_8.xml b/.idea/libraries/Maven__org_rendersnake_rendersnake_1_8.xml new file mode 100644 index 0000000..7251aaa --- /dev/null +++ b/.idea/libraries/Maven__org_rendersnake_rendersnake_1_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_scala_lang_modules_scala_java8_compat_2_11_0_7_0.xml b/.idea/libraries/Maven__org_scala_lang_modules_scala_java8_compat_2_11_0_7_0.xml new file mode 100644 index 0000000..f4b289b --- /dev/null +++ b/.idea/libraries/Maven__org_scala_lang_modules_scala_java8_compat_2_11_0_7_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_8.xml b/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_8.xml new file mode 100644 index 0000000..fc8460c --- /dev/null +++ b/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_13.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_13.xml new file mode 100644 index 0000000..4fd913d --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_13.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_21.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_21.xml new file mode 100644 index 0000000..1b644dd --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_21.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_25.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_25.xml new file mode 100644 index 0000000..20e8163 --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_25.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml new file mode 100644 index 0000000..1e67260 --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__redis_clients_jedis_2_8_1.xml b/.idea/libraries/Maven__redis_clients_jedis_2_8_1.xml new file mode 100644 index 0000000..f12195b --- /dev/null +++ b/.idea/libraries/Maven__redis_clients_jedis_2_8_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__stax_stax_api_1_0_1.xml b/.idea/libraries/Maven__stax_stax_api_1_0_1.xml new file mode 100644 index 0000000..0b13335 --- /dev/null +++ b/.idea/libraries/Maven__stax_stax_api_1_0_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__xml_apis_xml_apis_1_0_b2.xml b/.idea/libraries/Maven__xml_apis_xml_apis_1_0_b2.xml new file mode 100644 index 0000000..c36e717 --- /dev/null +++ b/.idea/libraries/Maven__xml_apis_xml_apis_1_0_b2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..cc1f0b7 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,15 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..272b874 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..dc408e4 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..19f4360 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,1218 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileHelper + contain + Time + getSub + Filenamef + .keys + write + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1550924800456 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 25 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 83 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java + 26 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java + 33 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 77 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 80 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java + 27 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java + 79 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java + 77 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java + 37 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java + 94 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java + 104 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 72 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java + 85 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 84 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java + 72 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java + 34 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java + 54 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java + 33 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java + 84 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java + 23 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java + 116 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java + 47 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 140 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java + 78 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java + 114 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java + 131 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java + 184 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java + 61 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java + 28 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java + 35 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java + 41 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java + 56 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java + 98 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java + 141 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java + 114 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java + 146 + + + + + + + + + folder.getParent() + JAVA + CODE_FRAGMENT + + + shape.getParent() + JAVA + CODE_FRAGMENT + + + cluster.getName() + JAVA + CODE_FRAGMENT + + + shape.getPath() + JAVA + CODE_FRAGMENT + + + cluster.getPath() + JAVA + CODE_FRAGMENT + + + actionSet.toString() + JAVA + CODE_FRAGMENT + + + file.getName() + JAVA + CODE_FRAGMENT + + + file. + JAVA + CODE_FRAGMENT + + + inner.get(prefix+"-"+firstValue); + JAVA + CODE_FRAGMENT + + + + + 1 + JAVA + EXPRESSION + + + "JDT" + JAVA + EXPRESSION + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + No facets are configured + + + + + + + + + + + + + + + 1.8 + + + + + + + + client + + + + + + + + 1.8 + + + + + + + + Maven: ch.qos.logback:logback-classic:1.1.2 + + + + + + + + \ No newline at end of file diff --git a/FixPatternMiner.iml b/FixPatternMiner.iml index a7d8955..82122fc 100644 --- a/FixPatternMiner.iml +++ b/FixPatternMiner.iml @@ -1,6 +1,6 @@ - + @@ -10,7 +10,17 @@ + + + + + + + + + + @@ -18,18 +28,9 @@ - - - - - - - - - diff --git a/difffile b/difffile new file mode 100644 index 0000000..4dc952b --- /dev/null +++ b/difffile @@ -0,0 +1,210 @@ +diff --git a/.gitignore b/.gitignore +index 24cae48..27c7ba3 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -14,6 +14,7 @@ GumTreeInput/ + GumTreeInput1/ + GumTreeOutput1/ + GumTreeResults/ ++.idea/* + + */Test.java + *TestParser.java +diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +index 4d885ea..eb1eca1 100644 +--- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java ++++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +@@ -1,5 +1,9 @@ + package edu.lu.uni.serval.FixPatternParser.violations; + ++import com.github.gumtreediff.actions.model.Delete; ++import com.github.gumtreediff.actions.model.Insert; ++import com.github.gumtreediff.actions.model.Move; ++import com.github.gumtreediff.actions.model.Update; + import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; + + import java.io.*; +@@ -30,27 +34,29 @@ public class FixedViolationHunkParser extends FixedViolationParser { + public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project) { + List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); + if (actionSets.size() != 0) { +-// boolean isUpdate = +-// actionSets.stream().allMatch(p -> p.getAction() instanceof Update); +-// boolean isInsert = +-// actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); +-// boolean isDelete = +-// actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); +-// boolean isMove = +-// actionSets.stream().allMatch(p -> p.getAction() instanceof Move); + int hunkSet = 0; + // if (isUpdate || isInsert || isDelete || isMove) { + for (HierarchicalActionSet actionSet : actionSets) { +- String folder = "/ALL/"; +-// if (isUpdate) { +-// folder = "/UPD/"; +-// } else if (isDelete) { +-// folder = "/DEL/"; +-// } else if (isInsert) { +-// folder = "/INS/"; +-// } else if (isMove) { +-// folder = "/MOV/"; +-// } ++ boolean isUpdate = ++ actionSets.stream().allMatch(p -> p.getAction() instanceof Update); ++ boolean isInsert = ++ actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); ++ boolean isDelete = ++ actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); ++ boolean isMove = ++ actionSets.stream().allMatch(p -> p.getAction() instanceof Move); ++ String folder = "/MIX/"; ++ if (isUpdate) { ++ folder = "/UPD/"; ++ } else if (isDelete) { ++ folder = "/DEL/"; ++ } else if (isInsert) { ++ folder = "/INS/"; ++ } else if (isMove) { ++ folder = "/MOV/"; ++ } else{ ++ folder = "/MIX/"; ++ } + + + FileOutputStream f = null; +diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java +index 3084dbe..ffe1b01 100644 +--- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java ++++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java +@@ -61,7 +61,7 @@ public class ParseFixPatternWorker extends UntypedActor { + int testInfos = 0; + int timeouts = 0; + StringBuilder builder = new StringBuilder(); +- ++// log.info("Work queue size " + files.size()); + for (MessageFile msgFile : files) { + File revFile = msgFile.getRevFile(); + File prevFile = msgFile.getPrevFile(); +diff --git a/src/main/java/edu/lu/uni/serval/config/Configuration.java b/src/main/java/edu/lu/uni/serval/config/Configuration.java +index 4421c72..7162180 100644 +--- a/src/main/java/edu/lu/uni/serval/config/Configuration.java ++++ b/src/main/java/edu/lu/uni/serval/config/Configuration.java +@@ -2,6 +2,6 @@ package edu.lu.uni.serval.config; + + public class Configuration { + +- public static final long SECONDS_TO_WAIT = 900L; ++ public static final long SECONDS_TO_WAIT = 90000L; + + } +diff --git a/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java +index 43895e0..066b07d 100644 +--- a/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java ++++ b/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java +@@ -5,7 +5,7 @@ import akka.actor.ActorSystem; + import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; + import edu.lu.uni.serval.MultipleThreadsParser.ParseFixPatternActor; + import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; +-import edu.lu.uni.serval.utils.FileHelper; ++import edu.lu.uni.serval.FixPattern.utils.FileHelper; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + +@@ -13,6 +13,7 @@ import java.io.File; + import java.util.ArrayList; + import java.util.Arrays; + import java.util.List; ++import java.util.Locale; + import java.util.stream.Collectors; + import java.util.stream.Stream; + +@@ -31,6 +32,7 @@ public class EnhancedASTDiff { + Stream stream = Arrays.stream(listOfFiles); + List folders = stream + .filter(x -> !x.getName().startsWith(".")) ++// .filter(x -> x.getName().startsWith("JDT")) + .collect(Collectors.toList()); + + +@@ -49,11 +51,11 @@ public class EnhancedASTDiff { + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); +- FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ALL"); ++ FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MIX"); + + + int a = 0; +- ++ log.info("Work queue size " + msgFiles.size()); + ActorSystem system = null; + ActorRef parsingActor = null; + final WorkMessage msg = new WorkMessage(0, msgFiles); +@@ -80,6 +82,11 @@ public class EnhancedASTDiff { + if (revFiles.length >= 0) { + for (File revFile : revFiles) { + String fileName = revFile.getName(); ++ if(fileName.toLowerCase(Locale.ENGLISH).contains("test")){ ++ log.info(fileName); ++ continue; ++ } ++ + File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file + fileName = fileName.replace(".java", ".txt"); + File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file +diff --git a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java +index 44575a7..88f36c6 100644 +--- a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java ++++ b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java +@@ -5,7 +5,7 @@ import akka.actor.ActorSystem; + import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; + import edu.lu.uni.serval.MultipleThreadsParser.ParseFixPatternActor; + import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; +-import edu.lu.uni.serval.utils.FileHelper; ++import edu.lu.uni.serval.FixPattern.utils.FileHelper; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + +diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java +index 9c1d05d..3dceda0 100644 +--- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java ++++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java +@@ -37,7 +37,7 @@ public class CallShell { + } + // Thread.sleep(Integer.valueOf(serverWait)); + +- String cmd = "redis-cli -p %s ping"; ++ String cmd = "bash source activate redisEnv && redis-cli -p %s ping"; + String cmd1 = String.format(cmd,Integer.valueOf(port)); + runPing(cmd1); + +diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java +index 39bba51..8657dce 100644 +--- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java ++++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java +@@ -52,13 +52,15 @@ public class StoreEDiffInCache { + .filter(x -> x.getName().startsWith("UPD") || + x.getName().startsWith("INS") || + x.getName().startsWith("DEL") || ++ x.getName().startsWith("MIX") || + x.getName().startsWith("MOV")) + .collect(Collectors.toList()); + File[] files1 = fs.get(0).listFiles(); + File[] files2 = fs.get(1).listFiles(); + File[] files3 = fs.get(2).listFiles(); + File[] files4 = fs.get(3).listFiles(); +- dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new); ++ File[] files5 = fs.get(4).listFiles(); ++ dumps = Stream.of(files1, files2, files3,files4,files5).flatMap(Stream::of).toArray(File[]::new); + }else{ + fs = fileStream + .filter(x -> x.getName().startsWith(operation)) +@@ -70,7 +72,7 @@ public class StoreEDiffInCache { + for (File f : dumps) { + String name = f.getName(); + +- String key = pjName + "/"+ operation+"/" + name; ++ String key = pjName + f.getPath().replace(pj.getPath(),""); + String result = key +","+f.getPath(); + workList.add(result); + } diff --git a/pom.xml b/pom.xml index 20628f7..e783f0d 100644 --- a/pom.xml +++ b/pom.xml @@ -25,11 +25,11 @@ 1.2 - - - - - + + edu.lu.uni + simple-utils + 0.0.1-SNAPSHOT + @@ -135,7 +135,7 @@ - edu.lu.uni.serval.fixminer.Launcher + edu.lu.uni.serval.fixminer.akka.compare.CompareTrees diff --git a/src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiff.java b/src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiff.java deleted file mode 100644 index acec7ff..0000000 --- a/src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiff.java +++ /dev/null @@ -1,98 +0,0 @@ -package edu.lu.uni.serval.FixPattern.utils; - -import com.github.gumtreediff.actions.model.*; -import com.github.gumtreediff.tree.ITree; -import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; - -import java.io.IOException; -import java.util.List; - -/** - * Created by anilkoyuncu on 17/09/2018. - */ -public class EDiff { - private static Logger log = LoggerFactory.getLogger(EDiff.class); - public static ITree getSimpliedTree(String fn, JedisPool outerPool) { - - ITree tree = null; - Jedis inner = null; - try { - inner = outerPool.getResource(); - while (!inner.ping().equals("PONG")){ - log.info("wait"); - } - inner.select(1); - String dist2load = inner.get(fn); - inner.select(0); - String s = inner.get(dist2load); - HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); - - ITree parent = null; - ITree children =null; - TreeContext tc = new TreeContext(); - tree = EDiffHelper.getASTTree(actionSet, parent, children,tc); - tree.setParent(null); - tc.validate(); - - } catch (IOException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - }finally { - if (inner != null) { - inner.close(); - } - } - return tree; - - } - public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){ - - int newType = 0; - - Action action = actionSet.getAction(); - if (action instanceof Update){ - newType = 101; - }else if(action instanceof Insert){ - newType =100; - }else if(action instanceof Move){ - newType = 102; - }else if(action instanceof Delete){ - newType=103; - }else{ - new Exception("unknow action"); - } - if(actionSet.getParent() == null){ - //root - - parent = tc.createTree(newType, "", null); - tc.setRoot(parent); - -// parent = new Tree(newType,""); - }else{ - children = tc.createTree(newType, "", null); - children.setParentAndUpdateChildren(parent); -// children = new Tree(newType,""); -// parent.addChild(children); - } - List subActions = actionSet.getSubActions(); - if (subActions.size() != 0){ - for (HierarchicalActionSet subAction : subActions) { - - if(actionSet.getParent() == null){ - children = parent; - } - getActionTree(subAction,children,null,tc); - - } - - - } - return parent; - } -} diff --git a/src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiffHelper.java b/src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiffHelper.java deleted file mode 100644 index b6f7980..0000000 --- a/src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiffHelper.java +++ /dev/null @@ -1,88 +0,0 @@ -package edu.lu.uni.serval.FixPattern.utils; - -import com.github.gumtreediff.tree.ITree; -import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * Created by anilkoyuncu on 17/09/2018. - */ -public class EDiffHelper { - private static Logger log = LoggerFactory.getLogger(EDiffHelper.class); - /** Read the object from Base64 string. */ - public static Object fromString( String s ) throws IOException, - ClassNotFoundException { - byte [] data = Base64.getDecoder().decode( s ); - ObjectInputStream ois = new ObjectInputStream( - new ByteArrayInputStream( data ) ); - Object o = ois.readObject(); - ois.close(); - return o; - } - - /** Write the object to a Base64 string. */ - public static String toString( Serializable o ) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream( baos ); - oos.writeObject( o ); - oos.close(); - return Base64.getEncoder().encodeToString(baos.toByteArray()); - } - - public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children, TreeContext tc){ - - int newType = 0; - - String astNodeType = actionSet.getAstNodeType(); - List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); - - if(keysByValue.size() != 1){ - log.error("More than 1"); - } - newType = keysByValue.get(0); - if(actionSet.getParent() == null){ - //root - -// parent = new Tree(newType,""); - parent = tc.createTree(newType, "", null); - tc.setRoot(parent); - }else{ -// children = new Tree(newType,""); -// parent.addChild(children); - children = tc.createTree(newType, "", null); - children.setParentAndUpdateChildren(parent); - } - List subActions = actionSet.getSubActions(); - if (subActions.size() != 0){ - for (HierarchicalActionSet subAction : subActions) { - - if(actionSet.getParent() == null){ - children = parent; - } - getASTTree(subAction,children,null,tc); - - } - - - } - return parent; - } - - - public static List getKeysByValue(Map map, E value) { - return map.entrySet() - .stream() - .filter(entry -> Objects.equals(entry.getValue(), value)) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - } -} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/CUCreator.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/CUCreator.java deleted file mode 100644 index ec47e1a..0000000 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/CUCreator.java +++ /dev/null @@ -1,55 +0,0 @@ -package edu.lu.uni.serval.FixPatternParser; - -import org.eclipse.jdt.core.dom.AST; -import org.eclipse.jdt.core.dom.ASTParser; -import org.eclipse.jdt.core.dom.CompilationUnit; - -import java.io.*; - -/** - * Creator of a CompilationUnit. - * - * @author kui.liu - * - */ -public class CUCreator { - - public CompilationUnit createCompilationUnit(File javaFile) { - CompilationUnit unit = null; - try { - char[] sourceCode = readFileToCharArray(new FileReader(javaFile)); - ASTParser parser = createASTParser(sourceCode); - parser.setKind(ASTParser.K_COMPILATION_UNIT); - unit = (CompilationUnit) parser.createAST(null); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - - return unit; - } - - private ASTParser createASTParser(char[] javaCode) { - ASTParser parser = ASTParser.newParser(AST.JLS8); - parser.setSource(javaCode); - - return parser; - } - - private char[] readFileToCharArray(FileReader fileReader) throws IOException { - StringBuilder fileData = new StringBuilder(); - BufferedReader br = new BufferedReader(fileReader); - - char[] buf = new char[10]; - int numRead = 0; - while ((numRead = br.read(buf)) != -1) { - String readData = String.valueOf(buf, 0, numRead); - fileData.append(readData); - buf = new char[1024]; - } - br.close(); - - return fileData.toString().toCharArray(); - } -} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java deleted file mode 100644 index 4d885ea..0000000 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +++ /dev/null @@ -1,87 +0,0 @@ -package edu.lu.uni.serval.FixPatternParser.violations; - -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; - -import java.io.*; -import java.util.List; - - - -/** - * Parse fix violations with GumTree in terms of multiple statements. - * - * @author kui.liu - * - */ -public class FixedViolationHunkParser extends FixedViolationParser { - - public String testingInfo = ""; - - public int nullMappingGumTreeResult = 0; - public int pureDeletions = 0; - public int largeHunk = 0; - public int nullSourceCode = 0; - public int nullMatchedDiffEntry = 0; - public int testInfos = 0; - - public String unfixedViolations = ""; - - @Override - public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project) { - List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); - if (actionSets.size() != 0) { -// boolean isUpdate = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Update); -// boolean isInsert = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); -// boolean isDelete = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); -// boolean isMove = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Move); - int hunkSet = 0; -// if (isUpdate || isInsert || isDelete || isMove) { - for (HierarchicalActionSet actionSet : actionSets) { - String folder = "/ALL/"; -// if (isUpdate) { -// folder = "/UPD/"; -// } else if (isDelete) { -// folder = "/DEL/"; -// } else if (isInsert) { -// folder = "/INS/"; -// } else if (isMove) { -// folder = "/MOV/"; -// } - - - FileOutputStream f = null; - - try { -// String pj = diffentryFile.getParent().split("Defects4J")[1]; - String datasetName = project; - String[] split1 = diffentryFile.getParent().split(datasetName); - String root = split1[0]; - String pj = split1[1].split("/")[1]; - - String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/" + pj + folder + diffentryFile.getName() + "_" + String.valueOf(hunkSet); - f = new FileOutputStream(new File(hunkTreeFileName)); - ObjectOutputStream o = new ObjectOutputStream(f); - o.writeObject(actionSet); - - o.close(); - f.close(); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - hunkSet++; - } - -// } - } - - } - - - -} diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java deleted file mode 100644 index 3084dbe..0000000 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java +++ /dev/null @@ -1,139 +0,0 @@ -package edu.lu.uni.serval.MultipleThreadsParser; - -import akka.actor.Props; -import akka.actor.UntypedActor; -import akka.japi.Creator; -import edu.lu.uni.serval.FixPatternParser.RunnableParser; -import edu.lu.uni.serval.FixPatternParser.violations.FixedViolationHunkParser; -import edu.lu.uni.serval.config.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.List; -import java.util.concurrent.*; - -public class ParseFixPatternWorker extends UntypedActor { - private static Logger log = LoggerFactory.getLogger(ParseFixPatternActor.class); - - private String project; - - - public ParseFixPatternWorker(String project) { - this.project = project; - } - - public static Props props(final String project) { - return Props.create(new Creator() { - - private static final long serialVersionUID = -7615153844097275009L; - - @Override - public ParseFixPatternWorker create() throws Exception { - return new ParseFixPatternWorker(project); - } - - }); - } - - @Override - public void onReceive(Object message) throws Exception { - if (message instanceof WorkMessage) { - WorkMessage msg = (WorkMessage) message; - List files = msg.getMsgFiles(); - StringBuilder editScripts = new StringBuilder(); - StringBuilder patchesSourceCode = new StringBuilder(); - StringBuilder sizes = new StringBuilder(); - StringBuilder tokens = new StringBuilder(); - StringBuilder testingInfo = new StringBuilder(); - - int id = msg.getId(); - int counter = 0; - - int nullGumTreeResults = 0; - int noSourceCodeChanges = 0; - int noStatementChanges = 0; - int nullDiffEntry = 0; - int nullMappingGumTreeResults = 0; - int pureDeletion = 0; - int largeHunk = 0; - int nullSourceCode = 0; - int testInfos = 0; - int timeouts = 0; - StringBuilder builder = new StringBuilder(); - - for (MessageFile msgFile : files) { - File revFile = msgFile.getRevFile(); - File prevFile = msgFile.getPrevFile(); - File diffentryFile = msgFile.getDiffEntryFile(); - - - FixedViolationHunkParser parser = new FixedViolationHunkParser(); - - final ExecutorService executor = Executors.newSingleThreadExecutor(); - // schedule the work - final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project)); - try { - // wait for task to complete - future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); - - nullDiffEntry += parser.nullMatchedDiffEntry; - nullMappingGumTreeResults += parser.nullMappingGumTreeResult; - pureDeletion += parser.pureDeletions; - largeHunk += parser.largeHunk; - nullSourceCode += parser.nullSourceCode; - testInfos += parser.testInfos; - testingInfo.append(parser.testingInfo); - builder.append(parser.unfixedViolations); - - String editScript = parser.getAstEditScripts(); - if ("".equals(editScript)) { - - } else { - editScripts.append(editScript); - patchesSourceCode.append(parser.getPatchesSourceCode()); - sizes.append(parser.getSizes()); - tokens.append(parser.getTokensOfSourceCode()); - - counter ++; - if (counter % 100 == 0) { - editScripts.setLength(0); - patchesSourceCode.setLength(0); - sizes.setLength(0); - tokens.setLength(0); - log.info("Worker #" + id +" finialized parsing " + counter + " files..."); - testingInfo.setLength(0); - } - } - } catch (TimeoutException e) { - future.cancel(true); - System.err.println("#Timeout: " + revFile.getName()); - } catch (InterruptedException e) { - System.err.println("#TimeInterrupted: " + revFile.getName()); - e.printStackTrace(); - } catch (ExecutionException e) { - System.err.println("#TimeAborted: " + revFile.getName()); - e.printStackTrace(); - } finally { - executor.shutdownNow(); - } - } - - if (sizes.length() > 0) { - editScripts.setLength(0); - patchesSourceCode.setLength(0); - sizes.setLength(0); - tokens.setLength(0); - - testingInfo.setLength(0); - } - - log.info("Worker #" + id +"finialized parsing " + counter + " files..."); - log.info("Worker #" + id + " finialized the work..."); - this.getSender().tell("STOP", getSelf()); - } else { - unhandled(message); - } - } - -} diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/WorkMessage.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/WorkMessage.java deleted file mode 100644 index 9d30eb6..0000000 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/WorkMessage.java +++ /dev/null @@ -1,24 +0,0 @@ -package edu.lu.uni.serval.MultipleThreadsParser; - -import java.util.List; - -public class WorkMessage { - - private int id; - private List msgFiles; - - public WorkMessage(int id, List msgFiles) { - super(); - this.id = id; - this.msgFiles = msgFiles; - } - - public int getId() { - return id; - } - - public List getMsgFiles() { - return msgFiles; - } - -} \ No newline at end of file diff --git a/src/main/java/edu/lu/uni/serval/config/Configuration.java b/src/main/java/edu/lu/uni/serval/config/Configuration.java deleted file mode 100644 index 4421c72..0000000 --- a/src/main/java/edu/lu/uni/serval/config/Configuration.java +++ /dev/null @@ -1,7 +0,0 @@ -package edu.lu.uni.serval.config; - -public class Configuration { - - public static final long SECONDS_TO_WAIT = 900L; - -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java deleted file mode 100644 index 43895e0..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java +++ /dev/null @@ -1,101 +0,0 @@ -package edu.lu.uni.serval.fixminer; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; -import edu.lu.uni.serval.MultipleThreadsParser.ParseFixPatternActor; -import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; -import edu.lu.uni.serval.utils.FileHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class EnhancedASTDiff { - - private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class); - - public static void main(String inputPath, String outputPath,String numOfWorkers,String project) { - - - String parameters = String.format("\nInput path %s \nOutput path %s",inputPath,outputPath); - log.info(parameters); - - File folder = new File(inputPath); - File[] listOfFiles = folder.listFiles(); - Stream stream = Arrays.stream(listOfFiles); - List folders = stream - .filter(x -> !x.getName().startsWith(".")) - .collect(Collectors.toList()); - - - for (File target : folders) { - String pjName = target.getName(); - - - final List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" - System.out.println(msgFiles.size()); - if(msgFiles.size() == 0) - continue; - - String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; - - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ALL"); - - - int a = 0; - - ActorSystem system = null; - ActorRef parsingActor = null; - final WorkMessage msg = new WorkMessage(0, msgFiles); - try { - log.info("Akka begins..."); - system = ActorSystem.create("Mining-FixPattern-System"); - - parsingActor = system.actorOf(ParseFixPatternActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor"); - parsingActor.tell(msg, ActorRef.noSender()); - } catch (Exception e) { - system.shutdown(); - e.printStackTrace(); - } - - } - } - - - private static List getMessageFiles(String gumTreeInput) { - String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile - File revFilesPath = new File(inputPath + "revFiles/"); - File[] revFiles = revFilesPath.listFiles(); // project folders - List msgFiles = new ArrayList<>(); - if (revFiles.length >= 0) { - for (File revFile : revFiles) { - String fileName = revFile.getName(); - File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file - fileName = fileName.replace(".java", ".txt"); - File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file - - MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); - - msgFiles.add(msgFile); - - } - - return msgFiles; - } - else{ - return null; - } - } - - -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java index d4cf22b..79e1c22 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java @@ -1,6 +1,7 @@ package edu.lu.uni.serval.fixminer; -import edu.lu.uni.serval.fixminer.cluster.*; +import edu.lu.uni.serval.fixminer.jobs.*; +import edu.lu.uni.serval.utils.CallShell; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,34 +27,36 @@ public class Launcher { appProps.load(new FileInputStream(appConfigPath)); String portInner = appProps.getProperty("portInner","6380"); -// String serverWait = appProps.getProperty("serverWait", "50000"); String numOfWorkers = appProps.getProperty("numOfWorkers", "10"); String jobType = appProps.getProperty("jobType","ALL"); String portDumps = appProps.getProperty("portDumps","6399"); String pythonPath = appProps.getProperty("pythonPath","/Users/anilkoyuncu/bugStudy/code/python"); String datasetPath = appProps.getProperty("datasetPath","/Users/anilkoyuncu/bugStudy/dataset"); String pjName = appProps.getProperty("pjName","allDataset"); -// String dbNo = appProps.getProperty("dbNo","0"); String actionType = appProps.getProperty("actionType","ALL"); String threshold = appProps.getProperty("threshold","1"); String cursor = appProps.getProperty("cursor","10000000"); String chunk = appProps.getProperty("chunk","1.txt"); + String eDiffTimeout = appProps.getProperty("eDiffTimeout","900"); + String isBig = appProps.getProperty("isBigPair","true"); + String parallelism = appProps.getProperty("parallelism","FORKJOIN"); + boolean isBigPair = Boolean.parseBoolean(isBig); String parameters = String.format("\nportInner %s " + "\nnumOfWorkers %s " + "\njobType %s \nport %s " + "\npythonPath %s \ndatasetPath %s" + - "\npjName %s \nactionType %s \nthreshold %s \ncursor %s" - , portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor); + "\npjName %s \nactionType %s \nthreshold %s \ncursor %s \neDiffTimeout %s \nisBigPair %s \nparallelism %s" + , portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,eDiffTimeout,isBig,parallelism); log.info(parameters); - mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk); + mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk,eDiffTimeout,isBigPair,parallelism); } - public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk){ + public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk, String eDiffTimeout, boolean isBigPair, String parallelism){ String dbDir; @@ -65,35 +68,37 @@ public class Launcher { gumInput = datasetPath +"/"+pjName+"/"; gumOutput = datasetPath + "/EnhancedASTDiff" + pjName; dbDir = datasetPath + "/redis"; - pairsPath = datasetPath + "/pairsImport"+pjName; + pairsPath = datasetPath + "/pairs"; dumpsName = "dumps-"+pjName+".rdb"; + int iCursor = Integer.valueOf(cursor); try { switch (jobType) { case "ENHANCEDASTDIFF": - EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName); + EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType,parallelism); break; case "CACHE": StoreEDiffInCache.main(gumOutput, portDumps, dbDir, actionType+dumpsName,actionType); break; case "SI": - CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType); - ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath); +// CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor); + ImportPairs2DB.main(pairsPath, portInner, dbDir,datasetPath,chunk); break; case "SIMI": - AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,cursor,chunk); + AkkaTreeLoader.main(portInner, dbDir, "pairs.rdb" , portDumps, actionType+dumpsName,pairsPath,numOfWorkers,iCursor,chunk,eDiffTimeout,parallelism); break; case "LEVEL1": - level1(portInner, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput); + String indexFile = pairsPath+actionType + "/" +pjName+actionType+".index"; + level1(portInner, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, gumInput,iCursor,indexFile); break; //CALC python abstractPatch.py to from cluster folder case "LEVEL2": - level2(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); + level2(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput,iCursor); break; //CALC via python case "LEVEL3": - level3(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); + level3(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput,iCursor); break; case "EXTRACTPATTERN": PatternExtractor.mainLaunch(portInner,numOfWorkers,jobType,portDumps,pythonPath,datasetPath,pjName,actionType,threshold); @@ -111,17 +116,17 @@ public class Launcher { } - private static void level1(String portInner, String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String pairsPath, String dumpsName, String gumInput) throws Exception { + private static void level1(String portInner, String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String gumInput,int cursor,String indexFile) throws Exception { - TreeLoaderClusterL1.main(portInner, port, dbDir, "level1-"+pjName+ actionType+".rdb", dbDir ,pjName + actionType); + TreeLoaderClusterL1.main(portInner, port, dbDir, "level1-"+pjName+ actionType+".rdb", dbDir ,pjName + actionType,cursor); CallShell cs1 =new CallShell(); String db1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String db11 = String.format(db1, dbDir,"level1-"+pjName+ actionType+".rdb" ,Integer.valueOf(port)); cs1.runShell(db11, port); - String runpy = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s"; - String formatRunPy = String.format(runpy,pythonPath +"/abstractPatch.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, "matches" + pjName + actionType, threshold); + String runpy = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s"; + String formatRunPy = String.format(runpy,pythonPath +"/abstractPatch.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, "matches" + pjName + actionType, threshold,indexFile); cs1.runShell(formatRunPy); String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; @@ -129,18 +134,18 @@ public class Launcher { cs1.runShell(stopServer, port); } - private static void level2(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput) throws Exception { + private static void level2(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput,int cursor) throws Exception { String stopServer; - MultiThreadTreeLoaderCluster.calculatePairsOfClusters(datasetPath + "/cluster"+pjName+ actionType, datasetPath,actionType); +// MultiThreadTreeLoaderCluster.calculatePairsOfClusters(datasetPath + "/cluster"+pjName+ actionType, datasetPath,actionType); - MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath+"/pairs"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType); + MultiThreadTreeLoaderCluster.mainCompare(port, datasetPath+"/pairsAction", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6380",actionType,cursor); CallShell cs3 =new CallShell(); String db22 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String db1b = String.format(db22, dbDir,"clusterl1-"+pjName+actionType+".rdb",Integer.valueOf(port)); cs3.runShell(db1b, port); - String runpy2 = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s"; - String formatRunPy1a = String.format(runpy2,pythonPath +"/abstractPatchCluster.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold); + String runpy2 = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s"; + String formatRunPy1a = String.format(runpy2,pythonPath +"/abstractPatchCluster.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold,"dummy"); cs3.runShell(formatRunPy1a); String stopServer1a = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; @@ -148,18 +153,18 @@ public class Launcher { cs3.runShell(stopServer, port); } - private static void level3(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput) throws Exception { + private static void level3(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput,int cursor) throws Exception { String stopServer; MultiThreadTreeLoaderCluster3.calculatePairsOfClusters(datasetPath + "/cluster-2l"+pjName+ actionType, datasetPath,actionType); - MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath+"/pairs-2l"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType); + MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath+"/pairs-2l"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType,cursor); CallShell cs5 =new CallShell(); String dba = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String dbaa = String.format(dba, dbDir,"clusterl2-"+pjName+actionType+".rdb",Integer.valueOf(port)); cs5.runShell(dbaa, port); - String runpya = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s"; - String formatRunPya = String.format(runpya,pythonPath +"/abstractPatchClusterLevel3.py", gumInput, datasetPath + "/cluster-3l"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold); + String runpya = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s"; + String formatRunPya = String.format(runpya,pythonPath +"/abstractPatchClusterLevel3.py", gumInput, datasetPath + "/cluster-3l"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold,"dummy"); cs5.runShell(formatRunPya); String stopServera = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java deleted file mode 100644 index 44575a7..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java +++ /dev/null @@ -1,239 +0,0 @@ -package edu.lu.uni.serval.fixminer; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; -import edu.lu.uni.serval.MultipleThreadsParser.ParseFixPatternActor; -import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; -import edu.lu.uni.serval.utils.FileHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class TestHunkParserSingleFile { - - private static Logger log = LoggerFactory.getLogger(TestHunkParserSingleFile.class); - public static void main(String[] args) { - // input data - -// String rootPath = "/Users/anilkoyuncu/bugStudy"; - String inputPath; - String outputPath; - if(args.length > 0){ - inputPath = args[1]; - outputPath = args[0]; - }else{ -// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4"; - inputPath = "/Users/anilkoyuncu/bugStudy/dataset/allDataset"; -// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/"; - outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle"; - } - - //5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL - - File folder = new File(inputPath); - File[] listOfFiles = folder.listFiles(); - Stream stream = Arrays.stream(listOfFiles); - List folders = stream - .filter(x -> !x.getName().startsWith(".")) - - .collect(Collectors.toList()); - -// List targetList = new ArrayList(); -// for (File f:folders){ -// for(File f1 :f.listFiles()){ -// if (!f1.getName().startsWith(".")){ -// targetList.add(f1); -// } -// } -// } - -// List pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT"); - List files = new ArrayList(); -// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java"); - files.add("d6c5e5_9f96d9_hbase-server#src#main#java#org#apache#hadoop#hbase#master#RegionStates.java"); - for(String f : files){ - String pjName = "HBASE"; -// for (File target : folders) { -// String pjName = target.getName(); -// if (!pjList.contains(pjName)){ -// continue; -// } - -// final List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" - MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f); - - List msgFiles = new ArrayList<>(); - msgFiles.add(messageFile); - String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; - final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list"; - final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list"; - final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list"; - final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv"; - final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list"; - - - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps"); - FileHelper.deleteDirectory(editScriptsFilePath); - FileHelper.deleteDirectory(patchesSourceCodeFilePath); - FileHelper.deleteDirectory(buggyTokensFilePath); - FileHelper.deleteDirectory(editScriptSizesFilePath); - FileHelper.deleteDirectory(alarmTypesFilePath); - - StringBuilder astEditScripts = new StringBuilder(); - StringBuilder tokens = new StringBuilder(); - StringBuilder sizes = new StringBuilder(); - StringBuilder patches = new StringBuilder(); - StringBuilder alarmTypes = new StringBuilder(); - -// int a = 0; - - ActorSystem system = null; - ActorRef parsingActor = null; - final WorkMessage msg = new WorkMessage(0, msgFiles); - try { - log.info("Akka begins..."); - system = ActorSystem.create("Mining-FixPattern-System"); - parsingActor = system.actorOf(ParseFixPatternActor.props(1, "dataset"), "mine-fix-pattern-actor"); - parsingActor.tell(msg, ActorRef.noSender()); - } catch (Exception e) { - system.shutdown(); - e.printStackTrace(); - } - -// int counter = 0; -// for (MessageFile msgFile : msgFiles) { -// FixedViolationHunkParser parser = new FixedViolationHunkParser(); -// -// final ExecutorService executor = Executors.newSingleThreadExecutor(); -// // schedule the work -// final Future future = executor.submit(new RunnableParser(msgFile.getPrevFile(), -// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser)); -// try { -// // where we wait for task to complete -// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); -// String editScripts = parser.getAstEditScripts(); -// if (!editScripts.equals("")) { -// astEditScripts.append(editScripts); -// tokens.append(parser.getTokensOfSourceCode()); -// sizes.append(parser.getSizes()); -// patches.append(parser.getPatchesSourceCode()); -// alarmTypes.append(parser.getAlarmTypes()); -// -// a++; -// if (a % 100 == 0) { -// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); -// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); -// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); -// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); -// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); -// astEditScripts.setLength(0); -// tokens.setLength(0); -// sizes.setLength(0); -// patches.setLength(0); -// alarmTypes.setLength(0); -// System.out.println("Finish of parsing " + a + " files......"); -// } -// } -// } catch (TimeoutException e) { -// err.println("task timed out"); -// future.cancel(true /* mayInterruptIfRunning */); -// } catch (InterruptedException e) { -// err.println("task interrupted"); -// } catch (ExecutionException e) { -// err.println("task aborted"); -// } finally { -// executor.shutdownNow(); -// } -// } - - FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); - FileHelper.outputToFile(buggyTokensFilePath, tokens, true); - FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); - FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); - FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); - astEditScripts.setLength(0); - tokens.setLength(0); - sizes.setLength(0); - patches.setLength(0); - alarmTypes.setLength(0); - - -// classifyByAlarmTypes(); - } - } - - - private static List getMessageFiles(String gumTreeInput) { - String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile - File revFilesPath = new File(inputPath + "revFiles/"); - File[] revFiles = revFilesPath.listFiles(); // project folders - List msgFiles = new ArrayList<>(); - - // gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/ - // fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java - if (revFiles.length >= 0) { - for (File revFile : revFiles) { -// if (revFile.getName().endsWith(".java")) { - String fileName = revFile.getName(); - File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file - fileName = fileName.replace(".java", ".txt"); - File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file - File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file - MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); - msgFile.setPositionFile(positionFile); - msgFiles.add(msgFile); -// } - } - - return msgFiles; - } - else{ - return null; - } - } - - private static MessageFile getMessageFile(String gumTreeInput, String fileName) { -// String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile -// File revFilesPath = new File(inputPath + "revFiles/"); -// File[] revFiles = revFilesPath.listFiles(); // project folders -// List msgFiles = new ArrayList<>(); - - // gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/ - // fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java -// if (revFiles.length >= 0) { -// for (File revFile : revFiles) { -// if (revFile.getName().endsWith(".java")) { -// String fileName = revFile.getName(); - File revFile = new File(gumTreeInput + "revFiles/"+fileName); - File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file - fileName = fileName.replace(".java", ".txt"); - File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file -// File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file - MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); - return msgFile; -// msgFile.setPositionFile(positionFile); -// msgFiles.add(msgFile); -// } -// } -// -// return msgFiles; -// } -// else{ -// return null; -// } - } - - - - - - - -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/BaseMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/BaseMessage.java new file mode 100644 index 0000000..dfdbbde --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/BaseMessage.java @@ -0,0 +1,46 @@ +package edu.lu.uni.serval.fixminer.akka; + +/** + * Created by anilkoyuncu on 18/09/2018. + */ +public class BaseMessage { + + private long SECONDS_TO_WAIT; + private int id; + private int threadPoolSize; + + + public BaseMessage(int id,Long timeout) { + this.id = id; + this.SECONDS_TO_WAIT = timeout; +// this.threadPoolSize = threadPoolSize; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + + + public long getSECONDS_TO_WAIT() { + return SECONDS_TO_WAIT; + } + + public void setSECONDS_TO_WAIT(long SECONDS_TO_WAIT) { + this.SECONDS_TO_WAIT = SECONDS_TO_WAIT; + } + + public int getThreadPoolSize() { + return threadPoolSize; + } + + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } + + +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java new file mode 100644 index 0000000..6a23292 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java @@ -0,0 +1,106 @@ +package edu.lu.uni.serval.fixminer.akka.compare; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.ScanParams; +import redis.clients.jedis.ScanResult; + +import java.util.List; + +/** + * Created by anilkoyuncu on 12/09/2018. + */ +public class AkkaTreeParser { + + private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class); + + + public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, int cursor, String eDiffTimeout, String parallelism){ + + final List listOfPairs = getMessages(innerPool,cursor); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" + + + switch (parallelism){ + case "AKKA": + ActorSystem system = null; + ActorRef parsingActor = null; + final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout); + try { + log.info("Akka begins..."); + system = ActorSystem.create("Compare-EnhancedDiff-System"); + + parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor"); + parsingActor.tell(msg, ActorRef.noSender()); + } catch (Exception e) { + system.shutdown(); + e.printStackTrace(); + } + break; + case "FORKJOIN": + int counter = new Object() { + int counter = 0; + + { + listOfPairs.stream(). + parallel(). + peek(x -> counter++). + forEach(m -> + { + Compare compare = new Compare(); + compare.coreCompare(m, innerPool, outerPool); + if (counter % 10 == 0) { + log.info("Finalized parsing " + counter + " files... remaing " + (listOfPairs.size() - counter)); + } + } + ); + } + }.counter; + log.info("Finished parsing {} files",counter); + break; + default: + log.error("Unknown parallelism {}", parallelism); + break; + } + + + + + + } + + public static List getMessages(JedisPool innerPool, int cursor){ + + + ScanResult scan; + + try (Jedis inner = innerPool.getResource()) { + while (!inner.ping().equals("PONG")){ + log.info("wait"); + } + + ScanParams sc = new ScanParams(); + //150000000 + log.info("Scanning "); + sc.count(cursor); + + sc.match("*"); +// sc.match("pair_[0-9]*"); + + scan = inner.scan("0", sc); + int size = scan.getResult().size(); + log.info("Scanned " + String.valueOf(size)); + } + List result = scan.getResult(); + log.info("Getting results"); + return result; + + + + + + } +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java similarity index 79% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java index 49c03af..911d50b 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java @@ -1,11 +1,11 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.akka.compare; import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.matchers.Matcher; import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.FixPattern.utils.EDiff; +import edu.lu.uni.serval.utils.EDiffHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -32,21 +32,18 @@ public class Compare { try { jedis = innerPool.getResource(); -// resultMap = jedis.hgetAll(name); String[] split = name.split("_"); String i = split[1]; String j = split[2]; + String keyName = split[0]; -// String firstValue = resultMap.get("0"); -// String secondValue = resultMap.get("1"); + oldTree = EDiffHelper.getSimpliedTree(keyName,i,outerPool); - oldTree = EDiff.getSimpliedTree(i,outerPool); - - newTree = EDiff.getSimpliedTree(j,outerPool); + newTree = EDiffHelper.getSimpliedTree(keyName,j,outerPool); Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); m.match(); @@ -70,8 +67,8 @@ public class Compare { if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { - String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); -// log.info(matchKey); + String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); + log.info("{} tagged to be similar" ,matchKey); jedis.select(1); jedis.set(matchKey, result); @@ -80,13 +77,13 @@ public class Compare { jedis.select(0); - jedis.del("pair_" + (String.valueOf(i)) + "_" + String.valueOf(j)); + jedis.del(keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j)); } catch (Exception e) { - log.error(e.toString() + " {}", name); + log.debug("{} not comparable", name); // e.printStackTrace(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java new file mode 100644 index 0000000..510230e --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java @@ -0,0 +1,197 @@ +package edu.lu.uni.serval.fixminer.akka.compare; + +import com.github.gumtreediff.actions.ActionGenerator; +import com.github.gumtreediff.actions.model.Action; +import com.github.gumtreediff.matchers.Matcher; +import com.github.gumtreediff.matchers.Matchers; +import com.github.gumtreediff.tree.ITree; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.PoolBuilder; +import org.apache.commons.text.similarity.JaroWinklerDistance; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static edu.lu.uni.serval.fixminer.jobs.MultiThreadTreeLoaderCluster3.getNames; + + +/** + * Created by anilkoyuncu on 03/04/2018. + */ +public class CompareTrees { + + private static Logger log = LoggerFactory.getLogger(CompareTrees.class); + + + public static void main(String[] args) throws IOException { + + String portInner = "6380"; + String port = "6399"; +// + JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000); + + JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000); + + List listOfPairs = AkkaTreeParser.getMessages(innerPool,100000000); +// listOfPairs.stream().parallel(). + + int counter = new Object() { + int counter = 0; + + { + + listOfPairs.parallelStream(). + peek(x -> counter++). + forEach(m -> + { +// Compare compare = new Compare(); + coreCompare(m, args[0],innerPool, outerPool); + if (counter % 1000 == 0) { + log.info("Finalized parsing " + counter + " files... remaing " + (listOfPairs.size() - counter)); + } + } + ); + } + }.counter; +// coreCompare(args[0],args[1],args[2],args[3]); + } + public static void coreCompare(String pairName, String treeType,JedisPool innerPool,JedisPool outerPool ) { + + +// String portInner = "6380"; +// String port = "6399"; + +// JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000); +// +// JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000); + + Map resultMap; + Jedis jedis = null; + ITree oldTree = null; + ITree newTree = null; + + try { + jedis = innerPool.getResource(); + + String[] split = pairName.split("_"); + + + String i = split[1]; + String j = split[2]; + String keyName = split[0]; + + switch (treeType) { + case "shape": + oldTree = EDiffHelper.getShapes(keyName, i, outerPool); + newTree = EDiffHelper.getShapes(keyName, j, outerPool); + break; + case "action": + oldTree = EDiffHelper.getActions(keyName, i, outerPool,innerPool); + newTree = EDiffHelper.getActions(keyName, j, outerPool,innerPool); + break; + case "token": + oldTree = EDiffHelper.getTokens(keyName, i, outerPool,innerPool); + newTree = EDiffHelper.getTokens(keyName, j, outerPool,innerPool); + List oldTokens = new ArrayList<>(); + List newTokens = new ArrayList<>(); + + + + + oldTokens = getNames(oldTree,oldTokens); + newTokens = getNames(newTree,newTokens); + + + CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]); + CharSequence[] newSequences = newTokens.toArray(new CharSequence[newTokens.size()]); + JaroWinklerDistance jwd = new JaroWinklerDistance(); + + Double overallSimi = Double.valueOf(0); + if(oldSequences.length > 0 && (oldSequences.length == newSequences.length)){ + for (int idx = 0; idx < newSequences.length; idx++) { + Double simi = jwd.apply(newSequences[idx], oldSequences[idx]); + overallSimi = simi + overallSimi; + } + overallSimi = overallSimi / oldSequences.length; + }else{ + overallSimi = Double.valueOf(0); +// if(oldSequences.length != 0) { +// log.info("ERROR"); +// } + } + + int retval = Double.compare(overallSimi, Double.valueOf(0.8)); + String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); + if(retval >= 0){ + + + + String result = i + "," + j + ","+String.join(",", oldTokens); + jedis.select(2); + jedis.set(matchKey, result); + } + jedis.select(0); + jedis.del(matchKey); + + return; + default: + break; + } + + Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); + m.match(); + + + ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); + ag.generate(); + List actions = ag.getActions(); + + double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree); + String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1); + double diceSimilarity1 = m.diceSimilarity(oldTree, newTree); + String diceSimilarity = String.format("%1.2f", diceSimilarity1); + double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree); + String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); + + String editDistance = String.valueOf(actions.size()); + + String result = i + "," + j + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; + + String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); + + if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) + || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { +// log.info("{} tagged to be similar" ,matchKey); + + jedis.select(2); + jedis.set(matchKey, result); + + } + + + jedis.select(0); + jedis.del(matchKey); + + + + } catch (Exception e) { + + log.debug("{} not comparable", pairName); +// e.printStackTrace(); + + + }finally { + if (jedis != null) { + jedis.close(); + } + } + } + + +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/RunnableCompare.java similarity index 90% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/RunnableCompare.java index f51ca09..4ee2ced 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/RunnableCompare.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.akka.compare; import redis.clients.jedis.JedisPool; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java similarity index 86% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeActor.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java index 939c129..24b2045 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java @@ -1,11 +1,10 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.akka.compare; import akka.actor.ActorRef; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; import akka.routing.RoundRobinPool; -import edu.lu.uni.serval.fixminer.cluster.akka.TreeMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.JedisPool; @@ -61,15 +60,15 @@ public class TreeActor extends UntypedActor { int toIndex = (i + 1) * average + counter; List pairsOfWorkers = pairs.subList(fromIndex, toIndex); - final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool); + final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool,((TreeMessage) message).getSECONDS_TO_WAIT()); mineRouter.tell(workMsg, getSelf()); - logger.info("Assign a task to worker #" + (i + 1) + "..."); + logger.info("Assign {} task to worker #" + (i + 1) + "...",pairsOfWorkers.size()); } } else if ("STOP".equals(message.toString())) { counter ++; - logger.info(counter + " workers finailized their work..."); + logger.info(counter + " workers finalized their work..."); if (counter >= numberOfWorkers) { - logger.info("All workers finailized their work..."); + logger.info("All workers finalized their work..."); this.getContext().stop(mineRouter); this.getContext().stop(getSelf()); this.getContext().system().shutdown(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeMessage.java similarity index 64% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeMessage.java index b23bfe8..a99752c 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeMessage.java @@ -1,5 +1,6 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.compare; +import edu.lu.uni.serval.fixminer.akka.BaseMessage; import redis.clients.jedis.JedisPool; import java.util.List; @@ -7,25 +8,27 @@ import java.util.List; /** * Created by anilkoyuncu on 12/09/2018. */ -public class TreeMessage { - private int id; +public class TreeMessage extends BaseMessage{ private List name; private JedisPool innerPool; private JedisPool outerPool; - public TreeMessage(int id, List name, JedisPool innerPool, JedisPool outerPool) { - this.id = id; + + public TreeMessage(int id, List name, JedisPool innerPool, JedisPool outerPool,String eDiffTimeout) { + super(id,new Long(eDiffTimeout)); + this.name = name; this.innerPool = innerPool; this.outerPool = outerPool; - } - public int getId() { - return id; } + public TreeMessage(int id, List name, JedisPool innerPool, JedisPool outerPool,Long eDiffTimeout) { + super(id,eDiffTimeout); + + this.name = name; + this.innerPool = innerPool; + this.outerPool = outerPool; - public void setId(int id) { - this.id = id; } public List getName() { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java new file mode 100644 index 0000000..ae79022 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java @@ -0,0 +1,82 @@ +package edu.lu.uni.serval.fixminer.akka.compare; + +import akka.actor.Props; +import akka.actor.UntypedActor; +import akka.japi.Creator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.JedisPool; + +import java.util.List; +import java.util.concurrent.*; + +public class TreeWorker extends UntypedActor { + private static Logger log = LoggerFactory.getLogger(TreeWorker.class); + + + public TreeWorker() { + + } + + public static Props props() { + return Props.create(new Creator() { + + private static final long serialVersionUID = -7615153844097275009L; + + @Override + public TreeWorker create() throws Exception { + return new TreeWorker(); + } + + }); + } + + + + @Override + public void onReceive(Object message) throws Exception { + if(message instanceof TreeMessage) { + + TreeMessage msg = (TreeMessage) message; + List files = msg.getName(); + JedisPool innerPool = msg.getInnerPool(); + JedisPool outerPool = msg.getOuterPool(); + + int id = msg.getId(); + int counter = 0; + + for (String name : files) + { + + + Compare compare = new Compare(); + final ExecutorService executor = Executors.newSingleThreadExecutor(); +// // schedule the work + final Future future = executor.submit(new RunnableCompare(name, innerPool, outerPool, compare)); + try { +// wait for task to complete + future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); + counter++; + if (counter % 100 == 0) { + log.info("Worker #" + id +" finalized parsing " + counter + " pairs... remaing "+ (files.size() - counter)); + } + } catch (TimeoutException e) { + future.cancel(true); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } finally { + executor.shutdownNow(); + } + } + + log.info("Worker #" + id + " finalized the work..."); + this.getSender().tell("STOP", getSelf()); + }else{ + unhandled(message); + } + } + + +} diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java similarity index 61% rename from src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternActor.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java index 46ea6df..5d16315 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.MultipleThreadsParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import akka.actor.ActorRef; import akka.actor.Props; @@ -10,29 +10,29 @@ import org.slf4j.LoggerFactory; import java.util.List; -public class ParseFixPatternActor extends UntypedActor { +public class EDiffActor extends UntypedActor { - private static Logger logger = LoggerFactory.getLogger(ParseFixPatternActor.class); + private static Logger logger = LoggerFactory.getLogger(EDiffActor.class); private ActorRef mineRouter; private final int numberOfWorkers; private int counter = 0; - public ParseFixPatternActor(int numberOfWorkers, String project) { + public EDiffActor(int numberOfWorkers, String project) { mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers) - .props(ParseFixPatternWorker.props(project)), "mine-fix-pattern-router"); + .props(EDiffWorker.props(project)), "mine-fix-pattern-router"); this.numberOfWorkers = numberOfWorkers; } public static Props props(final int numberOfWorkers, final String project) { - return Props.create(new Creator() { + return Props.create(new Creator() { private static final long serialVersionUID = 9207427376110704705L; @Override - public ParseFixPatternActor create() throws Exception { - return new ParseFixPatternActor(numberOfWorkers, project); + public EDiffActor create() throws Exception { + return new EDiffActor(numberOfWorkers, project); } }); @@ -41,8 +41,8 @@ public class ParseFixPatternActor extends UntypedActor { @SuppressWarnings("deprecation") @Override public void onReceive(Object message) throws Exception { - if (message instanceof WorkMessage) { - List files = ((WorkMessage) message).getMsgFiles(); + if (message instanceof EDiffMessage) { + List files = ((EDiffMessage) message).getMsgFiles(); int size = files.size(); int average = size / numberOfWorkers; int reminder = size % numberOfWorkers; @@ -54,15 +54,15 @@ public class ParseFixPatternActor extends UntypedActor { int toIndex = (i + 1) * average + counter; List filesOfWorkers = files.subList(fromIndex, toIndex); - final WorkMessage workMsg = new WorkMessage(i + 1, filesOfWorkers); + final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getActionType()); mineRouter.tell(workMsg, getSelf()); - logger.info("Assign a task to worker #" + (i + 1) + "..."); + logger.info("Assign {} task to worker #" + (i + 1) ,filesOfWorkers.size()); } } else if ("STOP".equals(message.toString())) { counter ++; - logger.info(counter + " workers finailized their work..."); + logger.info(counter + " workers finalized their work..."); if (counter >= numberOfWorkers) { - logger.info("All workers finailized their work..."); + logger.info("All workers finalized their work..."); this.getContext().stop(mineRouter); this.getContext().stop(getSelf()); this.getContext().system().shutdown(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java new file mode 100644 index 0000000..667680b --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java @@ -0,0 +1,110 @@ +package edu.lu.uni.serval.fixminer.akka.ediff; + +import com.github.gumtreediff.actions.model.Delete; +import com.github.gumtreediff.actions.model.Insert; +import com.github.gumtreediff.actions.model.Move; +import com.github.gumtreediff.actions.model.Update; +import edu.lu.uni.serval.utils.FileHelper; + +import java.io.*; +import java.util.List; + + + +/** + * Parse fix violations with GumTree in terms of multiple statements. + * + * @author kui.liu + * + */ +public class EDiffHunkParser extends EDiffParser { + + + @Override + public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project,String actionType) { + List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); + if (actionSets.size() != 0) { +// String folder= null; + boolean processActionSet = true; +// switch (actionType){ +// case "ALL": +// if(actionType.equals("ALL")){ +// folder = "/ALL/"; +// processActionSet = true; +// }else if(actionType.equals("UPD") || actionType.equals("INS") || actionType.equals("DEL") || actionType.equals("MOV")|| actionType.equals("MIX")){ +// boolean isUPD = actionSets.stream().allMatch(p -> p.getAction() instanceof Update); +// boolean isINS = actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); +// boolean isDEL = actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); +// boolean isMOV = actionSets.stream().allMatch(p -> p.getAction() instanceof Move); +// if(isUPD){ +// folder = "/UPD/"; +// processActionSet = true; +// }else if(isINS){ +// folder = "/INS/"; +// processActionSet = true; +// }else if(isDEL){ +// folder = "/DEL/"; +// processActionSet = true; +// }else if(isMOV){ +// folder = "/MOV/"; +// processActionSet = true; +// }else{ +// folder = "/MIX/"; +// processActionSet = true; +// } +// }else{ +// +// +// processActionSet = false; +// System.err.print(actionType + "not known"); +// +// } + + + int hunkSet = 0; + if(processActionSet){ + for (HierarchicalActionSet actionSet : actionSets) { + + + + + FileOutputStream f = null; + + try { + + String astNodeType = actionSet.getAstNodeType(); + int size = actionSet.toString().split("\\n").length; + +// int size = actionSet.strList.size(); + + String datasetName = project; + String[] split1 = diffentryFile.getParent().split(datasetName); + String root = split1[0]; + String pj = split1[1].split("/")[1]; + + File file = new File(root + "EnhancedASTDiff" + datasetName + "/"+astNodeType+"/"+String.valueOf(size)+"/"); + file.mkdirs(); + String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/"+astNodeType+"/"+String.valueOf(size)+"/" + pj +"_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet); +// String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/" + pj + folder + diffentryFile.getName() + "_" + String.valueOf(hunkSet); + f = new FileOutputStream(new File(hunkTreeFileName)); + ObjectOutputStream o = new ObjectOutputStream(f); + o.writeObject(actionSet); + + o.close(); + f.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + hunkSet++; + } + + } + } + + } + + + +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java new file mode 100644 index 0000000..ac5e11d --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java @@ -0,0 +1,45 @@ +package edu.lu.uni.serval.fixminer.akka.ediff; + +import edu.lu.uni.serval.fixminer.akka.BaseMessage; + +import java.util.List; + +public class EDiffMessage extends BaseMessage{ + + + private List msgFiles; + + + + private String actionType; + + + + + + public EDiffMessage(int id, List msgFiles,String eDiffTimeout,String actionType) { + super(id,new Long(eDiffTimeout)); + this.msgFiles = msgFiles; + this.actionType = actionType; + } + public EDiffMessage(int id, List msgFiles,Long eDiffTimeout,String actionType) { + super(id,eDiffTimeout); + this.msgFiles = msgFiles; + this.actionType = actionType; + } + + public String getActionType() { + return actionType; + } + + public void setActionType(String actionType) { + this.actionType = actionType; + } + + + + public List getMsgFiles() { + return msgFiles; + } + +} \ No newline at end of file diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java similarity index 72% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java index 3bbc4eb..8e0e9d7 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java @@ -1,10 +1,7 @@ -package edu.lu.uni.serval.FixPatternParser.violations; +package edu.lu.uni.serval.fixminer.akka.ediff; import com.github.gumtreediff.actions.model.Action; -import edu.lu.uni.serval.FixPatternParser.Parser; import edu.lu.uni.serval.gumtree.GumTreeComparer; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; import edu.lu.uni.serval.utils.ListSorter; import java.io.File; @@ -17,7 +14,7 @@ import java.util.List; * @author kui.liu * */ -public class FixedViolationParser extends Parser { +public class EDiffParser extends Parser { /* * ResultType: @@ -29,11 +26,7 @@ public class FixedViolationParser extends Parser { */ public int resultType = 0; - protected String violationTypes = ""; - @Override - public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project) { - } /** * Regroup GumTree results without remove the modification of variable names. @@ -68,16 +61,11 @@ public class FixedViolationParser extends Parser { } } + @Override + public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, String actionType) { + + } - -// @Override -// public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { -// -// } - -// public void setUselessViolations(List uselessViolations) { -// this.uselessViolations = uselessViolations; -// } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java new file mode 100644 index 0000000..77a30b0 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java @@ -0,0 +1,86 @@ +package edu.lu.uni.serval.fixminer.akka.ediff; + +import akka.actor.Props; +import akka.actor.UntypedActor; +import akka.japi.Creator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.List; +import java.util.concurrent.*; + +public class EDiffWorker extends UntypedActor { + private static Logger log = LoggerFactory.getLogger(EDiffActor.class); + + private String project; + + + public EDiffWorker(String project) { + this.project = project; + } + + public static Props props(final String project) { + return Props.create(new Creator() { + + private static final long serialVersionUID = -7615153844097275009L; + + @Override + public EDiffWorker create() throws Exception { + return new EDiffWorker(project); + } + + }); + } + + @Override + public void onReceive(Object message) throws Exception { + if (message instanceof EDiffMessage) { + EDiffMessage msg = (EDiffMessage) message; + List files = msg.getMsgFiles(); + + int id = msg.getId(); + int counter = 0; + + for (MessageFile msgFile : files) { + File revFile = msgFile.getRevFile(); + File prevFile = msgFile.getPrevFile(); + File diffentryFile = msgFile.getDiffEntryFile(); + + + + EDiffHunkParser parser = new EDiffHunkParser(); + + final ExecutorService executor = Executors.newSingleThreadExecutor(); + // schedule the work + final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getActionType())); + try { + // wait for task to complete + future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); + + counter ++; + if (counter % 10 == 0) { + log.info("Worker #" + id +" finalized parsing " + counter + " files... remaing "+ (files.size() - counter)); + } + } catch (TimeoutException e) { + future.cancel(true); + System.err.println("#Timeout: " + revFile.getName()); + } catch (InterruptedException e) { + System.err.println("#TimeInterrupted: " + revFile.getName()); + e.printStackTrace(); + } catch (ExecutionException e) { + System.err.println("#TimeAborted: " + revFile.getName()); + e.printStackTrace(); + } finally { + executor.shutdownNow(); + } + } + + log.info("Worker #" + id + " finalized the work..."); + this.getSender().tell("STOP", getSelf()); + } else { + unhandled(message); + } + } + +} diff --git a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalActionSet.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalActionSet.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java index 4b5fb60..62f44e4 100644 --- a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalActionSet.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.gumtree.regroup; +package edu.lu.uni.serval.fixminer.akka.ediff; import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.tree.ITree; @@ -168,7 +168,7 @@ public class HierarchicalActionSet implements Comparable, return this.startPosition.compareTo(o.startPosition);//this.action.compareTo(o.action); } - + private List strList = new ArrayList<>(); @Override diff --git a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java index db1e1db..5549e2f 100644 --- a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java @@ -1,8 +1,8 @@ -package edu.lu.uni.serval.gumtree.regroup; +package edu.lu.uni.serval.fixminer.akka.ediff; import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; +import edu.lu.uni.serval.utils.ASTNodeMap; import edu.lu.uni.serval.utils.ListSorter; import java.util.ArrayList; diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/MessageFile.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java similarity index 93% rename from src/main/java/edu/lu/uni/serval/MultipleThreadsParser/MessageFile.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java index de224ef..b51e597 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/MessageFile.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.MultipleThreadsParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import java.io.File; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java similarity index 92% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java index 2b7367b..21bd9b4 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import java.io.File; @@ -18,7 +18,7 @@ public abstract class Parser implements ParserInterface { protected String originalTree = ""; // Guide of generating patches. protected String actionSets = ""; // Guide of generating patches. - public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile,String project); + public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile,String project,String actionType); @Override diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java similarity index 88% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java index 2a4c517..aac3e0f 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; public interface ParserInterface { diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java similarity index 80% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java index 891af4f..b74726f 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import java.io.File; @@ -9,6 +9,7 @@ public class RunnableParser implements Runnable { private File diffentryFile; private Parser parser; private String project; + private String actionType; public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) { this.prevFile = prevFile; @@ -17,16 +18,17 @@ public class RunnableParser implements Runnable { this.parser = parser; } - public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser,String project) { + public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser,String project,String actionType) { this.prevFile = prevFile; this.revFile = revFile; this.diffentryFile = diffentryFile; this.parser = parser; this.project = project; + this.actionType= actionType; } @Override public void run() { - parser.parseFixPatterns(prevFile, revFile, diffentryFile,project); + parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,actionType); } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/AkkaTreeLoader.java deleted file mode 100644 index 2cc115b..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/AkkaTreeLoader.java +++ /dev/null @@ -1,114 +0,0 @@ -package edu.lu.uni.serval.fixminer.cluster; - -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Pattern; - -import static edu.lu.uni.serval.fixminer.cluster.akka.AkkaTreeParser.akkaCompare; - -/** - * Created by anilkoyuncu on 19/03/2018. - */ -public class AkkaTreeLoader { - - private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class); - - - public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, String cursor,String chunk) throws Exception { - - - String parameters = String.format("\nportInner %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,chunkName,dbDir,dumpsName); - log.info(parameters); - - - CallShell cs = new CallShell(); - String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; - String cmd1 = String.format(cmd, dbDir,dumpsName,Integer.valueOf(port)); -// - cs.runShell(cmd1,port); - String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; - String cmd2 = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner)); - log.info(cmd1); - log.info(cmd2); -// - cs.runShell(cmd2, portInner); - JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000); - JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000); - - - - String pairsIndexFile = pairsPath + "/"+ chunkName; - //0.txt.rdb - pairsIndexFile = pairsIndexFile.replace(chunk+".rdb",".index"); - - - Pattern pattern = Pattern.compile(","); - String csvFile = pairsIndexFile; - try { - try (BufferedReader in = new BufferedReader(new FileReader(csvFile));){ - Map namefreq = in - .lines() - .map(x -> pattern.split(x)) - .collect(HashMap::new, (map, x) -> - map.put(x[0], x[1]), - Map::putAll); - - Jedis inner = null; - try { - inner = outerPool.getResource(); - - for (Map.Entry entry : namefreq.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - inner.select(1); - inner.set(key,value); - } - - - }finally { - if (inner != null) { - inner.close(); - } - } - - - } - } catch (IOException e) { - e.printStackTrace(); - } - - - - - akkaCompare(innerPool,outerPool,numOfWorkers,cursor); - - - - - } - - - - - - - - - - - - -} - - - - diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/ImportPairs2DB.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/ImportPairs2DB.java deleted file mode 100644 index 6249a4a..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/ImportPairs2DB.java +++ /dev/null @@ -1,63 +0,0 @@ -package edu.lu.uni.serval.fixminer.cluster; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - - -/** - * Created by anilkoyuncu on 05/04/2018. - */ -public class ImportPairs2DB { - private static Logger log = LoggerFactory.getLogger(ImportPairs2DB.class); - - public static void main(String csvInputPath,String portInner,String dbDir,String datasetPath) throws Exception { - - - String parameters = String.format("\nInput path %s \nportInner %s \ndbDir %s",csvInputPath,portInner,dbDir); - log.info(parameters); - - - File folder = new File(csvInputPath); - File[] subFolders = folder.listFiles(); - Stream stream = Arrays.stream(subFolders); - List pjs = stream - .filter(x -> x.getName().endsWith(".txt")) - .collect(Collectors.toList()); - Integer portInt = Integer.valueOf(portInner); - - for (File pj : pjs) { - - String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; - cmd = String.format(cmd, dbDir,pj.getName() +".rdb", portInt); - log.info(cmd); - CallShell cs = new CallShell(); - cs.runShell(cmd, portInner); - - cmd = "bash "+datasetPath + "/redisSingleImport.sh" +" %s %s"; - - cmd = String.format(cmd, pj.getPath(), portInt); - log.info(cmd); - cs.runShell(cmd); - - String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - String stopServer2 = String.format(stopServer,portInt); - cs.runShell(stopServer2, portInner); - - portInt++; - - - - } - - - log.info(parameters); - } - - -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoader.java deleted file mode 100644 index e4e0441..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoader.java +++ /dev/null @@ -1,600 +0,0 @@ -package edu.lu.uni.serval.fixminer.cluster; - -import com.github.gumtreediff.actions.ActionGenerator; -import com.github.gumtreediff.actions.model.Action; -import com.github.gumtreediff.matchers.Matcher; -import com.github.gumtreediff.matchers.Matchers; -import com.github.gumtreediff.tree.ITree; -import com.github.gumtreediff.tree.Tree; -import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; - -import java.io.*; -import java.nio.file.Files; -import java.time.Duration; -import java.util.*; -import java.util.concurrent.Executors; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Created by anilkoyuncu on 19/03/2018. - */ -public class MultiThreadTreeLoader { - - private static class StreamGobbler implements Runnable { - private InputStream inputStream; - private Consumer consumer; - - public StreamGobbler(InputStream inputStream, Consumer consumer) { - this.inputStream = inputStream; - this.consumer = consumer; - } - - @Override - public void run() { - new BufferedReader(new InputStreamReader(inputStream)).lines() - .forEach(consumer); - } - } - - private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoader.class); - - - public static void loadRedis(String cmd,String serverWait){ - Process process; - - try { -// String comd = String.format(cmd, f.getAbsoluteFile()); - process = Runtime.getRuntime() - - .exec(cmd); - - - StreamGobbler streamGobbler = - new StreamGobbler(process.getInputStream(), System.out::println); - Executors.newSingleThreadExecutor().submit(streamGobbler); -// int exitCode = process.waitFor(); -// assert exitCode == 0; - Thread.sleep(Integer.valueOf(serverWait)); - - } catch (IOException e) { - e.printStackTrace(); - } - catch (InterruptedException e) { - e.printStackTrace(); - } - log.info("Load done"); - } - - public static void main(String[] args) { - - - String inputPath; - String outputPath; - String port; - String portInner; - String pairsCSVPath; - String importScript; - String pairsCompletedPath; - String serverWait; - String option; - String dbDir; - String chunkName; - if (args.length > 0) { - inputPath = args[0]; - port = args[1]; - portInner = args[2]; - serverWait = args[3]; - option = args[4]; - chunkName = args[5]; - dbDir = args[6]; -// pairsCSVPath = args[3]; -// importScript = args[4]; -// pairsCompletedPath = args[3]; - } else { - inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; - outputPath = "/Users/anilkoyuncu/bugStudy/dataset/"; - port = "6379"; - portInner = "6380"; - serverWait = "10000"; - option = "COMP"; - pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test"; - importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh"; - pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed"; - chunkName ="chunk5.rdb"; - dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; - } - - if (option.equals("CALC")) { - calculatePairs(inputPath, port); - log.info("Calculate pairs done"); - }else { - comparePairs(port, inputPath, portInner, serverWait,chunkName,dbDir); - } - - - } - - - public static void comparePairs(String port,String inputPath, String innerPort,String serverWait,String chunkName, String dbDir){ -// String cmd; -// cmd = "bash " + importScript +" %s"; - - - - List dir; - List path; - - String orgDbname; - - - File files = new File(dbDir); - File[] listFiles = files.listFiles(); - - - Stream stream = Arrays.stream(listFiles); - List folders = stream - .filter(x -> x.getName().equals(chunkName)) - .collect(Collectors.toList()); - for (File folder : folders) { - - - - - String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; - cmd = String.format(cmd, dbDir,folder.getName(),Integer.valueOf(innerPort)); - loadRedis(cmd,serverWait); - - - JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000); - ScanResult scan; - try (Jedis inner = pool.getResource()) { - while (inner.ping()== "PONG"){ - log.info("wait"); - } - - ScanParams sc = new ScanParams(); - sc.count(150000000); - sc.match("pair_*"); - - scan = inner.scan("0", sc); - int size = scan.getResult().size(); - log.info("Scanning " + String.valueOf(size)); - } - scan.getResult().parallelStream() - .forEach(m -> coreCompare(m, inputPath, innerPort)); - - - String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - stopServer = String.format(stopServer,Integer.valueOf(innerPort)); - loadRedis(stopServer,serverWait); - - } - - - - - } - - private static void coreCompare(String name , String inputPath, String innerPort) { - JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000); - Map resultMap; - try (Jedis jedis = pool.getResource()) { - resultMap = jedis.hgetAll(name); - } - String[] split = name.split("_"); - - - String i = split[1]; - String j = split[2]; - String firstValue = resultMap.get("0"); - String secondValue = resultMap.get("1"); - - String[] firstValueSplit = firstValue.split("GumTreeOutput2"); - String[] secondValueSplit = secondValue.split("GumTreeOutput2"); - - if (firstValueSplit.length == 1) { - firstValue = inputPath + firstValueSplit[0]; - } else { - firstValue = inputPath + firstValueSplit[1]; - } - - if (secondValueSplit.length == 1) { - secondValue = inputPath + secondValueSplit[0]; - } else { - secondValue = inputPath + secondValueSplit[1]; - } - - try { - ITree oldTree = getSimpliedTree(firstValue); - - ITree newTree = getSimpliedTree(secondValue); - - Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); - m.match(); - - - ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); - ag.generate(); - List actions = ag.getActions(); - - double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree); - String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1); - double diceSimilarity1 = m.diceSimilarity(oldTree, newTree); - String diceSimilarity = String.format("%1.2f", diceSimilarity1); - double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree); - String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); - - String editDistance = String.valueOf(actions.size()); - - String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; - - - if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) - || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { - String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); - - try (Jedis jedis = pool.getResource()) { - jedis.select(1); - jedis.set(matchKey, result); - } - } - - - try (Jedis jedis = pool.getResource()) { - jedis.del("pair_"+ (String.valueOf(i)) + "_" + String.valueOf(j)); - } - - - - - }catch (Exception e){ - log.warn(e.toString() + " {}",(name)); - - - } - - - - - - } - - - - public static void calculatePairs(String inputPath,String port) { - File folder = new File(inputPath); - File[] listOfFiles = folder.listFiles(); - Stream stream = Arrays.stream(listOfFiles); - List pjs = stream - .filter(x -> !x.getName().startsWith(".")) - .collect(Collectors.toList()); - - List fileToCompare = new ArrayList<>(); - for (File pj : pjs) { - File[] files = pj.listFiles(new FilenameFilter() { - public boolean accept(File dir, String name) { - return name.startsWith("ActionSetDumps"); - } - }); - Collections.addAll(fileToCompare, files[0].listFiles()); - - } - System.out.println("a"); -// compareAll(fileToCompare); - readMessageFiles(fileToCompare,port); - } - -// public static void processMessages(String inputPath, String outputPath) { -// File folder = new File(outputPath + "pairs_splitted/"); -// File[] listOfFiles = folder.listFiles(); -// Stream stream = Arrays.stream(listOfFiles); -// List pjs = stream -// .filter(x -> !x.getName().startsWith(".")) -// .collect(Collectors.toList()); -// FileHelper.createDirectory(outputPath + "comparison_splitted/"); -// pjs.parallelStream() -// .forEach(m -> coreLoop(m, outputPath,inputPath)); -// } - - - public static ITree getSimpliedTree(String fn) { - HierarchicalActionSet actionSet = null; - try { - FileInputStream fi = new FileInputStream(new File(fn)); - ObjectInputStream oi = new ObjectInputStream(fi); - actionSet = (HierarchicalActionSet) oi.readObject(); - oi.close(); - fi.close(); - - - } catch (FileNotFoundException e) { - log.error("File not found"); - e.printStackTrace(); - } catch (IOException e) { - log.error("Error initializing stream"); - e.printStackTrace(); - } catch (ClassNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - ITree parent = null; - ITree children =null; - ITree tree = getASTTree(actionSet, parent, children); - tree.setParent(null); - - return tree; - - } - - public static List getKeysByValue(Map map, E value) { - return map.entrySet() - .stream() - .filter(entry -> Objects.equals(entry.getValue(), value)) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - } - - public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ - - int newType = 0; - - String astNodeType = actionSet.getAstNodeType(); - List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); - - if(keysByValue.size() != 1){ - log.error("Birden cok astnodemapmapping"); - } - newType = keysByValue.get(0); - if(actionSet.getParent() == null){ - //root - - parent = new Tree(newType,""); - }else{ - children = new Tree(newType,""); - parent.addChild(children); - } - List subActions = actionSet.getSubActions(); - if (subActions.size() != 0){ - for (HierarchicalActionSet subAction : subActions) { - - if(actionSet.getParent() == null){ - children = parent; - } - getASTTree(subAction,children,null); - - } - - - } - return parent; - } - -// public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ -// -// int newType = 0; -// -// Action action = actionSet.getAction(); -// if (action instanceof Update){ -// newType = 101; -// }else if(action instanceof Insert){ -// newType =100; -// }else if(action instanceof Move){ -// newType = 102; -// }else if(action instanceof Delete){ -// newType=103; -// }else{ -// new Exception("unknow action"); -// } -// if(actionSet.getParent() == null){ -// //root -// -// parent = new Tree(newType,""); -// }else{ -// children = new Tree(newType,""); -// parent.addChild(children); -// } -// List subActions = actionSet.getSubActions(); -// if (subActions.size() != 0){ -// for (HierarchicalActionSet subAction : subActions) { -// -// if(actionSet.getParent() == null){ -// children = parent; -// } -// getActionTree(subAction,children,null); -// -// } -// -// -// } -// return parent; -// } - - - - private static void coreLoop(File mes, String outputPath,String inputPath) { - try { - - log.info("Starting in coreLoop"); - - BufferedReader br = null; - String sCurrentLine = null; - BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "comparison_splitted/" + "output_" + mes.getName())); - - br = new BufferedReader( - new FileReader(mes)); - while ((sCurrentLine = br.readLine()) != null) { - String currentLine = sCurrentLine; - String[] split = currentLine.split("\t"); - String i = split[0]; - String j = split[1]; - String firstValue = split[2]; - String secondValue = split[3]; - - firstValue = inputPath + firstValue.split("GumTreeOutput2")[1]; - secondValue = inputPath + secondValue.split("GumTreeOutput2")[1]; - - ITree oldTree = getSimpliedTree(firstValue); - - ITree newTree = getSimpliedTree(secondValue); - - Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); - m.match(); - - ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); - ag.generate(); - List actions = ag.getActions(); - writer.write(String.valueOf(i)); - writer.write("\t"); - writer.write(String.valueOf(j)); - writer.write("\t"); - - writer.write(String.format("%1.2f", m.chawatheSimilarity(oldTree, newTree))); - writer.write("\t"); - writer.write(String.format("%1.2f", m.diceSimilarity(oldTree, newTree))); - writer.write("\t"); - writer.write(String.format("%1.2f", m.jaccardSimilarity(oldTree, newTree))); - writer.write("\t"); - writer.write(String.valueOf(actions.size())); - writer.write("\t"); - writer.write(firstValue); - writer.write("\t"); - writer.write(secondValue); - writer.write("\n"); - - - } - writer.close(); - } catch (FileNotFoundException e) { - log.error("File not found"); - e.printStackTrace(); - } catch (IOException e) { - log.error("Error initializing stream"); - e.printStackTrace(); - - } - log.info("Completed output_" + mes.getName()); - } - - private static void readMessageFiles(List folders,String port) { - - List treesFileNames = new ArrayList<>(); - - - for (File target : folders) { - - treesFileNames.add(target.toString()); - } -// FileHelper.createDirectory(outputPath + "pairs/"); - log.info("Calculating pairs"); -// treesFileNames = treesFileNames.subList(0,100); - byte [] buf = new byte[0]; - String line = null; - - -// FileChannel rwChannel = new RandomAccessFile(outputPath + "pairs/" +"textfile.txt", "rw").getChannel(); -// ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE); - int fileCounter = 0; - - JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); - try (Jedis jedis = jedisPool.getResource()) { - List dir = null; - List path = null; - for (int i = 0; i < treesFileNames.size(); i++) { - for (int j = i + 1; j < treesFileNames.size(); j++) { - - - // do operations with jedis resource - - String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j); -// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1]; -// jedis.set(key,value); - - jedis.hset(key,"0",treesFileNames.get(i).split("GumTreeOutput2")[1]); - jedis.hset(key,"1",treesFileNames.get(j).split("GumTreeOutput2")[1]); - //10000000 - if(Integer.compare(jedis.dbSize().intValue(),10000000) == 0){ - dir = jedis.configGet("dir"); - path = jedis.configGet("dbfilename"); - File dbPath = new File(dir.get(1)+"/"+path.get(1)); - File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb"); - try { - jedis.save(); - while (jedis.ping()== "PONG"){ - log.info("wait"); - } - - - - Files.copy(dbPath.toPath(),savePath.toPath()); - } catch (IOException e) { - - e.printStackTrace(); - } - fileCounter++; - jedis.flushDB(); - - } - - - } - } - jedis.save(); - fileCounter++; - File dbPath = new File(dir.get(1)+"/"+path.get(1)); - File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb"); - try { - - while (jedis.ping()== "PONG"){ - log.info("wait"); - } - - - - Files.copy(dbPath.toPath(),savePath.toPath()); - } catch (IOException e) { - - e.printStackTrace(); - } - jedis.flushDB(); - } - - - - - log.info("Done pairs"); - } - - static final JedisPoolConfig poolConfig = buildPoolConfig(); - - - private static JedisPoolConfig buildPoolConfig() { - final JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(128); - poolConfig.setMaxIdle(128); - poolConfig.setMinIdle(16); - poolConfig.setTestOnBorrow(true); - poolConfig.setTestOnReturn(true); - poolConfig.setTestWhileIdle(true); - poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); - poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); - poolConfig.setNumTestsPerEvictionRun(3); - poolConfig.setBlockWhenExhausted(true); - - return poolConfig; - } - - - -// return msgFiles; -} - - - - diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeWorker.java deleted file mode 100644 index cfc7bb9..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeWorker.java +++ /dev/null @@ -1,224 +0,0 @@ -package edu.lu.uni.serval.fixminer.cluster; - -import akka.actor.Props; -import akka.actor.UntypedActor; -import akka.japi.Creator; -import edu.lu.uni.serval.config.Configuration; -import edu.lu.uni.serval.fixminer.cluster.akka.TreeMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.JedisPool; - -import java.util.List; -import java.util.concurrent.*; - -public class TreeWorker extends UntypedActor { - private static Logger log = LoggerFactory.getLogger(TreeWorker.class); - - -// private JedisPool innerPool; -// private JedisPool outerPool; -// -// public TreeWorker(String innerPort,String outerPort) { -//// this.innerPool = innerPool; -//// this.outerPool = outerPool; -// this.outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(outerPort),20000000); -// this.innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(innerPort),20000000); -// -// -// } -// -// public static Props props(final String innerPort,final String outerPort) { -// return Props.create(new Creator() { -// -// private static final long serialVersionUID = -7615153844097275009L; -// -// @Override -// public TreeWorker create() throws Exception { -// return new TreeWorker(innerPort,outerPort); -// } -// -// }); -// } - - public TreeWorker() { - - } - - public static Props props() { - return Props.create(new Creator() { - - private static final long serialVersionUID = -7615153844097275009L; - - @Override - public TreeWorker create() throws Exception { - return new TreeWorker(); - } - - }); - } - - - - @Override - public void onReceive(Object message) throws Exception { - if(message instanceof TreeMessage) { - - -// if (message instanceof edu.lu.uni.serval.MultipleThreadsParser.WorkMessage) { - TreeMessage msg = (TreeMessage) message; - List files = msg.getName(); - JedisPool innerPool = msg.getInnerPool(); - JedisPool outerPool = msg.getOuterPool(); - - int id = msg.getId(); -// int counter = new Object() { - int counter = 0; - - // - for (String name : files) - { -// files.stream(). -// parallel(). -// peek(x -> counter++). -// forEach(m -> -// { -// Compare compare = new Compare(); -// compare.coreCompare(m, innerPool, outerPool); -// } -// ); -// } -// }.counter; - -// - final ExecutorService executor = Executors.newFixedThreadPool(20); -// // schedule the work - final Future future = executor.submit(new RunnableCompare(name, innerPool, outerPool, new Compare())); - try { -// wait for task to complete - future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); - Compare compare = new Compare(); - compare.coreCompare(name, innerPool, outerPool); - counter++; -// nullDiffEntry += parser.nullMatchedDiffEntry; -// nullMappingGumTreeResults += parser.nullMappingGumTreeResult; -// pureDeletion += parser.pureDeletions; -// largeHunk += parser.largeHunk; -// nullSourceCode += parser.nullSourceCode; -// testInfos += parser.testInfos; -// testingInfo.append(parser.testingInfo); -// builder.append(parser.unfixedViolations); -// -// String editScript = parser.getAstEditScripts(); -// if ("".equals(editScript)) { -//// if (parser.resultType == 1) { -//// nullGumTreeResults += countAlarms(positionFile, ""); -//// } else if (parser.resultType == 2) { -//// noSourceCodeChanges += countAlarms(positionFile, ""); -//// } else if (parser.resultType == 3) { -//// noStatementChanges += countAlarms(positionFile, ""); -//// } -// } else { -// editScripts.append(editScript); -// patchesSourceCode.append(parser.getPatchesSourceCode()); -// sizes.append(parser.getSizes()); -// tokens.append(parser.getTokensOfSourceCode()); -// -// counter ++; -// if (counter % 100 == 0) { -// FileHelper.outputToFile(editScriptsFilePath + "edistScripts_" + id + ".list", editScripts, true); -// FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true); -// FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true); -// FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true); -// editScripts.setLength(0); -// patchesSourceCode.setLength(0); -// sizes.setLength(0); -// tokens.setLength(0); -// log.info("Worker #" + id +" finialized parsing " + counter + " files..."); -// FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); -// testingInfo.setLength(0); -// } -// } - } catch (TimeoutException e) { - future.cancel(true); -////// timeouts += countAlarms(positionFile, "#Timeout:"); -// System.err.println("#Timeout: " + name); - } catch (InterruptedException e) { -////// timeouts += countAlarms(positionFile, "#TimeInterrupted:"); -//// System.err.println("#TimeInterrupted: " + revFile.getName()); - e.printStackTrace(); - } catch (ExecutionException e) { -////// timeouts += countAlarms(positionFile, "#TimeAborted:"); -//// System.err.println("#TimeAborted: " + revFile.getName()); - e.printStackTrace(); - } finally { - executor.shutdownNow(); - } - } - - log.info("bitti"); - log.info("Worker #" + id +"finialized parsing " + counter + " files..."); - log.info("Worker #" + id + " finialized the work..."); - this.getSender().tell("STOP", getSelf()); -// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; -// stopServer = String.format(stopServer,Integer.valueOf(innerPort)); -// loadRedis(stopServer,serverWait); -// -// if (sizes.length() > 0) { -// FileHelper.outputToFile(editScriptsFilePath + "editScripts_" + id + ".list", editScripts, true); -// FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true); -// FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true); -// FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true); -// editScripts.setLength(0); -// patchesSourceCode.setLength(0); -// sizes.setLength(0); -// tokens.setLength(0); -// -// FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); -// testingInfo.setLength(0); -// } -// String statistic = "\nNullGumTreeResults: " + nullGumTreeResults + "\nNoSourceCodeChanges: " + noSourceCodeChanges + -// "\nNoStatementChanges: " + noStatementChanges + "\nNullDiffEntry: " + nullDiffEntry + "\nNullMatchedGumTreeResults: " + nullMappingGumTreeResults + -// "\nPureDeletion: " + pureDeletion + "\nLargeHunk: " + largeHunk + "\nNullSourceCode: " + nullSourceCode + -// "\nTestingInfo: " + testInfos + "\nTimeout: " + timeouts; -// FileHelper.outputToFile("OUTPUT/statistic_" + id + ".list", statistic, false); -// FileHelper.outputToFile("OUTPUT/UnfixedV_" + id + ".list", builder, false); -// -// log.info("Worker #" + id +"finialized parsing " + counter + " files..."); -// log.info("Worker #" + id + " finialized the work..."); -// this.getSender().tell("STOP", getSelf()); -// } else { - }else{ - unhandled(message); - } - } - -// static final JedisPoolConfig poolConfig = buildPoolConfig(); -// -// -// private static JedisPoolConfig buildPoolConfig() { -// final JedisPoolConfig poolConfig = new JedisPoolConfig(); -// poolConfig.setMaxTotal(128); -// poolConfig.setMaxIdle(128); -// poolConfig.setMinIdle(16); -// poolConfig.setTestOnBorrow(true); -// poolConfig.setTestOnReturn(true); -// poolConfig.setTestWhileIdle(true); -// poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); -// poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); -// poolConfig.setNumTestsPerEvictionRun(3); -// poolConfig.setBlockWhenExhausted(true); -// -// return poolConfig; -// } - - - - - - - - - - -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java deleted file mode 100644 index f009b25..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java +++ /dev/null @@ -1,51 +0,0 @@ -package edu.lu.uni.serval.fixminer.cluster; - -import java.util.List; - -public class WorkMessage { - - private int id; - private List msgFiles; - - public String getDbDir() { - return dbDir; - } - - public String getServerWait() { - return serverWait; - } - - private String dbDir; - private String serverWait; - - public String getInputPath() { - return inputPath; - } - - private String inputPath; - - public String getInnerPort() { - return innerPort; - } - - private String innerPort; - - public WorkMessage(int id, List msgFiles,String innerPort,String inputPath,String dbDir,String serverWait) { - super(); - this.id = id; - this.msgFiles = msgFiles; - this.innerPort = innerPort; - this.inputPath = inputPath; - this.dbDir = dbDir; - this.serverWait = serverWait; - } - - public int getId() { - return id; - } - - public List getMsgFiles() { - return msgFiles; - } - -} \ No newline at end of file diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java deleted file mode 100644 index d76b532..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java +++ /dev/null @@ -1,75 +0,0 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import edu.lu.uni.serval.fixminer.cluster.TreeActor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.ScanParams; -import redis.clients.jedis.ScanResult; - -import java.util.List; - -/** - * Created by anilkoyuncu on 12/09/2018. - */ -public class AkkaTreeParser { - - private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class); - - - public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, String cursor){ - - final List listOfPairs = getMessages(innerPool,cursor); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" - - - - - ActorSystem system = null; - ActorRef parsingActor = null; - final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool); - try { - log.info("Akka begins..."); - system = ActorSystem.create("Compare-EnhancedDiff-System"); - - parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor"); - parsingActor.tell(msg, ActorRef.noSender()); - } catch (Exception e) { - system.shutdown(); - e.printStackTrace(); - } - } - - public static List getMessages(JedisPool innerPool, String cursor){ - - - ScanResult scan; - - try (Jedis inner = innerPool.getResource()) { - while (!inner.ping().equals("PONG")){ - log.info("wait"); - } - - ScanParams sc = new ScanParams(); - //150000000 - log.info("Scanning "); - sc.count(Integer.valueOf(cursor)); - - sc.match("pair_[0-9]*"); - - scan = inner.scan("0", sc); - int size = scan.getResult().size(); - log.info("Scanned " + String.valueOf(size)); - } - List result = scan.getResult(); - log.info("Getting results"); - return result; - - - - - - } -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java new file mode 100644 index 0000000..915e4bc --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java @@ -0,0 +1,149 @@ +package edu.lu.uni.serval.fixminer.jobs; + +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.PoolBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser.akkaCompare; + +/** + * Created by anilkoyuncu on 19/03/2018. + */ +public class AkkaTreeLoader { + + private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class); + + + public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, int cursor, String chunk, String eDiffTimeout, String parallelism) throws Exception { + + + String parameters = String.format("\nportInner %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,chunkName,dbDir,dumpsName); + log.info(parameters); + + + CallShell cs = new CallShell(); + String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; + String cmd1 = String.format(cmd, dbDir,dumpsName,Integer.valueOf(port)); +// + cs.runShell(cmd1,port); + String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; + String cmd2 = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner)); + log.info(cmd1); + log.info(cmd2); +// + cs.runShell(cmd2, portInner); + JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000); + JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000); + + + +// String pairsIndexFile = pairsPath + "/"+ chunkName; + //0.txt.rdb +// pairsIndexFile = pairsIndexFile.replace(chunk+".rdb",".index"); + + File folder = new File(pairsPath); + File[] rootsFolder = folder.listFiles(); + + Stream stream = Arrays.stream(rootsFolder); + List roots = stream + .filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + + for(File root:roots) { + + File[] files = root.listFiles(); + Stream fileStream = Arrays.stream(files); + List pairsIndexFiles = fileStream + .filter(x -> !x.getName().startsWith(".")) + .filter(x -> x.getName().endsWith(".index")) + .collect(Collectors.toList()); + for (File pairsIndexFile : pairsIndexFiles) { + + Pattern pattern = Pattern.compile(","); + String csvFile = pairsIndexFile.getPath(); + try { + try (BufferedReader in = new BufferedReader(new FileReader(csvFile));) { + Map namefreq = in + .lines() + .map(x -> pattern.split(x)) + .collect(HashMap::new, (map, x) -> + map.put(root.getName()+"-"+pairsIndexFile.getName().split("\\.")[0]+"-"+x[0], x[1]), + Map::putAll); + + Jedis inner = null; + try { + inner = outerPool.getResource(); + + for (Map.Entry entry : namefreq.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + inner.select(1); + inner.set(key, value); + } + + + } finally { + if (inner != null) { + inner.close(); + } + } + + + } + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + + + akkaCompare(innerPool,outerPool,numOfWorkers,cursor,eDiffTimeout,parallelism); + +// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; +// String stopServer2 = String.format(stopServer,Integer.valueOf(port)); +// +// cs.runShell(stopServer2,port); +// +// +// String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); +// +// cs.runShell(stopServer1,portInner); + + + + + } + + + + + + + + + + + + +} + + + + diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java similarity index 66% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java index 87ab887..494857d 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java @@ -1,7 +1,8 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; +import edu.lu.uni.serval.utils.CallShell; import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -21,7 +22,7 @@ import java.util.List; public class CalculatePairs { private static Logger log = LoggerFactory.getLogger(CalculatePairs.class); - public static void main(String dbDir,String chunkName,String port,String outputPath,String pjName) throws Exception { + public static void main(String dbDir,String chunkName,String port,String outputPath,String pjName,boolean isBigPair,int cursor) throws Exception { String parameters = String.format("\nport %s \nchunkName %s \ndbDir %s",port,chunkName,dbDir); @@ -45,7 +46,7 @@ public class CalculatePairs { ScanParams sc = new ScanParams(); //150000000 - sc.count(1500000000); + sc.count(cursor); sc.match("*"); scan = outer.scan("0", sc); @@ -59,31 +60,59 @@ public class CalculatePairs { byte [] buf = new byte[0]; String line = null; + if(isBigPair) { + bigPair(outputPath, pjName, result, buf, line); + }else{ + smallPair(outputPath, pjName, result, buf, line); + } + +// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; +// String stopServer2 = String.format(stopServer,Integer.valueOf(port)); +//// loadRedis(stopServer2,serverWait); +// cs.runShell(stopServer2,serverWait); + log.info("Done pairs"); + } + + private static void smallPair(String outputPath, String pjName, List result, byte[] buf, String line) { try { -// FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv"); -// DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); -// + FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv"); + DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); + + FileOutputStream fosIndex = new FileOutputStream(outputPath + "/" +pjName+".index"); + DataOutputStream outStreamIndex = new DataOutputStream(new BufferedOutputStream(fosIndex)); + + for (int i = 0; i < result.size(); i++) { + line = String.valueOf(i) +"," + result.get(i)+"\n"; + outStreamIndex.write(line.getBytes()); + + for (int j = i + 1; j < result.size(); j++) { + + + + line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n"; + outStream.write(line.getBytes()); + + } + } + outStream.close(); + outStreamIndex.close(); + + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + }catch (java.nio.BufferOverflowException e) { + log.error(line); + log.error(String.valueOf(buf.length)); + e.printStackTrace(); + } + } + + private static void bigPair(String outputPath, String pjName, List result, byte[] buf, String line) { + try { + FileOutputStream fosIndex = new FileOutputStream(outputPath + "/" +pjName+".index"); DataOutputStream outStreamIndex = new DataOutputStream(new BufferedOutputStream(fosIndex)); -// -// -// -// -// for (int i = 0; i < result.size(); i++) { -// line = String.valueOf(i) +"," + result.get(i)+"\n"; -// outStreamIndex.write(line.getBytes()); -// -// for (int j = i + 1; j < result.size(); j++) { -// -// -// -// line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n"; -// outStream.write(line.getBytes()); -// -// } -// } -// outStream.close(); -// outStreamIndex.close(); int fileCounter = 0; FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +pjName +String.valueOf(fileCounter)+".txt", "rw").getChannel(); int maxSize = 500*1000000; @@ -96,8 +125,6 @@ public class CalculatePairs { for (int j = i + 1; j < result.size(); j++) { - -// line = String.valueOf(i) +"\t" + String.valueOf(j)+"\n"; // + "\t" + result.get(i) + "\t" + result.get(j)+"\n"; line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n"; buf = line.getBytes(); if(wrBuf.remaining() > 500) { @@ -125,12 +152,6 @@ public class CalculatePairs { log.error(String.valueOf(buf.length)); e.printStackTrace(); } - -// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; -// String stopServer2 = String.format(stopServer,Integer.valueOf(port)); -//// loadRedis(stopServer2,serverWait); -// cs.runShell(stopServer2,serverWait); - log.info("Done pairs"); } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java new file mode 100644 index 0000000..f2d3c44 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java @@ -0,0 +1,141 @@ +package edu.lu.uni.serval.fixminer.jobs; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage; +import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile; +import edu.lu.uni.serval.utils.FileHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class EnhancedASTDiff { + + private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class); + + public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType, String parallelism) { + + + String parameters = String.format("\nInput path %s \nOutput path %s",inputPath,outputPath); + log.info(parameters); + + File folder = new File(inputPath); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List folders = stream + .filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + + + List allMessageFiles = new ArrayList<>(); + for (File target : folders) { + String pjName = target.getName(); + + + List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" + allMessageFiles.addAll(msgFiles); +// System.out.println(msgFiles.size()); + if (msgFiles.size() == 0) + continue; + + String GUM_TREE_OUTPUT = outputPath + "/" + pjName + "/"; + +// a + +// if (actionType.equals("ALL")) { +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/" + actionType); +// } else { +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MIX"); +// } + } + + switch (parallelism){ + case "AKKA": + ActorSystem system = null; + ActorRef parsingActor = null; + final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,actionType); + try { + log.info("Akka begins..."); + log.info("{} files to process ...", allMessageFiles.size()); + system = ActorSystem.create("Mining-FixPattern-System"); + + parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor"); + parsingActor.tell(msg, ActorRef.noSender()); + } catch (Exception e) { + system.shutdown(); + e.printStackTrace(); + } + break; + case "FORKJOIN": + int counter = new Object() { + int counter = 0; + + { + allMessageFiles.stream(). + parallel(). + peek(x -> counter++). + forEach(m -> + { + EDiffHunkParser parser = new EDiffHunkParser(); + parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,actionType); + if (counter % 10 == 0) { + log.info("Finalized parsing " + counter + " files... remaing " + (allMessageFiles.size() - counter)); + } + } + ); + } + }.counter; + log.info("Finished parsing {} files",counter); + break; + default: + log.error("Unknown parallelism {}", parallelism); + break; + } + + + + + + } + + + + private static List getMessageFiles(String gumTreeInput) { + String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile + File revFilesPath = new File(inputPath + "revFiles/"); + File[] revFiles = revFilesPath.listFiles(); // project folders + List msgFiles = new ArrayList<>(); + if (revFiles.length >= 0) { + for (File revFile : revFiles) { + String fileName = revFile.getName(); + File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file + fileName = fileName.replace(".java", ".txt"); + File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file + + MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); + + msgFiles.add(msgFile); + + } + + return msgFiles; + } + else{ + return null; + } + } + + +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java new file mode 100644 index 0000000..585d082 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java @@ -0,0 +1,80 @@ +package edu.lu.uni.serval.fixminer.jobs; + +import edu.lu.uni.serval.utils.CallShell; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +/** + * Created by anilkoyuncu on 05/04/2018. + */ +public class ImportPairs2DB { + private static Logger log = LoggerFactory.getLogger(ImportPairs2DB.class); + + public static void main(String csvInputPath, String portInner, String dbDir, String datasetPath, String chunk) throws Exception { + + + String parameters = String.format("\nInput path %s \nportInner %s \ndbDir %s",csvInputPath,portInner,dbDir); + log.info(parameters); + + String[] splits = chunk.split("\\."); + String chunkType = splits[splits.length-1]; + log.info("Chunk type {}",chunkType); + + File folder = new File(csvInputPath); + File[] subFolders = folder.listFiles(); + Stream stream = Arrays.stream(subFolders); + List roots = stream +// .filter(x -> x.getName().endsWith(chunkType)) + .collect(Collectors.toList()); + + Integer portInt = Integer.valueOf(portInner); + + String cmd = "bash " + dbDir + "/" + "startServer.sh" + " %s %s %s"; + cmd = String.format(cmd, dbDir,"pairs.rdb", portInt); + log.info(cmd); + CallShell cs = new CallShell(); + cs.runShell(cmd, portInner); + + for (File root : roots) { + + File[] files = root.listFiles(); + Stream fileStream = Arrays.stream(files); + List pairs = fileStream + .filter(x -> x.getName().endsWith(chunkType)) + .collect(Collectors.toList()); + + for(File pj:pairs) { + + + + cmd = "bash " + datasetPath + "/redisSingleImport.sh" + " %s %s %s"; + + cmd = String.format(cmd, pj.getPath(), portInt,root.getName()+"-"+pj.getName().split("\\.")[0]); + + log.info(cmd); + cs.runShell(cmd,portInner); + + + } + + + + } + + String stopServer = "bash " + dbDir + "/" + "stopServer.sh" + " %s"; + String stopServer2 = String.format(stopServer, portInt); + cs.runShell(stopServer2, portInner); + +// portInt++; + log.info(parameters); + } + + +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java similarity index 62% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java index fa6ed79..84e487d 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.model.Action; @@ -6,11 +6,11 @@ import com.github.gumtreediff.matchers.Matcher; import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.FixPattern.utils.EDiff; -import edu.lu.uni.serval.FixPattern.utils.EDiffHelper; -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.EDiffHelper; import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.javatuples.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,10 +20,8 @@ import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import java.io.*; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -35,7 +33,7 @@ public class MultiThreadTreeLoaderCluster { private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoaderCluster.class); - public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type) throws Exception { + public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type,int cursor) throws Exception { CallShell cs = new CallShell(); String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; @@ -49,8 +47,8 @@ public class MultiThreadTreeLoaderCluster { - String cmd3; - cmd3 = "bash " + importScript +" %s %s"; +// String cmd3; +// cmd3 = "bash " + importScript +" %s %s"; JedisPool jedisPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000); @@ -65,40 +63,74 @@ public class MultiThreadTreeLoaderCluster { .filter(x -> !x.getName().startsWith(".")) .collect(Collectors.toList()); - for (File f:folders){ + for (File f:folders) { // if(f.getName().startsWith("cluster0")) { + File[] files = f.listFiles(); + Stream fileStream = Arrays.stream(files); + + List pairs = fileStream + .filter(x -> !x.getName().startsWith(".")) + .filter(x -> !x.getName().endsWith(".index")) + .collect(Collectors.toList()); + for (File pair : pairs) { try (Jedis jedis = jedisPool.getResource()) { // do operations with jedis resource ScanParams sc = new ScanParams(); - sc.count(150000000); - sc.match("pair_[0-9]*"); + sc.count(cursor); + sc.match(f.getName()+"*"); log.info("Scanning"); ScanResult scan = jedis.scan("0", sc); int size = scan.getResult().size(); if (size == 0) { - String comd = String.format(cmd3,f.getPath(),portInner); + String cmd3 = "bash " + importScript + " %s %s %s"; + + cmd3 = String.format(cmd3, pair.getPath(), portInner,f.getName()+"-"+pair.getName().split("\\.")[0]); +// String comd = String.format(cmd3, f.getPath(), portInner); // loadRedis(comd); - cs.runShell(comd); + log.info("Importing {} pairs for cluster {}", size, f.getName()); + cs.runShell(cmd3, portInner); scan = jedis.scan("0", sc); size = scan.getResult().size(); } - log.info("Scanned " + String.valueOf(size)); + log.info("Scanned {} for cluster {}", String.valueOf(size), f.getName()); + + Pattern pattern = Pattern.compile(","); + String csvFile = pair.getPath(); + csvFile = csvFile.replace("txt","index"); + try (BufferedReader in = new BufferedReader(new FileReader(csvFile));) { + Map namefreq = in + .lines() + .map(x -> pattern.split(x)) + .collect(HashMap::new, (map, x) -> + map.put(f.getName()+"-"+pair.getName().split("\\.")[0]+"-"+x[0], x[1]), + Map::putAll); + + for (Map.Entry entry : namefreq.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + jedis.select(1); + jedis.set(key, value); + } - String clusterName = f.getName().replaceAll("[^0-9]+", ""); + + } + + +// String clusterName = f.getName().replaceAll("[^0-9]+", ""); //76 scan.getResult().parallelStream() - .forEach(m -> coreCompare(m, jedisPool, clusterName,outerPool,type)); + .forEach(m -> coreCompare(m, jedisPool, f.getName(), outerPool, type)); jedis.save(); @@ -107,6 +139,7 @@ public class MultiThreadTreeLoaderCluster { // } + } } String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); @@ -121,33 +154,47 @@ public class MultiThreadTreeLoaderCluster { - public static Pair getTree(String firstValue, JedisPool outerPool,String type){ + public static Pair getTree(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool){ ITree tree = null; Jedis inner = null; - String[] split2 = firstValue.split("/"); - - String fullFileName = split2[split2.length-1]; - String[] split = fullFileName.split(".txt_"); - String pureFileName = split[0]; - String[] splitPJ = split[1].split("_"); - String project = splitPJ[1]; - String actionSetPosition = splitPJ[0]; + Jedis outer = null; +// String[] split2 = firstValue.split("/"); +// +// String fullFileName = split2[split2.length-1]; +// String[] split = fullFileName.split(".txt_"); +// String pureFileName = split[0]; +// String[] splitPJ = split[1].split("_"); +// String project = splitPJ[1]; +// String actionSetPosition = splitPJ[0]; try { - inner = outerPool.getResource(); - String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition; - String si= inner.get(filename); + inner = innerPool.getResource(); + inner.select(1); + String dist2load = inner.get(prefix+"-"+firstValue); + outer = outerPool.getResource(); + outer.select(0); + String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load); + Set keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load); + if(keys.size() == 1) { + s = (String) keys.toArray()[0]; + }else{ + throw new Error("cok key"); + } + + +// String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition; + String si= outer.get(s); HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si); ITree parent = null; ITree children =null; TreeContext tc = new TreeContext(); - tree = EDiff.getActionTree(actionSet, parent, children,tc); + tree = EDiffHelper.getActionTree(actionSet, parent, children,tc); tree.setParent(null); tc.validate(); @@ -160,8 +207,11 @@ public class MultiThreadTreeLoaderCluster { if (inner != null) { inner.close(); } + if (outer != null) { + outer.close(); + } } - Pair pair = new Pair<>(tree,project); + Pair pair = new Pair<>(tree,null); return pair; @@ -196,19 +246,20 @@ public class MultiThreadTreeLoaderCluster { catch (Exception e){ e.printStackTrace(); } - String firstValue = resultMap.get("0"); - String secondValue = resultMap.get("1"); +// String firstValue = resultMap.get("0"); +// String secondValue = resultMap.get("1"); try { - Pair oldPair = getTree(firstValue, outerPool,type); - Pair newPair = getTree(secondValue, outerPool,type); + String keyName = split[0]; + Pair oldPair = getTree(keyName,i, outerPool,jedisPool); + Pair newPair = getTree(keyName,j, outerPool,jedisPool); ITree oldTree = oldPair.getValue0(); ITree newTree = newPair.getValue0(); - String oldProject = oldPair.getValue1(); - String newProject = newPair.getValue1(); +// String oldProject = oldPair.getValue1(); +// String newProject = newPair.getValue1(); @@ -230,23 +281,24 @@ public class MultiThreadTreeLoaderCluster { String editDistance = String.valueOf(actions.size()); // jedis.select(1); - String result = resultMap.get("0") + "," + oldProject +"," + resultMap.get("1") + "," +newProject+ "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; + String result = i+ "," + j + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; // jedis.set(resultKey, result); + String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { - String matchKey = "match-"+clusterName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); - jedis.select(1); + jedis.select(2); jedis.set(matchKey, result); } jedis.select(0); - String pairKey = "pair_" + (String.valueOf(i)) + "_" + String.valueOf(j); - jedis.del(pairKey); + + jedis.del(matchKey); // log.info("Completed " + resultKey); }catch (Exception e){ - log.warn(e.toString() + " {}",(name)); + log.debug(e.toString() + " {}",(name)); + e.printStackTrace(); } @@ -328,7 +380,7 @@ public class MultiThreadTreeLoaderCluster { e.printStackTrace(); } - log.info("Done pairs"); + log.info("Done pairs of cluster {}",cluster); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster3.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java similarity index 97% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster3.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java index 661595a..e542cc5 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster3.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java @@ -1,11 +1,12 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.FixPattern.utils.EDiffHelper; -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.EDiffHelper; import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.similarity.JaroWinklerDistance; import org.slf4j.Logger; @@ -32,7 +33,7 @@ public class MultiThreadTreeLoaderCluster3 { private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoaderCluster3.class); - public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type) throws Exception { + public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type,int cursor) throws Exception { CallShell cs = new CallShell(); String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; @@ -68,7 +69,7 @@ public class MultiThreadTreeLoaderCluster3 { try (Jedis jedis = jedisPool.getResource()) { // do operations with jedis resource ScanParams sc = new ScanParams(); - sc.count(150000000); + sc.count(cursor); sc.match("pair_[0-9]*"); log.info("Scanning"); @@ -77,7 +78,7 @@ public class MultiThreadTreeLoaderCluster3 { if (size == 0) { String comd = String.format(cmd3, f.getPath(), portInner); - cs.runShell(comd); + cs.runShell(comd,portInner); scan = jedis.scan("0", sc); @@ -148,7 +149,7 @@ public class MultiThreadTreeLoaderCluster3 { ITree parent = null; ITree children =null; TreeContext tc = new TreeContext(); - tree = EDiffHelper.getASTTree(actionSet, parent, children,tc); + tree = EDiffHelper.getTokenTree(actionSet, parent, children,tc); // tree.setParent(null); tc.validate(); @@ -412,7 +413,7 @@ public class MultiThreadTreeLoaderCluster3 { } // if (oldTokens.size() == 0 ) {// && (oldTree.getType() != 41 && oldTree.getType() != 21 && oldTree.getType() !=17 && oldTree.getType()!=60 && oldTree.getType() != 46)){ -// log.info("dur bakalim nereye!???"); +// log.info("!???"); // } return oldTokens; } @@ -525,7 +526,7 @@ public class MultiThreadTreeLoaderCluster3 { .collect(Collectors.toList()); FileHelper.createDirectory(outputPath + "/pairs-2l"+type); - + log.info("Calculating pairs"); for (File pj : pjs) { File[] files = pj.listFiles(); List fileList = Arrays.asList(files); @@ -535,6 +536,7 @@ public class MultiThreadTreeLoaderCluster3 { } File[] clusterFiles = cluster.listFiles(); List clusterFilesL = Arrays.asList(clusterFiles); + readMessageFilesCluster(clusterFilesL, outputPath, inputPath, pj.getName(), cluster.getName(),type); @@ -558,7 +560,7 @@ public class MultiThreadTreeLoaderCluster3 { treesFileNames.add(target.toString()); } - log.info("Calculating pairs"); + // treesFileNames = treesFileNames.subList(0,100); String filename = "cluster" + cluster + "_" + subCluster; @@ -608,7 +610,7 @@ public class MultiThreadTreeLoaderCluster3 { e.printStackTrace(); } - log.info("Done pairs"); + log.info("Done pairs of {}",filename); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/PatternExtractor.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java similarity index 80% rename from src/main/java/edu/lu/uni/serval/fixminer/PatternExtractor.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java index bbc33a5..7c7d3bc 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/PatternExtractor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java @@ -1,20 +1,19 @@ -package edu.lu.uni.serval.fixminer; +package edu.lu.uni.serval.fixminer.jobs; -import edu.lu.uni.serval.FixPattern.utils.Checker; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; +import edu.lu.uni.serval.utils.Checker; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; -import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -65,36 +64,9 @@ public class PatternExtractor { public static void loadDB(String inputPath,String portInner,String dbDir,String chunkName,String operation,List fixes) throws Exception { -// String inputPath; -// String portInner; -// String serverWait; -// String dbDir; -// String chunkName; -// String numOfWorkers; -// if (args.length > 0) { -// inputPath = args[0]; -// portInner = args[1]; -// serverWait = args[2]; -// chunkName = args[3]; -// numOfWorkers = args[4]; -// dbDir = args[5]; -// } else { -// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; -// portInner = "6399"; -// serverWait = "10000"; -// chunkName ="dumps.rdb"; -// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; -// numOfWorkers = "1"; -// } -// String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s \noperation %s", inputPath, portInner, serverWait, chunkName, dbDir, operation); -// log.info(parameters); -// CallShell cs = new CallShell(); -// String cmd = "bash " + dbDir + "/" + "startServer.sh" + " %s %s %s"; -// cmd = String.format(cmd, dbDir, chunkName, Integer.valueOf(portInner)); -//// loadRedis(cmd,serverWait); -// cs.runShell(cmd, serverWait); - JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(6399),20000000); + + JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(6399),20000000); @@ -134,11 +106,11 @@ public class PatternExtractor { s = inner.get(fn); if (s != null) { - HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); BufferedWriter writer = new BufferedWriter(new FileWriter(savePath+saveFN)); - writer.write(toString(actionSet)); + writer.write(EDiffHelper.toString(actionSet)); writer.close(); @@ -207,7 +179,7 @@ public class PatternExtractor { String saveFN = pattern.getName() + "_" + project + "_" + operation + "_" + joinFN; try{ String content = new String(Files.readAllBytes(Paths.get(savePath + saveFN))); - HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(content); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(content); int astType = actionSet.getNode().getType(); if (Checker.isStatement(astType) || astType == 23 //FieldDeclaration @@ -264,41 +236,7 @@ public class PatternExtractor { - static final JedisPoolConfig poolConfig = buildPoolConfig(); - private static String toString( Serializable o ) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream( baos ); - oos.writeObject( o ); - oos.close(); - return Base64.getEncoder().encodeToString(baos.toByteArray()); - } - - /** Read the object from Base64 string. */ - private static Object fromString( String s ) throws IOException , - ClassNotFoundException { - byte[] data = Base64.getDecoder().decode(s); - ObjectInputStream ois = new ObjectInputStream( - new ByteArrayInputStream(data)); - Object o = ois.readObject(); - ois.close(); - return o; - } - private static JedisPoolConfig buildPoolConfig() { - final JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(128); - poolConfig.setMaxIdle(128); - poolConfig.setMinIdle(16); - poolConfig.setTestOnBorrow(false); - poolConfig.setTestOnReturn(false); - poolConfig.setTestWhileIdle(true); - poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); - poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); -// poolConfig.setNumTestsPerEvictionRun(3); - poolConfig.setBlockWhenExhausted(true); - - return poolConfig; - } public static List readIdList(String fileName) { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java similarity index 57% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java index 39bba51..e04f260 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java @@ -1,8 +1,9 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; -import edu.lu.uni.serval.FixPattern.utils.EDiffHelper; -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -37,42 +38,55 @@ public class StoreEDiffInCache { File folder = new File(inputPath); File[] subFolders = folder.listFiles(); Stream stream = Arrays.stream(subFolders); - List pjs = stream + List roots = stream .filter(x -> !x.getName().startsWith(".")) .collect(Collectors.toList()); List workList = new ArrayList(); File[] dumps; - for (File pj : pjs) { - String pjName = pj.getName(); - File[] files = pj.listFiles(); - Stream fileStream = Arrays.stream(files); - List fs; - if (operation.equals("ALLOP")){ - fs= fileStream - .filter(x -> x.getName().startsWith("UPD") || - x.getName().startsWith("INS") || - x.getName().startsWith("DEL") || - x.getName().startsWith("MOV")) + for (File root : roots) { + String pjName = root.getName(); + File[] sizes = root.listFiles(); + Stream sizesStream = Arrays.stream(sizes); + List sizeFolders = sizesStream + .filter(x -> !x.getName().startsWith(".")) .collect(Collectors.toList()); - File[] files1 = fs.get(0).listFiles(); - File[] files2 = fs.get(1).listFiles(); - File[] files3 = fs.get(2).listFiles(); - File[] files4 = fs.get(3).listFiles(); - dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new); - }else{ - fs = fileStream - .filter(x -> x.getName().startsWith(operation)) - .collect(Collectors.toList()); - dumps = fs.get(0).listFiles(); - } + + for (File sf:sizeFolders){ + File[] files = sf.listFiles(); + int length = files.length; + if (length == 1) + continue; + + Stream fileStream = Arrays.stream(files); + List fs; + // if (operation.equals("ALLOP")){ + // fs= fileStream + // .filter(x -> x.getName().startsWith("UPD") || + // x.getName().startsWith("INS") || + // x.getName().startsWith("DEL") || + // x.getName().startsWith("MOV")) + // .collect(Collectors.toList()); + // File[] files1 = fs.get(0).listFiles(); + // File[] files2 = fs.get(1).listFiles(); + // File[] files3 = fs.get(2).listFiles(); + // File[] files4 = fs.get(3).listFiles(); + // dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new); + // }else{ +// fs = fileStream +// .filter(x -> x.getName().startsWith(operation)) +// .collect(Collectors.toList()); +// dumps = fs.get(0).listFiles(); + // } - for (File f : dumps) { - String name = f.getName(); + for (File f : files) { + String name = f.getName(); + + String key = pjName + "/"+ sf.getName()+"/" + name; + String result = key +","+f.getPath(); + workList.add(result); + } - String key = pjName + "/"+ operation+"/" + name; - String result = key +","+f.getPath(); - workList.add(result); } } @@ -111,10 +125,10 @@ public class StoreEDiffInCache { } catch (FileNotFoundException e) { - log.error("File not found"); + log.error("File not found {}" , path); e.printStackTrace(); } catch (IOException e) { - log.error("Error initializing stream"); + log.error("Error initializing stream {}", path); e.printStackTrace(); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeLoaderClusterL1.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java similarity index 94% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeLoaderClusterL1.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java index 36a480a..3c0818f 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeLoaderClusterL1.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java @@ -1,6 +1,7 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.PoolBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -21,7 +22,7 @@ public class TreeLoaderClusterL1 { private static Logger log = LoggerFactory.getLogger(TreeLoaderClusterL1.class); - public static void main(String portInner,String port,String inputPath,String level1DB,String level1Path,String innerTypePrefix) throws Exception { + public static void main(String portInner,String port,String inputPath,String level1DB,String level1Path,String innerTypePrefix,int cursor) throws Exception { @@ -58,7 +59,7 @@ public class TreeLoaderClusterL1 { inner.select(1); ScanParams sc = new ScanParams(); //150000000 - sc.count(150000000); + sc.count(cursor); sc.match("match_[0-9]*"); ScanResult scan; scan = inner.scan("0", sc); diff --git a/src/main/java/edu/lu/uni/serval/FixPattern/utils/ASTNodeMap.java b/src/main/java/edu/lu/uni/serval/utils/ASTNodeMap.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/FixPattern/utils/ASTNodeMap.java rename to src/main/java/edu/lu/uni/serval/utils/ASTNodeMap.java index 266063c..c2d5bf8 100644 --- a/src/main/java/edu/lu/uni/serval/FixPattern/utils/ASTNodeMap.java +++ b/src/main/java/edu/lu/uni/serval/utils/ASTNodeMap.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPattern.utils; +package edu.lu.uni.serval.utils; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java b/src/main/java/edu/lu/uni/serval/utils/CallShell.java similarity index 94% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java rename to src/main/java/edu/lu/uni/serval/utils/CallShell.java index 9c1d05d..85c76ab 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java +++ b/src/main/java/edu/lu/uni/serval/utils/CallShell.java @@ -1,9 +1,12 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.utils; /** * Created by anilkoyuncu on 17/04/2018. */ +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -11,6 +14,7 @@ import java.util.concurrent.TimeUnit; public class CallShell { + private static Logger log = LoggerFactory.getLogger(CallShell.class); @@ -28,6 +32,8 @@ public class CallShell { public static void runShell(String command, String port) throws Exception { + log.info(command); + Process process = Runtime.getRuntime().exec(command); BufferedReader reader = new BufferedReader(new InputStreamReader( process.getInputStream())); diff --git a/src/main/java/edu/lu/uni/serval/FixPattern/utils/Checker.java b/src/main/java/edu/lu/uni/serval/utils/Checker.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/FixPattern/utils/Checker.java rename to src/main/java/edu/lu/uni/serval/utils/Checker.java index 83e1e3a..89752fa 100644 --- a/src/main/java/edu/lu/uni/serval/FixPattern/utils/Checker.java +++ b/src/main/java/edu/lu/uni/serval/utils/Checker.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPattern.utils; +package edu.lu.uni.serval.utils; public class Checker { diff --git a/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java b/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java new file mode 100644 index 0000000..0285940 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java @@ -0,0 +1,71 @@ +package edu.lu.uni.serval.utils; + +import com.github.gumtreediff.tree.ITree; +import com.github.gumtreediff.tree.TreeContext; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class ClusterToPattern { + + public static void main(String[] args) throws IOException, ClassNotFoundException { + String shapesFolder = "/Users/anil.koyuncu/projects/fixminer-all/enhancedASTDiff/python/data/shapes"; + + String port = "6399"; + JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000); + + File folder = new File(shapesFolder); + + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List shapes = stream + .filter(x -> !x.getName().startsWith(".")) + .filter(x -> !x.getName().endsWith(".pickle")) + .collect(Collectors.toList()); + for(File shape:shapes){ + File[] clusters = shape.listFiles(); + Stream fileStream = Arrays.stream(clusters); + List fileList = fileStream.filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + for(File cluster:fileList){ + File[] files = cluster.listFiles(); + File file = files[0]; + String key = shape.getName() + "/*/"+file.getName(); + Jedis outer = outerPool.getResource(); + String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load); + Set keys = outer.keys(key); + if (keys.size() == 1) { + s = (String) keys.toArray()[0]; + } else { + throw new Error("cok key"); + } + + String si = outer.get(s); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si); + + + String s1 = actionSet.toString(); + File saveFile = new File(folder.getParent()+"/shapePatterns/" +shape.getName()+"/"+cluster.getName()+".pattern"); + saveFile.getParentFile().mkdirs(); + BufferedWriter writer = new BufferedWriter(new FileWriter(saveFile)); + writer.write(s1); + + writer.close(); + + + } + + } + + } +} diff --git a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java new file mode 100644 index 0000000..766f154 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java @@ -0,0 +1,352 @@ +package edu.lu.uni.serval.utils; + +import com.github.gumtreediff.actions.model.*; +import com.github.gumtreediff.tree.ITree; +import com.github.gumtreediff.tree.TreeContext; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import java.io.*; +import java.util.*; +import java.util.stream.Collectors; + +/** + * Created by anilkoyuncu on 17/09/2018. + */ +public class EDiffHelper { + private static Logger log = LoggerFactory.getLogger(EDiffHelper.class); + /** Read the object from Base64 string. */ + public static Object fromString( String s ) throws IOException, + ClassNotFoundException { + byte [] data = Base64.getDecoder().decode( s ); + ObjectInputStream ois = new ObjectInputStream( + new ByteArrayInputStream( data ) ); + Object o = ois.readObject(); + ois.close(); + return o; + } + + /** Write the object to a Base64 string. */ + public static String toString( Serializable o ) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream( baos ); + oos.writeObject( o ); + oos.close(); + return Base64.getEncoder().encodeToString(baos.toByteArray()); + } + + + public static ITree getTokenTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){ + + int newType = 0; + + String astNodeType = actionSet.getAstNodeType(); + + String label = actionSet.getAction().toString(); + List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); + + if(keysByValue.size() != 1){ + log.error("More than 1"); + } + newType = keysByValue.get(0); + if(actionSet.getParent() == null){ + //root + + parent = tc.createTree(newType, label, null); + tc.setRoot(parent); + }else{ + + children = tc.createTree(newType, label, null); + children.setParentAndUpdateChildren(parent); + } + List subActions = actionSet.getSubActions(); + if (subActions.size() != 0){ + for (HierarchicalActionSet subAction : subActions) { + + if(actionSet.getParent() == null){ + children = parent; + } + getTokenTree(subAction,children,null,tc); + + } + + + } + return parent; + } + + + public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children, TreeContext tc){ + + int newType = 0; + + String astNodeType = actionSet.getAstNodeType(); + List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); + + if(keysByValue.size() != 1){ + log.error("More than 1"); + } + newType = keysByValue.get(0); + if(actionSet.getParent() == null){ + //root + +// parent = new Tree(newType,""); + parent = tc.createTree(newType, "", null); + tc.setRoot(parent); + }else{ +// children = new Tree(newType,""); +// parent.addChild(children); + children = tc.createTree(newType, "", null); + children.setParentAndUpdateChildren(parent); + } + List subActions = actionSet.getSubActions(); + if (subActions.size() != 0){ + for (HierarchicalActionSet subAction : subActions) { + + if(actionSet.getParent() == null){ + children = parent; + } + getASTTree(subAction,children,null,tc); + + } + + + } + return parent; + } + + + public static List getKeysByValue(Map map, E value) { + return map.entrySet() + .stream() + .filter(entry -> Objects.equals(entry.getValue(), value)) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } + + public static ITree getSimpliedTree(String prefix, String fn, JedisPool outerPool) { + + ITree tree = null; + Jedis inner = null; + try { + inner = outerPool.getResource(); + while (!inner.ping().equals("PONG")){ + log.info("wait"); + } + inner.select(1); + String dist2load = inner.get(prefix+"-"+fn); + inner.select(0); + String s = inner.get(prefix.replace("-","/") +"/"+dist2load); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); + + ITree parent = null; + ITree children =null; + TreeContext tc = new TreeContext(); + tree = EDiffHelper.getASTTree(actionSet, parent, children,tc); + tree.setParent(null); + tc.validate(); + + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + }finally { + if (inner != null) { + inner.close(); + } + } + return tree; + + } + + + public static ITree getShapes(String prefix, String fn, JedisPool outerPool) { + + ITree tree = null; + Jedis inner = null; + try { + inner = outerPool.getResource(); + while (!inner.ping().equals("PONG")){ + log.info("wait"); + } + inner.select(1); + String dist2load = inner.get(prefix+"-"+fn); + inner.select(0); + String s = inner.get(prefix.replace("-","/") +"/"+dist2load); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); + + ITree parent = null; + ITree children =null; + TreeContext tc = new TreeContext(); + tree = EDiffHelper.getASTTree(actionSet, parent, children,tc); + //tree.setParent(null); + tc.validate(); + + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + }finally { + if (inner != null) { + inner.close(); + } + } + return tree; + + } + + + public static ITree getActions(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool) { + + + ITree tree = null; + Jedis inner = null; + Jedis outer = null; + + + try { + inner = innerPool.getResource(); + inner.select(1); + String dist2load = inner.get(prefix + "-" + firstValue); + outer = outerPool.getResource(); + outer.select(0); + String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load); + Set keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load); + if (keys.size() == 1) { + s = (String) keys.toArray()[0]; + } else { + throw new Error("cok key"); + } + + +// String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition; + String si = outer.get(s); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si); + + + ITree parent = null; + ITree children = null; + TreeContext tc = new TreeContext(); + tree = EDiffHelper.getActionTree(actionSet, parent, children, tc); + //tree.setParent(null); + tc.validate(); + + + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } finally { + if (inner != null) { + inner.close(); + } + if (outer != null) { + outer.close(); + } + } + + return tree; + } + + + public static ITree getTokens(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool) { + + + ITree tree = null; + Jedis inner = null; + Jedis outer = null; + + + try { + inner = innerPool.getResource(); + inner.select(1); + String dist2load = inner.get(prefix + "-" + firstValue); + outer = outerPool.getResource(); + outer.select(0); + String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load); + Set keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load); + if (keys.size() == 1) { + s = (String) keys.toArray()[0]; + } else { + throw new Error("cok key"); + } + + +// String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition; + String si = outer.get(s); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si); + + + ITree parent = null; + ITree children = null; + TreeContext tc = new TreeContext(); + tree = EDiffHelper.getTokenTree(actionSet, parent, children, tc); + tree.setParent(null); + tc.validate(); + + + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } finally { + if (inner != null) { + inner.close(); + } + if (outer != null) { + outer.close(); + } + } + + return tree; + } + + + + + public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){ + + int newType = 0; + + Action action = actionSet.getAction(); + if (action instanceof Update){ + newType = 101; + }else if(action instanceof Insert){ + newType =100; + }else if(action instanceof Move){ + newType = 102; + }else if(action instanceof Delete){ + newType=103; + }else{ + new Exception("unknow action"); + } + if(actionSet.getParent() == null){ + //root + + parent = tc.createTree(newType, "", null); + tc.setRoot(parent); + + }else{ + children = tc.createTree(newType, "", null); + children.setParentAndUpdateChildren(parent); + + } + List subActions = actionSet.getSubActions(); + if (subActions.size() != 0){ + for (HierarchicalActionSet subAction : subActions) { + + if(actionSet.getParent() == null){ + children = parent; + } + getActionTree(subAction,children,null,tc); + + } + + + } + return parent; + } +} diff --git a/src/main/java/edu/lu/uni/serval/FixPattern/utils/PoolBuilder.java b/src/main/java/edu/lu/uni/serval/utils/PoolBuilder.java similarity index 95% rename from src/main/java/edu/lu/uni/serval/FixPattern/utils/PoolBuilder.java rename to src/main/java/edu/lu/uni/serval/utils/PoolBuilder.java index fba3f04..31562f2 100644 --- a/src/main/java/edu/lu/uni/serval/FixPattern/utils/PoolBuilder.java +++ b/src/main/java/edu/lu/uni/serval/utils/PoolBuilder.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPattern.utils; +package edu.lu.uni.serval.utils; import redis.clients.jedis.JedisPoolConfig;