changes

python/common/commons.py

@@ -276,7 +276,7 @@ def shellGitCheckout(cmd,timeout =600,enc='utf-8'):
             logging.warning(cmd +'\t'+str(t))
     return output,errors
 
-def callSpinfer(cmd,timeout =600,enc='utf-8'):
+def callSpinfer(cmd,timeout =900,enc='utf-8'):
     output = ''
     errors = ''
     # logging.debug(cmd)

python/sprinferIndex.py

@@ -6,6 +6,7 @@ SPINFER_INDEX_PATH = os.environ["dataset"]
 COCCI_PATH = join(os.environ["coccinelle"],'spatch')
 DATASET = os.environ["dataset"]
 ROOT_DIR = os.environ["ROOT_DIR"]
+REDIS_PORT = os.environ["REDIS_PORT"]
 
 
 def indexCore():
@@ -163,7 +164,7 @@ def runSpinfer():
     #     #     logging.info(cmd)
     #     #     output, e = shellGitCheckout(cmd)
     #     #     logging.info(output)
-    parallelRun(callSpinfer,bigCmdList)
+    parallelRun(callSpinfer,bigCmdList,max_workers=8)
 
     # if not os.path.exists(join(DATA_PATH,'cocci')):
     #     os.mkdir(join(DATA_PATH,'cocci'))
@@ -253,6 +254,54 @@ def getFreqPatterns():
     re.search(r"// Recall:(.*), Precision:(.*), Matching recall:(.*)")
 
 
+
+def getPatternTypes():
+
+    if isfile(join(DATA_PATH,'allCocciPatternsLast.pickle')):
+        coccis = load_zipped_pickle(join(DATA_PATH,'allCocciPatternsLast.pickle'))
+    else:
+        commentPattern = r"(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)"
+        DATA = '/Users/anil.koyuncu/projects/fixminer/fixminer-data/patches'
+        coccis =os.listdir(join(DATA, 'cocci'))
+        cocciPatterns = pd.DataFrame(columns=['cid', 'pattern','inferedFrom','recall','precision','matchingRecall'])
+        ind = 0
+        for cocci in coccis:
+            try:
+                if cocci == '.DS_Store':
+                    continue
+                with open(join(DATA, 'cocci', cocci), 'r') as iFile:
+                    idx = iFile.read()
+                    idx
+                    inferedFrom = re.search(r"// Infered from:(.*)\n",idx).groups()
+                    recall,precision, matchingRecall = re.search(r"// Recall:(.*), Precision:(.*), Matching recall:(.*)",idx).groups()
+                    pattern = re.sub(commentPattern, '', idx, re.DOTALL)
+                    cocciPatterns.loc[ind] = [cocci,pattern,inferedFrom,recall.strip(),precision.strip(),matchingRecall.strip()]
+                    ind = ind +1
+            except Exception as e:
+                # shutil.move(join(DATA,'merged',cocci),join(DATA,'mergedBroken',cocci))
+                continue
+                # logging.error(e)
+        cocciPatterns['iFiles'] = cocciPatterns.inferedFrom.apply(lambda x: getInferred(x[0]))
+
+        cocciPatterns['freq'] = cocciPatterns.iFiles.apply(lambda x: len(x))
+        cocciPatterns['project'] = cocciPatterns.iFiles.apply(lambda x: list(set([i.split('/{')[0].replace('(','') for i in x])))
+        cocciPatterns.sort_values(by='freq', inplace=True, ascending=False)
+        # save_zipped_pickle(cocciPatterns,join(DATA_PATH,'allCocciPatterns.pickle'))
+        save_zipped_pickle(cocciPatterns,join(DATA_PATH,'allCocciPatternsLast.pickle'))
+
+    coccis
+
+    port = REDIS_PORT
+    import redis
+    redis_db = redis.StrictRedis(host="localhost", port=port, db=0)
+
+    redis_db
+
+    # def getPatternFromRedis(x):
+    #     lines = redis_db.hget(dKey, 'actionTree')
+
+
+
 def removeDuplicates2():
     commentPattern = r"(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)"
     DATA = '/Users/anil.koyuncu/projects/fixminer/fixminer-data2'

python/test_patched_file.py

@@ -73,19 +73,22 @@ def testCore(t):
                 # print("container is ready")
 
                 # print("First_test:", end=' ')
-                output += 'First_test:'
+                # output += 'First_test:'
-                pre_test_outcomes = {}
+                # pre_test_outcomes = {}
-                pre_failure_cases, pre_failure, total, pre_test_outcomes = test_all(bug, container, client)
+                # pre_failure_cases, pre_failure, total, pre_test_outcomes = test_all(bug, container, client)
-                # print("@fail:{}@total:{}".format(pre_failure, total), end=' ')
+                # # print("@fail:{}@total:{}".format(pre_failure, total), end=' ')
-                # print("@pre_failure_cases:{}".format(pre_failure_cases), end=' ')
+                # # print("@pre_failure_cases:{}".format(pre_failure_cases), end=' ')
-                if pre_failure == 0:
+                # if pre_failure == 0:
-                    logging.error(bugName + ' no failed test initially')
+                #     logging.error(bugName + ' no failed test initially')
-                    return ''
+                #     return ''
-                output += '@fail:' + str(pre_failure) + '@total:' + str(total) + ', '
+                # output += '@fail:' + str(pre_failure) + '@total:' + str(total) + ', '
 
-                spfiles = listdir(join(DATASET, 'cocci'))
+                # spfiles = listdir(join(DATASET, 'cocci'))
+                spfiles = load_zipped_pickle(join(DATA_PATH, 'uniquePatternsL.pickle'))
+                spfiles.sort_values(by='uFreq', inplace=True, ascending=False)
+                spfiles = spfiles[['uid']]
                 # print("patching... " + bugName)
-                for idx,spfile in enumerate(spfiles):
+                for idx,spfile in enumerate(spfiles.uid.values.tolist()):
                     if spfile == '.DS_Store':
                         continue