From 057f5d9136f1245d028e7fe6593e77e4809c559d Mon Sep 17 00:00:00 2001 From: anil Date: Wed, 19 Aug 2020 14:53:20 +0200 Subject: [PATCH] changes --- python/common/commons.py | 2 +- python/sprinferIndex.py | 51 ++++++++++++++++++++++++++++++++++++- python/test_patched_file.py | 25 ++++++++++-------- 3 files changed, 65 insertions(+), 13 deletions(-) diff --git a/python/common/commons.py b/python/common/commons.py index f0f15f5..a081168 100644 --- a/python/common/commons.py +++ b/python/common/commons.py @@ -276,7 +276,7 @@ def shellGitCheckout(cmd,timeout =600,enc='utf-8'): logging.warning(cmd +'\t'+str(t)) return output,errors -def callSpinfer(cmd,timeout =600,enc='utf-8'): +def callSpinfer(cmd,timeout =900,enc='utf-8'): output = '' errors = '' # logging.debug(cmd) diff --git a/python/sprinferIndex.py b/python/sprinferIndex.py index f0ad7ac..84cce83 100644 --- a/python/sprinferIndex.py +++ b/python/sprinferIndex.py @@ -6,6 +6,7 @@ SPINFER_INDEX_PATH = os.environ["dataset"] COCCI_PATH = join(os.environ["coccinelle"],'spatch') DATASET = os.environ["dataset"] ROOT_DIR = os.environ["ROOT_DIR"] +REDIS_PORT = os.environ["REDIS_PORT"] def indexCore(): @@ -163,7 +164,7 @@ def runSpinfer(): # # logging.info(cmd) # # output, e = shellGitCheckout(cmd) # # logging.info(output) - parallelRun(callSpinfer,bigCmdList) + parallelRun(callSpinfer,bigCmdList,max_workers=8) # if not os.path.exists(join(DATA_PATH,'cocci')): # os.mkdir(join(DATA_PATH,'cocci')) @@ -253,6 +254,54 @@ def getFreqPatterns(): re.search(r"// Recall:(.*), Precision:(.*), Matching recall:(.*)") + +def getPatternTypes(): + + if isfile(join(DATA_PATH,'allCocciPatternsLast.pickle')): + coccis = load_zipped_pickle(join(DATA_PATH,'allCocciPatternsLast.pickle')) + else: + commentPattern = r"(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)" + DATA = '/Users/anil.koyuncu/projects/fixminer/fixminer-data/patches' + coccis =os.listdir(join(DATA, 'cocci')) + cocciPatterns = pd.DataFrame(columns=['cid', 'pattern','inferedFrom','recall','precision','matchingRecall']) + ind = 0 + for cocci in coccis: + try: + if cocci == '.DS_Store': + continue + with open(join(DATA, 'cocci', cocci), 'r') as iFile: + idx = iFile.read() + idx + inferedFrom = re.search(r"// Infered from:(.*)\n",idx).groups() + recall,precision, matchingRecall = re.search(r"// Recall:(.*), Precision:(.*), Matching recall:(.*)",idx).groups() + pattern = re.sub(commentPattern, '', idx, re.DOTALL) + cocciPatterns.loc[ind] = [cocci,pattern,inferedFrom,recall.strip(),precision.strip(),matchingRecall.strip()] + ind = ind +1 + except Exception as e: + # shutil.move(join(DATA,'merged',cocci),join(DATA,'mergedBroken',cocci)) + continue + # logging.error(e) + cocciPatterns['iFiles'] = cocciPatterns.inferedFrom.apply(lambda x: getInferred(x[0])) + + cocciPatterns['freq'] = cocciPatterns.iFiles.apply(lambda x: len(x)) + cocciPatterns['project'] = cocciPatterns.iFiles.apply(lambda x: list(set([i.split('/{')[0].replace('(','') for i in x]))) + cocciPatterns.sort_values(by='freq', inplace=True, ascending=False) + # save_zipped_pickle(cocciPatterns,join(DATA_PATH,'allCocciPatterns.pickle')) + save_zipped_pickle(cocciPatterns,join(DATA_PATH,'allCocciPatternsLast.pickle')) + + coccis + + port = REDIS_PORT + import redis + redis_db = redis.StrictRedis(host="localhost", port=port, db=0) + + redis_db + + # def getPatternFromRedis(x): + # lines = redis_db.hget(dKey, 'actionTree') + + + def removeDuplicates2(): commentPattern = r"(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)" DATA = '/Users/anil.koyuncu/projects/fixminer/fixminer-data2' diff --git a/python/test_patched_file.py b/python/test_patched_file.py index 0d5b946..b1f5246 100755 --- a/python/test_patched_file.py +++ b/python/test_patched_file.py @@ -73,19 +73,22 @@ def testCore(t): # print("container is ready") # print("First_test:", end=' ') - output += 'First_test:' - pre_test_outcomes = {} - pre_failure_cases, pre_failure, total, pre_test_outcomes = test_all(bug, container, client) - # print("@fail:{}@total:{}".format(pre_failure, total), end=' ') - # print("@pre_failure_cases:{}".format(pre_failure_cases), end=' ') - if pre_failure == 0: - logging.error(bugName + ' no failed test initially') - return '' - output += '@fail:' + str(pre_failure) + '@total:' + str(total) + ', ' + # output += 'First_test:' + # pre_test_outcomes = {} + # pre_failure_cases, pre_failure, total, pre_test_outcomes = test_all(bug, container, client) + # # print("@fail:{}@total:{}".format(pre_failure, total), end=' ') + # # print("@pre_failure_cases:{}".format(pre_failure_cases), end=' ') + # if pre_failure == 0: + # logging.error(bugName + ' no failed test initially') + # return '' + # output += '@fail:' + str(pre_failure) + '@total:' + str(total) + ', ' - spfiles = listdir(join(DATASET, 'cocci')) + # spfiles = listdir(join(DATASET, 'cocci')) + spfiles = load_zipped_pickle(join(DATA_PATH, 'uniquePatternsL.pickle')) + spfiles.sort_values(by='uFreq', inplace=True, ascending=False) + spfiles = spfiles[['uid']] # print("patching... " + bugName) - for idx,spfile in enumerate(spfiles): + for idx,spfile in enumerate(spfiles.uid.values.tolist()): if spfile == '.DS_Store': continue