azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge python scripts

Browse Source
This commit is contained in:
fixminer
2020-04-06 21:30:39 +02:00
parent 61a9612345
commit c5463f91f8
75 changed files with 95073 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/patchManyBugs.py
+206
View File
@@ -0,0 +1,206 @@
from common.commons import *
DATA_PATH = os.environ["DATA_PATH"]
SPINFER_PATH = os.environ["spinfer"]
ROOT_DIR = os.environ["ROOT_DIR"]
SPINFER_INDEX_PATH = os.environ["dataset"]
COCCI_PATH = join(os.environ["coccinelle"],'spatch')
DATASET = os.environ["dataset"]
import bugzoo
from bugzoo import server, Container
from bugzoo import Patch, Client
import signal
sosbugs = ['manybugs:gmp:14166-14167', 'manybugs:gzip:2009-08-16-3fe0caeada-39a362ae9d',
'manybugs:gzip:2009-10-09-1a085b1446-118a107f2d', 'manybugs:gzip:2010-02-19-3eb6091d69-884ef6d16c',
'manybugs:libtiff:2005-12-21-3b848a7-3edb9cd', 'manybugs:libtiff:2006-03-03-a72cf60-0a36d7f',
'manybugs:libtiff:2006-03-03-eec4c06-ee65c74', 'manybugs:libtiff:2007-07-13-09e8220-f2d989d',
'manybugs:libtiff:2007-11-02-371336d-865f7b2', 'manybugs:libtiff:2009-02-05-764dbba-2e42d63',
'manybugs:libtiff:2009-08-28-e8a47d4-023b6df', 'manybugs:libtiff:2010-11-27-eb326f9-eec7ec0',
'manybugs:libtiff:2006-02-23-b2ce5d8-207c78a', 'manybugs:lighttpd:2661-2662', 'manybugs:lighttpd:2254-2259',
'manybugs:lighttpd:2785-2786', 'manybugs:lighttpd:1948-1949',
'manybugs:php:2011-12-10-74343ca506-52c36e60c4', 'manybugs:php:2011-04-02-70075bc84c-5a8c917c37',
'manybugs:php:2011-03-25-8138f7de40-3acdca4703', 'manybugs:php:2011-12-04-1e6a82a1cf-dfa08dc325',
'manybugs:php:2012-02-08-ff63c09e6f-6672171672', 'manybugs:php:2011-11-19-eeba0b5681-d3b20b4058',
'manybugs:php:2011-04-07-77ed819430-efcb9a71cd', 'manybugs:php:2011-02-01-01745fa657-1f49902999',
'manybugs:php:2012-03-12-7aefbf70a8-efc94f3115', 'manybugs:php:2011-10-15-0a1cc5f01c-05c5c8958e',
'manybugs:php:2011-01-30-5bb0a44e06-1e91069eb4', 'manybugs:php:2011-02-01-fefe9fc5c7-0927309852',
'manybugs:php:2011-02-27-e65d361fde-1d984a7ffd', 'manybugs:php:2011-03-19-5d0c948296-8deb11c0c3',
'manybugs:php:2011-03-23-63673a533f-2adf58cfcf', 'manybugs:php:2011-04-06-187eb235fe-2e25ec9eb7',
'manybugs:php:2011-04-09-db01e840c2-09b990f499', 'manybugs:php:2011-05-17-453c954f8a-daecb2c0f4',
'manybugs:php:2011-05-24-b60f6774dc-1056c57fa9', 'manybugs:php:2011-10-16-1f78177e2b-d4ae4e79db',
'manybugs:php:2011-10-31-2e5d5e5ac6-b5f15ef561', 'manybugs:php:2011-10-31-c4eb5f2387-2e5d5e5ac6',
'manybugs:php:2011-11-01-ceac9dc490-9b0d73af1d', 'manybugs:php:2011-11-11-fcbfbea8d2-c1e510aea8',
'manybugs:php:2011-11-15-236120d80e-fb37f3b20d', 'manybugs:php:2011-11-16-55acfdf7bd-3c7a573a2c',
'manybugs:php:2011-11-22-ecc6c335c5-b548293b99', 'manybugs:php:2011-11-23-eca88d3064-db0888dfc1',
'manybugs:php:2012-01-27-544e36dfff-acaf9c5227', 'manybugs:php:2012-01-30-9de5b6dc7c-4dc8b1ad11',
'manybugs:php:2012-02-25-c1322d2505-cfa9c90b20', 'manybugs:php:2012-03-04-60dfd64bf2-34fe62619d',
'manybugs:php:2012-03-08-0169020e49-cdc512afb3', 'manybugs:php:2012-03-11-3954743813-d4f05fbffc',
'manybugs:php:2012-03-12-438a30f1e7-7337a901b7', 'manybugs:python:69223-69224',
'manybugs:python:69368-69372', 'manybugs:python:70098-70101', 'manybugs:python:70056-70059',
'manybugs:wireshark:37112-37111', 'manybugs:wireshark:37122-37123', 'manybugs:gmp:13420-13421',
'manybugs:gzip:2009-09-26-a1d3d4019d-f17cbd13a1', 'manybugs:lighttpd:1913-1914',
'manybugs:php:2011-11-19-51a4ae6576-bc810a443d', 'manybugs:php:2011-03-11-d890ece3fc-6e74d95f34',
'manybugs:php:2011-11-19-eeba0b5681-f330c8ab4e', 'manybugs:php:2012-01-01-80dd931d40-7c3177e5ab']
def cocciCore(t):
cmd, manybug, patchName, spfile,srcPath = t
# logging.info(cmd)
output, e = shellGitCheckout(cmd)
# logging.info(output)
patchSize = os.path.getsize(join(DATA_PATH,"manybugs",manybug,'patches',patchName+spfile+'.txt'))
if patchSize == 0 :
os.remove(join(DATA_PATH,"manybugs",manybug,'patches',patchName+spfile+'.txt'))
else:
cmd = 'patch -d '+srcPath.replace(patchName,'') + ' -i ' + join(DATA_PATH,"manybugs",manybug,'patches',patchName+spfile+'.txt') + ' -o ' + join(DATA_PATH,"manybugs",manybug,'patched',patchName+spfile+'.c')
output, e = shellGitCheckout(cmd)
output,e
print(output)
def buildAll():
bugList = []
port = 6000
for i in sosbugs:
t = i,port
bugList.append(t)
if port == 6300:
port = 6000
port += 1
parallelRun(checkBuild, bugList, max_workers=4)
def checkBuild(t):
bugName, port = t
# port = 3000
# cmd = 'bash {} {}'.format(join(DATA_PATH, 'startBugzoo.sh'), port)
cmd = 'bash {} {}'.format(join(ROOT_DIR,'data', 'buildBugzoo.sh'), bugName)
out, e = shellGitCheckout(cmd)
# container = None
# with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) as process:
#
# # o,e = shellGitCheckout(cmd)
# url = "http://127.0.0.1:{}".format(port)
# timeout_connection = 3000
# client = Client(url, timeout_connection=timeout_connection)
#
# try:
# bug = client.bugs[bugName]
# if not client.bugs.is_installed(bug):
# print(' building ' + bugName)
# o = client.bugs.build(bug)
#
# # client.bugs.download(bug)
# # print("the image is not installed :'(")
#
# # print("creating container...")
# container = client.containers.provision(bug)
#
# except Exception as e:
# logging.error(e)
# # continue
# finally:
# # ''
# cmd = 'docker stop {}'.format(container.id)
# out, e = shellGitCheckout(cmd)
# client.shutdown()
# os.killpg(process.pid, signal.SIGTERM)
def patchCore():
try:
pathManyBug = join(DATA_PATH, "manybugs")
manybugs = listdir(pathManyBug)
manybugs = [i for i in manybugs if i in sosbugs]
spfiles = listdir(join(DATASET,'cocci'))
from sprinferIndex import filterPatterns
filteredPattern = filterPatterns()
spfiles = [i for i in spfiles if i in filteredPattern]
workList = []
for manybug in manybugs:
if manybug == '.DS_Store':
continue
# files = listdir(join(join(DATA_PATH,"manybugs",manybug,'diffs')))
if os.path.exists(join(pathManyBug, manybug, 'patches')):
shutil.rmtree(join(pathManyBug, manybug, 'patches'))
os.mkdir(join(pathManyBug, manybug, 'patches'))
else:
os.mkdir(join(pathManyBug, manybug, 'patches'))
if os.path.exists(join(pathManyBug, manybug, 'patched')):
shutil.rmtree(join(pathManyBug, manybug, 'patched'))
os.mkdir(join(pathManyBug, manybug, 'patched'))
else:
os.mkdir(join(pathManyBug, manybug, 'patched'))
bug, fix = manybug.split(':')[-1].split('-')[-2:]
files = get_filepaths(join(pathManyBug, manybug,'diffs'), bug)
sources = [i for i in files if not i.endswith('oracle.c')]
for src in sources:
# srcPath = src.replace('/diffs/','/src/')
srcPath = src
patchName = src.split('/')[-1]
for spfile in spfiles:
cmd = COCCI_PATH + ' --sp-file ' + join(DATASET,'cocci',spfile) + ' ' + srcPath + ' --patch -o' + join(pathManyBug,manybug,'patches',patchName) + ' > ' + join(pathManyBug, manybug, 'patches', patchName + spfile + '.txt')
# cmd = COCCI_PATH + ' --sp-file ' + join(DATASET,'cocci',spfile) + ' ' + srcPath + ' -o ' + join(DATA_PATH,"introclass",manybug,'patches',patchName+spfile+'.txt')
t = cmd,manybug,patchName,spfile,srcPath
workList.append(t)
except Exception as e:
logging.error(e)
parallelRun(cocciCore,workList)
def exportSosPatches():
pathManyBug = join(DATA_PATH, "manybugs")
pathManyBugSos = join(DATA_PATH, "manybugs_sos")
manybugs = listdir(pathManyBug)
manybugs = [i for i in manybugs if i in sosbugs]
for manybug in manybugs:
# shutil.copytree(join(pathManyBug,manybug,'patched'),join(pathManyBugSos,manybug,'patched'))
shutil.copytree(join(pathManyBug,manybug,'patches'),join(pathManyBugSos,manybug,'patches'))
def patched():
pathManyBug = join(DATA_PATH, "manybugs")
manybugs = listdir(pathManyBug)
manybugs = [i for i in manybugs if i in sosbugs]
# spfiles = listdir(join(DATASET,'cocci'))
workList = []
for manybug in manybugs:
if manybug == '.DS_Store':
continue
# files = listdir(join(join(DATA_PATH,"manybugs",manybug,'diffs')))
if os.path.exists(join(DATA_PATH, "manybugs_patched", manybug, 'patches')):
shutil.rmtree(join(DATA_PATH, "manybugs_patched", manybug, 'patches'))
os.makedirs(join(DATA_PATH, "manybugs_patched", manybug, 'patches'))
else:
os.makedirs(join(DATA_PATH, "manybugs_patched", manybug, 'patches'))
files = get_filepaths(join(DATA_PATH, "manybugs", manybug), '.c')
sources = [i for i in files if not i.endswith('oracle.c')]
bug, fix = manybug.split(':')[-1].split('-')[-2:]
files = get_filepaths(join(pathManyBug, manybug, 'diffs'), bug)
sources = [i for i in files if not i.endswith('oracle.c')]
for src in sources:
# srcPath = src.replace('/diffs/','/src/')
srcPath = src
patchName = src.split('/')[-1]
spfiles = listdir(join(DATA_PATH,"manybugs",manybug,'patches'))
for spfile in spfiles:
spfile =spfile.replace(patchName,'').replace('.txt','')
cmd = COCCI_PATH + ' --sp-file ' + join(DATASET,'cocci',spfile) + ' ' + srcPath + ' -o ' + join(DATA_PATH,"manybugs_patched",manybug,'patches',patchName+spfile+'.c')
t = cmd,manybug,patchName,spfile
workList.append(t)
parallelRun(cocciCore,workList)