From cb9abe7d3e8408e08b482ca539b1a787e1504988 Mon Sep 17 00:00:00 2001 From: "Azalea (on HyDEV-Daisy)" Date: Mon, 16 May 2022 02:16:49 -0400 Subject: [PATCH] [F] Start redis from python --- config.yml | 1 - python/bugstats.py | 2 +- python/common/commons.py | 23 ++++++---- python/main.py | 45 +++++-------------- python/pairs.py | 8 ++-- redis.conf => python/redis.conf | 0 python/stats.py | 2 +- .../serval/richedit/jobs/CompareTrees.java | 20 +++------ .../serval/richedit/jobs/EnhancedASTDiff.java | 4 -- 9 files changed, 38 insertions(+), 67 deletions(-) rename redis.conf => python/redis.conf (100%) diff --git a/config.yml b/config.yml index a6934e0..a60f3c5 100644 --- a/config.yml +++ b/config.yml @@ -10,7 +10,6 @@ dataset: repo: /workspace/EECS-Research/data/1080/datasets fixminer: - srcPath: /workspace/EECS-Research/fixminer_source projectType : java datapath: /workspace/EECS-Research/data/1080 diff --git a/python/bugstats.py b/python/bugstats.py index aa9be1c..e625ba8 100644 --- a/python/bugstats.py +++ b/python/bugstats.py @@ -13,7 +13,7 @@ def bStats(): dbDir = join(DATA_PATH, 'redis') portInner = '6399' - startDB(dbDir, portInner, PROJECT_TYPE) + redis_start(dbDir, portInner, PROJECT_TYPE) import redis diff --git a/python/common/commons.py b/python/common/commons.py index 4e8a72d..3417740 100644 --- a/python/common/commons.py +++ b/python/common/commons.py @@ -493,20 +493,24 @@ def get_class_weights(y): return {cls: round(float(majority) / float(count), 2) for cls, count in counter.items()} -def stopDB(dbDir, portInner): - # cmd = "bash " + dbDir + "/" + "stopServer.sh " + " " + portInner; - cmd = "redis-cli -p " + portInner + " shutdown save" +def redis_shutdown(port: int): + print(f'Shutting down redis {port}...') + cmd = f"redis-cli -p {port} shutdown save" o, e = shellGitCheckout(cmd) logging.info(o) + print('> Shutdown complete.') -def startDB(dbDir, portInner, projectType): - dbName = "dumps-" + projectType + ".rdb" - # portInner = '6380' - cmd = "bash " + dbDir + "/" + "startServer.sh " + dbDir + " " + dbName + " " + portInner; +def redis_start(root_dir: str, db_dir: str, port: int): + Path(db_dir).mkdir(exist_ok=True, parents=True) + + print(f'Starting redis {port}...') + cmd = f"redis-server {root_dir}/redis.conf --dir {db_dir} --dbfilename redis.rdb --port {port} --daemonize yes" o, e = shellGitCheckout(cmd) - ping = "redis-cli -p " + portInner + " ping" + assert not e, e + + ping = f"redis-cli -p {port} ping" o, e = shellGitCheckout(ping) m = re.search('PONG', o) @@ -515,7 +519,8 @@ def startDB(dbDir, portInner, projectType): logging.info('Waiting for checkout') o, e = shellGitCheckout(ping) m = re.search('PONG', o) - print(o) + + print('> Redis started.') def unique_everseen(iterable, key=None): diff --git a/python/main.py b/python/main.py index 2b8a7d0..52f3374 100644 --- a/python/main.py +++ b/python/main.py @@ -32,10 +32,15 @@ def job_dataset4c(): core() +def job_start_redis(): + db_dir = join(DATA_PATH, 'redis') + redis_shutdown(REDIS_PORT) + redis_start(ROOT_DIR, db_dir, REDIS_PORT) + + def job_richedit(): - dbDir = join(DATA_PATH, 'redis') - stopDB(dbDir, REDIS_PORT) - cmd = f"JAVA_HOME='{jdk8}' java -jar '{JAR_PATH}' {args.prop} RICHEDITSCRIPT " + job_start_redis() + cmd = f"JAVA_HOME='{jdk8}' java -jar '{JAR_PATH}' {args.prop} RICHEDITSCRIPT" output = shellCallTemplate(cmd) logging.info(output) @@ -49,11 +54,8 @@ def job_actionSI(): def job_compare(): - # cmd = "mvn exec:java -f '/data/fixminer_source/' - # -Dexec.mainClass='edu.lu.uni.serval.richedit.akka.compare.CompareTrees' - # -Dexec.args='"+ " shape " + join(DATA_PATH,"redis") +" ALLdumps-gumInput.rdb " + - # "clusterl0-gumInputALL.rdb /data/richedit-core/python/data/richEditScript'" - cmd = f"JAVA_HOME='{jdk8}' java -jar '{JAR_PATH}' {args.prop} COMPARE " + job_start_redis() + cmd = f"JAVA_HOME='{jdk8}' java -jar '{JAR_PATH}' {args.prop} COMPARE" output = shellCallTemplate4jar(cmd) logging.info(output) @@ -62,7 +64,7 @@ def job_cluster(): from abstractPatch import cluster dbDir = join(DATA_PATH, 'redis') - startDB(dbDir, REDIS_PORT, PROJECT_TYPE) + redis_start(dbDir, REDIS_PORT, PROJECT_TYPE) cluster(join(DATA_PATH, 'actions'), join(DATA_PATH, 'pairs'), 'actions') @@ -77,7 +79,7 @@ def job_clusterTokens(): from abstractPatch import cluster dbDir = join(DATA_PATH, 'redis') - startDB(dbDir, REDIS_PORT, PROJECT_TYPE) + redis_start(dbDir, REDIS_PORT, PROJECT_TYPE) cluster(join(DATA_PATH, 'tokens'), join(DATA_PATH, 'pairsToken'), 'tokens') @@ -95,16 +97,6 @@ def job_indexClusters(): divideCoccis() removeDuplicates() - # from patchManyBugs import patchCore - # patchCore() - # # from patchManyBugs import patched - # # patched() - # from patchManyBugs import exportSosPatches - # exportSosPatches() - # from validate_manybugs import validate - # - # validate() - def job_patternOperations(): from sprinferIndex import patternOperations @@ -117,27 +109,14 @@ def job_patchManyBugs(): buildAll() - # from patchManyBugs import patchCore - # patchCore() - # # from patch_validate import patch_validate_mine - # # patch_validate_mine() - # from patchManyBugs import patched - # patched() - # from patchManyBugs import exportSosPatches - # exportSosPatches() - def job_patchIntro(): from sprinferIndex import patchCoreIntro patchCoreIntro() - # from sprinferIndex import patched - # patched() def job_validateIntro(): - # from patch_validate_introClass2 import patch_validate - # patch_validate() from test_patched_file import patch_validate patch_validate() diff --git a/python/pairs.py b/python/pairs.py index f339a69..52899e8 100644 --- a/python/pairs.py +++ b/python/pairs.py @@ -12,7 +12,7 @@ def importTokens(): # portInner = '6380' # startDB(dbDir, portInner, "clusterl1-gumInputALL.rdb") portInner = REDIS_PORT - startDB(dbDir, portInner, PROJECT_TYPE) + redis_start(dbDir, portInner, PROJECT_TYPE) import redis # import pairs @@ -48,7 +48,7 @@ def importAction(): # startDB(dbDir, portInner, "clusterl0-gumInputALL.rdb") portInner = REDIS_PORT - startDB(dbDir, portInner, PROJECT_TYPE) + redis_start(dbDir, portInner, PROJECT_TYPE) import redis pairsShapes = join(DATA_PATH, 'pairs') @@ -140,7 +140,7 @@ def actionPairs(): dbDir = join(INNER_DATA_PATH, 'redis') portInner = REDIS_PORT - startDB(dbDir, portInner, PROJECT_TYPE) + redis_start(dbDir, portInner, PROJECT_TYPE) import redis redis_db = redis.StrictRedis(host="localhost", port=portInner, db=0) @@ -244,4 +244,4 @@ def createPairs(matches): for row in pairs: a, b = row out.write(a + ',' + b + '\n') - return False \ No newline at end of file + return False diff --git a/redis.conf b/python/redis.conf similarity index 100% rename from redis.conf rename to python/redis.conf diff --git a/python/stats.py b/python/stats.py index b2f11e3..414a1cf 100644 --- a/python/stats.py +++ b/python/stats.py @@ -266,7 +266,7 @@ def defects4jStats(isFixminer=False): dbDir = join(DATA_PATH, 'redis') portInner = REDIS_PORT - startDB(dbDir, portInner, PROJECT_TYPE ) + redis_start(dbDir, portInner, PROJECT_TYPE) import redis diff --git a/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/CompareTrees.java b/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/CompareTrees.java index 21f6f79..7d15b88 100755 --- a/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/CompareTrees.java +++ b/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/CompareTrees.java @@ -23,16 +23,9 @@ public class CompareTrees { private static final Logger log = LoggerFactory.getLogger(CompareTrees.class); - public static void main(String redisPath, String portDumps, String dumpsName, String numOfWorkers) throws Exception + public static void main(String redisPath, String port, String dumpsName, String numOfWorkers) throws Exception { - String port = portDumps; //"6399"; - CallShell cs = new CallShell(); - String cmd = "bash " + redisPath + "/" + "startServer.sh" + " %s %s %s"; - cmd = String.format(cmd, redisPath, dumpsName, Integer.valueOf(port)); - log.info(cmd); - CallShell.runShell(cmd, port); - - final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "localhost", Integer.valueOf(port), 20000000); + final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "localhost", Integer.parseInt(port), 20000000); HashMap filenames = getFilenames(outerPool); String job = getLevel(outerPool); @@ -46,12 +39,11 @@ public class CompareTrees } IntStream stream = IntStream.range(0, compare.intValue()); - String finalJob = job; ProgressBar.wrap(stream.parallel(), "Task").forEach(m -> - { - newCoreCompare(finalJob, errorPairs, filenames, outerPool); - } - ); + { + newCoreCompare(job, errorPairs, filenames, outerPool); + } + ); log.info("End process"); } diff --git a/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/EnhancedASTDiff.java b/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/EnhancedASTDiff.java index 8fcf524..3bc7552 100755 --- a/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/EnhancedASTDiff.java +++ b/richedit/src/main/java/edu/lu/uni/serval/richedit/jobs/EnhancedASTDiff.java @@ -33,10 +33,6 @@ public class EnhancedASTDiff String parameters = String.format("\nInput path %s", inputPath); log.info(parameters); - // Run Redis - String cmd = String.format("redis-server %s/redis.conf --dir %s --dbfilename redis.rdb --port %s --daemonize yes", srcPath, dbDir, redisPort); - CallShell.runShell(cmd, redisPort); - JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1", Integer.parseInt(redisPort), 20000000); boolean isJava = projectType.equals("java");