random key for compare tree

src/main/java/edu/lu/uni/serval/fixminer/Launcher.java

@@ -75,10 +75,10 @@ public class Launcher {
         try {
             switch (jobType) {
                 case "ENHANCEDASTDIFF":
-                    EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType,parallelism);
+                    EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType,parallelism,portDumps, dbDir, actionType+dumpsName);
                     break;
                 case "CACHE":
-                    StoreEDiffInCache.main(gumOutput, portDumps, dbDir, actionType+dumpsName,actionType);
+                    StoreEDiffInCache.main(gumOutput, portDumps, dbDir, actionType+dumpsName);
                     break;
                 case "SI":
 //                    CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor);

src/main/java/edu/lu/uni/serval/fixminer/akka/DocumentSimilarity.java

@@ -0,0 +1,601 @@
+package edu.lu.uni.serval.fixminer.akka;
+
+//Import the Java utility classes
+
+import java.io.BufferedReader;
+
+import java.io.FileReader;
+
+import java.io.PrintWriter;
+
+import java.util.ArrayList;
+
+import java.util.HashMap;
+
+import java.util.HashSet;
+
+import java.util.Map;
+
+import java.util.Set;
+
+//Import the Apache Commons Maths library classes to handle the term-vectors from Apache Lucene
+
+import org.apache.commons.math3.linear.ArrayRealVector;
+
+import org.apache.commons.math3.linear.RealVector;
+
+//Import the Apache Lucene classes
+
+import org.apache.lucene.analysis.Analyzer;
+
+import org.apache.lucene.analysis.core.SimpleAnalyzer;
+
+import org.apache.lucene.document.Document;
+
+import org.apache.lucene.document.Field;
+
+import org.apache.lucene.document.FieldType;
+
+import org.apache.lucene.index.AtomicReader;
+
+import org.apache.lucene.index.DirectoryReader;
+
+import org.apache.lucene.index.DocsEnum;
+
+import org.apache.lucene.index.IndexReader;
+
+import org.apache.lucene.index.IndexWriter;
+
+import org.apache.lucene.index.IndexWriterConfig;
+
+import org.apache.lucene.index.MultiFields;
+
+import org.apache.lucene.index.Term;
+
+import org.apache.lucene.index.Terms;
+
+import org.apache.lucene.index.TermsEnum;
+
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+
+import org.apache.lucene.search.similarities.TFIDFSimilarity;
+
+import org.apache.lucene.store.Directory;
+
+import org.apache.lucene.store.RAMDirectory;
+
+import org.apache.lucene.util.Bits;
+
+import org.apache.lucene.util.BytesRef;
+
+import org.apache.lucene.util.Version;
+
+
+
+import com.thoughtworks.xstream.core.util.Fields;
+
+
+
+//A class to find the similarity of all text documents referenced in a text file
+
+public class DocumentSimilarity {
+
+    private static final String CONTENT = "Content";  /*the name of the field stored by Apache Lucene which includes the text from the
+
+text documents.*/
+
+    private Set<String> terms = new HashSet<>(); //changed state with new terms after each call to getTermFrequencies() method
+
+
+
+    private ArrayList<Integer> directoryIndex; //The Apache Lucene index for documents to mine for similarities
+
+    /* Indexed, tokenized, stored. */
+
+    public final FieldType TYPE_STORED = new FieldType(); //This is the field profiled by Apache Lucene to collect its term frequencies
+
+    private HashMap<String,Integer> termsCount; //stores the counts of all terms found in the corpus
+
+    private ArrayList<Integer> scannedDocs; //stores the IDs of docs already scanned and stored in termsCount
+
+
+
+
+
+    DocumentSimilarity(String lookupFile) {
+
+        //Initialise the field to profile in each document
+
+        TYPE_STORED.setIndexed(true);
+
+        TYPE_STORED.setTokenized(true);
+
+        TYPE_STORED.setStored(true);
+
+        TYPE_STORED.setStoreTermVectors(true);
+
+        TYPE_STORED.setStoreTermVectorPositions(true);
+
+        TYPE_STORED.freeze();
+
+
+
+        termsCount = new HashMap<String,Integer>();
+
+        scannedDocs = new ArrayList<Integer>();
+
+
+
+        try {
+
+            //initialise an in-memory (RAM-based) index of the documents in the input folder using Apache Lucene
+
+            Directory directory = createIndex(lookupFile);
+
+            IndexReader reader = DirectoryReader.open(directory);
+
+
+
+            String outputFile = "C:/Users/default/test/docs_similarity.csv"; /* the output file where the document similarities will be stored */
+
+            PrintWriter writer;
+
+            writer = new PrintWriter(outputFile, "UTF-8");
+
+            writer.println("doc1,doc2,similarity");
+
+
+
+
+
+            //loop on the documents in the folder and compare them together
+
+            for (int i=0; i<directoryIndex.size(); i++) {
+
+                for (int j=i+1; j<directoryIndex.size(); j++) {
+
+                    Map<String, Double> f1 = getTermFrequencies(reader, i); //get the term frequencies profile of the first document
+
+                    RealVector v1 = toRealVector(f1); //convert term frequencies profile to a vector
+
+                    Map<String, Double> f2 = getTermFrequencies2(reader, j); //get the term frequencies profile of the second document
+
+                    RealVector v2 = toRealVector(f2); //convert term frequencies profile to a vector
+
+                    double sim = getCosineSimilarity(v1, v2);  //compute the cosine similarity of the documents pair using their terms frequencies profiles
+
+                    writer.println(directoryIndex.get(i)+","+directoryIndex.get(j)+","+sim); //write the similarity to an output CSV file
+
+                }
+
+                terms = new HashSet<>();
+
+            }
+
+            reader.close();
+
+            writer.close();
+
+        } catch (Exception e) {
+
+            e.printStackTrace();
+
+        }
+
+
+
+        writeTermsCount();
+
+    }
+
+
+
+
+
+    //A method to initialise an in-memory (RAM-based) index of the documents in the input file using Apache Lucene
+
+    Directory createIndex(String lookupFile)  {
+
+        try {
+
+            Directory directory = new RAMDirectory();
+
+
+
+            /* Initialize the analyzer which profiles the text inside the documents. This analyser does all the pre-processing of the
+
+             * text including stop-word removal, tokenization, etc. depending on the type of analyzer used.
+
+             * Check the reference website for different types of analyzers: https://www.tutorialspoint.com/lucene/lucene_analysis.htm
+
+             */
+
+            Analyzer analyzer = new SimpleAnalyzer(Version.LUCENE_CURRENT); //create a new SimpleAnalyzer
+
+            IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_CURRENT,
+
+                    analyzer);
+
+            IndexWriter writer = new IndexWriter(directory, iwc); //write the index of the documents with their analysis in-memory
+
+
+
+            int did=1; //documents are numbered by a document ID as 1,2,3...n in a lookup file of the documents to be indexed
+
+            BufferedReader dsLookup;
+
+            dsLookup = new BufferedReader(new FileReader(lookupFile));
+
+            String ds; //The document number in each line
+
+
+
+            ArrayList<Integer> directoryIndex = new ArrayList<Integer>();
+
+
+
+            while ((ds = dsLookup.readLine()) != null) {
+
+                //Set the document id to the string read from the line in lookup file
+
+                try {
+
+
+
+                    String documentText = ""; //stores the text from the document
+
+
+
+                    FileReader fr = new FileReader(did+".txt");
+
+                    BufferedReader br = new BufferedReader(fr);
+
+                    String sCurrentLine;
+
+                    while ((sCurrentLine = br.readLine()) != null) {
+
+                        documentText += (sCurrentLine);
+
+                    }
+
+
+
+                    //add the document to the index
+
+                    addDocument(writer, documentText);
+
+                    directoryIndex.add(did);
+
+                } catch (Exception e) {
+
+                    System.out.println("!!! Document "+did+" has an error and was skipped !!!");
+
+                    continue;
+
+                }
+
+            }
+
+            writer.close();
+
+            this.directoryIndex = directoryIndex;
+
+            return directory;
+
+        } catch (Exception e) {
+
+            e.printStackTrace();
+
+        }
+
+        return null;
+
+    }
+
+
+
+
+
+    //a method to add a document to the index by writing its text as a "content" field in Apache Lucene
+
+    void addDocument(IndexWriter writer, String content) {
+
+        try {
+
+            Document doc = new Document(); //create a new Apache Lucene document
+
+            Field field = new Field(CONTENT, content, TYPE_STORED); //Add the field with the text content to the index
+
+            doc.add(field);
+
+            writer.addDocument(doc);
+
+        } catch (Exception e) {
+
+            e.printStackTrace();
+
+        }
+
+
+
+    }
+
+
+
+
+    //a method to get the cosine similarity between two term-frequency vectors from a pair of documents
+
+    public double getCosineSimilarity(RealVector v1, RealVector v2) {
+
+        return (v1.dotProduct(v2)) / (v1.getNorm() * v2.getNorm());
+
+    }
+
+
+
+
+
+    //a method to get the term frequencies from a document
+
+    Map<String, Double> getTermFrequencies(IndexReader reader, int docId) {
+
+        try {
+
+            Terms vector = reader.getTermVector(docId, CONTENT);
+
+            TermsEnum termsEnum = null;
+
+            termsEnum = vector.iterator(termsEnum);
+
+            Map<String, Double> frequencies = new HashMap<>();
+
+            BytesRef text = null;
+
+            TFIDFSimilarity tfidfSim = new DefaultSimilarity();
+
+            boolean scannedDoc = scannedDocs.contains(docId);
+
+
+
+            while ((text = termsEnum.next()) != null) {
+
+                String term = text.utf8ToString();
+
+                org.apache.lucene.index.Fields fields = reader.getTermVectors(0);
+
+                Term termInstance = new Term("Content", term);
+
+                long indexDf = reader.docFreq(termInstance);
+
+                int docCount = reader.numDocs();
+
+
+
+                //increment the term count in the terms count lookup if doc not scanned before
+
+                if(!scannedDoc) {
+
+                    if(termsCount.containsKey(termInstance.toString())) {
+
+                        Integer cnt = termsCount.get(termInstance.toString());
+
+                        cnt++;
+
+                        termsCount.replace(termInstance.toString(), cnt);
+
+                    } else {
+
+                        termsCount.put(termInstance.toString(), 1);
+
+                    }
+
+                }
+
+
+
+
+
+                DocsEnum docs = termsEnum.docs(MultiFields.getLiveDocs(reader),null,0);
+
+
+
+                //calculate the TF-IDF of the term, as compared to all documents in the corpus (the Apache Lucene Index)
+
+                double tfidf = 0.0;
+
+                while(docs.nextDoc() != DocsEnum.NO_MORE_DOCS)  {
+
+                    tfidf = tfidfSim.tf(docs.freq()) * tfidfSim.idf(docCount, indexDf);
+
+                }
+
+
+
+
+
+                frequencies.put(term, tfidf);
+
+                scannedDocs.add(docId);
+
+                terms.add(term);
+
+            }
+
+            return frequencies;
+
+        } catch (Exception e) {
+
+            e.printStackTrace();
+
+        }
+
+        return null;
+
+    }
+
+
+
+    Map<String, Double> getTermFrequencies2(IndexReader reader, int docId) {
+
+        try {
+
+            Terms vector = reader.getTermVector(docId, CONTENT);
+
+            TermsEnum termsEnum = null;
+
+            termsEnum = vector.iterator(termsEnum);
+
+            Map<String, Double> frequencies = new HashMap<>();
+
+            BytesRef text = null;
+
+            TFIDFSimilarity tfidfSim = new DefaultSimilarity();
+
+            boolean scannedDoc = scannedDocs.contains(docId);
+
+            int docCount = reader.numDocs();
+
+
+
+            while ((text = termsEnum.next()) != null) {
+
+                String term = text.utf8ToString();
+
+                Term termInstance = new Term("Content", term);
+
+                long indexDf = reader.docFreq(termInstance);
+
+
+
+
+
+                //increment the term count in the terms count lookup if doc not scanned before
+
+                if(!scannedDoc) {
+
+                    if(termsCount.containsKey(termInstance.toString())) {
+
+                        Integer cnt = termsCount.get(termInstance.toString());
+
+                        cnt++;
+
+                        termsCount.replace(termInstance.toString(), cnt);
+
+                    } else {
+
+                        termsCount.put(termInstance.toString(), 1);
+
+                    }
+
+                }
+
+
+
+                DocsEnum docs = termsEnum.docs(MultiFields.getLiveDocs(reader),null,0);
+
+                double tfidf = 0.0;
+
+                while(docs.nextDoc() != DocsEnum.NO_MORE_DOCS)  {
+
+                    tfidf = tfidfSim.tf(docs.freq()) * tfidfSim.idf(docCount, indexDf);
+
+                }
+
+
+
+
+
+                frequencies.put(term, tfidf);
+
+                scannedDocs.add(docId);
+
+
+
+            }
+
+            return frequencies;
+
+        } catch (Exception e) {
+
+            e.printStackTrace();
+
+        }
+
+        return null;
+
+    }
+
+
+
+
+
+    //convert the term-frequencies extracted to a real vector
+
+    RealVector toRealVector(Map<String, Double> map) {
+
+        RealVector vector = new ArrayRealVector(terms.size());
+
+        int i = 0;
+
+        for (String term : terms) {
+
+            double value = map.containsKey(term) ? map.get(term) : 0.0;
+
+            vector.setEntry(i++, value);
+
+        }
+
+        return (RealVector) vector.mapDivide(vector.getL1Norm());
+
+    }
+
+
+
+
+    //Write the terms-frequencies from the documents to a CSV file
+
+    private void writeTermsCount() {
+
+        try {
+
+            String outputFile = "C:/Users/default/test/document_terms_count.csv";
+
+            PrintWriter writer;
+
+            writer = new PrintWriter(outputFile, "UTF-8");
+
+            writer.println("term,count");
+
+
+
+            for (String key : termsCount.keySet()) {
+
+                writer.println(key+","+termsCount.get(key));
+
+            }
+
+
+
+            writer.close();
+
+
+
+        } catch (Exception e) {
+
+            e.printStackTrace();
+
+        }
+
+    }
+
+
+
+    //MAIN class to execute with the .txt-file lookup of the documents to index
+
+    public static void main(String[] args) {
+
+        new DocumentSimilarity("documents_lookup.txt");
+
+    }
+
+}

src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java

@@ -26,19 +26,19 @@ public class AkkaTreeParser {
 
         switch (parallelism){
             case "AKKA":
-                ActorSystem system = null;
-                ActorRef parsingActor = null;
-                final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout);
-                try {
-                    log.info("Akka begins...");
-                    system = ActorSystem.create("Compare-EnhancedDiff-System");
-
-                    parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor");
-                    parsingActor.tell(msg, ActorRef.noSender());
-                } catch (Exception e) {
-                    system.shutdown();
-                    e.printStackTrace();
-                }
+//                ActorSystem system = null;
+//                ActorRef parsingActor = null;
+//                final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout);
+//                try {
+//                    log.info("Akka begins...");
+//                    system = ActorSystem.create("Compare-EnhancedDiff-System");
+//
+//                    parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor");
+//                    parsingActor.tell(msg, ActorRef.noSender());
+//                } catch (Exception e) {
+//                    system.shutdown();
+//                    e.printStackTrace();
+//                }
                 break;
             case "FORKJOIN":
                 int counter = new Object() {
@@ -70,6 +70,15 @@ public class AkkaTreeParser {
 
 
 
+    }
+    public static List<String> getRMessages(JedisPool innerPool, int cursor){
+        try (Jedis inner = innerPool.getResource()) {
+            while (!inner.ping().equals("PONG")){
+                log.info("wait");
+            }
+            List<String> pairs = inner.srandmember("pairs", cursor);
+            return pairs;
+        }
     }
 
     public static List<String> getMessages(JedisPool innerPool, int cursor){
@@ -81,7 +90,6 @@ public class AkkaTreeParser {
             while (!inner.ping().equals("PONG")){
                 log.info("wait");
             }
-
             ScanParams sc = new ScanParams();
             //150000000
             log.info("Scanning ");
@@ -91,6 +99,7 @@ public class AkkaTreeParser {
 //            sc.match("pair_[0-9]*");
 
             scan = inner.scan("0", sc);
+
             int size = scan.getResult().size();
             log.info("Scanned " + String.valueOf(size));
         }
@@ -102,5 +111,27 @@ public class AkkaTreeParser {
 
 
 
+    }
+
+    public static String getMessage(JedisPool innerPool){
+
+
+        ScanResult<String> scan;
+
+        try (Jedis inner = innerPool.getResource()) {
+            while (!inner.ping().equals("PONG")){
+                log.info("wait");
+            }
+
+            String myset = inner.spop("pairs");
+            return myset;
+
+        }
+
+
+
+
+
+
     }
 }

src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java

@@ -1,19 +1,25 @@
 package edu.lu.uni.serval.fixminer.akka.compare;
 
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
 import com.github.gumtreediff.actions.ActionGenerator;
 import com.github.gumtreediff.actions.model.Action;
 import com.github.gumtreediff.matchers.Matcher;
 import com.github.gumtreediff.matchers.Matchers;
 import com.github.gumtreediff.tree.ITree;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
+import edu.lu.uni.serval.utils.CallShell;
 import edu.lu.uni.serval.utils.EDiffHelper;
 import edu.lu.uni.serval.utils.PoolBuilder;
 import org.apache.commons.text.similarity.JaroWinklerDistance;
+import org.javatuples.Pair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import redis.clients.jedis.Jedis;
 import redis.clients.jedis.JedisPool;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -29,36 +35,88 @@ public class CompareTrees {
     private static Logger log = LoggerFactory.getLogger(CompareTrees.class);
 
 
-    public static void main(String[] args) throws IOException {
+    public static void main(String[] args) throws Exception {
+
+
 
         String portInner = "6380";
         String port = "6399";
+        CallShell cs = new CallShell();
+        String cmd = "bash "+args[1] + "/" + "startServer.sh" +" %s %s %s";
+        cmd = String.format(cmd, args[1],args[2],Integer.valueOf(port));
+        log.info(cmd);
+        cs.runShell(cmd, port);
+
+        String cmdInner = "bash "+args[1] + "/" + "startServer.sh" +" %s %s %s";
+        cmdInner = String.format(cmdInner, args[1],args[3],Integer.valueOf(portInner));
+        log.info(cmdInner);
+        cs.runShell(cmdInner, portInner);
+
+        String numOfWorkers = args[4];
+        String host = args[5];
+//  -Djava.util.concurrent.ForkJoinPool.common.parallelism=256
+
+        final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000);
+
+        final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(port),20000000);
+
+        List<String> listOfPairs = AkkaTreeParser.getRMessages(innerPool,Integer.valueOf(numOfWorkers));
+
+//        String pair = AkkaTreeParser.getMessage(innerPool);
+
+
+
+//        ActorSystem system = null;
+//        ActorRef parsingActor = null;
+//        final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,timeout,args[0]);
+//        try {
+//            log.info("Akka begins...");
+//            Config load = ConfigFactory.load();
+//            system = ActorSystem.create("Compare-EnhancedDiff-System");
 //
-        JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
+//            parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor");
+//            parsingActor.tell(msg, ActorRef.noSender());
+//        } catch (Exception e) {
+//            system.shutdown();
+//            e.printStackTrace();
+//        }
+//        try {
+//            coreCompare(pair, args[0], innerPool, outerPool);
+//        } catch (Exception e) {
+//            try (Jedis inner = innerPool.getResource()) {
+//
+//                inner.sadd("pairs",pair);
+//
+//
+//            }
+//        }
 
-        JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000);
+        listOfPairs.stream().parallel().forEach(m->coreCompare(m, args[0],innerPool, outerPool));
+//        listOfPairs.parallelStream().forEach(m->coreCompare(m, args[0],innerPool, outerPool));
 
-        List<String> listOfPairs = AkkaTreeParser.getMessages(innerPool,100000000);
-//        listOfPairs.stream().parallel().
+//        for (String listOfPair : listOfPairs) {
+//            coreCompare(listOfPair, args[0],innerPool, outerPool);
+//        }
 
-        int counter = new Object() {
-            int counter = 0;
 
-            {
-
-                listOfPairs.parallelStream().
-                        peek(x -> counter++).
-                        forEach(m ->
-                                {
-//                                    Compare compare =  new Compare();
-                                    coreCompare(m, args[0],innerPool, outerPool);
-                                    if (counter % 1000 == 0) {
-                                        log.info("Finalized parsing " + counter + " files... remaing " + (listOfPairs.size() - counter));
-                                    }
-                                }
-                        );
-            }
-        }.counter;
+//        int counter = new Object() {
+//            int counter = 0;
+//
+//            {
+//
+//                listOfPairs.parallelStream().
+//                        peek(x -> counter++).
+//                        forEach(m ->
+//                                {
+////                                    Compare compare =  new Compare();
+//                                    coreCompare(m, args[0],innerPool, outerPool);
+//                                    if (counter % 1000 == 0) {
+//                                        log.info("Finalized parsing " + counter + " files... remaing " + (listOfPairs.size() - counter));
+//                                    }
+//                                }
+//                        );
+//            }
+//        }.counter;
 //        coreCompare(args[0],args[1],args[2],args[3]);
     }
     public static void coreCompare(String pairName, String treeType,JedisPool innerPool,JedisPool outerPool ) {
@@ -75,6 +133,8 @@ public class CompareTrees {
         Jedis jedis = null;
         ITree oldTree = null;
         ITree newTree = null;
+        Pair<ITree, HierarchicalActionSet> oldPair = null;
+        Pair<ITree, HierarchicalActionSet> newPair = null;
 
         try {
             jedis = innerPool.getResource();
@@ -88,45 +148,55 @@ public class CompareTrees {
 
             switch (treeType) {
                 case "shape":
-                    oldTree = EDiffHelper.getShapes(keyName, i, outerPool);
-                    newTree = EDiffHelper.getShapes(keyName, j, outerPool);
+                    oldTree = EDiffHelper.getShapes(keyName, i, outerPool,innerPool);
+                    newTree = EDiffHelper.getShapes(keyName, j, outerPool,innerPool);
                     break;
                 case "action":
-                    oldTree = EDiffHelper.getActions(keyName, i, outerPool,innerPool);
-                    newTree = EDiffHelper.getActions(keyName, j, outerPool,innerPool);
+                    oldPair = EDiffHelper.getActions(keyName, i, outerPool,innerPool);
+                    newPair = EDiffHelper.getActions(keyName, j, outerPool,innerPool);
+                    oldTree = oldPair.getValue0();
+                    newTree = newPair.getValue0();
+
+
                     break;
                 case "token":
                     oldTree = EDiffHelper.getTokens(keyName, i, outerPool,innerPool);
                     newTree = EDiffHelper.getTokens(keyName, j, outerPool,innerPool);
-                    List<String> oldTokens = new ArrayList<>();
-                    List<String> newTokens = new ArrayList<>();
+//                    List<String> oldTokens = new ArrayList<>();
+//                    List<String> newTokens = new ArrayList<>();
 
 
 
 
-                    oldTokens = getNames(oldTree,oldTokens);
-                    newTokens = getNames(newTree,newTokens);
+                    String oldTokens = EDiffHelper.getNames2(oldTree);
+                    String newTokens = EDiffHelper.getNames2(newTree);
 
 
-                    CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]);
-                    CharSequence[] newSequences = newTokens.toArray(new CharSequence[newTokens.size()]);
+//                    CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]);
+//                    CharSequence[] newSequences = newTokens.toArray(new CharSequence[newTokens.size()]);
                     JaroWinklerDistance jwd = new JaroWinklerDistance();
 
-                    Double overallSimi = Double.valueOf(0);
-                    if(oldSequences.length > 0 && (oldSequences.length == newSequences.length)){
-                        for (int idx = 0; idx < newSequences.length; idx++) {
-                            Double simi = jwd.apply(newSequences[idx], oldSequences[idx]);
-                            overallSimi = simi + overallSimi;
-                        }
-                        overallSimi = overallSimi / oldSequences.length;
-                    }else{
-                        overallSimi = Double.valueOf(0);
-//                    if(oldSequences.length != 0) {
-//                        log.info("ERROR");
-//                    }
-                    }
 
-                    int retval = Double.compare(overallSimi, Double.valueOf(0.8));
+                    Double overallSimi = Double.valueOf(0);
+
+                    if (! (oldTokens.trim().isEmpty() || newTokens.trim().isEmpty())){
+                        overallSimi = jwd.apply(oldTokens, newTokens);
+
+                    }
+//                    if(oldSequences.length > 0 && (oldSequences.length == newSequences.length)){
+//                        for (int idx = 0; idx < newSequences.length; idx++) {
+//                            Double simi = jwd.apply(newSequences[idx], oldSequences[idx]);
+//                            overallSimi = simi + overallSimi;
+//                        }
+//                        overallSimi = overallSimi / oldSequences.length;
+//                    }else{
+//                        overallSimi = Double.valueOf(0);
+////                    if(oldSequences.length != 0) {
+////                        log.info("ERROR");
+////                    }
+//                    }
+
+                    int retval = Double.compare(overallSimi, Double.valueOf(1));
                     String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j);
                     if(retval >= 0){
 
@@ -137,7 +207,8 @@ public class CompareTrees {
                         jedis.set(matchKey, result);
                     }
                     jedis.select(0);
-                    jedis.del(matchKey);
+                    jedis.srem("pairs",matchKey);
+//                    jedis.del(matchKey);
 
                     return;
                 default:
@@ -152,31 +223,81 @@ public class CompareTrees {
             ag.generate();
             List<Action> actions = ag.getActions();
 
-            double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree);
-            String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1);
-            double diceSimilarity1 = m.diceSimilarity(oldTree, newTree);
-            String diceSimilarity = String.format("%1.2f", diceSimilarity1);
-            double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree);
-            String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1);
+//            double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree);
+//            String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1);
+//            double diceSimilarity1 = m.diceSimilarity(oldTree, newTree);
+//            String diceSimilarity = String.format("%1.2f", diceSimilarity1);
+//            double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree);
+//            String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1);
 
-            String editDistance = String.valueOf(actions.size());
-
-            String result = i + "," + j + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
+            String editDistance;
+//            if(keyName.split("-")[1].equals("1")){
+//                if(oldTree.getType() == newTree.getType()){
+//                    editDistance = "0";
+//                }else{
+//                    editDistance = "1";
+//                }
+//            }else {
+//                editDistance = String.valueOf(actions.size());
+//            }
+            editDistance = String.valueOf(actions.size());
+//            String result = i + "," + j + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
+            String result = i + "," + j + "," +  editDistance;
 
             String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j);
 
-            if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
-                    || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) {
+//            if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
+//                    || ((Double) jaccardSimilarity1).equals(1.0) || editDistance.equals("0")) {
+            if (editDistance.equals("0")) {
 //                log.info("{} tagged to be similar" ,matchKey);
 
-                jedis.select(2);
-                jedis.set(matchKey, result);
+                if (treeType.equals("action")) {
+//                    log.info(editDistance);
+
+                    HierarchicalActionSet oldProject = oldPair.getValue1();
+                    HierarchicalActionSet newProject = newPair.getValue1();
+
+//                    oldTree = EDiffHelper.getTargets(keyName, i, outerPool,innerPool);
+                    oldTree = EDiffHelper.getTargets(oldProject);
+//                    newTree = EDiffHelper.getTargets(keyName, j, outerPool,innerPool);
+                    newTree = EDiffHelper.getTargets(newProject);
+
+                    m = Matchers.getInstance().getMatcher(oldTree, newTree);
+                    m.match();
+
+
+                    ag = new ActionGenerator(oldTree, newTree, m.getMappings());
+                    ag.generate();
+                    actions = ag.getActions();
+
+                    if(keyName.split("-")[1].equals("1")){
+                        if(oldTree.getType() == newTree.getType()){
+                            editDistance = "0";
+                        }else{
+                            editDistance = "1";
+                        }
+                    }else {
+                        editDistance = String.valueOf(actions.size());
+                    }
+
+                    if(editDistance.equals("0")){
+                        jedis.select(2);
+                        jedis.set(matchKey, result);
+                    }
+
+
+                }else{
+
+                    jedis.select(2);
+                    jedis.set(matchKey, result);
+                }
 
             }
 
 
             jedis.select(0);
-            jedis.del(matchKey);
+            jedis.srem("pairs",matchKey);
+//            jedis.del(matchKey);
 
 
 
@@ -184,6 +305,7 @@ public class CompareTrees {
 
             log.debug("{} not comparable", pairName);
 //            e.printStackTrace();
+//            throw e;
 
 
         }finally {

src/main/java/edu/lu/uni/serval/fixminer/akka/compare/RunnableCompare.java

@@ -7,17 +7,19 @@ public class RunnableCompare implements Runnable {
 	private String name;
 	private JedisPool innerPool;
 	private JedisPool outerPool;
-	private Compare comparer;
+	private CompareTrees comparer;
+	private String type;
 
-	public RunnableCompare(String name , JedisPool innerPool, JedisPool outerPool, Compare comp) {
+	public RunnableCompare(String name , JedisPool innerPool, JedisPool outerPool,String type) {
 		this.name = name;
 		this.innerPool = innerPool;
 		this.outerPool = outerPool;
-		this.comparer = comp;
+		this.type = type;
+
 	}
 
 	@Override
 	public void run() {
-		comparer.coreCompare(name, innerPool, outerPool);
+		comparer.coreCompare(name,type, innerPool, outerPool);
 	}
 }

src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java

@@ -4,6 +4,7 @@ import akka.actor.ActorRef;
 import akka.actor.Props;
 import akka.actor.UntypedActor;
 import akka.japi.Creator;
+import akka.routing.BalancingPool;
 import akka.routing.RoundRobinPool;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -21,7 +22,7 @@ public class TreeActor extends UntypedActor {
 
 
 	public TreeActor(int numberOfWorkers) {
-		mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers)
+		mineRouter = this.getContext().actorOf(new BalancingPool(numberOfWorkers)
 				.props(TreeWorker.props()), "tree-router");
 		this.numberOfWorkers = numberOfWorkers;
 	}
@@ -47,6 +48,7 @@ public class TreeActor extends UntypedActor {
 			List<String> pairs = ((TreeMessage) message).getName();
 			JedisPool innerPool = ((TreeMessage) message).getInnerPool();
 			JedisPool outerPool = ((TreeMessage) message).getOuterPool();
+			String type = ((TreeMessage) message).getType();
 
 
 			int size = pairs.size();
@@ -60,7 +62,7 @@ public class TreeActor extends UntypedActor {
 				int toIndex = (i + 1) * average + counter;
 
 				List<String> pairsOfWorkers = pairs.subList(fromIndex, toIndex);
-				final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool,((TreeMessage) message).getSECONDS_TO_WAIT());
+				final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool,((TreeMessage) message).getSECONDS_TO_WAIT(),type);
 				mineRouter.tell(workMsg, getSelf());
 				logger.info("Assign {} task to worker #" + (i + 1) + "...",pairsOfWorkers.size());
 			}

src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeMessage.java

@@ -13,21 +13,32 @@ public class TreeMessage extends BaseMessage{
     private JedisPool innerPool;
     private JedisPool outerPool;
 
+    public String getType() {
+        return type;
+    }
 
-    public TreeMessage(int id, List<String> name, JedisPool innerPool, JedisPool outerPool,String eDiffTimeout) {
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    private String type;
+
+
+    public TreeMessage(int id, List<String> name, JedisPool innerPool, JedisPool outerPool,String eDiffTimeout,String treeType) {
         super(id,new Long(eDiffTimeout));
 
         this.name = name;
         this.innerPool = innerPool;
         this.outerPool = outerPool;
-
+        this.type = treeType;
     }
-    public TreeMessage(int id, List<String> name, JedisPool innerPool, JedisPool outerPool,Long eDiffTimeout) {
+    public TreeMessage(int id, List<String> name, JedisPool innerPool, JedisPool outerPool,Long eDiffTimeout,String treeType) {
         super(id,eDiffTimeout);
 
         this.name = name;
         this.innerPool = innerPool;
         this.outerPool = outerPool;
+        this.type = treeType;
 
     }
 

src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java

@@ -41,6 +41,7 @@ public class TreeWorker extends UntypedActor {
 			List<String> files = msg.getName();
 			JedisPool innerPool = msg.getInnerPool();
 			JedisPool outerPool = msg.getOuterPool();
+			String type = msg.getType();
 
 			int id = msg.getId();
 			int counter = 0;
@@ -49,19 +50,21 @@ public class TreeWorker extends UntypedActor {
 				{
 
 
-				Compare compare = new Compare();
-				final ExecutorService executor = Executors.newSingleThreadExecutor();
+
+
+				final ExecutorService executor = Executors.newFixedThreadPool(1);
 //				// schedule the work
-				final Future<?> future = executor.submit(new RunnableCompare(name, innerPool, outerPool, compare));
+				final Future<?> future = executor.submit(new RunnableCompare(name, innerPool, outerPool,type));
 				try {
 //					 wait for task to complete
 					future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS);
 					counter++;
-						if (counter % 100 == 0) {
+						if (counter % 1000 == 0) {
 							log.info("Worker #" + id +" finalized parsing " + counter + " pairs... remaing "+ (files.size() - counter));
 						}
 				} catch (TimeoutException e) {
 					future.cancel(true);
+					System.err.println("#Timeout: " + name);
 				} catch (InterruptedException e) {
 					e.printStackTrace();
 				} catch (ExecutionException e) {

src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java

@@ -54,7 +54,7 @@ public class EDiffActor extends UntypedActor {
 				int toIndex = (i + 1) * average + counter;
 				
 				List<MessageFile> filesOfWorkers = files.subList(fromIndex, toIndex);
-				final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getActionType());
+				final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getInnerPool());
 				mineRouter.tell(workMsg, getSelf());
 				logger.info("Assign {} task to worker #" + (i + 1) ,filesOfWorkers.size());
 			}

src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java

@@ -4,9 +4,13 @@ import com.github.gumtreediff.actions.model.Delete;
 import com.github.gumtreediff.actions.model.Insert;
 import com.github.gumtreediff.actions.model.Move;
 import com.github.gumtreediff.actions.model.Update;
+import edu.lu.uni.serval.utils.EDiffHelper;
 import edu.lu.uni.serval.utils.FileHelper;
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
 
 import java.io.*;
+import java.util.ArrayList;
 import java.util.List;
 
 
@@ -21,7 +25,7 @@ public class EDiffHunkParser extends EDiffParser {
 	
 
 	@Override
-	public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project,String actionType) {
+	public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool) {
 		List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile);
 		if (actionSets.size() != 0) {
 //			String folder= null;
@@ -73,8 +77,9 @@ public class EDiffHunkParser extends EDiffParser {
 					try {
 
 						String astNodeType = actionSet.getAstNodeType();
-						int size = actionSet.toString().split("\\n").length;
-
+//						int size = actionSet.toString().split("\\n").length;
+						actionSet.toString();
+						int size = actionSet.getActionSize();
 //						int size = actionSet.strList.size();
 
 						String datasetName = project;
@@ -82,19 +87,30 @@ public class EDiffHunkParser extends EDiffParser {
 						String root = split1[0];
 						String pj = split1[1].split("/")[1];
 
-						File file = new File(root + "EnhancedASTDiff" + datasetName + "/"+astNodeType+"/"+String.valueOf(size)+"/");
-						file.mkdirs();
-						String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/"+astNodeType+"/"+String.valueOf(size)+"/" + pj +"_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
+//						File file = new File(root + "EnhancedASTDiff" + datasetName + "/"+astNodeType+"/"+String.valueOf(size)+"/");
+//						file.mkdirs();
+//						String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/"+astNodeType+"/"+String.valueOf(size)+"/" + pj +"_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
 //						String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/" + pj + folder + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
-						f = new FileOutputStream(new File(hunkTreeFileName));
-						ObjectOutputStream o = new ObjectOutputStream(f);
-						o.writeObject(actionSet);
 
-						o.close();
-						f.close();
+//						f = new FileOutputStream(new File(hunkTreeFileName));
+//						ObjectOutputStream o = new ObjectOutputStream(f);
+//						o.writeObject(actionSet);
+//
+//						o.close();
+//						f.close();
+
+//						String name = f.getName();
+
+						String key = astNodeType+"/"+String.valueOf(size)+"/" + pj +"_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
+
+						try (Jedis inner = innerPool.getResource()) {
+
+							inner.set(key, EDiffHelper.toString(actionSet));
+						}
+
 					} catch (FileNotFoundException e) {
 						e.printStackTrace();
-					} catch (IOException e) {
+					} catch (Exception e) {
 						e.printStackTrace();
 					}
 					hunkSet++;

src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java

@@ -1,6 +1,7 @@
 package edu.lu.uni.serval.fixminer.akka.ediff;
 
 import edu.lu.uni.serval.fixminer.akka.BaseMessage;
+import redis.clients.jedis.JedisPool;
 
 import java.util.List;
 
@@ -13,19 +14,31 @@ public class EDiffMessage extends BaseMessage{
 
 	private String actionType;
 
+	public JedisPool getInnerPool() {
+		return innerPool;
+	}
+
+	public void setInnerPool(JedisPool innerPool) {
+		this.innerPool = innerPool;
+	}
+
+	private JedisPool innerPool;
+
 
 
 
 	
-	public EDiffMessage(int id, List<MessageFile> msgFiles,String eDiffTimeout,String actionType) {
+	public EDiffMessage(int id, List<MessageFile> msgFiles,String eDiffTimeout,JedisPool pool) {
 		super(id,new Long(eDiffTimeout));
 		this.msgFiles = msgFiles;
 		this.actionType = actionType;
+		this.innerPool = pool;
 	}
-	public EDiffMessage(int id, List<MessageFile> msgFiles,Long eDiffTimeout,String actionType) {
+	public EDiffMessage(int id, List<MessageFile> msgFiles,Long eDiffTimeout,JedisPool pool) {
 		super(id,eDiffTimeout);
 		this.msgFiles = msgFiles;
 		this.actionType = actionType;
+		this.innerPool = pool;
 	}
 
 	public String getActionType() {

src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java

@@ -3,6 +3,7 @@ package edu.lu.uni.serval.fixminer.akka.ediff;
 import com.github.gumtreediff.actions.model.Action;
 import edu.lu.uni.serval.gumtree.GumTreeComparer;
 import edu.lu.uni.serval.utils.ListSorter;
+import redis.clients.jedis.JedisPool;
 
 import java.io.File;
 import java.util.ArrayList;
@@ -62,7 +63,7 @@ public class EDiffParser extends Parser {
 	}
 
 	@Override
-	public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, String actionType) {
+	public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool) {
 
 	}
 

src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java

@@ -5,6 +5,7 @@ import akka.actor.UntypedActor;
 import akka.japi.Creator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import redis.clients.jedis.JedisPool;
 
 import java.io.File;
 import java.util.List;
@@ -41,7 +42,8 @@ public class EDiffWorker extends UntypedActor {
 
 			int id = msg.getId();
 			int counter = 0;
-			
+			JedisPool innerPool = msg.getInnerPool();
+
 			for (MessageFile msgFile : files) {
 				File revFile = msgFile.getRevFile();
 				File prevFile = msgFile.getPrevFile();
@@ -53,7 +55,7 @@ public class EDiffWorker extends UntypedActor {
 
 				final ExecutorService executor = Executors.newSingleThreadExecutor();
 				// schedule the work
-				final Future<?> future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getActionType()));
+				final Future<?> future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getInnerPool()));
 				try {
 					// wait for task to complete
 					future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS);

src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java

@@ -171,6 +171,10 @@ public class HierarchicalActionSet implements Comparable<HierarchicalActionSet>,
 
 	private List<String> strList = new ArrayList<>();
 
+	public int getActionSize(){
+		return strList.size();
+	}
+
 	@Override
 	public String toString() {
 		String str = actionString;

src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java

@@ -187,7 +187,7 @@ public class HierarchicalRegrouper {
 		if (action instanceof Addition) {
 			parent = ((Addition) action).getParent(); // parent in the fixed source code tree
 		}
-		
+
 		if (parent.getType() == 55)  {
 			int type = action.getNode().getType();
 			// Modifier, NormalAnnotation, MarkerAnnotation, SingleMemberAnnotation

src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java

@@ -1,5 +1,7 @@
 package edu.lu.uni.serval.fixminer.akka.ediff;
 
+import redis.clients.jedis.JedisPool;
+
 import java.io.File;
 
 /**
@@ -18,7 +20,7 @@ public abstract class Parser implements ParserInterface {
 	protected String originalTree = ""; 		// Guide of generating patches.
 	protected String actionSets = ""; 		// Guide of generating patches.
 
-	public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile,String project,String actionType);
+	public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool);
 	
 
 	@Override

src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java

@@ -1,5 +1,7 @@
 package edu.lu.uni.serval.fixminer.akka.ediff;
 
+import redis.clients.jedis.JedisPool;
+
 import java.io.File;
 
 public class RunnableParser implements Runnable {
@@ -9,7 +11,7 @@ public class RunnableParser implements Runnable {
 	private File diffentryFile;
 	private Parser parser;
 	private String project;
-	private String actionType;
+	private JedisPool pool;
 	
 	public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) {
 		this.prevFile = prevFile;
@@ -18,17 +20,17 @@ public class RunnableParser implements Runnable {
 		this.parser = parser;
 	}
 
-	public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser,String project,String actionType) {
+	public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool) {
 		this.prevFile = prevFile;
 		this.revFile = revFile;
 		this.diffentryFile = diffentryFile;
 		this.parser = parser;
 		this.project = project;
-		this.actionType= actionType;
+		this.pool = innerPool;
 	}
 
 	@Override
 	public void run() {
-		parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,actionType);
+		parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool);
 	}
 }

src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java

@@ -6,9 +6,12 @@ import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor;
 import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser;
 import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage;
 import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile;
+import edu.lu.uni.serval.utils.CallShell;
 import edu.lu.uni.serval.utils.FileHelper;
+import edu.lu.uni.serval.utils.PoolBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import redis.clients.jedis.JedisPool;
 
 import java.io.File;
 import java.util.ArrayList;
@@ -21,12 +24,20 @@ public class EnhancedASTDiff {
 
 	private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class);
 
-	public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType, String parallelism) {
+	public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType, String parallelism, String portInner, String dbDir, String chunkName) throws Exception {
 
 
 		String parameters = String.format("\nInput path %s \nOutput path %s",inputPath,outputPath);
 		log.info(parameters);
 
+		CallShell cs = new CallShell();
+		String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
+		cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
+
+		cs.runShell(cmd, portInner);
+
+		JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
+
 		File folder = new File(inputPath);
 		File[] listOfFiles = folder.listFiles();
         Stream<File> stream = Arrays.stream(listOfFiles);
@@ -41,10 +52,11 @@ public class EnhancedASTDiff {
 
 
 			List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
-			allMessageFiles.addAll(msgFiles);
+
 //            System.out.println(msgFiles.size());
-			if (msgFiles.size() == 0)
+			if (msgFiles == null)
 				continue;
+			allMessageFiles.addAll(msgFiles);
 
 			String GUM_TREE_OUTPUT = outputPath + "/" + pjName + "/";
 
@@ -65,7 +77,7 @@ public class EnhancedASTDiff {
 			case "AKKA":
 				ActorSystem system = null;
 				ActorRef parsingActor = null;
-				final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,actionType);
+				final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool);
 				try {
 					log.info("Akka begins...");
 					log.info("{} files to process ...", allMessageFiles.size());
@@ -79,25 +91,25 @@ public class EnhancedASTDiff {
 				}
 				break;
 			case "FORKJOIN":
-				int counter = new Object() {
-					int counter = 0;
-
-					{
-						allMessageFiles.stream().
-								parallel().
-								peek(x -> counter++).
-								forEach(m ->
-										{
-											EDiffHunkParser parser =  new EDiffHunkParser();
-											parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,actionType);
-											if (counter % 10 == 0) {
-												log.info("Finalized parsing " + counter + " files... remaing " + (allMessageFiles.size() - counter));
-											}
-										}
-								);
-					}
-				}.counter;
-				log.info("Finished parsing {} files",counter);
+//				int counter = new Object() {
+//					int counter = 0;
+//
+//					{
+//						allMessageFiles.stream().
+//								parallel().
+//								peek(x -> counter++).
+//								forEach(m ->
+//										{
+//											EDiffHunkParser parser =  new EDiffHunkParser();
+//											parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,msg.getInnerPool());
+//											if (counter % 10 == 0) {
+//												log.info("Finalized parsing " + counter + " files... remaing " + (allMessageFiles.size() - counter));
+//											}
+//										}
+//								);
+//					}
+//				}.counter;
+//				log.info("Finished parsing {} files",counter);
 				break;
 			default:
 				log.error("Unknown parallelism {}", parallelism);
@@ -115,26 +127,29 @@ public class EnhancedASTDiff {
 	private static List<MessageFile> getMessageFiles(String gumTreeInput) {
 		String inputPath = gumTreeInput; // prevFiles  revFiles diffentryFile positionsFile
 		File revFilesPath = new File(inputPath + "revFiles/");
-		File[] revFiles = revFilesPath.listFiles();   // project folders
-		List<MessageFile> msgFiles = new ArrayList<>();
-        if (revFiles.length >= 0) {
-            for (File revFile : revFiles) {
-                String fileName = revFile.getName();
-                File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
-                fileName = fileName.replace(".java", ".txt");
-                File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file
+		log.info(revFilesPath.getPath());
+		File[] revFiles = revFilesPath.listFiles();
+		if (revFiles!= null ){
+	//		List<File> collect = Arrays.stream(revFiles).filter(x -> x.getName().startsWith("b50867_6e80c3_src#main#java#org#apache#hadoop#hbase#regionserver#HRegion"))
+	//				.collect(Collectors.toList());// project folders
+			List<MessageFile> msgFiles = new ArrayList<>();
+			for (File revFile : revFiles) {
+	//		for (File revFile : collect) {
+				String fileName = revFile.getName();
+				File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
+				fileName = fileName.replace(".java", ".txt");
+				File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file
 
-                MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
+				MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
 
-                msgFiles.add(msgFile);
+				msgFiles.add(msgFile);
 
-            }
+			}
 
-            return msgFiles;
-        }
-        else{
-            return null;
-        }
+			return msgFiles;
+		}else{
+			return null;
+		}
 	}
 
 

src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java

@@ -2,6 +2,7 @@ package edu.lu.uni.serval.fixminer.jobs;
 
 import com.github.gumtreediff.tree.ITree;
 import com.github.gumtreediff.tree.TreeContext;
+import com.github.gumtreediff.tree.TreeUtils;
 import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
 import edu.lu.uni.serval.utils.CallShell;
 import edu.lu.uni.serval.utils.EDiffHelper;
@@ -171,6 +172,7 @@ public class MultiThreadTreeLoaderCluster3 {
 
 
 
+
     public static List<String> getNames(ITree oldTree, List<String> oldTokens){
 
         List<ITree> descendants = oldTree.getDescendants();
@@ -209,17 +211,26 @@ public class MultiThreadTreeLoaderCluster3 {
                 List<String> m = new ArrayList<String>();
                 if(label.startsWith("UPD")){
                     upd = true;
-                    java.util.regex.Matcher matcher;
-                    if(sType.equals("53")){
-                        String timeRegex = ".*@@(ClassInstanceCreation:new [a-zA-Z0-9]+).*@TO@\\s(ClassInstanceCreation:new [a-zA-Z0-9]+).*";
-                        Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
-                        matcher= pattern.matcher(split[1]);
-                    }else {
-                        String timeRegex = "@@(.*)@TO@(.*)";
-                        Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
-                        matcher = pattern.matcher(split[1]);
-                    }
+
+                    String timeRegex = "UPD (AnonymousClassDeclaration|ArrayAccess|ArrayCreation|ArrayInitializer|ArrayType|AssertStatement|Assignment|Block|BooleanLiteral|BreakStatement|CastExpression|CatchClause|CharacterLiteral|ClassInstanceCreation|CompilationUnit|ConditionalExpression|ConstructorInvocation|ContinueStatement|DoStatement|EmptyStatement|ExpressionStatement|FieldAccess|FieldDeclaration|ForStatement|IfStatement|ImportDeclaration|InfixExpression|Initializer|Javadoc|LabeledStatement|MethodDeclaration|MethodInvocation|NullLiteral|NumberLiteral|PackageDeclaration|ParenthesizedExpression|PostfixExpression|PrefixExpression|PrimitiveType|QualifiedName|ReturnStatement|SimpleName|SimpleType|SingleVariableDeclaration|StringLiteral|SuperConstructorInvocation|SuperFieldAccess|SuperMethodInvocation|SwitchCase|SwitchStatement|SynchronizedStatement|ThisExpression|ThrowStatement|TryStatement|TypeDeclaration|TypeDeclarationStatement|TypeLiteral|VariableDeclarationExpression|VariableDeclarationFragment|VariableDeclarationStatement|WhileStatement|InstanceofExpression|LineComment|BlockComment|TagElement|TextElement|MemberRef|MethodRef|MethodRefParameter|EnhancedForStatement|EnumDeclaration|EnumConstantDeclaration|TypeParameter|ParameterizedType|QualifiedType|WildcardType|NormalAnnotation|MarkerAnnotation|SingleMemberAnnotation|MemberValuePair|AnnotationTypeDeclaration|AnnotationTypeMemberDeclaration|Modifier|UnionType|Dimension|LambdaExpression|IntersectionType|NameQualifiedType|CreationReference|ExpressionMethodReference|SuperMethodReference|TypeMethodReference)@@(.*)@TO@(.*)@AT@";
+//                    java.util.regex.Matcher matcher;
+//                    if(sType.equals("53")){
+//                        String timeRegex = ".*@@(ClassInstanceCreation:new [a-zA-Z0-9]+).*@TO@\\s(ClassInstanceCreation:new [a-zA-Z0-9]+).*";
+//                        Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
+//                        matcher= pattern.matcher(split[1]);
+//                    }else {
+//                        String timeRegex = "@@(.*)@TO@(.*)";
+//                        Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
+//                        matcher = pattern.matcher(split[1]);
+//                    }
+                    Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
+                    java.util.regex.Matcher matcher = pattern.matcher(label);
+
+
                     if (matcher.matches()) {
+                        String group = matcher.group(2);
+                        String group1 = matcher.group(3);
+
                         String hours = matcher.group(1);
                         String to = matcher.group(2);
                         if(sType.equals("31")){
@@ -296,11 +307,16 @@ public class MultiThreadTreeLoaderCluster3 {
 
                     }
                 }else if(label.startsWith("INS") && upd == false){
-                    String timeRegex = "@@(.*)@TO@(.*)";
+                    String timeRegex ="MOV (AnonymousClassDeclaration|ArrayAccess|ArrayCreation|ArrayInitializer|ArrayType|AssertStatement|Assignment|Block|BooleanLiteral|BreakStatement|CastExpression|CatchClause|CharacterLiteral|ClassInstanceCreation|CompilationUnit|ConditionalExpression|ConstructorInvocation|ContinueStatement|DoStatement|EmptyStatement|ExpressionStatement|FieldAccess|FieldDeclaration|ForStatement|IfStatement|ImportDeclaration|InfixExpression|Initializer|Javadoc|LabeledStatement|MethodDeclaration|MethodInvocation|NullLiteral|NumberLiteral|PackageDeclaration|ParenthesizedExpression|PostfixExpression|PrefixExpression|PrimitiveType|QualifiedName|ReturnStatement|SimpleName|SimpleType|SingleVariableDeclaration|StringLiteral|SuperConstructorInvocation|SuperFieldAccess|SuperMethodInvocation|SwitchCase|SwitchStatement|SynchronizedStatement|ThisExpression|ThrowStatement|TryStatement|TypeDeclaration|TypeDeclarationStatement|TypeLiteral|VariableDeclarationExpression|VariableDeclarationFragment|VariableDeclarationStatement|WhileStatement|InstanceofExpression|LineComment|BlockComment|TagElement|TextElement|MemberRef|MethodRef|MethodRefParameter|EnhancedForStatement|EnumDeclaration|EnumConstantDeclaration|TypeParameter|ParameterizedType|QualifiedType|WildcardType|NormalAnnotation|MarkerAnnotation|SingleMemberAnnotation|MemberValuePair|AnnotationTypeDeclaration|AnnotationTypeMemberDeclaration|Modifier|UnionType|Dimension|LambdaExpression|IntersectionType|NameQualifiedType|CreationReference|ExpressionMethodReference|SuperMethodReference|TypeMethodReference)@@(.*)@TO@ (AnonymousClassDeclaration|ArrayAccess|ArrayCreation|ArrayInitializer|ArrayType|AssertStatement|Assignment|Block|BooleanLiteral|BreakStatement|CastExpression|CatchClause|CharacterLiteral|ClassInstanceCreation|CompilationUnit|ConditionalExpression|ConstructorInvocation|ContinueStatement|DoStatement|EmptyStatement|ExpressionStatement|FieldAccess|FieldDeclaration|ForStatement|IfStatement|ImportDeclaration|InfixExpression|Initializer|Javadoc|LabeledStatement|MethodDeclaration|MethodInvocation|NullLiteral|NumberLiteral|PackageDeclaration|ParenthesizedExpression|PostfixExpression|PrefixExpression|PrimitiveType|QualifiedName|ReturnStatement|SimpleName|SimpleType|SingleVariableDeclaration|StringLiteral|SuperConstructorInvocation|SuperFieldAccess|SuperMethodInvocation|SwitchCase|SwitchStatement|SynchronizedStatement|ThisExpression|ThrowStatement|TryStatement|TypeDeclaration|TypeDeclarationStatement|TypeLiteral|VariableDeclarationExpression|VariableDeclarationFragment|VariableDeclarationStatement|WhileStatement|InstanceofExpression|LineComment|BlockComment|TagElement|TextElement|MemberRef|MethodRef|MethodRefParameter|EnhancedForStatement|EnumDeclaration|EnumConstantDeclaration|TypeParameter|ParameterizedType|QualifiedType|WildcardType|NormalAnnotation|MarkerAnnotation|SingleMemberAnnotation|MemberValuePair|AnnotationTypeDeclaration|AnnotationTypeMemberDeclaration|Modifier|UnionType|Dimension|LambdaExpression|IntersectionType|NameQualifiedType|CreationReference|ExpressionMethodReference|SuperMethodReference|TypeMethodReference)@@(.*)@AT@";
+//
+//                    String timeRegex = "@@(.*)@TO@(.*)";
                     Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
-                    java.util.regex.Matcher matcher = pattern.matcher(split[1]);
+                    java.util.regex.Matcher matcher = pattern.matcher(label);
 
                     if (matcher.matches()) {
+                        String group = matcher.group(2);
+                        String group1 = matcher.group(4);
+
                         String hours = matcher.group(1);
                         if (hours.startsWith("MethodName:")) {
                             String methodName = hours.split(":")[1];
@@ -313,11 +329,13 @@ public class MultiThreadTreeLoaderCluster3 {
                     }
 
                 }else if(label.startsWith("DEL") && upd == false){
-                    String timeRegex = "@@(.*)";
+                    String timeRegex = "DEL (AnonymousClassDeclaration|ArrayAccess|ArrayCreation|ArrayInitializer|ArrayType|AssertStatement|Assignment|Block|BooleanLiteral|BreakStatement|CastExpression|CatchClause|CharacterLiteral|ClassInstanceCreation|CompilationUnit|ConditionalExpression|ConstructorInvocation|ContinueStatement|DoStatement|EmptyStatement|ExpressionStatement|FieldAccess|FieldDeclaration|ForStatement|IfStatement|ImportDeclaration|InfixExpression|Initializer|Javadoc|LabeledStatement|MethodDeclaration|MethodInvocation|NullLiteral|NumberLiteral|PackageDeclaration|ParenthesizedExpression|PostfixExpression|PrefixExpression|PrimitiveType|QualifiedName|ReturnStatement|SimpleName|SimpleType|SingleVariableDeclaration|StringLiteral|SuperConstructorInvocation|SuperFieldAccess|SuperMethodInvocation|SwitchCase|SwitchStatement|SynchronizedStatement|ThisExpression|ThrowStatement|TryStatement|TypeDeclaration|TypeDeclarationStatement|TypeLiteral|VariableDeclarationExpression|VariableDeclarationFragment|VariableDeclarationStatement|WhileStatement|InstanceofExpression|LineComment|BlockComment|TagElement|TextElement|MemberRef|MethodRef|MethodRefParameter|EnhancedForStatement|EnumDeclaration|EnumConstantDeclaration|TypeParameter|ParameterizedType|QualifiedType|WildcardType|NormalAnnotation|MarkerAnnotation|SingleMemberAnnotation|MemberValuePair|AnnotationTypeDeclaration|AnnotationTypeMemberDeclaration|Modifier|UnionType|Dimension|LambdaExpression|IntersectionType|NameQualifiedType|CreationReference|ExpressionMethodReference|SuperMethodReference|TypeMethodReference)@@(.*)@AT@";
+//                    String timeRegex = "@@(.*)";
                     Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
-                    java.util.regex.Matcher matcher = pattern.matcher(split[1]);
+                    java.util.regex.Matcher matcher = pattern.matcher(label);
 
                     if (matcher.matches()) {
+                        String group = matcher.group(2);
                         String hours = matcher.group(1);
                         if (hours.startsWith("MethodName:")){
                             String methodName = hours.split(":")[1];
@@ -327,11 +345,15 @@ public class MultiThreadTreeLoaderCluster3 {
                         }
                     }
                 }else if(label.startsWith("MOV")  && upd == false){
-                    String timeRegex = "@@(.*)@TO@(.*)";
+                    String timeRegex ="MOV (AnonymousClassDeclaration|ArrayAccess|ArrayCreation|ArrayInitializer|ArrayType|AssertStatement|Assignment|Block|BooleanLiteral|BreakStatement|CastExpression|CatchClause|CharacterLiteral|ClassInstanceCreation|CompilationUnit|ConditionalExpression|ConstructorInvocation|ContinueStatement|DoStatement|EmptyStatement|ExpressionStatement|FieldAccess|FieldDeclaration|ForStatement|IfStatement|ImportDeclaration|InfixExpression|Initializer|Javadoc|LabeledStatement|MethodDeclaration|MethodInvocation|NullLiteral|NumberLiteral|PackageDeclaration|ParenthesizedExpression|PostfixExpression|PrefixExpression|PrimitiveType|QualifiedName|ReturnStatement|SimpleName|SimpleType|SingleVariableDeclaration|StringLiteral|SuperConstructorInvocation|SuperFieldAccess|SuperMethodInvocation|SwitchCase|SwitchStatement|SynchronizedStatement|ThisExpression|ThrowStatement|TryStatement|TypeDeclaration|TypeDeclarationStatement|TypeLiteral|VariableDeclarationExpression|VariableDeclarationFragment|VariableDeclarationStatement|WhileStatement|InstanceofExpression|LineComment|BlockComment|TagElement|TextElement|MemberRef|MethodRef|MethodRefParameter|EnhancedForStatement|EnumDeclaration|EnumConstantDeclaration|TypeParameter|ParameterizedType|QualifiedType|WildcardType|NormalAnnotation|MarkerAnnotation|SingleMemberAnnotation|MemberValuePair|AnnotationTypeDeclaration|AnnotationTypeMemberDeclaration|Modifier|UnionType|Dimension|LambdaExpression|IntersectionType|NameQualifiedType|CreationReference|ExpressionMethodReference|SuperMethodReference|TypeMethodReference)@@(.*)@TO@ (AnonymousClassDeclaration|ArrayAccess|ArrayCreation|ArrayInitializer|ArrayType|AssertStatement|Assignment|Block|BooleanLiteral|BreakStatement|CastExpression|CatchClause|CharacterLiteral|ClassInstanceCreation|CompilationUnit|ConditionalExpression|ConstructorInvocation|ContinueStatement|DoStatement|EmptyStatement|ExpressionStatement|FieldAccess|FieldDeclaration|ForStatement|IfStatement|ImportDeclaration|InfixExpression|Initializer|Javadoc|LabeledStatement|MethodDeclaration|MethodInvocation|NullLiteral|NumberLiteral|PackageDeclaration|ParenthesizedExpression|PostfixExpression|PrefixExpression|PrimitiveType|QualifiedName|ReturnStatement|SimpleName|SimpleType|SingleVariableDeclaration|StringLiteral|SuperConstructorInvocation|SuperFieldAccess|SuperMethodInvocation|SwitchCase|SwitchStatement|SynchronizedStatement|ThisExpression|ThrowStatement|TryStatement|TypeDeclaration|TypeDeclarationStatement|TypeLiteral|VariableDeclarationExpression|VariableDeclarationFragment|VariableDeclarationStatement|WhileStatement|InstanceofExpression|LineComment|BlockComment|TagElement|TextElement|MemberRef|MethodRef|MethodRefParameter|EnhancedForStatement|EnumDeclaration|EnumConstantDeclaration|TypeParameter|ParameterizedType|QualifiedType|WildcardType|NormalAnnotation|MarkerAnnotation|SingleMemberAnnotation|MemberValuePair|AnnotationTypeDeclaration|AnnotationTypeMemberDeclaration|Modifier|UnionType|Dimension|LambdaExpression|IntersectionType|NameQualifiedType|CreationReference|ExpressionMethodReference|SuperMethodReference|TypeMethodReference)@@(.*)@AT@";
+//                    String timeRegex = "@@(.*)@TO@(.*)";
                     Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
-                    java.util.regex.Matcher matcher = pattern.matcher(split[1]);
+                    java.util.regex.Matcher matcher = pattern.matcher(label);
 
                     if (matcher.matches()) {
+                        String group = matcher.group(2);
+                        String group1 = matcher.group(4);
+
                         String hours = matcher.group(1);
                         if (hours.startsWith("MethodName:")){
                             String methodName = hours.split(":")[1];

src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java

@@ -25,9 +25,9 @@ public class StoreEDiffInCache {
     private static Logger log = LoggerFactory.getLogger(StoreEDiffInCache.class);
 
 
-    public static void main(String inputPath,String portInner,String dbDir,String chunkName,String operation) throws Exception {
+    public static void main(String inputPath,String portInner,String dbDir,String chunkName) throws Exception {
 
-        String parameters = String.format("\nInput path %s \nportInner %s \nchunkName %s \ndbDir %s \noperation %s",inputPath,portInner,chunkName,dbDir,operation);
+        String parameters = String.format("\nInput path %s \nportInner %s \nchunkName %s \ndbDir %s \noperation %s",inputPath,portInner,chunkName,dbDir);
         log.info(parameters);
         CallShell cs = new CallShell();
         String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";

src/main/java/edu/lu/uni/serval/utils/CallShell.java

@@ -81,6 +81,7 @@ try{
 
                     runPing(command);
                 }else{
+                    TimeUnit.MINUTES.sleep(1);
                     System.out.println(e);
 
                 }

src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java

@@ -3,16 +3,12 @@ package edu.lu.uni.serval.utils;
 import com.github.gumtreediff.tree.ITree;
 import com.github.gumtreediff.tree.TreeContext;
 import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
+import org.apache.commons.io.FileUtils;
 import redis.clients.jedis.Jedis;
 import redis.clients.jedis.JedisPool;
 
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
+import java.io.*;
+import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -22,8 +18,80 @@ public class ClusterToPattern {
         String shapesFolder = "/Users/anil.koyuncu/projects/fixminer-all/enhancedASTDiff/python/data/shapes";
 
         String port = "6399";
-        JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000);
+//        JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000);
+        Jedis jedis = new Jedis("127.0.0.1",Integer.valueOf(port),20000000);
+//        writeTofFile(shapesFolder, outerPool);
 
+        File file = new File(shapesFolder);
+
+
+        File[] files = file.listFiles();
+        ArrayList<String> strings = new ArrayList<>();
+        List<File> folders = Arrays.stream(files).filter(x -> x.isDirectory())
+                .collect(Collectors.toList());
+        for (File target : folders) {
+            File[] files2 = target.listFiles();
+            for (File file1 : files2) {
+                File[] files1 = file1.listFiles();
+                strings.add(((File)files1[0]).listFiles()[0].getName());
+//                System.out.println();
+
+            }
+
+//            Collection<File> files1 = FileUtils.listFiles(target, null, true);
+
+        }
+
+        for (String string : strings) {
+            export("*"+string,jedis);
+        }
+
+//        String export = export("VariableDeclarationStatement"+"/*/camel_8f0c15_ab3e17_camel-core#src#main#java#org#apache#camel#component#bean#BeanInfo.txt_0", jedis);
+//        String export = export(args[0], jedis);
+//        System.out.println(export);
+
+    }
+
+    private static void export(String filename,Jedis outer) throws IOException, ClassNotFoundException {
+
+//        String key = shape.getName() + "/*/" + file.getName();
+//        Jedis outer = outerPool.getResource();
+        String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load);
+        Set<String> keys = outer.keys(filename);
+        if (keys.size() == 1) {
+            s = (String) keys.toArray()[0];
+        } else {
+            throw new Error("cok key");
+        }
+
+        String si = outer.get(s);
+        HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si);
+        ITree tree = null;
+        ITree parent = null;
+        ITree children =null;
+        TreeContext tc = new TreeContext();
+        tree = EDiffHelper.getASTTree(actionSet, parent, children,tc);
+        tree.setParent(null);
+        tc.validate();
+
+        String s2 = tree.toStaticHashString();
+        try(FileWriter fw = new FileWriter("myfile.txt", true);
+            BufferedWriter bw = new BufferedWriter(fw);
+            PrintWriter out = new PrintWriter(bw))
+        {
+            out.println(s2);
+            //more code
+        } catch (IOException e) {
+            //exception handling left as an exercise for the reader
+        }
+
+//        String s1 = actionSet.toString();
+//        outer.close();
+//        return s1;
+    }
+
+
+    private static void writeTofFile(String shapesFolder, JedisPool outerPool) throws IOException, ClassNotFoundException {
         File folder = new File(shapesFolder);
 
         File[] listOfFiles = folder.listFiles();
@@ -39,8 +107,10 @@ public class ClusterToPattern {
                     .collect(Collectors.toList());
             for(File cluster:fileList){
                 File[] files = cluster.listFiles();
+//                for (File f:files) {
+//                    File file = f.listFiles()[0];
                 File file = files[0];
-                String key = shape.getName() + "/*/"+file.getName();
+                String key = shape.getName() + "/*/" + file.getName();
                 Jedis outer = outerPool.getResource();
                 String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load);
                 Set<String> keys = outer.keys(key);
@@ -55,17 +125,18 @@ public class ClusterToPattern {
 
 
                 String s1 = actionSet.toString();
-                File saveFile = new File(folder.getParent()+"/shapePatterns/" +shape.getName()+"/"+cluster.getName()+".pattern");
+//                File saveFile = new File(folder.getParent() + "/actionPatterns/" + shape.getName() + "/" + cluster.getName() +"/"+f.getName()+ ".pattern");
+                File saveFile = new File(folder.getParent() + "/shapePatterns/" + shape.getName() + "/" + cluster.getName() + ".pattern");
                 saveFile.getParentFile().mkdirs();
                 BufferedWriter writer = new BufferedWriter(new FileWriter(saveFile));
                 writer.write(s1);
 
-                writer.close();
+                    writer.close();
+//                }
 
 
             }
 
         }
-
     }
 }

src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java

@@ -3,14 +3,18 @@ package edu.lu.uni.serval.utils;
 import com.github.gumtreediff.actions.model.*;
 import com.github.gumtreediff.tree.ITree;
 import com.github.gumtreediff.tree.TreeContext;
+import com.github.gumtreediff.tree.TreeUtils;
 import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import redis.clients.jedis.Jedis;
 import redis.clients.jedis.JedisPool;
+import org.javatuples.Pair;
 
 import java.io.*;
 import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
@@ -79,6 +83,66 @@ public class EDiffHelper {
     }
 
 
+
+    public static ITree getTargetTree(HierarchicalActionSet actionSet, ITree parent, ITree children, TreeContext tc){
+
+        int newType = 0;
+
+        String astNodeType = null;
+
+        Action action = actionSet.getAction();
+        if (action instanceof Update){
+            astNodeType = actionSet.getAstNodeType();
+            List<Integer> keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType);
+
+            if(keysByValue.size() != 1){
+                log.error("More than 1");
+            }
+            newType = keysByValue.get(0);
+        }else if(action instanceof Insert){
+            newType = ((Insert)action).getParent().getType();
+        }else if(action instanceof Move){
+            newType = ((Move)action).getParent().getType();
+        }else if(action instanceof Delete){
+            astNodeType = actionSet.getAstNodeType();
+            List<Integer> keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType);
+
+            if(keysByValue.size() != 1){
+                log.error("More than 1");
+            }
+            newType = keysByValue.get(0);
+        }
+
+
+
+        if(actionSet.getParent() == null){
+            //root
+
+//            parent = new Tree(newType,"");
+            parent = tc.createTree(newType, "", null);
+            tc.setRoot(parent);
+        }else{
+//            children = new Tree(newType,"");
+//            parent.addChild(children);
+            children = tc.createTree(newType, "", null);
+            children.setParentAndUpdateChildren(parent);
+        }
+        List<HierarchicalActionSet> subActions = actionSet.getSubActions();
+        if (subActions.size() != 0){
+            for (HierarchicalActionSet subAction : subActions) {
+
+                if(actionSet.getParent() == null){
+                    children = parent;
+                }
+                getTargetTree(subAction,children,null,tc);
+
+            }
+
+
+        }
+        return parent;
+    }
+
     public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children, TreeContext tc){
 
         int newType = 0;
@@ -163,19 +227,21 @@ public class EDiffHelper {
     }
 
 
-    public static ITree getShapes(String prefix, String fn, JedisPool outerPool) {
+    public static ITree getShapes(String prefix, String fn, JedisPool outerPool,JedisPool innerPool) {
 
         ITree tree = null;
         Jedis inner = null;
+        Jedis outer = null;
         try {
-            inner = outerPool.getResource();
+            inner = innerPool.getResource();
             while (!inner.ping().equals("PONG")){
                 log.info("wait");
             }
             inner.select(1);
             String dist2load = inner.get(prefix+"-"+fn);
-            inner.select(0);
-            String s = inner.get(prefix.replace("-","/") +"/"+dist2load);
+            outer = outerPool.getResource();
+            outer.select(0);
+            String s = outer.get(prefix.replace("-","/") +"/"+dist2load);
             HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s);
 
             ITree parent = null;
@@ -193,39 +259,150 @@ public class EDiffHelper {
             if (inner != null) {
                 inner.close();
             }
+            if (outer != null) {
+                outer.close();
+            }
         }
         return tree;
 
     }
 
 
-    public  static ITree getActions(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool) {
-
+    public static ITree getTargets(String prefix, String fn, JedisPool outerPool,JedisPool innerPool) {
 
         ITree tree = null;
-        Jedis inner = null;
-        Jedis outer = null;
-
-
+//        Jedis inner = null;
+//        Jedis outer = null;
         try {
-            inner = innerPool.getResource();
-            inner.select(1);
-            String dist2load = inner.get(prefix + "-" + firstValue);
-            outer = outerPool.getResource();
-            outer.select(0);
-            String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load);
-            Set<String> keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load);
-            if (keys.size() == 1) {
-                s = (String) keys.toArray()[0];
-            } else {
-                throw new Error("cok key");
+            String dist2load =  null;
+            String si = null;
+            try(Jedis inner = innerPool.getResource()){
+                inner.select(1);
+                dist2load = inner.get(prefix+"-"+fn);
+
+            }
+//            while (!inner.ping().equals("PONG")){
+//                log.info("wait");
+//            }
+            try(Jedis outer = outerPool.getResource()){
+                outer.select(0);
+                String s = null;
+                Set<String> keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load);
+                if (keys.size() == 1) {
+                    s = (String) keys.toArray()[0];
+                } else {
+                    throw new Error("cok key");
+                }
+                si = outer.get(s);
+
             }
 
 
 //            String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition;
-            String si = outer.get(s);
             HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si);
 
+            ITree parent = null;
+            ITree children =null;
+            TreeContext tc = new TreeContext();
+            tree = EDiffHelper.getTargetTree(actionSet, parent, children,tc);
+            //tree.setParent(null);
+            tc.validate();
+
+        } catch (IOException e) {
+            e.printStackTrace();
+        } catch (ClassNotFoundException e) {
+            e.printStackTrace();
+        }
+        return tree;
+
+    }
+
+
+    public static ITree getTargets(HierarchicalActionSet actionSet) {
+
+        ITree tree = null;
+//        Jedis inner = null;
+//        Jedis outer = null;
+        try {
+//            String dist2load =  null;
+//            String si = null;
+//            try(Jedis inner = innerPool.getResource()){
+//                inner.select(1);
+//                dist2load = inner.get(prefix+"-"+fn);
+//
+//            }
+////            while (!inner.ping().equals("PONG")){
+////                log.info("wait");
+////            }
+//            try(Jedis outer = outerPool.getResource()){
+//                outer.select(0);
+//                String s = null;
+//                Set<String> keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load);
+//                if (keys.size() == 1) {
+//                    s = (String) keys.toArray()[0];
+//                } else {
+//                    throw new Error("cok key");
+//                }
+//                si = outer.get(s);
+//
+//            }
+//
+//
+////            String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition;
+//            HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si);
+
+            ITree parent = null;
+            ITree children =null;
+            TreeContext tc = new TreeContext();
+            tree = EDiffHelper.getTargetTree(actionSet, parent, children,tc);
+            //tree.setParent(null);
+            tc.validate();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+//
+//        } catch (IOException e) {
+//            e.printStackTrace();
+//        } catch (ClassNotFoundException e) {
+//            e.printStackTrace();
+//        }
+        return tree;
+
+    }
+
+
+    public  static Pair<ITree,HierarchicalActionSet> getActions(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool) {
+
+
+        ITree tree = null;
+//        Jedis inner = null;
+//        Jedis outer = null;
+        HierarchicalActionSet actionSet = null;
+
+        try {
+            String dist2load = null;
+            String si = null;
+            try (Jedis inner = innerPool.getResource()){
+
+            inner.select(1);
+            dist2load = inner.get(prefix + "-" + firstValue);
+            }
+            try(Jedis outer = outerPool.getResource()) {
+                outer.select(0);
+                String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load);
+                Set<String> keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load);
+                if (keys.size() == 1) {
+                    s = (String) keys.toArray()[0];
+                } else {
+                    throw new Error("cok key");
+                }
+                si = outer.get(s);
+            }
+
+
+//            String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition;
+            actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si);
+
 
             ITree parent = null;
             ITree children = null;
@@ -239,18 +416,24 @@ public class EDiffHelper {
             e.printStackTrace();
         } catch (ClassNotFoundException e) {
             e.printStackTrace();
-        } finally {
-            if (inner != null) {
-                inner.close();
-            }
-            if (outer != null) {
-                outer.close();
-            }
         }
-
-        return tree;
+        Pair<ITree, HierarchicalActionSet> pair = new Pair<>(tree,actionSet);
+        return pair;
+//        return tree;
     }
 
+    public static void getLeaves(ITree tc){
+
+        int height = tc.getHeight();
+       if(height == 0){
+           log.info(tc.getLabel());
+       }else{
+            List<ITree> children = tc.getChildren();
+            for (ITree child:children){
+                getLeaves(child);
+            }
+        }
+    }
 
     public  static ITree getTokens(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool) {
 
@@ -286,6 +469,7 @@ public class EDiffHelper {
             tree = EDiffHelper.getTokenTree(actionSet, parent, children, tc);
             tree.setParent(null);
             tc.validate();
+//            getLeaves(tree);
 
 
         } catch (IOException e) {
@@ -349,4 +533,150 @@ public class EDiffHelper {
         }
         return parent;
     }
+
+
+    public static List<ITree> retainLeaves(List<ITree> trees) {
+        Iterator<ITree> tIt = trees.iterator();
+        while (tIt.hasNext()) {
+            ITree t = tIt.next();
+            if (!t.isLeaf()) tIt.remove();
+        }
+        return trees;
+    }
+
+    public static String getTokenAtNode(ITree leaf,String token){
+//        if (leaf.getParent() == null) {
+            String label = leaf.getLabel();
+            String key = label.split(String.valueOf(leaf.getType()))[0];
+            String group="";
+            Pattern pattern;
+            java.util.regex.Matcher matcher;
+
+            List<ITree> parents;
+            List<String> actionList = new ArrayList<>();
+            Set<String> uniqueGas = new HashSet<String>(actionList);
+            switch (key.trim()) {
+
+                case "DEL":
+                    parents = leaf.getParents();
+
+                    for (ITree parent : parents) {
+                        String s = parent.getLabel().split(String.valueOf(parent.getType()))[0];
+                        actionList.add(s);
+                    }
+                    actionList.add(key);
+
+                    if (uniqueGas.size() == 1){
+                        Optional<ITree> first = parents.stream().filter(p -> p.isRoot()).findFirst();
+                        if(first.isPresent()){
+                            ITree parent = first.get();
+                            label = parent.getLabel();
+                        }
+                    }
+
+                    pattern = Pattern.compile(delPattern);
+                    matcher = pattern.matcher(label);
+                    if (matcher.matches()) {
+                        group = matcher.group(2);
+
+                    }
+                    break;
+                case "INS":
+                    parents = leaf.getParents();
+
+                    for (ITree parent : parents) {
+                        String s = parent.getLabel().split(String.valueOf(parent.getType()))[0];
+                        actionList.add(s);
+                    }
+                    actionList.add(key);
+
+                    if (uniqueGas.size() == 1){
+                        Optional<ITree> first = parents.stream().filter(p -> p.isRoot()).findFirst();
+                        if(first.isPresent()){
+                            ITree parent = first.get();
+                            label = parent.getLabel();
+                        }
+                    }
+
+//
+                    pattern = Pattern.compile(insPattern);
+                    matcher = pattern.matcher(label);
+                    if (matcher.matches()) {
+                        group = matcher.group(2);// +" " + matcher.group(3);
+
+                    }
+                    break;
+                case "MOV":
+                    parents = leaf.getParents();
+
+                    for (ITree parent : parents) {
+                        String s = parent.getLabel().split(String.valueOf(parent.getType()))[0];
+                        actionList.add(s);
+                    }
+                    actionList.add(key);
+
+                    if (uniqueGas.size() == 1){
+                        Optional<ITree> first = parents.stream().filter(p -> p.isRoot()).findFirst();
+                        if(first.isPresent()){
+                            ITree parent = first.get();
+                            label = parent.getLabel();
+                        }
+                    }
+
+                    pattern = Pattern.compile(movPattern);
+                    matcher = pattern.matcher(label);
+                    if (matcher.matches()) {
+                        group = matcher.group(2);
+                        String group1 = matcher.group(3);
+
+                    }
+                    break;
+                case "UPD":
+                    pattern = Pattern.compile(updPattern);
+                    matcher = pattern.matcher(label);
+                    if (matcher.matches()) {
+                        group = matcher.group(2) +" " + matcher.group(3);
+
+                    }
+                    break;
+
+            }
+            token = group;
+            return group;
+//        }else{
+//            getTokenAtNode(leaf.getParent(),token);
+//            return token;
+//        }
+    }
+
+    static String movPattern = "MOV (.*)@@(.*)@TO@ (.*)@@(.*)@AT@.*";
+    static String delPattern = "DEL (.*)@@(.*)@AT@.*";
+    static String insPattern = "INS (.*)@@(.*)@TO@ (.*)@@(.*)@AT@.*";
+    static String updPattern = "UPD (.*)@@(.*)@TO@(.*)@AT@.*";
+
+    public static String getNames2(ITree oldTree){
+
+        List<ITree> leaves = retainLeaves(TreeUtils.postOrder(oldTree));
+        List<String> res = new ArrayList<>();
+        String resu="";
+        for (ITree leaf : leaves) {
+            String tokenAtNode = getTokenAtNode(leaf, resu);
+
+            res.add(tokenAtNode);
+        }
+
+//        log.info("s");
+
+        if (res.size() == 1){
+            final String regex = "MethodName:([a-zA-Z0-9]+):.*MethodName:([a-zA-Z0-9]+):.*";
+            Pattern compile = Pattern.compile(regex);
+            Matcher matcher = compile.matcher(res.get(0));
+            if(matcher.matches()){
+                res.set(0,matcher.group(1) +" " + matcher.group(2));
+            }
+        }
+
+        return String.join(" ",res);
+
+    }
 }

src/main/java/edu/lu/uni/serval/utils/PoolBuilder.java

@@ -17,17 +17,18 @@ public class PoolBuilder {
 
     private static JedisPoolConfig buildPoolConfig() {
         final JedisPoolConfig poolConfig = new JedisPoolConfig();
-        poolConfig.setMaxTotal(128);
-        poolConfig.setMaxIdle(128);
-        poolConfig.setMinIdle(16);
-        poolConfig.setTestOnBorrow(true);
-        poolConfig.setTestOnReturn(true);
-        poolConfig.setTestWhileIdle(true);
+        poolConfig.setMaxTotal(1024);
+        poolConfig.setMaxIdle(1024);
+//        poolConfig.setMinIdle(16);
+//        poolConfig.setTestOnBorrow(true);
+//        poolConfig.setTestOnReturn(true);
+//        poolConfig.setTestWhileIdle(true);
         poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis());
         poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis());
         poolConfig.setNumTestsPerEvictionRun(3);
         poolConfig.setBlockWhenExhausted(true);
 
+
         return poolConfig;
     }
 }