package feedback;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.StringTokenizer;
import java.util.HashMap;
import java.util.Random;
import java.util.Calendar;
import java.text.SimpleDateFormat;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.FilterIndexReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.HitCollector;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryTermVector;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.TopDocCollector;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermFreqVector;

/**
 *
 * @author greg
 */
public class RatingTool {
    
    public static ArrayList<Float> scores = new ArrayList<Float>();
    public static ArrayList<Float> fakeScores = new ArrayList<Float>();
    public static HashMap noiseCache = new HashMap();

    public static void main(String[] args) throws Exception {
        String usage =
                "Usage:\tYou didn't enter any arguments. Try again.";
        if (args.length > 0 && ("-h".equals(args[0]) || "-help".equals(args[0]))) {
            System.out.println(usage);
            System.exit(0);
        }

        String index = "index";
        String field = "contents";
        String newQuery="";
        boolean raw = true;
        String normsField = null;
        int hitsPerPage = 10;
        String relevant = "";
        String irrelevant = "";
        String skipped = "";
        int numRel = 0;
        int numIr = 0;
        int numRanked=0,numRounds=1, numUndo=0, numReranked=0;
        int rankMode=5;
        int topTerms=20;
        int target=-1;
        float a=(float)0.25;
        float b=(float)0.25;
        float c=(float)0.75;
        int numPerRound=1;
        Boolean display=false;
        IndexReader reader = IndexReader.open(index);
        int numDocs= reader.numDocs();
        int maxResults=1000;
        ArrayList<Integer> targetMethods = new ArrayList<Integer>();
        String queryFile="";

        for (int i = 0; i < args.length; i++) {
            if ("-index".equals(args[i])) {
                index = args[i + 1];
                i++;
            } else if ("-a".equals(args[i])){
                a=Float.parseFloat(args[i+1]);
                i++;
            } else if ("-b".equals(args[i])){
                b=Float.parseFloat(args[i+1]);
                i++;
            } else if ("-c".equals(args[i])){
                c=Float.parseFloat(args[i+1]);
                i++;
            } else if ("-rank".equals(args[i])) {
                numPerRound=Integer.parseInt(args[i+1]);
                i++;
            } else if ("-maxDocs".equals(args[i])) {
                maxResults=Integer.parseInt(args[i+1]);
                i++;
            }else if ("-field".equals(args[i])) {
                field = args[i + 1];
                i++;
            } else if ("-norms".equals(args[i])) {
                normsField = args[i + 1];
                i++;
            }else if ("-includeTerms".equals(args[i])) {
               rankMode=Integer.parseInt(args[i + 1]);
               topTerms=Integer.parseInt(args[i+2]);
                i+=2;
            }else if("-display".equals(args[i])){
                display=true;
            }else if("-query".equals(args[i])){
                queryFile=args[i+1];
                i++;
            }
        }
        
        System.out.println("Your settings: a="+a+" b="+b+" c"+c+" methods per round="+numPerRound+
                "\nmax similar results="+maxResults+" term ranking= method "+rankMode+","+topTerms+" top terms.");

        int numTerms=0;
        HashMap termDict=new HashMap(0);
        for(int g=0;g<numDocs;g++){
            TermFreqVector test = reader.getTermFreqVector(g,"contents");
            String[] docTerms = test.getTerms();
            int[] freqs = test.getTermFrequencies();
            numTerms+=docTerms.length;
            
            for(int h=0;h<docTerms.length;h++){
                if(!termDict.containsKey(docTerms[h])){
                    termDict.put(docTerms[h], freqs[h]);
                }else{
                    int freq=(Integer) termDict.get(docTerms[h]);
                    freq+=freqs[h];
                    termDict.remove(docTerms[h]);
                    termDict.put(docTerms[h],freq);
                }
            }
        }
        
        int[] rel = new int[numDocs + 1];
        int[] ir = new int[numDocs + 1];
        int[] skip = new int[numDocs + 1];

        //intialize rating arrays
        for (int i = 0; i <= numDocs; i++) {
            rel[i] = 0;
            ir[i] = 0;
            skip[i]=0;
            scores.add((float) 0.0);
            fakeScores.add((float) 0.0);
        }

        if (normsField != null) {
            reader = new OneNormsReader(reader, normsField);
        }

        Searcher searcher = new IndexSearcher(reader);
        Analyzer analyzer = new StandardAnalyzer();

        BufferedReader in =  new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
        QueryParser parser = new QueryParser(field, analyzer);

        
        System.out.println("Please enter your last name: \n");
        String name = in.readLine();
        name = name.trim();
        System.out.print("Please enter the name of your query file: \n");
        queryFile=in.readLine();
        queryFile=queryFile.trim();
        
        String line="";
    
        Boolean done = false;
        BufferedReader qf = new BufferedReader(new FileReader(queryFile));
        String cline="";
        int now=0;
        try{
            while((cline=qf.readLine())!=null){
                now++;
                if(now==1){
                    line=cline;
                    line=line.trim();
                    System.out.println("Your initial query is:"+line);
                }
                else if(now>=3){
                    targetMethods.add(Integer.parseInt(cline)-839);
                }
            }
        }finally{
            qf.close();
         }
        
        //Parse query.
        Query query = parser.parse(line);
        QueryTermVector queryTermVector = new QueryTermVector( line, analyzer );
        String[] queryTerms = queryTermVector.getTerms();
        int[] queryFreqs = queryTermVector.getTermFrequencies();
    
        ArrayList<String> newTerms = new ArrayList<String>();
        ArrayList<Integer> newFreqs = new ArrayList<Integer>();
        ArrayList<String> log = new ArrayList<String>();
        
        
        //Remove terms with df of 0 (aka: aren't in the corpus)
        for(int i=0;i<queryTerms.length;i++){
            int df = reader.docFreq(new Term("contents",queryTerms[i]));
            if(df>0){
                newTerms.add(queryTerms[i]);
                newFreqs.add(queryFreqs[i]);
            }
        }
        String revisedQuery="";
        for(int j=0;j<newTerms.size();j++){
            for(int k=0;k<newFreqs.get(j);k++){
                revisedQuery=revisedQuery+newTerms.get(j)+" ";
            }
        }
        String originalQuery=revisedQuery;
        query = parser.parse(revisedQuery);
        log.add("Initial Query: "+originalQuery);
        queryTermVector = new QueryTermVector( revisedQuery, analyzer );
        System.out.println("Searching for: " + query.toString(field));
        System.out.print("\nGetting initial search results. Please be patient.\n");

        doStreamingSearch(searcher, query);
        ArrayList<Integer> ranks=sortAndDisplay(numDocs, reader,maxResults, rel, ir, skip,0);

        String command = "";
        String rate = "";
        int docNum = 0;
        String rating = "";
        
        //Get initial positions
        ArrayList<Integer> positions = findTargets(ranks,targetMethods,maxResults);
        
        String out="The initial position(s) of your target method(s) are:";
        for(int i=0;i<targetMethods.size();i++){
                out=out+" "+(positions.get(i)+1);
        }
        log.add(out);
        if(display){
            System.out.println(out);
        }

        Calendar cal = Calendar.getInstance();
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH:mm:ss");

        
        //Feedback loop
        done=false;
        while(!done) {
            
            System.out.print("\n\n\n\n\n");
            System.out.println("\nWelcome to round "+numRounds+". Type R for relevant, I for irrelevant, or S for skip.\n"+
                    "If you think a method is the target method, type T.\n"+
                    "To rerank a previous method, type UNDO\n"+
                    "Please be patient. The rating process can take some time.");
            log.add("Round "+numRounds+":");
            int unknown=0;
            ArrayList<Integer> thisRound = new ArrayList<Integer>();
            for(int ranked=0;ranked<numPerRound;ranked++){
                thisRound.add(ranks.get(ranked+unknown));
                Document doc = reader.document(ranks.get(ranked+unknown));
                out="Document #"+ranks.get(ranked+unknown)+" "+doc.get("title");
                System.out.println("\n"+out+"\nSource code:\n");
                printMethodText(doc.get("title"));
                System.out.println("\nEnter a ranking ->     ");
                String ranking = in.readLine();
                if((ranking.toLowerCase().equals("r"))||(ranking.toLowerCase().equals("relevant"))){
                    rel[ranks.get(ranked+unknown)]=1;
                    ir[ranks.get(ranked+unknown)]=0;
                    skip[ranks.get(ranked+unknown)]=0;
                    numRel++;
                    out=out+" = Relevant";
                    cal = Calendar.getInstance();
                    out=sdf.format(cal.getTime())+" "+out;
                    log.add(out);
                    numRanked++;
                }
                else if((ranking.toLowerCase().equals("i"))||(ranking.toLowerCase().equals("irrelevant"))){
                    rel[ranks.get(ranked+unknown)]=0;
                    ir[ranks.get(ranked+unknown)]=1;
                    skip[ranks.get(ranked+unknown)]=0;
                    numIr++;
                    out=out+" = Irrelevant";
                    cal = Calendar.getInstance();
                    out=sdf.format(cal.getTime())+" "+out;
                    log.add(out);
                    numRanked++;
                }
                else if((ranking.toLowerCase().equals("s"))||(ranking.toLowerCase().equals("skip"))){
                    rel[ranks.get(ranked+unknown)]=0;
                    ir[ranks.get(ranked+unknown)]=0;
                    skip[ranks.get(ranked+unknown)]=1;
                    out=out+" = Skipped";
                    cal = Calendar.getInstance();
                    out=sdf.format(cal.getTime())+" "+out;
                    log.add(out);
                    unknown++;
                    ranked--;
                    numRanked++;
                }else if((ranking.toLowerCase().equals("t"))||(ranking.toLowerCase().equals("target"))){
                    target=ranks.get(ranked+unknown);
                    out=out+" = Target Method";
                    cal = Calendar.getInstance();
                    out=sdf.format(cal.getTime())+" "+out;
                    log.add(out);
                    done=true;
                    numRanked++;
                    break;
                }else if((ranking.toLowerCase().equals("undo"))||(ranking.toLowerCase().equals("u"))){
                    ranked--;
                    numUndo++;
                    String rerank="";
                    int changed=0;
                    while(!rerank.toLowerCase().equals("done")){
                         System.out.println("\nHere are the documents you have previously rated.\n"+
                            "To change the rating, type the document number followed by R, I, S, or T and press enter. (Ex: 12345 R)\n"+
                            "To view the source code of a method, type the document number followed by show.\n"+
                            "Type done when finished rerating.\n");
                        for(int docCounter=0;docCounter<numDocs;docCounter++){
                            if(rel[docCounter]==1){
                                Document prev = reader.document(docCounter);
                                System.out.println("Document #"+docCounter+" "+prev.get("title")+" = Relevant");
                            }else if(ir[docCounter]==1){
                                Document prev = reader.document(docCounter);
                                System.out.println("Document #"+docCounter+" "+prev.get("title")+" = Irrelevant");
                            }else if(skip[docCounter]==1){
                                Document prev = reader.document(docCounter);
                                System.out.println("Document #"+docCounter+" "+prev.get("title")+" = Skipped");
                            }
                        }
                        System.out.println("\nEnter a ranking ->    ");
                        rerank=in.readLine();
                        if(!rerank.toLowerCase().equals("done")){
                            int num = Integer.parseInt(rerank.substring(0,rerank.indexOf(" ")));
                            num= Math.abs(num);
                            String rank=rerank.substring(rerank.indexOf(" ")+1);
                            if((num<numDocs)&&((rank.toLowerCase().equals("r"))||rank.toLowerCase().equals("relevant"))){
                                rel[num]=1;
                                if(ir[num]==1)
                                    numIr--;
                                ir[num]=0;
                                skip[num]=0;
                                numRel++;
                                numReranked++;
                                cal = Calendar.getInstance();
                                log.add(sdf.format(cal.getTime())+" Document "+num+" reranked as relevant.");
                                if(thisRound.indexOf(num)==-1)
                                    changed++;
                            }else if((num<numDocs)&&((rank.toLowerCase().equals("i"))||rank.toLowerCase().equals("irrelevant"))){
                                if(rel[num]==1)
                                    numRel--;
                                rel[num]=0;
                                ir[num]=1;
                                numIr++;
                                skip[num]=0;
                                numReranked++;
                                cal = Calendar.getInstance();
                                log.add(sdf.format(cal.getTime())+" Document "+num+" reranked as irrelevant.");
                                if(thisRound.indexOf(num)==-1)
                                    changed++;
                            }else if((num<numDocs)&&((rank.toLowerCase().equals("s"))||rank.toLowerCase().equals("skip"))){
                                rel[num]=0;
                                ir[num]=0;
                                skip[num]=1;
                                numReranked++;
                                cal = Calendar.getInstance();
                                log.add(sdf.format(cal.getTime())+" Document "+num+" reranked as skipped.");
                                if(thisRound.indexOf(num)==-1)
                                    changed++;
                            }else if((num<numDocs)&&((rank.toLowerCase().equals("t"))||rank.toLowerCase().equals("target"))){
                                target=num;
                                done=true;
                                numReranked++;
                                cal = Calendar.getInstance();
                                log.add(sdf.format(cal.getTime())+" Document "+num+" reranked as a target method.");
                                break;
                            }else if((num<numDocs)&&(rank.toLowerCase().equals("show"))){
                                Document show = reader.document(num);
                                printMethodText(show.get("title"));
                                System.out.println("Document "+num+": "+show.get("title"));
                            }else{
                                System.out.println("That wasn't an acceptable rating or the document number doesn't exist.");
                            }
                            if(done==true)
                                break;
                        }else{
                            System.out.println("You are now done reranking.");
                        }
                        if(done==true)
                            break;
                    }
                    if(done==true)
                        break;
                    //Go back to original query because we have changed our minds.
                    if(changed>0){
                        queryTermVector = new QueryTermVector( originalQuery, analyzer);
                    }
                }else{
                    System.out.println("That wasn't an acceptable rating.");
                    ranked--;
                }
                
                if(done==true)
                    break;
            }

            //Build new query
            if(done==false){
                System.out.println("Building a new query. Please be patient.\n");
                newQuery= buildNewQuery(queryTermVector,reader,
                        rel,ir,a,b,c,
                        numDocs,numRel,numIr,
                        maxResults,numTerms,termDict,
                        rankMode,topTerms);
                newQuery = newQuery.trim();

                query = parser.parse(newQuery);
                queryTermVector = new QueryTermVector( newQuery, analyzer );

                doStreamingSearch(searcher, query);
                ranks=sortAndDisplay(numDocs, reader,maxResults, rel, ir, skip,0);
                positions = findTargets(ranks,targetMethods,maxResults);
                
                out="The new position(s) of your target method(s) are:";
                for(int i=0;i<targetMethods.size();i++){
                      out=out+" "+(positions.get(i)+1);
                 }
                log.add(out);
                if(display){
                        System.out.println(out);
                }
                

                numRounds++;

                if (numRanked>=50) {
                    out="Fifty methods ranked. Quitting.";
                    System.out.println(out);
                    log.add(out);
                    done=true;
                }
            }
        } 

        relevant = "";
        irrelevant = "";
        skipped="";

        for (int j = 0; j <= numDocs; j++) {
            if (rel[j] == 1) {
                relevant = relevant + " " + j;
            }
            if (ir[j] == 1) {
                irrelevant = irrelevant + " " + j;
            }
            if (skip[j] == 1) {
                skipped = skipped + " " + j;
            }
        }
        out="\nA total of "+numRanked+" methods were ranked over "+numRounds+" rounds.";
        System.out.println(out);
        log.add(out);
        
        out="Relevant Documents:" + relevant;
        System.out.println(out);
        log.add(out);
        out="Irrelevant Documents:" + irrelevant;
        System.out.println(out);
        log.add(out);
        out="Skipped Documents:"+ skipped;
        System.out.println(out);
        log.add(out);
        out="Target Method:"+target;
        System.out.println(out);
        log.add(out);

        out="\nThe final position(s) of your target method(s) are:";
        String finalTars="";
        Boolean id=false;
        for(int i=0;i<targetMethods.size();i++){
            if(target==targetMethods.get(i))
                id=true;
            finalTars=finalTars+(positions.get(i)+1)+",";
        }
        finalTars=finalTars.substring(0,(finalTars.length()-1));
        if(id==false){
            log.add(out+finalTars);
            if(display)
                System.out.println(out+finalTars);
        }else{
            out="Correctly identified the target method.";
            log.add(out);
            if(display)
                System.out.println(out);
        }
       
        out="\n Your final query is:\n"+newQuery;
        System.out.println(out);
        log.add(out);

        reader.close();
        in.close();
        
        //Print log to a file
        PrintWriter pw = new PrintWriter(new FileWriter(name+"_"+"log_"+queryFile));
        for(int current=0;current<log.size();current++){
            pw.print(log.get(current)+"\n");
        }
        pw.print("Summary;\n");
        pw.write(a+","+b+","+c+","+numPerRound+","+rankMode+","+topTerms+","+numRel+","+numIr+","+(numRanked-numRel-numIr)
                +","+numUndo+","+numReranked+","+numRanked+","+numRounds+","+finalTars);
        pw.close();
    }

    /** Use the norms from one field for all fields.  Norms are read into memory,
     * using a byte of memory per document per searched field.  This can cause
     * search of large collections with a large number of fields to run out of
     * memory.  If all of the fields contain only a single token, then the norms
     * are all identical, then single norm vector may be shared. */
    private static class OneNormsReader extends FilterIndexReader {

        private String field;

        public OneNormsReader(IndexReader in, String field) {
            super(in);
            this.field = field;
        }

        public byte[] norms(String field) throws IOException {
            return in.norms(this.field);
        }
    }

    public static void printMethodText(String fileName) throws IOException{
        String[] words = fileName.split("\\.");
        String className="./corpus";
        for(int i=4;i<words.length-3;i++){
            className=className+"/"+words[i];
        }
        className=className+".java";
        String methodName=words[words.length-1];
        if(methodName.indexOf("_")!=-1&&methodName.indexOf("_")!=0)
            methodName=methodName.substring(0,methodName.indexOf("_"));
        
        BufferedReader qf = new BufferedReader(new FileReader(className));
        String cline="";
        String printVer="";
        Boolean printing=false,isClass=false,override=false,runnable=false;
        String firstWord="";
        
        try{
            while((cline=qf.readLine())!=null){
                printVer=cline;
                cline=cline.trim();
                String[] wordList=cline.split(" ");
                firstWord=wordList[0];
                if((firstWord.indexOf("(")!=-1)&&(firstWord.indexOf("(")!=0))
                    firstWord=firstWord.substring(0,firstWord.indexOf("("));
                if(firstWord.equals("public")||firstWord.equals("private")||firstWord.equals("protected")||firstWord.equals("static")||firstWord.equals("void")||firstWord.equals("class")){
                    if(printing==true&&override==false&&runnable==false){
                        if(wordList[wordList.length-1].indexOf(";")==-1)
                            printing=false;
                    }else{
                        if(runnable==true)
                            runnable=false;
                        isClass=false;
                        for(int j=0;j<wordList.length;j++){
                            if(wordList[j].equals("class"))
                                isClass=true;
                            if(wordList[j].equals("Runnable"))
                                runnable=true;
                            if(wordList[j].indexOf("(")!=-1)
                                wordList[j]=wordList[j].substring(0,wordList[j].indexOf("("));
                            if(wordList[j].equals(methodName)&&isClass==false&&j>0&&cline.indexOf(";")==-1)
                                printing=true;
                        }
                    }
                    override=false;
                }else if(firstWord.equals("@Override")){
                    override=true;
                }
                
                if(firstWord.equals("//{{{")){
                    if(wordList[wordList.length-1].equals("method")||wordList[wordList.length-1].equals("class")||wordList[wordList.length-1].equals("members"))
                        printing=false;
                }
                
                if(printing==true)
                    System.out.println(printVer);
            }
        }finally{
            qf.close();
        }
        
    }
    public static ArrayList<Integer> sortAndDisplay(int numDocs, IndexReader reader, int maxResults, int[] rel, int[] ir, int[] skip, int testMode) throws Exception{
        ArrayList<Float> copy;
        if(testMode==0)
            copy = scores;
        else
            copy = fakeScores;
        
            ArrayList<Integer> cantBe = new ArrayList<Integer>();
            ArrayList<Integer> ranks = new ArrayList<Integer>();
            float max;
            int maxDoc;
            int numDisplayed=0;
            
            //Filter out previously rated documents
            for(int i=0;i<=numDocs;i++){
                if(rel[i]==1||ir[i]==1||skip[i]==1){
                    cantBe.add(i);
                }
            }

            while((cantBe.size()<=numDocs)&&(numDisplayed<=maxResults)){
                maxDoc=-1;
                max=0;
                for(int i=0;i<numDocs;i++){
                    if((copy.get(i)>max)&&(cantBe.indexOf(i)==-1)){
                        max=copy.get(i);
                        maxDoc=i;
                    }     
                }
                if(maxDoc!=-1){
                    Document doc = reader.document(maxDoc);
                    //System.out.println("Document #"+maxDoc+" "+doc.get("title")+", Score: "+max); 
                    numDisplayed++;
                    ranks.add(maxDoc);
                }
                cantBe.add(maxDoc);
            }

       return ranks;
    }
    
    /*
     * Simple method to find the target methods
     */
    
    public static ArrayList<Integer> findTargets(ArrayList<Integer> ranks,ArrayList<Integer> targets, int maxResults){
        ArrayList<Integer> where = new ArrayList<Integer>();
        for(int k=0;k<targets.size();k++){
            where.add(maxResults+1);
        }
        
        for(int i=0;i<ranks.size();i++){
            for(int j=0;j<targets.size();j++){
                if(ranks.get(i).equals(targets.get(j))){
                    where.set(j,i);
                }
            }
        }
        
        return where;
    }
    
    /*
     * This method takes your old query and builds a new one based on your liked and disliked documents.
     * A,B, and C are user-supplied constants to represent weights. Defaults are 1. 
     */
     
    public static String buildNewQuery(QueryTermVector oldQuery, IndexReader reader, int[] rel, int[] ir, float a, float b, float c, int numDocs, int numLiked, int numDisliked, int maxResults, int numTerms,HashMap termDict,int rankMode, int topTerms) throws Exception{
        
        String[] queryTerms = oldQuery.getTerms();
        int[] queryFreqs = oldQuery.getTermFrequencies();
    
        ArrayList<String> newTerms = new ArrayList<String>();
        ArrayList<Float> newFreqs = new ArrayList<Float>();
        ArrayList<String> newQueryTerms = new ArrayList<String>();
        ArrayList<Float> newQueryFreqs = new ArrayList<Float>();
        ArrayList<String> removeTerms = new ArrayList<String>();
        ArrayList<Float> removeFreqs = new ArrayList<Float>();

        
        //Load in terms/freqs from original query
        for(int i=0;i<queryTerms.length;i++){
            int df = reader.docFreq(new Term("contents",queryTerms[i]));
            float idf= (float)Math.log(numDocs/(1+df));

            newQueryTerms.add(queryTerms[i]);
            //newFreqs.add(a*queryFreqs[i]*idf);
            newQueryFreqs.add(a*queryFreqs[i]);
        }

        
        //Compile list of liked and disliked terms
        if(numLiked>0||numDisliked>0){
            for(int j=0;j<numDocs;j++){
                if(rel[j]==1){
                    //Grab term list/freqs
                    TermFreqVector test = reader.getTermFreqVector(j,"contents");
                    String[] docTerms = test.getTerms();
                    int[] freqs = test.getTermFrequencies();
                    
                    //Go through list of terms
                    for(int k=0;k<docTerms.length;k++){
                        //See if term is already in list. Add it if not. Adjust freq if so.
                        if(newTerms.indexOf(docTerms[k])==-1){
                            newTerms.add(docTerms[k]);
                            int df = reader.docFreq(new Term("contents",docTerms[k]));
                            float idf= (float)Math.log(numDocs/(1+df));
                            //newFreqs.add(((b/numLiked)*(freqs[k]*idf)));
                            newFreqs.add((b/numLiked)*freqs[k]);
                            //System.out.println("REL: new term: "+newFreqs.get(k));
                        } else{
                            int index = newTerms.indexOf(docTerms[k]);
                            float numToAdd = newFreqs.get(index);
                            int df = reader.docFreq(new Term("contents",docTerms[k]));
                            float idf= (float)Math.log(numDocs/(1+df));
                            //numToAdd = numToAdd + ((b/numLiked)*(freqs[k]*idf));
                            numToAdd = numToAdd + ((b/numLiked)*freqs[k]);
                            newFreqs.set(index,numToAdd);
                            //System.out.println("REL: old term: "+newFreqs.get(k));
                        }
                    }
                    
                }else if(ir[j]==1){
                    //Grab term list/freqs
                    TermFreqVector test = reader.getTermFreqVector(j,"contents");
                    String[] docTerms = test.getTerms();
                    int[] freqs = test.getTermFrequencies();
                    
                    //Go through list of terms
                    for(int k=0;k<docTerms.length;k++){
                        //See if term is already in list. Add it if not. Adjust freq if so.
                        if(removeTerms.indexOf(docTerms[k])==-1){
                            removeTerms.add(docTerms[k]);
                            int df = reader.docFreq(new Term("contents",docTerms[k]));
                            float idf= (float)Math.log(numDocs/(1+df));
                            //removeFreqs.add(((c/numDisliked)*(freqs[k]*idf)));
                            removeFreqs.add((c/numDisliked)*freqs[k]);
                            //System.out.println("IR: new term: "+removeFreqs.get(k));
                        } else{
                            int index = removeTerms.indexOf(docTerms[k]);
                            float numToRemove = removeFreqs.get(index);
                            int df = reader.docFreq(new Term("contents",docTerms[k]));
                            float idf= (float)Math.log(numDocs/(1+df));
                            //numToAdd = numToAdd + ((b/numLiked)*(freqs[k]*idf));
                            numToRemove = numToRemove + ((c/numDisliked)*freqs[k]);
                            removeFreqs.set(index,numToRemove);
                            //System.out.println("IR: old term: "+removeFreqs.get(k));
                        }
                    }
                    
                }
            }
        }
        
      
        //Rank relevant terms
        String newQuery="";
        ArrayList<Double> rankScores=new ArrayList<Double>();
        ArrayList<String> bestTerms = new ArrayList<String>();
        
        //switch based on sort order    
        if(rankMode==1){
            //compute noise
            for(int n=0;n<newTerms.size();n++){
                double noise=0;
                if(noiseCache.containsKey(newTerms.get(n))){
                    noise=(Double) noiseCache.get(newTerms.get(n));
                }else{
                    for(int o=0;o<numDocs;o++){
                        TermFreqVector test = reader.getTermFreqVector(o,"contents");
                        String[] docTerms = test.getTerms();
                        int[] freqs = test.getTermFrequencies();
                        for(int p=0;p<docTerms.length;p++){
                            if(docTerms[p].equals(newTerms.get(n))){
                                double newNoise=(double) (freqs[p]*numTerms)/(Integer) termDict.get(newTerms.get(n));
                                newNoise*=Math.log((Integer)termDict.get(newTerms.get(n))*freqs[p]);
                                noise+=newNoise;
                                break;
                            }
                        }
                    }
                    noiseCache.put(newTerms.get(n),noise);
                }
                rankScores.add(noise);
            }
        }else if(rankMode==2){
            //Mode 2 is Pk (num documents in relevant set containing term k
            for(int n=0;n<newTerms.size();n++){
                double times=0;
                
                for(int o=0;o<rel.length;o++){
                    //Is the document relevant?
                    if(rel[o]==1){
                        TermFreqVector test = reader.getTermFreqVector(o,"contents");
                        String[] docTerms = test.getTerms();
                        int[] freqs = test.getTermFrequencies();

                        //Does it contain the term?
                        for(int p=0;p<docTerms.length;p++){
                            if(docTerms[p].equals(newTerms.get(n))){
                                times++;
                                break;
                            }
                        }
                    }
                }
                rankScores.add(times);
            }
        }else if(rankMode==3){
            //Mode 3 is modified noise (noise within relevant set)
            
            for(int n=0;n<newTerms.size();n++){
                double noise=0;
                int times=0;
                
                //First pass to get Pk
                for(int o=0;o<rel.length;o++){
                    //Is the document relevant?
                    if(rel[o]==1){
                        TermFreqVector test = reader.getTermFreqVector(o,"contents");
                        String[] docTerms = test.getTerms();
                        int[] freqs = test.getTermFrequencies();

                        //Does it contain the term?
                        for(int p=0;p<docTerms.length;p++){
                            if(docTerms[p].equals(newTerms.get(n))){
                                times++;
                                break;
                            }
                        }
                    }
                }
                
                //Second pass to collect modified noise
                 for(int o=0;o<rel.length;o++){
                    //Is the document relevant?
                    if(rel[o]==1){
                        TermFreqVector test = reader.getTermFreqVector(o,"contents");
                        String[] docTerms = test.getTerms();
                        int[] freqs = test.getTermFrequencies();
                        for(int p=0;p<docTerms.length;p++){
                            if(docTerms[p].equals(newTerms.get(n))){
                                double newNoise=(double) (freqs[p]*times)/(Integer) termDict.get(newTerms.get(n));
                                newNoise*=Math.log((Integer)termDict.get(newTerms.get(n))*freqs[p]);
                                noise+=newNoise;
                                break;
                            }
                        }
                    }
                 }
                rankScores.add(noise);
            }
        }else if(rankMode==4){
            //Mode 4 is noise*num occurences of k in the relevant set
           
            for(int n=0;n<newTerms.size();n++){
                double noise=0;
                int times=0;
                 //First pass to collect noise.
                if(noiseCache.containsKey(newTerms.get(n))){
                    noise=(Double) noiseCache.get(newTerms.get(n));
                }else{
                    for(int o=0;o<numDocs;o++){
                        TermFreqVector test = reader.getTermFreqVector(o,"contents");
                        String[] docTerms = test.getTerms();
                        int[] freqs = test.getTermFrequencies();
                        for(int p=0;p<docTerms.length;p++){
                            if(docTerms[p].equals(newTerms.get(n))){
                                double newNoise=(double) (freqs[p]*numTerms)/(Integer) termDict.get(newTerms.get(n));
                                newNoise*=Math.log((Integer)termDict.get(newTerms.get(n))*freqs[p]);
                                noise+=newNoise;
                                break;
                            }
                        }
                    }
                    noiseCache.put(newTerms.get(n),noise);
                }
                
                //Second pass to collect RTFk
                for(int o=0;o<rel.length;o++){
                    //Is the document relevant?
                    if(rel[o]==1){
                        TermFreqVector test = reader.getTermFreqVector(o,"contents");
                        String[] docTerms = test.getTerms();
                        int[] freqs = test.getTermFrequencies();

                        //Does it contain the term?
                        for(int p=0;p<docTerms.length;p++){
                            if(docTerms[p].equals(newTerms.get(n))){
                                times+=freqs[p];
                                break;
                            }
                        }
                    }
                }
                rankScores.add(noise*times);
            }
        }else if(rankMode==5){
             //Mode 5 is noise*num occurences of k in the collection*nom docs in rel set containing k
           
            for(int n=0;n<newTerms.size();n++){
                double noise=0;
                int times=0;
                 //First pass to collect noise.
                if(noiseCache.containsKey(newTerms.get(n))){
                    noise=(Double) noiseCache.get(newTerms.get(n));
                }else{
                    for(int o=0;o<numDocs;o++){
                        TermFreqVector test = reader.getTermFreqVector(o,"contents");
                        String[] docTerms = test.getTerms();
                        int[] freqs = test.getTermFrequencies();
                        for(int p=0;p<docTerms.length;p++){
                            if(docTerms[p].equals(newTerms.get(n))){
                                double newNoise=(double) (freqs[p]*numTerms)/(Integer) termDict.get(newTerms.get(n));
                                newNoise*=Math.log((Integer)termDict.get(newTerms.get(n))*freqs[p]);
                                noise+=newNoise;
                                break;
                            }
                        }
                    }
                    noiseCache.put(newTerms.get(n),noise);
                }
                
                //Second pass to collect Pk
                for(int o=0;o<rel.length;o++){
                    //Is the document relevant?
                    if(rel[o]==1){
                        TermFreqVector test = reader.getTermFreqVector(o,"contents");
                        String[] docTerms = test.getTerms();
                        int[] freqs = test.getTermFrequencies();

                        //Does it contain the term?
                        for(int p=0;p<docTerms.length;p++){
                            if(docTerms[p].equals(newTerms.get(n))){
                                times++;
                                break;
                            }
                        }
                    }
                }
                rankScores.add(noise*(Integer) termDict.get(newTerms.get(n))*times);
            }
        }else if(rankMode==6){
             //Mode 5 is noise*num occurences of k in the collection
           
            for(int n=0;n<newTerms.size();n++){
                double noise=0;
                 //First pass to collect noise.
                if(noiseCache.containsKey(newTerms.get(n))){
                    noise=(Double) noiseCache.get(newTerms.get(n));
                }else{
                    for(int o=0;o<numDocs;o++){
                        TermFreqVector test = reader.getTermFreqVector(o,"contents");
                        String[] docTerms = test.getTerms();
                        int[] freqs = test.getTermFrequencies();
                        for(int p=0;p<docTerms.length;p++){
                            if(docTerms[p].equals(newTerms.get(n))){
                                double newNoise=(double) (freqs[p]*numTerms)/(Integer) termDict.get(newTerms.get(n));
                                newNoise*=Math.log((Integer)termDict.get(newTerms.get(n))*freqs[p]);
                                noise+=newNoise;
                                break;
                            }
                        }
                    }
                    noiseCache.put(newTerms.get(n),noise);
                }
                
                rankScores.add(noise*(Integer) termDict.get(newTerms.get(n)));
            }
        }
        //sort
        for(int o=0;o<topTerms;o++){
            double best=0;
            int pos=-1;
            for(int n=0;n<rankScores.size();n++){
                if(rankScores.get(n)>best&&(!bestTerms.contains(newTerms.get(n)))){
                    best=rankScores.get(n);
                    pos=n;
                }
            }
            if(pos>=0){
                bestTerms.add(newTerms.get(pos));
                //System.out.println(newTerms.get(pos));
            }else{
                break;
            }
        }

        //Add relevant terms if they are high ranked
        for(int n=0;n<newTerms.size();n++){
            String term = newTerms.get(n);
            float freq = newFreqs.get(n);
            if(freq>0){
               if(bestTerms.contains(term)){
                    if(newQueryTerms.indexOf(term)==-1){
                        newQueryTerms.add(term);
                        newQueryFreqs.add(freq);
                        //System.out.println("Term added "+term+"-"+freq);
                    }else{
                        float oldFreq=newQueryFreqs.get(newQueryTerms.indexOf(term));
                        freq+=oldFreq;
                        newQueryFreqs.set(newQueryTerms.indexOf(term),freq);
                        //System.out.println("Rel term already there: "+term+"-"+freq);
                    }
               }
            }
        }
        
        //Remove irrelevant terms
         for(int n=0;n<removeTerms.size();n++){
            String term = removeTerms.get(n);
            float freq = removeFreqs.get(n);
            if(freq>0){
                if(newQueryTerms.indexOf(term)!=-1){
                    float oldFreq=newQueryFreqs.get(newQueryTerms.indexOf(term));
                    oldFreq-=freq;
                    newQueryFreqs.set(newQueryTerms.indexOf(term),oldFreq);
                    //System.out.println("Ir term:"+term+"-"+freq+" now:"+oldFreq);
                }
            }
        }
        
        //Now, build the string
        for(int n=0;n<newQueryTerms.size();n++){
            String term = newQueryTerms.get(n);
            float freq = newQueryFreqs.get(n);
            for(int o=0;o<freq;o++){
                newQuery=newQuery+term+" ";
            }
        }

        //System.out.println("\n"+newQuery+"\n");
        return newQuery;
    }
    /**
     * This method uses a custom HitCollector implementation which simply prints out
     * the docId and score of every matching document. 
     * 
     *  This simulates the streaming search use case, where all hits are supposed to
     *  be processed, regardless of their relevance.
     */
    public static void doStreamingSearch(final Searcher searcher, Query query) throws IOException {
        HitCollector streamingHitCollector = new HitCollector() {
            // simply print docId and score of every matching document
            public void collect(int doc, float score) {
                scores.set(doc,score);
                //System.out.println("doc=" + doc + " score=" + score);
                
            }
        };

        searcher.search(query, streamingHitCollector);
    }
    
     public static void doStreamingSearch(final Searcher searcher, Query query, int test) throws IOException {
        HitCollector streamingHitCollector = new HitCollector() {
            // simply print docId and score of every matching document
            public void collect(int doc, float score) {
                fakeScores.set(doc,score);
                //System.out.println("doc=" + doc + " score=" + score);
                
            }
        };

        searcher.search(query, streamingHitCollector);
    }

    /**
     * This demonstrates a typical paging search scenario, where the search engine presents 
     * pages of size n to the user. The user can then go to the next page if interested in
     * the next hits.
     * 
     * When the query is executed for the first time, then only enough results are collected
     * to fill 5 result pages. If the user wants to page beyond this limit, then the query
     * is executed another time and all hits are collected.
     * 
     */
    public static void doPagingSearch(BufferedReader in, Searcher searcher, Query query,
            int hitsPerPage, boolean raw, boolean interactive) throws IOException {

        // Collect enough docs to show 5 pages
        TopDocCollector collector = new TopDocCollector(5 * hitsPerPage);
        searcher.search(query, collector);
        ScoreDoc[] hits = collector.topDocs().scoreDocs;

        int numTotalHits = collector.getTotalHits();
        System.out.println(numTotalHits + " total matching documents");

        int start = 0;
        int end = Math.min(numTotalHits, hitsPerPage);

        while (true) {
            if (end > hits.length) {
                System.out.println("Only results 1 - " + hits.length + " of " + numTotalHits + " total matching documents collected.");
                System.out.println("Collect more (y/n) ?");
                String line = in.readLine();
                if (line.length() == 0 || line.charAt(0) == 'n') {
                    break;
                }

                collector = new TopDocCollector(numTotalHits);
                searcher.search(query, collector);
                hits = collector.topDocs().scoreDocs;
            }

            end = Math.min(hits.length, start + hitsPerPage);

            for (int i = start; i < end; i++) {
                if (raw) {                              // output raw format
                    System.out.println("doc=" + hits[i].doc + " score=" + hits[i].score);
                    continue;
                }

                Document doc = searcher.doc(hits[i].doc);
                String path = doc.get("path");
                if (path != null) {
                    System.out.println((i + 1) + ". " + path);
                    String title = doc.get("title");
                    if (title != null) {
                        System.out.println("   Title: " + doc.get("title"));
                    }
                } else {
                    System.out.println((i + 1) + ". " + "No path for this document");
                }

            }

            if (!interactive) {
                break;
            }

            if (numTotalHits >= end) {
                boolean quit = false;
                while (true) {
                    System.out.print("Press ");
                    if (start - hitsPerPage >= 0) {
                        System.out.print("(p)revious page, ");
                    }
                    if (start + hitsPerPage < numTotalHits) {
                        System.out.print("(n)ext page, ");
                    }
                    System.out.println("(q)uit or enter number to jump to a page.");

                    String line = in.readLine();
                    if (line.length() == 0 || line.charAt(0) == 'q') {
                        quit = true;
                        break;
                    }
                    if (line.charAt(0) == 'p') {
                        start = Math.max(0, start - hitsPerPage);
                        break;
                    } else if (line.charAt(0) == 'n') {
                        if (start + hitsPerPage < numTotalHits) {
                            start += hitsPerPage;
                        }
                        break;
                    } else {
                        int page = Integer.parseInt(line);
                        if ((page - 1) * hitsPerPage < numTotalHits) {
                            start = (page - 1) * hitsPerPage;
                            break;
                        } else {
                            System.out.println("No such page");
                        }
                    }
                }
                if (quit) {
                    break;
                }
                end = Math.min(numTotalHits, start + hitsPerPage);
            }

        }

    }
}