package nxm.sys.lib;

import java.awt.Component;
import java.awt.HeadlessException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;

import javax.swing.DefaultListCellRenderer;
import javax.swing.DefaultListModel;
import javax.swing.JList;

import nxm.sys.inc.AsciiMap;
import nxm.sys.inc.Commandable;
import nxm.sys.inc.InternalUseOnly;
import nxm.sys.lib.FileName.FNCase;
import nxm.sys.lib.FileUtil.FnFilter;
import tools.BuildProps;

/** Search Logic Help Panel to search help index and display results (hits) to user and allow user
    to view result contents in their browser window. This is just a placeholder for new search
    logic that is not Lucene jar dependent

    @author Jeanette McCall, Zach Matthews, Lina Romeo
 */
@SuppressWarnings({"serial"})
public class HelpSearchEngine extends javax.swing.JPanel implements AsciiMap {

  /** JList model for search results */ protected DefaultListModel<Document> resultsModel;
  /** results list */                   protected javax.swing.JList<Document> results;

  /** The Midas Context */
  private Midas M;

  /** Help index searcher instance */
  private Searcher searcher;
  /** pre-compiled pattern for HTML &lt;pre&gt; tag */
  private static final Pattern PRE_OPEN_REGEX  = Pattern.compile("(?i)<pre>");
  /** pre-compiled pattern for HTML &lt;/pre&gt; tag */
  private static final Pattern PRE_CLOSE_REGEX  = Pattern.compile("(?i)</pre>");
  /** pre-compiled pattern for HTML &lt;title&gt; tag */
  private static final Pattern TITLE_OPEN_REGEX  = Pattern.compile("(?i)<title>");
  /** pre-compiled pattern for HTML &lt;/title&gt; tag */
  private static final Pattern TITLE_CLOSE_REGEX  = Pattern.compile("(?i)</title>");
  /** pre-compiled pattern for HTML &lt;script tag */
  private static final Pattern SCRIPT_OPEN_REGEX  = Pattern.compile("(?i)<script");
  /** pre-compiled pattern for HTML &lt;/script&gt; tag */
  private static final Pattern SCRIPT_CLOSE_REGEX  = Pattern.compile("(?i)</script>");
  /** pre-compiled pattern for HTML &lt;style tag */
  private static final Pattern STYLE_OPEN_REGEX  = Pattern.compile("(?i)<style");
  /** pre-compiled pattern for HTML &lt;/style&gt; tag */
  private static final Pattern STYLE_CLOSE_REGEX  = Pattern.compile("(?i)</style>");
  /** Implementation does not have lucene.jar dependency */
  public static final boolean NO_LUCENE_DEPENDENCY = true;

  private static final String[] RESERVED_WORDS = {"A","AN","AND","ARE","AS","AT",
      "BE","BUT","BY","FOR","IF","IN","INTO","IS","IT","NO","NOT","OF","ON","OR",
      "SUCH","THAT","THE","THEIR","THEN","THERE","THESE","THEY","THIS","TO","WAS",
      "WILL","WITH"};

  private static final String RESERVED_WORDS_REGEX = ".*\\b(A|AN|AND|ARE|AS|AT|BE|BUT|" +
      "BY|FOR|IF|IN|INTO|IS|IT|NO|NOT|OF|ON|OR|SUCH|THAT|THE|THEIR|THEN|THERE|THESE|" +
      "THEY|THIS|TO|WAS|WILL|WITH)\\b.*";

  private static final int[] INTS_TO_KEEP = {1000,1999,2000,3000,4000,5000,5010,6000};

  private static final String ONE_OR_TWO_BACKSLASH = "\\\\{1,2}";

  /** Table with locations of the html files to search */
  private static Table indexLocations = new Table();

  private static boolean debug = false;

  /** Constructor - calls super(), create empty resultsModel, then initialize. */
  public HelpSearchEngine () throws HeadlessException {
    super();
    M = Shell.getMidasContext();
  }

  /** Results Model to be used by results JList during initComponents. */
  protected DefaultListModel<Document> getResultsModel () { return resultsModel; }

  protected String findHits (HelpSearchEngine.Searcher searcher, String queryString, int maxResults) throws ParseException {
    List<Document> hits = searcher.query(queryString,maxResults+1); // execute search
    queryString = queryString.trim();
    int len = hits.size();
    String statusMessage;
    if (len == 0 || len == 1)   { statusMessage = len + " result found for: "  + queryString; }
    else if (len <= maxResults || maxResults < 0) { statusMessage = len + " results found for: " + queryString; }
    else { // else number results > maxResult (limiting result list)
      len = maxResults; // show only maxResults
      statusMessage = len +"+ results found for: " + queryString;
    }

    for (int ii=0; ii<len; ii++) {
      resultsModel.addElement(hits.get(ii));
    } // end for loop
    return statusMessage;
  }

  /** show contents of selected result's hits using the registered document viewer
    (via {@link #setViewer(nxm.sys.libg.MJHelp.DocViewerIF)}. e.g. show it in
    the browser.
   */
  protected String findResultHitPath () {
    String path = "";
    int i = results.getSelectedIndex();
    if (i<0) return path;
    try {
      Document doc = resultsModel.get(i);
      // display document in viewer if it exists
      if (doc != null) {
        path = doc.getPath();
      }
    } catch (ArrayIndexOutOfBoundsException e) {
      // ignore this as another query may have changed the index
      Shell.warnStackTrace("showResultHit Exception: "+e, e);
    }
    return path;
  }

  /** List results
   *  @param list     Numbered list of search entries to the screen
   *  @param query    query string to search for (this cannot be null)
   *  @param showpath Show path to files for query results
   */
  private void listResults (List<Document> list, String query, boolean showpath) {
    int  len  = list.size();
    String statusMessage = len + " result";
    if (len == 0 || len == 1) statusMessage += " found for: ";
    else                      statusMessage += "s found for: ";
    M.more(Terminal.INIT);
    M.more(statusMessage + query);
    String nmroot = Shell.getNmRoot();
    for (int i=0; i<len && M.more(); i++) {
      Document doc = list.get(i);
      String doctitle = doc.getTitle();// get its title
      String path     = doc.getPath();  // get its path field
      String filename = path.substring(path.lastIndexOf('/') + 1);
      String pathFromNMROOT = filename;
      int nmrootIndex = path.indexOf(nmroot);
      if (nmrootIndex != -1) pathFromNMROOT = path.substring(nmroot.length());    // move past NMROOT in path
      if ((doctitle == null) || doctitle.equals("")) { //use the path if it has no title
        doctitle = pathFromNMROOT;
      } else if (showpath) {
        doctitle += " (" + pathFromNMROOT +")";
      } else {
        doctitle += " (" + filename +")";
      }
      String index = " "+i+") "; if (i<10) index = " "+index;
      M.more(index + " " + doctitle);
    } // end for loop
    M.more(Terminal.EXIT);
  }

  private List<Document> searchIndex (String queryStr,BuildProps sysBuildProps) {
    List<Document> results = new ArrayList<>(0);
    if (searcher == null) {
      String indexPath = sysBuildProps.getPath("help.index.dir");
      if (indexPath == null || indexPath.length()==0) M.error("help.index.dir not set in build.props!");
      try {
         searcher = new Searcher(indexPath);
      } catch (IOException ioe) {
        M.warning("Cannot initialize help index search. " + ioe);
      }
    }

    try {
      if (searcher != null) results = searcher.query(queryStr, 100); // execute search (limit to 100 results)
    } catch (ParseException pe) {
      String message = "Invalid search string. " + pe.getMessage();
      M.warning(message);
    }
    return results;
  }

  /** Index files for help and searches
   *
   *  @param sysBuildProps The BuildProps for the SYS Option Tree
   *  @param filter        Optional override of the filename filter
   *  @param moreDirs      '|' separated list of additional directories to include
   *  @param debug         The debug flag
   *  @return FINISH
   */
  @InternalUseOnly
  public int runHelpIndex (BuildProps sysBuildProps, String filter, String moreDirs, boolean debug) {
    String  indexPath      = sysBuildProps.getPath("help.index.dir");
    int     maxFieldLength = Integer.parseInt(sysBuildProps.getProperty("help.maxFieldLength", "100000"));
    String  locations      = sysBuildProps.getProperty("help.index.locations");
    String  useParentDirs  = sysBuildProps.getProperty("help.index.useparent");
    if ((filter==null) || filter.isEmpty()) filter = sysBuildProps.getProperty("help.index.filter", filter);
    if (indexPath==null || indexPath.length()==0) M.error("help.index.dir not set in build.props!");
    if (moreDirs!=null) locations += "|" + moreDirs;
    createIndexLocationTable(locations,useParentDirs);

    M.info("Creating help index at " + indexPath + " filter=" + filter +
        "\nThis may take a minute or two...");
    long startTime = System.currentTimeMillis();

    Indexer indexer = new Indexer(indexPath,filter);
    indexer.runIndexer(indexPath);

    long endTime = System.currentTimeMillis();
    int numDocs = indexer.getDocsCount(); // total number of documents indexed
    M.info("Help index created. Indexed "+ numDocs +" files in "+ (endTime - startTime)/1000 +" seconds.");

    return Commandable.FINISH;
  }

  private void createIndexLocationTable (String locations, String useParentDirs) {
    if (locations!=null && locations.length()>0) {
      if (debug) M.info("indexing dirs = "+locations+"  -  useParentDirs ="+useParentDirs);
      locations.replaceAll(ONE_OR_TWO_BACKSLASH, "/");
      String[] dirs = locations.split("[|]");
      for (String dir : dirs) {
        dir = dir.trim();
        if (dir.length()>0) {
          boolean useParentDir = (useParentDirs.indexOf(dir) != -1) ; // false for most use cases
          if (useParentDir) {
            int beforeSub = dir.lastIndexOf('/');
            dir = dir.substring(0, beforeSub);
          }
          String key = dir.substring(dir.lastIndexOf('/')+1);
          indexLocations.put(key.toUpperCase(),dir);
        }
      }
    }
  }

  /** This method gets called when a new opt is built after the initial build.
   *  This updates the index files and search_words_to_files.tbl
   *  @param opt             The Option Tree getting built after the initial build
   *  @param sysBuildProps   The BuildProps for the newly added Option Tree
   *  @param filter          Optional override of the filename filter
   *  @param moreDirs        '|' separated list of additional directories to include
   *  @return FINISH
   */
  @InternalUseOnly
  public int updateHelpIndex (String opt, BuildProps sysBuildProps, String filter, String moreDirs) {
    String  indexPath      = sysBuildProps.getPath("help.index.dir");
    if (indexPath == null || indexPath.length()==0) M.error("help.index.dir not set in build.props!");
    String  locations      = sysBuildProps.getProperty("help.index.locations");
    String  useParentDirs  = sysBuildProps.getProperty("help.index.useparent");
    if ((filter==null) || filter.isEmpty()) filter = sysBuildProps.getProperty("help.index.filter", filter);
    if (moreDirs!=null) locations += "|" + moreDirs;
    createIndexLocationTable(locations,useParentDirs);

    Indexer indexer = new Indexer(indexPath,filter);
    if (searcher == null) {
      try {
         searcher = new Searcher(indexPath);
      } catch (IOException ioe) {
        M.warning("Cannot initialize help index search. " + ioe);
      }
    }
    indexer.updateHelpIndex(opt, searcher.searchWordsTable, searcher.titleCharTable);
    searcher.loadSearchWordsTable();
    return Commandable.FINISH;
  }

  /** Handle user's interaction with search results from terminal, such as selecting and
   *  viewing the search result contents. (CLI - command line interface)
   *
   *  @param sysBuildProps The BuildProps for the SYS Option Tree
   *  @param queryStr      Search string
   *  @param showpath      Show path to files for query results
   *  @return FINISH
   */
  @InternalUseOnly
  @SuppressWarnings("deprecation") // for M.terminal
  public int runHelpCLI (BuildProps sysBuildProps, String queryStr, boolean showpath) {
    if (queryStr==null || queryStr.trim().length() == 0) {
      queryStr = M.terminal.getInput("Enter search string: ", "").trim();
      if (queryStr.length()==0) return Commandable.FINISH; // nothing to search for
    }

    // search help index for specified query string
    List<Document> results = searchIndex(queryStr,sysBuildProps);

    int len = results.size();
    if (len > 1) listResults(results,queryStr,showpath); // output the list of matches if more than 1 result

    // loop on what to do next
    for (int ilist=0; ilist<len; ilist++) {
      String ans="N";
      if (len>1) ans = M.terminal.getInput("Enter item number or [N]ext/Quit/List ?: ","");
      if (M.terminal.getInterrupt()>0) {               break; }

      if (ans.length()==0) ans="N";
      else ans = ans.trim().toUpperCase();

      if (ans.startsWith("Q")) {
                                                       break;
      }
      else if (ans.startsWith("L")) {
        listResults(results,queryStr,showpath);
        ilist--;                                       continue;
      }
      else if (ans.startsWith("N")) {
        if (ilist==len) {                              break; }
      }
      else {
        try {
          ilist = Integer.parseInt(ans);
        } catch (NumberFormatException nfe) {
          ilist--;                                     continue;
        }
      }

      if (ilist >= len) {                              break; }
      Object obj = results.get(ilist);

      // display result content if it is a Document Hit
      if (obj instanceof Document) {
        Document doc   = (Document)obj;
        String docPath = doc.getPath();  // get its path field
        TextFile tf    = new TextFile(M,new FileName(docPath,FNCase.KeepCase));

        if (!tf.open(TextFile.NOABORT)) continue;
        M.more(Terminal.INIT);
        M.more("Displaying file: " + docPath);
        String text;
        boolean isHtml = tf.getMimeType().equals("text/html");
        boolean insidePreTag = false;
        boolean insideScriptTag = false;
        boolean insideStyleTag = false;
        while (M.more() && (text=tf.read())!=null) {
          if (isHtml) {
            if (!insidePreTag && PRE_OPEN_REGEX.matcher(text).matches())  insidePreTag = true;
            if ( insidePreTag && PRE_CLOSE_REGEX.matcher(text).matches()) insidePreTag = false;
            if (!insideScriptTag && SCRIPT_OPEN_REGEX.matcher(text).find()) insideScriptTag = true;
            if ( insideScriptTag && SCRIPT_CLOSE_REGEX.matcher(text).find()) insideScriptTag = false;
            if (!insideStyleTag && STYLE_OPEN_REGEX.matcher(text).find()) insideStyleTag = true;
            if ( insideStyleTag && STYLE_CLOSE_REGEX.matcher(text).find()) insideStyleTag = false;
            text = StringUtil.html2Text(text);                      // convert HTML to text
            if (!insidePreTag && text.trim().length()==0) continue; // ignore empty lines outside <pre> tags
            if (!insidePreTag && (insideScriptTag || insideStyleTag)) continue; // ignore script and style tags outside pre tags
          }

          M.more(text);
        }
        M.more(Terminal.EXIT);
        tf.close();
        M.println("For better formatting of the contents, view the following in your browser.\n "+docPath);  // using M.println instead of M.terminal.writeln to allow logging - NXM-1813
      }
    } // end for loop

    return Commandable.FINISH;
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // Inner classes
  //////////////////////////////////////////////////////////////////////////////////////////////////

  /** Creates index files that contain hash numbers for the files
   *  These are stored under htdocs/index/ and named with the
   *  first 3 Hex characters after the 'S' score mentioned below.
   *  <pre>
   *    |-hash-|
   *  0xS0010000-htdocs/help/sys_
   *  0xS0020000-htdocs/help/dsp_
   *  0xS1000000-htdocs/api/
   *  0xS1010000-htdocs/api/sys/
   *  0xS2000000-htdocs/training
   *  0xS3000000-htdocs/usersguide
   *  0xS4010000-nxm/sys/docs
   *    |tbl|
   *  </pre>
   *  Each of these tables list the files in that area giving them
   *  a Hex number 0000 - FFFF
   */
  private static class Indexer {
    private int docsCount = 0;
    private String indexPath = null;
    private FnFilter fnFilter = null;  // filename filter for files to index

    private Midas midas = Shell.getMidasContext();
    private String nmroot = Shell.getNmRoot();

    private static final Pattern NON_ALPHA_NUM_REGEX = Pattern.compile("\\P{Alnum}");

    private static final Pattern INT_REGEX = Pattern.compile("\\p{Digit}+");
    private static final int INT_MAX_DIGITS = 4;

    // Listed in priority number
    private static final String HELP_TYPE_NUM    = "0";
    private static final String API_TYPE_NUM     = "1";
    private static final String TRAIN_TYPE_NUM   = "2";
    private static final String USERS_TYPE_NUM   = "3";
    private static final String OPTDOCS_TYPE_NUM = "4";

    private static final double STARTING_NON_TITLE_SCORE = 6;
    private static final double MIN_NON_TITLE_SCORE = 1;
    private static final double TITLE_SCORE = 0;

    private static final double BONUS_INCREMENT = 300;
    private static final String SEARCH_WORDS_FILE_NAME = "search_words_to_files.tbl";
    private static final String TITLE_CHAR_FILE_NAME = "title_char_to_files.tbl";

    private static final String OPT_TREE_TABLE = "optTreeIndex.tbl";

    private Indexer (String indexPath, String filter) {
      this.indexPath = indexPath;
      fnFilter = FileUtil.createFnFilter(filter,"*.html|*.htm|*.txt");
    }

    private void runIndexer (String indexPath) {
      makeIndexFiles(); // Make location files each with a list of files in that area of the help system
      makeSearchWords(); // Generate the master list of searchable words
    }

    /** This method gets called when a new opt is built after the initial build
     *  New index files are created and the master list of searchable words are updated
     *
     *  @param opt       The Option Tree getting built after the initial build
     *  @param searchWordsTable search words table
     *  @param titleCharTable   title starting char table
     */
    public int updateHelpIndex (String opt, Table searchWordsTable, Table titleCharTable) {
      // read in optTreeIndex and if it's there return
      // Not rebuilding help index for opt... to do so make help all
      Table optIndexTbl = Table.fromTableFile(midas,new FileName(indexPath+File.separator+OPT_TREE_TABLE,FNCase.KeepCase));
      if (optIndexTbl.containsKey(opt)) {
        midas.info("Not rebuilding help index for OPT:" + opt + " to do so run: make help all");
        return Commandable.FINISH;
      }
      makeIndexFiles(opt); // Make location files each with a list of files in that area of the help system
      updateSearchWords(opt, searchWordsTable, titleCharTable); // Update the master list of searchable words
      return Commandable.FINISH;
    }

    /** Each table contains a list of filtered text-based files for a given searchable area
     *  (i.e. help sys, api dsp, training, etc.)
     *  These tables get mapped to a three-digit hex number representing the searchable area
     */
    private void makeIndexFiles () {
      String[] opts = midas.results.get("OPT.PATH").toString().toLowerCase().split(",");
      opts = Util.reverseArray(opts, 0);
      Table optIndexTbl = new Table();
      int opIdx = 1;
      for (String opt : opts) {
        optIndexTbl.put(opt.toUpperCase(), String.format("%02x",opIdx++));
      }

      // API
      // arrays are number of opts + 1 (some docs may be under api but not be part of an option tree)
      Table[] apiTbls = createAPITables(opts, true);

      // Help
      // arrays are number of opts + 1 (some docs may be under help but not be part of an option tree)
      Table[] helpTbls = createHelpTables(opts, true);

      // training
      Table trainTbl = createMainDocsTable("training");

      // usersguide
      Table usersTbl = createMainDocsTable("usersguide");

      // opt docs (nxm/<opt>/docs)
      // arrays are same size as number of opts (all are part of an option tree)
      int[] optDocsTblSizes = new int[opts.length];
      Arrays.fill(optDocsTblSizes,0);
      Table[] optDocsTbls = new Table[opts.length];
      Arrays.setAll(optDocsTbls,i->new Table());
      for (int optIdx = 0; optIdx < opts.length; optIdx++) {
        String opt = opts[optIdx];
        File optDocsDir = new File(nmroot+"nxm/"+opt+"/docs");
        File optDocsFiles[] = FileUtil.findFiles(optDocsDir,fnFilter);
        for (File optDocsFile : optDocsFiles) {
          String absPath = optDocsFile.getAbsolutePath();
          String absPathSTD = absPath.replaceAll(ONE_OR_TWO_BACKSLASH, "/");
          String relPath = absPathSTD.substring(absPathSTD.indexOf("/nxm/"+opt+"/docs/")+1);
          String title = readTitle(absPath);
          Table fileTbl = new Table("{PATH="+relPath+",TITLE="+title+"}"); // relPath CS & UNIX separators
          optDocsTbls[optIdx].put(String.format("%04x",optDocsTblSizes[optIdx]++),fileTbl);
        }
      }

      // Read final results
      File indexDir = new File(indexPath);
      if (indexDir.exists()) FileUtil.recursiveDelete(indexDir,false);
      if (indexDir.exists() || indexDir.mkdir()) {
        for (int apiTblIdx = 0; apiTblIdx < apiTbls.length; apiTblIdx++) {
          docsCount += apiTbls[apiTblIdx].size();
          apiTbls[apiTblIdx].toTextFile(new TextFile(midas,new FileName(indexPath+File.separatorChar+API_TYPE_NUM+String.format("%02x",apiTblIdx)+".tbl",FNCase.KeepCase)));
        }
        for (int helpTblIdx = 0; helpTblIdx < helpTbls.length; helpTblIdx++) {
          docsCount += helpTbls[helpTblIdx].size();
          helpTbls[helpTblIdx].toTextFile(new TextFile(midas,new FileName(indexPath+File.separatorChar+HELP_TYPE_NUM+String.format("%02x",helpTblIdx)+".tbl",FNCase.KeepCase)));
        }
        docsCount += trainTbl.size();
        trainTbl.toTextFile(new TextFile(midas,new FileName(indexPath+File.separator+TRAIN_TYPE_NUM+"00.tbl",FNCase.KeepCase)));
        docsCount += usersTbl.size();
        usersTbl.toTextFile(new TextFile(midas,new FileName(indexPath+File.separator+USERS_TYPE_NUM+"00.tbl",FNCase.KeepCase)));
        for (int optDocsTblIdx = 0; optDocsTblIdx < optDocsTbls.length; optDocsTblIdx++) {
          docsCount += optDocsTblSizes[optDocsTblIdx];

          // opt docs doesn't have no-option-tree case, so the index of the Array of Tables is one behind that of the Map
          optDocsTbls[optDocsTblIdx].toTextFile(new TextFile(midas,new FileName(indexPath+File.separator+OPTDOCS_TYPE_NUM+String.format("%02x",optDocsTblIdx+1)+".tbl",FNCase.KeepCase)));
        }
        optIndexTbl.toTextFile(new TextFile(midas,new FileName(indexPath+File.separator+OPT_TREE_TABLE,FNCase.KeepCase)));
      }
    }

    private Table[] createAPITables (String[] opts, boolean includeMainDir) {
      String docPath = indexLocations.getS("API");
      int offset = (includeMainDir) ? 1 : 0;
      Table[] tbls = new Table[opts.length+offset];
      Arrays.setAll(tbls,i->new Table());
      if (docPath==null) return tbls;
      String docPathSTD = docPath.toLowerCase().replaceAll(ONE_OR_TWO_BACKSLASH, "/");
      int indexRelPath = docPathSTD.substring(0,docPathSTD.lastIndexOf("/api")).lastIndexOf('/') + 1;
      if (includeMainDir) {
        File typeDir = new File(docPath);
        addFiles(tbls,0,typeDir,indexRelPath,false);
      }
      for (int optIdx = 0; optIdx < opts.length; optIdx++) {
        String opt = opts[optIdx];
        File typeDir = new File(docPath+"/"+opt.toLowerCase());
        addFiles(tbls,optIdx+offset,typeDir,indexRelPath,true);
      }
      return tbls;
    }

    private Table[] createHelpTables (String[] opts, boolean includeMainDir) {
      String docPath = indexLocations.getS("HELP");
      int offset = (includeMainDir) ? 1 : 0;
      Table[] tbls = new Table[opts.length+offset];
      Arrays.setAll(tbls,i->new Table());
      if (docPath==null) return tbls;
      String docPathSTD = docPath.replaceAll(ONE_OR_TWO_BACKSLASH, "/");
      int indexRelPath = docPathSTD.substring(0,docPathSTD.lastIndexOf("/help")).lastIndexOf('/') + 1;
      File helpDir = new File(docPath);
      File helpFiles[] = FileUtil.findFiles(helpDir,fnFilter);
      for (File helpFile : helpFiles) {
        String absPath = helpFile.getAbsolutePath();
        String absPathSTD = absPath.replaceAll(ONE_OR_TWO_BACKSLASH, "/");
        String relPath = absPathSTD.substring(indexRelPath);
        String title = readTitle(absPath);
        Table fileTbl = new Table("{PATH="+relPath+",TITLE="+title+"}"); // relPath CS & UNIX separators
        boolean fileInOpt = false;
        for (int optIdx = 0; optIdx < opts.length; optIdx++) {
          if (absPathSTD.contains("help/"+opts[optIdx]+"_")) { // not using toLowerCase because opt is not necessarily LC
            tbls[optIdx+offset].put(String.format("%04x",tbls[optIdx+offset].size()),fileTbl);
            fileInOpt = true;
            break;
          }
        }

        if (!fileInOpt && includeMainDir) tbls[0].put(String.format("%04x",tbls[0].size()),fileTbl);
      }
      return tbls;
    }

    private Table createMainDocsTable (String type) {
      String docPath = indexLocations.getS(type.toUpperCase());
      Table tbl = new Table();
      if (docPath==null) return tbl;
      String docPathSTD = docPath.toLowerCase().replaceAll(ONE_OR_TWO_BACKSLASH, "/");
      int indexRelPath = docPathSTD.substring(0,docPathSTD.lastIndexOf("/"+type.toLowerCase())).lastIndexOf('/') + 1;
      File docDir = new File(docPath);
      File docFiles[] = findFilesInDir(docDir);
      for (File finalFile : docFiles) {
        String absPath = finalFile.getAbsolutePath();
        String title = readTitle(absPath);
        String relPath = absPath.substring(indexRelPath).replaceAll(ONE_OR_TWO_BACKSLASH, "/");
        Table fileTbl = new Table("{PATH="+relPath+",TITLE="+title+"}"); // relPath CS & UNIX separators
        tbl.put(String.format("%04x",tbl.size()),fileTbl);
      }
      return tbl;
    }

    /** Add File Entries to the Table at the given index
     *
     *  @param tbls          Tables array to modify
     *  @param index         index of the Table in array to modify
     *  @param typeDir       directory in which to look for files
     *  @param indexRelPath  index of relative path ASSUMING path is specified with UNIX file separators
     *  @param recursive     true to not only include files in current directory but also sub directories
     */
    private void addFiles (Table[] tbls, int index, File typeDir, int indexRelPath, boolean recursive) {
      File typeFiles[] = (recursive) ? FileUtil.findFiles(typeDir, fnFilter) : findFilesInDir(typeDir);
      for (File file : typeFiles) {
        String absPath = file.getAbsolutePath().replaceAll(ONE_OR_TWO_BACKSLASH, "/");;
        String relPath = absPath.substring(indexRelPath);
        String title = readTitle(absPath);
        Table fileTbl = new Table("{PATH="+relPath+",TITLE="+title+"}"); // relPath CS & UNIX separators
        tbls[index].put(String.format("%04x",tbls[index].size()),fileTbl);
      }
    }

    private File[] findFilesInDir (File dir) {
      List<File> dirFiles = new ArrayList<>();
      if (dir.isDirectory() && dir.exists()) {
        File[] files = dir.listFiles();
        for (int i=0; i<files.length; i++) {
          if ((fnFilter==null) || fnFilter.accept(files[i])) {
            dirFiles.add(files[i]);
          }
        }
      }
      return dirFiles.toArray(new File[dirFiles.size()]);
    }

    /** Each table contains a list of filtered text-based files for the given option tree
     *  if not already available
     *  These tables get mapped to a three-digit hex number representing the searchable area
     */
    private void makeIndexFiles (String opt) {
      Table optIndexTbl = Table.fromTableFile(midas,new FileName(indexPath+File.separator+OPT_TREE_TABLE,FNCase.KeepCase));
      int numKeys = optIndexTbl.size();
      optIndexTbl.put(opt, String.format("%02x",numKeys+1));
      opt = opt.toLowerCase();
      String[] opts = new String[] {opt};

      // API
      Table[] apiTbl = createAPITables(opts,false);

      // Help
      Table[] helpTbl = createHelpTables(opts,false);

      // opt docs (nxm/<opt>/docs)
      int optDocsTblSize = 0;
      Table optDocsTbl = new Table();
      File optDocsDir = new File(nmroot+"nxm/"+opt+"/docs");
      File optDocsFiles[] = FileUtil.findFiles(optDocsDir,fnFilter);
      for (File optDocsFile : optDocsFiles) {
        String absPath = optDocsFile.getAbsolutePath();
        String absPathSTD = absPath.replaceAll(ONE_OR_TWO_BACKSLASH, "/");
        String relPath = absPathSTD.substring(absPathSTD.indexOf("/nxm/"+opt+"/docs/")+1);
        String title = readTitle(absPath);
        Table fileTbl = new Table("{PATH="+relPath+",TITLE="+title+"}"); // relPath CS & UNIX separators
        optDocsTbl.put(String.format("%04x",optDocsTblSize++),fileTbl);
      }

      // Read final results
      File indexDir = new File(indexPath);

      if (indexDir.exists()) {
        apiTbl[0].toTextFile(new TextFile(midas,new FileName(indexPath+File.separatorChar+API_TYPE_NUM+String.format("%02x",numKeys+1)+".tbl",FNCase.KeepCase)));
        helpTbl[0].toTextFile(new TextFile(midas,new FileName(indexPath+File.separatorChar+HELP_TYPE_NUM+String.format("%02x",numKeys+1)+".tbl",FNCase.KeepCase)));
        // opt docs doesn't have no-option-tree case, so the index of the Array of Tables is one behind that of the Map
        optDocsTbl.toTextFile(new TextFile(midas,new FileName(indexPath+File.separator+OPTDOCS_TYPE_NUM+String.format("%02x",numKeys+1)+".tbl",FNCase.KeepCase)));
      }
      optIndexTbl.toTextFile(new TextFile(midas,new FileName(indexPath+File.separator+OPT_TREE_TABLE,FNCase.KeepCase)));
    }

    private void makeSearchWords () {
      Map<String,List<String>> masterIndexSearchWords = new TreeMap<>(); // Map of words to ArrayList of file id (SLOC####)
      Map<Integer,HashSet<String>> masterTitleCharWords = new TreeMap<>();
      for (int ch = 'A'; ch <= 'Z'; ch++) {
        masterTitleCharWords.put(ch, new HashSet<String>());
      }

      try (Stream<Path> stream = Files.list(Paths.get(indexPath))) {
        stream.forEach(p -> {
          File locationFile = p.toFile();
          String locationFileName = locationFile.getName();
          if (locationFileName.endsWith(".tbl") && !locationFileName.equals(SEARCH_WORDS_FILE_NAME) && !locationFileName.equals(OPT_TREE_TABLE)) {
            String location = locationFileName.substring(0,locationFileName.indexOf(".tbl"));
            Table locationFileTable = new Table((new TextFile(midas,new FileName(locationFile.getAbsolutePath(),FNCase.KeepCase))));
            for (String fileIdx : locationFileTable.getKeys()) {
              Table fileTbl = (Table)locationFileTable.get(fileIdx);
              String title = (String)fileTbl.get("TITLE");
              String relPath = (String)fileTbl.get("PATH");
              String absPath = nmroot+relPath;
              String[] titleWords = (!"".equals(title)) ? toWords(title) : null;
              String[] textWords = null;
              TextFile indexFile = new TextFile(midas,new FileName(absPath,FNCase.KeepCase));
              if (titleWords != null) {
                for (String word : titleWords) {
                  int ch = word.toUpperCase().charAt(0);
                  if ((ch>='A') && (ch<='Z')) {
                    HashSet<String> hs = masterTitleCharWords.get(ch);
                    if (hs!=null) {
                      hs.add("0" + location + fileIdx);
                      masterTitleCharWords.put(ch, hs);
                    }
                  }
                }
              }
              if (indexFile.open()) {
                String rawText = indexFile.readAll();
                if (indexFile.getMimeType().equals("text/html")) {
                  String textFromHtml = textFromHtml(rawText);
                  textWords = toWords(textFromHtml);
                }
                else {
                  textWords = toWords(rawText);
                }
                indexFile.close();

                if ((textWords != null) && (textWords.length > 0)) {
                  double bonusScore = BONUS_INCREMENT/textWords.length;
                  Map<String,Double> indexFileWords = new HashMap<>(); // Map of words in current file with each word's score for the current file
                  for (String word : textWords) {
                    if (skipInt(word) || wordInFilter(RESERVED_WORDS,word)) continue;

                    Double score = indexFileWords.get(word);
                    if (score == null) {
                      double startingScore = (wordInFilter(titleWords,word) ? TITLE_SCORE : STARTING_NON_TITLE_SCORE);
                      indexFileWords.put(word,startingScore);
                    }
                    else if (score > MIN_NON_TITLE_SCORE) {
                      // The score needs to be capped in case it goes below the minimum allowable non-title score
                      double adjustedScore = Math.max(score-bonusScore,MIN_NON_TITLE_SCORE);
                      indexFileWords.put(word,adjustedScore);
                    }
                  }

                  // Add words from current index file to master search words
                  for (String word : indexFileWords.keySet()) {
                    int score = indexFileWords.get(word).intValue();
                    String wordEntry = Integer.toString(score)+location+fileIdx;
                    List<String> wordEntriesList = masterIndexSearchWords.get(word);
                    if (wordEntriesList == null) {
                      wordEntriesList = new ArrayList<>();
                      masterIndexSearchWords.put(word,wordEntriesList);
                    }
                    wordEntriesList.add(wordEntry);
                  }
                }
              }
            }
          }
        });

      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }

      // Sort List of entries in order for each word
      TextFile searchWordsFile = new TextFile(midas,new FileName(indexPath+File.separator+SEARCH_WORDS_FILE_NAME,FNCase.KeepCase));
      searchWordsFile.open(BaseFile.OUTPUT);
      for (String word : masterIndexSearchWords.keySet()) {
        List<String> wordEntriesList = masterIndexSearchWords.get(word);
        Collections.sort(wordEntriesList);
        String listStr = wordEntriesList.toString();
        int listStrLen = listStr.length();
        if (listStrLen > 2) listStr = listStr.substring(1,listStrLen-1);
        searchWordsFile.write(word+"=\""+listStr.replace(" ","")+"\"\n");
      }

      // Sort Set entries for each starting character
      TextFile titleCharFile = new TextFile(midas,new FileName(indexPath+File.separator+TITLE_CHAR_FILE_NAME,FNCase.KeepCase));
      titleCharFile.open(BaseFile.OUTPUT);
      for (int ch : masterTitleCharWords.keySet()) {
        HashSet<String> titleCharSet = masterTitleCharWords.get(ch);
        List<String> sortedList = new ArrayList<>(titleCharSet);
        Collections.sort(sortedList);
        String listStr = sortedList.toString();
        int listStrLen = listStr.length();
        if (listStrLen > 2) listStr = listStr.substring(1,listStrLen-1);
        char letter = (char)ch;
        if (listStr!="[]") titleCharFile.write(letter+"=\""+listStr.replace(" ","")+"\"\n");
      }

      searchWordsFile.close();
      titleCharFile.close();
    }

    private void updateSearchWords (String opt, Table searchWordsTable, Table titleCharTable) {
      Map<String,List<String>> starterIndexSearchWords = new TreeMap<>(); // Map of words to ArrayList of file id (SLOC####)
      if (searchWordsTable!=null) {
        for (String key : searchWordsTable.getKeys()) {
          String entry = searchWordsTable.getS(key);
          if (entry==null) continue;
          String[] hashScores = entry.split(",");
          ArrayList<String> mapEntry = new ArrayList<>(Arrays.asList(hashScores));
          starterIndexSearchWords.put(key, mapEntry);
        }
      }
      Map<String,List<String>> masterIndexSearchWords = new TreeMap<>(starterIndexSearchWords);

      Map<Integer,HashSet<String>> starterTitleCharWords = new TreeMap<>();
      if (titleCharTable!=null) {
        for (String key : titleCharTable.getKeys()) {
          String entry = titleCharTable.getS(key);
          if (entry==null) continue;
          String[] hashScores = entry.split(",");
          HashSet<String> mapEntry = new HashSet<>(Arrays.asList(hashScores));
          int ch = key.charAt(0);
          starterTitleCharWords.put(ch, mapEntry);
        }
      }
      Map<Integer,HashSet<String>> masterTitleCharWords = new TreeMap<>(starterTitleCharWords);

      Table optIndexTbl = Table.fromTableFile(midas,new FileName(indexPath+File.separator+OPT_TREE_TABLE,FNCase.KeepCase));
      final String opt_index = optIndexTbl.containsKey(opt) ? optIndexTbl.getS(opt) : ""; // finds the option tree index files

      try (Stream<Path> stream = Files.list(Paths.get(indexPath))) {
        stream.forEach(p -> {
          File locationFile = p.toFile();
          String locationFileName = locationFile.getName();
          if (locationFileName.endsWith(opt_index + ".tbl") && !locationFileName.equals(SEARCH_WORDS_FILE_NAME)) {
            String location = locationFileName.substring(0,locationFileName.indexOf(".tbl"));
            Table locationFileTable = new Table((new TextFile(midas,new FileName(locationFile.getAbsolutePath(),FNCase.KeepCase))));
            for (String fileIdx : locationFileTable.getKeys()) {
              Table fileTbl = (Table)locationFileTable.get(fileIdx);
              String title = (String)fileTbl.get("TITLE");
              String relPath = (String)fileTbl.get("PATH");
              String absPath = nmroot+relPath;
              String[] titleWords = (!"".equals(title)) ? toWords(title) : null;
              String[] textWords = null;
              TextFile indexFile = new TextFile(midas,new FileName(absPath,FNCase.KeepCase));
              if (titleWords != null) {
                for (String word : titleWords) {
                  int ch = word.toUpperCase().charAt(0);
                  if ((ch>='A') && (ch<='Z')) {
                    HashSet<String> hs = masterTitleCharWords.get(ch);
                    if (hs!=null) {
                      hs.add("0" + location + fileIdx);
                      masterTitleCharWords.put(ch, hs);
                    }
                  }
                }
              }
              if (indexFile.open()) {
                String rawText = indexFile.readAll();
                if (indexFile.getMimeType().equals("text/html")) {
                  String textFromHtml = textFromHtml(rawText);
                  textWords = toWords(textFromHtml);
                }
                else {
                  textWords = toWords(rawText);
                }
                indexFile.close();

                if ((textWords != null) && (textWords.length > 0)) {
                  double bonusScore = BONUS_INCREMENT/textWords.length;
                  Map<String,Double> indexFileWords = new HashMap<>(); // Map of words in current file with each word's score for the current file
                  for (String word : textWords) {
                    if (skipInt(word) || wordInFilter(RESERVED_WORDS,word)) continue;
                    Double score = indexFileWords.get(word);
                    if (score == null) {
                      double startingScore = (wordInFilter(titleWords,word) ? TITLE_SCORE : STARTING_NON_TITLE_SCORE);
                      indexFileWords.put(word,startingScore);
                    }
                    else if (score > MIN_NON_TITLE_SCORE) {
                      // The score needs to be capped in case it goes below the minimum allowable non-title score
                      double adjustedScore = Math.max(score-bonusScore,MIN_NON_TITLE_SCORE);
                      indexFileWords.put(word,adjustedScore);
                    }
                  }

                  // Add words from current index file to master search words
                  for (String word : indexFileWords.keySet()) {
                    int score = indexFileWords.get(word).intValue();
                    String wordEntry = Integer.toString(score)+location+fileIdx;
                    List<String> wordEntriesList = masterIndexSearchWords.get(word);
                    if (wordEntriesList == null) {
                      wordEntriesList = new ArrayList<>();
                      masterIndexSearchWords.put(word,wordEntriesList);
                    }
                    wordEntriesList.add(wordEntry);
                  }
                }
              }
            }
          }
        });

      } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }

      // Sort List of entries in order for each word
      TextFile searchWordsFile = new TextFile(midas,new FileName(indexPath+File.separator+SEARCH_WORDS_FILE_NAME,FNCase.KeepCase));
      searchWordsFile.open(BaseFile.OUTPUT);
      for (String word : masterIndexSearchWords.keySet()) {
        List<String> wordEntriesList = masterIndexSearchWords.get(word);
        Collections.sort(wordEntriesList);
        String listStr = wordEntriesList.toString();
        int listStrLen = listStr.length();
        if (listStrLen > 2) listStr = listStr.substring(1,listStrLen-1);
        searchWordsFile.write(word+"=\""+listStr.replace(" ","")+"\"\n");
      }

      // Sort Set entries for each starting character
      TextFile titleCharFile = new TextFile(midas,new FileName(indexPath+File.separator+TITLE_CHAR_FILE_NAME,FNCase.KeepCase));
      titleCharFile.open(BaseFile.OUTPUT);
      for (int ch : masterTitleCharWords.keySet()) {
        HashSet<String> titleCharSet = masterTitleCharWords.get(ch);
        List<String> sortedList = new ArrayList<>(titleCharSet);
        Collections.sort(sortedList);
        String listStr = sortedList.toString();
        int listStrLen = listStr.length();
        if (listStrLen > 2) listStr = listStr.substring(1,listStrLen-1);
        char letter = (char)ch;
        if (listStr!="[]") titleCharFile.write(letter+"=\""+listStr.replace(" ","")+"\"\n");
      }

      searchWordsFile.close();
      titleCharFile.close();
    }

    /** May want to move some of this into StringUtil, since
     *  evidently the regular html2text was not sufficient
     *  @param textFromHtml text from an HTML file
     *  @return text with HTML stripped
     */
    private static String textFromHtml (String textFromHtml) {

      // Strip out raw title so it isn't included in the remaining text
      Matcher titleOpen = TITLE_OPEN_REGEX.matcher(textFromHtml);
      Matcher titleClose = TITLE_CLOSE_REGEX.matcher(textFromHtml);
      if (titleOpen.find() && titleClose.find()) {
        String rawTitle = textFromHtml.substring(titleOpen.start(),titleClose.end());
        textFromHtml = textFromHtml.replace(rawTitle,"");
      }

      Matcher scriptOpen = SCRIPT_OPEN_REGEX.matcher(textFromHtml);
      Matcher scriptClose = SCRIPT_CLOSE_REGEX.matcher(textFromHtml);
      int offset = 0;
      while (scriptOpen.find() && scriptClose.find()) {
        String rawScript = textFromHtml.substring(scriptOpen.start()-offset,scriptClose.end()-offset);
        textFromHtml = textFromHtml.replace(rawScript,"");
        offset += rawScript.length();
      }

      Matcher styleOpen = STYLE_OPEN_REGEX.matcher(textFromHtml);
      Matcher styleClose = STYLE_CLOSE_REGEX.matcher(textFromHtml);
      offset = 0;
      while (styleOpen.find() && styleClose.find()) {
        String rawStyle = textFromHtml.substring(styleOpen.start()-offset,styleClose.end()-offset);
        textFromHtml = textFromHtml.replace(rawStyle,"");
        offset += rawStyle.length();
      }
      textFromHtml = StringUtil.html2Text(textFromHtml);
      return textFromHtml;
    }

    private int getDocsCount () {
      return docsCount;
    }

    private String readTitle (String absFilePath) {
      String title = "";
      TextFile tf = new TextFile(midas,new FileName(absFilePath,FNCase.KeepCase));
      if (tf.open()) {
        if (tf.getMimeType().equals("text/html")) {
          String rawText = tf.readAll();
          Matcher titleOpen = TITLE_OPEN_REGEX.matcher(rawText);
          Matcher titleClose = TITLE_CLOSE_REGEX.matcher(rawText);
          if (titleOpen.find() && titleClose.find()) {
            title = rawText.substring(titleOpen.end(),titleClose.start());
          }
        }
        tf.close();
      }
      return title;
    }

    private static String[] toWords (String text) {
      text = text.toUpperCase();
      text = NON_ALPHA_NUM_REGEX.matcher(text).replaceAll(" ");
      String[] words = StringUtil.cleanSplit(text);
      return words;
    }

    private boolean wordInFilter (String[] wordFilter, String word) {
      if (wordFilter == null) return false;
      for (String stopWord : wordFilter) {
        if (stopWord.equals(word)) return true;
      }
      return false;
    }

    private boolean skipInt (String word) {
      Matcher matcher = INT_REGEX.matcher(word);
      if (!matcher.matches()) return false; // If not all digits, don't skip
      if (word.length() > INT_MAX_DIGITS) return true; // If more than four digits, return true
      int wordInt = Integer.parseInt(word);
      for (int num : INTS_TO_KEEP) {
        if (num == wordInt) return false; // if in include list, don't skip
      }
      return ((wordInt < 2) || (Long.bitCount(wordInt) != 1)); // if power of 2, don't skip
    }
  }

  /** Help index searcher for querying/finding content in the Help index.
      This class public because it is used by Help.
   */
  @InternalUseOnly("Moved from intr.Help")
  public static class Searcher {
    private Midas  midas = Shell.getMidasContext();
    private String nmroot = Shell.getNmRoot();
    private String indexPath = null;
            Table  searchWordsTable = null;
            Table  titleCharTable = null;
    private String[] searchWordsTableKeys = null;
    private Table  expressionsTable = null;
    private static final int  EXPRESSION_KEY_1 = 10000;
    private int expressionNum;
    private static int minStartsWith = 5; // Search word of this length or longer treated as startsWith
    private static final char WILDCARD = '?';
    private static final char MULT_WILDCARD = '*';

    /** Construct a help searcher for querying the content in the specified help index.
     *
     *  @param indexPath path to directory to lucene help index
     *  @throws IOException &nbsp;
     */
    public Searcher (String indexPath) throws IOException {
      setIndexPath(indexPath);
      loadSearchWordsTable();
    }

    /** Minimum number of character search term to have search be startsWith
     *  without regex
     *  @param minStartsWith min number of characters to trigger startsWith
     */
    public static void setStartsWithMin (int minStartsWith) {
      Searcher.minStartsWith = minStartsWith;
    }

    private void loadSearchWordsTable () {
      TextFile searchWordsFile = new TextFile(midas,new FileName(indexPath+File.separator+Indexer.SEARCH_WORDS_FILE_NAME,FNCase.KeepCase));
      if (searchWordsFile.exists() && searchWordsFile.open()) {
        searchWordsTable = new Table(searchWordsFile);
        searchWordsTableKeys = searchWordsTable.getKeys();
      }

      TextFile titleCharFile = new TextFile(midas,new FileName(indexPath+File.separator+Indexer.TITLE_CHAR_FILE_NAME,FNCase.KeepCase));
      if (titleCharFile.exists() && titleCharFile.open()) {
        titleCharTable = new Table(titleCharFile);
      }
    }

    /** Queries a specific string/phrase and returns a list of documents that contain the string.
     *  This method is public because it is used by Junit test
     *
     *  @param queryString the string that will be searched
     *  @param maxResults the max number of results specified to be found
     *  @return the list of documents that contain the queried string
     */
    public List<Document> query (String queryString, int maxResults) throws ParseException {
      expressionsTable = new Table();
      expressionNum = EXPRESSION_KEY_1;
      List<Document> list = new ArrayList<>(); // list of results to return
      ArrayList<String> filesList = new ArrayList<>();
      if (searchWordsTable==null) return list;
      if (maxResults <= 0) maxResults = Integer.MAX_VALUE;     // effectively unlimited results

      if ((queryString != null) && queryString.length() > 0) {
        String queryStringUC = queryString.toUpperCase();
        int iSpace = -1;
        String justTheQuotes = queryStringUC.replaceAll("[^\"]","");
        if (justTheQuotes.length()>0) {
          if ((justTheQuotes.length()%2)==0) queryStringUC = preprocessPhrases(queryStringUC);
          else return list;
        }
        if (debug) System.out.println("query string after quotes stripped ["+ queryStringUC+"]");
        boolean hasWildCard = queryStringUC.contains(""+WILDCARD) || queryStringUC.contains(""+MULT_WILDCARD);
        if (queryStringUC.indexOf(LPAREN) >= 0) {
          // we have parenthesis grouping
          filesList = evalComplexSearch(queryStringUC);
        }
        else if ((iSpace = queryStringUC.indexOf(SPACE))>=0) {
          if (iSpace==queryStringUC.length()-1) { // just 1 word and then space
            if (hasWildCard) filesList = evalWildcard(queryStringUC.trim());
            else filesList = fullWordSearch(queryStringUC.trim());
          }
          // we have multiple words or a boolean expression
          else {
            String processedExpression = preprocessExpression(queryStringUC);
            if (processedExpression.indexOf(SPACE)==(processedExpression.length()-1)) {
              if (hasWildCard) filesList = evalWildcard(processedExpression.trim());
              else filesList = fullWordSearch(processedExpression.trim());
            }
            else {
              filesList = evalExpression(processedExpression);
            }
          }
        }
        else if (queryStringUC.contains(""+WILDCARD) || queryStringUC.contains(""+MULT_WILDCARD)) {
          // We have a LUCENE Wildcard Word
          filesList = evalWildcard(queryStringUC);
        }
        else if (queryString.length()>=minStartsWith) filesList = evalWildcard(queryStringUC+MULT_WILDCARD);
        else if (queryString.length()>1) filesList = fullWordSearch(queryStringUC.trim());
        else filesList = searchTitles(queryString);
      }
      if (debug) System.out.println("query filesList:"+filesList);
      boolean filesListNotEmpty = (filesList!=null) && !filesList.isEmpty() && !(filesList.size()==1 && filesList.get(0).equals(""));
      if (filesListNotEmpty) {
        String[] wordEntriesList = filesList.toArray(new String[filesList.size()]);
        for (int i = 0; i < wordEntriesList.length && i < maxResults; i++) {
          String wordEntry = wordEntriesList[i];
          if (wordEntry.length()==0) continue;
          String location = wordEntry.substring(1,4);
          TextFile locationFile = new TextFile(midas,new FileName(indexPath+File.separator+location+".tbl",FNCase.KeepCase));
          if (locationFile.exists() && locationFile.open()) {
            String fileIdx = wordEntry.substring(4);
            Table locationFileTbl = new Table(locationFile);
            Table indexFileTbl = (Table)locationFileTbl.get(fileIdx);
            String path = (String)indexFileTbl.get("PATH");
            String title= (String)indexFileTbl.get("TITLE");
            Document doc = new Document(nmroot+path,title);
            list.add(doc);
            locationFile.close();
          }
        }
      }
      return list;
    }

    /** Pre-process the expression so that implied ANDs are inserted
     *  and || becomes OR and && becomes AND and AND NOT becomes ANDNOT
     *
     * @param queryStringUC
     * @return
     */
    private String preprocessExpression (String queryStringUC) {
      if (debug) System.out.println("preprocessExpression queryStringUC["+queryStringUC+"]");
      String[] expressionParts = queryStringUC.split("\\s+");
      String cleanExpression = "";
      boolean opAllowed = false;
      for (int i=0; i<expressionParts.length; i++) {
        String curPart = expressionParts[i];
        if ("|".equals(curPart) || "||".equals(curPart) || "OR".equals(curPart)) {
          if (!opAllowed) return invalidExpression(queryStringUC);
          cleanExpression += " OR";
          opAllowed = false;
        }
        else if ("+".equals(curPart) || "&&".equals(curPart) || "AND".equals(curPart)) {
          if (!opAllowed) return invalidExpression(queryStringUC);

          if (i+1<expressionParts.length && "NOT".equals(expressionParts[i+1]) || "!".equals(expressionParts[i+1])) {
              i++; // consume the NOT
              cleanExpression += " ANDNOT";
          }
          else {
            cleanExpression += " AND";
          }
          opAllowed = false;
        }
        else if ("-".equals(curPart) || "ANDNOT".equals(curPart)) {
          cleanExpression += " ANDNOT";
          opAllowed = false;
        }
        else if ((i==expressionParts.length-1) && curPart.equals("|") || curPart.equals("O") || curPart.equals("&") ||
            curPart.equals("A") || curPart.equals("AN") || curPart.equals("ANDN") || curPart.equals("ANDNO")) {
          // user might be in middle of typing BOOLEAN
          cleanExpression += " ";
          opAllowed = false;
        }
        else {
          if (opAllowed) cleanExpression += " AND "+curPart;
          else if (i==0) cleanExpression += curPart;
          else cleanExpression += " " + curPart;
          opAllowed=true;
        }
      }
      if (queryStringUC.charAt(queryStringUC.length()-1)!=' ' && opAllowed
          && cleanExpression.charAt(cleanExpression.length()-1)!= ' ') cleanExpression += "*";
      if (debug) System.out.println("cleanExpression["+cleanExpression+"]");
      return cleanExpression;
    }

    private static String invalidExpression (String queryStringUC) {
      return "";
    }

    /** Get the List of files for one specific word
     *
     *  @param word       word to look up as key in search table
     *  @return ArrayList of files (in hash form) that contain the word
     */
    private  ArrayList<String> fullWordSearch (String word) {
      if (debug) System.out.println("fullWordSearch :"+word);
      ArrayList<String> filesList = new ArrayList<>(); // results from this word
      String wordFileNums;
      if (isExpressionWord(word)) {
        wordFileNums = expressionsTable.getS(word);
      }
      else {
        wordFileNums = searchWordsTable.getS(word);
      }
      if (wordFileNums != null) {
        String[] fileNums = (wordFileNums).split(",");
        filesList = new ArrayList<>(Arrays.asList(fileNums));
      }
      return filesList;
    }

    private ArrayList<String> searchTitles (String letter) {
      if (titleCharTable==null) return new ArrayList<>();
      String entry = titleCharTable.getS(letter.toUpperCase());
      if (entry==null) return new ArrayList<>();
      String[] titleCharHash = entry.split(",");
      if ((titleCharHash==null) || (titleCharHash.length==0)) return new ArrayList<>();
      if (debug) System.out.println("searchTitles returning ArrayList from "+Arrays.asList(titleCharHash));
      return new ArrayList<>(Arrays.asList(titleCharHash));
    }

    /** Evaluate an Word with wildcard characters (ex: p*t)
     *  Lucene: words are always treated like word* even when there are spaces after them
     *          and even when they are followed by another word or boolean expression
     *  NXM: Since this does not allow a way to search for an specific word,
     *       we will treat the last word on the line as word*,
     *       but all others will be treated as the word themselves
     *  Lucene '?' becomes NXM '.' and Lucene * become .*
     *
     *   @param luceneRegex Lucene style regex
     *   @return ArrayList of files (scoreHash) that have a word with that pattern
     */
   private ArrayList<String> evalWildcard (String luceneRegex) {
     if (debug) System.out.println("evalWildcard "+luceneRegex);
     if (!luceneRegex.contains("?") && (luceneRegex.indexOf("*")==(luceneRegex.length()-1))) {
       // we match word that starts with this
       return evalStartsWith(luceneRegex.substring(0,luceneRegex.length()-1));
     }
     // convert to Java regex
     String javaRegex = luceneRegex.replaceAll("\\?", "\\.");
     javaRegex = javaRegex.replaceAll("\\*", "\\."+"*");

     // calls evalWildcard or fullWordSearch as needed
     return evalRegexMatch(javaRegex);
   }

   /** Short cut for startsWith case
    *
    *  @param prefix     prefix the word needs to start with
    *  @return ArrayList of file scoreHashes that have words that startWith this prefix
    */
   private ArrayList<String> evalStartsWith (String prefix) {
     if (debug) System.out.println("evalStartsWith prefix["+prefix+"]");
     int idx = Arrays.binarySearch(searchWordsTableKeys,prefix); // get location or (-(insertion point) - 1)
     if (idx < 0) idx = -(idx+1); // first word to check is at insertion_point
     char nextChar = (char)(prefix.charAt(prefix.length()-1) + 1);
     String beyond = (prefix.length()>1) ? prefix.substring(0,prefix.length()-1) + nextChar : "" + nextChar;
     int idxBeyond = Arrays.binarySearch(searchWordsTableKeys,idx,searchWordsTableKeys.length,beyond); // get location or (-(insertion point) - 1)
     if (idxBeyond < 0) idxBeyond = -(idxBeyond+1); // first word to NOT check
     HashMap<String,Integer> filesMatch = new HashMap<>();
     // create table with best matching score for each file that has one of the words
     for (int i=idx; i < idxBeyond; i++) {
       if (searchWordsTableKeys[i].startsWith(prefix)) {
         populateFilesMatch(i,filesMatch);
       }
       else {
         System.out.println("looking for prefix:"+prefix+" on word "+searchWordsTableKeys[i]);
       }
     }

     return bestMatches(filesMatch);
   }

   /** Find the files that have a word matching the Java RegEx
    *
    *  @param javaRegex Java Regular Expression
    *  @return ArrayList of file scoreHashes that have a word that matches the RegEx
    */
   private ArrayList<String> evalRegexMatch (String javaRegex) {
     if (debug) System.out.println("evalRegexMatch javaRegex:"+javaRegex);
     HashMap<String,Integer> filesMatch = new HashMap<>();
     // create table with best matching score for each file that has one of the words
     for (int i=0; i < searchWordsTableKeys.length -1; i++) {
       if (searchWordsTableKeys[i].matches(javaRegex)) populateFilesMatch(i,filesMatch);
     }
     return bestMatches(filesMatch);
   }

   /** Populate the filesMatch HashMap with files from this word
    *
    *  @param i           index of word
    *  @param filesMatch  HashMap to update
    */
   private void populateFilesMatch (int i, HashMap<String,Integer> filesMatch) {
     if (debug) System.out.println("populateFilesMatch i:"+i);
     String entry = searchWordsTable.getS(searchWordsTableKeys[i]);
     if (entry==null) return;
     String[] wordEntriesList = entry.split(",");
     populateFilesMatch(wordEntriesList,filesMatch);
   }

   /** Populate the filesMatch HashMap with files from this word
   *
   *  @param i           index of word
   *  @param filesMatch  HashMap to update
   */
  private void populateFilesMatch (String[] wordEntriesList, HashMap<String,Integer> filesMatch) {
    if (debug) System.out.println("begin populateFilesMatch");
    for (int file=0; file < wordEntriesList.length; file++) {
      // get previous score if file is already in our list or -1 if missing
      if (wordEntriesList[file].length()==0) continue;
      String fileHash = wordEntriesList[file].substring(1);
      Integer prevScore = filesMatch.getOrDefault(fileHash,Integer.MAX_VALUE);
      int currScore = Integer.valueOf(wordEntriesList[file].substring(0,1));
      if (currScore<prevScore) filesMatch.put(fileHash, currScore);
    }
    if (debug) System.out.println("end populateFilesMatch");
  }

   /** Get the best matches from all the files matched by the regex
    *
    *  @param filesMatch HashMap containing files that match and their score
    *  @return ArrayList of file scoreHashes in order best match first
    */
   private ArrayList<String> bestMatches (HashMap<String,Integer> filesMatch) {
     // create expression entry
     ArrayList<String> scoreHash = new ArrayList<>();
     for (String key : filesMatch.keySet()) {
       scoreHash.add("" + filesMatch.get(key) + key);
     }
     Collections.sort(scoreHash);
     return scoreHash;
   }

    /** Evaluate a Boolean Expression (ex: plot AND panel)
     *  Lucene: word1 word2 is treated as word1* AND word2*
     *  NXM: word1 word2 will be treated as word1 AND word2*
     *  CURRENTLY: Just AND and OR or supported
     *  Note: Needs to handle lookups from expressionsTable
     *  Future: word1 AND NOT word2  will also be supported
     *  but not NOT word
     *
     *  @param expression Lucene style expression
     *  @return ArrayList of file scoreHashes that match the expression
     */
    private ArrayList<String> evalExpression (String expression) {
      if (debug) System.out.println("evalExpression:"+expression);
      String[] expParts = expression.split("\\s+");
      String orOnlyExpression = "";
      // Process all the OPERATORS and get this to an OR only expression
      for (int i=1; i < expParts.length-1; i+=2) {
        ArrayList<String> scoreHash = null;
        boolean newKey = false;
        switch (expParts[i]) {
          case "AND": {
            scoreHash = andOp(expParts[i-1],expParts[i+1]);
            if (isExpressionsTableEntry(expParts[i-1])) {
              int newLength = orOnlyExpression.length() - expParts[i-1].length() - 1;
              newLength = Math.max(0, newLength);
              orOnlyExpression = orOnlyExpression.substring(0, newLength);
              if (debug) System.out.println(expParts[i-1]+" AND "+expParts[i+1]+" orOnlyExpression:"+orOnlyExpression);
            }
            newKey=true;
            break;
          }
          case "ANDNOT": {
            scoreHash = andNotOp(expParts[i-1],expParts[i+1]);
            if (isExpressionsTableEntry(expParts[i-1])) {
              int newLength = orOnlyExpression.length() - expParts[i-1].length() - 1;
              newLength = Math.max(0, newLength);
              orOnlyExpression = orOnlyExpression.substring(0, newLength);
              if (debug) System.out.println(expParts[i-1]+" ANDNOT "+expParts[i+1]+" orOnlyExpression:"+orOnlyExpression);
            }
            newKey=true;
            break;
          }
          case "OR": {
            if (!orOnlyExpression.isEmpty()) {
              orOnlyExpression += ("OR " + expParts[i+1]+ " ");
            }
            else {
              orOnlyExpression += (expParts[i-1] + " OR " + expParts[i+1]+ " ");
            }
            if (debug) System.out.println(expParts[i-1]+" OR "+expParts[i+1]+" orOnlyExpression:"+orOnlyExpression);
            break;
          }
        }
        if (newKey) {
          // 1) assign the result to a new key in the expression Table ex:"10000", "10001"
          String expressionKey = toExpressionsTableEntry(scoreHash);
          // 2) update the OR only expression with this new KEY
          orOnlyExpression += expressionKey + " ";
          if (debug) System.out.println("newKey orOnlyExpression:"+orOnlyExpression);
          expParts[i+1] = expressionKey; // This expression entry is now the last word
        }
      }
      orOnlyExpression.trim();
      // Process the OR only expression calling evalWildcard or fullWordSearch as needed
      if (debug)System.out.println("orOnlyExpression:"+orOnlyExpression);
      String[] orExpParts = orOnlyExpression.split("\\s+");
      if (orExpParts.length==1) {
        String word = orExpParts[0];
        String[] wordScoreHash = null;
        if (debug)System.out.println("orExpParts.length==1 word:"+word+" expressionWord:"+isExpressionWord(word));
        String entry = expressionsTable.getS(word);
        if (entry==null) return new ArrayList<>();
        if (isExpressionWord(word)) wordScoreHash = entry.split(",");
        else wordScoreHash = entry.split(",");
        if (debug)System.out.println("evalExpression returning ArrayList from "+Arrays.asList(wordScoreHash));
        return new ArrayList<>(Arrays.asList(wordScoreHash));
      }
      else {
        HashMap<String,Integer> filesMatch = getFilesMatchWord(orExpParts[0]);
        for (int i=2; i < orExpParts.length; i+=2) {
          filesMatch = orOp(filesMatch,orExpParts[i]);
        }
        if (debug) System.out.println("evalExpression returning bestMatches");
        return bestMatches(filesMatch);
      }
    }

    private boolean isExpressionsTableEntry (String word) {
      return StringUtil.isInteger(word) && Integer.valueOf(word)>=EXPRESSION_KEY_1;
    }

    private boolean isExpressionWord (String word) {
      return (word.length()==5) && word.matches("[0-9]+"); // temp word that is really expression
    }

    /** assign the result to a new key in the expression Table ex:"10000", "10001"
     *
     * @param scoreHash ArrayList of scoreHash values
     * @return Expressions Table key
     */
    private String toExpressionsTableEntry (ArrayList<String> scoreHash) {
      // 1) assign the result to a new key in the expression Table ex:"10000", "10001"
      String expressionKey = Integer.toString(expressionNum++);
      String value = scoreHash.toString().replaceAll(" ","");
      value = value.substring(1, value.length()-1);
      expressionsTable.put(expressionKey,value);
      if (debug) System.out.println("expressionsTable:"+expressionsTable);
      return expressionKey;
    }

    private ArrayList<String> andOp (String word1, String word2) {
      if (debug) System.out.println("andOp word1["+word1+"] word2["+word2+"]");
      HashMap<String,Integer> filesMatchW1 = getFilesMatchWord(word1);
      HashMap<String,Integer> filesMatchW2 = getFilesMatchWord(word2);
      HashMap<String,Integer> filesMatchBoth = new HashMap<>();
      Set<String> keysW1 = filesMatchW1.keySet();
      if (debug) System.out.println("andOp keysW1"+keysW1);
      for (String key : keysW1) {
        if (filesMatchW2.containsKey(key)) {
          int scoreW1 = filesMatchW1.get(key);
          int scoreW2 = filesMatchW2.get(key);
          int avgScore = ((scoreW1+scoreW2)/2);
          filesMatchBoth.put(key,avgScore);
        }
      }
      if (debug) System.out.println("andOp filesMatchBoth:"+filesMatchBoth);
      return bestMatches(filesMatchBoth);
    }

    private ArrayList<String> andNotOp (String word1, String word2) {
      HashMap<String,Integer> filesMatchW1 = getFilesMatchWord(word1);
      HashMap<String,Integer> filesMatchW2 = getFilesMatchWord(word2);
      HashMap<String,Integer> filesMatchW1NW2 = new HashMap<>();
      Set<String> keysW1 = filesMatchW1.keySet();
      for (String key : keysW1) {
        if (!filesMatchW2.containsKey(key)) {
          int scoreW1 = filesMatchW1.get(key);
          filesMatchW1NW2.put(key,scoreW1);
        }
      }
      return bestMatches(filesMatchW1NW2);
    }

    private HashMap<String,Integer> orOp (HashMap<String,Integer> filesMatchW1, String word2) {
      if (debug) System.out.println("begin orOp word2:"+word2);
      HashMap<String,Integer> filesMatchW2 = getFilesMatchWord(word2);
      HashMap<String,Integer> filesMatchEither = new HashMap<>(filesMatchW1);
      Set<String> keysW2 = filesMatchW2.keySet();
      for (String key : keysW2) {
        if (!filesMatchW1.containsKey(key)) {
          filesMatchEither.put(key,filesMatchW2.get(key));
        }
        else { // duplicate file
          int scoreW1 = filesMatchW1.get(key);
          int scoreW2 = filesMatchW2.get(key);
          filesMatchEither.put(key,Math.max(scoreW1, scoreW2));
        }
      }
      if (debug) System.out.println("end orOp word2:"+word2);
      return filesMatchEither;
    }

    /** Get Files that match a Word in the Search Table or an Expression
     *  in the Expression table
     *
     *  @param word either a single word or a 10000-19999 number that
     *              is an entry in the expressionTable
     *  @return HashMap of file number and score
     */
    private HashMap<String,Integer> getFilesMatchWord (String word) {
      if (debug) System.out.println("getFilesMatchWord ["+word+"]");
      boolean wordWC = word.contains(""+WILDCARD) || word.contains(""+MULT_WILDCARD);
      String[] wordScoreHash = null;
      String wordFileNums;
      ArrayList<String> wordFiles;
      HashMap<String,Integer> filesMatchW = new HashMap<>();
      if (!wordWC) {
        boolean expressionWord = (word.length()==5) && word.matches("[0-9]+"); // temp word that is really expression
        if (expressionWord) wordFileNums = expressionsTable.getS(word);
        else wordFileNums = searchWordsTable.getS(word);
        if (wordFileNums != null) {
          wordScoreHash = (wordFileNums).split(",");
          populateFilesMatch(wordScoreHash, filesMatchW);
          return filesMatchW;
        }
        else return filesMatchW;
      }
      else {
        wordFiles = evalWildcard(word);
        if (wordFiles != null) {
          wordScoreHash = wordFiles.toArray(new String[0]);
          populateFilesMatch(wordScoreHash, filesMatchW);
          return filesMatchW;
        }
        else return filesMatchW;
      }
    }

    /** Evaluate a complex expression containing parenthesis
     *
     *  @param complexExpression complex Lucene style expression
     *  @return ArrayList of file scoreHashes that match the expression
     */
   private ArrayList<String> evalComplexSearch (String complexExpression) {
     String fullExpression = complexExpression;
     boolean noError = true;
     // evaluates each expression contained in parentheses and combines lists
     // calls evalExpression evalWildcard or fullWordSearch as needed
     while (complexExpression.contains(""+LPAREN) && noError) {
       // 1) find first index of RPAREN
       int iRight = complexExpression.indexOf(RPAREN);
       if (iRight>0) {
         // 2) find last index of LPAREN before that index
         int iLeft = complexExpression.substring(0,iRight).lastIndexOf(LPAREN);
         if (iLeft>=0) {
           // 3) send off that section to evalExpression (maybe make sure it is not a single word first)
           ArrayList<String> sectionList = evalExpression(complexExpression.substring(iLeft+1,iRight));
           // 4) assign the result to a an expression List ex:"10000", "10001"
           String expressionKey = Integer.toString(expressionNum++);
           String value = sectionList.toString().replaceAll(" ","");
           value = value.substring(1, value.length()-1);
           expressionsTable.put(expressionKey,value);
           // 5) modify the (..) portion we processed to be the expression key number
           complexExpression = complexExpression.substring(0,iLeft) + expressionKey + complexExpression.substring(iRight+1);
         }
         else {
           noError= false;
           Shell.warning("Invalid Lucene expression:"+fullExpression);
         }
       }
       else {
         noError= false;
         Shell.warning("Invalid Lucene expression:"+fullExpression);
       }
     }

     return evalExpression(complexExpression);
   }

   /** ONLY the phrases need to be processed at this point
   *
   *  @param queryStringUC query string that contains phrases
   *  @return query string with expression table entries in place of phrases
   */
    private String preprocessPhrases (String queryStringUC) {
      String processedString = "";
      String nextSection = queryStringUC;
      int idxQuoteOpen = queryStringUC.indexOf(QUOTE);
      while (idxQuoteOpen>=0) {
        String after1st = nextSection.substring(idxQuoteOpen+1);
        int idxQuoteClose = after1st.indexOf(QUOTE);
        String thisPhrase = after1st.substring(0,idxQuoteClose);
        if (thisPhrase.matches(RESERVED_WORDS_REGEX)) {
          Shell.warning("phrase ["+thisPhrase+"] contains a reserved Word. Reserved words are:"+
              Arrays.toString(RESERVED_WORDS));
        }
        else if (thisPhrase.indexOf(' ')>0) {
          ArrayList<String> scoreHash = evalPhrase(after1st.substring(0,idxQuoteClose));
          processedString += toExpressionsTableEntry(scoreHash);
          }
        else {
          processedString += thisPhrase + " ";
          }
        nextSection = after1st.substring(idxQuoteClose+1);
        idxQuoteOpen = nextSection.indexOf("\"");
        if (nextSection.length()>0 && idxQuoteOpen>=0) processedString +=  nextSection.substring(0,idxQuoteOpen);
      }
      processedString = processedString.trim();
      processedString += (nextSection.length()>0) ? nextSection : " ";
      return processedString;
    }


   /** Evaluates a group of words (phrase) surrounded by double quotes
    *  1) Initially convert ''word1 word2 word3'' to word1 AND word2 AND word3
    *     and find which files contain all 3 words
    *  2) For each file that contains all the words
    *     a) open file and grab all the file's text
    *     b) use textFromHtml to get pure text with HTML stripped
    *     c) find if the phrase exists in the file, return files
    *     d) count occurances and create score if exists -- Not currently implemented
    *
    * @param phrase words that were contained within double quotes
    * @return ArrayList of file scoreHashes that match the phrase
    */
   private ArrayList<String> evalPhrase (String phrase) {
    Integer maxResults = Integer.MAX_VALUE;     // effectively unlimited results
    String fullExpression = phrase.replaceAll(" ", " AND ");
    ArrayList<String> filesList = evalExpression(fullExpression);
    ArrayList<String> results = new ArrayList<>();
    if (filesList != null) {
      String[] wordEntriesList = filesList.toArray(new String[filesList.size()]);
      for (int i = 0; i < wordEntriesList.length && i < maxResults; i++) {
      String wordEntry = wordEntriesList[i];
      if (wordEntry.length()==0) continue;
        String location = wordEntry.substring(1,4); // gives 3 digit of location of table score= index 0, location= index 1-4
        TextFile locationFile = new TextFile(midas,new FileName(indexPath+File.separator+location+".tbl",FNCase.KeepCase));
        if (locationFile.exists() && locationFile.open()) {
          String fileIdx = wordEntry.substring(4);
          Table locationFileTbl = new Table(locationFile);
          Table indexFileTbl = (Table)locationFileTbl.get(fileIdx);
          String path = (String)indexFileTbl.get("PATH");
          String absPath = nmroot+path;
          TextFile indexFile = new TextFile(midas,new FileName(absPath,FNCase.KeepCase));
          if (indexFile.open()) {
            String rawText = indexFile.readAll();
            if (indexFile.getMimeType().equals("text/html")) {
              String textFromHtml = Indexer.textFromHtml(rawText);
              if (textFromHtml.toUpperCase().contains(phrase)) {
                results.add(wordEntriesList[i]);
              }
            }
            indexFile.close();
          }
        }
      }
    }
    if (debug) System.out.println("evalPhrase returning:"+results);
    return results;
  }

    /** Use Shell to display info/warn messages if Shell.getNmRoot() is set,
     *  otherwise display to standard output.
     *  @param msg String to display as an info message
     *  @param ex  if exception exists then this becomes a warning message
     */
    private void displayMessage (String msg, Exception ex) {
      if (Shell.getNmRoot()!=null) {
        if (ex!=null) Shell.printStackTrace(msg, ex);
        else          Shell.info(msg);
      } else {
        String prepend = (ex!=null) ? "WARN: " : "INFO: ";
        System.out.println(prepend+msg);
        if (ex!=null) ex.printStackTrace();
      }
    }

    /** Set the path to help index for searches.
     *  @param indexDir path to directory containing the help index
     */
    private void setIndexPath (String indexPath) {
      displayMessage("HELP: setting indexPath to " + indexPath, null);
      this.indexPath = indexPath;
    }
  } // end class Searcher

  /** Customer Cell Renderer for List to display search results stored in {@link Document} class. */
  @InternalUseOnly
  public class ResultsRenderer extends DefaultListCellRenderer {
   /** default constructor */
   public ResultsRenderer () {}

   /** Display value as "index. title" when showPathInResult is false (default), otherwise
       use "index. title - path". Also add the following tooltip "index. path (score)".
    */
    public Component getListCellRendererComponent (JList<?> list, Object value, int index,
       boolean isSelected, boolean cellHasFocus) {
      Component comp = super.getListCellRendererComponent(list,value,index,isSelected,cellHasFocus);
      return comp;
    }
  } // end class ResultsRenderer

  /** Specifies information about files/documents including the file path, title, and filename
   *  This class is public because it is used by Junit test
   */
  @InternalUseOnly
  public static class Document {
    private String path = null; // relative path
    private String title = null;
    private String fileName = null;

    private Document (String path, String title) {
      this.path = path;
      this.title = title;
      this.fileName = path.substring(path.lastIndexOf(File.separator)+1);
    }

    private String getPath () {
      return path;
    }

    private String getTitle () {
      return title;
    }

    public String toString() {
      return (title != null) ? title : fileName;
    }
  }

  /** This class will be moving once implemented */
  @InternalUseOnly
  public class ParseException extends MidasException{
    /** Place Holder */
    ParseException () {}
  }
}
