package nxm.sys.lib;

import static java.lang.Math.max;
import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static nxm.sys.lib.LogUtilities.log;
import static nxm.sys.lib.Util.getShortClassName;

import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.math.BigInteger;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.CopyOption;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileStore;
import java.nio.file.FileSystemException;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.logging.Level;
import java.util.regex.Pattern;

import nxm.sys.inc.InternalUseOnly;
import nxm.sys.inc.ListFile;
import nxm.sys.inc.MidasReference;
import nxm.sys.inc.PlotFile;
import nxm.sys.inc.PlotFileNH;
import nxm.sys.inc.ProvisionalUseOnly;
import nxm.sys.inc.Units;
import nxm.sys.lib.FileName.FNCase;
/** A collection of utility methods useful when working with files.

    @author  J. Czechowski IV
    @since   NeXtMidas 2.3.0
 */
public final class FileUtil {
  /** comma (',') or OS path-separator (Unix=':' or DOS/Window=';') regex (compiled) */
  public static final Pattern COMMA_OR_PATH_SEPARATOR = Pattern.compile("[,"+File.pathSeparator+"]");

  /** A file name separator '/' or '\' regex (compiled) */
  private static final Pattern FILE_SEPARATOR = Pattern.compile("[/\\\\]");
  /** Starts with Windows drive letter regex (compiled) */
  private static final Pattern WINDOWS_DRIVE  = Pattern.compile("^[a-zA-Z]:.*");
  /** Starts with '/' or '\' then Windows drive letter regex (compiled)*/
  private static final Pattern SLASH_WINDOWS_DRIVE = Pattern.compile("^[/\\\\][a-zA-Z]:.*");

  /** Only used by getLocalPrefixList() */
  private static String[] localPrefixList = null;

  /** Used with {@link #copyListFile(Midas,Object,Object,Object[])} to indicate that column
      definitions should be propagated from the input file to the output file.
   */
  public static final Object[] PROPAGATE = new Object[0];

  /** Get Deprecate 2 Arg URL Constructor */
  private static Constructor<?> urlConstructor2Arg = ReflectUtil.getConstructorOfClass(URL.class, URL.class, String.class);
  /** Use URL API deprecated in JDK20 */
  private static boolean useDeprecatedUrlAPI = urlConstructor2Arg != null;

  /** Prevent instantiation since all methods are static. */
  private FileUtil() { }

  private static final CopyOption[]             PRESERVE        = { COPY_ATTRIBUTES };
  private static final CopyOption[]             NO_PRESERVE     = { };
  private static final Set<PosixFilePermission> DEF_PERMISSIONS = getDefaultPosixFilePermissions();

  /** This will perform a binary copy from one file to another. This opens (if necessary) and
      closes the input and output files.<br>
      Note: Since NeXtMidas 2.9.1 the inFile and outFile can be either a file name or {@link BaseFile}.<br>
      WARN: inFile and outFile <b>should NOT be</b> a subclass of BaseFile (such as DataFile) as it's reads/writes
            will skip to the data offset and will probably result in undesired behavior.
      @param midas The midas context to use.
      @param inFile  The file name (or {@link BaseFile} instance - NOT subclass) of the source file to read the data from.
      @param outFile The file name (or {@link BaseFile} instance - NOT subclass) of the destination file to write/copy the data to.
   */
  public static void copyBaseFile (Midas midas, Object inFile, Object outFile) {
    copyBaseFile(midas, inFile, outFile, BaseFile.BUFFER_SIZE);
  }

  /** This will perform a binary copy from one file to another. This opens (if necessary) and
      closes the input and output files.<br>
      Note: Since NeXtMidas 2.9.1 the inFile and outFile can be either a file name or {@link BaseFile}.<br>
      WARN: inFile and outFile <b>should NOT be</b> a subclass of BaseFile (such as DataFile) as it's reads/writes
            will skip to the data offset and will probably result in undesired behavior.
      @param midas The midas context to use.
      @param inFile  The file name (or {@link BaseFile} instance - NOT subclass) of the source file to read the data from.
      @param outFile The file name (or {@link BaseFile} instance - NOT subclass) of the destination file to write/copy the data to.
      @param bufSize transfer length (buffer size) in bytes
      @since NeXtMidas 3.1.2
   */
  public static void copyBaseFile (Midas midas, Object inFile, Object outFile, final int bufSize) {
    byte[]   buf   = new byte[bufSize];
    boolean  done  = false;
    BaseFile in    = (inFile  instanceof BaseFile)? (BaseFile) inFile  : new BaseFile(midas, inFile);
    BaseFile out   = (outFile instanceof BaseFile)? (BaseFile) outFile : new BaseFile(midas, outFile);
    double   pause = (midas != null)? midas.pause : 0.0125;

    if (!in.isOpen() && !in.open(BaseFile.INPUT)) {
      throw new MidasException("BaseFile: Error opening URL="+in.getURL()+" for INPUT.");
    }
    if (!out.isOpen() && !out.open(BaseFile.OUTPUT)) {
      throw new MidasException("BaseFile: Error opening URL="+out.getURL()+" for OUTPUT.");
    }

    // since 3.1.2: Pre-allocate output file length to input file's length.
    //              This can be disabled via {PREALLOCATELENGTH=false} qualifier
    if (out.getQualifiers().getState("PREALLOCATELENGTH", true)) {
      long inFileLen = in.getResource().getLength();
      IOResource ores = out.getResource();
      ores.setLength(inFileLen);
      if (ores.getLength() < inFileLen) {       // if output file length does not match
        out.getResource().setLength(inFileLen); // setLength again (e.g. for RamFile)
      }
    } // end since 3.1.2

    while (!done) {
      int bytesRead = in.read(buf, 0, buf.length);

      if (bytesRead == 0) {
        Time.sleep(pause);
      }
      else if (bytesRead < 0) {
        done = true;
      }
      else {
        out.write(buf,0,bytesRead);
      }
    } // end while loop

    in.close();
    out.close();
  }

  /** This will copy the data from one {@link ListFile} to another. This is the same as
      <code>copyListFile(midas,inFile,outFile,PROPAGATE)</code>.
      @param midas    The midas context to use.
      @param inFile   The file name of the source file to read the data from.
      @param outFile  The file name of the destination file to write/copy the data to.
      @throws ClassCastException If either of the file names does not match an instance of
                                 {@link ListFile}.
   */
  public static void copyListFile (Midas midas, Object inFile, Object outFile) {
    copyListFile(midas,inFile,outFile,PROPAGATE);
  }

  /** This will copy the data from one {@link ListFile} to another. Since NeXtMidas 3.3.0,
      this is the same as <code>copyListFile(midas,inFile,outFile,PROPAGATE,0)</code>.
      @param midas   The midas context to use.
      @param inFile  The file name of the source file to read the data from.
      @param outFile The file name of the destination file to write/copy the data to.
      @param cols    The names of the columns to include (or one of the special values listed above).
      @throws ClassCastException If either of the file names does not match an instance of
                                 {@link ListFile}.
   */
  public static void copyListFile (Midas midas, Object inFile, Object outFile, Object[] cols) {
    // Note: It is intentional that this is not a vararg function.
    copyListFile(midas, inFile, outFile, cols, 0);
  }

  /** This will copy the data from one {@link ListFile} to another. This is the same as:
      <pre>
        int propmask = Parser.mask(DataFile.propagationMaskList, hdrPropMask, Parser.MASK_ALL);
        copyListFile(midas,inFile,outFile,PROPAGATE,propmask);
      </pre>
      @param midas       The midas context to use.
      @param inFile      The file name of the source file to read the data from.
      @param outFile     The file name of the destination file to write/copy the data to.
      @param cols        The names of the columns to include (or one of the special values listed above).
      @param hdrPropMask The set of flags controlling header propagation to the output file.
      @throws ClassCastException If either of the file names does not match an instance of
                                 {@link ListFile}.
      @since NeXtMidas 3.3.0
   */
  public static void copyListFile (Midas midas, Object inFile, Object outFile, Object[] cols,
                                   String hdrPropMask) {
    int propmask = Parser.mask(DataFile.propagationMaskList, hdrPropMask, Parser.MASK_ALL);
    copyListFile(midas, inFile, outFile, cols, propmask);
  }

  /** This will copy the data from one {@link ListFile} to another. Unlike {@link #copyBaseFile}
      this is NOT a binary copy and can be used to copy data between two different file types (e.g.
      {@link DataFile} to {@link CsvFile}). <br>
      <br>
      There are two special values used when defining the columns in the output file:
      <pre>
        cols = null      - Columns are already defined (e.g. via a qualifier on fn2).
        cols = {@link #PROPAGATE} - Propagate column definitions from input file to output file.
                           (Since NeXtMidas 3.3.0, if TIME_NAME=, ABSC_NAME=, or INDEX_NAME= are
                           specified, they will be "virtually" propagated as required.)
      </pre>
      Since NeXtMidas 3.3.0 it is possible to propagate the time, abscissa, and index values
      to the output file. This is done by adding the following qualifiers to the output file:
      <pre>
        TIME_NAME=[col]  - Name of the column to be populated with the computed
                           time values. The time value inserted will be a {@link Time}
                           object (which will convert to String/number as needed).
                           This is only applicable when the input is:
                             - {@link DataFile} (time value taken from {@link DataFile#getTimeAt}), or
                             - {@link PlotFile} with {@link PlotFile#getUnits()} equal to
                               {@link Units#TIME} or {@link Units#TIMECODE} (value is computed based
                               on {@link PlotFile#getStart()} and {@link PlotFile#getDelta()})

        ABSC_NAME=[col]  - Name of the column to be populated with the computed
                           abscissa values. This is only applicable when the input
                           is:
                             - {@link DataFile} or {@link PlotFile} (value is computed based
                               on {@link PlotFile#getStart()} and {@link PlotFile#getDelta()})

        INDEX_NAME=[col] - Name of the column to be populated with the computed
                           index value. This applies to all file types and is
                           automatically computed based on the (zero-based) row number.
      </pre>
      The primary use of <code>TIME_NAME=</code>, and <code>ABSC_NAME=</code> is in a {@link DataFile}
      to {@link CsvFile} conversion where this information would otherwise be lost. If
      <code>TIME_NAME=</code>, or <code>ABSC_NAME=</code> are used in a situation where they are not
      applicable, the default value for that column (0 or "") will generally be inserted. <br>
      <br>
      When using <code>TIME_NAME=</code>, <code>ABSC_NAME=</code> or <code>INDEX_NAME=</code>, if the
      column identified already exists in the input file, the value from the input file
      will be ignored in favor of the computed value (where applicable) -- EXCEPT when
      the column name is given with a "+" prefix (e.g. <code>TIME_NAME=+TIME</code>) in which
      case the value from the input file will be kept. <i>(The use of the "+" prefix to
      indicate optional column data is patterned off of <code>M$TRANSFORM</code> in X-Midas.)</i>
      @param midas       The midas context to use.
      @param inFile      The file name of the source file to read the data from.
      @param outFile     The file name of the destination file to write/copy the data to.
      @param cols        The names of the columns to include (or one of the special values listed above).
      @param hdrPropMask The set of flags controlling header propagation to the output file. This
                         takes the same values as used with {@link DataFile#propagationMaskList}.
                         If the input and output files are both instances of {@link DataFile} the
                         {@link DataFile#propagate} function is used for this. Otherwise if both
                         are instances of {@link PlotFile} the following fields will be set via
                         a set-keys call: XDELTA, XFRAME, XSTART, XUNITS, YDELTA, YFRAME, YSTART,
                         YUNITS, <b>TYPE, DELTA, START, UNITS</b>. The propagation of additional
                         fields and/or for additional file-type combinations may be added in the
                         future without warning, so it is important to set this to 0 if such is
                         not desired.
      @throws ClassCastException If either of the file names does not match an instance of {@link ListFile}.
      @since NeXtMidas 3.3.0
   */
  public static void copyListFile (Midas midas, Object inFile, Object outFile, Object[] cols,
                                   int hdrPropMask) {
    ListFile in  = (ListFile)BaseFile.getInstanceFor(midas,inFile);  // The input file
    ListFile out = (ListFile)BaseFile.getInstanceFor(midas,outFile); // The output file

    // OPEN INPUT FILE
    if (!in.open(BaseFile.INPUT)) {
      throw new MidasException("BaseFile: Error opening URL="+in.getURL()+" for INPUT.");
    }

    // PROPAGATE HEADER & KEYWORDS
    // This is done between the opening of the input and output files so that any user-specified
    // qualifiers controlling the output file will be applied AFTER the propagated settings.
    if (hdrPropMask == 0) {
      // nothing to propagate
    }
    else if ((in instanceof DataFile) && (out instanceof DataFile)) {
      DataFile df = (DataFile)out;
      df.propagate((DataFile)in, hdrPropMask);
      df.setSubRecords(""); // <-- do not keep old definitions
      df.setSize(0.0);      // <-- set size=0 since we use append calls below
    }
    else if ((in instanceof PlotFile) && (out instanceof PlotFile)
                                      && ((hdrPropMask&DataFile.PM_HEADER) != 0)) {
      Table quals = new Table();
      quals.put("XDELTA", ((PlotFile)in).getXDelta());
      quals.put("XFRAME", ((PlotFile)in).getXFrame());
      quals.put("XSTART", ((PlotFile)in).getXStart());
      quals.put("XUNITS", ((PlotFile)in).getXUnits());
      quals.put("YDELTA", ((PlotFile)in).getYDelta());
      quals.put("YFRAME", ((PlotFile)in).getYFrame());
      quals.put("YSTART", ((PlotFile)in).getYStart());
      quals.put("YUNITS", ((PlotFile)in).getYUnits());

      quals.put("TYPE",   ((PlotFile)in).getType());
      quals.put("DELTA",  ((PlotFile)in).getDelta());
      quals.put("START",  ((PlotFile)in).getStart());
      quals.put("UNITS",  ((PlotFile)in).getUnits());

      KeyObject.setKeys(out, quals, null, KeyObject.SOFT);
    }

    // OPEN OUTPUT FILE
    if (!out.open(BaseFile.OUTPUT)) {
      throw new MidasException("BaseFile: Error opening URL="+out.getURL()+" for OUTPUT.");
    }

    // INITIALIZE COLUMN DEFINITIONS
    Table colDefs;

    if (cols == PROPAGATE) {
      colDefs = in.getRecordDefs();  // <-- taken from input file
    }
    else if (cols == null) {
      colDefs = out.getRecordDefs(); // <-- taken from output qualifiers
    }
    else {
      Table inDefs = in.getRecordDefs();

      colDefs = new Table();
      for (int i = 0; i < cols.length; i++) {
        String col = cols[i].toString();
        Table  def = inDefs.getTable(col);
        if (def == null) def = recDef(col, null, null, null, null);
        colDefs.put(col, def);
      }
    }

    // DETERMINE COMPUTED COLUMNS
    // Use "" rather than null so we don't need additional "if" checks the code below loop code.
    String time  = Convert.o2s(out.getName().getQualifier("TIME_NAME",  ""));
    String absc  = Convert.o2s(out.getName().getQualifier("ABSC_NAME",  ""));
    String index = Convert.o2s(out.getName().getQualifier("INDEX_NAME", ""));

    if (time.startsWith("+")) {
      time = time.substring(1);
      if (in.getRecordDefs().containsKey(time)) time = ""; // don't override
    }
    if (absc.startsWith("+")) {
      absc = absc.substring(1);
      if (in.getRecordDefs().containsKey(absc)) absc = ""; // don't override
    }
    if (index.startsWith("+")) {
      index = time.substring(1);
      if (in.getRecordDefs().containsKey(index)) index = ""; // don't override
    }

    if (cols == PROPAGATE) {
      // "virtual" propagation of the special TIME, ABSC, and INDEX values
      if (!time.isEmpty() && !colDefs.containsKey(time)) {
        String form = (out instanceof DataFile)? "SD" : null;
        colDefs.put(time, recDef(time, form, null, Units.SCALAR, Units.TIME));
      }
      if (!absc.isEmpty() && !colDefs.containsKey(absc)) {
        Object units = (in instanceof PlotFile)? ((PlotFile)in).getUnits() : null;
        colDefs.put(absc, recDef(absc, "SD", null, Units.SCALAR, units));
      }
      if (!index.isEmpty() && !colDefs.containsKey(index)) {
        String form = (in.getNumberOfRows() <= Integer.MAX_VALUE)? "SL" : "SX";
        colDefs.put(index, recDef(index, form, null, Units.SCALAR, null));
      }
    }

    // SET COLUMN DEFINITIONS
    out.setRecordDefs(colDefs);

    // COPY DATA OVER
    // For performance reasons the main part of this is split into three blocks based on
    // input file type (DataFile/PlotFile/other) rather than using a single loop with
    // a large number of "if" statements.
    if (in instanceof DataFile) {
      DataFile df   = (DataFile)in;
      Table    data = new Table();
      DataFile dftc = null;

      // TODO: Rethink this check... goal is to prevent creating a TimeLine in
      // the output file when one did not exist in the input file.
      if (((hdrPropMask&DataFile.PM_TIMECODE)!=0) && (out instanceof DataFile)
          && (df.getTimeLineHandler() != null)) {
        dftc = (DataFile)out;
      }

      df.seek(0.0);
      for (long row = 0; true; row++) {
        Time   _time   = df.getTimeAtCurrent();
        double _absc   = df.getStart() + df.getDelta() * row;
        int    numRead = df.readDataTable(data);

        if (numRead < 0) break;

        data.put(time,  _time);
        data.put(absc,  _absc);
        data.put(index, row);

        if (dftc != null) {
          dftc.setTimeAt(_time);
        }
        out.insertData(-1.0, data);
      }
    }
    else if (in instanceof PlotFile) {
      PlotFile pf = (PlotFile)in;

      if ((pf.getUnits() != Units.TIME) || (pf.getUnits() != Units.TIMECODE)) {
        time = ""; // absc value != time value, don't include
      }

      pf.seek(0.0);
      for (long row = 0; row < in.getNumberOfRows(); row++) {
        double _absc = pf.getStart() + pf.getDelta() * row;

        Table data = in.getDataTable((double)row);

        data.put(time,  new Time(_absc));
        data.put(absc,  _absc);
        data.put(index, row);

        out.insertData(-1.0, data);
      }
    }
    else {
      for (long row = 0; row < in.getNumberOfRows(); row++) {
        Table data = in.getDataTable((double)row);

        data.put(index, row);

        out.insertData(-1.0, data);
      }
    }

    // CLOSE FILES
    in.close();
    out.close();
  }

  /** Creates a directory symbolic link (symlink) to specified target directory. <br>
   *  In Windows 7/2008 or later, if user does not have "SeCreateSymbolicLinkPrivilege", which is granted only to
   *  administrators by default then provide a fallback mechanism by attempting to create a directory junction point
   *  using Windows OS command: <code>MKLINK/J</code>.
   *  @param linkName The path to name of the symlink that is being created
   *  @param targetOfLink The path to the target directory the symlink (linkName) points to
   *  @return true if the symlink was created, otherwise false (e.g. directory already exists at linkName, permission issue, ...)
   *  @throws MidasException if targetOfLink is not a directory
   *  @since NeXtMidas 3.7.0
   */
  public static boolean makeDirSymLink (String linkName, String targetOfLink) throws MidasException {
    Path linkNamePath     = Paths.get(linkName);
    Path targetOfLinkPath = Paths.get(targetOfLink);

    if (!Files.isDirectory(targetOfLinkPath))
      throw new MidasException("Unable to create dir symlink '" + linkName + "' since targetOfLink is not a directory: " + targetOfLink);

    if (Files.exists(linkNamePath)) {
      String existType = "file";
      if (Files.isSymbolicLink(linkNamePath))   { existType = "symlink";   }
      else if (Files.isDirectory(linkNamePath)) { existType = "directory"; }
      Shell.warning("Cannot make symbolic link because a " + existType + " exists at " + linkName + " targetOfLink=" + targetOfLink);
      return false;
    }

    try {
      Files.createSymbolicLink(linkNamePath, targetOfLinkPath);
      return true;                          // success in creating symbolic link
    }
    catch (FileAlreadyExistsException ex) {
      Shell.warning("Symbolic link was not created at '" + linkName + "' " + ex);
    }
    catch (FileSystemException ex) {
      if (Shell.isWindows()) { // Windows requires privileges above a standard user, try creating a junction point
        String cmdline = "cmd /c mklink /j " + linkName + " " + targetOfLink;
        StringBuilder stdout = new StringBuilder();
        StringBuilder stderr = new StringBuilder();

        int exitCode = Foreign.runInternal(cmdline, stdout, stderr);
        if (exitCode == 0) { return true; } // success in creating junction point
        else Shell.warning("Unable to create directory junction point. mklink returned exit code: " + exitCode
            + "\n === stdout ===\n" + stdout + "\n === stderr ===\n" + stderr);
      }
      else Shell.warnStackTrace("Symbolic link was not created at '" + linkName + "' to " + targetOfLink, ex);
    }
    catch (IOException | UnsupportedOperationException ex) {
      Shell.warnStackTrace("Symbolic link was not created. at '" + linkName + "' to " + targetOfLink, ex);
    }
    return false;
  }

  /** Creates a record definition for use with {@link ListFile} or {@link DataFile}.
      @param name   The name of the column.            (required)
      @param format The format (e.g. "SF" or "VD").    (optional, but strongly encouraged)
      @return The record definition.
   */
  public static Table recDef (String name, String format) {
    if (name == null) throw new NullPointerException("Name required, given null.");
    Table def = new Table();
    def.put("NAME", name);
    if (format != null) def.put("FORMAT", format);
    return def;
  }

  /** Creates a record definition for use with {@link ListFile} or {@link DataFile}.
      The 'type' and 'units' can either be the formal names as strings or the
      numeric constant from {@link Units}.
      @param name   The name of the column.            (required)
      @param format The format (e.g. "SF" or "VD").    (optional, but strongly encouraged)
      @param offset The sub-record offset.             (optional, only applies to Type 3000 files)
      @param type   The sub-record type.               (optional, only applies to Type 5000/6000 files)
      @param units  The sub-record units.              (optional, only applies to Type 5000/6000 files)
      @return The record definition.
   */
  public static Table recDef (String name, String format, Integer offset, Object type, Object units) {
    Table def = recDef(name, format);
    if (offset != null) def.put("OFFSET", offset);
    if (type   != null) def.put("TYPE",   type);
    if (units  != null) def.put("UNITS",  units);
    return def;
  }

  /** Makes a file accessible via the {@link ListFile} interface.
      @param file The file to use via the {@link ListFile} interface.
      @return Returns the following...
              <pre>
                (1) If file implements {@link ListFile}, file is returned, cast to
                    {@link ListFile}.
                (2) if file implements {@link PlotFileNH}, a read-only view of the
                    file is returned that conforms to the {@link ListFile} interface.
                (3) Returns null if unable to convert to a {@link ListFile}.
              </pre>
   */
  public static ListFile asListFile (Object file) {
    if (file instanceof ListFile  ) return (ListFile)file;
    if (file instanceof PlotFileNH) return new PlotFileAsListFile((PlotFileNH)file);

    return null;
  }

  /** Makes a file accessible via the {@link ListFile} interface.
      @param file The file to use via the {@link ListFile} interface.
      @return Returns the following...
              <pre>
                (1) If file implements {@link ListFile}, file is returned, cast to
                    {@link ListFile}.
                (2) if file implements {@link PlotFileNH}, a read-only view of the
                    file is returned that conforms to the {@link ListFile} interface.
                (3) Returns null if unable to convert to a {@link ListFile}.
              </pre>
   */
  public static PlotFileNH asPlotFileNH (Object file) {
    if (file instanceof PlotFileNH) return (PlotFileNH)file;

    if (file instanceof PlotFile  ) {
      if (file instanceof ListFile) return new PlotFileWrapper2((PlotFile)file, (ListFile)file);
      else                          return new PlotFileWrapper((PlotFile)file);
    }

    return null;
  }

  /** Resolves the canonical file name. Unlike {@link File#getCanonicalFile()} this will fall-back
      and use {@link File#getAbsoluteFile()} if there is an {@link IOException} thrown by the
      {@link File#getCanonicalFile()}. <br>
      <br>
      The canonical file name is the absolute file name after all symbolic links, <code>/./</code>
      and <code>/../</code> entries have been removed. See {@link File#getCanonicalFile()} for
      more details.
      @param file The file to get the canonical name for.
      @return The canonical name for the given file.
      @throws SecurityException If either the {@link File#getCanonicalFile()} or
              {@link File#getAbsoluteFile()} can not execute due to security limitations.
   */
  public static File getCanonicalFile (File file) {
    File canonical;
    try {
      // The next line resolves any symbolic links and any ../ entries, it may fail on some systems.
      canonical = file.getCanonicalFile();
    } catch (IOException e) {
      // Unable to resolve links, just fix any ../ entries.
      canonical = file.getAbsoluteFile();
    }
    return canonical;
  }

  /** Resolves the canonical URL. This is simply...
      <pre>
        URI uri = new URI(url.toString());
        uri = uri.normalize(); // Removes any ./ or ../ entries
        url = uri.toURL();
      </pre>
      ...except that in the event of an exception, the original URL is just returned. <br>
      <br>
      The canonical URL is the absolute file name after all <code>/./</code> and <code>/../</code>
      entries have been removed. See {@link URI#normalize()} for more details.
      @param url The URL to get the canonical form for.
      @return The canonical form of the URL.
      @since NeXtMidas 2.5.3
   */
  public static URL getCanonicalUrl (URL url) {
    try {
      URI uri = new URI(url.toString());
      uri = uri.normalize(); // Removes any ./ or ../ entries
      url = uri.toURL();
    } catch(URISyntaxException e) {
      // Unable to convert URL->URI
    } catch(MalformedURLException e) {
      // Unable to convert URI->URL
    }
    return url;
  }

  /** Resolves the canonical path. If the path is a local path (see {@link #isLocalFileName})
      then {@link #getCanonicalFile} is used; otherwise {@link #getCanonicalUrl} is used. In
      either case, if an error occurs, the original path is returned.
      @param path The path to get the canonical form for. Both the linux file pattern (/../../) and
                  the Windows file pattern (\\..\\..\\) can be passed in as the path.
      @return The canonical form of the path.
      @since NeXtMidas 2.5.3
   */
  public static String getCanonicalPath (String path) {
    try {
      String local = toLocalFileName(path);

      if (local != null) {
        path = getCanonicalFile(new File(local)).toString();
      }
      else {
        path = getCanonicalUrl((new URI(path)).toURL()).toString();
      }
    } catch(Exception e) {
      // Unable to convert
    }
    return path;
  }

  /** Gets the relative directory for src relative to dest. For example:
      <pre>
        getRelativeDir("/home/smith/nxm230/nxm/sys/lib/",
                       "/home/smith/nxm230/nxm/sys/lib/")    = ""

        getRelativeDir("/home/smith/nxm230/nxm/",
                       "/home/smith/nxm230/nxm/sys/lib/")    = "sys/lib/"

        getRelativeDir("/home/smith/nxm230/nxm/sys/lib/",
                       "/home/smith/nxm230/nxm/")            = "../../"
      </pre>
      @param src  The source DIRECTORY (the return value is relative to this).
      @param dest The destination DIRECTORY (this is the directory we are trying to get to from src).
      @return The relative directory.
   */
  public static String getRelativeDir(File src, File dest) {
    return getRelativeDir(FileName.terminatePath(src.toString()),
                          FileName.terminatePath(dest.toString()),
                          File.separatorChar);
  }

  /** Gets the relative directory for src relative to dest. For example:
      <pre>
        getRelativeDir("http://alice.com/nextmidas/",
                       "http://alice.com/nextmidas/")            = ""

        getRelativeDir("http://alice.com/nextmidas/snapshots/",
                       "http://alice.com/nextmidas/")            = "../"

        getRelativeDir("http://alice.com/nextmidas/",
                       "http://alice.com/nextmidas/snapshots/")  = "snapshots/"

        getRelativeDir("http://alice.com:8080/nextmidas/",
                       "http://alice.com/nextmidas/")
                                          = "http://alice.com/nextmidas/"

        getRelativeDir("http://68.167.166.136/nextmidas/",
                       "http://alice..com/nextmidas/")
                                          = "http://alice.com/nextmidas/"

        getRelativeDir("http://alice.com/nextmidas/",
                       "ftp://alice.com/nextmidas/")
                                          = "ftp://alice.com/nextmidas/"

        getRelativeDir("http://alice.com/nextmidas/",
                       "http://www.nextmidas.com/")
                                          = "http://www.nextmidas.com/"
      </pre>
      At this time no attempt is made to resolve host names (e.g. 68.167.166.136 and
      alice.com are treated as unique although they point to the same system).
      This may be changed in the future.
      @param src  The source DIRECTORY (the return value is relative to this).
      @param dest The destination DIRECTORY (this is the directory we are trying to get to from src).
      @return The relative directory.
   */
  public static String getRelativeUrl(URL src, URL dest) {
    String relDir;
    int    srcPort  = src.getPort();
    int    destPort = dest.getPort();
    String srcStr   = src.toString();
    String destStr  = dest.toString();

    if (!srcStr.endsWith("/"))  { srcStr  = srcStr  + "/"; }
    if (!destStr.endsWith("/")) { destStr = destStr + "/"; }

    if (src.getProtocol().equals(dest.getProtocol()) && (srcPort == destPort)
     && src.getHost().equals(dest.getHost())) {
      relDir = getRelativeDir(srcStr, destStr, '/');
    } else {
      relDir = destStr;
    }
    return relDir;
  }

  /** Gets the relative directory for src relative to dest. This is only for use by the other
      getRelativeDir(..) methods.
   */
  private static String getRelativeDir(Object src, Object dest, char separator) {
    StringBuilder relDir  = new StringBuilder(); // dest relative to content
    try {
      String[] srcPath   = splitPath(src.toString());
      String[] destPath  = splitPath(dest.toString());
      int      sameCount = 1; // Use sameCount=1 to skip "/" directory

      while ((sameCount < srcPath.length) && (sameCount < destPath.length)
             && srcPath[sameCount].equals(destPath[sameCount])) {
        sameCount++;
      }

      for (int i = sameCount; i < srcPath.length; i++) {
        relDir.append("..").append(separator);
      }

      for (int i = sameCount; i < destPath.length; i++) {
        relDir.append(destPath[i]).append(separator);
      }

    } catch (Exception e) {
      Shell.printStackTrace(e);
    }
    return relDir.toString();
  }

  /** Converts a given URL to a local file name
      @param url The given URL
      @return The file name without any "file:" prefix or null if it is not a local file
      @since NeXtMidas 3.1.0
   */
  public static String toLocalFileName (URL url) {
    return toLocalFileName(url.toString());
  }

  /** Indicates if a given URL represents a local file name
      @param url The given URL
      @return true if it is a local file name, false if not
      @see #isLocalFileName(String)
      @since NeXtMidas 3.1.0
   */
  public static boolean isLocalFileName (URL url) {
    return isLocalFileName(url.toString());
  }

  /** Splits a string that contains items that are comma-separator of OS path-separator separated
      into an array of Strings. This never returns null so that client usage of this API is simplified.
      @param entries comma or OS path-separator separated string list of entries
      @return string array of entries (empty if entries is null or empty)
      @since NeXtMidas 3.0.1
   */
  public static String[] splitEntries (String entries) {
    if (entries == null) return StringUtil.EMPTY_STRING_ARRAY;
    return COMMA_OR_PATH_SEPARATOR.split(entries, 0);
  }

  /** Splits a path into its component parts.
      @param path is first converted to a String (cannot be null!) then split up
   */
  private static String[] splitPath(Object path) {
    return FILE_SEPARATOR.split(path.toString());
  }

  /** Gets the path separator used in the given path based on the first
      path separator character occurrence in the given path.
      @param path A path.
      @return The path separator used in path or null if unknown.
      @since NeXtMidas 2.5.3
   */
  public static String getPathSeparator (String path) {
    if (path.indexOf('/')  >= 0) return "/";  // UNIX or URL
    if (path.indexOf('\\') >= 0) return "\\"; // DOS
    if (path.indexOf('[')  >= 0) return "[";  // VMS
    return null;
  }

  /** Add required terminating directory separator if not already present.
      The end terminator is gleaned from the input path. If the separator
      cannot be gleaned from input path, the local system's file separator
      (i.e. {@link File#separator}) will be used.
      @param path The path to terminate.
      @return The terminated path or path if already terminated.
      @since NeXtMidas 2.5.3
  */
  public static String terminatePath (String path) {
    String term = getPathSeparator(path);

    if (term == null) {
      // term = "/";  // DEFAULTS to UNIX prior to NeXtMidas 2.9.0
      term = File.separator; // use local system's file/directory separator
      // 2010-07-02 NTN: I commented out the below warning since Shell.warning() tries to get a Midas context
      //                 which caused it to boot up a NeXtMidas session when using this class as a library.
      // TODO: we should use a more standard logging framework like SLF4J to log messages at the API level
      // Shell.warning("No known delimiter in path '"+path+"', using UNIX");
    }

    return (path.endsWith(term))? path : path+term;
  }

  /** Converts the separators in a file path to the correct version. This changes any slashes
      ('/') or back-slashes ('\') to a given character. This is commonly used in one of two ways:
      <pre>
        changePathSeparator(path, File.pathSeparator) - Converts to local file system
        changePathSeparator(path, "/")                - Converts to URL style
      </pre>
      @param path      The file or directory path.
      @param separator The separator to use.
      @return The path with the correct separator in use.
   */
  public static String changePathSeparator (String path, String separator) {
    String repl = StringUtil.textToReplacement(separator);
    return FILE_SEPARATOR.matcher(path).replaceAll(repl);
  }

  /** Converts a given path to a local file name. For example:
      <pre>
        toLocalFileName("/home/smith/myfile.prm")             = "/home/smith/myfile.prm"
        toLocalFileName("file:/home/smith/myfile.prm")        = "/home/smith/myfile.prm"
        toLocalFileName("file:///home/smith/myfile.prm")      = "/home/smith/myfile.prm"
        toLocalFileName("ramd:/midas/icedisk/myfile.prm")     = "/midas/icedisk/myfile.prm"
        toLocalFileName("http://alice.com/myfile.prm")        = null
        toLocalFileName("c:\data\myfile.prm")                 = "c:\data\myfile.prm"
        toLocalFileName("file:/c:/data/myfile.prm")           = "c:\data\myfile.prm" (Since 2.9.0)
      </pre>
      Note that the local file name will use the local path separator:
      <pre>
        Unix:
          toLocalFileName("file:/home/smith/myfile.prm")      = "/home/smith/myfile.prm"
        Windows:
          toLocalFileName("file:/home/smith/myfile.prm")      = "\home\smith\myfile.prm"
      </pre>
      @param path The file name either as a normal path or with the "file:" prefix.
      @return The file name without any "file:" prefix or null if it is not a local file.
      @see #isLocalFileName(String)
   */
  public static String toLocalFileName (String path) {
    String prefix = getLocalPrefix(path);
    String fname  = null;

    if (prefix != null) {
      fname = path.substring(prefix.length()); // remove the "file:"

      if (fname.startsWith("///")) {
        fname = fname.substring(2); // if it was "file:///home/junk" make it "/home/junk"
      }
      else if (SLASH_WINDOWS_DRIVE.matcher(fname).matches()) {
        fname = fname.substring(1); // remove slash before windows drive letter
      }
      fname = changePathSeparator(fname, File.separator);
    }
    return fname;
  }

  /** Indicates if a given path to a local file name. For example:
      <pre>
        isLocalFileName("/home/smith/myfile.prm")             = true
        isLocalFileName("file:/home/smith/myfile.prm")        = true
        isLocalFileName("file:///home/smith/myfile.prm")      = true
        isLocalFileName("http://alice.com/myfile.prm")        = false
        isLocalFileName("c:\data\myfile.prm")                 = true
        isLocalFileName("file:/c:/data/myfile.prm")           = true
        // NOTE: "ramd:" is a special case for ICE/Midas RamDisk
        isLocalFileName("ramd:/midas/icedisk/myfile.prm")     = true
      </pre>
      @param path The file name either as a normal path or with the "file:" prefix.
      @return true if it is a local file name, false if not.
      @see #toLocalFileName(String)
   */
  public static boolean isLocalFileName(String path) {
    return (getLocalPrefix(path) != null);
  }

  /**
   * Create URL from parameter filename. File.toURI.toURL converts DOS and
   * Unix/Linus formats to proper URL format.
   * @param  fileName file name
   * @return URL of file
   * @throws MalformedURLException if there are problems with the protocol
   * @since NeXtMidas 3.7.1
   */
  public static URL toURL (String fileName) throws MalformedURLException {
    URL url = null;
    try {
      if (isLocalFileName(fileName)) {
        url = new File(fileName).toURI().toURL();
      }
      else if (fileName.startsWith("classpath:")) {
        // handle new "classpath:" case, since 3.7.3 for NXM-3099
        IOResource ior = Shell.getMidasContext().io.findResource(fileName,0);
        String urlStr = ior.getURL();
        url = (new URI(urlStr)).toURL();
      }
      else {
        url = (new URI(fileName)).toURL();
      }
    }
    catch (URISyntaxException e) {
      throw new MalformedURLException("Forming URL via URI failed:"+e.getMessage()+" "+e.getStackTrace());
    }

    return url;
  }

  /**
   * Create URL from parameter filename relative to context file.
   * File.toURI.toURL converts DOS and Unix/Linus formats to proper URL format.
   * @param  contextFile  context to generate file's path
   * @param  fileName     file name
   * @return URL of file
   * @throws MalformedURLException if there are issues with the protocol
   * @since NeXtMidas 3.7.1
   */
  public static URL toURL (String contextFile, String fileName) throws MalformedURLException {
    URL url = null;
    boolean nonStandardURL = false;
    try {
      if (isLocalFileName(contextFile)) {

        // adjust file name to account for Windows separators and drive letter
        String filePath = fileName.replace("\\", "/");
        if (filePath.charAt(1) == ':' ) filePath = "file:" + filePath;
        url = new File(contextFile).toURI().resolve(filePath).toURL();
      }
      else {
        if ( !contextFile.startsWith("file:") ) contextFile = "file:" + contextFile;
        if (useDeprecatedUrlAPI) {
          URL contextURL = (new URI(contextFile)).toURL();
          try { // NXM-3950 Use deprecated for removal URL constructor via reflection
            return (URL)urlConstructor2Arg.newInstance(contextURL, fileName);
          } catch (InstantiationException | IllegalAccessException | IllegalArgumentException
              | InvocationTargetException e) {
            // switch to new code, since this did not work
            Shell.info("File.toURL("+contextFile+","+fileName+") switching to alternate code for deprecated URL constructor");
          }
        }
        // Work with non-deprecated URL/URI API
        if (fileName.contains(":")) {
          contextFile=fileName;
          nonStandardURL = true;
        }
        else if (contextFile.startsWith("file:ram:")) {
          contextFile="file:"+fileName;
          nonStandardURL = true;
        }
        else {
          int idx = contextFile.indexOf('/');
          if ((idx > 0) && (contextFile.charAt(idx-1)==':') && (contextFile.indexOf(':') < idx-1)) {
            // we have a context of the pattern first:second:/........
            // in this case, we just want to eliminate the filename in the context and append it with the new filename
            int indexLastSlash = contextFile.lastIndexOf("/");
            if (indexLastSlash > 0) {
              contextFile = contextFile.substring(0,indexLastSlash+1) + fileName;
              nonStandardURL = true;
            }
          }
        }

        if (nonStandardURL) {
          URI tryURI = new URI(contextFile);
          url = tryURI.toURL();
        }
        else {
          URL contextURL = (new URI(contextFile)).toURL();
          Path finalPath = Paths.get(contextURL.toString(), fileName);
          URI tryURI = new URI(finalPath.toString());
          url = tryURI.toURL();
        }
      }
    }
    catch (URISyntaxException e) {
      throw new MalformedURLException("Forming URL via URI failed:"+e.getMessage()+" "+e.getStackTrace());
    }

    return url;
  }

  /** Determine which local prefix is used.
      @param path The path.
      @return null=NotLocal, ""=AlreadyLocal, "xxx:"=Prefix is "xxx:"
   */
  private static String getLocalPrefix(String path) {
    String prefix = null;
    if (path.indexOf(':') < 0) {                        // No ':' in the path name, assume local
      prefix = "";
    } else if (WINDOWS_DRIVE.matcher(path).matches()) { // Starts with a Windows drive letter
      prefix = "";
    } else {                                            // Check to see if it begins with "file:",...
      String[] list = getLocalPrefixList();
      for (int i = 0; (prefix == null) && (i < list.length); i++) {
        if (path.startsWith(list[i])) {
          prefix = list[i];
        }
      }
    }
    return prefix;
  }

  /** Indicates if a given path starts with a local path separator character(s) (aka the root directory).
      For example:
      <pre>
        startsWithRootDir("/home/smith/myfile.prm")            = true only for unix
        startsWithRootDir("\data\myfile.prm")                  = true only for windows
        startsWithRootDir("home/smith/myfile.prm")             = false
        startsWithRootDir("data\myfile.prm")                   = false
        startsWithRootDir("file:/home/smith/myfile.prm")       = false
        startsWithRootDir("http://alice.com")                  = false
        startsWithRootDir("dd:\data\myfile.prm")               = false
        // NOTE: the two below is true under unix, but
        startsWithRootDir("c:\data\myfile.prm")                = true for both unix/windows
        startsWithRootDir("x:\data\myfile.prm")                = true for both unix/windows
        // NOTE: the below path usage is not recommended without the "\" after the drive letter
        startsWithRootDir("c:data\myfile.prm")                  = true
        startsWithRootDir("ramd:/midas/icedisk/myfile.prm")     = false
      </pre>
      @param path The file path to check
      @return true it it starts with the local path separator, otherwise false.
   */
  public static boolean startsWithRootDir(String path) {
    return (path.startsWith(File.separator))         // it starts with a '/' or a '\'
        || WINDOWS_DRIVE.matcher(path).matches();    // if starts with a Windows drive letter
  }

  /** Gets the list of prefixes (e.g. "file:", "ramd:") that are used with local files. */
  private static String[] getLocalPrefixList() {
    if (localPrefixList == null) {
      // JC4: This is where we should load the list of local file prefixes allowed. For now
      //      I've just hard-coded in the two most common ones.
      localPrefixList = new String[] { "file:", "ramd:", "ice:" };
    }
    return localPrefixList;
  }

  /** Recursively finds files on the local disk. The include/exclude filters allow filtering of
      the files. If both filters are null, all files will be included. Note that the filters are
      also applied to directory names even though the directories will not be included in the list
      (this allows exclusions of given directories while searching, e.g. "CVS").
      @param dir    The directory to search in.
      @param filter The filter that indicates the files to include (null=Include All).
      @return The files found.
      @since NeXtMidas 2.5.0
   */
  public static File[] findFiles (File dir, FileFilter filter) {
    ArrayList<File> allFiles = new ArrayList<>();

    if (dir.isDirectory() && dir.exists()) {
      HashMap<File,File> dirMap = new HashMap<>();
      File can = getCanonicalFile(dir);
      dirMap.put(can, dir); // put canonical dir path into Map to prevent going into backlinks (BUG 2444)
      findFiles(dirMap, allFiles, dir, filter);
    }
    return allFiles.toArray(new File[allFiles.size()]);
  }

  /** Recursive method used to find files. Only used by findFiles(..). */
  private static void findFiles (HashMap<File,File> dirMap, List<File> allFiles, File dir, FileFilter filter) {
    File[] files = dir.listFiles();

    for (int i = 0; i < files.length; i++) {
      File file = files[i];
      if ((filter == null) || filter.accept(file)) {
        allFiles.add(file);
      }
      if (file.isDirectory()) {
        File can = getCanonicalFile(file);
        if (dirMap.get(can) == null) { // never revisit a directory (in case of back links, etc)
          dirMap.put(can,file);
          findFiles(dirMap,allFiles,file,filter);
        }
      }
    }
  }

  /** Provide a platform neutral way to delete a file, because not all Operating
      Systems provide file deletion capabilities outside of the OS shell, and
      macros cannot run OS shell commands (only native OS commands can be run
      from macros).  This method is recursive because it deletes directories
      as well as child files under directories. This method will NOT follow
      symbolic links, but will delete it.
      @param file file or directory to recursively delete
      @return count of items deleted (zero for none or when file is null)
      @since NeXtMidas 3.1.0
   */
  public static int recursiveDelete (File file) {
    if (file == null) return 0;

    int count = 0;
    if (file.isDirectory()) {
      for (File child : file.listFiles()) {
        count += recursiveDelete(child);
      }
    }
    if (file.delete()) count++;
    // else  System.err.println("Could not delete: " + file);
    return count;
  }

  /** Provide a platform neutral way to recursively delete specified path.
      @param path path to file or directory to recursively delete (cannot be null)
      @return count of items deleted (zero for none)
      @see #recursiveDelete(File)
      @since NeXtMidas 3.1.0
   */
  public static int recursiveDelete (String path) {
    return recursiveDelete(new File(path));
  }

  /** Creates a file name filter. This is a short form of {@link #createFnFilter(String,String,String)}
      that only supports file name comparisons.
      @param filter The filter string passed in by the user.
      @param def    The default filter.
      @return The filter that accepts/rejects the files given.
      @throws NullPointerException If the filter string has a '/' in it.
      @since NeXtMidas 2.5.0
   */
  public static FnFilter createFnFilter (String filter, String def) {
    return createFnFilter(null,filter,def);
  }

  /** Creates a file name filter. By default, all files not matching one of the filters given
      are excluded. There are special keywords/characters in the filter string to allow a user
      to easily include/exclude files.
      <pre>
        Special Characters (must be first character in an entry in filter string):
          '-'    - Exclude any file that matches this.
          '+'    - Include any file that matches this.

        Special Keywords (must be first entry in filter string):
            ALL      - Same as "*"
            DEF      - Include the defaults
            DEFAULT  - Include the defaults
            NONE     - Does nothing (all files excluded by default)

        Note: If filter is blank ("") or null the default filter will be used.
      </pre>
      Some examples:
      <pre>
        createFnFilter("/data/",   "*.xml|*.csv",     "*.tmp|*.prm")   -&gt; Accepts *.xml or *.csv
        createFnFilter("/data/",   "+*.xml",          "*.tmp|*.prm")   -&gt; Accepts *.xml, *.tmp, or *.prm
        createFnFilter("/data/",   "DEF|*.xml",       "*.tmp|*.prm")   -&gt; Accepts *.xml, *.tmp, or *.prm
        createFnFilter("/data/",   "NONE|*.xml",      "*.tmp|*.prm")   -&gt; Accepts *.xml only
        createFnFilter("/data/",   "-*.tmp",          "*.tmp|*.prm")   -&gt; Accepts *.prm only
        createFnFilter("/data/",   "-*.xml",          "*.tmp|*.prm")   -&gt; Accepts *.tmp or *.prm
        createFnFilter("/data/",   "-*.xml",          "")              -&gt; All files rejected
        createFnFilter("/data/",   "",                "")              -&gt; All files rejected
        createFnFilter("/data/",   "-*",              "*.tmp|*.prm")   -&gt; All files rejected
        createFnFilter("/data/",   "NONE",            "*.tmp|*.prm")   -&gt; All files rejected
        createFnFilter("/data/",   "*",               "*.tmp|*.prm")   -&gt; Accepts all files
        createFnFilter("/data/",   "*|-*.xml",        "*.tmp|*.prm")   -&gt; Accepts all files that are not *.xml
        createFnFilter("/data/",   "*|*.xml",         "*.tmp|*.prm")   -&gt; Accepts all files
        createFnFilter("/data/",   "foo/*.txt|*.prm", "*.tmp|*.prm")   -&gt; Accepts all *.prm files and all
                                                                          *.txt files in /data/foo/
        createFnFilter("c:\data\", "foo/*.txt|*.prm", "*.tmp|*.prm")   -&gt; Accepts all *.prm files and all
                                                                          *.txt files in c:\data\foo\
      </pre>
      @param root   The root directory (should use '\' on Unix, '/' on Windows).
      @param filter The filter string passed in by the user (always uses '/'). (null=UseDefault)
      @param def    The default filter (null="*").
      @return The filter that accepts/rejects the files given.
      @throws NullPointerException If root is null and the filter string has a '/' in it.
      @since NeXtMidas 2.5.0
   */
  public static FnFilter createFnFilter (String root, String filter, String def) {
    List<String> incPath = new ArrayList<>();
    List<String> excPath = new ArrayList<>();
    List<String> incFile = new ArrayList<>();
    List<String> excFile = new ArrayList<>();
    boolean  includeDef  = false;
    boolean  includeAll  = false;

    if ((filter == null) || (filter.length() == 0)) {
      filter = (def != null)? def : "*";
    }

    String[] filtList = filter.split("[|]");
    int index = 0;

    if (filtList[0].equalsIgnoreCase("ALL") || filtList[0].equalsIgnoreCase("*")) {
      includeAll = true;
      index++;
    } else if (filtList[0].equalsIgnoreCase("NONE")) {
      index++;
    } else if (filtList[0].equalsIgnoreCase("DEF") || filtList[0].equalsIgnoreCase("DEFAULT")) {
      includeDef = true;
      index++;
    } else if (filtList[0].startsWith("+") || filtList[0].startsWith("-")) {
      includeDef = true;
    }

    if (includeDef) {
      if (def == null) {
        includeAll = true;
      } else {
        String[] defList = def.split("[|]");
        includeAll = makeFilterList(incPath,incFile,excPath,excFile,defList,0) || includeAll;
      }
    }
    includeAll = makeFilterList(incPath,incFile,excPath,excFile,filtList,index) || includeAll;

    if (includeAll) { incFile = null; incPath = null; } // make it faster for this special case

    return new FnFilterImpl(root,incPath,incFile,excPath,excFile);
  }

  /** Only used by createFileNameFilter(..) */
  private static boolean makeFilterList (List<String> incPath, List<String> incFile,
                                         List<String> excPath, List<String> excFile, String[] list, int index) {
    boolean includeAll = false;

    for (; index < list.length; index++) {
      String  filt = list[index];
      boolean excl = false;

           if (filt.startsWith("-")) { filt = filt.substring(1); excl = true;  }
      else if (filt.startsWith("+")) { filt = filt.substring(1); excl = false; }

      if (filt.length() == 0) {
        // ignore
      } else if (excl) {
        if (filt.indexOf('/') >= 0) excPath.add(filt);
        else                        excFile.add(filt);
      } else if (filt.equals("*")) { // must be after excl but before incl
        includeAll = true;
      } else {
        if (filt.indexOf('/') >= 0) incPath.add(filt);
        else                        incFile.add(filt);
      }
    }
    return includeAll;
  }

  /** A filter that is compatible with both {@link FilenameFilter}, and {@link FileFilter}.
      @since NeXtMidas 2.5.0
   */
  public static interface FnFilter extends FilenameFilter, FileFilter {
    @Override
    public String toString();
  }

  /** Extract the modified date from a file using the HTTP header info. If the
      file is a local file, this method will use {@link File#lastModified()}
      method. If the file is not local, a URL connection is made
      and {@link URLConnection#getLastModified()} is used.
      @param url The URL of the file to check
      @return Seconds since 1970 to the URL modified date, or -1 if not found
      @since NeXtMidas 2.7.1
    */
  public static long lastModified (URL url) {
    long dateInSeconds = -1;

    if (isLocalFileName(url.toString())) {
      dateInSeconds = (long) ((new File(url.getPath())).lastModified() * .001);
    } else {
      try {
        URLConnection conn;
        conn = url.openConnection();
        dateInSeconds = (long)(conn.getLastModified() * .001);
      }
      catch (IOException e) {
        Shell.printStackTrace(e);
      }
    }

    return dateInSeconds;
  }

  /** Copies data from an input to an output.
   *  @param in  The input.
   *  @param out The output.
   *  @throws IOException If an i/o error is thrown/
   */
  private static void copy (InputStream in, OutputStream out) throws IOException {
    byte[] buf = new byte[23768];
    int    numRead;

    while ((numRead = in.read(buf)) > 0) {
      out.write(buf, 0, numRead);
    }
  }

  /** Tests keyword conversion
   *
   *  @param in  input file name
   *  @param out output file name
   */
  @InternalUseOnly("Since created")
  public static void test (String in, String out) {
    FileName keyConvFileName = new FileName("nxm.sys.dat.keyconv_plat2blue.tbl",FNCase.KeepCase);
    Table kwconv = new Table(new TextFile(keyConvFileName));
    kwconv.put("TOA_SIGMA.VALUE", 1);
    kwconv.put("FOA_SIGMA.VALUE", 2);

    convertKeywords(null, new FileName(in), new FileName(out), kwconv);
  }

  /** Creates a copy of a DataFile with a new set of keywords. This method works by copying the file
   *  and then doing the conversion on the new file with {@link #convertKeywordsInPlace}. See
   *  {@link #convertKeywordsInPlace} for details on the conversion description to use.
   *  @param ref    The Midas reference to be used during the conversion (can be null).
   *  @param in     The name of the input file. (not null).
   *  @param out    The name of the output file. (not null).
   *  @param kwconv The keyword conversion description (see above) (not null).
   *  @since NeXtMidas 3.5.4
   */
  @ProvisionalUseOnly("Added in NeXtMidas 3.5.4")
  @SuppressWarnings({"BroadCatchBlock", "TooBroadCatch"})
  public static void convertKeywords (MidasReference ref, FileName in, FileName out, Table kwconv) {
    // Copy contents of file over to new temp file. Use special copy procedure below to avoid
    // warnings that would otherwise show up in NeXtMidas. Note that we still use BaseFile since
    // it will allow reading from inside a TAR file.
    try (DataFile inDF    = new DataFile(ref, in);
         BaseFile inFile  = new BaseFile(ref, in);
         BaseFile outFile = new BaseFile(ref, out); ) { // Java 7 try-with-resource statements
      inDF.open(); // 1st validates that input file is a Midas BLUE file (since 3.7.0)

      inFile.open(BaseFile.INPUT);
      outFile.open(BaseFile.OUTPUT);
      copy(inFile.io.getInputStream(), outFile.io.getOutputStream());
    }
    catch (Exception e) {
      throw new MidasException("Unable to create " + out + " from " + in, e);
    }

    // Fix keywords in the new file
    DataFile df = new DataFile(ref, out);
    df.setQualifier("PROT","FALSE"); // We need to set "PROT=false" to override the protect flag
                                     // in the header of the copied file.
    convertKeywordsInPlace(ref, df, kwconv);
  }

  /** Converts the keywords in a DataFile from one format to another format using a provided
   *  conversion description. The conversion description is a table (usually the contents of
   *  a table file) with an _OPTIONS_ sub-table plus a sub-table for each of the keywords
   *  (or set of keywords) being generated. <br>
   *  <br>
   *  The _OPTIONS_ sub-table has the following keys:
   *  <pre>
   *    +------------------+----------------------------------------------------------+------------+
   *    |     KEY NAME     |                   DESCRIPTION                            |   DEFAULT  |
   *    +------------------+----------------------------------------------------------+------------+
   *    | KEEP_EXISTING    | Keep existing file keywords?                             | FALSE      |
   *    +------------------+----------------------------------------------------------+------------+
   *    | VERIFIER         | The name of a Java method to call for verification prior | no verifier|
   *    |                  | writing the keywords to the file. The signature must     |            |
   *    |                  | match that shown below. This is used to handle any       |            |
   *    |                  | complex inter-dependencies in the conversion that can't  |            |
   *    |                  | be addressed using the standard conversion description   |            |
   *    |                  | mechanisms.                                              |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | COMMENT          | A free-form comment related to the file (ignored).       |            |
   *    +------------------+----------------------------------------------------------+------------+
   *  </pre>
   *  Each of the other sub-tables defines a keyword (or set of keywords) to be inserted into
   *  the file. The entries are processed in order and are last-one-wins; this allows multiple
   *  possible settings for a single key to be specified with the last one being the value set.
   *  Each of the subtables should have the following keys:
   *  <pre>
   *    +------------------+----------------------------------------------------------+------------+
   *    |     KEY NAME     |                   DESCRIPTION                            |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | KEYS             | The comma-separated list of keys this entry represents.  | Same as the|
   *    |                  | If omitted, the value is equal to the top-level table    | key in the |
   *    |                  | key (this is the common case). If more then one entry    | primary    |
   *    |                  | is specified, the results will be put in each of the     | table      |
   *    |                  | keys.                                                    |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | TYPE             | The keyword data target type (e.g. "D","L","S"). If TYPE |            |
   *    |                  | is omitted the type will be auto-determined by Keywords. |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | SOURCE           | The name of the keyword in the original file that will   |            |
   *    |                  | be read in.                                              |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | VALUE            | A fixed value for the keyword. This overrides any SOURCE,|            |
   *    |                  | CONVERSION or DEFAULT settings (but not TYPE) and is     |            |
   *    |                  | frequently set when overriding an entry in a standard    |            |
   *    |                  | conversion, for example:                                 |            |
   *    |                  |   Table conv = new Table(new TextFile(ref,"myconv.tbl"));|            |
   *    |                  |   conv.set("FOO.VALUE", 2); // override FOO              |            |
   *    |                  |   convertKeywordsInPlace(ref, df, conv);                 |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | DEFAULT          | A default value for the keyword if not found.            |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | REQUIRED         | Is the keyword required to be in the output file? If     | FALSE      |
   *    |                  | set to "TRUE" and the value for the keyword can not be   |            |
   *    |                  | determined an exception is thrown. If "FALSE" nothing    |            |
   *    |                  | will be inserted if the value can't be determined.       |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | CONVERSION       | The conversion method to use if a direct setting of the  | NONE       |
   *    |                  | keyword can not be done. This should have one of the     |            |
   *    |                  | following values:                                        |            |
   *    |                  |   "NONE"        - No conversion, copy value to new key   |            |
   *    |                  |   "INLINE_FUNC" - A NeXtMidas in-line expression handles |            |
   *    |                  |                   the conversion                         |            |
   *    |                  |   "JAVA_FUNC"   - A Java method handles the conversion   |            |
   *    |                  |   "MAPPING"     - A simple replacement mapping handles   |            |
   *    |                  |                   the conversion                         |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | JAVA_FUNC        | Used when CONVERSION is "JAVA_FUNC", this specifies the  |            |
   *    |                  | method to be called. See below for details on the method |            |
   *    |                  | signature required for the method being called.          |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | JAVA_FUNC_OPTS   | Used when CONVERSION is "JAVA_FUNC" to pass a table of   |            |
   *    |                  | options to the Java method.                              |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | MAPPING          | Used when CONVERSION is "MAPPING", this specified the    |            |
   *    |                  | mapping between input values and output values. This     |            |
   *    |                  | will be a Table where the key is the input value and     |            |
   *    |                  | the corresponding value is the output value. If an       |            |
   *    |                  | input does not match any of the entries in the MAPPING   |            |
   *    |                  | table, the DEFAULT will be used.                         |            |
   *    |                  |                                                          |            |
   *    |                  | Example:                                                 |            |
   *    |                  |   {1="AC",2="AC",3="DC"} Convert an input of "1" or "2"  |            |
   *    |                  |                          to "AC" and an input of "3" to  |            |
   *    |                  |                          "DC" with an input of 4 taking  |            |
   *    |                  |                          on the (separate) DEFAULT value |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | INLINE_FUNC      | Used when CONVERSION is "INLINE_FUNC", this provides the |            |
   *    |                  | NeXtMidas in-line function used to convert input value   |            |
   *    |                  | to output value. The syntax is the same as the CALC(..), |            |
   *    |                  | SEDIT(..), and TEST(..) in-line functions used within a  |            |
   *    |                  | NeXtMidas macro, except that any parameter that starts   |            |
   *    |                  | with a '^' will be subject to a keyword lookup with a    |            |
   *    |                  | zero ('0') used in place of any keywords that are not    |            |
   *    |                  | found.                                                   |            |
   *    |                  |                                                          |            |
   *    |                  | Examples:                                                |            |
   *    |                  |   CALC(^{FREQ},1e6,*)       Convert FREQ from MHz to Hz  |            |
   *    |                  |   CALC(^{A},^{B},+)         Add the values of A and B    |            |
   *    |                  |   SEDIT(^{A},APPEND,^{B})   Append B to the end of  A    |            |
   *    |                  |   TEST(^{FREQ},LT,30e6)     Is FREQ below VHF range?     |            |
   *    |                  |                                                          |            |
   *    |                  | When "INLINE_FUNC" is used the DEFAULT will be set if    |            |
   *    |                  | the SOURCE is not found; however it is not required that |            |
   *    |                  | SOURCE be the subject of the function operation.         |            |
   *    |                  |                                                          |            |
   *    |                  | If TEST(..) is used, it will return "true" or "false" if |            |
   *    |                  | the TYPE is "S" and 1 or 0 if TYPE is numeric.           |            |
   *    +------------------+----------------------------------------------------------+------------+
   *    | COMMENT          | A free-form comment related to the keyword (ignored).    |            |
   *    +------------------+----------------------------------------------------------+------------+
   *  </pre>
   *  Any Java methods called for verification need the following signature:
   *  <pre>
   *    /## Handles the verification...
   *     #  +param ref    The Midas reference to be used.
   *     #  +param df     The DataFile to modify the keywords in.  <b>Must be treated as read-only.</b>
   *     #  +param kw     The Keywords from the DataFile.          <b>Must be treated as read-only.</b>
   *     #  +param kwconv The keyword conversion description.              <b>Must be treated as read-only.</b>
   *     #  +param kwds   The new keyword values.                  <b>Can be modified.</b>
   *     #  +throws RuntimeException If verification fails.
   *     #/
   *    public static void verify (MidasReference ref, DataFile df, Keywords kw, Table kwconv,
   *                               Table kwds, Table types) { ... }
   *  </pre>
   *  Any Java methods called to handle the conversion need the following signature:
   *  <pre>
   *    /## Handles the conversion...
   *     #  +param ref    The Midas reference to be used.
   *     #  +param df     The DataFile to modify the keywords in.  <b>Must be treated as read-only.</b>
   *     #  +param kw     The Keywords from the DataFile.          <b>Must be treated as read-only.</b>
   *     #  +param kwconv The keyword conversion description.              <b>Must be treated as read-only.</b>
   *     #  +param key    The key for the current conversion.      <b>Must be treated as read-only.</b>
   *     #  +param kwds   The new keyword values.                  <b>Should be modified.</b>
   *     #  +return The map containing the keywords to insert where the keys are the keyword names
   *     #          (with optional type specifier) and the values are the values to insert.
   *     #/
   *    public static void foo (MidasReference ref, DataFile df, Keywords kw, Table kwconv,
   *                            String key, Table kwds, Table types) { ... }
   *  </pre>
   *  The <code>$NMROOT/nxm/sys/dat/kwconv_plat2blue.tbl</code> file provides an example of the
   *  conversion description for a conversion from PLATINUM keywords to BLUE keywords.
   *  @param ref    The Midas reference to be used during the conversion (can be null).
   *  @param df     The DataFile to modify the keywords in (not null).
   *  @param kwconv The keyword conversion description (see above) (not null).
   *  @since NeXtMidas 3.5.4
   */
  @ProvisionalUseOnly("Added in NeXtMidas 3.5.4")
  @SuppressWarnings( {"unchecked", "BroadCatchBlock", "TooBroadCatch", "UseSpecificCatch"})
  public static void convertKeywordsInPlace (MidasReference ref, DataFile df, Table kwconv) {
    Keywords kw      = df.getKeywordsObject();
    boolean  closeDF = false;

    if (!df.isOpen()) {
      if (!kwconv.getState("_WARN_MISSING_", false)) {
        df.setAutoConvertPlatinum(false);
      }
      df.open(DataFile.INOUT);
      closeDF = true;
    }

    convertKeywordsInPlace(ref, df, kw, kwconv);

    if (closeDF) {
      df.close();
    }
  }

  static void convertKeywordsInPlace (MidasReference ref, DataFile df, Keywords kw, Table kwconv) {
    Table    kwds    = new Table();
    Table    opts    = kwconv.getTable("_OPTIONS_", new Table());
    boolean warnMissing = kwconv.getState("_WARN_MISSING_",false);

    // ==== GENERATE THE NEW KEYWORDS ==============================================================
    String keys = null;
    try {
      // -- Process Keyword Entries ----------------------------------------------------------------
      for (String key : kwconv.getKeys()) {
        if (key.startsWith("_") && key.endsWith("_")) {
          continue; // "_OPTIONS_" or other "hidden" field
        }
        Table   ct           = kwconv.getTable(key);
                keys         = ct.getS("KEYS",         key);
        String  type         = ct.getS("TYPE",         null);
        String  source       = ct.getS("SOURCE",       null);
        String  sourceAlt    = ct.getS("SOURCE_ALTERNATE",   null);
        String  conversion   = ct.getS("CONVERSION",   "NONE");
        Object  val          = ct.get( "VALUE",        null);
        String  inlineFunc   = ct.getS("INLINE_FUNC",  null);
        String  javaFunc     = ct.getS("JAVA_FUNC",    null);
        boolean required     = ct.getState("REQUIRED", false);
        Table   mapping      = ct.getTable("MAPPING",  null);

        if (ct.containsKey("VALUE")) {
          // The VALUE overrides everything. This is frequently seen in user-provided overrides that
          // want to force a setting regardless of the other conversion options and defaults.
        }
        else if (conversion.equals("MAPPING")) {
          if (mapping == null) throw new NullPointerException("Missing MAPPING=");

          val = kw.getKey(source);
          if (val != null) val = mapping.get(val.toString());
          if (val == null) val = ct.get("DEFAULT");
        }
        else if (conversion.equals("INLINE_FUNC")) {
          if (inlineFunc == null) throw new NullPointerException("Missing INLINE_FUNC=");

          val = kw.getKey(source);
          if (val != null) {
            // TODO: We should be able to do an "evaluate carets" function call if we do this...
            //   Midas m = new Midas();
            //   m.getResults().putAll(kw.makeTable());
            // However kw.makeTable() doesn't work with PLATINUM files right now, and we don't
            // want to fix it until early in a beta series (as it might break others).
            Midas m = null; // Midas ref

            // Evaluate the carets in the function
            int    i; // start index
            int    j; // end index
            String k; // keyword name
            while ((i = inlineFunc.indexOf('^')) >= 0) {
              if ((i < inlineFunc.length()-1) && (inlineFunc.charAt(i+1) == '{')) {
                j = inlineFunc.indexOf('}', i+1) + 1; // additional +1 to skip over '}' below
                if (j <= 0) throw new IllegalArgumentException("Missing '}' in "+inlineFunc);
                k = inlineFunc.substring(i+2, j-1);
              }
              else {
                j = i+1;
                while ((j < inlineFunc.length()) && inlineFunc.substring(j,j+1).matches("[A-Za-z0-9_.]")) {
                  // Find end of keyword name, assume alphanumeric name with '_' and '.' as valid
                  // parts of the keyword name (note that '.' is common in PLATINUM).
                  j++;
                }
                k = inlineFunc.substring(i+1, j);
              }
              Object v = kw.getKey(k);
              if (v == null) v = "0";
              inlineFunc = inlineFunc.substring(0,i) + v + inlineFunc.substring(j);
            }
                 if (inlineFunc.startsWith("CALC(" )) val = new Data(Convert.inLineCalc(m,inlineFunc));
            else if (inlineFunc.startsWith("SEDIT(")) val = Convert.inLineSedit(m,inlineFunc);
            else if (inlineFunc.startsWith("TEST(" )) val = Convert.inLineTest(m,inlineFunc);
            else throw new IllegalArgumentException("Unsupported INLINE_FUNC="+inlineFunc);
          }
          else {
            val = ct.get("DEFAULT");
          }
        }
        else if (conversion.equals("JAVA_FUNC")) {
          if (javaFunc == null) throw new NullPointerException("Missing JAVA_FUNC=");
          int      indexDot   = javaFunc.lastIndexOf('.');
          String   className  = javaFunc.substring(0,indexDot);
          String   methodName = javaFunc.substring(indexDot+1);
          Class<?> classObj   = Class.forName(className);
          Method   method     = classObj.getMethod(methodName, MidasReference.class, DataFile.class,
                                                               Keywords.class, Table.class, String.class,
                                                               Table.class);
          method.invoke(null, ref, df, kw, kwconv, key, kwds);
          continue; // skip the value setting below
        }
        else if (!conversion.equals("NONE")) {
          throw new IllegalArgumentException("Unsupported CONVERSION="+conversion);
        }
        else if (source == null) {
          // This is a normal look-up entry, but the LOOKUP is missing. This will happen when users
          // provide a DEFAULT but no VALUE or LOOKUP. In this case the DEFAULT is used. This differs
          // from providing a VALUE in that a user could override the default conversion (e.g. merge
          // their custom options on top of a normal conversion description)  with a LOOKUP whereas
          // once VALUE is set, it overrides everything else.
          val = ct.get("DEFAULT");
        }
        else { // normal source look-up
          val = kw.getKey(source);
          if (val == null && sourceAlt !=null) val = kw.getKey(sourceAlt);
          if (val == null) val = ct.get("DEFAULT");
        }

        if (val != null) {
          for (String k : keys.split(",")) {
            String keyname = (type == null)? k : (type + ":" + k);
            kwds.put(keyname, val);
          }
        }
        else if (required) {
          for (String k : keys.split(",")) {
            if (!hasKeyword(kwds, k)) {
              if (warnMissing) {
                ref.getMidas().warning("INCOMPLETE conversion of PLATINUM file. Source file is missing source key "+source+" for "+key+
                                       ".\n      file:"+df.filename);
              }
              else {
                throw new RuntimeException("No value found.");
              }
            }
          }
        }
      }
      keys = null;

      // -- Verify Keyword Entries -----------------------------------------------------------------
      String verifier = opts.getS("VERIFIER");

      if ((verifier != null) && !warnMissing) {
        int      indexDot   = verifier.lastIndexOf('.');
        String   className  = verifier.substring(0,indexDot);
        String   methodName = verifier.substring(indexDot+1);
        Class<?> classObj   = Class.forName(className);
        Method   method     = classObj.getMethod(methodName, MidasReference.class, DataFile.class,
                                                 Keywords.class, Table.class, Table.class);
        method.invoke(null, ref, df, kw, kwconv, kwds);
      }
    }
    catch (InvocationTargetException e) {
      if (keys == null) keys = "keywords";
      throw new MidasException("Could not set "+keys+" in "+df.getName(), e.getCause());
    }
    catch (Exception e) {
      if (keys == null) keys = "keywords";
      throw new MidasException("Could not set "+keys+" in "+df.getName(), e);
    }

    // ==== SET THE NEW KEYWORDS ===================================================================
    boolean keepExisting = opts.getState("KEEP_EXISTING", false);  // corrected to KEEP_EXISTING in 3.7.0

    if (!keepExisting) {
      kw.deleteScope("EXT");
    }
    kw.setScope("EXT");
    kw.putAll(kwds);

  }

  /** Indicates if a table of keyword entries (which may have a type specifier) contains a
   *  given key.
   *  @param kwds The table of keywords.
   *  @param key  The key to look for.
   *  @return true if found, false otherwise.
   */
  @InternalUseOnly // Added in 3.7.0 while J. Czechowski fixes table issue, remove before end of 3.7.x series
  public static boolean hasKeyword (Table kwds, String key) {
    return (getKeywordKey(kwds,key) != null);
  }

  /** Gets the canonical key in a table of keyword entries (which may have a type specifier) for a
   *  given key.
   *  @param kwds The table of keywords.
   *  @param key  The key to look for.
   *  @return The canonical key if found, false otherwise.
   */
  @InternalUseOnly // Added in 3.7.0 while J. Czechowski fixes table issue, remove before end of 3.7.x series
  public static String getKeywordKey (Table kwds, String key) {
    // Must go in reverse order since keywords is last-one-wins
    String[] keys = kwds.getKeys();
    for (int i = keys.length-1; i >= 0; i--) {
      String k = keys[i];
      if (k.equals(key)     ) return k;
      if (k.length()  <=  2 ) continue; // no type specifier
      if (k.charAt(1) != ':') continue; // no type specifier
      if (k.substring(2).equals(key)) return k;
    }
    return null;
  }

  /** Removes all entries matching a canonical key in a table of keyword entries
   *  (which may have a type specifier) for a given key.
   *  @param kwds The table of keywords.
   *  @param key  The key to look for.
   */
  @InternalUseOnly // Added in 3.7.0 while J. Czechowski fixes table issue, remove before end of 3.7.x series
  public static void removeKeyword (Table kwds, String key) {
    // Remove all instances of the key
    kwds.remove(key);
    for (String k : kwds.getKeys()) {
      if (k.length()  <=  2 ) continue; // no type specifier
      if (k.charAt(1) != ':') continue; // no type specifier
      if (k.substring(2).equals(key)) kwds.remove(k);
    }
  }

  /** Handles the verification of keywords for the {@link #convertKeywordsInPlace} method.
   *  @param ref    The Midas reference to be used.
   *  @param df     The DataFile to modify the keywords in.
   *  @param kw     The Keywords from the DataFile.
   *  @param kwconv The keyword conversion description.
   *  @param kwds   The new keyword values.
   *  @since NeXtMidas 3.5.4
   */
  @InternalUseOnly("Called via reflection by convertKeywordsInPlace(..) to support kwconv_plat2blue.tbl")
  public static void kwconvVerifyBlueKW (MidasReference ref, DataFile df, Keywords kw,
                                    Table kwconv, Table kwds) {
    String antKey = getKeywordKey(kwds,"ANTENNA");

    if (hasKeyword(kwds,"ANT_LAT") || hasKeyword(kwds,"ANT_LON") || hasKeyword(kwds,"ANT_ALT")) {
      // Must be set as a trio
      if (!hasKeyword(kwds,"ANT_LAT")) throw new MidasException("No TERMINAL.LAT found");
      if (!hasKeyword(kwds,"ANT_LON")) throw new MidasException("No TERMINAL.LON found");
      if (!hasKeyword(kwds,"ANT_ALT")) throw new MidasException("No TERMINAL.ALT found");

      // Don't let ANTENNA default to "NONE" in this case
      if ((antKey != null) && kwds.getS(antKey).equalsIgnoreCase("NULL")) {
        removeKeyword(kwds, "ANTENNA");
      }
    }
    else if ((antKey == null) || kwds.getS(antKey).equalsIgnoreCase("NULL")) {
      // Default ANTENNA to NONE
      kwds.put("S:ANTENNA", "NONE");
    }

    if (hasKeyword(kwds,"SYSTEM_TOA_SIGMA") || hasKeyword(kwds,"SYSTEM_FOA_SIGMA")) {
      // Must be specified as a pair
      if (!hasKeyword(kwds,"SYSTEM_TOA_SIGMA")) throw new MidasException("No SYSTEM_TOA_SIGMA found");
      if (!hasKeyword(kwds,"SYSTEM_FOA_SIGMA")) throw new MidasException("No SYSTEM_FOA_SIGMA found");
    }

    if (hasKeyword(kwds,"POS_COVAR_MTX11") || hasKeyword(kwds,"TOA_SIGMA") || hasKeyword(kwds,"FOA_SIGMA")) {
      // All be specified whenever COVARIANCE matrix is set
      if (!hasKeyword(kwds,"POS_COVAR_MTX11")) throw new MidasException("No COV found");
      if (!hasKeyword(kwds,"TOA_SIGMA"      )) throw new MidasException("No TOA_SIGMA found");
      if (!hasKeyword(kwds,"FOA_SIGMA"      )) throw new MidasException("No FOA_SIGMA found");
    }
  }

  /** Handles the conversion of keywords for the {@link #convertKeywordsInPlace} method. This
   *  method assumes the StateVector keywords already exist in the file (just in the wrong form).
   *  The entry should have a <code>JAVA_FUNC_OPTS</code> table with a <code>FORMAT</code> entry set to
   *  one of the following entries specifying the new format the keywords should take on:
   *  <pre>
   *    EPH  - Conversion using {@link StateVector#toKeywordsEph()} (this is the default)
   *    PLAT - Conversion using {@link StateVector#toKeywordsPlat(DataFile)} (passing in the df reference)
   *    VEC  - Conversion using {@link StateVector#toKeywordsVec()}
   *  </pre>
   *  @param ref    The Midas reference to be used.
   *  @param df     The DataFile to modify the keywords in.
   *  @param kw     The Keywords from the DataFile.
   *  @param kwconv The keyword conversion description.
   *  @param key    The key for the current conversion.
   *  @param kwds   The new keyword values.
   *  @since NeXtMidas 3.5.4
   */
  @InternalUseOnly("Called via reflection by convertKeywordsInPlace(..) to support kwconv_plat2blue.tbl")
  public static void kwconvConvertSV (MidasReference ref, DataFile df, Keywords kw, Table kwconv,
                                 String key, Table kwds) {
    Table       ct  = kwconv.getTable(key);
    String      fmt = ct.getS("JAVA_FUNC_OPTS.FORMAT", null);
    StateVector sv;
    try {
      sv = new StateVector(df);
    }
    catch (MidasException nosv){
      Shell.warning("INCOMPLETE conversion of PLATINUM file.missing StateVector data." +
                    "\n      file:"+df.filename);
      return;
    }

         if (fmt == null       ) kwds.putAll(sv.toKeywordsEph());
    else if (fmt.equals("EPH" )) kwds.putAll(sv.toKeywordsEph());
    else if (fmt.equals("PLAT")) kwds.putAll(sv.toKeywordsPlat(df));
    else if (fmt.equals("VEC" )) kwds.putAll(sv.toKeywordsVec());
    else throw new MidasException("Could not set StateVector keys "+df.getName()+": Unsupported format "+fmt);
  }

  /** Handles the conversion of PLATINUM TIMETAG.TIME and TIMETAG.OFFSET values to BLUE TCSAMPLE,
   *  TC_WHOLE, and TC_FRAC values for the {@link #convertKeywordsInPlace} method.
   *  @param ref    The Midas reference to be used.
   *  @param df     The DataFile to modify the keywords in.
   *  @param kw     The Keywords from the DataFile.
   *  @param kwconv The keyword conversion description.
   *  @param key    The key for the current conversion.
   *  @param kwds   The new keyword values.
   *  @since NeXtMidas 3.5.4
   */
  @InternalUseOnly("Called via reflection by convertKeywordsInPlace(..) to support kwconv_plat2blue.tbl")
  public static void kwconvConvertTC (MidasReference ref, DataFile df, Keywords kw, Table kwconv,
                                 String key, Table kwds) {

    // The new timeline format is BLUE_HIGH_PRECISION - since 3.7.2 for NXM-3010
    // This is now all that is needed to force writing of BLUE TC keywords
    // (unless the file qualifier contradicts this)
    if (df.timeLine !=null) {
      df.timeLine.format = TimeLine.Format.BLUE_HIGH_PRECISION;
    }
  }

  /** Handles the conversion of PLATINUM COV values to BLUE POS_COVAR* and VEL_COVAR* values for the
   *  {@link #convertKeywordsInPlace} method.
   *  @param ref    The Midas reference to be used.
   *  @param df     The DataFile to modify the keywords in.
   *  @param kw     The Keywords from the DataFile.
   *  @param kwconv The keyword conversion description.
   *  @param key    The key for the current conversion.
   *  @param kwds   The new keyword values.
   *  @since NeXtMidas 3.5.4
   */
  @InternalUseOnly("Called via reflection by convertKeywordsInPlace(..) to support kwconv_plat2blue.tbl")
  public static void kwconvConvertCOVAR (MidasReference ref, DataFile df, Keywords kw, Table kwconv,
                                 String key, Table kwds) {
    Data cov = (Data)kw.getKey("COV");
    if (cov == null) return;

    // In PLATINUM the covariance matrix is a 6x6 diagonal matrix which includes both POS and VEL
    // (see Section 2.4.3 in PLATINUM Vol. 2). In BLUE there are two separate 3x3 matricies for
    // POS and VEL *except* that some BLUE implementations support the optional entries that
    // make up the remainder of the 6x6 matrix as these are used by a number of applications
    // (see JICD 4.2).
    final String[] KEYS = {
      "POS_COVAR_MTX11","POS_COVAR_MTX12","POS_COVAR_MTX13","POS_COVAR_MTX14","POS_COVAR_MTX15","POS_COVAR_MTX16",
                        "POS_COVAR_MTX22","POS_COVAR_MTX23","POS_COVAR_MTX24","POS_COVAR_MTX25","POS_COVAR_MTX26",
                                          "POS_COVAR_MTX33","POS_COVAR_MTX34","POS_COVAR_MTX35","POS_COVAR_MTX36",
                                                            "VEL_COVAR_MTX11","VEL_COVAR_MTX12","VEL_COVAR_MTX13",
                                                                              "VEL_COVAR_MTX22","VEL_COVAR_MTX23",
                                                                                                "VEL_COVAR_MTX33",
    };
    for (int i = 0; i < KEYS.length; i++) {
      kwds.put("D:"+KEYS[i], cov.getNumber(i));
    }
  }

  /** Handles the conversion to SBT from the PLATINUM keyword ACQETF or alternates
   *  {@link #convertKeywordsInPlace} method.
   *  Priority of equations - based on Platinum ICD Vol 2 Revision 4 Tuning Keywords section
   *  1st) SBT = ACQETF
   *  2nd) SBT = COL_RF + Id * SBT_OFFSET                          (equation 4)
   *  3rd) SBT = COL_LOCKED_LO + GND_LOCKED_LO + Id * OUTPUT_IF    (equation 3)
   *  @param ref    The Midas reference to be used.
   *  @param df     The DataFile to modify the keywords in.
   *  @param kw     The Keywords from the DataFile.
   *  @param kwconv The keyword conversion description.
   *  @param key    The key for the current conversion.
   *  @param kwds   The new keyword values.
   *  @since NeXtMidas 3.7.0
   */
  @InternalUseOnly("Called via reflection by convertKeywordsInPlace(..) to support kwconv_plat2blue.tbl")
  public static void kwconvConvertSBT (MidasReference ref, DataFile df, Keywords kw, Table kwconv,
                                 String key, Table kwds) {
    Double sbt = kw.getD("ACQETF", null);
    if (sbt == null) {
      Integer specSenseMultiplier =  getSpectralSenseMultiplier(kw);
      if (specSenseMultiplier != null) {
        // first try SBT = COL_RF + Id * SBT_OFFSET                 (equation 4)
        Double sbt_offset = getSbtOffset(kw);
        Double col_rf = kw.getD("COL_RF", null);
        if (sbt_offset !=null && col_rf != null) {
          sbt = col_rf + specSenseMultiplier * sbt_offset;
        }
        else { // try SBT = COL_LOCKED_LO + GND_LOCKED_LO + Id * OUTPUT_IF (equation 3)
          Double col_locked_lo = kw.getD("COL_LOCKED_LO", null);
          Double gnd_locked_lo = kw.getD("GND_LOCKED_LO", null);
          Double fTIF = kw.getD("OUTPUT_IF", null);
          if (col_locked_lo != null && gnd_locked_lo != null && fTIF != null) {
            sbt = col_locked_lo + gnd_locked_lo + specSenseMultiplier * fTIF;
          }
        }
      }
    }
    if (sbt == null) {
      if (kwconv.getState("_WARN_MISSING_",false)) {
        Shell.warning("INCOMPLETE conversion of PLATINUM file. No ACQETF or COL_RF found in PLATINUM source file to calculate SBT."+
                      ".\n      file:"+df.filename);
      }
      else {
        throw new NullPointerException("No ACQETF or COL_RF to calculate SBT in PLATINUM file");
      }
    }
    else {
      kwds.put("D:SBT", sbt);
    }
  }

  /** Handles the conversion to VRF from the PLATINUM keyword COL_RF or alternates
   *  {@link #convertKeywordsInPlace} method.
   *  Priority of equations - based on Platinum ICD Vol 2 Revision 4 Tuning Keywords section
   *  1st) VRF = COL_RF
   *  2nd) VRF = COL_LOCKED_LO + GND_LOCKED_LO + Id * COL_IF_CENTER (equation 3)
   *  @param ref    The Midas reference to be used.
   *  @param df     The DataFile to modify the keywords in.
   *  @param kw     The Keywords from the DataFile.
   *  @param kwconv The keyword conversion description.
   *  @param key    The key for the current conversion.
   *  @param kwds   The new keyword values.
   *  @since NeXtMidas 3.7.0
   */
  @InternalUseOnly("Called via reflection by convertKeywordsInPlace(..) to support kwconv_plat2blue.tbl")
  public static void kwconvConvertVRF (MidasReference ref, DataFile df, Keywords kw, Table kwconv,
                                   String key, Table kwds) {
    Double vrf = kw.getD("COL_RF", null);
    if (vrf == null) {
      Double sbt_offset = getSbtOffset(kw);
      Integer specSenseMultiplier =  getSpectralSenseMultiplier(kw);
      if (specSenseMultiplier != null) {
        // tray alternate: VRF = COL_LOCKED_LO + GND_LOCKED_LO + Id * COL_IF_CENTER      (equation
        Double col_locked_lo = kw.getD("COL_LOCKED_LO", null);
        Double gnd_locked_lo = kw.getD("GND_LOCKED_LO", null);
        Double fNOM = kw.getD("COL_IF_CENTER", null);
        if (col_locked_lo != null && gnd_locked_lo != null && fNOM != null) {
          vrf = col_locked_lo + gnd_locked_lo + specSenseMultiplier * fNOM;
        }
      }
    }
    if (vrf == null) {
      if (kwconv.getState("_WARN_MISSING_",false)) {
        Shell.warning("INCOMPLETE conversion of PLATINUM file. No COL_RFF or keyword pairs found in PLATINUM source file to calculate VRF"+
                      ".\n      file:"+df.filename);
      }
      else {
        throw new NullPointerException("No COL_RFF or keyword pairs to calculate VRF in PLATINUM file");
      }
    }
    else {
      kwds.put("D:VRF", vrf);
    }
  }

  /** Handles the conversion to FNOM from the PLATINUM keyword COL_IF_CENTER or alternates
   *  {@link #convertKeywordsInPlace} method.
   *  Priority of equations - based on Platinum ICD Vol 2 Revision 4 Tuning Keywords section
   *  1st) FNOM = COL_IF_CENTER
   *  2nd) FNOM = OUTPUT_IF - SBT_OFFSET                            (equation 4)
   *  3rd) FNOM = (ACQETF - (COL_LOCKED_LO + GND_LOCKED_LO)) * Id   (equation 3)
   *  @param ref    The Midas reference to be used.
   *  @param df     The DataFile to modify the keywords in.
   *  @param kw     The Keywords from the DataFile.
   *  @param kwconv The keyword conversion description.
   *  @param key    The key for the current conversion.
   *  @param kwds   The new keyword values.
   *  @since NeXtMidas 3.7.0
   */
  @InternalUseOnly("Called via reflection by convertKeywordsInPlace(..) to support kwconv_plat2blue.tbl")
  public static void kwconvConvertFNOM (MidasReference ref, DataFile df, Keywords kw, Table kwconv,
                                   String key, Table kwds) {
    Double fNOM = kw.getD("COL_IF_CENTER", null);
    if (fNOM == null) {
      // first alternate: try FNOM = OUTPUT_IF - SBT_OFFSET       (equation 4)
      Double fTIF = kw.getD("OUTPUT_IF", null);
      Double sbt_offset = getSbtOffset(kw);
      if (fTIF != null && sbt_offset != null) {
        fNOM = fTIF - sbt_offset;
      }
      else { // second alternate: try (ACQETF - (COL_LOCKED_LO + GND_LOCKED_LO)) * Id  (equation 3)
        Integer specSenseMultiplier =  getSpectralSenseMultiplier(kw);
        Double col_locked_lo = kw.getD("COL_LOCKED_LO", null);
        Double gnd_locked_lo = kw.getD("GND_LOCKED_LO", null);
        Double sbt = kw.getD("ACQETF", null);
        if (specSenseMultiplier != null && col_locked_lo != null &&
            gnd_locked_lo != null && sbt != null) {
          fNOM = (sbt - (col_locked_lo + gnd_locked_lo)) * specSenseMultiplier;
        }
      }
    }

    if (fNOM == null) {
      if (kwconv.getState("_WARN_MISSING_",false)) {
        Shell.warning("INCOMPLETE conversion of PLATINUM file. No COL_IF_CENTER or keyword pairs found in PLATINUM source file to calculate FNOM"+
            ".\n      file:"+df.filename);
      }
      else {
        throw new NullPointerException("No COL_IF_CENTER or keyword pairs to calculate FNOM in PLATINUM file");
      }
    }
    else {
      kwds.put("D:FNOM", fNOM);
    }
  }

  /**
   * Get SBT_OFFSET if valid
   * @param kw keywords
   * @return value of SBT_OFFSET or null if value is not valid
   * @since NeXtMidas 3.7.0
   */
  private static Double getSbtOffset (Keywords kw) {
    Double sbt_offset = null;
    Integer sbt_offset_correct = kw.getL("SBT_OFFSET_CORRECT", null);
    if (sbt_offset_correct != null && sbt_offset_correct.intValue() == 1) {
      sbt_offset = kw.getD("SBT_OFFSET", null);
    }
    return sbt_offset;
  }

  /**
   * Get the multiplier for spectral sense if valid
   * @param kw keywords
   * @return value of multiplier (1: upright, -1: inverted) or null if value is not valid
   * @since NeXtMidas 3.7.0
   */
  private static Integer getSpectralSenseMultiplier (Keywords kw) {
    Integer multiplier = null;
    Integer data_inversion_flag = kw.getL("DATA_INVERSION_FLAG", null);
    if (data_inversion_flag != null) {
      int flagValue = data_inversion_flag.intValue();
      switch (flagValue) {
        case 0 : multiplier = 1;  // upright
                 break;
        case 1 : multiplier = -1; // inverted
                 break;
      }
    }
    return multiplier;
  }

  /** Coverts BLUE file header, keyword and record data to JSON
  *
  *  @param ref           MidasReference
  *  @param datafileName  DataFile FileName
  *  @param textfileName  JSON TextFile FileName
  *  @param convTbl       Conversion table
  */
  @InternalUseOnly
  public static void convertBlue2JSON (MidasReference ref, FileName datafileName,
                                       FileName textfileName, Table convTbl) {
    DataFile df = new DataFile(ref, datafileName);
    DataFileHeader dfHeader = new DataFileHeader(df);
    Table dfHeaderKeywordData = dfHeader.getHeaderAndKeywordsAsTable();
    Table dfRecordData = null;

   Table    kwds    = new Table();
    Table    opts    = convTbl.getTable("_OPTIONS_", new Table());
    boolean warnMissing = convTbl.getState("_WARN_MISSING_",false);

    //MapKBT nestedKwds = MapKBT.newMap();
    Map<String,Object> nestedKwds = new HashMap<>();

    // ==== GENERATE THE NEW KEYWORDS ==============================================================
    String keys = null;
    try {
      // -- Process Keyword Entries ----------------------------------------------------------------
      for (String key : convTbl.getKeys()) {
        if (key.startsWith("_") && key.endsWith("_")) {
          continue; // "_OPTIONS_" or other "hidden" field
        }
        Table   ct   = convTbl.getTable(key);
        buildStructure(nestedKwds, key, df, ct, dfHeaderKeywordData, dfRecordData);
      }
      keys = null;

    }
    catch (InvocationTargetException e) {
      if (keys == null) keys = "keywords";
     throw new MidasException("2JSON InvocationTargetException failed with exception:", e);
    }
    catch (Exception e) {
      if (keys == null) keys = "keywords";
      System.out.println("convertHeadKey2JSON exception information:"+e.getSuppressed()+" "+e.getCause()+" "+e.getStackTrace());
      throw new MidasException("2JSON failed with exception:", e);
    }

    String json = JsonUtilities.toJSON(nestedKwds).toString();
    TextFile out = new TextFile(textfileName);
    out.open(BaseFile.OUTPUT);
    out.write(json);
    out.close();

  }

  private static void buildStructure (Map<String,Object> nestedMap, String key,
      DataFile df, Table ct, Table dfHeaderKeywordData, Table dfRecordData)
      throws Exception {
    String  type = ct.getS("TYPE", null); // now only checked to determine the structure of the JSON output
    if ("MAP".equals(type)) {
      Map<String,Object> subMap = new HashMap<>();
      for (String subKey : ct.getKeys()) {
        if (!"TYPE".equals(subKey)) {
          Table ctSub = ct.getTable(subKey);
          String subType = ctSub.getS("TYPE",null);
          if (typeIsStructure(subType)) {
            buildStructure(subMap, subKey, df, ctSub, dfHeaderKeywordData, dfRecordData);
          }
          else {
            subMap.put(subKey,retrieveValue(ctSub, dfHeaderKeywordData, dfRecordData, subKey));
          }
        }
      }
      nestedMap.put(key,subMap);
    }
    else if ("ARRAY_MAP".equals(type)) {
      ArrayList<Map<String,Object>> mapArray = new ArrayList<>();
      for (String tempMapKey : ct.getKeys()) {
        if (!"TYPE".equals(tempMapKey)) {
          Table ctMap = ct.getTable(tempMapKey);
          Map<String,Object> subMap = new HashMap<>();
          for (String subKey : ctMap.getKeys()) {
            if (!"TYPE".equals(subKey)) {
              Table ctSub = ctMap.getTable(subKey);
              String subType = ctSub.getS("TYPE",null);
              if (typeIsStructure(subType)) {
                buildStructure(subMap, subKey, df, ctSub, dfHeaderKeywordData, dfRecordData);
              }
              else {
                subMap.put(subKey,retrieveValue(ctSub, dfHeaderKeywordData, dfRecordData, subKey));
              }
            }
          }
          mapArray.add(subMap);
        }
      }
      nestedMap.put(key,mapArray);
    }
    else if ("ARRAY_RECORDS".equals(type)) {
      ArrayList<Map<String,Object>> mapArray = new ArrayList<>();
      if (dfRecordData == null) dfRecordData = getRecordDataAsTable(df);
      for (String row : dfRecordData.getKeys()) {
        Map<String,Object> rowMap = new HashMap<>();
        Table rowTable = dfRecordData.getTable(row);
        for (String recMapKey : ct.getKeys()) {
          if (!"TYPE".equals(recMapKey)) {
            Table ctMap = ct.getTable(recMapKey);
            rowMap.put(recMapKey,retrieveValue(ctMap, rowTable, dfRecordData.getTable(row), recMapKey));
          }
        }
        mapArray.add(rowMap);
      }
      nestedMap.put(key,mapArray);
    }
    else {
      nestedMap.put(key,retrieveValue(ct, dfHeaderKeywordData, dfRecordData, key));
    }
  }

  private static boolean typeIsStructure (String type) {
    return ("ARRAY_MAP".equals(type) || "MAP".equals(type));
  }

  /** Retrieve a Value from the appropriate source<br>
   *  this includes converting the value to a new form, if needed
   *  @param ct    conversion table
   *  @param dfHeaderKeywordTable  BLUE Header and Keyword Information in a Table
   *  @param dfRecordDataTable     BLUE Record Data in a Table
   *  @param key   key being looked up (only needed here if we add
   *               additional capability found convertKeywordsInPlace)
   *  @return value
   *  @throws Exception throws IllegalArgumentException and can throw others
   */
  private static Object retrieveValue (Table ct, Table dfHeaderKeywordTable, Table dfRecordDataTable,
      String key) throws Exception{

    String  keys         = ct.getS("KEYS",         key); // allows multiple key sources to be listed - not currently used
    String  source       = ct.getS("SOURCE",       null);
    String  location     = ct.getS("LOCATION",     "EXTENDED");
    if ("EXT".equals(location)) location = "EXTENDED"; // allow the abbreviation
    String  conversion   = ct.getS("CONVERSION",   "NONE");
    Object  val          = ct.get( "VALUE",        null);
    String  inlineFunc   = ct.getS("INLINE_FUNC",  null);
    String  javaFunc     = ct.getS("JAVA_FUNC",    null);
    boolean required     = ct.getState("REQUIRED", false); // do we want this check in the future?
    Table   mapping      = ct.getTable("MAPPING",  null);
    location = location.toUpperCase();

    if (ct.containsKey("VALUE")) {
      // The VALUE overrides everything. This is frequently seen in user-provided overrides that
      // want to force a setting regardless of the other conversion options and defaults.
    }
    else if (conversion.equals("MAPPING")) {
      if (mapping == null) throw new NullPointerException("Missing MAPPING=");

      if (!"RECORD".equals(location)) {
        val = ((Table)dfHeaderKeywordTable.get(location)).getKey(source);
      }
      else {
        val = dfRecordDataTable.get(source);
      }
      if (val != null) val = mapping.get(val.toString());
      if (val == null) val = ct.get("DEFAULT");
    }
    else if (conversion.equals("INLINE_FUNC")) {
      if (inlineFunc == null) throw new NullPointerException("Missing INLINE_FUNC=");

        if (!"RECORD".equals(location)) {
          val = ((Table)dfHeaderKeywordTable.get(location)).getKey(source);
        }
       else {
         val = dfRecordDataTable.get(source);
        }
        if (val != null) {
        // TODO: We should be able to do an "evaluate carets" function call if we do this...
        //   Midas m = new Midas();
        //   m.getResults().putAll(kw.makeTable());
        // However kw.makeTable() doesn't work with PLATINUM files right now, and we don't
        // want to fix it until early in a beta series (as it might break others).
        Midas m = null; // Midas ref

        // Evaluate the carets in the function
        int    i; // start index
        int    j; // end index
        String k; // keyword name
        while ((i = inlineFunc.indexOf('^')) >= 0) {
          if ((i < inlineFunc.length()-1) && (inlineFunc.charAt(i+1) == '{')) {
            j = inlineFunc.indexOf('}', i+1) + 1; // additional +1 to skip over '}' below
            if (j <= 0) throw new IllegalArgumentException("Missing '}' in "+inlineFunc);
            k = inlineFunc.substring(i+2, j-1);
          }
          else {
            j = i+1;
            while ((j < inlineFunc.length()) && inlineFunc.substring(j,j+1).matches("[A-Za-z0-9_.]")) {
              // Find end of keyword name, assume alphanumeric name with '_' and '.' as valid
              // parts of the keyword name (note that '.' is common in PLATINUM).
              j++;
            }
           k = inlineFunc.substring(i+1, j);
          }
          Object v =  ((Table)dfHeaderKeywordTable.get(location)).getKey(source);
          if (v == null) v = "0";
          inlineFunc = inlineFunc.substring(0,i) + v + inlineFunc.substring(j);
        }
             if (inlineFunc.startsWith("CALC(" )) val = new Data(Convert.inLineCalc(m,inlineFunc));
        else if (inlineFunc.startsWith("SEDIT(")) val = Convert.inLineSedit(m,inlineFunc);
        else if (inlineFunc.startsWith("TEST(" )) val = Convert.inLineTest(m,inlineFunc);
        else throw new IllegalArgumentException("Unsupported INLINE_FUNC="+inlineFunc);
      }
      else {
        val = ct.get("DEFAULT");
      }
    }
    else if (conversion.equals("JAVA_FUNC")) {
      if (javaFunc == null) throw new NullPointerException("Missing JAVA_FUNC=");
      int      indexDot   = javaFunc.lastIndexOf('.');
      String   className  = javaFunc.substring(0,indexDot);
      String   methodName = javaFunc.substring(indexDot+1);
      Class<?> classObj   = Class.forName(className);
      Method   method     = classObj.getMethod(methodName, Table.class,
          Table.class, Table.class, String.class);
      val = method.invoke(null, ct, dfHeaderKeywordTable, dfRecordDataTable, key);
    }
    else if (!conversion.equals("NONE")) {
      throw new IllegalArgumentException("Unsupported CONVERSION="+conversion);
    }
    else if (source == null) {
      // This is a normal look-up entry, but the LOOKUP is missing. This will happen when users
      // provide a DEFAULT but no VALUE or LOOKUP. In this case the DEFAULT is used. This differs
      // from providing a VALUE in that a user could override the default conversion (e.g. merge
      // their custom options on top of a normal conversion description)  with a LOOKUP whereas
      // once VALUE is set, it overrides everything else.
      val = ct.get("DEFAULT");
    }
    else { // normal source look-up

      if (!"RECORD".equals(location)) {
        val = ((Table)dfHeaderKeywordTable.get(location)).getKey(source);
      }
      else {
        val = dfRecordDataTable.get(source);
      }

      if (val == null) val = ct.get("DEFAULT");
    }

    return val;
  }

  /** Convert by Mapping Values and then Concatinating them
   *
   *  @param ct                   conversion table
   *  @param dfHeaderKeywordTable Table containing header and keyword data
   *  @param dfRecordDataTable    Table containing BLUE Record data
   *  @param key                  key to look up (not used)
   *  @return String value formed
   *  @throws Exception throws IllegalArgument and possible other Exceptions
   */
  @InternalUseOnly("Called via reflection by to support HEADKEY2JSON")
  public static String convMapConcat (Table ct, Table dfHeaderKeywordTable, Table dfRecordDataTable,
      String key) throws Exception{
    Table mapCatOpts = ct.getTable("JAVA_FUNC_OPTS");
    String concat = "";
    for (String mapKey : mapCatOpts.getKeys()) {
      Table mappingInfo = mapCatOpts.getTable(mapKey);
      Object valObj = retrieveValue(mappingInfo, dfHeaderKeywordTable, dfRecordDataTable, key);
      if (!(valObj instanceof String)) valObj = Convert.o2s(valObj);
      String valToMap = (String)valObj;
      Table   mapping      = mappingInfo.getTable("MAPPING",  null);
      if (mapping !=null) {
        if (valToMap != null ) {
        concat += mapping.get(valToMap);
        }
      }
      else {
        concat += valToMap;
      }
    }
    return concat;
  }

  /** Convert a double value to the appropriate Time Format
   *  @param ct                   conversion table
   *  @param dfHeaderKeywordTable Table containing header and keyword data
   *  @param dfRecordDataTable    Table containing BLUE Record data
   *  @param key                  key to look up (not used)
   *  @return String with Time formatted
   *  @throws Exception throws Exception if something goes wrong in conversion
   */
  @InternalUseOnly("Called via reflection by to support HEADKEY2JSON")
  public static String convDoubleTimeToFmt (Table ct, Table dfHeaderKeywordTable, Table dfRecordDataTable,
      String key) throws Exception{;
    Table timeMapOpts = ct.getTable("JAVA_FUNC_OPTS");
    Double doubleTime = 0.0;
    if (!timeMapOpts.containsKey("TIME1")) {
      Object valTime = retrieveValue(timeMapOpts, dfHeaderKeywordTable, dfRecordDataTable, key);
      if (!(valTime instanceof Double)) valTime = Convert.o2d(valTime);
      doubleTime = (Double)valTime;
    }
    else {
      int timeNum = 1;
      while (timeMapOpts.containsKey("TIME"+timeNum)) {
        Table thisTime = timeMapOpts.getTable("TIME"+timeNum);
        Object valTime = retrieveValue(thisTime, dfHeaderKeywordTable, dfRecordDataTable, key);
        if (!(valTime instanceof Double)) valTime = Convert.o2d(valTime);
        doubleTime += (Double)valTime;
        timeNum++;
      }
    }
    String format = timeMapOpts.getS("FORMAT", "STD");
    return Time.toString (doubleTime, format , 6);
  }

   /** Get BLUE Record Data in Table form
   *
   *  @param df the DataFile
   *  @return Table containing record data
   */
  @InternalUseOnly
  public static Table getRecordDataAsTable (DataFile df) {
    Table tblRecordData = new Table();
    df.open();
    if (df.isRecordBased()) {
      int numRows = (int)df.getNumberOfRows();
      for (int i=0; i < numRows; i++) {
        tblRecordData.put("ROW_"+i, df.getDataTable(i));
      }
    }
    df.close();
    return tblRecordData;
  }

   /** Get BLUE Record Data in List of Tables form
   *
   *   @param df the DataFile
   *   @return Table containing record data
   */
  @InternalUseOnly
  public static List<Table> getRecordDataAsList (DataFile df) {
    List<Table> listRecordData = new ArrayList<>();
    df.open();
    if (df.isRecordBased()) {
      int numRows = (int)df.getNumberOfRows();
      for (int i=0; i < numRows; i++) {
        listRecordData.add(df.getDataTable(i));
      }
    }
    df.close();
    return listRecordData;
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // Nested Classes
  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** A private implementation of {@link FileUtil.FnFilter} */
  private static final class FnFilterImpl implements FnFilter {
    private final StringBuffer description;  // description for toString()
    private final Pattern[]    pathAccept;   // Acceptance based on full path
    private final Pattern[]    pathReject;   // Rejection based on full path
    private final Pattern[]    fileAccept;   // Acceptance based on name only
    private final Pattern[]    fileReject;   // Rejection based on name only

    /** Only used by createFileNameFilter(..) */
    FnFilterImpl (String root, List<String> accPath, List<String> accFile, List<String> rejPath, List<String> rejFile) {
      this.pathAccept  = (accPath == null)? null           : new Pattern[accPath.size()];
      this.fileAccept  = (accFile == null)? null           : new Pattern[accFile.size()];
      this.pathReject  = (rejPath == null)? new Pattern[0] : new Pattern[rejPath.size()];
      this.fileReject  = (rejFile == null)? new Pattern[0] : new Pattern[rejFile.size()];
      this.description = new StringBuffer();
      description.append("FnFilter: (accept: ");

      if (pathAccept == null) {
        description.append("* ");
      } else {
        compileRegex(description, fileAccept, accFile, "");
        compileRegex(description, pathAccept, accPath, root);
      }
      description.append("reject:");
      compileRegex(description, fileReject, rejFile, "");
      compileRegex(description, pathReject, rejPath, root);
      description.append(")");
    }

    private void compileRegex (StringBuffer descr, Pattern[] pat, List<String> wild, String root) {
      for (int i = 0; i < pat.length; i++) {
        String wc = wild.get(i);
        if (wc.indexOf('/') >= 0) {
          if ((root == null) || root.isEmpty()) {
            throw new NullPointerException("FileUtil.createFnFilter: found '"+wc+"', but root was null.");
          }
          wc = terminatePath(root) + wc;
        }
        wc = wc.replace('/', File.separatorChar);
        pat[i] = Pattern.compile(StringUtil.wildcardsToRegex(wc));
        descr.append(wc).append(' ');
      }
    }

    @Override
    public boolean accept (File dir, String name) {
      return accept(new File(dir,name));
    }

    @Override
    public boolean accept (File file) {
      String  path = file.getPath();
      String  name = file.getName();
      boolean ok   = (fileAccept == null);

      if (file.isDirectory()) {
        path = path + File.separator;
        for (int i = 0; !ok && (i < pathAccept.length); i++)   ok =  pathAccept[i].matcher(path).matches();
        for (int i = 0;  ok && (i < pathReject.length); i++)   ok = !pathReject[i].matcher(path).matches();
      } else {
        for (int i = 0; !ok && (i < fileAccept.length); i++)   ok =  fileAccept[i].matcher(name).matches();
        for (int i = 0; !ok && (i < pathAccept.length); i++)   ok =  pathAccept[i].matcher(path).matches();
        for (int i = 0;  ok && (i < fileReject.length); i++)   ok = !fileReject[i].matcher(name).matches();
        for (int i = 0;  ok && (i < pathReject.length); i++)   ok = !pathReject[i].matcher(path).matches();
      }
      return ok;
    }

    @Override
    public String toString() {
      return description.toString();
    }
  }


  /** Wraps a PlotFileNH as a read-only ListFile. */
  private static class PlotFileAsListFile implements ListFile {
    private final PlotFileNH pf;

    /** Creates a new instance. */
    PlotFileAsListFile (PlotFileNH pf) {
      this.pf = pf;
    }

    @Override public boolean  isOpen()                 { return pf.isOpen();            }
    @Override public boolean  open(int flags)          { return pf.open(flags);         }
    @Override public String   getURL()                 { return pf.getURL();            }
    @Override public FileName getName()                { return pf.getName();           }
    @Override public double   getNumberOfRows()        { return pf.getSize();           }
    @Override public boolean  getProtected()           { return true;                   } // Always read-only
    @Override public int      getRecordDefCount()      { return pf.getRecordDefCount(); }
    @Override public Table    getRecordDef(int i)      { return pf.getRecordDef(i);     }
    @Override public Table    getRecordDefs()          { return pf.getRecordDefs();     }
    @Override public Table    getDataTable(double off) { return pf.getDataTable(off);   }
              public int      seek (double off)        { return pf.seek(off);           }
              public double   seek()                   { return pf.seek();              }
              public double   avail()                  { return pf.avail();             }
    @Override public void     close()                  { pf.close();                    }
              public int      readDataTable(Table tbl) { return pf.readDataTable(tbl);  }
    @Override public void     removeData (double off)  { removeData(off,1);             }

    @Override
    public void setRecordDefs (Table defs) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    @Override
    public int setData (double offset, Table table) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    @Override
    public void setData (double element, String fieldName, Object obj) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    @Override
    public void insertData (double offset, Object obj) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    @Override
    public void removeData (double offset, long count) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    public int writeDataTable (Table tbl) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    @Override
    public void update() { // since 3.1.1
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
  }

  /** Wraps a PlotFileNH as a read-only ListFile. */
  @InternalUseOnly
  public static class PlotFileWrapper extends PlotFileGetLayerModSwap implements ListFile {
    protected final PlotFile pf;

    /** Creates a new instance. */
    protected PlotFileWrapper (PlotFile pf) {
      super(pf);
      this.pf = pf;
      if (pf == null) throw new NullPointerException("pf can not be null");
    }

    @Override public String   getURL()       { return pf.getURL();       }
    @Override public String   getTag()       { return pf.getTag();       }
    @Override public double   getSize()      { return pf.getSize();      }
    @Override public String   getFormat()    { return pf.getFormat();    }
    @Override public double   getStart()     { return pf.getStart();     }
    @Override public double   getDelta()     { return pf.getDelta();     }
    @Override public int      getUnits()     { return pf.getUnits();     }
    @Override public double   getXStart()    { return pf.getXStart();    }
    @Override public double   getXDelta()    { return pf.getXDelta();    }
    @Override public int      getXUnits()    { return pf.getXUnits();    }
    @Override public int      getXFrame()    { return pf.getXFrame();    }
    @Override public double   getYStart()    { return pf.getYStart();    }
    @Override public double   getYDelta()    { return pf.getYDelta();    }
    @Override public int      getYUnits()    { return pf.getYUnits();    }
    @Override public int      getYFrame()    { return pf.getYFrame();    }
    @Override public int      getType()      { return pf.getType();      }
    @Override public int      getMode()      { return pf.getMode();      }

    @Override public FileName getName()      { return pf.getName();      }
    @Override public boolean  isStream()     { return pf.isStream();     }
    @Override public boolean  isStreaming()  { return pf.isStreaming();  }
    @Override public boolean  isOpen()       { return pf.isOpen();       }
    @Override public double   seek()         { return pf.seek();         }
    @Override public double   avail()        { return pf.avail();        }

    @Override public Data    getDataBuffer (int size)   { return pf.getDataBuffer(size); }
    @Override public double  getTimeAt (double offset)  { return pf.getTimeAt(offset);   }
    @Override public Object  getQualifier (String key)  { return pf.getQualifier(key);   }
    @Override public boolean find (int dir)             { return pf.find(dir);           }
    @Override public boolean open (int flags)           { return pf.open(flags);         }
    @Override public int     read (Data data, int elem) { return pf.read(data,elem);     }
    @Override public int     seek (double off)          { return pf.seek(off);           }
    @Override public double  getNumberOfRows()          { return pf.getSize();           }
    @Override public boolean getProtected()             { return true;                   }
    @Override public int     getRecordDefCount()        { return pf.getXFrame();         }

    @Override public void    init (Object ref, Object fname) { pf.init(ref,fname); }
    @Override public void    connect (int mode)              { pf.connect(mode);   }
    @Override public void    close()                         { pf.close();         }
    @Override public void    removeData (double off)         { removeData(off,1);  }

    @Override
    public void setRecordDefs (Table defs) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    @Override
    public int setData (double offset, Table table) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    @Override
    public void setData (double element, String fieldName, Object obj) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    @Override
    public void insertData (double offset, Object obj) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    @Override
    public void removeData (double offset, long count) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    /** Throws Unsupported Operation Exception in FileUtil
        @param tbl NeXtMidas Table
        @return throws an exception and never a regular return
     */
    public int writeDataTable (Table tbl) {
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }
    @Override
    public void update() {  // since 3.1.1
      throw new UnsupportedOperationException("Can not write to "+getName()+" file is read-only.");
    }

    @Override
    public Table getRecordDef (int i) {
      Table def = null;

      if ((i >= 0) && (i < getRecordDefCount())) {
        def = new Table();
        def.put("NAME",   "F"+(i+1));
        def.put("FORMAT", getFormat());
      }
      return def;
    }

    @Override
    public Table getRecordDefs() {
      Table tbl = new Table();

      for (int i = 0; i < getRecordDefCount(); i++) {
        tbl.put(Integer.toString(i), getRecordDef(i));
      }
      return tbl;
    }

    @Override
    public Table getDataTable (double offset) {
      double seek = seek();
      Table  tbl  = null;

      seek(offset);
      Data buf = getDataBuffer(1);
      int  num = pf.read(buf, 1);

      if (num != 1) {
        tbl = new Table();
        for (int i = 0; i < buf.getAPE(); i++) {
          tbl.put(("F"+(i+1)), buf.getAtomAt(i));
        }
      }
      seek(seek+1);
      return tbl;
    }

    @Override
    public int readDataTable (Table tbl) {
      Table row = getDataTable(seek());
      if (row == null) return -1;

      tbl.merge(row);
      return 1;
    }
  }

  /** Wraps a PlotFileNH as a read-only ListFile. */
  private static class PlotFileWrapper2 extends PlotFileWrapper {
    protected final ListFile lf;

    /** Creates a new instance. */
    PlotFileWrapper2 (PlotFile pf, ListFile lf) {
      super(pf);
      this.lf = lf;
      if (lf == null) throw new NullPointerException("lf can not be null");
      if (lf != pf  ) throw new IllegalArgumentException("lf must match pf");
    }

    @Override public double  getNumberOfRows()                   { return lf.getNumberOfRows();     }
    @Override public boolean getProtected()                      { return lf.getProtected();        }
    @Override public int     getRecordDefCount()                 { return lf.getRecordDefCount();   }
    @Override public Table   getRecordDef(int i)                 { return lf.getRecordDef(i);       }
    @Override public Table   getRecordDefs()                     { return lf.getRecordDefs();       }
    @Override public Table   getDataTable (double off)           { return lf.getDataTable(off);     }
    @Override public int     setData (double off, Table tbl)     { return lf.setData(off,tbl);      }

    @Override public void setRecordDefs (Table defs)                    { lf.setRecordDefs(defs);   }
    @Override public void removeData (double off)                       { lf.removeData(off);       }
    @Override public void insertData (double off, Object obj)           { lf.insertData(off,obj);   }
    @Override public void removeData (double off, long count)           { lf.removeData(off,count); }
    @Override public void setData (double off, String name, Object obj) { lf.setData(off,name,obj); }

    @Override
    public int writeDataTable (Table tbl) {
      double seek = seek();
      lf.setData(seek,tbl);
      seek(seek+1);
      return 0;
    }

    @Override
    public int readDataTable (Table tbl) {
      double seek = seek();
      Table row = getDataTable(seek);
      if (row == null) return -1;

      tbl.merge(row);
      seek(seek+1);
      return 1;
    }
  }

  /** Opens an input stream for a file with check for classpath and then for local file.
   *  @param fileName The file name.
   *  @return The input stream (not null).
   *  @throws IOException If the stream could not be created (usually file-not-found).
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static InputStream getInputStreamFor (String fileName) throws IOException {
    return getInputStreamFor(FileUtil.class.getClassLoader(), fileName);
  }

  /** Opens an input stream for a file with check for classpath and then for local file.
   *  @param classLoader The class loader to use (not null).
   *  @param fileName    The file name (not null).
   *  @return The input stream (not null).
   *  @throws IOException If the stream could not be created (usually file-not-found).
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static InputStream getInputStreamFor (ClassLoader classLoader, String fileName) throws IOException {
    InputStream in = classLoader.getResourceAsStream(fileName);
    return (in != null)? in : new FileInputStream(fileName);
  }

  /** Generates a same-directory temp file name using the convention that the temp version of a
   *  file has the same name, but with a dot (".") prefix which is used on a lot of POSIX-style
   *  systems that are trying to do atomic copy/move operations.
   *  @param fname  The file name.
   *  @return The version with the dot prefix.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static Path toLocalTemp (Path fname) {
    return fname.resolveSibling("."+fname.getFileName());
  }

  /** Generates a same-directory temp file name using the convention that the temp version of a
   *  file has the same name, but with a dot (".") prefix which is used on a lot of POSIX-style
   *  systems that are trying to do atomic copy/move operations.
   *  , although it *may* work on
   *  Windows,
   *  @param fname  The file name.
   *  @return The version with the dot prefix.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static File toLocalTemp (File fname) {
    return new File(fname.getParentFile(), "."+fname.getName());
  }

  /** Creates a new file, overwriting any existing one.
   *  @param file The file.
   *  @throws IOException If there is an I/O error.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void createFile (File file) throws IOException {
    createFile(file.toPath());
  }

  /** Creates a new file, overwriting any existing one.
   *  @param file The file.
   *  @throws IOException If there is an I/O error.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void createFile (Path file) throws IOException {
    deleteIfExists(file);
    log(Level.FINER, "Creating new file {0}", file);
    Files.createFile(file);
  }

  /** Deletes all files within a directory.
   *  @param dir The directory.
   *  @throws IOException If there is an I/O error.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void deleteDir (File dir) throws IOException {
    deleteIfExists(dir.toPath());
  }

  /** Deletes all files within a directory.
   *  @param dir The directory.
   *  @throws IOException If there is an I/O error.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void deleteDir (Path dir) throws IOException {
    FileVisitor<Path> visitor = new FileVisitor<Path>() {
      @Override
      public FileVisitResult preVisitDirectory (Path dir, BasicFileAttributes attrs) throws IOException {
        return FileVisitResult.CONTINUE;
      }

      @Override
      public FileVisitResult visitFile (Path file, BasicFileAttributes attrs) throws IOException {
        deleteIfExists(file);
        return FileVisitResult.CONTINUE;
      }

      @Override
      public FileVisitResult postVisitDirectory (Path dir, IOException e) throws IOException {
        deleteIfExists(dir);
        return FileVisitResult.CONTINUE;
      }

      @Override
      public FileVisitResult visitFileFailed (Path file, IOException e) throws IOException {
        log(Level.SEVERE, "File visit failed for {0}", e, file);
        return FileVisitResult.CONTINUE;
      }
    };

    log(Level.FINE, "Deleting directory {0}", dir);
    Files.walkFileTree(dir, visitor);
    deleteIfExists(dir);
  }

  /** Deletes a file, if it exists.
   *  @param file The file.
   *  @throws IOException If there is an I/O error.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void deleteIfExists (File file) throws IOException {
    deleteIfExists(file.toPath());
  }

  /** Deletes a file, if it exists.
   *  @param file The file.
   *  @throws IOException If there is an I/O error.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void deleteIfExists (Path file) throws IOException {
    if (Files.deleteIfExists(file)) {
      log(Level.FINE, "Deleting {0}", file);
    }
  }

  /** Moves a file, replacing any prior file. This will attempt an atomic move, if that fails
   *  (or if moving between file systems), this will first copy to a file named {@code ".<out>"}
   *  in the output directory (overwriting it if it exists) and then doing a (likely) atomic
   *  move to the final name.
   *  @param in  The original file.
   *  @param out The new file.
   *  @return The output file (may differ from {@code out} if {@code out} is a directory).
   *  @throws IOException If there is an I/O error.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static File move (File in, File out) throws IOException {
    return move(in.toPath(), out.toPath()).toFile();
  }

  /** Moves a file, replacing any prior file. This will attempt an atomic move, if that fails
   *  (or if moving between file systems), this will first copy to a file named {@code ".<out>"}
   *  in the output directory (overwriting it if it exists) and then doing a (likely) atomic
   *  move to the final name.
   *  @param in  The original file.
   *  @param out The new file.
   *  @return The output file (may differ from {@code out} if {@code out} is a directory).
   *  @throws IOException If there is an I/O error.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static Path move (Path in, Path out) throws IOException {
    if (out.toFile().isDirectory()) {
      out = out.resolve(in.getFileName());
    }

    // ==== TRY ATOMIC MOVE ========================================================================
    FileStore inFS  = Files.getFileStore(in.getParent());
    FileStore outFS = Files.getFileStore(out.getParent());

    if (Objects.equals(inFS, outFS)) {
      // Normal atomic move (same disk mount (AKA "file stores"))...
      log(Level.FINE, "Moving {0} to {1} (via atomic move)", in, out);
      try {
        Files.move(in, out, ATOMIC_MOVE, REPLACE_EXISTING);
        return out;
      }
      catch (AtomicMoveNotSupportedException e) {
        // This should be rare since we check the file disk mounts (AKA "file stores") first...
        log(Level.WARNING, "Moving {0} to {1} (via atomic move) failed", e, in, out);
      }
    }
    else {
      // Move+rename for moving accross disk mounts...
      log(Level.FINE, "Moving {0} to {1} (via move+rename)", in, out);
      Path temp = toLocalTemp(out);
      try {
        Files.move(in, temp, REPLACE_EXISTING);
        Files.move(temp, out, ATOMIC_MOVE, REPLACE_EXISTING);
        return out;
      }
      catch (AtomicMoveNotSupportedException e) {
        // This should be rare since temp and out should have the same file disk mount...
        log(Level.WARNING, "Moving {0} to {1} (via move+rename) failed", e, in, out);
      }
    }

    // ==== USE COPY-THEN-MOVE =====================================================================
    Path temp = toLocalTemp(out);
    log(Level.FINE, "Moving {0} to {1} (via {2})", in, out, temp);
    _copy(in, out, temp, PRESERVE);
    Files.delete(in);
    return out;
  }

  /** Moves a file, replacing any prior file. This will move via a copy operation thereby ensuring
   *  that the new file gets the file permissions that match the owner.
   *  @param in  The original file.
   *  @param out The new file.
   *  @return The output file (may differ from {@code out} if {@code out} is a directory).
   *  @throws IOException If there is an I/O error.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static File moveByCopy (File in, File out) throws IOException {
    return move(in.toPath(), out.toPath()).toFile();
  }

  /** Moves a file, replacing any prior file. This will move via a copy operation thereby ensuring
   *  that the new file gets the file permissions that match the owner.
   *  @param in  The original file.
   *  @param out The new file.
   *  @return The output file (may differ from {@code out} if {@code out} is a directory).
   *  @throws IOException If there is an I/O error.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static Path moveByCopy (Path in, Path out) throws IOException {
    if (out.toFile().isDirectory()) {
      out = out.resolve(in.getFileName());
    }

    // ==== USE COPY-THEN-MOVE =====================================================================
    Path temp = toLocalTemp(out);
    log(Level.FINE, "Moving {0} to {1} (via {2})", in, out, temp);
    _copy(in, out, temp, PRESERVE);
    Files.delete(in);
    return out;
  }

  /** Copies a file, replacing any prior file. This will *not* copy via an intermediate file.
   *  @param in  The original file.
   *  @param out The new file.
   *  @throws IOException If there is an I/O error.
   *  @see #copy(File,File)
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void quickCopy (File in, File out) throws IOException {
    quickCopy(in.toPath(), out.toPath());
  }

  /** Copies a file, replacing any prior file. This will *not* copy via an intermediate file.
   *  @param in  The original file.
   *  @param out The new file.
   *  @throws IOException If there is an I/O error.
   *  @see #copy(Path,Path)
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void quickCopy (Path in, Path out) throws IOException {
    if (out.toFile().isDirectory()) {
      out = out.resolve(in.getFileName());
    }
    log(Level.FINE, "Copying {0} to {1}", in, out);
    _quickCopy(in, out, NO_PRESERVE);
  }

  /** Copies a file, replacing any prior file. This will *not* copy via an intermediate file.
   *  This will attempt to preserve any file permissions just like {@code cp -p}.
   *  @param in  The original file.
   *  @param out The new file.
   *  @throws IOException If there is an I/O error.
   *  @see #copy(File,File)
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void quickCopyP (File in, File out) throws IOException {
    quickCopyP(in.toPath(), out.toPath());
  }

  /** Copies a file, replacing any prior file. This will *not* copy via an intermediate file.
   *  This will attempt to preserve any file permissions just like {@code cp -p}.
   *  @param in  The original file.
   *  @param out The new file.
   *  @throws IOException If there is an I/O error.
   *  @see #copy(Path,Path)
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void quickCopyP (Path in, Path out) throws IOException {
    if (out.toFile().isDirectory()) {
      out = out.resolve(in.getFileName());
    }
    log(Level.FINE, "Copying {0} to {1}", in, out);
    _quickCopy(in, out, PRESERVE);
  }

  /** Copies a file, replacing any prior file. This will first copy to a file named {@code ".<out>"}
   *  in the output directory (overwriting it if it exists) and then do a (likely) atomic move
   *  to the final name.
   *  @param in  The original file.
   *  @param out The new file.
   *  @throws IOException If there is an I/O error..
   *  @see #quickCopy(File,File)
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void copy (File in, File out) throws IOException {
    copy(in.toPath(), out.toPath());
  }

  /** Copies a file, replacing any prior file. This will first copy to a file named {@code ".<out>"}
   *  in the output directory (overwriting it if it exists) and then do a (likely) atomic move
   *  to the final name.
   *  @param in  The original file.
   *  @param out The new file.
   *  @throws IOException If there is an I/O error.
   *  @see #quickCopy(Path,Path)
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void copy (Path in, Path out) throws IOException {
    if (out.toFile().isDirectory()) {
      out = out.resolve(in.getFileName());
    }
    Path temp = toLocalTemp(out);
    log(Level.FINE, "Copying {0} to {1} (via {2})", in, out, temp);
    _copy(in, out, temp, NO_PRESERVE);
  }

  /** Copies a file, replacing any prior file. This will first copy to a file named {@code ".<out>"}
   *  in the output directory (overwriting it if it exists) and then do a (likely) atomic move
   *  to the final name. This will attempt to preserve any file permissions just like {@code cp -p}.
   *  @param in  The original file.
   *  @param out The new file.
   *  @throws IOException If there is an I/O error..
   *  @see #quickCopy(File,File)
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void copyP (File in, File out) throws IOException {
    copy(in.toPath(), out.toPath());
  }

  /** Copies a file, replacing any prior file. This will first copy to a file named {@code ".<out>"}
   *  in the output directory (overwriting it if it exists) and then do a (likely) atomic move
   *  to the final name. This will attempt to preserve any file permissions just like {@code cp -p}.
   *  @param in  The original file.
   *  @param out The new file.
   *  @throws IOException If there is an I/O error.
   *  @see #quickCopy(Path,Path)
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static void copyP (Path in, Path out) throws IOException {
    if (out.toFile().isDirectory()) {
      out = out.resolve(in.getFileName());
    }
    Path temp = toLocalTemp(out);
    log(Level.FINE, "Copying {0} to {1} (via {2})", in, out, temp);
    _copy(in, out, temp, PRESERVE);
  }

  /** This should only be used internal to copy(..) and move(..). */
  private static void _copy (Path in, Path out, Path temp, CopyOption[] opts) throws IOException {
    Files.deleteIfExists(out);
    Files.deleteIfExists(temp);
    _quickCopy(in, temp, opts);
    try {
      Files.move(temp, out, ATOMIC_MOVE);
    }
    catch (AtomicMoveNotSupportedException e) {
      // This is not an error since it can happen for reasons that are unrelated
      // to a "failure" condition. However, such should be exceedingly rare, so
      // we want to log the exception for debugging purposes.
      log(Level.FINE, "Atomic move within same directory failed", e);
      Files.move(temp, out);
    }
  }

  /** This should only be used internal to copy(..) and move(..). */
  private static void _quickCopy (Path in, Path out, CopyOption[] opts) throws IOException {
    Files.deleteIfExists(out);
    Files.copy(in, out, opts);
    if (DEF_PERMISSIONS != null) {
      if (opts == NO_PRESERVE) Files.setPosixFilePermissions(out, DEF_PERMISSIONS);
    }
  }

  /** Gets the user's default POSIX file permissions. These will typically equate to what is
   *  set via {@code umask}.
   *  @return The default permissions or null (if Windows will return null)
   *  @since NeXtMidas 3.9.0
   */
  @SuppressWarnings({"UseSpecificCatch", "BroadCatchBlock", "TooBroadCatch"})
  private static Set<PosixFilePermission> getDefaultPosixFilePermissions () {
    // There isn't a good way to get this, so we just create a temp file (which should get
    // the default permissions) and then read back the permissions. Note that we explicitly
    // create the temp file using File.createTempFile(..) not the NIO  Files.createTempFile(..)
    // since the latter ignores umask and makes a temp file that is only accessible by the user
    // (which is good for temp files, but bad for this trick).
    Set<PosixFilePermission> permissions = null;
    if (!Shell.isWindows()) {
      try {
        File f = File.createTempFile("Utils", ".tmp");
        permissions = Files.getPosixFilePermissions(f.toPath());
        f.delete();
        log(Level.FINEST, "Default POSIX file permissions: {0}", permissions);
      }
      catch (Exception e) {
        log(Level.SEVERE, "Unable to determine default POSIX file permissions", e);
      }
    }
    return permissions;
  }

  /** Computes the MD5SUM for a file.
   *  @param inFile The file.
   *  @return The MD5SUM value.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static BigInteger md5sum (Path inFile) {
    return md5sum(inFile.toFile());
  }

  /** Computes the MD5SUM for a file.
   *  @param inFile The file.
   *  @return The MD5SUM value.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static BigInteger md5sum (File inFile) {
    try (InputStream in = new FileInputStream(inFile)) {
      return md5sum(inFile.toString(), in);
    }
    catch (IOException e) {
      throw new RuntimeException("Unable to compute MD5 sum for "+inFile, e);
    }
  }

  /** Computes the MD5SUM for a file.
   *  @param in The input stream.
   *  @return The MD5SUM value.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static BigInteger md5sum (InputStream in) {
    String name = getShortClassName(in.getClass());
    return md5sum(name, in);
  }

  /** Computes the MD5SUM for a file.
   *  @param inFile The file.
   *  @return The MD5SUM value.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  private static BigInteger md5sum (String name, InputStream in) {
    try {
      MessageDigest md = MessageDigest.getInstance("MD5");

      byte[] buf = new byte[64436];
      int    num;
      while ((num = in.read(buf)) > 0) {
        md.update(buf, 0, num);
      }
      return new BigInteger(1, md.digest());
    }
    catch (IOException|NoSuchAlgorithmException e) {
      throw new RuntimeException("Unable to compute MD5 sum for "+name, e);
    }
  }

  /** Reads all content from a file.
   *  @param inFile The file.
   *  @return The content read in.
   *  @throws IOException Any exceptions during reading.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static byte[] readAll (Path inFile) throws IOException {
    return readAll(inFile.toFile());
  }

  /** Reads all content from a file.
   *  @param inFile The file.
   *  @return The content read in.
   *  @throws IOException Any exceptions during reading.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static byte[] readAll (File inFile) throws IOException {
    try (InputStream in = new FileInputStream(inFile)) {
      return readAll(in, (int)inFile.length());
    }
  }

  /** Reads all content from an input stream.
   *  @param c The connection to read from.
   *  @return The content read in.
   *  @throws IOException Any exceptions during reading.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static byte[] readAll (HttpURLConnection c) throws IOException {
    int         code    = c.getResponseCode();
    boolean     http200 = (code >= 200) && (code <= 299);
    InputStream in      = (http200)? c.getInputStream() : c.getErrorStream();

    return readAll(in, c.getContentLength());
  }

  /** Reads all content from an input stream.
   *  @param in The input stream.
   *  @return The content read in.
   *  @throws IOException Any exceptions during reading.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static byte[] readAll (InputStream in) throws IOException {
    return readAll(in, -1);
  }

  /** Reads all content from an input stream.
   *  @param in       The input stream.
   *  @param sizeHint The estimated size.
   *  @return The content read in.
   *  @throws IOException Any exceptions during reading.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static byte[] readAll (InputStream in, int sizeHint) throws IOException {
    // readAll(InputStream,int) and readAllChars(InputStream,int) are identical, except for
    // the last line where they return either a normalized buf[] or String.
    byte[] buf = new byte[max(64436, sizeHint)];
    int    len = 0;
    int    n;

    while ((n = in.read(buf, len, buf.length-len)) > 0) {
      len += n;

      if (len == buf.length) {
        buf = Arrays.copyOf(buf, len + len/2);
      }
    }
    return Arrays.copyOf(buf, len);
  }

  /** Reads all content from a file.
   *  @param inFile The file.
   *  @return The content read in.
   *  @throws IOException Any exceptions during reading.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static CharSequence readAllChars (Path inFile) throws IOException {
    return readAllChars(inFile.toFile());
  }

  /** Reads all content from a file.
   *  @param inFile The file.
   *  @return The content read in.
   *  @throws IOException Any exceptions during reading.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static CharSequence readAllChars (File inFile) throws IOException {
    try (InputStream in = new FileInputStream(inFile)) {
      return readAllChars(in, (int)inFile.length());
    }
  }

  /** Reads all content from an input stream.
   *  @param in The input stream.
   *  @return The content read in.
   *  @throws IOException Any exceptions during reading.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static CharSequence readAllChars (InputStream in) throws IOException {
    return readAllChars(in, -1);
  }

  /** Reads all content from an input stream.
   *  @param in       The input stream.
   *  @param sizeHint The estimated size.
   *  @return The content read in.
   *  @throws IOException Any exceptions during reading.
   *  @since NeXtMidas 3.9.0
   */
  @ProvisionalUseOnly("until adequately tested")
  public static CharSequence readAllChars (InputStream in, int sizeHint) throws IOException {
    // readAll(InputStream,int) and readAllChars(InputStream,int) are identical, except for
    // the last line where they return either a normalized buf[] or String.
    byte[] buf = new byte[max(64436, sizeHint)];
    int    len = 0;
    int    n;

    while ((n = in.read(buf, len, buf.length-len)) > 0) {
      len += n;

      if (len == buf.length) {
        buf = Arrays.copyOf(buf, len + len/2);
      }
    }
    return new String(buf, 0, len);
  }

  /** returns the filename extension
      @param string Full file name
      @return File name extension
      @since NeXtMidas 3.9.2 moved here for JDK modularity
   */
  public static String ext (String string) {
    return new FileName(string).getExt();
  }

  /** returns the path of a filename
      @param string Full file name
      @return Head or path of file name
      @since NeXtMidas 3.9.2 moved here for JDK modularity
   */
  public static String head (String string) {
    FileName fn = new FileName(string);
    return fn.getPath();
  }

  /** returns the filename without extension of a full file name
      @param string Full file name
      @return File name without path or extension
      @since NeXtMidas 3.9.2 moved here for JDK modularity
   */
  public static String root (String string) {
    FileName fn = new FileName(string);
    return fn.getRoot();
  }

  /** returns the filename and extension of a full file name
      @param string Full file name
      @return File name root and extension
      @since NeXtMidas 3.9.2 moved here for JDK modularity
   */
  public static String tail (String string) {
    FileName fn = new FileName(string);
    return fn.getRoot()+"."+fn.getExt();
  }

  /** Set whether to use the URL API deprecated in JDK20
   *  @param useDeprecatedUrlAPI Whether to use the URL API deprecated in JDK20
   */
  @InternalUseOnly
  public static void setUseDeprecatedUrlAPI (boolean useDeprecatedUrlAPI) {
    FileUtil.useDeprecatedUrlAPI = useDeprecatedUrlAPI;
  }

  /** Get whether to use the URL API deprecated in JDK20
   *  @return true if using the URL API deprecated in JDK20
   */
  @InternalUseOnly
  public static boolean getUseDeprecatedUrlAPI () {
    return FileUtil.useDeprecatedUrlAPI;
  }
}
