package nxm.sys.lib;

import java.io.Closeable;
import java.io.File;
import java.util.Map;

import nxm.sys.inc.AsciiMap;
import nxm.sys.inc.Commandable;
import nxm.sys.inc.DataTypes;
import nxm.sys.inc.Debug;
import nxm.sys.inc.InternalUseOnly;
import nxm.sys.inc.KeyObjectNames;
import nxm.sys.inc.MidasReference;
import nxm.sys.inc.ProvisionalUseOnly;
import nxm.sys.lib.FileName.FNCase;
import nxm.sys.libm.Tolerance;

/** A class to define a basic file. <br>
 *  <br>
 *  Since NeXtMidas 3.7.0 this implements the {@link java.io.Closeable} interface.
 *
 *  @author Jeff Schoen
 */
public class BaseFile extends Object implements AsciiMap, DataTypes, Closeable {
  /** The default buffer size used by many file types. */
  public static int BUFFER_SIZE = 32768;

  /** The comma-separated list of properties a file can have. */
  public static final String propertyList = "Name,Size,Comment,Qualifiers";

  /** File class identifier for a "Base File".          */
  public static final int BASE  = 0;
  /** File class identifier for a "Data File".          */
  public static final int DATA  = 1;
  /** File class identifier for a "Text File".          */
  public static final int TEXT  = 2;
  /** File class identifier for a "Image File".         */
  public static final int IMAGE = 3;
  /** File class identifier for a "Shape File".         */
  public static final int SHAPE = 4;
  /** File class identifier for an "ESRI DBF File".     */
  public static final int DBF   = 5;
  /** File class identifier for a "Zip File".           */
  public static final int ZIP   = 6;
  /** File class identifier for a "tar Archive".        */
  public static final int TAR   = 7;
  /** File class identifier for a "Java Archive".       */
  public static final int JAR   = 8;

  /** The comma-separated list I/O-open-mode flags. */
  public static final String flagsList =
        "Input,Output,Wrap,Append,Flush,NoAbort,Optional,Native,"                // 0x000001 - 0x000080
      + "ReOpen,Wait,HeaderOnly,Truncate,MkDir,ForceAbort,AlwaysReOpen,NoOpen,"  // 0x000100 - 0x008000
      + "PktHdr,WarnEmpty,EmbeddedLF,MaintainCase,DoNotAddExtension";                                                      // 0x010000 - ...

  /** I/O open flag: File used as an input file.        */
  public static final  int INPUT   = 0x0001;
  /** I/O open flag: File used as an output file.       */
  public static final int OUTPUT  = 0x0002;
  /** I/O open flag: File should use "wrapping".        */
  public static final int WRAP    = 0x0004;
  /** I/O open flag: File should be appended to. This flag works as follows:
      <pre>
        {@link #INOUT}|{@link #APPEND}   - Append to an existing file, error out if it does not exist.
        {@link #OUTPUT}|{@link #APPEND}  - Append to a file, create a new file if necessary.
        {@link #INPUT}|{@link #APPEND}   - Illegal flag combination.
      </pre>
   */
  public static final int APPEND  = 0x0008;
  /** I/O open flag: File should automatically be flushed after every write. */
  public static final int FLUSH   = 0x0010;
  /** I/O open flag: Methods (specifically {@link #open()} and {@link #open(int)}) should
                     not abort with an error when an exception is encountered.
   */
  public static final int NOABORT = 0x0020;
  /** I/O open flag: Use of the file is optional.       */
  public static final int OPTIONAL= 0x0040;
  /** I/O open flag: File is a native file.             */
  public static final int NATIVE  = 0x0080;
  /** I/O open flag: File is in the process of being REOPENed. This flag should only be set
                     internal to the file system to flag a file (typically a {@link Pipe})
                     that is in the process of being reopened. (See also {@link #ALWAYSREOPEN}.)
   */
  public static final int REOPEN  = 0x0100;
  /** I/O open flag: Wait for file to exist during open.
      @since NeXtMidas 2.5.0
   */
  public static final int WAIT    = 0x0200;
  /** I/O open flag: Open file for header access only. Any data-specific operations
      (such as opening a detached file header) can be skipped over.
      @since NeXtMidas 2.5.0
   */
  public static final int HEADERONLY = 0x0400;
  /** I/O open flag: Truncate existing file data. This causes any existing file with the same name
      to be cleared out (or deleted) before opening for {@link #OUTPUT}. When this is NOT set, an
      existing file may be overwritten by inserting the new data over top of existing data (this
      default matches X-Midas). This flag is ignored if the {@link #INPUT} or {@link #INOUT} flags
      are set. See "Direction Values" at the top of {@link IOResource} for details.
      @since NeXtMidas 2.5.0
   */
  public static final int TRUNCATE = 0x0800;
  /** I/O open flag: File's directory should be made.   */
  public static final int MKDIR   = 0x1000;
  /** I/O open flag: Used *only* by {@link TextFile} to force {@link #open()} to abort when unable
      to open the file. This is the default behavior whenever {@link #NOABORT} is not set. This flag
      is only used to work around a bug in {@link TextFile} (see {@link TextFile#open()} for details).
      Note that {@link #NOABORT} and {@link #FORCEABORT} are intended to be mutually exclusive, if
      combined {@link #NOABORT} will be used.
      @since NeXtMidas 2.5.0
   */
  public static final int FORCEABORT = 0x2000;
  /** I/O open flag: File will always be reopened following {@link #REOPEN}ed being set. When a
      {@link Pipe} needs to be reopened the {@link #REOPEN} flag is set and then the {@link Pipe}
      waits for the reopen. However, the pipe has a safety check to detect cases where a
      {@link Primitive} does not reconnect to the pipe following a reopen (an example would be a
      {@link Primitive} that disconnects from <code>_pipeA</code> due to getting a reopen, but rather
      than reconnecting to <code>_pipeA</code> it connects to <code>_pipeB</code>). If the {@link Pipe}
      detects that the {@link Primitive} has completed its reopen and has not reconnected to the
      {@link Pipe}, it discards that connection (preventing the pipe from blocking). This flag
      disables that safety check and tells the {@link Pipe} to wait for this file to reopen, without
      regard to the state of the {@link Primitive}. This flag is used where pipe restarts have
      special handling.
      @since NeXtMidas 2.5.5
   */
  public static final int ALWAYSREOPEN = 0x4000;
  /** I/O open flag: File should not be opened.         */
  public static final int NOOPEN  = 0x8000; // JC4: Is anyone still using this?
  /** I/O open flag: File contains only Packet Headers. */
  public static final int PKTHDR  = 0x10000;
  /** I/O open flag: Warn if file is empty. */
  public static final int WARNEMPTY  = 0x20000; // since 2.7.3
  /** I/O open flag: File has embedded linefeed(s) in data  */
  public static final int EMBEDDEDLF = 0x40000;
  /** I/O flag to maintain case when creating file */
  public static final int MAINTAINCASE = 0x80000;
  /** I/O flag to prevent the adding of extension when creating file */
  public static final int DONOTADDEXTENSION = 0x100000;
  /** I/O open flag: File used for input and output. Same as <code>{@link #INPUT}|{@link #OUTPUT}</code>. */
  public static final int INOUT = INPUT|OUTPUT;

  /** Default I/O open flags: Currently set to <code>DEF_FLAGS = {@link #TRUNCATE}</code>.
      @since NeXtMidas 2.5.0
   */
  public static final int DEF_FLAGS = TRUNCATE;

  /** Equivalence test flag: Test binary equivalence.   */
  public static final int EQ_BIN    = 0x0001;
  /** Equivalence test flag: Test file size.            */
  public static final int EQ_SIZE   = 0x0002;
  /** Equivalence test flag: Test Blue file header.     */
  public static final int EQ_HDR    = 0x0004;
  /** Equivalence test flag: Test Blue file data.       */
  public static final int EQ_DATA   = 0x0008;
  /** Equivalence test flag: Test Blue file keywords.   */
  public static final int EQ_KEY    = 0x0010;
  /** Equivalence test flag: Test (Relative) Tolerance. */
  public static final int EQ_TOL    = 0x0020;
  /** Equivalence test flag: Test Magnitude.            */
         static final int EQ_MAG    = 0x0040; // package-private reserved for use by Data
  /** Equivalence test flag: Case-sensitive strings.    */
         static final int EQ_CS     = 0x0080; // package-private reserved for use by Data
  /** Equivalence test flag: Vector equality.           */
         static final int EQ_VEQ    = 0x0100; // package-private reserved for use by Data
  /** Equivalence test flag: Test Absolute Tolerance. Since NeXtMidas 3.3.2 */
         static final int EQ_ABSTOL = 0x0200; // package-private reserved for use by Data
  /** Equivalence test flag: Test with Length Limit. Since NeXtMidas 3.7.0 */
  public static final int EQ_LENLIM = 0x0400; // package-private reserved for use by Data and DataFile
  /** Equivalence test flag: Use "default" tests.       */
  public static final int EQ_DEF    = 0x8000;

  /** Requests a radix of 10 (decimal) is to be used.    */
  public static final int RADIX_10 = 0x0;
  /** Requests a radix of 16 (hexadecimal) is to be used.*/
  public static final int RADIX_16 = 0x1;
  /** Requests a radix of 2 (binary) is to be used.      */
  public static final int RADIX_2  = 0x2;
  /** Requests a radix of 8 (octal) is to be used.       */
  public static final int RADIX_8  = 0x4;

  /** Value used as initial size of byte buffer */
  private static final int MAX_BUF_SIZE = 32678;

  /** The size of the file (in bytes).             */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public    double     size=0.0;
  /** The current offset into the file (in bytes). */
  protected double     offset=0.0;
  /** The file name of the file.                   */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public    String     filename;
  /** The "tag" for the file.                      */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public    String     tag;
  /** The aux used for the file.                   */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public    String     aux;
  /** The comments associated with the file.       */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public    String     comment;
  /** Indicates if the file is open.               */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public    boolean    isOpen=false;
  /** Indicates if the file has been found.        */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public    boolean    isFound=false;
  /** Indicates if the file is an input file.      */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public    boolean    isInput=false;
  /** Indicates if the file is an output file.     */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  @KeyObjectNames({"isOutput"}) // suppress acronym "IO" to disambiguate with {@link #io}
  public    boolean    isOutput=false;
  /** The flags used when opening the file.        */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public    int        flags=DEF_FLAGS;
  /** Indicates linefeed(s) are present in data    */
  @InternalUseOnly("Since NeXtMidas 3.7.1")
  public    boolean    hasLinefeed=false;
  /** The current midas context.                   */
  @InternalUseOnly("Since NeXtMidas 3.5.1, this has always been meant for read-only use by callers")
  public    Midas      M;   // this field should be final, but cannot since it is initialized in init(..) method
  /** The current command.                         */
  @InternalUseOnly("Since NeXtMidas 3.5.1, this has always been meant for read-only use by callers")
  public    Command    cmd; // this field should be final, but cannot since it is initialized in init(..) method
  /** The file name of the file. deprecated Since 3.5.0, use {@link #getFileName()} or {@link #getName()} instead. */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public    FileName   fn; // TODO: add deprecated annotation in 3.7+
  /** The {@link IOResource} to use.               */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public    IOResource io;
  /** File Comparison Tolerance.
      @deprecated Since 3.5.0, use {@link #getFeqTolerance()} or {@link #setFeqTolerance(double)} instead.
   */
  @Deprecated
  public    double     tolerance = 0; // TODO: make private in 3.9.0+

  private    double    absTolerance = 0;

  /** Initialize new instance. */
  public BaseFile() { }

  /** Initialize new instance with filename.
      @param filename The name of the file to use.
   */
  public BaseFile (String filename) {
    this(null, filename);
  }

  /** Initialize new instance with reference and filename.
      @param ref      The reference to use.
      @param filename The name of the file to use.
      deprecated Since 3.5.0 use {@link #BaseFile(MidasReference, Object)}
   */
  // TODO: add deprecated annotation in 3.7+
  @InternalUseOnly("deprecated Since NeXtMidas 3.5.0 use BaseFile(MidasReference, Object)")
  public BaseFile (Object ref, Object filename) {
    this();
    init(Convert.ref2MidasReference(ref), filename);
  }

  /** Initialize new instance with reference and filename.
    @param ref      The midas reference to use.
    @param filename The name of the file to use.
    @since 3.5.0
   */
  public BaseFile (MidasReference ref, Object filename) {
    this();
    init(ref, filename);
  }

  /** Initialize references and filename.
      @param ref      The reference to use (Ideally a {@link Command} for proper cmdline args picking).
      @param filename The name of the file to use.
      deprecated Since 3.5.0 use {@link #init(MidasReference, Object)}
   */
  // TODO: add deprecated annotation in 3.7+
  @InternalUseOnly("deprecated Since NeXtMidas 3.5.0 use init(MidasReference, Object)")
  public void init (Object ref, Object filename) {
    init(Convert.ref2MidasReference(ref), filename);
  }

  /** Initialize references and filename.
      @param ref      The midas reference to use (Ideally a {@link Command} for proper cmdline args picking)
      @param filename The name of the file to use.
      @since 3.5.0
   */
  public void init (MidasReference ref, Object filename) {
    M = Convert.ref2Midas(ref);
    if (ref!=null) {
      if ((M!=null) && M.isJavaBehavior()) { // NXM-4097 base MAINTAINCASE on Java vs Macro usage
        flags |= MAINTAINCASE;
      }
    }
    init(ref,M,filename);
  }

  /** Initialize references and filename.
   *  This init is for cases where flags have already been handled
   *  @param ref The midas reference to use (Ideally a {@link Command} for proper cmdline args picking)
   *  @param M   The Midas
   *  @param filename The name of the file to use.
   */
  void init (MidasReference ref, Midas M, Object filename) {
    this.M = M; // may have been initialized already, but make sure
    if (ref instanceof Command) { cmd = (Command)ref; }
    else                        { cmd = null;         }
    if (cmd != null && cmd.args.find("/AUX")) aux = cmd.args.getS("/AUX");
    setName(filename);
  }

  /** Test for the existence of the file for a given I/O direction.
      @param dir I/O direction. See "Direction Values" at the top of {@link IOResource} for details.
      @return <code>true</code> if the file exists, <code>false</code> if not.
  */
  public boolean find (int dir) {
    IOResource oldio = io;
    io = findResource(dir);
    if (oldio  != null) oldio.close();
    return io.exists() && !io.isContainer(); // since 3.1.1 added second test to match DataFile.exists() (BUG 2569)
  }

  IOResource findResource (int dir) { // package-private only used by BaseFile and FileName
    return M.io.findResource(fn,dir,"",aux);
  }

  /** Test for the existence of the file. This is the same as using <code>find(IOResource.DIR_INPUT)</code>.
      @return <code>true</code> if the file exists, <code>false</code> if not.
      @since NeXtMidas 2.1.1
  */
  public final boolean exists () {
    return find(IOResource.DIR_INPUT);
  }

  /** Open a data connection with specified flags. This is the same as <code>setFlags(flag)</code>
      followed by {@link #open()}. Any subclasses should override {@link #open()} rather than this
      method.
      @param flag The I/O open flags to use.
      @return <code>true</code> if the file is successfully opened and <code>false</code> if not.
  */
  public boolean open (int flag) {
    setFlags(flag);
    return open();
  }

  /** Gets I/O direction, used by open() in BaseFile and DataFile. */
  int getDir (boolean isInput, boolean isOutput, boolean isFound) {
    if (isInput && isOutput && isFound) return IOResource.DIR_INOUT;

    if (isInput) return ((flags&TRUNCATE)!=0)? IOResource.DIR_INPUT_TRUNC  : IOResource.DIR_INPUT;
    else         return ((flags&TRUNCATE)!=0)? IOResource.DIR_OUTPUT_TRUNC : IOResource.DIR_OUTPUT;
  }

  /** Open the file with the flags that have already been set. Note that when the {@link #WAIT}
      flag is set (or the file is a pipe) this method will wait for an input file to be created
      if it does not already exist.
      @return <code>true</code> if the file is successfully opened and <code>false</code> if not.
  */
  public boolean open () {
    isOpen = false;

    if ((flags&INOUT) != 0);
    else flags |= INPUT;

    if (tag.length()==0 && filename.length()==0) {
      if ((flags&OPTIONAL)!=0) return isOpen;
      throw new MidasException("BaseFile: No FileName specified");
    }

    if ((flags&EMBEDDEDLF) != 0) {
      hasLinefeed = true;
    }

    //*** NOTE: start DataFile matches this block of code
    if ((flags&APPEND) != 0) {
      isInput  = ((flags&INPUT)!=0) || exists();
      isOutput = true;
    } else {
      isInput  = (flags&INPUT)  != 0;
      isOutput = (flags&OUTPUT) != 0;
    }
    int dir = getDir(isInput,isOutput,false);  // Initial dir before setting isFound
    isFound = find(dir);
    //*** NOTE: end DataFile matches this block of code

    if ((M.debug&Debug.IO)!=0) M.info("OpenBF tag="+tag+" Path="+getURL());
    if (isInput && !isFound && cmd!=null && cmd.isPiped && ((flags&WAIT)!=0)) {
      waitForFile(); // Waits for an input file to be created
      if (!isFound && (flags&NOABORT)==0 ) { close();  return isOpen; }// wait aborted...(since 3.3.0 - Bug 2686)
    }
    dir = getDir(isInput,isOutput,isFound); // Update dir based on isFound

    try {
      if (isInput && !isFound) {
        if ((flags&NOABORT)!=0) return isOpen;
        throw new CouldNotFindException();
      }
      isOpen = io.open(dir);
      if (!isOpen) throw new MidasException("Could not open");

      setInternals();

      Table qualifiers = fn.getQualifierTable();

      if (KeyObject.setKeys(this,qualifiers,M,KeyObject.SOFT) > 0) {
        setInternals(); // apply changes done by the filename qualifiers
      }

      if ((flags&APPEND)!=0) seek(getSize()); // since 3.1.3: seek to EOF when appending (BUG 2624)
    }
    catch (MidasException e) {
      io.close(); isOpen=false;
      if ((flags&NOABORT)==0) {
        throw new MidasException("Could not open "+getName()+": "+e.getMessage(),
                                 "flags="+getFlagsString(), e);
      }
      if ((M.debug&Debug.IO) != 0) M.printStackTrace(e);
    }
    return isOpen;
  }

  /** Waits for an input file to be created. This method should only be called from the
      {@link #open()} method and should be called just after {@link #isInput} and {@link #isFound}
      have been set. This is separate from open() since DataFile and BaseFile both use it, but
      under different conditions. Note that (cmd!=null && cmd.isPiped) must be true before calling
      this method.
      @return true if the file was found, false if the method aborted (added in 3.3.0)
      @since NeXtMidas 2.5.0
   */
  protected void waitForFile () {
    if ((M.debug&Debug.IO)!=0) M.info ("Waiting for file: "+tag);
    int oldState=0;
    if (cmd!=null) { oldState = cmd.state; cmd.state = Commandable.WAIT; }
    // check for getInterrupt and finishing added in 3.3.0 for Bug 2686
    while (!cmd.getInterrupt() && !cmd.finishing() &&  M.isPiped() && !isFound) {
      Time.sleep(0.1);
      isFound = find(IOResource.DIR_INPUT);
    }
    if (isFound && cmd!=null) cmd.state = oldState;
  }

  /** Sets the flags to be used for opening a file.
      @param flags The flags as defined in {@link #flagsList}.
      @see #flagsList
  */
  public void setFlags (String flags) {
    setFlags( Parser.mask(flagsList,flags,0) );
  }

  /** Adds to the set of flags to be used for opening a file.
      @param flag The flags individual flags to use OR'ed together (e.g. to use both {@link #INPUT}
                  and {@link #OUTPUT} use <code>setFlags(INPUT | OUTPUT)</code>).
  */
  public void setFlags (int flag) {
    flags |= flag;
    if ((flags&NATIVE)!=0) setQualifier("USENATIVE","Y");
    if ((flags&MKDIR)!=0) setQualifier("MKDIR","Y");
  }

  /** Gets the file flags.
      @return the file flags.
      @since NeXtMidas 2.1.3
   */
  public int getFlags () {
    return flags;
  }

  /** Gets the file flags as a string.
      @return the file flags.
      @since NeXtMidas 2.1.3
   */
  public String getFlagsString() {
    return Parser.mask2s(flagsList, getFlags());
  }

  /** Gets these flags as a string.
      @param flags the flags to get the string value of
      @return string representation of the flags
      @since NeXtMidas 4.1.4
   */
  @InternalUseOnly
  public static String getFlagsString(int flags) {
    return Parser.mask2s(flagsList, flags);
  }

  /** Update a data header. */
  public void update () {
  }

  /** Flush any cached data to disk and update header.
      @todo Consider changing this to include a call to {@link #update()}
            or Consider changing the comments so they do not imply that such
            is being done when it is not.
  */
  public void flush () {
  }

  /** Close a data connection. */
  public void close () {
    if (!isOpen) return;
    if ((M.debug&Debug.IO)!=0) M.info("CloseBF tag="+tag);
    if (isOutput) update();
    if (io!=null) io.close();
    isOpen = false;
  }


  /** Erase the data file.
      @param force This parameter is not used.
      @return <code> 0</code> if the file is not found;
              <code>-1</code> if the file can not be deleted; or
              <code> 1</code> if the file is successfully deleted.
      @todo Consider removing the force parameter since it is never used.
      @todo Change the return value to associate it with a pre-defined constant.
  */
  public int erase (boolean force) {
    if (!find(IOResource.DIR_INPUT)) return 0;
    if (!io.delete()) return -1;
    return 1;
  }

  /** This method is a safety measure that may not be needed
   *  Only if setName is called after the no-arg constructor
   *  and without calling init, then this setName with a
   *  MidasReference must be used to set the FileName flags
   *  according to whether this is Java usage or not.
   *
   *  @param ref      The reference to use.
   *  @param filename The name of the file to use.
   */
  @ProvisionalUseOnly("API may be removed at a later date")
  public void setName (MidasReference ref, Object filename) {
    M = Convert.ref2Midas(ref);
    if (ref!=null) {
      if ((M!=null) && M.isJavaBehavior()) { // NXM-4097 base MAINTAINCASE on Java vs Macro usage
        flags |= MAINTAINCASE;
      }
    }
    setName(filename);
  }

  /** Set the file name. This will set the file name parsing out qualifiers, trimmers, etc...
      This should only be used directly if filename flags have already been adjusted
      for Java versus Macro usage.
      @param filename The file name to use (this can be a {@link FileName}, {@link String} or other
                      {@link Object} which will return a file name when its {@link Object#toString()}
                      method is called.
  */
  public void setName (Object filename) {
    if (Midas.isJavaVsMidasDebug()) M.info("#JvsM BaseFile.setName filename:"+filename+
        " flags:"+getFlagsString(flags)+" is FileName:"+(filename instanceof FileName));
    if (filename instanceof FileName) {
      fn = (FileName)filename;
    }
    else fn = getFileNameFor(M,Convert.o2s(filename), flags);
    if (fn.hasQualifiers()) {
      // 2010-10-19 NTN: since 3.1.0, pass in default aux, tag, flags string so that these values
      //                 DO NOT get overwritten if there is NO qualifiers overriding it, respectively
      setAux(Convert.o2s(fn.getQualifier("AUX",aux),M));
      setTag(Convert.o2s(fn.getQualifier("TAG",tag),M));
      setFlags(Convert.o2s(fn.getQualifier("FLAGS",getFlagsString()),M));
    }
    if (tag==null||tag.length()==0) tag = fn.getTag();
    this.filename = fn.getFullName();
  }

  /** Get the file name for the file, with any qualifiers and trimmers.
      @return The file name, or null if not set yet.
      @since NeXtMidas 2.3.1
   */
  public FileName getName () {
    return fn;
  }

  /** Get the file name for the file, with any qualifiers and trimmers.
      This is same as {@link #getName()}, but needed so that we can replace the deprecated public fn field.
      @return The file name, or null if not set yet.
      @since 3.5.0
   */
  public FileName getFileName () {
    return fn;
  }

  /** Gets the MIME type for the given file type. BaseFile always assumes a basic binary data set
      and returns <code>"application/octet-stream"</code>, however many sub-classes override this.
      @return The applicable MIME type.
      @since NeXtMidas 2.7.0
   */
  public String getMimeType () {
    return "application/octet-stream";
  }

  /** Gets the MIME type based on a file name.
      @param ref      The reference object.
      @param filename The file name.
      @return The applicable MINE type.
      @since NeXtMidas 2.7.0
   */
  public static String getMimeType (Object ref, Object filename) {
    BaseFile bf = BaseFile.getInstanceFor(ref, filename);
    return bf.getMimeType();
  }

  /** Set/update the internal fields to reflect header block. */
  protected void setInternals () {
    size = (double)io.getLength();
  }

  /** Write specified bytes at next address. This will start with byte
      <code>buf[boff]</code> and end with <code>buf[boff + bytes - 1]</code>.
      @param buf   The buffer of bytes.
      @param boff  The offset into <code>buf</code> to begin writing with.
      @param bytes The total number of bytes to write.
      @return The status indicator.
      @todo Explain what "status indicator" means as a return value.
  */
  public int write (byte[] buf, int boff, int bytes) {
    if ((M.debug&Debug.IO)!=0)                  // report run-time IO debug
      M.info("WriteBF "+tag+" bytes="+bytes);
    int status = io.write( buf, boff, bytes);
    if (status>0) offset += status;
    return (status);
  }

  /** Read specified bytes at next address. This will start with byte
      <code>buf[boff]</code> and end with <code>buf[boff + bytes - 1]</code>.
      @param buf   The buffer of bytes.
      @param boff  The offset into <code>buf</code> to begin writing with.
      @param bytes The total number of bytes to read.
      @return The actual number of bytes read or -1 if at the End-Of-File. If the number of bytes
              available is less than the number of bytes to read this will read only those bytes
              that are actually available and then return.
  */
  public int read (byte[] buf, int boff, int bytes) {
    if ((M.debug&Debug.IO)!=0){    // report run-time IO debug
      M.info("ReadBF "+tag+" bytes="+bytes);
    }
    int status = io.read( buf, boff, bytes);
    if (status>0) offset += status;
    else if (status==CoreIO.REOPEN && cmd!=null) cmd.setState(Commandable.RESTART);
    return (status);
  }

  /** Set the tolerance used for file comparisons.
      First checks to see if tolerance is set using /TOL switch. If not,
      gets the current System tolerance from the Tolerance class.
      @since NeXtMidas 2.7.0
   */
  public void setFeqTolerance () {
    if (cmd != null && cmd.args.find("/TOL")) tolerance = cmd.args.getD("/TOL");
    else tolerance = Tolerance.getTolerance();
  }

  /** Set the tolerance used for file comparisons
      @param tol The new value for the tolerance
      @since NeXtMidas 2.7.0
   */
  public void setFeqTolerance (double tol) {
    tolerance = tol;
  }

  /** Get the tolerance used for file comparisons
      @return tolerance The tolerance used for file comparisons
      @since NeXtMidas 2.7.0
   */
  public double getFeqTolerance () {
    return tolerance;
  }

  /** Set the absolutetolerance used for file comparisons.
      First checks to see if tolerance is set using /ABSTOL switch.
      If not, it is set to 0

  */
  void setFeqAbsTolerance () {
    if (cmd != null && cmd.args.find("/DELTA")) {
      absTolerance = cmd.args.getD("/DELTA");
    }
    else absTolerance = 0;
  }

  /** Set the absolute tolerance used for file comparisons
      @param tol The new value for the tolerance

  */
  void setFeqAbsTolerance (double tol) {
    absTolerance = tol;
  }

  /** Get the absolute tolerance used for file comparisons
      @return The absolute tolerance used for file comparisons
   */
  double getFeqAbsTolerance () {
    return absTolerance;
  }

  /** Convenience method for reading one byte at a time.
      @return The next byte of data from the input file.
      @see #read(byte[],int,int)
  */
  public byte readByte () {
    byte[] buf = new byte[1];
    read (buf,0,1);
    return buf[0];
  }

  /** Move next read/write pointer to specified element offset.
      @param off The offset to move to.
      @return Always returns <code>0</code>.
      @see #seek()
      @todo Consider making the return type void.
  */
  public int seek (double off) {
    offset = off;
    if (io!=null) io.seek((long)off);
    return 0;
  }

  /** Get the location of the read/write pointer.
      @return The offset of the read/write pointer (in bytes).
      @see #seek(double)
      @todo Consider making the return type long since it is not possible have
            a seek value that is a fraction of a byte.
  */
  public double seek () { return (double)io.seek(); }

  /** Gets the number of elements currently available for read/write.
      For most file types, the number of elements is the same the number of bytes.
      For Midas BLUE files (DataFile), this returns the number of elements available.
      @return The number of elements currently available for read/write. When the underlining
              IO Resource is a PipeResource this can also return negative when
              the PIPE is {@link CoreIO#CLOSED}, {@link CoreIO#REOPEN}, or {@link CoreIO#ERROR}
  */
  public double avail () { return (double)io.avail(); }

  /** Check to see if the specified number of elements are available for reading. This method is
        intended to be used within the <code>process()</code> method of a primitive:
        <pre>
          public int process () {
            if (!inFile.isReady(transferLength)) return NOOP;
              int numRead = inFile.read(..); // this may read less than one transfer length if at end of the file
            ...
            return NORMAL;
          }
        </pre>
      @param numElements The number of elements to check for reading.
      @return true if the number of elements specified are available for reading or if the resource
              ends/restarts before the specified number of elements will be available; false if
              not near the end of the resource and the specified elements are not available yet
      @throws MidasException if the number of elements specified exceeds the size of the pipe.
   */
  public boolean isReady (double numElements) {
    int numBytes = (int)Math.ceil(numElements); // numElements == numBytes in BaseFile
    return io.isReady(numBytes);
  }

  /** List the contents of the header for inline calls.
      @return The contents of the header for inline calls.
  */
  public String status () {
    return toString()+"\n"+listHeader();
  }

  /**  @return A short synopsis of file. */
  public String toString () {
    // since 3.1.1: format is now tag=value and file's basename is included
    String str = "Basic Resource=" + ((io!=null)? io.getTypeString() : " ") +
                 " Name=" + ((fn!=null)? fn.getBasename() : "");
    return str;
  }

  /** List the contents of the header if one exists.
      Otherwise, list the file name, size and any comments.
      @return The contents of the header.
      @see #getURL()
      @see #getSize()
      @see #getComment()
  */
  public String listHeader () {
    String list =
      "FileName    :  "+getURL()+"\n"
    + "Size        :  "+getSize()+"\n"
    + "Comment     :  "+getComment()+"\n";
    return list;
  }

  /** Returns a list of the keywords defined.
      @return The keywords defined or an message stating that no keywords are defined.
  */
  public String listKeywords () {
    return "No keywords defined for " + Util.getClassName(this) + " class\n";
  }

  /** This will list the elements of the file.
      @param start    The byte to seek to before beginning the listing.
      @param elements The number of elements to include in the listing.
      @param format   The format to use when displaying the elements.
      @param flags    The radix to use when listing the file. Should be one of {@link #RADIX_2},
                      {@link #RADIX_8}, {@link #RADIX_10}, or {@link #RADIX_16}.
      @return The listing of the elements of the file.
      @todo Consider renaming flags to something more appropriate such as
            "radixFlag", especially since flags implies that more than one can
            be specified when this is not the case.
  */
  public String listElements (double start, int elements, String format, int flags) {
    // DATALIST (traditional)
    if (format != null) {
      int radix = flagsToRadix(flags);
      Data data = new Data(format,elements);
      seek(start*data.bpe);
      elements = read(data.buf,0,elements*data.bpe) / data.bpe;
      if (elements<=0) return null;
      return data.toString(0,elements*data.spa,radix)+"\n";
    }

    // HEXDUMP (hex + ascii)
    int num = (elements > 16)? 16 : elements;
    Data data = new Data("SB",num);
    seek(start*data.bpe);
    num = read(data.buf,0,num);
    if (num<=0) return null;

    StringBuilder hex   = new StringBuilder();
    StringBuilder ascii = new StringBuilder();
    StringBuilder tail  = new StringBuilder();
    for (int i = 0; i < 16; i++) {
      if (i == 8) hex.append(' ');
      hex.append(' ');
      if (i < num) {
        byte b = data.buf[i];
        hex.append(Integer.toHexString((b>>4)&0x0F));
        hex.append(Integer.toHexString(b     &0x0F));

             if (b <  0x20) ascii.append('.');
        else if (b == 0x7f) ascii.append('.');
        else                ascii.append((char)b);
      }
      else {
        hex.append("  ");
        tail.append(" ");
      }
    }
    hex.append("  |").append(ascii).append("|").append(tail).append('\n');

    if (elements > 16) {
      String next = listElements(start+num, elements-num, format, flags);
      if (next != null) hex.append(next);
    }
    return hex.toString();
  }

  /** Convert a radix flag into the appropriate radix. (For example using
      {@link #RADIX_8} will produce <code>8</code>.) If the radix flag passed
      in does not match a standard radix then <code>10</code> will be returned.
      @param flags The radix flag to convert to it's radix value. Should be one of {@link #RADIX_2},
                   {@link #RADIX_8}, {@link #RADIX_10}, or {@link #RADIX_16}.
      @return The radix matching the radix flag or <code>10</code> if the flag
              does not match a known radix.

  */
  protected int flagsToRadix (int flags) {
    int radix=10;
    if ((flags&RADIX_16)!=0) radix=16;
    if ((flags&RADIX_8)!=0) radix=8;
    if ((flags&RADIX_2)!=0) radix=1;
    return radix;
  }

  /** Compute the number of elements that can be displayed per line. This is
      computed with <code>listElementsPerLine = lineWidth / 4</code>.
      @param lineWidth The width of the line.
      @param format    <i>This parameter is not used.</i>
      @param flags     <i>This parameter is not used.</i>
      @return The number of elements that can be displayed per line.
      @todo Consider removing the unused parameter (flags).
  */
  public int listElementsPerLine (int lineWidth, String format, int flags) {
    return (format != null)? lineWidth/4 : 16;
  }

  /** Get the size value from the file's header.
      For most file types, the number of elements is the same as the number of bytes.
      For Midas BLUE files (DataFile), this returns the number of elements in the file.
      @return The size of the file in bytes for most files, in elements for DataFile.
  */
  public double getSize () { return size; }

  /** Get the current offset into the file, in bytes.
      @return the current file {@link #offset} in bytes
      @see #seek()
      @see #seek(double)
  */
  public double getOffset () { return offset; }

  /** Get the AUX used.
      @return the AUX used. ({@link #aux})
   */
  public String getAux () { return aux; }

  /** Get the file tag to used.
      @return the file tag used. ({@link #tag})
   */
  public String getTag () { return tag; }

  /** Get the comment associated with the file.
      @return the comment associated with the file. ({@link #comment})
   */
  public String getComment() { return comment; }

  /** Get the file name of the file. In order to maximize portability this will
      return the name of the file as a URL.
      @return The file name of the file as a URL.
  */
  public String getURL () {
    if (io==null) return fn.getFullName(); return io.getURL();
  }

  /** Get the table of qualifiers from the file's file name.
      @return The table of qualifiers for the file.
      @see #getQualifier(String)
  */
  public Table getQualifiers () { return fn.getQualifierTable(); }

  /** Get the value of a specific qualifier from the file's file name.
      @param key The qualifier's key.
      @return The qualifier value associated with the key, <code>key</code>.
      @see #getQualifiers()
      @see #setQualifier(String,Object)
  */
  public Object getQualifier (String key) { return fn.getQualifier(key); }


  /** Set the value of a specific qualifiers in the file's file name.
      @param key   The qualifier's key.
      @param value The qualifier's (new) value.
      @see #getQualifier(String)
  */
  public void setQualifier (String key, Object value) {
    fn.setQualifier(key,value);
  }

  /** Add the specified default qualifier (key/value) in the file's file name (which can still be overridden by user).
      @param key   The qualifier's key.
      @param value The qualifier's (new) value.
      @return true if key/value is added to the file's qualifiers table, otherwise false
                   (e.g. there is an existing user specified qualifier for that key).
      @see Table#addIfNotPresent(String,Object)
      @since NeXtMidas 3.1.0
   */
  public boolean setDefaultQualifier (String key, Object value) {
    return getQualifiers().addIfNotPresent(key, value);
  }

  /** Add following Map of key/value,...,keyN,valueN to the default qualifiers in the file's file
      name (which can still be overridden by user).
      @param defQualsMap default qualifiers Map (e.g. Table) to add to file
      @return true if any key/value from defQualsMap is added to the file's qualifiers table.
      @see #setDefaultQualifier(String, Object)
      @see Table#addIfNotPresent(Map)
      @since NeXtMidas 3.1.0
   */
  public boolean setDefaultQualifiers (Map<String, Object> defQualsMap) {
    return getQualifiers().addIfNotPresent(defQualsMap);
  }

  /** Get the {@link IOResource} in use.
      @return the {@link IOResource} in use.
   */
  public IOResource getResource () { return io; }

  /** Tells if the file is open.
      @return true if {@link #isOpen} is true, false otherwise.
   */
  public boolean isOpen () { return isOpen; }

  /** Tells if the I/O resource being used is a stream.
      @return <code>true</code> if a stream is being used, <code>false</code> otherwise.
      @see #getResource()
      @see IOResource#isStream()
  */
  public boolean isStream () { return io.isStream(); }

  /** Tells if the I/O resource is streaming data.
      @return <code>true</code> if the resource is streaming, <code>false</code> otherwise.
      @see #getResource()
      @see IOResource#isStreaming()
  */
  public boolean isStreaming () { return io.isStreaming(); }

  /** Tells if the file is a directory.
      @return <code>true</code> if file is a directory, <code>false</code> otherwise.
  */
  public boolean isDir () {
    // if not a local filename (eg. http://) then use the exists()
    if (!FileUtil.isLocalFileName(filename)) {
          return exists();
    } else {
      CoreIO io = (M!=null)? M.io : new CoreIO();
      return io.dexists(fn.getFullName());
    }
  }

  /** Set the file header to indicate a specified file size.
      @param value The new file size to use (in bytes).
      @todo Consider making this a long since files can not have an offset that
            includes a fractional number of bytes.
  */
  public void setSize (double value) { size = value; }

  /** Set AUX to use.
      @param value The new AUX to use.
      @see #setAux(int)
  */
  public void setAux (String value) {
    if ((value == null) || (Shell.getAuxPath(value) != null)) {
      aux = value;
    } else {
      throw new MidasException("Can not set AUX to '"+value+"' for "+getName()+" unknown AUX.");
    }
  }

  /** Set the file tag to use.
      @param value The new file tag.
  */
  public void setTag (String value) { tag = value; }

  /** Set the file's comment.
      @param value The new file comment.
  */
  public void setComment (String value) { comment = value; }

  /** Set the file name extension.
      @param ext The file name extension to use.
      @see #setExtDefault(String)
  */
  public void setExt (String ext) { fn.setExt(ext); }

  /** Set the default file name extension. If the file name extension is not
      yet set (i.e. it has a length of <code>0</code>) then it will be set to
      <code>ext</code>, otherwise it will be left as is.
      @param ext The default file name extension to use.
      @see #setExt(String)
  */
  public void setExtDefault (String ext) {
    if (fn.getExt().length()==0) fn.setExt(ext);
  }

  /** Set AUX used.
      @param value The new AUX to use.
      @see #setAux(String)
   */
  public void setAux (int value) {
    // aux = ""+value; // prior to 3.1.0
    setAux(Integer.toString(value));
  }

  /** Set the file's use as an output file, provided that it was originally
      opened with the {@link #OUTPUT} flag. Turning off the output mode always
      works, regardless of how the file was opened; but turning it on only works
      if the file was originally opened with the {@link #OUTPUT} flag.
      @param value Turns the use of the file's output capabilities on
                   (<code>true</code>) or off (<code>false</code>).
      @return Whether or not output mode is on after attempting to change it.
  */
  public boolean setOutput (boolean value) {
    if (value==true && (flags&OUTPUT)!=0) isOutput = true;
    else isOutput = false;
    return isOutput;
  }

  /** This will rename a file.
      @param newUrl the new file URL.
      @return <code> 0</code> if the file is not found, 1 if it was renamed, -1 if not renamed.
  */
  public int rename (String newUrl) {
    if (!find(IOResource.DIR_INPUT)) return 0;
    boolean ok = CoreIO.rename (getURL(),newUrl);
    return (ok)? 1 : -1;
  }

  /** Return a {@link FileName} object given a string filename. The filename is completely parsed
      and interpreted in the context of the specified reference object. Most of the time this method
      is not called directly, this method is usually called by {@link #getInstanceFor(Object,Object)}.
      @param ref      The reference to use (usually an instance of {@link Midas} or {@link Command}).
      @param filename The filename string to parse.
      @return The parsed file name.
      @deprecated since 3.5.0 use {@link #getFileNameFor(MidasReference, String)}
   */
  @Deprecated
  public static FileName getFileNameFor (Object ref, String filename) {
    return getFileNameFor(Convert.ref2MidasReference(ref), filename);
  }

  /** Return a {@link FileName} object given a string filename. The filename is completely parsed
      and interpreted in the context of the specified reference object. Most of the time this method
      is not called directly, this method is usually called by {@link #getInstanceFor(Object,Object)}.
      @param ref      The reference to use (usually an instance of {@link Midas} or {@link Command}).
      @param filename The filename string to parse.
      @return the FileName
      @since 3.5.0
   */
  public static FileName getFileNameFor (MidasReference ref, String filename) {
    Midas refM = Convert.ref2Midas(ref);
    if ((refM!=null) && refM.isJavaBehavior()) { // NXM-4097 base MAINTAINCASE on Java vs Macro usage
      return getFileNameFor(ref, filename,MAINTAINCASE);
    }
    return getFileNameFor(ref, filename, 0);
  }

  /** Return a {@link FileName} object given a string filename and int flags. The filename is completely parsed
      and interpreted in the context of the specified reference object. Most of the time this method
      is not called directly, this method is usually called by {@link #getInstanceFor(Object,Object)}.
      @param ref      The reference to use (usually an instance of {@link Midas} or {@link Command}).
      @param filename The filename string to parse.
      @param flags    Flags for setting the case.
      @return the FileName
   */
  public static FileName getFileNameFor (MidasReference ref, String filename, int flags) {
    // degenerate case (encountered for optional files)
    if (filename==null || filename.length()==0) return new FileName("");

    // if this is a quoted string we don't need to try resolving names
    if (filename.charAt(0)==QUOTE) return new FileName(filename);

    // we have to see if this name or at least the part before trimmers and qualifiers
    // can be resolved in the results table.  We also need to get a decent TAG value
    // before substitution for labeling this file
    String tag = null;
    int ls = filename.length();
    int lr = Results.validNameLength(filename,0);
    if (lr > 0 && filename.charAt(lr-1) == ')') {
      // If there is a semicolon or equals to the right of a paren
      // Adjust valid name length to NOT include trimmer or old style qualifier.
      int lp = filename.indexOf('(');
      if (filename.indexOf(':',lp) > 0) { lr = lp; }
      if (filename.indexOf('=',lp) > 0) { lr = lp; }
    }
    if (lr==0);  // no chance of being a result name
    else if (lr==ls) tag = filename;  // could be entire name
    else if (filename.charAt(lr)=='{') tag = filename.substring(0,lr);  // pre qualifier
    else if (filename.charAt(lr)=='(') tag = filename.substring(0,lr);  // pre trimmer

    Results rt = Convert.ref2Results(ref);
    if (tag!=null && rt!=null) {    // look up results
      tag = tag.toUpperCase();
      Object value = rt.get(tag);
      if (value instanceof String ||
          (value instanceof Data && ((Data)value).isString())) {
        if (lr==ls) filename = value.toString();
        else filename = value.toString() + filename.substring(lr);
      }
      else if (value instanceof FileName) {
        FileName fn = ((FileName)value).cloneOf();  // do not modify actual FileName results entry
        if (lr<ls) fn.append(filename.substring(lr));
        return fn;
      }
      else tag=null;
    }
    if (Midas.isJavaVsMidasDebug()) Midas.log("#JvsM BaseFile.getFileNameFor "+filename
        +" flags:"+getFlagsString(flags), Midas.INFO);
    FNCase fnCase = (flags & MAINTAINCASE) != 0 ? FNCase.KeepCase : FileName.getDefaultFNCase();
    FileName fn = new FileName(filename, fnCase);
    if (tag!=null) fn.setTag(tag);

    return fn;
  }

  /** Get a new instance of {@link BaseFile} for a given reference and file name. This is the method
      that checks the file handlers table in <code>REG.HANDLERS.FILE</code>. By default this looks for
      the entry in the  <code>REG.HANDLERS.FILE</code> that matches the file extension. However, the
      <code>FG=</code> qualifier can be used to override this. Since NeXtMidas 2.7.0 it is also possible
      to give the fully-qualified name of the specific class to be used as the handler via the
      <code>FG=</code> qualifier (e.g. <code>FG=nxm.myopt.lib.MyFileHandler</code>). <br>
      <br>
      Once the specific file handler is identified, a new instance is created using its no-argument
      constructor and it is then initialized using the {@link #init(Object,Object)} method.
      @param ref      The reference to use (usually an instance of {@link Midas} or {@link Command}).
      @param filename The file name for the new file.
      @return The sub-class of {@link BaseFile} applicable to the given file, if unknown, the default
              is just to return an instance of {@link BaseFile}.
      @see #init(Object,Object)
      @deprecated since 3.5.0 use {@link #getInstanceFor(MidasReference, Object)}
   */
  @Deprecated
  public static BaseFile getInstanceFor (Object ref, Object filename) {
    return getInstanceFor(Convert.ref2MidasReference(ref), filename);
  }

  /** Get a new instance of {@link BaseFile} for a given reference and file name. This is the method
      that checks the file handlers table in <code>REG.HANDLERS.FILE</code>. By default this looks for
      the entry in the  <code>REG.HANDLERS.FILE</code> that matches the file extension. However, the
      <code>FG=</code> qualifier can be used to override this. Since NeXtMidas 2.7.0 it is also possible
      to give the fully-qualified name of the specific class to be used as the handler via the
      <code>FG=</code> qualifier (e.g. <code>FG=nxm.myopt.lib.MyFileHandler</code>). <br>
      <br>
      Once the specific file handler is identified, a new instance is created using its no-argument
      constructor and it is then initialized using the {@link #init(Object,Object)} method.
      @param ref      The reference to use (usually an instance of {@link Midas} or {@link Command}).
      @param filename The file name for the new file.
      @return The sub-class of {@link BaseFile} applicable to the given file, if unknown, the default
              is just to return an instance of {@link BaseFile}.
      @see #init(Object,Object)
      @since 3.5.0
   */
  public static BaseFile getInstanceFor (MidasReference ref, Object filename) {
    BaseFile file=null;
    FileName fn;

    if (filename instanceof FileName) fn = (FileName)filename;
    else fn = getFileNameFor(ref,Convert.o2s(filename));

    String fg  = Convert.o2s(fn.getQualifier("FG"));
    String ext = (fg == null)? fn.getExt() : fg; // Use FG= if given, otherwise use file extension

    // shortcut for most common case (pipes,datafiles)
    //if (ext.length()==0) return new DataFile(ref,filename); // prior to 2.9.3
    if ( (ext.length()==0) && (fn.getRoot().length()>0) ) return new DataFile(ref,filename); // since 2.9.3 for BUG 1898

    String className;
    if ((fg != null) && (fg.indexOf('.') > 0)) { // <-- check of fg (not ext) is intentional here!
      className = fg; // assume FG= set to name of class to use (e.g. FG="nxm.sys.lib.CsvFile")
    }
    else {
      className = Registry.getHandlerName("FILE."+ext.toUpperCase());
    }

    if (className==null) { // not specified, assume text file
      file = new BaseFile(ref,filename);
    } else {               // otherwise load according to table of extensions/groups
      file = (BaseFile)Shell.getInstanceForClassName(className);
      file.init(ref,filename);
    }
    // Assume that users of this method want all file types to act the same on open(). To do this
    // we need to add FORCEABORT to TextFile instances until we fix the NOABORT behavior in TextFile.
    if (file instanceof TextFile) file.setFlags(TextFile.FORCEABORT);
    return file;
  }

  /** Converts a string used on FEQ tests to an integer flag.
      @param feqString The file equality definition string. This string will specify the equality
                       tests that will be used when comparing two files.
      @return The results of or'ing together all of the comparison flags indicated in
              <code>feqString</code> or {@link #EQ_DEF} if none are specified.
   */
  public static int parseFileTestFlag (String feqString) {
     int iSlash = feqString.indexOf("/");
     if (iSlash < 0) return EQ_DEF; // No flags
     String str = feqString.substring(iSlash+1).replace('/','|');
     return Parser.mask("B,S,H,D,K,T,,,,,L", str, 0); // omits 'M','C' and 'V' used in Data

  }

  /** Computes a hashcode associated with the contents of the BaseFile. This is
      implemented to satisfy the requirement that classes that override
      {@link java.lang.Object#equals(Object)} should also override
      {@link java.lang.Object#hashCode()}.<br>
      <br>
      This method is included for Java 8 compliance only and not intended as added
      functionality. Consequently, in future releases, the value returned by this
      method is subject to change.
      @since 3.5.1
      @return hashcode
   */
  @Override
  public int hashCode () {
    return (int) this.getSize();
  }

  /** Test for equivalence of two BaseFiles. If the input object is not a BaseFile class, returns
      file. This is the same as using <code>this.equals(inobj, {@link #EQ_DEF})</code>.
      @param inobj The file to compare to this one.
      @return <code>true</code> if the given file is equal to this file based
              on the default equality tests, <code>false</code> otherwise.
  */
  @Override
  public boolean equals (Object inobj) {
    return this.equals( inobj, EQ_DEF);
  }

  /** Test for equivalence of two BaseFiles. Used by FEQ logical test.
      @param inobj The file to compare to this one.
      @param flags The equality comparison flags to use. Should be a combination of {@link #EQ_BIN},
                   {@link #EQ_SIZE}, {@link #EQ_HDR}, {@link #EQ_DATA}, {@link #EQ_KEY}, and
                   {@link #EQ_DEF}.
      @return <code>true</code> if the given file is equal to this file based
              on the specified equality tests, <code>false</code> otherwise.
  */
  public boolean equals (Object inobj, int flags) {
    return equalsBF(inobj, flags);
  }

  /** Test for equivalence of two BaseFiles. <br>
      <br>
      Implementation note: <i>Need this method to invoke from DataFile or other
      children.  Invoking the equals method causes a stack overflow.</i>
      @param inobj The file to compare to this one.
      @param flags The equality comparison flags to use (same as equals(Object,int) method).
      @return <code>true</code> if the given file is equal to this file based
              on the specified equality tests, <code>false</code> otherwise.
  */
  public boolean equalsBF (Object inobj, int flags) {
    if (this == inobj) {
      return true;
    }
    if (inobj == null) {
      return false;
    }
    boolean theSame = false;
    if (inobj.getClass() == getClass()) {
      BaseFile bf1 = this;
      BaseFile bf2 = (BaseFile) inobj;
      bf1.open(); bf2.open();
      if (flags == EQ_DEF) flags = EQ_BIN + EQ_SIZE;

      // size equality test (defaults true in case we're only checking the other case)
      boolean sameSize = true;
      if ((flags & EQ_SIZE) != 0) {
        sameSize = (bf1.getSize() == bf2.getSize());
      }

      // binary equality test (defaults true in case we're only checking the other case)
      boolean sameBinary = true;
      if (theSame && ((flags & EQ_BIN) != 0) || ((flags & EQ_LENLIM) != 0)) {
        // Since NeXtMidas 3.7.0 for BUG 2861
        // *NOTE* --- change in DEFAULT behavior!
        // 1. /L modifier allows data comparison to be limited to the size of the smaller file (if applicable)
        //    otherwise, the two files MUST have the same length!
        // 2. If one file has zero length, the files are not equal
        if(bf1.getSize() != 0 && bf2.getSize() != 0) { // both files have a length
          if((flags & EQ_LENLIM) != 0) {
            double size = Math.min(bf1.getSize(),bf2.getSize());
            sameBinary = (compare(bf1,bf2,0.0,size) == 0);
          }
          else if(bf1.getSize() == bf2.getSize()) {
            sameBinary = (compare(bf1,bf2,0.0,bf1.getSize()) == 0);
          }
          else {
            sameBinary = false;
          }
        }
        else if(bf1.getSize() != 0 || bf2.getSize() != 0) { // one file has 0 length
          sameBinary = false;
        }
      }
      theSame = sameSize && sameBinary;
      bf1.close(); bf2.close();
    }
    return theSame;
  }

  /** Compares a segment of data within two files.
      @param bf1    The first file.
      @param bf2    The second file.
      @param offset The offset into the data to begin comparing at (in bytes).
      @param size   The number of bytes to compare.
      @return <code>0</code> if both segments in the files are identical,
              <code>1</code> if the two segments differ.
  */
  public static int compare (BaseFile bf1, BaseFile bf2, double offset, double size) {
    int bytesPer = (int)Math.min((double)BUFFER_SIZE,size);
    byte[] buf1 = new byte[bytesPer];
    byte[] buf2 = new byte[bytesPer];
    bf1.seek(offset); bf2.seek(offset);
    for (double off=0; off<size;) {
      int bytes = (int)Math.min((double)bytesPer,size-off);

      if(readBytes(buf1,bf1,bytes) < 0) return 1;
      if(readBytes(buf2,bf2,bytes) < 0) return 1;

      if (!java.util.Arrays.equals(buf1,buf2)) return 1;
      off += bytes;
    }
    return 0;
  }

  /** Override by ZipFile and TarFile as a way to get the file entries
      @return array of entries for containers (ZIP, JAR, TAR) or null
      @since NeXtMidas 3.3.0
   */
  public Object[] getEntries () {
    return null;
  }

  /** Reads bytes from a base file. Unlike the normal read method this method will keep reading
      until the specified number of bytes are read or until the end of file is reached.
      @param buf   The buffer to read the bytes into.
      @param bf    The file to read from.
      @param bytes The number of bytes to read.
      @return Number of bytes read or -1*(number of bytes read) if End-Of-File reached.
   */
  private static int readBytes(byte[] buf, BaseFile bf, int bytes) {
    boolean eof    = false; // End of file reached
    int     read   = 0;     // Number of bytes read on a given pass
    int     toRead = bytes; // Number of bytes to read

    while((toRead > 0) && !eof) {
      read = bf.read(buf,(bytes-toRead),toRead);
      if(read < 0) {
        eof = true;
      } else if(read == 0) {
        Time.sleep(Shell.getSharedMidasContext().pause);
      } else {
        toRead -= read;
      }
    }

    return (eof)? -(bytes-toRead): bytes;
  }

  /** This will copy the data from one base file to another.
      @param bf1 The source file to read the data from.
      @param bf2 The destination file to write/copy the data to.
      @return Always returns <code>0</code>.
      @deprecated Since NeXtMidas 2.5.0: This method does not work properly with many of the
                  subclasses of {@link BaseFile} (e.g. {@link DataFile}). The
                  {@link FileUtil#copyBaseFile(Midas,Object,Object)} should be used to copy
                  a file.
   */
  @Deprecated public static int copy (BaseFile bf1, BaseFile bf2) {
    byte[]  buf  = new byte[BUFFER_SIZE];
    boolean done = false;

    while(!done) {
      int bytesRead = bf1.read(buf,0,buf.length);

      if(bytesRead == 0) {
        Time.sleep(Shell.getSharedMidasContext().pause);
      } else if(bytesRead < 0) {
        done = true;
      } else {
        bf2.write(buf,0,bytesRead);
      }
    }
    return 0;
  }

  /** Delete (recursively) the file or directory pointed to by the given path. If the path is a
      directory then the contents of that directory are deleted and then the directory is. The
      method returns the total number of files deleted. <br>
      <br>
      <b>Warning: This method kills with extreme prejudice so be sure that you actually want to
                  call this method before you do.</b>
      @param path The path to the file or directory to delete.
      @return The total number of files deleted.
  */
  public static int delete (String path) {
    int ndel = 0;

    File file = new File ( path );
    if ( file.isDirectory() ) {
      String[] list = file.list();
      for (int i = 0; i < list.length; i++) {
        ndel += delete ( "" + new File(path, list[i]) );
      }
      if (file.delete())  ndel++;
    }
    else {
      if (file.delete()) ndel++;
    }

    return ndel;
  }

  /** Temporary utility method to output warning messages.<br>
   * This is NOT public API since in the beginning of a future beta series (e.g. 3.7),
   * we can make sure this.M is never null (TODO).<br>
   * Then, we would just use M.warning(msg) instead.<br>
   * FYI: There are already many places in current code (BaseFile and DataFile) that assumes this.M is not null, e.g.
   *   {@code M.io..., if ((M.debug&Debug.IO)!=0)...}, etc. so it would probably BREAK (with NPE) if this.M is null.
   * @param msg warning message to output
   * @since NeXtMidas 3.5.4 for package use only, see note above for future plans
   */
  @InternalUseOnly("Since NeXtMidas 3.5.4, this will probably be removed in a future beta series")
  void warning (CharSequence msg) {     // package-private since initial version
    Midas midas = Convert.ref2Midas(M); // <-- this NeXtMidas 3.5.0 API never returns null
    midas.warning(msg);
  }

  /** Allows us to filter exceptions better. */
  static class CouldNotFindException extends MidasException { // package-private, used in BaseFile,
    private static final long serialVersionUID = 100271L;
    public CouldNotFindException() {
      super("Could not find");
    }
  } // end inner static class CouldNotFindException

  /**
   * Inserts the specified number of bytes into the file. No effort is made to clear (or zero-fill) the "gap" created by
   * the insert.
   * @param off The offset at which to perform the insert.
   * @param num The number of bytes to insert.
   */
  protected void insertBytes (long off, long num) {
    if (off > io.getLength()) { // subtract 1 from the length, length is not 0 based
      // When we want to add it at the last spot, it comes in as the final byte in the buffer. It can be equal to the
      // length.
      // Add to end of file...
      io.setLength(off + num);
    }
    else {
      // Insert into file...
      // Get the old length
      long oldLength = io.getLength();
      // Get the new length
      long newLength = oldLength + num;
      // Create temp buffer to help shift the data. use the factor to set up the max size of the buffer.
      byte[] buf = new byte[MAX_BUF_SIZE];
      // get the position we are starting at.
      long pos = oldLength;
      // set the new length
      io.setLength(newLength);

      while (pos > off) {
        // To read is calculated based on the size of the temporary buffer.
        // If the amount to move is too large for the buffer we need to increment the shift.
        // read the size of the buffer or the size to shift, whichever is smaller.
        int toRead = (int) Math.min(pos - off, buf.length);
        int numRead = io.read(buf, 0, toRead, pos - toRead);

        if (numRead != toRead) {
          throw new MidasException("Error while trying to insert content into " + getName());
        }
        // shift pos for next read if needed
        pos -= numRead;
        // Insert location needs to be shifted by the num variable. pos + num is he shift
        io.write(buf, 0, numRead, pos + num);
      }
    }
  }

  /**
   * Deletes the specified number of bytes from the file.
   * @param off The offset at which to perform the remove.
   * @param num The number of bytes to remove.
   */
  protected void removeBytes (long off, long num) {
    if (off > io.getLength()) {
      // Past the end of file, nothing to do...
    }
    else if (off + num > io.getLength()) {
        // At the end of file, just trim file size...
        io.setLength(off);
    }
    else {
      // Delete from file...
      // Size before removal
      long oldLength = io.getLength();
      // The size after removing - num is the number of records to remove in bytes.
      long newLength = oldLength - num;
      // create the temp buffer to read data to shift over the removed records.
      byte[] buf = new byte[MAX_BUF_SIZE];
   // The position to start reading in
      long pos = off + num;
      // get all data from pos (the offset plus the number of bytes to remove) to the end of the original.
      while (pos < oldLength) {
        // This will account for large amounts of data. It
        // will max out the buffer size at 32768 so if it has
        // more data to remove it will need to do more loops
        int toRead = (int) Math.min(oldLength - pos, buf.length);
        int numRead = io.read(buf, 0, toRead, pos); // read from the

        if (numRead != toRead) {
          throw new MidasException("Error while trying to remove content from " + getName());
        }

        io.write(buf, 0, numRead, pos - num); // write the data
        pos += numRead;
      }
      // zero out the data removed. 0 out the bytes from
      // newLength to oldLength
      io.write(new byte[MAX_BUF_SIZE], 0, (int) num, newLength);
      // set the length accordingly.
      io.setLength(newLength);
    }
  }
}
