package nxm.sys.lib;

import static nxm.sys.lib.Util.asSortedSet;

import java.io.File;
import java.net.MalformedURLException;
import java.nio.file.Path;
import java.util.Map;
import java.util.SortedSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import nxm.sys.inc.Commandable;
import nxm.sys.inc.DataTypes;
import nxm.sys.inc.Debug;
import nxm.sys.inc.Installation;
import nxm.sys.inc.InternalUseOnly;
import nxm.sys.inc.KeyObjectNames;
import nxm.sys.inc.ListFile;
import nxm.sys.inc.MidasReference;
import nxm.sys.inc.PacketHandler;
import nxm.sys.inc.ProvisionalUseOnly;
import nxm.sys.inc.Units;
import nxm.sys.lib.CoreIO.IOOptions;
import nxm.sys.lib.TimeLine.Format;
import nxm.sys.lib.TimeLine.NotifyInvalidEntry;
import nxm.sys.libm.Tolerance;

/** A class to define and handle a Midas Data file.
    <br>
    WARNING: The read/write/seek/avail/etc. methods are NOT synchronized for performance reasons.
             If reading or writing from multiple threads, caller MUST synchronize on this object.
             The find/open/close methods are synchronized since they generally are NOT called many times.

    <pre>
    EXAMPLES:

    Modifying Type 1000 file using setData

      DataFile fileToUpdate = new DataFile(M, "my_1000_file.tmp");
      fileToUpdate.open(DataFile.INOUT);
      Keywords kw = fileToUpdate.getKeywordsObject();
      kw.setKey("S:TESTKEY", "This is a test");

      // Set Data to different values than original data
      Data newFirstElem = new Data(0.25,0.5);
      Data newSecondElem = new Data(1.25,1.5);
      fileToUpdate.setData(0,newFirstElem);
      fileToUpdate.setData(1,newSecondElem);
      fileToUpdate.close();


    Create Type 3000 file by copying header information from another 3000 file
      DataFile in = new DataFile(M,"orig_3000_file.prm");
      in.open();
      DataFile out = new DataFile(M, "new_3000_file.tmp",in,DataFile.OUTPUT);
      out.open();
      Keywords kw = out.getKeywordsObject();
      kw.setKey("S:3000KEY", "3000 test value");

      out.setSubRecords("(SR2A|2A,SRSL|SL)");
      out.setData (0, "SR2A", "FIRST");
      out.setData (0, "SRSL", 1);
      out.setData (1, "SR2A", "SECOND");
      out.setData (1, "SRSL", 2);
      out.close();
      in.close();
    </pre>

    @author  Jeff Schoen / Jay Hallam / J. Czechowski IV / Neon Ngo
*/
public class DataFile extends BaseFile implements DataFileModSwapInterface, ListFile, Cloneable {
  /** The comma-separated list of properties (to show for dynamic introspection from GQuery and {@link Query}). */
  public static final String propertyList = "Name,Type,Format,Size,Qualifiers,Keywords,Extended";

  /** The comma-separated list of fields for the propagation mask. */
  public static final String propagationMaskList = "Header,Keywords,KeywordMain,KeywordExt,TimeCode,PacketHandler";

  /** Propagate main and adjunct header option                 */ public static final int PM_HEADER        =             0x1;
  /** Propagate keywords (in main and extended header) option  */ public static final int PM_KEYWORDS      =             0x2;
  /** Propagate keywords (in main header) option               */ public static final int PM_KEYWORDSMAIN  =             0x4;
  /** Propagate keywords (in extended header) option           */ public static final int PM_KEYWORDSEXT   =             0x8;
  /** Propagate timecode option                                */ public static final int PM_TIMECODE      =            0x10;
  /** Propagate PacketHandler option                           */ public static final int PM_PACKETHANDLER =            0x20;
  /** Propagate Group option: Default propagation mask is ALL  */ public static final int PM_DEFAULT       = Parser.MASK_ALL;

  /** quadword indicies */
  public static final int Q_RF=0, Q_ALT=1, Q_LAT=2, Q_LON=3,
    Q_AZIM=4, Q_ELEV=5, Q_ROLL=6, Q_YEAR=9, Q_SEC=10, Q_GHA=11;

  /** Byte Block Size */  public static final int BYTE_BLOCK_SIZE = 512;

  /** BLUE Header size */ public static final int HEADER_SIZE     = 512;

  /** BLUE Header version offset   */ public static final int OFF_VERSION   =   0;
  /** BLUE Header headrep offset   */ public static final int OFF_HEADREP   =   4;
  /** BLUE Header datarep offset   */ public static final int OFF_DATAREP   =   8;
  /** BLUE Header detached offset  */ public static final int OFF_DETACHED  =  12;
  /** BLUE Header protected offset */ public static final int OFF_PROTECTED =  16;
  /** BLUE Header pipe offset      */ public static final int OFF_PIPE      =  20;
  /** BLUE Header extstart offset  */ public static final int OFF_EXTSTART  =  24;
  /** BLUE Header extsize offset   */ public static final int OFF_EXTSIZE   =  28;
  /** BLUE Header datastart offset */ public static final int OFF_DATASTART =  32;
  /** BLUE Header datasize offset  */ public static final int OFF_DATASIZE  =  40;
  /** BLUE Header type offset      */ public static final int OFF_TYPE      =  48;
  /** BLUE Header format offset    */ public static final int OFF_FORMAT    =  52;
  /** BLUE Header flagmask offset  */ public static final int OFF_FLAGMASK  =  54;
  /** BLUE Header timecode offset  */ public static final int OFF_TIMECODE  =  56;
  /** BLUE Header inlet offset     */ public static final int OFF_INLET     =  64;
  /** BLUE Header outlets offset   */ public static final int OFF_OUTLETS   =  66;
  /** BLUE Header outmask offset   */ public static final int OFF_OUTMASK   =  68;
  /** BLUE Header pipeloc offset   */ public static final int OFF_PIPELOC   =  72;
  /** BLUE Header pipesize offset  */ public static final int OFF_PIPESIZE  =  76;
  /** BLUE Header inbyte offset    */ public static final int OFF_INBYTE    =  80;
  /** BLUE Header outbyte offset   */ public static final int OFF_OUTBYTE   =  88;
  /** BLUE Header outbytes offset  */ public static final int OFF_OUTBYTES  =  96;
  /** BLUE Header keylgth offset   */ public static final int OFF_KEYLGTH   = 160;
  /** BLUE Header keywords offset  */ public static final int OFF_KEYWORDS  = 164;

  /** Offset of adjunct header in file header (corresponds to xstart) */
  public static final int OFF_ADJUNCT = 256;

  /** Type 1000 2000 adjunct xstart offset */ public static final int ADJUNCT_XSTART_OFF  =  0;
  /** Type 1000 2000 adjunct xdelta offset */ public static final int ADJUNCT_XDELTA_OFF  =  8;
  /** Type 1000 2000 adjunct xunits offset */ public static final int ADJUNCT_XUNITS_OFF  = 16;

  /** Type 2000 adjunct subsize offset */ public static final int ADJUNCT_SUBSIZE_OFF = 20;
  /** Type 2000 adjunct ystart offset  */ public static final int ADJUNCT_YSTART_OFF  = 24;
  /** Type 2000 adjunct ydelta offset  */ public static final int ADJUNCT_YDELTA_OFF  = 32;
  /** Type 2000 adjunct yunits offset  */ public static final int ADJUNCT_YUNITS_OFF  = 40;

  /** type 3000(6000), 4000, 5000 adjunct reclgth offset */ public static final int ADJUNCT_RECLGTH_OFF = 44;

  /** type 3000(6000) adjunct rstart offset  */ public static final int ADJUNCT_RSTART_OFF  =  0;
  /** type 3000(6000) adjunct rdelta offset  */ public static final int ADJUNCT_RDELTA_OFF  =  8;
  /** type 3000(6000) adjunct runits offset  */ public static final int ADJUNCT_RUNITS_OFF  = 16;
  /** type 3000(6000) adjunct subrec offset  */ public static final int ADJUNCT_SUBREC_OFF  = 20;
  /** type 3000(6000) adjunct r2start offset */ public static final int ADJUNCT_R2START_OFF = 24;
  /** type 3000(6000) adjunct r2delta offset */ public static final int ADJUNCT_R2DELTA_OFF = 32;
  /** type 3000(6000) adjunct r2units offset */ public static final int ADJUNCT_R2UNITS_OFF = 40;
  /** type 3000(6000) adjunct substr offset  */ public static final int ADJUNCT_SUBRSTR_OFF = 48;

  /** type 4000 adjunct vrstart offset  */ public static final int ADJUNCT_VRSTART_OFF  =  0;
  /** type 4000 adjunct vrdelta offset  */ public static final int ADJUNCT_VRDELTA_OFF  =  8;
  /** type 4000 adjunct vrunits offset  */ public static final int ADJUNCT_VRUNITS_OFF  = 16;
  /** type 4000 adjunct nrecord offset  */ public static final int ADJUNCT_NRECORD_OFF  = 20;
  /** type 4000 adjunct vr2start offset */ public static final int ADJUNCT_VR2START_OFF = 24;
  /** type 4000 adjunct vr2delta offset */ public static final int ADJUNCT_VR2DELTA_OFF = 32;
  /** type 4000 adjunct vr2units offset */ public static final int ADJUNCT_VR2UNITS_OFF = 40;

  /** type 4000 adjunct tstart offset     */ public static final int ADJUNCT_TSTART_OFF     =   0;
  /** type 4000 adjunct tdelta offset     */ public static final int ADJUNCT_TDELTA_OFF     =   8;
  /** type 4000 adjunct tunits offset     */ public static final int ADJUNCT_TUNITS_OFF     =  16;
  /** type 4000 adjunct components offset */ public static final int ADJUNCT_COMPONENTS_OFF =  20;
  /** type 4000 adjunct t2start offset    */ public static final int ADJUNCT_T2START_OFF    =  24;
  /** type 4000 adjunct t2delta offset    */ public static final int ADJUNCT_T2DELTA_OFF    =  32;
  /** type 4000 adjunct t2units offset    */ public static final int ADJUNCT_T2UNITS_OFF    =  40;
  /** type 4000 adjunct tcomp offset      */ public static final int ADJUNCT_COMP_OFF       =  48;
  /** type 4000 adjunct quadword offset   */ public static final int ADJUNCT_QUADWORD_OFF   = 160;

  /** Header version char array length */ public static final int CHARS_VERSION = 4;
  /** Representation char array length */ public static final int CHARS_REP     = 4;
  /** Format char array length         */ public static final int CHARS_FORMAT  = 2;

  /** Microsecond timeline qualifier */ public static final String QUALIFIER_USEC_TIMELINE     = "USEBLUEUSECTIMELINE";
  /** BLUE TC timeline qualifier     */ public static final String QUALIFIER_TC_TIMELINE       =   "USEBLUETCTIMELINE";
  /** Platinum timeline qualifier    */ public static final String QUALIFIER_PLATINUM_TIMELINE = "USEPLATINUMTIMELINE";

  /** A Blank Type 6000 subrecord. */
  private static final String BLANK_T6_SUBREC = "                        " // name(24)
                                              + "000000000000000000000000" // minval(24)
                                              + "000000000000000000000000" // maxval(24)
                                              + "00000000"+"0001"+"0000"   // offset(8), num_elts(4), units(4)
                                              + "SB"+"000"+"000";          // format(2), uprefix(3), reserved(3)

  private static final String BLUE_VER_ORIG      = "1.0";
          static final String BLUE_VER_DEFAULT   = "1.1";
          static final String BLUE_VER_Z_DATA    = "1.2";
  private static final String BLUE_VER_PLATINUM  = "2.0";
  /** Blue Version numbers are of the form 1.d, where d is a digit */
  private static final Pattern BLUE_VER_PATTERN = Pattern.compile("1.\\d");
  /** Platinum Version numbers are of the form 2.d, where d is a digit */
  private static final Pattern PLATINUM_VER = Pattern.compile("2.\\d");

  /** bytesPerScalar */
  @InternalUseOnly public int bps=1;

  /** bytesPerAtom */
  @InternalUseOnly public int bpa=1;

  /** scalarsPerAtom */
  @InternalUseOnly public int spa=1;

  /** atomsPerElement */
  @InternalUseOnly public int ape=1;

  /** Bytes per element*/
  @InternalUseOnly public double dbpe = 1.0;

  /** Allowed types for this data file. For example, 1000, 2000,... */
  @InternalUseOnly public String types;

  /** Allowed formats for this data file. For example, SB, SF,... */
  @InternalUseOnly public String formats;

  /** Transfer length */
  @InternalUseOnly public int xferLength=-1;

  /** Consume length */
  @InternalUseOnly public int consLength=-1;

  /** Mode part of FORMAT digraph */
  @InternalUseOnly public byte dataMode;

  /** Type part of FORMAT digraph */
  @InternalUseOnly public byte dataType;

  /** Type code class of this data file. For example, 1 for type 1000 files, 2 for type 2000,... */
  @KeyObjectNames({"typeClass"}) // suppress acronym "TC" to disambiguate with {@link #getTimeCode()}
  @InternalUseOnly public int typeClass=1;

  /** Pointer to next DataFile (real-time header changes). */
  private DataFile next; // private since NeXtMidas 3.3.1, it was deprecated in 2.5.0: Use {@link #getNextDataFile()}.
  /** Pointer to previous DataFile (real-time header changes). */
  private DataFile previous;

  /** Default pipe size in bytes */
  @InternalUseOnly public int defPipeSize=131072;

  /** The IOResource for the header (when detached, io=DATA ioh=HEADER) */
  @InternalUseOnly public  IOResource   ioh;
  /** The IOResource used, cast to an instance of PipeResource */
  private PipeResource iop;

  /** Packetized data handler */
  @InternalUseOnly public PacketHandler pkh;

  /** Detached packet meta-data file */
  @InternalUseOnly public DataFile hp;

  /** Time Line information, use {@link #getTimeLineHandler()} or {@link #setTimeLineHandler(TimeLine)} instead */
  @InternalUseOnly public TimeLine timeLine; // TODO: make this protected (but we need getTimeLine() method, otherwise status infile timeline=theTimeline FAILS) BUT then it cause ambiguous match with getTransLength() (TL)
  /** Time of last data {@link #read(Data, int)} when there is a Time Line  */
  private Time time = new Time();

  /** The byte buffer holding the header. */
  @InternalUseOnly public byte[] hb = new byte[HEADER_SIZE];

  /** DataFile keywords */
  @InternalUseOnly public Keywords keywords = new Keywords(this);

  /** DataFile Data buffer */
  @InternalUseOnly public  Data buffer;     // JGS??? only used by fcalc?

  /** TEMPORARY - for testing purposes ONLY!!!!!
   *  Added in 3.8.0 to give more time to evaluate new code that should only be needed
   *  when a timeline entry was added without setting the header time
   */
  @InternalUseOnly public boolean enableNewRestartCode = false;

                                                                              private double       firstbyte;
  /** size in elements of data being written (often wsize==size)           */ private double       wsize;
                                                                              private double       cfsize;
                                                                              private double       cftop;
  /** Have the trimmers been set?                                          */ private boolean      trimSet = false;
  /** The actual trim value (lower)                                        */ private double       trim1;
  /** The actual trim value (upper)                                        */ private double       trim2;
                                                                              private boolean      isStream;
                                                                              private int          listLineWidth = 80;
  /** Used for handling TL= CL= see read(..)                               */ private double       saveOffset;
  /** Used for handling TL= CL= see read(..)                               */ private int          saveLength;
  /** Used for handling TL= CL= see read(..)                               */ private byte         saveBuf[];
  /** Temp buffer used by write(..) for converting byte rep on output      */ private byte         tempBuf[];
  /** Preserve data buffer - JGS??? unused (TODO remove?)                  */ private boolean      preserveDB;

  /** Data representation of the DATA ('I'=IEEE/'E'=EEEI/...)              */ private byte         dataRep;
  /** Data representation of the HEADER (package-private used by Keywords) */         byte         headRep;
  /** Type 6000 Subrecords (local temp buffer)                             */ private StringBuffer t6s;
  /** Long names of Type 3000/5000/6000 subrecords                         */ private String[]     recNames;
  /** Temp buffer used by packD/unpackD with VMS header                    */ private byte[]       tempVms       = new byte[8];
  /** 3.9.4 file is already up versioned to VER 1.2                        */ private boolean      isBlueVersion1dot2  = false;
  /** timeline autocorrection is on for wrap case                          */ private boolean      offset0AutoCorrectIsEnabled;

  /** Since 3.9.3 allow some minimal support for some unsupported data types */
  private boolean         minimalSupportForUnsupportedDataTypes = false;
  /** since 3.9.3 for NXM-3532 to allow propagation of TimeLine keywords without modification */
  private boolean         sameTimelineKeywords = false;
  /** since 4.1.0 by default, automatically convert PLATINUM files, including keywords, to BLUE files */
  private static volatile boolean  autoConvertPlatinumDefault  = true;
  /** since 4.1.0 by default, automatically convert PLATINUM files, including keywords, to BLUE files */
  private volatile boolean  autoConvertPlatinum  = autoConvertPlatinumDefault;
  /** Is this a PLATINUM file we are allowing to be converted */
  private boolean         allowablePlatinum = false;
  /** Raster displays should have IF axis and RF readout, based on SBT and JTIME keywords */
  private boolean         supportTuneAdjustmentsOnRaster = false;
  /** Tune Adjustment Data to support accurate Raster RF readouts */
  private TuneAdjust      tuneAdjustments = null;

  /** save off source file TimeLine.format in order to propagate in pipe */ Format sourceTimeLineFormat;
  /** save off source file Timeline tolr in order to propagate in pipe   */ double sourceTimeLineTolr;

  /** The magic numbers at the start of a valid BLUE file (see {@code isBlueFileMagic(..)})
      @since 3.9.0 for NXM-3264
   */
  private static final SortedSet<String> BLUE_MAGIC = asSortedSet("BLUEIEEEIEEE",
                                                                  "BLUEIEEEEEEI",
                                                                  "BLUEEEEIIEEE",
                                                                  "BLUEEEEIEEEI");

  /** This is the message returned by {@code getPartialBlueFileError(..)} when an exceptional
   *  condition is detected (see {@code getPartialBlueFileError(..)} for details). The exact
   *  wording within this may change between releases.
   *  @since NeXtMidas 3.9.0
   */
  public static final String BLUE_FILE_READ_ERROR = "Error reading BLUE file";

  /** Scalar values separator for listing type 1000 and default elements, defaults to '|'.
   *  @since NeXtMidas 4.1.2
   */
  private static String scalarSeparator = "|";

  /** Data elements separator for listing type 1000 and default elements, defaults to ','.
   *  @since NeXtMidas 4.1.2
   */
  private static String elementSeparator = ",";

  /** The default constructor for the class, just sets the default flags. Note that unlike
      {@link BaseFile#DEF_FLAGS} the {@link #TRUNCATE} flag is NOT set by default.
   */
  public DataFile () {
    flags = DEF_FLAGS & ~TRUNCATE; // Can not add MAINTAINCASE here, as would break Macro usage
  }

  /** Creates a new instance and then initializes it.
      <br><b>deprecated since NeXtMidas 3.5.0 use {@link #DataFile(MidasReference, Object)}</b>
      @param ref      The reference to use.
      @param filename The name of the file to use.
   */
  @InternalUseOnly("deprecated Since NeXtMidas 3.5.0") // TODO: add deprecated annotation in 3.7.1+ (still used by ICE/lib/ICEExtract.java)
  public DataFile (Object ref, Object filename) {
    this(Convert.ref2MidasReference(ref), filename);
  }

  /** Creates a new instance and then initializes it.
      <br>
      This is the same as:
      <pre>
        DataFile df = new DataFile();
        df.init(ref,filename);
      </pre>
      @param ref      The reference to use.
      @param filename The name of the file to use.
      @see #DataFile()
      @see #init(MidasReference,Object)
      @since NeXtMidas 3.5.0
   */
  public DataFile (MidasReference ref, Object filename) {
    this();
    int _flags = 0;
    if (ref!=null) { // NXM-4097
      Midas refM = ref.getMidas();
      if ((refM!=null) && refM.isJavaBehavior()) {
        _flags = MAINTAINCASE;
      }
    }

    setFlags(_flags);
    if (Midas.isJavaVsMidasDebug()) Midas.log("#JvsM DataFile 2-arg constructor filename:"
        +filename+" flags:"+getFlagsString(this.flags),Midas.INFO);
    init(ref,filename);
  }

  /** Creates a new instance and then initializes it.
      @param ref        The reference to use
      @param filename   The name of the file to use
      @param flags      The flags for the DataFile object
      @since NeXtMidas 3.9.1
   */
  public DataFile (MidasReference ref, Object filename, int flags) {
    this();
    setFlags(flags); // Since 3.9.1 set the flags for file creation
    if (Midas.isJavaVsMidasDebug()) Midas.log("#JvsM DataFile 3-arg constructor filename:"
        +filename+" flags:"+getFlagsString(flags), Midas.INFO);
    init(ref,filename, "", "", 0); // NXM-4097 needs to go directly to this init due to Midas vs Java flag handling
  }

  /** Creates a DataFile and then initializes it by calling <code>init(..)</code> with type/format checking.
      <br><b>deprecated since NeXtMidas 3.5.0 use {@link #DataFile(MidasReference, Object, String, String, int)}</b>
      @param ref     The reference to use.
      @param name    The name of the file (can be a {@link FileName} or a {@link String}).
      @param types   The types for the data in the file.
      @param formats The formats for the data in the file.
      @param flags   Any flags to open the file with (e.g.
                     {@link #OUTPUT}, {@link #APPEND}).
   */
  @InternalUseOnly("deprecated Since NeXtMidas 3.5.0") // TODO: add deprecated annotation in 3.7.1+ (still used by ICE/lib/Archiver.java)
  public DataFile (Object ref,Object name,String types,String formats,int flags) {
    this(Convert.ref2MidasReference(ref), name, types, formats, flags);
  }

  /** Creates a DataFile and then initializes it by calling <code>init(..)</code>
      with type/format checking. This is a convenience method equal to
      <code>DataFile(name,types,formats,flags,-1.0,null)</code>.
      @param ref     The reference to use.
      @param name    The name of the file (can be a {@link FileName} or a {@link String}).
      @param types   The types for the data in the file.
      @param formats The formats for the data in the file.
      @param flags   Any flags to open the file with (e.g.
                     {@link #OUTPUT}, {@link #APPEND}).
      @see #DataFile(MidasReference,Object,String,String,int,double,DataFile)
      @since NeXtMidas 3.5.0
  */
  public DataFile (MidasReference ref,Object name,String types,String formats,int flags) {
    this(ref,name,types,formats,flags,-1.0,null);
  }

  /** Creates a DataFile for output based on input DataFile and then initializes it by calling <code>init(..)</code>.
      <br><b>deprecated since NeXtMidas 3.5.0 use {@link #DataFile(MidasReference, Object, DataFile, int)}</b>
      @param ref     The reference to use.
      @param name    The name of the file (can be a {@link FileName} or a {@link String}).
      @param hcbin   An existing file to copy the header from.
      @param flags   Any additional flags to open the file with ({@link #APPEND}
                     is automatically included).
   */
  @InternalUseOnly("deprecated Since NeXtMidas 3.5.0") // TODO: add deprecated annotation in 3.7.1+ (still used by ICE/lib/Archiver.java)
  public DataFile (Object ref, Object name, DataFile hcbin, int flags) {
    this(Convert.ref2MidasReference(ref), name, hcbin, flags);
  }

  /** Creates a DataFile for output based on input DataFile and then initializes
      it by calling <code>init(..)</code>. This is a convenience method equal to
      <code>DataFile(name,"","",flags|OUTPUT,-1.0,hcbin)</code>.
      @param ref     The reference to use.
      @param name    The name of the file (can be a {@link FileName} or a {@link String}).
      @param hcbin   An existing file to copy the header from.
      @param flags   Any additional flags to open the file with ({@link #APPEND}
                     is automatically included).
      @see #DataFile(MidasReference,Object,String,String,int,double,DataFile)
      @since NeXtMidas 3.5.0
  */
  public DataFile (MidasReference ref, Object name, DataFile hcbin, int flags) {
    this(ref, name, "", "", flags|OUTPUT, -1.0, hcbin);
  }

  /** Creates a DataFile for input/output and then initializes it by calling <code>init(..)</code>.
      deprecated since NeXtMidas 3.5.0 use {@link #DataFile(MidasReference, Object, String, String, int, double, DataFile)}
      @param ref     The reference to use.
      @param name    The name of the file (can be a {@link FileName} or a {@link String}).
      @param types   The type for the data in the file (e.g. 1000, 2000, 3001, etc.).
      @param formats The format for the data in the file (e.g. "SF", CD", "VL", etc.)
      @param flags   Any flags to open the file with (e.g. {@link #OUTPUT}, {@link #APPEND}).
      @param size    The size of the file (this is ignored if it is not greater than zero).
      @param hcbin   An existing file to propagate/copy header information from (if not null).
   */
  @InternalUseOnly("deprecated Since NeXtMidas 3.5.0") // TODO: add deprecated annotation in 3.7+
  public DataFile (Object ref, Object name, String types, String formats,
                   int flags, double size, DataFile hcbin) {
    this(Convert.ref2MidasReference(ref), name, types, formats, flags, size, hcbin);
  }

  /** Creates a DataFile for input/output and then initializes it by calling
      <code>init(..)</code>.
      @param ref     The reference to use.
      @param name    The name of the file (can be a {@link FileName} or a {@link String}).
      @param types   The type for the data in the file (e.g. 1000, 2000, 3001, etc.).
      @param formats The format for the data in the file (e.g. "SF", CD", "VL", etc.)
      @param flags   Any flags to open the file with (e.g. {@link #OUTPUT}, {@link #APPEND}).
      @param size    The size of the file (this is ignored if it is not greater than zero).
      @param hcbin   An existing file to propagate/copy header information from (if not null).
      @see DataTypes
      @see DataFile#init(MidasReference,Object,DataFile,int)
      @see DataFile#setSize(double)
      @since NeXtMidas 3.5.0
  */
  public DataFile (MidasReference ref, Object name, String types, String formats,
      int flags, double size, DataFile hcbin) {
    this();
    if (Midas.isJavaVsMidasDebug()) Midas.log("DataFile constructor 4 Arg filename:"+
        filename+" flags:"+getFlagsString(flags), Midas.INFO);
//Bad    setFlags(flags); // NXM-4097 Since 4.1.4 set the flags for file creation
    init(ref,name,types,formats,flags,size,hcbin);
  }

  /** Returns BLUE class version
      @return BLUE class version
   */
  public String getClassVersion() {
    return "BLUE";
  }

  /** Initializes data main header fields to defaults */
  public void initMain () {
    setVersion("BLUE");
    setHeadRep(Shell.osrep);
    setDataRep(Shell.osrep);
    setDataStart(hb.length);
    setDataSize(0.0);
    setTimeCode(0.0);
    setFormat("SF");
    setType(1000);
    setXStart(0.0);
    setXDelta(1.0);
    setYStart(0.0);
    setYDelta(1.0);
    setInByte(0.0);
    for (int i=0; i<=8; i++) setOutByte(i,0.0);
  }

  /** Initialized data header fields.
      @param ref     The reference to use.
      @param name    The name of the file (can be a {@link FileName} or a {@link String}).
      @param types   The type for the data in the file (e.g. 1000, 2000, 3001, etc.).
      @param formats The format for the data in the file (e.g. "SF", CD", "VL", etc.)
      @param flags   Any flags to open the file with (e.g. {@link #OUTPUT}, {@link #APPEND}).
      @param size    The size of the file (this is ignored if it is not greater than zero).
      @param hcbin   An existing file to propagate/copy header information from (if not null).
      @since NeXtMidas 3.5.0
  */
  public void init (MidasReference ref, Object name, String types, String formats,
      int flags, double size, DataFile hcbin) {

    if (hcbin!=null) {
      init(ref,name,hcbin,flags|OUTPUT);
      // BUG 2106: user specified a file type and format, so that should override propagation from hcbin
      if (types!=null && types.length()==4) { this.types = types; setType(Convert.s2l(types)); }
      if (formats!=null && formats.length()==2) { this.formats = formats; setFormat(formats); }
    }
    else {
      init(ref,name,types,formats,flags);
    }

    if (size>0) setSize(size);
  }

  /** Initializes data header fields.
      <br><b>Deprecated Since NeXtMidas 3.7.0 use {@link #init(MidasReference, Object, String, String, int)}</b>
      @param ref      Reference to use when opening the file.
      @param filename The file name.
      @param types    The list of valid types to use when opening a file.
      @param formats  The list of valid formats to use when opening a file.
      @param flags    File flags (0=Not specified at this time). This ONLY adds to the
                      flags mask (i.e. IT DOES NOT remove any bits in the mask).
   */
  @InternalUseOnly("deprecated Since NeXtMidas 3.5.0") // TODO: add deprecated annotation in 3.7.1+
  public void init (Object ref, Object filename, String types, String formats, int flags) {
    // used in 2 places in DataFile - init(Object, ...) and used by ICE/lib/Archiver.java
    init(Convert.ref2MidasReference(ref), filename, types, formats, flags);
  }

  /** Initializes data header fields. There are two ways of using <code>types</code> and <code>formats</code>:
      <pre>
        INPUT FILE:
          - types is a list of valid input file types (e.g. "1000,2000"), if the
            type of the file does not match one of the types, {@link #open()} errors out.
              - Use "" to allow all file types
              - Use "-[type]" to disallow a specific type
          - formats is a list of valid input file data formats (e.g. "SF,SD"), if the
            data format of the file does not match one of the formats, {@link #open()}
            errors out.
              - Use "" to allow all data formats
              - Use "-[format]" to disallow a specific format (e.g. "-SF")
              - Use "#x" to allow any format where type is t (e.g. "#F" accepts "SF","CF",...)
              - Use "n*" to allow any format where the mode is n (e.g. "S*" accepts "SF","SB",...)

         OUTPUT FILE:
           - If types is a single file type (e.g. "3000"), {@link #setType(int)} is called
             with that type.
           - If formats is a single numeric format (e.g. "SF"), {@link #setFormat(String)} is
             called with that format.
      </pre>
      Note that <code>init(ref,filename,"","",0)</code> and <code>init(ref,filename)</code> are functionally
      identical. <br>
      Since 2.9.1: If the ENV.IOOPTIONS has the TruncateBLUE mask flag enabled, then TRUNCATE will
                   be added to the flags.
      @param ref      Reference to use when opening the file (usually an instance of {@link Midas} or
                      {@link Primitive}).
      @param filename The file name.
      @param types    The list of valid types to use when opening a file.
      @param formats  The list of valid formats to use when opening a file.
      @param flags    File flags (0=Not specified at this time). This ONLY adds to the
                      flags mask (i.e. IT DOES NOT remove any bits in the mask).
      @since NeXtMidas 3.5.0 to replace init(Object ref,...)
   */
  public void init (MidasReference ref, Object filename, String types, String formats, int flags) {
    initMain(); // TODO: should move this line after the super.init(..) line?
    M = Convert.ref2Midas(ref);
    super.init(ref,M,filename);
    if (Midas.isJavaVsMidasDebug()) Midas.log("#JvsM DataFile init that calls BaseFile init filename:"
        +filename+" flags:"+getFlagsString(flags), Midas.INFO);
    if ((M != null) && (M.io != null)) {

      if (M.io.isOptionSet(CoreIO.IOOptions.TruncateBLUE)) {
        flags |= TRUNCATE; // Since 2.9.1 for BUG 2175 Should use TRUNCATE by default (NTN: via ENV setting for now)
      }
      if (M.io.isOptionSet(CoreIO.IOOptions.MinimalSupportForUnsupportedDataTypes)) {
        // add option for minimal support of unsigned types in record data - since 3.9.3 for NXM-3021
        minimalSupportForUnsupportedDataTypes = true;
      }
      if (M.io.isOptionSet(CoreIO.IOOptions.SameTimelineKeywords)) {
        // add option for just leaving timeline keywords alone .. treat like other keywords - since 3.9.3
        sameTimelineKeywords = true;
      }
      offset0AutoCorrectIsEnabled =
          !M.io.isOptionSet(CoreIO.IOOptions.DisableTimeLineAutoCorrectNoOffset0Entry) &&
          !M.io.isOptionSet(CoreIO.IOOptions.SameTimelineKeywords);
    }
    setFlags(flags);
    this.types = types;
    this.formats = formats;
    if (formats.length()==2) setFormat(formats);
    if (types.length()==4) setType(Convert.s2l(types));
    if (cmd!=null) defPipeSize = cmd.args.getL("/PS",defPipeSize);
  }

  /** same as <code>init(ref, filename, "", "", 0)</code>.
      <br><b>deprecated Since NeXtMidas 3.5.0 use {@link #init(MidasReference, Object)}</b>
      @see #init(MidasReference, Object, String, String, int)
   */
  @InternalUseOnly("deprecated Since NeXtMidas 3.5.0") // TODO: add deprecated annotation in 3.7+
  @Override
  public void init (Object ref, Object filename) { // (only used by PlotFileWrapper.init(Object,Object)
    init(Convert.ref2MidasReference(ref), filename);
  }

  /** same as <code>init(ref, filename, "", "", 0)</code>.
      @see #init(MidasReference, Object, String, String, int)
      @since NeXtMidas 3.5.0
   */
  @Override
  public void init (MidasReference ref, Object filename) {
    M = Convert.ref2Midas(ref);
    if (ref!=null) {
      if ((M!=null) && M.isJavaBehavior()) { // NXM-4097 base MAINTAINCASE on Java vs Macro usage
        flags |= MAINTAINCASE;
      }
    }
    init(ref, filename, "", "", 0);
  }

  /// Initializes data header fields of an {@link #OUTPUT} file by propagating header/keyword information

  /** Initializes data header fields of an {@link #OUTPUT} file by propagating header and
      keywords (main + extended) information from the given <code>HCB</code>.
      @param ref   Reference to use when opening the file for output.
      @param fname The file name (either a {@link FileName} or a {@link String}).
      @param hcb   The file to propagate header information from (can not be null). In the case
                   of an input file, the file must be opened first.
      @param flags File flags (0=Not specified at this time). <i>(Note that {@link #OUTPUT} is
                   automatically included when this method is called.)</i>
      @since NeXtMidas 3.7.0 to replace init (Object ref, Object filename, DataFile hcb, int flags)
   */
  public void init (MidasReference ref, Object fname, DataFile hcb, int flags) {
    init(ref, fname, "", "", flags|OUTPUT);
    propagate(hcb, PM_DEFAULT);
  }

  /** Initializes data header fields of an {@link #OUTPUT} file by propagating header and
      keywords (main + extended) information from the given <code>HCB</code>.
      <br><b>deprecated Since NeXtMidas 3.7.0 use {@link #init(MidasReference, Object, DataFile, int)}</b>
      @param ref      Reference to use when opening the file (usually an instance of {@link Midas} or {@link Primitive}).
      @param filename The file name.
      @param hcb      The file to propagate header information from (can not be null). In the case
                      of an input file, the file must be opened first otherwise
      @param flags    File flags (0=Not specified at this time). <i>(Note that {@link #OUTPUT} is
                      automatically included when this method is called.)</i>
   */
  // TODO: add @Deprecated in 3.7.1+: use above method that takes in MidasReference (no longer used in baseline - still used by ICE/lib/Archive.java
  public void init (Object ref, Object filename, DataFile hcb, int flags) {
    init (ref, filename, "", "", flags|OUTPUT);
    propagate(hcb, PM_DEFAULT);
  }

  /** Initializes data header fields of an {@link #OUTPUT} file by propagating header information
      from the given header block. <b>This is a special-purpose method for advanced users only, in
      most cases you want to use {@link #init(MidasReference,Object,DataFile,int)}.</b>
      @param ref      Reference to use when opening the file.
      @param filename The file name.
      @param hb       The header block.
      @param flags    File flags (0=Not specified at this time). <i>(Note that {@link #OUTPUT} is
                      automatically included when this method is called.)</i>
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - has alway been for internal use by Rmif.java which is in a different package")
  public void init (Object ref, Object filename, byte[] hb, int flags) { // should only be used by Rmif.procPacket(..)
    init (ref, filename, "", "", flags|OUTPUT);
    copyHeader (hb);
    setInternals();
  }

  /** Propagate fields (propagation mask) from given <code>HCB</code> (DataFile) to this one.
      This is very similar to M$PROPAGATE but with a propagation mask option to specify
      what is desired to be propagated.
      @param hcb  The file to propagate information from (can not be null). In the case
                  of an input file, the file should be opened first otherwise only
                  programmatically set information will get propagated.
      @param propmaskStr Propagation mask of fields ('|' separated) to propagate from hcb to
                         this file. See {@link #propagationMaskList} for possible mask values.
                         If null or empty, will default to "ALL"
      @since NeXtMidas 2.8.0
   */
  public void propagate (DataFile hcb, String propmaskStr) {
    int propmask = Parser.mask(propagationMaskList, propmaskStr, PM_DEFAULT);
    propagate(hcb, propmask);
  }

  /** Propagate fields (propagation mask) from given <code>HCB</code> (DataFile) to this one.
      This is very similar to M$PROPAGATE but with a propagation mask option to specify
      the fields to be propagated.
      It is not an error for <code>this.isOpen</code> to be <code>false</code>, as may happen during a call to
      {@link Args#getDataFile(String,DataFile,int)}.
      @param hcb  The file to propagate information from (can not be null). In the case
                  of an input file, the file should be opened first otherwise only
                  programmatically set information will get propagated.
      @param propmask Propagation mask of fields to propagate from hcb to this file.
                      See {@link #propagationMaskList} for possible mask values.
      @since NeXtMidas 2.8.0, most of the code was moved from init(Object, Object, DataFile, int) to here
   */
  public void propagate (DataFile hcb, int propmask) {
    // save off timeline format and tolr from source file NXM-3639 3.9.4
    if (hcb.timeLine!=null) {
      this.sourceTimeLineFormat = hcb.timeLine.format;
      this.sourceTimeLineTolr = hcb.timeLine.getTolerance();
    }
    if (hcb.sourceTimeLineFormat != null ) {
      this.sourceTimeLineFormat = hcb.sourceTimeLineFormat;
      this.sourceTimeLineTolr = hcb.sourceTimeLineTolr;
    }

    if (propmask==PM_KEYWORDSEXT) { // need to handle this as a special case so as not to disrupt
      propagateExtKeywords(hcb);    // the following code
      return;
    }
    boolean propHeader     = (propmask&PM_HEADER)!=0;                     // is propagating header?
    boolean propMainKeys   = (propmask&(PM_KEYWORDS|PM_KEYWORDSMAIN))!=0; // is propagating keywords in main header?
    boolean propKeywords   = (propmask&(PM_KEYWORDS|PM_KEYWORDSEXT))!=0;  // is propagating keywords in extended header?
    boolean propNoVolatile = M.io.isOptionSet(CoreIO.IOOptions.DoNotPropagateVolatileKeywords);          // do not propagate "volatile" extended header keywords
    boolean propTimecode   = (propmask&PM_TIMECODE)!=0;                   // is propagating timecode in main header?
    String inFileName      = hcb.getFileName().getFullName();             // input file name
    boolean inFnNotEmpty   = inFileName.length() > 0;
    boolean isSameFN       = inFileName.equals(getFileName().getFullName()); // is in HCB and this the same file? (cannot use URL, since it could be slightly different)

    if (!hcb.isOpen  &&   // BUG 2103 - print warning when input file has not been opened to propagate headers
        inFnNotEmpty &&   //            AND input file name is not empty
        !isSameFN) {      // BUG 2144 - AND if not the same file name (for when pipe restarts)
      M.warning("Cannot propagate 'ALL' header/keyword to ["+getURL()+"] from UNOPENED file ["+hcb.getURL()+"]");
    }
    if (propHeader) {                     // propagate header
      copyHeader (hcb.hb);
      headRep = hcb.headRep;              // copy over header rep
    }
    if (propMainKeys && !propHeader) {    // propagating keywords in main and if NOT propagating header
      setKeywords(hcb.getKeywords());     // then have to copy over keywords in main header
    }
    if (propKeywords) {                   // propagate keyword in main and extended header
      Keywords kw = Keywords.convertedCopyOfKeywords(hcb.keywords, hcb.headRep, Shell.rep); // NXM-3633 init with copy with right rep
      keywords.init(kw.buf, kw.size, Shell.rep); // for type 6000, need to propagate keywords from extended header before calling setInternals()
      boolean forceQual = Convert.o2z(hcb.getQualifier("FORCE"));
      if (autoConvertPlatinum && PLATINUM_VER.matcher(hcb.getBlueVer()).matches() && !forceQual) {
        M.info("Performing auto-conversion Platinum to Blue in keyword propagation from file:"+hcb.filename+" to:"+filename);
        Table plat2blueTable = Table.fromTableFile(M, "nxm.sys.dat.kwconv_plat2blue.tbl");
        plat2blueTable.put("_WARN_MISSING_", true);
        FileUtil.convertKeywordsInPlace(M, hcb, keywords, plat2blueTable);
      }
      if (propNoVolatile) keywords.deleteVolatileKeywords(); // since 3.9.1, get rid of keywords with VOLATILE_KW_ prefix
    }
    if (propHeader) {                     // propagate header (continued)
      convertHeaderRep(Shell.osrep);      // convert header rep to OS rep (after copying Keywords)
      setDataRep(Shell.osrep);            // reset the following header fields
      setProtected(false);
      setDetached(0);
      setPipeId(0);
      setDataStart(hb.length);
      setInternals();                     // this should be called after keywords.init(..) for type 6000 files
      size = hcb.size;
    }
    if (propTimecode && !propHeader) {    // propagate timecode and if not propagating header
      setTimeCode(hcb.getTimeCode());     // then have to explicitly set timecode
    }
    if ((propmask&PM_PACKETHANDLER)!=0) { // propagate PacketHandler
      pkh = hcb.pkh;
    }
  }

  /** Propagate extended header keywords without also propagating the main header.
      This is a reimplementation of {@link Keywords#init(Keywords)}, handled as a special case of
      {@link #propagate(DataFile,int)}, to ensure that if the two DataFiles do not have the same
      representation (EEEI/IEEE), the extended header keywords are still properly converted.
      @param dfSrc The file to propagate extended keywords from (can not be null). In the case
                   of an input file, the file should be opened first otherwise only
                   programmatically set information will get propagated.
      @since NeXtMidas 3.7.1, particularly for use by XBC
   */
  private void propagateExtKeywords (DataFile dfSrc) {
    DataFile dfDest = this;
    String fnSrc = dfSrc.getFileName().getFullName(); // source file name
    String fnDest = dfDest.getFileName().getFullName(); // destination file name
    if (!dfSrc.isOpen     &&     // BUG 2103 - print warning when input file has not been opened to propagate headers
         fnSrc.length()>0 &&     //            AND input file name is not empty
        !fnSrc.equals(fnDest)) { // BUG 2144 - AND if not the same file name (for when pipe restarts)
      M.warning("Cannot propagate keywords to ["+getURL()+"] from UNOPENED file ["+dfSrc.getURL()+"]");
    }

    Keywords kwSrc = dfSrc.keywords;
    Keywords kw = new Keywords();
    if (kwSrc.size>0) {
      kw.size = kwSrc.size;
      kw.buf = new byte[kwSrc.buf.length];
      System.arraycopy (kwSrc.buf,0, kw.buf,0, kwSrc.size); // copy keyword buffer (in extended header)
      byte repSrc = (byte)dfSrc.getHeadRep().charAt(0);
      byte repDest = (byte)dfDest.getHeadRep().charAt(0);
      if (repSrc!=repDest) {
        kw.convertExtRep(repSrc, repDest);
      }
    }
    Keywords kwDest = dfDest.keywords;
    kwDest.size = kw.size;
    kwDest.buf = kw.buf;
    kwDest.clearScope();

    // If we reached this point via a call to NmJackets#propagateExtKeywords(long,int,int), t6s will
    // not be null and so #setInternals() will not accomplish this one thing we need it to do:
    if (typeClass == 6) t6s = readT6def(false);
  }

  /** Copies the specified header to that of this data file. <b>This is a special-purpose method for
      advanced users only, in most cases you want to use {@link #init(MidasReference,Object,DataFile,int)}.</b>
      This uses a byte-for-byte copy and does not do any conversions.
      @param buf The new header to use.
   */
  public void copyHeader (byte[] buf) {
    int bytes = buf.length;
    extendHeader(bytes);
    System.arraycopy (buf,0, hb,0, bytes);
  }

  @Override
  public synchronized boolean find (int dir) {
// TODO: call super.find(dir); and override necessary stuff afterwards (for 3.7+ beta series)
    // Since 3.1.1: This method was made synchronized so that when multiple threads
    //              find/open this DataFile, they don't step over each other for BUG 2457.
    //  Q for Jeff: Why are this.io and this.ioh ALWAYS REPLACED when find(.) is called?
    IOResource oldio = io, oldioh = ioh;
    io = ioh = M.io.findResource(getFileName(), dir, "", aux, (flags & DONOTADDEXTENSION)!=0);
    if (isPipe()) iop=(PipeResource)io;
    else { // 2011-03-16 NTN: only close old IO only if they are NOT pipes
      if (oldio  != null) oldio.close();
      if (oldioh != null && oldioh != oldio) oldioh.close();
    }
    return io.exists() && !io.isContainer();
  }

  @Override
  public synchronized boolean open () {
    // Since 3.1.1: This method was made synchronized so that when multiple threads
    //              find/open this DataFile, they don't step over each other for BUG 2457.
    isOpen = false;
    wsize = 0.0;

    FileName fn = getFileName();
    if ((flags&INOUT) != 0) { }
    else if (size>0.0) flags |= OUTPUT;
    else flags |= INPUT;
    if (tag.length()==0 || tag.equals("NULL") || fn.getRoot().equalsIgnoreCase("null")) {
      if ((flags&OPTIONAL)!=0) return isOpen;
      if (tag.length()!=0 && (flags&OUTPUT)!=0) return isOpen;
      M.error("No FileName specified");
    }

    //*** NOTE: start BaseFile matches this block of code
    if ((flags&APPEND) != 0) {
      isInput  = ((flags&INPUT)!=0) || exists();
      isOutput = true;
    } else {
      isInput  = (flags&INPUT)  != 0;
      isOutput = (flags&OUTPUT) != 0;
    }
    int dir = getDir(isInput,isOutput,false);
    isFound = find(dir);
    // NOTE: end BaseFile matches this block of code

    String url = getURL();
    if ((M.debug&Debug.IO)!=0) M.info("OpenDF tag="+tag+" Path="+url);

    // Unlike BaseFile, always allow a wait on a pipe (even if WAIT is not set).
    if (isInput && !isFound && cmd!=null && cmd.isPiped && (((flags&WAIT)!=0) || isPipe())) {
      waitForFile(); // Waits for an input file to be created
      if (!isFound && (flags&NOABORT)==0 ) { close(); return isOpen; } // wait aborted...this step added in 3.3.0 for Bug 2686
    }
    dir = getDir(isInput,isOutput,isFound); // Update dir based on isFound

    if (isPipe()) {
      setDataStart(0); setPipeId(0);
      int ps = Convert.o2l(fn.getQualifier("PS"),M);
      if (ps>0) defPipeSize=ps;
      iop.df=this; iop.pipeSize=defPipeSize;
    }

    try {
      if (isOutput && !isInput) {
        setBlueVerIO();
        setSize(size);
        // assume it could be attached or detached
        io.setLength( (long)(getDataStart()+getDataSize()) );
      }

      // since 2.2.0: Ignore/Override protection flag in header for FILE UNPROTECT (Bug 473)
      boolean noProtect = !Convert.o2z(fn.getQualifier("PROT",true));

      if (isInput && isOutput && isFound) {
        if (!io.open(IOResource.DIR_INOUT)) throw new MidasException("Could not open "+url+" for input/output");
        readHeader();
        if (getProtected() && !noProtect) throw new MidasException("Output is protected");
      }
      else if (isInput && isFound) {
        if (!io.open(IOResource.DIR_INPUT)) throw new MidasException("Could not open "+url+" for input");
        readHeader();
      }
      else if (isInput) {
        if ((flags&NOABORT)!=0) return isOpen;
        throw new CouldNotFindException();
      }
      else if (isOutput && isFound) {
        // line below changed to open IOResource using dir for Bug 1166 TRUNCATE flag not handled properly
        // if (!io.open(IOResource.DIR_INOUT)) throw new MidasException("Could not open "+url+" for output");
        if (!io.open(dir)) throw new MidasException("Could not open "+url+" for output");
        byte[] hbt = new byte[HEADER_SIZE];
        readHeader(hbt);  // read header into temp buffer
        if (!((flags&TRUNCATE) !=0) && !Convert.unpackS(hbt,0,4).equals(getClassVersion())) { // is it a BLUE file?
          // Check to see if the file has any data in it.  Since 3.9.3
          String localFileName = "";
          try {
            localFileName = BaseFile.getFileNameFor(M, getFileName().toString()).toLocalFileName();
          } catch (MalformedURLException e) {
            M.warning(e); // Can this even be thrown? The file has been found at this point.
          }
          File f = new File(localFileName);
          // Throw warning for non-BLUE file that has data in it.
          if (f.length() != 0) M.warning("DataFile: Overwriting non-"+getClassVersion()+" file: "+url);  // warn for BUG 1974 - Since 3.9.3 - added check to see if the file is empty before printing the warning.
        }
        else if ((hbt[16]+hbt[19]!=0) && !noProtect && getClassVersion().equals("BLUE")) {             // is it protected?
          throw new MidasException("Output is protected");         // only check protected fields for BLUE file
        }
        else if (StringUtil.isEmpty(FileName.getExt(url)) && io.isFile() && io.getType()!=IOResource.RAM) {
          // warn for BUG 1974 (but not for RamFile since it does not have extensions (dot is replace with a dash))
          M.warning("DataFile: Overwriting extensionless BLUE file: "+url);
        }
        // If this is a RESTART, set TIMELINE based on previous DataFile's timeline
        // Since 3.7.2 for NXM-3040
        // Out of caution, this was switched to being generally disabled in 3.8.0, as this was needed only for a
        // special case where the first timeline value was set, but the starttime in the header was not (ex: waveform)
        if (enableNewRestartCode && (previous !=null)) {
          Time stopTime  = previous.getTimeAt(previous.seek(), null);
          Time startTime = stopTime.addSec(previous.getDelta());

          newTimeLineFromDF(previous);
          // Currently not notifying about invalid TimeLine entries created during restarts,
          // until those issues have been resolved NXM-3040
          setTimeAt(0.0, startTime, NotifyInvalidEntry.WARNING);
        }
        if ((flags&APPEND)!=0 && getClassVersion().equals("BLUE")) {
          if (hb[52]!=hbt[52]) throw new MidasException("Cannot APPEND with formatMode="+Convert.unpackS(hbt,52,1));
          if (hb[53]!=hbt[53]) throw new MidasException("Cannot APPEND with formatType="+Convert.unpackS(hbt,53,1));
          readHeader();   // read header into internal buffer
          setReps();
          readKeywords(getBlueVer(), getBlueIO());
        }
        else writeHeader();
      }
      else if (isOutput) { // for output ONLY
        // line below changed to open IOResource using dir for Bug 1166 TRUNCATE flag not handled properly
        if (!io.open(dir)) throw new MidasException("Could not open "+url+" for output");
        // insert extended keywords between header and data
        if (io.isStream() && keywords.size>0 && getDetached()==0) {
          long start = ((hb.length+511)/HEADER_SIZE + (keywords.size+511)/HEADER_SIZE) * HEADER_SIZE;
          setDataStart(start);
          writeHeader();
          setDetached(1); // to place ext keywords directly after header
          writeKeywords();
          setDetached(0);
          io.seek(start);
        } else {
          writeHeader();
        }
      }
      isOpen = true;
// *****************************************************************************
// ** file is OPEN after this point, unless an exception is thrown            **
// *****************************************************************************

      // ==== Begin do not change the order of the next several lines!!! ====
      setReps(); // needed by keyword routines
      final String origBlueVer = getBlueVer();
      final String origBlueIO = getBlueIO();
      if (isInput && !checkBlueVer()) {
        if (autoConvertPlatinum && PLATINUM_VER.matcher(origBlueVer).matches()) {
          M.info("DataFile: Does not have full support PLATINUM files VER='"+origBlueVer
                    +"' IO='"+origBlueIO+"' found for "+getName()+".");
        }
        else {
          M.warning("DataFile: Unsupported BLUE version VER='"+origBlueVer
                    +"' IO='"+origBlueIO+"' found for "+getName()+".");
        }
      }
      if (isOutput && isInput) setBlueVerIO(); // Change IO= and VER= since we are now the writer of the file.
      // ==== End do not change the order of the above lines!!! ====

      // allow override of PacketHandler
      String qpacket = (String)fn.getQualifier("PACKET");
      if (qpacket!=null) setPacket (qpacket);

      if ((flags&HEADERONLY)!=0) {
        io = null;           // Only read header file (read/write of data will cause NullPointerException)
        isStream = false;
      } else {
        openPacketHandler(); // Open packet handler (if applicable)
        isStream = io.isStream();
        if (io.isStreaming()) setDataStart(io.seek());
      }
      setInternals();

      // some special pipe setup
      if (getPipeId()<0) {
        cfsize = getOutByte(0)/dbpe;
        cftop  = cfsize * Math.floor(getInByte()/dbpe/cfsize);
        flags |= WRAP;
      }

      // Init internal write size, so existing files modified in middle have correct size at close
      if (isOutput && isFound && (isInput || (flags&APPEND)!=0)) wsize = size;

      if (isInput) {
        readKeywords(origBlueVer, origBlueIO); // 2010-05-12 exception is caught & warning is printed inside readKeywords() for BUG 2141
        if (setTrim(fn.getTrimmers())<0) M.warning("DataFile: Bad trim string="+fn.getTrimmers());
        // all non-NeXtMidas versions create IEEE SP only, even if header says VAX|EEEI
        //if (dataType=='P' && !getCreator().startsWith("NXM")) setDataRep("IEEE");
        if (dataType=='P' && !getBlueIO().equals("NeXtMidas")) setDataRep("IEEE");
      }
      if (fn.hasQualifiers()) {
        Table qualifiers = fn.getQualifierTable();
        // Note: qualifiers can be used that match capital letters of setter method names.
        // This matches "most file types will accept any qualifier that matches one of the
        // settable fields for the class", but it also means other setter methods can be
        // accessed via qualifiers (ex: setTimeLineLength())
        KeyObject.setKeys(this, qualifiers, M, KeyObject.SOFT); // no error on setKeys
        setInternals(); // in case qualifiers modify
        if (isOutput) {
          if (isPipe() || getDetached()!=0) { // since 3.1.1, flush keywords to main/extended
            writeKeywords();                  // headers for PIPES -or- detached headers for BUG 2512
          }
          writeHeader();
        }
      }
      if (isOpen && !isPipe() && getDetached()!=0) {
        // Allow override, but default Data Start to Zero for detached files. This is done even
        // if HEADERONLY is set since it is included in the header.
        if (dir>0) {
          Object ds        = fn.getQualifier("DS");            // 2nd precedence
          Object datastart = fn.getQualifier("DATASTART");     // 1st precedence
          Object dsval = (datastart != null)? datastart : ds;  // since 2.9.1
          setDataStart( Convert.o2l(dsval, M, 0) );
        }
        if ((flags&HEADERONLY)==0) {
          io = M.io.findResource(getDetachName(),"","",fn.getQualifierTable(),dir,true);
          io.setLength( (long)(getDataStart()+getDataSize()) );
          if (isInput && isOutput && io.exists()) {
            if (!io.open(IOResource.DIR_INOUT)) throw new MidasException("Could not open detached data: "+io.getURL());
          } else {
            if (!io.open(dir)) throw new MidasException("Could not open detached data: URL="+io.getURL());
          }
        }
      }
      if ((flags&APPEND)==0)    seek(0.0);    // no append
      else if ((flags&WRAP)==0) seek(wsize);  // append to normal file
      else seek(getInByte()/dbpe);            // append to RT file

      if (isInput && isOpen) {
        boolean warn = (flags&APPEND)!=0;
        if (!checkType(getType(),types)) {
          if (warn) M.warning("DataFile: Unallowed Type="+getType()+" allowable="+types+" for "+getName());
          else throw new MidasException("Unallowed Type="+getType()+" allowable="+types);
        }
        if (!checkFormat(getFormat(),formats,minimalSupportForUnsupportedDataTypes)) { // 3.9.3 add minmal support for unsigned
          if (warn) M.warning("DataFile: Unallowed Format="+getFormat()+" allowable="+formats+" for "+getName());
          else throw new MidasException("Unallowed Format="+getFormat()+" allowable="+formats);
        }
//      if ((flags&WARNEMPTY)!=0 && size<=0) M.warning("No data elements in URL="+url); // 2.7.3 to 3.3.1
        // since 3.3.2: do not warn for pipe as size may not be known yet for streaming data (BUG 2745)
        if ((flags&WARNEMPTY)!=0 && size<=0 && !isPipe()) M.warning("No data elements in URL="+url);
      }

      if (isInput && isOpen && isCreator("<=NXM2.6.0") && (typeClass == 5)) {
        Time.checkQuadwords(M,this);
      }

      if (isPipe() && timeLine != null) {
        // since 3.3.0: to FIX PIPE RESTART issue for PLOT/RMIF (BUG 2685)
        // NOTE1: Cannot call #getTimeLineDelta() since we might have TimeLine with out-dated delta
        //        in the case of a RESTART of a PIPE since PLOT and RMIF re-uses the DataFile instance
        //        vs creating a new DataFile instance like normal primitives such as FFT and NOOP.
        // NOTE2: We cannot do this in the PipeResource.open() since the headers and qualifiers has
        //        not been updated yet.
        timeLine.setDelta(getDelta() / dbpe);
      }
    } catch (MidasException e) {
      if (io != null) io.close();
      if (ioh != null && ioh != io) ioh.close();
      isOpen=false;
      if ((flags&NOABORT)==0) {
        throw new MidasException("Could not open "+getName()+": "+e.getMessage(),
                                 "flags="+getFlagsString(), e);
      }
      if ((M.debug&Debug.IO) != 0) M.printStackTrace(e);
    }
    return isOpen;
  }

  @Override
  public void update () {
    resetDataSize();
    if (isStream && !isPipe()) return; // cannot update streams that are NOT pipes
    writeKeywords();
    writeHeader();
  }

  @Override
  public void flush () {
    if (!isOpen) return;
    if (hp!=null) hp.flush();
    double ds1 = getDataSize();
    double ds2 = resetDataSize();
    if (isStream && !isPipe()) return; // cannot flush streams that are NOT pipes
    if (timeLine!=null && iop==null && ioh!=io) writeKeywords();
    if (ds2>ds1 || (flags&WRAP)!=0) writeHeader();
  }

  /** recalculate the current data size */
  private double resetDataSize () {
    double writebyte = wsize*dbpe;
    if (isOutput) {
      if (pkh!=null) writebyte = pkh.dataToPacketOffset(this,writebyte);
      if (isInput && writebyte<=getDataSize()) { } // don't truncate input/output data files
      else setDataSize(writebyte, false);          // set data size but don't reset the internals
      if ((flags&WRAP)!=0) setInByte( (cftop+offset)*dbpe );
    }
    return writebyte;
  }

  @Override
  public synchronized void close () {
    // Since 3.1.1: This method was made synchronized so that when multiple threads
    //              find/open this DataFile, they don't step over each other for BUG 2457.
    if (!isOpen) return;
    if ((M.debug&Debug.IO)!=0) M.info("CloseDF tag="+tag);
    if (cmd!=null && cmd.state==Commandable.RESTART) flags |= REOPEN;
    if (isOutput) update();
    if (isOutput && (flags&WARNEMPTY)!=0 && getOffset()==0) M.warning("No output written to URL="+getURL());
    if (pkh!=null) pkh.close(this);
    if (io!=null) io.close();
    if (ioh!=io && ioh!=null) ioh.close();
    isOpen = false;
  }

  /** constants for values return by {@link #erase(boolean)}. */
  public static final int E_HCB_OK=0x1, E_HCB_FAIL=0x2, E_DET_OK=0x4,
    E_DET_FAIL=0x8, E_PKT_OK=0x10, E_PKT_FAIL=0x20, E_PROT_FAIL=0x40;

  @Override
  public int erase (boolean force) {
    int status = 0;
    if (!find(-1)) return 0;
    open();
    close();
    if (!force && getProtected()) return E_PROT_FAIL;
    status |= (ioh.delete())? E_HCB_OK: E_HCB_FAIL;
    if (getDetached()!=0) status |= (io.delete())? E_DET_OK: E_DET_FAIL;
    if (hp!=null) status |= (hp.io.delete())? E_PKT_OK: E_PKT_FAIL;
    return status;
  }

  /** Erases the current DataFile
      @param force This parameter is not used
      @return int code indicating whether or not the file was erased
   */
  public int eraseFileName (boolean force) {
    DataFile htmp = new DataFile(M, getFileName());
    return htmp.erase(force);
  }

  @Override
  public int rename (String newName) {
    int status = 0; boolean ok;
    if (!find(-1)) return 0;
    open(); close(); // get header info on auxiliary files
    FileName fni = new FileName(getURL());
    FileName fno = new FileName(newName);
    ok = CoreIO.rename (fni.getFullName(), fno.getFullName() );
    status |= (ok)? E_HCB_OK: E_HCB_FAIL;
    if (getDetached()!=0) {
      fni.setExt(getDetachExt());
      fno.setExt(getDetachExt());
      ok = CoreIO.rename (fni.getFullName(), fno.getFullName() );
      status |= (ok)? E_DET_OK: E_DET_FAIL;
    }
    if (hp!=null) {
      fni.setExt("pkt");
      fno.setExt("pkt");
      ok = CoreIO.rename (fni.getFullName(), fno.getFullName() );
      status |= (ok)? E_PKT_OK: E_PKT_FAIL;
    }
    return status;
  }

  /**
   * check a type against an allow/disallow list
   * @param type     file type (ex: 1000, 2000, ...)
   * @param typeList allow/disallow list
   * @return true if allowed type
   */
  public static boolean checkType (int type, String typeList) {
    int i, typeClass=type/1000;
    if (typeList==null||typeList.length()==0) return true;
    i = typeList.indexOf(""+type);
    if (i<0) i = typeList.indexOf(""+typeClass+"000");
    if (i<0 && typeList.charAt(0)!='-') return false;
    if (i>0 && typeList.charAt(i-1)=='-') return false;
    return true;
  }

  /** check a format against an allow/disallow list.
      NOTE: This method is DEPRECATED from public use, it is only intended for
            internal usage in SYS). Use Args.testFormat(String format, String allowable) instead.
      @param format     input format to check (must be two character digraph string)
      @param formatList option string of allowable types (if null or blank, then
                         format testing is bypassed)
      @return true if format is valid against allow/disallow list (formatList), otherwise false
    */
  public static boolean checkFormat (String format, String formatList) {
    return checkFormat(format,formatList,false);
  }

  /** check a format against an allow/disallow list.
      @param format     input format to check (must be two character digraph string)
      @param formatList option string of allowable types (if null or blank, then
                         format testing is bypassed)
      @param minimalSupportOfUnsupportedDataTypes true if should support
      @return true if format is valid against allow/disallow list (formatList), otherwise false
   */
  private static boolean checkFormat (String format, String formatList, boolean minimalSupportOfUnsupportedDataTypes) {
    char mode=format.charAt(0), type=format.charAt(1);
    if (formatList==null||formatList.length()==0) return true;
    int i = formatList.indexOf(format);
    if (i<0 && NUMERIC_DATA_TYPE_LIST.indexOf(type)>=0) i = formatList.indexOf( mode+"#" );
    if (i<0) { // since 3.9.3 add minimal support for some unofficially supported data types
      if (!minimalSupportOfUnsupportedDataTypes) {
        if (DATA_TYPE_LIST.indexOf(type)>=0) i = formatList.indexOf( mode+"*" );
      }
      else {
        if (EXTENDED_DATA_TYPE_LIST.indexOf(type)>=0) i = formatList.indexOf( mode+"*" );
      }
    }
    if (i<0 && MULT_DATA_MODE_LIST.indexOf(mode)>=0) i = formatList.indexOf( "#"+type );
    if (i<0 && formatList.charAt(0)!='-') return false;
    if (i>0 && formatList.charAt(i-1)=='-') return false;
    return true;
  }

  /** called to modify Packet handler for input|output
   *  @param config Packet handler configuration
   */
  public void setPacket (String config) {
    if (config.equals("NONE") || config.equals("NULL")) {
      keywords.deleteMain("PACKET");
      pkh = null;
      return;
    }
    // if starts with / we are just adding parameters to existing config
    // add to the left so they will override previous values
    if (config.charAt(0)=='/') {
      String s = getPacket();
      int i = s.indexOf('/');
      if (i>0) config = s.substring(0,i)+config+s.substring(i);
      else config = s+config;
    }
    keywords.putMain("PACKET",config);
  }

  /** get packet
   *  @return value of keyword 'PACKET'
   */
  public String getPacket() {
    return keywords.getMain("PACKET");
  }

  /** set packet handler
   *  @param packetHandler packet handler
   */
  public void setPacketHandler (PacketHandler packetHandler) {
    if (isOpen) M.error("Cannot set PacketHandler on opened URL="+getURL());
    pkh = packetHandler;
    keywords.deleteMain ("PACKET");
    if (pkh!=null) {
      String config = pkh.getConfiguration(this);
      if (config!=null) keywords.putMain ("PACKET",config);
    }
  }

  /** get packet handler
   *  @return packet handler
   */
  public PacketHandler getPacketHandler () { return pkh; }

  /** Only called from {@link #open()} */
  private void openPacketHandler () {
    try {
      String packetConfig = keywords.getMain("PACKET");
      if (packetConfig==null) return;
      if ((M.debug&Debug.IO)!=0) M.info("Open PacketHandler Cfg="+packetConfig);
      Table pt=null;
      boolean strip=false;
      int ip = packetConfig.indexOf('/');
      if (ip>0) {
        pt = new Table();
        Parser.switchString(packetConfig,pt);
        packetConfig = packetConfig.substring(0,ip);
      }
      if (pkh==null) {
        String name = packetConfig.toUpperCase();
        pkh = (PacketHandler)Registry.getHandler("PACKET."+name);
        if (pkh==null) throw new MidasException("Unable to load packet handler class for type '"+name+"'.");
      }
      if (pt!=null) {
        String paux = (String)pt.get("DET");
        if (paux==null || Convert.isFalseString(paux)) hp=null;
        else {
          FileName pfn = new FileName(io.getURL());
          if (!Convert.isTrueString(paux)) {
            pfn.setPath (M.io.getAuxPath(paux));
            hp.aux = paux;
          }
          pfn.setExt("pkt");
          pkh.setFileName(this,pfn);
        }
        pt.remove("DET");
        strip = pt.getState("STRIP");
        if (strip) pt.remove("STRIP");
        if (pt.getSize()>0) KeyObject.setKeys(pkh,pt);
      }
      pkh.open(this);
      if (strip) keywords.deleteMain("PACKET");
      else keywords.putMain ("PACKET",pkh.getConfiguration(this));
    }
    catch (RuntimeException e) {
      if ((flags&NOABORT)!=0) {
        M.warning("DataFile: Unable to open packet handler for "+getName()+": "+e.getMessage());
        pkh = null; // <-- Prevents NullPoinbterException during read/write.
      } else {
        throw e;
      }
    }
  }

  /** get packet handler flags
   *  @return packet handler flags (as int)
   */
  public int getPacketHandlerFlags () {
    return (flags|PKTHDR) & ~WRAP;
  }

  /** Designed to generate trimmers needed to get the portion of a DataFile
   *  that is within the specified time boundaries
   *
   *  @param fileName  name of DataFile that needs trimmers
   *  @param tmin      min time as double
   *  @param  tmax     max time as double
   *  @return trimmers as String <code>{&lt;1st_elem&gt;,&lt;last_elem&gt;}</code>
   */
  @ProvisionalUseOnly("In evaluation phase")
  public static String trimmersFromTimes (String fileName, double tmin, double tmax) {
    DataFile df = new DataFile(Shell.getMidasContext(), fileName);
    df.open();
    double firstElement = df.getIndexAt(tmin);
    double secondElement = df.getIndexAt(tmax);
    df.close();
    String fileTrimmers = "("+firstElement+":"+secondElement+")";
    return fileTrimmers;
  }

  /** set file trimming settings on current input file */
  private int setTrim (String trim) {
    trim1=0.0; trim2=size;
    // Since 3.3.4: added && !isOuput on next if line for appending (output) on circular (WRAP)
    //              input file AND no file trimmers are set on filename (BUG 2772)
    if (trim==null && (flags&WRAP)!=0 && !isOutput) trim = "(:)";
    if (trim==null) return 0;
    int ic,ie=trim.length()-1;
    ic = trim.indexOf(':'); if (ic<=0) return -1;
    // since 3.3.4: pass in ADDR_UNKNOWN instead of ADDR_INDEX so that file trimmer with tilde
    //              use ADDR_ABSCISSA_LEGACY mode, otherwise use ADDR_INDEX mode (BUG 2771)
    double d1 = getIndex(trim.substring(1,ic), ISTART, ADDR_UNKNOWN);      // parse start
    double d2 = getIndex(trim.substring(ic+1,ie), IEND, d1, ADDR_UNKNOWN); // parse end
    return setTrimByIndex (d1,d2);
  }

  private int setTrimByAbsc (double d1, double d2) {
    return setTrimByIndex(getIndex(d1),getIndex(d2));
  }

  private int setTrimByIndex (double d1, double d2) {
    double minOffset, maxOffset;
    if ((flags&WRAP)==0) { minOffset=0.0; maxOffset=Math.max(0.0,size); }
    else { maxOffset=getInByte()/dbpe; minOffset=Math.max(0.0,maxOffset-cfsize); }
    if (d1<minOffset) { M.warning("Trim start="+d1+" before file start="+minOffset); d1=minOffset; }
    if (d2>maxOffset) { M.warning("Trim end="+d2+" past file end="+maxOffset); d2=maxOffset; }
    if (d1>d2) { M.warning("Trim start="+d1+" after trim end="+d2); d1=d2; }
    trim1=d1; trim2=d2; size=d2-d1;
    // Need to flag that trimmers have been set because 0:0 is valid and the default.
    // The setSize() method needs to know when trimmers have been set.
    trimSet = true;
    seek(0.0); // seek to trim1
    // fix start to reflect trim offset
    if ((flags&WRAP)==0) setStart( getStart()+d1*getDelta() );
    // if wrapped fix time code and reset start
    else { setStart(0.0); setTimeCode(getTimeAtCurrent()); }
    return 1;
  }

  /** Indexing modes for <code>getIndex(..)</code> methods.
      <pre>
        ISTART - start indexing mode.
        IEND   - end indexing mode.
        IDELTA - index delta indexing mode.
        RDELTA - abscissa delta indexing mode.
        JSTART - Frame start indexing mode (for type 2000 file).
        JEND   - Frame end   indexing mode (for type 2000 file).
        JDELTA - Frame delta indexing mode (for type 2000 file) e.g. increment between samples in frame.
      </pre>
   */
  public static final int ISTART=1, IEND=2, IDELTA=3, RDELTA=4, JSTART=5, JEND=6, JDELTA=7;

  /** The list of addressing modes that can be used by the <code>getIndex(..)</code>.
      Note: ABSCISSA_LEGACY mode is deprecated.
      @see #getIndex(String, int, double, int)
      @since NeXtMidas 2.9.1
   */
  public static final String addressingModeList = "INDEX,ABSCISSA,ABSCISSA_AUTO,TIMECODE,ABSCISSA_LEGACY";

  /** Addressing mode for <code>getIndex(..)</code> methods.
      <pre>
        ADDR_UNKNOWN=0       - Unknown addressing mode (since 3.3.4):
                               if ~ is specified, then use ADDR_ABSCISSA_LEGACY mode,
                               otherwise defaults to ADDR_INDEX mode.
        ADDR_INDEX           - Absolute indexing.
        ADDR_ABSCISSA        - Abscissa indexing only (e.g. physical time or frequency).
        ADDR_ABSCISSA_AUTO   - Same as ADDR_TIMECODE mode when abscissa units is Time, otherwise same a ADDR_ABSCISSA mode.
        ADDR_TIMECODE        - Indexing using time: abscissa address + (TimeLine (if exists) or timecode, in that order).
        ADDR_ABSCISSA_LEGACY - Similar to ADDR_ABSCISSA_AUTO, but ONLY use abscissa + TimeLine/timecode when
                               abscissa units is Time and when return index is greater than 100e6 seconds (3 yrs, 3 mons, 3+ days).
      </pre>
      @since NeXtMidas 2.9.1
   */
  public static final int ADDR_UNKNOWN=0, ADDR_INDEX=1, ADDR_ABSCISSA=2, ADDR_ABSCISSA_AUTO=3,
                          ADDR_TIMECODE=4, ADDR_ABSCISSA_LEGACY=5;

  /** Additional options (mask) for {@link DataFile#getIndex(double, int, double, int, int)} method. <br>
      0x0       - none/no options (default=none). <br>
      IMODEDEF  - default index based on mode. <br>
        &nbsp;    ISTART - start of file/pipe <br>
        &nbsp;    IEND   - end of file/pipe <br>
        &nbsp;    IDELTA/JDELTA - 1 (delta absolute index) <br>
        &nbsp;    RDELTA - 0 <br>
        &nbsp;    JSTART - 0 (frame start index) <br>
        &nbsp;    JEND   - &lt;frame size&gt; (frame end index) <br>
      ITILDE    - tilde (~) abscissa addressing mode. <br>
      IRELATIVE - use relative addressing (from relOff). <br>
      IFROMSIZE - address from size of file (i.e. from end of file).
      @since NeXtMidas 2.9.1
   */
  public static final int IMODEDEF=0x1, ITILDE=0x2, IRELATIVE=0x4, IFROMSIZE=0x8;

  /** Gets the absolute index address from an input value with options mask for using
      default index for specified mode parameter ({@linkplain #IMODEDEF}),
      relative ({@linkplain #IRELATIVE}), from end of file (size) ({@linkplain #IFROMSIZE}),
      or tilde (~) abscissa ({@linkplain #ITILDE}) addressing. Base on addressingMode
      it can account for Timeline/Timecode, see {@link #addressingModeList} for more info.
      This method is similar to the X-Midas M$INDEX API.
      @param value          double value to get the absolute index address from this file
      @param mode           indexing mode (one of {@link #ISTART}, {@link #IEND}, {@link #IDELTA},
                            {@link #RDELTA}, {@link #JSTART}, {@link #JEND}, or {@link #JDELTA})
      @param relOff         relative offset
      @param addressingMode addressing mode to use (one of {@link #ADDR_INDEX}, {@link #ADDR_ABSCISSA},
                            {@link #ADDR_ABSCISSA_AUTO}, {@link #ADDR_TIMECODE})
      @param  options       additional options to apply when getting the absolute index (mask of
                            {@link #IMODEDEF}, {@link #ITILDE}, {@link #IRELATIVE}, {@link #IFROMSIZE})
      @return absolute (index) address into this DataFile for the specified arg
      @since NeXtMidas 2.9.1 - moved code from {@link #getIndex(String, int, double, boolean)} to here
                               and {@link #getIndex(String, int, double, int)}.
      @deprecated Since NeXtMidas 3.3.4, this is meant for internal use only
   */
  @InternalUseOnly
  @Deprecated
  public double getIndex (double value, int mode, double relOff, int addressingMode, int options) {
    double delta, start, size;
    boolean isDelta = (mode==IDELTA) || (mode==JDELTA) || (mode==RDELTA);
    boolean isWrapping = ((flags&WRAP) != 0); // is this a wrapping file/pipe?
    int units=0;

    if (mode>=JSTART) { delta = getXDelta(); start = getXStart(); size = getSubSize(); units=getXUnits(); }
    else              { delta = getDelta();  start = getStart();  size = getSize(); units=getUnits();   }
    if (mode<JSTART && isWrapping) { size = getInByte()/dbpe; }

    if ((options&IMODEDEF) != 0) {
      double defIndex;
      if      (mode==IEND   || mode==JEND)   defIndex = size;
      else if (mode==ISTART || mode==JSTART) defIndex = 0;
      else if (mode==IDELTA || mode==JDELTA) defIndex = 1;
      else                                   defIndex = 0;

      if      (!isWrapping)  { } // non wrapping file, does not need additional adjustments
      else if (mode==IEND)   { defIndex = getInByte()/dbpe; }
      else if (mode==ISTART) { defIndex = Math.max(0.0, getInByte()/dbpe - cfsize);
                               relOff  += defIndex;
      }
      return defIndex;
    }
    else if (isWrapping && (mode==ISTART)) {
      relOff += Math.max(0.0, getInByte()/dbpe - cfsize);
    }

    boolean relative = (options&IRELATIVE)!=0 ||  isDelta;
    boolean fromsize = (options&IFROMSIZE)!=0 && !isDelta;
    boolean abscissa = (options&ITILDE)   !=0 || (addressingMode >= ADDR_ABSCISSA);

    double retIndex = value;
    if (relative) {
      if (abscissa) retIndex = retIndex / delta; // adjust abscissa value to absolute index value
      retIndex += relOff;
    }
    else {
      if (abscissa) { // need to adjust abscissa value to absolute index value
        if (addressingMode == ADDR_UNKNOWN) addressingMode = ADDR_ABSCISSA_LEGACY; // since 3.3.4 (BUG 2771)
        boolean abscOnlyMode = (addressingMode == ADDR_ABSCISSA); // addressing by abscissa only
        boolean timecodeMode = (addressingMode == ADDR_TIMECODE); // addressing by TimeLine/timecode
        if (units == Units.TIME) {
          abscOnlyMode |= (addressingMode == ADDR_ABSCISSA_LEGACY) && (retIndex <= 100e6);
          timecodeMode |= (addressingMode >= ADDR_ABSCISSA_AUTO); // e.g. ABSCISSA_AUTO(2), TIMECODE(3), or ABSCISSA_LEGACY(4)
        }
        if (!timecodeMode) {   // fall back to abscissa only, if not TimeLine/timecode addressing
          abscOnlyMode = true; // i.e. for ABSCISSA_AUTO, ABSCISSA_LEGACY and tilde (abscissa) modes
        }

        if      (abscOnlyMode) {  retIndex = (retIndex - start) / delta;                 }
        else if (timecodeMode) {
          if (timeLine != null) { retIndex = timeLine.getOffsetAt(retIndex) / dbpe;      }
          else                  { retIndex = (retIndex - getTimeCode() - start) / delta; }
        }
      } // end if (abscissa)
      if (fromsize) retIndex = (size - retIndex);
    }
    if (mode!=RDELTA) retIndex = Math.floor(retIndex+.5);
    if (pkh!=null && !isDelta) {
      retIndex = Math.floor( pkh.naturalDataOffset(this, retIndex*dbpe) / dbpe );
    }
    return retIndex;
  }

  /** Gets the absolute index address from a command line arg syntax, by parsing the
      (+) relative, (-) from end of file (size),  or (~) abscissa prefixes and
      defaulting/checking ranges of the input arg value (index, abscissa, timecode).
      This method is similar to the X-Midas M$INDEX API.
      @param arg    desired addressing arguments from command line
      @param mode   indexing mode (one of {@link #ISTART}, {@link #IEND}, {@link #IDELTA},
                    {@link #RDELTA}, {@link #JSTART}, {@link #JEND}, or {@link #JDELTA})
      @param relOff relative offset
      @param addressingMode  addressing mode to use (one of {@link #ADDR_INDEX}, {@link #ADDR_ABSCISSA},
                             {@link #ADDR_ABSCISSA_AUTO}, {@link #ADDR_TIMECODE})
      @return absolute (index) address into this DataFile for the specified arg
      @see #getIndex(double, int, double, int, int)
      @since NeXtMidas 2.9.1 - moved code from {@link #getIndex(String, int, double, boolean)} to here
                               and {@link #getIndex(double, int, double, int, int)}
   */
  public double getIndex (String arg, int mode, double relOff, int addressingMode) {
    if (arg==null || arg.length()==0 || arg.equals("*")) {
      return getIndex(0.0, mode, relOff, addressingMode, IMODEDEF);
    }

    int ii=0;
    int optionsMask = 0;
    boolean isDelta = (mode==IDELTA) || (mode==JDELTA) || (mode==RDELTA);
    if (arg.charAt(ii)=='+')             { ii++; optionsMask |= IRELATIVE; }
    if (arg.charAt(ii)=='-' && !isDelta) { ii++; optionsMask |= IFROMSIZE; }
    if (arg.charAt(ii)=='~')             { ii++; optionsMask |= ITILDE;    }
    double d = Convert.s2d(arg.substring(ii), M);
    return getIndex(d, mode, relOff, addressingMode, optionsMask);
  }

  /** DEPRECATED: Gets the absolute index address from a command line arg syntax.
   *  @param arg    desired addressing arguments from command line
   *  @param mode   indexing mode (one of {@link #ISTART}, {@link #IEND}, {@link #IDELTA}
   *  @param relOff relative offset
   *  @param absc true for {@link #ADDR_ABSCISSA} otherwise {@link #ADDR_INDEX} addressing mode
   *  @return absolute (index) address into this DataFile for the specified arg
   *  @deprecated Since NeXtMidas 2.9.1, use {@link #getIndex(String, int, double, int)}
   */
  @Deprecated public double getIndex (String arg, int mode, double relOff, boolean absc) {
    int addressingMode = (absc? ADDR_ABSCISSA : ADDR_INDEX);
    return getIndex(arg, mode, relOff, addressingMode);
  }

  /** Same as <code>getIndex(arg, mode, 0.0, {@link #ADDR_INDEX}})</code>.
      @param arg    desired addressing arguments from command line
      @param mode   indexing mode (one of {@link #ISTART}, {@link #IEND}, {@link #IDELTA}
      @return absolute (index) address into this DataFile for the specified arg
      @see #getIndex(String, int, double, int)
      @deprecated Since NeXtMidas 3.3.4, use {@link #getIndex(String, int, int)},
                  this method was meant for internal use only but is no longer in use
   */
  @Deprecated public double getIndex (String arg, int mode) {
    return getIndex(arg, mode, 0.0, ADDR_INDEX);
  }

  /** Same as <code>getIndex(arg, mode, 0.0, addressingMode)</code>.
      @param arg    desired addressing arguments from command line
      @param mode   indexing mode (one of {@link #ISTART}, {@link #IEND}, {@link #IDELTA},
                    {@link #RDELTA}, {@link #JSTART}, {@link #JEND}, or {@link #JDELTA})
      @param addressingMode  addressing mode to use (one of {@link #ADDR_INDEX}, {@link #ADDR_ABSCISSA},
                             {@link #ADDR_ABSCISSA_AUTO}, {@link #ADDR_TIMECODE})
      @return absolute (index) address into this DataFile for the specified arg
      @see #getIndex(String, int, double, int)
   */
  public double getIndex (String arg, int mode, int addressingMode) {
    return getIndex(arg, mode, 0.0, addressingMode);
  }

  /** Same as <code>getIndex(dvalue, mode, 0.0, addressingMode, 0x0)</code>.
      @param dvalue double value to get the absolute index address from this file
      @param mode   indexing mode (one of {@link #ISTART}, {@link #IEND}, {@link #IDELTA},
                    {@link #RDELTA}, {@link #JSTART}, {@link #JEND}, or {@link #JDELTA})
      @param addressingMode  addressing mode to use (one of {@link #ADDR_INDEX}, {@link #ADDR_ABSCISSA},
                             {@link #ADDR_ABSCISSA_AUTO}, {@link #ADDR_TIMECODE})
      @return absolute (index) address into this DataFile for the specified arg
      @see #getIndex(double, int, double, int, int)
      @since NeXtMidas 2.9.1
   */
  public double getIndex (double dvalue, int mode, int addressingMode) {
    return getIndex(dvalue, mode, 0.0, addressingMode, 0x0);
  }

  /** DEPRECATED: Gets the absolute index address from a command line arg syntax.
      @param arg    desired addressing arguments from command line
      @param mode   indexing mode (one of {@link #ISTART}, {@link #IEND}, {@link #IDELTA},
                    {@link #RDELTA}, {@link #JSTART}, {@link #JEND}, or {@link #JDELTA})
      @param absc true for {@link #ADDR_ABSCISSA} otherwise {@link #ADDR_INDEX} addressing mode
      @return absolute (index) address into this DataFile for the specified arg
      @see #getIndex(String, int, double, int)
      @deprecated Since NeXtMidas 2.9.1, use {@link #getIndex(String, int, int)}
   */
  @Deprecated public double getIndex (String arg, int mode, boolean absc) {
    int addressingMode = (absc? ADDR_ABSCISSA : ADDR_INDEX);
    return getIndex(arg, mode, 0.0, addressingMode);
  }

  /** Same as <code>getIndex(arg, mode, relOff, false)</code>.
      @param arg    desired addressing arguments from command line
      @param mode   indexing mode (one of {@link #ISTART}, {@link #IEND}, {@link #IDELTA},
                    {@link #RDELTA}, {@link #JSTART}, {@link #JEND}, or {@link #JDELTA})
      @param relOff relative offset
      @return absolute (index) address into this DataFile for the specified arg
      @see #getIndex(String, int, double, int)
   */
  public double getIndex (String arg, int mode, double relOff) {
    return getIndex(arg, mode, relOff, ADDR_INDEX);
  }

  /** Gets absolute index from specified abscissa value based on {@link #getStart()} and
      {@link #getDelta()} header and without any TimeLine/timecode adjustments.
      Use {@link #getIndex(String, int, double, int)} if you need the addressing to include TimeLine or timecode.
      @param absc abscissa value to convert to absolute index in file
      @return absolute (index) address into this DataFile for the specified arg
      @see #getIndex(double, int, double, int, int)
   */
  public double getIndex (double absc) {
    return Math.floor((absc - getStart()) / getDelta() + 0.5); // +0.5 to round
  }

  /** align headers
   *
   *  @param h2   other DataFile
   *  @param absc true if by absiccca
   *  @param tc   unused
   */
  public void align (DataFile h2, boolean absc, boolean tc) {
    DataFile h1 = this;
    double size1 = h1.getSize();
    double size2 = h2.getSize();
    if (absc) {
      double d1=h1.getDelta(), d2=h2.getDelta(), dc=d2-d1;
      double s1=h1.getStart(), s2=h2.getStart(), sc=s2-s1, sx=Math.max(s1,s2);
      double e1=s1+d1*size1, e2=s2+d2*size2, ex=Math.min(e1,e2);
      if (d1==0 || d2==0) M.error ("Cannot abscissa align files with delta=0");
      if (Math.abs(dc/d1)>1e-6) M.error ("Cannot abscissa align files with delta mismatch > 1e-6");
      h1.setTrimByAbsc(sx,ex);
      h2.setTrimByAbsc(sx,ex);
    }
    else {
      double _size = Math.min(size1,size2);
      h1.setTrimByIndex(0,_size);
      h2.setTrimByIndex(0,_size);
    }
  }

  private void setReps () {
    headRep = hb[OFF_HEADREP];
    dataRep = hb[OFF_DATAREP];
  }

  @Override
  public void setInternals () {
    setReps();
    typeClass = getType()/1000;
    if (typeClass==2 && getSubSize()<=0) setSubSize(1);
    dataType = getFormatType();
    dataMode = getFormatMode();
    getBPS();
    getSPA();
    getBPE();
    firstbyte = getDataStart();
    if ((typeClass == 6) && (t6s==null)) t6s = readT6def(false); // Type changed to 6000.
    if ((typeClass != 6) && (t6s!=null)) t6s = null;             // Type changed from 6000.
    boolean setwsize = (wsize!=0) && (wsize==size);  // if size and wsize are in step, keep them in step below after setSize()
    if (isOpen && (isInput || (flags&APPEND)!=0) && dbpe>0.0) setSize();
    if (isOpen && setwsize) wsize = size;
  }

  /** Set Data FrameSize for data transfer operations only - do not change header.
      Note: This is NOT the same as force1000 or force2000 in XMidas
      @param fs Frame size (0=Treat as Type 1000).
   */
  public void setDFS (int fs) {
    if (trimSet) { trim1 *= ape; trim2 *= ape; }
    if (fs==0) {
      ape = 1;
      dbpe = bpa;
      typeClass = 1;
      if (trimSet) seek(0.0);
    }
    else {
      ape = fs;
      dbpe = fs*bpa;
      typeClass = 2;
    }
    if (trimSet) { trim1 /= ape; trim2 /= ape; }
    if (dbpe<0) dbpe = -dbpe/8;
    setSize();
    if ((M.debug&Debug.IO)!=0) M.info("FrameDF tag="+tag+" Frame="+ape+" Size="+size);
  }

  /** Add a SubRecord to a type 3000/5000/6000 file header. This is the same as
      <code>addSubRec(name,format,-1,{@link Units#SCALAR},{@link Units#UNDEFINED})</code>.
      @param name     Name of the SubRecord.
      @param format   The format of the SubRecord (e.g. "SF").
      @return Index of the new SubRecord.
   */
  public int addSubRec (String name, String format) {
    return setSubRec(-1, name, format, -1, Units.SCALAR, Units.UNDEFINED, true);
  }

  /** Add a SubRecord to a type 3000/5000/6000 file header. This is the same as
      <code>addSubRec(name,format,offset,{@link Units#SCALAR},{@link Units#UNDEFINED})</code>.
      @param name     Name of the SubRecord.
      @param format   The format of the SubRecord (e.g. "SF").
      @param offset   Byte offset of the SubRecord (-1=AutoCompute).
      @return Index of the new SubRecord.
   */
  public int addSubRec (String name, String format, int offset) {
    return setSubRec(-1, name, format, offset, Units.SCALAR, Units.UNDEFINED, true);
  }

  /** Add a SubRecord to a type 3000/5000/6000 file header.
      @param name     Name of the SubRecord. Name length is limited to 4 characters for Type 3000/5000
                      files and 24 characters for Type 6000 files. Names that exceed this length will
                      be truncated automatically.
      @param format   The format of the SubRecord (e.g. "SF").
      @param offset   Byte offset of the SubRecord (-1=AutoCompute). This is is ignored by Type 5000 files.
      @param type     The type (see {@link Units}, 0=UNKNOWN). This is ignored by Type 3000/6000 files.
      @param units    The units for the component (see {@link Units}, 0={@link Units#NONE}). This is ignored by Type 3000 files.
      @return Index of the new SubRecord.
      @since NeXtMidas 2.7.0
   */
  public int addSubRec (String name, String format, int offset, int type, int units) {
    return setSubRec(-1, name, format, offset, type, units, true);
  }

  /** Add a SubRecord to a type 3000/5000/6000 file header.
      @param name     Name of the SubRecord. Name length is limited to 4 characters for Type 3000/5000
                      files and 24 characters for Type 6000 files. Names that exceed this length will
                      be truncated automatically.
      @param format   The format of the SubRecord (e.g. "SF").
      @param offset   Byte offset of the SubRecord (-1=AutoCompute). This is is ignored by Type 5000 files.
      @param type     The type (see {@link Units}, 0=UNKNOWN). This is ignored by Type 3000/6000 files.
      @param units    The units for the component (see {@link Units}, 0={@link Units#NONE}). This is ignored by Type 3000 files.
      @return Index of the new SubRecord.
      @since NeXtMidas 2.7.0
   */
  public int addSubRec (String name, String format, int offset, String type, String units) {
    int t = getCompTypeID(type, Units.SCALAR);
    int u = getUnitsID(units, Units.UNDEFINED);
    return setSubRec(-1, name, format, offset, t, u, true);
  }

  /** Add a Component to a type 3000/5000/6000 file header. This is the same as
      <code>addSubRec(name,format,-1,type,units)</code>.
      @param name     Name of the SubRecord.
      @param format   The format of the SubRecord (e.g. "SF").
      @param type     The type (see {@link Units}).
      @param units    The units for the component (see {@link Units}).
      @return Index of the new Component.
      @since NeXtMidas 2.7.0
   */
  public int addComp (String name, String format, String type, String units) {
    int t = getCompTypeID(type, Units.SCALAR);
    int u = getUnitsID(units, Units.UNDEFINED);
    return setSubRec(-1, name, format, -1, t, u, true);
  }

  /** Add a Component to a type 3000/5000/6000 file header. This is the same as
      <code>addSubRec(name,format,-1,type,units)</code>.
      @param name     Name of the SubRecord.
      @param format   The format of the SubRecord (e.g. "SF").
      @param type     The type (see {@link Units}).
      @param units    The units for the component (see {@link Units}).
      @return Index of the new Component.
   */
  public int addComp (String name, String format, int type, int units) {
    return setSubRec(-1, name, format, -1, type, units, true);
  }

  /** Sets a SubRecord to a type 3000/5000/6000 file header.
      @param n        The SubRec number (-1 to add to end).
      @param name     Name of the SubRecord. Name length is limited to 4 characters for Type 3000/5000
                      files and 24 characters for Type 6000 files. Names that exceed this length will
                      be truncated automatically.
      @param format   The format of the SubRecord (e.g. "SF").
      @param offset   Byte offset of the SubRecord (-1=AutoCompute). This is is ignored by Type 5000 files.
      @param type     The type (see {@link Units}). This is ignored by Type 3000/6000 files.
      @param units    The units for the component (see {@link Units}). This is ignored by Type 3000 files.
      @param setInt   Set internals? (Set this to false if called in a loop and setInternals() is
                      called at the end.)
      @return Index of the new/updated SubRecord.
      @since NeXtMidas 2.7.0
   */
  private int setSubRec (int n, String name, String format, int offset, int type, int units, boolean setInt) {
    if (n < 0) { // add to end
      n = getSubSize();
      setSubSize(n+1);
      int boff = getRecLength();
      if (offset<0) offset = boff;
      if (n==0) setFormat(format);
      if (typeClass == 5) boff += Data.getBPA(format);
      else                boff = Math.max(boff,offset+Data.getBPA(format));
      setRecLength(boff);
    }
    // Bug 2665: This code moved to setRecFormat(..)
    //String form = getFormat();
    //if (!form.equals("NH") && !form.equals(format)) setFormat("NH");
    setRecName(n,name);
    setRecFormat(n,format);
    setRecType(n,(byte)type);
    setRecUnits(n,(byte)units);
    setRecOffset(n,(short)offset);
    if (setInt) setInternals();
    return n+1;
  }

  /** Write the header from local buffer */
  public void writeHeader () {
    writeHeader(hb);
  }

  /** Write the header from supplied buffer */
  private int writeHeader (byte[] buf) {
    int bytes=buf.length;
    if (isPipe()) {
      bytes = iop.writeHeader(buf);
    }
    else {
      long lseek = (io==ioh)? io.seek() : 0L;
      bytes = ioh.write (buf,0,bytes,0L);
      if (io==ioh) io.seek(lseek);
    }
    return bytes;
  }

  /** Internal use only: Get the MINIMUM header length in bytes that must be read in.
      Note that this will frequently return a value less than the "fixed" header
      length required by the BLUE ICD.
      @param tc type class (1=Type 1000, 2=Type 2000, etc.)
   */
  private int getHeaderLength (int tc) {
    int bytes = 256;
    int subs = getSubSize();
    switch (tc) {
      case 1: bytes += 20; break;
      case 2: bytes += 44; break;
      case 3: bytes += 48; bytes += subs*8; break;
      case 4: bytes += 20; break;
      case 5: bytes += 256; if (subs>14) bytes += (subs-14)*8; break;
      case 6: bytes += 48; break;  // JPH: This may need to change
    }
    return bytes;
  }

  /** Read the header into local buffer, extend if necessary */
  protected void readHeader () {
    readHeader(hb);
    String version = getVersion();
    if (version.equals("BLUE")) {
      int tc = getType()/1000; // need to compute since internals not set yet
      int bytes = getHeaderLength(tc);
      if (bytes>hb.length) {
        extendHeader(bytes);
        readHeader(hb);
      }
    }
    else throw new MidasException("DataFile "+getURL()+" does not have a valid BLUE Header");
  }

  /** Read the header into supplied buffer */
  private int readHeader (byte[] buf) {
    int bytes=buf.length;
    if (isPipe())
      iop.readHeader(buf);
    else
      bytes = ioh.read(buf,0,bytes,0L);
    return bytes;
  }

  private void extendHeader () {
    int tc = getType()/1000;
    int subs = getSubSize();
    if (tc==3 || tc==5) extendHeader(iSub(subs)); // JC4: In some cases we may want to do this for Type 6000
    if (t6s != null) checkT6cap(subs);
    if ((recNames != null) && (recNames.length < subs)) {
      int count = (subs+35)/36*36; // round up to nearest 36
      String[] names = new String[count];
      System.arraycopy(recNames, 0, names, 0, recNames.length);
      recNames = names;
    }
  }

  /** Extends the main header to the given number of bytes (rounded up to a multiple of 512), used
      with Extended Type 3000 files (>26 subrecords), extended Type 5000 files (>14 components).
   */
  private int extendHeader (int bytes) {
    if (bytes<=hb.length) return hb.length;
    bytes = (bytes+511)/HEADER_SIZE*HEADER_SIZE;
    byte[] hbt = new byte[bytes];
    System.arraycopy(hb,0,hbt,0,hb.length);
    hb = hbt;
    if (!isPipe()) setDataStart(bytes); // <-- JC4: should probably not do this if detached
    return bytes;
  }

  /** Writes elements of data at the current file position.
      @param data The data to write.
      @return The status of the file.
      @see #write(Data,int)
   */
  public int write (Data data) {
    int elements=data.size;
    if (data.ape!=ape) elements=(data.size*data.ape)/ape;
    return write (data, elements);
  }

  /** Writes elements of data at a specific offset. <br>
      This is identical to <code>write(data,index,data.size)</code>.
      @param data     The data to write.
      @param index    The start position in the file to start at.
      @return The status of the file.
      @see #write(Data,double,int)
   */
  public int write (Data data, double index) {
    seek(index);
    return write (data, data.size);
  }

  /** Writes elements of data at a specific offset. <br>
      Unlike some of the other <code>write(..)</code> methods, this will shift the
      current write position (see {@link #seek(double)}) as necessary before writing.
      @param data     The data to write.
      @param index    The start position in the file to start at.
      @param elements The number of elements (from <code>data</code>) to write.
      @return The status of the file.
      @see #seek(double)
      @see #write(Data,int)
   */
  public int write (Data data, double index, int elements) {
    seek(index);
    return write (data, elements);
  }

  /** Writes elements of data at the current file position.
      @param data     The data to write.
      @param elements The number of elements (from <code>data</code>) to write.
      @return The status of the file.
      @see #write(Data,double,int)
   */
  public int write (Data data, int elements) {
    if (!isOpen) { offset+=elements; return 0; }
    if ((data.writeMult!=1) && (elements%data.writeMult!=0)) {
      M.error("DataFile.write of "+data.getFormat()+" requires writes in multiples of "+data.writeMult+" file:"+this); //NXM-3894 Jeff 12-bit support
    }
    byte[] dbuf = data.buf;
    int boff = data.boff;
    if (elements > data.size) elements = data.size;  // truncate to buffer
    if (data.type!=dataType) { // NOTE: this DOES NOT support ASCII / ASCII 256 conversion - need DR if want
      int tbsize = (int)(elements*dbpe);
      if (dataType=='J') tbsize = tbsize*4/3;	// need romm for inplace conversion
      if (tempBuf==null || tempBuf.length<tbsize) tempBuf = new byte[tbsize];
      if (boff>0) boff /= data.bps;
      Convert.type( dbuf,boff,data.type, tempBuf,0,dataType, elements*ape*spa);
      dbuf = tempBuf; boff = 0;
    }
    if (data.rep!=dataRep) {      // convert from local data rep
      // since 3.7.0, use convertRecordRep for Type 6000 as some subrecords can have different num_elts (BUG 2959) and/or gaps
      if (dataType==NONHOMOGENEOUS || typeClass==6) convertRecordRep(dbuf,boff,elements,data.rep,dataRep);
      else Convert.rep( dbuf,boff,dataType, elements*ape*spa, data.rep, dataRep);
    }
    int status, bytes=(int)(elements*dbpe);  // convert elements to bytes
    if ((M.debug&Debug.IO)!=0)       // report run-time IO debug
      M.info("WriteDF tag="+tag+" Offset="+offset+" elem="+elements);
    Object dh=data.getHeader();
    if (dh instanceof Time) setTimeAt((Time)dh, NotifyInvalidEntry.WARNING);
    if (pkh==null) status = io.write (dbuf, boff, bytes);  // write bytes to raw device
    else status = pkh.write (this, dbuf, boff, bytes, (long)0);  // write bytes to pkt device
    if (status != bytes) elements = (int)((double)status/dbpe);
    if (elements>0) offset += elements;    // normal offset increment
    if (offset>wsize) wsize=offset;
    if ((flags&WRAP)!=0 && offset>=cfsize) wrapCircularFile();
    if ((flags&FLUSH)!=0) flush();
    return (elements);
  }

  /** This write supports type 3000/5000 writes of fields. <br>
      Unlike some of the other <code>write(..)</code> methods, this will shift the
      current write position (see {@link #seek(double)}) as necessary before writing. <br>
      <b>Note:</b> The data in the buffer must be in the same data format (IEEE/EEEI/...)
      as the file.
      @param buf           The buffer containing the data to write out.
      @param boff          The offset into the buffer to begin writing from.
      @param bytes         The number of bytes to write out.
      @param index         The element offset.
      @param recByteOffset The offset within the record.
      @return The status of the file.
   */
  public int write (byte[] buf, int boff, int bytes, double index, int recByteOffset) {
    index += ((double)recByteOffset/dbpe);
    if (index!=offset) seek(index);
    return write(buf,boff,bytes);
  }

  /** Writes a number of bytes to the file. <br>
      <b>Note:</b> The data in the buffer must be in the same data rep (IEEE/EEEI/...) as the file.
      @param buf   The buffer containing the data to write out.
      @param boff  The offset into the buffer to begin writing from.
      @param bytes The number of bytes to write out.
      @return The status of the file.
   */
  @Override
  public int write (byte[] buf, int boff, int bytes) {
    // Except for the three lines marked with "// **" this matches the write(long,int,int,boolean) method
    int status;
    if (!isOpen) { offset += (bytes/dbpe); return 0; }
    if ((M.debug&Debug.IO)!=0) M.info("WriteDF "+tag+" bytes="+bytes); // ** report run-time IO debug
    if (pkh==null) status = io.write (buf, boff, bytes);               // ** write bytes to raw device
    else status = pkh.write (this, buf, boff, bytes, (long)0);         // ** write bytes to raw device
    if (status>0) offset = Math.round(status + offset*dbpe) / dbpe;    // Round to nearest byte, see BUG 1173
    if (offset>wsize) wsize=offset;
    if ((flags&WRAP)!=0 && offset>=cfsize) wrapCircularFile();
    if ((flags&FLUSH)!=0) flush();
    return status;
  }

    /** Writes a number of bytes to the file from a native buffer. <br>
      This will start at memory address <code>lbuf+boff</code>.
      <b>Note:</b> The data in the buffer must be in the same data rep
      (IEEE/EEEI/...) as the file. If you want the data rep conversion to be
      done automatically, then call  {@link #write(long, int, int, boolean)}
      with true for the convert parameter.
      @param lbuf  The native pointer (address) to the buffer.
      @param boff  The offset into the buffer to begin writing from.
      @param bytes The number of bytes to write out.
      @return The status of the file.
   */
  public int write (long lbuf, int boff, int bytes) {
    // Except for the three lines marked with "// **" this matches the write(byte[],int,int) method
    int status;
    if (!isOpen) { offset += (bytes/dbpe); return 0; }
    if ((M.debug&Debug.IO)!=0) M.info("WriteDFn "+tag+" bytes="+bytes); // ** report run-time IO debug
    if (pkh==null) status = io.write (lbuf, boff, bytes);               // ** write bytes to raw device
    else status = pkh.write (this, null, boff, bytes, lbuf);            // ** write bytes to raw device
    if (status>0) offset = Math.round(status + offset*dbpe) / dbpe;     // Round to nearest byte, see BUG 1173
    if (offset>wsize) wsize=offset;
    if ((flags&WRAP)!=0 && offset>=cfsize) wrapCircularFile();
    if ((flags&FLUSH)!=0) flush();
    return status;
  }

  /** Writes a number of bytes to the file from a native buffer. <br>
      This will start at memory address <code>lbuf+boff</code>.
      @param lbuf  The native pointer (address) to the buffer.
      @param boff  The offset into the buffer to begin writing from.
      @param bytes The number of bytes to write out.
      @param convert Set to true to convert the data if the data representation
                     does not match the OS (machine) rep.
      @return The status of the file.
   */
  public int write (long lbuf, int boff, int bytes, boolean convert) {
    if (isOpen && convert && Shell.rep!=dataRep) {  // ** convert from local machine rep to file data rep
      int elements = (int)(bytes/dbpe);
      if (dataType==NONHOMOGENEOUS) convertRecordRep(lbuf,boff,elements,Shell.rep,dataRep); // ** convert data rep
      else Convert.rep(lbuf,boff,dataType, elements*ape*spa, Shell.rep, dataRep);           // ** convert data rep
    }
    return write(lbuf,boff,bytes);
  }

  /** Read data at the current address. This will read *up to* the number of elements in the transfer
      length or (if transfer length is -1) the number of elements that will fit in the data buffer.
      @param data The data buffer to read data into.
      @return The actual number of elements read in or N&lt;0 if at end of file.
   */
  public int read (Data data) {
    int elements = xferLength;
    if (elements<0) elements = data.size;
    if (data.ape!=ape) elements=(data.size*data.ape)/ape;
    return read(data, elements);
  }

  /** Read data at the specified address. This will read *up to* the number of elements in the transfer
      length or (if transfer length is -1) the number of elements that will fit in the data buffer.
      @param data  The data buffer to read data into.
      @param index File index to start reading at (0..N-1).
      @return The actual number of elements read in or N&lt;0 if at end of file.
   */
  public int read (Data data, double index) {
    int elements = xferLength;
    if (elements<0) elements = data.size;
    seek(index);
    return read(data, elements);
  }

  /** Read data at the specified address.
      @param data     The data buffer to read data into.
      @param index    File index to start reading at (0..N-1).
      @param elements The max number of elements to read in.
      @return The actual number of elements read in or N&lt;0 if at end of file.
   */
  public int read (Data data, double index, int elements) {
    seek(index);
    return read(data, elements);
  }

  @Override
  public int read (Data data, int elements) {
    if (!isOpen || elements<=0) return 0;
    int boff=data.boff;
    int bytes;
    int soff;
    int status;
    int cons=consLength;
    if (elements > data.size) elements = data.size; // truncate to buffer
    if (!isStream) { // check for end past EOF or CFL and truncate
      if (cftop+offset+elements>trim1+size) elements = (int)Math.max(0.0,trim1+size-cftop-offset);
      if (cfsize>0 && offset+elements>cfsize) elements = (int)Math.max(0.0,cfsize-offset);
      if (elements<=0) return -1;  // end of data stream
    }
    if ((M.debug&Debug.IO)!=0)     // report run-time IO debug
      M.info("ReadDF tag="+tag+" Offset="+offset+" elem="+elements);

    if (dataType==PACKED || dataType==NIBBLE || dataType==JINT){ // JINT added for NXM-3894
      double tmp = elements * dbpe;  // convert elements to number of bytes
      bytes = ((tmp % 1) != 0)? (int)tmp +1: (int)tmp; // partial bytes (PACKED and NIBBLE data) need a full byte returned
    } else bytes = (int)(elements * dbpe);   // convert elements to bytes

    if (saveLength>0) {     // implement consume length pre-seed of buffer
      soff = (int)((offset-saveOffset)*dbpe); boff=Math.max(0,saveLength-soff);
      System.arraycopy (saveBuf,soff, data.buf,0, boff); bytes-=boff;
    }
    if (pkh==null) status = io.read (data.buf, boff, bytes); // read bytes from raw device
    else status = pkh.read (this, data.buf, boff, bytes, (long)0); // read bytes from pkt device
    if (timeLine!=null) { time = getTimeAtCurrent(time); data.setHeader(time); }
    if (status<0) {       // check for pipe cycling restart sequence
      if (status==CoreIO.REOPEN && cmd!=null) cmd.setState(Commandable.RESTART);
      return (status);
    }
    if (status!=bytes) {       // calculate actual elements transfered
      if (elements == data.size && !this.isPipe()) {
        M.warning("DataFile "+fn+" does not have expected size only able to read "+((status+boff)/dbpe)+" out of "+elements+" elements");
      }
      elements = (int)((double)(status+boff)/dbpe);
    }
    if (cons>elements) {     // handle consume larger than transfer
      seek(cftop+offset+cons-trim1); saveLength=0;
    }
    else if (cons>=0) {     // handle consume less than transfer - save off overlap
      saveOffset = offset+cons;
      soff = (int)(cons*dbpe)+data.boff; saveLength = bytes+boff-soff;
      if (saveBuf==null || saveBuf.length<saveLength) saveBuf = new byte[saveLength];
      System.arraycopy (data.buf,soff, saveBuf,0, saveLength);
      offset += cons;
    }
    else offset += elements;     // normal offset increment
    if ((flags&WRAP)!=0 && offset>=cfsize) wrapCircularFile();
    if (data.rep!=dataRep) {     // convert to local data rep
      boff = data.boff; if (boff>0) boff /= bps;
      // since 3.7.0, use convertRecordRep for Type 6000 as some subrecords can have different num_elts (BUG 2958) and/or gaps
      if (dataType==NONHOMOGENEOUS || typeClass==6) convertRecordRep(data.buf,data.boff,elements,dataRep,data.rep);
      else Convert.rep(data.buf,boff,dataType, elements*ape*spa, dataRep,data.rep);
    }
    if (data.type!=dataType) {     // convert to local data type
      boff = soff = data.boff; if (boff>0) { boff /= bps; soff /= data.bps; }
      Convert.type(data.buf,boff,dataType, data.buf,soff,data.type, elements*ape*spa);
    }
    return (elements);
  }

  /** Read specified bytes at next address. This will copy the data to a byte array. <b>This
      is a special-purpose method for advanced users only.</b>
      <b>Note:</b> The data read may need to be converted if the file's data rep
      (IEEE/EEEI/...) is different than the OS (machine) data rep.
      @param buf   The byte buffer to read into.
      @param boff  The offset into <code>buf</code> to begin writing with.
      @param bytes The total number of bytes to read.
      @return The actual number of bytes read or -1 if at the End-Of-File.
  */
  @Override
  public int read (byte[] buf, int boff, int bytes) {
    // Except for the three lines marked with "// **" this matches the read(long,int,int,boolean) method
    int status;
    if (!isStream) { // check for end past EOF or CFL and truncate
      int elem = (int)(bytes/dbpe), elements=elem;
      if (cftop+offset+elements>trim1+size) elements = (int)Math.max(0.0,trim1+size-cftop-offset);
      if (cfsize>0 && offset+elements>cfsize) elements = (int)Math.max(0.0,cfsize-offset);
      if (elements<=0 && elem>0) return -1;  // end of data stream
      if (elem!=elements) bytes = (int)(elements*dbpe);
    }
    if ((M.debug&Debug.IO)!=0) M.info("ReadDFb "+tag+" bytes="+bytes); // ** report run-time IO debug
    if (pkh==null) status = io.read (buf, boff, bytes);                // ** read raw data
    else status = pkh.read (this, buf, boff, bytes, (long)0);          // ** read pkt data
    if (status>0) offset = Math.round(status + offset*dbpe) / dbpe;    // Round to nearest byte, see BUG 1173
    else if (status==CoreIO.REOPEN && cmd!=null) cmd.setState(Commandable.RESTART);
    if ((flags&WRAP)!=0 && offset>=cfsize) wrapCircularFile();
 //   if (status>0 && offset>size && !isStream) status -= (offset-size)*dbpe;
    return status;
  }

    /** Read specified bytes at next address. This will copy the data to a native pointer. <b>This
      is a special-purpose method for advanced users only.</b>
      <b>Note:</b> The data read may need to be converted if the file's data rep
      (IEEE/EEEI/...) is different than the OS (machine) data rep. If you want the
      data rep conversion to be done automatically, then call
      {@link #read(long, int, int, boolean)} with true for the convert parameter.
      @param lbuf  Native pointer to buffer.
      @param boff  The offset into <code>buf</code> to begin writing with.
      @param bytes The total number of bytes to read.
      @return The actual number of bytes read or -1 if at the End-Of-File.
  */
  public int read (long lbuf, int boff, int bytes) {
    return read(lbuf, boff, bytes, false);
  }

  /** Read specified bytes at next address. This will copy the data to a native pointer. <b>This
      is a special-purpose method for advanced users only.</b>
      @param lbuf  Native pointer to buffer.
      @param boff  The offset into <code>buf</code> to begin writing with.
      @param bytes The total number of bytes to read.
      @param convert Set to true to convert the data if the data representation
                     does not match the OS (machine) rep.
      @return The actual number of bytes read or -1 if at the End-Of-File.
  */
  public int read (long lbuf, int boff, int bytes, boolean convert) {
    // Except for the three lines marked with "// **" this matches the read(byte[],int,int) method
    int status;
    if (!isStream) { // check for end past EOF or CFL and truncate
      int elem = (int)(bytes/dbpe), elements=elem;
      if (cftop+offset+elements>trim1+size) elements = (int)Math.max(0.0,trim1+size-cftop-offset);
      if (cfsize>0 && offset+elements>cfsize) elements = (int)Math.max(0.0,cfsize-offset);
      if (elements<=0 && elem>0) return -1;  // end of data stream
      if (elem!=elements) bytes = (int)(elements*dbpe);
    }
    if ((M.debug&Debug.IO)!=0) M.info("ReadDFn "+tag+" bytes="+bytes); // ** report run-time IO debug
    if (pkh==null) status = io.read (lbuf, boff, bytes);               // ** read raw data
    else status = pkh.read (this, null, boff, bytes, lbuf);            // ** read pkt data
    if (status>0) offset = Math.round(status + offset*dbpe) / dbpe;    // Round to nearest byte, see BUG 1173
    else if (status==CoreIO.REOPEN && cmd!=null) cmd.setState(Commandable.RESTART);
    if ((flags&WRAP)!=0 && offset>=cfsize) wrapCircularFile();
//    if (status>0 && offset>size && !isStream) status -= (offset-size)*dbpe;

    if (convert && status>0 && dataRep!=Shell.rep) {  // ** convert to local machine data rep
      int elements = (int)(status/dbpe);
      if (dataType==NONHOMOGENEOUS) convertRecordRep(lbuf, boff, elements, dataRep, Shell.rep);
      else Convert.rep(lbuf, boff, dataType, elements*ape*spa, dataRep, Shell.rep);
    }

    return status;
  }

  /** Seek to a given element in a frame of data.
      @param offset   The offset into the file to the start of the frame.
      @param fieldNum The field number (0..n-1).
   */
  private void seekF (double offset, int fieldNum) {
    int fieldOffset = getRecOffset(fieldNum);
    offset += ((double)fieldOffset/dbpe);
    if (offset!=this.offset) seek(offset);
  }

  @Override
  public int seek (double value) {
    offset = value+trim1-cftop; // store in instance variable
    if (cfsize<=0) { } // no WRAP
    else if (offset<0 && cftop>=cfsize) { cftop-=cfsize; offset+=cfsize; }
    else if (offset>=cfsize) { cftop+=cfsize; offset-=cfsize; }
    // seek on an output file acts like a write of 0 bytes
    if (isOutput && offset>wsize) wsize = offset;
    double boffset = offset*dbpe; // calc in bytes
    if (pkh!=null) pkh.seek(this,boffset);
    else if (io!=null) io.seek((long)(firstbyte+boffset+0.1)); // +round to eliminate rounding errors
    return 0;
  }

  @Override
  public double seek () { return offset+cftop-trim1; }

  @Override
  public double avail () {
    if (isStream) {
      double bytes = (double)io.avail();
      if (bytes<=0) return bytes;
      if (bytes<dbpe) return 1;
      return (bytes/dbpe);
    }
    return (trim1+size) - (offset+cftop);
  }

  @Override
  public boolean isReady (double numElements) {
    double availTrimmers = (trim1+size) - (offset+cftop);
    if (availTrimmers < numElements) {
      numElements = availTrimmers; // EOF is "early" due to file trimmers, only check for that number of bytes
    }
    int numBytes = (int)Math.ceil(numElements * dbpe);
    return io.isReady(numBytes);
  }

  /** Return the number of process loops available to read/write data, based on a given transfer size. <br>
      <br>
      For pipes, this is typically the number of bytes available divided by the number of bytes in each transfer.
      <br>
      For static files, the returned value is typically the number of elements divided by the number of elements given in xfer.
      <br>
      In each case, if the number of available bytes or elements is lower than the xfer size, this method will return 1.
      @param xfer Number of elements per process cycle
      @return the number of process loops available to read/write data, based on the given transfer size
   */
  @SuppressWarnings("deprecation") // for usage of pipe fields which will become package-private
  public int processReady (int xfer) {
    if (isStream) {
      double bytes = (double)io.avail();
      double bpx = xfer*dbpe;
      if (bytes<bpx && isPipe() && iop.pipe.inlet==0) return 1; // last read before pipe restart
      if (bytes<=0) return (int)bytes;                          // pipe restart or EOF
      return (int)(bytes/bpx);
    } else {
      double elements = (trim1+size) - (offset+cftop);
      if (elements<xfer) return 1;
      return (int)(elements/xfer); // TODO: should change to ceil for static files otherwise will need to run another process loop?
                                   // JGS: does FFT need this floor to handle partial frames differently?
    }
  }

  /** skip the specified number of elements.
      When skipping beyond the end of file (EOF) this does NOT guarantee zeros to be written,
      although when working with native file resources (NFileResource) this might happen.
      @param elements the number of logical elements to skip without reading
      @return number of elements skipped. For PipeResource it can also return negative when
              the PIPE is {@link CoreIO#CLOSED}, {@link CoreIO#REOPEN}, or {@link CoreIO#ERROR}
   */
  public int skip (int elements) {
    double avail = avail();
    if (avail<0) return (int)avail;
    if (elements<0) elements = (int)avail;

    if (elements<=0) {
      // nothing to do
    }
    else if (iop!=null && pkh==null) {
      // bypass other stuff in seek() for speed on pipes
      offset+=elements;
      iop.seek((long)(firstbyte+(trim1+offset)*dbpe));
    }
    else {
      // use seek() to handle everything else
      seek(cftop+offset+elements-trim1);
    }
    return elements;
  }

  @Override
  public void connect (int flag) {
    if (isPipe()) iop.connect(flag);
  }

  /** handles reOpening this file
   *  @return DataFile that has been reopened
   */
  public DataFile reOpen () {
    close();
    MidasReference ref = cmd;
    if (ref==null) ref = M;
    DataFile df = new DataFile(ref, getFileName());
    df.flags = flags;
    df.open();
    return df;
  }

  /** get the Pipe for this class
   *  @return Pipe for this class
   */
  public Pipe getPipe () {
    if (iop==null) return null;
    return iop.pipe;
  }

  @Override
  public String toString () {
    // since 3.1.1: format is now tag=value and file's basename is included
    FileName fn = getFileName();
    String str = "Type="+getType()
               + " Resource=" + ((io!=null)? io.getTypeString() : " ")
               + " Name=" + ((fn!=null)? fn.getBasename() : "")
               + " Flags="+getFlagsString();
    if (M!=null && (M.debug&Debug.IO)!=0) str += " @"+Integer.toHexString(hashCode()); // <-- unique object hashcode for DEBUG
    return str;
  }

  @Override
  public String listHeader () {
    // Print additional main headers with /ALL switch
    boolean all   = (cmd != null) && cmd.args.getState("/ALL"); // since 3.1.1
    boolean quads = all; // add check for /QUADS to match X-Midas?

    String list = "DataFile ("+getVersion()+")    :  "+ioh.getURL()+"\n";

    if (getDetached()!=0)  list += "DataFile (Detached):  "+io.getURL()+"\n";
    if (getPacket()!=null) list += "Packet Config      :  "+getPacket()+"\n";

    Time    _time         = getTime();      // get time from timecode field + TC_PREC keyword
    boolean hasTimecode   = !_time.isZero() && (getTimeCode() != 0);
    boolean hasTC_PREC    = (getTCPREC() != 0);
    int     decimalPlaces = (hasTC_PREC)? 12 : Time.getDecimalPrecision(_time.getSec()); // picosecond precision if have TC_PREC, otherwise use utility method to figure it out

    if ((cmd != null) && cmd.args.isPresent("/PREC")) { // since 3.3.0: to specify number of decimal places for time strings
      decimalPlaces = cmd.args.getL("/PREC",decimalPlaces);
    }

    if (hasTimecode || hasTC_PREC || all) { // since 3.3.0 also list if /ALL
      list += "Timecode           :  "+_time.toString(decimalPlaces)+"\n";
    }

    int xUnits = getXUnits();
    int yUnits = getYUnits();
    double xStart = getXStart();
    double yStart = getYStart();
    StringBuffer xsAsTimeSb = new StringBuffer(36); // 36 for max of " (1950:02:09::17:59:03.123456789012)"
    StringBuffer ysAsTimeSb = new StringBuffer(36); // 36 for max of " (1950:02:09::17:59:03.123456789012)"

    list += extractStartTime(_time, hasTimecode, hasTC_PREC, decimalPlaces,
                             xUnits, yUnits, xStart, yStart, xsAsTimeSb, ysAsTimeSb);

    list += "Data Rep (H/D)     :  "+getHeadRep()+" / "+getDataRep()+"\n"
          + "Number of Elements :  "+getSize()+"\n"
          + "File Type          :  "+getType()+"\n"
          + "Data Format        :  "+getFormat()+" - "+
              getFormatModeName(getFormatMode())+" "+
              getFormatTypeName(getFormatType())+"\n"
          + "Write Protect      :  "+getProtected()+"\n"
          + "Abscissa units     :  "+getUnitsName(xUnits)+"\n"
          + "Abscissa start     :  "+xStart+xsAsTimeSb+"\n"
          + "Abscissa delta     :  "+getXDelta()+"\n";

    if (all || (getUnits() == Units.TIME_S)) {
      if (typeClass == 1) list += "Sample Rate (1/xd) :  "+(1/getDelta())+" Hz\n";
      if (typeClass == 2) list += "Frame Rate  (1/yd) :  "+(1/getDelta())+" Hz\n";

      list += "Duration (sz*delta):  "+(getSize() * getDelta())+" sec\n";
      list += "Duration (end-start): "+getTimeDuration()       +" sec\n";
    }

    if (typeClass == 2) {
      list += "Frame Length       :  "+getSubSize()+"\n"
            + "Secondary units    :  "+getUnitsName(yUnits)+"\n"
            + "Secondary start    :  "+yStart+ysAsTimeSb+"\n"
            + "Secondary delta    :  "+getYDelta()+"\n";
    }

    if (typeClass == 3) {
      list += "Number Subrecords  :  "+getSubSize()+"\n"
            + "Bytes per Record   :  "+getRecLength()+"\n";
      for (int i=0; i<getSubSize(); i++) {
        list += "SubRecord Name="+getRecName(i)+"  Format="+getRecFormat(i)+
                "  Offset="+getRecOffset(i)+"\n";
      }
    }

    if (typeClass == 6) {
      list += "Number Subrecords  :  "+getSubSize()+"\n"
            + "Bytes per Record   :  "+getRecLength()+"\n";
      for (int i=0; i<getSubSize(); i++) {
        list += "SubRecord Name="+getRecName(i)+"  Format="+getRecFormat(i)+
                "  Offset="+getRecOffset(i) + "  Elem="+getRecElements(i)+
                "  Min="+getRecMinimum(i)+"  Max="+getRecMaximum(i)+
                "  Units="+getRecUnitsName(i) + "\n";
      }
    }

    if (typeClass == 5) {
      list += "Number Components  :  "+getSubSize()+"\n"
            + "Bytes per Record   :  "+getRecLength()+"\n"
            + "Frame of Reference :  "+getReferenceFrame()+"\n";
      if (quads || getReferenceFrame().startsWith("TOP")) {
        list += "  Altitude         :  "+getQuadword(Q_ALT)+" meters\n"
              + "  Latitude         :  "+getQuadword(Q_LAT)+" deg\n"
              + "  Longitude        :  "+getQuadword(Q_LON)+" deg\n";
      }
      if (quads || getReferenceFrame().equals("TOP")) {
        list += "  Azimuth          :  "+getQuadword(Q_AZIM)+" deg\n"
              + "  Elevation        :  "+getQuadword(Q_ELEV)+" deg\n"
              + "  Roll             :  "+getQuadword(Q_ROLL)+" deg\n";
      }
      if (quads || getReferenceFrame().equals("ECI") || getQuadword(Q_YEAR)!=0) {
        list += "  Epoch Year       :  "+getQuadword(Q_YEAR)+"\n"
              + "  Epoch Seconds    :  "+getQuadword(Q_SEC)+"\n"
              + "  Epoch GHA        :  "+getQuadword(Q_GHA)+" radians\n";
      }
      for (int i=0; i<getSubSize(); i++) {
        list += "Component Name="+getRecName(i)+"   Format="+getRecFormat(i)+
                "  Type="+getRecCompTypeName(i)+
                "  Units="+getRecUnitsName(i)+"\n";
      }
    }

    int ihs=getKeyLength();
    if (ihs>0 || all) {
      list += "IntHeader KeySize  :  "+ihs+" bytes\n";
    }

    int extHdrSize=getExtSize();
    if (extHdrSize>0 || all) {
      list += "ExtHeader Size     :  "+extHdrSize+" bytes\n";
    }

    if (io != null && isPipe()) {
      list += "Pipe Size          :  "+iop.pipe.getSize()+" bytes\n";
      list += "Pipe inByte/outByte:  "+getInByte()+" / "+getOutByte(0)+"\n";

      for (int i = 0; i <= iop.pipe.getMaxOutlets(); i++) {
        String pName = (i==0)? "Inlet": "Outlet "+i;
        String pStat = "<not attached>";

        if (iop.pipe.isAttached(i)) {
          pStat = ""+getOutByte(i) + " / " + Convert.ref2String(iop.pipe.getOwner(i));
        }
        list += "  outByte/"+StringUtil.padRight(pName,9)+":  "+pStat+"\n";
      }
    }

    if ((flags&WRAP)!=0) {
      list += "CirFile inByte/size:  "+getInByte()+" / "+getOutByte(0)+"\n";
    }

    // since 3.1.1: print additional main headers for debugging when /ALL switch is given
    if (all) {
      list += "ExtHeader Start    :  "+getExtStart()+" (512-byte) blocks\n"
            + "Data start         :  "+getDataStart()+" bytes\n"
            + "Data size          :  "+getDataSize()+" bytes\n"
            + "Detached header    :  "+getDetached()+"\n";
      // below fields are only used by X-Midas pipes and is NOT applicable
      // to BLUE files (but possbily could be set by SOURCEFILE/RT in future)
      list += "pipe mode          :  "+getPipeId()+"\n"
            + "flagmask           :  "+getDataMask()+"\n"
            + "inlet              :  "+getInlet()+"\n"
            + "outlets            :  "+getOutlets()+"\n"
            + "outmask (async)    :  "+getOutMask()+"\n"
            + "pipeloc            :  "+getPipeLoc()+"\n";

      if (!isPipe()) { // these values are included above when IO is a pipe
        list += "pipesize           :  "+getPipeSize()+" bytes\n"
              + "in_byte            :  "+getInByte()+"\n"
              + "out_byte           :  "+getOutByte(0)+"\n";
      }
      // end fields are only used by X-Midas pipes and is NOT applicable to
      // BLUE files (but possbily could be set by SOURCEFILE/RT in future)
    }

    try {
      Object _comment = keywords.get("COMMENT");
      if (_comment!=null) list += "Comment            :  "+_comment+"\n";
    }
    catch (Exception e) {
      // continue to list header even if we have corrupted keyword block to fix BUG 2141
      M.warning("DataFile: Unable to list 'COMMENT' keyword for " + getFileName().getFullName());
    }

    return list;
  }

  /** Extract/get Start Time of file and update xsAsTimeSb and ysAsTimeSb for {@link #listHeader()}.
      @param time       Initially Time object from getTime() (i.e. from Timecode + TC_PREC)
      @param xsAsTimeSb X Start as Time StringBuffer (output)
      @param ysAsTimeSb Y Start as Time StringBuffer (output)
      @return string to append to listHeader / STATUS output. Empty string if no start time.
      @since NeXtMidas 3.1.1 (extracted from listHeader)
   */
  private String extractStartTime (Time time, boolean hasTimecode, boolean hasTC_PREC,
      int decimalPlaces, int xUnits, int yUnits, double xStart, double yStart,
      StringBuffer xsAsTimeSb, StringBuffer ysAsTimeSb) {
    Time startTime;
    StringBuilder startTimeSource = new StringBuilder(23); // 23 for max of "(TimeCode+TC_PREC+XStart)"

    if (timeLine!=null) {
      startTime = timeLine.getTimeAt(0, null); // get start time from TimeLine (i.e. at offset 0)
      startTimeSource.append("TimeLine");
      if (timeLine.format != TimeLine.Format.MICROSECOND_PRECISION) decimalPlaces = 12; // since 3.3.0
    }
    else {
      startTime = new Time(time);
      if (hasTimecode) startTimeSource.append("TimeCode");
      if (hasTC_PREC) {
        if (startTimeSource.length()>0) startTimeSource.append('+');
        startTimeSource.append("TC_PREC");
      }
      if (typeClass==2) {
        if (yUnits==Units.TIME_S && yStart!=0) {
          Time yTime = Time.toTimeAndStringBuffer(yStart, ysAsTimeSb, " (", ")");
          startTime.addSec(yTime.getSec());
          if (startTimeSource.length()>0) startTimeSource.append('+');
          startTimeSource.append("YStart");
        }
      }
      else {
        if (xUnits==Units.TIME_S && xStart!=0) {
          Time xTime = Time.toTimeAndStringBuffer(xStart, xsAsTimeSb, " (", ")");
          startTime.addSec(xTime.getSec());
          if (startTimeSource.length()>0) startTimeSource.append('+');
          startTimeSource.append("XStart");
        }
      }
    }
    if (startTimeSource.length()>0) {
      decimalPlaces = Math.max(decimalPlaces, Time.getDecimalPrecision(startTime.getSec()));
      return "Start time         :  "+startTime.toString(decimalPlaces)+" ("+startTimeSource+")\n";
    }
    return "";
  }

  @Override
  public String listKeywords () {
    StringBuilder sb = new StringBuilder(4096);
    for (Map.Entry<String,Object> kwd : keywords.getAll("MAIN", false)) {
      sb.append(Keywords.list(kwd, "MAIN", 18, 0)).append('\n');
    }
    for (Map.Entry<String,Object> kwd : keywords.getAll("EXT", false)) {
      sb.append(Keywords.list(kwd, "EXT", 18, 0)).append('\n');
    }
    return sb.toString();
  }

  /**
   * WARNING: for recorded-based files (e.g. type 3000, 5000, 6000) current implementation
   *          only lists the first record/element read.
   */
  @Override
  public String listElements (double start, int elements, String format, int flags) {
    String line=""; int i,ioff,len=0, radix=flagsToRadix(flags);
    Data data = getDataBuffer(elements);
    seek(start);
    elements = read(data,elements);
    if (elements<=0) return null;
    switch (typeClass) {
      case 1: default:
        // Updated in NeXtMidas 4.1.2 to use different API which allows
        // scalar and element separators to be specified
        line = data.toString(null,0,elements,radix,true,scalarSeparator,elementSeparator,false).toString();
        break;
      case 2:
        int ape=data.ape; data.ape=1;
        int j=getSubSize(), j1=Math.max(1,j-4);
        line = ")";  // construct end of array line
        for (; j>j1; ) {
          String s = data.toString((j-1)*data.spa,1,radix);
          if (line.length()+s.length()>listLineWidth/2) break;
          line = ','+s+line; j--;
        }
        String eline = line;
        line = "(";  // construct start of array line
        for (i=0; i<j; i++) {
          String s = data.toString(i*data.spa,1,radix);
          if (line.length()+eline.length()+s.length()+4>listLineWidth) break;
          line += s; line += ',';  // +4 is command and 3 dots
        }
        // merge start and end if possible
        if (i==j) line += eline.substring(1);
        else line += "..."+eline;
        data.ape = ape;
        break;
      case 3:
      case 6:
        for (i=0,ioff=0; i<getSubSize(); i++) {
          ioff = getRecOffset(i);
          int n = getRecElements(i);
          Data subdata = data.getAtomAt(ioff,getRecFormat(i),n);
          String subtext = getRecName(i);
          if (subdata.type==DOUBLE && subtext.equals("TIME") && subdata.toD()>1e6)
            subtext += "="+Time.format(subdata.toD(),Time.STD,6);
          else
            subtext += "="+subdata.toString(0,subdata.size,radix);
          if (len==0) { }
          else if (len+subtext.length()>listLineWidth) { line+="\n     "; len=5; }
          else line+="  ";
          line += subtext; len += subtext.length();
        }
        break;
      case 4:
        break;
      case 5:
        for (i=0,ioff=0; i<getSubSize(); i++) {
          Data subdata = data.getAtomAt(ioff,getRecFormat(i));
          String subtext = getRecName(i)+"="+subdata.toString(0,subdata.size,radix);
          if (len==0) { }
          else if (len+subtext.length()>listLineWidth) { line+="\n     "; len=5; }
          else line+="  ";
          line += subtext; len += subtext.length();
          ioff += subdata.bpe;
      }
      break;
    }
    return line+"\n";
  }

  @Override
  public int listElementsPerLine (int lineWidth, String form, int flags) {
    this.listLineWidth = lineWidth;
    int ncols;
    byte formatType = getFormatType();
         if (formatType==ASCII   ) ncols=(int)((lineWidth-6)/(1+dbpe));
    else if (formatType==ASCII256) ncols=(int)((lineWidth-6)/(1+dbpe)); // 3.9.4 NXM-3570
    else if ((flags&RADIX_16)!=0 ) ncols=(lineWidth-6)/(3+Math.max(1,2*bps));
    else if ((flags&RADIX_2 )!=0 ) ncols=(lineWidth-6)/(3+Math.max(1,8*bps));
    else if (formatType==NIBBLE  ) ncols=lineWidth/5;
    else if (formatType==BYTE    ) ncols=lineWidth/5;
    else if (formatType==INT     ) ncols=lineWidth/7;
    else if (formatType==JINT    ) ncols=lineWidth/7;    // NXM-3894 Jeff 12-bit J
    else if (formatType==LONG    ) ncols=lineWidth/10;
    else if (formatType==FLOAT   ) ncols=lineWidth/10;
    else if (formatType==DOUBLE  ) ncols=lineWidth/20;
    else if (formatType==PACKED  ) ncols=lineWidth/9*8;
    else                          ncols=8;

    if (typeClass>=2) ncols = -getSubSize();
    else              ncols = Math.max(1,ncols/getSPA());

    if (formatType==PACKED) ncols=Math.max(1,ncols/8)*8;
    return ncols;
  }

  /** Read keywords from file into memory.
      As of NeXtMidas 2.8.1, exception is caught and warning is printed inside
      this method to resolve BUG 2141. This allows opening a BLUE file with corrupted
      keywords in the extended headers beyond 1024 offset from start of extended
      header and reading the main header.
      @return size of keyword, negative on error
   */
  private int readKeywords (String origBlueVer, String origBlueIO) {
    int extSize  = getExtSize();
    int extStart = getExtStart();
    long off = ((long)extStart)*BYTE_BLOCK_SIZE;
    try {
      if (extSize==0) keywords.init(null,0,headRep);
      else if (iop!=null) keywords.init(iop.pipe.getDataFile().keywords);
      else {
        int len = (extSize+511)/BYTE_BLOCK_SIZE*BYTE_BLOCK_SIZE;
        byte[] buf = new byte[len];
        int bytes = ioh.read(buf,0,extSize,off);
        if (bytes!=extSize) M.warning("Size mismatch in reading extended header for file=" + getFileName().getFullName());
        keywords.init(buf,bytes,headRep);
      }
      if (typeClass==6) t6s = readT6def(true);
      Data tld = (Data)keywords.get("TIMELINE");

      // Read TimeLine Keywords
      // Allow other keywords to be read, even when the TimeLine ERRORS out
      try {
        if (tld!=null) newTimeLineFromData(tld);
        handlePlatinumTimelineKeywords(origBlueVer, origBlueIO); // since 3.3.0 process/check Platinum timeline keywords for BUG 2675
        handleBlueTCTimelineKeywords(origBlueVer, origBlueIO);   // since 3.7.2 process/check BLUE TC timeline keywords for NXM-3010
        if (timeLine != null) { // since 3.9.4 set timeLine format
          timeLine.format = overrideTimeLineFormat(timeLine.format); // switch to new override format if needed
        }
      }
      catch (MidasException me) {
        M.warnStackTrace("Invalid TimeLine for file:"+this.getFileName(), me);
      }

      if (isRecordBased()) { // load long subrecord names
        String scope = keywords.getScope();
        keywords.clearScope();
        Keywords.Iterator iter = keywords.iterator("SECTION-=SUBRECORD_NAMES");
        if (iter != null) {
          initRecNames();
          while (iter.hasNext()) {
            Keywords.Key key = iter.next();
            int sr = Convert.s2l(key.name.substring(2)) - 1; // -1 to convert to zero-based
            recNames[sr] = key.value.toString();
          }
        }
        keywords.setScope(scope);
      }
    }
    catch (Exception e) {
      extSize = -1; // signal error with negative return value
      M.warning("DataFile: Unable to read keywords for file=" + getFileName().getFullName()
          +" reason="+ (e instanceof MidasException? e.getMessage() : e));
      if ((M.debug&Debug.IO)!=0) M.printStackTrace(e);
    }
    if (allowablePlatinum && !Convert.o2z(getFileName().getQualifier("FORCE"))) {
      M.info("Performing auto-conversion Platinum to Blue on file:"+filename);
      Table plat2blueTable = Table.fromTableFile(M, "nxm.sys.dat.kwconv_plat2blue.tbl");
      plat2blueTable.put("_WARN_MISSING_", true);
      FileUtil.convertKeywordsInPlace(M, this, plat2blueTable);
      keywords.putMain("VER", BLUE_VER_DEFAULT, true);
    }
    return extSize;
  }

  protected int writeKeywords () {
    // Update TIMELINE (if applicable)
    if (timeLine!=null && timeLine.getSize()>0 && iop==null  && !sameTimelineKeywords && !keywords.getScope().equals("MAIN")) {
    //if (timeLine!=null && timeLine.getSize()>0 && iop==null  && !sameTimelineKeywords) {
      boolean debug = (M.debug&Debug.IO) != 0;
      if ((flags&WRAP)!=0) timeLine.purge(getInByte()-getOutByte(0));
      if (usePlatinumTimeline()) { // since 3.3.0: use existing Platinum high precision timeline keywords (BUG 2675)
        if (debug) M.info("DEBUG: writing high precision Platinume time line keywords... to "+getName());
        double defTimeDelta = 0.0; // default to 0.0 to use seconds since epoch (if TIME_DELTA keyword is not set)
        Time timeEpoch = null;
        Object tmpobj = keywords.get("TIME_EPOCH");
        if (tmpobj instanceof String) { // epoch time specified
          try {
            timeEpoch = new Time((String)tmpobj, Time.ISO8601);
            defTimeDelta = 1e-9;        // default to nanosecond precision
          } catch (Exception e) {
            M.warning("Invalid TIME_EPOCH ["+tmpobj+"] specified! Falling back to start of today for epoch. "+e);
          }
        }
        if (timeEpoch == null) {
          // no epoch, use file start time as epoch
          // Note: The high precision is not used as epoch is only stored in whole seconds
          timeEpoch = getTimeAt(0, null);
          defTimeDelta = 0.0;    // default to seconds since file start time as epoch (allows for very high precision)
        }
        double timeDelta = keywords.getD("TIME_DELTA", defTimeDelta);
        Map<String,Object> tlKeywords = timeLine.toPlatinumKeywords(timeEpoch, timeDelta, dbpe);
        keywords.delete("TIMELINE"); // get rid of these if we are switching to high precision
        keywords.putAll(tlKeywords);
      }
      else if (useBlueTCTimeline()) {
        if (debug) M.type("DEBUG: writing high precision BLUE TC time line keywords... to "+getName());
        Map<String,Object> tlKeywords = timeLine.toBlueTCKeywords(dbpe);
        keywords.delete("TIMELINE"); // get rid of these if we are switching to high precision
        keywords.putAll(tlKeywords);
      }
      else { // use microsecond precision TIMELINE keyword
        if (debug) M.type("DEBUG: writing microsecond precision TIMELINE keyword... to "+getName());
        keywords.put("TIMELINE", timeLine.toData());
      }
    }

    // Update Type 6000 subrec definitions (if applicable)
    if (typeClass==6) {
      int len = getSubSize();
      checkT6cap(len);
      keywords.put("SUBREC_DESCRIP","TYPE0");
      keywords.put("SUBREC_DEF",t6s.substring(0,len*96)); // don't save "extra" entries
    }

    // Update long subrecord names in extended header (clear if not applicable)
    if (isRecordBased()) {
      String scope = keywords.getScope();
      keywords.clearScope();
      keywords.deleteScope("SECTION=-SUBRECORD_NAMES"); // delete old record names, clears current scope
      if (recNames != null) { // save long record names
        int     max = (t6s==null)? 4 : 24;
        boolean any = false;
        keywords.setScope("EXT");
        for (int sr = 0; sr < getSubSize(); sr++) {
          if (recNames[sr].length() <= max) continue; // short name -> don't save
          if (!any) {
            keywords.add("SECTION", "SUBRECORD_NAMES");
            any = true;
          }
          keywords.add("SR"+(sr+1), recNames[sr]); // +1 to convert to one-based
        }
        if (any) keywords.add("SECTION", "END");
      }
      keywords.setScope(scope);
    }

    // Write out keywords
    int size = keywords.size;
    int start = (hb.length+511)/HEADER_SIZE;
    if (getDetached()==0) start = (int)((getDataStart()+getDataSize()+511)/HEADER_SIZE);
    if (size == 0) start = 0;  // JC4: if size=0 start must be 0 as per BLUE File CCD
    String kwsb = keywords.getMain("KWSB"); // since 3.7.0 for BUG 2847
    if (kwsb!=null) start = Convert.s2l(kwsb);
    setExtSize(size);
    setExtStart(start);
    if (size==0 || iop!=null) return 0;  // PIPE or no keywords
    long coffset = ioh.seek(); // save off current data seek
    long offset = ((long)start)*HEADER_SIZE;
    int len = (size+511)/BYTE_BLOCK_SIZE*BYTE_BLOCK_SIZE;
    int bytes = ioh.write (keywords.buf,0,len,offset);
    if (io==ioh) ioh.seek(coffset); // restore current data seek
    if (bytes!=len) M.warning("Err writing extended header to "+getName()+" wrote="+bytes+" expected="+len+" bytes");
    return bytes;
  }

  /** set KWSB Main Header Keyword which is used to set start position of the
   *  extended header (in 512-octet blocks).
   *  @param blk value for KWSB Main Header Keyword
   *  @since NeXtMidas 3.7.0
   */
  public void setKWSB (int blk) {
    keywords.putMain("KWSB",""+blk,false);
  }

  /** Initialize the set of record names based on current short names. */
  private void initRecNames () {
    int subs = getSubSize();
    int min  = (typeClass == 5)? 14 : 26;
    int num  = (subs > min)? subs : min;
    recNames = new String[num];

    for (int i = 0; i < subs; i++) {
      recNames[i] = getRecName(i,true);
    }
  }

  /** Get default detached file name (with extension).
   *  @return default detached file name (with extension)
   */
  public String getDetachName () {
    String dname = io.getURL();
    int i = dname.lastIndexOf('.');
    if (i<0) return dname;
    return dname.substring(0,i+1)+getDetachExt();
  }

  /** Get default detached name extension.
   *  @return default detached name extension
   */
  public String getDetachExt () {
    int lval = getDetached();
//jph -- 31jul08 -- According to XM DETACH explain file, a -1 in the detach field
//jph    if (lval>=0 && lval<128) return "det";
    if (lval == -1 || (lval>=0 && lval<128)) return "det";
    else return unpackS(12,4).toLowerCase();
  }

  /** Set keyword scope
   *  @param scope keyword scope
   */
  public void setKeyScope (String scope) {
    keywords.setScope(scope);
  }
  /** Get keyword scope
   *  @return keyword scope
   */
  public String getKeyScope () {
    return keywords.getScope();
  }
  /** Get a named keyword
   *  @param name name of keyword
   *  @return value of named keyword
   */
  public Object getKeyWord (String name) {
    return keywords.get(name);
  }
  /** Get a named keyword with scope
   *  @param name name of keyword
   *  @param scope scope within extended header
   *  @return value of named keyword within specified scope
   */
  public Object getKeyWord (String name, String scope) {
    keywords.setScope(scope);
    Object value = keywords.get(name);
    keywords.clearScope();
    return value;
  }

  /** Get an element of Data from the file
      @param offset The offset (0 to n-1) into the file
      @return The element of Data from the file or null if the offset is at/past
              the end of file (EOF). <i>Prior to NeXtMidas 3.3.0 this function
              returned a {@link Data} object regardless of whether EOF had been
              reached, but in the event that EOF had been reached the {@link Data}
              object would contain a buffer with junk values. The current behavior
              matches {@link #getDataTable(double)}.</i>
   */
  public Data getData (double offset) {
    Data data = getDataBuffer(1);
    int  ok   = read(data,offset,1);
    // Bug 2667: Should return null at EOF (following Bug 2666, failure to return
    // null at EOF will cause an OutOfMomoryError in %test_file_byte_order)
    // return data;
    return (ok < 0)? null : data;
  }

  /** Get an object of data from the file. For record-based files (see {@link #isRecordBased()})
      this will be a the same as {@link #getDataTable(double)}; for all others this will be the
      same as {@link #getData(double)}
      @param offset The offset (0 to n-1) into the file.
      @return The data at the given offset.
   */
  private Object getDataObject (double offset) {
    if (isRecordBased()) {
      return getDataTable(offset);
    } else {
      return getData(offset);
    }
  }

  @Override
  public Table getDataTable (double offset) {
    Table tbl = new Table();
    int   ok  = getDataTable(offset,tbl);
    return (ok < 0)? null : tbl;
  }

  @Override
  public int readDataTable (Table tbl) {
    tbl.clear();
    return getDataTable(seek(),tbl);
  }

  private int getDataTableOld (double offset, Table tbl) { // old impl of getDataTable(..) prior to 3.7.0
    if (offset < 0) return CoreIO.ERROR;

    if ((typeClass == 1) || (typeClass == 2)) {
      // Use a single read/convert for frame-based data.
      Data data = getData(offset);
      if (data == null) return CoreIO.CLOSED;
      if (data.type==PACKED || data.type==NIBBLE) data = getBitsFromByte(data, offset);  // BUG 1593
      for (int i = 0; i < data.getAPE(); i++) {
        tbl.put(("F"+(i+1)), data.getAtomAt(i));
      }
    }
    else {
      // need to read each element individually for record-based
      for (int i=0; i<getSubSize(); i++) {
        Data data = getData(offset,i);
        if (data == null) return CoreIO.CLOSED;
        if (data.type==PACKED || data.type==NIBBLE) data = getBitsFromByte(data, offset);  // BUG 1593
        tbl.put(getRecName(i), data);
      }
    }
    return 1;
  }

  /** Read a row of data into a table. */
  private int getDataTable (double offset, Table tbl) {
    if (offset < 0) return CoreIO.ERROR;
//  if (offset>=size) return CoreIO.CLOSED; // TODO: ask Jeff why this was added, as it caused type 3000 pipes to fail (run %test_plot_layermulti)

    Data data = getData(offset);  // converts Rep
    if (data == null) return CoreIO.CLOSED;

    if ((typeClass == 1) || (typeClass == 2)) {
      // Use a single read/convert for frame-based data.
      if (data.type==PACKED || data.type==NIBBLE) data = getBitsFromByte(data, offset);  // BUG 1593
      if (data.type==JINT && (dbpe==1.5)) data = getDataFromSJ(data, offset);
      for (int i=0; i<data.getAPE(); i++) {
        tbl.put(("F"+(i+1)), data.getAtomAt(i));
      }
    }
    else {
      // need to read each element individually for record-based
      for (int i=0; i<getSubSize(); i++) {
        int fieldOffset = getRecOffset(i);
        int numElements = getRecElements(i);
        String fieldName = getRecName(i);
        String fieldFormat = getRecFormat(i);
        Data dataf = null;
        char formatType = fieldFormat.charAt(1);
        if (!minimalSupportForUnsupportedDataTypes || (numElements > 1) || !((formatType == 'U') || (formatType == 'V'))) {
          dataf = new Data(fieldFormat,numElements);
          System.arraycopy(data.buf,fieldOffset, dataf.buf,0, dataf.buf.length);
          if (dataf.type==PACKED || dataf.type==NIBBLE) dataf = getBitsFromByte(dataf, 0);  // BUG 1593
        }
        else {
          // Minimal support for records data of unsigned types 'U' and "V' added in 3.9.3 for NXM-3021
          // note: this could possibly expanded to allow other undefined types to become strings
          // TODO: Add support for type 6000 with multiple elements per column
          dataf = moveUnsignedIntoLargerSigned(fieldFormat,data,numElements,fieldOffset);
        }
        tbl.put(fieldName, dataf);
      }
    }
    return 1;
  }

  /** Adds minimal support of unsigned record data by moving unsigned data into larger signed type
   *  Only designed to be called with 'U' or "V" type data
   *
   *  @param fieldFormat the format for this field in the record
   *  @param data        the data object holding 'U' or "V' data
   *  @param numElements NEEDS to be 1 for now ... can add support for more later
   *  @param fieldOffset offset of this field
   *  @return Data object containing signed values matching the original Data objects unsigned
   *  @since 3.9.3
   */
  private Data moveUnsignedIntoLargerSigned (String fieldFormat, Data data, int numElements, int fieldOffset) {
    char mode = fieldFormat.charAt(0);
    char type = fieldFormat.charAt(1);
    String newFormat = (type == 'U') ? mode + "L" : mode + "X";
    int spa = Data.getSPA(mode);
    int bps = Data.getBPS(type);
    Data dataf = new Data(newFormat,numElements,0); // note: type 6000 could be an additional issue # elements needs to be 1 for this logic
    for (int s = 0; s < spa; s++) {
      System.arraycopy(data.buf,fieldOffset+(s*bps), dataf.buf, s*bps*2, bps);
    }
    return dataf;
  }

  /** Grab the relevant bits out of a byte. This is only called from {@link #getDataTable(double, Table)}
      and only when the data is contained in a single byte (PACKED bits and NIBBLE data)
      @param data The data object
      @param offset The offset
      @since NeXtMidas 2.7.3
   */
  private Data getBitsFromByte (Data data, double offset) {
    if (data.type==PACKED || data.type==NIBBLE) {
      byte newByte;
      if (data.type==PACKED) {
        int start = (int) (offset % 8);
        if (getDataRep().equalsIgnoreCase("IEEE")) start = 7 - start; // adjust offset if IEEE
        /* Code below taken from Convert.unpackP but used here without the method call (for perf) */
        newByte = (byte)((data.buf[(start)>>3]>>((start)&0x7))&0x1);
      }
      else {
        int start = (int) (offset % 2);
        // TODO: BUG 709: UNcomment the line below when we are ready to fully support NIBBLE
        //if (getDataRep().equalsIgnoreCase("IEEE")) start = 1 - start; // adjust offset if IEEE
        /* Code below taken from Convert.unpackN but used here without the method call (for perf) */
        newByte = (byte)((data.buf[(start)>>1]<<((((start)&1)==0)?28:24) )>>28);
      }
      return new Data(newByte);
    }
   return data;  // if this method was called for non-nibble or packed data, just return the original
  }

  /** Similiar to PACKED and NIBBLE, JINT needs special unpacking
   *
   *  @param data   Data object with 'J' data
   *  @param offset offset into data buffer
   *  @return new INT Data object
   *  @since NeXtMidas 4.1.2 for NXM-3894
   */
  private Data getDataFromSJ (Data data, double offset) {
    Data newData;
    if (offset%2==0) newData = new Data((short)((data.buf[1]<<12)| ((0xFF&data.buf[0])<<4))); // convert to INT
    else newData = new Data((short)(data.buf[1]<<8 | (0xF0&(data.buf[0])))); // convert to INT
    return newData;
  }

  /** Writes a row of data to the end of file.
      @param tbl The table to put the data into.
      @return &gt;0 if successful, &lt;0 if unsuccessful.
      @since NeXtMidas 2.5.4
   */
  public int writeDataTable (Table tbl) {
    return setData(seek(),tbl);
  }

  /** This is the same as using <code>getDataTable(offset)</code>.
   *  @param offset The offset (0 to n-1) into the file. This must be an integral value (the input
   *                is a double rather than a long is done for historical reasons).
   *  @return The data at the given offset or null if at end of the file.
   */
  public final Table getDataT (double offset) { return getDataTable(offset); }

  /** This is the same as using <code>getDataTable(offset)</code>.
   *  @param offset The offset (0 to n-1) into the file. This must be an integral value (the input
   *                is a double rather than a long is done for historical reasons).
   *  @return The data at the given offset or null if at end of the file.
   */
  public final Table getRecord (double offset) { return getDataTable(offset); }

  /** Get an element of Data from a named field in a record. This is the same as
      <code>getData(offset,findRec(fieldName))</code>.
      @param offset    The offset (0 to n-1) into the file.
      @param fieldName The name of the field/record.
      @return The element of data of null if the field name is invalid.
   */
  public Data getData (double offset, String fieldName) {
    return getData(offset,findRec(fieldName));
  }

  /** Get an element of Data from with a field index in a record.
      @param offset   The offset (0 to n-1) into the file.
      @param fieldNum The index of the field/record (0..N-1).
      @return The element or data of null if the field number is invalid.
      @since NeXtMidas 2.7.0
   */

  public Data getDataZ (double offset, int fieldNum) {
    Data data = null;

    if ((fieldNum >= 0) && (fieldNum < getRecordDefCount())) {
      String fieldFormat = getRecFormat(fieldNum);
      int numElements = getRecElements(fieldNum); // since 3.6.0 to properly read all element data in Type 6000 files with num_elts > 1 (Bug 2958)

      data = new Data(fieldFormat, numElements);

      seekF(offset,fieldNum);

      int status = read(data.buf,0,data.buf.length);
      if (status != data.buf.length) {
        data = null;
      } else if (data.rep!=dataRep) {  // convert to proper data rep.
        Convert.rep(data.buf, 0, fieldFormat, numElements, dataRep, data.rep);
      }
    }
    return data;
  }

  /** gets a Data element from the file
      @param offset   (0...n-1) into the file
      @param fieldNum the index of a subrecord (0...n-1)
      @return the element of Data from the file
   */
  public Data getData (double offset, int fieldNum) {
    if (fieldNum<0 || fieldNum>=getRecordDefCount()) return null;

    // dont want to support fractional seek, so read one whole element
    Data data = getData(offset);
    if (data == null) return null;

    int fieldOffset = getRecOffset(fieldNum);
    int numElements = getRecElements(fieldNum);
    String fieldFormat = getRecFormat(fieldNum);
    Data dataf = new Data(fieldFormat,numElements);
    System.arraycopy(data.buf,fieldOffset, dataf.buf,0, dataf.buf.length);

    return dataf;
  }

  @Override
  public void insertData (double offset, Object obj) {
    if (offset < 0) { offset = getSize(); }

    Data buf = getDataBuffer(1);
    while (getSize() < offset) {
      write(buf, getSize()); // Increases file size by one
      setSize(wsize);        // Forces the file size to update.
    }
    write(buf, getSize());   // Increases file size by one
    setSize(wsize);          // Forces the file size to update.

    for (double pos = getSize()-2.0; pos >= offset; pos--) {
      setData(pos+1,getDataObject(pos)); // Copy the data from (pos) to (pos+1)
    }
    setData(offset,obj); // Writes the new data
  }

  @Override
  public void removeData (double offset) {
    removeData(offset, 1);
  }

  @Override
  public void removeData (double offset, long count) {
    if ((offset < getSize()) && (count != 0L)) {
      if (wsize==0) wsize = getSize(); // If this is not set, we need to set it to getSize() so that it works correctly - Since 3.9.1
      if (count > 0L) {
        for (double pos = offset; pos < (getSize() - count); pos++) {
          setData(pos,getDataObject(pos+count));
        }
        wsize = wsize - count; // Decreases file size by count
      } else {
        wsize = offset;        // Decreases file size to match offset
      }
      setSize(wsize); // Forces the file size to update.
    }
  }

  /** Set an element of Data in the file. If the data is a {@link Table} then it will be inserted
      using {@link #setData(double,Table)}, otherwise the object will be converted to the
      appropriate data type (numeric or ASCII) and inserted at the given position.
      @param offset The offset (0 to n-1) into the file.
      @param obj    The data to insert.
   */
  public void setData (double offset, Object obj) {
    if (obj instanceof Table) {
      setData (offset, (Table)obj);
    } else {
      Data data = Convert.o2Data(obj,(char)dataType,M);
      write (data,offset,1);
    }
  }

  /** Sets an element of Data in a field of a file.
      @param offset    The offset (0 to n-1) into the file.
      @param fieldName The name of the field.
      @param obj       The data.
   */
  @Override
  public void setData (double offset, String fieldName, Object obj) {
    int fieldNum = findRec(fieldName);
    if (fieldNum >= 0) {
      setData (offset, fieldNum, obj);
    } else {
      M.warning("DataFile: Invalid Field name '"+fieldName+"'");
    }
  }

  /** Sets an element of Data in a field of a file. This method is usually called by
      {@link #setData(double,String,Object)}, which takes in a field name, rather than the index.
      @param offset   The offset (0 to n-1) into the file.
      @param fieldNum The field index (should get from the name).
      @param obj      The data.
   */
  public void setData (double offset, int fieldNum, Object obj) {
    if ((fieldNum >= 0) && (fieldNum < getRecordDefCount())) {
      String fieldFormat = getRecFormat(fieldNum);
      int    numElements = getRecElements(fieldNum); // since 3.6.0 for Bug 2959
      if (minimalSupportForUnsupportedDataTypes) {
        char type = fieldFormat.charAt(1);
        if (type=='U' || type=='V') {
          M.warning("DataFile: No support for modifying type "+type+" fields");
          return;
        }
      }
      Data   data        = Data.fromObject(obj, fieldFormat, numElements, M);

      if (data.rep!=dataRep) {  // convert to proper data rep.
        Convert.rep(data.buf, 0, fieldFormat, numElements, dataRep, data.rep);
      }

      seekF(offset,fieldNum);
      write(data.buf,0,data.buf.length);
    } else {
      M.warning("DataFile: Invalid Field number "+fieldNum);
    }
  }

  @Override
  public int setData (double offset, Table table) {
    for (int i=0; i<getRecordDefCount(); i++) {
      setData(offset,i,table.get(getRecName(i)));
    }
    return 1; // <-- This matches NeXtMidas 2.3.1.
  }

  /** Gets the frame size of a Type 1000/2000 file.
      @return The frame size.
      @see #setFrameSize(int)
   */
  @KeyObjectNames({"getFrameSize"}) // suppress acronym "FS" to disambiguate with {@link #getFS()}
  public int getFrameSize () { return getSubSize(); }

  /** Sets the frame size of a Type 1000/2000 file. <br>
      This can be used to quickly convert to/from Type 1000/2000 file format, where Type 1000 files
      have a frame size of one and Type 2000 files have a frame size greater than one.<br>
      This should not be used to set the sub-element/frame-size alone, as it does much more.
      @param fs The frame size.
      @see #getFrameSize()
   */
  @KeyObjectNames({"setFrameSize"}) // suppress acronym "FS" to disambiguate with {@link #setFS()}
  public void setFrameSize (int fs) {
    boolean setwsize = (wsize!=0) && (wsize==size);  // check to see if size and wsize are "in step"
    int tc = getType()/1000;
    double dbpei = dbpe;
    if (fs==0 && tc==2) {
      setType(1000);
    }
    else if (fs>0 && tc==1) {
      setType(2000);
      setYStart(getXStart());
      setYDelta(fs*getXDelta());
      setYUnits(getXUnits());
      setXStart(0.0);
      setSubSize(fs);
    }
    else if (fs>0 && tc==2) {
      setSubSize(fs);
    }
    typeClass = getType()/1000;
    getBPE();
    setSize();
    trim1 = trim1*dbpei/dbpe;
    trim2 = trim2*dbpei/dbpe;
    if (isOpen && setwsize) wsize = size;  // keep size and wsize "in step" if they started in step
  }

  /** Get unique record name without issuing conflict warnings
   *
   *  @param prefix prefix used in creation of unique record name that will be
   *                created if there is a conflict with the original record name
   *  @param irec   integer used in creation of unique record name
   *  @param oname  original name (used if unique)
   *  @return       unique record name
   */
  public String getUniqueRecName (String prefix, int irec, String oname) {
    return getUniqueRecName (prefix, irec, oname, false);
  }

  /** Get unique record name with control over whether to issue conflict warnings
   *
   *  @param prefix prefix used in creation of unique record name that will be
   *                created if there is a conflict with the original record name
   *  @param irec   integer used in creation of unique record name
   *  @param oname  original name (used if unique)
   *  @param warn   whether to warn if there was a conflict with the original name
   *  @return       unique record name
   */
  public String getUniqueRecName (String prefix, int irec, String oname, boolean warn) {
    String name;
    if (irec>99) name = prefix.substring(0,1)+irec;
    else if (irec>9) name = prefix.substring(0,2)+irec;
    else name = prefix.substring(0,3)+irec;
    if (oname!=null) {
      for (int i=0; i<getSubSize(); i++) {
        if (oname.equals(getRecName(i))) {
          if (warn) M.warning ("SubRecName "+oname+" conflicts. Rename to "+name);
          oname = name;
        }
      }
      name = oname;
    }
    return name;
  }

  /** Set subrecords for a Type 3000/5000/6000 file. If the file is not one of the types listed
      (e.g. 1000 or 2000) the type will be changed using <code>setType(3000)</code>. The subrecords
      are defined by a string in the form:
      <pre>
             (NAME1/FORMAT1/OFFSET1/TYPE1/UNITS1,NAME2/FORMAT2/OFFSET2/TYPE2/UNITS2,...)
        or   (NAME1/FORMAT1,NAME2/FORMAT2,...)
        or   (NAME1|FORMAT1,NAME2|FORMAT2,...)
        or  +(NAMEx|FORMATx,NAMEy|FORMATy,...)  <i>adds to current</i>
      </pre>
      The following parameters are permitted for each SubRec (in order):
      <pre>
        Name   | Description                   | File Types     | Default
        -------+-------------------------------+----------------+-----------------------
        NAME   | Name of the SubRec (required) | 3000 5000 6000 |
        FORMAT | Format digraph for SubRec     | 3000 5000 6000 | "SF"
        OFFSET | SubRec offset                 | 3000      6000 | automatically computed
        TYPE   | Component type                |      5000      | {@link Units#UNDEFINED}
        UNITS  | SubRec units                  |      5000 6000 | {@link Units#NONE}
      </pre>
      Parameters not applicable to a given file type are ignored. See
      {@link #addSubRec(String,String,int,int,int)} for more details.
      @param value The subrecord specifier string, the parenthesis are optional.
      @see #setComponents(String)
  */
  public void setSubRecords (String value) {
    if (!isRecordBased()) setType(3000); // Set type if necessary
    setSubRecords(value, false);
  }

  /** Internal: setSubrecords (comp = use old type 5000 syntax). */
  private void setSubRecords (String value, boolean comp) {
    // incremental set ?
    if (value.startsWith("+")) {
      value = value.substring(1);
    } else {
      setSubSize(0);
      setRecLength(0);
    }

    // Remove the ()'s if necessary
    int i = value.indexOf('(');
    int j = value.lastIndexOf(')');
    if (i==0 && j>0) value = value.substring(1,j);


    if (value.length() > 0) {
      // Parse the comma delimited string
      Parser st = new Parser (value);
      for (int k=1; k<=st.elements(); k++) {
        String recString = st.get(k);
        setSubRec(-k, recString, comp);
      }
    }
    setInternals();
  }

  /** Updates a SubRec definition (n=-1 for adding a new one), must call setInternals() after this.
      @param n         The subrecord number.
      @param recString The subrecord definition.
      @param comp Set to true for old 5000 syntax.
   */
  private void setSubRec (int n, String recString, boolean comp) {
    Parser recTokens = new Parser (recString);
    if (recString.indexOf('/')>=0) recTokens.setDelimiter('/');
    if (recString.indexOf('|')>=0) recTokens.setDelimiter('|');
    int nt = recTokens.elements();

    String recName;
    String recFormat;
    int recOffset;
    int recType;
    int recUnits;
    if (n < 0) {
      int k = -n;
      recName   = (comp)? "C"+k : "SR"+k; // default to index
      recFormat = "SF"; // default to float
      recOffset = -1;   // default to end of last
      recType   = (comp)? Units.SCALAR : Units.UNDEFINED;
      recUnits  = Units.NONE;
    }
    else {
      recName   = getRecName(n);
      recFormat = getRecFormat(n);
      recOffset = (typeClass == 5)? -1 : getRecOffset(n); // not applicable for Type 5000 (don't compute)
      recType   = getRecCompType(n);
      recUnits  = getRecUnits(n);
    }
    if (nt>0) recName   = recTokens.get(1);
    if (nt>1) recFormat = recTokens.get(2);

    if (comp) {
      if (nt>2) recType  = getCompTypeID(recTokens.get(3),recType);
      if (nt>3) recUnits = getUnitsID(recTokens.get(4),recUnits);
    }
    else {
      if (nt>2) recOffset = Convert.s2l(recTokens.get(3));
      if (nt>3) recType   = getCompTypeID(recTokens.get(4),recType);
      if (nt>4) recUnits  = getUnitsID(recTokens.get(5),recUnits);
    }
    setSubRec(n, recName, recFormat, recOffset, recType, recUnits, false);
  }

  /** Sets a subrecord definition for a Type 3000 file.
      @param n     The subrecord number.
      @param value The subrecord definition in the form <code>"(NAME,FORMAT,OFFSET)"</code>.
      @see #setSubRecords(String)
   */
  @KeyObjectNames({"setSubRec"}) // suppress acronym "SR" to disambiguate with {@link #setSR()}
  public void setSubRec (int n, String value) {
    // Remove the ()'s if necessary
    int i = value.indexOf('(');
    int j = value.lastIndexOf(')');
    if (i==0 && j>0) value = value.substring(1,j);
    setSubRec(n, value, false);
    setInternals();
  }

  /** Set components for a Type 5000 file. If the file is not a type 5000 file
      (i.e. <code>{@link #getType()} = 5xxx</code>) the type will be changed using
      <code>setType(5000)</code>. The components are defined by a string in the form:
      <pre>
             (NAME1/FORMAT1/COMPTYPE1,UNITS1,NAME2/FORMAT2/COMPTYPE2/UNITS2...)
        or   (NAME1/FORMAT1,NAME2/FORMAT2,...)
        or   (NAME1|FORMAT1,NAME2|FORMAT2,...)
        or  +(NAMEx|FORMATx,NAMEy|FORMATy,...)  <i>adds to current</i>
      </pre>
      If a format is not specified it is defaulted to <code>"SD"</code>, if a comptype is not
      specified it is defaulted to {@link Units#SCALAR}, if a units is not specified it is
      defaulted to {@link Units#NONE}.
      @param value The component specifier string, the parenthesis are optional.
      @see #setSubRecords(String)
  */
  public void setComponents (String value) {
    if (typeClass != 5) setType(5000); // Set type if necessary
    setSubRecords(value, true);
  }

  /** Sets a component definition for a Type 5000 file.
      @param n     The component number.
      @param value The component definition in the form <code>"(NAME,FORMAT,COMPTYPE,UNITS)"</code>.
      @see #setComponents(String)
   */
  @KeyObjectNames({"setComp"}) // suppress acronym "C" to disambiguate with {@link #setC(int, String)}
  public void setComp (int n, String value) {
    // Remove the ()'s if necessary
    int i = value.indexOf('(');
    int j = value.lastIndexOf(')');
    if (i==0 && j>0) value = value.substring(1,j);
    setSubRec(n, value, true);
    setInternals();
  }

  /** This creates a new Data buffer for IO and only uses the DataFile object to initialize the buffer's create parameters.
      If reformatting is anticipated use {@link #getDataBuffer(int,byte)}.
      @param  elements The object size
      @return the Data object
   */
  @Override
  public Data getDataBuffer (int elements) {
    return getDataBuffer(elements,dataType);
  }

  /** This creates a new Data buffer for IO and only uses the DataFile object to initialize the buffer's create parameters.
      If reformatting is anticipated use {@link #createDataBuffer(int,byte)}.
      For historic reasons we are keeping {@link #getDataBuffer(int)}.
      @since NeXtMidas 3.7.1
      @param elements The object size
      @return the Data object
   */
  public final Data createDataBuffer (int elements) {
    return getDataBuffer(elements);
  }

  /** This creates a Data buffer for IO with reformat and only uses the DataFile object to initialize the buffer's create parameters.
      This is a convenience method that is identical to calling {@link #createDataBuffer(int,char)}.
      @since NeXtMidas 3.3.0
      @param elements The object size
      @param type Part of FORMAT digraph
      @return the Data object
   */
  public Data getDataBuffer (int elements, char type) {
    return getDataBuffer(elements, (byte)type);
  }

  /** This creates a Data buffer for IO with reformat and only uses the DataFile object to initialize the buffer's create parameters.
      For historic reasons we are keeping {@link #getDataBuffer(int, char)}.
      @since NeXtMidas 3.7.1
      @param elements The object size
      @param type Part of FORMAT digraph
      @return the Data object
  */
  public final Data createDataBuffer (int elements, char type) {
    return getDataBuffer(elements, type);
  }

  /** This creates a Data buffer for IO with reformat and only uses the DataFile object to initialize the buffer's create parameters.
      @param elements The object size
      @param type Part of FORMAT digraph
      @return the Data object
  */
  public Data getDataBuffer (int elements, byte type) {
    int fbytes,dbytes;
    Data data = new Data();
    if (dataType==PACKED) fbytes = (int)(8*dbpe+.9);
    else fbytes = (int)dbpe;
    if (type!=dataType) dbytes = ape * Data.getSPA(dataMode) * Data.getBPS(type);
    else dbytes = fbytes;
    data.setFormatType(type);
    data.setFormatMode(dataMode);
    if (typeClass==2) data.setFrame(ape);
    if (isRecordBased()) data.setRecLength(dbytes);
    data.setSize(elements);
    if (fbytes>dbytes) data.buf = new byte[elements*fbytes];
    xferLength=elements;
    return data;
  }

  /** This creates a Data buffer for IO with reformat and only uses the DataFile object to initialize the buffer's create parameters.
      For historic reasons we are keeping {@link #getDataBuffer(int, byte)}.
      @since NeXtMidas 3.7.1
      @param elements The object size
      @param type Part of FORMAT digraph
      @return the Data object
  */
  public final Data createDataBuffer (int elements, byte type) {
    return getDataBuffer(elements, type);
  }

  /*JGS - why is this here??? */
  /** Toggles whether or not to preserve the data buffer
      @param state toggles preserving data buffer
   */
  public void setPreserveDataBuffer (boolean state) {
    preserveDB = state;
  }

  /** Get bytes per element
   *  @return bytes per element
   */
  public double getBPE () {
    if (typeClass >= 2) ape = getSubSize(); else ape=1;
    if (typeClass >= 3) dbpe = (double)getRecLength();
    else if (typeClass >= 2) dbpe = ape * getBPA();
    else dbpe = getBPA();
    if (dbpe<0) dbpe = -dbpe/8;
    return dbpe;
  }

  /** Get bytes per atom
   *  @return bytes per atom
   */
  public int getBPA () {
    bpa = Data.getSPA(dataMode) * Data.getBPS(dataType);
    if (bpa<0 && -bpa%8==0) bpa = -bpa/8; // NXM-3894 Jeff 12-bit J
    return bpa;
  }

  /** Get atoms per element
   * @return atoms per element
   */
  public int getAPE () {
    if (typeClass >= 2) ape = getSubSize(); else ape = 1;
    return ape;
  }

  /** Get scalars per atom
   * @return scalars per atom
   */
  public int getSPA () {
    spa = Data.getSPA( dataMode );
    return spa;
  }

  /** Get bytes per scalar
   * @return bytes per scalar
   */
  public int getBPS () {
    bps = Data.getBPS( dataType );
    return bps;
  }

  private void wrapCircularFile () {
    double cur=seek();
    if (wsize>cfsize) {  // force wrap on transfer boundary
      cfsize=wsize;
      setOutByte(0,cfsize*dbpe);
    }
    cftop += cfsize;
    seek(cur);
  }

  /** Set Circular File Length
   *  @param length Circular File length
   */
  public void setCircularFileLength (double length) {
    if (length>0) {  // turn WRAP on
      setPipeId(-1);
      flags |= WRAP;
      cfsize = Math.max(cfsize,Math.round(length/getDelta()));
    } else {    // turn it off
      setPipeId(0);
      flags &= ~WRAP;
      cfsize = 0;
    }
    setOutByte(0,cfsize*dbpe);
  }

  /** get Circular File input pointer
   *  @return Circular File input pointer
   */
  public double getCircularFileOffset () {
    return (getInByte()/dbpe)*getDelta();
  }

    /** Sets the time line length.
      @param tll The time line length (-1 = Grow as needed).
      @see TimeLine#setAlloc(int)
      @since NeXtMidas 2.5.0
   */
  public void setTimeLineLength (int tll) {
    if (timeLine == null) newTimeLine();
    timeLine.setAlloc(tll);
  }

  /** Gets the time line length (-1 = Grow as needed).
      @return length of the time line (-1 = Grow as needed).
      @see TimeLine#getAlloc()
      @since NeXtMidas 2.5.0
   */
  public int getTimeLineLength () {
    return getTimeLineLength(TimeLine.getUsecPrecisionAllocDefault());
  }

  /** Gets the time line length (-1 = Grow as needed).
      @param tll The default value.
      @see TimeLine#getAlloc()
      @since NeXtMidas 2.5.0
   */
  private int getTimeLineLength (int tll) {
    if (timeLine != null) {
      tll = timeLine.getAlloc();
    }
    else if ((cmd != null) && cmd.args.find("/TLL")) {
      int tllSwitch = cmd.args.getL("/TLL"); // Can be -1
      // If the switch is less than TimeLine.DEF_ALLOC or greater than -TimeLine.DEF_ALLOC then use the value that is effectively larger
      if ((tllSwitch > -TimeLine.DEF_ALLOC) && (tllSwitch < TimeLine.DEF_ALLOC)) {
        tll = ((tllSwitch == TimeLine.GROW_AS_NEEDED) || (tllSwitch < -TimeLine.DEF_ALLOC))? tllSwitch: Math.max(tll,tllSwitch); // special handling for default value
      }
      // In 4.0.0 for NXM-3639, give preference to switch value over the global default
      // this is needed, since the global default can new be greater than TimeLine.DEF_ALLOC
      else tll = tllSwitch;
    }
    return tll;
  }

  /** Sets the time line tolerance.
      @param tolr The time line tolerance
      @see TimeLine#setTolerance(double)
      @since NeXtMidas 2.5.0
   */
  public void setTimeLineTolerance (double tolr) {
    if (timeLine == null) newTimeLine();
    timeLine.setTolerance(tolr);
  }

  /** Gets the time line tolerance.
      @see TimeLine#getTolerance()
      @return timeline tolerance
      @since NeXtMidas 2.5.0
   */
  public double getTimeLineTolerance () {
    double tlt = TimeLine.DEF_TOLERANCE;

    if (timeLine != null) {
      tlt = timeLine.getTolerance();
    }
    else if ((cmd != null) && cmd.args.find("/TLT")) {
      tlt = cmd.args.getD("/TLT");
    }
    return tlt;
  }

  /** Set the TimeLine format
   *  @since 3.9.4 for NXM-3639
   */
  void setTimeLineFormat (TimeLine.Format format) {
    if (timeLine == null) newTimeLine();
    timeLine.format = format;
  }

  /** Gets the time line delta. Note that this is in sec/byte.
      @see TimeLine#getDelta()
      @return timeline delta
      @since NeXtMidas 2.5.0
   */
  private double getTimeLineDelta () { // JC4: this should not be public
    return (timeLine == null)? (getDelta()/dbpe): timeLine.getDelta();
  }

  /** Creates a new time line with defaults based on keywords and source information. */
  private void newTimeLine () {

    if (dbpe == 0) {
      // This happens when format=NH and there are no sub-record definitions.
      // Give a friendlier error msg vs TimeLine.delta = infinity invalid arg error msg.
      throw new MidasException("Cannot initialize TimeLine until after sub-records are defined for type="
                             + getType() + " with format="+getFormat()+ " for "+getName());
    }
    TimeLine.Format formatOfKeywords = timeLineFormatFromKeywords();
    TimeLine.Format format = TimeLine.Format.MICROSECOND_PRECISION;
    if (timeLine == null) {
      Object tllQual = this.getQualifier("TLL");
      int tll = (tllQual!=null) ? Convert.o2l(tllQual) : getTimeLineLength(0); // get tll /TLL switch if available
      double tolr = TimeLine.DEF_TOLERANCE;
      TimeLine.Format overrideFormat = overrideTimeLineFormat(null);
      if (overrideFormat != null) {
        if (overrideFormat == TimeLine.Format.MICROSECOND_PRECISION) {
          if (tll==0) tll = TimeLine.getUsecPrecisionAllocDefault();
          if (this.sourceTimeLineTolr   != 0) tolr = Math.max(this.sourceTimeLineTolr,TimeLine.MICROSECOND_TOLERANCE);
        }
        else {
          if (tll==0) tll = TimeLine.getHighPrecisionAllocDefault();
          tolr = TimeLine.QUARTER_NANOSECOND_TOLERANCE;
        }
        format = overrideFormat;
      }
      else if (formatOfKeywords!=null) {
        if (formatOfKeywords == TimeLine.Format.MICROSECOND_PRECISION) {
          if (tll==0) tll = TimeLine.getUsecPrecisionAllocDefault();
          Data tld = (Data)keywords.get("TIMELINE");
          tolr = tld.getD(1);
        }
        else {
          if (tll==0) tll = TimeLine.getHighPrecisionAllocDefault(); // fixed in 4.0.0
          tolr = TimeLine.QUARTER_NANOSECOND_TOLERANCE;
        }
        format = formatOfKeywords;
      }
      else if (this.sourceTimeLineFormat != null) {
        if (this.sourceTimeLineFormat == TimeLine.Format.MICROSECOND_PRECISION) {
          if (tll==0) tll = TimeLine.getUsecPrecisionAllocDefault();
          if (this.sourceTimeLineTolr   != 0) tolr = this.sourceTimeLineTolr;
        }
        else {
          if (tll==0) tll = TimeLine.getHighPrecisionAllocDefault();
          tolr = TimeLine.QUARTER_NANOSECOND_TOLERANCE;
        }
        format = this.sourceTimeLineFormat;
      }
      if (tll==0) tll = TimeLine.DEF_ALLOC;
      timeLine = new TimeLine(tll, getTimeLineDelta(), tolr);
    }
    else {
      M.info("This DataFile has existing TimeLine, but a new one is being created :"+this);
      Object tllQual = this.getQualifier("TLL");
      int tll = (tllQual!=null) ? Convert.o2l(tllQual) : getTimeLineLength(); // get tll /TLL switch if available
      timeLine = new TimeLine(tll, getTimeLineDelta(), TimeLine.DEF_TOLERANCE);
      format = timeLine.format;
    }
    timeLine.format = format;
    if (!sameTimelineKeywords) {
      deleteTCKeywords();  // delete pre-existing TC keywords
    }
  }

  /** Update timeLine tolerance and format from pre-existing keywords
   *  This is designed to be called when a timeLine is being updated
   *  The pre-existing keywords were just a copy over from the input
   *  file, but they do contain valuable information on timeline tolerance
   *  and format
   *  @return true if the timeLine format and tolerance have been updated
   *  @since NeXtMidas 3.9.4
   */
  private TimeLine.Format timeLineFormatFromKeywords() {
    TimeLine.Format format = null;
    Data tld = (Data)keywords.get("TIMELINE");
    if (tld != null) {
      format = TimeLine.Format.MICROSECOND_PRECISION;
    }
    else if (keywords.get("TCSAMPLE_1") != null) {
      format = TimeLine.Format.BLUE_HIGH_PRECISION;
    }
    else if (keywords.get("TIMETAG.OFFSET") != null) {
      // Note: this path not fully tested
      format = TimeLine.Format.PLATINUM_HIGH_PRECISION;
    }
    return format;
  }

  /**  Delete the pre-existing Blue high precision Platinum Keywords in the file
   *   in preparation for writing out updated/correct keywords
   *   @return true if keywords were deleted
   *   @since NeXtMidas 3.9.0
   */
  private boolean deletePlatinumTimeLineKeywords () {

    if (keywords.delete("TIMETAG.OFFSET") != 0) {
      // we have Platinum timeline Keywords, so delete all of them
      keywords.delete("TIMETAG.TIME");        // array of times type X or D (long or double)
      keywords.delete("TIMETAG.TIME.UNITS");  // type L (integer)
      keywords.delete("TIME_EPOCH");          // string in ISO8601 format
      keywords.delete("TIME_DELTA");          // type D (double)
      keywords.delete("TIME_TICKS");          // type X (long) (inverse of delta)

      if (timeLine != null) {
        // Preserve the fine tolerance / high precision setting
        timeLine.format = TimeLine.Format.PLATINUM_HIGH_PRECISION;
        timeLine.setTolerance(TimeLine.QUARTER_NANOSECOND_TOLERANCE);
      }
      return true;
    }
    else {
      return false;
    }
  }

  /**  Delete the pre-existing Blue high precision TC Keywords in the file
   *   in preparation for writing out updated/correct keywords
   *   @return true if keywords were deleted
   */
  private boolean deleteTCKeywords () {
    int i = 1;  // # of first TC sample

    // while there are more samples in the file, delete the TC Keyword pair
    while (keywords.delete("TCSAMPLE_"+i) != 0) {
      keywords.delete("TC_WHOLE_"+i);
      keywords.delete("TC_FRAC_"+i);
      i++;
    }

    boolean keywordsDeleted = i > 1; // true if keywords were found (and deleted) - since 3.8.0
    if (keywordsDeleted & (timeLine != null)) {
      // Preserve the fine tolerance / high precision setting - since 3.9.0
      timeLine.format = TimeLine.Format.BLUE_HIGH_PRECISION;
      timeLine.setTolerance(TimeLine.QUARTER_NANOSECOND_TOLERANCE);
    }

    return keywordsDeleted; // true if keywords were found (and deleted) - since 3.8.0
  }

  /** Delete all timeline keywords from this DataFile
   *  @return true if some keywords were deleted
   *  @since NeXtMidas 3.8.0
   */
  @InternalUseOnly
  public boolean deleteTimeLineKeywords () {
    boolean someKeywordsRemoved = keywords.delete("TIMELINE") != 0;
    someKeywordsRemoved |= deletePlatinumTimeLineKeywords(); // added in 3.9.0
    someKeywordsRemoved |= deleteTCKeywords(); // putting this last in case there are a mix
    return someKeywordsRemoved;
  }

  /** Delete all non-timeline keywords from this DataFile
   *  @return true if some keywords were deleted
   *  @since NeXtMidas 3.9.4
   */
  @InternalUseOnly
  public boolean deleteNonTimeLineKeywords () {
    boolean keywordsWereDeleted = false;
    String[] extKeys = keywords.getKeys("EXT");
    for (String key : extKeys) {
      Matcher matchTC = TimeLine.TIMELINE_KEYWORD_PATTERN.matcher(key);
      if (!matchTC.matches()) {
         keywords.delete(key);
         keywordsWereDeleted = true;
      }
    }
    return keywordsWereDeleted;
  }

  /** Initialize a timeline based on a given file to propagate timeline info from. This is used
      by PipeResource to initialize the timeline in a pipe reader based on the writer of the pipe.
   */
  void newTimeLineFromDF (DataFile in) {   // package-private, only called from PipeResource
    // NOTE: See related test cases in test_timeline_restart.mm
    timeLine = new TimeLine(in.getTimeLineLength(), in.getTimeLineDelta(), in.getTimeLineTolerance());
  }

  /** Initialize a timeline based on a given Data object (containing microsecond precision time) of TIMELINE keyword. */
  private void newTimeLineFromData (Data data) {
    newTimeLine();
    CoreIO coreIO = Shell.getMidasContext().io;
    int timeLineSize = (data.size*data.spa-2)/2;
    // When the timeline does not contain an entry for the first data element and the
    // timecode is not the default 1950, allocate a size of 1 bigger than expected size
    // to allow for inserting entry for 1st element that is based on the timecode
    // Since 3.7.3 to match behavior of TC Keywords (
    if (!TimeLine.dataObjectTimelineStartsWithFirstElement(data)  && (getTime().getSec() > 0)) {
      boolean disableErrorMsg = coreIO.isOptionSet(
          CoreIO.IOOptions.SuppressInvalidTimeLineEntryErrors);
      if (!disableErrorMsg) {
        String warning = "TimeLine entries for "+fn+" do not start at the first element.";
        if (offset0AutoCorrectIsEnabled) warning += "  Inserting entry for offset 0 based on TimeCode";
        M.warning(warning);
      }
      if (offset0AutoCorrectIsEnabled) timeLineSize++; // need additional space when adding auto-correct value
    }
    timeLine.setAlloc(timeLineSize);
    timeLine.fromData(data, getTime().addSec(getStart())); // NXM-3972
  }


  /** Re-create this.timeLine from keywords<br>
   *  This is intended to be used in the case where a DataFile is opened, its
   *  timeline keywords are modified, and the desire is for that update to be reflected
   *  in the timeLine and in the keywords present when the file is closed.<br>
   *  This is not intended to be used on a file that is being created within a pipe,
   *  as PipeResource handles the setting of time entries in that case.
   *  @since 3.9.1 for NXM-3185
   */
  @ProvisionalUseOnly("This API is still under evaluation")
  public void updateTimeLineFromKeywords () {
    this.timeLine = null; // starting from scratch
    Data tld = (Data)keywords.get("TIMELINE");
    if (tld!=null) newTimeLineFromData(tld);
    handlePlatinumTimelineKeywords(getBlueVer(), getBlueIO());
    handleBlueTCTimelineKeywords(getBlueVer(), getBlueIO());
  }

  /** Combines functionality of {@link nxm.sys.lib.Keywords#putAll(Map)} and
   *  {@link #updateTimeLineFromKeywords()}<br>
   *  Updates the keywords and then recreates timeLine based on the timeline keywords
   *  @param map Map containing keywords to be updated
   *  @since 3.9.1 for NXM-3185
   */
  @ProvisionalUseOnly("This API is still under evaluation")
  public void updateKeywordsAndTimeLine (Map<String,Object> map) {
    keywords.putAll(map);
    updateTimeLineFromKeywords();
  }

  /**
   * get TimeLine handler class
   * @return TimeLine handler
   */
  public TimeLine getTimeLineHandler () {
    return timeLine;
  }

  /** Set TimeLine handler class. This method is used by the <code>KEYWORD</code> command when deleting
      the time line (otherwise DataFile will automatically write out its copy of the TimeLine when
      the file is closed). At this time, only erasing the time line (<code>tl=null</code>) is supported,
      the {@link #setTimeAt} method should be used to create a time line.
      @param tl The time line (must be null).
      @since NeXtMidas 2.5.0
   */
  void setTimeLineHandler (TimeLine tl) { // package-private, only used by Keywords at this time
    if (tl != null) {
      throw new MidasException("DataFile: Use TimeLine.setTimeLineObject(..) or the setTimeAt(..) method to alter the TimeLine.");
    }
    timeLine = tl;
  }

  /** Set / overwrite the TimeLine handler class.
      @param tl The new TimeLine object
      @since NeXtMidas 3.7.2
    */
  public void setTimeLineObject (TimeLine tl) {
    timeLine = tl;
  }

  /** Indicates if the data file is a local file.
   *  @return true if the data file is a local file
   */
  public boolean isFile () { return (io!=null && io.isFile()); }

  /** Indicates if this resource is a JAR file
   *  @return true if the data file is a local file
   *  @since NeXtMidas 4.1.4
   */
  public boolean isJarFile() { return (io!=null && io.isJarFile());}

  /** Indicates if the data file is a pipe.
   * @return true if the data file is a pipe
   */
  public boolean isPipe () { return (io!=null && io.isPipe()); }

  /** Gets the file version in the header (this is ALWAYS equal to "BLUE" for Midas Blue files).
   *  @return file version in the header
   */
  public String getVersion () { return unpackS(OFF_VERSION,4); }

  /** Gets the byte-ordering representation for the main and extended header (usually "IEEE" or "EEEI").
   *  @return the byte-ordering representation for the main and extended header
   */
  public String getHeadRep () { return unpackS(OFF_HEADREP,4); }

  /** Gets the byte-ordering representation for the data (usually "IEEE" or "EEEI").
   *  @return the byte-ordering representation for the data
   */
  public String getDataRep () { return unpackS(OFF_DATAREP,4); }

  /** Gets the "Detached Header" field in the header. This field is set to zero
      when the header is not detached and a non-zero value when the header is
      detached. The non-zero values have special meaning that can alter the
      name of the detached file name, as such they should not be interpreted
      directly from this field, but should be obtained by calling
      {@link #getDetachName()}.
      @return The detached header status (0 for not detached).
   */
  public int getDetached () { return unpackL(OFF_DETACHED); }

  @Override
  public boolean getProtected () { return (unpackL(OFF_PROTECTED)!=0); }

  /** Gets the Pipe-ID of the file.
   *  @return Pipe-ID of the file
   */
  public int getPipeId () { return unpackL(OFF_PIPE); }

  /** Gets the start position of the extended header (in 512-octet blocks). The
      use of "blocks" rather than "bytes" in this function mirrors the field in
      the main header of the BLUE file. Example (from BLUE 1.0 ICD):
      <pre>
        getExtStart() -&gt; 1000 = extended header starts on 512,000th byte of file
      </pre>
      Note that the type and units of this field impose a size restriction of
      ~1 TiB on the data portion of a BLUE file. However, this restriction can
      be worked-around (within the scope of the BLUE 1.0 ICD) by either using a
      detached file for the data *or* by relocating the extended header to follow
      the main header (i.e. before the data section).
      @return The start position of the extended header (in 512-octet blocks).
   */
  public int getExtStart () { return unpackL(OFF_EXTSTART); }

  /** Gets the size/length of the extended header (in bytes).
      @return The size/length of the extended header (in bytes).
   */
  public int getExtSize () { return unpackL(OFF_EXTSIZE); }

  /** Gets the start position of the data (in bytes).
   *  @return The start position of the data (in bytes)
   */
  public double getDataStart () { return unpackD(OFF_DATASTART); }

  /** Gets the size/length of the data (in bytes).
   * @return The size/length of the data (in bytes)
   */
  public double getDataSize () { return unpackD(OFF_DATASIZE); }

  @Override
  public int getType () { return unpackL(OFF_TYPE); }

  /* NTN: This method have been renamed to getTypeCodeClass() so that it does NOT
     conflict with getTimeCode() for macros that uses the TC abbreviation.
     In the future, we can look into adding a Java5 annotation to specify a
     different abbreviation for this instead of the default (e.g. "TypeClass" vs "TC")
     public int getTypeClass() { return typeClass; }
   */

  /** Gets the type code class based on the file type. This will return 1 for
      all type 1000 files (1000 through 1999), 2 for all type 2000 files
      (2000 through 2999), 3 for all type 3000 files, etc.
      @return type code class based on the file type
      @since NeXtMidas 2.7.1
   */
  public int getTypeCodeClass () { return typeClass; }

  @Override
  public String getFormat () { return unpackS(OFF_FORMAT,2); }

  /** Gets the data format mode ('S' for scalar, 'C' for complex, etc.).
   *  @return data format mode
   */
  public byte getFormatMode () { return unpackB(OFF_FORMAT); }

  /** Gets the data format type ('B' for byte, 'D' for double, etc.).
   *  @return data format type
   */
  public byte getFormatType () { return unpackB(OFF_FORMAT + 1); }

  /** Sets the data flag mask.
      @return The flag mask.
   */
  public short getDataMask () { return unpackI(OFF_FLAGMASK); }

  /** Gets the time-code of the file, this only gets the "timecode" filed in the
      main header of the file. In many cases a higher-precision time-code may
      be available from {@link #getTime()}.
      @return The value for the "timecode" field in the main header.
   */
  public double getTimeCode () { return unpackD(OFF_TIMECODE); }

  /** get inlet owner
   *  @return inlet owner
   */
  public short  getInlet ()        { return unpackI(OFF_INLET); }
  /** get number of outlets
   *  @return number of outlets
   */
  public short  getOutlets ()      { return unpackI(OFF_OUTLETS); }
  /** get outlet async mask
   *  @return outlet async mask
   */
  public int    getOutMask ()      { return unpackL(OFF_OUTMASK); }
  /** get pipe location
   *  @return pipe location
   */
  public int    getPipeLoc ()      { return unpackL(OFF_PIPELOC); }
  /** get pipe size
   *  @return pipe size
   */
  public int    getPipeSize ()     { return isPipe()? iop.pipe.getSize()     : unpackL(OFF_PIPESIZE); }
  /** get next input byte
   *  @return next input byte
   */
  public double getInByte ()       { return isPipe()? iop.pipe.getInByte()   : unpackD(OFF_INBYTE); }
  /** get next out byte (cummulative)
   *  @param i outlet number
   *  @return next out byte
   */
  public double getOutByte (int i) { return isPipe()? iop.pipe.getOutByte(i) : unpackD(OFF_OUTBYTE+i*8); }

  /** Get the length of the MAIN header keyword block.
      Not intended for general use. Use {@link #getKeywordsObject()} instead.
      @return length of the MAIN header keyword block
   */
  public int getKeyLength () { return unpackL(OFF_KEYLGTH); }
  /** Get the MAIN header keyword block as a string.
      Not intended for general use. Use {@link #getKeywordsObject()} instead.
      @return  MAIN header keyword block as a string
   */
  public String getKeywords () { return new String(hb,OFF_KEYWORDS,getKeyLength()); } // unpackS strips trailing NULL
  /** Get the keywords Object. Repeated calls will return the same reference
   * @return The keywords Object
   */
  public Keywords getKeywordsObject () { return keywords; }

  @Override public double getXStart () { return unpackD(OFF_ADJUNCT + ADJUNCT_XSTART_OFF); }
  @Override public double getXDelta () { return unpackD(OFF_ADJUNCT + ADJUNCT_XDELTA_OFF); }
  @Override public int    getXUnits () { return unpackL(OFF_ADJUNCT + ADJUNCT_XUNITS_OFF); }
  /** Gets the variable name for the X-axis units
      @return variable name for X-axis units
   */
  public String getXUnitsName () { return getUnitsVarName(getXUnits()); }

  /** Gets the sub-element/frame size for Type 1000 and Type 2000 files. In most cases it is best
      to use the {@link #getFrameSize()} method in place of this one.
      @return sub-element/frame size for Type 1000 and Type 2000 files
   */
  public int getSubSize () { return unpackL(OFF_ADJUNCT + ADJUNCT_SUBSIZE_OFF); }

  @Override public double getYStart () { return unpackD(OFF_ADJUNCT + ADJUNCT_YSTART_OFF); }
  @Override public double getYDelta () { return unpackD(OFF_ADJUNCT + ADJUNCT_YDELTA_OFF); }
  @Override public int    getYUnits () { return unpackL(OFF_ADJUNCT + ADJUNCT_YUNITS_OFF); }
  /** Gets the variable name for the Y-axis units
      @return variable name for Y-axis units
   */
  public String getYUnitsName () { return getUnitsVarName(getYUnits()); }

  /** Gets the total number of bytes in each record for a Type 3000/5000/6000 file.
   *  @return total number of bytes in each record for a Type 3000/5000/6000 file
   */
  public int getRecLength () { return unpackL(OFF_ADJUNCT + ADJUNCT_RECLGTH_OFF); }

  /** Gets the length of the main header. This is 512 for all Type 1000/2000/6000 files. For "normal"
      type 3000/5000 files this will also be 512; however NeXtMidas "extended" Type 3000/5000 files
      may be greater than 512.
      @return The length of the main header in bytes.
   */
  public int getHeaderLength () {
    int len = getHeaderLength(typeClass); // <-- may return value < 512
    return Math.max(HEADER_SIZE, len);
  }

  /** Gets the name of a subrecord in a Type 3000/5000/6000 file. This is the same as
      <code>getRecName(i,false)</code>.
      @param i The index of the subrecord (0..n-1).
      @return The name of the subrecord.
   */
  public String getRecName (int i) {
    return getRecName(i,false);
  }

  /** Gets the name of a subrecord in a Type 3000/5000/6000 file.
      @param i The index of the subrecord (0..n-1).
      @param raw Get the raw name (set this to <code>true</code> to disable lookup for long subrecord names).
      @return The name of the subrecord.
      @since NeXtMidas 2.7.0
   */
  public String getRecName (int i, boolean raw) {
    if (!raw && (recNames != null)) return recNames[i];
    if (!isRecordBased()) return "F"+(i+1); // Type 1000/2000 use "F1", "F2", "F3", ...
    return (t6s!=null)? getT6s(i,0,24).trim() : unpackS(iSub(i),4);
  }

  /** Gets the format of a subrecord in a Type 3000/5000/6000 file.
      @param i The index of the subrecord (0..n-1).
      @return The format of the subrecord/component (e.g. "SB", "VD", etc.).
   */
  public String getRecFormat (int i) {
    if (isRecordBased()) {
      return (t6s!=null) ? getT6s(i,88,2) : unpackS(iSub(i)+4,2); // Type 3000/5000 match in position
    } else {
      return getFormat();
    }
  }

  /** Gets the format mode of a subrecord in a Type 3000/5000/6000 file.
      @param i The index of the subrecord (0..n-1).
      @return The format mode of the subrecord/component ('S' for scalar, 'C' for complex, etc.).
   */
  public byte getRecFormatMode (int i) { return (byte)getRecFormat(i).charAt(0); }

  /** Gets the format type of a subrecord in a Type 3000/5000/6000 file.
      @param i The index of the subrecord (0..n-1).
      @return The format type of the subrecord/component ('B' for byte, 'D' for double, etc.).
   */
  public byte getRecFormatType (int i) { return (byte)getRecFormat(i).charAt(1); }

  /** Gets the offset of a subrecord in a Type 3000/5000/6000 file. For Type 3000/6000 files this
      may be un-aligned to allow "gaps" between subrecords (commonly used with detached files). For
      other types, this is automatically computed.
      @param i The index of the subrecord (0..n-1).
      @return The byte offset of the subrecord.
   */
  public short getRecOffset (int i) {
    if (t6s!=null) return (short)getT6l(i,72,8);     // Type 6000 -> need to look up
    if (typeClass == 3) return unpackI(iSub(i)+6);   // Type 3000 -> need to look up

    String form = getFormat();
    if (form.equals("NH")) {                         // Type 5000 (NH) -> iterative computation
      short ioff = 0;
      while (i>0) ioff += Data.getBPA(getRecFormat(--i));
      return ioff;
    }
    return (short)(Data.getBPA(form) * i);           // Type 1000/2000/5000 homogeneous -> just multiply
  }

  /** Gets the number elements (num_elts) of a subrecord in a Type 6000 file.
      For all other file types this returns 1.
      @param i The index of the subrecord (0..n-1).
      @return The number elements in the subrecord.
   */
  public int getRecElements (int i) { return (t6s!=null)? getT6l(i,80,4) : 1; }

  /** Sets the number elements (num_elts) of a subrecord in a Type 6000 file.
      @param i   The index of the subrecord (0..n-1).
      @param val The number elements in the subrecord.
   */
  public void setRecElements (int i, int val) {
    if (val != 1) M.warning("NeXtMidas does not support Type 6000 files with "+val+" rec elements."); // num_elts (num_samps in Platinum) in this field
    if (t6s!=null) setT6l(i,80,4,val);
  }

  /** Gets the units of a subrecord in a Type 5000/6000 file. For all other file types this returns
      {@link #getUnits()}.
      @param i The index of the subrecord (0..n-1).
      @return The units of the subrecord.
   */
  public int getRecUnits (int i) {
    if (t6s!=null) {
      return getT6l(i,84,4);
    } else if (typeClass==5) {
      return unpackB(iSub(i)+7);
    } else {
      return getUnits();
    }
  }

  /** Gets the name of the units of a subrecord in a Type 5000/6000 file.
      @param i The index of the subrecord (0..n-1).
      @return The units of the subrecord.
      @see #getRecUnits(int)
      @since NeXtMidas 2.5.0
   */
  public final String getRecUnitsName (int i) { return getUnitsVarName(getRecUnits(i)); }

  /** Gets the comp-type of a component in a Type 5000 file. For all other file types this is
      returns {@link Units#UNDEFINED}.
      @param i The index of the component (0..n-1).
      @return The comp-type of the component (see {@link Units}).
      @since NeXtMidas 2.5.0
   */
  public int getRecCompType (int i) {
    if (typeClass==5) {
      return unpackB(iSub(i)+6);
    } else {
      return Units.UNDEFINED;
    }
  }

  /** Gets the name of the comp-type of a component in a Type 5000 file.
      @param i The index of the component (0..n-1).
      @return The comp-type of the component (see {@link Units}).
      @see #getRecCompType(int)
      @since NeXtMidas 2.5.0
   */
  public String getRecCompTypeName (int i) {
    return getCompTypeName((byte)getRecCompType(i), true);
  }

  /** Gets the minimum value for a Type 6000 subrecord (0 if unknown).
   * @param i subrecord number
   * @return minimum value for a Type 6000 subrecord (0 if unknown)
   */
  public double getRecMinimum (int i) { return (t6s!=null)? getT6d(i,24,24) : 0; }
  /** Gets the maximum value for a Type 6000 subrecord (0 if unknown).
   * @param i subrecord number
   * @return maximum value for a Type 6000 subrecord (0 if unknown)
   */
  public double getRecMaximum (int i) { return (t6s!=null)? getT6d(i,48,24) : 0; }

  /** Sets the minimum value for a Type 6000 subrecord
   *  @param i   subrecord number
   *  @param val minimum value
   */
  public void setRecMinimum (int i, double val) { if (t6s!=null) setT6d(i,24,24,val); }
  /** Sets the maximum value for a Type 6000 subrecord.
   *  @param i   subrecord number
   *  @param val maximum value
   */
  public void setRecMaximum (int i, double val) { if (t6s!=null) setT6d(i,48,24,val); }

  /** Gets the units prefix (i.e. "uprefix" char[3] field) of a subrecord definition in a Type 6000 file.
   *  NOTE: per the BLUE ICD, use of this field is discouraged.
   * @param i The index of the subrecord (0..n-1).
   * @return units prefix string (char[3}) for specified subrecord in Type 6000 -or- empty string for all other file types.
   * @since NeXtMidas 3.6.0
   */
  @InternalUseOnly("Since initial version, use of this field is discouraged in BLUE ICD.")
  public String getRecUnitsPrefix (int i) { return (t6s != null)? getT6s(i, 90, 3) : ""; }

  /** Sets the units prefix (i.e. "uprefix" char[3] field) of a subrecord definition in a Type 6000 file.
   * NOTE: per the BLUE ICD, use of this field is discouraged.
   * @param i The index of the subrecord (0..n-1).
   * @param val The value to set (will be truncated or padded with zero(s) to length of 3 per BLUE ICD spec (cannot be null).
   * @since NeXtMidas 3.6.0
   */
  @InternalUseOnly("Since initial version, use of this field is discouraged in BLUE ICD.")
  public void setRecUnitsPrefix (int i, String val) {
    if (t6s != null) {
      final int fieldLen = 3;
      if (val.length() < fieldLen) val = StringUtil.padRight(val, fieldLen, '0');
      setT6s(i, 90, fieldLen, val);
    }
  }

  /** Gets the reserved field (char[3]) of a subrecord definition in a Type 6000 file.
   *  NOTE: per the BLUE ICD, this is not used (it exists to pad to 8-byte boundary).
   *  @param i The index of the subrecord (0..n-1).
   *  @return reserved string (char[3}) for specified subrecord in Type 6000 -or- empty string for all other file types.
   *  @since NeXtMidas 3.6.0
   */
  @InternalUseOnly("Since initial version, use of this field is reserved in BLUE ICD (to pad to 8-byte boundary).")
  public String getRecReserved (int i) { return (t6s != null)? getT6s(i, 93, 3) : ""; }

  /** Sets the reserved field (char[3] field) of a subrecord definition in a Type 6000 file.
   *  NOTE: per the BLUE ICD, this is not used (it exists only to pad to 8-byte boundary).
   *  @param i The index of the subrecord (0..n-1).
   *  @param val The value to set (will be truncated or padded with zero(s) to length of 3 per BLUE ICD spec.
   *  @since NeXtMidas 3.6.0
   */
  void setRecReserved (int i, String val) { // package-private since this field is reserved (only used for padding)
    if (t6s != null) {
      final int fieldLen = 3;
      if (val.length() < fieldLen) val = StringUtil.padRight(val, fieldLen, '0');
      setT6s(i, 93, fieldLen, val);
    }
    if (t6s != null) setT6s(i, 93, 3, val);
  }

  /** Unpack a string from Type 6000 subrecord definition.
      @param index  Subrecord number.
      @param offset Starting offset.
      @param len    Length in bytes.
   */
  private String getT6s (int index, int offset, int len) {
    int off = 96*index + offset;
    return t6s.substring(off,off+len);
  }

  /** Unpack a double from Type 6000 subrecord definition.
      @param index  Subrecord number.
      @param offset Starting offset.
      @param len    Length in bytes.
   */

  private double getT6d (int index, int offset, int len) {
    return Convert.s2d(getT6s(index,offset,len));
  }

  /** Unpack a 32-bit integer (L) from Type 6000 subrecord definition.
      @param index  Subrecord number.
      @param offset Starting offset.
      @param len    Length in bytes.
   */
  private int getT6l (int index, int offset, int len) {
    return Convert.s2l(getT6s(index,offset,len));
  }

  /** Pack a string into Type 6000 subrecord definition.
      @param index  Subrecord number.
      @param offset Starting offset.
      @param len    Length in bytes.
   */
  private void setT6s (int index, int offset, int len, String str) {
    if (str.length() > len) str = str.substring(0,len);
    if (str.length() < len) str = StringUtil.padRight(str,len);
    checkT6cap(index+1);

    int off = 96*index + offset;
    t6s.replace(off, off+len, str);
  }

  /** Pack a double into Type 6000 subrecord definition.
      @param index  Subrecord number.
      @param offset Starting offset.
      @param len    Length in bytes.
   */
  private void setT6d (int index, int offset, int len, double val) {
    String str = StringUtil.padLeft(Double.toString(val), len, '0');
    setT6s(index, offset, len, str);
  }

  /** Pack a 32-bit integer (L) into Type 6000 subrecord definition.
      @param index  Subrecord number.
      @param offset Starting offset.
      @param len    Length in bytes.
   */
  private void setT6l (int index, int offset, int len, int val) {
    String str = StringUtil.padLeft(Integer.toString(val), len, '0');
    setT6s(index, offset, len, str);
  }

  /** Reads the Type 6000 SUBREC_DEF from the extended header. */
  private StringBuffer readT6def (boolean warn) {
    String des = (String)keywords.get("SUBREC_DESCRIP");
    String def = (String)keywords.get("SUBREC_DEF");
    if ((des != null) && !des.equals("TYPE0")) { // if null assume "TYPE0"
      M.warning("DataFile: Unknown Type 6000 definition SUBREC_DESCRIP='"+des+"'");
    }
    if ((def==null) && warn) {
      M.warning("DataFile: No SUBREC_DEF keyword found in type 6000 file");
    }
    return (def==null)? new StringBuffer() : new StringBuffer(def);
  }

  /** Checks length of t6s and increases if not enough for given number of subrecs. */
  private void checkT6cap (int nsubrec) {
    int len = 96*nsubrec;
    while (t6s.length() < len) {
      t6s.append(BLANK_T6_SUBREC);
    }
  }

  /** This is the same as <code>getRecName(i)</code>.
   * @param i The index of the subrecord (0..n-1).
   * @return The name of the subrecord
   */
  public final String getCompName (int i) { return getRecName(i); }
  /** This is the same as <code>getRecFormat(i)</code>.
   * @param i The index of the subrecord (0..n-1).
   * @return The format of the subrecord/component (e.g. "SB", "VD", etc.).
   */
  public final String getCompFormat (int i) { return getRecFormat(i); }
  /** This is the same as <code>getRecType(i)</code>.
      @param i The index of the component (0..n-1).
      @return The comp-type of the component (see {@link Units}).
   */
  public final byte getCompType (int i) { return (byte)getRecCompType(i); }
  /** This is the same as <code>getRecUnits(i)</code>.
      @param i The index of the subrecord (0..n-1).
      @return The units of the subrecord.
   */
  public final byte getCompUnits (int i) { return (byte)getRecUnits(i); }
  /** This is the same as <code>getRecUnitsName(i)</code>.
      @param i The index of the subrecord (0..n-1).
      @return The units of the subrecord.
   */
  public final String getCompUnitsName (int i) { return getRecUnitsName(i); }

  /** Gets the reference frame for a Type 5000 file. Although this is generally
      read via the functions in {@link Transform}, it is possible to read this
      directly.
      @return The reference frame. This should be one of "ECI", "ECR",
              "TOP" or "TOPOCENT" (other values are legal, but discouraged
              by the BLUE 1.0 ICD).
   */
  public String getReferenceFrame () {
    // TODO: This should return null if not a Type 5000 file.
    return unpackS(416,8);
  }

  /** Gets a "quadword" for a Type 5000 file. Although this is generally
      read via the functions in {@link Transform}, it is possible to read this
      directly (e.g. via HEADERMOD).
      @param i     The quadword number (1..11). Although it technically legal to
                   use i=0, doing so is highly discouraged as i=0 is generally
                   a string describing the reference frame (see {@link #getReferenceFrame}).
      @return The quadword value.
   */
  public double getQuadword (int i) {
    // TODO: This should return 0.0 or NaN if not a Type 5000 file and should
    //       error out if i<0 or i>11.
    return unpackD((OFF_ADJUNCT+ADJUNCT_QUADWORD_OFF)+i*8);
  }

  /** Gets the file size from the file header (in number of elements).
      @return size The file size (in number of elements).
   */
  @Override
  public double getSize () { return size; }

  /** Gets the transfer length in elements. (See also {@link #getConsLength()})
      @return number of elements
   */
  public int getXferLength ()  { return xferLength; }

  /** Gets the transfer length in elements. (This is identical to {@link #getXferLength()}
   *  which is the more commonly used because of the  XFER= qualifier)
      @return number of elements
   */
  public int getTransLength () { return xferLength; }

  /** Gets the consume length in elements. (See also {@link #getXferLength()}).
      In most situations the transfer length and consume length will be identical.
      In situations where data is to be skipped (e.g. a block thinning of the data)
      the consume length will be larger than the transfer length. In situations
      where there is data overlap (e.g. an FFT with overlap) the consume length
      will be smaller than the transfer length.
      @return number of elements
   */
  public int getConsLength ()  { return consLength; }

  /** Returns MAIN + EXTENDED Header keywords. This does not match the intended
      behavior of returning just the EXTENDED Header, and therefore has been deprecated

      @deprecated Since NeXtMidas 3.5.1: Use {@link #getKeywordsObject()}
      and limit the scope to just the extended header using {@link Keywords#setScope(String)}<br>
      For example:<br>
         <code>Keywords kw = df.getKeywordsObject();<br>
         kw.setScope("EXT");</code>
      @return MAIN + EXTENDED Header keyword
   */
  @Deprecated public Keywords getExtended() { return keywords; }

  /** Gets the number of bytes that have been written so far. In the case of pipes this
      is the total number of bytes written to the pipe, including ALL bytes written
      even if there was a pipe restart. This method explicitly takes into account any
      {@link PacketHandler} that may be present.
      @return The number of bytes that have been written so far
   */
  public double getDataStop () {
    // JC4/JGS: Not tested using packet handler with restarts.
    double writebyte = wsize*dbpe;
    if (pkh!=null) writebyte = pkh.dataToPacketOffset(this,writebyte);
    return firstbyte+writebyte;
  }

  /** Gets a quick approximation of the nyquist for a file. This is computed with
      <code>nyquist = 0.5/getXDelta()</code> regardless of the file type. This is
      rarely used with file types other than Type 1000/2000.
      @return a quick approximation of the nyquist for a file
   */
  public double getNyquist() { return (0.5/getXDelta()); }

  /** Get X-axis rate
      @return X-axis rate
   */
  public double getXRate() { return (1.0/getXDelta()); }
  /** Get Y-axis rate
      @return Y-axis rate
   */
  public double getYRate() { return (1.0/getYDelta()); }
  /** Set X-axis rate
      @param rate new X-axis rate
   */
  public void setXRate (double rate) { setXDelta(1.0/rate); }
  /** Set Y-axis rate
      @param rate new Y-axis rate
   */
  public void setYRate (double rate) { setYDelta(1.0/rate); }

  @Override public int getMode ()   { return getSPA(); }
  @Override public int getXFrame () { return getSubSize(); }
  @Override public int getYFrame () { return (int)getSize(); }

  /** get length of file
   *  @return length of file (size * delta)
   */
  public double getLength () { return getSize()*getDelta(); }

  /** Gets the file URL. When the file is detached, this returns the URL for the
      header (not the detached data).
   */
  @Override
  public String getURL () {
    if (ioh==null) {
      return getFileName().getFullName();
    } else {
      return ioh.getURL();
    }
  }

  /** Sets the file version in the header (this is ALWAYS equal to "BLUE" for
      Midas Blue files). See {@link #setBlueVer} for setting the version of a
      BLUE file. <br>
      <br>
      Note: The confusing naming convention of "version" and "BlueVer" (or "VER=")
      stems from the fact that the idea of a keyword describing the BLUE version
      (i.e. VER=) was introduced more than a decade after the version field was
      included in the main header before BLUE 1.1. Prior to that time, variations
      in version were accomplished by changing this field (e.g. the predecessor
      to "BLUE" was "GOLD").
      @param value The version (this is ALWAYS equal to "BLUE" for Midas Blue files).
   */
  public void setVersion (String value) { packS(OFF_VERSION,CHARS_VERSION,value); }

  /** Sets the byte-ordering representation for the main and extended header.
      Note: Setting this DOES NOT automatically convert byte order of multi-byte fields in the header,
      call {@link #convertHeaderRep(String)} to actually convert the header's rep.
      @param value The byte-ordering for the header (usually "IEEE" or "EEEI", see {@link DataTypes}).
   */
  public void setHeadRep (String value) { packS(OFF_HEADREP,OFF_HEADREP,value); setReps(); }
  /** Sets the byte-ordering representation for the main and extended header.
   *  @param value The byte-ordering for the header
   */
  public void setHeadRep (byte value) { packB(OFF_HEADREP, value); setReps();}
  /** Sets the byte-ordering representation for the data.
      Note: Setting this DOES NOT automatically convert the byte order of existing data.
      @param value The byte-ordering for the data (usually "IEEE" or "EEEI", see {@link DataTypes}).
   */
  public void setDataRep (String value) { packS(OFF_DATAREP,OFF_HEADREP,value); setReps(); }

  /** Sets the "Detached Header" field in the header.
      @param value The detached header status (0 for not detached).
   */
  public void setDetached (int value) { packL(OFF_DETACHED,value); }

  /** Sets the "Detached Header" field in the header.
      @param value The detached header status ("0" for not detached).
   */
  public void setDetached (String value) {
    if (value.equals("0")) setDetached(0);
    else if (value.equals("1")) setDetached(1);
    else packS(12,4,value);
  }

  /** Sets the write-protect flag in the file.
      @param value Indicates if the file should be treated as write-protected.
   */
  public void setProtected (boolean value) { int i=value? 1:0; packL(OFF_PROTECTED,i); }

  /** Sets the Pipe-ID of the file.
      @param value The Pipe-ID.
   */
  public void setPipeId (int value) { packL(OFF_PIPE,value); }

  /** Sets the start position of the extended header (in 512-octet blocks). See
      {@link #getExtStart()} for details on the value used in this field.
      @param value The start position of the extended header (in 512-octet blocks).
   */
  public void setExtStart (int value) { packL(OFF_EXTSTART,value); }

  /** Sets the size/length of the extended header (in bytes).
      @param value The size/length of the extended header (in bytes).
   */
  public void setExtSize (int value) { packL(OFF_EXTSIZE,value); }

  /** Sets the start position of the data (in bytes).
      @param value The start position of the data (in bytes).
   */
  public void setDataStart (double value) { packD(OFF_DATASTART,value); firstbyte=value; }

  /** Sets the start position of the data (in bytes).
   *  @param value The start position of the data (in bytes).
   */
  public void setDS (double value) { setDataStart(value); }

  /** Sets the size/length of the data (in bytes). This will call {@link #setInternals()}.
      @param value The size/length of the data (in bytes).
   */
  public void setDataSize (double value) { setDataSize(value, true); }

  /** Sets the size/length (in bytes) of the data. This method is for advanced users
      who have changed some internals, but do not want them to be reset
      when the data size (in bytes) is changed.
      @param value The size/length of the data (in bytes).
      @param resetInternals Set to true if the {@link #setInternals()} method should be called.
      @since NeXtMidas 2.7.0
   */
  protected void setDataSize (double value, boolean resetInternals) {
    packD(OFF_DATASIZE,value);
    if (resetInternals) setInternals();
  }

  /** Sets the Blue File Type.
      @param value The Blue File Type (e.g. 1000 for Type 1000 file, 3000 for Type 3000 file, etc.).
   */
  public void setType (int value) { packL(OFF_TYPE,value); setInternals(); }

  /** Sets the data format mode.
      @param value The data format mode ('S' for scalar, 'C' for complex, etc.).
   */
  public void setFormatMode (byte value) { packB(OFF_FORMAT,value); setInternals(); }

  /** Sets the data format type.
      @param value The data format type ('B' for byte, 'D' for double, etc.).
   */
  public void setFormatType (byte value) {
    if (!isBlueVersion1dot2 && (value == DataFile.ASCII256)) {
      // We have the new 'Z' Data Type added in BLUE ICD 1.2, so file needs VER of 1.2
      setBlueVer("1.2");
    }
    packB(OFF_FORMAT+1,value); setInternals();
  }

  /** Sets the data format.
      @param value The data format (e.g. "SB", "VD", "8A", etc.).
   */
  public void setFormat (String value) {
    if (!isBlueVersion1dot2 && (value.charAt(1) == DataFile.ASCII256)) {
      // We have the new 'Z' Data Type added in BLUE ICD 1.2, so file needs VER of 1.2
      setBlueVer("1.2");
    }
    packS(OFF_FORMAT,CHARS_FORMAT,value); setInternals();
  }

  /** Sets the data flag mask.
      @param value The flag mask.
   */
  public void setDataMask (short value) { packI(OFF_FLAGMASK,value); }

  /** Sets the time code of the file, this only sets the timecode filed in the file, it does NOT
      update <code>TC_PREC</code>, see {@link #setTime(double,double)} for details.
      @param value The value for the timecode field.
   */
  public void setTimeCode (double value) { packD(OFF_TIMECODE,value); }

  /** Sets the time code of the file, this only sets the "timecode" field in the file,
      it does NOT update <code>TC_PREC</code> (see {@link #setTime(double,double)} for details).
      This uses {@link Time#getSec()} for up to microsecond precision (since J1950)
      in the "timecode" field.
      @param time The time for the timecode field (cannot be null)
      @since NeXtMidas 3.3.0
   */
  public void setTimeCode (Time time) { setTimeCode(time.getSec()); }

  /** pack in Inlet owner in BLUE header
   *  @param value Inlet owner
   */
  public void setInlet   (short value) { packI(OFF_INLET,value); }
  /** pack in number of Outlets in BLUE header
   *  @param value Outlets
   */
  public void setOutlets (short value) { packI(OFF_OUTLETS,value); }
  /** pack in Outlet async mask in BLUE header
   *  @param value OutMask
   */
  public void setOutMask (int value)   { packL(OFF_OUTMASK,value); }
  /** pack in Pipe Location in BLUE header
   *  @param value Pipe Location
   */
  public void setPipeLoc (int value)   { packL(OFF_PIPELOC,value); }

  /** If this is a pipe, set its size.
      @param value The new size for the pipe.
   */
  public void setPipeSize (int value) { packL(OFF_PIPESIZE,value); if (isPipe()) iop.pipe.setSize(value); }

  /** Calls {@link Pipe#setInByte(double)} to set the index of the next byte to be written
      @param value index of the next byte to be written
   */
  public void setInByte (double value) { packD(OFF_INBYTE,value); if (isPipe()) iop.pipe.setInByte(value); }

  /** Calls {@link Pipe#setInByte(double)} to set the index of the next byte to be read by a given outlet
      @param i     outlet number (1..N) or 0 for inlet.
      @param value The new index.
   */
  public void setOutByte (int i,double value) { packD(OFF_OUTBYTE+i*8,value); if (isPipe()) iop.pipe.setOutByte(i,value); }

  /** Sets the length of the keyword string. <br>
      It is best not to call this directly, rather one should use {@link #setKeywords(String)} which
      will set the keyword string and set this value to match the length of the string.
      @param value The length of the keyword string in bytes.
   */
  public void setKeyLength (int value) { packL(OFF_KEYLGTH,value); }

  /** Sets the keywords string.
      @param value The keyword string.
   */
  public void setKeywords (String value) {
    // TODO: JC4 - this should do a setKeyLength(min(92, value.length())
    packS(OFF_KEYWORDS,92,value);
    setKeyLength(value.length());
  }

  /** Sets extended header keywords from a string<br>
   *  This was specifically added to support setting
   *  keywords via the file qualifier EXTKW=...
   *
   *  @param value string with comma separated list of kEY=VALUE
   *  @since NeXtMidas 3.9.3 for NXM-3009
   */
  public void setExtKw (String value) {
    String[] extKeywords = value.split(",");
    for (String keyvalue : extKeywords) {
      String[] kv = keyvalue.split("=");
      keywords.setKey(kv[0].trim(), kv[1].trim());
    }
  }

  /** Sets main header keywords from a string<br>
   *  This was specifically added to support setting
   *  keywords via the file qualifier MAINKW=...<br>
   *  NOTE: In the situation where the output is a
   *  pipe file (i.e. _pipename), changes to core
   *  keywords (ex: VER, CREATOR) will not propagate
   *
   *  @param value string with comma separated list of kEY=VALUE
   *  @since NeXtMidas 3.9.3 for NXM-3009
   */
  public void setMainKw (String value) {
    String[] mainKeywords = value.split(",");
    for (String keyvalue : mainKeywords) {
     String[] kv = keyvalue.split("=");
     keywords.putMain(kv[0].trim(), kv[1].trim());
    }
  }

  /** Propagate Timeline Keywords WITHOUT any adjustment or modification
   *
   *  @param dontTouch "true" if the Timeline Keywords should exactly match those
   *                   of the reference file
   *  @since NeXtMidas 3.9.3
   */
  public void setSameTLKW (String dontTouch) {
    sameTimelineKeywords = dontTouch.toLowerCase().startsWith("true");
  }

  /** Propagate Timeline Keywords WITHOUT any adjustment or modification
   *
   *  @param dontTouch true if the Timeline Keywords should exactly match those
   *                   of the reference file
   *  @since NeXtMidas 3.9.3
   */
  public void setSameTLKW (boolean dontTouch) {
    sameTimelineKeywords = dontTouch;
  }

  /** Propagate Timeline Keywords WITHOUT any adjustment or modification
   *
   *  @param dontTouch "true" if the Timeline Keywords should exactly match those
   *                   of the reference file
   */
  public void setSameTimeLineKeywords (String dontTouch) {
    setSameTLKW(dontTouch);
  }

  /** Propagate Timeline Keywords WITHOUT any adjustment or modification
   *
   *  @param dontTouch true if the Timeline Keywords should exactly match those
   *                   of the reference file
   */
  public void setSameTimeLineKeywords (boolean dontTouch) {
    setSameTLKW(dontTouch);
  }

  /** Support accurate <b>raster</b> RF readouts by honoring Extended Header
   *  tune adjustment keywords, displaying in IF, but having an RF readout
   *
   *  @param supportTuneAdjustmentsOnRaster true for tune adjustment support
   */
  @InternalUseOnly
  public void setRasterTuneAdjust (boolean supportTuneAdjustmentsOnRaster) {
    if (tuneAdjustments == null) {
      // Load Tune Adjustments from keywords
      try {
        tuneAdjustments = new TuneAdjust(keywords, getXStart(), getYStart());
      }
      catch (Exception e) {
        System.out.println("exception "+e);
      }
    }
    this.supportTuneAdjustmentsOnRaster = supportTuneAdjustmentsOnRaster;
  }

  /** Get if accurate <b>raster</b> RF readouts by honoring Extended Header
   *  tune adjustment keywords, displaying in IF, but having an RF readout
   *  is enabled
   *
   *  @return if raster tune adjustment support is enabled
   */
  @InternalUseOnly
  public boolean getRasterTuneAdjust () {
    return this.supportTuneAdjustmentsOnRaster;
  }

  /** Get Tune Adustments
   *  designed to be used by Layer2D
   *  @return object with tune adjustments
   */
  @InternalUseOnly
  public TuneAdjust getTuneAdjustments() {
    return this.tuneAdjustments;
  }

  /** Sets the start position along the X-axis.
      @param value The start position.
   */
  public void setXStart (double value) { packD(OFF_ADJUNCT+ADJUNCT_XSTART_OFF,value); }

  /** Sets the delta between points along the X-axis.
      @param value The delta.
   */
  public void setXDelta (double value) { packD(OFF_ADJUNCT+ADJUNCT_XDELTA_OFF,value); }

  /** Sets the units for values along the X-axis.
      @param value The units (see {@link Units}).
   */
  public void setXUnits (int value) { packL(OFF_ADJUNCT+ADJUNCT_XUNITS_OFF,value); }

  /** Sets the units for values along the X-axis.
      @param value The units (see {@link Units}).
   */
  public void setXUnits (String value) { setXUnits(getUnitsID(value)); }

  /** Sets the sub-element/frame size for Type 1000 and Type 2000 files. In most cases it is best
      to use the {@link #setFrameSize(int)} method in place of this one.
      @param value The sub-element/frame size.
   */
  public void setSubSize (int value) { packL(OFF_ADJUNCT+ADJUNCT_SUBSIZE_OFF,value); extendHeader(); }

  /** Sets the start position along the Y-axis.
      @param value The start position.
   */
  public void setYStart (double value) { packD(OFF_ADJUNCT+ADJUNCT_YSTART_OFF,value); }

  /** Sets the delta between points along the Y-axis.
      @param value The delta.
   */
  public void setYDelta (double value) { packD(OFF_ADJUNCT+ADJUNCT_YDELTA_OFF,value); }

  /** Sets the units for values along the Y-axis.
      @param value The units (see {@link Units}).
   */
  public void setYUnits (int value) { packL(OFF_ADJUNCT+ADJUNCT_YUNITS_OFF,value); }

  /** Sets the units for values along the Y-axis.
      @param value The units (see {@link Units}).
   */
  public void setYUnits (String value) { setYUnits(getUnitsID(value)); }

  /** Sets the number of subrecords/components in a Type 3000/5000/6000 file.
      @param value The number of subrecords/components.
   */
  public void setRecLength (int value) { packL(OFF_ADJUNCT+ADJUNCT_RECLGTH_OFF,value); }

  /** Sets the name of a subrecord in a Type 3000/5000/6000 file.
      @param i     The index of the subrecord (0..n-1).
      @param value The name of the subrecord.
   */
  public void setRecName (int i, String value) {
    int max = (t6s!=null)? 24 : 4;
    int off = 0;
    int isub = iSub(i);

    if (value.length() > max) {
      off = 1;
      if (recNames == null) initRecNames();
    }

    if (t6s!=null) {
      if (off == 1) setT6s(i,0,1,"~");
      setT6s(i,off,24-off,value);
    }
    if ((t6s==null) || (hb.length >= iSub(i+1))) {
      if (off == 1) hb[isub] = '~';
      packS(isub+off,4-off,value);
    }
    if (recNames != null) recNames[i] = value;
  }

  /** Sets the format of a subrecord/component in a Type 3000/5000/6000 file.
      @param i     The index of the subrecord/component (0..n-1).
      @param value The format of the subrecord/component (e.g. "SB", "VD", etc.).
   */
  public void setRecFormat (int i, String value) {

    if (!isBlueVersion1dot2 && (value.charAt(1) == DataFile.ASCII256)) {
      // We have the new 'Z' Data Type added in BLUE ICD 1.2, so file needs VER of 1.2 - NXM-3570
      setBlueVer("1.2");
    }

    if (t6s!=null) {
      setT6s(i,88,2,value);
      if (hb.length >= iSub(i+1)) packS(iSub(i)+4,2,value); // Include Type 3000 entry if space available
    } else {
      packS(iSub(i)+4,2,value); // Types 3000/5000 in same position
    }

    // Bug 2665 - need to check format when setting sub-record format
    String form = getFormat();
    if (!form.equals("NH") && !form.equals(value)) {
      setFormat("NH");
    }
  }

  /** Sets the offset of a subrecord in a Type 3000/6000 file.
      @param i     The index of the subrecord (0..n-1).
      @param value The byte offset of the subrecord.
   */
  public void setRecOffset (int i, short value) {
    if (t6s!=null) {
      setT6l(i,72,8,value);
      if (hb.length >= iSub(i+1)) packI(iSub(i)+6,value); // Include Type 3000 entry if space available
    } else if (typeClass == 3) {
      packI(iSub(i)+6,value);
    }
  }

  /** Sets the comp-type of a subrecord in a Type 5000 file.
      @param i     The index of the component (0..n-1).
      @param value The comp-type of the component (see {@link Units}).
   */
  public void setRecType (int i, byte value) {
    if (typeClass == 5) packB(iSub(i)+6,value);
  }

  /** Sets the units of a subrecord in a Type 5000/6000 file.
      @param i     The index of the component (0..n-1).
      @param value The units of the component (see {@link Units}).
   */
  public void setRecUnits (int i, byte value) {
    if (t6s!=null) setT6l(i,84,4,value);
    if (typeClass == 5) packB(iSub(i)+7,value);
  }

  /** Sets the name of a component in a Type 5000 file.
      @param i     The index of the component (0..n-1).
      @param value The name of the component.
   */
  public void setCompName (int i, String value) { setRecName(i,value); }

  /** Sets the format of a component in a Type 5000 file.
      @param i     The index of the component (0..n-1).
      @param value The format of the component (e.g. "SB", "VD", etc.).
   */
  public void setCompFormat (int i, String value) { setRecFormat(i,value); }

  /** Sets the comp-type of a component in a Type 5000 file.
      @param i     The index of the component (0..n-1).
      @param value The comp-type of the component (see {@link Units}).
   */
  public void setCompType (int i, byte value) { setRecType(i,value); }

  /** Sets the units of a component in a Type 5000 file.
      @param i     The index of the component (0..n-1).
      @param value The units of the component (see {@link Units}).
   */
  public void setCompUnits (int i, byte value) { setRecUnits(i,value); }

  /** Sets the reference frame for a Type 5000 file. Although this is generally
      set via the functions in {@link Transform}, it is possible to set this
      directly (e.g. via HEADERMOD).
      @param value The reference frame. This should be one of "ECI", "ECR",
                   "TOP" or "TOPOCENT" (other values are legal, but discouraged
                   by the BLUE 1.0 ICD).
   */
  public void setReferenceFrame (String value) {
    // TODO: This should error out if not a Type 5000 file as calling this will
    //       corrupt the definitions for a Type 3000 file.
    packS(416,8,value);
  }

  /** Sets a "quadword" for a Type 5000 file. Although this is generally
      set via the functions in {@link Transform}, it is possible to set this
      directly (e.g. via HEADERMOD).
      @param i     The quadword number (1..11). Although it technically legal to
                   use i=0, doing so is highly discouraged as i=0 is generally
                   a string describing the reference frame (see {@link #setReferenceFrame}).
      @param value The quadword value.
   */
  public void setQuadword (int i, double value) {
    /* TODO: This should error out if not a Type 5000 file as calling this will
             corrupt the definitions for a Type 3000 file. This should also error
             out if i<0 or i>11 as these will corrupt even a Type 5000 file.
     */
    packD((OFF_ADJUNCT+ADJUNCT_QUADWORD_OFF)+i*8,value);
  }

  /** Get the comment from the file (or user specified)
      NOTE: File must be open() or this method will always return null
      @since NeXtMidas 2.7.3
   */
  @Override
  public String getComment () {
    return (String) keywords.get("COMMENT");
  }

  /** Sets the <code>COMMENT=</code> keyword in the file. This is usually used for user-defined comments
      about the file.
   */
  @Override
  public void setComment (String value) {
    super.setComment(value);
    keywords.put("COMMENT",value);
  }

  /** Gets the ID of the file's creator.
      @return The files creator or "UNKNOWN" if unknown.
   */
  public String getCreator () {
    String creator = keywords.getMain("CREATOR");
    if (creator==null) creator = "UNKNOWN";
    return creator;
  }

  /** Checks the value of the creator keyword.
      @param  str Creator to check for (e.g. {@code "<NXM2.4.2", "<=NXM2.4.1", ">=NXM1.8.2" or "=NXM2.6.0"}).
      @return true if it is a match, false if not
   */
  private boolean isCreator (String str) {
    String creator = getCreator();

    if (creator.equals("UNKNOWN")) return false;

    boolean lt = false;
    boolean gt = false;
    boolean eq = false;

    if (str.startsWith("<")) { lt = true; str = str.substring(1); }
    if (str.startsWith(">")) { gt = true; str = str.substring(1); }
    if (str.startsWith("=")) { eq = true; str = str.substring(1); }
    String expTri = str.substring(0,3); // Trigraph for creator ("NXM" or "XM ")
    String expVer = str.substring(3);   // Version number
    String actVer;

    if (!creator.startsWith(expTri)) {
      // Different frameworks --> not a match
      return false;
    }
    else if (creator.matches("...[0-9]+[.][0-9]+[.][0-9]+")) {
      // Current version number scheme (e.g. "2.6.0")
      actVer = creator.substring(3);
    }
    else if (creator.matches("...[0-9][0-9][0-9]")) {
      // Convert old number scheme (e.g. "180" to "1.8.0")
      actVer = "" + creator.charAt(3) + "." + creator.charAt(4) + "." + creator.charAt(5);
    }
    else if (creator.matches("...[0-9][0-9][0-9][a-zA-Z]")) {
      // Convert extremely old number scheme with suffix (e.g. "160a" to "1.6.0")
      actVer = "" + creator.charAt(3) + "." + creator.charAt(4) + "." + creator.charAt(5);
    }
    else {
      // Does not match numbering scheme
      return false;
    }

    int comp = StringUtil.compareVersions(actVer, expVer);
    return (eq && (comp == 0)) || (lt && (comp < 0)) || (gt && (comp > 0));
  }

  /** Turn off or on the default autoconversion of PLATINUM files to BLUE files
   *  @param autoConvertPlatinum true  - autoconvertion is on
   *                             false - autoconversion is off
   */
  public static synchronized void setAutoConvertPlatinumDefault (boolean autoConvertPlatinum) {
    DataFile.autoConvertPlatinumDefault = autoConvertPlatinum;
  }

  /** Get if autoconversion of PLATINUM files to BLUE files is default
   *  @return true if enabled
   */
  public static boolean getAutoConvertPlatinumDefault () {
    return DataFile.autoConvertPlatinumDefault;
  }

  /** Turn off or on the default autoconversion of PLATINUM files to BLUE files
   *  @param autoConvertPlatinum true  - autoconvertion is on
   *                             false - autoconversion is off
   */
  public synchronized void setAutoConvertPlatinum (boolean autoConvertPlatinum) {
    this.autoConvertPlatinum = autoConvertPlatinum;
  }

  /** Get if autoconversion of PLATINUM files to BLUE files is default
   *  @return true if enabled
   */
  public boolean getAutoConvertPlatinum () {
    return this.autoConvertPlatinum;
  }

  /** Sets the default VER=, IO= and CREATOR= keywords.  */
  protected void setBlueVerIO () {
    // Use file's current version (NXM-3570 3.9.4)
    // may just be adding data where 'Z' type already used
    // If no BLUE VER is present or version is 1.0, default to 1.1 as in past
    String currentVer = keywords.getMain("VER");
    String defaultVersion = (currentVer==null || currentVer.contentEquals(BLUE_VER_ORIG)) ?
        BLUE_VER_DEFAULT : currentVer;
    setBlueVer(defaultVersion);
    setBlueIO("NeXtMidas");

    // The following is for backwards compatibility with NXM 2.4.2
    String value = "NXM"+Installation.NM_VERSION;
    keywords.putMain("CREATOR",value,false);
  }

  /** Sets the version of the BLUE file ICD used when writing the file.
      @param blueVer The version to use (must be "1.0","1.1", or "1.2").
   */
  public final void setBlueVer (String blueVer) { setBlueVer(blueVer,false); }

  /** Sets the name of the I/O library that created the file.
      @param blueIO  The name of the I/O library that created the file (must be "NeXtMidas").
   */
  public final void setBlueIO (String blueIO) { setBlueIO(blueIO,false); }

  /** <b>INTERNAL USE ONLY:</b> Sets the version of the BLUE file ICD used when writing the file.
      @param blueVer The version to use (must be "1.0", "1.1", or "1.2").
      @param force   Force the version to be set even if invalid. Setting this to true violates
                     the BLUE CCD. (This is here for testing purposes ONLY.)
   */
  @InternalUseOnly
  public void setBlueVer (String blueVer, boolean force) {
    boolean forceQual = Convert.o2z(getFileName().getQualifier("FORCE"));
    boolean forcing = force || forceQual;
    allowablePlatinum = !forcing && autoConvertPlatinum && PLATINUM_VER.matcher(blueVer).matches();
    if (!forcing) {
      if (allowablePlatinum) {
        blueVer = BLUE_VER_DEFAULT;
      }
      // the FORCE qualifier is for INTERNAL USE only (to create test data) and CAN be changed/removed at any time
      else if (!checkBlueVer(blueVer)) {
        throw new MidasException("DataFile: Can not set VER="+blueVer+", version number is invalid.");
      }
    }
    isBlueVersion1dot2 = blueVer.equals(BLUE_VER_Z_DATA);
    keywords.putMain("VER", blueVer, true);
  }

  /** <b>INTERNAL USE ONLY:</b> Sets the name of the I/O library that created the file.
      @param blueIO  The name of the I/O library that created the file (must be "NeXtMidas").
      @param force   Force the library name to be set even if invalid. Setting this to true violates
                     the BLUE CCD. (This is here for testing purposes ONLY.)
   */
  @InternalUseOnly
  public void setBlueIO (String blueIO, boolean force) {
    // the FORCE qualifier is for INTERNAL USE only (to create test data) and CAN be changed/removed at any time
    if (!force && !Convert.o2z(getFileName().getQualifier("FORCE")) && !blueIO.equals("NeXtMidas")) {
      throw new MidasException("DataFile: Can not set IO="+blueIO+", I/O library name is invalid.");
    }
    keywords.putMain("IO", blueIO, true);
  }



  /** Gets the version of the BLUE file ICD used when writing the file (as identified by the
      <code>VER=</code> keyword in the main header). If the <code>IO=</code> keyword is not in the header,
      version <code>"1.0"</code> is assumed.
      @return The version of the BLUE file ICD used when writing the file.
      @since NeXtMidas 2.5.3
   */
  public String getBlueVer () {
    String val = keywords.getMain("VER");
    return (val==null)? "1.0" : val;
  }

  /** Checks to see if a compatible VER= is given. */
  private boolean checkBlueVer () { return checkBlueVer(getBlueVer()); }

  /** Checks to see if a compatible VER= is given. */
  private boolean checkBlueVer (String ver) {
    return ver.equals(BLUE_VER_ORIG) || ver.equals(BLUE_VER_DEFAULT) ||
           ver.contentEquals(BLUE_VER_Z_DATA);
  }

  /** Gets the name of the I/O library that created the file (as identified by the <code>IO=</code>
      keyword in the main header). If the <code>IO=</code> keyword is not in the header, but the
      <code>CREATOR=</code> keyword identifies the creator as an older version of NeXtMidas,
      <code>"NeXtMidas"</code> is returned. Otherwise, if the <code>IO=</code> keyword is not in the header,
      <code>"UNKNOWN"</code> will be returned.
      @return The name of the I/O library that created the file.
      @since NeXtMidas 2.5.3
   */
  public String getBlueIO () {
    String val = keywords.getMain("IO");
    if (val == null) {
      String creator = keywords.getMain("CREATOR");
      if ((creator != null) && creator.startsWith("NXM")) val = "NeXtMidas";
    }
    return (val==null)? "UNKNOWN" : val;
  }

  /** Gets the byte index of the start of a Type 3000 subrecord definition or a Type 5000 component definition.
      @param i The subrecord/component index (0..n-1).
      @return The byte-index of the start of the subrecord/component in the adjunct header.
   */
  private int iSub (int i) {
    if (i>=14 && typeClass==5) return 512+(i-14)*8;
    return 304+i*8;
  }

  /** Sets the sub-element/frame size for Type 1000 and Type 2000 files. <br>
      This method just calls {@link #setSubSize(int)}, and is provided for use in legacy primitives,
      all new primitives should use {@link #setSubSize(int)} in place of this.
      @param value The sub-element/frame size.
   */
  @KeyObjectNames() // hide to disambiguate with {@link #setSubSize()}
  public void setSubsize (int value) { setSubSize(value); }

  /** Gets the sub-element/frame size for Type 1000 and Type 2000 files. <br>
      This method just calls {@link #getSubSize()}, and is provided for use in legacy primitives,
      all new primitives should use {@link #getSubSize()} in place of this.
      @return The sub-element/frame size.
   */
  public int getSubsize () { return getSubSize(); }

  /** Set number of components for Type 5000 file. Same as {@link #setSubSize(int)}.
   *  @param value number of components
   */
  public void setNumberComponents (int value) { setSubSize(value); }

  /** Get number of components for Type 5000 file. Same as {@link #getSubSize()}.
   *  @return number of components
   */
  public int getNumberComponents () { return getSubSize(); }

  /** Set number of SubRecords for Type 3000/6000 file. Same as {@link #setSubSize(int)}.
   *  @param value number of SubRecords
   */
  public void setNumberSubRecords (int value) { setSubSize(value); }

  /** Get number of SubRecords for Type 3000/6000 file. Same as {@link #getSubSize()}.
   *  @return number of SubRecords
   */
  public int getNumberSubRecords () { return getSubSize(); }

  @Override public double getStart () { return (typeClass==2)? getYStart() : getXStart(); }
  @Override public double getDelta () { return (typeClass==2)? getYDelta() : getXDelta(); }
  @Override public int    getUnits () { return (typeClass==2)? getYUnits() : getXUnits(); }

  /** Gets the Y-Rate or the X-Rate depending on the BLUE file type
   *  @return Y-Rate or X-Rate depending on the BLUE file type
      @since NeXtMidas 3.6.0
   */
  public double getRate () { return (typeClass==2)? getYRate() : getXRate(); }

  /** This is the same as <code>getUnitsName((int)key)</code>.
      @param key The type of units.
      @return The name of the units.
   */
  public static String getCompUnitsName (byte key) {
    return getUnitsName ((int)key);
  }

  /** Get the name string for a specified type of units. For example
      <code>getUnitsName({@link Units#TIME})</code> returns <code>"Time (sec)"</code>.
      @param key   The type of units.
      @param cmult The multiplier string.
      @return The name of the units.
      @see Units
   */
  public static String getUnitsName (int key, String cmult) {
    String name = "Unknown (U)";

    if (key == Units.FREQUENCY) {
      name = "Frequency (" + cmult + "Hz)"; // JC4: Old special-case... should be changed.
    }
    else if ((key >= 0) && (key < Units.UNITS_INFO.length) && (Units.UNITS_INFO[key][0] != null)) {
      if (Units.UNITS_INFO[key][2] != null) {
        name = Units.UNITS_INFO[key][1] + " (" + Units.UNITS_INFO[key][2] +")";
      } else {
        name = Units.UNITS_INFO[key][1] + " (U)";
      }
    }

    return name;
  }

  /** Get the variable id for a specified Type 5000 component type.
      @param name The type of units.
      @return The name of the units.
      @see Units
   */
  public static int getCompTypeID (String name) {
    return getCompTypeID(name, Units.NONE);
  }

  /** Get the variable id for a specified Type 5000 component type.
      @param name The type of units.
      @param def  The default if not found.
      @return The name of the units.
      @see Units
   */
  private static int getCompTypeID (String name, int def) {
    name = name.trim().toUpperCase();
    return Parser.find(Units.compTypeList, name, def);
  }

  /** <b>Work in progress may change within this beta series:</b>
      Get the variable id for a specified type of units. For example
      <code>getUnitsID("TIME")</code> and <code>getUnitsID(1)</code> both return 1.
      @param name The type of units.
      @return The ID of the units. If the name is null, {@link Units#NONE} is
              returned; if the units are otherwise unknown, {@link Units#UNKNOWN}
              is returned.
      @see Units
      @since NeXtMidas 3.3.0
   */
  public static int getUnitsID (Object name) {
    if (name == null) return Units.NONE;
    if (name instanceof Number) {
      int id = ((Number)name).intValue();
      return (id < 0)? Units.UNKNOWN : id;
    }
    return getUnitsID(name.toString());
  }

  /** Get the variable id for a specified type of units. For example <code>getUnitsID("TIME")</code>
      returns <code>1</code>.
      @param name The type of units.
      @return The name of the units.
      @see Units
   */
  public static int getUnitsID (String name) {
    return getUnitsID(name, Units.UNKNOWN);
  }

  /** Get the variable id for a specified type of units. For example <code>getUnitsID("TIME")</code>
      returns <code>1</code>.
      @param name The type of units.
      @param def  The default if not found.
      @return The name of the units.
      @see Units
   */
  private static int getUnitsID (String name, int def) {
    int id = -1;

    name = name.trim().toUpperCase();

    for (int i = 0; (id < 0) && (i < Units.UNITS_ABBR.length); i++) {
      if (name.equals(Units.UNITS_ABBR[i][0])) {
        id = ((Integer)Units.UNITS_ABBR[i][1]).intValue();
      }
    }
    for (int i = 0; (id < 0) && (i < Units.UNITS_INFO.length); i++) {
      if (name.equals(Units.UNITS_INFO[i][0])) {
        id = i;
      }
    }
    if ((id < 0) && StringUtil.isNumber(name)) {
      id = Convert.o2l(name);
    }

    return (id < 0)? Units.UNKNOWN: id;
  }

  /** Get the variable name for a specified type of units. For example <code>getUnitsVarName(1)</code>
      returns <code>"TIME_S"</code>.
      @param key The type of units.
      @return The name of the units.
      @see Units
   */
  public static String getUnitsVarName (int key) {
    String varName = null;

    if ((key >= 0) && (key < Units.UNITS_INFO.length)) {
      varName = Units.UNITS_INFO[key][0];
    }

    return (varName == null)? "UNKNOWN": varName;
  }

  /** Get the name string for a specified type of units. For example
      <code>getUnitsName({@link Units#TIME})</code> returns <code>"Time (sec)"</code>.
      @param key The type of units.
      @return The name of the units.
      @see Units
   */
  public static String getUnitsName (int key) {
    return getUnitsName(key,"");
  }

  /** Get the name string for a specified type of units. For example
      <code>getUnitsName({@link Units#TIME})</code> returns <code>"sec"</code>.
      @param key   The type of units.
      @param cmult The multiplier string.
      @return The name of the units or <code>""</code> for unknown units or for
              those with no unit name.
      @see Units
   */
  public static String getUnitsNameShort (int key, String cmult) {
    String name = "";

    if (key == Units.FREQUENCY) {
      name = cmult + "Hz"; // JC4: Old special-case... should be changed.
    }
    else if ((key >= 0) && (key < Units.UNITS_INFO.length) && (Units.UNITS_INFO[key][2] != null)) {
      name = Units.UNITS_INFO[key][2];
    }

    return name;
  }

  /** Get the name string for a specified type of units. For example
      <code>getUnitsName({@link Units#TIME})</code> returns <code>"sec"</code>.
      @param key The type of units.
      @return The name of the units or <code>""</code> for unknown units or for
              those with no unit name.
      @see Units
   */
  public static String getUnitsNameShort (int key) {
    return getUnitsNameShort(key, "");
  }

  /** Invokes {@link Data#getFormatTypeName(byte)}
      @param key The type of data.
      @return The name of the data type.
   */
  public static String getFormatTypeName (byte key) {
    return Data.getFormatTypeName(key);
  }

  /** Invokes {@link Data#getFormatModeName(byte)}
      @param key The type of array.
      @return The name of the array type.
   */
  public static String getFormatModeName (byte key) {
    return Data.getFormatModeName(key);
  }

  /** Get the name string for a specified computation type. For example
      <code>getCompTypeName({@link Units#SCALAR})</code> returns <code>"Scalar"</code>.
      @param key The type of computation.
      @return The name of the computation type.
      @see Units
   */
  public static String getCompTypeName (byte key) {
    return getCompTypeName(key,false);
  }

  /** Get the name string for a specified computation type. For example
      <code>getCompTypeName({@link Units#SCALAR})</code> returns <code>"Scalar"</code>.
      @param key The type of computation.
      @return The name of the computation type.
      @see Units
   */
  private static String getCompTypeName (byte key, boolean upcase) {
    String name = null;

    if ((key >= 0) && (key < Units.COMP_TYPE_INFO.length)) {
      name = (upcase)? Units.COMP_TYPE_INFO[key][1]: Units.COMP_TYPE_INFO[key][0];
    }

    return (name != null)? name: getCompTypeName((byte)0,upcase);
  }

  /** Sets the X or Y start position based on the Blue file type
      @param start The start position
      @since NeXtMidas 3.2.1
   */
  public void setStart (double start) {
    if (typeClass==2) setYStart(start); else setXStart(start);
  }
  /** Sets the delta between points along the X-axis or Y-axis based on the Blue file type
      @param delta The delta between points
      @since NeXtMidas 3.2.1
    */
  public void setDelta (double delta) {
    if (typeClass==2) setYDelta(delta); else setXDelta(delta);
  }
  /** Sets the units for values along the X-axis or Y-axis based on the Blue file type
      @param units The units for data values
      @since NeXtMidas 3.2.1
    */
  public void setUnits (int units) {
    if (typeClass==2) setYUnits(units); else setXUnits(units);
  }

  /** This is the same as <code>return getRecOffset(i)</code>.
      @param i The index of the subrecord (0..n-1).
      @return The byte offset of the subrecord.
   */
  public final short getCompOffset (int i) { return getRecOffset(i); }

  /** Validate a user format input and convert to internal form
   * @param format user data format
   * @param allowable allow/disallow list to validate against
   */
  public void setFormat (String format, String allowable) {
    if (!checkFormat(format,allowable)) M.error("Unallowed Format: "+format);
    setFormat(format);
  }

  /** Validate a user type input and convert to internal form
   * @param type      sser Blue File type
   * @param allowable allow/disallow list to validate against
   */
  public void setType (int type, String allowable) {
    if (!checkType(type,types)) M.error("Unallowed Type: "+type);
    setType(type);
  }

  /** Set the size of the file in logical elements. */
  @Override
  public void setSize (double value) {
    size = value;
    getBPE();
    double bsize = size*dbpe;
    if (pkh!=null) bsize = pkh.dataToPacketOffset(this,bsize);
    setDataSize(bsize);
    if (isOpen) wsize=size;
  }

  /** set the size of the file in logical elements to reflect current bytes */
  public void setSize() {
    double bsize = getDataSize();
    if (pkh!=null) bsize = pkh.packetToDataOffset(this,bsize);
    if (trimSet) size = trim2-trim1;
    else if (dbpe>0) size = bsize/dbpe;
  }

  /** Sets the header timecode and always put <code>TC_PREC</code> keyword (even if it is zero) see {@link #getTime()} for details.
      @param time The {@link Time} object with <code>wsec</code> and <code>fsec</code>.
   */
  public void setTime (Time time) { setTime(time.getWSec(),time.getFSec(), true); }

  /** Sets the header timecode and <code>TC_PREC</code> see {@link #getTime()} for details.
      @param time The {@link Time} object with <code>wsec</code> and <code>fsec</code>.
      @param alwaysPutTCPREC true to always put <code>TC_PREC</code> keyword, false to only put it when it is not zero.
      @since NeXtMidas 3.5.4
   */
  public void setTime (Time time, boolean alwaysPutTCPREC) { setTime(time.getWSec(),time.getFSec(), alwaysPutTCPREC); }

  /** Sets the header timecode and always put <code>TC_PREC</code> keyword (even if it is zero) see {@link #getTime()} for details.
      @param wsec Whole seconds since 1 Jan 1950.
      @param fsec Fractional seconds since 1 Jan 1950.
*/
  public void setTime (double wsec, double fsec) {
    setTime(wsec, fsec, true);
  }

  /** Sets the header timecode and <code>TC_PREC</code> see {@link #getTime()} for details.
      @param wsec Whole seconds since 1 Jan 1950.
      @param fsec Fractional seconds since 1 Jan 1950.
      @param alwaysPutTCPREC true to always put <code>TC_PREC</code> keyword, false to only put it when it is not zero.
      @since NeXtMidas 3.5.4 moved code from original setTime(double,double) and added alwaysPutTCPREC param
   */
  public void setTime (double wsec, double fsec, boolean alwaysPutTCPREC) {
    double timeu = 1e-6  * Math.floor(fsec*1e6+.5);
    double timep = 1e-12 * Math.floor((fsec-timeu)*1e12+.5);
    setTimeCode(wsec+timeu);
    if (alwaysPutTCPREC || timep != 0) { // some apps require TC_PREC so default is to always include TC_PREC
      keywords.putMain("TC_PREC",""+timep);
    }
  }

  /** Get TC_PREC double value from <code>TC_PREC</code> keyword in main header holds precision time.
      @return timecode high precision time as a double, 0.0 if zero or not set.
   */
  private double getTCPREC () {
    double val = 0;
    String tc_prec = keywords.getMain("TC_PREC");
    if (tc_prec!=null) {
      // The try..catch was added for BUG 1003 to gracefully handle invalid TC_PREC.
      try {
        val = Convert.s2d(tc_prec);
      } catch (Exception e) {
        M.warning("Unable to add TC_PREC="+tc_prec+" to time_code value: "+e.getMessage());
      }
    }
    return val;
  }

  /** Gets the header timecode (and <code>TC_PREC</code> keyword, if applicable) and returns the time
      as a {@link Time} object. Within a BLUE file time timecode with microsecond precision is
      stored in a field of the main header in Midas time (seconds since 1 Jan 1950). If a higher
      level of precision is required that information is stored via the main header keyword
      <code>TC_PREC</code>. The absolute value of <code>TC_PREC</code> should be less than 10<sup>-6</sup>
      since the <code>timecode</code> header field already contains a value precise to the microsecond.
      The <code>TC_PREC</code> value should not be used to provide times finer than picosecond precision
      (10<sup>-12</sup>). See section 3.4.1 of the BLUE ICD for more details.
      The resulting time (timecode + <code>TC_PREC</code>) is the "epoch" for the file.
      The "starting time" of the file (time corresponding to the first element) is equal to
      the "epoch" plus {@link #getStart()}; EXCEPT in cases where the file has a <code>TIMELINE</code>
      (if a <code>TIMELINE</code> is present, it overrides the header fields). These computations along
      with <code>TIMELINE</code> handling are handled by the {@link #getTimeAt(double)} method.
      @return The {@link Time} object with <code>wsec</code> and <code>fsec</code>.
   */
  public Time getTime() {
    Time tc = new Time(getTimeCode());
    tc.addSec(getTCPREC());
    return tc;
  }

  /** Gets the high precision time in the file at the specified element index in file
      @param index file index (in elements)
      @param time {@link Time} object to populate (null to create a new instance)
      @return high precision {@link Time} object at specified index (same as input param when not null)
      @since NeXtMidas 3.3.0
   */
  public Time getTimeAt (double index, Time time) {
    if (time == null) time = new Time();

    if (index < 0) {
      index = index + seek();
      if (index < 0) {
        if (getPreviousDataFile() != null) {
          time = getPreviousDataFile().getTimeAt(index, time); // Before latest pipe restart.
        } else {
          time.fromJ1950x(0.0, 0.0); // Before start of data.
        }
        return time;
      }
    }

    index += trim1;
    if (timeLine != null) {
      time = timeLine.getTimeAt(index*dbpe, time);
    } else {
      time.fromTime(getTime());
      time.addSec( getStart() + index*getDelta() );  // this includes TIMECODE + TC_PREC + XSTART
    }
    return time;
  }

  /** Gets the high precision time at the current {@link #seek()} index.
      @return high precision {@link Time} object at current index/offset in file.
      @since NeXtMidas 3.3.0
      @see #getTimeAt() getTimeAt() for lower (microsecond) precision version
   */
  public Time getTimeAtCurrent() {
    return getTimeAt(cftop+offset-trim1, null);
  }

  /** Gets the high precision time at the current {@link #seek()} index.
      @param time {@link Time} object to populate (null to create a new instance)
      @return high precision {@link Time} object at current index/offset in file.
      @since NeXtMidas 3.3.1
   */
  public Time getTimeAtCurrent (Time time) {
    return getTimeAt(cftop+offset-trim1, time);
  }

  /** This is a less-precise (up to microsecond precision) alternative to {@link #getTimeAt(double, Time)}
      that can be used in situations where high-precision is not a concern.
      @param index file index (in elements).
      @return time at specified index (in seconds).
      @see #getTimeAt(double, Time) getTime(double) for a high precision version
   */
  @Override
  public double getTimeAt (double index) {
    return getTimeAt(index, null).getSec();
  }

  /** Gets the less precise (up to microsecond precision) time at the current index.
      @return time at current index (in seconds)
      @see #getTimeAtCurrent() getTimeAtCurrent() for a high precision version
   */
  public double getTimeAt() {
    return getTimeAt(cftop+offset-trim1);
  }

  /** Sets the high precision time at a given index.
      @param index file index (in elements)
      @param time  The time
      @since NeXtMidas 3.3.0
                    (null and zero/negative entries are ignored).
      @throws MidasException for invalid entries, unless
                    {@link IOOptions#SuppressInvalidTimeLineEntryErrors}
                    is set
   */
  public void setTimeAt (double index, Time time) {
    // Since 3.7.2 for NXM-3040 setting invalid entries via this public API
    // will thow a MidasException unless IOOPtion.SuppressInvalidTimeLineEntryErrors
    // is set
    setTimeAt(index, time, NotifyInvalidEntry.EXCEPTION);
  }

  /**
   * Sets the high precision time at a given index.
   * @param index file index (in elements)
   * @param time  The time
   * @param notifyLevel the level of reporting for invalid timeline entries
   *                    {@link TimeLine.NotifyInvalidEntry}
   * @since NeXtMidas 3.7.3
   */
  @InternalUseOnly
  public void setTimeAt  (double index, Time time, NotifyInvalidEntry notifyLevel) {
    index += trim1;
    if (timeLine==null) newTimeLine();
    timeLine.setTimeAt(index, dbpe, time, notifyLevel);
  }

  /** Sets the time (up to microsecond precision) at a given index.
      @param index file index (in elements)
      @param time  The time (seconds since J1950)
      @see #setTimeAt(double, Time) setTime(double index, Time time) for a high precision version
   */
  public void setTimeAt (double index, double time) {
    setTimeAt(index, new Time(time));
  }

  /** Sets the high precision time at a current offset.
      @param time  The time
      @since NeXtMidas 3.3.0
   */
  public void setTimeAt (Time time) {
    setTimeAt(cftop+offset-trim1,time);
  }

  /** Sets the high precision time at a current offset.
      @param time  The time
      @param notifyLevel the level of reporting for invalid timeline entries
                         {@link TimeLine.NotifyInvalidEntry}
      @since NeXtMidas 3.7.3
   */
  public void setTimeAt (Time time, NotifyInvalidEntry notifyLevel) {
    setTimeAt(cftop+offset-trim1,time, notifyLevel);
  }

  /** Set the time at a current offset.
      @param time  The time (seconds since J1950)
      @see #setTimeAt(Time) setTimeAt(Time) for a high precision version
   */
  public void setTimeAt (double time) {
    setTimeAt(cftop+offset-trim1,time);
  }

  /** Gets the file index/offset (in elements) at a given time.
      @param time  The time
      @return N = file index/offset in elements (e.g. frame) in the file or -N = frames before start of this file.
      @since NeXtMidas 3.3.0
   */
  public double getIndexAt (Time time) {
    return getIndexAt(time, 0.0);
  }

  /** Gets the file index/offset (in elements) at a given time.
      @param time  The time (seconds since J1950)
      @return N = file index/offset in elements (e.g. frame) in the file or -N = frames before start of this file.
      @see #getIndexAt(Time) getIndexAt(Time) for a high precision version
   */
  public double getIndexAt (double time) {
    return getIndexAt(new Time(time), 0.0);
  }

  /** <b>Internal use only:</b> Gets the index at a given time. This version uses count to walk
      backwards through previous files (prior to the last pipe restart).
      @param time  The time.
      @param count The running count of frames (i.e. offset in elements) to subtract
      @return N = file index/offset in elements (e.g. frame) in the file or -N = frames before start of this file.
      @since NeXtMidas 3.3.0
   */
  private double getIndexAt (Time time, double count) {
    double index;

//  if (time < getTimeAt(0.0)) { // prior to 3.3.0
    if (time.diff(getTimeAt(0.0, null)) < 0) {
      DataFile df = getPreviousDataFile();
      if (df != null) {
        index = df.getIndexAt(time, df.seek()+count); // Before latest pipe restart.
      } else {
        index = Double.NaN; // Before start of data.
      }
    }
    else {
      double off;
      if (timeLine!=null) {
        off = timeLine.getOffsetAt(time)/dbpe;
      } else {
//      offset = (time-getTimeCode()-getStart())/getDelta(); // prior to 3.3.0 (does not include TC_PREC - b/c precision is lost after microseconds for secs since J1950
        off = (time.diff(getTime()) - getStart()) / getDelta(); // includes TIMECODE + TC_PREC + XSTART
      }
      index = Math.floor(off-trim1) - count;
    }
    return index;
  }

  /** set the nominal transfer length
   *  @param elements nominal transfer length in elements
   */
  public void setXferLength (int elements) { xferLength = elements; }
  /** set the nominal transfer length
   *  @param elements nominal transfer length in elements
   */
  public void setTransLength (int elements) { xferLength = elements; }
  /** set the nominal consume length
   *  @param elements nominal consume length in elements
   */
  public void setConsLength (int elements) { consLength = elements; }

  /** Test for the existence of a named record in a Type 3000/5000 file and return its index.
      @param name The record name.
      @return Returns -1 if the named record does not exist, otherwise returns the record's index.
   */
  public int findRec (String name) {
    if (isRecordBased()) {
      for (int i=0; i<getSubSize(); i++) {
        if (name.equals(getRecName(i))) return i;
      }
    }
    else {
      // JC4: Fast version for Type 1000/2000 files... it looks more complicated but is
      //      needed for performance reasons.
      if ((name.charAt(0) == 'F') || (name.charAt(0) == 'f')) {
        String num = name.substring(1);
        if (StringUtil.isNumber(num)) {
          int index = Convert.s2l(num);

          if ((index >= 0) && (index < getSubSize())) {
            return index;
          }
        }
      }
    }
    return -1;
  }

  /** Test for the existence of a named record in a Type 3000/6000 file and return its byte offset.
      @param name The record name.
      @return Returns -1 if the named record does not exist, otherwise returns the record's byte offset.
   */
  public int findRecOffset (String name) {
    int i = findRec(name);
    return (i < 0)? -1 : getRecOffset(i);
  }

  /** DEPRECATED
   *  @param name The record name.
   *  @return Returns -1 if the named record does not exist, otherwise returns the record's index.
   *  @deprecated NeXtMidas 2.5.0 use {@link #findRec(String)}. */
  @Deprecated public int findIndex (String name) {
    return findRec(name);
  }

  // JC4 (28DEC07): I updated the pack/unpack methods below to just call the methods in Convert
  // that take in the byte rep (these methods were added to Convert a few years ago and we had
  // just not updated the methods below.) This eliminates (1) extra method call and (2) memory
  // copy to/from hbt in cases where we have a foreign header (except for VMS double/float),
  // (3) the localRep variable. Also hbt (512-bytes) is replaced by tempVms (8-bytes).
  /** unpack data elements from the main header */
  private byte   unpackB (int off) { return hb[off]; }
  private short  unpackI (int off) { return Convert.unpackI(hb,off,headRep); }
  private int    unpackL (int off) { return Convert.unpackL(hb,off,headRep); }
  private String unpackS (int off, int len) { return Convert.unpackS(hb,off,len); }
  private double unpackD (int off) {
    if (headRep!='V') return Convert.unpackD(hb,off,headRep); // Fast conversion when not using VMS
    System.arraycopy(hb,off,tempVms,0,8);
    Convert.rep(tempVms,0,DOUBLE,1,headRep,Shell.rep);
    return Convert.unpackD(tempVms,0);
  }

  /** pack data elements from the main header */
  private void packB (int off, byte   val) { hb[off] = val; }
  private void packI (int off, short  val) { Convert.packI(hb,off,val,headRep); }
  private void packL (int off, int    val) { Convert.packL(hb,off,val,headRep); }
  private void packS (int off, int len, String val) { Convert.packS(hb, off, len, val); }
  private void packD (int off, double val) {
    if (headRep!='V') { Convert.packD(hb,off,val,headRep); return; } // Fast conversion when not using VMS
    Convert.packD(tempVms,0,val);
    Convert.rep(tempVms,0,DOUBLE,1,Shell.rep,headRep);
    System.arraycopy(tempVms,0,hb,off,8);
  }

  /** convert data representation for a Subrecord of Interest (SOI)
      for multiple records (wrapper for {@link  nxm.sys.lib.Convert#rep(byte[], int, byte, int, byte, byte, int, int)})
      @param buf  byte array containing records
      @param off  Offset into byte array of 1st SOI
      @param n    Number of scalars per SOI
      @param from Input representation
      @param to   Output representation
      @since NeXtMidas 3.2.1
    */
  public void convertRecordRep (byte[] buf, int off, int n, byte from, byte to) {
    // Except for the three lines marked with "// **" this matches
    // the convertRecordRep(long lbuf, int off, int n, byte from, byte to) method
    byte type,mode;
    int spa,bpa, bpe=(int)dbpe;
    for (int i=0; i<getSubSize(); i++) {
      type = getRecFormatType(i);
      mode = getRecFormatMode(i);
      spa  = Data.getSPA(mode) * getRecElements(i);   // since 3.7.0: multiply num_elts (could be > 1 )for Bug 2958
      bpa  = spa*Data.getBPS(type);
      Convert.rep(buf,off,type,spa, from,to, n,bpe);  // ** convert data rep
      off += bpa;
    }
  }

  /** This is the same as {@link #convertRecordRep(byte[], int, int, byte, byte)},
      except that it takes in a native pointer to a data buffer to work on.
      @since NeXtMidas 2.7.0
   */
  void convertRecordRep (long lbuf, int off, int n, byte from, byte to) {
    // Except for the three lines marked with "// **" this matches
    // the convertRecordRep(byte[] buf, int off, int n, byte from, byte to) method
    byte type,mode;
    int spa,bpa, bpe=(int)dbpe;
    for (int i=0; i<getSubSize(); i++) {
      type = getRecFormatType(i);
      mode = getRecFormatMode(i);
      spa  = Data.getSPA(mode) * getRecElements(i);   // since 3.7.0: multiply num_elts (could be > 1 )for Bug 2958
      bpa  = spa*Data.getBPS(type);
      Convert.rep(lbuf,off,type,spa, from,to, n,bpe); // ** convert data rep
      off += bpa;
    }
  }

  /** Test for valid data representation string (error=0)
   *  @param rep data representation in String form
   *  @return byte form of character representation (error=0)
   */
  public static byte testRep (String rep) {
    if (rep.startsWith("VAX")) return VAX;
    else if (rep.equals("IEEE")) return IEEE;
    else if (rep.equals("EEEI")) return EEEI;
    else return 0;
  }

  /** Convert header to a new representation
   *  @param rep new header representation
   */
  public void convertHeaderRep (String rep) {
    byte from = headRep;
    byte to = testRep(rep);
    convertHeaderRep(from,to);
    setHeadRep(rep);
    // headRep = to; // since 3.1.1 (this is set in convertHeaderRep(..)
  }

  /** Converts the file's MAIN and EXTENDED headers from one representation to another.
      @param from Current buffer representation ({@link #IEEE}, {@link #EEEI}, ...).
      @param to   New buffer representation ({@link #IEEE}, {@link #EEEI}, ...).
   */
  public void convertHeaderRep (byte from, byte to) {
    if (to==from) return;
    convertHeaderRep(hb, from, to);
    if (keywords!=null) keywords.convertExtRep(from,to);
    headRep = to; // since 3.1.1 for BUG 1992 (otherwise subsequent byte-order conversion in keywords will pick up incorrect header rep)
  }

  /** Converts a MAIN header from one representation to another.
      @param buf  The header buffer.
      @param from Current buffer representation ({@link #IEEE}, {@link #EEEI}, ...).
      @param to   New buffer representation ({@link #IEEE}, {@link #EEEI}, ...).
   */
  public void convertHeaderRep (byte[] buf, byte from, byte to) {
    Convert.rep(buf,  12, Convert.LONG,   5, from, to);
    Convert.rep(buf, OFF_DATASTART, Convert.DOUBLE, 2, from, to);  // data_start, data_size
    Convert.rep(buf, OFF_TYPE,      Convert.LONG,   1, from, to);  // file type
    Convert.rep(buf, OFF_FLAGMASK,  Convert.INT,    1, from, to);
    Convert.rep(buf, OFF_TIMECODE,  Convert.DOUBLE, 1, from, to);  // timecode
    Convert.rep(buf, OFF_INLET,     Convert.INT,    2, from, to);
    Convert.rep(buf, OFF_OUTMASK,   Convert.LONG,   3, from, to);
    Convert.rep(buf, OFF_INBYTE,    Convert.DOUBLE,10, from, to);  // in_byte, out_bytes
    Convert.rep(buf, OFF_KEYLGTH,   Convert.LONG,   1, from, to);
    Convert.rep(buf, OFF_ADJUNCT+ADJUNCT_XSTART_OFF, Convert.DOUBLE, 2, from, to);  // adjunct: Xstart, Xdelta
    Convert.rep(buf, OFF_ADJUNCT+ADJUNCT_XUNITS_OFF, Convert.LONG,   2, from, to);
    Convert.rep(buf, OFF_ADJUNCT+ADJUNCT_YSTART_OFF, Convert.DOUBLE, 2, from, to);  // Ystart, Ydelta
    Convert.rep(buf, OFF_ADJUNCT+ADJUNCT_YUNITS_OFF, Convert.LONG,   2, from, to);

    int tc = Convert.unpackL(buf, OFF_TYPE, to) / 1000;
    switch (tc) {
      case 3: case 6:
        for (int i=0; i<26; i++) {
          Convert.rep(buf, (OFF_ADJUNCT+ADJUNCT_SUBRSTR_OFF+6)+i*8, Convert.INT, 1, from, to); // RecOffset (Type 3000) - pretty sure these offsets are what formed the 310 (6=offset in type 3000 subrecstruct)
        }
        break;
      case 5:
        Convert.rep(buf, (OFF_ADJUNCT+ADJUNCT_QUADWORD_OFF+8), Convert.DOUBLE, 11, from, to); //Quadwords
        break;
    }
  }

  @Override
  public boolean equals (Object infile) {
    // Since 3.5.1 ,changed default to check /D/H/S/K for BUG 2840
    return this.equals(infile, EQ_DATA + EQ_HDR + EQ_SIZE + EQ_KEY);
  }

  /** Determine if two DataFiles are equal (with 0 tolerance). Used by FEQ logical test.
  @param infile The other datafile to compare to
  @param flags which parts of the DataFiles to compare, i.e. Size, header,
               data, keywords.
  @return <code>TRUE</code> if the two files are equal (with 0 tolerance)
   */
  @Override
  public boolean equals (Object infile, int flags) {
    boolean status = false;
    if (infile instanceof DataFile) {
      status = this.equals((DataFile) infile, flags);
    }
    return status;
  }

  /** Determine if two DataFiles are equal (within tolerance)
      @param df2 The other datafile to compare to
      @param flags which parts of the DataFiles to compare, i.e. Size, header,
                   data, keywords.
      @return <code>TRUE</code> if the two files are equal (within tolerance)
   */
  public boolean equals (DataFile df2, int flags) {
    // If using tolerance through macro, look for the /TOL value
    double tolerance;
    if ((flags & EQ_TOL) == EQ_TOL) {
      setFeqTolerance(); // parse /TOL= switch (if present)
      tolerance = getFeqTolerance();
    }
    else if ((flags & EQ_ABSTOL) == EQ_ABSTOL) {
      setFeqAbsTolerance(); // parse /ABSTOL= switch (if present)
      tolerance = getFeqAbsTolerance();
    }
    else {
      tolerance = 0; // if not using tolerance, set it to 0
    }
    return equals(df2, flags, tolerance);
  }

  /** Determine if two DataFiles are equal (within tolerance)
      @param df2 The other datafile to compare to
      @param flags which parts of the DataFiles to compare, i.e. Size, header,
                   data, keywords.
      @param tolerance The comparison tolerance. Use '-1' to ignore
      @return <code>TRUE</code> if the two files are equal (within tolerance)
   */
  public boolean equals (DataFile df2, int flags, double tolerance) {
    if (flags == EQ_DEF) flags=EQ_DATA + EQ_HDR;
    boolean absoluteTol = ((flags&BaseFile.EQ_ABSTOL)!=0);
    boolean theSame;

    // Binary comparison?  If so, ignore all other flags
    if ( (flags & EQ_BIN) != 0 ) {
      // Use the BaseFile class for binary comparison.
      BaseFile bf1 = (BaseFile) this;
      BaseFile bf2 = (BaseFile) df2;
      theSame = bf1.equalsBF(bf2, flags);
    }
    else {
      DataFile df1 = this;
      df1.open(); df2.open();

      // Compare sizes?
      boolean sameSize = true;
      if ( (flags & EQ_SIZE) != 0 ) {
        sameSize = (df1.getSize() == df2.getSize());
      }
      theSame = sameSize;

      // Compare headers?
      boolean sameHdr = true;
      if (theSame && ((flags & EQ_HDR) != 0 )) {
        sameHdr = df1.equalsHeader(df2);
      }
      theSame = theSame && sameHdr;


      // Compare data?
      boolean sameData = true;
      if (theSame && ((flags & EQ_DATA) != 0) || ((flags & EQ_TOL) != 0) || ((flags & EQ_LENLIM) != 0)) {
        // Since NeXtMidas 3.7.0 for BUG 2861
        // *NOTE* --- change in DEFAULT behavior!
        // 1. /L modifier allows data comparison to be limited to the size of the smaller file (if applicable)
        //    otherwise, the two files MUST have the same length!
        // 2. If one file has zero length, the files are not equal
        if(df1.getSize() != 0 && df2.getSize() != 0) { // both files have a length
          double _offset = 0.0;  // TBD to use trimmers
          if((flags & EQ_LENLIM) != 0) {
            double _size = Math.min(df1.getSize(),df2.getSize());
            sameData = compare(df1,df2,_offset,_size, tolerance, absoluteTol);
          }
          else if(df1.getSize() == df2.getSize()) {
            sameData = compare(df1,df2,_offset,df1.getSize(), tolerance,absoluteTol);
          }
          else {
            sameData = false;
          }
        }
        else if(df1.getSize() != 0 || df2.getSize() != 0) { // one file has 0 length
          sameData = false;
        }
      }
      theSame = theSame && sameData;

      // Compare keywords?
      boolean sameKey = true;
      if (theSame && ((flags & EQ_KEY) != 0)) {
        sameKey = df1.equalsKeywordsExt(df2);
      }
      if ((M.debug&Debug.IO)!=0) {
        M.type("DataFile.equals: "+"size? "+sameSize+", hdr? "+sameHdr
                                +", data? "+sameData+", key? "+sameKey);
      }
      theSame = theSame && sameKey;
      df1.close(); df2.close();
    }
    return theSame;
  }

  /** Compare two DataFile objects
      @param df1    First DataFile
      @param df2    Second DataFile
      @param offset Offset into files to start at, elements 0..size-1
      @param size   Number of elements to compare
      @return true if they match, false otherwise.
   */
  public static boolean compare (DataFile df1, DataFile df2, double offset, double size) {
    return compare(df1, df2, offset, size, 0);
  }

  /** Compare two DataFile objects
      @param df1       First DataFile
      @param df2       Second DataFile
      @param offset    Offset into files to start at, elements 0..size-1
      @param size      Number of elements to compare
      @param tolerance The comparison tolerance
      @return <code>true</code> if they match, false otherwise.
  */
  public static boolean compare (DataFile df1, DataFile df2, double offset, double size, double tolerance) {
    return compare(df1, df2, offset, size, tolerance, false);
  }

  /** Compare two DataFile objects
      @param df1       First DataFile
      @param df2       Second DataFile
      @param offset    Offset into files to start at, elements 0..size-1
      @param size      Number of elements to compare
      @param tolerance The comparison tolerance
      @param absoluteTol true if this tolerance is an absolute tolerance
      @return <code>true</code> if they match, false otherwise.
   */
  public static boolean compare (DataFile df1, DataFile df2, double offset, double size,
      double tolerance, boolean absoluteTol) {
    Data data1,data2;
    int elemsPer = (int)Math.max(1.0,Math.min(131072.0,size*df1.dbpe)/df1.dbpe);
    boolean type3000orGreater = (df1.getType()/1000 > 2) || (df2.getType()/1000 > 2);
    if (type3000orGreater) {
      data1 = df1.getDataBuffer(elemsPer);
      data2 = df2.getDataBuffer(elemsPer);
    } else {
      byte type1 = df1.getFormatType();
      byte type2 = df2.getFormatType();
      byte type = (type1 == type2) ? type1 : Data.promoteType (type1, type2);
      data1 = df1.getDataBuffer(elemsPer, type);
      data2 = df2.getDataBuffer(elemsPer, type);
    }
    if (tolerance > 0) {
      int elements1 = (int)df1.getSize();             // # of elements in the files
      int elements2 = (int)df2.getSize();
      if (elements1 != elements2) return false;       // if element # is diff, files not equal
      for (int zz=0; zz<elements1;) {                 // Go through the elements of the file
        df1.read(data1,offset+zz,elemsPer);           // Load given number of elements into the buffers
        df2.read(data2,offset+zz,elemsPer);
        int subElems1 = data1.getSize();              // Get the number of elems in the buffer

        for (int xx=0; xx< subElems1; xx++) {         // Go through each element
          Data elemData1 = data1.getIndex(xx);        // Select the element
          Data elemData2 = data2.getIndex(xx);
          int ape = elemData1.getAPE();               // Get the number of Atoms in that element
          for (int yy=0; yy< ape; yy++) {             // Go through each Atom
            Data value1 = elemData1.getAtomAt(yy);    // Get the value of the atom
            Data value2 = elemData2.getAtomAt(yy);
            double d1 = value1.toD();                 // Convert to double
            double d2 = value2.toD();
            if (!absoluteTol) {
              if (!Tolerance.equalsWithTolerance(d1, d2, tolerance)) {   // compare doubles with tolerance
                return false;
              }
            }
            else {
              if (!Tolerance.equalsWithAbsTol(d1, d2, tolerance)) {   // compare doubles with absolute tolerance
                return false;
              }
            }
          }
        }
      zz += elemsPer;                                 // increment elements
      }
    } else {
      for (double off=offset; off<size+offset;) {
        int elements = (int)Math.min((double)elemsPer,size-off);
        if (elements == 0) return false;
        int status1 = df1.read(data1,off,elements);
        int status2 = df2.read(data2,off,elements);
        if (!java.util.Arrays.equals(data1.buf,data2.buf)) return false;
        off += elements;
      }
    }
    return true;
  }

  /** Determines if two file headers are "equal" by comparing the "appropriate" fields in the
      header dependent on the file type (1000, 2000,...). Note that size is not one of the
      fields compared.
      <pre>
        Tested Fields are:
          For All files: type, format and time code

          Type 1000: xstart, xdelta, xunits
          Type 2000: xstart, xdelta, xunits, subsize, ystart, ydelta, yunits
          Type 3000: xstart, xdelta, xunits, subsize
          Type 5000: xstart, xdelta, xunits, subsize, quadwords
          Type 3000: xstart, xdelta, xunits, subsize
      </pre>
      @param df2 The file to compare to this one.
      @return true if they match, false otherwise.
   */
  public boolean equalsHeader (DataFile df2) {
    boolean theSame = false;
    DataFile df1 = this;
    // Compare the pertinent fields for ALL data files...
    if (df1.getType()       ==  df2.getType()       &&
        df1.getFormat().equals(df2.getFormat())     &&
        df1.getTimeCode()   ==  df2.getTimeCode() ) {
      int fileType = df1.getType()/1000;
      int nbytes = 0;
      boolean doSubRecs = (fileType>2);
      int nSubrecs = df1.getSubSize();

      byte df1rep = (byte)df1.getHeadRep().charAt(0);
      byte df2rep = (byte)df2.getHeadRep().charAt(0);
      df1.convertHeaderRep (df1rep,Shell.rep); // restore byte-order of file1
      df2.convertHeaderRep (df2rep,Shell.rep); // restore byte-order of file2
      // Compare only the fields in the adjunct header that are important
      // for the given file type.
      if (fileType == 1) {
        nbytes = 20; // Xstart, Xdelta, Xunits
      } else if (fileType == 2) {
        nbytes = 44; // Xstart, Xdelta, Xunits, subsize, Ystart, Ydelta, Yunits
      } else {
        nbytes = 24; // Xstart, Xdelta, Xunits, subsize
        doSubRecs = true; // Compare the sub records for 3000 and higher
      }
      byte[] buf1 = new byte[nbytes];
      byte[] buf2 = new byte[nbytes];
      System.arraycopy(df1.hb, OFF_ADJUNCT, buf1, 0, nbytes);
      System.arraycopy(df2.hb, OFF_ADJUNCT, buf2, 0, nbytes);
      df1.convertHeaderRep (Shell.rep,df1rep);
      df2.convertHeaderRep (Shell.rep,df2rep);
      if (java.util.Arrays.equals(buf1,buf2)) {
        if (doSubRecs) {
          // Compare the subrecords.  Can not get all the subrecords
          // at once...must get the offset for each individually.
          boolean subRecsSame = true;
          for (int i = 0; i < nSubrecs; i++) {
             if ( df1.getRecOffset(i) != df2.getRecOffset(i)     ||
                  !df1.getRecName(i).equals(df2.getRecName(i))   ||
                  !df1.getRecFormat(i).equals(df2.getRecFormat(i)) ) {
               subRecsSame = false;
               break;
             }
          }
          theSame = subRecsSame;
        } else {
          theSame = true;
        }
      }

      // Compare quadwords (added in NeXtMidas 3.3.0)
      if ((getTypeCodeClass() != 5) || !theSame) {
        // Not a Type 5000 file and/or not equal, skip quadwords checks
      }
      else if ((M != null) && (M.io != null) && M.io.isOptionSet(IOOptions.DisableQuadwordsEqualityChecks)) {
        // Check of quadwords disabled (matches NeXtMidas 3.2 and earlier)
      }
      else if ((getType() == 5001) /*|| (getType() == 5010)*/) {
        // For a Type 5001 file, only compare the fields pertinent to the given
        // reference frame as defined by the BLUE ICD. (Note that Type 5010 is
        // also handled here as -- although not indicated in the BLUE 1.0 ICD,
        // there is sufficient evidence that it should be handled this way, and
        // is likely to be included in the BLUE 1.1. ICD).
        if (!df1.getReferenceFrame().equals(df2.getReferenceFrame())) {
          theSame = false;
        }
        else if (df1.getReferenceFrame().equals("ECI")) {
          theSame = (df1.getQuadword(Q_YEAR) == df2.getQuadword(Q_YEAR))
                 && (df1.getQuadword(Q_SEC ) == df2.getQuadword(Q_SEC ))
                 && (df1.getQuadword(Q_GHA ) == df2.getQuadword(Q_GHA ));
        }
        else if (df1.getReferenceFrame().equals("ECR")) {
          // only QW(0)=ECR is applicable
        }
        else if (df1.getReferenceFrame().equals("TOP") ||
                 df1.getReferenceFrame().equals("TOPOCENT")) {
          theSame = (df1.getQuadword(Q_ALT ) == df2.getQuadword(Q_ALT ))
                 && (df1.getQuadword(Q_LAT ) == df2.getQuadword(Q_LAT ))
                 && (df1.getQuadword(Q_LON ) == df2.getQuadword(Q_LON ))
                 && (df1.getQuadword(Q_AZIM) == df2.getQuadword(Q_AZIM))
                 && (df1.getQuadword(Q_ELEV) == df2.getQuadword(Q_ELEV))
                 && (df1.getQuadword(Q_ROLL) == df2.getQuadword(Q_ROLL));
        }
        else {
          // Custom reference frame, check all of them
          for (int i = 416; (i < 512) && theSame; i++) {
            theSame = (df1.unpackB(i) == df2.unpackB(i));
          }
        }
      }
      else {
        // For all other Type 5000 files, check all fields
        for (int i = 416; (i < 512) && theSame; i++) {
          theSame = (df1.unpackB(i) == df2.unpackB(i));
        }
      }
    }

    return theSame;
  }

  /** Compare two DataFile keywords in EXTENDED HEADER ONLY!
      @param df2 The file to compare to this one.
      @return true if they match, false otherwise.
   */
  public boolean equalsKeywordsExt (DataFile df2) {
    DataFile df1 = this;
    final boolean debug = (M != null) && (M.debug&Debug.IO) != 0;
    if (debug) {
      Keywords.Iterator kwi;
      nxm.sys.lib.Keywords.Key key;
      M.type("DataFile.equalsKeywordsExt(): size keys in this file '" + df1.getFileName().getFullName() + "' = " + df1.keywords.size);
      for (kwi=df1.keywords.iterator(); (key=kwi.next())!=null;) {
        M.type( key.name+"="+key.value);
      }
      M.type("DataFile.equalsKeywordsExt(): size keys in other file '" + df2.getFileName().getFullName() + "' = " + df2.keywords.size);
      for (kwi=df2.keywords.iterator(); (key=kwi.next())!=null;) {
        M.type( key.name+"="+key.value);
      }
    }

    boolean status = false;
    if (df1.keywords.size == df2.keywords.size) {
      status = true;
      df1.keywords.setScope ("EXT");
      df2.keywords.setScope ("EXT");
      Keywords.Iterator kwi1 = df1.keywords.iterator();
      Keywords.Iterator kwi2 = df2.keywords.iterator();
      nxm.sys.lib.Keywords.Key key1,key2;
      for (; (key1=kwi1.next())!=null && (key2=kwi2.next())!=null; ) {
        if (debug) {
          M.type("DataFile.equalsKeywordsExt(): key1 name '"
                 +key1.name+"', value '"+key1.value
                 +"', key2 name '"+key2.name+"' value '"+key2.value+"'");
        }
        if (!key1.name.equals(key2.name) || !key1.value.equals(key2.value)) {
          if (debug) {
            M.type("DataFile.equalsKeywordsExt(): name '"
                   +key1.name+"' != '"+key2.name
                   +"', or value \n'"+key1.value+"' != \n'"+key2.value+"'");
          }
          status = false;
          break;
        }
      }
    }
    return status;
  }

  /** Flags a file as contiguous. Since contiguous files are not supported on Linux/Windows, this
      method does nothing. This method allows use of the <code>{CTG=...}</code> file qualifier that was
      available in X-Midas (in X-Midas, contiguous files were supported under VMS only).
      @param value DOES NOTHING
   */
  public void setCTG (int value) { }

  /** Shorthand for {@link #setQuadword(int, double)} used by HEADERMOD.
      @param index The index (quadword number plus 1)
      @param value The quadword value.
   */
  public void setQW (int index, double value) { setQuadword(index-1,value); }
  /** Shorthand for {@link #getQuadword(int)} used by HEADERMOD.
   * @param index The index (quadword number plus 1)
   * @return The quadword value
   */
  public double getQW (int index) { return getQuadword(index-1); }
  /** Shorthand for {@link #getFrameSize()} used by HEADERMOD and qualifiers.
   * @return frame size
   */
  public int getFS () { return getFrameSize(); }
  /** Shorthand for {@link #setFrameSize(int)} used by HEADERMOD and qualifiers.
      @param fs The frame size.
   */
  public void setFS (int fs) { setFrameSize(fs); }
  /** Shorthand for {@link #setComp(int,String)} used by HEADERMOD.
      @param index The index (component number plus 1)
      @param fields The component definition in the form <code>"(NAME,FORMAT,COMPTYPE,UNITS)"</code>.
   */
  public void setC (int index, String fields) { setComp(index-1,fields); }
  /** Shorthand for {@link #setSubRec(int,String)} used by HEADERMOD.
      @param index The index (subrecord number plus 1)
      @param fields The subrecord definition in the form <code>"(NAME,FORMAT,OFFSET)"</code>.
   */
  public void setSR (int index, String fields) { setSubRec(index-1,fields); }

  @Override
  public double getNumberOfRows () { return getSize(); }

  @Override
  public int getRecordDefCount () {
    return (typeClass==1)? 1: getSubSize();
  }

  @Override
  public void setRecordDefs (Table defs) {
    String[] keys = defs.getKeys();

    if (typeClass <= 2) setType(3000);

    setSubSize(keys.length);

    short offset = 0;
    for (int i = 0; i < keys.length; i++) {
      String name = keys[i];
      Table def = defs.getTable(name);

      String form  = def.getS("FORMAT","XA");
      short  off   = def.getI("OFFSET",offset); // byte offset
      String comp  = def.getS("TYPE",null);
      String units = def.getS("UNITS",null);
              name = def.getS("NAME", name); // since 3.5.4 as part of BUG 2903 (this change makes it reflective of getRecordDefs())

      setRecName(i, name);
      setRecFormat(i,form);
      setRecOffset(i,off);
      if (comp !=null) setRecType(i,(byte)getCompTypeID(comp)); // for Type 5000 file
      if (units!=null) setRecUnits(i,(byte)getUnitsID(units));

      int numElementsPerSubRecord = 1;
      if (t6s != null) { // since 3.6.0 to better support Type 6000 files (Bug 2960)
        double minval   = def.getD("MINVAL", Double.NaN);
        double maxval   = def.getD("MAXVAL", Double.NaN);
        int   numElts   = def.getL("NUM_ELTS", -1); // this field should contain unsigned integer
        String uprefix  = def.getS("UPREFIX", null);
        String reserved = def.getS("RESERVED", null);

        if (!Double.isNaN(minval)) setRecMinimum (i, minval);
        if (!Double.isNaN(maxval)) setRecMaximum (i, maxval);
        if (uprefix != null)       setRecUnitsPrefix(i, uprefix);
        if (reserved != null)      setRecReserved(i, reserved);
        if (numElts > 0) {         setRecElements(i, numElts);
          numElementsPerSubRecord = numElts;
        }
      }

      offset = (short)(off + Data.getBPA(form)*numElementsPerSubRecord);
    } // end for loop
    setRecLength(offset);
    setInternals();
  }

  @Override
  public Table getRecordDefs () {
    int subsize = getRecordDefCount();
    Table table = new Table();
    for (int i = 0; i < subsize; i++) {
      table.put(getRecName(i), getRecordDef(i));
    }
    return table;
  }

  @Override
  public Table getRecordDef (int i) {
    Table table = null;
    if ((i >= 0) && (i < getRecordDefCount())) {
      table = new Table();
      table.put("NAME",   getRecName(i));
      table.put("FORMAT", getRecFormat(i));
      table.put("OFFSET", getRecOffset(i));
      table.put("TYPE",   getRecCompTypeName(i)); // TODO: for 3.7+ beta series? only do this for Type 5000 (not relevant to Type 3000/6000)
      table.put("UNITS",  getRecUnitsName(i));

      if (t6s != null) { // since 3.6.0 include Type 6000 fields (Bug 2960)
        table.put("MINVAL",   getRecMinimum(i));
        table.put("MAXVAL",   getRecMaximum(i));
        table.put("NUM_ELTS", getRecElements(i));
        table.put("UPREFIX",  getRecUnitsPrefix(i));
        table.put("RESERVED", getRecReserved(i));
      }
    }
    return table;
  }

  /** Indicates if the file is typically accessed in a record-based manner. This is true for
      Type 3000, Type 5000 and Type 6000 files.
      @return true if the file is typically accessed in a record-based manner
      @since NeXtMidas 2.5.0
   */
  public boolean isRecordBased () {
    return ((typeClass == 3) || (typeClass == 5) || (typeClass == 6));
  }

  /** Gets the pointer to next DataFile (real-time header changes).
      @return The next (newer) data file in the chain or null if not applicable.
      @since NeXtMidas 2.5.0
   */
  public DataFile getNextDataFile () {
    return next;
  }

  /** Gets the pointer to previous DataFile (real-time header changes).
      @return The next (older) data file in the chain or null if not applicable.
      @since NeXtMidas 2.5.0
   */
  public DataFile getPreviousDataFile () {
    return previous;
  }

  /** Sets the pointer to next DataFile (real-time header changes).
      @since NeXtMidas 2.5.0
   */
  void setNextDataFile (DataFile df) { // Package-private, only used in DataFile and Pipe
    next = df;
  }

  /** Sets the pointer to previous DataFile (real-time header changes).
      @since NeXtMidas 2.5.0
   */
  void setPreviousDataFile (DataFile df) { // Package-private, only used in DataFile and Pipe
    previous = df;
  }

  DataFile copy () {  // package-private only used by Pipe - may be removed!
    try {
      return (DataFile)clone();
    } catch (CloneNotSupportedException e) {
      throw new MidasException("DataFile: Can not clone.",e);
    }
  }

  /** Should Platinum time line keywords be used (vs NeXtMidas microsecond precision TIMELINE keyword)?
      Precedence is:<pre>
        1. True if reading/writing 2.0 Platinum BLUE file (for partial 2.0 support)
        2. Allow a timeline file qualifier to override #3 - #5.
           This does not override #1 which is required by Platinum 2.0 ICD.
           a. "USEPLATINUMTIMELINE" or
           b. "USEBLUETCTIMELINE" if true
        3. Input/Ouput from existing file with TimeLine from Platinum timeline keywords
TODO--> 4. Has Platinum BLUE 2.0 time line keywords (propagated from input to output file)
        5. PreferPlatinumTimeLine in ENV.IOPTIONS is set
      </pre>
      @return true to use Platinum time line keywords
      @since NeXtMidas 3.3.0
   */
  private boolean usePlatinumTimeline () {
    Object overrideQualifier = getFileName().getQualifier("USEPLATINUMTIMELINE", null);
    if ("2.0".equals(getBlueVer())) {
      return true;                             // #1 Platinum BLUE 2.0 file
    }
    else if (overrideQualifier != null) {
      return Convert.o2z(overrideQualifier);   // #2 fn qualifier override
    }
    else {
      // Make sure qualifier for BLUE TC Timeline is not true - since 3.7.2 for NXM-3010
      Boolean tcTimeLineQualifier = getBooleanQualifierValue(QUALIFIER_TC_TIMELINE);
      Boolean usecTimeLineQualifier = getBooleanQualifierValue(QUALIFIER_USEC_TIMELINE);
      if ((tcTimeLineQualifier != null && tcTimeLineQualifier) || (usecTimeLineQualifier != null && usecTimeLineQualifier)) {
        return false;
      }
      boolean fromPlatinumTimeline = (timeLine != null && timeLine.format == TimeLine.Format.PLATINUM_HIGH_PRECISION);
      return fromPlatinumTimeline                                   // #3 from Platinum timeline keywords
             || M.io.isOptionSet(IOOptions.PreferPlatinumTimeLine); // #5 global IO options
    }
  }

  /** Handle (read, check and process) high precision timeline keywords based on the
      Platinum BLUE Data Exchange Format Standard rev 4.
      <b>NOTE: MUST be called after {@link #setInternals()}</b>
      @param origBlueVer original value of VER= keyword in main header
      @param origBlueIO  original value of IO= keyword in main header
      @since NeXtMidas 3.3.0 for BUG 2675
   */
  private void handlePlatinumTimelineKeywords (String origBlueVer, String origBlueIO) {
    TimeLine platTimeLine = TimeLine.fromPlatinumKeywords(keywords, getDelta(), dbpe);
    if (platTimeLine != null) { // extraction of Platinum timeline keywords succeeded
      boolean isPlatinum = "2.0".equals(origBlueVer);
      boolean usePlatinumTimeline = isPlatinum || isCreator(">=NXM3.3.0") || usePlatinumTimeline(); // overlapping condition ver 2.0
      if (this.timeLine == null) { // no conflicting NeXtMidas TIMELINE in file
        // Note: Not even setting PreferPlatinumTimeLine in ENV.IOPTIONS or
        // a file qualifier to USEPLATINUMTIMELINE=true will have the Platinum
        // timeline get used if the microsecond TIMELINE keywords exist
        if (usePlatinumTimeline) {
          this.timeLine = platTimeLine; // use Platinum high precision timeline Keywords
        }
        else {
          // Note: As or 3.7.1, it seems this else will only be reached if:
          // 1) the file has the qualifier USEPLATINUMTIMELINE=false, and
          // 2) the file version is NOT 2.0
          // 3) the creator is NXM 3.2.x or earlier
          // Otherwise, either platTimeline will be null and this code will not
          // be reached, or fromPlatinumKeywords will have set timeline.format
          // to Format.PLATINUM_HIGH_PRECISION and therefore usePlatinumTimeline()
          // and usePlatinumTimeline will be true.
          // Currently PreferPlatinumTimeLine in ENV.IOPTIONS has no effect.
          M.warning("Ignoring Platinum timeline keywords since it has been modified by IO="
              + getBlueIO() + (("UNKNOWN".equals(getCreator())) ? "" : " CREATOR=" + getCreator())
              + " in file " + getName());
        }
      }
      else {
        if (keywords.find("TIMELINE") != null) { // only warn in this case (since 3.9.1)
          // have existing NeXtMidas microsecond precision TIMELINE keyword
          M.warning("File contains NeXtMidas TIMELINE with microsecond precision and Platinum high "
            + "precision timeline keywords, using TIMELINE = "+ timeLine + " for file="+getName());
        }
        else {
          // we may have timeline from previous, so we are not initializing again
        }
        //QUESTION: TODO: Why are we not forcing use of platinum if specified
      }

      // Time midasStartTime = getTime(0.0); // <-- this can use TIMELINE keyword so CANNOT be use here
      final double start = getStart();
      Time blueStartTime = getTime().addSec(start);           // BLUE start time = TIMECODE + TC_PREC + (XSTART or YSTART)
      Time platStartTime = platTimeLine.getTimeAt(0.0, null); // Platinum start time (from TIMETAG.TIME etc. keywords)
      double tdiff = blueStartTime.diff(platStartTime);
      if (Math.abs(tdiff) > 1e-12) {     // if time diff is greater than 1 picosecond (else it is OK)
        Time tc_tcprec_Time = getTime(); // JUST: timecode + TC_PREC from main header
        if (isPlatinum && (tc_tcprec_Time.isZero() || tdiff < 1e-6)) {
          // Platinum file w/o timecode & TC_PREC is OK since it follows Platinum standard and
          // user's cannot be confused between which start time to use
          if (start != 0) { // XStart/YStart is NOT zero (need to adjust time)
            platStartTime.addSec(-start); // subtract XStart/YStart from Platinum start time for (timecode + TC_PREC)
          }
          M.warning("Internally fixing BLUE TimeCode+TC_PREC to "+platStartTime.toString(12)+" for "+getName());
          setTime(platStartTime); // fix timecode & TC_PREC internally in main header
        }
        else {
          M.warning("Midas TimeCode+TC_PREC+Start =" + blueStartTime.toString(12)+ " does not match"
            + "\n      Platinum TIMETAG.TIME keyword=" + platStartTime.toString(12)
            + "\n      diff from Midas=" + tdiff + " ; VER="+origBlueVer+" IO="+origBlueIO+" for "+getName());
        }
      }
    }
  }

  /** Should TC time line keywords be used (vs NeXtMidas microsecond precision TIMELINE keyword)?
      Precedence is:<pre>
        1. Allow a timeline file qualifier to override #2 - #3.
           a. "USEBLUETCTIMELINE" or
           b. "USEPLATINUMTIMELINE" if true
        2. File has timeline format set to TimeLine.Format.BLUE_HIGH_PRECISION
           due to creation from file with TimeLine from BLUE TC timeline keywords
        3. PreferBlueTCTimeLine in ENV.IOOPTIONS is set
      </pre>
      @return true to use TC time line keywords
      @since NeXtMidas 3.7.2 for NXM-3010
  */
  private boolean useBlueTCTimeline () {
    Boolean tcTimeLineQualifier = getBooleanQualifierValue(QUALIFIER_TC_TIMELINE);
    Boolean usecTimeLineQualifier = getBooleanQualifierValue(QUALIFIER_USEC_TIMELINE);
    if ((usecTimeLineQualifier != null) && usecTimeLineQualifier) {
      return false;
    }
    else if (tcTimeLineQualifier != null) {
      return tcTimeLineQualifier;
    }
    else {
      // Make sure qualifier for PLATINUM Timeline is not true
      Boolean platTimeLineQualifier = getBooleanQualifierValue(QUALIFIER_PLATINUM_TIMELINE);
      if (platTimeLineQualifier != null && platTimeLineQualifier) {
        return false;
      }
      // #2 from BLUE TC timeline keywords
      boolean fromBlueTCTimeline = (timeLine != null && timeLine.format ==
          TimeLine.Format.BLUE_HIGH_PRECISION);
      // #3 global IO options
      boolean preferTCSet = M.io.isOptionSet(IOOptions.PreferBlueTCTimeLine);
      return fromBlueTCTimeline || preferTCSet;
    }
  }

  /** Handle (read, check and process) BLUE TC timeline keywords as outlined
      in the JICD 4.2 ICD
     <b>NOTE: MUST be called after {@link #setInternals()}</b>
      @param origBlueVer original value of VER= keyword in main header
      @param origBlueIO  original value of IO= keyword in main header
      @since NeXtMidas 3.7.2
  */
  private void handleBlueTCTimelineKeywords (String origBlueVer, String origBlueIO) {
    TimeLine blueTCTimeLine = TimeLine.fromBlueTCKeywords(keywords, getTime().addSec(getStart()), getDelta(), dbpe); // NXM-3972
    if (blueTCTimeLine != null) { // extraction of BLUE TC timeline keywords succeeded
      boolean isBlue = BLUE_VER_PATTERN.matcher(origBlueVer).matches();
      boolean useBlueTCTimeline = useBlueTCTimeline();  // ONLY used as check if trying to force
      // should probably overwrite
      if (this.timeLine == null || useBlueTCTimeline) { // no other timeline or prefer Blue TC timeline
        this.timeLine = blueTCTimeLine;
      }
      else { // have existing NeXtMidas microsecond precision TIMELINE keyword
        M.warning("File contains NeXtMidas TIMELINE with microsecond precision and BLUE TC high "
          + "precision timeline keywords, using TIMELINE = "+ timeLine + " for file="+getName()); // TODO update warning
      }
    }
  }

  /** Determines the new format of the current TimeLine based on the qualifier in use.
   *  @param original TimeLine format
   *  @return new TimeLine format
   *  @since NeXtMidas 3.9.4
   */
  private TimeLine.Format overrideTimeLineFormat (TimeLine.Format original) {
    TimeLine.Format format = original;
    if (usePlatinumTimeline()) format = TimeLine.Format.PLATINUM_HIGH_PRECISION;
    if (useBlueTCTimeline()) format = TimeLine.Format.BLUE_HIGH_PRECISION;
    Boolean usecOverride = getBooleanQualifierValue(QUALIFIER_USEC_TIMELINE);
    if (usecOverride!= null && usecOverride) {
      format = TimeLine.Format.MICROSECOND_PRECISION;
    }
    return format;
  }

  /** Checks if a given TimeLine qualifier is in use.
   *  @param TimeLine qualifier
   *  @return true if the specified TimeLine qualifier is in use
   *  @since NeXtMidas 3.9.4
   */
  private Boolean getBooleanQualifierValue (String qualifier) {
    Object overrideQualifier = getFileName().getQualifier(qualifier, null);
    if (overrideQualifier != null) {
      return Convert.o2z(overrideQualifier);
    }
    else return null;
  }

  /** Time of the first element
   *  @return Time of the first element
   *  @since 3.9.1
   */
  public Time getTimeAtStart () {
    return getTimeAt(0, null);
  }

  /** Time of the last element
   *  @return Time of the last element
   *  @since 3.9.1
   */
  public Time getTimeAtEnd () {
    return getTimeAt(getSize(), null);
  }

  /** Get the duration
   *  @return the duration
   *  @since 3.9.1
   */
  public double getTimeDuration () {
    return getTimeAtEnd().diff(getTimeAtStart());
  }

  /** Indicates if a BLUE file *appears* to be complete. This can come up in cases where an external
   *  process is writing/transferring a BLUE file that will be used by this process. This method is
   *  NOT fool-proof and is merely looking for common indicators that the file is incomplete (such
   *  as header length and sizes that don't match up).
   *  @param p The file path.
   *  @return true if {@code getPartialBlueFileError(p)} would return null (i.e. no issues), and
   *          false otherwise.
   *  @since NeXtMidas 3.9.0
   */
  public static boolean isCompleteBlueFile (Path p) {
    return isCompleteBlueFile(p.toFile());
  }

  /** Indicates if a BLUE file *appears* to be complete. This can come up in cases where an external
   *  process is writing/transferring a BLUE file that will be used by this process. This method is
   *  NOT fool-proof and is merely looking for common indicators that the file is incomplete (such
   *  as header length and sizes that don't match up).
   *  @param f The file.
   *  @return true if {@code getPartialBlueFileError(f)} would return null (i.e. no issues), and
   *          false otherwise.
   *  @since NeXtMidas 3.9.0
   */
  public static boolean isCompleteBlueFile (File f) {
    return (getPartialBlueFileError(f) == null);
  }

  /** Filters out partial BLUE files. This can come up in cases where an external
   *  process is writing/transferring a BLUE file that will be used by this process.
   *  @param p The file path.
   *  @return null if ok, otherwise a short "error" message in a human-readable form. Specific
   *          "errors" that relate to an incomplete BLUE file being found are not enumerated and
   *          can change between releases. A general error in accessing the file that causes an
   *          condition while inspecting the file will result in a {@link #BLUE_FILE_READ_ERROR}
   *          error since it can be due to the BLUE file being incomplete or can result from any
   *          number of unrelated errors (e.g. disk read error).
   * @since NeXtMidas 3.9.0
   */
  public static String getPartialBlueFileError (Path p) {
    return getPartialBlueFileError(p.toFile());
  }

  /** Indicates if a BLUE file *appears* to be complete. This can come up in cases where an external
   *  process is writing/transferring a BLUE file that will be used by this process. This method is
   *  NOT fool-proof and is merely looking for common indicators that the file is incomplete (such
   *  as header length and sizes that don't match up).
   *  @param f The file.
   *  @return null if ok, otherwise a short "error" message in a human-readable form. Specific
   *          "errors" that relate to an incomplete BLUE file being found are not enumerated and
   *          can change between releases. A general error in accessing the file that causes an
   *          condition while inspecting the file will result in a {@link #BLUE_FILE_READ_ERROR}
   *          error since it can be due to the BLUE file being incomplete or can result from any
   *          number of unrelated errors (e.g. disk read error).
   *  @since NeXtMidas 3.9.0
   */
  public static String getPartialBlueFileError (File f) {
    long len = f.length();

    if (len < 256) {
      // Header will always be >256
      return "Incomplete MAIN header (way too short)";
    }

    try (nxm.sys.lib.DataFile df = new nxm.sys.lib.DataFile(null, f)) {
      df.open(nxm.sys.lib.DataFile.NOABORT|nxm.sys.lib.DataFile.HEADERONLY);
      if (!df.isOpen()) {
        return "Failed to open file";
      }

      long   extStart  = df.getExtStart();
      long   extSize   = df.getExtSize();
      long   dataStart = (long)df.getDataStart();
      long   dataSize  = (long)df.getDataSize();
      long   mainSize  = df.getHeaderLength();
      double size      = df.getSize();
      long   sizeBytes = (long)Math.ceil(size * df.getBPE());

      if (len < mainSize) {
        return "Incomplete MAIN header (too short)";
      }
      if ((size == 0) && (dataSize != 0)) {
        // Common case where main header is initially written with size=0 and size is not updated
        // until file gets closed.
        return "Incomplete MAIN header (incomplete data)";
      }
      if (len < dataStart+dataSize) {
        return "Incomplete DATA (by offset check)";
      }
      if (len < dataStart+sizeBytes) {
        return "Incomplete DATA (by element count)";
      }
      if ((extSize > 0) && (len < extStart+extSize)) {
        return "Incomplete EXTENDED header";
      }
    }
    catch (Exception e) {
      // TODO: Log the exception as a low-level message
      return BLUE_FILE_READ_ERROR;
    }
    return null;
  }

  /** Determines if the given file is a BLUE file. This will return true if either
   *  {@code isBlueFileName(fname)} *or* {@code isBlueFileMagic(ref,fname)} return true.
   *  @param ref   The Midas reference (can be null if a full path/URL is given).
   *  @param fname The file name.
   *  @return true if a BLUE file, false otherwise.
   *  @since NeXtMidas 3.9.0
   */
  public static boolean isBlueFile (MidasReference ref, Object fname) {
    return isBlueFileName(fname) || isBlueFileMagic(ref, fname);
  }

  /** Determines if the given file is a BLUE file by checking for the "Magic number" in the header
   *  at the start of the file. Specifically this looks at the first 12 octets of the file and looks
   *  for them to match one of the following values:
   *  <pre>
   *     Version   Head Rep  Data Rep
   *    +---------+---------+---------+
   *    | B L U E | I E E E | I E E E |
   *    +---------+---------+---------+
   *
   *    +---------+---------+---------+
   *    | B L U E | I E E E | E E E I |
   *    +---------+---------+---------+
   *
   *    +---------+---------+---------+
   *    | B L U E | E E E I | I E E E |
   *    +---------+---------+---------+
   *
   *    +---------+---------+---------+
   *    | B L U E | E E E I | E E E I |
   *    +---------+---------+---------+
   *  </pre>
   *  Note: These checks omit the antiquated (though strictly valid) combinations involving
   *  CRAY-/VAX-format data, but since these have little-to-no support in NeXtMidas (and current
   *  versions of X-Midas), they represent a novelty and not a real-world use-case.
   *  @param ref   The Midas reference (can be null if a full path/URL is given).
   *  @param fname The file name.
   *  @return true if a BLUE file, false otherwise.
   *  @since NeXtMidas 3.9.0
   */
  public static boolean isBlueFileMagic (MidasReference ref, Object fname) {
    try (BaseFile bf = new BaseFile(ref, fname)) {
      if (!bf.exists()) return false;
      // Although we typically check the return value on the read(..) it doesn't matter here
      // since anything other than 12 will give us a buffer that fails the checks immediately
      // below.
      bf.open();
      byte[] buf = new byte[12];
      bf.read(buf, 0, buf.length);
      return BLUE_MAGIC.contains(new String(buf));
    }
  }

  /** Determines if the given file name matches a BLUE file.
   *  @param fname The file name.
   *  @return true if a BLUE file, false otherwise.
   *  @since NeXtMidas 3.9.0
   */
  public static boolean isBlueFileName (Object fname) {
    FileName fn  = FileName.toFileName(fname);
    String   ext = fn.getExt();

    switch (ext) {
      // Normal BLUE files...
      case "tmp":   return true;
      case "prm":   return true;

      // Platinum files...
      case "bdif":  return true;
      case "cdif":  return true;
      case "cwdw":  return true;
      case "ncwdw": return true;
      case "npdw":  return true;
      case "pdw":   return true;
      case "ppdw":  return true;
      case "rpdw":  return true;
      case "sdw":   return true;
      case "spdw":  return true;
      case "wpdw":  return true;
      case "wcwdw": return true;

      // Other...
      default:      return false;
    }
  }

  /** Set the flags for the DataFile
      @param flags      The flags to be set
   */
  public void setFlags(int flags) {

    CoreIO io = Shell.getSharedMidasContext().io;
    if(io.isOptionSet(CoreIO.IOOptions.CreateFileMaintainCase)) flags |= MAINTAINCASE;
    if(io.isOptionSet(CoreIO.IOOptions.CreateFileWithoutTmpExtenstion)) flags |= DONOTADDEXTENSION;
    if (Midas.isJavaVsMidasDebug()) Midas.log("#JvsM DataFile.setFlags fileName:"
        +filename+" flags:"+getFlagsString(flags), Midas.INFO);
    super.setFlags(flags);
  }

  /** Returns the total number of timestamps in a TC timeline
      @return TimeLine.getNumTCEntries(keywords) number of timestamps
      @since NeXtMidas 3.9.4
   */
  public int getNumTimelineEntries () {
    return TimeLine.getNumTCEntries(keywords);
  }
  /** Returns whether or not timeline autocorrection is enabled
      @return offset0AutoCorrectIsEnabled whether or not autocorrection is enabled
      @since NeXtMidas 3.9.4
   */
  public boolean isAutoCorrectEnabled () {
    return offset0AutoCorrectIsEnabled;
  }

  /** Get the scalar values separator for listing type 1000 and default elements
   *  @return The scalar values separator for listing type 1000 and default elements
   *  @since NeXtMidas 4.1.2
   */
  @InternalUseOnly
  public static String getScalarSeparator () {
    return scalarSeparator;
  }

  /** Sets the scalar values separator for listing type 1000 and default elements
   *  @param sSep The new scalar values separator for listing type 1000 and default elements
   *  @since NeXtMidas 4.1.2
   */
  @InternalUseOnly
  public static void setScalarSeparator(String sSep) {
    scalarSeparator = sSep;
  }

}
