package nxm.sys.lib;

import static java.lang.Math.signum;
import static nxm.sys.libm.Tolerance.equalsWithAbsTolerance;
import static nxm.sys.libm.Tolerance.equalsWithTolerance;

import java.io.IOException;
import java.io.ObjectInputStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Type;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.Vector;

import nxm.sys.inc.AsciiMap;
import nxm.sys.inc.DataTypes;
import nxm.sys.inc.Indexable;
import nxm.sys.inc.InternalUseOnly;
import nxm.sys.inc.MidasReference;
import nxm.sys.inc.ProvisionalUseOnly;
import nxm.sys.libm.Tolerance;


/** A class to hold a data segment in a buffer. This is the basic data object for all I/O
    and Math functions.
    <pre>
      Definitions:
        DATA   - An array of 1 or more ATOMS, typically held via a {@link Data} object.

        ATOM   - An N-dimensional quantity defined using a format digraph (mode &amp; type).

        MODE   - The multiplicity of the ATOM ('S'=Scalar, 'C'=Complex, 'V'=Vector, ...).

        TYPE   - The scalar numeric type ('D'=DOUBLE, 'F'=FLOAT, 'I'=INT, ...).

                 It is important to note this class uses the Midas names for the types
                 (DOUBLE,FLOAT,XLONG,LONG,INT,BYTE) rather than the Java names for the
                 types (double,float,long,int,short,byte). This is done for historical
                 reasons and for compatibility with other Midas frameworks.

        FORMAT - The FORMAT (or FORM) is the digraph representing mode and type (e.g.
                 "SF" for "ScalarFloat" or "VD" for "VectorDouble").

        REP    - The data representation (byte ordering used). In most cases this will
                 be equal to the byte representation of the local system. In the case
                 of BLUE files {@link DataFile}, handles the applicable conversion as part of
                 the read/write.
    </pre>

    <h2>Using Data Buffers When Reading/Writing Files</h2>
    Data buffers are often used when reading and writing binary data files (e.g. BLUE
    files). In the case of a BLUE file, the data buffer used should usually be created
    via the {@link DataFile#getDataBuffer(int)} or {@link DataFile#getDataBuffer(int,byte)}
    method. <br>
    <br>
    For a more instructive explanation with examples, please see the
    "<a href="../../../../../training/PrimitivesFiles.html">Files in a Primitive</a>"
    section in the "Primitives" module of the <i>NeXtMidas Training</i>.

    <h2><a id="AccessingValues">Accessing Values</a></h2>
    The pack/unpack methods provide access to a scalar value in the data buffer based on byte
    offset into the buffer. The set/get methods provide access to a scalar value in the data
    buffer based on element offset into the buffer.<br>
    <br>
    <table border="1">
      <caption style="visibility:hidden">""</caption>
      <tr><th>Type                   </th><th colspan="2">Using Byte Offset                                                                          </th></tr>
      <!--**************************************************************************************************************************************************-->
      <tr><td>{@link #DOUBLE DOUBLE}      </td><td>{@link #packD packD(int boff, double val)}         </td><td>{@link #unpackD unpackD(int boff)}         </td></tr>
      <tr><td>{@link #FLOAT  FLOAT}       </td><td>{@link #packF packF(int boff, float val)}          </td><td>{@link #unpackF unpackF(int boff)}         </td></tr>
      <tr><td>{@link #XLONG  XLONG}       </td><td>{@link #packX packX(int boff, long val)}           </td><td>{@link #unpackX unpackX(int boff)}         </td></tr>
      <tr><td>{@link #LONG   LONG}        </td><td>{@link #packL packL(int boff, int val)}            </td><td>{@link #unpackL unpackL(int boff)}         </td></tr>
      <tr><td>{@link #INT    INT}         </td><td>{@link #packI packI(int boff, short val)}          </td><td>{@link #unpackI unpackI(int boff)}         </td></tr>
      <tr><td>{@link #BYTE   BYTE}        </td><td>{@link #packB packB(int boff, byte val)}           </td><td>{@link #unpackB unpackB(int boff)}         </td></tr>
      <tr><td>{@link #NIBBLE NIBBLE}      </td><td>n/a <i>(not on byte boundaries)</i>                </td><td>n/a <i>(not on byte boundaries)</i>        </td></tr>
      <tr><td>{@link #PACKED PACKED}      </td><td>n/a <i>(not on byte boundaries)</i>                </td><td>n/a <i>(not on byte boundaries)</i>        </td></tr>
      <tr><td>{@link #ASCII    ASCII}*    </td><td>n/a <i>(block of 8 characters)</i>                 </td><td>n/a <i>(block of 8 characters)</i>         </td></tr>
      <tr><td>{@link #ASCII256 ASCII256}* </td><td>n/a <i>(block of 256 characters)</i>               </td><td>n/a <i>(block of 256 characters)</i>         </td></tr>
      <tr><td>{@link CharSequence String}*</td><td>{@link #packS packS(int boff, int len, CharSequence val)}</td><td>{@link #unpackS unpackS(int boff, int len)}</td></tr>
      <tr><td>{@link Character char}*     </td><td>{@link #packC packC(int boff, char val)}           </td><td>{@link #unpackC unpackC(int boff)}         </td></tr>
      <tr><td>{@link Data    Data}        </td><td>{@link #packData packData(int boff, Data val)}     </td><td>TBD                                        </td></tr>
      <!--**************************************************************************************************************************************************-->
      <tr><th>Type                   </th><th colspan="2">Using Element Offset                                                                       </th></tr>
      <!--**************************************************************************************************************************************************-->
      <tr><td>{@link #DOUBLE   DOUBLE}   </td><td>{@link #setD setD(int off, double val)}            </td><td>{@link #getD getD(int off)}                </td></tr>
      <tr><td>{@link #FLOAT    FLOAT}    </td><td>{@link #setF setF(int off, float val)}             </td><td>{@link #getF getF(int off)}                </td></tr>
      <tr><td>{@link #XLONG    XLONG}    </td><td>{@link #setX setX(int off, long val)}              </td><td>{@link #getX getX(int off)}                </td></tr>
      <tr><td>{@link #LONG     LONG}     </td><td>{@link #setL setL(int off, int val)}               </td><td>{@link #getL getL(int off)}                </td></tr>
      <tr><td>{@link #INT      INT}      </td><td>{@link #setI setI(int off, short val)}             </td><td>{@link #getI getI(int off)}                </td></tr>
      <tr><td>{@link #BYTE     BYTE}     </td><td>{@link #setB setB(int off, byte val)}              </td><td>{@link #getB getB(int off)}                </td></tr>
      <tr><td>{@link #NIBBLE   NIBBLE}   </td><td>{@link #setN setN(int off, byte val)}              </td><td>{@link #getN getN(int off)}                </td></tr>
      <tr><td>{@link #PACKED   PACKED}   </td><td>{@link #setP setP(int off, byte val)}              </td><td>{@link #getP getP(int off)}                </td></tr>
      <tr><td>{@link #ASCII    ASCII}*   </td><td>{@link #setA setA(int off, int len, byte[] val)}   </td><td>{@link #getA getA(int off, int len)}       </td></tr>
      <tr><td>{@link #ASCII256 ASCII256}*</td><td>{@link #setZ setZ(int off, int len, byte[] val)}   </td><td>{@link #getZ getZ(int off, int len)}       </td></tr>
      <tr><td>{@link String    String}*  </td><td>n/a                                                </td><td>n/a                                        </td></tr>
      <tr><td>{@link Character char}*    </td><td>{@link #setC setC(int off, char val)}              </td><td>{@link #getC getC(int off)}                </td></tr>
      <tr><td>{@link Data      Data}     </td><td>n/a                                                </td><td>n/a                                        </td></tr>
      <!--**************************************************************************************************************************************************-->
      <tr><td colspan="3">* {@link #packS packS}/{@link #unpackS unpackS} are used for packing/unpacking
              Fortran strings with a fixed length (padded with spaces) to/from a Java {@link String}
              (UNICODE characters are not allowed). {@link #packC packC}/{@link #unpackC unpackC} are
              used for packing/unpacking a single ASCII character as a Java <code>char</code>. The
              {@link #setA setA}/{@link #getA getA} methods set/get a Fortran string that is already
              packed in a byte buffer.</td></tr>
    </table>

    <h2><a id="AccessingArrays">Accessing Arrays</a></h2>
    While the pack/unpack and get/set methods are good for accessing individual values in the data
    buffer, they are not very efficient when it comes to accessing large arrays of data (such as
    a frame of data). The cast/uncast methods allow the entire frame of data to be converted to/from
    a Java array in an efficient manner.
    <pre>
      Definitions:
        CAST   - Temporary conversion from Data buffer to Java array
        UNCAST - Restores the Data buffer and ends use of Java array
    </pre>
    <table border="1">
      <caption style="visibility:hidden">""</caption>
      <tr><th>Type                  </th><th>Cast                           </th><th>Uncast                        </th></tr>
      <tr><td>{@link #DOUBLE DOUBLE}</td><td><code>double[] array = </code>{@link #castD(boolean) castD(boolean copy)} </td>
                                         <td>{@link #uncast(double[],boolean) uncast(double[] array, boolean copy)}</td></tr>
      <tr><td>{@link #FLOAT  FLOAT} </td><td><code>float[]  array = </code>{@link #castF(boolean) castF(boolean copy)} </td>
                                         <td>{@link #uncast(float[],boolean)  uncast(float[]  array, boolean copy)}</td></tr>
      <tr><td>{@link #XLONG  XLONG} </td><td><code>long[]   array = </code>{@link #castX(boolean) castX(boolean copy)} </td>
                                         <td>{@link #uncast(long[],boolean)   uncast(long[]   array, boolean copy)}</td></tr>
      <tr><td>{@link #LONG   LONG}  </td><td><code>int[]    array = </code>{@link #castL(boolean) castL(boolean copy)} </td>
                                         <td>{@link #uncast(int[],boolean)    uncast(int[]    array, boolean copy)}</td></tr>
      <tr><td>{@link #INT    INT}   </td><td><code>short[]  array = </code>{@link #castI(boolean) castI(boolean copy)} </td>
                                         <td>{@link #uncast(short[],boolean)  uncast(short[]  array, boolean copy)}</td></tr>
      <tr><td>{@link #BYTE   BYTE}  </td><td><code>byte[]   array = </code>{@link #castB(boolean) castB(boolean copy)} </td>
                                         <td>{@link #uncast(byte[],boolean)   uncast(byte[]   array, boolean copy)}</td></tr>
      <tr><td>{@link #NONHOMOGENEOUS NONHOMOGENEOUS}  </td><td><code>{@link MidasException} (not possible)</code></td>
                                         <td>{@link MidasException} (not possible)</td></tr>
      <tr><td colspan="3">
            The boolean <code>copy</code> flag on the cast/uncast methods permits skipping a memory copy
            in cases where it is unnecessary. Setting the copy flag to <code>false</code> on a cast can
            greatly improve speed by skipping the copy of the {@link Data} buffer to the array.
            Setting the copy flag to <code>false</code> on an uncast greatly improves speed by
            discarding the values in the local array without copying them back. In cases where a
            memory mapping is available (see below), the <code>copy</code> flag is ignored.</td></tr>
    </table>
    <br>
    In order to maximize efficiency, the cast/uncast will try to perform a direct memory mapping
    of the buffer. The memory mapping must be done <u>via native JNI code</u> and only works when
    the buffer REP matches the native REP, the array type matches the underlying data and the bytes
    are properly aligned (some platforms only allow a direct mapping when the buffer is 8-byte
    aligned).<br>
    <br>
    In cases where a direct memory mapping is not possible, a copy is required. Memory copy always
    works and does not require native code; however, it is significantly slower than a memory
    mapping. Because of this, the conversion routines will perform a memory mapping whenever
    possible (the choice between memory mapping and copy is automatic).<br>
    <br>
    Because of the two possibilities (memory mapping vs copy) it is important to remember to do
    an uncast immediately after using the array (and before writing the buffer to disk). The uncast
    releases the mapped memory (if memory mapping is used) or copies back the data to the byte
    array (if a copy is used).<br>
    <br>
    Within a {@link Primitive}, the cast and corresponding uncast should always occur within the
    <code>process()</code> method. Once the uncast has been called, the array created by the cast should
    no longer be used. <span style="color:blue">Failure to properly do an uncast or use of the array after
    the uncast may cause exceptions (including {@link ArrayIndexOutOfBoundsException}). Always do
    an uncast immediately after use of the array.</span><br>
    <br>
    <b>Standard Usage in a Primitive <i>(separate input and output buffers)</i>:</b>
    <pre>
      public int process () {
        // Read from file into inputBuffer

        double[] inArray  = inputBuffer.castD(true);   // copy=true, copy values from byte buffer into array
        double[] outArray = outputBuffer.castD(false); // copy=false, don't care about initial values

        // do work on arrays

        inputBuffer.uncast(inArray, false);            // copy=false, values in array can be discarded
        outputBuffer.uncast(outArray, true);           // copy=true, copy values from array into byte buffer

        // Write outputBuffer to file
        return status;
      }
    </pre>
    <b>Standard Usage in a Primitive <i>(shared buffer for input and output)</i>:</b>
    <pre>
      public int process () {
        // Read from file into inOutBuffer

        double[] array = inOutBuffer.castD(true);   // copy=true, copy values from byte buffer into array

        // do work on array

        inOutBuffer.uncast(array, true);            // copy=true, copy values from array into byte buffer

        // Write inOutBuffer to file
        return status;
      }
    </pre>

    <!-- JC4: I only included the most common read/write methods in DataFile. -->
    @see DataTypes
    @see DataFile#getDataBuffer(int)
    @see DataFile#getDataBuffer(int,byte)
    @see DataFile#read(Data)
    @see DataFile#write(Data,int)
    @see nxm.sys.inc.PlotFile#read(Data,int)

    @author Jeff Schoen
*/
public class Data extends Object
  implements Indexable, DataTypes, AsciiMap, java.io.Serializable, Comparable<Object> {

  /** NOTE: this should be changed if the structure/fields of this class changes. */
  private static final long serialVersionUID = 5144324106233708397L;

  /** Equivalence test flag: Test binary equivalence.   */
  public static final int EQ_BIN     = BaseFile.EQ_BIN;
  /** Equivalence test flag: Test buffer size.          */
  public static final int EQ_SIZE    = BaseFile.EQ_SIZE;
  /** Equivalence test flag: Test (Relative) Tolerance. */
  public static final int EQ_TOL     = BaseFile.EQ_TOL;
  /** Equivalence test flag: Test Magnitude of Data.    */
  public static final int EQ_MAG     = BaseFile.EQ_MAG;
  /** Equivalence test flag: Case-sensitive strings.    */
  public static final int EQ_CS      = BaseFile.EQ_CS;
  /** Equivalence test flag: Vector equality (compare all values in Data). */
  public static final int EQ_VEQ     = BaseFile.EQ_VEQ;
  /** Equivalence test flag: Test Absolute Tolerance. public since NeXtMidas 3.7.0, added in 3.3.2. */
  public static final int EQ_ABSTOL  = BaseFile.EQ_ABSTOL;
  /** Equivalence test flag: Test with Length Limit. Since NeXtMidas 3.7.0 */
         static final int EQ_LENLIM  = BaseFile.EQ_LENLIM;
  /** Equivalence test flag: Use "default" tests (can not be combined with other mask values). */
  public static final int EQ_DEF     = BaseFile.EQ_DEF;
  /** Equivalence test flag: The "default" tests (mask value) = {@link #EQ_SIZE}|{@link #EQ_CS}|{@link #EQ_VEQ}. */
  public static final int EQ_DEFAULT = EQ_SIZE|EQ_CS|EQ_VEQ;

  /** The maximum number of bytes in {@link #buf} when computing the hash code for Data objects */
  private static final int MAX_BYTES_FOR_HASHCODE = 32;

  /** common format modes, used in {@link #setSPA(int, char)} and {@link #resetMode()} */
  private static final String MODES = "SCVQ5678MX";

  /** These are the format modes which will be considered valid by this class.  Note that 'K' and 'N'
      are not valid BLUE format modes, but for compatibility reasons they are accepted here when
      paired with types 'W' and 'H', respectively.
   */
  private static final String DATA_MODES = DataTypes.DATA_MODE_LIST+DataTypes.MULT_DATA_MODE_LIST+"KN";

  /** These are the format types which will be considered valid by this class.  Note that 'W' and 'H'
      are not valid BLUE format types, but for compatibility reasons they are accepted here when
      paired with modes 'K' and 'N', respectively.  Additionally, 'O' is being tolerated here to
      avoid breaking the implementation of XMValue.
   */
  private static final String DATA_TYPES = DataTypes.DATA_TYPE_LIST+"WHO";

  /** These are the format types which will be considered valid by this class.  Note that 'U', 'V'
      are not valid BLUE format types, but are only allowed here for the purposes of generating
      test files.
   */
  private static final String EXTENDED_DATA_TYPES = DataTypes.EXTENDED_DATA_TYPE_LIST+"WHO";

  /** Number of elements in data segment */
  @InternalUseOnly("Since NeXtMidas 3.5.1, use getSize() and/or setSize()")
  public int size;

  /** Atomic mode.
      @see nxm.sys.inc.DataTypes#DATA_MODE_LIST
  */
  @InternalUseOnly("Since NeXtMidas 3.5.1, use getFormatMode() and/or setFormatMode()")
  public byte mode;

  /** Atomic type.
      @see nxm.sys.inc.DataTypes#DATA_TYPE_LIST
  */
  @InternalUseOnly("Since NeXtMidas 3.5.1, use getFormatType(), setFormatType(), and/or getFormatTypeChar()")
  public byte type;

  /** data representation (byte ordering used).
      @see nxm.sys.inc.DataTypes#repList
  */
  @InternalUseOnly("Since NeXtMidas 3.5.1, use getRep() and/or setRep()")
  public byte rep;

  /** Attributes (Read,Write,Delete,Hide) */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public byte attr;

  /** References to data buffer */
  @InternalUseOnly("Since NeXtMidas 3.5.1, use getBuf()")
  public byte[] buf;

  /** data start offset in buf[] in bytes */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public int boff=0;

  /** bytes per element in data buffer */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public int bpe=1;

  /** atoms per element in data buffer */
  @InternalUseOnly("Since NeXtMidas 3.5.1, use getAPE() andor setAPE()")
  public int ape=1;

  /** scalars per atom in data buffer */
  @InternalUseOnly("Since NeXtMidas 3.5.1, use getSPA() and/or setSPA()")
  public int spa=1;

  /** bytes per scalar in data buffer */
  @InternalUseOnly("Since NeXtMidas 3.5.1")
  public int bps=1;

  /** must write in multiples of this */
  @InternalUseOnly
  public int writeMult =1; // to support 'J' Data Type NXM-3894

  /** References to Java working array, null if none. */
  private transient Object array;
  /** format type of {@link #array}, 0 if the Java array is null (e.g. is not used). */
  private byte atype;

  /** Reference to header data object */
  private transient Object header;

  /** Copy of header object, when header is actually a Time (standard usage) */
  private Time headerTime;

  /** Construct a Data object */
  public Data () {
    this(SCALAR, BYTE, 0);
  }

  /**
   * Added for easy access to the necessary methods for getting Number type data out of the data buffer.
   * @since 3.9.3
   */
  private final static Map<Byte,Method> FROM_BUF_METHODS = getFromBufMethodsBuilder();

  /**
   * Added for dynamic access to methods for casting Number to different number types.
   * @since 3.9.3
   */
  private final static Map<Class<?>, Method> CASTING_METHODS = getCastingMethods();
  /** Construct a Data buffer for future use
      @param newMode The atomic mode ('S' for scalar, 'C' for complex, etc.).
      @param newType The atomic type ('B' for byte, 'S' for short, 'D' for double, etc.)
      @param newSize The new size in elements.
      @since NeXtMidas 3.7.1
   */
  private Data (byte newMode, byte newType, int newSize) {
    checkFormatPairing(newMode, newType);
    mode = modeFromMode(newMode);
    type = typeFromType(newType);
    bps  = bpsFromType(type);
    spa  = spaFromMode(mode);
    bpe  = bps*spa*ape;
    writeMult = ((newType==JINT) && (spa%2!=0)) ? 2:1;  // NXM-3894 12-bit 'J'
    rep  = Shell.rep;
    attr = 0;

    setSize(sizeFromSize(newSize));
  }

  /** Construct a Data buffer for future use
      @param format The data format (e.g. "SB", "VD", "8A", etc.).
      @param size   The initial size in elements.
   */
  public Data (String format, int size) {
    this((byte)format.charAt(0), (byte)format.charAt(1), size);
  }

  /** Construct a Data buffer for future use. If the default value is null
      or zero this constructor is identical to {@link Data#Data(String,int)}.
      @param format   The buffer format (e.g. "SF").
      @param size     The size of the buffer.
      @param defValue The default value for the scalar values in the buffer.
      @since NeXtMidas 3.3.0
   */
  public Data (String format, int size, Number defValue) {
    this(format, size);
    if ((defValue != null) && (defValue.doubleValue() != 0.0) && (size > 0)) {
      if (bps >= 1) {
        // In most cases we can bypass many of the packing steps since we are
        // merely repeating values. We do this by packing the first value and
        // thereafter copying the packed bytes of that value to the rest of
        // the buffer.
        setNumber(0, defValue);
        int start = boff + bps; // skip first scalar
        int end   = boff + size*bpe;
        for (int i=start,j=0; i < end; i++,j++) {
          buf[i] = buf[boff+(j%bpe)];
        }
      }
      else {
        // For sub-byte types (e.g. "SN" or "SP") things get trickier since we
        // need to handle the start and end of the "fast copy" with some extra
        // checks. For now we will leave this as a "future enhancement."
        for (int i=0; i < size*spa; i++) {
          setNumber(i, defValue);
        }
      }
    }
  }

  /** Construct a Data object from a String with conversion
      @param string     String to convert
      @param conversion Character representing conversion (e.g. 'D','F',...)
      @param ref        Reference object (often a Midas context or a Command)
      @return           Converted object
   */
  public static Data fromString (String string, char conversion, Object ref) {
    // Passing newMode=' ' on the next line causes Convert.s2o(..) to try and auto-detect
    // the mode. This matches the NeXtMidas 3.7.0 behavior.
    return fromString(string, (byte)' ', (byte)conversion, Convert.ref2MidasReference(ref));
  }

  /** Construct a Data object from a String with conversion
      @param string  String to convert
      @param newMode The atomic mode ('S' for scalar, 'C' for complex, etc.).
      @param newType The atomic type ('B' for byte, 'S' for short, 'D' for double, etc.)
      @param ref     Reference object (often a Midas context or a Command)
      @return        Converted object
      @since NeXtMidas 3.7.1
   */
  // TODO: consider making this method public
  private static Data fromString (String string, byte newMode, byte newType, MidasReference ref) {
    Object obj = Convert.s2o(string, getFormat(newMode, newType), ref);
    if (obj instanceof Data) return (Data)obj;
    else if (obj instanceof String) return new Data((String)obj,newType==ASCII256); // make either 'A' or 'Z' type NXM-3570
    else return new Data(Convert.o2d(obj));
  }

  /** Construct a Data object from an Object with conversion
      @param obj         The object (value) to pack.
      @param format      The format of the data.
      @param numElements Number of elements to pack from obj into dataOut (since 3.6.0 for Bug 2959), 1 in most scenarios
      @param ref         Reference object (often a Midas context or a Command)
      @return            Converted object
      @since NeXtMidas 3.7.1
   */
  // this method replaces the former DataFile.packDataIntoBuffer()
  static Data fromObject (Object obj, String format, int numElements, MidasReference ref) {
    Data dataOut = new Data(format, numElements);
    try {
      if (dataOut.isString()) {
        int _bpa = getBPA(format);
        if (obj == null) { obj = ""; }
        dataOut.packS(0, _bpa, Convert.o2s(obj));
      }
      else {
        if (obj == null) { obj = new Data(format, numElements); } // Created a zero-filled data object
        obj = Convert.o2Data(obj, format, ref); // NXM-3570 force conversion to Data obj, so 'Z' interpreted as 256 ASCII
        dataOut.fillBufferFrom((Data)obj, numElements);
      }
    }
    catch (MidasException e) {
      throw e; // Assume the message already describes the problem.
    }
    catch (RuntimeException e) {
      throw new MidasException("Data: unable to convert value "+obj+" to format "+format, e);
    }
    return dataOut;
  }

  /** Construct a Data from a byte buffer with a given representation. <br> <br>
      A copy of the byte buffer will be made before any conversions are made.  <br> <br>
      The representation ({@link #IEEE} or {@link #EEEI}) of the underlying data
      in the {@link Data} object returned is independent of the representation
      of the data in the buffer passed to this method.

      @param buffer The byte buffer to get the bytes from.
      @param rep    The byte representation of the data in the buffer (usually
                    {@link #IEEE} or {@link #EEEI}).
      @param offset The offset into the byte buffer to begin reading from.
      @param ele    The number of elements to read from the buffer.
      @param itype  The data type of the data in the buffer (usually
                    {@link #DOUBLE} or {@link #LONG}).
      @param spa    The number of scalars per atom.

      @return A data object initialized with the data in the buffer.

      @see Convert#rep(byte[],int,byte,int,byte,byte)
      @see #Data(byte[],int,int,int,int)
   */
  public static Data fromBuffer(byte[] buffer, byte rep, int offset,
                                int ele, byte itype, int spa) {
    final int    bytes = ele * getBPS(itype);
    final byte[] buf   = new byte[bytes];

    System.arraycopy(buffer, offset, buf, 0, bytes);

    Convert.rep(buf, 0, itype, ele, rep, Shell.rep);

    return new Data(buf, 0, buf.length, itype);
  }

  /** Construct a Data from a byte buffer with a given representation. <br> <br>
      A copy of the byte buffer will be made before any conversions are made.  <br> <br>
      The representation ({@link #IEEE} or {@link #EEEI}) of the underlying data
      in the {@link Data} object returned is independent of the representation
      of the data in the buffer passed to this method.

      @param buffer The byte buffer to get the bytes from.
      @param rep    The byte representation of the data in the buffer (usually
                    {@link #IEEE} or {@link #EEEI}).
      @param offset The offset into the byte buffer to begin reading from.
      @param ele    The number of elements to read from the buffer.
      @param form   The format of the data in the buffer.

      @return A data object initialized with the data in the buffer.

      @see Convert#rep(byte[],int,byte,int,byte,byte)
      @see #fromBuffer(byte[],byte,int,int,byte,int)
   */
  public static Data fromBuffer(byte[] buffer, byte rep, int offset,
                                int ele, String form) {
    byte itype = (byte)form.charAt(1);
    int  spa   = Data.getSPA(form.charAt(0));

    return fromBuffer(buffer, rep, offset, ele, itype, spa);
  }

  /** Construct a Data from a byte buffer with a given representation. <br> <br>
      This is identical to <code>fromBuffer(buffer,rep,offset,ele,itype,-1)</code>.

      @param buffer The byte buffer to get the bytes from.
      @param rep    The byte representation of the data in the buffer (usually
                    {@link #IEEE} or {@link #EEEI}).
      @param offset The offset into the byte buffer to begin reading from.
      @param ele    The number of elements to read from the buffer.
      @param itype  The data type of the data in the buffer (usually
                    {@link #DOUBLE} or {@link #LONG}).

      @return A data object initialized with the data in the buffer.

      @see #fromBuffer(byte[],byte,int,int,byte,int)
   */
  public static Data fromBuffer(byte[] buffer, byte rep, int offset,
                                int ele, byte itype) {
    return fromBuffer(buffer,rep,offset,ele,itype,-1);
  }

  /** Construct a Data object from a String, an array of numbers, or a numeric Object.
      @param value the object to construct the Data object from.
      @throws MidasException for un-supported Objects
    */
  public Data (Object value) {
    this();
         if (value instanceof String)    initTo( (String)value );
    else if (value instanceof double[])  initTo( (double[])value );
    else if (value instanceof float[])   initTo( (float[])value );
    else if (value instanceof long[])    initTo( (long[])value );
    else if (value instanceof int[])     initTo( (int[])value );
    else if (value instanceof short[])   initTo( (short[])value );
    else if (value instanceof byte[])    initTo( (byte[])value );
    else if (value instanceof Double)    initTo( ((Double)value).doubleValue());
    else if (value instanceof Float)     initTo( ((Float)value).floatValue());
    else if (value instanceof Long)      initTo( ((Long)value).longValue());
    else if (value instanceof Integer)   initTo( ((Integer)value).intValue());
    else if (value instanceof Short)     initTo( ((Short)value).shortValue());
    else if (value instanceof Byte)      initTo( ((Byte)value).byteValue());
    else if (value instanceof List<?>)   fromList(castObjectToList (value)); // Added in 3.9.3 to support the use of ArrayList types
    else throw new MidasException("Invalid object type passed to Data constructor");
  }

  /** Construct a Data object from a String, an array of numbers, or a numeric Object with specified SPA
      @param value  the object to construct the Data object from.
      @param spa    scalars per atom
   */
  public Data (Object value, int spa) {
    this(value);
    setSPA(spa);
  }

  /** Construct Data object looking into a byte array with specified SPA
      @param buffer   The buffer to construct the Data object from.
      @param offset   The byte offset
      @param bytes    The number of bytes to read from the buffer
      @param itype    The atomic type ('B' for byte, 'S' for short, 'D' for double, etc.)
      @param spa      Scalars per atom
   */
  public Data (byte[] buffer, int offset, int bytes, int itype, int spa) {
    this();
    buf = buffer;
    boff = offset;
    setFormatType((byte)itype);
    setSize(bytes/bpe);
    if (spa<0) resetMode();
    else if (spa>1) setSPA(spa);
  }

  /** Construct Data object looking into a byte array
      @param buffer   The buffer to construct the Data object from.
      @param offset   The byte offset
      @param bytes    The number of bytes to read from the buffer
      @param itype    The atomic type ('B' for byte, 'S' for short, 'D' for double, etc.)
   */
  public Data (byte[] buffer, int offset, int bytes, int itype) {
    this(buffer,offset,bytes,itype,-1);
  }

  /** Construct a Data object from a String
      @param value the String to construct the Data object from.
   */
  public Data (String value) { this(); initTo(value); }

  /** Construct a Data object from a String
      @param value the String to construct the Data object from.
      @param isASCII256 true if should be 'Z' 256 ASCII type
    */
  public Data (String value, boolean isASCII256) {
    this();
    if (isASCII256) {
      initToASCII256(value);
    }
    else {
      initTo(value);
    }
  }

  /** Construct a Data object from a double
      @param value the double to construct the Data object from.
   */
  public Data (double value) { this(); initTo(value); }

  /** Construct a Data object from a float
      @param value the float to construct the Data object from.
   */
  public Data (float value) { this(); initTo(value); }

  /** Construct a Data object from an eXtra long (Java long)
      @param value the long to construct the Data object from.
   */
  public Data (long value) { this(); initTo(value); }

  /** Construct a Data object from a long (Java int)
      @param value the int to construct the Data object from.
   */
  public Data (int value) { this(); initTo(value); }

  /** Construct a Data object from a short
      @param value the short to construct the Data object from.
   */
  public Data (short value) { this(); initTo(value); }

  /** Construct a Data object from a byte
      @param value the byte to construct the Data object from.
   */
  public Data (byte value) { this(); initTo(value); }

  /** Construct a Data object from an array of double.
      @param value the double[] to construct the Data object from.
      @since NeXtMidas 3.3.0
   */
  public Data (double ... value) { this(); initTo(value); }

  /** Construct a Data object from an array of float.
      @param value the float[] to construct the Data object from.
      @since NeXtMidas 3.3.0
   */
  public Data (float ... value) { this(); initTo(value); }

  /** Construct a Data object from an array of eXtra long (Java long).
      @param value the long[] to construct the Data object from.
      @since NeXtMidas 3.3.0
   */
  public Data (long ... value) { this(); initTo(value); }

  /** Construct a Data object from an array of long (Java int).
      @param value the int[] to construct the Data object from.
      @since NeXtMidas 3.3.0
   */
  public Data (int ... value) { this(); initTo(value); }

  /** Construct a Data object from an array of short.
      @param value the short[] to construct the Data object from.
      @since NeXtMidas 3.3.0
   */
  public Data (short ... value) { this(); initTo(value); }

  /** Construct a Data object from an array of byte.
      @param value the byte[] to construct the Data object from.
      @since NeXtMidas 3.3.0
   */
  public Data (byte ... value) { this(); initTo(value); }

  /** Creates a copy of this data buffer. The new instance will have its own buffer (boff=0)
      initialized with a copy of the data in this object's buffer. Note that all instances of
      <code>Data</code> are unique with respect to the <code>cast</code> and <code>uncast</code> methods
      and arrays cast from one instance can not be uncast from another instance even if it is
      a copy.
      @return A copy of the data object.
      @since NeXtMidas 2.7.1
   */
  public Data copy () {
    Data copy = new Data();
    int  len  = size*bpe;

    copy.size   = this.size;
    copy.mode   = this.mode;
    copy.type   = this.type;
    copy.rep    = this.rep;
    copy.attr   = this.attr;
    copy.buf    = new byte[len];  // <-- new buffer (initialized below)
    copy.boff   = 0;              // <-- new buffer starts at 0 (saves space)
    copy.bpe    = this.bpe;
    copy.ape    = this.ape;
    copy.spa    = this.spa;
    copy.bps    = this.bps;
    copy.writeMult = this.writeMult; // NXM-3894 12-bit 'J'
    copy.array  = null;           // <-- do not copy (doing so could cause problems with cast/uncast)
    copy.atype  = 0;              // <-- do not copy (doing so could cause problems with cast/uncast)
    copy.header = this.header;
    copy.headerTime = this.headerTime; // copy used in serialization, when header is a Time Object

    System.arraycopy(buf, boff, copy.buf, 0, len);

    return copy;
  }

  private void initTo (double[] value)  { setFormatType(DOUBLE); size=value.length; buf=new byte[size*bpe]; fromArray(value); }
  private void initTo (float[] value)   { setFormatType(FLOAT);  size=value.length; buf=new byte[size*bpe]; fromArray(value); }
  private void initTo (long[] value)    { setFormatType(XLONG);  size=value.length; buf=new byte[size*bpe]; fromArray(value); }
  private void initTo (int[] value)     { setFormatType(LONG);   size=value.length; buf=new byte[size*bpe]; fromArray(value); }
  private void initTo (short[] value)   { setFormatType(INT);    size=value.length; buf=new byte[size*bpe]; fromArray(value); }
  private void initTo (byte[] value)    { setFormatType(BYTE);   size=value.length; buf=value; }

  private void initTo (double value)  { setFormatType(DOUBLE); size=1; buf=new byte[bpe]; packD(0,value); }
  private void initTo (float value)   { setFormatType(FLOAT);  size=1; buf=new byte[bpe]; packF(0,value); }
  private void initTo (long value)    { setFormatType(XLONG);  size=1; buf=new byte[bpe]; packX(0,value); }
  private void initTo (int value)     { setFormatType(LONG);   size=1; buf=new byte[bpe]; packL(0,value); }
  private void initTo (short value)   { setFormatType(INT);    size=1; buf=new byte[bpe]; packI(0,value); }
  private void initTo (byte value)    { setFormatType(BYTE);   size=1; buf=new byte[bpe]; packB(0,value); }

  private void initTo (String value)  { setFormatType(ASCII);  size=(value.length()+7)/8;
          buf=new byte[size*bpe]; packS(0,size*bpe,value);}

  private void initToASCII256 (String value) {
          setFormatType(ASCII256);
          size=(value.length()+255)/256;
          buf=new byte[size*bpe]; packS(0,size*bpe,value);
  }

  /** Sets the data format.
      @param format The data format (e.g. "SB", "VD", "8A", etc.).
   */
  public void setFormat (String format) {
    setFormatMode((byte)format.charAt(0));
    setFormatType((byte)format.charAt(1));
  }

  /** Set the atom type. This is an older form of {@link #setFormatType(byte)} that is maintained
      for backwards-compatibility. New code should use {@link #setFormatType(byte)} in place of this.
      @param type The atomic type ('B' for byte, 'S' for short, 'D' for double, etc.)
   */
  public void setType (byte type) {
    setFormatType(type);
  }

  /** Set the atom type. (This method uses a <code>byte</code> rather than a <code>char</code> for
      consistency with {@link DataFile} which matches the BLUE header.)
      @param type The atomic type ('B' for byte, 'S' for short, 'D' for double, etc.)
      @since NeXtMidas 2.7.0
      @see #setFormat(String)
   */
  public void setFormatType (byte type) {
    this.type = type;
    bps = getBPS(type);
    bpe = ape*spa*bps;
    writeMult = ((type==JINT) && (spa%2!=0)) ? 2:1; // NXM-3894 12-bit 'J'
  }

  /** Set the atom type. This will cast the char to a byte and call {@link #setFormatType(byte)}.
      @param type The atomic type ('B' for byte, 'S' for short, 'D' for double, etc.)
      @since NeXtMidas 3.3.0
   */
  public void setFormatType (char type) {
    setFormatType((byte)type);
  }

  /** Get the atom type. (This method uses a <code>byte</code> rather than a <code>char</code> for
      consistency with {@link DataFile} which matches the BLUE header.)
      @return The atomic type ('B' for byte, 'S' for short, 'D' for double, etc.)
      @since NeXtMidas 2.7.0
      @see #getFormat()
   */
  public byte getFormatType () {
    return type;
  }

  /** Get the atom type (casting byte as a char).
      @return The atomic type ('B' for byte, 'S' for short, 'D' for double, etc.)
      @since NeXtMidas 3.3.0
   */
  public char getFormatTypeChar () {
    return (char)type;
  }

  /** Get the atom type letter. This is an older form of {@link #getFormatType()} that is maintained
      for backwards-compatibility. New code should use {@link #getFormatType()} in place of this.
      @return The atomic type ('B' for byte, 'D' for double, etc.)
      @deprecated Since NeXtMidas 2.7.0: Use {@link #getFormatType()} in place of this.
   */
  @Deprecated public char getTypeLetter () {
    // JC4: I deprecated this method since it was only being used within NeXtOpt and that code
    // now used getFormatType(). I doubt it was in common use elsewhere.
    return this.getFormat().charAt(1);
  }

  /** Check for valid Data type character.
      @param type Possible type letter ('B' for byte, 'D' for double, etc.).
      @return True if the type if valid, false if it is not.
   */
  public static boolean validType (char type) {
    return (DataTypes.DATA_TYPE_LIST.indexOf(type) >=0);
  }

  /** Set the atom mode. This is an older form of {@link #setFormatMode(byte)} that is maintained
      for backwards-compatibility. New code should use {@link #setFormatMode(byte)} in place of this.
      @param mode The atomic mode ('S' for scalar, 'C' for complex, etc.).
   */
  public void setMode (byte mode) {
    setFormatMode(mode);
  }

  /** Set the atom mode. (This method uses a <code>byte</code> rather than a <code>char</code> for
      consistency with {@link DataFile} which matches the BLUE header.)
      @param mode The atomic mode ('S' for scalar, 'C' for complex, etc.).
      @since NeXtMidas 2.7.0
      @see #setFormat(String)
   */
  public void setFormatMode (byte mode) {
    this.mode = mode;
    spa = getSPA(mode);
    bpe = ape*spa*bps;
    writeMult = ((type==JINT) && (spa%2!=0)) ? 2:1; // NXM-3894 12-bit 'J'
  }

  /** Set the atom mode. This will cast the char to a byte and call {@link #setFormatMode(byte)}.
      @param mode The atomic mode ('S' for scalar, 'C' for complex, etc.).
      @since NeXtMidas 3.3.0
   */
  public void setFormatMode (char mode) {
    setFormatMode((byte)mode);
  }

  /** Get the atom mode. (This method uses a <code>byte</code> rather than a <code>char</code> for
      consistency with {@link DataFile} which matches the BLUE header.)
      @return The atomic mode ('S' for scalar, 'C' for complex, etc.).
      @since NeXtMidas 2.7.0
      @see #getFormat()
   */
  public byte getFormatMode () {
    return mode;
  }

  /** Get the atom mode (casting byte as a char).
      @return The atomic mode ('S' for scalar, 'C' for complex, etc.).
      @since NeXtMidas 3.3.0
   */
  public char getFormatModeChar () {
    return (char) mode;
  }

  /** set the scalars per atom
      @param spa the value to set scalars per atom.
   */
  public void setSPA (int spa) {
    setSPA(spa, ' ');
  }

  /** set the scalars per atom */
  void setSPA (int spa, char mode) {
    if (mode == ' ') {
      if (spa>0 && spa<=10) this.mode = (byte)MODES.charAt(spa-1);
      else if (spa==16) this.mode = TRANSMATRIX;
      else { this.mode = SCALAR; spa = 1; }

    } else if (spa == -1) {
      spa = getSPA((byte)mode);

    } else if (getSPA((byte)mode) != spa) {
      throw new MidasException("Data: Illegal combination mode='"+mode+"' spa="+spa);
    }

    if (ape>1) {
      ape = ape*this.spa/spa;
    } else {
      size = size*this.spa/spa;
      bpe = bpe*spa/this.spa;
    }
    this.spa = spa;
    writeMult = ((type==JINT) && (spa%2!=0)) ? 2:1; // NXM-3894 12-bit 'J'
  }

  /** auto set the atom mode */
  public void resetMode() {
    if (size>0 && size<=4) setFormatMode((byte)MODES.charAt(size-1));
    else setFormatMode(SCALAR);
    if (size==ape) setSize(1);
    else setSize(size/spa);
  }

  /** Set frame size given atoms per element.
      NOTE: This does not change the buffer size.  Call {@link #setSize(int)}
      to expand the buffer if needed.
      @param ape atoms per element
   */
  public void setFrame (int ape) {
    this.ape = ape;
    bpe = ape*spa*bps;
  }

  /** set frame size or record length given given bytes per element
      @param bpe bytes per element
   */
  public void setRecLength (int bpe) {
    this.bpe = bpe;
    if (bpe<0) {
      if (bps<0) ape = bpe/(spa*bps);
      else ape = bpe/(spa*bps*-8);
    }
    else {
      if (bps<0) ape = -8*bpe/(spa*bps);
      else ape = bpe/(spa*bps);
    }
  }

  /** Size the object to hold requested number of elements. Since 2.7.1 the existing data within the limits
      of both the old and new buffers is preserved. If the size is increased the new values will
      be initialized to 0.
      @param size The new size in elements.
   */
  public void setSize (int size) {
    this.size = size;
    int bytes = boff;
    if (bpe>0) bytes += size*bpe;
    else       bytes += (-size*bpe+7)/8;
    try {
      if (buf == null) {
        buf = new byte[bytes];
      }
      else if (bytes > buf.length) {    // allocate more memory only if buffer not big enough
        // Preserve existing data, see Bug 1748
        byte[] buffer = new byte[bytes];
        System.arraycopy(buf, 0, buffer, 0, Math.min(buf.length, bytes));
        buf = buffer;
      }
    } catch (OutOfMemoryError e) {
      // This is is likely to happen when DataFile is trying to allocate a data buffer
      // for a Type 2000 file with a huge frame size (e.g. FS=1M), see Bug 885.
      throw new MidasException("Data.setSize("+size+"): not enough memory to allocate a "+bytes+" byte "
                              +"array for SIZE="+size+" APE="+getAPE()+" FORM="+getFormat()+". "
                              +"Current free memory is "+Runtime.getRuntime().freeMemory()+" bytes.",e);
    }
    prepArray(atype);
  }

  /** Returns the format string.
      @return The data format (e.g. "SB", "VD", "8A", etc.).
   */
  public String getFormat() {
    byte form[] = { mode, type };
    return new String(form);
  }

  /** Returns the specs string
      @return the spec String object.
   */
  public String getSpecs() {
    String specs = "";
    if (size>1) specs += size;
    if (mode!=SCALAR) specs += (char)mode;
    specs += (char)type;
    return specs;
  }

  /** Get bytes per atom for specified format String.
      @param format The data format (e.g. "SF", "CD", "VD", "9A", ...)
      @return bytes per atom for the specified format
      @throws MidasException since NeXtMidas 2.7.2 if format String is invalid
   */
  public static int getBPA (String format) {
    if (format==null || format.length()<2) throw new MidasException("Data.getBPA() format='"
        + format + "' must contain at least 2 characters!");
    byte dataMode = (byte)format.charAt(0);
    byte dataType = (byte)format.charAt(1);
    int bpa = getSPA(dataMode) * getBPS(dataType);
    if (bpa<0 && (-bpa%8)==0) bpa = -bpa/8;  // NXM-3894 Jeff 12-bit J
    return bpa;
  }

  /** Get bytes per scalar given type character F,D,...
      @param ctype the type (F,D,...)
      @return the value of bytes per scalar as an integer.
   */
  public static int getBPS (char ctype) {
    byte btype = (byte)ctype;
    return getBPS (btype);
  }

  static Type getJavaType (char type) {
    switch (type) {
      case FLOAT:    return Float.TYPE;
      case XLONG:    return Long.TYPE;
      case LONG:     return Integer.TYPE;
      case INT:      return Short.TYPE;
      case BYTE:     return Byte.TYPE;
      case DOUBLE:   return Double.TYPE;
      case ASCII:    return String.class;
      case ASCII256: return String.class;
      case 'S':      return String.class; // result type String
      case 'T':      return Table.class;  // result type Table
      default: return Object.class;
    }
  }

  /** Get the name string for a specified data format. For example
      <code>getFormatTypeName('L')</code> returns <code>"LongInt"</code>.
      @param key The type of data.
      @return The name of the data type.
      @see DataTypes
   */
  public static String getFormatTypeName (byte key) {
    String name;
    switch (key) {
      case FLOAT:    name = "Float"; break;
      case XLONG:    name = "XLongInt"; break;
      case LONG:     name = "LongInt"; break;
      case INT:      name = "Integer"; break;
      case JINT:     name = "Int12Packed"; break;  // NXM-3894 Jeff 12-bit J
      case BYTE:     name = "Byte"; break;
      case 'M': name = "MuLaw"; break;
      case 'O': name = "OffsetByte"; break;
      case NIBBLE:   name = "NibblePacked"; break;
      case PACKED:   name = "BitPacked"; break;
      case DOUBLE:   name = "Double"; break;
      case ASCII:    name = "Ascii"; break;
      case ASCII256: name = "Ascii256"; break; // NXM-3570
      case 'H': name = "Homogeneous"; break;
      default: name = "Undefined"; break;
    }
    return name;
  }

  /** Get the name string for a specified array. For example
      <code>getFormatModeName('S')</code> returns <code>"Scalar"</code>. <br> <br>
      Note that the special types (<code>'S'</code>, <code>'C'</code>, etc.)
      and the basic types (<code>'1'</code>, <code>'2'</code>, etc.) will not
      produce the same results although they represent an array with the same
      dimensions.
      @param key The type of array.
      @return The name of the array type.
      @see DataTypes
   */
  public static String getFormatModeName (byte key) {
    String name;
    switch (key) {
      case SCALAR:      name = "Scalar"; break;
      case COMPLEX:     name = "Complex"; break;
      case VECTOR:      name = "Vector"; break;
      case QUAD:        name = "Quad"; break;
      case MATRIX:      name = "Matrix"; break;
      case XMODE:       name = "Ten"; break;
      case TRANSMATRIX: name = "TransMatrix"; break;
      case AMODE:       name = "Thirty-Two"; break;  // missed earlier = added in 3.9.4
      case 'N': name = "Non"; break;
      case '1': name = "One"; break;
      case '2': name = "Two"; break;
      case '3': name = "Three"; break;
      case '4': name = "Four"; break;
      case '5': name = "Five"; break;
      case '6': name = "Six"; break;
      case '7': name = "Seven"; break;
      case '8': name = "Eight"; break;
      case '9': name = "Nine"; break;
      default: name = "Undefined"; break;
    }
    return name;
  }

  /** Get bytes per scalar given the type. This is same as M$NPT function except: <br>
      1. returns -4 for type 'N' (nibble) <br>
      2. one (instead of zero) is returned for unknown types
      @param type The scalar type
      @return the value of bytes per scalar as an integer.
   */
  public static int getBPS (byte type) {
    int bps;
    switch (type) {
      case FLOAT:  bps = 4; break;
      case DOUBLE: bps = 8; break;
      case XLONG:  bps = 8; break;
      case LONG:   bps = 4; break;
      case ULONG:  bps = 4; break;  // since 3.9.3 add minimal unsigned support for NXM-3021
      case INT:    bps = 2; break;
      case UINT:   bps = 2; break;
      case JINT:   bps = -12; break; // NXM-3894 Jeff 12-bit J
      case BYTE:   bps = 1; break;
      case NIBBLE: bps =-4; break;
      case PACKED: bps =-1; break;
      case 'M':    bps = 1; break;
      case ASCII:  bps = 8; break;
      case ASCII256: bps = 256; break; // NXM-3570
      default : bps = 1; break;
    }
    return bps;
  }

  /** Get scalars per atom for the specified mode.
      This is same as M$NPM function except: <br>
      1. one (instead of zero) is returned for unknown modes <br>
      2. for values (128-255), does NOT return the ordinal of the number minus 127<br>
      modes: S,C,V,Q,M,X,T,A =  1,2,3,4,9,10,16,32 <br>
      modes: 0-9             =  0-9 <br>
      mode 'A' = 32 (since NeXtMidas 3.1.0 to match X-Midas 4.6+)
      @param mode The data mode ('S' for scalar, 'V' for vector, etc.).
      @return The number of scalars per atom for the given mode.
  */
  public static int getSPA (byte mode) {
    int spa;
    if (mode>='1' && mode<='9')
      spa = mode - (int)'0';
    else switch (mode) {
      case SCALAR:      spa = 1; break;
      case COMPLEX:     spa = 2; break;
      case VECTOR:      spa = 3; break;
      case QUAD:        spa = 4; break;
      case MATRIX:      spa = 9; break;
      case XMODE:       spa = 10; break;
      case TRANSMATRIX: spa = 16; break;
      case AMODE:       spa = 32; break; // since 3.1.0 to match X-Midas 4.6.0+ // jtm correction
      default : spa = 1; break;
    }
    return spa;
  }

  /** Get scalars per atom for the specified mode.
      @param mode The data mode ('S' for scalar, 'V' for vector, etc.).
      @return The number of scalars per atom for the given mode.
      @see #getSPA(byte)
   */
  public static int getSPA (char mode) {
    return getSPA((byte)mode);
  }

  /** Get the max value for this Data type
      @return the max value for this Data type.
   */
  public double getTypeMax () { return getTypeMax(type); }

  /** Get the max value for the given type
    @param type The scalar type
    @return the max value for the given type.
  */
  public static double getTypeMax (byte type) {
    double d=0;
    switch (type) {
      case DOUBLE: d =         DataTypes.DMAX; break;
      case FLOAT:  d = (double)DataTypes.FMAX; break;
      case XLONG:  d = (double)DataTypes.XMAX; break;
      case LONG:   d = (double)DataTypes.LMAX; break;
      case INT:    d = (double)DataTypes.IMAX; break;
      case BYTE:   d = (double)DataTypes.BMAX; break;
    }
    return d;
  }

  /** Get the min value for this Data type
      @return the min value for this Data type.
   */
  public double getTypeMin () { return getTypeMin(type); }

  /** Get the min value for the given type
    @param type The scalar type
    @return the min value for given type.
  */
  public static double getTypeMin (byte type) {
    double d=0;
    switch (type) {
      case DOUBLE: d =         DataTypes.DMIN; break;
      case FLOAT:  d = (double)DataTypes.FMIN; break;
      case XLONG:  d = (double)DataTypes.XMIN; break;
      case LONG:   d = (double)DataTypes.LMIN; break;
      case INT:    d = (double)DataTypes.IMIN; break;
      case BYTE:   d = (double)DataTypes.BMIN; break;
    }
    return d;
  }

  /** Gets the representation of the data used internally.
    @return The representation of the data used internally, this will usually be either
            {@link DataTypes#IEEE} or {@link DataTypes#EEEI}.
    @see #getRep(String)
  */
  public byte getRep () {
    return this.rep;
  }

  /** Gets the representation of the data used internally. <br>
      Unlike this method, {@link DataFile#testRep(String)} will only check for representations that
      are supported by {@link DataFile} while this method will check for all known representations.
      Additionally this method is case-insensitive while that in {@link DataFile}  is not.
      @param rep The string describing the representation (such as IEEE or EEEI).
      @return The representation of the data used internally, this will usually be either
              {@link DataTypes#IEEE} or {@link DataTypes#EEEI} or zero for unknown representations.
      @see #getRepName(byte)
  */
  public static byte getRep (String rep) {
    rep = (rep == null)? "": rep.toUpperCase();

    if (rep.startsWith("VAX"))   return DataTypes.VAX;
    else if (rep.equals("IEEE")) return DataTypes.IEEE;
    else if (rep.equals("EEEI")) return DataTypes.EEEI;
    else if (rep.equals("CRAY")) return DataTypes.CRAY;
    else return 0;
  }

  /** Gets the name of representation of the data used internally. Usually either
      <code>"IEEE"</code> or <code>"EEEI"</code>.
      @see #getRepName(byte)
      @return the name of the data used internally.
  */
  public String getRepName () {
    return getRepName(this.getRep());
  }

  /** Gets the name of representation of the data used internally. <br> <br> This is setup such that
      the following will be true: <code> getRep(getRepName(rep)) == rep </code>
      @param rep The representation id (such as {@link DataTypes#IEEE} or {@link DataTypes#EEEI}).
      @return The name of representation of the data used internally, this will usually be either
              <code>"IEEE"</code> or <code>"EEEI"</code>. If <code>rep</code> represents an unknown
              representation then <code>null</code> will be returned.
      @see #getRep(String)
  */
  public static String getRepName (byte rep) {
    switch(rep) {
      case DataTypes.VAX:   return "VAX";
      case DataTypes.IEEE:  return "IEEE";
      case DataTypes.EEEI:  return "EEEI";
      case DataTypes.CRAY:  return "CRAY";
      default:              return null;
    }
  }

  /** Get the size in elements
      @return the size of this Data object in number of elements.
   */
  public int getSize () { return size; }

  /** Get the Atoms per Element
      @return the number of atoms per element for this Data object.
   */
  public int getAPE () { return ape; }

  /** Get the Scalars per Atom
      @return the number of scalars per atom for this Data object.
   */
  public int getSPA () { return spa; }

  /** Get the value of the Object
      @return the value of this object.
   */
  public String getValue () { return toString(); }

  /** Sets the atoms per element to the specified value and changes the size to
      fit the buffer
      @param ape The atoms per element to set the Data object to
      @since NeXtMidas 3.1.0
   */
  public void setAPE (int ape) {
    double factor = ((double)this.ape) / ape;
    int newSize = (int)Math.ceil(size*factor);
    setFrame(ape);
    setSize(newSize);
  }

  /** Get an element or elements from non-homogeneous data buffer. This is
      typically used when accessing Type 3000/5000/6000 files.
      @param byteoffset Byte offset at the start of the atom.
      @param format     Format digraph for the atom (e.g. "SF" or "VD").
      @param nelement   The number of elements to get.
      @return the element with the given parameters.
      @since NeXtMidas 3.1.0
   */
  public Data getElementAt (int byteoffset, String format, int nelement) {
    Data element = new Data(format, nelement);
    element.setFrame(ape);
    element.setSize(nelement);
    System.arraycopy (buf,boff+byteoffset, element.buf,0, bpe*nelement);
    return element;
  }

  /** Get an atom from homogeneous data buffer.
      @param index Element offset at the start of the atom.
      @return the atom at index.
   */
  public Data getAtomAt (int index) {
    Data atom = new Data();
    atom.setFormatType(type);
    atom.setFormatMode(mode);
    atom.setSize(1);
    /* The next two lines ensure a length of at least 1 when dealing with PACKED and NIBBLE data */
    int localbpe = (atom.bpe<0)? (int)(-atom.bpe/8 + .9): atom.bpe;
    int localbps = (bps<0)? (int)(-bps/8 + .9): bps;
    System.arraycopy (buf,boff+index*spa*localbps, atom.buf,0,localbpe);

    return atom;
  }

  /** Get an atom from non-homogeneous data buffer. This is typically used when accessing
      Type 3000/5000/6000 files. This is the same as <code>getAtomAt(byteoffset,format,1)</code>.
      @param byteoffset Byte offset at the start of the atom.
      @param format     Format digraph for the atom (e.g. "SF" or "VD").
      @return the atom found with the given parameters.
   */
  public Data getAtomAt (int byteoffset, String format) {
// JC4: This method is rarely used (just in LIST and DATALIST), we should probably change it to
// return getAtomAt(byteoffset,format,1);
    Data atom = new Data();
    atom.setFormat(format);
    atom.setSize(1);
    System.arraycopy (buf,boff+byteoffset, atom.buf,0, atom.bpe);
    return atom;
  }

  /** Get an atom or atoms from non-homogeneous data buffer. This is typically used when accessing
      Type 3000/5000/6000 files.
      @param byteoffset Byte offset at the start of the atom.
      @param format     Format digraph for the atom (e.g. "SF" or "VD").
      @param natom      The number of atoms to get.
      @return the atom(s) found at with the given parameters.
   */
  public Data getAtomAt (int byteoffset, String format, int natom) {
    Data atom = new Data(format, natom);
    System.arraycopy (buf,boff+byteoffset, atom.buf,0, atom.bpe*natom);
    return atom;
  }

  /** Get a scalar from data buffer.
      @param index Scalar offset into the buffer.
      @return the scalar found at the given index.
   */
  public Data getScalarAt (int index) {
    Data scalar = new Data();
    scalar.setFormatMode(SCALAR);
    scalar.setFormatType(type);
    scalar.setSize(1);
    System.arraycopy (buf,index*bps, scalar.buf,0, scalar.bps);
    return scalar;
  }

  /** Sets the scalar at the specified offset in bytes.
      @throws MidasException for un-supported Objects
      @since NeXtMidas 3.1.0 (implemented)
      @param off    Byte offset at the start of the scalar.
      @param value  The Data object.
   */
  public void setScalarAt (int off, Object value) {
    Data data;
    if (value instanceof Data) {
      data = (Data) value;
    }
    else {
      data = new Data(value);
    }
    setScalarAt(off, data);
  }

  /** Sets the scalar at the specified offset in bytes.  Note that this method
      will throw an ArrayStoreException if the passed Object cannot be
      interpreted as an array of the appropriate type.
      @param off    Byte offset at the start of the scalar.
      @param data  The Data object.
      @since NeXtMidas 3.1.0
   */
  public void setScalarAt (int off, Data data) {
    int localbps = (bps<0)? (int)(-bps/8 + .9): bps;
    System.arraycopy (data.buf,0, buf,boff+off, localbps);
  }

  /** Sets the atom at the specified index to the given atom.
      Uses a calculation similar to getAtomAt(int index) for positioning.
      @param index The index in atoms (not elements) of the atom to set
      @param atom The data to use to set the atom.
      @since NeXtMidas 3.1.0
  */
  public void setAtomAt (int index, Data atom) {
    int localbps = (bps<0)? (int)(-bps/8 + .9): bps;
    int atomSize = spa*localbps;
    System.arraycopy (atom.buf,atom.boff, buf,boff+index*atomSize, atomSize);
  }

  /** Sets the element at the specified index to the given element.
      Uses a calculation similar to getAtomAt(int index) for positioning.
      @param index The index in elements (not atoms) of the element to set
      @param element The data to use to set the element.
      @since NeXtMidas 3.1.0
   */
  public void setElementAt (int index, Data element) {
    int localbpe = (bpe<0)? (int)(-bpe/8 + .9): bpe;
    System.arraycopy (element.buf,element.boff, buf,boff+index*localbpe,localbpe);
  }

  /** Get the size of the buffer allocated to hold the data. This is the "total size" of the buffer
      not the "available" size ("total size" differs from "available" in cases where the initial
      offset into the buffer is non-zero).
      @return the buffer size (bytes)
   */
  public int getBufLength () {
    return buf.length;
  }

  /** Get the buffer allocated to hold the data.
      @return the buffer contents.
   */
  public byte[] getBuf () {
    return buf;
  }

  /** Returns the current contents of all class variables
      @return a String detailing the current values for {@link #size}, {@link #mode}, {@link #type},
      {@link #rep}, {@link #attr}, buf.length, {@link #buf}, {@link #boff}, {@link #bpe},
      {@link #ape}, {@link #spa}, and {@link #bps}.
   */
  @InternalUseOnly("Since NeXtMidas 3.7.1")
  // TODO: better printing of buf when it contains binary data
  public String dump () {
     return "size="+size+" mode="+(char)mode+" type="+(char)type+" rep="+(char)rep+" attr="+attr
          +" (bufLen="+buf.length+") buf=["+ printableBuf() +"] boff=" +boff
          + " bpe="+bpe+" ape="+ape+" spa="+spa+" bps="+bps;
  }

  private StringBuilder printableBuf () {
    StringBuilder sb = new StringBuilder();
    for (byte b : buf) sb.append( (b==0) ? '`' : (b<32 || 126<b) ? '.' : (char)b );
    return sb;
  }

  /** Converts the data to string, ensuring that the text is always padded to end on an 8-byte
      boundary (trailing spaces are added). This is the same as <code>toASCII(0,size,10)</code>.
      @return the Data object converted to a String.
   */
  public String toASCII () { return toASCII(0,size,10); }

  /** Converts the data to string, ensuring that the text is always padded to end on an 8-byte
      boundary (trailing spaces are added in in case they were removed during processing). This
      method is used mainly to read Midas ASCII files, which require each character to be 8 bytes.
      (i.e. "me" would consume 16 bytes (2 chars * 8 bytes/char)).<br>
      <br>
      NOTE: for ASCII, if the text is not on an 8-byte boundary, the text is padded out to end
      on an 8-byte boundary and then the offset is applied.
      NOTE: for ASCII256, if the text is not on an 256-byte boundary, the text is padded out to end
      on an 8-byte boundary and then the offset is applied.
      @param offset   byte offset denoting the start of the string
      @param elements the number of elements to convert to ASCII in the case of a numeric data buffer
      @param radix    the radix to use in the case of a numeric data buffer
      @return the Data object converted to a String.
   */
  public String toASCII (int offset, int elements, int radix) {
    if (!this.isString())  return toString(offset,elements,radix);
    return new String(buf,boff+offset,size*bpe-offset);
  }

  /** Convert to a string, trim trailing spaces from ASCII data
      @return the Data object converted to a String (with trailing spaces removed).
   */
  @Override
  public String toString () { return toString(0,size,10); }

  /** Convert to a string with the given pattern acceptable to
      the {@link nxm.sys.lib.MFormat} class.

    @param pattern Format pattern in Java or Fortran form
    @return the Data object converted to a String with the given format.
    @see nxm.sys.lib.MFormat
    @see java.text.DecimalFormat
  */
  public String toString (String pattern) {
    if (pattern.startsWith("(") && pattern.endsWith(")")) {
      pattern = MFormat.convertFortranPattern(pattern);
    }
    double dval = this.toD();
    MFormat mfmt = MFormat.getNumberFormatFor(pattern);
    if (pattern.equalsIgnoreCase("DMS") ||
        pattern.equalsIgnoreCase("LAT") ||
        pattern.equalsIgnoreCase("LON")) {
      return Convert.deg2dms(dval);
    } else {
      return mfmt.format(dval).trim(); //jph -- should not have to do this
    }
  }

  /** Convert the data to a string. If the type is ASCII, any trailing spaces will be dropped
      consistent with the way strings are represented in a BLUE file. This is identical to using
      <code>toString(null, offset, elements, radix, null, null, null, false)</code>.
      @param offset     Scalar element offset into the buffer (usually 0). This should generally
                        be a multiple of SPA for numeric values, and the character (byte) offset
                        for ASCII values.
      @param elements   Number of elements to convert.
      @param radix      The radix to use (16=Hexadecimal, 10=Decimal, 8=Octal, 2=Binary). This is
                        ignored for ASCII data.
      @return the Data object converted to a String.
   */
  public String toString (int offset, int elements, int radix) {
    return toString(null, offset, elements, radix, null, null, null, false).toString();
  }

  /** Convert the data to a string. If the type is ASCII, any trailing spaces will be dropped
      consistent with the way strings are represented in a BLUE file. <br>
      <br>
      Note: When the radix is 10, this function handles all {@link #PACKED} values as if they were
            "SP" with the bits grouped into bytes separated by the scalar separator (<code>sSep</code>).
      @param str        The {@link StringBuilder} to append the string to allowing chaining. If
                        null a new {@link StringBuilder} will be created.
      @param offset     Scalar element offset into the buffer (usually 0). This should generally
                        be a multiple of SPA for numeric values, and the character (byte) offset
                        for ASCII values.
      @param elements   Number of elements to convert.
      @param radix      The radix to use (16=Hexadecimal, 10=Decimal, 8=Octal, 2=Binary). This is
                        ignored for ASCII data.
      @param parens     Include parenthesis before/after any numeric values? Setting this to null
                        will enable parenthesis any time multiple scalar values are displayed. This
                        is ignored for ASCII data.
      @param sSep       Separator to use between two scalar values of the same numeric element
                        (default is ",").
      @param eSep       Separator to use between two numeric elements (default is ",").
      @param error      Throw exception on error? Setting this to false will cause unsupported radix
                        and unknown conversion errors to be sent to the output rather than causing a
                        {@link MidasException} (this feature is used by the RESULTS command).
      @return The {@link StringBuilder} the content was written to. This will never be null.
      @throws IllegalArgumentException when giving invalid arguments (i.e. when sSep or eSep are empty strings)
      @since NeXtMidas 3.5.4
   */
  public StringBuilder toString (StringBuilder str, int offset, int elements, int radix,
                                 Boolean parens, CharSequence sSep, CharSequence eSep,
                                 boolean error) {
    if (this.isString()) {
      // JC4: Note, we CANNOT quote this value here... doing so is NOT backwards-compatible.
      if (str == null) str = new StringBuilder(size*bpe);
      return str.append(Convert.unpackS(buf,offset,size*bpe));
    }

    int nscalar = elements*ape*spa;
    int i       = offset;
    int n       = i+nscalar;

    if (str    == null) str    = new StringBuilder(nscalar*12); // nscalar*12 is good starting size
    if (parens == null) parens = (nscalar > 1);
    if (sSep   == null) sSep   = ",";
    if (eSep   == null) eSep   = ",";
    // TODO: consider excluding type==PACKED in the two below checks (particularly in eSep, maybe not in sSep since we always set it to "" below)
    if (sSep.length() == 0) throw new IllegalArgumentException("scalar separator cannot be empty");
    if (eSep.length() == 0) throw new IllegalArgumentException("element separator cannot be empty");

    if (parens) str.append('(');

    if (radix==16) {
      str.append(Convert.bb2hex(buf,i*bps,bps)); i++;
      for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(Convert.bb2hex(buf,i*bps,bps));
    }
    else if (radix==8) {
      str.append(Convert.bb2oct(buf,i*bps,bps)); i++;
      for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(Convert.bb2oct(buf,i*bps,bps));
    }
    else if ((radix==1) || (radix==2)) { // 1=Binary is old convention
      str.append(Convert.bb2bin(buf,i*bps,bps)); i++;
      for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(Convert.bb2bin(buf,i*bps,bps));
    }
    else if (radix!=10) {
      if (error) throw new MidasException("Unsupported Radix "+radix);
      return str.append("Unsupported Radix ").append(radix);
    }
    else {
      switch (type) {
        case DOUBLE: str.append(getD(i++)); for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(getD(i)); break;
        case FLOAT:  str.append(getF(i++)); for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(getF(i)); break;
        case LONG:   str.append(getL(i++)); for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(getL(i)); break;
        case INT:    str.append(getI(i++)); for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(getI(i)); break;
        case JINT:   str.append(getJ(i++)); for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(getJ(i)); break;  // NXM-3894 Jeff 12-bit J
        case BYTE:   str.append(getB(i++)); for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(getB(i)); break;
        case XLONG:  str.append(getX(i++)); for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(getX(i)); break;
        case NIBBLE: str.append(getN(i++)); for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(getN(i)); break;
        case PACKED: str.append(getP(i++)); for (; i<n; i++) str.append(((i%8  )==0)? eSep : ""  ).append(getP(i)); break;
        case ULONG:  str.append(getUL(i++)); for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(getUL(i)); break; // minimal unsigned support in 3.9.3
        case UINT:   str.append(getUI(i++)); for (; i<n; i++) str.append(((i%spa)==0)? eSep : sSep).append(getUI(i)); break;
        default:
          if (error) throw new MidasException("Unknown Conversion '"+(char)type+"'");
          return str.append("Unknown Conversion '").append((char)type).append("'");
      }
    }
    if (parens) str.append(')');

    return str;
  }

  /**
   * Convert data in String into this data object without any result translation. This can do the reverse of
   * {@link #toString(StringBuilder, int, int, int, Boolean, CharSequence, CharSequence, boolean)}.
   * @param string  String containing data values to be parsed (cannot be null).
   * @param delims  String data value delimiters (when type is not numeric) (can specify multiple delimiters)
   * @since NeXtMidas 3.5.4
   */
  @InternalUseOnly("provisional API - subject to change in future beta series")
  public void fromString (String string, String delims) {
    if (this.isString()) {
      fromString(string);
      return;
    }
// POSSIBLE CODE FOR FUTURE support of (type === STRING)
//    else if (type == STRING) {
//      byte[] bytes = string.getBytes("UTF-8");
//      int len = bytes.length;
//      setSize(len); // grow buffer is necessary
//      System.arraycopy(bytes,0, buf,0, len);
//    }
    final Object ref = null;
    StringTokenizer st = new StringTokenizer(string, delims); // " ,;"); // allow space, comma, and semi-colon as delimiters
    final int numTokens = st.countTokens();
    final int newSize = numTokens / ape / spa;
    setSize(newSize); // grow buffer is necessary

    int i;
    switch (type) {
      case DOUBLE: for (i=0; st.hasMoreTokens(); i++) setD(i,         Convert.s2d(st.nextToken(),ref));  break;
      case FLOAT:  for (i=0; st.hasMoreTokens(); i++) setF(i, (float) Convert.s2d(st.nextToken(),ref));  break;
      case LONG:   for (i=0; st.hasMoreTokens(); i++) setL(i,         Convert.s2l(st.nextToken(),ref));  break;
      case INT:    for (i=0; st.hasMoreTokens(); i++) setI(i, (short) Convert.s2l(st.nextToken(),ref));  break;
      case BYTE:   for (i=0; st.hasMoreTokens(); i++) setB(i, (byte)  Convert.s2l(st.nextToken(),ref));  break;
      case XLONG:  for (i=0; st.hasMoreTokens(); i++) setX(i,         Convert.s2xl(st.nextToken(),ref)); break;
      case NIBBLE: for (i=0; st.hasMoreTokens(); i++) setN(i, (byte)  Convert.s2l(st.nextToken(),ref));  break;
      case PACKED: for (i=0; st.hasMoreTokens(); i++) setP(i, (byte)  Convert.s2l(st.nextToken(),ref));  break;
      default:
        throw new MidasException("Unknown Conversion from String to '" + (char)type + "' numTokens=" + numTokens);
    }
  }

  /** Returns conversion to an Ascii string value
      @return the Data object converted to an ASCII String value.
   */
  public byte[] toA () {
    if (type==ASCII) return buf;
    else return toString().getBytes();
  }

  /** Returns conversion to a 1-bit integer value
      @return the Data object converted to a 1-bit integer value.
   */
  public byte toP () { return (byte)toD(); }

  /** Returns conversion to a 4-bit integer value
      @return the Data object converted to a 4-bit integer value.
   */
  public byte toN () { return (byte)toD(); }

  /** Returns conversion to a 8-bit integer value
      @return the Data object converted to a 8-bit integer value.
   */
  public byte toB () { return (byte)toD(); }

  /** Returns conversion to a 16-bit integer value
      @return the Data object converted to a 16-bit integer value.
   */
  public short toI () { return (short)toD(); }

  /** Returns conversion to a 32-bit integer value
      @return the Data object converted to a 32-bit integer value.
   */
  public int toL () { return (int)toD(); }

  /** Returns conversion to a 64-bit integer value

      (updated to use toNumber() for 3.5.1 bug 2833)

      @return the Data object converted to a 64-bit integer value.
   */
  public long toX () {
    return toNumber().longValue();
  }

  /** Returns conversion to a float value
      @return the Data object converted to a float value.
   */
  public float toF () { return (float)toD(); }

  /** Returns conversion to a double value
      @return the Data object converted to a double value.
   */
  public double toD () {
         if (type==DOUBLE)   return         unpackD(0);
    else if (type==FLOAT)    return (double)unpackF(0);
    else if (type==LONG)     return (double)unpackL(0);
    else if (type==XLONG)    return (double)unpackX(0);
    else if (type==INT)      return (double)unpackI(0);
    else if (type==JINT)     return (double)getJ(0);         // NXM-3894 Jeff 12-bit J
    else if (type==BYTE)     return (double)unpackB(0);
    else if (type==NIBBLE)   return (double)getN(0);
    else if (type==PACKED)   return (double)getP(0);
    else if (type==ASCII)    return Convert.s2d(toString()); // JC4: Added 16MAR06 (Bug 328)
    else if (type==ASCII256) return Convert.s2d(toString()); // 3.9.4 NXM-3570
    else return 0.0;
  }

  /** Returns conversion to a Number value
      @return the Data object converted to a Number value.
   */
  Number toNumber () {
    if (type==DOUBLE)        return unpackD(0);
    else if (type==FLOAT)    return unpackF(0);
    else if (type==LONG)     return unpackL(0);
    else if (type==XLONG)    return unpackX(0);
    else if (type==INT)      return unpackI(0);
    else if (type==JINT)     return getJ(0);     // NXM-3894 Jeff 12-bit J
    else if (type==BYTE)     return unpackB(0);
    else if (type==NIBBLE)   return getN(0);
    else if (type==PACKED)   return getP(0);
    else if (type==ASCII)    return Convert.s2d(toString()); // JC4: Added 16MAR06 (Bug 328)
    else if (type==ASCII256) return Convert.s2d(toString()); // 3.9.4 NXM-3570
    else return 0.0;
}

  /** Convert a single-element scalar object from one type to another. This is a non-destructive
      method in that the current {@link Data} object is NOT altered. This is the same as
      <code>convertTo(newType,-1,true)</code>.
      <pre>
        NOTE: Prior to NeXtMidas 2.5.1 this method only converted the first
              scalar value and discarded all other values in the buffer.
      </pre>
      @param newType The new type.
      @return <code>this</code> if there is no change in size/type, or a new {@link Data} object.
   */
  public Data convertTo (char newType) {
    return convertTo(newType,-1,true);
  }

  /** Convert a single-element scalar object from one type to another. This is a non-destructive
      method in that the current {@link Data} object is NOT altered. For convenience, this method
      takes in a new size to use when allocating the new buffer, passing -1 will allocate a buffer
      that matches the current size.
      @param newType The new type.
      @param bufSize The size of the new buffer in number of scalar values (-1=UseDefault). The
                     total number of bytes allocated is equal to <code>bufSize * getBPS(newType)</code>.
      @param fast    If <code>true</code> this method is allowed to skip conversion if the new type and
                     size match current values. In this case the method simply returns <code>this</code>.
      @return The converted {@link Data} object.
      @since NeXtMidas 2.5.1
   */
  public Data convertTo (char newType, int bufSize, boolean fast) {
    Data data;
    int  oldSize = size * spa;

    if (bufSize == -1) bufSize = oldSize;

    if (fast && (type == newType) && (oldSize == bufSize)) {
      data = this;
    } else {
      int    toCopy  = Math.min(oldSize, bufSize);
      byte[] buffer  = new byte[bufSize * getBPS(newType)];

      Convert.type(buf, boff, type, buffer, 0, (byte)newType, toCopy, Convert.ROUND_DOWN, true);

      data = new Data(buffer, 0, buffer.length, newType, spa);
      data.setFormatMode(mode);
    }
    return data;
  }

  /** Converts this data object to a new data object. This is only used by Convert.o2o(..) and
      Convert.s2o(..).
   */
  Data convertTo (int newSize, int newSPA, char newMode, char newType) { // package-private
    return convertTo(newSize, newSPA, (byte)newMode, (byte)newType);
  }

  /** Converts this data object to a new data object. This is only used by Convert.o2o(..) and
      Convert.s2o(..), via convertTo(int,int,char,char).
      It does NOT handle ASCII to ASCII256 type conversions, as that is not needed by the calling methods.
      @param newSize A non-negative size, or else the magic value <pre>-1</pre>.
      @param newSPA  A valid SPA value, or else the magic value <pre>-1</pre>.
      @param newMode An atomic mode (<pre>(byte)'S'</pre> for scalar, <pre>(byte)'C'</pre> for complex, etc.), or else the magic value <pre>(byte)' '</pre>.
      @param newType An atomic type (<pre>(byte)'B'</pre> for byte, <pre>(byte)'S'</pre> for short, <pre>(byte)'D'</pre> for double, etc.), or else the magic value <pre>(byte)'_'</pre>.
      @return        Converted object
      @since NeXtMidas 3.7.1
   */
  private Data convertTo (int newSize, int newSPA, byte newMode, byte newType) {
    Data data;
    Integer newSpaFromNewMode = nullOrSpaFromMode(newMode);
    Integer newSpaFromNewSpa = nullOrSpaFromSpa(newSPA);
    Byte newTypeFromNewType = nullOrTypeFromType(newType);
    Integer newSizeFromNewSize = nullOrSizeFromSize(newSize);

    // ensure the values of newMode and newSPA are legal and consistent
    if (null==newSpaFromNewMode && null==newSpaFromNewSpa) {
      newMode = mode;
      newSPA = spa;
    }
    else if (null!=newSpaFromNewMode && null!=newSpaFromNewSpa && newSpaFromNewMode!=newSpaFromNewSpa) {
      throw new IllegalArgumentException("Data: "+newSPA+" is not the SPA for mode '"+(char)newMode+"'");
    }
    else if (null==newSpaFromNewMode) {
      newMode = modeFromSpa(newSPA);
    }
    else if (null==newSpaFromNewSpa) {
      newSPA = newSpaFromNewMode;
    }

    // ensure the value of newType is legal
    if (null==newTypeFromNewType) {
      newType = type;
    }
    // Note: ASCII to ASCII256 could be handled, but it is unnecessary given the usage of this method
    if (newType!=type && (Data.isString(newType) || Data.isString(type) || newType==(byte)'O' || type==(byte)'O')) {
      throw new IllegalArgumentException("Data: Unable to convert from type '"+(char)type+"' to type '"+(char)newType+"'");
    }

    // ensure the pairing of newMode and newType is legal
    checkFormatPairing(newMode, newType);

    // ensure that newSize has an actual value
    if (null==newSizeFromNewSize) {
      newSize = (size*spa + newSPA-1) / newSPA;
    }

    /*
     * This fix was added during the release testing for NeXtMidas 3.7.1 The error was
     * in removing records from file type 2000. The ConvertTo function was getting
     * call incorrectly. Bellow change was originally on NextMidas 3.7.0 but somehow during
     * the NXM-2867 bug fix and error was introduced.
     */
    if ((size == newSize) && (spa == newSPA) && (mode == newMode) && (type == newType)) {
      data = this;
    }
    else {
      data = convertTo(newMode, newType, newSize);
    }
    return data;
  }

  /** Convert this object into a new Data object. This is only used by convertTo(int,int,byte,byte).
      @param newMode The atomic mode (<pre>(byte)'S'</pre> for scalar, <pre>(byte)'C'</pre> for complex, etc.).
      @param newType The atomic type (<pre>(byte)'B'</pre> for byte, <pre>(byte)'S'</pre> for short, <pre>(byte)'D'</pre> for double, etc.)
      @param newSize The new size in elements.
      @since NeXtMidas 3.7.1
   */
  private Data convertTo (byte newMode, byte newType, int newSize) {
    Data data = new Data(newMode, newType, newSize);
    int numScalars = Math.min(numScalars(), data.numScalars());
    Convert.type(buf, boff, type, data.buf, 0, newType, numScalars, Convert.ROUND_DOWN, true);
    return data;
  }

  /** Calculate the total number of scalars (byte, short, double, etc.) held as data by this object.
      @since NeXtMidas 3.7.1
   */
  private int numScalars () {
    return spa*ape*size;
  }

  /** Calculate the total number of bytes of data held by this object.  The returned value will always be less than
      the value of <pre>this.buf.length</pre> by at least the value of <pre>this.boff</pre>.
      @since NeXtMidas 3.7.1
   */
  private int numBytes () {
    return (bpe>0) ? bpe*size : (-bpe*size+7)/8;
  }

  /** Calculate the total number of bytes required to hold numElements of data, given the current value of <pre>this.bpe</pre>.
      @param numElements Any non-negative count of elements
      @since NeXtMidas 3.7.1
   */
  private int numBytes (int numElements) {
    return (bpe>0) ? bpe*numElements : (-bpe*numElements+7)/8;
  }

  /** Perform a safe byte-level copy from <pre>src.buf</pre> to <pre>this.buf</pre>, given the current settings for <pre>src.boff</pre> and <pre>this.boff</pre>.
      If <pre>src.size</pre> is shorter than <pre>numElements</pre>, a zero-fill is performed to ensure that <pre>(src.bpe*numElements)</pre> bytes
      are written to <pre>this.buf</pre>.  If <pre>this.buf</pre> is less than <pre>(src.bpe*numElements)</pre> bytes long, it will be replaced.
      If any bookkeeping changes are required to maintain internal consistency among the fields of this object,
      those changes must be performed separately.
      @param src         Another Data to be used as a source of elements
      @param numElements Any non-negative count of elements to write into <pre>this.buf</pre>
      @since NeXtMidas 3.7.1
   */
  private void fillBufferFrom (Data src, int numElements) {
    int numBytesToWrite = src.numBytes(numElements);
    int numBytesOfData = Math.min(src.numBytes(), numBytesToWrite); // src may be shorter than numElements, in which case a zero fill will be necessary
    int numBytesOfZeroes = numBytesToWrite - numBytesOfData;
    if (boff+numBytesToWrite > buf.length) { // allocate more memory if buffer not big enough
      buf = new byte[boff+numBytesToWrite];
      numBytesOfZeroes = 0; // new arrays are already zero-filled
    }
    System.arraycopy(src.buf, src.boff, buf, boff, numBytesOfData);
    if (numBytesOfZeroes>0) {
      Arrays.fill(buf, boff+numBytesOfData, boff+numBytesToWrite, (byte)0);
    }
  }

  /** Given a valid mode, return the number of scalars per atom (SPA) for that mode.  A mode is valid if it matches one of the characters in <pre>Data.DATA_MODES</pre>.
      Given the magic value <pre>(byte)' '</pre>, which indicates "use the default", return null.
      Otherwise, throw an IllegalArgumentException.
      @param  mode An atomic mode (<pre>(byte)'S'</pre> for scalar, <pre>(byte)'C'</pre> for complex, etc.), or else the magic value <pre>(byte)' '</pre>.
      @return The SPA for the given mode, or else null.
      @throws IllegalArgumentException if the given mode is neither a valid mode nor the magic value <pre>(byte)' '</pre>.
      @since NeXtMidas 3.7.1
   */
  private static Integer nullOrSpaFromMode (byte mode) { return ((byte)' '==mode) ? null : spaFromMode(mode); }

  /** Given a valid scalars per atom (SPA) value, return that value.  A SPA is valid if it matches the SPA corresponding with one of the characters in <pre>Data.DATA_MODES</pre>.
      Given the magic value <pre>-1</pre>, which indicates "use the default", return null.
      Otherwise, throw an IllegalArgumentException.
      @param  spa A valid SPA value, or else the magic value <pre>-1</pre>.
      @return The given SPA value, or else null.
      @throws IllegalArgumentException if the given SPA is neither a valid SPA nor the magic value <pre>-1</pre>.
      @since NeXtMidas 3.7.1
   */
  private static Integer nullOrSpaFromSpa (int spa) { return (-1==spa) ? null : spaFromSpa(spa); }

  /** Given a valid type, return that type.  A type is valid if it matches one of the characters in <pre>Data.DATA_TYPES</pre>.
      Given the magic value <pre>(byte)'_'</pre>, which indicates "use the default", return null.
      Otherwise, throw an IllegalArgumentException.
      @param  type An atomic type (<pre>(byte)'B'</pre> for byte, <pre>(byte)'S'</pre> for short, <pre>(byte)'D'</pre> for double, etc.), or else the magic value <pre>(byte)'_'</pre>.
      @return The given type value, or else null.
      @throws IllegalArgumentException if the given type is neither a valid type nor the magic value <pre>(byte)'_'</pre>.
      @since NeXtMidas 3.7.1
   */
  private static Byte nullOrTypeFromType (byte type) { return ((byte)'_'==type) ? null : typeFromType(type); }

  /** Given a valid size, return that size.  A size is valid if it is non-negative.
      Given the magic value <pre>-1</pre>, which indicates "use the default", return null.
      Otherwise, throw an IllegalArgumentException.
      @param  size A non-negative size, or else the magic value <pre>-1</pre>.
      @return The given size, or else null.
      @throws IllegalArgumentException if the given size is neither a valid size nor the magic value <pre>-1</pre>.
      @since NeXtMidas 3.7.1
   */
  private static Integer nullOrSizeFromSize (int size) { return (-1==size) ? null : sizeFromSize(size); }

  /** Utility function for converting a given mode and type into a standard NeXtMidas <pre>format</pre> String.
      No validity checking is performed.
      @param  mode An atomic mode
      @param  type An atomic type
      @return a two-character <pre>format</pre> String
      @since NeXtMidas 3.7.1
   */
  private static String getFormat (byte mode, byte type) {
    return "" + (char)mode + (char)type;
  }

  /** Verify that mode <pre>(byte)'N'</pre> and type <pre>(byte)'H'</pre> are only ever paired with each other.
      Verify that mode <pre>(byte)'K'</pre> and type <pre>(byte)'W'</pre> are only ever paired with each other.
      No other validity checking is performed.
      @param  mode An atomic mode
      @param  type An atomic type
      @throws IllegalArgumentException if the given mode is not a valid mode.
      @since NeXtMidas 3.7.1
   */
  private static void checkFormatPairing (byte mode, byte type) {
    if (
      (mode==(byte)'N' && type!=(byte)'H') ||
      (type==(byte)'H' && mode!=(byte)'N') ||
      (mode==(byte)'K' && type!=(byte)'W') ||
      (type==(byte)'W' && mode!=(byte)'K')
    ) throw new IllegalArgumentException("Data: \""+(char)mode+(char)type+"\" is not a valid FORMAT");
  }

  /** Check the validity of a given mode value.  A mode is valid if it matches one of the characters in <pre>Data.DATA_MODES</pre>.
      @param  mode An atomic mode (<pre>(byte)'S'</pre> for scalar, <pre>(byte)'C'</pre> for complex, etc.).
      @return The given mode value.
      @throws IllegalArgumentException if the given mode is not a valid mode.
      @since NeXtMidas 3.7.1
   */
  private static byte modeFromMode (byte mode) {
    if (DATA_MODES.indexOf((char)mode) >= 0) {
      return mode;
    }
    throw new IllegalArgumentException("Data: '"+(char)mode+"' is not a valid MODE");
  }

  /** Check the validity of a given type value.  A type is valid if it matches one of the characters in <pre>Data.DATA_TYPES</pre>.
      @param  type An atomic type (<pre>(byte)'B'</pre> for byte, <pre>(byte)'S'</pre> for short, <pre>(byte)'D'</pre> for double, etc.).
      @return The given type value.
      @throws IllegalArgumentException if the given type is not a valid type.
      @since NeXtMidas 3.7.1
   */
  private static byte typeFromType (byte type) {
    if (DATA_TYPES.indexOf((char)type) >= 0) {
      return type;
    }
    else if ((Shell.getMidasContext().io.isOptionSet(CoreIO.IOOptions.MinimalSupportForUnsupportedDataTypes)) && EXTENDED_DATA_TYPES.indexOf((char)type) >= 0) {
      // added to generate test fields with unsigned types 'U' and "V' - adding limited support in 3.9.3
      return type;
    }
    throw new IllegalArgumentException("Data: '"+(char)type+"' is not a valid TYPE");
  }

  /** Check the validity of a given scalars per atom (SPA) value.
      A SPA is valid if it matches the SPA corresponding with one of the characters in <pre>Data.DATA_MODES</pre>.
      @param  spa A valid SPA value.
      @return The given SPA value.
      @throws IllegalArgumentException if the given SPA is not a valid SPA.
      @since NeXtMidas 3.7.1
   */
  private static int spaFromSpa (int spa) {
    if (
      (0<spa && spa<11) // matches mode = '1' to '9' or 'S', 'C', 'V', 'Q', 'M', 'X'
      || spa==16 // matches mode = 'T'
      || spa==32 // matches mode = 'A' (since 3.1.0 to match X-Midas 4.6.0+)
      ) {
      return spa;
    }
    throw new IllegalArgumentException("Data: "+spa+" is not a valid SPA");
  }

  /** Check the validity of a given size.  A size is valid if it is non-negative.
      @param  size A non-negative size.
      @return The given size.
      @throws IllegalArgumentException if the given size is not a valid size.
      @since NeXtMidas 3.7.1
   */
  private static int sizeFromSize (int size) {
    if (-1<size) {
      return size;
    }
    throw new IllegalArgumentException("Data: "+size+" is not a valid SIZE");
  }

  /** Given a valid mode, return the number of scalars per atom (SPA) for that mode.
      A mode is valid if it matches one of the characters in <pre>Data.DATA_MODES</pre>.
      @param  mode An atomic mode (<pre>(byte)'S'</pre> for scalar, <pre>(byte)'C'</pre> for complex, etc.).
      @return The SPA for the given mode.
      @throws IllegalArgumentException if the given mode is not a valid mode.
      @since NeXtMidas 3.7.1
   */
  @SuppressWarnings("fallthrough")
  private static int spaFromMode (byte mode) {
    switch ((char)mode) {
      case SCALAR:
      case 'N':            // special case for 'NH' format
      case 'K':            // special case for 'KW' format
      case '1': return 1;
      case COMPLEX:
      case '2': return 2;
      case VECTOR:
      case '3': return 3;
      case QUAD:
      case '4': return 4;
      case '5': return 5;
      case '6': return 6;
      case '7': return 7;
      case '8': return 8;
      case MATRIX:
      case '9': return 9;
      case XMODE: return 10;
      case TRANSMATRIX: return 16;
      case AMODE: return 32; // since 3.1.0 to match X-Midas 4.6.0+
    }
    throw new IllegalArgumentException("Data: '"+(char)mode+"' is not a valid MODE");
  }

  /** Given a valid scalars per atom (SPA) value, return a mode which corresponds with that value.
      A SPA is valid if it matches the SPA corresponding with one of the characters in <pre>Data.DATA_MODES</pre>.
      @param  spa A valid SPA value.
      @return A mode which corresponds with the given SPA.
      @throws IllegalArgumentException if the given SPA is not a valid SPA.
      @since NeXtMidas 3.7.1
   */
  private static byte modeFromSpa (int spa) {
    switch (spa) {
      case  1: return SCALAR;
      case  2: return COMPLEX;
      case  3: return VECTOR;
      case  4: return QUAD;
      case  5: return '5';
      case  6: return '6';
      case  7: return '7';
      case  8: return '8';
      case  9: return MATRIX;
      case 10: return XMODE;
      case 16: return TRANSMATRIX;
      case 32: return AMODE; // since 3.1.0 to match X-Midas 4.6.0+
    }
    throw new IllegalArgumentException("Data: "+spa+" is not a valid SPA");
  }

  /** Given a valid type value, return the number of bytes per scalar (BPS) which corresponds with that value.
      A type is valid if it matches one of the characters in <pre>Data.DATA_TYPES</pre>.
      For scalars smaller than a byte, such as nibbles, the number of bits per scalar is returned instead.
      This change in units is flagged by changing the sign of the return value to negative.
      @param  type An atomic type (<pre>(byte)'B'</pre> for byte, <pre>(byte)'S'</pre> for short, <pre>(byte)'D'</pre> for double, etc.).
      @return The BPS for the given type.
      @throws IllegalArgumentException if the given type is not a valid type.
      @since NeXtMidas 3.7.1
   */
  @SuppressWarnings("fallthrough")
  private static int bpsFromType (byte type) {
    switch ((char)type) {
      case ASCII:
      case DOUBLE:
      case XLONG:  return  8; // bytes
      case FLOAT:
      case LONG:   return  4; // bytes
      case UINT:   return  2; // bytes
      case ULONG:  return  4; // bytes
      case INT:    return  2; // bytes
      case JINT:   return  -12; // bits   NXM-3894 Jeff 12-bit J
      case BYTE:
      case 'H':            // special case for 'NH' format
      case 'W':            // special case for 'KW' format
      case 'O':            // special case for XMValue.INT_U1
      case 'M':    return  1; // byte
      case NIBBLE: return -4; // bits
      case PACKED: return -1; // bit
      case ASCII256: return 256; // NXM-3570
    }
    throw new IllegalArgumentException("Data: '"+(char)type+"' is not a valid TYPE");
  }

  /** @deprecated Since NeXtMidas 2.5.5: Use {@link #getP(int)}, since this method DOES NOT do documented functionality.
      @param off Element offset into the byte buffer (0..n-1)
      @return Java byte.
   */
  @Deprecated public byte unpackP (int off) { return Convert.unpackP(buf,boff*8+off,rep); }

  /** @deprecated Since NeXtMidas 2.5.5: Use {@link #getN(int)}, since this method DOES NOT do documented functionality.
      @param off Element offset into the byte buffer (0..n-1)
      @return Java byte
   */
  @Deprecated public byte unpackN (int off) { return Convert.unpackN(buf,boff*4+off,rep); }

  /** Unpack a BYTE (Java byte) from data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @return Java byte
   */
  public byte unpackB (int off) { return buf[boff+off]; }

  /** Unpack an ASCII as Java char from data buffer at given byte offset.
      <b>WARNING this will choke if you use UNICODE characters.</b>
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @return Java char
   */
  public char unpackC (int off) { return (char)buf[boff+off]; }

  /** Unpack an INT (Java short) from data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @return Java int
   */
  public short unpackI (int off) { return Convert.unpackI(buf,boff+off); }

  /** Unpack a DOUBLE (Java double) from data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @return Java double
   */
  public int unpackL (int off) { return Convert.unpackL(buf,boff+off); }

  /** Unpack an XLONG (Java long) from data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @return Java long
   */
  public long unpackX (int off) { return Convert.unpackX(buf,boff+off); }

  /** Unpack a FLOAT (Java float) from data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @return Java float
   */
  public float unpackF (int off) { return Convert.unpackF(buf,boff+off); }

  /** Unpack a DOUBLE (Java double) from data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @return Java double
   */
  public double unpackD (int off) { return Convert.unpackD(buf,boff+off); }

  /** Unpack an ASCII String as a Java String from data buffer at given byte offset.
      <b>WARNING this will choke if you use UNICODE characters.</b>
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @param len Number of bytes to read into the String (same as number of ASCII characters)
      @return Java String
   */
  public String unpackS (int off, int len) { return Convert.unpackS(buf,boff+off,len); }

  /** @deprecated Since NeXtMidas 2.5.5: Use {@link #setP(int, byte)}, since this method DOES NOT do documented functionality.
      @param off  Byte offset into the byte buffer (0..n-1)
      @param val  value to be packed.
   */
  @Deprecated public void packP (int off, byte val) { Convert.packP(buf,boff*8+off,val,rep); } // <-- using element offset vs byte offset

  /** @deprecated Since NeXtMidas 2.5.5: Use {@link #setN(int, byte)}, since this method DOES NOT do documented functionality.
      @param off  Byte offset into the byte buffer (0..n-1)
      @param val  value to be packed.
   */
  @Deprecated public void packN (int off, byte val) { Convert.packN(buf,boff*4+off,val,rep); } // <-- using (2-bit element)? offset vs byte offset

  /** Pack a BYTE (Java byte) into data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void packB (int off, byte val) { buf[boff+off] = val; }

  /** Pack an ASCII as Java char into data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void packC (int off, char val) { buf[boff+off] = (byte)val; }

  /** Pack an INT (Java short) into data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void packI (int off, short val) { Convert.packI(buf,boff+off,val); }

  /** Pack a LONG (Java int) into data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void packL (int off, int val) { Convert.packL(buf,boff+off,val); }

  /** Pack an XLONG (Java long) into data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void packX (int off, long val) { Convert.packX(buf,boff+off,val); }

  /** Pack a FLOAT (Java float) into data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void packF (int off, float val) { Convert.packF(buf,boff+off,val); }

  /** Pack a DOUBLE (Java double) into data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void packD (int off, double  val) { Convert.packD(buf,boff+off,val); }

  /** Pack an ASCII String as a Java String into data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @param len Number of bytes to write from the String (same as number of ASCII characters)
      @param val Value to be packed
   */
  // NTN: If we remove this API, then classes compiled using previous version of NeXtMidas will get NoSuchMethodError
  public void packS (int off, int len, String val) { Convert.packS(buf,boff+off,len,val); }

  /** Pack an ASCII String as a Java String into data buffer at given byte offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into the byte buffer (0..n-1)
      @param len Number of bytes to write from the String (same as number of ASCII characters)
      @param val Value to be packed
   */
  public void packS (int off, int len, CharSequence val) { Convert.packS(buf,boff+off,len,val); }

  /** Pack bytes from a Data buffer into this data buffer.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Byte offset into <b>this</b> byte buffer (0..n-1)
      @param val Value(s) to be packed
   */
  public void packData (int off, Data val) { System.arraycopy(val.buf,val.boff, buf,boff+off, val.bpe); }

  /** Unpack a PACKED bit as a Java byte from data buffer at given bit offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return Java byte
   */
  public byte getP (int off) { return Convert.unpackP(buf,boff*8+off,rep); }

  /** Unpack a packed NIBBLE as a Java byte from data buffer at given nibble offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return Java byte
   */
  public byte getN (int off) { return Convert.unpackN(buf,boff*2+off,rep); }

  /** Unpack a BYTE (Java byte) from data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return Java byte
   */
  public byte getB (int off) { return buf[boff+off]; }

  /** Unpack an ASCII as Java char from data buffer at given element offset.
      <b>WARNING this will choke if you use UNICODE characters.</b>
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return Java char
   */
  public char getC (int off) { return (char)buf[boff+off]; }

  /** Unpack an JINT (Java short) from data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return java short
      @since 4.1.2 Jeff contribution for NXM-3894 12-bit J
   */
  public short getJ (int off) { return Convert.unpackJ(buf,(boff*2/3)+off); }

  /** Unpack an INT (Java short) from data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return java short
   */
  public short getI (int off) { return Convert.unpackI(buf,boff+(off<<1)); }

  /** Unpack an unsigned INT (Java short) from data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return java int
      @since 3.9.3
  */
  @InternalUseOnly
  public int getUI (int off) { return Convert.unpackU(buf,boff+(off<<1)); }

  /** Unpack a LONG (Java int) from data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return Java double
   */
  public int getL (int off) { return Convert.unpackL(buf,boff+(off<<2)); }

  /** Unpack an unsigned LONG (Java int) from data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return Java long
   */
  public long getUL (int off) { return Convert.unpackUL(buf,boff+(off<<2)); }

  /** Unpack an XLONG (Java long) from data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return Java long
   */
  public long getX (int off) { return Convert.unpackX(buf,boff+(off<<3)); }

  /** Unpack a FLOAT (Java float) from data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return Java float
   */
  public float getF (int off) { return Convert.unpackF(buf,boff+(off<<2)); }

  /** Unpack a DOUBLE (Java double) from data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @return java double
   */
  public double getD (int off) { return Convert.unpackD(buf,boff+(off<<3)); }

  /** Unpack a ASCII (Fortran String) from data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1).
                 (This is multiples of 8-character blocks)
      @param len Number of ASCII characters to read in (should be a multiple of 8)
      @return ASCII String.
   */
  public byte[] getA (int off, int len) { return Convert.unpackA(buf,boff+(off<<3),len); }

  /** Unpack a ASCII (Fortran String) from data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1).
                 (This is multiples of 256-character blocks)
      @param len Number of ASCII characters to read in (should be a multiple of 8)
      @return ASCII String.
      @since NeXtMidas 3.9.4 for NXM-3570
  */
  public byte[] getZ (int off, int len) { return Convert.unpackZ(buf,boff+(off<<3),len); }

  /** Pack a PACKET bit into data buffer at given bit offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void setP (int off, byte val) { Convert.packP(buf,boff*8+off,val,rep); }

  /** Pack a packet NIBBLE into data buffer at given nibble offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void setN (int off, byte val) { Convert.packN(buf,boff*2+off,val,rep); }

  /** Pack a BYTE (Java byte) into data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void setB (int off, byte val) { buf[boff+off] = val; }

  /** Pack an ASCII as Java char into data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void setC (int off, char val) { buf[boff+off] = (byte)val; }

  /** Pack an INT (Java short) into data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void setI (int off, short val) { Convert.packI(buf,boff+(off<<1),val); }

  /** Pack a LONG (Java int) into data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void setL (int off, int val) { Convert.packL(buf,boff+(off<<2),val); }

  /** Pack an XLONG (Java long) into data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void setX (int off, long val) { Convert.packX(buf,boff+(off<<3),val); }

  /** Pack a FLOAT (Java float) into data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void setF (int off, float val) { Convert.packF(buf,boff+(off<<2),val); }

  /** Pack a DOUBLE (Java double) into data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1)
      @param val Value to be packed
   */
  public void setD (int off, double  val) { Convert.packD(buf,boff+(off<<3),val); }

  /** Pack a ASCII (Fortran String) into data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1).
                 (This is multiples of 8-character blocks)
      @param len Number of ASCII characters to write out (should be a multiple of 8)
      @param val Value to be packed
   */
  public void setA (int off, int len, byte[] val) { Convert.packA(buf,boff+(off<<3),len,val); }

  /** Pack a ASCII (Fortran String) into data buffer at given element offset.
      See <a href="#AccessingValues">Accessing Values</a> for details.
      @param off Element offset into the byte buffer (0..n-1).
                 (This is multiples of 256-character blocks)
      @param len Number of ASCII characters to write out (should be a multiple of 256)
      @param val Value to be packed
      @since NeXtMidas 3.9.4 for NXM-3570
  */
  public void setZ (int off, int len, byte[] val) { Convert.packZ(buf,boff+(off<<3),len,val); }

  /** Gets the i'th scalar value as a number. <br>
      <br>
      <b>Note:</b> This function is less efficient than directly calling the normal get functions
      (e.g. {@link #getL}) and should be reserved for situations where performance is not critical.
      @param off Element offset into the buffer (0..n-1)
      @return The number unpacked or null if the buffer is holding a non-numeric
              array (e.g. a string or a non-homogenous buffer).
      @since NeXtMidas 3.3.0
   */
  public Number getNumber (int off) {
    switch (type) {
      case DOUBLE: return getD(off);
      case FLOAT:  return getF(off);
      case XLONG:  return getX(off);
      case LONG:   return getL(off);
      case INT:    return getI(off);
      case BYTE:   return getB(off);
      case NIBBLE: return getN(off);
      case PACKED: return getP(off);
      default:     return null;
    }
  }

  /** Sets the i'th scalar value as a number. Note that the value given will be
      converted to the value matching the buffer type using the applicable
      function in {@link Number} (e.g. <code>val.intValue()</code> for a "SL" buffer)
      which may incur a loss of precision and/or truncation. <br>
      <br>
      <b>Note:</b> This function is less efficient than directly calling the normal set functions
      (e.g. {@link #setL}) and should be reserved for situations where performance is not critical.
      @param off Element offset into the buffer (0..n-1)
      @param val The numeric value to set.
      @throws NullPointerException If val is null.
      @throws UnsupportedOperationException If the buffer is holding a
              non-numeric array (e.g. a string or a non-homogenous buffer).
      @since NeXtMidas 3.3.0
   */
  public void setNumber (int off, Number val) {
    if (val == null) {
      throw new NullPointerException("Expected a number but given null.");
    }
    switch (type) {
      case DOUBLE: setD(off, val.doubleValue()); break;
      case FLOAT:  setF(off, val.floatValue() ); break;
      case XLONG:  setX(off, val.longValue()  ); break;
      case LONG:   setL(off, val.intValue()   ); break;
      case INT:    setI(off, val.shortValue() ); break;
      case BYTE:   setB(off, val.byteValue()  ); break;
      case NIBBLE: setN(off, val.byteValue()  ); break;
      case PACKED: setP(off, val.byteValue()  ); break;
      default:     throw new UnsupportedOperationException("Can not set a numeric"
                                            + " value in a "+getFormat()+" buffer");
    }
  }

  /** allocate a Java working array */
  private void prepArray (byte ntype) {
    int n = size*ape*spa;
    if (ntype==0) return;
    if (ntype==atype && n<=java.lang.reflect.Array.getLength(array)) return;
    atype = ntype;
    switch (atype) {
      case PACKED: array = new byte[n/8]; break;
      case ULAW:   array = new byte[n]; break;
      case BYTE:   array = new byte[n]; break;
      case INT:    array = new short[n]; break;
      case LONG:   array = new int[n]; break;
      case XLONG:  array = new long[n]; break;
      case FLOAT:  array = new float[n]; break;
      case DOUBLE: array = new double[n]; break;
    }
  }

  /** Determines if the specified type is a floating point data type (i.e. 'F' or 'D').
      @since NeXtMidas 3.1.0
      @param type the type to be tested.
      @return true if the type is a floating point data type, false otherwise.
   */
  public static boolean isFloat (byte type) {
    return (type == FLOAT) || (type == DOUBLE);
  }

  /** Promote Midas format digraphs, returning the "greatest" of the input digraphs.
      This is similar to the M$PROMOTE_FORMAT function.<br>
      <pre>
  MODE HIERARCHY:  S - Scalar              &lt;-- lowest
                   C - Complex
                   V - Vector
                   Q - Quad
                   M - Matrix (3x3)
                   X - Ten
                   T - TransMatrix (4x4)
                   A - 32                  &lt;-- highest
       </pre>
      See {@link #promoteType(char, char, boolean, boolean)} for the TYPE HIERARCHY
      promotion rules.
      @param form1 format digraph 1 (required)
      @param form2 format digraph 2 (required)
      @param form3 format digraph 3 (optional)
      @see #promoteType(char, char, boolean, boolean)
      @see #getSPA(char)
      @throws NullPointerException if form1, form2 or form3 is null
      @throws StringIndexOutOfBoundsException if form1, form2 or form3 length is NOT 2+
      @since NeXtMidas 3.1.0
      @return the greatest of the input digraphs.
   */
  public static String promoteFormat (String form1, String form2, String form3) {
    char mode1 = form1.charAt(0);
    char mode2 = form2.charAt(0);
    char mode  = (getSPA(mode1) >= getSPA(mode2))? mode1 : mode2; // promote mode
    char type  = promoteType(form1.charAt(1), form2.charAt(1));   // promote type

    // promote mode/type against optional form3 parameter
    if (form3 != null && form3.length()>0) {
      char mode3 = form3.charAt(0);
      mode = (getSPA(mode) >= getSPA(mode3))? mode : mode3;
      type = promoteType(type, form3.charAt(1));
    }

    return "" + mode + type;
  }

  /** Promote type with no format override. This is the same as
      <code>(byte)promoteType((char)t1,(char)t2,false,false)</code>.
      @param t1 Format type
      @param t2 Second Format type
      @return A format type character
   */
  public static byte promoteType (byte t1, byte t2) {
    char ctype = promoteType((char)t1,(char)t2,false,false);
    return (byte) ctype;
  }

  /** Promote type with no format override. This is the same as <code>promoteType(t1,t2,false,false)</code>.
      @param t1 Format type
      @param t2 Second Format type
      @return A format type character
   */
  public static char promoteType (char t1, char t2) {
    return promoteType(t1,t2,false,false);
  }

  /** Promote type with no format override for three input types.
      Similar to M$PROMOTE_TYPE3 function.
      @see #promoteType(char, char, boolean, boolean)
      @since NeXtMidasd 3.1.0
      @param t1 Format type
      @param t2 Second Format type
      @param t3 Third Format type
      @return A format type character
   */
  public static char promoteType (char t1, char t2, char t3) {
    char t = promoteType(t1, t2, false, false);
    return promoteType(t, t3, false, false);
  }

  /** Determine the "greatest" of two Midas format types. The following hierarchy is used:
      <pre>{@code
          D - Double (64-bit float) &lt;-- Highest
          F - Float (32-bit float)
          X - XLong (64-bit integer)
          L - Long  (32-bit integer)
          I - Int   (16-bit integer)
          B - Byte    <-- Lowest returned when t1 != t2
          N - Nibble
          P - Packed  <-- Lowest
      }</pre>
      Because a Long has more mantissa precision than a float, one may desire to
      promote a Long/Float combination into a double. The formatOverride
      parameter may be used to return a double ('D') for L/F or F/L combinations. <br>
      <br>
      The double precision flag may be used to ALWAYS return a double precision
      format type in the event that the two types don't match.<br>
      <br>
      Note: Not able to promote 'A'=ASCII to any other types, or vice-versa but
      for the case where both t1=t2='A', then return 'A'.
      <br>
      This is similar to the M$PROMOTE_TYPE function.<br>
      Since NeXtMidas 3.1.0, types are promoted along two axes: precision and range (integer versus float).
      This guarantees that both are preserved as much as is possible.  For
      example, given 'F' and 'X', the 'F' causes promotion to a floating point
      type, while the 'X' causes promotion to an 8-byte size, resulting in an
      out type of 'D'.  For other combinations, the promotion is, in effect,
      linear ("BILFD", in ascending order) as with prior releases.
      <pre>{@code
                 precision
            +------------------>
       int  | B -> I -> L -> X
            |           |    |
            |           v    v
      float |           F -> D
            v
      }</pre>
      @param t1 Format type
      @param t2 Second Format type
      @param formatOverride Override hierarchy (for L/F to type D promotion)
      @param doublePrec Always return double precision
      @return A format type character
      @throws MidasException If attempting to convert between numeric and ASCII.
   */
  public static char promoteType (char t1, char t2,
                                  boolean formatOverride,
                                  boolean doublePrec) throws MidasException {
    char type = DOUBLE;
    if ( t1 == t2 ) {
      // if the two input types are the same, then no promotion is needed
      type = t1;
    } else if (Data.isString((byte)t1) || Data.isString((byte)t2)) { // not supporting promotion from A to Z at this time
      throw new MidasException ("promoteType: non-numeric format type cannot be promoted");
    } else if (!doublePrec) {
      // Output type is floating point if any of the inputs are floats; size is
      // the largest of all input sizes.
      boolean fp = isFloat((byte)t1) || isFloat((byte)t2);
      int   size = Math.max(getBPS(t1), getBPS(t2));

      // Map back to a type code from the size and int/float type.
      switch (size) {
        case 8: type = (char)(fp? DOUBLE : XLONG); break;
        case 4: type = (char)(fp? (formatOverride? DOUBLE:FLOAT) : LONG); break;
        case 2: type = (char)INT; break;
        case 1: // fall-through
        default:
                type = (char)BYTE; break;
      }
    }
    return type;
  }

  /** Converts an object to a given radix. If the obj is not a number or the radix is unset
      (radix=-1) the orig object is returned.
      @param radix  The radix to use (16=Hexadecimal, 10=Decimal, 8=Octal, 2=Binary). This is ignored for ASCII data.
      @param obj    The object to convert to a given radix
      @return either object itself if radix=-1 or String version of object with specified radix
   */
  // 3.9.1 - moved to lib from temporary intr.Results location
  @InternalUseOnly
  public static Object toRadix (int radix, Object obj) {
  if (radix != -1) {
         if (obj instanceof double[])  obj = new Data(obj);
    else if (obj instanceof float[])   obj = new Data(obj);
    else if (obj instanceof long[])    obj = new Data(obj);
    else if (obj instanceof int[])     obj = new Data(obj);
    else if (obj instanceof short[])   obj = new Data(obj);
    else if (obj instanceof byte[])    obj = new Data(obj);
    else if (obj instanceof Double)    obj = new Data(obj);
    else if (obj instanceof Float)     obj = new Data(obj);
    else if (obj instanceof Long)      obj = new Data(obj);
    else if (obj instanceof Integer)   obj = new Data(obj);
    else if (obj instanceof Short)     obj = new Data(obj);
    else if (obj instanceof Byte)      obj = new Data(obj);

    if (obj instanceof Data) {
      obj = ((Data)obj).toString(0,1,radix);
    }
  }
  return obj;
}

  /** Gets the multiplier for non-data instances.
      @param object The object.
      @return The multiplier.
   */
  //3.9.1 - moved to lib from temporary intr.Results location
  @InternalUseOnly
  public static int getMultiplier (Object object) {
         if (object instanceof String   ) return ((String)object).length();
    else if (object instanceof Object[] ) return ((Object[])object).length;
    else if (object instanceof double[] ) return ((double[])object).length;
    else if (object instanceof float[]  ) return ((float[])object).length;
    else if (object instanceof long[]   ) return ((long[])object).length;
    else if (object instanceof int[]    ) return ((int[])object).length;
    else if (object instanceof short[]  ) return ((short[])object).length;
    else if (object instanceof byte[]   ) return ((byte[])object).length;
    else if (object instanceof boolean[]) return ((boolean[])object).length;
    else    /* Generic object.         */ return -1;
  }

  /** Gets the data type letter for non-data instances.
      @param object The object.
      @return The type letter or -1 for none.
   */
  @InternalUseOnly
  //3.9.1 - moved to lib from temporary intr.Results location
  // this method should move to lib in 3.9.0
  // temporarily making public for NXM-3120
  public static char getDataTypeLetter (Object object) {
         if (object instanceof Table    ) return 'T';
    else if (object instanceof String   ) return 'S';
    else if (object instanceof KeyVector) return 'K';
    else if (object instanceof Message  ) return 'M';
    else if (object instanceof Double   ) return 'd';
    else if (object instanceof Float    ) return 'f';
    else if (object instanceof Long     ) return 'x';
    else if (object instanceof Integer  ) return 'l';
    else if (object instanceof Short    ) return 'i';
    else if (object instanceof Byte     ) return 'b';
    else if (object instanceof Boolean  ) return 'z';
    else if (object instanceof Object[] ) return 'O';
    else if (object instanceof double[] ) return 'd';
    else if (object instanceof float[]  ) return 'f';
    else if (object instanceof long[]   ) return 'x';
    else if (object instanceof int[]    ) return 'l';
    else if (object instanceof short[]  ) return 'i';
    else if (object instanceof byte[]   ) return 'b';
    else if (object instanceof boolean[]) return 'z';
    else    /* Generic object.         */ return 'O';
  }

  static char getDataTypeLetterPlus (Object object) {
    if ((object instanceof BigDecimal) || (object instanceof BigInteger)) return 'n';
    return getDataTypeLetter(object);
  }

  /** Cast this as a typed array - must be followed by uncast. */
  private Object cast (byte ntype, boolean copy) {
    if (type == NONHOMOGENEOUS || ntype == NONHOMOGENEOUS) {
      throw new MidasException("Cannot cast non-homogeneous data "+(char)type+"->"+(char)ntype);
    }
    if (type==ntype) {
      if (type==DOUBLE && Native.useCastX) return Native.castD(buf);
      if (type==FLOAT  && Native.useCast ) return Native.castF(buf);
//TBD if (type==XLONG  && Native.useCastX) return Native.castX(buf);
      if (type==LONG   && Native.useCast ) return Native.castL(buf);
      if (type==INT    && Native.useCast ) return Native.castI(buf);
      if (type==BYTE                     ) return buf;
    }
    if (ntype!=atype) prepArray(ntype);
    if (copy) Convert.bb2ja(buf,0,type, array,0,atype, size*ape*spa);
    return array;
  }

  /** UnCast this from a typed array - must be preceded by cast. */
  private void uncast (byte ntype, Object array, boolean copy) {
    if (type == NONHOMOGENEOUS || ntype == NONHOMOGENEOUS) {
      throw new MidasException("Cannot uncast non-homogeneous data "+(char)type+"->"+(char)ntype);
    }
    if (type==ntype) {
      if (type==DOUBLE && Native.useCastX) { Native.uncastD(buf); return; }
      if (type==FLOAT  && Native.useCast ) { Native.uncastF(buf); return; }
//TBD if (type==XLONG  && Native.useCastX) { Native.uncastX(buf); return; }
      if (type==LONG   && Native.useCast ) { Native.uncastL(buf); return; }
      if (type==INT    && Native.useCast ) { Native.uncastI(buf); return; }
      if (type==BYTE                     ) {                      return; }
    }
    if (copy) Convert.ja2bb(array,0,ntype, buf,0,type, size*ape*spa);
  }

  /** Cast this as a double array - must be followed by uncast.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @param copy Make a copy if true
      @return double array
   */
  public double[] castD (boolean copy) { return (double[])cast(DOUBLE,copy); }
  /** Cast this as a float array - must be followed by uncast.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @param copy Make a copy if true
      @return float array
   */
  public float[] castF (boolean copy) { return (float[])cast(FLOAT,copy); }
  /** Cast this as a long array - must be followed by uncast.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @param copy Make a copy if true
      @return long array
      @since NeXtMidas 2.7.1
   */
  public long[] castX (boolean copy) { return (long[])cast(XLONG,copy); }
  /** Cast this as a int array - must be followed by uncast.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @return int array
      @param copy Make a copy if true
   */
  public int[] castL (boolean copy) { return (int[])cast(LONG,copy); }
  /** Cast this as a short array - must be followed by uncast.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @return short array
      @param copy Make a copy if true
   */
  public short[] castI (boolean copy) { return (short[])cast(INT,copy); }
  /** Cast this as a byte array - must be followed by uncast.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @param copy Make a copy if true
      @return byte array
   */
  public byte[] castB (boolean copy) { return (byte[])cast(BYTE,copy); }

  /** Revert a casted array to its standard form following a {@link #castD}.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @param array Array to uncast
      @param copy Make a copy if true
   */
  public void uncast (double[] array, boolean copy) { uncast(DOUBLE,array,copy); }
  /** Revert a casted array to its standard form following a {@link #castF}.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @param array Array to uncast
      @param copy Make a copy if true
   */
  public void uncast (float[] array, boolean copy) { uncast(FLOAT,array,copy); }
  /** Revert a casted array to its standard form following a {@link #castX}.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @param array Array to uncast
      @param copy Make a copy if true
   */
  public void uncast (long[] array, boolean copy) { uncast(XLONG,array,copy); }
  /** Revert a casted array to its standard form following a {@link #castL}.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @param array Array to uncast
      @param copy Make a copy if true
   */
  public void uncast (int[] array, boolean copy) { uncast(LONG,array,copy); }
  /** Revert a casted array to its standard form following a {@link #castI}.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @param array Array to uncast
      @param copy Make a copy if true
   */
  public void uncast (short[] array, boolean copy) { uncast(INT,array,copy); }
  /** Revert a casted array to its standard form following a {@link #castB}.
      See <a href="#AccessingArrays">Accessing Arrays</a> for details.
      @param array Array to uncast
      @param copy Make a copy if true
   */
  public void uncast (byte[] array, boolean copy) { uncast(BYTE,array,copy); }

  /** Get data buffer into working array. This is the same as <code>toArray(array,0)</code>.
      @param array The primitive array to put the data into. This must be an instance of
                   <code>double[]</code>, <code>float[]</code>, <code>long[]</code>,
                   <code>short[]</code>, or <code>byte[]</code>.
   */
  public void toArray (Object array) {  toArray(array, 0); }

  /** Get data buffer into working array. <br><br>
      Example 1: Copy two VD values into an array (one after the other).
      <pre>
        Data first;
        Data second;
        ...
        double[] myArray = new double[6];
        first.toArray(myArray, 0);
        second.toArray(myArray, 3);
      </pre>
      Example 2: Copy the data into a primitive array that is not zero-based.
      <pre>
        Data myData;
        double[] oneBasedArray; // A one-based array
        ...
        myData.toArray(oneBasedArray, 1);
      </pre>
      @param array  The primitive array to put the data into. This must be an instance of
                    <code>double[]</code>, <code>float[]</code>, <code>long[]</code>,
                    <code>short[]</code>, or <code>byte[]</code>.
      @param offset The offset into the primitive array to start at.
      @since NeXtMidas 2.1.0
   */
  public void toArray (Object array, int offset) {
         if (array instanceof float[])  toArray(array,offset,FLOAT);
    else if (array instanceof double[]) toArray(array,offset,DOUBLE);
    else if (array instanceof long[])   toArray(array,offset,XLONG);
    else if (array instanceof int[])    toArray(array,offset,LONG);
    else if (array instanceof short[])  toArray(array,offset,INT);
    else if (array instanceof byte[])   toArray(array,offset,BYTE);
  }

  /** Get data buffer into working array.
      @param array The primitive array to put the data into.
      @param aoff  The offset into the array to start at.
      @param atype The array type.
   */
  public void toArray (Object array, int aoff, byte atype) {
    Convert.bb2ja(buf,0,type, array,aoff,atype, size*ape*spa);
  }

  /** copy data from the working array into the byte buffer (if necessary) */
  public void fromArray () {
    Convert.ja2bb(array,0,type, buf,0,type, size*ape*spa);
  }

  /** set data buffer from an array
      @param array the input array.
   */
  public void fromArray (Object array) {
         if (array == this.array)       fromArray();
    else if (array instanceof float[])  fromArray(array,0,FLOAT);
    else if (array instanceof double[]) fromArray(array,0,DOUBLE);
    else if (array instanceof long[])   fromArray(array,0,XLONG);
    else if (array instanceof int[])    fromArray(array,0,LONG);
    else if (array instanceof short[])  fromArray(array,0,INT);
    else if (array instanceof byte[])   fromArray(array,0,BYTE);
  }

  /** set data buffer from an array
      @param array Input array
      @param aoff  Array offset
      @param atype Array type
   */
  public void fromArray (Object array, int aoff, byte atype) {
    Convert.ja2bb(array,aoff,atype, buf,0,type, size*ape*spa);
  }

  /** set a Data object from a String
      @param string the input String
   */
  public void fromString (String string) {
    if (isString()) {
      int len = string.length();
      if (len>bpe) len=bpe;  // truncate so that we don't overrun buffer (TODO: 1. warn, 2. throw exception, 3. expand buffer, ?)
      byte[] bytes = string.getBytes();
      System.arraycopy (bytes,0, buf,0, len);
      for (int i=len; i<bpe; i++) buf[i]=(byte)SPACE; // pad rest of buffer with spaces
    }
    else {
      Shell.warning("Data.fromString(String) only Data objects that are ASCII types can be set with this method - not type '"+(char)type+"'");
    }
  }

  /** get the element at the given index
      @param index The given index
      @return the element at the given index
   */
  public Data getIndex (int index) {
    if (size>1) return getAtomAt(index);
    else return getScalarAt(index);
  }

  /** Internal Use ONLY!
   *  Get header as a Time object, if applicable
   *  @return header as a Time object
   */
  @InternalUseOnly
  public Time getHeaderTime () {
    return headerTime;
  }

  /** get the in-line Header information
      @return the header information.
   */
  public Object getHeader () {
    return header;
  }

  /** set the in-line Header information
      @param header the header information to set
   */
  public void setHeader (Object header) {
    this.header = header;
    // support deserialization of header in the standard case
    if ((header != null) && (header instanceof Time)) {
      headerTime = (Time) header;
    }
    else headerTime = null;
  }

  /** Not yet implemented - set the element at the specified index
      @param index  the index value of the element to be set
      @param value  the value to be set.
   */
  public void setIndex (int index, Object value) {
    Shell.warning("Data.setIndex: Not yet implemented. This does nothing");
  }

  /** get the index associated with the key field
      @param key  the key value.
      @return     the index value of the specified key.
   */
  public static int getKeyIndex (String key) {
    return KeyObject.getKeyIndex(key);
  }

  /** Indicates if this data object contains a string value
   * @return true if this is represented as a string, false if it is a number or set of numbers
   * @since NeXtMidas 3.8.0
   */
  public boolean isString () {
    return ((type == ASCII) || (type == ASCII256)); // added 256 ASCII type in 3.9.4 for NXM-3570
  }

  /** Indicates if the type is a String type string value
   *  @param typeByte character representing type
   *  @return true if this is represented as a string, false if it is a number or set of numbers
   *  @since NeXtMidas 3.9.4
   */
  @InternalUseOnly
  public static boolean isString (byte typeByte) {
    return ((typeByte == ASCII) || (typeByte == ASCII256));
  }

  /** Normalizes the data object by converting it to the the nearest Java type. This will
   *  return a {@link String} if {@link #isString()} is true, a {@link Number} if it
   *  holds a singular scalar value, and a {@code List<Number>} if it holds multiple
   *  numbers, and {@code null} if it is a numeric type and empty (this last case is
   *  very rare).
   *  @return The object type
   *  @since NeXtMidas 3.8.0
   */
  @ProvisionalUseOnly("Added as ProvisionalUseOnly in 3.8.0.  Intended to become public in a future release")
  public Object toObject () {
    if (this.isString()) return toString();

    switch (numScalars()) {
      case 0:  return null;
      case 1:  return toNumber();
      default: return toList();
    }
  }

  /** Returns the values as lists of number.  Unlike the {@link #toVector()} this method
   *  exclusively works with numeric values.
   *  @return The list of scalars
   *  @throws IllegalStateException if {@link #isString()} returns true.
   *  @since NeXtMidas 3.8.0
   */
  public List<Number> toList () {
    int nscalar = numScalars();
    int i;
    List<Number> v = new ArrayList<>(nscalar);
    // The code below matches toVector(), except for type 'A' and 'Z' NXM-3570
    switch (type) {
      case DOUBLE: for (i=0;i<nscalar;i++) v.add(unpackD(i*8)); break;
      case FLOAT:  for (i=0;i<nscalar;i++) v.add(unpackF(i*4)); break;
      case XLONG:  for (i=0;i<nscalar;i++) v.add(unpackX(i*8)); break;
      case LONG:   for (i=0;i<nscalar;i++) v.add(unpackL(i*4)); break;
      case INT:    for (i=0;i<nscalar;i++) v.add(unpackI(i*2)); break;
      case BYTE:   for (i=0;i<nscalar;i++) v.add(buf[boff+i]);  break;
      case NIBBLE: for (i=0;i<nscalar;i++) v.add(getN(i)); break;
      case PACKED: for (i=0;i<nscalar;i++) v.add(getP(i)); break;
      case ASCII:    throw new IllegalStateException("Can not convert String to List<Number>");
      case ASCII256: throw new IllegalStateException("Can not convert String to List<Number>"); // 3.9.4 NXM-3570
    }
    return v;
  }

  /** Returns the values as lists of lists size SPA.  Unlike the {@link #toVector()} this method
   *  exclusively works with numeric values.
   *  @return The list of elements
   *  @throws IllegalStateException if {@link #isString()} returns true.
   *  @since NeXtMidas 4.1.1
   */
  @InternalUseOnly
  public List<List<Number>> toListSPA () {
    int nelem = ape*size;
    int i = 0;
    List<List<Number>> nestedSPA = new ArrayList<>(nelem);
    switch (type) {
      case DOUBLE: {
        for (int e=0;e<nelem;e++) {
          List<Number> spaList = new ArrayList<>(spa);
          for (int s=0;s<spa;s++) {
            spaList.add(unpackD(i*8));
            i++;
          }
          nestedSPA.add(spaList);
        }
        break;
      }
      case FLOAT: {
        for (int e=0;e<nelem;e++) {
          List<Number> spaList = new ArrayList<>(spa);
          for (int s=0;s<spa;s++) {
            spaList.add(unpackF(i*4));
            i++;
          }
          nestedSPA.add(spaList);
        }
        break;
      }
      case XLONG: {
        for (int e=0;e<nelem;e++) {
          List<Number> spaList = new ArrayList<>(spa);
          for (int s=0;s<spa;s++) {
            spaList.add(unpackX(i*8));
            i++;
          }
          nestedSPA.add(spaList);
        }
        break;
      }
      case LONG: {
        for (int e=0;e<nelem;e++) {
          List<Number> spaList = new ArrayList<>(spa);
          for (int s=0;s<spa;e++) {
            spaList.add(unpackL(i*4));
            i++;
          }
          nestedSPA.add(spaList);
        }
        break;
      }
      case INT: {
        for (int e=0;e<nelem;e++) {
          List<Number> spaList = new ArrayList<>(spa);
          for (int s=0;s<spa;s++) {
            spaList.add(unpackI(i*2));
            i++;
          }
          nestedSPA.add(spaList);
        }
        break;
      }
      case BYTE: {
        for (int e=0;e<nelem;e++) {
          List<Number> spaList = new ArrayList<>(spa);
          for (int s=0;s<spa;s++) {
            spaList.add(buf[boff+i]);
            i++;
          }
          nestedSPA.add(spaList);
        }
        break;
      }
      case NIBBLE: {
        for (int e=0;e<nelem;e++) {
          List<Number> spaList = new ArrayList<>(spa);
          for (int s=0;s<spa;s++) {
            spaList.add(getN(i));
            i++;
          }
          nestedSPA.add(spaList);
        }
        break;
      }
      case PACKED: {
        for (int e=0;e<nelem;e++) {
          List<Number> spaList = new ArrayList<>(spa);
          for (int s=0;s<spa;s++) {
            spaList.add(getP(i));
            i++;
          }
          nestedSPA.add(spaList);
        }
        break;
      }
      case ASCII:    throw new IllegalStateException("Can not convert String to List<Number>");
      case ASCII256: throw new IllegalStateException("Can not convert String to List<Number>"); // 3.9.4 NXM-3570
    }
    return nestedSPA;
  }

  /** Returns the values as lists of number.  Unlike the {@link #toVector()} this method
   *  exclusively works with numeric values.
   *  @return The list of scalars
   *  @throws IllegalStateException if {@link #isString()} returns true.
   */
  @InternalUseOnly
  public List<String> toStringList () {
    int nscalar = numScalars();
    int numBytesString = spa*bps;
    int i;
    List<String> lst = new ArrayList<>(nscalar);
    switch (type) {
      case DOUBLE: for (i=0;i<nscalar;i++) lst.add(String.valueOf(unpackD(i*8))); break;
      case FLOAT:  for (i=0;i<nscalar;i++) lst.add(String.valueOf(unpackF(i*4))); break;
      case XLONG:  for (i=0;i<nscalar;i++) lst.add(String.valueOf(unpackX(i*8))); break;
      case LONG:   for (i=0;i<nscalar;i++) lst.add(String.valueOf(unpackL(i*4))); break;
      case INT:    for (i=0;i<nscalar;i++) lst.add(String.valueOf(unpackI(i*2))); break;
      case BYTE:   for (i=0;i<nscalar;i++) lst.add(String.valueOf(buf[boff+i]));  break;
      case NIBBLE: for (i=0;i<nscalar;i++) lst.add(String.valueOf(getN(i))); break;
      case PACKED: for (i=0;i<nscalar;i++) lst.add(String.valueOf(getP(i))); break;
      case ASCII:  for (i=0;i<nscalar/spa;i++) lst.add(unpackS(i*numBytesString,bps)); break;
      case ASCII256: for (i=0;i<nscalar/spa;i++) lst.add(unpackS(i*numBytesString,bps)); break;
    }
    return lst;
  }


  /**Returns the values as a list of numbers.  Unlike the newer {@link #toList} this method
   * will attempt to handle a string value, though it does so by breaking the strings up into
   * substrings (which is usually undesired, but matches legacy behavior).<br>
   * <br>
   * @return the Vector of scalars.
   */
  public Vector<Object> toVector () {
    int nscalar=size*ape*spa;
    int i;
    int ioff;
    Vector<Object> v = new Vector<>(nscalar);
    // The code below matches toList(), except for type 'A' and 'Z'
    switch (type) {
      case DOUBLE:   for (i=0;i<nscalar;i++) v.addElement(unpackD(i*8)); break; // 3.8.0 - removed manual boxing of primitive numeric types
      case FLOAT:    for (i=0;i<nscalar;i++) v.addElement(unpackF(i*4)); break;
      case XLONG:    for (i=0;i<nscalar;i++) v.addElement(unpackX(i*8)); break;
      case LONG:     for (i=0;i<nscalar;i++) v.addElement(unpackL(i*4)); break;
      case INT:      for (i=0;i<nscalar;i++) v.addElement(unpackI(i*2)); break;
      case BYTE:     for (i=0;i<nscalar;i++) v.addElement(buf[boff+i]);  break;
      case NIBBLE:   for (i=0;i<nscalar;i++) v.addElement(getN(i)); break;
      case PACKED:   for (i=0;i<nscalar;i++) v.addElement(getP(i)); break;
      case ASCII:    for (i=0,ioff=0;i<size;i++,ioff+=bpe) v.addElement(unpackS(ioff,bpe)); break;
      case ASCII256: for (i=0,ioff=0;i<size;i++,ioff+=bpe) v.addElement(unpackS(ioff,bpe)); break; // 3.9.4 NXM-3570
    }
    return v;
  }

  /** Compares another object to this one. <br>
      <br>
      If <code>obj</code> is a {@link Data} object then the comparison will be done by
      {@link #compareTo(Data,int)} using the comparison flag {@link #EQ_DEF}. Otherwise, both
      obj and this will be converted to strings and the result of that string comparison
      will be returned.
      <br>
      Prior to NeXtMidas 2.7.0 {@link Data} objects are treated as scalars when performing
      the comparison but with the note that "This may be changed in the future...".
      @param obj  The object to compare to this one.
      @return 0 if this and data are equal, &lt;0 if this is less than data and &gt;0 if this
              is greater than data.
      @since NeXtMidas 1.9.4
   */
  @Override
  public int compareTo (Object obj) {
    if (obj == this) return 0; // same
    if (obj == null) throw new NullPointerException("Can not compare "+this+" to "+obj+".");

    if (obj instanceof Data) {
      Data data = (Data)obj;
      if ((this.spa == 1) && (data.spa == 1)) {
        return compareTo((Data)obj, EQ_CS);
      }
      else {
        return compareTo((Data)obj, EQ_CS|EQ_MAG);
      }
    }
    return this.toString().compareTo(obj.toString());
  }


  /** Compares another data object to this one. <br>
      <br>
      If either object is of type ASCII, they will be compared using a string comparison. Otherwise
      a numerical comparison will be used. Since inequality comparisons between non-scalar numbers
      is not well-defined, this will throw an exception if a non-scalar value is passed in *except*
      if the MAGNITUDE flag is asserted (at which point the magnitude of the values is compared).
      @param data  The data object to compare to this one.
      @param flags The comparison flags:
      <pre>
        {@link #EQ_CS}     - Use case-sensitive check for ASCII types, otherwise
                   a case-insensitive check is used.
        {@link #EQ_MAG}    - Compare the magnitude of the values of each atom
                   (see {@link #getMagnitude()}). This can be combined with EQ_SIZE
                   and EQ_TOL.
        {@link #EQ_DEF}    - Use the default comparisons (EQ_DEFAULT) and only the
                   default comparisons (can not be combined with other flags).
                   This is the same as EQ_MAG|EQ_CS if either this or data is
                   non-scalar; otherwise just EQ_CS is used. This differs from
                   the behavior of EQ_DEF when passed to {@link #equals(Data,int,double)}.)
      </pre>
      @return 0 if this and data are equal, {@code <0} if this is less than data and {@code >0} if this is greater than data.
      @throws IllegalArgumentException If the size and number of atoms-per-element (APE) for either
                                       data object are not 1 or if the the number of scalars-per-atom
                                       (SPA) is not 1 and {@link #EQ_MAG} was not specified.
      @since NeXtMidas 2.7.0
   */
  public int compareTo (Data data, int flags) {
    if (data == this) return 0; // same
    if (data == null) throw new NullPointerException("Can not compare "+this+" to "+data+".");

    if (flags == EQ_DEF) {
           if (this.spa == 1) flags = EQ_CS;
      else if (data.spa == 1) flags = EQ_CS;
      else                    flags = EQ_CS|EQ_MAG;
    }

    if (this.isString() || data.isString())  {
      String t = this.toString();
      String d = data.toString();

      if ((flags & EQ_CS)==0) { // not case-sensitive
        t = t.toUpperCase();
        d = d.toUpperCase();
      }
      return t.compareTo(d);
    }

    if (flags == -1) { // DEPRECATED: Used by deprecated code in Args.java
      return (int)signum(this.toD() - data.toD());
    }

    if ((this.size != 1) || (data.size != 1) || (this.ape != 1) || (data.ape != 1)) {
      throw new IllegalArgumentException("Can not compare data buffers with more than one atom "
                                       + "given buffers with sizes "+this.size+" and "+data.size+" "
                                       + "and atom-per-element (APE) "+this.ape+" and "+data.ape+".");
    }
    if ((flags & EQ_MAG) != 0) { // check magnitude (mode is not important here)
      double[] t = this.getMagnitude(); // Since size=1 and ape=1 both of these this should have
      double[] d = data.getMagnitude(); // exactly 1 element in them
      return (int)signum(t[0] - d[0]);
    }
    if ((this.spa != 1) || (data.spa != 1)) {
      throw new IllegalArgumentException("Can not directly compare non-scalar values. Given values "
                                       + "with formats "+this.getFormat()+" and "+data.getFormat()+".");
    }
    return (int)signum(this.toD() - data.toD());
  }

  /** Tests to see if an object is equal to this data value. This works as follows:
      <pre>
        - If data is equal to this object, return true.
        - If data is equal to null, return false.
        - If either this or data is an ASCII type, compare both as Strings (see {@link #toString()}.
        - Otherwise do a numeric comparison based on the flags passed in.
      </pre>
      @param data  The Data object to compare with this one.
      @param flags The comparison flags:
      <pre>
        {@link #EQ_SIZE}   - Makes sure both have the same number of elements.
                    If not specified, the elements are checked up to the length of the shortest one.
                    This is intended for use when comparing large buffers of data.
        {@link #EQ_CS}     - Use case-sensitive check for ASCII types, otherwise a case-insensitive check is used.
        {@link #EQ_BIN}    - Use a binary data comparison. Use with caution since
                    this will return false if even the slightest thing differs.
        {@link #EQ_MAG}    - Compare the magnitude of the values of each atom (see {@link #getMagnitude()}).
                    This can be combined with EQ_SIZE and EQ_TOL.
        {@link #EQ_VEQ}    - Compare the each value in an atom one-by-one. This is the same as computing
                    vector equality in Euclidean space.
                    This *currently* does type promotion before comparing values.
                    This can be combined with EQ_SIZE and EQ_TOL.
        {@link #EQ_TOL}    - Do comparisons using a "relative" tolerance to negate floating-
                    point rounding errors (see {@link Tolerance#equalsWithTolerance}).
        {@link #EQ_ABSTOL} - Do comparisons using an "absolute" tolerance to negate floating-
                    point rounding errors (see {@link Tolerance#equalsWithAbsTolerance}).
        {@link #EQ_DEFAULT} - Use the default comparisons (can be combined with other test flags).
        {@link #EQ_DEF}     - Use the default comparisons (EQ_DEFAULT) and only the
                    default comparisons (can not be combined with other flags).
      </pre>
      @param tol   The tolerance to use for numerical tolerance if {@link #EQ_TOL} or {@link #EQ_ABSTOL} is specified.
      @return true if they are the same, false otherwise.
      @see #equals(Object)
   */
  public boolean equals (Data data, int flags, double tol) {
    if (data == this) return true;
    if (data == null) return false;

    if ((flags & (EQ_TOL|EQ_ABSTOL)) == 0) tol = 0; // ignore tolerance
    if (flags == EQ_DEF) flags = EQ_DEFAULT; // use default tests

    // STRING TESTS
    if (this.isString() || data.isString()) {
      if ((flags & EQ_CS)!=0) return toString().equals(data.toString());
      else                    return toString().equalsIgnoreCase(data.toString());
    }

    // NUMERIC TESTS (stand-alone)
    if ((flags&EQ_SIZE)!=0) {
      if (this.size != data.size) return false; // different number of elements
    }
    int size = Math.min(this.size, data.size);
    boolean absTol = ((flags & EQ_ABSTOL) != 0);

    // NUMERIC TESTS
    if ((flags & EQ_BIN) != 0) { // fast binary comparison (overrides EQ_MAG)
      if (this.bpe != data.bpe) return false; // different data sizes (same as checking mode, type, ape)
      if ((this.boff == data.boff) && (this.buf == data.buf)) return true;
      int i = this.boff;
      int j = data.boff;
      int ii = i + (size*ape*spa*bps);
      for (;i<ii;i++,j++) { if (this.buf[i] != data.buf[i]) return false; } // TODO: this is probably a BUG - should be ... != data.buf[j] (i.e. j not i)
      return true;
    }
    else if ((flags & EQ_VEQ) != 0) {
      if (this.ape != data.ape) return false; // different atoms per elements
      if (this.spa != data.spa) return false; // different scalars per atom

      byte type = this.type;
      if (this.type != data.type) {
        type = promoteType(this.type, data.type); // only promote if necessary
      }
      if ((type == PACKED) || (type == NIBBLE)) {
        // Can not cast 'P' or 'N' to a Java array, therefore we convert to a byte
        // since NeXtMidas 3.3.2 (see Bug 2532 and %TEST_DATALIST_PACKED_AND_NIBBLE).
        type = BYTE;
      }
      Object array1 = this.cast(type,true);
      Object array2 = data.cast(type,true);
      boolean equal = (absTol)? equalsWithAbsTolerance(array1, 0, array2, 0, size*ape*spa, tol)
                              : equalsWithTolerance(   array1, 0, array2, 0, size*ape*spa, tol);
      this.uncast(type,array1,false);
      data.uncast(type,array2,false);
      return equal;
    }
    else if ((flags & EQ_MAG) != 0) { // check magnitude (mode is not important here)
      if (this.ape != data.ape) return false; // different atoms per elements
      return (absTol)? equalsWithAbsTolerance(this.getMagnitude(), 0, data.getMagnitude(), 0, size*ape, tol)
                     : equalsWithTolerance(   this.getMagnitude(), 0, data.getMagnitude(), 0, size*ape, tol);
    }
    return true;
  }

  /** Converts a string used on EQ tests to an integer flag.
      @param eqString The equality definition string. This string will specify
                      the equality tests that will be used when comparing two
                      Data objects. The tests are those listed in the IF explain
                      file
      @param absTol   Is absolute tolerance being used (true) or relative
                      tolerance (false)?
      @return The results of or'ing together all of the comparison flags indicated
              in <code>eqString</code> or {@link #EQ_DEFAULT} if none are specified.
   */
  static int parseTestFlag (String eqString, boolean absTol) { // package-private only used in Args
    // Note that != and NE should change to == or EQ before being passed to this method.
    if (eqString.equals("==")) return EQ_DEFAULT;
    if (eqString.equals("~=")) return (absTol)? EQ_DEFAULT|EQ_ABSTOL
                                              : EQ_DEFAULT|EQ_TOL;
    int iSlash = eqString.indexOf('/');
    if (iSlash < 0) return EQ_DEFAULT; // No flags
    String str = eqString.substring(iSlash+1).replace('/','|');
    int mask = Parser.mask("B,S,,,,T,M,C,V", str, 0); // omits 'H', 'D' and 'K' used in BaseFile
    if ((mask & (EQ_MAG|EQ_BIN|EQ_VEQ)) == 0) {
      mask = mask | EQ_VEQ; // no numeric test specified, assume EQ_VEQ
    }

    if (absTol & ((mask&EQ_TOL)!=0)) {
      mask = (mask & ~EQ_TOL)|EQ_ABSTOL; // Swap relative tolerance with absolute
    }
    return mask;
  }

  /** Computes the magnitude for all atoms in the buffer. The magnitude is computed using the
      Euclidean norm formula:
      <pre>
        mag(x) = sqrt(x1^2 + x2^2 + ... + xN^2)
      </pre>
      Note that this computation is done using double-precision values.
      @return The array holding the magnitude for all elements in the buffer or <code>null</code>
              if this is called on an ASCII type.
      @since NeXtMidas 2.7.0
   */
  public double[] getMagnitude () {
    if (this.isString()) return null;
    double[] vals = castD(true);
    double[] mag = new double[vals.length/spa];
    for (int i=0,j=0; i < mag.length; i++) {
      double m = 0;
      for (int k = 0; k < spa; j++,k++) m += vals[j]*vals[j];
      mag[i] = Math.sqrt(m);
    }
    uncast(vals,false);
    return mag;
  }

  /** Returns a hash code value for the specified byte array. This will only
      compare up to the first MAX_BYTES_FOR_HASHCODE (32) bytes of the data
      buffer. This is done to reduce the time it takes to compute the hash
      value for large data buffers. The length of the buffer is also added to
      the output hash code to help with the hash code distribution.
      @param array the target array.
      @return hash code for the specified byte array
      @since NeXtMidas 2.7.0
   */
  public static int hashCodeByteArray (byte[] array) {
    int prime = 31;
    if (array == null) return 0;
    int result = 1;
    for (int ii=0; ii<array.length && ii<MAX_BYTES_FOR_HASHCODE; ii++) {
      result = (prime * result) + array[ii];
    }
    return (prime * result) + array.length;
  }

  /** Returns a hash code value for this Data object.
      @return hash code for this Data object.
      @see java.lang.Object#hashCode()
      @see java.lang.Object#equals(Object)
      @see #equals(Object)
      @since NeXtMidas 2.7.0
   */
  @Override
  public int hashCode () {
    final int prime = 31;
    int result = 1;
    result = (prime * result) + size;
    result = (prime * result) + mode;
    result = (prime * result) + type;
    // NTN: we cannot use Object's hashCode() on buf, since it is unique for each
    //      instance of the byte array, even if they contain the same contents
    result = (prime * result) + hashCodeByteArray(buf);
    return result;
  }

  /** Indicates whether some other object is "equal to" this one.
      Does a binary equals comparison of the other object with this one.
      This means that data containing the same values but in different format
      types (e.g. I, L, F, D, etc.) will NOT be equal to each other (i.e. no type promotion).
      Likewise, the same data in the same format type with different data
      modes (e.g. S, C, V, Q, M, etc.) will NOT be equal to each other.
      @see java.lang.Object#equals(java.lang.Object)
      @see java.lang.Object#hashCode()
      @see #hashCode()
      @since NeXtMidas 2.7.0
   */
  @Override
  public boolean equals (Object obj) {
    if (this == obj) return true;
    if (obj == null) return false;
    if (!(obj instanceof Data)) return false;
    Data other = (Data) obj;
    if (size != other.size) return false;
    if (mode != other.mode) return false;
    if (type != other.type) return false;
    return Arrays.equals(buf, other.buf);
  }

  /** Retrieve an List of type that extends the Number class. (Byte, Integer, Double, Long, Short, Float)
   *
   *  @param <T> The type of the input
   *  @param classType Class type for creating the list
   *  @return The ArrayList built from the Data buffer
   *  @since NeXtMidas 3.9.3 for NXM-3420
   */
  @SuppressWarnings("unchecked")
  public <T extends Number> List<T> toList (Class<T> classType){
    // do the specific method to get the right list.
    Class<?> currentClass = numberTypes.get(type);
    if (currentClass == null) {
      // warning for non number data type
      return null;
    }
    Method castMethod = CASTING_METHODS.get(classType);
    Method getFromBuf = FROM_BUF_METHODS.get(type);

    if (castMethod==null) {
      Midas.log("Converting Data object to ArrayList functionality not avaialbe for class " + classType, Midas.WARNING);
      return null;
    }
    if (getFromBuf==null) {
      Midas.log("Converting Data object to ArrayList functionality not avaialbe for type " + type, Midas.WARNING);
      return null;
    }
    List<T> result = new ArrayList<>();
    try {
      for (int i = 0; i < size; i++) {
        result.add((T) castMethod.invoke(getFromBuf.invoke(this, i)));
      }
    } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
      Midas.log("Converting Data object has failed for class " + classType + " with exception " + e, Midas.ERROR);
    }
    return result;
  }

  /**
   * Initialize data from List of type LONG (java type Integer)
   * @param valuesList The (java type Integer) List to populate the data buffer
   * @since 3.9.3
   */
  private void fromListL (List<Integer> valuesList){
    initTo(valuesList.stream().mapToInt(Integer::intValue).toArray());
  }

  /**
   * Initialize data from List of type INT (java type Short)
   * @param valuesList The INT (java type Short) List to populate the data buffer
   * @since 3.9.3
   */
  private void fromListI (List<Short> valuesList){
    short[] temp = new short[valuesList.size()];
    for(int i = 0; i< valuesList.size();i++) {
      temp[i]= valuesList.get(i);
    }
    initTo(temp);
  }

  /**
   * Initialize data from List of type XLONG (java type Long)
   * @param valuesList The XLONG (java type Long) List to populate the data buffer
   * @since 3.9.3
   */
  private void fromListX (List<Long> valuesList){
    initTo(valuesList.stream().mapToLong(Long::longValue).toArray());
  }

  /**
   * Initialize data from List of type FLOAT (java type Float)
   * @param valuesList The FLOAT (java type Float) List to populate the data buffer
   * @since 3.9.3
   */
  private void fromListF (List<Float> valuesList){
    float[] temp = new float[valuesList.size()];
    for(int i = 0; i< valuesList.size();i++) {
      temp[i]= valuesList.get(i);
    }
    initTo(temp);
  }

  /**
   * Initialize data from List of type DOUBLE (java type Double)
   * @param valuesList The DOUBLE (java type Double) List to populate the data buffer
   * @since 3.9.3
   */
  private void fromListD (List<Double> valuesList){
    initTo(valuesList.stream().mapToDouble(Double::doubleValue).toArray());
  }

  /**
   * Initialize data from List of type DOUBLE (java type Double)
   * @param valuesList The DOUBLE (java type Double) List to populate the data buffer
   * @since 3.9.3
   */
  private void fromListB (List<Byte> valuesList){
    byte[] temp = new byte[valuesList.size()];
    for(int i = 0;i<valuesList.size();i++) {
      temp[i] = valuesList.get(i);
    }
    initTo(temp);
  }

  /**
   * Initialize data from List (Can be of type Short, Integer, Float, Long, Double)
   * @param object The List to build the data buffer
   * @since 3.9.3 for NXM-3420
   */
  @SuppressWarnings("unchecked")
  public void fromList (List<?> object) {
    if(object.size()==0) {
      // warning about empty list - can't make data object if list is empty.
      return;
    }
    if(object.get(0) instanceof Byte)    fromListB((List<Byte>)object);
    if(object.get(0) instanceof Integer) fromListL((List<Integer>)object);
    if(object.get(0) instanceof Float)   fromListF((List<Float>)object);
    if(object.get(0) instanceof Long)    fromListX((List<Long>)object);
    if(object.get(0) instanceof Double)  fromListD((List<Double>)object);
    if(object.get(0) instanceof Short)   fromListI((List<Short>)object);

  }

  /**
   * Builds the map of casting methods used for casting Number objects to the primitive version.
   * @return the primitive version of the Number Object
   * @since 3.9.3
   */
  private static Map<Class<?>, Method> getCastingMethods () {
    Map<Class<?>, Method> temp = new HashMap<>();
    try {
       temp.put(Short.class,Number.class.getDeclaredMethod("shortValue"));
       temp.put(Byte.class,Number.class.getDeclaredMethod("byteValue"));
       temp.put(Float.class,Number.class.getDeclaredMethod("floatValue"));
       temp.put(Integer.class,Number.class.getDeclaredMethod("intValue"));
       temp.put(Long.class,Number.class.getDeclaredMethod("longValue"));
       temp.put(Double.class,Number.class.getDeclaredMethod("doubleValue"));

    } catch (NoSuchMethodException | SecurityException e) {
      // unable to cast to that value
    }
    return temp;
  }

  /**
   * Builds the map of getter methods used for getting the data form the byte buffer.
   * @return the primitive version of the Number Object
   * @since 3.9.3
   */
  private static Map<Byte,Method> getFromBufMethodsBuilder () {
    Map<Byte,Method> temp = new HashMap<>();
    try {
      temp.put(BYTE, Data.class.getDeclaredMethod("getB", int.class));
      temp.put(FLOAT, Data.class.getDeclaredMethod("getF", int.class));
      temp.put(INT, Data.class.getDeclaredMethod("getI", int.class));
      temp.put(LONG, Data.class.getDeclaredMethod("getL", int.class));
      temp.put(XLONG, Data.class.getDeclaredMethod("getX", int.class));
      temp.put(DOUBLE, Data.class.getDeclaredMethod("getD", int.class));
    } catch (NoSuchMethodException | SecurityException e) {
      // NO WAY TO GET VALUE FROM BUFFER
      temp = null;
    }
    return temp;
  }

  /**
   * This will cast an Object to a List of type T
   * @param <T> The type of the object
   * @param obj The object to be converted
   * @return the object casted to a List
   * @since 3.9.3
   */
  @SuppressWarnings("unchecked")
  private static <T extends List<?>> T castObjectToList (Object obj) {
    return (T) obj;
  }

  /** Overwrite default deserialization
  *
  *  @param ois Object Input Strean
  *  @throws ClassNotFoundException Exception thrown by readObject
  *  @throws IOException Exception thrown by readObject
  */
 private void readObject (ObjectInputStream ois) throws ClassNotFoundException, IOException {
   // default deserialization
   ois.defaultReadObject();
   // restor header from header saved as Time object
   this.header = this.headerTime;
 }

}
