package nxm.sys.lib;

import static nxm.sys.lib.DataFile.getUnitsID;

import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import nxm.sys.inc.AsciiMap;
import nxm.sys.inc.DataTypes;
import nxm.sys.inc.InternalUseOnly;
import nxm.sys.inc.Keyable;

/** A class to perform Keyword head block manipulation for a Midas BLUE file such as {@link DataFile}.
 *  Please see the BLUE 1.1 ICD (found in the docs area of SYS) for details.
 *  @author Jeff Schoen / Neon Ngo
 *  @see DataFile#getKeywordsObject()
 */
public class Keywords extends Object implements Keyable,DataTypes,AsciiMap {
  //Normal extended header format (see BLUE 1.1 ICD for details):
  //   Format {
  //     L:length - Total length of keyword (lkey) including header (8), data value, tag name, and padding.
  //     I:lext   - Length of extra bytes = length - length of data
  //     B:ltag   - Length of keyword tag name
  //     B:type   - A,B,I,L,...
  //     data     - Data (has associated REP). length of data = (length - lext)
  //     tag      - Tag name is after data
  //     pad      - PAD bytes (to reach a total keyword structure size that is a multiple of 8 bytes)
  //   }

  /** These are the only keywords that should appear in the MAIN header. This is as per version 1.1
   *  of the BLUE ICD. Any keyword not in this list, is subject to automatic deletion. (BLUE 1.1
   *  requires that the <code>VER=</code> and <code>IO=</code> keywords be written, {@link DataFile} will
   *  be forced to delete any extra keywords in the main header if it needs to make room for these.
   *  <pre>
   *    Main Header Keywords Listed in ICD:
   *      CREATOR=        - Identifier for the system that created the file.
   *      IO=             - Name of the I/O library that wrote the file (see {@link DataFile#getBlueIO}).
   *      PACKET=         - Used by NeXtMidas to indicate that a separate file contains packetized data
   *                        that controls the interpretation of data in this file. The value identifies a
   *                        packet handler and any switches that modify its behavior (e.g. ICE and SDDS).
   *      PKT_BYTE_COUNT= - Used by X-Midas to indicate that a file contains packetized data. The value
   *                        is the total number of bytes of packetized data contained in the file.
   *      TC_PREC=        - Precision time code (see section 3.4.1.1 of the BLUE ICD).
   *      VER=            - Version of BLUE ICD used when writing file (see {@link DataFile#getBlueVer}).
   *  </pre>
   *  @since NeXtMidas 2.5.3
   */
  @InternalUseOnly("Since NeXtMidas 3.3.0")
  public static final String[] MAIN_HEADER_KEYWORDS
          = { "CREATOR", "IO", "PACKET", "PKT_BYTE_COUNT", "TC_PREC", "VER" }; // Must be in alphabetical order!!!
  /** Keywords permitted in the MAIN header without warning on insertion. This list complements
   *  the {@link #MAIN_HEADER_KEYWORDS} list with some entries that are permitted by convention,
   *  even though they are not listed in the BLUE 1.0 or BLUE 1.1. ICD.
   *  <pre>
   *    ACQDATE=          - Acquisition date
   *    ACQTIME=          - Acquisition time
   *    COMMENT=          - User-supplied comment
   *  </pre>
   *  Unlike the entries in {@link #MAIN_HEADER_KEYWORDS} the entries here are not given
   *  "priority use" of the MAIN header any may be removed if necessary to accommodate
   *  one of the entries in {@link #MAIN_HEADER_KEYWORDS} (e.g. a long COMMENT may be
   *  removed to permit insertion of TC_PREC).
   *  @since NeXtMidas 3.3.0
   */
  @InternalUseOnly("Since created")
  public static final String[] MAIN_HEADER_IGNORE
          = { "ACQDATE", "ACQTIME", "COMMENT" };  // Must be in alphabetical order!!!

  /** byte offset in header of keywords string */
  private static final int IKOFF      =  164;
  /** Internal Keywords KMAX (bytes) */
  private static final int IKMAX      =   92;
  /** Allocation size  */
  private static final int EXTSIZE    = 2048;
  /** keyword header length of binary keyword block in Midas BLUE EXTENDED header */
  @InternalUseOnly("Since NeXtMidas 3.7.0") // TODO: should not be public?
  public  static final int HEADER_LEN =    8;

  /** Keyword scopes: SCOPE (only in current scope), MAIN (only in main header), or EXTENDED (only in extended header) */
  public static final int SCOPE=0, MAIN=0x1, EXTENDED=0x2;

  /** SET keyword flags: ADD (allow duplicate keys), REPLACE (overwrite existing key in current scope) */
  public static final int ADD=0x1, REPLACE=0x2;
  /** mask value for ADD|REPLACE */
  private static final int ADD_REPLACE = ADD|REPLACE;

  /** extract in place */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - this field should be private")
  public static final boolean EXTRACTINPLACE=false;

  /** The comma-separated list of possible Keyword flags. (since NeXtMidas 3.1.1) */
  public static final String flagsList = "NoAbort";
  /** Keywords flag: should not abort (throw Exception) when an invalid keyword block is found (since NeXtMidas 3.1.1) */
  public static final int NOABORT = 0x01;
  /** default Keywords flags: none (since 3.1.1) */
  private static final int DEF_FLAGS = 0;
  /** Substitute value for keyword containing dots */
  @InternalUseOnly("Since NeXtMidas 3.7.1")
  public static final String DOT_SUBSTITUTE = "->";
  /** changeable dotSubstitute value for user customization. Defaulted to DEFAULT_DOT_SUBSTITUTE. Since 3.9.1 */
  private static String dotSubstitute = DOT_SUBSTITUTE;
  // Public fields marked InternalUseOnly since 3.7.0 (TODO: These should not be public)
  /** Buffer for extended header */
  @InternalUseOnly("Since NeXtMidas 3.7.0")
  public  byte[]   buf;
  /** Buffer for main header (keywords start at IKOFF) */
  @InternalUseOnly("Since NeXtMidas 3.7.0")
  public  byte[]   ibuf;
  /** Size of extended header */
  @InternalUseOnly("Since NeXtMidas 3.7.0")
  public  int      size;
  /** Size of main header keywords (0..IKMAX). -1 to init this later (since main header has not been read yet) */
  @InternalUseOnly("Since NeXtMidas 3.7.0")
  public  int      isize;

  /** Byte ordering (not properly initialized, deprecated in NeXtMidas 2.5.2, use DataFile.getHeadRep()) */
  private byte     rep; // TODO remove in 3.9+
  /** Current scope string */
  private String   scope = "";
  /** Start of scope (-isize..size, -N is in main header) */
  private int      scope_start;
  /** End of scope   (-isize..size, -N is in main header) */
  private int      scope_end;
  /** current keyword returned from {@link #getNext()} */
  private Key      cur   = new Key();
  /** DataFile the keywords are associated with */
  private DataFile hcb;
  /** flags mask , see {@value #flagsList} (since NeXtMidas 3.1.1) */
  private int      flags = 0;
  /** methods where we have warned about invalid kwd blocks (since 3.1.1) */
  private HashSet<String> warnedForInvalidBlockSet = new HashSet<>();
  /** value of previous invalid DataFile (HCB)'s main header keylength field (so that we don't warn multiple times on same value). since 3.7.0 */
  private Integer prevInvalidHcbKeylength;

  /** String versions of "(" and ")", really belongs in AsciiMap */
  static final String LPAREN_S = "(", RPAREN_S = ")";

  /** Flags that control the handling of units. Values include:
      <pre>
        UNITS_WARN        - Issue warning on units mismatch.
        UNITS_LIST        - Show units with {@link #list(Map.Entry,String,int,int)}.
        UNITS_PREFER_UNIT - Prefer use of ".UNIT" rather than ".UNITS".
        UNITS_MATCH       - Match the choice of ".UNIT" or ".UNITS" based on
                            existing keyword if {@link #REPLACE} is used and
                            exactly one of them already exists.
      </pre>
   */
  @InternalUseOnly
  public static final int UNITS_WARN        = 0x0100,
                          UNITS_LIST        = 0x0200,
                          UNITS_PREFER_UNIT = 0x0400,
                          UNITS_MATCH       = 0x0800;

  /** Initialize a new empty Keyword object. */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - Keywords objects should only be initialized by DataFile")
  public Keywords () { // merge with constructor below
    super();
    size=isize=0;
    cur.offset=0;
    hcb=null;
  }

  /** Initialize a new Keyword object.
   *  @param hcb The associated DataFile.
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - Keywords objects should only be initialized by DataFile")
  public Keywords (DataFile hcb) {
    this();
    this.hcb   = hcb;
    this.ibuf  = hcb.hb;
    this.isize = -1; // init later since this is "normally" created with a fresh DataFile instance before reading of the main header
  }

  /** Initialize {@link #isize} and {@link #ibuf} from {@link #hcb} (which MUST not be null).
   *  This should only be called {@code if isize < 0}.
   *  @throws MidasException this.hcb is null
   *  @since NeXtMidas 3.7.0
   */
  private void initFromDataFile () throws MidasException {
    if (hcb == null) throw new MidasException("Cannot init Keywords's main header fields from a null DataFile!"); // since 3.7.0 for BUG 3010
    ibuf = hcb.hb;
    isize = hcb.getKeyLength();
    if (isize > IKMAX) { // since 3.7.0 - warn and ensure isize is in valid range of [0-IKMAX] - BUG 3010
      // only warn once for a particular invalid size (to reduce duplicate warnings)
      if (prevInvalidHcbKeylength == null || prevInvalidHcbKeylength != isize) {
        prevInvalidHcbKeylength = isize;
        warn("DataFile's main header has invalid keylength [" + isize + "]. Capping to max of " + IKMAX + " bytes.");
      }
      isize = IKMAX;
    }
  }

  /** Clears scope and clears keywords by setting main and extended header sizes to 0.<br>
   *  <br>
   *  This calls {@link #deleteAll()} which is the preferred method. This method is
   *  only provided for compatibility with other Java classes.
   */
  public void clear () { // TODO: This should be marked final
    deleteAll();
  }

  /** Clears scope and clears keywords by setting main and extended header sizes to 0.<br>
   *  <br>
   *  To delete just keywords within a scope, use {@link #deleteScope(String)}.<br>
   *  To delete a single keyword, use {@link #delete(String)}.
   *  @since NeXtMidas 3.5.1
   */
  public void deleteAll () {
    // Clear Keywords by setting main and extended header sizes to 0
    size=isize=cur.offset=0;
    // Clear the Scope
    scope_start=scope_end=0;
  }

  /** Clears the scope; effectively setting the scope to be the whole of the main
   *  and extended header. This can also be achieved by setting scope to an empty string.<br>
   *  <br>
   *  To clear keywords within a given scope, use {@link #deleteScope(String)}.
   */
  public void clearScope () {
    scope_start = -isize;
    scope_end = this.size;
  }

  /** Initialize the keywords for extended header.
   *  @param buf   Keyword buffer.
   *  @param size  Buffer size
   *  @param rep   Byte representation (deprecated).
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - Keywords objects should only be initialized by DataFile")
  public void init (byte[] buf, int size, byte rep) {
    this.size = size;
    this.buf = buf;
    if (hcb!=null) {
      initFromDataFile();        // keywords in DataFile's header (in main header)
    }
    if (rep!=0) this.rep = rep;  // <-- DEPRECATED since 2.5.3
    else this.rep = Shell.rep;   // <-- DEPRECATED since 2.5.3
    if (!chk(1024)) {
      String msg = "Bad Extended Header on URL="+((hcb!=null)? hcb.getURL() : "");
      warn(msg);
      this.size = 0;
    }
    clearScope();
  }

  /** Initialize the keywords for a pipe. This is a special purpose method that is used for
   *  propagating keywords across pipe restarts.
   *  @param kw The chained keyword object.
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - Keywords objects should only be initialized by DataFile")
  public void init (Keywords kw) {
    buf=null;
    if (kw.size>0) {
      buf = new byte[kw.buf.length];
      System.arraycopy (kw.buf,0, buf,0, kw.size); // copy keyword buffer (in extended header)
    }
    init(buf,kw.size,(byte)0);
  }

  /** Get current Keyword flags.
   *  @return The current flags.
   *  @since NeXtMidas 3.1.1
   */
  public int getFlags () { return flags; }

  /** Set current Keyword flags.
   *  @param newFlags The new flags.
   *  @since NeXtMidas 3.1.1
   */
  public void setFlags (int newFlags) { flags = newFlags; }

  /** Set current Keyword flags.
   *  @param newFlagsStr The new flag string.
   *  @since NeXtMidas 3.1.1
   */
  public void setFlags (String newFlagsStr) {
    setFlags(Parser.mask(flagsList, newFlagsStr, DEF_FLAGS));
  }

  /** Get current Keyword flags in string form.
   *  @return The current flags.
   *  @since NeXtMidas 3.1.1
   */
  public String getFlagsString () {
    return Parser.mask2s(flagsList, flags);
  }

  /** Get the current scope string.
   *  @return The current scope (see {@link #setScope(String)} for details).
   */
  public String getScope () { return scope; }

  /** Set the scope string. This only applies to the extended header as there is no scoping concept
   *  in the main header. However, there are three special scopes:
   *  <pre>
   *     MAIN   - scope to main header
   *     EXT    - scope to entire extended header
   *     ALL    - scope to everything in main and extended header
   *  </pre>
   *  This accepts input strings of the form:
   *  <pre>
   *    TAG=VAL                 - Single scope
   *    "TAG1=VAL1,TAG2=VAL2"   - Complex scope (quoted)
   *    (TAG=VAL1,TAG=VAL2)     - Complex scope (parenthesis)
   *  </pre>
   *  Note that if the scope is not found (0 is returned) the old scope is not preserved.
   *  @param newScope Input scope or "MAIN" or "EXT" or "ALL". A null, blank,
   *                  or white space only String is the same as "ALL".
   *  @return The number of levels if the scope is found or 0 if scope is not found. Note that this
   *          will always return 1 for "MAIN", "EXT", or "ALL".
   */
  public int setScope (String newScope) {
    int levels=0;
    scope = (newScope==null)? "" : newScope.trim();
    if (scope.equals("ALL") || scope.length()==0) { clearScope(); return 1; }
    scope_start=-isize; scope_end=0;
    if (scope.equals("MAIN")) { return 1; }
    scope_start=0; scope_end=size;
    if (scope.equals("EXT")) { return 1; }
    Parser parser;
    // Handle string of the form (TAG=VAL1,TAG=VAL2)
    if (scope.startsWith(LPAREN_S)) {
      if (scope.endsWith(RPAREN_S)) {
        parser = new Parser(scope.substring(1,scope.length()-1), AsciiMap.COMMA);
      } else {
        warn("Unbalanced parenthesis in '"+scope+"'");
        parser = new Parser(scope.substring(1), AsciiMap.COMMA);
      }
    } else {
      parser = new Parser(scope, AsciiMap.COMMA);
    }
    while (parser.more()) {
      String str = parser.next();
      int iEqual=str.indexOf('=');
      String name = str.substring(0,iEqual);
      String value = str.substring(iEqual+1);
      int found = setScope(name,value);
      // Any keyword not found for scope causes error
      if (found == 1) { levels++; }
      else            { levels = 0; break; }
    }
    return levels;
  }

  /** The real scope setting method, not published to the outside world */
  private int setScope (String name, String value) {
    boolean found=false;
    int i; Key k=new Key();
    byte[] bname = name.getBytes();
    int blen = bname.length;
    boolean skiptag = (bname[blen-1]=='-' || bname[blen-1]=='+');
    boolean nexttag = bname[blen-1]=='+';
    if (nexttag||skiptag) blen--;
    for (k.offset=scope_start; k.offset<scope_end; k.offset+=k.length) {
      String invalidDetails = unpack(k);
      if (invalidDetails != null) {           // since 3.1.1 (will get here if NoAbort is set)
        if (k.length >= HEADER_LEN) continue; // try next keyword block
        else                        break;    // bad length, can't recover from this (break out of loop)
      }
      if (k.ltag!=blen) continue;
      for (i=0; i<blen; i++)
        if (bname[i] != buf[k.offset + HEADER_LEN + k.ldat + i]) break;
      if (i!=blen) { } // no match
      else if (found) {
        scope_end=k.offset;
        if (nexttag) scope_end += k.length;
        return 1;
      } else {
        extractData(k);
        if (!Data.isString(k.type)) {
          // Numbers like 2 and 2.0 have to be treated as equal to match legacy X-Midas behavior
          if (Convert.o2d(k.value) == Convert.o2d(value)) {
            found=true; scope_start=k.offset;
            if (skiptag) scope_start += k.length;
          }
        } else {
          if (value.equals(k.value.toString())) {
            found=true; scope_start=k.offset;
            if (skiptag) scope_start += k.length;
          }
        }
      }
    }
    if (found) return 1;
    else return 0;
  }

  /** Get the next keyword in the MAIN or extended header. This method should NOT be used to iterate
   *  through all of the keywords (or all the keywords in scope), if used this way it may result a
   *  StringIndexOutOfBoundsException for some (but not all) files. The proper way to iterate over
   *  the keywords is as follows:
   *  <pre>
   *    if (keywds.setScope("FIELD="+i) == 1) { // setScope is optional but frequently used
   *      Keywords.Iterator kwi = keywds.iterator();
   *      Keywords.key      key = null;
   *      while (kwi.hasNext()) {
   *        key = kwi.next();
   *        // Process keyword here
   *      }
   *    }
   *  </pre>
   *  @return next Key, null if at end or if invalid block is found
   */
  public Key getNext () {
    cur.offset += cur.length;
    if (cur.offset>=scope_end) return null;
    String invalidDetails = unpack(cur);
    if (invalidDetails != null) { // since 3.1.1 (will get here is NoAbort is set)
      return null;                // return null if invalid keyword block
    }
    extractName(cur);
    extractData(cur);
    return cur;
  }

  /** Unpack keyword header block into specified Key object. <br>
   *  <br>
   *  Since 3.1.1: If NoAbort is NOT set in the flags mask AND an invalid keyword block is found
   *               then a MidasException will be thrown with the details of the bad block.
   *  @param k Key to unpack keyword block into
   *  @return null if successful, otherwise a String with more details of invalid keyword block (since 3.1.1)
   *  @throws MidasException when an invalid block has been unpacked, unless the {@link #NOABORT}
   *          flag has been set (since 3.1.1)
   *  @see #handleBadKeywordBlock(Key, String, CharSequence)
   */
  private String unpack (Key k) {
    return unpack(k,Shell.rep);
  }

  /** Unpack keyword header block into specified Key object. <br>
   *  <br>
   *  Since 3.1.1: If NoAbort is NOT set in the flags mask AND an invalid keyword block is found
   *               then a MidasException will be thrown with the details of the bad block.
   *  @param k Key to unpack keyword block into
   *  @param hcbNullRep representation to use if hcb is null
   *  @return null if successful, otherwise a String with more details of invalid keyword block (since 3.1.1)
   *  @throws MidasException when an invalid block has been unpacked, unless the {@link #NOABORT}
   *          flag has been set (since 3.1.1)
   *  @see #handleBadKeywordBlock(Key, String, CharSequence)
   */
  private String unpack (Key k, byte hcbNullRep) {
    int off = k.offset;
    short lext;
    if (off < 0) {  // negative offset is to unpack keyword block in BLUE file's main header
      int i;
      off += isize; // translate offset to internal buffer
      for (i=off; i<isize && ibuf[IKOFF+i]!=0; i++) { }             // find offset of NUL char
      k.length = i-off;
      for (i=off; i<off+k.length && ibuf[IKOFF+i]!=EQUALS; i++) { } // find offset of '=' char
      k.ltag = i-off;
      k.ldat = k.length-k.ltag-1;
      k.type = ASCII;                           // all internal fields in main header are ASCII
      if (ibuf[IKOFF+off+1]==COLON) k.type=ibuf[IKOFF+off];
      if (off+k.length < isize) k.length++;     // skip \0 delimiter
      else k.length = isize-off;
    }
    else { // else unpack keyword block in extended header
      byte _rep = (hcb==null)? hcbNullRep: hcb.headRep;
      k.length = Convert.unpackL(buf, off,   _rep); // int_4 for lkey
      lext     = Convert.unpackI(buf, off+4, _rep); // int_2 for lext
      k.ltag = buf[off+6]; // int_1
      k.type = buf[off+7]; // char
      k.ldat = k.length - lext;

      // since 3.1.1, check if unpacked keyword block is valid (in extended header) for BUG 2548
      if ( k.length < HEADER_LEN   // 1. lkey (in extended header) cannot be < 8 bytes (keyword header size)
          || lext    >  k.length   // 2. lext cannot be >  than lkey (this also covers when lext is negative)
          || k.ltag  >= k.length   // 3. ltag cannot be >= lkey
          || k.ldat  >= k.length   // 4. ldat cannot be >= lkey (this also covers when ldat is negative)
          ) {
        byte count = 0;
        StringBuilder details = new StringBuilder(88); // 88 is about the max size for all the error details below
        if (k.length < HEADER_LEN) details.append((++count>1? "," : "")).append(" lkey < ").append(HEADER_LEN); // 1
        if (lext > k.length)       details.append((++count>1? "," : "")).append(" lext > lkey");      // 2a
        if (lext < 0)              details.append((++count>1? "," : "")).append(" lext is negative"); // 2b
        if (k.ltag >= k.length)    details.append((++count>1? "," : "")).append(" ltag >= lkey");     // 3
        if (k.ldat >= k.length)    details.append((++count>1? "," : "")).append(" ldat >= lkey");     // 4a
        if (k.ldat < 0)            details.append((++count>1? "," : "")).append(" ldat is negative"); // 4b
        details.append(' ');
        handleBadKeywordBlock(k, "unpack", details);
        return details.toString();
      }
    }
    return null;
  }

  /** Find the named key in the current scope. This is identical to <code>find(name,SCOPE)</code>.
   *  @param name The keyword name.
   *  @return The key or null if not found.
   */
  public Key find (String name) { // TODO: This should be final
    return find(name,SCOPE);
  }

  /** Find the named key in the specified scope. If there are multiple instances the first one is
   *  returned.
   *  @param name  The keyword name
   *  @param where The keyword scope: {@link #SCOPE} (current scope), {@link #MAIN}, or {@link #EXTENDED}
   *  @return Key if found, null if not found
   */
  public Key find (String name, int where) {
    int i,off,len; Key k=new Key();
    byte[] tmp, bname=name.getBytes();
    int start=scope_start, end=scope_end;
    if (where==MAIN) { start=-isize; end=0; }
    else if (where==EXTENDED) { start=0; end=size; }
    for (k.offset=start; k.offset<end; k.offset+=k.length) {
      String invalidDetails = unpack(k);
      if (invalidDetails != null) {           // since 3.1.1 (will get here if NoAbort is set)
        if (k.length >= HEADER_LEN) continue; // try next keyword block
        else                        break;    // bad length, can't recover from this (break out of loop)
      }
      if (k.offset<0) { tmp=ibuf; off=IKOFF+k.offset+isize; len=k.ltag; }
      else { tmp=buf; off=k.offset+HEADER_LEN+k.ldat; len=k.ltag; }
      if (tmp[off+1]==COLON) { off+=2; len-=2; }
      if (len!=bname.length) continue;
      for (i=0; i<bname.length; i++)
        if (bname[i]!=tmp[off+i]) break;
      if (i==bname.length) return k;
    }
    return null;
  }

  /** Check validity of this block up to the specified length.
   *  @return true is valid (up to specified size), otherwise false for invalid block(s)
   */
  private boolean chk (int maxlen) {
    byte _rep = (hcb==null)? Shell.rep: hcb.headRep;
    Key k=new Key();
    if (size>8 && Convert.unpackL(buf,4,_rep)<200) convertMartes();
    int end = (maxlen>=0)?  Math.min(size,maxlen) : size;
    for (k.offset=0; k.offset<end; k.offset+=k.length) {
      String invalidDetails = unpack(k);
      if (invalidDetails != null) {                  // since 3.1.1 (will get here is NoAbort is set)
        return false;                                // 1. since 3.1.1, invalid blocks for BUG 2548
      }
      if ((k.offset + k.length) > size ||            // 2. bad length (beyond size of extended header)
          (k.offset + HEADER_LEN + k.ldat + k.ltag) > size) { // 3. bad tag length and/or data length (beyond size of extend header)
        handleBadKeywordBlock(k, "chk", "bad tag length and/or data length");
        return false;
      }
      for (int i=0,j=k.offset+HEADER_LEN+k.ldat; i<k.ltag; i++) {
        char c = (char)buf[j+i];
        if (c<' ' || c>'~') {                        // 4. bad char in (tag) name
          handleBadKeywordBlock(k, "chk", "bad char in tag name");
          return false;
        }
      }
    }
    return true;
  }

  /** Converts old MARTES keywords to the standard form. See the BLUE 1.0 ICD for details on the
   *  format of the MARTES keywords.
   */
  private void convertMartes () {
    byte _rep = (hcb==null)? Shell.rep: hcb.headRep;
    int off,next;
    byte type;
    byte[] tmp = new byte[256];
    Key k=new Key(); k.offset=0;
    for (off=0; off<size; off=next) {
      next = Convert.unpackL(buf,off+0,_rep);
      int ltag = Convert.unpackL(buf,off+4,_rep);
      int ldat = Convert.unpackL(buf,off+8,_rep);
      int idat = 12 + (ltag+3)&0xFC;
      // type is before colon if in name, else default to ASCII
      type = (buf[off+13]==COLON)? buf[off+12] : ASCII;
      System.arraycopy(buf,off+12, tmp,0, ltag);  // save off name
      k.length = (8+ldat+ltag+7)&0xFFFFFFF8; // round to next 8 byte boundary
      Convert.packL(buf,k.offset+0,k.length);
      Convert.packI(buf,k.offset+4,(short)(k.length-ldat));
      if (_rep!=Shell.rep) {
        Convert.rep(buf,k.offset+0, LONG,1, Shell.rep,_rep);
        Convert.rep(buf,k.offset+4, INT,1, Shell.rep,_rep);
      }
      buf[k.offset+6] = (byte)ltag;
      buf[k.offset+7] = type;
      System.arraycopy (buf,off+12+((ltag+3)&0xFC), buf,k.offset+8, ldat);
      System.arraycopy (tmp,0, buf,k.offset+8+ldat, ltag);
      k.offset += k.length;
    }
    size = k.offset;
  }

  /** Extracts the name for a key.
   *  @param k The key.
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - this is only used internal to Keywords and XBC")
  public void extractName (Key k) {
    if (k.ltag < 0) { // since 3.1.1 for BUG 2548
      handleBadKeywordBlock(k, "extractName", "negative ltag");
      return;
    }
    if (k.offset<0) k.name = new String(ibuf,IKOFF+k.offset+isize,k.ltag);
    else k.name = new String(buf, k.offset+HEADER_LEN+k.ldat, k.ltag);
    if (k.ltag>1 && k.name.charAt(1)==':') k.name = k.name.substring(2);
  }

  /** Extracts the data for a key.
   *  @param k The key.
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - this is only used internal to Keywords and XBC")
  public void extractData (Key k) {
    byte _rep = (hcb==null)? Shell.rep: hcb.headRep;
    int offset = k.offset + HEADER_LEN;
    if (k.offset<0) k.value = new String(ibuf,IKOFF+k.offset+isize+k.ltag+1,k.ldat);
    else if (Data.isString(k.type)) k.value = new String(buf,offset,k.ldat);
    else if (EXTRACTINPLACE && (_rep==Shell.rep||k.type==BYTE)) {
        k.value = new Data(buf,offset,k.ldat,k.type);
    } else {
      byte[] buffer = new byte[k.ldat];
      System.arraycopy(buf,offset, buffer,0, k.ldat);
      if (_rep!=Shell.rep&&k.type!=BYTE) {
        Convert.rep (buffer,0, k.type, k.ldat/Data.getBPS(k.type), _rep,Shell.rep);
      }
      k.value = new Data(buffer,0,k.ldat,k.type);
    }
  }

  /** Gets all of the keys in a collection (setting scope to "ALL"). This is
   *  identical to using <code>getAll("ALL",false)</code>.
   *  @return The collection containing the keys.
   *  @since NeXtMidas 3.3.0
   */
  public final Collection<Map.Entry<String,Object>> getAll () {
    return getAll("ALL",false);
  }

  /** Gets all of the keys in a collection. <br>
   *  <br>
   *  If a corrupted keyword block is detected, an exception will be thrown, except if the
   *  {@link #NOABORT} flag is set, in which case this function will return all of the keywords
   *  it was able to extract before encountering the error (sorted if applicable) followed by
   *  an entry with the key name {@code "<error>"}. Note that in the error case, calling this
   *  function to separately extract the main and extended header keywords (e.g. "MAIN",
   *  "EXT") *may* yield more keywords than calling this function with a scope of "ALL".
   *  @param scope The scope to use, one of: {@code "ALL", "MAIN", "EXT" or "EXTENDED", "<scope>"}.
   *  @param sort  Sort the entries before returning?
   *  @return The collection containing the keys.
   *  @since NeXtMidas 3.3.0
   */
  public final Collection<Map.Entry<String,Object>> getAll (String scope, boolean sort) {
    if (scope.startsWith("EXT")) scope = "EXT"; // since 3.9.1 support "EXTENDED" for NXM-3100
    Iterator iter = iterator(scope);
    boolean  err  = false;

    ArrayList<Map.Entry<String,Object>> keys = new ArrayList<>();
    try {
      while (iter.hasNext()) {
        Key k = iter.next();
        keys.add(new AbstractMap.SimpleEntry<>(k.name, k.value));
      }
    }
    catch (RuntimeException e) {
      // Note that we don't print out the exception since it should already have been printed as a warning
      if ((flags & NOABORT) == 0) throw e;
      err = true;
    }

    if (sort) {
      Collections.sort(keys, StringUtil.KEY_COMPARATOR);
    }

    if (err) {
      keys.add(new AbstractMap.SimpleEntry<String,Object>("<error>", "<corrupted keyword block>"));
    }
    return keys;
  }

  /** Gets (extracts) the value for the specified keyword name. If multiples are found this will
   *  return the first one.
   *  @param name keyword name to get
   *  @return value for specified keyword name, null if keyword is not found.
   */
  public Object get (String name) {
    Key k = find(name);
    if (k==null) return null;
    extractData(k);
    return k.value;
  }

  /** Gets (extracts) the value for the specified keyword name as an integer.
   *  @param name keyword name to get
   *  @param def  default value to return if keyword is not found
   *  @return integer value for specified keyword name
   */
  public int getL (String name, int def) {
    Object value = get(name);
    if (value==null) return def;
    else return Convert.o2l(value);
  }

  /** Gets (extracts) the value for the specified keyword name as an double.
   *  @param name keyword name to get
   *  @param def  default value to return if keyword is not found
   *  @return double value for specified keyword name
   */
  public double getD (String name, double def) {
    Object value = get(name);
    if (value==null) return def;
    else return Convert.o2d(value);
  }

  /** Gets (extracts) the .UNIT or .UNITS value for the specified keyword name as an integer.
   *  This is identical to <code>getUnitsFor(name,null,UNITS_WARN)</code>.
   *  @param name  keyword name to get the units for.
   *  @return Integer value for the units specifier or null if not found.
   *  @since NeXtMidas 3.3.0
   */
  public final Integer getUnitsFor (String name) {
    return getUnitsFor(name,null,UNITS_WARN);
  }

  /** Gets (extracts) the .UNIT or .UNITS value for the specified keyword name
   *  as an integer. This is identical to <code>getUnitsFor(name,def,UNITS_WARN)</code>.
   *  @param name  keyword name to get the units for.
   *  @param def   default value to return if keyword is not found
   *  @return Integer value for the units specifier or def if not found.
   *  @since NeXtMidas 3.3.0
   */
  public final Integer getUnitsFor (String name, Integer def) {
    return getUnitsFor(name,def,UNITS_WARN);
  }

  /** Gets (extracts) the .UNIT or .UNITS value for the specified keyword name
   *  as an integer. The Platinum ICD Vol. 1 calls out support for ".UNITS" in
   *  section 1.8.1 and ".UNITS" in section 1.8.2; as such this function checks
   *  for both of them. If both are present and don't match a warning may be
   *  issued.
   *  @param name  keyword name to get the units for (e.g. use name="ABC" to get
   *               the value of "ABC.UNITS" or "ABC.UNIT").
   *  @param def   default value to return if keyword is not found
   *  @param flags Controls the handling of mismatch between ".UNITS" and ".UNIT"
   *               in the event that both are present (see {@link #UNITS_WARN} and {@link #UNITS_PREFER_UNIT}).
   *  @return Integer value for the units specifier (see {@link nxm.sys.inc.Units}) -or- def if not found.
   *  @since NeXtMidas 3.3.0
   */
  public Integer getUnitsFor (String name, Integer def, int flags) {
    Object units = get(name+".UNITS");
    Object unit  = get(name+".UNIT");

    if ((unit == null) && (units == null)) return def;
    if ((unit == null) && (units != null)) return Convert.o2l(units);
    if ((unit != null) && (units == null)) return Convert.o2l(unit);

    int _units = Convert.o2l(units);
    int _unit  = Convert.o2l(unit);
    if (_units == _unit) return _units; // both match

    if ((flags&UNITS_WARN) != 0) {
      warn("Found keyword mismatch "+name+".UNIT="+_unit+" and "
          +name+".UNITS="+_units+" using "+name+".UNITS="+_units);
    }
    return ((flags&UNITS_PREFER_UNIT) != 0)? _unit : _units;
  }

  /** Gets (extracts) the value for the specified keyword name as an integer.
   *  @param name keyword name to get
   *  @param def  default value to return if keyword is not found
   *  @return integer value for specified keyword name
   *  @since NeXtMidas 3.3.0
   */
  public Integer getL (String name, Integer def) {
    Object value = get(name);
    if (value==null) return def;
    else return Convert.o2l(value);
  }

  /** Gets (extracts) the value for the specified keyword name as a double.
   *  @param name keyword name to get
   *  @param def  default value to return if keyword is not found
   *  @return double value for specified keyword name
   *  @since NeXtMidas 3.3.0
   */
  public Double getD (String name, Double def) {
    Object value = get(name);
    if (value==null) return def;
    else return Convert.o2d(value);
  }

  /** Gets (extracts) the value for the specified keyword name as a string.
   *  @param name keyword name to get
   *  @param def  default value to return if keyword is not found
   *  @return double value for specified keyword name
   *  @since NeXtMidas 3.5.4
   */
  public String getS (String name, String def) {
    Object value = get(name);
    if (value==null) return def;
    else return Convert.o2s(value);
  }

  /** Adds the specified keyword name/value to this (allowing duplicate keyword names).
   *  This is identical to <code>set(name,obj,ADD)</code>.
   *  @param name keyword name to add
   *  @param obj value for specified keyword name to add
   */
  public void add (String name, Object obj) { // TODO: Mark as final
    set(name,obj,ADD);
  }

  /** <b>Work in progress may change within this beta series:</b>
   *  Adds the specified keyword name/value/units to this based on the specified
   *  flags. This is identical to: <code>addWithUnits(name,value,units,0)</code>.
   *  @param name  keyword name to set.
   *  @param value value for specified keyword name to set.
   *  @param units The units accompanying the value.
   *  @since NeXtMidas 3.3.0
   */
  public final void addWithUnits (String name, Object value, Object units) {
    addWithUnits(name,value,units,0);
  }

  /** Adds the specified keyword name/value/units to this based on the specified
   *  flags.
   *  @param name  keyword name to set.
   *  @param value value for specified keyword name to set.
   *  @param units The units accompanying the value. This can either be the integer
   *               value from {@link nxm.sys.inc.Units} or the corresponding string.
   *  @param flags The flags specifying the choice of ".UNIT" or ".UNITS" (see
   *               {@link #UNITS_PREFER_UNIT}).
   *  @since NeXtMidas 3.3.0
   */
  public final void addWithUnits (String name, Object value, Object units, int flags) {
    setWithUnits(name,value,units,flags|ADD);
  }

  /** Copies all of the mappings from the specified map (allowing duplicate keyword names)
   *  to this using. This is identical to <code>setAll(map,ADD)</code>.
   *  @param map Mappings to be copied in this.
   *  @since NeXtMidas 3.1.2
   */
  public void addAll (Map<String,Object> map) { // TODO: Mark as final
    setAll(map,ADD);
  }

  /** Puts the specified keyword name/value to this (replacing any existing keyword name). This
   *  is identical to <code>set(name,obj,REPLACE)</code>.
   *  @param name keyword name to put
   *  @param obj value for specified keyword name to put
   */
  public void put (String name, Object obj) {
    set(name,obj,REPLACE);
  }

  /** Puts the specified keyword name/value/units to this based on the specified
   *  flags. This is identical to: <code>putWithUnits(name,value,units,UNITS_MATCH)</code>.
   *  @param name  keyword name to set.
   *  @param value value for specified keyword name to set.
   *  @param units The units accompanying the value.
   *  @since NeXtMidas 3.3.0
   */
  public final void putWithUnits (String name, Object value, Object units) {
    putWithUnits(name,value,units,UNITS_MATCH);
  }

  /** Puts the specified keyword name/value/units to this based on the specified
   *  flags. Note that the behavior of this method is undefined if multiple
   *  matches already exist.
   *  @param name  keyword name to set.
   *  @param value value for specified keyword name to set.
   *  @param units The units accompanying the value. This can either be the integer
   *               value from {@link nxm.sys.inc.Units} or the corresponding string.
   *  @param flags The flags specifying the choice of ".UNIT" or ".UNITS" (see
   *               {@link #UNITS_MATCH} and {@link #UNITS_PREFER_UNIT}).
   *  @since NeXtMidas 3.3.0
   */
  public final void putWithUnits (String name, Object value, Object units, int flags) {
    setWithUnits(name,value,units,flags|REPLACE);
  }

  /** Copies all of the mappings from the specified map. Duplicate keyword names will
   *  replace existing entry. This is identical to <code>setAll(map,REPLACE)</code>.
   *  @param map Mappings to be copied in this.
   *  @since NeXtMidas 3.1.2
   */
  public void putAll (Map<String,Object> map) { // TODO: Mark as final
    setAll(map,REPLACE);
  }

  /** Copies all of the mappings from the specified collection. Duplicate keyword names will
   *  replace existing entry. This is identical to <code>setAll(entries,REPLACE)</code>.
   *  @param entries Mappings to be copied in this.
   *  @since NeXtMidas 3.1.2
   */
  public void putAll (Collection<Map.Entry<String,Object>> entries) {
    setAll(entries,REPLACE);
  }

  /** Copies all of the mappings from the specified map to this using the specified flags. This
   *  is identical to <code>setAll(map.entrySet(),flags)</code> (except for additional checks for null).
   *  @param map   Mappings to be copied in this.
   *  @param flags The flags used when setting the keywords.
   *  @since NeXtMidas 3.1.2
   */
  public void setAll (Map<String,Object> map, int flags) { // TODO: Mark as final
    if (map==null) return; // if map is null, don't need to copy to prevent NPE
    setAll(map.entrySet(),flags);
  }

  /** Copies all of the mappings from the specified collection to this using the specified flags.
   *  @param entries Mappings to be copied in this set of keywords.
   *  @param flags   The flags used when setting the keywords.
   *  @since NeXtMidas 3.3.0
   */
  public void setAll (Collection<Map.Entry<String,Object>> entries, int flags) {
    if (entries==null) return; // if map is null, don't need to copy to prevent NPE
    for (Map.Entry<String,Object> e : entries) {
      set(e.getKey(), e.getValue(), flags);
    }
  }

  /** Sets (add or put) the specified keyword name/value/units to this based on
   *  the specified flags. Note that when using {@link #REPLACE}, the behavior
   *  of this method is undefined if multiple matches already exist.
   *  @param name  keyword name to set.
   *  @param value value for specified keyword name to set.
   *  @param units The units accompanying the value. This can either be the integer
   *               value from {@link nxm.sys.inc.Units} or the corresponding string.
   *  @param flags The flags specifying {@link #ADD} or {@link #REPLACE} plus the
   *               choice of ".UNIT" or ".UNITS" (see {@link #UNITS_MATCH} and
   *               {@link #UNITS_PREFER_UNIT}). If neither {@link #ADD} or
   *               {@link #REPLACE} are set, {@link #REPLACE} is assumed.
   *  @since NeXtMidas 3.3.0
   */
  @InternalUseOnly("Since NeXtMidas 3.3.0")
  private void setWithUnits (String name, Object value, Object units, int flags) {
    String  kwdName    = (name.length()>2 && name.charAt(1)==':')? name.substring(2) : name;
    boolean preferUnit = ((flags&UNITS_PREFER_UNIT)!=0);

    // Check the ADD/REPLACE flags
    if ((flags&(ADD_REPLACE)) == 0) {
      flags = flags & REPLACE;
    }
    else if ((flags&(ADD_REPLACE)) == ADD_REPLACE) {
      throw new MidasException("ADD and REPLACE flags are mutually exclusive, but both were given");
    }

    // If doing a REPLACE check for old entries
    if ((flags & REPLACE) != 0) {
      Key oldUnits = find(kwdName+".UNITS");
      Key oldUnit  = find(kwdName+".UNIT");

      // Match units if applicable
      if ((flags&UNITS_MATCH) != 0) {
        if ((oldUnits != null) && (oldUnit == null)) preferUnit = false;
        if ((oldUnits == null) && (oldUnit != null)) preferUnit = true;
      }

      // Delete the one we aren't going to use
      if ((oldUnits != null) &&  preferUnit) delete(oldUnits);
      if ((oldUnit  != null) && !preferUnit) delete(oldUnit);
    }

    // Set the keywords (note that only the ADD_REPLACE get passed down) the actual
    // type used for the units is arbitrary with "X:", "L:", "I:", and "B:" all
    // valid ('I:" and "L:" tend to be the most common), in this case we use "B:"
    // since it is the most space-efficient and matches the size to encode
    // sub-record units in Type 5000 files.
    if (preferUnit) {
      set(name,                  value,             flags&ADD_REPLACE);
      set("B:"+kwdName+".UNIT",  getUnitsID(units), flags&ADD_REPLACE);
    }
    else {
      set(name,                  value,             flags&ADD_REPLACE);
      set("B:"+kwdName+".UNITS", getUnitsID(units), flags&ADD_REPLACE);
    }
  }

  /** Sets the specified keyword name/value. This will either add (default) or put/replace (via
   *  {@link #REPLACE} flag) the keywords based on the specified flags.
   *  @param name   Keyword name to set. The name may contain an optional type prefix (e.g. "D:FOO")
   *                that will force a type conversion prior to insertion.
   *  @param obj    Value for specified keyword name to set.
   *  @param flags  SET keyword flags mask (e.g. {@link #REPLACE}).
   */
  public void set (String name, Object obj, int flags) {
    char type = '_';
    if (name.length()==0) return;
    if (name.length()>2 && name.charAt(1)==':') {
      type = name.charAt(0); name = name.substring(2);
      obj = Convert.o2o(obj,type,null);
    }
    else {
      // This section added in NeXtMidas 3.3.0
      if ((obj instanceof Data) || (obj instanceof String) || (obj instanceof Table)) {
        // keep as-is
      }
      else {
        // try to convert
        obj = Convert.o2o(obj,type,null);
      }
    }
    // If key name contains DOT_SUBSTITUTE, ALWAYS substitute with "."
    if (name.contains(getDotSubstitute())){
      name = name.replace(getDotSubstitute(), ".");
    }
    if (obj instanceof Data) {
      Data data = (Data)obj;
      if (data.isString()) {
        String string = ""+obj;
        set(name, data.type, string.getBytes(), string.length(), flags);
      } else {
        set(name, data.type, data.buf, data.size*data.bpe, flags);
      }
    }
    else if (obj instanceof String) {
      String string = (String)obj;
      set(name, ASCII, string.getBytes(), string.length(), flags);
    }
    else if (obj instanceof Table) {
      byte[] bbuf = ((Table)obj).toBytes();
      set(name, ASCII, bbuf, bbuf.length, flags);
    }
    else {
      throw new MidasException ("Illegal Keyword Data type: "+obj);
    }
  }

  /** Sets the specified keyword name/value. This is a special-purpose method and not for general
   *  use, see {@link #set(String,Object,int)}.
   *  @param name   keyword name (tag).
   *  @param type   type of keyword value.
   *  @param data   keyword value as byte array.
   *  @param len    length of data (in bytes).
   *  @param flags  SET keyword flags mask (e.g. {@link #REPLACE}).
   */
  public void set (String name, byte type, byte[] data, int len, int flags) {
    byte  _rep   = (hcb==null)? Shell.rep: hcb.headRep;
    int   off    = scope_end, olength = 0;    // default no replace
    byte  ltag   = (byte)name.length();
    int   length = (len+ltag+15)/8*8;
    short lext   = (short)(length-len);
    if (type==STRING) type=ASCII;
    boolean replacingEXT = false;  // default to not a replacement
    boolean replacingMAIN = false; // added support for MAIN keywords (NXM-4003)

    // find current location for replacement
    Key k = null;
    if ((flags&REPLACE)!=0) {
      k = find(name,SCOPE);

      if (k != null) { // we are replacing an existing keyword
        replacingMAIN = k.offset < 0; // Negative offset means the key was found in the main header (NXM-4003)
        replacingEXT  = !replacingMAIN;
      }
    }

    // *** check if should be a MAIN or EXT HEADER keyword ***
    if (replacingEXT) {
      off = k.offset; olength = k.length;
    }
    else if (replacingMAIN || (!scope.equals("EXT") && isReqMain(name))) {
      String value = convertDataToString(name, type, data, len);
      if (value != null) {
        // Even if we are not replacing a keyword, assume this key/value is for MAIN if:
        // 1) The scope included MAIN
        // 2) The key name is a standard MAIN header keyword
        // 3) and the Data type matches the keyword
        putMain(name,value); // MAIN header keywords should be set using putMain
        return;
      }
    }

    // *** setting EXTENDED HEADER keyword ***

    // expand buffer if necessary
    if (buf==null) buf = new byte[EXTSIZE];
    if (size+length-olength > buf.length) {
      byte[] bufo = buf;
      buf = new byte[ (size+length+EXTSIZE)/EXTSIZE*EXTSIZE ];
      System.arraycopy (bufo,0, buf,0, size);
    }
    // create/collapse the hole
    if (length!=olength && off<size) {
      System.arraycopy (buf,off+olength, buf,off+length, size-(off+olength));
    }
    Convert.packL(buf,off+0,length);
    Convert.packI(buf,off+4,lext);
    buf[off+6] = ltag;
    buf[off+7] = type;
    if (_rep!=Shell.rep) {
      if (!Data.isString(type) && type!=BYTE) {
        int nelems = len / Data.getBPS(type); // prior to 3.7.0 this was set to 1 (BUG 3009)
        Convert.rep(data,0,type,nelems,Shell.rep,_rep);
      }
      Convert.rep(buf,off,LONG,1,Shell.rep,_rep);
      Convert.rep(buf,off+4,INT,1,Shell.rep,_rep);
    }
    System.arraycopy (data,0, buf,off+HEADER_LEN, len);                            // set data
    for (int i=0,j=off+HEADER_LEN+len; i<ltag;) buf[j++] = (byte)name.charAt(i++); // set tag name
    scope_end += length-olength;
    size += length-olength;
  }

  private String convertDataToString (String name, byte type, byte[] data, int len) {
    String value = null;
    if (Data.isString(type)) {
      value = new String(data, 0, len);
    }
    else if (type != Data.NONHOMOGENEOUS) {
      if ("TC_PREC".equals(name) && Data.isFloat(type)) {
        Data thisValue = new Data(data);
        value = "" + thisValue.getD(0);
      }
      else if ("PKT_BYTE_COUNT".equals(name) && !Data.isFloat(type)) {
        Data thisValue = new Data(data);
        value = ""+thisValue.getL(0);
      }
    }
    return value;
  }

  /** Deletes a specific keyword.
   *  @param name keyword to delete
   *  @return 0, or length of key.
   *  @see #deleteScope(String)
   *  @see #deleteAll()
   */
  public int delete (String name) {
    if (name.equals("TIMELINE") && hcb!=null) {
      hcb.setTimeLineHandler(null); // <-- Since 2.5.0 (DataFile will re-insert it otherwise)
    }
    return delete(find(name));
  }

  /** Deletes a key. */
  @InternalUseOnly("Since NeXtMidas 3.3.0")
  private int delete (Key k) {  // Since NeXtMidas 3.3.0
    if (k==null) return 0;
    if (k.offset<0) return deleteMain(k.name);
    size -= k.length;  // new length, close the gap
    System.arraycopy(buf,k.offset+k.length, buf,k.offset, size-k.offset);
    if (scope_start>k.offset) scope_start -= k.length;
    if (scope_end>k.offset) scope_end -= k.length;
    return size; // corrected to size in 3.7.2
  }

  /** Deletes all keywords within a given scope. Before returning, this method will always call
   *  {@link #clearScope()} since the "current" scope was just deleted.
   *  @param scope The scope to delete.
   *  @return true
   *  @since NeXtMidas 2.7.0
   *  @see #delete(String)
   *  @see #deleteAll()
   */
  public boolean deleteScope (String scope) {
    int i = setScope(scope);
    if (i > 0) {
      if (buf == null) { // since 3.9.1 for NXM-3100
        Shell.info("No "+scope+" keywords deleted in header as keyword buffer is null");
        return true;
      }
      int len = size - scope_end;
      System.arraycopy(buf, scope_start, buf, scope_end, len);
      size = len;
    }
    clearScope();
    return true;
  }

  /** Deletes all extended header keywords with the prefix ""VOLATILE_KW_"
   *  @return true if volatile keywords were deleted
   */
  public boolean deleteVolatileKeywords () {
    boolean deleted = false;
    Keywords.Iterator kwi = this.iterator();
    List<Key> volatileKeys = new LinkedList<>();
    while (kwi.hasNext()) {
      Key key = kwi.next();
      if (key.name.startsWith("VOLATILE_KW_")) {
        delete(key); // This delete will prevent any index out of bounds issues
        volatileKeys.add(key);
        deleted = true;
      }
    }
    return deleted;
  }

  /** Put/replace a keyword into the MAIN header. If the keyword specified is included in
   *  {@link #MAIN_HEADER_KEYWORDS}, and the header is full, any keywords in the MAIN header that
   *  are not in {@link #MAIN_HEADER_KEYWORDS} are subject to automatic deletion as is needed to
   *  add the new keyword. If a keyword is deleted and <code>warn=true</code>, a warning will be issued.
   *  <br>
   *  This is not a general-purpose method. Users should rarely need to modify MAIN header keywords,
   *  and where modification is required the proper way to do this is via:
   *  <pre>
   *    kw.setScope("MAIN");
   *    kw.put(name,value);
   *  </pre>
   *  @param name  The keyword name.
   *  @param value The value of the keyword.
   *  @param warn  Issue warnings if (1) the main header is full, or (2) if inserting a keyword in
   *               the main header that belongs in the extended header (see {@link #MAIN_HEADER_KEYWORDS}).
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - this is only used internal to Keywords and DataFile")
  public void putMain (String name, String value, boolean warn) {
    if (name==null || name.length()<=0) return; // check for degenerate cases

    // strip type casting - internal keywords must be Ascii anyway
    if (name.length()>2 && name.charAt(1)==':') name = name.substring(2);

    if (!isOkMain(name) && warn) {
      warn(name+"="+value+" should only be set in the EXTENDED header");
    }
    if (isize<0) { initFromDataFile(); }
    deleteMain(name); // no duplicates in main header keywords
    // form the new entry
    String knv = name + "=" + value + NULL;
    int off = isize;
    int len = knv.length();
    if ((off+len>IKMAX) && isReqMain(name)) {
      // Delete as many keywords from MAIN as are necessary to make room. Keep only those listed
      // in MAIN_HEADER_KEYWORDS. Note that it is (remotely) possible that this will not find enough
      // room in the header.
      Table    main = makeTable(true,MAIN);
      String[] keys = main.getKeys();
      for (int i = keys.length-1; (i >= 0) && (off+len>IKMAX); i--) {
        if (!isReqMain(keys[i])) {
          if (warn) warn("No room in MAIN header for "+name+"='"+value+"', "+ "deleting "
                        + keys[i]+"='"+main.getS(keys[i])+"' to make room");
          deleteMain(keys[i]);
          off = isize;
        }
      }
    }
    if (off+len>IKMAX) {
      if (warn) warn("No room in MAIN header keywords for entry: "+name+"='"+value+"'");
      return;
    }
    System.arraycopy (knv.getBytes(),0, ibuf,IKOFF+off, len);
    isize += len;
    hcb.setKeyLength(isize);
  }

  /** Put/replace a keyword into the MAIN header. This is the same as <code>putMain(name,value,true)</code>.
   *  @param name  The keyword name.
   *  @param value The value of the keyword.
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - this is only used internal to Keywords and DataFile")
  public void putMain (String name, String value) {
    putMain(name,value,true);
  }

  /** Indicates if the given name is in MAIN_HEADER_KEYWORDS. */
  private static boolean isReqMain (String name) {
    return (Arrays.binarySearch(MAIN_HEADER_KEYWORDS, name) >= 0);
  }

  /** Indicates if the given name is in MAIN_HEADER_KEYWORDS or MAIN_HEADER_IGNORE. */
  private static boolean isOkMain (String name) {
    return isReqMain(name)
        || (Arrays.binarySearch(MAIN_HEADER_IGNORE, name) >= 0);
  }

  /** Handle how a bad keyword block found (e.g. throw MidasException unless the {@link #NOABORT}
   *  flag has been set. If not aborting, then display a warning once per fromMethod.
   *  @param key         keyword block that is invalid
   *  @param fromMethod  method where the warning is coming from
   *  @param moreDetails any additional details/information
   *  @throws MidasException unless the {@link #NOABORT} flag has been set.
   *  @since NeXtMidas 3.1.1
   */
  private void handleBadKeywordBlock (Key key, String fromMethod, CharSequence moreDetails) throws MidasException {
    if ((flags&NOABORT)==0) {
      throw new MidasException(fromMethod+" found invalid keyword block ("+moreDetails+") " + key.headerInfo());
    }
    else if (!warnedForInvalidBlockSet.contains(fromMethod)) { // otherwise only warn once per fromMethod
      warnedForInvalidBlockSet.add(fromMethod);
      warn(fromMethod + " invalid block (" + moreDetails + ") "+ key.headerInfo());
    }
  }

  /** Prints out a warning. */
  private void warn (String msg) {
    // String wmsg = "Keywords: " + msg + " file='" + (hcb==null? null : hcb.getName()) + "'."; // prior to 3.1.1
    String wmsg = "Keywords: " + msg + " file='" + hcb + "'."; // since 3.1. since DataFile has more informative toString()
    if ((hcb != null) && (hcb.M != null)) { hcb.M.warning(wmsg); }
    else                                  { Shell.warning(wmsg); }
  }

  /** Get a keyword from the MAIN header. Returns the FIRST occurrence of the keyword or NULL if not
   *  found. <br>
   *  <br>
   *  This is not a general-purpose method. Users should rarely need to modify MAIN header keywords,
   *  and where modification is required the proper way to do this is via:
   *  <pre>
   *    kw.setScope("MAIN");
   *    kw.getS(name,null);
   *  </pre>
   *  @param name The keyword name.
   *  @return The value or null if not found.
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - this is only used internal to Keywords and DataFile")
  public String getMain (String name) {
    if (isize<0) { initFromDataFile(); } // auto attach
    // check for degenerate cases
    if (isize<=0 || name==null || name.length()==0) return null;
    // strip type casting - internal keywords must be ASCII anyway
    if (name.length()>2 && name.charAt(1)==':') name = name.substring(2);
    Key k = find(name,MAIN);
    if (k==null) return null;
    extractData(k);
    return (String)k.value;
  }

  /** Delete a keyword from the MAIN header. <br>
   *  <br>
   *  This is not a general-purpose method. Users should rarely need to modify MAIN header keywords,
   *  and where modification is required the proper way to do this is via:
   *  <pre>
   *    kw.setScope("MAIN");
   *    kw.deleteKeyword(name);
   *  </pre>
   *  @param name The keyword name.
   *  @return The length of the old keyword of 0 if not found.
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - this is only used internal to Keywords and DataFile")
  public int deleteMain (String name) {
    if (isize<0) { initFromDataFile(); } // auto attach to DataFile
    // check for degenerate cases
    if (isize<=0 || name==null || name.length()==0) return 0;
    // strip type casting - internal keywords must be ASCII anyway
    if (name.length()>2 && name.charAt(1)==':') name = name.substring(2);
    Key k = find(name,MAIN);
    if (k==null) return 0;
    int off=k.offset+isize;
    int len=k.ltag+1+k.ldat+1;
    if (off+len<isize) System.arraycopy(ibuf,IKOFF+off+len, ibuf,IKOFF+off, isize-off-len);
    isize -= len;
    hcb.setKeyLength(isize);
    return len;
  }

  /** List a given keyword. */
  private String list (Key key) { // deprecated in NeXtMidas 3.3.0, marked private in 3.7.0
    // Legacy X-Midas: Use '=' for main and '==' for extended to distinguish the two
    String sep = (key.offset <0) ? "=" : "==";
    if (key.value instanceof String)
      return "S:"+key.name+sep+key.value;
    else if (key.value instanceof Data) {
      Data data = (Data)key.value;
      return (char)data.type+":"+key.name+sep+key.value;
    }
    return "?:"+key.name+"=unknown (invalid keyword block at offset="+key.offset+")";
  }

  /** List a given keyword. Intended for use by KEYWORD command and {@link DataFile}.
   *  @param kwd   Keyword to list
   *  @param scope The scope used.
   *  @param pad   The "pretty print" padding value (0=disable).
   *  @param flags Listing flags (e.g. {@link #UNITS_LIST}).
   *  @return The keyword listing.
   *  @since NeXtMidas 3.3.0
   */
  @InternalUseOnly("Since created")
  public static CharSequence list (Map.Entry<String,Object> kwd, String scope, int pad, int flags) {
    // Notes:
    //   1. Use '=' for MAIN and '==' for EXT to match X-Midas
    //   2. The type determination is taken from the old list(Key) function
    StringBuilder str = new StringBuilder(512); // 512 is a better default than 16
    String        sep = (scope.equals("MAIN"))? "= " : "==";
    String        key = kwd.getKey();
    Object        val = kwd.getValue();

         if (val instanceof String) str.append('S');
    else if (val instanceof Data  ) str.append(((Data)val).getFormatTypeChar());
    else                            str.append('?');

    str.append(':');
    str.append(key);
    if (pad > 0) {
      while (str.length() < pad) str.append(' ');
      str.append(' ').append(sep).append(' ');
    }
    else {
      str.append(sep);
    }
    str.append(val);

    if (((flags&UNITS_LIST)!=0) && (key.endsWith(".UNIT") || key.endsWith(".UNITS"))) {
      // Show units name in parenthesis (the use of ".UNIT" or ".UNITS" comes
      // from section 1.8.1 is the Platinum ICD Vol. 1). It this fails for any reason, just omit it.
      try {
        String units = DataFile.getUnitsVarName(Convert.o2i(val));
        str.append("\t(").append(units).append(')');
      }
      catch (Exception e) {
        // ignore
      }
    }
    return str;
  }

  /** Dump the keywords in the MAIN and EXTENDED header to a text file.
   *  @param tf The file to dump the keywords to.
   *  @see #load(TextFile)
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - this is only intended for use by the KEYWORDS command")
  public void dump (TextFile tf) {
    for (Iterator kwi=iterator(); kwi.getNext();) tf.writeln( kwi.list() );
  }

  /** Make a flat table out of the keywords. This is identical to <code>makeTable(true)</code>.
   *  @return A table containing all main and extended header keywords
   */
  public Table makeTable () { // TODO: Mark as final
    return makeTable(true);
  }

  /** Make a table out of the keywords in the MAIN and EXTENDED header. Table can be flat or
   *  have inner tables MAIN and EXTENDED
   *  @param flat Put main and extended keywords at the top level if true
   *  @return A table containing all main and extended header keywords
   */
  public Table makeTable (boolean flat) {
    return makeTable(flat,SCOPE);
  }

  /** Make a table out of the keywords in the MAIN and EXTENDED header.  Table can be flat or
   *  have inner tables MAIN and EXTENDED
   *  @param flat  Put main and extended keywords at the top level if true.
   *  @param scope {@link #SCOPE}=Include Current Scope Only, {@link #MAIN}=Include only MAIN header,
   *               {@link #EXTENDED}=Include only EXTENDED header, or <code>MAIN|EXTENDED</code>=Include
   *               both MAIN and EXTENDED header (i.e. include everything).
   *  @return A table the keywords.
   */
  private Table makeTable (boolean flat, int scope) {
    Table tb = new Table(Table.KV, Table.ADD);
    Table mainTab = null;
    Table extendTab = null;
    if (!flat) {
      mainTab   = new Table(Table.KV, Table.ADD); tb.put("MAIN", mainTab);
      extendTab = new Table(Table.KV, Table.ADD); tb.put("EXTENDED",extendTab);
    }
    for (Iterator kwi=iterator(scope); kwi.getNext();) {
      if (flat) {
        kwi.name = kwi.getName(true); // Handle Platinum, since 4.1.2;
        tb.putx(kwi.name, kwi.value);
      } else {
        if (kwi.curentKey.offset < 0) {
          mainTab.put(kwi.name, kwi.value);
        }
        else {
          // PLATINUM files can contain "." in the keyword which result in NULL values.
          // NXM-2849 substitute "." with DOT_SUBSTITUTE as of 1/27/2016.  This will change in the future.
          kwi.name = kwi.getName(true);
          extendTab.put(kwi.name, kwi.value);
        }
      }
    }
    return tb;
  }

  /** Load the keywords from a text file using = for main == or extended.
   *  @param tf text file to load keywords from (file must be open).
   *  @see #dump(TextFile)
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - this is only intended for use by the KEYWORDS command")
  public void load (TextFile tf) {
    int ie,off,len; char type; String text,name;
    while ( (text=tf.readProper()) != null) {
      off=0; len=text.length(); type='_';
      if (text.charAt(1) == ':') {
        type = text.charAt(0);
        if (Data.isString((byte)type)) type=DataTypes.STRING;
        off = 2;
      }
      ie = text.indexOf('=');
      // prior to 3.1.2 this check was not present and an ArrayIndexOutOfBoundsexception would be thrown
      if (ie<0) throw new MidasException("Invalid entry "+text+" found in "+tf.getName());
      name = text.substring(off,ie);
      if (((ie+1)<len) && (text.charAt(ie+1)=='=')) {  // extended header
        text = text.substring(ie+2);
        if (type==DataTypes.STRING) add(name,text);
        else add(name,Convert.s2o(text, type, null));
      }
      else {                                           // main header
        text = text.substring(ie+1);
        putMain(name,text);
      }
    }
  }

  /** Load the keywords from a text file into the extended header.
   *  @param tf text file to load keywords from (file must be open).
   *  @since NeXtMidas 3.1.2 (this is the original implementation of {@link #load(TextFile)})
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0 - this is only intended for use by the KEYWORDS command")
  public void loadExt (TextFile tf) {
    int ie,off,len; char type; String text,name;
    while ( (text=tf.readProper()) != null) {
      off=0; len=text.length(); type='_';
      if (text.charAt(1) == ':') {
        type = text.charAt(0);
        if (Data.isString((byte)type)) type=DataTypes.STRING;
        off = 2;
      }
      ie = text.indexOf('=');
      if (ie<0) throw new MidasException("Invalid entry "+text+" found in "+tf.getName());
      name = text.substring(off,ie);
      if (++ie<len && text.charAt(ie)=='=') ie++;
      if (ie>=len) text=""; else text = text.substring(ie);
      if (type==DataTypes.STRING) add (name,text);
      else add(name, Convert.s2o(text,type,null) );
    }
  }

  /** private since 3.7.0, was deprecated in 2.5.3: This method should not be called directly, use {@link DataFile#convertHeaderRep}. */
  private void convertRep (byte from, byte to) { // deprecated in NeXtMidas 2.5.3, marked private in 3.7.0 - TODO remove in 3.9+
    convertExtRep(from,to);
    rep = to;
  }

  /** Convert the representation of the keywords in the extended header.
      @param from From representation
      @param to   To representation
   */
  void convertExtRep (byte from, byte to) { // package-private, used by DataFile
    Key k=new Key();
    for (k.offset=0; k.offset<size; k.offset+=k.length) {
      String invalidDetails = unpack(k,from);
      if (invalidDetails != null) { // since 3.1.1 (will get here is NoAbort is set)
        if (k.length >= HEADER_LEN) continue; // try next keyword block
        else                        break;    // bad length, can't recover from this (break out of loop)
      }
      int off=k.offset;
      Convert.rep(buf,off,LONG,1,from,to);
      Convert.rep(buf,off+4,INT,1,from,to);
      if (!Data.isString(k.type) && k.type!=BYTE)
      Convert.rep(buf, off+HEADER_LEN, k.type,k.ldat/Data.getBPS(k.type),from,to);
    }
  }

  /** Create a copy of the provided keywords that in uses the proper byte ordering
   *
   *  @param kwSrc  source keywords
   *  @param srcRep source keyword data representation
   *  @param newRep new    keyword data representation
   *  @return copy of keywords with correct byte ordering
   */
  @InternalUseOnly
  static Keywords convertedCopyOfKeywords (Keywords kwSrc, byte srcRep, byte newRep) {
    Keywords kw = new Keywords();
    if (kwSrc.size>0) {
      kw.size = kwSrc.size;
      kw.buf = new byte[kwSrc.buf.length];
      System.arraycopy (kwSrc.buf,0, kw.buf,0, kwSrc.size); // copy keyword buffer (in extended header)
      if (srcRep!=newRep) {
        kw.convertExtRep(srcRep, newRep);
      }
    }
    return kw;
  }

  /** Gets all the keywords
   *  @return All the keywords
   *  @since NeXtMidas 3.9.4
   */
  @Override
  public String[] getKeys() {
    ArrayList<String> keys = new ArrayList<>();
    for (Iterator kwi=iterator(); kwi.getNext();) {
      keys.add(kwi.name);
    }
    return keys.toArray(new String[keys.size()]);
  }

  /** Gets all the keywords within a specified scope
   *  @param scope The scope in which to retrieve existing keywords
   *  @return All the keywords within the specified scope
   *  @since NeXtMidas 3.9.4
   */
  public String[] getKeys (String scope) {
    ArrayList<String> keys = new ArrayList<>();
    for (Iterator kwi=iterator(scope); kwi.getNext();) {
      keys.add(kwi.name);
    }
    return keys.toArray(new String[keys.size()]);
  }

  /** Gets all the keywords within a specified scope
   *  @param scope The scope in which to retrieve existing keywords
   *  @param keyPrefix what the key name must start with
   *  @return All the keywords within the specified scope that match
   */
  @InternalUseOnly
  public String[] getKeys (String scope, String keyPrefix) {
    ArrayList<String> keys = new ArrayList<>();
    for (Iterator kwi=iterator(scope); kwi.getNext();) {
      if (kwi.name.startsWith(keyPrefix)) keys.add(kwi.name);
    }
    return keys.toArray(new String[keys.size()]);
  }

  /** Sets the scope (if name="SCOPE") otherwise simply does a put.
      @param name  Key name
      @param value Key value
   */
  @Override
  public Object setKey (String name, Object value) {
    if (name.equals("SCOPE")) setScope((String)value);
    else put(name,value);
    return value;
  }

  /** Gets the scope (if name="SCOPE") otherwise simply does a get.
      @param name  Key name
   */
  @Override
  public Object getKey (String name) {
    if (name.equals("SCOPE")) return scope;
    else return get(name);
  }

  /** Class to hold a keyword */
  public static class Key {
    byte type;   // type of data in keyword data/value
    int  offset; // offset>=0 is relative to start of extended header; ovvset<0 is relative to end of keywords string in main header
    int  length; // Total length of "total" keyword block (lkey)
    int  ltag;   // Length of keyword tag
    int  ldat;   // Length of keyword data (value)
    /** The keyword name/tag */
    public String name;
    /** The keyword value/data */
    public Object value;

    /** default constructor */
    public Key () {}

    /** Gets the keyword type.
     *  @return The type char ('D','F','L',etc.)
     */
    public char getType() {
      return (char)type;
    }

    @Override
    public String toString() {
      String t = (type >= 'A' && type <= 'Z')? Character.toString((char)type): "n/a";
      return "Key: type="+t+" offset="+offset+" length="+length+" ltag="+ltag
            +" ldat="+ldat+" name="+name+" value="+value;
    }

    /** String representation of keyword header info (+offset) for warning/errors messages
        @since 3.1.1
     */
    private String headerInfo () {
      String t = (type >= 'A' && type <= 'Z')? Character.toString((char)type): "n/a";
      return "Key: lkey="+length +" lext="+(length-ldat) +" ltag="+ltag +" type="+t +" offset="+offset +" ldat="+ldat;
    }
  } // end inner class Key

  /** String representation of all keywords */
  @Override
  public String toString() {
    StringBuilder keywordsString = new StringBuilder(512); // 512 is a better default than 16
    for (Iterator kwi=iterator(); kwi.getNext();) {
      if (kwi.value instanceof String)
        keywordsString.append("S:");
      else if (kwi.value instanceof Data) {
        Data data = (Data)kwi.value;
        keywordsString.append((char)data.type).append(":");
      }
      keywordsString.append(kwi.name).append("=").append(kwi.value).append("\n");
    }
    return keywordsString.toString();
  }

  /** Gets an iterator that iterates over the keywords within the current scope. This is identical
   *  to <code>iterator(SCOPE)</code>.
   *  @return The appropriate iterator.
   */
  public Iterator iterator () { // TODO: Mark as final
    return iterator(SCOPE);
  }

  /** Gets an iterator that iterates over the keywords within a specified scope. This is effectively
   *  the same as doing a <code>setScope(..)</code> followed by a call to <code>iterator()</code>.
   *  @param scope The scope to get an iterator for.
   *  @return The applicable iterator or null if the given scope is not found.
   *  @since NeXtMidas 2.7.0
   */
  public Iterator iterator (String scope) {
    int i = setScope(scope);
    return (i <= 0)? null : iterator(SCOPE);
  }

  /** Gets an iterator that iterates over the keywords.
   *  @param scope {@link #SCOPE}=Include Current Scope Only, {@link #MAIN}=Include only MAIN header,
   *               {@link #EXTENDED}=Include only EXTENDED header, or <code>MAIN|EXTENDED</code>=Include
   *               both MAIN and EXTENDED header (i.e. include everything).
   *  @since NeXtMidas 2.5.3
   */
  private Iterator iterator (int scope) {
    int start = 0;
    int end   = 0;

    if (scope == SCOPE)      { start = scope_start;             end = scope_end;           }
    if ((scope&MAIN)!=0)     { start = Math.min(start, -isize); end = Math.max(end, 0);    }
    if ((scope&EXTENDED)!=0) { start = Math.min(start, 0);      end = Math.max(end, size); }

    return new Iterator(this, start, end);
  }

  /** Iterator used to iterate through the keyword block. */
  public static class Iterator implements java.util.Iterator<Key> {
    /** Keywords object to be iterated */
    public  Keywords kw;      // TODO: This should not be public
    /** name of current key */
    public  String   name;    // TODO: This should not be public
    /** value of current key */
    public  Object   value;   // TODO: This should not be public
    private int      end;
    /** current keyword block location (0 for first one) */
    private int      atBlock   = 0;
    private Key      curentKey = new Key();

    /** Iterator Constructor
        @param kw Keywords object
     */
    public Iterator (Keywords kw) {  // TODO: This should not be public
      this(kw, kw.scope_start, kw.scope_end);
    }

    Iterator (Keywords kw, int start, int end) {
      this.kw          = kw;
      this.end         = end;
      curentKey.offset = start;
      curentKey.length = 0;
    }
    @Override
    public boolean hasNext () {
      if (atBlock > 0 && curentKey.length < 1) {
        return false; // since 3.1.1 for BUG 2548 (break out if lkey is zero or negative
      }
      return curentKey.offset+curentKey.length < end;
    }
    /** @return next keyword block, null if at end -OR- if invalid block is found */
    @Override
    public Key next () {
      curentKey.offset += curentKey.length;
      if (curentKey.offset>=end) return null;
      String invalidDetails = kw.unpack(curentKey);
      atBlock++;
      if (invalidDetails != null) { // since 3.1.1 (will get here is NoAbort is set)
        curentKey.name  = null; // reset field
        curentKey.value = null; // reset field

        if (curentKey.length >= HEADER_LEN) {
          // might be able to recover from this, so let it continue to below
        }
        else { return null; } // bad length, can't recover from this (return)
      }
      kw.extractName(curentKey);
      kw.extractData(curentKey);
      return curentKey;
    }
    @Override
    public void remove () {
      kw.delete(name);
    }
    /** Calls {@link #next()} and then updates the internal 'name' and 'value' fields.
     *  @return The results of calling {@link #hasNext()} at the start of this method.
     */
    public boolean getNext () {
      if (!hasNext()) return false;
      next();
      name  = curentKey.name;
      value = curentKey.value;
      return true;
    }
    /** Get key name and return modified key name if parameter is set to true
     *  @param modify If true, substitute DOT_SUBSTITUTE for "." in key name
     *  @return name  Returns modified key name
     *  @since NeXtMidas 3.7.1
     */
    public String getName (boolean modify) {
      if (modify) {
        return name.replace(".", getDotSubstitute());
      }
      return name;
    }
    /** Lists the current key.
     *  @return A listing of the current key.
     */
    public String list () {
      return kw.list(curentKey);
    }
  } // end inner class Iterator

  /** Get the substitute for '.' in keywords
   *  @return the dot substitute
   *  @since 3.9.1
   */
  public static String getDotSubstitute () {
    return dotSubstitute;
  }

  /** Set the '.' substitute, since Table does nto work well with dots in keys
   *  @param dotSubstitute the '.' substitute
   *  @since 3.9.1
   */
  public static void setDotSubstitute (String dotSubstitute) {
    if(dotSubstitute==null) return;
    Keywords.dotSubstitute = dotSubstitute;
  }
}
