package nxm.sys.lib;

import static nxm.sys.lib.Convert.inLineFile;
import static nxm.sys.lib.StringUtil.getClosingFensePos;
import static nxm.sys.lib.StringUtil.stripQuotes;

import java.io.Externalizable;
import java.io.File;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.lang.reflect.Array;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.file.Path;
import java.util.AbstractSet;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.DoubleAccumulator;
import java.util.concurrent.atomic.DoubleAdder;
import java.util.concurrent.atomic.LongAccumulator;
import java.util.concurrent.atomic.LongAdder;

import nxm.sys.inc.AsciiMap;
import nxm.sys.inc.Filter;
import nxm.sys.inc.InternalUseOnly;
import nxm.sys.inc.Keyable;
import nxm.sys.inc.KeyableByType;
import nxm.sys.inc.MidasReference;
import nxm.sys.inc.Tablizable;

/** A class to perform Table put/get and maintenance functions. This class is used for result tables,
    parameters, switches, and other primitive specific tables. This class pre-dates most of the
    classes in the <code>java.util</code> package, and for years, was compatible with Java 1.1. Since
    NeXtMidas 2.2.0, this class has implemented the {@link Map} interface, but with a few caveats
    necessary for backwards-compatibility.<br>
    <br>
    This class will implements the {@link Map} interface, but with the following caveats:
    <pre>
      (1) All keys are converted to strings on insertion.
      (2) Any null values are converted to "NULL" on insertion (exception
          {@link #EXCEPTION_ON_NULL} throws exception rather than allowing
          null and PERMIT_NULL permits null).
      (3) Keys with a dot represent a put of a value into a sub-table (or
          sub-object using the methods in {@link KeyObject}). For example
          put("A.B","FOO") gets the table "A" from this table and sets "B"
          to "FOO" in that table.
    </pre>
    This class also extends the beyond the normal {@link Hashtable} class by adding typed methods
    for the {@link Data} and {@link File} classes, a {@link #get(String)} method that allows keys
    to be abbreviated, and follows links to other tables to resolve references in a hierarchy of
    tables. <br>
    <br>
    The serialized form of a table has the format:
    <pre>
      {key1=value1,...,keyN=valueN}
    </pre>
    and can include nested tables as follows:
    <pre>
      {key1=value1,...,keyT={keyT1=valueT1,...,keyTN=valueTN},...,keyN=valueN}
    </pre>
    The current implementation allows initialization from the serialized form directly into a
    byte buffer to provide for fast serialization if no alterations are made. This also preserves
    key ordering. <i>This disables validity checks for the table until a later time.</i>

    <h2><a id="TableModes">Table Modes</a></h2>
    The mode of the table identifies the underlying structure used to hold the keys and values.
    Prior to NeXtMidas 2.7.0, only three modes ({@link #FB}, {@link #KV},and {@link #HT}) were
    supported. Since NeXtMidas 2.7.0, a wide range of table modes are supported. The various
    modes are summarized in the following table.
    <table border="1">
      <caption style="visibility:hidden">""</caption>
      <tr><th>Mode                                     </th><th>Underlying Class  </th><th>Synchronized</th><th>Iteration Order</th><th>Performance<sup>1</sup></th></tr>
      <tr><td>FromBytes                 ({@link #FB})  </td><td><code>byte[]</code>            </td><td>n/a</td><td>n/a            </td><td>n/a             </td></tr>
      <tr><td>KeyVector                 ({@link #KV})  </td><td>{@link KeyVector}          </td><td>Yes</td><td>Insertion order</td><td>O(N)<sup>2</sup></td></tr>
      <tr><td>Hashtable                 ({@link #HT})  </td><td>{@link Hashtable}          </td><td>Yes</td><td>Unordered      </td><td>O(1)            </td></tr>
      <tr><td>HashMap<sup>3</sup>       ({@link #HM})  </td><td>{@link HashMap}            </td><td>No </td><td>Unordered      </td><td>O(1)            </td></tr>
      <tr><td>LinkedHashMap             ({@link #LHM}) </td><td>{@link LinkedHashMap}      </td><td>No </td><td>Insertion order</td><td>O(1)            </td></tr>
      <tr><td>SynchronizedLinkedHashMap ({@link #SLHM})</td><td>{@link LinkedHashMap}      </td><td>Yes</td><td>Insertion order</td><td>O(1)            </td></tr>
      <tr><td>TreeMap                   ({@link #TM})  </td><td>{@link TreeMap}            </td><td>No </td><td>Alphabetical   </td><td>O(LG(N))        </td></tr>
      <tr><td>SynchronizedTreeMap       ({@link #STM}) </td><td>{@link TreeMap}            </td><td>Yes</td><td>Alphabetical   </td><td>O(LG(N))        </td></tr>
      <tr><td>Map                       ({@link #MAP}) </td><td>{@link Map}<sup>4</sup>    </td><td>n/a</td><td>n/a            </td><td>n/a             </td></tr>
      <tr><td>Keyable                   ({@link #KEY}) </td><td>{@link Keyable}<sup>4</sup></td><td>n/a</td><td>n/a            </td><td>n/a             </td></tr>
      <tr><td>ConcurrentHashMap         ({@link #CHM}) </td><td>{@link ConcurrentHashMap}  </td><td>No<sup>5</sup></td><td>Unordered</td><td>O(1)       </td></tr>
      <tr><td colspan="5">
        <sup>1</sup> Average time cost for {@link #get(String)} and {@link #put(String,Object)}.<br>
        <sup>2</sup> The {@link #ADD} flag gives O(1) performance when inserting entries in
                     {@link #KV} mode since it disables checks for duplicate entries, performance
                     of other methods is unaffected by the {@link #ADD} flag.<br>
        <sup>3</sup> Hashtable is the synchronized version of HashMap.<br>
        <sup>4</sup> {@link Map} and {@link Keyable} are interfaces, see {@link #MAP} and {@link #KEY}
                     for details.<br>
        <sup>5</sup> {@link ConcurrentHashMap} is a hash table supporting full concurrency of
                     retrievals and adjustable expected concurrency for updates.
                     See java.util.concurrent.ConcurrentHashMap API for details. (Since NeXtMidas 3.1.1).
      </td></tr>
    </table>
    <br>
    <dl>
    <dt>NOTE1:</dt>
      <dd>The public methods in this class are NOT invokeable (by the INVOKE command) within the macro language.</dd>
    <dt><a id="dotsyntax">NOTE2:</a></dt>
      <dd>Most of the getXXX methods (e.g. {@link #getTable(String, boolean)}, {@link #get(String)})
          and putXXX methods (e.g. {@link #put(String, Object)}, {@link #put(String, double)})
          takes in a <b>"key"</b> parameter that is a String which can use the <b>dot syntax</b>
          to drill down/chain into the sub-Tables and/or Object properties/methods (if they exists). <br>
          BUT from this flexibility, those methods can throw a {@link MidasException} when drilling down
          into a Object's property/method that does NOT exist. The MidasException message might
          look something like &quot;<code>ERROR: KeyObject: Can not access '&lt;value after dot&gt;' in
          &lt;class name&gt; when given no arguments: No methods found.</code>&quot;.
          So care must be taken account for this or to avoid this scenario.<br>
          <br>
          Since NeXtMidas 2.9.2, the {@link #ONLY_DOT_INTO_SUBTABLES} flag can be used to tell the
          {@link #get(String)}, {@link #get(String, Object, int)}, and hence most of the getXXX methods
          to only to drill down into sub-Tables for the specified dotted key (i.e. do NOT drill down
          to the value object's properties and methods).
      </dd>
      <dt>NOTE3:</dt>
      <dd>Table's External Serialization covers Key Value pairs and flags.
      </dd>
     </dl>
    <br>
    Since 3.3.1: Table implements {@link java.lang.Iterable} interface so it can be use to
    iterate over the keys in the table using Java 5's foreach loop.

    @author  Jeff Schoen (initial version)
    @author  J. Czechowski IV (changes to implement {@link Map} and subsequent updates)
    @author  Neon Ngo (additional maintenance and fixes)
 */
@SuppressWarnings("UseOfObsoleteCollectionType") // suppress warnings about Hashtable
public class Table implements KeyableByType, AsciiMap, Externalizable, Map<String,Object>, Tablizable, Iterable<String> {
  // Implementation notes:
  //   (1) An AssertionError is thrown in cases where a genericMode is encountered that should be
  //       impossible. If an AssertionError is found, there is something wrong internal to table.
  //   (2) "Hashtable" was chosen over "HashTable" since it matches java.util.Hashtable. The
  //       abbreviation "HT" is maintained for backwards-compatibility (and because we like it).

  /** NOTE: this should be changed if the structure/fields of this class changes. */
  private static final long serialVersionUID = -732556679284448385L; // last updated in NeXtMidas 3.7.0

  /** Table Mode: Data is in a byte array ("From Bytes") (see <a href="#TableModes">Table Modes</a>).
      When a table in {@link #FB} mode is accessed (put, get, etc.) it is automatically converted
      to one of the "working" modes as specified by the default table mode.
   */
  public static final int FB=1;
  /** Table Mode: Data is in a {@link KeyVector} (see <a href="#TableModes">Table Modes</a>). */
  public static final int KV=2;
  /** Table Mode: Data is in a {@link Hashtable} (see <a href="#TableModes">Table Modes</a>). */
  public static final int HT=3;
  /** Table Mode: Data is in a {@link HashMap} (see <a href="#TableModes">Table Modes</a>).
      @since NeXtMidas 2.7.0
   */
  public static final int HM=4;
  /** Table Mode: Data is in a {@link LinkedHashMap} (see <a href="#TableModes">Table Modes</a>).
      @since NeXtMidas 2.7.0
   */
  public static final int LHM=5;
  /** Table Mode: Data is in a synchronized {@link LinkedHashMap} (see <a href="#TableModes">Table Modes</a>).
      @since NeXtMidas 2.7.0
   */
  public static final int SLHM=6;
  /** Table Mode: Data is in a {@link TreeMap} (see <a href="#TableModes">Table Modes</a>).
      @since NeXtMidas 2.7.0
   */
  public static final int TM=7;
  /** Table Mode: Data is in a synchronized {@link TreeMap} (see <a href="#TableModes">Table Modes</a>).
      @since NeXtMidas 2.7.0
   */
  public static final int STM=8;
  /** Table Mode: Data is in a {@link Map} (see <a href="#TableModes">Table Modes</a>).
      Since {@link Map} is an interface, when the mode is set to {@link #MAP}, {@link Table} will
      use a concrete implementation of {@link Map} in place of it. The implementation used is
      undefined, but {@link LinkedHashMap} is common.
      @since NeXtMidas 2.7.0
   */
  public static final int MAP=9;
  /** Table Mode: Data is in a {@link Keyable} (see <a href="#TableModes">Table Modes</a>).
      Since {@link Keyable} is an interface, when the mode is set to {@link #KEY}, {@link Table} will
      use a concrete implementation of {@link Keyable} in place of it. The implementation used is
      undefined, but {@link KeyVector} is common.
      @since NeXtMidas 2.7.0
   */
  public static final int KEY=10;
  /** Table Mode: Data is in a {@link ConcurrentHashMap} (see <a href="#TableModes">Table Modes</a>).
      @since NeXtMidas 3.1.1
   */
  public static final int CHM=11;


  private static final int FIRST_MODE = FB;    // Numerically the first valid table mode
  private static final int LAST_MODE  = CHM;   // Numerically the last  valid table mode

  /** The set of table modes in their abbreviated form. <i>This is maintained for
      backwards-compatibility with NeXtMidas 2.6.0 and earlier, most users will want to use
      {@link #modeList}.</i>
      <pre>tableModesList = {@value}</pre>
   */
  public static final String tableModesList = "FB,KV,HT,HM,LHM,SLHM,TM,STM,MAP,KEY,CHM";

  /** The set of table modes (see <a href="#TableModes">Table Modes</a>).
      <pre>modeList = {@value}</pre>
   */
  public static final String modeList = "FromBytes,KeyVector,Hashtable,HashMap,"
                                      + "LinkedHashMap,SynchronizedLinkedHashMap,"
                                      + "TreeMap,SynchronizedTreeMap,Map,Keyable,"
                                      + "ConcurrentHashMap";

  /** Table flag: Automatically read from previous table (see {@link #getPrevious()}). */
  public static final int AUTOREADPREV=0x1;

  /** Table flag: Automatically write to previous table (see {@link #getPrevious()}). */
  public static final int AUTOWRITEPREV=0x2;

  /** Table flag: All calls to <code>put()s</code> or <code>putx()s</code> may assume that the key being
      added is unique. This shifts the burden of uniqueness checks to the caller in
      cases where required for maximum performance. The behavior of the table is
      undefined if an non-unique key is inserted when the {@link #ADD} flag is set.<br>
      <br>
      Since NeXtMidas 2.7.1, if the ADD flag is specified, it will change this Table to
      KeyVector mode to allow duplicate keys in the Table. This behavior ensures
      backwards-compatibility with legacy applications expecting the ADD flag to
      allow duplicate keys.<br>
      <br>
      Typical usage:
      <pre>
        Table tbl = new Table(Table.KV, Table.ADD);
        for (..) {
          tbl.put(uniqueKey, val); // add unique key to table
        }
        tbl.setFlags(0);
      </pre>
   */
  public static final int ADD=0x4;

  /** Table flag: Table is read only. Any changes must be accompanied by the {@link #FORCE} flag. */
  public static final int READONLY=0x8;

  /** Table flag: Overrides the {@link #READONLY} flag and forces a change to a read-only table
                  (see {@link #put(String,Object,int)}. See <code>/FORCE</code> in <code>RESULTS</code>.
   */
  public static final int FORCE=0x10;

  /** Table flag: Add table roots as needed. See <code>/ADDROOT</code> in <code>RESULTS</code>. */
  public static final int ADDROOT=0x20;

  /** Table flag: Do not convert keys to upper case.
      @since NeXtMidas 2.5.0
   */
  public static final int CASESENSITIVE=0x40;

  /** Table flag: Do not insert <code>"NULL"</code> in place of <code>null</code>, rather throw a
                  {@link NullPointerException} if <code>null</code> is inserted.  This can not
                  be combined with {@link #PERMIT_NULL}
      @since NeXtMidas 2.7.1
   */
  public static final int EXCEPTION_ON_NULL=0x80;

  /** Table flag: When creating Tables from Strings or writing Tables as Strings, parse the String to handle escape characters
      @since NeXtMidas 2.9.0  (Bug 2292)
   */
  public static final int DISABLE_ESCAPE_SEQUENCES=0x100;

  /** Table flag: When using the <a href="#dotsyntax">dot syntax</a> in the key parameter for the
      {@link #get(String)} and hence most of the <b>getXXX</b> methods, only drill down to sub-Tables
      (i.e. do NOT drill down to object properties and methods).
      @since NeXtMidas 2.9.2 to address BUG 2333 and BUG 2334
  */
  public static final int ONLY_DOT_INTO_SUBTABLES=0x200;

  /** When creating a Table from a String, perform checks on the String to prevent some cases of malformed Tables
      @since NeXtMidas 3.3.0
   */
  public static final int CHECK_MALFORMED = 0x400;  // added for Bug 85 in 3.3.0

  /** When saving a Table, save Time Objects as T:yyyy:MM:dd::HH:mm:ss.SSSSSSSSSSSS
      @since NeXtMidas 3.5.3 (Bug 2876)
   */
  public static final int SERIALIZE_TIME_AS_TCOLON = 0x800;

  /** Any put where the key is null or the empty string will be silently ignored.
   *  @since NeXtMidas 3.7.0
   */
  public static final int IGNORE_EMPTY_INSERTS = 0x1000;

  /** When doing a put or get, check for a type specifier on the key name and do the appropriate
   *  type conversion. For example <code>put("L:FOO",3.14)</code> will convert the value to type "L"
   *  (i.e. 3) and then insert it into the table as "FOO".
   *  @since NeXtMidas 3.7.0
   */
  public static final int INLINE_TYPE_CONVERSION = 0x2000;

  /** When doing a put or get, perform special results-table specific conversions. This should only
   *  be used by {@link Results}; users looking for in-line type conversion should set the
   *  {@link #INLINE_TYPE_CONVERSION} flag.
   *  @since NeXtMidas 3.7.0
   */
  @InternalUseOnly("Since NeXtMidas 3.7.0: This flag should only be set by nxm.sys.lib.Results")
  public static final int INLINE_RESULTS_CONVERSION = 0x4000;

  /** Permit {@code null} values. Note this can not be combined with {@link #EXCEPTION_ON_NULL}
   *  or any of the table modes that do not support {@code null} values.  Also note that as values
   *  are serialized, a {@code null} becomes indistinguashable from the String {@code "null"}. <br>
   *  <br>
   *  This is unrelated to {@link #IGNORE_EMPTY_INSERTS} as this deals exclusively with values and
   *  {@link #IGNORE_EMPTY_INSERTS} deals exclusively with keys.
   *  @since NeXtMidas 3.7.3
   */
  public static final int PERMIT_NULL = 0x8000;

  /** Put all numeric values, both Data objects and Numbers, as a Java Number so that Tables
   *  and their re-serialized copy can be compared and equal tables will be considered equal
   *  @since NeXtMidas 3.8.0
   */
  public static final int KEEP_JAVA_NUMERIC_TYPE = 0X10000;

  /** Serialize tables strictly based on current table flags, without adding the
   *  DISABLE_ESCAPE_SEQUENCES flag
   *  @since NeXtMidas 3.8.0
   */
  public static final int SERIALIZE_WITH_ORIGINAL_FLAGS = 0x20000;

  /** To disable the new consistent handling of whitespace.
   *
   *  Ignore space after the = in a table specification (i.e. the space between = and 4 in
   *  the table <pre>res tbl {key4 = 4}</pre>)
   *  Ignore space between value end and comma (<pre>res tbl {key1 = 1  ,key2=2}</pre>
   *  @since NeXtMidas 3.8.1
   */
  public static final int DISABLE_CONSISTENT_HANDLING_OF_WHITESPACE = 0x40000;

  /** All numeric values will be serialized as as the upper case type.
   *  Upon re-serialization all numeric types will be serialized as Java
   *  Numbers.
   *  @since NeXtMidas 3.9.2
   */
  public static final int ONLY_JAVA_NUMERIC_SERIALIZATION = 0X80000;

  /** Table keys are quoted in serialization
   *  (allows fuller character range in keys and matches JSON behavior)
   *  @since NeXtMidas 4.0.0
   */
  public static final int QUOTED_KEYS_SERIALIZATION = 0X100000;

  /** Allows Table Keys to have dots in their name
   */
  @InternalUseOnly
  public static final int ALLOW_DOTS_IN_KEYS = 0X200000;

  /** Mask of contradictory null flags, that should never be set at the same time */
  public static final int CONTRADICTORY_NULL_FLAGS = EXCEPTION_ON_NULL + PERMIT_NULL;

  /** List of available table flags.
      <pre>flagsList = {@value}</pre>
      @since NeXtMidas 2.7.0
   */
  public static final String flagsList = "AutoReadPrev,AutoWritePrev,Add,ReadOnly,"     // 0x0001 - 0x0008
                                       + "Force,AddRoot,CaseSensitive,ExceptionOnNull," // 0x0010 - 0x0080
                                       + "DisableEscapeSequences,OnlyDotIntoSubTables," // 0x0100 - 0x0200
                                       + "CheckMalformed,SerializeTimeAsTColon,"        // 0x0400 - 0x0800
                                       + "IgnoreEmptyInserts,InlineTypeConversion,"     // 0x1000 - 0x2000
                                       + "InlineResultsConversion,PermitNull,"          // 0x04000 - 0x08000
                                       + "KeepJavaNumericType,SerializeWithOriginalFlags," // 0x10000 - 0x20000
                                       + "DisableConsistentHandlingOfWhiteSpace,"          // 0x40000
                                       + "OnlyJavaNumericSerialization,"                   // 0x80000
                                       + "QuotedKeysSerialization,"                        // 0x100000
                                       + "AllowDotsInKeys";                                // 0x200000

  private static int defaultMode  = SLHM;  // Default table mode (can not be set to FB)
  private static int defaultFlags = AUTOREADPREV; // Default table flags
  /** Tables with AllowDotsInKeys need to have the dots replaced when the Shell saves state - this is the default substitute */
  @InternalUseOnly
  public  static String SAVESTATE_KEY_DOT_SUBSTITUTE = "<DOT>";
  private static String saveStateKeyDotSubstitute = SAVESTATE_KEY_DOT_SUBSTITUTE;

  private   int                  genericMode  = FB;     // Generic type of table (FB,KV,MAP,KEY)
  private   int                  specificMode = FB;     // Specific type of table (any of the possible modes)
  private   byte[]               fb;                    // DATA: byte buffer holding data in FB mode (null=Empty)
  private   transient KeyVector            kv;          // DATA: KeyVector holding data in KV mode
  private   transient Keyable              kbl;         // DATA: Keyable holding data in KEY mode (not called key
                                                        //       since 'key' is a very common variable name)
  private   transient Map<String,Object>   map;         // DATA: Hashtable holding data in HT mode
  private   transient Listener             listener;    // Table listener
  private   Table                prev;                  // Link to the previous table
  protected int                  flags;                 // Table flags
  private   transient Enum                 en;          // Local enumeration used
  protected transient Object               ref;         // RESULTS: Reference object to use (MidasReference or Results)
  protected transient Map<String,Function> fht;         // RESULTS: HashTable for macro functions (internal use only for Results subclass)

  /** enables the use of the caret (^) to reference previous key/value pair in the table. */
  private boolean caretEnabled = true;

  /** Disable 4.0.0 fix that allowed keys with '(' or ')' in name */
  private static boolean doNotAllowRandomParen = false;

  /** Initialize an empty Table with the default mode and default flags. */
  public Table () {
    this(defaultMode, defaultFlags);
  }

  /** Initialize an empty Table with the given mode. <br><br>
      @param mode The table mode to use (see {@link #tableModesList}).
   */
  public Table (int mode) {
    this(mode,defaultFlags);
  }

  /** Initialize an empty Table.
      @param mode  The table mode to use (see {@link #tableModesList}).
      @param flags The table flags.
      @since NeXtMidas 2.5.0
   */
  @SuppressWarnings("OverridableMethodCallInConstructor")
  public Table (int mode, int flags) {
    setMode(mode);
    boolean currentNotNull = Shell.getCurrent() != null;

    // Need to check if shell exists. If it does, we set the "FavorJavaBehaviorOverTraditionalMidasBehavior" flag in Table based on the CoreIO option. Since 3.9.0
    if (currentNotNull && Shell.getMidasContext().io.isOptionSet(
        CoreIO.IOOptions.FavorJavaBehaviorOverTraditionalMidasBehavior)) {
      setFlags(flags | KEEP_JAVA_NUMERIC_TYPE | ONLY_JAVA_NUMERIC_SERIALIZATION);
    }
    else {
      setFlags(flags);
    }
    if (currentNotNull && Shell.getMidasContext().io.isOptionSet(
        CoreIO.IOOptions.DisableResultUsageWithCaretInTable)) {
      caretEnabled = false;
    }
  }

  /** Initialize an empty Table.
   *  @param mode  The table mode to user (see {@link #tableModesList}).
   *  @param flags The table flags
   *  @since NeXtMidas 3.7.3
   */
  @SuppressWarnings("OverridableMethodCallInConstructor")
  public Table (String mode, String flags) {
    this(modeStrToNum(mode), Parser.mask(flagsList,  flags,  defaultFlags));
  }

  /** Initialize table from a String in serialized form.  If the string is
      null then the constructor ignores the string and initializes itself to an empty table.
      @param string The data for the table (in the same format as used by {@link #toString()}).
  */
  public Table (String string) {
    this(FB);
    if (string!=null) fromBytes(string.getBytes());
  }

  /** Initialize table from a String in serialized form.  If the string is
      null then the constructor ignores the string and initializes itself to an empty table.
      @param string The data for the table (in the same format as used by {@link #toString()}).
      @param flags  The table flags
  */
  public Table (String string, int flags) {
    this(FB,flags);
    if (string!=null) fromBytes(string.getBytes());
  }

  /** Initialize table from a byte array in serialized form.  If the byte
      array is null or zero length then the constructor ignores the argument
      and initializes itself to an empty table.
      @param bytes The bytes that make up the table.
  */
  public Table (byte[] bytes) {
    this(FB);
    if (bytes!=null && bytes.length>0) fromBytes(bytes);
  }

  /** Initialize table from a byte array (specified range) in serialized form.
      If the byte array is null or zero length then the constructor ignores the argument
      and initializes itself to an empty table. If the start index and/or length is
      invalid (i.e. out of bounds) this is also initialize itself to an empty table.
      @param bytes The bytes that make up the table.
      @param start start index in byte array
      @param len   the desired length from start index in byte array
      @since NeXtMidas 2.7.1
  */
  public Table (byte[] bytes, int start, int len) {
    this(FB);
    if (bytes!=null && bytes.length>0 && start>=0 && len<=bytes.length-start) {
      fromBytes(bytes, start, start+len);
    }
  }

  /** Initialize table from a TextFile using default flags.
      Note: this forces this Table into KeyVector mode to allow duplicates to be
            loaded (legacy behavior).
      @param tf The text file defining the table.
      @see #fromTextFile(TextFile)
   */
  public Table (TextFile tf) {
    // fromTextFile(tf, defaultFlags); <-- NOT doing this b/c it would be a change in behavior
    this(FB);
    fromTextFile(tf); // <-- this will force Table to KV mode
  }

  /** Initialize table from a TextFile using specified flags mask and default table mode.
      @param tf The text file defining the table.
      @param flags Table flags mask to use for this
      @since NeXtMidas 3.1.2
   */
  public Table (TextFile tf, int flags) {
    this(FB, flags);
    fromTextFile(tf, flags);
  }

  /** Initialize a table from the contents of a Hashtable.  Upon completion
      this Table will contain references to the same keys and values as the
      given hashtable but will not have a reference to the hash itself.
      @param ht The hashtable file defining the table.
      @deprecated Since NeXtMidas 3.7.0: Use {@link #Table(Map)} to be consistent with the {@link Map} interface.
   */
  @Deprecated
  @SuppressWarnings("OverridableMethodCallInConstructor")
  public Table (Hashtable<String,?> ht) {
    this(FB);
    fromTable(ht);
  }

  /** Creates a new table with the specified content. Note that this is identical to calling
   *  <code>new Table()</code> followed by <code>putAll(map)</code>.
   *  @param map The values to insert into the table.
   *  @since NeXtMidas 3.7.0
   */
  @SuppressWarnings("OverridableMethodCallInConstructor")
  public Table (Map<String,?> map) {
    putAll(map);
  }

  /** Initialize a table with a given String. Unlike most other Table constructors,
      this constructor can create and load the Table right away so as to allow for checking
      the input String for formatting
      @param string  The String to convert to a Table
      @param load  Whether or not to parse the input String immediately and load the Table
      @since NeXtMidas 3.3.0
   */
  @SuppressWarnings("OverridableMethodCallInConstructor")
  public Table (String string, boolean load) {
    this(FB);
    if (string==null) return;
    if (!load) fromBytes(string.getBytes());
    else       fromBytes(string.getBytes(),0, string.length());
  }

  /** Converts a mode string to a number. This accepts either the mode abbreviation or full name. */
  private static int modeStrToNum (String mode) {
    // Since 3.7.3, allow default mode to be selected
    if (mode.equalsIgnoreCase("DEFAULT") || mode.equalsIgnoreCase("DEF")) return defaultMode;
    // Check tableModesList first so "KEY" matches "Keyable" not "KeyVector" and "HT" matches
    // "Hashtable". Checking tableModesList first is backwards-compatible with NeXtMidas 2.6.0.
    int m = Parser.find(tableModesList, mode, -1);
    if (m < 0) m = Parser.find(modeList, mode, -1);
    if (m < 0) throw new MidasException("Table: Invalid mode '"+mode+"' expected one of '"+modeList+"'.");
    return m;
  }

  /** Get default mode for tables.
      @return The default table mode to use (see <a href="#TableModes">Table Modes</a>).
      @since NeXtMidas 2.7.0
   */
  public static int getDefaultModeInt () {
    return defaultMode;
  }

  /** Get default mode for tables.
      @return One of the names listed in {@link #tableModesList}.
      @since NeXtMidas 2.7.0
   */
  public static String getDefaultModeName () {
    return Parser.get(tableModesList, getDefaultModeInt());
  }

  /** Get default mode for tables.
      @return One of the names listed in {@link #modeList}.
      @since NeXtMidas 2.7.0
   */
  public static String getDefaultMode () {
    return Parser.get(modeList, getDefaultModeInt());
  }

  /** Set default mode for tables. Note that in the case of {@link #MAP} and {@link #KEY}
      a specific implementation of those interfaces will be used. Note that {@link #FB} can not
      be used as a default Table mode.
      @param newMode The default table mode to use (see <a href="#TableModes">Table Modes</a>).
      @throws MidasException if passed an invalid mode or if passed "FB".
   */
  public static void setDefaultMode (String newMode) {
    setDefaultMode(modeStrToNum(newMode));
  }

  /** Set default mode for tables. Note that in the case of {@link #MAP} and {@link #KEY}
      a specific implementation of those interfaces will be used. Note that {@link #FB} can not
      be used as a default Table mode.
      @param newMode The default table mode to use (see <a href="#TableModes">Table Modes</a>).
      @throws MidasException if passed an invalid mode or if passed {@link #FB}.
   */
  public static void setDefaultMode (int newMode) {
    if (newMode == FB) {
      throw new MidasException("Table: Default table mode can not be set to FB.");
    }
    if ((newMode < FIRST_MODE) || (newMode > LAST_MODE)) {
      throw new MidasException("Table: Unknown table mode "+newMode+".");
    }
    defaultMode = newMode;
  }

  /** Set mode of table (see <a href="#TableModes">Table Modes</a>).
      @param newMode The table mode to use (see {@link #modeList} and {@link #tableModesList}).
   */
  public void setMode (String newMode) {
    setMode(modeStrToNum(newMode));
  }

  /** Get mode of table (see <a href="#TableModes">Table Modes</a>).
      @return The integer mask value corresponding with the names listed in {@link #modeList}.
      @since NeXtMidas 2.7.0
   */
  public int getModeInt () {
    return specificMode;
  }

  /** Get mode of table (see <a href="#TableModes">Table Modes</a>).
      @return One of the names listed in {@link #modeList}.
      @since NeXtMidas 2.7.0
   */
  public String getMode () {
    return Parser.get(modeList, getModeInt());
  }

  /** Get mode of table in its abbreviated form. <i>This is maintained for backwards-compatibility
      with NeXtMidas 2.6.0 and earlier, most users will want to use {@link #getMode()}.</i>
      @return One of the names listed in {@link #tableModesList}.
   */
  public String getModeName () {
    return Parser.get(tableModesList, getModeInt());
  }

  /** Checks to see if the Table is using a given mode (see <a href="#TableModes">Table Modes</a>).
      If mode is {@link #MAP} or {@link #KEY} this will check to see if the mode being used is
      compatible with that mode (e.g. <code>isMode({@link #MAP})</code> will return true for a mode
      of {@link #MAP}, {@link #HT}, {@link #HM}, etc. and <code>isMode({@link #KEY})</code> will
      return true for a mode of {@link #KEY} or {@link #KV}).
      @param mode The mode to check for (see {@link #modeList} and {@link #tableModesList}).
      @return true if that mode is being used, false otherwise.
      @since NeXtMidas 2.7.0
   */
  public boolean isMode (int mode) {
    if (mode == specificMode) return true;
    if (mode == MAP) return (genericMode == MAP);
    if (mode == KEY) return (genericMode == KEY) || (genericMode == KV);
    return false;
  }

  /** Checks to see if the Table is using a given mode (see <a href="#TableModes">Table Modes</a>).
      If mode is {@link #MAP} or {@link #KEY} this will check to see if the mode being used is
      compatible with that mode (e.g. <code>isMode({@link #MAP})</code> will return true for a mode
      of {@link #MAP}, {@link #HT}, {@link #HM}, etc. and <code>isMode({@link #KEY})</code> will
      return true for a mode of {@link #KEY} or {@link #KV}).
      @param mode The mode to check for (see {@link #modeList} and {@link #tableModesList}).
      @return true if that mode is being used, false otherwise.
      @since NeXtMidas 2.7.0
   */
  public boolean isMode (String mode) {
    return isMode(modeStrToNum(mode));
  }

  /** Set Mode of Table. Note that in the case of {@link #MAP} and {@link #KEY} a specific
      implementation of those interfaces will be used; as such a subsequent call to
      {@link #getMode()} may return a value other than {@link #MAP} or {@link #KEY}.
      @param newMode The table mode to use (see <a href="#TableModes">Table Modes</a>).
   */
  public void setMode (int newMode) {
    if (isMode(newMode)) return;  // No change

    // Converting to FB is a special case that needs to be handled apart from the rest
    if (newMode == FB) {
      fb = toBytes();
      specificMode = FB;
      genericMode  = FB;
      return;
    }

    int       oldMode = genericMode;
    byte[]    oldFB   = fb;
    KeyVector oldKV   = kv;
    Map<String,Object> oldMAP = map;
    Keyable   oldKEY  = kbl;

    fb  = null;
    kv  = null;
    map = null;
    kbl = null;

    if (newMode == MAP) newMode = SLHM; // Use implementation of Map     (may change in future)
    if (newMode == KEY) newMode = KV;   // Use implementation of Keyable (may change in future)

    specificMode = newMode;
    switch (newMode) {
      case KV:   genericMode = KV;  kv  = new KeyVector(32);                                break;
      case HT:   genericMode = MAP; map = new Hashtable<>();                   break;
      case HM:   genericMode = MAP; map = new HashMap<>();                     break;
      case LHM:  genericMode = MAP; map = new LinkedHashMap<>();               break;
      case SLHM: genericMode = MAP; map = Collections.synchronizedMap(new LinkedHashMap<String,Object>()); break;
      case TM:   genericMode = MAP; map = new TreeMap<>();                     break;
      case STM:  genericMode = MAP; map = Collections.synchronizedMap(new TreeMap<String,Object>());       break;
      case CHM:  genericMode = MAP; map = new ConcurrentHashMap<>();           break; // since 3.1.1
      default:   throw new IllegalArgumentException("Unknown table mode "+newMode);
    }

    switch (oldMode) {
      case FB:  if (oldFB != null) fromBytes(oldFB,0,oldFB.length); break;
      case KV:  merge(oldKV);                                       break;
      case MAP: putAll(oldMAP);                                     break;
      case KEY: merge(oldKEY);                                      break;
    }
  }

  /** get core object
   *  @return underlying Table
   */
  public Object getCore() {
    Object obj=null;
    switch (genericMode) {
      case FB:  obj=fb; break;
      case KV:  obj=kv; break;
      case MAP: obj=map; break;
      case KEY: obj=kbl; break;
    }
    return obj;
  }

  /** Ensures that the mode is not FB, if it is, change it. */
  private void checkModeFB () {
    if (genericMode<=FB) setMode(defaultMode);
  }

  /** Return the entry from the table that is located under the given key. <br>
      <br>
      Four particular key names (and their abbreviations) have special significance:
      <pre>
        "GETPREV", "Prev"  - same as {@link #getPrevious()}
        "GETKEYS", "Keys"  - same as {@link #getKeys()}
        "GETLIST", "List"  - same as {@link #getKeyList()}
        "GETSIZE". "Size"  - same as {@link #size()}
      </pre>
      This method is intended for Macro users, API users should call {@link #get(String)}.
      @param key The name of the object to look up.
      @return The object that is currently stored under the given key.
      @throws NullPointerException if <code>key</code> is <code>null</code>
   */
  @Override
  public Object getKey (String key) throws NullPointerException {
    if (key.equals("Prev") || key.equals("Keys") || key.equals("List") || key.equals("Size")) {
      Shell.deprecate("getKey('"+key+"') is deprecated. Use getKey('GET" +key.toUpperCase()+"')."); // for Bug 1749
    }

    int len = key.length();
    if (len!=7 && len!=4) return getx(key);
    else if (key.equalsIgnoreCase("GETPREV") || key.equals("Prev")) return getPrevious();
    else if (key.equalsIgnoreCase("GETKEYS") || key.equals("Keys")) return getKeys();
    else if (key.equalsIgnoreCase("GETLIST") || key.equals("List")) return getKeyList();
    else if (key.equalsIgnoreCase("GETSIZE") || key.equals("Size")) return size();
    else if (key.equalsIgnoreCase("GETCORE") || key.equals("Core")) return getCore();
    else return getx(key);
  }

  /** Sets the key in the table. This is the same as <code>put(key,value)</code>, except that it
   *  returns the inserted value following the put.
   *  @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
   *  @param value The value to insert (any <code>null</code> values will be changed to <code>"NULL"</code>
   *               on insertion unless {@link #EXCEPTION_ON_NULL} or {@link #PERMIT_NULL} is set).
   *  @return The inserted value. Normally this returns *exactly* the value passed in, and does not
   *          reflect any conversion from <code>null</code> to "NULL", etc. However, if
   *          {@link #INLINE_TYPE_CONVERSION} is set, this will return the value in the as-inserted
   *          (post type conversion) form (note that this behavior violates {@link Keyable}).
   *  @throws NullPointerException If <code>key</code> is <code>null</code> or if <code>value</code> is
   *                               <code>null</code> and {@link #EXCEPTION_ON_NULL} is set.
   */
  @Override
  public Object setKey (String key, Object value) {  // TODO: make final (but need to update XmAuxTable.java first
    put(key,value);
    return ((INLINE_TYPE_CONVERSION&flags)==0)? value : get(key);
  }

  /** Remove an entry from the table. This is identical to <code>remove(key,0)</code>.
      @param key The name of the key to remove. See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @return oldVal The value previously associated with that key (since NeXtMidas 3.1.0)
      @throws NullPointerException if <code>key</code> is <code>null</code>
   */
  public final Object remove (String key) throws NullPointerException {  // marked final since 3.7.0
    return remove(key,0);
  }

  /** Remove an entry from the table.
      @param key The name of the key to remove. See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @param attr  The attributes associated with the remove.
      @return oldVal The value previously associated with that key (since NeXtMidas 3.1.0)
      @throws NullPointerException if <code>key</code> is <code>null</code>
      @since NeXtMidas 2.5.5
   */
  public synchronized Object remove (String key, int attr) throws NullPointerException {
    // since 3.1.1: made method synchronized for thread safety (BUG 2500)
    if ((flags&READONLY)!=0 && (attr&FORCE)==0) {
      Shell.warning("Must use FORCE flag to remove tag="+key+" from ReadOnly table");
      return null; // Bug 1343
    }
    if (key == null) return null;
    Object oldVal;
    int i;
    Object getxVal = getx(key);
    if (((flags & ALLOW_DOTS_IN_KEYS) == 0) || (getxVal==null)) {
      oldVal = get(key);
      i = key.indexOf('.');
    }
    else {
      oldVal = getxVal;
      i = -1; // using full thing as key with dots in name
    }
    checkModeFB();
    if (i>0) {
      String keyroot = key.substring(0,i);
      Object root = getx(keyroot);
      if (root instanceof Table) {
        ((Table)root).remove(key.substring(i+1), attr);
        return oldVal;
      }
    }
    if (key.length() > 0) {
      switch (genericMode) {
        case KEY: kbl.setKey(key,null);  break;
        case KV:  kv.remove(key);        break;
        case MAP: {
          if (en !=null && en.iter != null) en.iter.remove(); // Bug 1791 Table.remove() incompatible with SLHM
          else map.remove(key);
          break;
        }
        default:  throw new AssertionError("Unknown table mode "+genericMode);
      }
    }
    return oldVal;
  }

  /** Put a generic entry into the table under the given key.
   *  This is identical to <code>put(key,value,0,ref)</code> (where <code>ref</code> is normally null).
   *  @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
   *  @param value The value to insert (any <code>null</code> values will be changed to <code>"NULL"</code>
   *               on insertion unless {@link #EXCEPTION_ON_NULL} or {@link #PERMIT_NULL} is set).
   *  @return The previous value associated with the key being set in order to match the {@link Map}
   *          interface (since NeXtMidas 3.3.3).
   *  @throws NullPointerException if <code>key</code> is <code>null</code> or if <code>value</code> is
   *                               <code>null</code> and {@link #EXCEPTION_ON_NULL} is set in flags.
   */
  @Override
  public final Object put (String key, Object value) throws NullPointerException { // marked final since 3.7.0
    return put(key,value,0,ref);
  }

  /** Put a generic entry into the table under the given key with flag attributes.
   *  This is identical to <code>put(key,value,attr,ref)</code> (where <code>ref</code> is normally null).
   *  @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
   *  @param value The value to insert (any <code>null</code> values will be changed to <code>"NULL"</code>
   *               on insertion unless {@link #EXCEPTION_ON_NULL} or {link #PERMIT_NULL}  is set).
   *  @param attr  The attributes associated with the put.
   *  @return The previous value associated with the key being set.
   *  @throws NullPointerException if <code>key</code> is <code>null</code> or if <code>value</code> is
   *                               <code>null</code> and {@link #EXCEPTION_ON_NULL} is set in flags.
   *  @since NeXtMidas 2.5.4
   */
  public final Object put (String key, Object value, int attr) throws NullPointerException { // marked final since 3.7.0
    return put(key,value,attr,ref);
  }

  /** Put a generic entry into the table under the given key with flag attributes.
   *  This is identical to <code>put(key,value,0,ref)</code>.
   *  @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
   *  @param value The value to insert (any <code>null</code> values will be changed to <code>"NULL"</code>
   *               on insertion unless {@link #EXCEPTION_ON_NULL} or {link #PERMIT_NULL}  is set).
   *  @param ref   The reference object for translation.
   *  @return The previous value associated with the key being set.
   *  @throws NullPointerException if <code>key</code> is <code>null</code> or if <code>value</code> is
   *                               <code>null</code> and {@link #EXCEPTION_ON_NULL} is set in flags.
   *  @since NeXtMidas 3.7.0 (was available in Results sub-class in NeXtMidas 3.6.0 and earlier)
   */
  public final Object put (String key, Object value, Object ref) throws NullPointerException { // marked final since 3.7.0
    return put(key,value,0,ref);
  }

  /** Put a generic entry into the table under the given key with flag attributes. Where it is
   *  necessary to add a sub-table (i.e. {@link #ADDROOT} is set), the new table will be added
   *  with the {@link #addTable(String)} function. <br>
   *  <br>
   *  Note: Prior to NeXtMidas 3.5.4 the {@link #ADDROOT} flag was handled inconsistently and
   *  the exceptions thrown were inconsistent (and didn't match the documentation).
   *  @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
   *  @param value The value to insert (any <code>null</code> values will be changed to <code>"NULL"</code>
   *               on insertion unless {@link #EXCEPTION_ON_NULL} or {link #PERMIT_NULL}  is set).
   *  @param attr  The attributes associated with the put.
   *  @param ref   The reference object for translation.
   *  @throws RuntimeException If one of the following is true:
   *               <pre>
   *                 1. key is null (and {@link #IGNORE_EMPTY_INSERTS} is not set)
   *                 2. value is null, and {@link #EXCEPTION_ON_NULL} is set in flags
   *                 3. a sub-entry is being set (e.g. "A.B" or "X.Y.Z"), and the
   *                    upper-level table is missing, and AddRoot fails due to:
   *                      a. The upper-level table is not a table (e.g. insert
   *                         of "X.Y.Z" where "X" is a generic Keyable)
   *                      b. ADDROOT is set either in attr or the flags of the
   *                         upper-level table
   *               </pre>
   *  @return Since 3.3.3: the previous value associated with key, or null if there was no mapping for key. (A null
   *      return can also indicate that the map previously associated null with key, if the
   *      implementation supports null values.)
   */
  public Object put (String key, Object value, int attr, Object ref) throws RuntimeException {
    int allFlags = flags | attr; // The union of the table flags and the attributes passed in

    // ==== CHECK FOR VALID KEY NAMES ==============================================================
    if (((allFlags&IGNORE_EMPTY_INSERTS)!=0) && ((key==null) || key.isEmpty())) {
      return null; // ignore empty insert
    }
    if (key==null) {
      // since NeXtMidas 3.3.3 for Bug 2699 (other NPE happens later on)
      throw new IllegalArgumentException("Table Key=null not allowed");
    }

    // ==== INLINE TYPE CONVERSION =================================================================
    if (((allFlags&INLINE_TYPE_CONVERSION)!=0) && ((key.length()>2) && (key.charAt(1)==':'))) {
      char type = key.charAt(0);
      key = key.substring(2);
      if (type == 'A') {
        Shell.deprecate("Since NeXtMidas 2.5.2: Change use of A:"+key+" to S:"+key+". "
                      + "The behavior of of A: will change in future releases.");
        type = 'S';
      }
      value = Convert.o2o(value,type,ref);
    }

    // ==== INLINE RESULTS CONVERSION ==============================================================
    // These conversions were internal to Results.java prior to NeXtMidas 3.7.0 but resulted in a
    // lot of duplicated code that wasn't always updated when Table was updated (e.g. Bug 2929).
    if ((allFlags&INLINE_RESULTS_CONVERSION)!=0) {
      int i=key.indexOf('.');
      int j=key.indexOf(')');
      if (i<0&&j<0) {                                             // pre-defined macro function
        Function f;
        if (fht!=null && (f=fht.get(key))!=null) {
          f.set(value);
          return value;
        }
        else {
          // JC4: This seems fundamentally wrong (i.e. we should just use the normal put code below)
          // but not having this causes issues for the confidence tests for some reason. We should
          // investigate this (and the corresponding getx(..) in the get(..) method) in the future.
          return putx(key,value,attr);
        }
      }
      else if (j>0 && key.startsWith("FILE(")) {                  // in-line file
        j = getClosingFensePos(key,"()",4);
        Midas midas = Convert.ref2Midas(ref);
        DataFile df = inLineFile(midas, key.substring(5,j), true);
        Object prevValue = KeyObject.setKey(df,key.substring(j+2),value,ref);
        df.close();
        return prevValue;
      }
      else if (j>=0) {                                           // full on setKey
        return KeyObject.setKey(this,key,value,ref);
      }
    }

    // ==== NORMAL TABLE INSERT ====================================================================
    checkModeFB();
    int i;
    Object root = this;
    String origKey = key;
    boolean findingNestedIsWorking = true;
    while (findingNestedIsWorking && (i = key.indexOf('.')) > 0) {
      String keyroot = key.substring(0,i);
      if (root instanceof Table) {
        Table tbl = (Table)root;
        root = tbl.getx(keyroot);

        if (root == null) {
          // We will add the root if AddRoot is passed in (attr) or if the table we are adding
          // to (tbl) has AddRoot set. Prior to NeXtMidas 3.5.4 (Bug 2929) only attr was checked
          // for AddRoot, and the check didn't apply to the top level (i.e. a put of "X.Y.Z" only
          // did AddRoot if "X.Y" was missing, but not "X").
          if (((attr&ADDROOT)!=0) || ((tbl.flags&ADDROOT)!=0)) {
            root = tbl.addTable(keyroot);
          }
          else if ((flags & ALLOW_DOTS_IN_KEYS) != 0) {
            // the nested put failed, so assuming this is a key with dots in it NXM-3772
            findingNestedIsWorking = false;
            root = this;
            key = origKey;
            i = -1;
          }
          else {
            // For ERROR reporting, figure out if the top level was a Table that exists
            int origDot = origKey.indexOf('.');
            String origKeyroot = (origDot > 0) ? origKey.substring(0,origDot):origKey; // get name of Table in RESULTS
            String topLevelMissing = (this.getx(origKeyroot) instanceof Table) ? "Key "+keyroot:"Table "+origKeyroot;
            // Throw appropriate message
            throw new MidasException(topLevelMissing+" does not exist, so unable to set value of "+origKey+" without forcing via /ADDROOT switch");
          }
        }
      }
      else {
        root = KeyObject.getKey(root,keyroot,null,ref);
        if (root == null) {
          throw new MidasException("Key "+keyroot+" does not exist, so unable to set value of "+origKey+" without forcing via /ADDROOT switch");
        }
      }
      key = key.substring(i+1);
    }
    return (root instanceof Table)? ((Table)root).putx(key,value,attr)
                                  : KeyObject.setKey(root,key,value);
  }

  /** Put a generic entry into the table under the given key if if there isn't an existing entry present at that key.
      This is similar to the idea in the {@link java.util.Set#add(Object)} method.
      @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @param value The value to insert (any <code>null</code> values will be changed to <code>"NULL"</code>
               on insertion unless {@link #EXCEPTION_ON_NULL} or {link #PERMIT_NULL}  is set).
      @return true if key/value is added, otherwise false (i.e. there is an existing entry at specified key).
      @throws NullPointerException if <code>key</code> is <code>null</code> or if <code>value</code> is
                               <code>null</code> and {@link #EXCEPTION_ON_NULL} is set in flags.
      @see #put(String, Object) for normal put use put(..) and its various overloaded methods.
      @since NeXtMidas 3.1.0
   */
  public boolean addIfNotPresent (String key, Object value) throws NullPointerException {
    if (containsKey(key)) return false; // return false: key/value was NOT added to table
    put(key,value,0,null);
    return true;                        // return true: key/value was added to table
  }

  /** Merge key/values from specified Map into this Table for any key which is NOT already present.
      @param map Map (e.g. Table) to merge in key/values that are not already present.
      @return true if any key/value map is added, otherwise false (i.e. there is an existing entry at specified key).
      @see #addIfNotPresent(String, Object)
      @since NeXtMidas 3.1.0

   */
  public boolean addIfNotPresent (Map<String,Object> map) throws NullPointerException {
    if (map==null) return false; // return false: no keys/values was NOT added to table
    boolean retval = false;
    java.util.Iterator<Map.Entry<String,Object>> iter = map.entrySet().iterator();

    while (iter.hasNext()) {
      Map.Entry<String, Object> entry = iter.next();
      retval |= addIfNotPresent(entry.getKey(), entry.getValue());
    }

    return retval;
  }

  /** Adds a new empty table to this table at the specified key if it is not already present.
      The table added will initially be empty, but will have the same mode and flags as this
      table. <br>
      <br>
      See important note on the <a href="#dotsyntax">dot syntax here</a> for key argument.
      @param key The name of the new table.
      @return A pointer to the new table (after adding it to this table) if key is not already present,
              otherwise a pointer to the existing value at that key in this Table.
   */
  public synchronized Object addTable (String key) {
    Object root = get(key);
    if (root!=null) return root;
    root = new Table(specificMode,flags);
    putx(key,root);
    return root;
  }

  /** Put an entry into *this* table. Unlike the various <code>put(..)</code> methods, the key must NOT
      contain any dots (One exception is a limited use case where dots will be part of the key name).
      @param key   The key to insert (not <code>null</code>).
      @param value The value to insert (any <code>null</code> values will be changed to <code>"NULL"</code>
                   on insertion unless {@link #EXCEPTION_ON_NULL} or {link #PERMIT_NULL}  is set).
      @throws NullPointerException if <code>key</code> is <code>null</code> or if <code>value</code> is
                                   <code>null</code> and {@link #EXCEPTION_ON_NULL} is set.
   */
  public void putx (String key, Object value) throws NullPointerException {
    putx(key,value,0);
  }

  /** Put an entry into *this* table. Unlike the various <code>put(..)</code> methods, the key must NOT
      contain any dots (One exception is a limited use case where dots will be part of the key name).
      @param key   The key to insert (not <code>null</code>).
      @param value The value to insert (any <code>null</code> values will be changed to <code>"NULL"</code>
                   on insertion unless {@link #EXCEPTION_ON_NULL} or {link #PERMIT_NULL}  is set)
      @param attr  The attributes associated with the put.
      @throws NullPointerException if <code>key</code> is <code>null</code> or if <code>value</code> is
                                   <code>null</code> and {@link #EXCEPTION_ON_NULL} is set.
      @return Since 3.3.3: the previous value associated with key, or null if there was no mapping
              for key. A null can be returned in the following situations:
              <pre>
              #1) The map previously associated null with key
              #2) The Table is ReadOnly and the FORCE flag is not set in attr.
              #3) There is a Table Listener which returned true on it's putAction.
              #4) The root is a Results and prev!=null and root==value indicating
                  that nothing has changed (this is an odd usage that applies only
                  Results and is historical in nature).
              </pre>
   */
  public synchronized Object putx (String key, Object value, int attr) throws NullPointerException {
    // since 3.1.1: made method synchronized for thread safety (BUG 2500)

    if ((flags&READONLY)!=0 && (attr&FORCE)==0) {
      Shell.warning("Must use FORCE flag to put tag="+key+" into ReadOnly table");
      return null; // return #2. Since 2.7.1 for Bug 1343
    }
    if (listener!=null && listener.putAction(key,value)) return null; // return #3
    checkModeFB();
    if (value==null && ((flags&PERMIT_NULL)==0)) {
      if ((flags & EXCEPTION_ON_NULL) != 0) {
        throw new NullPointerException("Table: Can not insert "+key+"="+value+" into table.");
      }
      value = "NULL";
    }

    // check for global table reference
    if (prev!=null) {
      Object root=null;
      switch (genericMode) {
        case KV:  root = kv.get(key);      break;
        case MAP: root = map.get(key);     break;
        case KEY: root = kbl.getKey(key);  break;
        default:  throw new AssertionError("Unknown table mode "+genericMode);
      }
      if (root instanceof Results) {
        // update value only if it differs from original
        if (root != value) return ((Table)root).putx(key,value,attr);
        return null; // return #4 (root == value)
      }
    }

    switch (genericMode) {
      case KV:  if ((flags&ADD)!=0) { kv.add(key,value); return null; }
                else                { return kv.put(key,value);       }
      case MAP: return map.put(key,value);
      case KEY: return kbl.setKey(key,value);
      default:  throw new AssertionError("Unknown table mode "+genericMode);
    }
  }

  /** Get a generic entry/value from the table using the specified key.
      @param key      The name of the entry to get. See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @param defValue The value to return if specified key is not found.
      @param options  Table flags/options to apply. E.g. {{@link #ONLY_DOT_INTO_SUBTABLES}.
      @return The value associated with the key or <code>defValue</code> if the specified key is not
              part of the table.
      @throws NullPointerException If the key is <code>null</code>.
      @see #containsKey(String)
      @since NeXtMidas 2.9.2
   */
  public Object get (String key, Object defValue, int options) {
    int     allFlags = flags | options; // The union of the table flags and the attributes passed in
    char    type     = ' ';   // (' '=No Type Conversion)
    Object  value    = null;
    boolean valueSet = false;

    // ==== CHECK FOR VALID KEY NAMES ==============================================================
    if (((allFlags&IGNORE_EMPTY_INSERTS)!=0) && ((key==null) || key.isEmpty())) {
      return null; // ignore empty get
    }
    if (key==null) {
      // since NeXtMidas 3.7.0 (other NPE happens later on)
      throw new NullPointerException("Table Key=null not allowed");
    }

    // ==== INLINE TYPE CONVERSION (1 of 2) ========================================================
    if (((allFlags&INLINE_TYPE_CONVERSION)!=0) && ((key.length()>2) && (key.charAt(1)==':'))) {
      type = key.charAt(0);
      key  = key.substring(2);
    }

    // ==== INLINE RESULTS CONVERSION ==============================================================
    // These conversions were internal to Results.java prior to NeXtMidas 3.7.0 but resulted in a
    // lot of duplicated code that wasn't always updated when Table was updated (e.g. Bug 2929).
    if ((allFlags&INLINE_RESULTS_CONVERSION)!=0) {
      Function f;
      int i=key.indexOf('.'); // Results should not have '.' in Key name, '.' is separator between keys
      int j=key.indexOf(')');
      if (i<0&&j<0) {                                             // simple case
        if (fht!=null && (f=fht.get(key))!=null) {
          value = f.get();
          valueSet = true;
        }
        else {
          // JC4: This seems fundamentally wrong (i.e. we should just use the normal get code below)
          // but not having this causes issues for the confidence tests for some reason. We should
          // investigate this (and the corresponding putx(..) in the put(..) method) in the future.
          value = getx(key);
          valueSet = true;
        }
      }
      else if (j<0) {                                             // quick 1st level
        Object root = getx(key.substring(0,i));
        value = KeyObject.getKey (root,key.substring(i+1),null,ref);
        valueSet = true;
      }
      else if (key.startsWith("FILE(")) {                         // in-line file
        j = getClosingFensePos(key,"()",4);
        Midas midas = Convert.ref2Midas(ref);
        DataFile df = inLineFile(midas, key.substring(5,j), false);
        if (j==key.length()-1) value = df;
        else value = KeyObject.getKey (df,key.substring(j+2),null,ref);
        valueSet = true;
        df.close();
      }
      else if (key.startsWith("CALC(")) {                         // in-line calc
        value = Convert.d2o(Convert.s2d(key,ref));
        valueSet = true;
      }
      else if (key.startsWith("TEST(")) {                         // in-line test
        value = Convert.d2o(Convert.s2l(key,ref));
        valueSet = true;
      }
      else if (key.startsWith("SEDIT(")) {                        // in-line SEDIT
        // TODO: This can be replaced with a call to Convert.inLineSedit(ref,key)
        Midas M = nxm.sys.lib.Convert.ref2Midas(ref);
        Parser p = new Parser(key.substring(6,key.length()-1), true);
        Args args = new Args(M,p);
        if (ref instanceof Args) ((Args)ref).copySwitchesTo(args);
        value = Convert.processSeditCommandLine(args.getCS(1),M,args,2,false);
        valueSet = true;
      }
      else {
        // full on getKey
        // NXM-2921 - fix for non-indexable keys, ex args with colons
        try {
          value = KeyObject.getKey(this,key,null,ref);
          valueSet = true;
        } catch (MidasException midasException) {
          value = null;
        }

      }
    }

    // ==== NORMAL TABLE INSERT ====================================================================
    // checkModeFB(); // don't need to call it here, since we call it in getx() below
    boolean allowingKeysWithDots = (flags & ALLOW_DOTS_IN_KEYS) != 0;
    boolean drillIntoProperties = !allowingKeysWithDots && ((options & ONLY_DOT_INTO_SUBTABLES) == 0); // for BUG 2333 and BUG 2334

    int ii;
    if (valueSet) {
      // already set
    }
    else if ((ii=key.indexOf('.')) > 0) {
      String keyroot = key.substring(0,ii);
      String subkey = key.substring(ii+1);
      Object root = getx(keyroot);
      if (root==null) {
        Object keyWithDotsValue = getx(key);
        if (allowingKeysWithDots && (keyWithDotsValue != null)) {
          return keyWithDotsValue;
        }
        return defValue;
      }
      while (root!=null && (ii=subkey.indexOf('.')) > 0 ) {
        keyroot = subkey.substring(0,ii);
        subkey  = subkey.substring(ii+1);
        if (root instanceof Table)    root = ((Table)root).getx(keyroot);
        else if (drillIntoProperties) root = KeyObject.getKey(root, keyroot);
      } // end while loop
      if (root==null) {
        Object keyWithDotsValue = getx(key);
        if (allowingKeysWithDots && (keyWithDotsValue != null)) {
          return keyWithDotsValue;
        }
        return defValue;
      }
      else if (root instanceof Table) value = ((Table)root).getx(subkey);
      else if (drillIntoProperties)   value = KeyObject.getKey(root, subkey);
      if (allowingKeysWithDots && (value==null)) value = getx(key);
    }
    else {
      value = getx(key);
    }

    // ==== INLINE TYPE CONVERSION (2 of 2) AND RETURN =============================================
    if (value == null) return defValue;
    if (type  != ' ' ) return Convert.o2o(value,type,ref);
    return value;
  }

  /** Same as <code>get(key, defValue, flags)</code>.
   *  @see #get(String, Object, int)
   *  @param key The name of the entry to get.
   *  @param defValue The default value.
   *  @return The value associated with the key or the default provided if the key is not found.
   *  @since NeXtMidas 2.9.2
   */
  public final Object get (String key, Object defValue) {  // marked final since 3.7.0
    return get(key, defValue, flags);
  }

  /** Get a generic entry from the table using the specified key.
      This is the same as <code>get(key, null, flags)</code>.
      @param key The name of the entry to get. See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @return The value associated with the key or <code>null</code> if the specified key is not
              part of the table.
      @throws NullPointerException If the key is <code>null</code>.
      @see #get(String, Object, int)
      @see #containsKey(String)
   */
  public Object get (String key) throws NullPointerException {
    return get(key, null, flags);
  }

  /** Gets an entry from *this* table. Unlike the <code>get(..)</code> method, the key must NOT
      contain any dots.
      @param key The key to get (not <code>null</code>).
      @return The value from the table (null if not found).
      @throws NullPointerException If the key is <code>null</code>.
   */
  public Object getx (String key) throws NullPointerException {
    checkModeFB();
    Boolean found;
    Object  value;

    switch (genericMode) {
      case KV:  value = kv.get(key);      found = (value != null) || (kv.find(key) >= 0);     break;
      case MAP: value = map.get(key);     found = (value != null) || (map.containsKey(key));  break;
      case KEY: value = kbl.getKey(key);  found = (value != null);                            break;
      default:  throw new AssertionError("Unknown table mode "+genericMode);
    }

    if (prev != null) {
      if (!found && (flags&AUTOREADPREV)!=0) value = prev.getx(key);
      else if (value instanceof Results)     value = ((Table)value).getx(key);
    }

    if (listener!=null) listener.getAction(key,value);
    return value;
  }

  /** Find the key matching the given abbreviation. The abbreviation for a key is the first key
      found that starts with the given abbreviation.
      @param abbr The abbreviation.
      @return The name of the key that matches the abbreviation or <code>null</code> if not found.
   */
  public String match (String abbr) {
    if (containsKey(abbr)) return abbr;  // found non-abbreviated
    for (Enumeration<String> e=keys(); e.hasMoreElements(); ) {
      String key = e.nextElement();
      if (key.startsWith(abbr)) return key;
    }
    return null;
  }

  /** Clear the Table of all entries. This is identical to <code>clear(0)</code>. */
  @Override
  public final void clear () {  // marked final since 3.7.0
    clear(0);
  }

  /** Clear the Table of all entries.
      @param attr The attributes associated with the clear.
      @since NeXtMidas 2.5.5
   */
  public void clear (int attr) {
    if ((flags&READONLY)!=0 && (attr&FORCE)==0) {
      Shell.warning("Must use FORCE flag to clear ReadOnly table");
      return; // Bug 1343
    }
    if (genericMode == KEY) {
      // Keyable has no clear method, so just iterate over all keys and remove each one.
      for (String key : kbl.getKeys()) {
        remove(key,attr);
      }
    }
    else {
      switch (genericMode) {
        case FB:  fb = null;       break;
        case KV:  kv.clear();      break;
        case MAP: map.clear();     break;
        case KEY: throw new AssertionError("This should be impossible."); // KEY handled above
        default:  throw new AssertionError("Unknown table mode "+genericMode);
      }
    }
  }

  /** Check if specified table is null or empty (via {@link #isEmpty()}.
      @param tbl Table object to check
      @return true if value is null or it is empty.
      @since NeXtMidas 3.3.2
   */
  public static boolean isNullOrEmpty (Table tbl) {
    return (tbl == null) || tbl.isEmpty();
  }

  @Override
  public final boolean isEmpty () {  // marked final since 3.7.0
    return size()==0;
  }

  @Override
  public int size () {
    if ((genericMode == FB) && (fb == null)) return 0; // Don't bother converting if it is empty
    checkModeFB();
    switch (genericMode) {
      case KV:  return kv.size();
      case MAP: return map.size();
      case KEY: return kbl.getKeys().length;
      default:  throw new AssertionError("Unknown table mode "+genericMode);
    }
  }

  /** Gets the number of elements in the table. This is identical to <code>size()</code>.
   *  @return The size.
   */
  public final int getSize () { return size(); }  // marked final since 3.7.0

  /** Indicates if the table contains the given key (should NOT contain dots) as this
      queries the underlining container directly.
      @param key The key to look for (not null, and no dots).
      @return <code>true</code> if Table contains this key, <code>false</code> otherwise.
      @see #containsKey(String)
      @since NeXtMidas 2.9.2 this was the original implementation of the <code>containsKey</code> method.
   */
  @InternalUseOnly("This method is for internal use only, use containsKey(..)") // marked since NeXtMidas 3.5.4
  public boolean containsKeyx (String key) {
    checkModeFB();
    switch (genericMode) {
      case KV:  return (kv.findKey(key) >= 0);
      case MAP: return map.containsKey(key);
      case KEY: return (kbl.getKey(key) != null);
      default:  throw new AssertionError("Unknown table mode "+genericMode);
    }
  }

  /** Indicates if the table contains the given key. Since NeXtMidas 2.9.2, the dot syntax can be
      used to drill down into keys for sub-Table(s) at any level. Note that this is different
      than the {@link #get(String)} method's <a href="#dotsyntax">dot syntax</a> behavior,
      since this forces the {@link #ONLY_DOT_INTO_SUBTABLES} option.
      Use {@link #containsKeyx(String)} for old behavior to NOT drill down into sub-Tables.
      @param key The key to look for (not null). The dot syntax can be used to drill
                 down into keys for any sub-Table (i.e. not object's properties/methods).
      @return <code>true</code> if Table contains this key, <code>false</code> otherwise.
      @see #containsValue(Object)
      @see #get(String, Object, int)
      @see #containsKeyx(String)
   */
  public boolean containsKey (String key) {
    Object val = get(key, null, flags|ONLY_DOT_INTO_SUBTABLES);
    return (val != null);
  }

  /** Indicates if the table contains the given value. <br>
      <br>
      <span style="color:red">Warning: The behavior for {@link #KV} mode is non-standard and does not
      match the {@link Map} interface. If the table is in {@link #KV} mode this does an identity
      check (<code>==</code>), rather than an equality (<code>.equals(..)</code>) check. (This is likely
      to change in future versions.)</span>
      @param value The value to look for.
      @return <code>true</code> if Table contains this value, <code>false</code> otherwise.
      @see #containsKey(String)
   */
  @Override
  public boolean containsValue (Object value) {
    checkModeFB();
    if (genericMode == KEY) {
      // No containsValue method in Keyable, need to iterate over each value and check it.
      for (String key : kbl.getKeys()) {
        if (value.equals(kbl.getKey(key))) return true;
      }
      return false;
    }
    else {
      switch (genericMode) {
        case KV:  return (kv.findValue(value) >= 0);
        case MAP: return map.containsValue(value);
        default:  throw new AssertionError("Unknown table mode "+genericMode);
      }
    }
  }

  /** <b>Legacy API:</b> Initializes an enumeration list of the table. The enumeration is the
   *  accessible using the {@link #more()} and {@link #next()} methods. <br>
   *  <br>
   *  <b>This is a Legacy API and should not be used in new code. The proper way to do this
   *  is via {@link #keySet()} or {@link #getKeys()} which follow the {@link Map} interface
   *  and can be used directly in an enhanced for loop.</b>
   *  @return The current size of the table.
   */
  @Deprecated
  public int enumeration () { // TODO: Deprecate this method
    en = new Enum(this);
    return size();
  }

  /** <b>Legacy API:</b> Return true if there are more elements in the enumeration list.
   *  Must use {@link #enumeration()} to initialize the enumeration first. <br>
   *  <br>
   *  <b>This is a Legacy API and should not be used in new code. The proper way to do this
   *  is via {@link #keySet()} or {@link #getKeys()} which follow the {@link Map} interface
   *  and can be used directly in an enhanced for loop.</b>
   *  @return <code>true</code> if the enumeration has more elements, <code>false</code> otherwise.
   *          (Returns <code>false</code> if {@link #enumeration()} has not been called.)
   *  @see #enumeration()
   *  @see #next()
   */
  @Deprecated
  public boolean more () { // TODO: Deprecate this method
    if (en==null) return false;
    return en.hasMoreElements();
  }

  /** <b>Legacy API:</b> Return the next key in the table enumeration list.
   *  Must use {@link #enumeration()} to initialize the enumeration first. <br>
   *  <br>
   *  <b>This is a Legacy API and should not be used in new code. The proper way to do this
   *  is via {@link #keySet()} or {@link #getKeys()} which follow the {@link Map} interface
   *  and can be used directly in an enhanced for loop.</b>
   *  @return The next key in the enumeration.
   *          (Returns <code>null</code> if {@link #enumeration()} has not been called.)
   *  @see #enumeration()
   *  @see #more()
   */
  @Deprecated
  public String next () { // TODO: Deprecate this method
    if (en==null) return null;
    return en.nextElement();
  }

  /** Dump the contents of the table to {@link System#out}. This is identical to
   *  <code>dump(System.out)</code> and is intended for debugging purposes only.
   */
  @SuppressWarnings("UseOfSystemOutOrSystemErr")
  public final void dump () { dump(System.out); }  // marked final since 3.7.0

  /** Dump the contents of the table to the given print stream. This is intended for debugging
      purposes only.
      @param dev The print stream (device) to dump the table to.
   */
  public void dump (java.io.PrintStream dev) {
    for (Iterator ti=iterator(); ti.getNext();) dev.println (ti.key+" = "+ti.value);
  }

  @Override
  public final String[] getKeys () {  // marked final since 3.7.0
    return getKeys(null);
  }

  /** Gets the list of keys as a comma separated list. This is the same as <code>getKeyList(null)</code>.
   *  @return The list of keys.
   */
  public final String getKeyList() {  // marked final since 3.7.0
    return getKeyList(null);
  }

  /** Gets the list of keys matching a filter (on values).
      @param valueFilter The filter to use (on values) to determine which keys to list.
                         (If this is <code>null</code> all keys are listed.)
      @return list of keys passing filter (empty list if none)
      @since NeXtMidas 3.3.1
   */
  public String[] getKeys (Filter valueFilter) {
    // TODO: JC4: This probably could be more efficient
    Iterator ti = iterator();
    if (valueFilter == null) return ti.keys;
    ArrayList<String> list = new ArrayList<>(ti.size);
    while (ti.getNext()) {
      if (valueFilter.accept(ti.value)) {
        list.add(ti.key);
      }
    }
    return list.toArray(StringUtil.EMPTY_STRING_ARRAY);
  }

  /** Gets the list of keys matching a filter (on values) as a comma separated list.
      @param valueFilter The filter to use (on values) to determine which keys to list.
                         (If this is <code>null</code> all keys are listed.)
      @return The list of keys (comma separated).
   */
  public String getKeyList (Filter valueFilter) {
    // TODO: JC4: This probably could be more efficient
    String list="";
    for (Iterator ti=iterator(); ti.getNext(); ) {
      if (valueFilter!=null && !valueFilter.accept(ti.value)) continue;
      if (list.length()==0) list = ti.key;
      else list += ","+ti.key;
    }
    return list;
  }


  /** Gets the list of the toString() of Values
      @return String array containing toString of values
  */
  @InternalUseOnly
  public String[] getValuesS () {
    Iterator ti = iterator();
    ArrayList<String> list = new ArrayList<>(ti.size);
    while (ti.getNext()) {
       list.add(this.get(ti.key).toString());
    }
    return list.toArray(StringUtil.EMPTY_STRING_ARRAY);
  }

  /** Converts this table to a String. This is the same as <code>toStringBuffer()</code>, except when
   *  the mode is {@link #FB}.
   *  @return This table as a string.
   */
  @Override
  public String toString () {
    if (fb!=null) return new String(fb);
    return toStringBuffer().toString();
  }

  /** Recursively goes through a Table and creates rows and tables for each sub-table and strings, returning an HTML
      representation for the whole thing.
      @param table A table to be converted into an HTML representation.
      @param fontColor The color of the text of the values.
      @return A string containing a HTML Table representation of the table
  */
  public static String toHTMLTable(final Table table, final String fontColor)
  {
    String displayText = "<table>";
    for (String key : table.keySet())
    {
       //If it's a table, parse out the keys and create rows by recursively going into the sub-table and making a new table for it.
       if(table.get(key) instanceof Table)
       {
         displayText += "<tr><td>" + key + "</td><td>" + toHTMLTable((Table)table.get(key), fontColor) + "</td></tr>";
       }
       //Otherwise create a row for the field if it's not a table.
       else
       {
         displayText += "<tr><td>" + key + "</td><td color=#"+fontColor + "><b>" + table.get(key) + "</b></td></tr>";
       }
    }
    displayText += "</table>";
    return displayText;
  }

  /** Converts this Table to a String. Same as {@link #toString()}, but this method
      translates escape sequences where needed (tabs, quotes, Unicode chars, etc)
      @return  The Table as a String
      @since NeXtMidas 2.9.0
   */
  public String getEscapedString () {
    return new String(toBytes());  // toBytes add the escapes back in
  }

  /** Serialize to a byte array.
   *  @return This table as a byte array.
   */
  public byte[] toBytes() {
    StringBuffer sb = new StringBuffer(1024);
    toStringBuffer(this,sb,flags);
    byte[] bytes = new byte[sb.length()];
    for (int i=0; i<bytes.length; i++) bytes[i] = (byte)sb.charAt(i);
    return bytes;
  }

  /** Serialize to a byte array. Used by Shell.saveState()
   *  @param replaceKeyDots replace any dots found in key names
   *  @return This table as a byte array.
   */
  @InternalUseOnly
  public byte[] toBytes(boolean replaceKeyDots) {
    StringBuffer sb = new StringBuffer(1024);
    toStringBuffer(this,sb,flags,replaceKeyDots);
    byte[] bytes = new byte[sb.length()];
    for (int i=0; i<bytes.length; i++) bytes[i] = (byte)sb.charAt(i);
    return bytes;
  }

  /** Converts the table to a string buffer. <br><br>
      <b>Note</b>: This method does NOT add any escape characters into the String.
      @return This table as a string buffer.
   */
  public synchronized StringBuffer toStringBuffer() {
    StringBuffer sb = new StringBuffer(1024);
    int flagsPlusDisableEscape = flags | DISABLE_ESCAPE_SEQUENCES; //Since 3.5.3 bug 2876
    int serializationFlags = ((flags & SERIALIZE_WITH_ORIGINAL_FLAGS) == 0) ?
        flagsPlusDisableEscape : flags;         // Since 3.8.0 allow serialization with just original flags
    toStringBuffer(this,sb,serializationFlags); // Since 3.5.3 new signature, no longer just passing esc = false
    return sb;
  }

  /** Adds the serialized string value of a given object to a string buffer. <br>
      <br>
      The object is added to the {@link StringBuffer} using the following formatting:
      <pre>
        <u>Class        Added As...                                  </u>
        <i>null</i>            "null"
        String          The string, enclosed in double-quotes.
        Data            In the format T:VAL, where T is the type
                        (D,F,X,L,I,B) and VAL is the value.
        KeyVector       Uses {@link KeyVector#toString()}.
        Table           Uses {@link KeyVector#toString()}.
        java.util.Map   Similar to that used for a Table.
        java.util.List  Similar to that used for an array.
        <i>array</i>    In the form (VAL0,VAL1,...,VALN)
        <i>all others</i> Uses {@link Object#toString()}.
      </pre>

      @param obj The object to add to it.
      @param sb  The string buffer to add to.
      @param esc Whether to use escape sequences.
      @deprecated Since NeXtMidas 3.5.3 use {@link #toStringBuffer(Object, StringBuffer, int)}
   */
  @Deprecated
  public static void toStringBuffer (Object obj, StringBuffer sb, boolean esc) {
    int tflags = esc ? 0 : DISABLE_ESCAPE_SEQUENCES; // Since 3.5.3 Bug 2876
// TODO: switch above line to next 3 lines
//    int tflags = Table.defaultFlags;
//    if (esc) tflags &= ~DISABLE_ESCAPE_SEQUENCES;
//    else     tflags |=  DISABLE_ESCAPE_SEQUENCES;
    // in 3.7.0 calls to non-deprecated toStringBuffer(...) Bug 2938
    toStringBuffer(obj, sb, tflags);
  }

  /** Adds the serialized string value of a given object to a string buffer. <br>
      <br>
      The object is added to the {@link StringBuffer} using the following formatting:
      <pre>
      <u>Class        Added As...                                  </u>
       <i>null</i>            "null"
       String          The string, enclosed in double-quotes.
       Data            In the format T:VAL, where T is the type
                       (D,F,X,L,I,B) and VAL is the value.
       KeyVector       Uses {@link KeyVector#toString()}.
       Table           Uses {@link KeyVector#toString()}.
       java.util.Map   Similar to that used for a Table.
       java.util.List  Similar to that used for an array.
       <i>array</i>    In the form (VAL0,VAL1,...,VALN)
       <i>all others</i> Uses {@link Object#toString()}.
      </pre>

      @param obj The object to add to it.
      @param sb  The string buffer to add to.
      @param int Table flags
      @since NeXtMidas 3.5.3 for Bug 2876
   */
  static void toStringBuffer (Object obj, StringBuffer sb, int tflags) {
    toStringBuffer(obj, sb, tflags, false);
  }

  /** Adds the serialized string value of a given object to a string buffer. <br>
      <br>
      The object is added to the {@link StringBuffer} using the following formatting:
      <pre>
      <u>Class        Added As...                                  </u>
       <i>null</i>            "null"
       String          The string, enclosed in double-quotes.
       Data            In the format T:VAL, where T is the type
                       (D,F,X,L,I,B) and VAL is the value.
       KeyVector       Uses {@link KeyVector#toString()}.
       Table           Uses {@link KeyVector#toString()}.
       java.util.Map   Similar to that used for a Table.
       java.util.List  Similar to that used for an array.
       <i>array</i>    In the form (VAL0,VAL1,...,VALN)
       <i>all others</i> Uses {@link Object#toString()}.
      </pre>

      @param obj The object to add to it.
      @param sb  The string buffer to add to.
      @param int Table flags
      @param replaceKeyDots replace any dots found in key names
      @since NeXtMidas 3.5.3 for Bug 2876
   */
  static void toStringBuffer (Object obj, StringBuffer sb, int tflags, boolean replaceKeyDots) {
    boolean dotGotReplaced = false;
    if (obj==null) {
      sb.append("null");
      return;
    }
    boolean esc = (tflags & DISABLE_ESCAPE_SEQUENCES) == 0;
    boolean saveTimeAsTColon = (tflags & SERIALIZE_TIME_AS_TCOLON) != 0;
    boolean keepJavaNumberType = (tflags & KEEP_JAVA_NUMERIC_TYPE) != 0;
    boolean onlyJavaNumericSerialization = (tflags & ONLY_JAVA_NUMERIC_SERIALIZATION) != 0; // 3.9.2 NXM-3437
    boolean quoteKeys = (tflags & QUOTED_KEYS_SERIALIZATION) != 0;
    boolean isTable=(obj instanceof Table);
    boolean addTblSig=esc && !isTable;
    Class<?> clazz = obj.getClass();
    if (isTable) {
      Table tab = (Table)obj;
           if (tab.fb!=null)         obj = tab.fb;
      else if (tab.genericMode==KV ) obj = tab.kv;
      else if (tab.genericMode==MAP) obj = tab.map;
      else if (tab.genericMode==KEY) obj = tab.kbl;
      else {
      sb.append("{}"); // An empty table
      return;
      }
    }
    if (obj instanceof KeyVector) {
      KeyVector kv = (KeyVector)obj;
      if (addTblSig) sb.append("O:KV:");
      sb.append(LCURLY);
      for (int i=0; i<kv.size(); ) {
        String key =  kv.getKey(i);
        if (quoteKeys) key = '"'+key+'"'; // since 4.0.0 for NXM-3715
        if (replaceKeyDots) {
          if (!dotGotReplaced && key.contains(".")) dotGotReplaced = true;
          key = key.replaceAll("\\.", saveStateKeyDotSubstitute);
        }
        sb.append(key);
        sb.append(EQUALS);
        toStringBuffer(kv.get(i),sb,tflags,replaceKeyDots);
        if (++i<kv.size()) sb.append(COMMA);
      }
      sb.append(RCURLY);
    }
    else if (obj instanceof Map) {
      if (addTblSig) {
             if (obj instanceof Hashtable        ) sb.append("O:HT:");
        else if (obj instanceof TreeMap          ) sb.append("O:TM:");
        else if (obj instanceof LinkedHashMap    ) sb.append("O:LHM:");
        else if (obj instanceof ConcurrentHashMap) sb.append("O:CHM:");
        else                                       sb.append("O:HM:");
      }
      final Map<?,?> map = (Map<?,?>)obj;
      final java.util.Iterator<?> entries = map.entrySet().iterator();
      sb.append(LCURLY);
      while (entries.hasNext()) {
        final Map.Entry<?,?> entry = (Map.Entry<?,?>)entries.next();
        Object origKey =  entry.getKey();
        String key = String.valueOf(origKey);
        if (replaceKeyDots) {
          if (!dotGotReplaced && key.contains(".")) dotGotReplaced = true;
          key = key.replaceAll("\\.", saveStateKeyDotSubstitute);
        }
        if (quoteKeys) key = '"'+key+'"'; // since 4.0.0 for NXM-3715
        final Object    value = entry.getValue();
        sb.append(key);
        sb.append(EQUALS);
        toStringBuffer(value,sb,tflags,replaceKeyDots);
        if (entries.hasNext()) sb.append(COMMA);
      }
      sb.append(RCURLY);
    }
    else if (obj instanceof List) {
      final java.util.List<?> list = (List<?>)obj;
      sb.append(LPAREN);
      for (int i=0; i<list.size(); i++) {
        if (i != 0) sb.append(COMMA);
        toStringBuffer(list.get(i),sb,tflags);
      }
      sb.append(RPAREN);
    }
    else if (obj instanceof byte[]) {
      byte[] fb = (byte[])obj;
      for (int i=0; i<fb.length; i++) sb.append((char)fb[i]);
    }
    else if (obj instanceof String) {
      String str = (String)obj;
      toStringBuffer(str,sb,tflags);
    }
    else if (obj instanceof Data) {
      Data data = (Data)obj;
      String str = data.toString();
      char type = (char)data.type;
      if (data.isString()) toStringBuffer (str,sb,tflags);
      else sb.append(type).append(':').append(str); // number with type prefix
    }
    else if (obj instanceof Keyable) {
      final Keyable  keyable = (Keyable)obj;
      final String[] keys    = keyable.getKeys();
      sb.append('{');
      for (int ii = 0; ii < keys.length; ii++) {
        String key   = (quoteKeys) ? '"'+keys[ii]+'"' : keys[ii];
        final Object value = keyable.getKey(key);
        if (ii != 0) sb.append(',');
        if (replaceKeyDots) {
          if (!dotGotReplaced && key.contains(".")) dotGotReplaced = true;
          key = key.replaceAll("\\.", saveStateKeyDotSubstitute);
        }
        sb.append(key); sb.append('='); toStringBuffer(value,sb,tflags,replaceKeyDots);
      }
      sb.append('}');
    }
   else if (clazz.isArray()) {
      final int length = Array.getLength(obj);
      sb.append('(');
      for (int ii = 0; ii < length; ii++) {
        if (ii != 0) sb.append(',');
        toStringBuffer(Array.get(obj,ii),sb,tflags);
       }
     sb.append(')');
    }
    else if ((clazz == Time.class) && saveTimeAsTColon) {   // Since 3.5.3 Bug 2876
      Time tobj = (Time)obj;
      String str = "T:" + tobj.toString("Full_Std",12);
      sb.append(str);  // needs to be pure T:.., no quotes, or it will be read back in as String
    }
    else if ((clazz == File.class) || (obj instanceof Path)) {
      // Convert any Windows backslash path delimiters to forward slashes
      // (Since 3.7.3 for NXM-3119)
      String str = obj.toString().replace('\\','/');
      toStringBuffer (str,sb,tflags);
    }
    else if (keepJavaNumberType && (obj instanceof Number)) {
      // Serialize JavaNumber objects with type prefixes, so they can be
      // re-serialized back into the appropriate type  - since 3.8.0 for NXM-3120
      obj  = handleOtherNumberClasses((Number)obj);
      String str = obj.toString();
      char type = Data.getDataTypeLetterPlus(obj);
      if (onlyJavaNumericSerialization) type = Character.toUpperCase(type); // 3.9.2 NXM-3437
      // If type is Double, Float, Long, Integer, Short, or Byte serialize with
      // Number lower-case prefix ('O' is default when type is not found)
      if (type != 'O') sb.append(type+":");
      sb.append(str);
    }
    else {
      String str = obj.toString();
      if (StringUtil.isNumber(str)) sb.append(str);
      else toStringBuffer(str,sb,tflags);
    }
    if (dotGotReplaced) {
      Shell.warning("In saving state table "+obj+" had the dots in the key names replaced with "+saveStateKeyDotSubstitute);
    }
  }

  /** Convert BigDecimals to Doubles and BigIntegers to Long if they fir
   *  Other Number types just get the Number returned
   *  @param javaNumber Java Number to check if it can fit in smaller type
   *  @return generally the argument passed in, but Double for BigDecimal
   *          and Long for BigInteger, if they fit.
   */
  private static Number handleOtherNumberClasses (Number javaNumber) {
    if (javaNumber instanceof BigDecimal) {
      Double doubleVal = ((BigDecimal)javaNumber).doubleValue();
      if ((doubleVal != Double.NEGATIVE_INFINITY)
          && (doubleVal != Double.POSITIVE_INFINITY)) {
        return doubleVal;
      }
      else return javaNumber; // value too large or small to fit in double
    }
    else if (javaNumber instanceof BigInteger) {
      try {
        return ((BigInteger)javaNumber).longValueExact();
      }
      catch (ArithmeticException ae) {
        // can not fit in long without losing information
        return javaNumber;
      }
    }
    else if (javaNumber instanceof AtomicInteger) {
      return ((AtomicInteger)javaNumber).intValue();
    }
    else if (javaNumber instanceof AtomicLong) {
      return ((AtomicLong)javaNumber).longValue();
    }
    else if (javaNumber instanceof DoubleAccumulator) {
      return ((DoubleAccumulator)javaNumber).doubleValue();
    }
    else if (javaNumber instanceof DoubleAdder) {
      return ((DoubleAdder)javaNumber).doubleValue();
    }
    else if (javaNumber instanceof LongAccumulator) {
      return ((LongAccumulator)javaNumber).longValue();
    }
    else if (javaNumber instanceof LongAdder) {
      return ((LongAdder)javaNumber).longValue();
    }
    else {
      return javaNumber;
    }
  }

  /** in 3.5.3, 3rd parameter became table flags, rather than just a boolean for escape sequences (Bug 2876 and Bug 2883) */
  private static void toStringBuffer (String str, StringBuffer sb, int tflags) {
    int nbs=0,nesc=0,nq=0,nc=0,iq=0,ip=0,ic=0,ls=str.length();
    boolean esc = (tflags&DISABLE_ESCAPE_SEQUENCES)==0;  //Since 3.5.3
    // decide if this string needs to be quoted or escaped
    for (int i=0; i<ls; i++) {
      char c=str.charAt(i);
           if (c==QUOTE) { nq++; iq=1-iq; }
      else if (c==BSLASH) nbs++;
      else if (c==COMMA  && iq==0 && ip==0 && ic==0) nc++;
      else if (c==LPAREN && iq==0) ip++;
      else if (c==RPAREN && iq==0) ip--;
      else if (c==LCURLY && iq==0) ic++;
      else if (c==RCURLY && iq==0) ic--;
      else if (c<SPACE || c>TILDE) nesc++;
    }
    // see if we can avoid escaping internal quotes
    if (nq>0 && nbs==0 && nc==0 && nesc==0) sb.append(str);
    // need to add escapes to quoted string (do here not toBytes() for performance and reuse of StringUtil)
    else if (nbs+nesc>0 && esc) sb.append('"').append(StringUtil.escapeString(str)).append('"');
    // quote all other strings for delimiters
    else sb.append('"').append(str).append('"');
  }

  /** Abbreviated form of {@link #toString()} used for debugging purposes only.
   *  @return The string form, abbreviated as necessary.
   */
  public String toStringAbbr () {
    String str = this.toString();
    if (str.length()<20) return str;
    return str.substring(0,20)+"...}";
  }

  /** De-serialize table entries from a byte array. This will clear the table and replace it with
   *  the given entries. Use {@link #append(String)} or {@link #fromBytes(byte[],int,int)} to just
   *  add in new entries. If the table mode is {@link #FB} this method will lazy-load the table.
   *  @param bytes The byte array.
   */
  public final void fromBytes (byte[] bytes) {  // marked final since 3.7.0 (avoids lint warnings in constructor)
    if (genericMode == FB) {  // lazy-load in FB mode
      fb = bytes;
    }
    else {                    // load immediately
      clear();
      fromBytes(bytes,0,bytes.length);
    }
  }

  /** Add table entries from the given table string.
   *  @param string The new entries to add.
   */
  public final void append (String string) {  // marked final since 3.7.0
    byte[] bytes = string.getBytes();
    fromBytes(bytes, 0, bytes.length);
  }

  /** De-serialize from a byte array (specified range). Unlike {@link #fromBytes(byte[])} this will
   *  NOT clear the table first.
   *  @param bytes  the byte array
   *  @param index1 start index in byte array
   *  @param index2 end (exclusive index in byte array
   *  @return status =0 good &lt;0 bad
   */
  public final int fromBytes (byte[] bytes, int index1, int index2) {  // marked final since 3.7.0 (avoids lint warnings in constructor)
    // developer's note: '(' and ')' can have special uses in a table
    // X-Midas file qualifier table can be bracketed with '(' and ')'
    // NeXMidas table keys can be a TEST
    //   {TEST(X,LT,3)=YELLOW}
    // NeXtMidas can also have keys where '(' or ')' is part of the key name - fixed in 4.0.0
    //   {1) first item=YELLOW}
    int ik1,ik2;  // index of key (start,end)
    int iv1,iv2;  // index of val (start,end)
    int ibr;      // number of braces
    int iparen;   // number of parenthesis not counted as braces
    int iqu;      // 1 for in quotes, 0 for out of quotes
    int i = index1;
    int n = index2;
    boolean escSeqEnabled  = (flags&DISABLE_ESCAPE_SEQUENCES)==0;
    boolean checkMalformed = (flags&CHECK_MALFORMED)!=0;
    boolean consistentHandlingWhiteSpace = (flags & DISABLE_CONSISTENT_HANDLING_OF_WHITESPACE) == 0;  // 3.8.1 fix for handling white space is not turned off
    boolean onlyJavaNumericSerialization = (flags & ONLY_JAVA_NUMERIC_SERIALIZATION) != 0;

    // trim off whitespace (before open/close braces) for BUG 2095
    while (i<n-1 && bytes[i]  <=SPACE) i++;                    // remove leading white space (left trim)
    while (i<n-1 && bytes[n-1]<=SPACE) n--;                    // remove trailing white space (right trim)

    // error checking
    boolean curlyTable = false;
    int length = n - i;
    if (length<2)  return -1;
         if (bytes[i]==LCURLY && bytes[n-1]==RCURLY) {curlyTable=true; } // tables must be bracketed by { }
    else if (bytes[i]==LPAREN && bytes[n-1]==RPAREN) { }                 // or by ( ) for X-Midas file qualifier syntax
    else {
      // exception/warning below added in 3.3.0 for Bug 85
      StringBuilder errmsg = new StringBuilder("Malformed Table (");
      if (bytes[i]!=LCURLY)   errmsg.append(" missing open '{' ,");
      if (bytes[n-1]!=RCURLY) errmsg.append(" missing close '}'");
      int limitedLen = Math.min(length, 1000); // limit error message size
      errmsg.append(" ) in table: ").append(new String(bytes, i, limitedLen));
      if (limitedLen < length) {
        errmsg.append(" [...truncated ").append(length - limitedLen).append(" bytes after 1000 limit...]");
      }
      if (checkMalformed) throw new MidasException(errmsg.toString());
      else Shell.warning(errmsg);
      return -1;
    }

    // trim off whitespace (within braces)
    while (i<n-1 && bytes[i+1]<=SPACE) i++;                    // remove white space at head (after  {)
    while (i<n-1 && bytes[n-2]<=SPACE) n--;                    // remove white space at tail (before })

    // parse loop
    for (i+=1,n-=1; i<n; ) {
      if (consistentHandlingWhiteSpace) { // new consistent handling of whitespace NXM-3143
        while (i<n && bytes[i]<=SPACE) i++;
        while (i<n && bytes[i]==RCURLY) {             // handle {A=2}{}{B=3,C=4} syntax
          int k = i+1;
          while (k<n && bytes[k]<=SPACE) k++;         // ignore whitespace between LCURLY and RCURLY
          if (k>=n || bytes[k]!=LCURLY) break;
          i=k+1;
          while (i<n && bytes[i]<=SPACE) i++;         // ignore whitespace immediately after LCURLY
        }
      }
      else {                                          // old pre 3.8.1 behavior
        while (i<n && bytes[i]==SPACE) i++;
        while (i<n && bytes[i]==RCURLY) {             // handle {A=2}{}{B=3,C=4} syntax
          int k = i+1;
          while (k<n && bytes[k]==SPACE) k++;         // ignore space between LCURLY and RCURLY
          if (k>=n || bytes[k]!=LCURLY) break;
          i=k+1;
          while (i<n && bytes[i]==SPACE) i++;         // ignore space immediately after LCURLY
        }
      }
      ik1=i;
      ik2=i;
      iv1=ik2+1;
      int nEq = 0;                                  // Count stray ='s which follow other ='s
      boolean inEscSeq = false;
      boolean beforeEquals = true;
      for (ibr=0,iqu=0,iparen=0; i<n && (iparen> 0 || ibr>0 || iqu>0 || bytes[i]!=COMMA); i++) {
        boolean inKeySection = curlyTable && beforeEquals; // NXM-3715
        byte ibyte = bytes[i];
        if (ibyte==QUOTE && (!escSeqEnabled || !inEscSeq)) {
          iqu = 1-iqu;         // toggle in/out of quotes
          inEscSeq = false;    // a start or end quote resets in escape seq
        }
        else if (iqu!=0) {     // bypass counters inside quotes
          inEscSeq = escSeqEnabled && !inEscSeq && (ibyte==BSLASH); // figure out if in esc seq
        }
        else if (ibyte==LCURLY) {beforeEquals=true; ibr++;}
        else if (ibyte==RCURLY) ibr--;
        else if (ibyte==LPAREN) {
          if (!inKeySection || doNotAllowRandomParen) ibr++;
          else iparen++;
        }
        else if (ibyte==RPAREN) {
          if (!inKeySection || doNotAllowRandomParen) ibr--;
          else iparen--;
        }
        else if (ibyte==EQUALS && ibr==0) {
          beforeEquals = false;
          if (++nEq==1) {
            ik2=i;  // found index for end of key (ik2)
            iv1=ik2+1;
            if (consistentHandlingWhiteSpace) { // since 3.8.1 for NXM-3143
              while (bytes[iv1] <= SPACE) {
                iv1++;  // get rid of whitespace between EQUALS and value
                i++;
              }
            }
          }
        }
        if (ibr<0) break;
      }
      if (i<=ik1) break;
      if (nEq==0) ik2=i;
      iv2=i;
      //iv1=ik2+1; // moved in 3.8.1 to add support for ignoring space after equals
      while (bytes[ik2-1]<=SPACE) ik2--;         // trim spaces between key and equals
      if (bytes[ik1]==QUOTE)  { ik1++; ik2--; }  // trim quotes around key names
      if(consistentHandlingWhiteSpace) {
        while(bytes[iv2-1]<=SPACE)iv2--;         // trim spaces at end of value 3.8.1 to fix issue with spaces between end of value and the comma
      }
      if (nEq>1) {
        //jph -- eliminated warning until I can filter valid case such as
        //   {SWITCHES=/VERBOSE=ON/STATE=10}
        //String msg = "Malformed Table (missing comma after key) in table: "+new String(bytes,iv1,len);
        //if (checkMalformed) throw new MidasException(msg);
        //else Shell.warning(msg);
      }

      // handle the key
      char type = '_';
      if (bytes[ik1+1]==COLON) { type = (char)bytes[ik1]; ik1+=2; }
      // using Convert.unpackS is faster than new String, because it assumes ASCII
      String key = Convert.unpackS(bytes,ik1,ik2-ik1);
      if ((flags&CASESENSITIVE) == 0) key = key.toUpperCase(); // not case sensitive - convert to uppercase

      // handle the value
      int len=iv2-iv1; // length of value
      if (len<=0) {
        put(key, "");
      }
      else if (bytes[iv1]==QUOTE) {
        // check below added to prevent malformed Tables when given bad input (Since 3.3.0 - Bug 85)
        if (bytes[iv2-1]!=QUOTE) { // missing closing quote
          String msg = "Malformed Table (improperly quoted) in table: "+new String(bytes,iv1,len);
          if (checkMalformed) throw new MidasException(msg);
          else Shell.warning(msg);
        }
        String res = new String(bytes,iv1+1,len-2);
        if (escSeqEnabled) res = StringUtil.unescapeString(res);
        put(key, res);
      }
      else if (bytes[iv1]==LCURLY) {
        Table sub = new Table(specificMode,flags);
        if(caretEnabled) sub.setPrevious(this);  // to allow self reference while reading (Jeff Change since 3.9.0)
        sub.fromBytes (bytes,iv1,iv2);
        if(caretEnabled) sub.setPrevious(null);  // undo self reference for normal operation (Jeff Change since 3.9.0)
        put(key, sub);
      }
      // This else if block is added by Jeff since 3.9.0
      else if (caretEnabled && bytes[iv1]==CARET) {       // allow entries to refer to previously defined members
        String fkey = new String(bytes,iv1+1,len-1);
        Table root = getRoot();
        Object rval = KeyObject.getKey(root,fkey);
        if (rval!=null) put(key, rval);
        else put(key, "^"+fkey);
      }
      else if (len>2 && bytes[iv1+1]==COLON) {
        String res = new String(bytes,iv1+2,len-2);
        char newType = (char)bytes[iv1];
        // since 3.9.2 - allow for all numeric types to be reserialized as Java Numbers
        char checkNumberType = (onlyJavaNumericSerialization) ? Character.toLowerCase(newType) : newType;
        // Allow serialization that preserves Java Number types - since 3.8.0 for NXM-3120  TODO 3.9.2 NXM-3437
        Number num;
        switch (checkNumberType) {
          case 'd': num = Double.valueOf(res); break;  // Java Double
          case 'f': num = Float.valueOf(res); break;   // Java Float
          case 'x': num = Long.valueOf(res); break;    // Java Long
          case 'l': num = Integer.valueOf(res); break; // Java Integer
          case 'i': num = Short.valueOf(res); break;   // Java Short
          case 'b': num = Byte.valueOf(res); break;    // Java Byte
          case 'n': num = new BigDecimal(res); break;  // Java BigDecimal NXM-4001
          default: num = null; break;
        }
        if (num != null) {
          put(key,num,null);
        }
        else {
          put(key, Convert.s2o(res,newType,null) );
        }
      }
      else {
        String res = new String(bytes,iv1,len);
        put(key, Convert.s2o(res,type,null) );
      }

      // special handling for merging tables (e.g. "{A=B,C=D}{E=F}" in table syntax)
      if (ibr<0 && i<n && bytes[i+1]==LCURLY) i++;

      // trim off whitespace (between key=value pairs)
      while (i<n && bytes[++i]==SPACE) { }
    }
    return 0;
  }

  /** Replace all occurrences of a regex in the keys with a String replacement
   *  @param regexToReplace  regex to be matched and replaced
   *  @param replacement     String replacement
   *  @since NeXtMidas 4.0.1
   */
  @InternalUseOnly
  public void modifyKeys (String regexToReplace, String replacement) {
    if ((regexToReplace==null) || (replacement==null)) return;
    boolean replacementContainsDots = replacement.contains(".");
    boolean regexIsDot = "\\.".equals(regexToReplace);
    if (replacementContainsDots) flags = flags| Table.ALLOW_DOTS_IN_KEYS;
    String[] origKeys = getKeys();
    String[] newKeys = new String[origKeys.length];
    Object[] newVals = new Object[origKeys.length];
    for (int i = 0; i < origKeys.length; i++) {
      String origKey = origKeys[i];
      Object origVal = getKey(origKey);
      if (origVal instanceof Table) {
        Table origValTbl = (Table)origVal;
        origValTbl.setFlags(getFlagsInt());
        origValTbl.modifyKeys(regexToReplace,replacement);
      }

      newKeys[i] = origKey.replaceAll(regexToReplace,replacement);
      newVals[i] = origVal;
    }

    clear();

    for (int i = 0; i < newKeys.length; i++) {
      putx(newKeys[i], newVals[i]);
    }
    if (regexIsDot && !replacementContainsDots) setFlags("-AllowDotsInKeys");
  }

  /** Prints out a warning message, along with the Table itself, if the Table has dots in keys
   *  @param tblName the name of the Table
   *  @since NeXtMidas 4.0.1
   */
  @InternalUseOnly
  public void hasDotsInKeys (String tblName) {
    boolean dotsInKeys = hasDotsInKeys();
    if (dotsInKeys) {
      Shell.warning("CONVERT XML2TAB was converting XML that had dots in key name. To avoid crashing,\n" +
          "      a table was created in a special limited function mode that allows dots in keys.\n" +
          "      Replace the '.' characters before serializing this table.\n "+
          "     " + tblName +" = " + this);
    }
  }

  private boolean hasDotsInKeys () {
    String[] tblKeys = getKeys();
    for (String key : tblKeys) {
      if (key.contains(".")) {
        return true;
      }
      else {
        Object objVal = getKey(key);
        if (objVal instanceof Table) {
          Table tblVal = (Table)objVal;
          boolean hasDottedKeys = tblVal.hasDotsInKeys();
          if (hasDottedKeys) return hasDottedKeys;
        }
      }
    }

    return false;
  }

  /** Set the contents of this table to the contents of the given hashtable. <br>
      <br>
      The Table will have its mode set to Hashtable mode ({@link #HT}) and its contents will
      be replaced with the contents of the parameter. <br>
      <br>
      This method performs a shallow copy of the parameter contents. Upon returning from this
      method the parameter can be modified without affecting the table. Modifying the contents
      of the hash will affect the table contents.
      @param table The hashtable containing the values to insert.
      @deprecated Since NeXtMidas 3.7.0: Use {@link #clear()} followed by {@link #putAll(Map)}
                  to be consistent with {@link Map} interface.
  */
  @Deprecated
  public void fromTable (Hashtable<?,?> table) {
    clear();
    setMode(HT);
    putAll(table);
  }

  /** Add entries to the table by de-serializing from a text file. This method allows duplicates by
   *  using {@link #KV} mode and adding the {@link #ADD} flag during loading. The original table
   *  flags will then be restored in the end (possibly causing duplicates to be dropped).
   *  @param tf    TextFile to read from (by convention the <code>.tbl</code> extension is used).
   *  @see #fromTextFile(TextFile, int)
   *  @see #fromTableFile
   */
  public final void fromTextFile (TextFile tf) {  // marked final since 3.7.0 (avoids lint warnings in constructor)
    int saveFlags = flags; // since 2.7.1: save current flags to restore at end of this method (BUG 1838)
    setMode(KV);           // since 2.7.1: by using KeyVector mode and ADD flag (on next line) (BUG 1839)
    fromTextFile(tf, flags|ADD); // allows duplicate keys in Table (KeyVector mode only)
    setFlags(saveFlags);   // restore flags after loading table to match 2.6.x behavior (BUG 1838)
  }

  /** Add entries to the table by de-serialize from a text file using specified table flags.
   *  <b>This method will set the table flags to the new flags provided.</b> If the flags contains
   *  the {@link #ADD} option, then table mode will be forced KeyVector mode to allow duplicate
   *  entries.
   *  @param tf    TextFile to read from (by convention the <code>.tbl</code> extension is used).
   *  @param flags Table flags to use (set) on this Table
   *  @see #fromTableFile
   *  @since NeXtMidas 3.1.2
   */
  public final void fromTextFile (TextFile tf, int flags) {  // marked final since 3.7.0 (avoids lint warnings in constructor)
    char lastchar=',';
    String line;
    StringBuilder sb = new StringBuilder(4096);
    sb.append('{');

    setFlags(flags);
    tf.open(TextFile.INPUT|TextFile.FORCEABORT);
    while ((line=tf.readProper())!=null) {
      line = line.trim();      // allow for indents
      // automatically add commas at end-of-previous line if appropriate
      if (line.charAt(0)=='}' || line.charAt(0)=='{') { }
      else if (lastchar=='{') { }
      else if (lastchar!=',') sb.append(',');
      sb.append(line); // add to buffer
      lastchar = line.charAt(line.length()-1);
    }
    tf.close();
    sb.append('}');
    String str = sb.toString();
    if (str.startsWith("{{") && str.endsWith("}}")) {
      // This happens when the text file includes the start/end braces (e.g. "{A=1,B2=}"). While
      // NeXtMidas will not write out a file like this, M2K will (Bug 965) and it is a common to
      // see in a hand-edited file.
      str = str.substring(1, str.length()-1);
    }
    fromBytes(str.getBytes());
  }

  /** Writes the table out to a text file. <b>This assumes the file has not been opened yet.</b>
   *  WARN: this will close the input text file.
   *  @param tf TextFile to write to (by convention the <code>.tbl</code> extension is used).
   */
  public void toTextFile (TextFile tf) {
    tf.open(TextFile.OUTPUT|TextFile.FORCEABORT);
    toTextFile(tf,this,"");
    tf.close();
  }

  /** Writes the table out to a section of a text file with specified indent. <b>This assumes the
   *  file has been opened prior to calling this method.</b>
   *  @param tf  The currently opened textfile to add to
   *  @param t   The table to list
   *  @param pad The String indent to use (usually spaces)
   */
  @InternalUseOnly("Used by PANEL when writing .mmp files") // Marked since NeXtMidas 3.5.4
  public static void toTextFile (TextFile tf, Table t, String pad) {
    Table.Iterator ti = t.iterator();
    for (int i=0; i<ti.size; i++) {
      String key = ti.keys[i];
      Object value = ti.values[i];
      if (value instanceof Table) {
        tf.writeln(pad+key+"={");
        toTextFile(tf,(Table)value,pad+"  ");
        tf.writeln(pad+"}");
      }
      else {
        final StringBuffer sb = new StringBuffer();
        toStringBuffer(value, sb, t.flags); // Since 3.5.3 pass in flags directly
        tf.writeln(pad+key+"="+sb);
      }
    }
  }

  /** Get a {@link Data} object from the table.
      @param key The name of the entry to get. See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @return The named entry as a {@link Data} object or <code>null</code> if not found.
   */
  public Data getData (String key) {
    Object entry = get(key);
    if (entry == null) return null;
    if (entry instanceof String) entry = new Data((String)entry);
    return (Data)entry;
  }

  /** Get a {@link Table} object from the table. The same as calling <code>getTable(key,false)</code>.
      @param key The name of the entry to get. See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @return The named entry as a {@link Table} object or null if not found. Also returns null if
              the entry is found, but is not an instance of {@link Table}.
      @throws MidasException when using the dot syntax in the key to drill down into desired
              object but does NOT exist (unless {@link #ONLY_DOT_INTO_SUBTABLES} flag is set).
      @see #getTable(String, Table)
   */
  public final Table getTable (String key) {  // marked final since 3.7.0
    return getTable(key,false);
  }

  /** Get a {@link Table} object from the table. If the object returned from
      the given key is not a {@link Table} and the checkVal param is true,
      this method will attempt to convert the Object to a Table.
      @param key The name of the entry to get. See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @param converto2t Whether or not to attempt to convert the returned entry to a {@link Table}
                        via {@link Convert#o2t(Object)} when it is not an instance of a Table.
      @return The named entry value as a {@link Table} object or null if not found. If entry is not
              a Table object and converto2t is true, then convert entry to a Table using
              {@link Convert#o2t(Object)}, otherwise return null.
      @throws MidasException when using the dot syntax in the key to drill down into desired
              object but does NOT exist (unless {@link #ONLY_DOT_INTO_SUBTABLES} flag is set).
      @see #getTable(String, Table)
      @since NeXtMidas 2.8.2
   */
  public Table getTable (String key, boolean converto2t) {  // This method added for BUG 2189
    Object entry = get(key);
    if (entry instanceof Table) return (Table)entry;
    if (converto2t)             return Convert.o2t(entry);
    return null;
  }

  /** Get a {@link Table} object from the table at the specified key. If the object returned from
      the given key is not a {@link Table} object, then the defValue is returned.
      The dot syntax can be used to drill down into keys for sub-Table(s) at any level (but NOT
      into the entry value's properties or methods). <br>
      Note that this is different from the standard <a href="#dotsyntax">dot syntax</a> behavior,
      since this forces the {@link #ONLY_DOT_INTO_SUBTABLES} option via call to
      <code>get(key, defValue, flags|ONLY_DOT_INTO_SUBTABLES)</code>.
      @param key The name of the entry to get Table value (not null). The dot syntax can be used to
                 drill down into keys for any sub-Table (but not object's properties/methods).
      @param defValue The default Table value for the parameter if the entry at the specified key does not exist.
      @return The named entry as a {@link Table} object or defValue if not found. Also returns defValue if
               the entry is found, but is not an instance of {@link Table}.
      @see #getTable(String)
      @see #get(String, Object, int)
      @since NeXtMidas 2.9.2
   */
  public Table getTable (String key, Table defValue) {
    Object entry = get(key, defValue, flags|ONLY_DOT_INTO_SUBTABLES);
    if (entry instanceof Table) return (Table)entry;
    else return defValue;
  }

  /** Get a {@link Table} object from the table at the specified key. Same as
      <code>return getTable(key,null)</code>.
      @param key The name of the entry to get Table value (not null). The dot syntax can be used to
                 drill down into keys for any sub-Table (but not object's properties/methods).
      @return The named entry as a {@link Table} object or null if not found. Also returns null if
               the entry is found, but is not an instance of {@link Table}.
      @see #getTable(String, Table)
      @see #get(String, Object, int)
      @since NeXtMidas 2.9.2
   */
  public final Table getTableOrNull (String key) {  // marked final since 3.7.0
    return getTable(key,null);
  }

  /** Get a {@link String} object from the table. <b>Note that this method does NOT attempt to convert
   *  a non-String to a String.</b>
   *  @param key The name of the entry to get. See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
   *  @return The named entry as a {@link String} object or null if not found. Also returns null if
   *          the entry is found, but is not an instance of {@link String}.
   *  @see #getS(String)
   */
  public String getString (String key) {
    // TODO: Change this to call getString(key,null)
    Object entry = get(key);
    if (entry instanceof String) return (String)entry;
    return null;
  }

  /** Get a {@link String} object from the table (drilling down into keys for any sub-Table (but
   *  not object's properties/methods). <b>Note that this method does NOT attempt to convert
   *  a non-String to a String.</b>
   *  @param key The name of the entry to get. The dot syntax can be used to
   *             drill down into keys for any sub-Table (but not object's properties/methods).
   *  @param defValue The value to return if specified key is not found.
   *  @return The named entry as a {@link String} object -or- defValue if key not found -or- null if entry
   *          value is NOT an instance of {@link String}.
   *  @see #getString(String)
   *  @see #getS(String,String)
   *  @since NeXtMidas 3.1.0
   */
  public String getString (String key, String defValue) {
    Object entry = get(key, defValue, flags|ONLY_DOT_INTO_SUBTABLES);
    if (entry instanceof String) return (String)entry;
    return null;
  }

  /** Get a {@link DataFile} object from the table.
      @param key The name of the entry to get. See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @return The named entry as a {@link DataFile} object or null if not found. Also returns null if
              the entry is found, but is not an instance of {@link DataFile}.
   */
  public DataFile getDataFile (String key) {
    Object entry = get (key);
    if (entry instanceof DataFile) return (DataFile)entry;
    return null;
  }

  /** Get a unique key instance name. This is identical to <code>getUniqueKey(key,"")</code>.
   *  <b>WARNING: This method was designed for use on small tables (usually less than 32 elements)
   *   and is extremely inefficient to use on large tables.</b>
   *  @param key The root name of the key.
   *  @return The unique form of the key name.
   */
  public final String getUniqueKey (String key) {  // marked final since 3.7.0
    return getUniqueKey(key,"");
  }

  /** Get a unique key instance name by adding the specified delimiter and the key number to make
   *  the key name unique. <b>WARNING: This method was designed for use on small tables
   *  (usually less than 32 elements) and is extremely inefficient to use on large tables.</b>
   *  @param key   The root name of the key.
   *  @param delim The delimiter to use between the root name of the key and the index number.
   *  @return The unique form of the key name.
   */
  public String getUniqueKey (String key, String delim) {
    String keyi = key;
    for (int i=1; containsKey(keyi); i++) keyi=key+delim+i;
    return keyi;
  }

  /** Renames an existing key. If the oldName is not found nothing is done. Note that this operation
      may or may not affect the iteration order (such is undefined and varies based on mode).
      @param oldName The old key name.
      @param newName The new key name.
   */
  public void rename (String oldName, String newName) {
    checkModeFB();
    if (genericMode == KV) {
      kv.rename(oldName, newName);
    }
    else if (genericMode == KEY) {
      Object val = kbl.getKey(oldName);
      if (val != null) kbl.setKey(newName, val);
    }
    else if (genericMode == MAP) {
      Object val = map.remove(oldName);
      if (val != null) map.put(newName, val);
    }
    else {
      throw new AssertionError("Unknown table mode "+genericMode);
    }
  }

  //////////////////////////////////////////////////////////////////////////////
  // Begin implementation of KeyableByType interface
  // See note on the <a href="#dotsyntax">dot syntax</a> for key parameter in
  // class description above.
  //////////////////////////////////////////////////////////////////////////////
  @Override public final String getS (String key) { return getS(key,null);     }
  @Override public final double getD (String key) { return getD(key,0.0d);     }
  @Override public final float  getF (String key) { return getF(key,0.0f);     }
  @Override public final long   getX (String key) { return getX(key,0L);       }
  @Override public final int    getL (String key) { return getL(key,0);        }
  @Override public final short  getI (String key) { return getI(key,(short)0); }
  @Override public final byte   getB (String key) { return getB(key,(byte)0);  }
  @Override public final Object getO (String key) { return getO(key,null);     }
  @Override public final boolean getState (String key)   { return getState(key,false);   }
  @Override public final boolean getNoState (String key) { return getNoState(key,false); }

  // The methods below marked final from NeXtMidas 3.7.0 (getO was final prior to 3.7.0)
  @Override public final String getS (String key, String def) { Object entry=get(key); return (entry==null)? def: Convert.o2s(entry); }
  @Override public final double getD (String key, double def) { Object entry=get(key); return (entry==null)? def: Convert.o2d(entry); }
  @Override public final float  getF (String key, float  def) { Object entry=get(key); return (entry==null)? def: Convert.o2f(entry); }
  @Override public final long   getX (String key, long   def) { Object entry=get(key); return (entry==null)? def: Convert.o2x(entry); }
  @Override public final int    getL (String key, int    def) { Object entry=get(key); return (entry==null)? def: Convert.o2l(entry); }
  @Override public final short  getI (String key, short  def) { Object entry=get(key); return (entry==null)? def: Convert.o2i(entry); }
  @Override public final byte   getB (String key, byte   def) { Object entry=get(key); return (entry==null)? def: Convert.o2b(entry); }
  @Override public final Object getO (String key, Object def) { Object entry=get(key); return (entry==null)? def: entry;              }

  @Override
  public boolean getState (String key, boolean def) {
    String str = getS(key,null);
    if (str == null)             return def;
    if (StringUtil.isTrue(str))  return true;
    if (StringUtil.isFalse(str)) return false;
    return (getD(key) > 0);
  }

  @Override
  public final boolean getNoState (String key, boolean def) {
    return !getState(key,!def);
  }

  /**
   * TODO this will be added in KeyableByType interface when java 7 is no longer supported
   * Get data object and return list
   * @param key   key associated with List
   * @return list
   */
 // @Override
  public List<?> getList (String key) {
    return Convert.o2List(getO(key));
  }

  //////////////////////////////////////////////////////////////////////////////
  // End implement KeyableByType interface
  //////////////////////////////////////////////////////////////////////////////

  /** Puts a numeric value into the table. <br>
      This is the same as <code>put(key,Convert.d2o(value));</code>.
      @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @param value The value to insert.
   */
  public void put (String key, double value) {
    if ((flags & KEEP_JAVA_NUMERIC_TYPE) != 0) {
      // Since 3.8.0 for NXM-3120 have option to store Table values as Number subclasses
      put(key,Double.valueOf(value));
    }
    else {
      put(key,Convert.d2o(value));
    }
  }

  /** Puts a numeric value into the table. <br>
      This is the same as <code>put(key,Convert.f2o(value));</code>.
      @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @param value The value to insert.
   */
  public void put (String key, float value) {
    if ((flags & KEEP_JAVA_NUMERIC_TYPE) != 0) {
      // Since 3.8.0 for NXM-3120 have option to store Table values as Number subclasses
      put(key,Float.valueOf(value));
    }
    else {
      put(key,Convert.f2o(value));
    }
  }

  /** Puts a numeric value into the table. <br>
      This is the same as <code>put(key,Convert.x2o(value));</code>.
      @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @param value The value to insert.
   */
  public void put (String key, long value) {
    if ((flags & KEEP_JAVA_NUMERIC_TYPE) != 0) {
      // Since 3.8.0 for NXM-3120 have option to store Table values as Number subclasses
      put(key,Long.valueOf(value));
    }
    else {
      put(key,Convert.x2o(value));
    }
  }

  /** Puts a numeric value into the table. <br>
      This is the same as <code>put(key,Convert.l2o(value));</code>.
      @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @param value The value to insert.
   */
  public void put (String key, int value) {
    if ((flags & KEEP_JAVA_NUMERIC_TYPE) != 0) {
      // Since 3.8.0 for NXM-3120 have option to store Table values as Number subclasses
      put(key,Integer.valueOf(value));
    }
    else {
      put(key,Convert.l2o(value));
    }
  }

  /** Puts a numeric value into the table. <br>
      This is the same as <code>put(key,Convert.i2o(value));</code>.
      @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @param value The value to insert.
   */
  public void put (String key, short value) {
    if ((flags & KEEP_JAVA_NUMERIC_TYPE) != 0) {
      // Since 3.8.0 for NXM-3120 have option to store Table values as Number subclasses
      put(key,Short.valueOf(value));
    }
    else {
      put(key,Convert.i2o(value));
    }
  }

  /** Puts a numeric value into the table. <br>
      This is the same as <code>put(key,Convert.b2o(value));</code>.
      @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @param value The value to insert.
   */
  public void put (String key, byte value) {
    if ((flags & KEEP_JAVA_NUMERIC_TYPE) != 0) {
      // Since 3.8.0 for NXM-3120 have option to store Table values as Number subclasses
      put(key,Byte.valueOf(value));
    }
    else {
      put(key,Convert.b2o(value));
    }
  }

  /** Puts a boolean value into the table. <br>
      This is the same as using <code>put(key,Boolean.TRUE)</code> or <code>put(key,Boolean.FALSE)</code>.
      @param key   The key to insert (not <code>null</code>). See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @param value The value to insert.
      @see #put(String, Object)
   */
  public void put (String key, boolean value) { put(key, (value)? Boolean.TRUE : Boolean.FALSE); }

  /** Starting with the current table, walk backward through the "previous" links until it finds
      the root table (located at the very end of the list because it was the first one created).
      The root table is the one whose "prev" link is set to null.
      @return The root table.
      @see #getPrevious()
   */
  public Table getRoot () {
    Table tbl = this;
    while (tbl.prev != null) tbl = tbl.prev;
    return tbl;
  }

  /** Update the "previous" link so that this table points back to its predecessor, maintaining
      the linked list of tables that eventually lead back to the global table.
      @param  prevTable to set "previous" link to
   */
  public void setPrevious (Table prevTable) {
    if (prev != null && prevTable != null && prevTable != prev) {
      Shell.warning("Table.setPrevious overwriting existing previous link");
    }
    prev = prevTable;
  }

  /** Gets the previous link, see {@link #setPrevious} for details.
      @return The previous link.
      @see #setPrevious(Table)
   */
  public Table getPrevious () { return prev; }

  /** Sets the default Table operating flags. Multiple flags can be set by OR'ing them together.
      Care must be taken in setting this since any change to the default flags could affect existing
      functionality that expects the old (NeXtMidas 2.6.0) default of {@link #AUTOREADPREV}.
      @param mask The flag mask.
      @since NeXtMidas 2.7.1
   */
  public static void setDefaultFlags (int mask) {
    // prevent contradictory setting on handling null values - since 3.7.3 for NXM-3122
    if ((mask & CONTRADICTORY_NULL_FLAGS) == CONTRADICTORY_NULL_FLAGS) {
      throw new IllegalArgumentException(
          "Illegal default flags mask (contains both PERMIT_NULL and EXCEPTION_ON_NULL):"
          + Integer.toHexString(mask));
    }
    defaultFlags = mask;
  }

  /** Sets the default Table operating flags. Multiple flags can be set by OR'ing them together.
      Care must be taken in setting this since any change to the default flags could affect existing
      functionality that expects the old (NeXtMidas 2.6.0) default of {@link #AUTOREADPREV}.
      @param mask The flag mask (see {@link #flagsList}).
      @since NeXtMidas 2.7.1
   */
  public static final void setDefaultFlags (String mask) {
    setDefaultFlags(Parser.mask(flagsList, mask, getDefaultFlagsInt()));
  }

  /** Gets the default Table flags.
      @return The flag mask (see {@link #flagsList}).
      @since NeXtMidas 2.7.1
   */
  public static int getDefaultFlagsInt () {
    return defaultFlags;
  }

  /** Gets the Table operating flags.
      @return The flag mask (see {@link #flagsList}).
      @since NeXtMidas 2.7.1
   */
  public static final String getDefaultFlags () {
    return Parser.mask2s(flagsList, getDefaultFlagsInt());
  }

  /** Sets the Table operating flags. Multiple flags can be set by OR'ing them together.
      Since NeXtMidas 2.7.1, if the ADD flag is specified, it will change this Table to
      KeyVector mode to allow duplicate keys in the Table.
      @param mask The flag mask.
   */
  public void setFlags (int mask) {
    // prevent contradictory setting on handling null values - since 3.7.3 for NXM-3122
    if ((mask & CONTRADICTORY_NULL_FLAGS) == CONTRADICTORY_NULL_FLAGS) {
      throw new IllegalArgumentException(
          "Illegal flags mask (contains both PERMIT_NULL and EXCEPTION_ON_NULL):"
          + Integer.toHexString(mask));
    }
    flags = mask;
    if ((flags&ADD)!=0) setMode(KV); // ADD flag implies KeyVector mode (Bug 1839)
  }

  /** Sets the Table operating flags.  Multiple flags can be set by OR'ing them together.
      @param mask The flag mask (see {@link #flagsList}).
      @see #setFlags(int)
   */
  public final void setFlags (String mask) {
    setFlags(Parser.mask(flagsList, mask, getFlagsInt()));
  }

  /** Gets the Table operating flags.
      @return The flag mask as an int (see {@link #flagsList}).
      @since NeXtMidas 2.5.0
   */
  public int getFlagsInt () {
    return flags;
  }

  /** Gets the Table operating flags.
      @return The flag mask as a String (see {@link #flagsList}).
      @since NeXtMidas 2.7.0
   */
  public final String getFlags () {
    return Parser.mask2s(flagsList, getFlagsInt());
  }

  /** set to false to turn off 4.0.0 fix that allows '(' and ")" as characters in a key
   *
   *  @param doNotAllowRandomParen true to disable fix and revert to old behavior
   */
  public static void setDoNotAllowRandomParen (boolean doNotAllowRandomParen) {
    Table.doNotAllowRandomParen = doNotAllowRandomParen;
  }

  /** Adds a Table listener. <br>
      @param listener The Table.Listener object.
   */
  public void addListener (Listener listener) {
    if (this.listener!=null && this.listener!=listener) {
      throw new MidasException("Table currently only supports one listener");
    }
    this.listener = listener;
  }

  /** Removes a Table listener. <br>
      @param listener The Table.Listener object.
   */
  public void removeListener (Listener listener) {
    // This method did nothing. Previous code: listener=null)
    if (listener==this.listener) this.listener=null;
  }

  /** Gets an enumeration of the keys in this table.
      @return An enumeration of the keys in this table.
   */
  public Enumeration<String> keys () { return new Enum(this); }

  /** Gets the hashtable that holds the data for this table. <br>
   *  If the mode is not set to hashtable ({@link #HT}), this method will set it to such.
   *  @return The hashtable that holds the data for this table.
   *  @deprecated Since NeXtMidas 3.7.0: This method was used to convert to a {@link Map} instance
   *              prior to Table implementing {@link Map} (which is has done since NeXtMidas 3.3.1).
   */
  @Deprecated
  public Hashtable<String,Object> getHashtable () {
    setMode(HT);
    return (Hashtable<String,Object>)map;
  }

  /** Returns an inverse of this table (that is value=key). Note that this will convert each value
   *  of the current table to a key prior to use in the new table (since keys need to be strings).
   *  This conversion may run into issues if the values contain special characters. Also the handling
   *  of duplicates is undefined.
   *  @return The inverse of this table.
   */
  public Table getInverse () {
    Table t = new Table(specificMode,flags);
    Iterator ti = this.iterator();
    for (int i=0; i<ti.size; i++) t.put( ti.values[i].toString(), ti.keys[i] );
    return t;
  }

  /** Merges a given keyable object (usually a Table) into this one. This copies all the values
      in the given keyable instance into this table overriding any existing entries (in this table)
      that match new entries in the keyable object. Note that this is a shallow merge in that it
      will not recursively enter any sub-tables. This method was added to speed up merging (vs
      using the {@link #putAll(Map)} method) of keyable objects. This speedup is most notable when
      the given a {@link KeyVector} or a Table in KV mode.
      @param keyable The keyable object to merge into this one.
      @since NeXtMidas 2.3.0
   */
  public void merge (Keyable keyable) {
    // JC4: Note that instances of KeyVector (or tables in KV mode) can be copied
    //      faster if they are accessed via index rather than key.
    KeyVector newKV  = null;
    Table     newTbl = null;

    if (keyable instanceof Table) {
      newTbl = (Table)keyable;
      newKV  = newTbl.kv;
    }
    else if (keyable instanceof KeyVector) {
      newKV = (KeyVector)keyable;
    }

    if (newKV != null) {
      if (newKV.size() == 0) {
        // do nothing
      }
      else if ((getSize() == 0) && (getModeInt() == KV)) {
        // If this is in KV mode and it is empty (e.g. called by copy()), just
        // copy the underlying KeyVector. Doing so is **MUCH** faster.
        this.kv = newKV.copy();
      }
      else {
        for (int i = 0; i < newKV.size(); i++) {
          putx(newKV.getKey(i), newKV.get(i));
        }
      }
    }
    else if (newTbl != null) {
      if (newTbl.isEmpty()) {
        // do nothing
      }
      else if (newTbl.fb != null) {
        if ((getModeInt() == FB) && (fb == null)) {
          fb = newTbl.fb.clone();
        }
        else {
          fromBytes(newTbl.fb, 0, newTbl.fb.length);
        }
      }
      else {
        Iterator iter = newTbl.iterator();
        for (int i = 0; i < iter.keys.length; i++) {
          putx(iter.keys[i], iter.values[i]);
        }
      }
    }
    else {
      for (String key : keyable.getKeys()) {
        putx(key, keyable.getKey(key));
      }
    }
  }

  /** Sorts the entries in the Table by key. If the table is not in KV mode, the mode will be
      changed to KV mode before sorting. <b>Since NeXtMidas 2.7.0: Setting the table to be in
      {@link #TM} or {@link #STM} will automatically sort the table by key (alphabetically).
      Using {@link #TM} or {@link #STM} can be significantly faster than using this method.</b>
      @param reverse Set this to true to reverse the sort order.
      @since NeXtMidas 2.3.0
   */
  public synchronized void sortByKey (boolean reverse) {
    if (genericMode != KV) { setMode(KV); }
    kv.sortByKey(reverse);
  }

  /** Makes a copy of the Table. This is a shallow copy in that it will copy the
      reference pointers to any nested Tables rather than copy their values to
      new tables; it does not make copies of the values in the table.
      @return A copy of the table.
      @since NeXtMidas 2.3.0
   */
  public synchronized Table copy () {
    return copyOf(this, specificMode);
  }

  /** Makes a copy of the Table to comply with the {@link Tablizable} interface (see
      {@link Tablizable#toTable()} for details). This is the same as calling {@link #copy()}.
      @return A copy of the table.
      @since NeXtMidas 2.7.1
   */
  @Override
  public final Table toTable () { // marked final since 3.7.0
    return copy();
  }

  /** Creates a table that is a copy of the given Keyable object. This is the same
      as <code>copyOf(keyable,KV)</code>.
      @param keyable The keyable object to copy.
      @return A copy of the Keyable as a table.
      @since NeXtMidas 2.3.0
   */
  public static Table copyOf (Keyable keyable) {
    return copyOf(keyable,KV);
  }

  /** Merges two tables into a new output table. Input tables <code>t1</code> and <code>t2</code> are
      not altered. Entries in <code>t1</code> are replaced by entries in <code>t2</code> if they have the
      same key. If the recurse flag is true, inner tables that exist in both <code>t1</code> and
      <code>t2</code> are merged as well (this only affects instances of {@link Table}). If recurse
      is false, inner tables in <code>t2</code> replace any existing inner tables of the same name in
      <code>t1</code> (this matches the behavior in {@link #merge(Keyable)}. <br>
      <br>
      For example:
      <pre>
        t1 = {ZERO=0,ONE=1,TWO=2,TABLE={A="A",B="B",C="C"}}
        t2 = {ONE="ABC",TWO="DEF",THREE="GHI",TABLE={B=2,C=3,D=4}}

        merge(t1,t2,false) -&gt; {ZERO=0,ONE="ABC",TWO="DEF",THREE="GHI",TABLE={B=2,C=3,D=4}}
        merge(t1,t2,true)  -&gt; {ZERO=0,ONE="ABC",TWO="DEF",THREE="GHI",TABLE={A="A",B=2,C=3,D=4}}
      </pre>
      @param t1 Table for which keys are replaced.
      @param t2 Replace or add these keys to t1.
      @param recurse - Recurse into inner tables if true
      @return A new table
  */
  public static Table merge (Table t1, Table t2, boolean recurse) {
    Table tout = t1.copy();
    if (!recurse) { tout.merge(t2); return tout; }

    for (String key : t2.getKeys()) {
      Object val2 = t2.get(key);
      if (val2 instanceof Table) {
        Object val1 = tout.get(key);
        if (val1 instanceof Table) {
          val2 = merge((Table)val1, (Table)val2, true);
        }
      }
      tout.put(key, val2);
    }
    return tout;
  }

  /** Creates a table that is a copy of the given Keyable object. This is a shallow
      copy in that it will copy the reference pointers to any nested Tables rather
      than copy their values to new tables; it does not make copies of the values in
      the table.
      @param keyable The keyable object to copy.
      @param mode    The table mode (any table mode other than FB).
      @return A copy of the Keyable as a table.
      @since NeXtMidas 2.3.0
   */
  public static Table copyOf (Keyable keyable, int mode) {
    Table tbl = new Table(mode);
    tbl.merge(keyable);
    return tbl;
  }

  /** Wraps a KeyVector as a table. This is ONLY used in a few special cases where it
      is required for performance reasons. The KeyVector must not contain any duplicate keys.
   */
  static Table wrap (KeyVector kv) { // Method is package-private to prevent accidental use.
    Table tbl = new Table(KV);
    tbl.kv = kv;
    return tbl;
  }

  /** Creates a new table from a table file. <br>
   *  <br>
   *  <i>Implementation Note: This is called <code>fromTableFile(..)</code> and not <code>fromTextFile(..)</code>
   *  to avoid confusion with the non-static methods.</i>
   *  @param ref      The reference to use (Ideally a {@link Command} for proper cmdline argument picking).
   *  @param fileName The name of the file to read (FileName, String, otherwise Object's toString() will be used).
   *  @return The new table.
   *  @see #fromTextFile(TextFile)
   *  @since NeXtMidas 3.7.0
   */
  public static Table fromTableFile (MidasReference ref, Object fileName) {
    Table t = new Table();
    t.fromTextFile(new TextFile(ref, fileName));
    return t;
  }

  /** Creates a new table from a config file. This is identical to <code>fromConfigFile(ref,fileName,false,false)</code>.
   *  @param ref      The reference to use (Ideally a {@link Command} for proper cmdline argument picking).
   *  @param fileName The name of the file to read (FileName, String, otherwise Object's toString() will be used).
   *  @return The new table.
   *  @see #fromConfigFile(String[],boolean,boolean)
   *  @since NeXtMidas 3.7.0
   */
  public static Table fromConfigFile (MidasReference ref, Object fileName) {
    return fromConfigFile(ref, fileName, false, false);
  }

  /** Creates a new table from a config file.
   *  @param ref      The reference to use (Ideally a {@link Command} for proper cmdline argument picking).
   *  @param fileName The name of the file to read (FileName, String, otherwise Object's toString() will be used).
   *  @param cs       Ignore normal upper-case-only rules for a Table?
   *  @param merge    Merge "DEFAULT" values into each of the sub-tables?
   *  @return The new table.
   *  @see #fromConfigFile(String[],boolean,boolean)
   *  @since NeXtMidas 3.7.0
   */
  public static Table fromConfigFile (MidasReference ref, Object fileName, boolean cs, boolean merge) {
    TextFile inFile = new TextFile(ref, fileName);
    inFile.open(TextFile.FORCEABORT);
    String[] allLines = inFile.readAllLines();
    inFile.close();
    return fromConfigFile(allLines,cs,merge);
  }

  /** Converts a config file to a table of tables. Given a config file of the form:
      <pre>
        [SECTION1]
        KEY1=VAL1
        KEY2=VAL2
        ...

        [SECTION2]
        KEY1=VAL1
        KEY2=VAL2
        ...
      </pre>
      This will produce a table of tables in the form:
      <pre>
        {SECTION1={KEY1=VAL1,KEY2=VAL2,...},
         SECTION2={KEY1=VAL1,KEY2=VAL2,...},
         ...}
      </pre>
      Note that a section named "DEFAULT" will be handled special. Also note that
      all keys and section names will be converted to upper case.

      @param allLines The lines from the configuration file
      @param cs       Ignore normal upper-case-only rules for a Table and bring
                      all of the keys in case-sensitive (does not apply to section names)
      @param merge    If this is true, "DEFAULT" values will be automatically merged
                      into each of the sub-tables. If this is false, the "DEFAULT"
                      values will simply remain in the separate "DEFAULT" table.
      @return         The table of tables based on the configuration file
      @see #toConfigFile()
      @since NeXtMidas 3.1.3
   */
  public static Table fromConfigFile (String[] allLines, boolean cs, boolean merge) {
    Table tbl     = new Table();
    Table defs    = null;
    Table section = null;

    for (int lineNum = 0; lineNum < allLines.length; lineNum++) {
      String line = allLines[lineNum].trim();

      // Strip comments
      if (line.startsWith("#") || line.startsWith(";")) {
        line = "";
      }
      int i = line.indexOf(" ;");
      if (i > 0) {
        line = line.substring(0,i).trim();
      }

      // Check line contents
      if (line.length()==0) {   // EMPTY LINE
        // do nothing (pass)
      }
      else if (line.startsWith("[") && line.endsWith("]")) {     // SECTION START
        String name = line.substring(1,line.length()-1).trim().toUpperCase();
        section = new Table();

        if (name.length()==0) throw new MidasException("Syntax error near line "+lineNum);
        if (name.equals("DEFAULT")) defs = section;
        tbl.put(name, section);
      }
      else if (section == null) {       // MISSING SECTION START
        throw new MidasException("Syntax error near line "+lineNum);

      }
      else {                          // KEY=VAL DEFINITION
        int eq = line.indexOf('=');
        if (eq < 0) {
          // Missing '=' in KEY=VAL
          throw new MidasException("Syntax error near line "+lineNum);
        }
        String key = line.substring(0,eq).trim();
        String val = line.substring(eq+1).trim();

        if (!cs) key = key.toUpperCase();
        section.put(key,val);
      }
    }

    // Merge in defaults
    if (merge && (defs != null)) {
      for (String key : tbl.getKeys()) {
        if (key.equals("DEFAULT")) continue;
        tbl.put(key, merge(defs,tbl.getTable(key),false));
      }
    }

    return tbl;
  }

  /**
   * Imports an OPAL table file into a new table.
   * @param ref      The reference to use (Ideally a {@link Command} for proper cmdline argument picking).
   * @param filename The name of the OPAL table file to read (FileName, String, otherwise Object's toString() will be used).
   * @param keepCase Ignore normal NeXtMidas rules and treat OPAL table keys as case-sensitive (this will make the table
   *          difficult to use within NeXtMidas).
   * @return The imported OPAL table as a NeXtMidas table.
   * @since NeXtMidas 3.7.0
   */
  public static Table fromOpalFile (MidasReference ref, Object filename, boolean keepCase) {
    return fromOpalFile(TextFile.readAllLines(ref, filename), keepCase);
  }

  /**
   * Imports an OPAL table file into a new table.
   * @param lines    The lines (text) from the input OPAL file.
   * @param keepCase Ignore normal NeXtMidas rules and treat OPAL table keys as case-sensitive (this will make the table
   *          difficult to use within NeXtMidas).
   * @return The imported OPAL table as a NeXtMidas table.
   * @since NeXtMidas 3.7.0
   */
  public static Table fromOpalFile (String[] lines, boolean keepCase) {
    List<String> tokens = new ArrayList<>();

    for (int lineNum = 0; lineNum < lines.length; lineNum++) {
      String line = lines[lineNum].trim();
      Parser p = new Parser(line, true);

      for (int i = 1; i <= p.elements(); i++) {
        String elem = p.get(i);

        if (elem.startsWith("{") && (elem.length() > 1)) {
          // Special case not handled in parser "{ // comment"
          tokens.add("{");
          elem = elem.substring(1).trim();
        }

        if (elem.startsWith("//")) {
          break;
        }

        if (elem.startsWith("=")) {
          tokens.add("=");
          elem = elem.substring(1);
        }

        Parser pp = new Parser(elem);
        pp.setDelimiter('=');
        pp.clean();

        if (pp.elements() == 1) {
          tokens.add(elem);
        }
        else {
          for (int j = 1; j <= pp.elements(); j++) {
            if (j > 1) tokens.add("=");
            tokens.add(pp.get(j));
          }
        }
      }
    }

    int first = 0;
    int last = tokens.size();
    if (tokens.get(first).equals("{") && tokens.get(last - 1).equals("}")) {
      // In general OPAL tables start and end with "{" and "}"
      first++;
      last--;
    }

    nxm.sys.lib.Table tbl = new nxm.sys.lib.Table();
    LinkedList<nxm.sys.lib.Table> stack = new LinkedList<>();
    stack.push(tbl);

    int i = first;
    while (i < last) {
      String token = tokens.get(i++);

      if (token.equals("}")) {
        stack.pop();
        continue;
      }
      String key = stripQuotes(token);
      String equals = tokens.get(i++);
      String val = tokens.get(i++);
      nxm.sys.lib.Table tab = stack.peek();

      if (!keepCase) {
        key = key.toUpperCase();
      }

      if (!equals.equals("=")) {
        throw new MidasException("Invalid OPAL table, missing '=' near " + key);
      }
      else
        if (val.equals("{")) {
          nxm.sys.lib.Table t = new nxm.sys.lib.Table();
          stack.push(t);
          tab.put(key, t);
        }
        else
          if (val.startsWith("@")) {
            String ref = stripQuotes(val.substring(1));
            if (!keepCase) {
              ref = ref.toUpperCase();
            }
            tab.put(key, tab.get(ref));
          }
          else
            if (val.startsWith("\"") && val.endsWith("\"")) {
              tab.put(key, stripQuotes(val));
            }
            else
              if ((val.length() > 2) && (val.charAt(1) == ':')) {
                char type = val.charAt(0);
                tab.put(key, Convert.o2o(val.substring(2), type, null));
              }
              else {
                tab.put(key, val);
              }
    }
    return tbl;
  }

  /** Initialize this Table's Keys and Values from JSON
   *  Current flags are honored and propagated to sub Tables
   *  Any preexisting Table values are cleared first
   *  @param json JSON
   */
  public void fromJSON (CharSequence json) {
    fromJSON(json, null);
  }

  /** Initializes this Table's Key and Values from JSON
   *  Current flags are honored and propagated to sub Tables
   *  Any preexisting Table values are cleared first
   *
   *  @param json JSON
   *  @param subForDots substitute any '.'s in key names with this (if not null)
   */
  public void fromJSON (CharSequence json, CharSequence subForDots) {
    boolean subNotNull = subForDots != null;
    boolean subNotHaveDots = (subNotNull && !subForDots.toString().contains("."));
    String  subDotString = (subNotNull) ? subForDots.toString() : null;
    Object val = JsonUtilities.fromJSON(json);
    this.clear();
    this.putAllAsPureTable((Map)val, subDotString, subNotHaveDots);
  }


  /** Converts a table of tables to a config file. This expects tables to be in
      the form:
      <pre>
        {SECTION1={KEY1=VAL1,KEY2=VAL2,...},
         SECTION2={KEY1=VAL1,KEY2=VAL2,...},
         ...}
      </pre>
      And will produce a configuration file in the form:
      <pre>
        [SECTION1]
        KEY1=VAL1
        KEY2=VAL2
        ...

        [SECTION2]
        KEY1=VAL1
        KEY2=VAL2
        ...
      </pre>
      Note that a section named "DEFAULT" will be handled special and will always
      be written to the top of the file.
      @return The lines of the configuration file
      @see #fromConfigFile(String[], boolean, boolean)
      @since NeXtMidas 3.1.3
   */
  public List<String> toConfigFile () {
    List<String> outFile = new LinkedList<>();

    // Always put DEFAULT first
    if (containsKey("DEFAULT")) {
      addSectionToConfigFile(outFile, "DEFAULT", getTable("DEFAULT"));
    }

    // Add all other sections
    for (String sectionName : getKeys()) {
      if (sectionName.equals("DEFAULT")) {
        // already done
      }
      else {
        addSectionToConfigFile(outFile, sectionName, getTable(sectionName));
      }
    }
    return outFile;
  }

  /** internal use only. Called from toConfigFile(..) */
  private void addSectionToConfigFile (List<String> outFile, String name, Table tbl) {
    if (tbl==null) throw new MidasException("Invalid input table. Expected "+name+" to be a sub-table.");

    outFile.add("["+name+"]");
    for (String key : tbl.getKeys()) {
      outFile.add(key + "=" + tbl.getS(key));
    }
    outFile.add("");
  }

  /** The {@link Enumeration} implementation (of keys) used by the {@link Table} class.
   *  @deprecated Since NeXtMidas 3.7.0: This is an internal class to table and will be marked
   *              private in a future release. Use {@link #keys()} in place of this.
   */
  @Deprecated
  public class Enum implements Enumeration<String> {
    private java.util.Iterator<String> iter;
    private KeyVector kv;
    private String keys[];
    private int item;

    /** Creates a new instance.
        @param table The table to create a new instance for.
        @see Table#keys()
     */
    public Enum (Table table) {
      table.checkModeFB();
      switch (table.genericMode) {
        case KV:  kv   = table.kv;                      break;
        case MAP: iter = table.map.keySet().iterator(); break;
        case KEY: keys = table.kbl.getKeys();           break;
        default:  throw new AssertionError("Unknown table mode "+genericMode);
      }
    }

    @Override
    public boolean hasMoreElements () {
           if (iter != null) return iter.hasNext();
      else if (kv   != null) return (item < kv.size());
      else                   return (item < keys.length);
    }

    @Override
    public String nextElement () {
           if (iter != null) return iter.next();
      else if (kv   != null) return kv.getKey(item++);
      else                   return keys[item++];
    }
  }

  /** Creates an iterator that iterates over the elements in this table.
      The return value of the Iterator's <code>{@link Table.Iterator#next()}</code> method is the key.
      To get the value at that key, use the iterator's <code>{@link Table.Iterator#getValue()}</code> method.
      @return An iterator that iterates over the elements in this table.
      @since NeXtMidas 1.7.0
   */
  @Override
  public synchronized Iterator iterator () { // since 3.1.1: made method synchronized for thread safety (BUG 2500)
    return new Iterator(this);
  }

  /** An iterator used by the {@link Table} class.
      The return value of the {@link #next()} is the key. To get the value at the key, use {@link #getValue()}.<br>
      Since 3.3.1: implements {@link java.util.Iterator} interface.<br>
      Since 3.5.4: implements {@link java.lang.Iterable} interface (so it can be used in Java 5+ for-each loop).<br>
      Since 3.5.4: for multiple use of this iterator, call {@link #reset()}.
      @since NeXtMidas 3.7.0
    */
  @InternalUseOnly("This class is not intended to be extended or instantiated directly") // marked since 3.7.0
  public class Iterator implements java.util.Iterator<String>, Iterable<String> {
    /** table to be referenced            */ public Table    table;
    /** key to be iterated                */ public String[] keys;
    /** values to be iterated             */ public Object[] values;
    /** offset of the current iteration   */ public int      offset;
    /** num keys in the current iteration */ public int      size;
    /** key in the current iteration      */ public String   key;
    /** value in the current iteration    */ public Object   value;

    protected Iterator (Table table, String[] keys, Object[] values) {
      this.table  = table;
      this.keys   = keys;
      this.values = values;
      this.offset = 0;
      this.size   = keys.length;

      if (values == null) {
        this.values = new Object[keys.length];
        for (int i = 0; i < this.size; i++) {
          this.values[i] = table.getKey(keys[i]);
        }
      }
    }

    /** Creates a new instance.
     *  @param table The table
     *  @deprecated Since NeXtMidas 3.7.0: Use {@link Table#iterator()} to create a new instance.
     */
    @Deprecated
    public Iterator (Table table) { // TODO: Mark protected in a future release
      this.table = table;
      table.checkModeFB();
      size = table.size();
      keys = new String[size];
      values = new Object[size];
      if (table.genericMode==MAP) {
        java.util.Iterator<Map.Entry<String, Object>> iter = table.map.entrySet().iterator();
        for (int i=0; i<size; i++) {
          Map.Entry<String,Object> e = iter.next();
          keys[i]=e.getKey();
          values[i]=e.getValue();
        }
      }
      else if (table.genericMode==KV) {
        for (int i=0; i<size; i++) {
          keys[i]=table.kv.getKey(i);
          values[i]=table.kv.get(i);
        }
      }
      else if (table.genericMode==KEY) {
        keys = table.kbl.getKeys();
        for (int i = 0; i < keys.length; i++) {
          values[i] = table.kbl.getKey(keys[i]);
        }
      }
      offset=0;
    }

    /** Determines whether or not there are more entries in the current iteration */
    @Override
    public boolean hasNext() {
      return offset<size;
    }
    /** Determines whether or not there are more entries in the current iteration
        @return whether or not there are remaining entries
     */
    public boolean getNext() {
      if (offset>=size) return false;
      next();
      return true;
    }
    /** Gets the next element in the iteration.
        @return key of next element. Use {@link #getValue()} to get value.
     */
    @Override
    public String next() {
      key = keys[offset];
      value = values[offset];
      offset++;
      return key;
    }
    /** Gets the key in the current iteration
        @return key in current iteration
     */
    public String getKey() {
      return key;
    }
    /** Gets the value in the current iteration
        @return value in current iteration
     */
    public Object getValue() {
      return value;
    }
    @Override
    public void remove() {
      table.remove(key);
    }

    /** @since NeXtMidas 3.5.4 for Iterable interface */
    @Override
    public java.util.Iterator<String> iterator () {
      return this;
    }

    /** Resets the iterator back to the position at which the iterator was created.
     *  @since NeXtMidas 3.5.4
     */
    public void reset () {
      offset = 0;
    }
  }

  /** A listener for the Table.
   *  @see Table#addListener
   */
  public interface Listener {
    /** Do something before the put.
     *  @param key   The key being put.
     *  @param value The value being put.
     *  @return true if done (i.e. if the put action has been completed) and false if incomplete
     *          (i.e. the remainder of the normal put method should be executed).
     */
    public boolean putAction (String key, Object value);
    /** Do something before after the get.
     *  @param key   The key being accessed.
     *  @param value The value that was obtained.
     *  @return <i>The return parallels {@link #putAction} but is currently ignored since this is
     *          called as the last line of the get method.</i>
     */
    public boolean getAction (String key, Object value);
  }

  /** A macro function quick reference.
   *  @see Results#addFunction
   */
  @InternalUseOnly("Used by Macro and Results")
  public interface Function {
    /** Performs a set.
     *  @param value The value to set.
     */
    public void set (Object value);
    /** Performs a get.
     *  @return The value obtained.
     */
    public Object get ();
    /** Executes the function.
     *  @param args The arguments.
     *  @return The return value.
     */
    public Object exec (Object args);
  }

  /** Writes the current contents of this table to the given output stream.
      Only the data of the table is written to the stream.  This method is
      required by the {@link Externalizable} interface and is used when doing
      serialization of the Table.
      @param out The stream to write the instance to.
      @throws IOException If an exception occurs while writing.
      @see #readExternal(ObjectInput)
   */
  @Override
  public void writeExternal (ObjectOutput out) throws IOException {
    out.writeObject( toString() );
    out.writeObject(flags); // added in NeXtMidas 4.1.1
  }

  /** Restore the state of this instance to the values that are contained in
      the input stream.  The input stream is required to have been written by
      the writeExternal method.  The data of the table is reset to the values
      in the stream and the other properties are initialized to their default
      state.
      @param in The stream to read the data from.
      @throws IOException            If an exception occurs while reading.
      @throws ClassCastException     If the input stream contains an object not written by
                                     {@link #writeExternal(ObjectOutput)}.
      @throws ClassNotFoundException If the input stream contains an object not written by
                                     {@link #writeExternal(ObjectOutput)}.
      @see #writeExternal(ObjectOutput)
   */
  @Override
  public void readExternal (ObjectInput in) throws IOException, ClassNotFoundException,
                                                   ClassCastException {
    String data = (String)in.readObject();
    flags = (int)in.readObject();
    fromBytes( data.getBytes() );
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // METHODS ADDED TO NeXtMidas 2.1.0 TO IMPLEMENT THE java.util.Map INTERFACE
  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** This is identical to <code>containsKey(key.toString())</code>.
      @param key The key to look for (should not be null).
      @throws NullPointerException if the specified key is null.
      @see #containsKey(String)
      @since NeXtMidas 2.1.0
   */
  @Override
  public boolean containsKey (Object key) {
    return containsKey(key.toString());
  }

  /** @since NeXtMidas 2.1.0 */
  @Override
  public Set<Map.Entry<String,Object>> entrySet () {
    final String[]    keys    = getKeys();
    final Map.Entry<String,Object>[] entries = new Entry[keys.length];

    for (int i = 0; i < keys.length; i++) {
      entries[i] = new Entry(this, keys[i]);
    }

    return new SimpleSet<Map.Entry<String, Object>>(this) {
      @Override public int size ()                              { return entries.length;                    }
      @Override public Map.Entry<String,Object> getItem (int i) { return entries[i];                        }
      @Override public Object removeItem (int i)                { return table.remove(entries[i].getKey()); }
    };
  }

  /** @since NeXtMidas 2.1.0 */
  @Override
  public Set<String> keySet () {
    final String[] keys = getKeys();

    return new SimpleSet<String>(this) {
      @Override public int size()               { return keys.length;           }
      @Override public String getItem(int i)    { return keys[i];               }
      @Override public Object removeItem(int i) { return table.remove(keys[i]); }
    };
  }

  /** Copies all of the mappings from the specified map to this map (see {@link Map#putAll(Map)}).
      This method is slower than using {@link #merge(Keyable)} when given a {@link KeyVector} or a
      {@link Table} in {@link #KV} mode.
      @param map Mappings to be stored in this map.
      @since NeXtMidas 2.1.0
   */
  @SuppressWarnings("rawtypes") // suppress warning as we cannot make it putAll(Hashtable<? extends String, ? extends Object> table)
                                // since Convert.o2t(..) calls this via new Table(Hashtable<?,?>
                                // and it DOES NOT have generic type (which it is erased at runtime)
  @Override
  public void putAll (Map map) {
    if (map==null) return; // if map is null, don't need to copy to fix NPE in Bug 2118
    java.util.Iterator<?> iter = map.entrySet().iterator();

    while (iter.hasNext()) {
      Map.Entry<?,?> entry = (Map.Entry<?,?>)iter.next();
      putx(entry.getKey().toString(), entry.getValue());
    }
  }

  /** Copies all of the mappings from the specified map to this map (see {@link Map#putAll(Map)}).
      This method is slower than using {@link #merge(Keyable)} when given a {@link KeyVector} or a
      {@link Table} in {@link #KV} mode.
      This stricter method uses put to error out when keys names have "."
      @param map       Mappings to be stored in this Table
      @param subForDot substitute for '.' in keys
      @param replacingDotsInKeys true if keys should have '.'s replaced
      @since NeXtMidas 4.1.2
    */
    @SuppressWarnings("rawtypes") // suppress warning as we cannot make it putAll(Hashtable<? extends String, ? extends Object> table)
                            // since Convert.o2t(..) calls this via new Table(Hashtable<?,?>
                            // and it DOES NOT have generic type (which it is erased at runtime)
  void putAllAsPureTable (Map map, String subForDot, boolean replacingDotsInKeys) {
    if (map==null) return; // if map is null, don't need to copy to fix NPE in Bug 2118
    java.util.Iterator<?> iter = map.entrySet().iterator();

    while (iter.hasNext()) {
      Map.Entry<?,?> entry = (Map.Entry<?,?>)iter.next();
      String key = entry.getKey().toString();
      if (key.contains(".")) {
        if (replacingDotsInKeys) key = key.replaceAll("\\.", subForDot);
        else if ((flags & ALLOW_DOTS_IN_KEYS) ==0) {
          // key with '.' is not allowed
          throw new MidasException("Copying Map entries into Table failed. Invalid Map key:"+key+" contains '.'");
        }
      }
      Object val = entry.getValue();
      if (val instanceof Map) {
        Table tab = new Table(this.specificMode,this.flags);
        tab.putAllAsPureTable((Map)val, subForDot, replacingDotsInKeys);
        putx(key,tab);
      }
      else {
        putx(key, entry.getValue());
      }
    }
  }

  @Override
  public Collection<Object> values () { // since NeXtMidas 2.1.0
    final String[] keys = getKeys(); // keys used to get value

    return new SimpleSet<Object>(this) {
      @Override public int size()               { return keys.length;                   }
      @Override public Object getItem(int i)    { return table.get(keys[i]);    }
      @Override public Object removeItem(int i) { return table.remove(keys[i]); }
    };
  }

  @Override
  public boolean equals (Object obj) { // since NeXtMidas 2.1.0
    if (obj == this)           return true;
    if (!(obj instanceof Map)) return false; // BUG 1358 - prevent NullPointerException & ClassCastException

    /* Copied from the Map Javadocs. */
    Map<?,?> t1 = this;
    Map<?,?> t2 = (Map<?,?>)obj;

    return t1.entrySet().equals(t2.entrySet());
  }

  @Override
  public int hashCode () { // since NeXtMidas 2.1.0
    // This is as defined in the Map Javadocs.
    final java.util.Iterator<Map.Entry<String, Object>> entries = entrySet().iterator();

    int hashcode = 0;
    while (entries.hasNext()) { hashcode += entries.next().hashCode(); }

    return hashcode;
  }

  /** Get a generic entry from the table using the specified key.
      This is the same as using <code>get(key.toString())</code>
      @param key The name of the entry to get. See note on the <a href="#dotsyntax">dot syntax</a> for this parameter.
      @return The value associated with the key (may be <code>null</code>) or <code>null</code> if
              the specified key is not part of the table.
      @see #put(String, Object)
   */
  @Override
  public final Object get (Object key) throws NullPointerException {
    if (key==null) return null;
    else return get (key.toString());
  }

  /** Remove the entry indexed under the specified key from the table. Since NeXtMidas 2.1.0 this
      returns an {@link Object} which is the previous value associated with the key being set.
      @param key The name of the key to remove. See note on the <a href="#dotsyntax">dot syntax</a> for key parameter.
      @return The value removed.
   */
  @Override
  public final Object remove (Object key) throws NullPointerException {
    if (key == null) return null;
    //final Object oldValue = get(key);  // prior to 3.1.0: changed when other remove method started returning an Object (Bug 2455)
    //return oldValue;                   // prior to 3.1.0
    return remove(key.toString());       // since 3.1.0 for BUG 2455
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // NESTED CLASSES USED TO IMPLEMENT THE java.util.Map INTERFACE
  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** A simple implementation of {@link Set} used in the implementation of {@link Map}. */
  private abstract class SimpleSet<E> extends AbstractSet<E> {
    protected final Table table; // The table

    /** Creates a new instance.
        @param table The table.
     */
    private SimpleSet(Table table) {
      this.table   = table;
    }

    /** Gets the i'th element.
        @param i The index.
        @return The item at the given index.
     */
    public abstract E getItem (int i);

    /** Removes the i'th element.
        @param i The index.
        @return the removed item at the i'th element, null if none removed.
     */
    public abstract Object removeItem (int i);

    @Override
    public java.util.Iterator<E> iterator () {
      return new java.util.Iterator<E>() {
        private int i = 0;
        @Override public boolean hasNext () { return (i < size()); }
        @Override public E       next ()    { i++; return getItem(i-1); }
        @Override public void    remove ()  { removeItem(i-1); }
      };
    }
  }

  /** A simple implementation of {@link java.util.Map.Entry}. */
  public static final class Entry implements Map.Entry<String,Object> {
    /** The table backing this Entry */ private final Table  table;
    /** The entry key                */ private final String key;

    /** Creates a new instance.
        @param key The key.
     */
    private Entry (Table table, String key) {
      this.table = table;
      this.key   = key;
    }

    @Override
    public boolean equals (Object obj) {
      if (obj == this) return true;                  // since 3.1.1
      if (!(obj instanceof Map.Entry)) return false; // since 3.1.1

      /* This is copied directly from the Map.Entry Javadocs */
      Map.Entry<?,?> e1 = this;
      Map.Entry<?,?> e2 = (Map.Entry<?,?>)obj;

      return (e1.getKey()==null   ? e2.getKey()==null   : e1.getKey().equals(e2.getKey()))  &&
             (e1.getValue()==null ? e2.getValue()==null : e1.getValue().equals(e2.getValue()));
    }

    @Override
    public int hashCode () {
      /* This is copied directly from the Map.Entry Javadocs */
      return (this.getKey()==null   ? 0 : this.getKey().hashCode()) ^
             (this.getValue()==null ? 0 : this.getValue().hashCode());
    }

    @Override public String getKey ()   { return key; }
    @Override public Object getValue () { return table.get(key); }
    @Override public Object setValue (Object value) { return table.put(key, value); }
  }
}
