package nxm.sys.lib;

import static java.util.Collections.singletonMap;
import static nxm.sys.inc.MapKBT.newMap;
import static nxm.sys.inc.MapKBT.wrapMap;
import static nxm.sys.lib.DataFile.Q_ALT;
import static nxm.sys.lib.DataFile.Q_AZIM;
import static nxm.sys.lib.DataFile.Q_ELEV;
import static nxm.sys.lib.DataFile.Q_GHA;
import static nxm.sys.lib.DataFile.Q_LAT;
import static nxm.sys.lib.DataFile.Q_LON;
import static nxm.sys.lib.DataFile.Q_ROLL;
import static nxm.sys.lib.DataFile.Q_SEC;
import static nxm.sys.lib.DataFile.Q_YEAR;

import java.util.ArrayList;
import java.util.Collection;
import java.util.EmptyStackException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Stack;
import java.util.StringTokenizer;
import java.util.regex.Pattern;

import nxm.sys.inc.Installation;
import nxm.sys.inc.InternalUseOnly;
import nxm.sys.inc.Keyable;
import nxm.sys.inc.ListFile;
import nxm.sys.inc.MapKBT;
import nxm.sys.inc.MidasReference;
import nxm.sys.inc.PlotFile;

/** Defines XML functions useful throughout NeXtMidas and NeXtMidas applications.

    @author  J. Czechowski IV / Neon Ngo
    @since   NeXtMidas 2.1.2
 */
public class XmlFile extends BaseFile {
  private static final Pattern XML_LT  = Pattern.compile("&lt;");  // Matches "&lt;" in the XML
  private static final Pattern XML_GT  = Pattern.compile("&gt;");  // Matches "&gt;" in the XML
  private static final Pattern XML_AMP = Pattern.compile("&amp;"); // Matches "&amp;" in the XML

  /** Key for root XML tag name - Used with "IncludeRootName" conversion option.
      @since NeXtMidas 3.3.3
   */
  public static final String KEY_ROOTNAME = "ROOTNAME"; // key for name of root xml tag

  /** XML to Table options enum (for setting in mask values).
      @since NeXtMidas 3.3.3
   */
  public static enum Xml2TblConversionOptions {
    /** Compress individual text records.  */
    Compression,
    /** Add an addition key/value to the Table containing the name of the root XML tag */
    IncludeRootName;
  };

  /** Default Xml2Tbl conversion options enum mask */
  private static final MaskValue<Xml2TblConversionOptions> DEF_XML2TBL_CONVERSION_OPTIONS = MaskValue.constant(Xml2TblConversionOptions.class);

  // begin constants since 3.5.4 for BUG 2903 (BLUE/File to XML and XML to BLUE/File conversions)
  /** XML content type for Midas XML files */
  public static final String   MIDAS_XML_CONTENT_TYPE = "application/x-midas-xml";
  /** must be in increasing order */
  public static final String[] MIDAS_XML_VERSIONS_SUPPORTED = { "1.0" };
  /** default XML version for Midas XML files */
  public static final String   DEFAULT_XML_VERSION    = "1.0";
  /** default outer tag for Midas XML files */
  public static final String   DEFAULT_OUTER_TAG      = "MidasXML";
  /** write type 1000 and type 2000 data as a block */
  public static final boolean  DEFAULT_BLOCKDATA      = true;
  /** generate XML with newlines and indentation */
  public static final boolean  DEFAULT_NEWLINES       = true;

  private static final String  XML2FILE_CNAME          = "CDATA";
  private static final String  XML2FILE_DUP_TAGS_DELIM = "_";
  private static final String  XML2FILE_DELIMITERS     = " ,;";  // allow space, comma, and semi-colon as delimiters for data values
  private static final String  DEF_DUPKEY_DELIM_TBL2XML = "__";
  private static boolean dotKeyFixEnabled = true; // enable fix for dots in keys

  // end constants since 3.5.4 for BUG 2903

  /** default constructor */
  public XmlFile () {}

  /** Opens a given text file and then reads it in as an XML file converting the XML to a Table.
      This is the same as <code>fastXmlFileToTable(new TextFile(ref,url),DEF_XML2TBL_CONVERSION_OPTIONS)</code>.
      @param ref The reference to use when opening the {@link TextFile} (usually a Midas instance).
      @param url The URL of the file.
      @return The table.
      @deprecated Since NeXtMidas 3.5.4 use {@link #fastXmlFileToTable(MidasReference, String)}.
   */
  @Deprecated
  public static Table fastXmlFileToTable (Object ref, String url) {
    return fastXmlFileToTable(new TextFile(Convert.ref2Midas(ref),url), null, null, 1, true);
  }

  /** Opens a given text file and then reads it in as an XML file converting the XML to a Table.
   *  This is the same as <code>fastXmlFileToTable(new TextFile(ref,url),DEF_XML2TBL_CONVERSION_OPTIONS)</code>.
   *  @param ref The midas reference to use when opening the {@link TextFile} (usually a Midas instance).
   *  @param url The URL of the file.
   *  @return The table.
   *  @since NeXtMidas 3.5.4
   */
  public static Table fastXmlFileToTable (MidasReference ref, String url) {
    return fastXmlFileToTable(new TextFile(ref,url), null, null, 1, true);
  }

  /** Opens a given text file and then reads it in as an XML file converting the XML to a Table.
      This is the same as <code>fastXmlFileToTable(tf,null,null,1,DEF_XML2TBL_CONVERSION_OPTIONS)</code>.
      @param tf The TextFile to read from.
      @return The table.
   */
  public static Table fastXmlFileToTable (TextFile tf) {
    return fastXmlFileToTable(tf,null,null,1,true);
  }

  /** Opens a given text file and then reads it in as an XML file converting the XML to a Table.
      @param tf       The TextFile to read from.
      @param delim    The delimiter between tag name and tag count (null=useDefault).
      @param cname    The name for the character data blocks (null=useDefault).
      @param start    The index of the first tag ({@code start<0} is special case).
      @param comp     Compress individual text records.
      @return The table.
   */
  public static Table fastXmlFileToTable (TextFile tf, String delim, String cname, int start, boolean comp) {
    MaskValue<Xml2TblConversionOptions> convOptions = DEF_XML2TBL_CONVERSION_OPTIONS.copy();

    if (comp) convOptions.set(Xml2TblConversionOptions.Compression);

    return fastXmlFileToTable(tf, delim,cname, start, convOptions);
  }

  /** Quickly converts a block of XML to a Table.
      @param xml      The XML packed in a string.
      @param delim    The delimiter between tag name and tag count (null=useDefault).
      @param cname    The name for the character data blocks (null=useDefault).
      @param start    The index of the first tag ({@code start<0} is special case).
      @param comp     Compress individual text records.
      @return The table.
   */
  public static Table fastXmlToTable (String xml, String delim, String cname, int start, boolean comp) {
    MaskValue<Xml2TblConversionOptions>  convOptions = DEF_XML2TBL_CONVERSION_OPTIONS.copy();

    if (comp) convOptions.set(Xml2TblConversionOptions.Compression);

    return fastXmlToTable(xml, delim,cname, start, convOptions);
  }

  /** Opens a given text file and then reads it in as an XML file converting the XML to a Table.
      This is the same as <code>fastXmlFileToTable(new TextFile(ref,url))</code>.
      @param ref The reference to use when opening the {@link TextFile} (usually a Midas instance).
      @param url The URL of the file.
      @param xml2TblConversionOptions Options for the conversion See {@link Xml2TblConversionOptions} for details
      @return The table.
      @deprecated since NeXtMidas 3.5.0 use {@link #fastXmlFileToTable(MidasReference, String, MaskValue)}
   */
  @Deprecated
  public static Table fastXmlFileToTable (Object ref, String url, MaskValue<Xml2TblConversionOptions> xml2TblConversionOptions) {
    return fastXmlFileToTable(new TextFile(Convert.ref2Midas(ref),url), xml2TblConversionOptions);
  }

  /** Opens a given text file and then reads it in as an XML file converting the XML to a Table.
      This is the same as <code>fastXmlFileToTable(new TextFile(ref,url))</code>.
      @param ref The Midas reference to use when opening the {@link TextFile} (usually a Midas instance).
      @param url The URL of the file.
      @param xml2TblConversionOptions Options for the conversion See {@link Xml2TblConversionOptions} for details
      @return The table.
      @since NeXtMidas 3.5.0
   */
  public static Table fastXmlFileToTable(MidasReference ref, String url, MaskValue<Xml2TblConversionOptions> xml2TblConversionOptions) {
    return fastXmlFileToTable(new TextFile(ref,url),xml2TblConversionOptions);
  }

  /** Opens a given text file and then reads it in as an XML file converting the XML to a Table.
      This is the same as <code>fastXmlFileToTable(tf,null,null,1,true)</code>.
      @param tf The TextFile to read from.
      @param xml2TblConversionOptions Options for the conversion See {@link Xml2TblConversionOptions} for details
      @return The table.
      @since NeXtMidas 3.3.3
   */
  public static Table fastXmlFileToTable(TextFile tf, MaskValue<Xml2TblConversionOptions> xml2TblConversionOptions) {
    return fastXmlFileToTable(tf,null,null,1,xml2TblConversionOptions);
  }

  /** Opens a given text file and then reads it in as an XML file converting the XML to a Table.
      This will open and close input file when done.
      @param tf       The TextFile to read from.
      @param delim    The delimiter between tag name and tag count (null=useDefault).
      @param cname    The name for the character data blocks (null=useDefault).
      @param start    The index of the first tag ({@code start<0} is special case).
      @param xml2TblConversionOptions Options for the conversion See {@link Xml2TblConversionOptions} for details
      @return The table.
      @since NeXtMidas 3.3.3
   */
  public static Table fastXmlFileToTable(TextFile tf, String delim, String cname, int start, MaskValue<Xml2TblConversionOptions> xml2TblConversionOptions) {
    tf.open(TextFile.INPUT|TextFile.FORCEABORT);
    String xml = tf.readAll();
    tf.close();
    return fastXmlToTable(xml, delim,cname, start, xml2TblConversionOptions );
  }

  /** Quickly converts a block of XML to a Table.
      @param xml      The XML packed in a string.
      @param delim    The delimiter between tag name and tag count (null=useDefault).
      @param cname    The name for the character data blocks (null=useDefault).
      @param start    The index of the first tag ({@code start<0} is special case).
      @param xml2TblConversionOptions Options for the conversion See {@link Xml2TblConversionOptions} for details (cannot be null).
      @return The table.
      @since NeXtMidas 3.3.3
   */
  public static Table fastXmlToTable(String xml, String delim, String cname, int start, MaskValue<Xml2TblConversionOptions> xml2TblConversionOptions) {
    // Determine if XML compression is requested
    boolean comp = xml2TblConversionOptions.isSet(Xml2TblConversionOptions.Compression);

    // Convert XML to Table
    XmlToTableHandler handler = new XmlToTableHandler(delim,cname,start,comp);
    parseXmlFast(xml, handler);
    Table tbl = handler.getTable();

    // If including root name, get the root XML tag name and include this
    // in the Table as an additional key value pair.
    if (xml2TblConversionOptions.isSet(Xml2TblConversionOptions.IncludeRootName)) {
      tbl.put(XmlFile.KEY_ROOTNAME, handler.getRootName());
    }

    return tbl;
  }

  /** Parses an XML string. This is a "fast" parser in that it does little or no error checking
      and does no XML validation. If an error occurs it will either go undetected or will cause
      an exception (of some form). This parser is intended for use in situations where the validity
      of the XML is guaranteed and speed is essential.
      @param xml     The raw XML.
      @param handler Handler that will be called to handle any tags/text found.
   */
  public static void parseXmlFast (String xml, SimpleXmlHandler handler) {

    int     cdataStart = 0;
    boolean upcaseTags = false;
    boolean upcaseKeys = false;
    char[]  chars      = xml.toCharArray(); // Using chars[i] is faster than using xml.charAt(i)
    int     tableMode  = Table.KV;

    if (handler instanceof NotSoSimpleXmlHandler) {
      upcaseTags = ((NotSoSimpleXmlHandler)handler).getUpcaseAllTags();
      upcaseKeys = ((NotSoSimpleXmlHandler)handler).getUpcaseAllKeys();
      tableMode  = ((NotSoSimpleXmlHandler)handler).getTableMode();
    }

    try {
      for(int i = 0; i < xml.length(); i++) {
        char thisChar = chars[i];

        if (thisChar == '<') {
          char nextChar = chars[i+1];

          if (cdataStart != i) {
            String cdata = xml.substring(cdataStart,i).trim();

            if (cdata.length() > 0) {
              handler.gotCharData(removeEscapeChars(cdata));
            }
          }

          if (nextChar == '/') {
            i+=2;
            int tagStart = i;
            i++;
            while (chars[i] != '>') { i++; }
            String tag = xml.substring(tagStart,i);
            if (upcaseTags) tag = toUpper(tag);
             handler.gotCloseTag(tag);

          } else if (nextChar == '!') {
            if ((i+6 < xml.length()) && (chars[i+2] == '-') && (chars[i+3] == '-')) {
              // An XML comment "<!-- ... -->" block
              i+=3;
              while ((chars[i]!='-') || (chars[i+1]!='-') || (chars[i+2]!='>')) { i++; }
              i+=2;

            } else if ((i+9 < xml.length()) && xml.substring(i,i+9).equals("<![CDATA[")) {
              // A CDATA section ("<![CDATA[ ... ]]>") that contains no XML or escape characters
              i+=9;
              cdataStart = i;
              while ((chars[i]!=']') || (chars[i+1]!=']') || (chars[i+2]!='>')) { i++; }
              handler.gotCharData(xml.substring(cdataStart,i));
              i+=2;

            } else if ((i+9 < xml.length()) && xml.substring(i,i+9).equals("<!DOCTYPE")) {
              // XML Document Type Definition ("<!DOCTYPE ... >")
              while ((chars[i] != '>')) {
                if (chars[i] == '[') { // Allows embedded DTD declarations inside the brackets.
                  while ((chars[i] != ']')) { i++; }
                }
                i++;
              }
            } else {
              throw new RuntimeException("Error unknown tag beginning with '<!'.");
            }

          } else if (nextChar == '?') {
            // An XML Processing Instruction "<? ... ?>"
            i+=2;
            while ((chars[i]!='?') || (chars[i+1]!='>')) { i++; }
            i++;

          } else {
            i++;
            int tagStart = i;
            while (isTag(chars[i])) { i++; }
            String  tag  = xml.substring(tagStart,i);;
            boolean end  = false;
            boolean done = false;
            Table   args = new Table(tableMode, Table.ADD);
            boolean argsNoDotsInKeys = true;

            while (!end && !done) {
              while (isWhiteSpace(chars[i])) { i++; }
              int keyStart = i;
              if (chars[i] == '/') {
                end = true;
                i++;
              } else if (chars[i] == '>') {
                done = true;
              } else {
                while (chars[i] != '=') { i++; }
                String key = xml.substring(keyStart,i);
                i++;
                char tic = chars[i]; // Either tic (') or quote (")
                i++;
                int valStart = i;
                while (chars[i] != tic) { i++; }
                String val = xml.substring(valStart,i);
                i++;
                if (upcaseKeys) key = toUpper(key);
                // putx is faster than args.put(key,val) and works in this simple case
                // though putx will considered dots part of the key name, args needs +AllowDotsInKeys for future use
                if (dotKeyFixEnabled && argsNoDotsInKeys && key.contains(".")) {
                  argsNoDotsInKeys = false;
                  args.setFlags("+AllowDotsInKeys");
                }
                args.putx(key, val, 0);
              }
            }
            args.setFlags("-Add"); // do not set to 0 anymore, as +AllowDotsInKeys may be set
            if (upcaseTags) tag = toUpper(tag);
            handler.gotOpenTag(tag, args);
            if (end) { handler.gotCloseTag(tag); }
          }
          cdataStart = i+1;
        } else if ((cdataStart == i) && (isWhiteSpace(thisChar))) {
          cdataStart = i+1;
        }
      }
    } catch (EmptyStackException e) {
      throw new MidasException("Malformed input XML file.", e);
    }
  }

  /** Fast toUpperCase(..) method for ASCII only. */
  private static String toUpper (String str) {
    char[]  chars = str.toCharArray();
    boolean mod   = false;

    for (int i = 0; i < chars.length; i++) {
      char c = chars[i];
      if ((c >= 'a') && (c <= 'z')) {
        chars[i] = (char)(c - 'a' + 'A');
        mod = true;
      }
    }
    return (mod)? new String(chars) : str;
  }

  /** Indicates if whitespace. Only used by parseXmlFast(String). */
  private static boolean isWhiteSpace(char c) {
    return (c == ' ') || (c == '\t') || (c == '\n');
  }

  /** Indicates if not part of a tag name. Only used by parseXmlFast(String). */
  private static boolean isTag(char c) {
    return (!isWhiteSpace(c)) && (c != '/') && (c != '>');
  }

  /** Removes escape characters from the XML. */
  private static String removeEscapeChars(String text) {
    // JC4: Pre-compiling the pattern is faster... especially when this is called often.
    //return text.replaceAll("&lt;",  "<")
    //           .replaceAll("&gt;",  ">")
    //           .replaceAll("&amp;", "&");
    text = XML_LT.matcher(text).replaceAll("<");
    text = XML_GT.matcher(text).replaceAll(">");
    text = XML_AMP.matcher(text).replaceAll("&");
    return text;
  }

  /** Converts a NeXtMidas table to XML. This is the same as
      <code>fastTableToXml(tab,null,null,null,false,newlines)</code>.
      @param tab      The table.
      @param newlines Should the XML be generated in a more readable format (newlines and
                      indentation), if this is false no extra whitespace will be added.
      @return The XML string.
      @since NeXtMidas 2.5.0
   */
  public static String fastTableToXml(Keyable tab, boolean newlines) {
    return fastTableToXml(tab,null,null,null,false,newlines);
  }

  /** Converts a NeXtMidas table to XML. This assumes that the table is using a similar mapping to
      XML that {@link #fastXmlToTable(String,String,String,int,boolean)} uses. This will just throw
      an exception is something looks wrong. Note that any entries that are {@link Keyable} will be
      expanded as sub-tables, everything else will be converted to a string.
      @param tab      The table.
      @param outerTag The desired outer XML tag (null = use default)
      @param delim    The delimiter used for duplicate tags (null = use default).
      @param cname    The name to use for CDATA sections (null = use default).
      @param compress Have CDATA sections been compressed (if true, it is not possible to go back,
                      all table entries are assumed to be attributes on the XML tags).
      @param newlines Should the XML be generated in a more readable format (newlines and
                      indentation), if this is false no extra whitespace will be added.
      @return The XML string.
      @since NeXtMidas 2.5.0
   */
  public static String fastTableToXml(Keyable tab, String outerTag, String delim,
                                      String cname, boolean compress, boolean newlines) {
    if (outerTag == null) outerTag = "XML";

    TableToXml tableToXml = new TableToXml(delim, cname, compress, newlines);

    return tableToXml.toXml(outerTag, tab);
  }

  /** Opens a given text file and then writes a table to it as XML. This is the same as
      <code>fastTableToXmlFile(new TextFile(ref,url),tab,newlines)</code>.
      @param ref The reference to use when opening the {@link TextFile} (usually a Midas instance).
      @param url The URL of the file.
      @param tab      The table.
      @param newlines Should the XML be generated in a more readable format (newlines and
                      indentation), if this is false no extra whitespace will be added.
      @since NeXtMidas 2.5.0
   */
  public static void fastTableToXmlFile (Object ref, String url, Keyable tab, boolean newlines) {
    fastTableToXmlFile(new TextFile(Convert.ref2Midas(ref),url),tab,newlines);
  }

  /** Opens a given text file and then writes a table to it as XML. This is the same as
      <code>fastTableToXmlFile(tf,tab,null,null,null,false,newlines)</code>.
      @param tf       The TextFile to write to.
      @param tab      The table.
      @param newlines Should the XML be generated in a more readable format (newlines and
                      indentation), if this is false no extra whitespace will be added.
      @since NeXtMidas 2.5.0
   */
  public static void fastTableToXmlFile(TextFile tf, Keyable tab, boolean newlines) {
    fastTableToXmlFile(tf,tab,null,null,null,false,newlines);
  }

  /** Opens a given text file and then writes a table to it as XML.
      @param tf       The TextFile to write to.
      @param tab      The table.
      @param outerTag The desired outer XML tag (null = use default)
      @param delim    The delimiter used for duplicate tags (null = use default).
      @param cname    The name to use for CDATA sections (null = use default).
      @param compress Have CDATA sections been compressed (if true, it is not possible to go back,
                      all table entries are assumed to be attributes on the XML tags).
      @param newlines Should the XML be generated in a more readable format (newlines and
                      indentation), if this is false no extra whitespace will be added.
      @since NeXtMidas 2.5.0
   */
  public static void fastTableToXmlFile(TextFile tf, Keyable tab, String outerTag, String delim,
                                        String cname, boolean compress, boolean newlines) {
    tf.open(TextFile.OUTPUT|TextFile.FORCEABORT);
    tf.write(fastTableToXml(tab, outerTag, delim, cname, compress, newlines));
    tf.close();
  }

  /** Converts a file to XML. e.g. BLUE to XML, CVS to XML, etc. This assumes that the table is using a similar mapping
   *  to XML that {@link #fastXmlToTable(String,String,String,int,boolean)} uses. This will just throw
   *  an exception is something looks wrong. Note that any entries that are {@link Keyable} will be
   *  expanded as sub-tables, everything else will be converted to a string.
   *  @param file    The file (it MUST implement {@link ListFile} or {@link PlotFile}).
   *  @param options The options controlling the conversion (omitting an option causes the default to be used).
   *                 Note that additional options may be added in the future. The current set of options:
   *                 <pre>
   *                   VERSION   - Version for the converted output (DEFAULT={@value #DEFAULT_XML_VERSION}).
   *                   OUTER_TAG - The desired outer XML tag (DEFAULT={@value #DEFAULT_OUTER_TAG}).
   *                   NEWLINES  - Should the XML be generated in a more readable format with
   *                               newlines and indentation? If this is false no extra
   *                               whitespace will be added (DEFAULT={@value #DEFAULT_NEWLINES}).
   *                   BLOCKDATA - Should the Type 1000 and Type 2000 data be written as a block?
   *                               (DEFAULT={@value #DEFAULT_BLOCKDATA}).
   *                 </pre>
   *  @return The XML string.
   *  @since NeXtMidas 3.5.4
   */
  public static CharSequence fileToXml (BaseFile file, Table options) {
    if (options == null) options = new Table();

    String        version = options.getS("VERSION", DEFAULT_XML_VERSION);
    String        ver     = null;
    StringBuilder xml     = new StringBuilder(65536); // start with 64K buffer

    // ==== DETERMINE THE VERSION NUMBER ===========================================================
    for (String v : MIDAS_XML_VERSIONS_SUPPORTED) {
      if (v.equals(version)) {
        ver = v;
        break;
      }
      if (StringUtil.compareVersions(version, v) < 0) {
        ver = v;
        Shell.warning("XML version '"+version+"' not supported, using version '"+ver+"'");
        break;
      }
    }
    if (ver == null) {
      ver = MIDAS_XML_VERSIONS_SUPPORTED[MIDAS_XML_VERSIONS_SUPPORTED.length-1];
      Shell.warning("XML version '"+version+"' not supported, using version '"+ver+"'");
    }

    // ==== LOAD USER OPTIONS ======================================================================
    String  outerTag  = options.getString("OUTER_TAG", DEFAULT_OUTER_TAG);
    boolean newlines  = options.getState( "NEWLINES",  DEFAULT_NEWLINES);
    boolean blockData = options.getState( "BLOCKDATA", DEFAULT_BLOCKDATA);
    String  newline   = (newlines)? "\n"     : "";
    String  nlOneIndent = (newlines)? "\n  "   : "";  // newline with one intent
    String  nlTwoIndent = (newlines)? "\n    " : "";  // newline with two indent

    // ==== DETERMINE INPUT FILE TYPE ==============================================================
    DataFile df = (file instanceof DataFile)? (DataFile)file : null;
    PlotFile pf = (file instanceof PlotFile)? (PlotFile)file : null;
    ListFile lf = (file instanceof ListFile)? (ListFile)file : null;

    if (lf == null) lf = FileUtil.asListFile(FileUtil.asPlotFileNH(file)); // try alternate conversion
    if (lf == null) throw new MidasException("Could not convert "+file+" to XML since it is not a ListFile or PlotFile");

    // ==== WRITE XML HEADER =======================================================================
    final BaseFile bf = file;
    if (ver.equals("1.0")) {
      String sourceType  = (df != null)? "application/x-midas-blue" : bf.getMimeType();
      String creator     = "NXM"+Installation.NM_VERSION; // matches DataFile CREATOR keyword

      xml.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
      xml.append(newline);
      xml.append('<').append(outerTag).append(' ')
         .append("ContentType='").append(MIDAS_XML_CONTENT_TYPE).append("' ")
         .append("Version='"    ).append(ver        ).append("' ")
         .append("SourceType='" ).append(sourceType ).append("' ")
         .append("Creator='"    ).append(creator    ).append("'>");
    }

    // ==== WRITE XML BODY =========================================================================
    if (df != null) {      // DataFile is more detailed than ListFile or PlotFile
      int typeClass = df.getTypeCodeClass();
      boolean isAscii = (Data.isString(df.getFormatType()));

      appendHeadXML(xml, df, version, nlOneIndent, nlTwoIndent);
      appendKeysXML(xml, df, version, nlOneIndent, nlTwoIndent);

      if (blockData && !isAscii && ((typeClass == 1) || (typeClass == 2))) {
        appendBlockDataXML(xml, df, version, nlOneIndent, nlTwoIndent);
      }
      else {
        appendRowDataXML(  xml, df, version, nlOneIndent, nlTwoIndent);
      }
    }
    else if (pf != null) { // PlotFile is more detailed than ListFile
      int typeClass = (pf instanceof ListFile)? -1 : pf.getType() / 1000;
      String format = pf.getFormat();
      boolean isAscii = (format.length() == 2) && (Data.isString((byte)format.charAt(1)));

      appendHeadXML(xml, pf, version, nlOneIndent, nlTwoIndent);
      appendKeysXML(xml, bf, version, nlOneIndent, nlTwoIndent);

      if (blockData && !isAscii && ((typeClass == 1) || (typeClass == 2))) {
        appendBlockDataXML(xml, pf, version, nlOneIndent, nlTwoIndent);
      }
      else {
        appendRowDataXML(  xml, lf, version, nlOneIndent, nlTwoIndent);
      }
    }
    else {                 // ListFile
      appendHeadXML(xml, lf, version, nlOneIndent, nlTwoIndent);
      appendKeysXML(xml, bf, version, nlOneIndent, nlTwoIndent);
      appendRowDataXML(xml, lf, version, nlOneIndent, nlTwoIndent);
    }

    // ==== WRITE XML FOOTER =======================================================================
    xml.append(newline);
    xml.append("</").append(outerTag).append('>').append(newline);
    return xml;
  }

  /** Appends header XML for a DataFile. Since 3.5.4. */
  private static void appendHeadXML (StringBuilder xml, DataFile df, String version, String nlOneIndent, String nlTwoIdent) {
    int    typeClass = df.getTypeCodeClass();
    Time   time      = df.getTime();
    String timeStr   = time.toString(Time.ISO8601, 12);

    xml.append(nlOneIndent).append("<FileHeader>");
    xml.append(nlTwoIdent).append("<Time>").append(timeStr     ).append("</Time>");
    xml.append(nlTwoIdent).append("<Size>").append(df.getSize()).append("</Size>");
    appendHeadXML(xml, (PlotFile)df, "-"+version, nlOneIndent, nlTwoIdent);

    if (typeClass == 5) {
      // Type 5000 reference frame quadwords
      String rf = df.getReferenceFrame();

      xml.append(nlTwoIdent).append("<ReferenceFrame Name='").append(rf).append("' ");
      if (rf.startsWith("TOP")) {
        xml.append("Altitude='" ).append(df.getQuadword(Q_ALT )).append("' ");
        xml.append("Latitude='" ).append(df.getQuadword(Q_LAT )).append("' ");
        xml.append("Longitude='").append(df.getQuadword(Q_LON )).append("' ");
      }
      if (rf.equals("TOP")) {
        xml.append("Azimuth='"  ).append(df.getQuadword(Q_AZIM)).append("' ");
        xml.append("Elevation='").append(df.getQuadword(Q_ELEV)).append("' ");
        xml.append("Roll='"     ).append(df.getQuadword(Q_ROLL)).append("' ");
      }
      if (rf.equals("ECI") || (df.getQuadword(Q_YEAR) != 0)) {
        xml.append("EpochYear='"   ).append(df.getQuadword(Q_YEAR)).append("' ");
        xml.append("EpochSeconds='").append(df.getQuadword(Q_SEC )).append("' ");
        xml.append("EpochGHA='"    ).append(df.getQuadword(Q_GHA )).append("' ");
      }
      xml.append("/>");
    }

    if ((typeClass == 3) || (typeClass == 5) || (typeClass == 6)) {
      // Record definitions
      xml.append(nlTwoIdent).append("<RecLength>").append(df.getRecLength()).append("</RecLength>");
      appendHeadXML(xml, (ListFile)df, "-"+version, nlOneIndent, nlTwoIdent);
    }
    xml.append(nlOneIndent).append("</FileHeader>");
  }

  /** Appends header XML for a PlotFile (use -version to skip outer tags, size and time). Since 3.5.4. */
  private static void appendHeadXML (StringBuilder xml, PlotFile pf, String version, String nlOneIndent, String nlTwoIndent) {
    String xUnits = DataFile.getUnitsVarName(pf.getXUnits());
    String yUnits = DataFile.getUnitsVarName(pf.getYUnits());

    if (!version.startsWith("-")) {
      Time   time    = new Time(pf.getTimeAt(0.0));
      String timeStr = time.toString(Time.ISO8601, 12);
      xml.append(nlOneIndent).append("<FileHeader>");
      xml.append(nlTwoIndent).append("<Time>"   ).append(timeStr     ).append("</Time>"  );
      xml.append(nlTwoIndent).append("<Size>"   ).append(pf.getSize()).append("</Size>"  );
    }

    xml.append(nlTwoIndent).append("<Type>"   ).append(pf.getType()      ).append("</Type>"  );
    xml.append(nlTwoIndent).append("<Format>" ).append(pf.getFormat()    ).append("</Format>");
    xml.append(nlTwoIndent).append("<XUnits>" ).append(xUnits            ).append("</XUnits>");
    xml.append(nlTwoIndent).append("<XStart>" ).append(pf.getXStart()    ).append("</XStart>");
    xml.append(nlTwoIndent).append("<XDelta>" ).append(pf.getXDelta()    ).append("</XDelta>");
    xml.append(nlTwoIndent).append("<SubSize>").append(pf.getXFrame()    ).append("</SubSize>"); // numSubRecords for 3000/6000, numComponents for 5000
    xml.append(nlTwoIndent).append("<YUnits>" ).append(yUnits            ).append("</YUnits>");
    xml.append(nlTwoIndent).append("<YStart>" ).append(pf.getYStart()    ).append("</YStart>");
    xml.append(nlTwoIndent).append("<YDelta>" ).append(pf.getYDelta()    ).append("</YDelta>");
//  xml.append(nlTwoIndent).append("<YFrame>" ).append(pf.getYFrame()    ).append("</YFrame>"); // not needed and probably more confusing to include

    if (!version.startsWith("-")) {
      xml.append(nlOneIndent).append("</FileHeader>");
    }
  }

  /** Appends header XML for a ListFile (use -version to skip outer tags, size and time). Since 3.5.4. */
  private static void appendHeadXML (StringBuilder xml, ListFile lf, String version, String nlOneIndent, String nlTwoIndent) {
    if (!version.startsWith("-")) {
      xml.append(nlOneIndent).append("<FileHeader>");
      xml.append(nlTwoIndent).append("<Size>").append(lf.getNumberOfRows()).append("</Size>");
    }

    final int recordDefintionCount = lf.getRecordDefCount();
    if (recordDefintionCount > 0) {
      xml.append(nlTwoIndent).append("<SubRecords>");
    }
    for (int i = 0; i < recordDefintionCount; i++) {
      Table def = lf.getRecordDef(i);
      xml.append(nlTwoIndent).append("  <SubRecord ");
      for (String key : def.getKeys()) {
        xml.append(key).append("='").append(def.get(key)).append("' ");
      }
      xml.append("/>");
    }
    if (recordDefintionCount > 0) {
      xml.append(nlTwoIndent).append("</SubRecords>");
    }

    if (!version.startsWith("-")) {
      xml.append(nlOneIndent).append("</FileHeader>");
    }
  }

  /** Appends header XML for a DataFile. Since 3.5.4. */
  private static void appendKeysXML (StringBuilder xml, DataFile df, String version, String nlOneIndent, String nlTwoIndent) {
    Keywords keywords = df.getKeywordsObject();
    xml.append(nlOneIndent).append("<FileKeywords>");
    for (Map.Entry<String,Object> kwd : keywords.getAll("MAIN", false)) {
      appendKeyword(xml, "MAIN", 'S', kwd.getKey(), kwd.getValue(), nlOneIndent, nlTwoIndent);
    }
    for (Map.Entry<String,Object> kwd : keywords.getAll("EXT", false)) {
      appendKeyword(xml, "EXT", '?', kwd.getKey(), kwd.getValue(), nlOneIndent, nlTwoIndent);
    }
    xml.append(nlOneIndent).append("</FileKeywords>");
  }

  /** Appends header XML for a BaseFile. Since 3.5.4. */
  private static void appendKeysXML (StringBuilder xml, BaseFile bf, String version, String nlOneIndent, String nlTwoIndent) {
    xml.append(nlOneIndent).append("<FileKeywords>");
    appendKeyword(xml, null, 'S', "COMMENT", bf.getComment(), nlOneIndent, nlTwoIndent);
    xml.append(nlOneIndent).append("</FileKeywords>");
  }

  /** Only used by appendKeysXML(..). Since 3.5.4. */
  private static void appendKeyword (StringBuilder xml, String scope, char type, String key, Object val, String nlOneIndent, String nlTwoIndent) {
    if (val == null) val = "";
    if (type == '?') {
      type = (val instanceof Data)? ((Data)val).getFormatTypeChar() : 'S';
    }
    String str = StringUtil.toHTML(val.toString()); // escape special XML characters

    // Note that we use double-quotes since those are the ones the StringUtil.toHTML(..) function handles
    xml.append(nlTwoIndent);
    xml.append("<KEY ");
    if (scope != null) xml.append("SCOPE=\"").append(scope).append("\" ");
    xml.append("TYPE=\"" ).append(type).append("\" ");
    xml.append("NAME=\"" ).append(key ).append("\" ");
    xml.append("VALUE=\"").append(str ).append("\"/>");
  }

  /** Appends XML data as elements for a ListFile. Since 3.5.4. */
  private static void appendRowDataXML (StringBuilder xml, ListFile lf, String version, String nlOneIndent, String nlTwoIndent) {
    xml.append(nlOneIndent).append("<FileData Type='ROW'>");
    for (int i = 0; i < lf.getNumberOfRows(); i++) {
      Table row = lf.getDataTable(i);
      xml.append(nlTwoIndent);
      xml.append("<!-- ").append(i).append(" --> ");
      xml.append("<ROW");
      for (String key : row.getKeys()) {
        String val = StringUtil.toHTML(row.getS(key));
        xml.append(" ").append(key).append("=\"").append(val).append('\"');
      }
      xml.append("/>");
    }
    xml.append(nlOneIndent).append("</FileData>");
  }

  /** Appends XML data as a block for a PlotFile. Since 3.5.4. */
  private static void appendBlockDataXML (StringBuilder xml, PlotFile pf, String version, String nlOneIndent, String nlTwoIndent) {
    int xf = pf.getXFrame();
    int tl = (xf > 1)? 1 : 32;

    Data data = pf.getDataBuffer(tl);
    xml.append(nlOneIndent).append("<FileData Type='BLOCK'>");

    int totRead = 0;
    int numRead;
    final String scalarSeparator  = " ";
    final String elementSeparator = ", ";
    final String frameSeparator   = ";";
    while ((numRead = pf.read(data, data.size)) >= 0) {
      xml.append(nlTwoIndent).append("<!-- ").append(totRead).append(" --> ");
      data.toString(xml, 0, numRead, 10, false, scalarSeparator, elementSeparator, true);
      if (xf > 1) xml.append(frameSeparator);
      totRead += numRead;
    }
    xml.append(nlOneIndent).append("</FileData>");
  }

  /**
   * Converts XML file to MIDAS BLUE file. This method will open and close both the input and output files.
   * @param inTextFile   input XML file
   * @param outDataFile  output BLUE file
   * @param options      The options controlling the conversion (omitting an option causes the default to be used).
   *                     Note that additional options may be added in the future.
   * @see #xmlToFile(String, DataFile, Table)
   * @since NeXtMidas 3.5.4
   */
  public static void xmlToFile (TextFile inTextFile, DataFile outDataFile, Table options) {
    inTextFile.open(TextFile.INPUT|TextFile.FORCEABORT);
    String xml = inTextFile.readAll();
    inTextFile.close();

    xmlToFile(xml, outDataFile, options);
  }

  /**
   * Converts XML file to MIDAS BLUE file.
   * This method will open and close the output file.
   * @param xml          input XML.
   * @param outDataFile  output MIDAS BLUE file.
   * @param options      The options controlling the conversion (omitting an option causes the default to be used).
   *                     Note that additional options may be added in the future. The current set of options:
   *                     <pre>
   *                     WARN        - Show warnings (DEFAULT=true).
   *                     </pre>
   * @since NeXtMidas 3.5.4
   */
  public static void xmlToFile (String xml, DataFile outDataFile, Table options) {
    if (options == null) options = new Table();
    // === LOAD DEFAULT OPTIONS ======================================================================
    options.addIfNotPresent("WARN", true);

    // ==== LOAD USER OPTIONS ======================================================================
    boolean showWarnings    = options.getState("WARN");

    MaskValue<Xml2TblConversionOptions> xml2TblConversionOptions = MaskValue.with(
        Xml2TblConversionOptions.IncludeRootName, Xml2TblConversionOptions.Compression);

    Table tab = fastXmlToTable(xml, XML2FILE_DUP_TAGS_DELIM, XML2FILE_CNAME, 1, xml2TblConversionOptions);

    checkContentTypeVersionSupported(tab, showWarnings);

    createFileHeader(outDataFile, tab, options);
    writeFileData(outDataFile, tab, options);
    writeFileKeywords(outDataFile, tab, options);
    outDataFile.close();
  }

  /** @since NeXtMidas 3.5.4 */
  private static boolean checkContentTypeVersionSupported (Table tab, boolean warn) {
    if (tab == null) return false;

    // ==== 1. check ContentType attribute ========================================================
    String contentType = tab.getS("CONTENTTYPE");
    boolean contentTypeCheck = MIDAS_XML_CONTENT_TYPE.equals(contentType);
    if (warn && !contentTypeCheck) {
      Shell.warning("checkContentTypeVersionSupported: Unsupported MidasXML: ContentType attribute [" + contentType + "]");
    }

    // ==== 2. check Version attribute ============================================================
    boolean versionCheck = false;
    String version = tab.getS("VERSION");
    for (String supportedVersion : MIDAS_XML_VERSIONS_SUPPORTED) {
      if (supportedVersion.equals(version)) {
        versionCheck = true;
        break;
      }
    }
    if (warn && !versionCheck) {
      Shell.warning("checkContentTypeVersionSupported: Unsupported MidasXML: Version attribute [" + version + "]");
    }

    // ==== 3. check root XML tag name ============================================================
    String rootTagName = tab.getS(KEY_ROOTNAME);
    boolean rootTagNameCheck = DEFAULT_OUTER_TAG.equalsIgnoreCase(rootTagName);
    if (warn && !rootTagNameCheck) {
      Shell.warning("checkContentTypeVersionSupported: Unsupported MidasXML: root tag name [" + rootTagName + "]");
    }

    return contentTypeCheck && versionCheck && rootTagNameCheck;
  }

  /**
   * Create/open BLUE file and setup MAIN header. flexible
   * @param df       output MIDAS BLUE file.
   * @param tab      input table with FileHeader to create
   * @param options  The options controlling the conversion (omitting an option causes the default to be used).
   * @since NeXtMidas 3.5.4
   */
  private static void createFileHeader (DataFile df, Table tab, Table options) {
    Table header = tab.getTable("FILEHEADER", null);
    if (header != null) {
      String tmpstr = header.getS("TIME", null);
      if (tmpstr != null) {
        df.setTime(new Time(tmpstr), false); // <-- more flexible than Time(tmpstr,Time.ISO8601) since it supports both ISO8601 and STD formats
      }
      // TODO: should we allow missing tags - yes,
      df.setSize(header.getD("SIZE"));
      df.setType(header.getL("TYPE", 1000)); // TODO: or default to 0?
      df.setFormat(header.getS("FORMAT"));
      df.setXUnits(header.getS("XUNITS"));
      df.setXStart(header.getD("XSTART"));
      df.setXDelta(header.getD("XDELTA"));
      df.setSubSize(header.getL("SUBSIZE"));
      df.setYUnits(header.getS("YUNITS"));
      df.setYStart(header.getD("YSTART"));
      df.setYDelta(header.getD("YDELTA"));

      int typeClass = df.getTypeCodeClass();
      if (typeClass == 5) {
        tab = header.getTable("REFERENCEFRAME");
        if (tab != null) {
          String rf = tab.getS("NAME");
          df.setReferenceFrame(rf);
          if (rf.startsWith("TOP")) {
            df.setQuadword(Q_ALT, tab.getD("ALTITUDE"));
            df.setQuadword(Q_LAT, tab.getD("LATITUDE"));
            df.setQuadword(Q_LON, tab.getD("LONGITUDE"));
          }
          if (rf.equals("TOP")) {
            df.setQuadword(Q_AZIM, tab.getD("AZIMUTH"));
            df.setQuadword(Q_ELEV, tab.getD("ELEVATION"));
            df.setQuadword(Q_ROLL, tab.getD("ROLL"));
          }
          if (rf.equals("ECI") || tab.getD("EPOCHYEAR") != 0) {
            df.setQuadword(Q_YEAR, tab.getD("EPOCHYEAR"));
            df.setQuadword(Q_SEC, tab.getD("EPOCHSECONDS"));
            df.setQuadword(Q_GHA, tab.getD("EPOCHGHA"));
          }
        }
      } // end if type 5000

      if ((typeClass == 3) || (typeClass == 5) || (typeClass == 6)) {
        df.setRecLength(header.getL("RECLENGTH"));
        Table recDefsTable = header.getTable("SUBRECORDS");
        if (recDefsTable != null) {
          df.setRecordDefs(recDefsTable);
        }
        // -or- loop through <SubRecord ...> tags // TODO remove below
//        int ii = 0;
//        String recName = "NAME";
//        String recFormat = "2A";
//        short  recOffset = 0;
//        byte   recType   = 0;
//        byte   recUnits  = 0;
//        df.setRecName  (ii, recName);
//        df.setRecFormat(ii, recFormat);
//        df.setRecOffset(ii, recOffset);
//        df.setRecType  (ii, recType); /* or */ df.setCompType (ii, recType);
//        df.setRecUnits (ii, recUnits);
      }
    }
    else if (options.getState("WARN")) {
      Shell.warning("createFileHeader: MidasXML missing <FileHeader> tag.");
    }
    df.open(BaseFile.OUTPUT);
  }

  /** Write file data to BLUE file.
   * @param df       output MIDAS BLUE file.
   * @param tab      input table with FileData to write out to file
   * @param options  The options controlling the conversion (omitting an option causes the default to be used).
   * @since NeXtMidas 3.5.4
   * @see #appendBlockDataXML(StringBuilder, PlotFile, String, String, String)
   * @see #appendRowDataXML(StringBuilder, ListFile, String, String, String)
   */
  private static void writeFileData (DataFile df, Table tab, Table options) {
    Table filedata = tab.getTable("FILEDATA", null);
    final boolean warn = options.getState("WARN");
    if (filedata != null) {
      String type = filedata.getS("TYPE", null);
      filedata.remove("TYPE"); // temporary remove so below loops don't have to skip the Type attribute
      if ("ROW".equals(type)) {        // ROW data
        for (Object value : filedata.values()) {
          if (value instanceof Table) {
            df.writeDataTable((Table) value);
          }
        }
      }
      else if ("BLOCK".equals(type)) { // BLOCK data
        int subsize = df.getSubSize();
        int xferlen = (subsize > 1)? 1 : 32; // matches logic in appendBlockDataXML(..)
        Data data = df.getDataBuffer(xferlen);

        for (Object value : filedata.values()) {
          if (value instanceof String) {
            String datastr = (String) value;
            data.fromString(datastr, XML2FILE_DELIMITERS);
            df.write(data);
          }
          else if (warn) {
            Shell.warning("writeFileData: MidasXML <FileData Type='BLOCK'> with unsupported data value of class: " + value.getClass());
          }
        }
      }
      else if (warn) {                 // UNKNOWN data format
        Shell.warning("writeFileData: MidasXML's <FileData> tag has unsupported Type attribute: " + type);
      }
      if (type != null) filedata.put("TYPE", type); // restore Type attribute
    }
    else if (warn) {
      Shell.warning("writeFileData: MidasXML missing <FileData> tag.");
    }
  }

  /** Write keywords to BLUE file's extender header.
   * @param df       output MIDAS BLUE file.
   * @param tab      input table with FileKeywords to write out to file
   * @param options  The options controlling the conversion (omitting an option causes the default to be used).
   * @since NeXtMidas 3.5.4
   */
  private static void writeFileKeywords (DataFile df, Table tab, Table options) {
    Table filekeywords = tab.getTable("FILEKEYWORDS", null);
    if (filekeywords != null) {
      Keywords keywords = df.getKeywordsObject();
      for (Object keyword : filekeywords.values()) {
        if (keyword instanceof Table) {
          Table kwtbl = (Table) keyword;
          String scope = kwtbl.getS("SCOPE");
          String type  = kwtbl.getS("TYPE");
          String name  = kwtbl.getS("NAME");
          Object value = kwtbl.get("VALUE");
          if (value instanceof String) {
            value = StringUtil.fromHTML((String) value); // un-escape special chars in XML
          }
          if ("MAIN".equals(scope)) { // write keyword to main header
            keywords.putMain(name, value.toString());
          }
          else {                      // else add (allowing dups) to extended header
            name = type + ":" + name; // so that value will be saved with proper data type
            keywords.add(name, value);
          }
        }
        // else ignore attributes in <FileKeywords> tag
      } // end for loop
    }
    else if (options.getState("WARN")) {
      Shell.warning("writeFileKeywords: MidasXML missing <FileKeywords> tag.");
    }
  }


  /** de-serialize from an XML format
   *  @param bufferString            XML string
   *  @param output                  Table with deserialized XML
   *  @param duplicateKeyDelim       duplicate key delimiter
   *  @throws NoSuchElementException thrown if unable to serialize element
   *  @since NeXtMidas 3.9.2 moved here from prim.convert
   */
  public static void fromXML(String bufferString, Table output, String duplicateKeyDelim)
         throws NoSuchElementException {
    Table  tagCounts = new Table();
    String tableName;
    String newName;
    StringTokenizer lineST;
    StringTokenizer mainST = new StringTokenizer(bufferString,"<>",false);

    while (mainST.hasMoreTokens()) {
      String line = mainST.nextToken().trim();

      // Single line entry with "name" and multiple key=value pairs
      // Example:  <GEO GMODE="4" GTHR="1" />
      if (line.endsWith("/")) {
        line = line.substring(0,line.length()-1);
        lineST = new StringTokenizer(line);
        tableName = lineST.nextToken();
        newName = getUniqueKey(tagCounts, tableName, duplicateKeyDelim);
        output.put(newName,makeXMLTableFromLine(lineST));
      } else if (line.length() <= 0){
        // Blank string, no action required
      } else {
        // Handle multi-line entries
        lineST = new StringTokenizer(line);
        if (lineST.countTokens() > 1) {
          tableName = lineST.nextToken();
          newName = getUniqueKey(tagCounts, tableName, duplicateKeyDelim);
          output.put(newName,makeXMLTableFromLine(lineST));
        } else {
          tableName = lineST.nextToken();
          newName = output.getUniqueKey(tableName);
          if (!tableName.equals("/XML")) output.addTable(newName);
        }

        String endtable="/"+tableName;
        boolean foundEnd = false;
        StringBuilder innerTable = new StringBuilder();
        while (mainST.hasMoreTokens() && !foundEnd) {
          line = mainST.nextToken();
          if (line.equals(endtable))
            foundEnd = true;
          else
            innerTable.append(line).append("<");
        }
        if (innerTable.length()>0 && foundEnd) {
          fromXML(innerTable.toString(),output.getTable(newName));
        }
      }
    }
  }

  /** Makes an XML table from the given String
   *  @param st string with tokens to convert to XML table
   *  @return XML Table
   *  @since NeXtMidas 3.9.2 moved here from prim.convert
   */
  public static Table makeXMLTableFromLine (StringTokenizer st) {
    String ss = "{";
    if (st.hasMoreTokens()) {
      String nt = st.nextToken();
      //jph -- This makes sure keys like A='3' are handled properly
      nt = nt.replace(nxm.sys.inc.AsciiMap.FTIC,nxm.sys.inc.AsciiMap.QUOTE);
      ss = ss.concat(nt);
    }
    while (st.hasMoreTokens()) {
      String nt = st.nextToken();
      if (nt.indexOf('=') > 0) {
        ss = ss.concat(",");
      }
      else {
        ss = ss.concat(" ");
      }
      //jph -- This makes sure keys like A='3' are handled properly
      nt = nt.replace(nxm.sys.inc.AsciiMap.FTIC,nxm.sys.inc.AsciiMap.QUOTE);
      ss = ss.concat(nt);
    }
    ss=ss.concat("}");

    Table newTable = new Table(ss);
    return newTable;
  }

  /** de-serialize from an XML format
   *  @param bufferString string
   *  @param output       Table with deserialized XML
   *  @throws NoSuchElementException thrown if unable to serialize element
   *  @since NeXtMidas 3.9.1 moved here from prim.convert
   */
  public static void fromXML(String bufferString, Table output) throws NoSuchElementException {
    fromXML(bufferString, output, DEF_DUPKEY_DELIM_TBL2XML);
  }

  /** Gets a unique key. This is just included as a patch to NeXtMidas 2.1.1 and 2.0.2, the real
      fix will be included in release 2.1.2.
   */
  private static String getUniqueKey(Table tagCounts, String key, String delim) {
    String str   = key;
    int    count = tagCounts.getL(key);

    if (count > 0) {
      str = key + delim + count;
    }
    tagCounts.put(key, count+1);

    return str;
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // Nested Classes
  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** Internal class used to convert a table to XML. */
  private static final class TableToXml {
    private final String INDENT = "  ";

    private StringBuffer xml;
    private String       delim;
    private String       cname;
    private boolean      compress;
    private String       newline;

    public TableToXml(String delim, String cname, boolean compress, boolean newlines) {
      this.xml = new StringBuffer();
      this.delim = (delim != null) ? delim: "_";
      this.cname = (cname != null) ? cname: "CDATA";
      this.compress = compress;
      this.newline  = (newlines)? "\n" : "";
    }

    /** Converts a table to XML. */
    public String toXml(String outerTag, Keyable tab) {
      return toXml(outerTag, tab, "").toString();
    }

    /** Recursive method... only called internally. */
    private StringBuffer toXml(String topLevel, Keyable tab, String indent) {
      StringBuffer xml       = new StringBuffer(); // The body xml.
      StringBuffer attr      = new StringBuffer(); // The attributes in the open tag.
      String[]     keys      = tab.getKeys();      // Table entries
      String       indt1     = "";                 // First-level indent
      String       indt2     = "";                 // Second-level indent
      boolean      hasNested = false;              // Are there nested keys?

      if (newline.length() > 0) {
        indt1 = newline + indent;
        indt2 = indt1 + INDENT;
      }
      attr.append(indt1).append("<").append(topLevel); // Add in "<topLevel" to front of attrs.
      int lenDelim = delim.length();
      for (int i = 0; i < keys.length; i++) {
        String key   = keys[i];
        Object value = tab.getKey(key); // Do this BEFORE stripping off the delim
        int idelim = key.lastIndexOf(delim);
        if (idelim > 0) { // Strip off the one-up counter necessary for a table, but not xml
          String count = key.substring(idelim+lenDelim);
          if (StringUtil.isInteger(count)) {
            key = key.substring(0,idelim);
          } else {
            throw new MidasException("XmlFile: Error converting table key '"+keys[i]+"' to XML, "
                                   + "when using delim='"+delim+"', invalid count="+count);
          }
        }

        if (value == null) {
          // do nothing
        } else if (key.equals(cname) && !compress) {
          value = StringUtil.toHTML(value.toString()); // Change '<' to '&lt;', '>' to '&gt;', etc.
          xml.append(indt2).append(value);

        } else {
          if (value instanceof Keyable) {
            xml.append(toXml(key, (Keyable)value, indent+INDENT));
            hasNested = true;
          } else {
            // Append the tag value: <tag>='<value>' to the top tag.
            attr.append(" ").append(key).append("=\"").append(value).append("\"");
          }

        }
      }

      if (xml.length() == 0) {
        attr.append("/>");
      } else if (!hasNested && (xml.length() < 40)) { // Tighten up when xml is just a small CDATA
        attr.append(">");
        attr.append(xml.substring(indt2.length()));
        attr.append("</").append(topLevel).append(">");
      } else {
        attr.append(">");
        attr.append(xml);
        attr.append(indt1).append("</").append(topLevel).append(">");
      }
      return attr;
    }
  } // end inner class TableToXml

  /** Defines the interface the XML parser will use to process the XML tags. */
  public static interface SimpleXmlHandler {
    /** This method is called when the XML parser finds an open tag.
        @param tag  The name of the tag.
        @param args The arguments associated with the tag (if any).
     */
    public void gotOpenTag(String tag, Table args);

    /** This method is called when the XML parser finds a close tag.
        @param tag  The name of the tag.
     */
    public void gotCloseTag(String tag);

    /** This method is called when the parser finds character data.
        @param cdata The character data.
     */
    public void gotCharData(String cdata);
  } // end inner interface SimpleXmlHandler

  /** Defines the interface the XML parser will use to process the XML tags.
      @since NeXtMidas 2.5.5
   */
  public static abstract class NotSoSimpleXmlHandler implements SimpleXmlHandler {

    /** default constructor */
    public NotSoSimpleXmlHandler () {}

    /** Indicates if all keys should be converted to uppercase before calling
        {@link #gotOpenTag} should be passed a table where all keys have been
        converted to upper case.
        @return true if all keys should be upper-cased, otherwise false
        @since NeXtMidas 2.5.5
     */
    public boolean getUpcaseAllKeys() {
      return false;
    }

    /** Indicates if the tags should be converted to uppercase before calling
        {@link #gotOpenTag} or {@link #gotCloseTag}.
        @return true if all tags should be upper-cased, otherwise false
        @since NeXtMidas 2.5.5
     */
    public boolean getUpcaseAllTags() {
      return false;
    }


    /** Indicates the table mode to use with the table passed to {@link #gotOpenTag}.
        @return table mode to use
        @since NeXtMidas 2.5.5
     */
    public int getTableMode() {
      return Table.KV;
    }
  } // end inner class NotSoSimpleXmlHandler

  /** An implementation of SimpleXmlHandler that fills in a {@link Table}. */
  public static class XmlToTableHandler extends NotSoSimpleXmlHandler {
    private final Stack<Table> tables = new Stack<>(); // The stack of tables.
    private final Stack<Table> counts = new Stack<>(); // The stack of tag counts.
    private final Stack<String> names = new Stack<>(); // The stack of names.
    private final String  delim;                  // Delimiter
    private final String  cname;                  // Name of CDATA field.
    private final int     start;                  // Start index (usually 1 or 0; -1 and -2 are special)
    private final boolean compress;               // Compress single entries?
    private final int     mode;                   // Table mode to use.
    private       String  cdata    = null;        // Any character data.
    private       Table   root     = null;        // Root table
    private       String  rootName = null;        // Root Name (since 3.3.3 for BUG 2752)
    private       boolean noKeysWithDots = true;  // None of the keys so far contain dots

    /** Creates a new instance using XmlToTableHandler(null,null,1,false,Table.KV). */
    public XmlToTableHandler() {
      this(null,null,1,false,Table.KV);
    }

    /** Creates a new instance using XmlToTableHandler(delim,cname,start,false,Table.KV).
        @param delim   The delimiter between tag name and tag count (null=useDefault).
        @param cname   The name for the character data blocks (null=useDefault).
        @param start   The index of the first tag.
     */
    public XmlToTableHandler(String delim, String cname, int start) {
      this(delim,cname,start,false,Table.KV);
    }

    /** Creates a new instance using XmlToTableHandler(delim,cname,start,compress,Table.KV).
        @param delim    The delimiter between tag name and tag count (null=useDefault).
        @param cname    The name for the character data blocks (null=useDefault).
        @param start    The index of the first tag.
        @param compress Compress individual text records?
     */
    public XmlToTableHandler(String delim, String cname, int start, boolean compress) {
      this(delim,cname,start,compress,Table.KV);
    }

    /** Creates a new instance. The <code>start</code> value indicates how duplicate elements are
        handled since all Table entries must be unique.
        <pre>{@code
          start = -2    - 0-based counter (always use counter)
          start = -1    - First has no counter, others start with 1 (legacy
                          functionality that matches old XML2TBL conversions)
          start =  0    - 0-based counter (use only if duplicates are found)
          start =  1    - 1-based counter (use only if duplicates are found)
          start =  N    - N-based counter (where N > 0)
        }</pre>
        Compression will take any single record with just character data and compress it. For
        example:
        <pre>{@code
          <XML>
            <greeting>Hello</greeting>
          </XML>

          Without Compression:  {GREETING={CDATA="Hello"}}
          With Compression:     {GREETING="Hello"}
        }</pre>
        @param delim    The delimiter between tag name and tag count (null=useDefault).
        @param cname    The name for the character data blocks (null=useDefault).
        @param start    The index of the first tag.
        @param compress Compress individual text records?
        @param mode     The default mode to use for tables ({@link Table#KV} or {@link Table#HT}).
        @since NeXtMidas 2.5.5
     */
    public XmlToTableHandler(String delim, String cname, int start, boolean compress, int mode) {
      this.delim    = (delim == null)? "_": delim;
      this.cname    = (cname == null)? "CDATA": cname.toUpperCase();
      this.start    = start;
      this.compress = compress;
      this.mode     = mode;
    }

    public boolean getUpcaseAllKeys () { return true; }
    public boolean getUpcaseAllTags () { return true; }
    public int     getTableMode ()     { return mode; }

    /** Clears and resets the handler so it can be used again. JC4: I added this method
        since I was parsing hundreds of XML messages a second and wanted to just "clear"
        the previous run and reuse the same handler (to eliminate the extra object creation time).
        @since NeXtMidas 2.3.0
     */
    public void clear() {
      tables.clear();
      counts.clear();
      names.clear();
      cdata = null;
      root  = null;
      rootName = null; // since 3.3.3 for BUG 2752
    }

    public void gotOpenTag(String tag, Table args) {
      if (!tables.empty()) tag = addUnique(tag, args); // Add if not first in file.

      tables.push(args);
      counts.push(new Table(Table.HT));
      names.push(tag);
    }

    public void gotCloseTag(String tag) {
      Table  top  = tables.pop();
      Table  cnt  = counts.pop();
      String name = names.pop();

      if (compress && (top.size() == 1)) { // If only one entry, compress it from Table to record.
        String key = top.getKeys()[0];
        Object val = top.get(key);

        if (key.equals(cname)) {
          if (!tables.empty()) {
            Table prev = tables.peek();
            prev.remove(name);
            prev.put(name, val);
          } else {
            top.rename(top.getKeys()[0], name);
          }
        }
      }

      // The final item on stack represents the root table
      if (tables.empty()) {
        root = top;
        rootName = name;  // since 3.3.3 for BUG 2752
      }
    }

    public void gotCharData(String cdata) {
      addUnique(cname, cdata);
    }

    /** This should only be called after XML parsing is done.
     *  @return the root table.
     **/
    public Table getTable() {
      return root;
    }

    /** Returns the root name. This should only be called after XML parsing is done.
        @return The name of the root element in the XML file.
        @since NeXtMidas 3.3.3
     */
    public String getRootName() {
      return rootName;
    }

    /** Adds a unique entry to the table on the top of the stack.
        @param key The key.
        @param val The value.
        @return The unique key used.
     */
    private String addUnique(String key, Object val) {
      Table  table = new Table();
      table = tables.peek();
      Table  count = counts.peek();

      int    index = count.getL(key, start);
      String ukey  = key + delim + index;

      if (index == -1) {               // Used by XML2TBL in CONVERT (no counter on first one)
        ukey = key;
        index = 0;
      } else if (index == -2) {        // Always add 0-based counter
        ukey  = key + delim + 0;
        index = 0;
      } else if (index == start) {     // Don't add one-up-counter until there are duplicates
        ukey = key;
      } else if (index == (start+1)) { // Found first duplicate, rename old one.
        table.rename(key, (key + delim + start));
      }

      if (dotKeyFixEnabled && noKeysWithDots && key.contains(".")) { // NXM-3772
        noKeysWithDots = false;
        table.setFlags("+AllowDotsInKeys");
        count.setFlags("+AllowDotsInKeys");
      }

      if (dotKeyFixEnabled) {
        count.putx(key,  index+1);
        table.putx(ukey, val);
      }
      else {
        count.put(key,  index+1);
        table.put(ukey, val);
      }

      return ukey;
    }
  } // end inner class XmlToTableHandler

  /** Converts XML to objects similar to what is done for JSON in {@code JsonUtilities}.
   *  @param xml The XML text.
   *  @return The result.
   *  @since NeXtMidas 3.9.0
   */
  public static MapKBT fromXML (String xml) {
    return fromXML(xml, false, null);
  }

  /** Converts XML to objects similar to what is done for JSON in {@code JsonUtilities}.
   *  @param xml          The XML text.
   *  @param upcase       Convert tag names to upper case?
   *  @param cdataTagName The CDATA tag name.
   *  @return The result.
   *  @since NeXtMidas 3.9.0
   */
  public static MapKBT fromXML (String xml, boolean upcase, String cdataTagName) {
    FastXmlHandler handler = new FastXmlHandler(upcase, cdataTagName);
    parseXmlFast(xml, handler);
    return handler.getValue();
  }

  /** Set whether the Dots in Keys fix is enabled or not
   *  @param fixEnabled true if enabled
   */
  @InternalUseOnly
  public static void setDotKeyFix (boolean fixEnabled) {
    XmlFile.dotKeyFixEnabled = fixEnabled;
  }

  /** Get whether the Dots in Keys fix is enabled or not
   *  @return true if fix enabled
   */
  @InternalUseOnly
  public static boolean getDotKeyFixEnabled () {
    return XmlFile.dotKeyFixEnabled;
  }

  /** Handler used with {@code fromXML(..)}. */
  private static class FastXmlHandler extends NotSoSimpleXmlHandler {
    private final LinkedList<XmlToken> tokens;
    private final boolean              upcase;
    private final String               cdataTagName;
    private final String               valueTagName;

    /** Creates a new instance.
     *  @param cdataTagName The CDATA tag name. If null, this will use the name 'value' unless
     *                      another 'value' is present, in which case it will use 'CDATA'.
     */
    FastXmlHandler (boolean upcase, String cdataTagName) {
      this.tokens       = new LinkedList<>();
      this.upcase       = upcase;
      this.cdataTagName = cdataTagName;
      this.valueTagName = (upcase)? "VALUE" : "value";
    }

    @Override
    public boolean getUpcaseAllKeys() {
      return upcase;
    }

    @Override
    public boolean getUpcaseAllTags() {
      return upcase;
    }

    @Override
    public void gotOpenTag (String tag, Table args) {
      tokens.push(new XmlToken(TokenType.OPEN, tag, args));
    }

    @Override
    public void gotCloseTag (String tag) {
      tokens.push(new XmlToken(TokenType.CLOSE, tag, null));
    }

    @Override
    public void gotCharData (String value) {
      tokens.push(new XmlToken(TokenType.CDATA, value, null));
    }

    /** Gets the resulting value. Note that this can get called only once, and must be called
     *  after parseXmlFast(..).
     *  @return The value.
     */
    public MapKBT getValue () {
      tokens.push(new XmlToken(TokenType.DONE, null, null));
      Object val = getValue(null, null);
      return (val instanceof MapKBT)? (MapKBT)val : wrapMap(singletonMap(cdataTagName, val));
    }

    /** Gets the next value.
     *  @param tagName The tag name.
     *  @param args    The args.
     *  @return The value.
     */
    @SuppressWarnings({"unchecked", "fallthrough"})
    private Object getValue (String tagName, Table args) {
      if (tokens.isEmpty()) return null;

      MapKBT   values = newMap();
      String   cdata  = null;
      XmlToken token;

      if (args != null) {
        values.putAll(args);
      }
      // This originally did tokens.pop which will do it backwards because it acts as a stack (LIFO). It needs to go backwards like a queue. (FIFO)
      while ((token = tokens.removeLast()) != null) {
        switch (token.type) {
          case OPEN:
            Object val = getValue(token.value, token.args);
            Object old = values.get(token.value);

            if (old == null) {
              values.put(token.value, val);
            }
            else if (old instanceof Collection) {
              ((Collection<Object>)old).add(val);
            }
            else {
              List<Object> all = new ArrayList<>(8);
              all.add(old);
              all.add(val);
              values.put(token.value, all);
            }
            break;

          case CLOSE:
            if (tagName!=null && !tagName.equals(token.value)) {
              throw new RuntimeException("Mismatched <"+tagName+"> and </"+token.value+">");
            }
            // FALLTHROUGH

          case DONE:
            if (cdata == null) {
              return values;
            }
            else if (values.isEmpty()) {
              return cdata;
            }
            else if (cdataTagName != null) {
              values.put(cdataTagName, cdata);
              return values;
            }
            else if (!values.containsKey(valueTagName)) {
              values.put(valueTagName, cdata);
              return values;
            }
            else {
              values.put("CDATA", cdata);
              return values;
            }

          case CDATA:
            cdata = (cdata == null)? token.value : cdata+" "+token.value;
            break;
        }
      }
      throw new RuntimeException("Missing </"+tagName+">");
    }
  }

  /** Token type used with {@link XmlToken}. */
  private static enum TokenType { OPEN, CLOSE, CDATA, DONE };

  /** A token. */
  private static class XmlToken {
    final TokenType type;
    final String    value;
    final Table     args;

    XmlToken (TokenType type, String value, Table args) {
      this.type  = type;
      this.value = value;
      this.args  = args;
    }

    public String toString() {
      return "XmlToken type:"+type+" value:"+value+" args:"+args;
    }
  }

}
