package nxm.sys.lib;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Vector;

import nxm.sys.inc.ListFile;
import nxm.sys.inc.MidasReference;
import nxm.sys.inc.Units;

/** A class to read and write a CSV (comma-separated-value) file. <br>
    <br>
    By default CSV files have no header information. However, it is common to include some amount
    of header information in the file. Where this is done the <code>HEADERROWS=</code> qualifier can
    be used to indicate this:
    <pre>
      HEADERROWS=0 <i>(Default)</i>
        There is no header information for the data. Unfortunately this
        is the default in the CSV specs, but this is almost universally
        ignored. To provide a header, use {@link #setColumnNames(String)}.

      HEADERROWS=1 <i>(Not default, but extremely common)</i>
        This is the most common case. Here the first row is the column
        names with the data following on the remaining rows. To override
        these names, use {@link #setColumnNames(String)}.

      HEADERROWS=N (N &gt; 1)
        This is a common case for some applications. In this case rows
        0..N-2 are header information (free form, user specific), row N-1
        is the column names with the data immediately following.
    </pre>
    As a rule, CSV files are used due to their simplicity and standardization, not because they
    are efficient for read/write access (which they are not). The main cause of this inefficiency
    is the fact that rows of data are not aligned to any particular byte-boundaries. As such, the
    only way to know where in the file a given row begins is to read all of the data in the file
    prior to that row (counting the newlines as you go). The only way to determine the number of
    rows in the file is to read the entire file (counting all of the newlines in it). Intermediate
    updates to the file is even less efficient.<br>
    <br>
    This class assumes that users are more interested in the speed of accessing the data rather
    than the speed of {@link #open()} and {@link #close()}. To this end, this class will read in
    the entire file on {@link #open()} and parse it into easily-accessible Java data structures.
    Any subsequent reads from the file will just return the data already in memory, and any writes
    will merely update the Java data structures. On {@link #close()} all of the updated data will
    be written out at once (note that the {@link #FLUSH} flag can be used to cause incremental
    writes, but it is very inefficient due to the file format). As such, read/write access to an
    open CSV file is generally faster than {@link DataFile} (which is dependent on disk i/o).
    However this comes at the cost of additional RAM used and a significantly slower
    {@link #open()} and {@link #close()}.
    @author  J. Czechowski IV / J. Hallam
    @since   NeXtMidas 2.1.1
*/
public class CsvFile extends BaseFile implements ListFile, CsvFileModSwapInterface {
  private static final ColumnDef[] NO_COLUMNS = new ColumnDef[0];

  private final Vector<Table>  rows    = new Vector<>();  // Holds all the rows, each row is a Table.
  private final Vector<String> head    = new Vector<>(); // Holds any extra header information.
  private       boolean     upToDate   = false;        // Does the file match the data stored in rows?
  private       int         headerRows = -1;           // Number of header rows (0=no header).
  private       int         pad        = 16;           // Number of spaces to allocate per row.
  private       double      seek       = 0.0;          // Current seek position (0..size-1)
  private       double      xStart     = 0.0;          // X-axis start
  private       double      xDelta     = 1.0;          // X-axis delta
  private       int         xUnits     = Units.NONE;   // X-axis units
  private       double      yStart     = 0.0;          // Y-axis start
  private       double      yDelta     = 1.0;          // Y-axis delta
  private       int         yUnits     = Units.NONE;   // Y-axis units
  private       String      defRowName = "ROW";        // The default row name.
  private       ColumnDef[] cols       = NO_COLUMNS;   // The column definitions.

  /** Initialize new instance */
  public CsvFile() { }

  /** Initialize new instance with filename
   *  IMPORTANT: Please use a CsvFile constructor with a MidasReference
   *  as the default for null currently does not have MAINTAINCASE, but
   *  that is anticipated to change.
      @param filename the file name.
   */
  public CsvFile (String filename) {
    init (null,filename);
  }

  /** Initialize new instance with FileName
      @param fn the file name.
   */
  public CsvFile (FileName fn) {
    init (null,fn);
  }

  /** Initialize new instance with reference and filename
      @deprecated since NeXtMidas 3.5.0 use {@link #CsvFile(MidasReference, Object)}
      @param filename the file name to use.
      @param ref the reference to use.
   */
  @Deprecated
  public CsvFile (Object ref, Object filename) {
    if (ref instanceof MidasReference) {
      init ((MidasReference) ref,filename);
    }
    else {
      init (Convert.ref2Midas(ref), filename);
    }
  }

  /** Initialize new instance with reference and filename
      @since NeXtMidas 3.5.0
      @param ref the reference to use.
      @param filename the file name to use.
   */
  public CsvFile (MidasReference ref, Object filename) {
    init (ref,filename);
  }

  /** Trims a string and removes quotes. */
  private static String removeQuotes(String str) {
    if (str != null) {
      str = str.trim();

      if (str.startsWith("\"") && str.endsWith("\"")) {
        str = str.substring(1, str.length()-1);  // Remove quotes
      }
    }
    return str;
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // Methods overridden from BaseFile
  //////////////////////////////////////////////////////////////////////////////////////////////////
  @Override
  public int seek(double val) { seek = val; return 1;}
  @Override
  public double seek()    { return seek; }
  @Override
  public double avail()   { return getSize() - seek(); }
  @Override
  public double getSize() { return rows.size(); }

  @Override
  public String getMimeType () {
    return "text/csv";
  }

  @Override
  public boolean open() {
    if (super.open()) {
    // 2011-10-21 NTN: the below two lines are done in the super class (BaseFile)'s open(), no need to do it again.
    // Peel off all of the qualifiers and invoke any set methods in this class
    //      Table qualsTab = fn.getQualifierTable();
    //      KeyObject.setKeys(this, qualsTab, M, KeyObject.SOFT);

      if (isInput) {
        readAll();   //replaces readFile() to support multi-line fields - since 3.7.1 for NXM-2931
      }
      setInternals();
    }
    return isOpen;
  }

  @Override
  public void close() {
    super.close();

    // Clear out any cached data
    cols = NO_COLUMNS;
    rows.clear();
    head.clear();
  }

  @Override
  public void update () {
    flush();
  }

  @Override
  public void flush() {
    if (isOutput && !upToDate && isOpen) {
      PrintStream out = new PrintStream(io.getOutputStream());

      for (int i = 0; i < getHeaderRows(); i++) {
        out.println(getHeaderData(i));
      }

      for (Table row : rows) {
        out.println(colValuesToString(cols,pad,row));
      }
    }
  }

  /** Converts the column values to a string. */
  private static Table colValuesFromString (Midas midas, int lineNum, ColumnDef[] cols, Parser parser) {
    Table row = new Table(Table.KV, Table.ADD);

    for (int i = 0; i < cols.length; i++) {
      String name = cols[i].name;

      if (i < parser.elements()) {
        row.put(name, removeQuotes(parser.get(i+1)));
      }
      else if (midas != null) {
        midas.warning("CsvFile: Missing value on line "+lineNum+": " + name);
      }
    }
    row.setFlags(0);
    return row;
  }

  /** Converts the column values to a string. */
  private static Table colValuesFromString (Midas midas, int lineNum, ColumnDef[] cols, ArrayList<String> str) {
    Table row = new Table(Table.KV, Table.ADD);

    for (int i = 0; i < cols.length; i++) {
      String name = cols[i].name;
      if (i < str.size()) {
        row.put(name, str.get(i));
      }
      else if (midas != null) {
        midas.warning("CsvFile: Missing value on line "+lineNum+": " + name);
      }
    }
    row.setFlags(0);
    return row;
  }

  /** Converts the header row to a string. */
  private static String colHeadersToString (ColumnDef[] cols, int pad) {
    StringBuilder str = new StringBuilder();
    for (int i = 0; i < cols.length; i++) {
      String name = cols[i].name;
      String fmt  = cols[i].format;

      if (fmt != null) { name = fmt + ":" + name; }

      name = "\"" + name + "\"";
      if (i != (cols.length-1)) { name += ","; }

      str.append(StringUtil.padRight(name, pad));
    }
    return str.toString();
  }

  /** Converts the column values to a string. */
  private static String colValuesToString (ColumnDef[] cols, int pad, Table row) {
    StringBuilder str = new StringBuilder();

    for (int j = 0; j < cols.length; j++) {
      String name = cols[j].name;
      String val  = row.getS(name);

      if (!StringUtil.isNumber(val)) { val = "\"" + val + "\""; }
      if (j != (cols.length-1)) { val += ","; }

      str.append(StringUtil.padRight(val, pad));
    }
    return str.toString();
  }

  /** Converts a table to data to a CSV string. This is the same as <code>toCsv(defs,data,0)</code>.
      @param defs The column definitions. This uses the same syntax as {@link #getRecordDefs()}.
      @param data The data.
      @return The CSV string with embedded "\n" characters marking line breaks.
      @since NeXtMidas 2.5.0
   */
  public static String tableToCsv (Table defs, Table data) {
    return tableToCsv(defs,data,0);
  }

  /** Converts a table to data to a CSV string. This is the same as
      <code>toCsv(defs,data,pad,"\n")</code>.
      @param defs The column definitions. This uses the same syntax as {@link #getRecordDefs()}.
      @param data The data.
      @param pad  The number of spaces to allocate for each column so they line up nicely.
      @return The CSV string with embedded "\n" characters marking line breaks.
      @since NeXtMidas 2.5.0
   */
  public static String tableToCsv (Table defs, Table data, int pad) {
    return tableToCsv(defs,data,pad,"\n");
  }

  /** Converts a table to data to a CSV string.
      @param defs The column definitions. This uses the same syntax as {@link #getRecordDefs()}.
      @param data The data. This should be a table of tables in the form:
                  <pre>
                    {ROW_1={&lt;col1&gt;=&lt;data1&gt;,&lt;col2&gt;=&lt;data2&gt;,...},
                     ROW_2={&lt;col1&gt;=&lt;data1&gt;,&lt;col2&gt;=&lt;data2&gt;,...},
                     ...}
                  </pre>
      @param pad  The number of spaces to allocate for each column so they line up nicely. Use
                  <code>pad=0</code> to disable.
      @param cr   The CR/LF characters to use (usually "\n").
      @return The CSV string with embedded line breaks.
      @since NeXtMidas 2.5.0
      @see #tablesToCsv(Table,Table[],int,String)
   */
  public static String tableToCsv (Table defs, Table data, int pad, String cr) {
    String[] rowNumbers = data.getKeys();
    Table[]  rows       = new Table[rowNumbers.length];

    for (int i = 0; i < rows.length; i++) {
      rows[i] = data.getTable(rowNumbers[i]);
    }
    return tablesToCsv(defs,rows,pad,cr);
  }

  /** Converts a table to data to a CSV string. This is the same as <code>tablesToCsv(defs,rows,0)</code>.
      @param defs The column definitions. This uses the same syntax as {@link #getRecordDefs()}.
      @param rows The data.
      @return The CSV string with embedded "\n" characters marking line breaks.
      @since NeXtMidas 2.5.0
   */
  public static String tablesToCsv (Table defs, Table[] rows) {
    return tablesToCsv(defs,rows,0);
  }

  /** Converts a table to data to a CSV string. This is the same as
      <code>tablesToCsv(defs,rows,pad,"\n")</code>.
      @param defs The column definitions. This uses the same syntax as {@link #getRecordDefs()}.
      @param rows The data.
      @param pad  The number of spaces to allocate for each column so they line up nicely.
      @return The CSV string with embedded "\n" characters marking line breaks.
      @since NeXtMidas 2.5.0
   */
  public static String tablesToCsv (Table defs, Table[] rows, int pad) {
    return tablesToCsv(defs,rows,pad,"\n");
  }

  /** Converts a table to data to a CSV string.
      @param defs The column definitions. This uses the same syntax as {@link #getRecordDefs()}.
      @param rows The data. This should be a table of tables in the form:
                  <pre>
                    data[0]={&lt;col1&gt;=&lt;data1&gt;,&lt;col2&gt;=&lt;data2&gt;,...}
                    data[1]={&lt;col1&gt;=&lt;data1&gt;,&lt;col2&gt;=&lt;data2&gt;,...},
                     ...
                  </pre>
      @param pad  The number of spaces to allocate for each column so they line up nicely. Use
                  <code>pad=0</code> to disable.
      @param cr   The CR/LF characters to use (usually "\n").
      @return The CSV string with embedded line breaks.
      @since NeXtMidas 2.5.0
      @see #tablesToCsv(Table,Table[],int,String)
   */
  public static String tablesToCsv (Table defs, Table[] rows, int pad, String cr) {
    ColumnDef[]  cols = toColsList(defs);
    StringBuilder str = new StringBuilder();

    str.append(colHeadersToString(cols,pad)).append(cr);

    for (int i = 0; i < rows.length; i++) {
      str.append(colValuesToString(cols,pad,rows[i])).append(cr);
    }
    return str.toString();
  }

  /** Converts a CSV string to a table of data. Note that if <code>defs==null</code> and
      <code>headerRows!=0</code>, the definitions will be read from the header row.
      @param defs        The column definitions. This uses the same syntax as {@link #getRecordDefs()}.
      @param str         The CSV data.
      @param headerRows  The number of header rows in the data.
      @return The Tables formed from parsing the CSV string.
      @since NeXtMidas 2.5.0
   */
  public static Table[] csvToTables (Table defs, String str, int headerRows) {
    String[]    lines    = (str == null)? new String[0] : str.split("[\n\r]+");
    int         rowCount = Math.max(0, lines.length - headerRows);
    Table[]     allRows  = new Table[rowCount];

    if ((defs == null) && (headerRows != 0) && (lines.length > headerRows)) {
      Parser parser = new Parser(lines[headerRows]);
      defs = new Table();
      for (int i = 1; i <= parser.elements(); i++) {
        ColumnDef def = getColumnDef(parser.get(i));
        defs.put(def.name, def.toTable());
      }
    }

    ColumnDef[] cols = toColsList(defs);
    for (int i = headerRows,j=0; i < lines.length; i++,j++) {
      Parser parser = new Parser(lines[i], ',');
      parser.setUseSpaceDelimiter(false);

      allRows[j] = colValuesFromString(null,i,cols,parser);
    }

    return allRows;
  }

  /** Converts a definition table to the appropriate list. */
  private static ColumnDef[] toColsList (Table defs) {
    String[] colNames = defs.getKeys();
    ArrayList<ColumnDef> cols = new ArrayList<>(colNames.length);

    for (String colName : colNames) {
      cols.add(new ColumnDef(defs.getTable(colName)));
    }
    return cols.toArray(NO_COLUMNS);
  }


  /** Give the CSV file name */
  public String toString () { return "CsvFile : " + getURL(); }

  /** List the text URL and size */
  public String listHeader () {
    StringBuilder str = new StringBuilder(256);

    str.append("CsvFile     : ").append(getURL()).append("\n");
    str.append("Rows        : ").append(rows.size()).append("\n");
    str.append("Columns     : ").append(cols.length).append("\n");

    str.append("Header      : ");
    int hr = getHeaderRows();
         if (hr == 0) { str.append("None\n"); }
    else if (hr == 1) { str.append("First Row\n"); }
    else              { str.append("First ").append(hr).append(" Rows\n"); }
    str.append("User Header : ");
    for (int i = 0; i < hr-1; i++) {
      if (i > 0) str.append("\n              ");
      str.append(getHeaderData(i));
    }
    str.append("\n");

    for (int i = 0; i < cols.length; i++) {
      String name   = cols[i].name;
      String format = cols[i].format;

      if (format == null) format = "??";

      str.append("Column Name=").append(StringUtil.padRight(name, 12));
      str.append("    Format=").append(format).append("\n");
    }
    return str.toString();
  }

  public String listElements (double start, int elem, String format, int flags) {
    String str = null;
    if (start < rows.size()) {
      str = getDataTable(start).toString() + "\n";
    }
    return str;
  }

  public int listElementsPerLine (int lineWidth, String format, int flags) {
    return 1;
  }


  //////////////////////////////////////////////////////////////////////////////////////////////////
  // Methods for ListFile
  //////////////////////////////////////////////////////////////////////////////////////////////////
  public double  getNumberOfRows ()   { return getSize();   }
  public boolean getProtected ()      { return true;        } // JC4: This should probably be false
  public int     getRecordDefCount () { return cols.length; }

  public Table getDataTable (double offset) {
    if (offset >= rows.size()) return null; // end-of-file
    Table row = rows.get((int)offset);
    return row.copy();
  }

  public int setData (double offset, Table table) {
    putData((int)offset, table, false);
    return 1;
  }

  public void setData (double element, String fieldName, Object obj) {
    Table tbl = rows.get((int)offset);
    tbl.put(fieldName, obj);
    setData(offset, tbl);
  }

  public void insertData (double offset, Object obj) {
    putData((int)offset, Convert.o2t(obj), true);
  }

  public final void removeData (double offset) {
    removeData(offset, 1L);
  }

  public void removeData (double offset, long count) {
    if (Util.removeListElements(rows, (int)offset, (int)count)) {
      upToDate = false;
    }
  }

  public void setRecordDefs (Table defs) {
    String[] keys = defs.getKeys();
    ArrayList<ColumnDef> list = new ArrayList<>();

    if (headerRows < 0) {
      setHeaderRows(1);
    }

    for (String key : keys) {
      Table  def = defs.getTable(key);
      String str = def.getS("NAME",key);
      String fmt = def.getS("FORMAT",null);

      list.add(getColumnDef(str,fmt));
    }
    cols = list.toArray(NO_COLUMNS);
  }

  public Table getRecordDefs () {
    Table tbl = new Table();
    for (int i = 0; i < getRecordDefCount(); i++) {
      Table def = getRecordDef(i);
      tbl.put(def.getS("NAME"), def);
    }
    return tbl;
  }

  public Table getRecordDef (int i) {
    return cols[i].toTable();
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // Methods for PlotFile
  //////////////////////////////////////////////////////////////////////////////////////////////////
  public double getStart()  { return getXStart(); }
  public double getDelta()  { return getXDelta(); }
  public int    getUnits()  { return getXUnits(); }
  public double getXStart() { return xStart; }
  public double getXDelta() { return xDelta; }
  public int    getXUnits() { return xUnits; }
  public int    getXFrame() { return (int)getNumberOfRows(); }
  public double getYStart() { return yStart; }
  public double getYDelta() { return yDelta; }
  public int    getYUnits() { return yUnits; }
  public int    getYFrame() { return 1; }
  public String getFormat() { return "NH"; }
  public int    getMode()   { return (int)getFormat().charAt(0); }
  public int    getType()   { return (int)getFormat().charAt(1); }

  public void connect (int mode) { }

  public Data getDataBuffer (int elements) {
    Data data = new Data();

    int numBytes = 0;
    for (ColumnDef colDef : cols) {
      String fmt = colDef.getFormat();

      numBytes += Data.getBPA(fmt);
    }
    data.setFormat("NH");
    data.setRecLength(numBytes);
    data.setSize(elements);

    return data;
  }

  public double getTimeAt (double offset) {
    return getXStart() + (getXDelta() * offset);
  }

  public int read (Data data, int elements) {
    int numRead = 0;
    int offset  = 0;

    elements = Math.min(elements, data.size);

    for (numRead = 0; (numRead < elements) && (avail() > 0.0); numRead++) {
      Table tbl = getDataTable(seek());
      seek(seek() + 1.0);

      for (ColumnDef colDef : cols) {
        String name = colDef.name;
        String fmt  = colDef.getFormat();
        int    bps  = Data.getBPA(fmt);

        switch (fmt.charAt(1)) {
          case DOUBLE:   data.packD(offset,    tbl.getD(name));  break;
          case FLOAT:    data.packF(offset,    tbl.getF(name));  break;
          case XLONG:    data.packX(offset,    tbl.getX(name));  break;
          case LONG:     data.packL(offset,    tbl.getL(name));  break;
          case INT:      data.packI(offset,    tbl.getI(name));  break;
          case BYTE:     data.packB(offset,    tbl.getB(name));  break;
          case ASCII:    data.packS(offset,bps,tbl.getS(name));  break;
          case ASCII256: data.packS(offset,bps,tbl.getS(name));  break; // NXM-3570
        }
        offset += bps;
      }
    }

    return numRead;
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // New Methods
  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** Sets xStart.
      @param val the value to set.
   */
  public void setXStart (double val) { xStart = val; }
  /** Sets xDelta.
      @param val the value to set.
   */
  public void setXDelta (double val) { xDelta = val; }
  /** Sets xUnits (see {@link Units}).
      @param val the value to set.
   */
  public void setXUnits (int    val) { xUnits = val; }
  /** Sets yStart.
      @param val the value to set.
   */
  public void setYStart (double val) { yStart = val; }
  /** Sets yDelta.
      @param val the value to set.
   */
  public void setYDelta (double val) { yDelta = val; }
  /** Sets yUnits (see {@link Units}).
      @param val the value to set.
   */
  public void setYUnits (int    val) { yUnits = val; }

  /** Sets the units for values along the X-axis.
      @param value The units (see {@link Units}).
   */
  public void setXUnits (String value) { setXUnits(DataFile.getUnitsID(value)); }

  /** Sets the units for values along the Y-axis.
      @param value The units (see {@link Units}).
   */
  public void setYUnits (String value) { setYUnits(DataFile.getUnitsID(value)); }

  /** Return the data as a single table.
      @return the data as a {@link Table} object.
   */
  public Table toTable () {
    return toTable(null);
  }

  /** Return the data as a single table.
      @param prefix The prefix on the row name (null = use default).
      @return the data as a {@link Table} object.
   */
  public Table toTable (String prefix) {
    if (prefix == null) { prefix = defRowName + "_"; }

    Table tbl = new Table();
    for (int i = 0; i < getNumberOfRows(); i++) {
      tbl.put(prefix + (i+1), getDataTable(i));
    }
    return tbl;
  }

  /** Return the data as XML.
      @return the data XML as a String
   */
  public String toXml () {
    return "<XML>\n"+toXmlBrief(2)+"</XML>";
  }

  /** Return the data as a XML but omit the top/bottom &lt;XML&gt; tags to allow insertion elsewhere
      in an XML file.
      @param indent The number of spaces to indent (0=noIndent, -1=allOneLine).
      @return the data XML as a String without enveloping &lt;XML&gt; tags.
   */
  public String toXmlBrief (int indent) {
    return toXmlBrief(null, indent);
  }

  /** Return the data as a XML but omit the top/bottom &lt;XML&gt; tags to allow insertion elsewhere
      in an XML file.
      @param tag    The XML tag name (null = use default).
      @param indent The number of spaces to indent (0=noIndent, -1=allOneLine).
      @return the data XML as a String without enveloping &lt;XML&gt; tags.
   */
  public String toXmlBrief(String tag, int indent) {
    if (tag == null) { tag = defRowName; }
    StringBuilder str = new StringBuilder();
    String indentStr = StringUtil.padLeft("", indent);

    for (int i = 0; i < getNumberOfRows(); i++) {
      Table row = getDataTable(i);

      str.append(indentStr).append("<").append(tag);

      for (ColumnDef colDef : cols) {
        String name = colDef.name;
        String val  = row.getS(name);

        str.append(' ').append(name).append("=\"").append(val).append("\"");
      }

      str.append("/>");
      if (indent != -1) { str.append("\n"); }
    }

    return str.toString();
  }

  /** Gets the size of the header in number of rows.
      @return the number of rows in the header.
   */
  public int getHeaderRows() {
    return (headerRows < 1)? 0 : headerRows;
  }

  /** Sets the size of the header in number of rows (0=NoHeader).
      @param rows the number of rows to set.
   */
  public void setHeaderRows(int rows) {
    headerRows = rows;
  }

  /** Gets a row of data from the header. The length of the header is specified in
      {@link #getHeaderRows()} (see {@link CsvFile} for detailed description). If <code>row</code>
      is equal to <code>getHeaderRows()-1</code> the line containing all of the column names is
      returned; if <code>row</code> is less than that the associated row is returned.
      @param row Header row number.
      @return The associated header data or <code>null</code> if <code>row</code> is invalid.
   */
  public String getHeaderData (int row) {
    if (headerRows <= 0 ) return null;
    if (row < 0         ) return null;
    if (row > headerRows) return null;

    if (row == headerRows-1) return colHeadersToString(cols,pad); // last one with column names
    if (row >= head.size())  return ""; // blank

    Object data = head.get(row);
    return (head == null)? "" : data.toString();
  }

  /** Sets a row of data in the header. The length of the header is specified in
      {@link #getHeaderRows()} (see {@link CsvFile} for detailed description). <b>The last row in
      the header (<code>row = getHeaderRows()-1</code>) is automatically set based on the column names
      and can not be modified using this method.</b> All other header data is free-form and can be
      set to whatever the user wants (note that it can *not* contain any return characters).
      @param row  Header row number.
      @param data The header data (can *not* contain any return characters). A value of <code>null</code>
                  is considered identical to a blank line (<code>""</code>).
      @return <code>true</code> if the header was set, <code>false</code> if not. (This always returns
              <code>false</code> if the file is read-only or if <code>row</code> is invalid).
   */
  public boolean setHeaderData (int row, String data) {
    if (!isOutput          ) return false;
    if (headerRows <= 0    ) return false;
    if (row <  0           ) return false;
    if (row >  headerRows  ) return false;
    if (row == headerRows-1) return false;

    if ((data != null) && (data.indexOf('\n') > 0)) {
      return false; // contains return character
    }

    while (head.size() <= row) {
      head.add(null);
    }
    head.set(row, data);
    return true;
  }

  /** Gets the column definition table.
      @return the column definition table.
   */
  public Table getColumnNamesTable() {
    Table tbl = new Table();
    for (ColumnDef colDef : cols) {
      String name = colDef.name;

      tbl.put(name, colDef.toTable());
    }

    return tbl;
  }

  /** Gets the column names.
      @return a String object containing the column names.
   */
  public String getColumnNames() {
    StringBuilder str = new StringBuilder();

    for (int i = 0; i < cols.length; i++) {
      String name = cols[i].name;
      String fmt  = cols[i].format;

      if (i   != 0   ) { str.append(","); }
      if (fmt != null) { str.append(fmt).append(":"); }
      str.append(name);
    }

    return str.toString();
  }

  /** Sets the column names.
      @param str The column names. Names should be separated with a comma (',') or a bar ('|'). Each
                 name may include an optional "&lt;FMT&gt;:" prefix giving the format of the data in the
                 column (e.g. "SF:LATITUDE|SF:LONGITUDE|CITY_NAME"). Note that all names will be
                 converted to upper case and any special characters other than a letter, number,
                 underscore, or dash will be converted to an underscore.
   */
  public void setColumnNames(String str) {
    ArrayList<ColumnDef> list = new ArrayList<>();
    if (str != null) {
      str = str.replaceAll("[|]", ","); // Convert "|" to ","
      Parser parser = new Parser(str);

      for (int i = 1; i <= parser.elements(); i++) {
        list.add(getColumnDef(parser.get(i)));
      }
    }
    cols = list.toArray(NO_COLUMNS);
  }

  /** Gets the default row prefix to use with {@link #toTable()} and {@link #toXml()}.
      @return the default row prefix to use with {@link #toTable()} and {@link #toXml()}.
   */
  public String getRowName() {
    return defRowName;
  }

  /** Sets the default row prefix to use with {@link #toTable()} and {@link #toXml()}.
      @param str the default row prefix to use with {@link #toTable()} and {@link #toXml()}.
   */
  public void setRowName(String str) {
    if ((str != null) && (str.length() > 0)) {
      defRowName = str;
    }
  }

  /** Creates a default column definition table for the given column number.
      @param colNum The column number.
   */
  private static ColumnDef getColumnDef(int colNum) {
    return getColumnDef("COL_"+colNum,null);
  }

  /** Creates a column definition table.
      @param def The column definition ("<NAME>", "<FMT>:<NAME>").
   */
  private static ColumnDef getColumnDef (String def) {
    Table  col  = new Table();
    String name = removeQuotes(def).toUpperCase();
    String fmt  = null;

    if ((name.length() > 2) && (name.charAt(2) == ':')) {
      fmt  = name.substring(0,2);
      name = name.substring(3);
    }
    else if ((name.length() > 1) && (name.charAt(1) == ':')) {
      fmt  = "S"+name.substring(0,1).toUpperCase(); // e.g. 'D:' -> 'SD'
      name = name.substring(2);
    }
    return getColumnDef(name,fmt);
  }

  /** Creates a column definition table.
      @param name The column name (not null).
      @param fmt  The column format (can be null).
   */
  private static ColumnDef getColumnDef(String name, String fmt) {
    name = name.toUpperCase().replaceAll("[^A-Z0-9_.-]", "_");
    if (name.contains(".")) name = name.replace(".", "->");  // since 3.7.1 for NXM-2931
    return new ColumnDef(name, fmt);
  }

  /** Sets a row in the file.
      @param index   The row number (0..n-1, -1 or n = addToEnd).
      @param tbl     The row.
      @param insert  Indicates if inserting a new row rather than modifying an existing one.
   */
  private void putData(int index, Table tbl, boolean insert) {
    Table row = new Table(Table.KV, Table.ADD);

    if (tbl == null) {
      // JC4: This tripped me up one time... so I added a helpful explanation.
      throw new MidasException("CsvFile: Attempting to set values using a null table.");
    }

    for (ColumnDef colDef : cols) {
      String name = colDef.name;
      String fmt  = colDef.format;
      Object val  = removeQuotes(tbl.getS(name));

      if ((fmt != null) && (fmt.length() == 2)) {
        if (val == null) {
          switch (fmt.charAt(1)) {
            case ASCII:     val = "";  break;
            case ASCII256:  val = "";  break; // NXM-3570 - column names should only use valid Data Formats
            case STRING:    val = "";  break;
            default:  val = "0"; break;
          }
        }
        val = Convert.o2Data(val, fmt.charAt(1), null); // NXM-3570 'Z' format should be treated as Data
      }
      else {
        if (val == null) {
          val = "";
        }
      }

      if (val != null) {
        row.putx(name, val, 0);
      }
    }
    row.setFlags(0);

    while (index > rows.size()) {
      putData(-1, new Table(), false);
    }

    insert = insert || (index==rows.size());

         if (index < 0) { rows.add(row);        }
    else if (insert)    { rows.add(index, row); }
    else                { rows.set(index, row); }

    upToDate = false;
  }

  /** Read the file and store it in the rows vector.
   *  TODO: this should be deprecated.
   */
  private void readFile() {
    if (isOpen) {
      try {
        BufferedReader in = new BufferedReader(new InputStreamReader(io.getInputStream()));
        String line = in.readLine();
        String nextLine = null;
        int lineNum = 0;

        while (line != null) {
          lineNum++;

          Parser parser = new Parser(line, ',');
          parser.setUseSpaceDelimiter(false);

          if ((lineNum == 0) && (headerRows == -1)) {
            // TBD -- Try to be smart about HEADERROWS=0 vs HEADERROWS=1
          }

          if (lineNum < getHeaderRows()) {
            head.add(line);

          }
          else if (lineNum == getHeaderRows()) {
            if (cols.length == 0) {
              ArrayList<ColumnDef> list = new ArrayList<>();
              for (int i = 1; i <= parser.elements(); i++) {
                list.add(getColumnDef(parser.get(i)));
              }
              cols = list.toArray(NO_COLUMNS);
            }
          }
          else {
            if (cols.length == 0) {
              ArrayList<ColumnDef> list = new ArrayList<>();
              for (int i = 1; i <= parser.elements(); i++) {
                list.add(getColumnDef(i));
              }
              cols = list.toArray(NO_COLUMNS);
            }

            Table row = colValuesFromString(M,lineNum,cols,parser);
            putData(-1, row, false);
          }
          if (nextLine != null) {
            line = nextLine;
            nextLine = null;
          }
          else {
            line = in.readLine();
          }
        }

      }
      catch (IOException e) {
        throw new MidasException("Error writing "+this+": "+e, e);
      }
    }
    upToDate = true;
  }

  /** Read the file and store it in the rows vector. Read all characters in data file that has embedded linefeeds
   *  if qualifier flag is set: FLAGS=EMBEDDEDLF.
   *  CSV special characters defined in RFC 4180 handled: DQUOTE *(TEXTDATA / COMMA / CR / LF / 2DQUOTE) DQUOTE
   *  Windows CRLF handled: CRLF = CR LF ;as per section 6.1 of RFC 2234
   *  @since NeXtMidas 3.7.1 for NXM-2931
   */
  @SuppressWarnings("fallthrough")
  private void readAll () {
    if (isOpen) {
      StringBuilder sb = new StringBuilder();                // hold cell data while reading
      ArrayList<String> data = new ArrayList<>();      // hold complete cell data for adding row
      BufferedReader in = null;                              // read file one character at a time
      int lineNum = 0;
      String line = "";
      boolean fieldWithQuote = false;                        // are we within a field thats value is quoted

      try {
        in = new BufferedReader(new InputStreamReader(io.getInputStream()));
        lineNum = 1;                                            // start reading at line one to get header row

        while ((lineNum < getHeaderRows()) && ((line = in.readLine()) != null)) {
          // Any lines before the header row just get picked up as-is with no special parsing
          head.add(line);
          lineNum++;
        }

        int currentChar;
        int nextChar = in.read();

        while (nextChar >= 0 || !data.isEmpty()) {  // Updated in 3.7.3 for NXM-3155 to ensure last row is added
          if (nextChar >= 0) {
            currentChar = nextChar;
            nextChar = in.read();
          }
          else {
            // we have one last line of data, so effectively add a '\n' to process
            currentChar = '\n';
            nextChar = -1;
          }

          if (currentChar == CR && nextChar == LF) {
            // Windows CRLF = CR LF ( RFC 2234 )
            currentChar = '\n';
            nextChar = in.read();
          }

          switch (currentChar) {
            case '"':                                         // e.g. "0\n1\n2" or "are, there more"
              if (fieldWithQuote) {
                Shell.warning("Multiple values found in a single column on line "+lineNum);
              }
              fieldWithQuote = true;

              if (hasLinefeed) {
                while (nextChar >= 0) {                       // read data between double quotes
                  currentChar = nextChar;
                  nextChar = in.read();

                  if (currentChar == CR && nextChar == LF) {
                    // Windows CRLF = CR LF ( RFC 2234 )
                    currentChar = '\n';
                    nextChar = in.read();
                  }

                  if ((currentChar == '"') && (nextChar == '"')) {
                    // Pair of double-quotes becomes a single quote (this is the only special character recognized).
                    sb.append('"');
                    nextChar = in.read(); // skip over second '"'
                  }
                  else if (currentChar == '"') {
                    // cell data complete (break out and wait for ',' or '\n')
                    break;
                  }
                  else {
                    sb.append((char)currentChar);
                  }
                }
              }
              else {
                // csv files not containing line feeds in cell data
                while (nextChar >= 0) {
                  currentChar = nextChar;
                  nextChar = in.read();

                  if (currentChar == CR && nextChar == LF) {
                    // Windows CRLF = CR LF ( RFC 2234 )
                    currentChar = '\n';
                    nextChar = in.read();
                  }

                  if ((currentChar == '\n') || (currentChar == '"')) {
                    // cell data complete (break out and wait for ',' or '\n')
                    break;
                  }
                  else {
                    sb.append((char) currentChar);
                  }
                }
              }
              break;

            case ',':
              data.add(sb.toString());
              sb.delete(0, sb.length());
              fieldWithQuote = false; // end of field, so not within field that has its value in quotes
              break;

            case '\n':
              data.add(sb.toString());
              sb.delete(0, sb.length());

              if ((lineNum == 0) && (headerRows == -1)) {
                // TBD -- Try to be smart about HEADERROWS=0 vs HEADERROWS=1
              }

              if (lineNum < getHeaderRows()) {
                throw new AssertionError("This should be impossible!");
              }
              else if (lineNum == getHeaderRows()) {
                if (cols.length == 0) {
                  ArrayList<ColumnDef> list = new ArrayList<>();
                  for (String col : data) {
                    list.add(getColumnDef(col));
                  }
                  cols = list.toArray(NO_COLUMNS);
                }
              }
              else {
                if (cols.length == 0) {
                  ArrayList<ColumnDef> list = new ArrayList<>();
                  for (int i = 0; i < data.size(); i++) {
                    list.add(getColumnDef(data.get(i)));
                  }
                  cols = list.toArray(NO_COLUMNS);
                }
                Table row = colValuesFromString(M, lineNum, cols, data);
                putData(-1, row, false);
              }

              data.clear();
              fieldWithQuote = false; // end of field, so not within field that has its value in quotes
              lineNum++;
              break;

            default:
              if (currentChar!=SPACE && currentChar!=HTAB && fieldWithQuote) {
                Shell.warning("Multiple values found in a single column on line "+lineNum);
              }
              sb.append((char)currentChar);
              break;
          }
        }
      }
      catch (IOException e) {
        throw new MidasException("Error writing " + this + ": " + e, e);
      }
      finally {
        try {
          if (in != null) in.close();
        }
        catch (IOException ex) {
          throw new MidasException("Error closing buffered reader");
        }
      }
    }
    upToDate = true;
  }

  /** Column definition. */
  private static final class ColumnDef {
    final String name;
    final String format;

    ColumnDef(String name, String format) {
      this.name   = name;
      this.format = format;
    }

    ColumnDef(Table tbl) {
      this(tbl.getS("NAME"), tbl.getS("FORMAT"));
    }

    String getName() { return name; }
    String getFormat() { return (format == null)? "XA" : format; }

    Table toTable() {
      Table table = new Table(Table.KV, Table.ADD);
      table.put("NAME",   name);
      if (format != null) table.put("FORMAT", format);
      table.setFlags(0);
      return table;
    }
  }
}
