package nxm.sys.libg;

import java.awt.BorderLayout;
import java.awt.Color;
import java.awt.Component;
import java.awt.Container;
import java.awt.Font;
import java.awt.datatransfer.DataFlavor;
import java.awt.datatransfer.StringSelection;
import java.awt.datatransfer.Transferable;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.HierarchyBoundsListener;
import java.awt.event.HierarchyEvent;
import java.lang.reflect.Constructor;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.Vector;

import javax.swing.DefaultCellEditor;
import javax.swing.DropMode;
import javax.swing.JCheckBox;
import javax.swing.JComponent;
import javax.swing.JLabel;
import javax.swing.JMenu;
import javax.swing.JMenuBar;
import javax.swing.JMenuItem;
import javax.swing.JOptionPane;
import javax.swing.JScrollPane;
import javax.swing.JTable;
import javax.swing.JTextField;
import javax.swing.JToggleButton;
import javax.swing.JViewport;
import javax.swing.SwingConstants;
import javax.swing.TransferHandler;
import javax.swing.UIManager;
import javax.swing.border.Border;
import javax.swing.border.EmptyBorder;
import javax.swing.table.AbstractTableModel;
import javax.swing.table.DefaultTableCellRenderer;
import javax.swing.table.DefaultTableColumnModel;
import javax.swing.table.TableCellEditor;
import javax.swing.table.TableCellRenderer;
import javax.swing.table.TableColumn;
import javax.swing.table.TableModel;

import nxm.sys.inc.DataTypes;
import nxm.sys.inc.InternalUseOnly;
import nxm.sys.inc.ListFile;
import nxm.sys.inc.ListInterface;
import nxm.sys.inc.MessageHandler;
import nxm.sys.lib.BaseFile;
import nxm.sys.lib.Cache;
import nxm.sys.lib.Convert;
import nxm.sys.lib.Data;
import nxm.sys.lib.DataFile;
import nxm.sys.lib.FileName;
import nxm.sys.lib.KeyObject;
import nxm.sys.lib.MFormat;
import nxm.sys.lib.Message;
import nxm.sys.lib.Midas;
import nxm.sys.lib.MidasException;
import nxm.sys.lib.Parser;
import nxm.sys.lib.RecordComparator;
import nxm.sys.lib.Shell;
import nxm.sys.lib.StringUtil;
import nxm.sys.lib.Table;
import nxm.sys.lib.Time;
import nxm.sys.lib.Util;

/** A {@link JTable} based implementation of {@link ListInterface} that has been extended (by
    {@link LargeJTable}) to allow files larger than 2GiB.

    NOTE: MJList does NOT currently support Java Serialization, even though it extends classes that support it.

    @see ListInterface

    @author  J. Czechowski IV
    @since   NeXtMidas 2.1.0
 */
public class MJList extends LargeJTable implements ListInterface {
  /** NOTE: this should be changed if the structure/fields of this class changes. */
  private static final long serialVersionUID = 2012021030311L; // for Serializable interface

  /** The Midas context to use                                                   */ private transient Midas    midas         =  null;
  /** The file being listed                                                      */ private transient ListFile listFile      =  null;
  /** Names of all columns in the file                                           */ private           String[] columnNames   =  null;
  /** Formats of all columns in the file                                         */ private           String[] columnFormats =  null;
  /** The shade color to use                                                     */ private           Color    shadeColor    =  null;
  /** The shade interval (0 for off)                                             */ private           int      shadeInterval =     0;
  /** The macro row offset, 1 for 1-based indexing                               */ private           long     rowOffset     =     1;
  /** The macro column offset, 1 for 1-based indexing                            */ private           long     colOffset     =     1;
  /** whether or not the cell is editable                                        */ private           boolean  editable      =  true;
  /** Adjust selected rows/cells up/down when adding or removing rows (BUG 2109) */ private           boolean  adjSelCells   = false;

  /** The default background color                        */ private           String   defFgColor      = null;
  /** The default foreground color                        */ private           String   defBgColor      = null;
  /** The default row height                              */ private           int      defRowHeight    =   -1;
  /** The default foreground color (from template)        */ private           String   globalFgColor   = null;
  /** The default background color (from template)        */ private           String   globalBgColor   = null;
  /** The default row height (from template)              */ private           int      globalRowHeight =   -1;
  /** Allow column dragging?                              */ private           boolean  colDrag         = true;
  /** Automatically hide columns not in template?         */ private           boolean  autoHide        = true;
  /** Automatically reorder columns in template?          */ private           boolean  autoOrder       = true;
  /** The current template                                */ private           Table    savedTemplate   = null;
  /** The set of colors used for COLOR=FIELDn             */ private           Table    mcolorTbl       = null;
  /** The primary key for updateRow(..), (null=UseRowNum) */ private           String   primaryKey      = null;
  /** The cache of primary key indicies                   */ private transient Cache    priKeyCache     = null;

  /** Menu bar "internal" to MJList        */ private JMenuBar      menuBar  = null;
  /** Parent container                     */ private Container     parent   = null;
  /** The button in the upper-left corner  */ private JToggleButton ulButton = null;
  /** The button in the upper-right corner */ private JToggleButton urButton = null;
  /** The button in the lower-left corner  */ private JToggleButton llButton = null;
  /** The button in the lower-right corner */ private JToggleButton lrButton = null;

  /** Message listener (null=UseDefault) */ private transient ListMessageListener msgListener = null;
  /** Message handler (usually list2)    */ private transient MessageHandler      msgHandler  = null;

  /** Send messages                                     */ private static final int SEL_MSGS        = 0x01;
  /** clear current selections                          */ private static final int SEL_CLEAR       = 0x02;
  /** ignore case when selecting by value (since 3.1.0) */ private static final int SEL_IGNORE_CASE = 0x04;

  /** Action listener that sends all actions to the menuAction method. */
  private transient final ActionListener MENU_ACTION = new ActionListener() {
    public void actionPerformed(ActionEvent e) {
      if (msgHandler != null) {
        msgHandler.processMessage(new Message("MENU", 0, e.getActionCommand(), msgHandler, this));
      }
    }
  };

  /** The default formats for given column types.
      <pre>
        DEFAULT_FORMATS[2*i+0] = column type
        DEFAULT_FORMATS[2*i+1] = default format for type
      </pre>
   */
  private static final String[] DEFAULT_FORMATS = { "STRING", null,
                                                    "NUMBER", "GEN",
                                                    "DATE",   "STD",
                                                    "ANGLE",  "DMS" };

  /** Used as a marker */
  @InternalUseOnly private static final String FORCE_UPDATE = "Force Update";

  /** Drag and Drop enabled?  (since 3.3.0 - Bug 2120) */
  private boolean dragDrop = false;

  /** Creates a new instance of MJList.
      @param maxSortSize The maximum sortable table size (-1 to disable sorting).
   */
  private MJList(int maxSortSize) {
    /* The order of the next lines is important, do not change the ordering of them! */
    this.setModel(new TableSorter(this, new NmTableModel(this), getTableHeader(), maxSortSize));
    super.setColumnModel(new NmColumnModel());

    /* The order of the next lines is irrelevant. */
    ulButton = new JToggleButton(); ulButton.setEnabled(false);
    urButton = new JToggleButton(); urButton.setEnabled(false);
    llButton = new JToggleButton(); llButton.setEnabled(false);
    lrButton = new JToggleButton(new Symbol.ScrollEnd());
    lrButton.setEnabled(true);
    lrButton.setBorderPainted(false);
    lrButton.setToolTipText("Click to enable/disable ScrollEnd.");

    defFgColor      = MColor.toString(getForeground());
    defBgColor      = MColor.toString(getBackground());
    defRowHeight    = getRowHeight();
    globalRowHeight = defRowHeight;
    mcolorTbl       = new Table();

    setDoubleBuffered(true); // <-- JC4: Added as per Bug 997 - I think this needs to go elsewhere.

    /* JC4: If autoCreateColumnsFromModel is set to true JTable will automatically re-create the
            columns when it receives a fireTableStructureChanged() from the model. -- We don't
            want this, so set it to false.
     */
    super.autoCreateColumnsFromModel = false;

    /** JC4: This tries to work around a refresh issue with the AWT graphics. */
    super.addHierarchyBoundsListener(new HierarchyBoundsListener() {
      public void ancestorMoved(HierarchyEvent e)   { if (isShowing()) { hack(); revalidate(); } }
      public void ancestorResized(HierarchyEvent e) { if (isShowing()) { hack(); revalidate(); } }
    });
  }

  /** Creates a new instance of MJList.
      @param mw          The window to add the list to.
      @param maxSortSize The maximum sortable table size (-1 to disable sorting).
      @param baseIndex   Set this to 1 for 1-based indexing, 0 for 0-based indexing.
   */
  public MJList(MWindow mw, int maxSortSize, int baseIndex) {
    this(null,mw,maxSortSize,baseIndex);
  }

  /** Creates a new instance of MJList.
      @param midas       The midas context to use.
      @param mw          The window to add the list to.
      @param maxSortSize The maximum sortable table size (-1 to disable sorting).
      @param baseIndex   Set this to 1 for 1-based indexing, 0 for 0-based indexing.
   */
  public MJList(Midas midas, MWindow mw, int maxSortSize, int baseIndex) {
    this(maxSortSize);
    this.midas = (midas == null)? Shell.getSharedMidasContext(): midas;

    if ((baseIndex >= 0) && (baseIndex <= 1)) {
      this.rowOffset = baseIndex;
      this.colOffset = baseIndex;
    } else {
      this.showErrorMsg("Illegal base index '"+baseIndex+"', using default.");
    }

    JScrollPane scrollPane = new JScrollPane(this, JScrollPane.VERTICAL_SCROLLBAR_ALWAYS,
                                                   JScrollPane.HORIZONTAL_SCROLLBAR_AS_NEEDED);

    scrollPane.setVerticalScrollBar(getVertScrollBar());
    setAutoResizeMode(AUTO_RESIZE_OFF);
    setVisible(true);
    setRowLabelsVisible(true);
    setCellSelectionEnabled(true);

    /* If we do not add these buttons the corners are hard to distinguish with the Gear3 theme. */
    scrollPane.setCorner(JScrollPane.UPPER_LEFT_CORNER,  ulButton);
    scrollPane.setCorner(JScrollPane.UPPER_RIGHT_CORNER, urButton);
    scrollPane.setCorner(JScrollPane.LOWER_LEFT_CORNER,  llButton);
    scrollPane.setCorner(JScrollPane.LOWER_RIGHT_CORNER, lrButton);

    if (mw != null) {
      addAllListeners(mw);

      parent   = mw.getPanel();
      menuBar  = createMenu();

      parent.setLayout(new java.awt.BorderLayout());
      parent.add(menuBar, BorderLayout.NORTH);
      parent.add(scrollPane, BorderLayout.CENTER);

      scrollPane.setBounds(parent.getBounds());
      parent.setBounds(parent.getBounds());
      setMenuVisible(false); // Hides menu and revalidates (the revalidation is MANDATORY)
    }
  }

// START - DIRTY HACK /////////////////////////////////////////////
  /** This is a dirty hack to address resize issues in the 2.5.0 release that are caused by
      problems in MWindow. The likely problem is that MJPanel and MWindow use a null layout
      (PANEL and GWidget do their own layout) that prevents the resizes from flowing through.
      <pre>
        (1) Set Bounds (sets value of new bounds and calls invalidate())
            (a) A call to Component.setBounds(..) calls Component.reshape(..) (deprecated)
            (b) In turn Component.reshape(..) calls invalidate() if the size has changed
            (c) Container.invalidate() calls super.invalidate() and invalidates its self
                in the LayoutManager if it implements LayourManager2

        (2) Invalidate causes a call to Component.validate() to occurr

        (3) Validate (apply the new bounds)
            (a) Container.validate() overrides Component.validate() and includes a call to
                Container.validateTree()
            (b) Container.validateTree() handles validation of the container's children
            (c) Container.validateTree() calls doLayout() which calls layout() (deprecated)
            (d) Container.layout() then uses LayoutManager.layoutContainer(this) to layout
                the container, if the layour manager is null, layout() does nothing
      </pre>
   */
  private void hack() {
    parent.invalidate();
    parent.validate();
  }
// END - DIRTY HACK /////////////////////////////////////////////

  /** Gets a thread-safe instance of this class.
      @return a thread-safe instance
      @see GraphicsUtil#getThreadSafeInstance(Object,Class)
   */
  public ListInterface getThreadSafeInstance() {
    return (ListInterface)GraphicsUtil.getThreadSafeInstance(this, ListInterface.class);
  }

  public void setMessageHandler (MessageHandler msgHandler) {
    this.msgHandler = msgHandler;
  }

  public MessageHandler getMessageHandler () {
    return msgHandler;
  }

  /** Convenience metnod used by createMenu() */
  private JMenuItem newMenuItem (String action, String text, int mnemonic) {
    JMenuItem menuItem = new JMenuItem(text, mnemonic);

    menuItem.setActionCommand(action);
    menuItem.addActionListener(MENU_ACTION);

    return menuItem;
  }

  /** Convenience metnod used by createMenu() */
  private JMenu newMenu (String action, String text, int mnemonic) {
    JMenu menu = new JMenu(text);

    menu.setMnemonic(mnemonic);
    menu.setActionCommand(action);

    return menu;
  }

// JC4: Jay has indicated that he thinks this type of method (createManu()) should
// be public to allow the menu to be attached to another location. I'm not sure about
// how exactly we want to do that, so this method is private for now; however I tried
// to write it in a way that it could be made public in the future.

  /** Creates a menu bar for this instance.
      @since NeXtMidas 2.5.0
   */
  private JMenuBar createMenu () {
    // JC4: For menu order I looked at NetBeans 5.5:
    //          'File', 'Edit', 'View', 'Navigate', ..., 'Tools', 'Window', 'Help'
    //      and Mozilla/SeaMonkey 1.0.8:
    //          'File', 'Edit', 'View', 'Go', ..., 'Tools', 'Window', 'Help'
    //
    //      I also put 'Preferences' under 'Edit' to match Mozilla. Also, 'Go to Line' and
    //      'Keyboard Shortcuts' match NetBeans.
    JMenuBar menuBar   = new JMenuBar();
    JMenu    fileMenu  = newMenu("FILE", "File",     'F');   menuBar.add(fileMenu);
    JMenu    editMenu  = newMenu("EDIT", "Edit",     'E');   menuBar.add(editMenu);
    //JMenu    viewMenu  = newMenu("VIEW", "View",     'V');   menuBar.add(viewMenu);
    JMenu    naviMenu  = newMenu("NAVI", "Navigate", 'N');   menuBar.add(naviMenu);
    JMenu    tempMenu  = newMenu("TEMP", "Template", 'T');   menuBar.add(tempMenu);
    //JMenu    toolMenu  = newMenu("TOOL", "Tools",    'T');   menuBar.add(toolMenu);
    //JMenu    windMenu  = newMenu("WIND", "Window",   'W');   menuBar.add(windMenu);
    JMenu    helpMenu  = newMenu("HELP", "Help",     'H');   menuBar.add(helpMenu);

    fileMenu.add(newMenuItem("FILE.LOAD",      "Load...",              'L'));
    fileMenu.add(newMenuItem("FILE.REREAD",    "Reread",               'R'));
    fileMenu.add(newMenuItem("FILE.SAVE_COPY", "Save Copy...",         'S'));
    fileMenu.addSeparator();
    fileMenu.add(newMenuItem("FILE.EXIT",      "Exit LIST2",           'X'));

    editMenu.add(newMenuItem("EDIT.COPY",      "Copy",                 'C'));
    editMenu.add(newMenuItem("EDIT.PASTE",     "Paste...",             'P'));
    editMenu.add(newMenuItem("EDIT.INSERT_ROW","Insert Row...",        'I'));
    editMenu.add(newMenuItem("EDIT.INSERT_VAL","Insert Values...",     'V'));
    editMenu.add(newMenuItem("EDIT.DELETE",    "Delete Row...",        'D'));
    editMenu.addSeparator();
    editMenu.add(newMenuItem("EDIT.PREF",      "Preferences...",       'P'));

    tempMenu.add(newMenuItem("TEMP.LOAD",      "Load...",              'L'));
    tempMenu.add(newMenuItem("TEMP.SAVE_COPY", "Save Copy...",         'S'));
    tempMenu.addSeparator();
    tempMenu.add(newMenuItem("TEMP.COPY",      "Copy",                 'C'));
    tempMenu.add(newMenuItem("TEMP.PASTE",     "Paste",                'P'));
    tempMenu.addSeparator();
    tempMenu.add(newMenuItem("TEMP.EDIT",      "Edit...",              'E'));
    tempMenu.add(newMenuItem("TEMP.DEF",       "Restore Default",      'R'));

    naviMenu.add(newMenuItem("NAVI.GOTO",      "Go to Line...",        'L'));

    helpMenu.add(newMenuItem("HELP.ABOUT",     "About",                'A'));
    helpMenu.add(newMenuItem("HELP.KEYS",      "Keyboard Shortcuts",   'K'));

    return menuBar;
  }

  /** Is the 'File' menu visible?
      @return true if 'File' menu is visible
      @since NeXtMidas 2.5.0
   */
  public boolean isMenuVisible() {
    return (menuBar == null)? false : menuBar.isVisible();
  }

  /** Set the visibility of the 'File' menu.
      @param visible visibility of the 'File' menu
      @since NeXtMidas 2.5.0
   */
  public void setMenuVisible(boolean visible) {
    if (menuBar != null) {
      menuBar.setVisible(visible);
      parent.invalidate();
      parent.validate();
    }
  }

  /** Sets an alternate listener for list messages.
      @param listener The listener to use (null=DoDefaultThing).
   */
  public void setListMessageListener(ListMessageListener listener) {
    msgListener = listener;
  }

  public long getRowOffset () {
    return rowOffset;
  }

  public long getColOffset () {
    return colOffset;
  }

  /** set if the MJList is editable
   *  @param val editable if true
   */
  public void setEditable (boolean val)  { editable = val; }

  /** is this MJList editable
   *  @return true if editable
   */
  public boolean isEditable () { return editable; }

  /** Gets the table sorter model being used. */
  private TableSorter getSortingModel() {
    return (TableSorter)getModel();
  }

  /** Gets the data model being used. */
  private TableModel getDataModel() {
    return getSortingModel().getTableModel();
  }

  /** Adds the standard set of mouse, key, focus, etc listeners for the MJList.
      @param mw The MWindow to use.
   */
  public void addAllListeners(MWindow mw) {
    if (mw != null) {
      addAllListeners(this,            mw);
      addAllListeners(getScrollPane(), mw);
    }
  }

  /** Adds the standard set of mouse, key, focus, etc listeners for the MJList.
      @param mw The MWindow to use.
   */
  public void addMouseKeyListeners(MWindow mw) {
    if (mw != null) {
      addMouseKeyListeners(this,            mw);
      addMouseKeyListeners(getScrollPane(), mw);
    }
  }

  /** Adds the standard set of ALL listeners for the MJList.
      @param comp The component to add the listeners to.
      @param mw   The MWindow to use.
   */
  private static void addAllListeners(Component comp, MWindow mw) {
    addMouseKeyListeners(comp, mw);
    comp.addComponentListener(mw);
    comp.addComponentListener(mw);
    comp.addFocusListener(mw);
  }

  /** Adds the standard set of mouse, and key listeners for the MJList.
      @param comp The component to add the listeners to.
      @param mw   The MWindow to use.
   */
  private static void addMouseKeyListeners(Component comp, MWindow mw) {
    comp.addKeyListener(mw.hkl);
    comp.addKeyListener(mw);
    comp.addMouseListener(mw);
    comp.addMouseMotionListener(mw);
  }

  /** Set whether or not to adjust selected cells and rows up or down when adding
      and removing rows.
      @param value True to adjust the cells
   */
  public void setAdjSelCells (boolean value) { adjSelCells = value; }

  /** Get if selected cells are adjusted when adding and removing rows
      @return TRUE if selected cells are adjusted when adding and removing rows
   */
  public boolean getAdjSelCells () { return adjSelCells; }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // Interface Methods
  //   Note that most of these methods simply call a corresponding method from inside the Swing/AWT
  //   event thread. This has two benefits, synchronization and performance improvements (the
  //   redraws can be merged).
  //////////////////////////////////////////////////////////////////////////////////////////////////
  @Override
  public ListFile getListFile () {
    return listFile;
  }

  @Override
  public String getColumnOrder () {
    return ((NmColumnModel)getColumnModel()).getColumnOrder();
  }

  @Override
  public int getSelectionMode () {
    return getSelectionModel().getSelectionMode();
  }

  @Override
  public final void setSelectMode (String mode) {
    int i = Parser.find(selectModeList, mode, getSelectionMode(), -1);
    if (i < 0) {
      throw new MidasException("Illegal selection mode '"+mode+"', expected one of '"+selectModeList+"'.");
    }
    setSelectionMode(i);
  }

  @Override
  public final String getSelectMode () {
    return Parser.get(selectModeList, getSelectionMode(), -1);
  }

  /** poll data
   *  @param poll    force warning if unpollable file
   *  @return number read
   */
  public double pollData (boolean poll) {
    double numRead = 0.0;

    if (listFile == null) {
      /* Do nothing -- this happens when polling in the middle of a file open. */

    } else if (listFile instanceof PollsOwnData) {
      numRead = ((PollsOwnData)listFile).pollData(poll);

    } else if (poll) {
      /* Do not use a popup since this is likely to occur once per second. */
      midas.warning("LIST2: Do not know how to poll '"+listFile.getURL()+"'.");
    }

    return numRead;
  }

  @Override
  public void setListFile (ListFile listFile) {
    setListFile(listFile, false);
  }

  @Override
  public void setListFile (ListFile listFile, boolean readOnly) {
    if (listFile != null) {
      if (!listFile.isOpen()) {
        listFile.open(BaseFile.INPUT);
      }
      if (listFile instanceof DataFile) {
        DataFile dataFile = (DataFile)listFile;

        if (dataFile.isPipe()) {
          listFile = new PipedDataFile(midas,dataFile,this);
          readOnly = true;
        } else {
          listFile = new CachedDataFile(midas,dataFile,this);
        }
      }
      if (!listFile.getProtected() && !readOnly) {
        listFile.open(BaseFile.INOUT);
      }
    }

    this.listFile = listFile;

    // load column definitions
    if (listFile != null) {
      int numCols = listFile.getRecordDefCount();
      columnNames   = new String[numCols];
      columnFormats = new String[numCols];

      for (int i = 0; i < numCols; i++) {
        Table def = listFile.getRecordDef(i);
        columnNames[i]   = def.getS("NAME");
        columnFormats[i] = def.getS("FORMAT");
      }
    }
    else {
      columnNames   = new String[0];
      columnFormats = new String[0];
    }

    // Reload template
    doApplyTemplate(savedTemplate,true,true);

    // Refresh and show row 0.
    getLargeTableModel().fireStructureChanged();
    showCell(0L,0L);
  }

  /** get template
   *  @return Table with template settings
   */
  public Table getTemplate () {
    Table template = new Table();

    if (savedTemplate != null) {
      String[] keys = savedTemplate.getKeys();
      for (String key : keys) {
        template.put(key, savedTemplate.getTable(key).copy());
      }
    }
    return template;
  }

  /** set template
   *  @param template table containing template settings
   */
  public void setTemplate (Table template) {
    doApplyTemplate(template,true,false);
  }

  /** set template
   *  @param table Object that is a table containing template settings
   */
  public void setTemplate (Object table) {
    setTemplate(Convert.o2t(table, this.midas));
  }

  /** update template
   *  @param template table with template information
   */
  public void updateTemplate (Table template) {
    doApplyTemplate(template,false,false);
  }

  /** update template
   *  @param template replacement template object
   */
  public void updateTemplate (Object template) {
    updateTemplate(Convert.o2t(template, this.midas));
  }

  /** get number of rows
   *  @return number of rows
   */
  public long getNumberOfRows () {
    return (listFile!=null)? (long)listFile.getNumberOfRows(): 0L;
  }

  public String[] getColumnNames() {
    String[] names = new String[getColumnCount()];
    for (int i = 0; i < names.length; i++) {
      names[i] = getColumnName(i);
    }
    return names;
  }

  public Table getRow (long row) {
    return (listFile != null)? listFile.getDataTable((double)getRealRowIndex(row)): null;
  }

  public Table getRow (Table row) {
    if (listFile==null) return null;
    long rowNum = getRowNumFrom(row);
    return listFile.getDataTable((double)rowNum);
  }

  public Table getAllRows () {
    Table tab = new Table();
    for (long i = 0; i < getNumberOfRows(); i++) {
      tab.put(""+(getMacroRowIndex(i)), listFile.getDataTable((double)i));
    }
    return tab;
  }

  public final void addRow (Table data) {
    int rowPos = data.getL("LIST__POS", -999);
    if (rowPos != -999) {
      midas.deprecate("LIST2: use addRow(long,Table) instead of LIST__POS "+rowPos);
      addRow(rowPos, data);
    } else {
      addRow(-1L, data);
    }
  }

  public void addRow (long row, Table data) {
    if ((listFile != null) && (data != null)) {
      long rowNum = (row < 0)? getNumberOfRows(): getRealRowIndex(row);
      doAddRow(rowNum, data, true);
    }
  }

  public final void addRows (Table rows) {
    if (rows != null) {
      String[] keys  = rows.getKeys();
      Table[]  array = new Table[keys.length];

      for (int i = 0; i < keys.length; i++) {
        array[i] = rows.getTable(keys[i]);
      }
      addRows(array);
    }
  }

  public final void addRows (Table[] rows) {
    addRows(-1L, rows);
  }

  public void addRows (long row, Table[] rows) {
    if ((listFile != null) && (rows != null) && (rows.length > 0)) {
      long first = -1L;
      long last  = -1L;

      for (Table tbl : rows) {
        if (tbl != null) {
          if (first == -1L) {
            first = last = (row < 0)? getNumberOfRows(): getRealRowIndex(row);
          } else {
            last = last+1L;
          }
          doAddRow(last, tbl, false);
        }
      }
      if (first != -1L) {
        getLargeTableModel().fireRowsInserted(first,last);
      }
    }
  }

  /** Only used by addRow(..) and addRows(..) */
  private void doAddRow(long rowNum, Table data, boolean fireInsert) {
    listFile.insertData((double)rowNum, data);
    if (fireInsert) {
      getLargeTableModel().fireRowsInserted(rowNum, rowNum);
    }
    if (rowNum != -1 && adjSelCells) {
      adjustSelectedCells(rowNum, true);
    }
  }

  /** Gets a list (array) of all the selected cells (including selected rows) and
      then adjust the selected cells up or down, based on whether there is a
      row of data being added or removed below the selected cell.

      @param rowNum The row number of the row being added or removed
      @param add true if adding a row, false if removing a row
      @since NeXtMidas 2.8.0
   */
  private void adjustSelectedCells (long rowNum, boolean add) {
    if (rowNum < 0) return; // nothing to do

    // Since 3.1.0 for BUG 2428
    //*/
    if ( !add ) { getTableSelectionModel().doRemoveRowSelection(rowNum-1, rowNum-1, null, false); }
    int move = add ? 1 : -1;
    for ( RowSelection row : getTableSelectionModel().getCurrentSelection() ) {
      if ( row.getIndex() >= rowNum ) {
        row.setIndex(row.getIndex()+move);
      }
    }
    // End of fix for BUG 2428

    /*/ // prior to 3.1.0
    // get the range of possible selected cells
    RowSelection[] rowSel = getTableSelectionModel().getCurrentSelection();
    if (rowSel.length==0) return;  // nothing is selected
    long rowStart = rowSel[0].getIndex();
    long rowEnd   = rowSel[rowSel.length-1].getIndex();
    int  rows     = (int) (rowEnd-rowStart) +1;
    int  cols     =  getColumnCount();
    int possibles = (rows * cols) ;  // max number of cells that could possibly be selected

    long[] selectedRows = new long[possibles];  // array of row positions
    int[]  selectedCols = new int[possibles];   // array of col positions
    int     counter = 0;                        // index for each array

    // loop through all of the possible selected rows/cells and add to the arrays if selected
    for (long r=rowStart; r<=rowEnd; r++) {
      if (getTableSelectionModel().isRowSelected(r)) {  // add the whole row then continue
        selectedRows[counter] = r;
        selectedCols[counter] = -1;   // -1 means select/deselect the entire row
        counter++;
        continue;
      }
      for (int c=0; c<cols; c++) {
        if (isCellSelected(r, c)) {
          selectedRows[counter] = r;
          selectedCols[counter] = c;
          counter++;
        }
      }
    }

    if (counter > 0) {
      if (add) {   // need to count backwards fro ADDs
        for (int i=counter-1; i>=0; i--) {
          if ( selectedRows[i] < rowNum) continue;
          adjustSelectedCell(rowNum, selectedRows[i], selectedCols[i], add);
        }
      } else {
        for (int i=0; i<counter; i++) {
          if ( selectedRows[i] < rowNum-1) continue;
          adjustSelectedCell(rowNum, selectedRows[i], selectedCols[i], add);
        }
      }
    } //*/ // Prior to 3.1.0
  }

  private void adjustSelectedCell (long changedRow, long selRow, int selCol, boolean add) {
    getTableSelectionModel().changeCellSelection(selRow, selCol, LargeTableSelectionModel.CTRL_CLICK, false);
    if (changedRow == selRow+1) return;  // removed a selected row
    int offset = add? 1: -1;
    getTableSelectionModel().changeCellSelection(selRow+offset, selCol, LargeTableSelectionModel.CTRL_CLICK, false);
  }

  public void removeAllRows () {
    doRemoveRows(getMacroRowIndex(0L), -1L);
  }

  public final void removeRow (long row) {
    removeRows(row,row);
  }

  public void removeRow (Table data) {
    long rowNum = getRowNumFrom(data);

    if (rowNum >=0) {
      listFile.removeData((double)rowNum);
      getLargeTableModel().fireRowsDeleted(rowNum, rowNum);
      if (adjSelCells) {
        adjustSelectedCells(rowNum+1, false);
      }

      if (getTopRowIndex() >= getNumberOfRows()) {
        setTopRowIndex(getTopRowIndex() - (getNumberOfVisibleRows()/2));
      }
    }
  }

  public void removeRows (long start, long end) {
    long count = end - start + 1;

    if (count < 0) {
      throw new MidasException("LIST2: removeRows got invalid start and end row combination: "
                             + "start="+start+" end="+end);
    }
    doRemoveRows(start, count);
  }

  public void removeRows (String rowNumbers) {
    if (rowNumbers != null) {
      Parser parser = new Parser(rowNumbers);
      long[] rows   = new long[parser.elements()];

      for (int i = 0; i < rows.length; i++) {
        rows[i] = Convert.o2x(parser.get(i+1));
      }
      doRemoveRows(rows);
    }
  }

  @Override
  public void removeRows (java.util.List<?> rowNumbers) {
    if ((rowNumbers != null) && (rowNumbers.size() > 0)) {
      long[] rows = new long[rowNumbers.size()];

      for (int i = 0; i < rows.length; i++) {
        rows[i] = Convert.o2x(rowNumbers.get(i));
      }
      doRemoveRows(rows);
    }
  }

  /** Similar to removeRows(..) but uses (start,count) not (start,end). */
  private void doRemoveRows (final long row, final long count) {
    long previousRowCount = getNumberOfRows();
    if (listFile != null) {
      long rowNum = (row < 0)? getNumberOfRows(): getRealRowIndex(row);

      if (count == 0) {
        // Do nothing
      }
      else if (count > 0) {
        listFile.removeData((double)rowNum, count);
        getLargeTableModel().fireRowsDeleted(rowNum, rowNum+count);
        if (adjSelCells) {
          for (long i = count-1; i >= 0; i--) {
            adjustSelectedCells(rowNum+1+i, false);
          }
        }
        else {
          // we are not shifting the selections so we need to unselect rows that no longer exist Since 3.9.1
          getTableSelectionModel().doRemoveRowSelection(getNumberOfRows(), previousRowCount, null, false);
        }
      }
      else {
        listFile.removeData((double)rowNum, -1L);
        getLargeTableModel().fireRowsDeleted(rowNum, -1L);
      }

      if (getTopRowIndex() >= getNumberOfRows()) {
        setTopRowIndex(getTopRowIndex() - (getNumberOfVisibleRows()/2));
      }

    }
  }

  /** Removes a set of rows. */
  private void doRemoveRows (long[] rows) {
    Arrays.sort(rows); // Remove from bottom to top to prevent indexing errors.

    if (listFile != null) {
      long previousRowCount = getNumberOfRows();
      for (int i = rows.length-1; i >= 0; i--) {
        long rowNum = getRealRowIndex(rows[i]);

        listFile.removeData((double)rowNum);
        getLargeTableModel().fireRowsDeleted(rowNum, rowNum);
        if (adjSelCells) adjustSelectedCells(rowNum+1, false);   // BUG 2109

      }
      // Do here so we aren't looping multiple times - it is only necessary to do this step once.
      if(!adjSelCells) {
        // we are not shifting the selections so we need to unselect rows that no longer exist Since 3.9.1
        getTableSelectionModel().doRemoveRowSelection(getNumberOfRows(), previousRowCount, null, false);
      }
    }

    if (getTopRowIndex() >= getNumberOfRows()) {
      setTopRowIndex(getTopRowIndex() - (getNumberOfVisibleRows()/2));
    }
  }

  public final void sortData (String keys) {
    sortData(keys.split("[,|]"));
  }

  public void sortData (String[] keys) {
    sortData(new RecordComparator(keys));
  }

  /** Sorts the data file (this sorts the underlying file, not just the view).
      @param comparator The comparator to use.
   */
  private void sortData (java.util.Comparator<Object> comparator) {
    if (getNumberOfRows() < 65536) {
      Table[] rows = new Table[(int)getNumberOfRows()];
      for (int i = 0; i < rows.length; i++) {
        rows[i] = listFile.getDataTable((double)i);
      }
      Arrays.sort(rows, comparator);
      removeAllRows();
      addRows(rows);
    } else {
      showWarningMsg("Unable to sort data, file is too large.");
    }
  }

  public void redraw (long row) {
    redraw(row,row);
  }

  public void redraw (long startRow, long endRow) {
    long top = getRealRowIndex(startRow);
    long btm = getRealRowIndex(endRow);
    getLargeTableModel().fireNeedsRedraw(top, btm);
  }

  public final void setCell (Table tbl) {
    String col = tbl.getS("COLUMN");

    if (col == null) { col = tbl.getS("COL"); }

    setCell(tbl.getX("ROW"), col, tbl.get("VALUE"));
  }

  public void setCell (long row, String col, Object val) {
    if (listFile != null) {
      long rowNum = getRealRowIndex(row);

      listFile.setData((double)rowNum, col, val);
      getLargeTableModel().fireNeedsRedraw(rowNum, rowNum);
    }
  }

  public void setRow (long row, Table data) {
    doSetRow(getRealRowIndex(row), data);
  }

  public void setRow (Table data) {
    doSetRow(getRowNumFrom(data), data);
  }

  /** Sets a row.
      @param rowNum The internal row number (not Macro row number).
      @param data   The data to set.
   */
  private void doSetRow (long rowNum, Table data) {
    if ((listFile != null) && (data != null)) {
      if ((rowNum < 0) || (rowNum >= getNumberOfRows())) {
        // Workaround for issues with DataFile updating file size after insert.
        doAddRow(rowNum, data, true);
      } else {
        listFile.setData((double)rowNum, data);
        getLargeTableModel().fireNeedsRedraw(rowNum, rowNum);
      }
    }
  }

  public void updateRow (long row, Table data) {
    long  rowNum  = getRealRowIndex(row);
    Table rowData = mergeData(rowNum, data);
    doSetRow(rowNum,rowData);
  }

  public void updateRow (Table data) {
    long  rowNum  = getRowNumFrom(data);
    Table rowData = mergeData(rowNum, data);
    doSetRow(rowNum,rowData);
  }

  public final void updateRows (Table[] rows) {
    for (Table tbl : rows) {
      updateRow(tbl);
    }
  }

  /** Gets the row number from a table of data. This looks up the primary key where applicable. */
  private long getRowNumFrom (Table data) {
    long rowNum = -1;

    if ((listFile == null) || (data == null)) {
      // do nothing
    } else if (primaryKey != null) {
      String keyVal = data.getS(primaryKey);

      if (keyVal == null) {
        throw new MidasException("updateRow(..) called, but primary key "+primaryKey+"= not found in "+data);
      } else {
        Number num = (Number)priKeyCache.get(keyVal);

        if ((num != null) && (num.longValue() >= 0L) && (num.longValue() < getNumberOfRows())) {
          Table oldRow = listFile.getDataTable(num.doubleValue());

          if (keyVal.equals(oldRow.getS(primaryKey))) {
            rowNum = num.longValue();
          } else {
            num = null;
          }
        }

        for (long row = 0; (rowNum == -1) && (row < getNumberOfRows()); row++) {
          Table oldRow = listFile.getDataTable((double)row);

          if (keyVal.equals(oldRow.getS(primaryKey))) {
            rowNum = row;
          }
        }

        if (rowNum == -1) {
          priKeyCache.remove(keyVal);
        } else {
          if (num == null) { num = Long.valueOf(rowNum); }
          priKeyCache.put(keyVal, num);
        }
      }
    } else {
      String row = data.getS("ROW");
      if (row != null) {
        rowNum = getRealRowIndex(Convert.o2x(row));
      } else {
        throw new MidasException("updateRow(..) called, but no primary key specified and no ROW= in "+data);
      }
    }
    return rowNum;
  }

  /** Merges data for a row, keeping existing values where applicable.
      @param rowNum The internal row number (not Macro row number).
      @param data   The data to insert.
      @return The data to be set or null if no values need to be set.
   */
  private Table mergeData (long rowNum, Table data) {
    Table tab;

    if ((listFile == null) || (data == null)) {                // Do nothing
      tab = null;

    } else if ((rowNum < 0) || (rowNum > getNumberOfRows())) { // New row, no merge needed
      tab = data;

    } else {                                                   // Preserve values not updated
      boolean update = false;
      Table   oldRow = listFile.getDataTable((double)rowNum);
      if (oldRow == null) return data; // This must be a new row

      for (String key : oldRow.getKeys()) {
        Object newVal = data.getO(key);

        if (newVal != null) {
          update = update || !Util.equals(newVal, oldRow.getO(key));
          oldRow.put(key, newVal);
        }
      }
      tab = (update)? oldRow : null; // Only update if there are changes
    }
    return tab;
  }

  public void setPrimaryKey (String key) {
    primaryKey  = (key == null)? null: key.toUpperCase();
    priKeyCache = (key == null)? null: new Cache(32);
  }

  public String getPrimaryKey () {
    return primaryKey;
  }

  /** Gets all the selected cells in the table.<br><br>
      Uses the current row selection to shorten the range of cells
      to be checked by {@link LargeJTable#isCellSelected(long, long)}
      @return A table of cells selected in the format {rowID={cellID=value}}
   */
  public Table getAllSelectedCells () {
    Table selCells = new Table();  // the selected cells table
    Table inner    = new Table();  // the cell to value

    // The following variables are used to keep track of the actual data
    // corresponding to the data as it is displayed
    long rowNum;
    long rowIndex;
    String colName;

    Object value;   // the value in the selected cell
    RowSelection[] rowSel = getTableSelectionModel().getCurrentSelection();
    if (rowSel.length==0) return selCells;  // nothing is selected
    long rowStart = rowSel[0].getIndex();
    long rowEnd   = rowSel[rowSel.length-1].getIndex();
    int  cols     =  getColumnCount();

    for (long row = rowStart; row <= rowEnd; row++) {
      for (long col=0; col <= cols; col++) {
        // check for entire row selected first to speed up process and avoid NPE
        if (getTableSelectionModel().isRowSelected(row)) {
          rowNum    = getSortingModel().toReal(row);
          rowIndex  = (row < 0L)? -1L: getMacroRowIndex(rowNum);
          value = getRow(rowIndex);
          selCells.put(Long.toString(rowIndex), value);
        } else if (getTableSelectionModel().isCellSelected(row, col)) {
          //translate the displayed selected cell index to the real one
          rowNum    = getSortingModel().toReal(row);
          rowIndex  = (row < 0L)? -1L: getMacroRowIndex(rowNum);
          colName   = getColID(col);


          value = getRow(rowIndex).get(colName);
          inner.put(colName, value);  // create the inner table to store the col->value
          selCells.put(Long.toString(rowIndex), inner); // add to the selected Cells

          inner    = new Table();  // reset so that it does not add same value twice
        }
      }
    }
    return selCells;
  }

  public Table getAllSelectedRows () {
    RowSelection[] rowSel = getTableSelectionModel().getCurrentSelection();
    Table tbl = new Table();

    for (RowSelection aRowSel : rowSel) {
      long   row   = getMacroRowIndex(getSortingModel().toReal(aRowSel.getIndex()));
      Object value = getRow(row);

      tbl.setKey(Long.toString(row), value);
    }

    return tbl;
  }

  /////// INDEX CONVERSIONS ////////////////////////////////////////////////////////////////////////
  public int  getModelRowIndex(int viewIndex)   { return (int)getSortingModel().toReal(viewIndex); }
  public int  getViewRowIndex(int modelIndex)   { return (int)getSortingModel().toView(modelIndex);}

  // DEPRECATED: The next six lines are useless, except that they are a workaround for
  //             a bug in the Java 1.4.1 compiler which would error out otherwise.
  public long getRealRowIndex(int modelIndex)   { return super.getRealRowIndex(modelIndex);       }
  public int  getModelRowIndex(long realIndex)  { return super.getModelRowIndex(realIndex);       }
  public int  getModelColIndex(int viewIndex)   { return super.getModelColIndex(viewIndex);       }
  public long getRealColIndex(int modelIndex)   { return super.getRealColIndex(modelIndex);       }
  public int  getModelColIndex(long realIndex)  { return super.getModelColIndex(realIndex);       }
  public int  getViewColIndex(int viewIndex)    { return super.getViewColIndex(viewIndex);        }

  public long getMacroRowIndex(long fileIndex)  { return fileIndex + rowOffset;                    }
  public long getRealRowIndex(long macroIndex)  { return macroIndex - rowOffset;                   }
  public long getMacroColIndex(long fileIndex)  { return fileIndex + colOffset;                    }
  public long getRealColIndex(long macroIndex)  { return macroIndex - colOffset;                   }

  long getMacroColIndex(String colName) { // package-private, at least for now
    long index = getColumnIndex(colName);
    return (index == -1)? -1L : getMacroColIndex(index);
  }

  public void showCell(long row, long column) {
    super.showCell(getRealRowIndex(row), column);
  }

  protected String getRowID(long row) {
    String id = null;
    if (row < 0) {
      id = super.getRowID(row);
    } else {
      id = String.valueOf(getMacroRowIndex(getSortingModel().toReal(row)));
    }
    return id;
  }

  /** get index of named column
   *  @param col coulumn name
   */
  long getColumnIndex(String col) {
    int index = -1;

    if ((col != null) && (listFile != null)) {
      col = col.toUpperCase(); // <-- Should be case-insensitive.

      // Check for matching name (default to case-sensitive)
      for (int i = 0; i < columnNames.length; i++) {
        if (col.equals(columnNames[i])) {
          index = i;
          break;
        }
      }

      // Check index value ("1" or "1.0" or "_1_")
      if (index < 0) {
        String num = col;

        if (col.startsWith("_") && col.endsWith("_")) {
          num = col.substring(1, col.length()-1);
        }
        if (StringUtil.isNumber(num)) {
          int val = Convert.o2l(num);

          if ((val >= 0) && (val < columnNames.length)) {
            index = val;
          }
        }
      }
    }
    return index;
  }

  /////// DISPLAY METHODS //////////////////////////////////////////////////////////////////////////
  /** Is grid visible
   *  @return true if grid is visible */
  public boolean isGridVisible()       { return getShowHorizontalLines() && getShowVerticalLines();}

  /** get row label visibility
   *  @return true if row labels are visible
   */
  public boolean getRowLabelsVisible() { return getRowHeader().isVisible(); }

  /** get column label visibility
   *  @return true if column labels are visible
   */
  public boolean getColLabelsVisible() { return getTableHeader().isVisible(); }

  /** set grid visibility
   *  @param showGrid true to shoe grid
   */
  public void setGridVisible(boolean showGrid) { setShowGrid(showGrid); }

  /** set cell foreground color - BROKEN and needs fixing
   *
   * @param row   row
   * @param col   column
   * @param color Color
   */
  public void setCellForeground(int row, int col, Color color) {
Shell.warning("setCellForeground(..)  Fix This >>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
// XXX Fix This
  }

  /////// SELECTION METHODS ////////////////////////////////////////////////////////////////////////
  public final void selectRow (long row)  throws IllegalArgumentException {
    selectRow(row, true);
  }

  public void selectRow (long row, boolean clearCurrentSelection) throws IllegalArgumentException {
    checkRowNumber(row);   // BUG 2137
    long rownum = getRealRowIndex(row);
    if (clearCurrentSelection) getTableSelectionModel().clearSelection(true);
    getTableSelectionModel().changeRowSelection(rownum, rownum, LargeTableSelectionModel.ADD, false);
  }

  public void selectRow (long row, String flags) throws IllegalArgumentException {
    // two lines below are because the macro language does not know boolean from String
    if (flags.equalsIgnoreCase("true"))       { selectRow(row, true); return; }
    else if (flags.equalsIgnoreCase("false")) { selectRow(row, false); return; }

    checkRowNumber(row);    // BUG 2137
    long rownum = getRealRowIndex(row);
    int sel = Parser.mask(selectList,flags, 0);
    boolean msg   = (sel&SEL_MSGS)!=0;
    boolean clear = (sel&SEL_CLEAR)!=0;
    if (clear) getTableSelectionModel().clearSelection(msg);
    getTableSelectionModel().changeRowSelection(rownum, rownum, LargeTableSelectionModel.ADD, msg);
  }

  public void selectRow (String column, String value, boolean ignoreCase) {

    String flags = (ignoreCase) ? "-CLEAR|+IGNORECASE" : "-CLEAR|-IGNORECASE";
    selectRowByColVal(column,value,flags);

    // Changed in 3.1.0 to call the new method
//    if (listFile != null) {
//      long colNo = getColumnIndex(column);
//      for (long row = 0; row < listFile.getNumberOfRows(); row++) {
//
//        // Bug 2467 Changes START
//        boolean sorted = getSortingModel().isSorting();
//        Object data = null;
//        if (sorted) data = getSortingModel().getValueAt(row, colNo) ;
//        else        data = listFile.getDataTable((double)row).getKey(column) ;  // changed in 3.1.0 for Bug 2467
//        // Object data = listFile.getDataTable((double)row).getKey(column);     // prior to Bug 2467
//        // Bug 2467 Changes END
//
//        boolean found = false;
//        if (ignoreCase) { found = ((data!=null) && value.equalsIgnoreCase(data.toString())); }
//        else            { found = value.equals(data); }
//        if (found) {
//          // This method is always zero-based
//          //addRowSelectionInterval(row, row);
//          //selectRow(row,new String("+MSG"));
//          getTableSelectionModel().changeRowSelection(row, row, LargeTableSelectionModel.ADD, false);
//        }
//      }
//    }
  }

  public void selectRowByColVal (String column, String value, String flags) {
    // two lines below because macro language does not know boolean from String
    if (flags.equalsIgnoreCase("true"))       flags = "-CLEAR|+MSG|+IGNORECASE";
    else if (flags.equalsIgnoreCase("false")) flags = "-CLEAR|+MSG|-IGNORECASE";

    if (flags==null || flags.equals("")) flags = selectList; // default
    if (listFile != null) {
      long colNo = getColumnIndex(column);
      int sel = Parser.mask(selectList,flags, 0);
      boolean msg = (sel&SEL_MSGS)!=0;
      boolean clear = (sel&SEL_CLEAR)!=0;
      boolean ignoreCase = (sel&SEL_IGNORE_CASE)!=0 ;  // backwards compatibility

      if (clear) getTableSelectionModel().clearSelection(msg); // first get rid of current selection, if desired

      for (long row = 0; row < listFile.getNumberOfRows(); row++) {

        // Bug 2467 Changes START
        boolean sorted = getSortingModel().isSorting();
        Object data = null;
        if (sorted) data = getSortingModel().getValueAt(row, colNo) ;
        else        data = listFile.getDataTable((double)row).getKey(column) ;  // changed in 3.1.0 for Bug 2467
        // Object data = listFile.getDataTable((double)row).getKey(column);     // prior to Bug 2467
        // Bug 2467 Changes END
        boolean found = false;
        if (ignoreCase) { found = ((data!=null) && value.equalsIgnoreCase(data.toString())); }
        else            { found = value.equals(data); }
        if (found) {
          // This method is always zero-based
          //addRowSelectionInterval(row, row);  // changed to call below to pass through msg value

          getTableSelectionModel().changeRowSelection(row, row, LargeTableSelectionModel.ADD, msg);
        }
      }
    }
  }

  /** Check that the passed in row number is equal to or greater than 0. If it is not, throw
      an illegalArgumentException
      @param row the row number
      @since NeXtMidas 2.8.1
   */
  private void checkRowNumber (long row) {
    if (row < 0L) {
      throw new IllegalArgumentException("Row numbers must be at least 0");
    }
  }

  /////// MESSAGE DISPLAY METHODS //////////////////////////////////////////////////////////////////
  /** Outputs an error message. */
  void showErrorMsg(String msg) {
    if (msgListener != null) {
      msgListener.showErrorMsg(msg);
    } else if (isVisible()) {
      midas.warning("MJList: " + msg);
      JOptionPane.showMessageDialog(this, ("ERROR: "+msg), "MJList", JOptionPane.ERROR_MESSAGE);
    } else {
      midas.warning("MJList: " + msg);
    }
  }

  /** Outputs an warning message. */
  void showWarningMsg(String msg) {
    if (msgListener != null) {
      msgListener.showWarningMsg(msg);
    } else if (isVisible()) {
      midas.warning("MJList:" + msg);
      JOptionPane.showMessageDialog(this, ("WARNING: "+msg), "MJList", JOptionPane.WARNING_MESSAGE);
    } else {
      midas.warning("MJList: " + msg);
    }
  }

  /** Outputs an deprecation message. */
  void showDeprecatedMsg(String msg) {
    if (msgListener != null) {
      msgListener.showDeprecatedMsg(msg);
    } else {
      midas.deprecate("MJList: " + msg);
    }
  }


  /////// DATA MANAGMENT METHODS ///////////////////////////////////////////////////////////////////
  /** Gets the scroll pane being used (or <code>null</code> if none). */
  public JScrollPane getScrollPane() {
    java.awt.Component parent = getParent();

    if ((parent != null) && (parent instanceof JViewport)) {
      parent = parent.getParent();

      if ((parent != null) && (parent instanceof JScrollPane)) {
        return (JScrollPane)parent;
      }
    }
    return null;
  }

  ////////////////// FILE ACCESS METHODS ///////////////////////////////////////////////////////////
  /** Gets the default column width for a given data format. If there are at least one rows in the
      file this will be based on the actual width of the first 2*N rows (where N is the number of
      visible rows). If there are less than one rows in the file then this will be based on
      estimated values.
      @param colName Name of the column (in the file), not null.
      @param fmt     The data format (e.g. "SF"), can be null.
      @param format  The text format to use, can be null.
      @param label   The display label at the top of the column, can be null.
      @return The default column width in characters.
   */
  private int getDefaultColWidth(String colName, String fmt, java.text.Format format, String label) {
    if (label == null) { label = colName; }
    int charWidth = 0;

    int rowsToCheck = (int)Math.min(listFile.getNumberOfRows(), (double)getNumberOfVisibleRows()*2);
    if (rowsToCheck > 0) {
      for (int i = 0; i < rowsToCheck; i++) {
        Object val = listFile.getDataTable((double)i).get(colName);
        String str = (format != null)? format.format(val): val.toString();
        if (str.equals("()")) { // handle case where format was not successful NXM-3948 (4.1.2)
          str = val.toString();
        }
        charWidth  = Math.max(charWidth, str.length());
      }
      charWidth = Math.max(charWidth, 4);

    } else {
      if (fmt == null) {
        charWidth = 20; // Assume String (was 80 in release 2.3.0)

      } else if (Data.isString((byte)fmt.charAt(1))) {
        charWidth = Data.getBPA(fmt);

      } else {
        int scalarWidth;
        int spa = Data.getSPA(fmt.charAt(0));

        switch(fmt.charAt(1)) {
          case DataTypes.DOUBLE: scalarWidth = 14; break;
          case DataTypes.FLOAT:  scalarWidth = 14; break;
          case DataTypes.XLONG:  scalarWidth =  8; break;
          case DataTypes.LONG:   scalarWidth =  6; break;
          case DataTypes.INT:    scalarWidth =  4; break;
          case DataTypes.JINT:   scalarWidth =  4; break;  // 12-bit 'J' support added for NXM-3894
          case DataTypes.BYTE:   scalarWidth =  3; break;
          default:   scalarWidth =  8; break;
        }
        charWidth = ((scalarWidth + 2) * spa) - 2;
      }
    }
    return Math.max(charWidth,label.length());
  }

  /** Gets a default text format to use for a given data format.
      @param fmt      The data format.
      @return An appropriate text format.
   */
  private static String getDefaultFormat(String type, String fmt) {
    for (int i = 0; i < DEFAULT_FORMATS.length; i+=2) {
      if (type.equals(DEFAULT_FORMATS[i])) {
        return DEFAULT_FORMATS[i+1];
      }
    }
    return null; // No matches found
  }

  /** Gets a default type to use for a given data format.
      @param fmt      The data format.
      @param numForm  NUMBER if not null
      @param enumTbl  ENUM if not null
      @return An appropriate type. NUMBER, ENUM, STRING, or TOGGLE.
   */
  private static String getDefaultType(String fmt, String numForm, Table enumTbl) {
    if (numForm != null)    return "NUMBER";
    if (enumTbl != null)    return "ENUM";
    if (fmt     == null)    return "STRING";
    if (fmt.equals("SZ"))   return "TOGGLE";
    if (Data.isString((byte)fmt.charAt(1))) return "STRING";
    if (fmt.charAt(1)=='Z') return "STRING"; // JC4: Not sure what to do for this, use STRING for now
    return "NUMBER";
  }

  /** Gets a default alignment.
      @param fmt  The Midas data format (e.g. "2A", "SB", "VD", etc.).
      @return An appropriate alignment LEFT,RIGHT or CENTER.
   */
  private static String getDefaultAlignment(String fmt) {
    if (fmt == null         ) return "LEFT"; // Assume String
    if (Data.isString((byte)fmt.charAt(1))) return "LEFT";
    if (fmt.charAt(0) == Data.SCALAR) return "RIGHT";
    if (fmt.charAt(0) == '1') return "RIGHT";
    return "CENTER";
  }

  /** Updates the column definitions. */
  private Table getDefaultColumnTemplate(int col) {
    Table template = new Table();

    if (listFile != null) {
      /* Load the column information from the file. */
      String  name   = columnNames[col];
      String  fmt    = columnFormats[col];
      String  type   = getDefaultType(fmt,null,null);
      String  format = getDefaultFormat(type,fmt);
      MFormat mfmt   = getDefaultMFormat(type,format,fmt);
      int     width  = getDefaultColWidth(name,fmt,mfmt,null);
      String  align  = getDefaultAlignment(fmt);

      if (width   >    0) { template.put("WIDTH", width ); }
      if (align  != null) { template.put("ALIGN", align ); }
      if (type   != null) { template.put("TYPE",  type  ); }
      if (format != null) { template.put("FORMAT",format); }
    }
    return template;
  }

  private MFormat getDefaultMFormat(String type, String format, String fmt) {
    MFormat mfmt = null;

         if (type.equals("STRING")) mfmt = MFormat.getStringFormat();
    else if (type.equals("NUMBER")) mfmt = MFormat.getNumberFormatFor(format,fmt,true);
    else if (type.equals("DATE"  )) mfmt = MFormat.getDateFormatFor(format);
    else if (type.equals("ANGLE" )) mfmt = MFormat.getAngleFormatFor(format);

    return mfmt;
  }

  /** Adds a column, ready for use.
      @param col The column number.
   */
  private void addColumn(int col) {
    TableCellRenderer tcr = new NmTableCellRenderer();
    TableCellEditor   tce = new DefaultCellEditor(new JTextField()); // XXX Fix This

    int width = 8 * getTextWidth();
    addColumn(new TableColumn(col, width, tcr, tce));
  }

  /** Removes all current columns. */
  private void removeAllColumns() {
    ((NmColumnModel)getColumnModel()).removeAllColumns();
  }

  /** Applies a template to the file. This should only be called from applyTemplate(Table,boolean).
      @param template The template.
      @param override Indicates if overriding old settings.
   */
  private void doApplyTemplate(Table template, boolean override, boolean update) {
    if ((savedTemplate == null) || override) {
      if (template != null) {
        savedTemplate = template.copy();
      } else {
        savedTemplate = new Table();
      }
    } else {
      String[] keys = template.getKeys();
      for (String key : keys) {
        Table oldEntry = savedTemplate.getTable(key);
        Table newEntry = template.getTable(key).copy();
        if (oldEntry == null) {
          oldEntry = new Table();
          savedTemplate.put(key, oldEntry);
        }
        oldEntry.merge(newEntry);
      }
    }

    if (override) {
      mcolorTbl.clear();
      removeAllColumns();
    }

    if (listFile != null) {
      Table   globalTempl = getPutT(savedTemplate, "_GLOBAL_", "{TYPE=LIST2,AUTOHIDE=TRUE,AUTOORDER=TRUE}");
      String  type        = getPutS(globalTempl,"TYPE","LIST2");
      int     templCols   = savedTemplate.size() - 1; // Columns in template

      if (type.equals("LIST2")) {
        applyGlobalTemplate(globalTempl, override);
      } else if (type.equals("XDATALIST")) {
        applyXmGlobalTemplate(globalTempl, override);
      } else {
        showErrorMsg("Invalid template type '"+type+"'.");
      }
      boolean ahide  = autoHide  && (templCols != 0);
      boolean aorder = autoOrder && (templCols != 0) && override && (template != null) && !update;

      for (int col = 0; col < columnNames.length; col++) {
        String colName  = columnNames[col];
        String colNum   = "_" + col + "_";
        Table  templ    = null;

        if (templ == null) { templ = savedTemplate.getTable(colName); }
        if (templ == null) { templ = savedTemplate.getTable(colNum ); }

        if (!ahide && (templ == null)) { // Not hidden, no template... create a default one.
          templ = getPutT(savedTemplate, colName, getDefaultColumnTemplate(col));
        }


        if (templ == null) {
          // Hidden
        } else {
          if (override) { addColumn(col); }

          if (type.equals("LIST2")) {
            applyColumnTemplate(col, templ, override, ahide);
          } else if (type.equals("XDATALIST")) {
            applyXmColumnTemplate(col, templ, override, ahide);
          } else {
            showWarningMsg("Unknown template TYPE='"+type+"', attempting to use it as TYPE=LIST2.");
            applyColumnTemplate(col, templ, override, ahide);
          }
        }
      }

      // Reorder columns to match the order in the template.
      if (aorder) {
        String[]      colNames = savedTemplate.getKeys();
        NmColumnModel colModel = (NmColumnModel)getColumnModel();

        for (int i = 0, col = 0; i < colNames.length; i++, col++) {
          int index = colModel.findColumnIndex(colNames[i]);

          if (index >= 0) { moveColumn(index, col); } // Move the column
          else            { col--;                  } // Don't count junk columns
        }
      }
      // Redraw everything
      resizeAndRepaint();
    }
  }

  private String getPutS(Table tbl, String key, String def) {
    String val = def;
    if (tbl != null) {
      val = tbl.getS(key,def);
      if ((val == null) && (def != null)) {
        val = def;
        tbl.put(key,def);
      }
    }
    return val;
  }

  private Table getPutT(Table tbl, String key, String def) {
    return getPutT(tbl, key, new Table(def));
  }

  private Table getPutT(Table tbl, String key, Table def) {
    Table val = def;
    if (tbl != null) {
      val = tbl.getTable(key);
      if ((val == null) && (def != null))  {
        val = def;
        tbl.put(key,def);
      }
    }
    return val;
  }

  /** This should only be called from applyTemplate(Table). */
  private void applyXmGlobalTemplate(Table template, boolean override) {
    Table tbl = template.copy();

    tbl.setKey("TYPE", "LIST2");
    String indexStr = tbl.getS("INDEX", "RIGHT").toUpperCase();
    boolean rows = indexStr.equals("RIGHT") || indexStr.equals("LEFT");
    boolean cols = tbl.getState("LABELS", true);

         if (rows && cols) { tbl.setKey("LABELS", "BOTH"); }
    else if (rows)         { tbl.setKey("LABELS", "ROW" ); }
    else if (cols)         { tbl.setKey("LABELS", "COL" ); }
    else                   { tbl.setKey("LABELS", "NONE"); }

    if (template.containsKey("INDEX")) {
      tbl.remove("INDEX");
    }

    if (tbl.containsKey("COLOR")) {
      tbl.setKey("FGCOLOR", convertColor(tbl.getS("COLOR")));
      tbl.remove("COLOR");
    }

    applyGlobalTemplate(tbl, override);
  }

  /** This should only be called from applyTemplate(Table,boolean). */
  private void applyXmColumnTemplate(int column, Table template, boolean override, boolean autoHide) {
    Table tbl = new Table(template.toString());

    if (tbl.containsKey("COLOR")) {
      tbl.setKey("FGCOLOR", convertColor(tbl.getS("COLOR")));
      tbl.remove("COLOR");
    }

    if (tbl.containsKey("COLUMNTYPE")) {
      String type = tbl.getS("COLUMNTYPE");
      tbl.setKey("TYPE", type);
      tbl.remove("COLUMNTYPE");

      if (type.equals("TIME")) {
        String timeFormat = tbl.getS("TIMEFMT", "STD");

        if (StringUtil.isNumber(timeFormat)) {
          timeFormat = Parser.get(Time.formatList, Convert.o2l(timeFormat));
        }

        tbl.setKey("TYPE",   "NUMBER");
        tbl.setKey("FORMAT", timeFormat);
        tbl.remove("TIMEFMT");
      }

    } else if (tbl.containsKey("DISPFORM")) {
      tbl.setKey("TYPE",   "NUMBER");
      tbl.setKey("FORMAT", tbl.getS("DISPFORM"));
      tbl.remove("DISPFORM");

    } else if (tbl.containsKey("TIMEFMT")) {
      tbl.setKey("TYPE",   "NUMBER");
      tbl.setKey("FORMAT", tbl.getS("TIMEFMT", "STD"));
      tbl.remove("TIMEFMT");
    }

    if (tbl.containsKey("MENU")) {
      String menu      = tbl.getS("MENU");
      Table  enumTable = new Table();
      int    enumCount = (new Parser(menu)).elements();
      String colors    = tbl.getS("MCOLORS", null);
      String values    = tbl.getS("MVALUES", null);

      tbl.remove("MENU");
      //tbl.remove("MCOLORS");
      tbl.remove("MVALUES");

      for (int i = 1; i <= enumCount; i++) {
        String name = "E"+i;
        enumTable.addTable(name);
        if (menu   != null) { enumTable.put(name + ".LABEL", Parser.get(menu,   i)); }
        if (colors != null) { enumTable.put(name + ".COLOR", Parser.get(colors, i)); }
        if (values != null) { enumTable.put(name + ".VALUE", Parser.get(values, i)); }
      }

      tbl.setKey("TYPE", "ENUM");
      tbl.setKey("ENUM", enumTable);
    }

    if (tbl.getS("TYPE","").equals("NUMBER")) {
      String format = tbl.getS("FORMAT","");

      if (format.startsWith("(") && format.endsWith(")")) {
        tbl.setKey("FORMAT", MFormat.convertFortranPattern(format));
      }
    }

    applyColumnTemplate(column, tbl, override, autoHide);
  }

  private String convertColor(String color) {
    if (StringUtil.isNumber(color)) {
      color = MColor.toString(MColor.getColorByIndex(Convert.o2l(color)));
    }
    return color;
  }

  /** This should only be called from applyTemplate(Table). */
  private void applyGlobalTemplate(Table template, boolean override) {
    String labels     = template.getS("LABELS");
    String grid       = template.getS("GRID");
    String fgColor    = template.getS("FGCOLOR");
    String bgColor    = template.getS("BGCOLOR");
    String colorMap   = template.getS("COLORMAP");
    int    numColors  = -1; // Number of colormap colors (may add this to template in the future).
    String shadeColor = template.getS("SHADECOLOR");
    String allowSort  = template.getS("SORTENABLED");
    String sort       = template.getS("SORT");
    String colDrag    = template.getS("COLDRAG");
    String autoHide   = template.getS("AUTOHIDE");
    String autoOrder  = template.getS("AUTOORDER");
    int    shadeInt   = template.getL("SHADEINT",  -1);
    float  fontSize   = template.getF("FONTSIZE",  -1f);
    String fontStyle  = template.getS("FONTSTYLE", null);
    String rowHtStr   = template.getS("ROWHEIGHT", null);

    String adjSelCellsStr = template.getS("ADJSELCELLS", "OFF");  // JWB added for BUG 2109
    int    rowHeight  = -1;

    if (override) {
      if (labels         == null) { labels         = "BOTH";       }
      if (grid           == null) { grid           = "TRUE";       }
      if (fgColor        == null) { fgColor        = defFgColor;   }
      if (bgColor        == null) { bgColor        = defBgColor;   }
      if (allowSort      == null) { allowSort      = "TRUE";       }
      if (sort           == null) { sort           = "";           }
      if (colDrag        == null) { colDrag        = "TRUE";       }
      if (autoHide       == null) { autoHide       = "TRUE";       }
      if (autoOrder      == null) { autoOrder      = "TRUE";       }
      if (shadeColor     == null) { shadeColor     = defBgColor;   }
      if (rowHtStr       == null) { rowHtStr       = "-1";         }
    }

    if (rowHtStr != null) {
      rowHeight = Convert.s2l(rowHtStr);
      if ((rowHeight != -1) && (rowHeight < 0)) {
        showWarningMsg("Invalid global template entry, ROWHEIGHT must be\n"
                     + "greater than 0, ROWHEIGHT=-1 to use default height.");
        rowHtStr  = FORCE_UPDATE;
        rowHeight = -1;
      }
    }

    if ((allowSort == null) && (sort != null)) { allowSort = "TRUE"; }

    if (labels != null) {
      labels = labels.toUpperCase();
           if (labels.equals("ROW"))  { setRowLabelsVisible(true);  setColLabelsVisible(false); }
      else if (labels.equals("COL"))  { setRowLabelsVisible(false); setColLabelsVisible(true);  }
      else if (labels.equals("BOTH")) { setRowLabelsVisible(true);  setColLabelsVisible(true);  }
      else if (labels.equals("NONE")) { setRowLabelsVisible(false); setColLabelsVisible(false); }
      else { showWarningMsg("Invalid global template entry 'LABELS=" + labels + "'.\n"
                          + "Possible values for LABELS are 'ROW', 'COL', 'BOTH' or 'NONE'.");   }
    }

    if (allowSort  != null) { setSortingEnabled(StringUtil.isTrue(allowSort)); }
    if (grid       != null) { setGridVisible(StringUtil.isTrue(grid)); }
    if (sort       != null) { setSort(sort);  }
    if (fgColor    != null) { globalFgColor   = fgColor;   }
    if (bgColor    != null) { globalBgColor   = bgColor;   }
    if (rowHtStr   != null) { globalRowHeight = rowHeight; }
    if (colDrag    != null) { this.colDrag    = StringUtil.isTrue(colDrag);  }
    if (autoHide   != null) { this.autoHide   = StringUtil.isTrue(autoHide); }
    if (autoOrder  != null) { this.autoOrder  = StringUtil.isTrue(autoOrder);}
    if (shadeColor != null) { this.shadeColor = MColor.getColor(shadeColor); }

    this.adjSelCells = adjSelCellsStr.equalsIgnoreCase("ON")? true: false;  // BUG 2109

    if (shadeInt >= 0) {
      if ((shadeInt == 0) || (shadeInt > 1)) {
        this.shadeInterval = shadeInt;
      } else {
        showWarningMsg("Invalid global template entry, SHADEINT must be\n"
                     + "at least 2 use SHADEINT=0 to turn shading off.");
      }
    }

    if (fontSize > 0) {
      Font newFont = getFont().deriveFont(fontSize);

      int oldHeight = getFontMetrics(getFont()).getHeight();
      int newHeight = getFontMetrics(newFont).getHeight();
      defRowHeight  = defRowHeight - oldHeight + newHeight;
      rowHtStr      = FORCE_UPDATE;
      setFont(newFont);
    }

    if (fontStyle != null) {
      int  oldMask = getFont().getStyle();
      int  newMask = Parser.mask(Line.fontStyleList,fontStyle,oldMask);
      Font newFont = getFont().deriveFont(newMask);
      setFont(newFont);
    }

    if (rowHtStr != null) { // Setting row height must come after setting of font size
      if (rowHeight == -1) rowHeight = defRowHeight;
      setRowHeight(rowHeight);
    }

    setColorMap("_GLOBAL_", colorMap, numColors);
  }

  /** This should only be called from applyTemplate(Table,boolean). */
  private void applyColumnTemplate(int column, Table template, boolean override, boolean autoHide) {
    // Note that HIDE is handled in applyTemplate0(..)
    String  colName    = columnNames[column];
    String  colFormat  = columnFormats[column];
    String  label      = template.getS("LABEL");
    int     width      = template.getL("WIDTH", -1);
    String  fgColor    = template.getS("FGCOLOR", globalFgColor);
    String  bgColor    = template.getS("BGCOLOR", globalBgColor);
    String  colorMap   = template.getS("COLORMAP");
    int     numColors  = -1; // Number of colormap colors (may add this to template in the future).
    String  renderer   = template.getS("RENDERER");
    Table   renderOpts = template.getTable("RENDEROPTS");
    Table   enumTbl    = template.getTable("ENUM");
    String  defFormat  = template.getS("FORMAT", null);
    String  defType    = template.getS("TYPE",   null);
    String  timeZone   = template.getS("TIMEZONE","UTC");
    String  type       = getPutS(template, "TYPE",   getDefaultType(colFormat,defFormat,enumTbl));
    String  format     = getPutS(template, "FORMAT", getDefaultFormat(type,colFormat));
    String  align      = getPutS(template, "ALIGN",  getDefaultAlignment(colFormat));
    String  mcolors    = template.getS("MCOLORS");
    boolean hide       = template.getState("HIDE",false);

    NmColumnModel     colModel     = (NmColumnModel)getColumnModel();
    TableColumn       tableColumn  = colModel.findColumn(column);
    if (tableColumn==null) return;    // column does not exist in the model
    TableCellRenderer cellRenderer = tableColumn.getCellRenderer();
    java.text.Format  mformat      = null;


    if ((renderer != null) && (renderer.length() == 0)) {
      renderer = null;
    }

    // Set cell renderer/editor
    if (renderer != null) {
      cellRenderer = createRenderer(renderer,renderOpts);

    } else if (type != null) {
      type = type.toUpperCase();

      if (type.equals("STRING")) {
        mformat = MFormat.getStringFormat();

      } else if (type.equals("NUMBER")) {
        mformat = MFormat.getNumberFormatFor(format,colFormat,true);

      } else if (type.equals("DATE")) {
        mformat = MFormat.getDateFormatFor(format,timeZone);

      } else if (type.equals("ANGLE")) {
        mformat = MFormat.getAngleFormatFor(format);

      } else if (type.equals("TOGGLE")) {
        cellRenderer = new NmCheckboxRenderer();

      } else if (type.equals("ENUM")) {
        if (enumTbl != null) {
          mformat = MFormat.getEnumFormat(enumTbl);
        } else {
          showWarningMsg("Invalid column template entry for column "+colName+", given 'TYPE=" + type + "'\n"
                       + "but no ENUM table was found.");
        }
      } else {
        showWarningMsg("Invalid column template entry for column "+colName+" 'TYPE=" + type + "'.");
      }
    }

    if ((mformat != null) && (renderer == null)) {
      if (cellRenderer instanceof NmTableCellRenderer) {
        ((NmTableCellRenderer)cellRenderer).setFormat(mformat);
      } else {
        cellRenderer = new NmTableCellRenderer(mformat);
      }
    } else if (cellRenderer == null) {
      cellRenderer = new NmTableCellRenderer();
    }

    // Set column width
    int prefWidth = width;
    int defWidth  = width;
    if (width < 0) {
      prefWidth = (colModel.findColumn(column).getPreferredWidth() / getTextWidth()) - 2;
      defWidth  = getDefaultColWidth(colName,format,mformat,label);
    }
    int pixWidth  = (Math.max(defWidth, prefWidth)+2) * getTextWidth(); // in pixels
    //template.put("WIDTH", defWidth);   // no need to save defaults (BUG 1380)
    tableColumn.setPreferredWidth(pixWidth);

    // Set column title
    if (label != null) { tableColumn.setHeaderValue(label); }

    // Set colors
    setColorMap(colName, colorMap, numColors);
    if (mcolors != null) {
      Parser  parser  = new Parser(mcolors);
      Color[] colors  = new Color[parser.elements()+1];

      colors[0] = null; // Element 0 in the color map is special use (null=UseDefaultFgColor).
      for (int i = 1; i < colors.length; i++) {
        colors[i] = MColor.getColor(parser.get(i));
      }
      mcolorTbl.put(colName, colors);
    }

    if (cellRenderer instanceof Component) {
      Component comp = (Component)cellRenderer;

      setColumnColor(comp, fgColor, false);
      setColumnColor(comp, bgColor, true);

      if ((align != null) && (comp instanceof JLabel)) {
        JLabel jlabel = (JLabel)comp;
        String algn   = align.toUpperCase();

             if (algn.equals("LEFT"  )) { jlabel.setHorizontalAlignment(SwingConstants.LEFT);   }
        else if (algn.equals("RIGHT" )) { jlabel.setHorizontalAlignment(SwingConstants.RIGHT);  }
        else if (algn.equals("CENTER")) { jlabel.setHorizontalAlignment(SwingConstants.CENTER); }
        else { showWarningMsg("Invalid column template entry 'ALIGN=" + align + "' for column "+colName+".\n"
                          + "Possible values for ALIGN are 'LEFT,RIGHT,CENTER'."); }
      }
    }
    tableColumn.setCellRenderer(cellRenderer);
    colModel.setColumnVisible(column,!hide);
  }

  /** Creates a new cell renderer.
      @param className The class name.
      @param options   Any options to set.
      @return The renderer or null if none is available.
   */
  private TableCellRenderer createRenderer (String className, Table options) {
    TableCellRenderer cellRenderer = null;
    try {
      Class<?> clazz = Class.forName(className);
      Constructor<?>[] consts = clazz.getConstructors();
      Constructor<?> constMidas = null;
      Constructor<?> constEmpty = null;

      // Find and call the best constructor, in preference this is:
      //   1. MyClass(Midas)
      //   2. MyClass()
      //
      for (Constructor<?> constructor : consts) {
        Class<?>[] types = constructor.getParameterTypes();
        if (types.length == 0) {
          constEmpty = constructor;
        } else if ((types.length == 1) && (types[0].equals(Midas.class))) {
          constMidas = constructor;
        }
      }

      if (constMidas != null) {
        cellRenderer = (TableCellRenderer)constMidas.newInstance(new Object[] { midas });
      } else if (constEmpty != null) {
        cellRenderer = (TableCellRenderer)constEmpty.newInstance((Object[])null);
      } else {
        midas.warning("MJList: could not find constructor "+className+"() or "+className+"(Midas).");
        cellRenderer = null;
      }
    } catch (Exception e) {
      midas.printStackTrace("MJList: Error instantiating cell renderer "+className,e);
      cellRenderer = null;
    }

    if (cellRenderer != null) {
      KeyObject.setKeys(cellRenderer, options, midas, KeyObject.SOFTWARN);
    }
    return cellRenderer;
  }

  /** Sets the column color. Only used by applyColumnTemplate(..). */
  private void setColumnColor(Component comp, String color, boolean background) {
    if (color == null) return; // Nothing to do.

    NmTableCellRenderer renderer    = null;
    String              overrideCol = null;

    if (comp instanceof NmTableCellRenderer) {
      renderer = (NmTableCellRenderer)comp;
    }

    if (color.startsWith("FIELD_")) {          // Use named column (NeXtMidas 2.3.0+)
      overrideCol = color.substring(6);
    } else if (color.startsWith("FIELD")) {    // Used numbered column (XDATALIST)
      int colNum  = Convert.o2l(color.substring(5));
      overrideCol = columnNames[colNum-1];
    }

    if (overrideCol == null) {
      if (background) { comp.setBackground(getColor(color,null)); }
      else            { comp.setForeground(getColor(color,null)); }
    } else if (renderer != null) {
      if (background) { renderer.setBgOverrideCol(overrideCol); }
      else            { renderer.setFgOverrideCol(overrideCol); }
    } else {
      showWarningMsg("Can not set color to "+color+" renderer does not extend NmTableCellRenderer.");
    }
  }

  /** Sets the color map for a column.
      @param col       The column name (or "_GLOBAL_").
      @param colorMap  The color map.
      @param numColors The number of colors to use in a named color map.
   */
  private void setColorMap(String col, String colorMap, int numColors) {
    if ((colorMap != null) && (colorMap.length() > 0)) {
      if (colorMap.equalsIgnoreCase("DEFAULT")) {
        if (mcolorTbl.containsKey(col)) { mcolorTbl.remove(col); }
      } else if (colorMap.startsWith("{")) {
        Table tbl = Convert.o2t(colorMap);
        Table eqs = new Table();
        String[] keys = tbl.getKeys();
        for (String key : keys) {
          String val = tbl.getS(key);
          Object value = null;

          if (val.startsWith("FIELD_")) {
            value = val;
          } else {
            value = MColor.getColor(val);
          }

          if (key.startsWith("TEST(")) {
            tbl.remove(key);
            eqs.put(key,value);
          } else {
            tbl.put(key, value);
          }
        }
        if (eqs.size() > 0) {
          tbl.put("_EQUATIONSET_", new EquationSet(eqs));
        }
        mcolorTbl.put(col, tbl);
      } else {
        mcolorTbl.put(col, MColor.getColorMap(colorMap, numColors));
      }
    }
  }

  private static final class EquationSet {
    private final Midas    midas;
    private final String[] equations;
    private final Object[] values;

    public EquationSet (Table tbl) {
      midas     = new Midas();
      equations = tbl.getKeys();
      values    = new Object[equations.length];

      for (int i = 0; i < equations.length; i++) {
        values[i] = tbl.get(equations[i]);
      }
    }

    /** Tests the equation set with the given value.
        @param x The value to test with.
        @return The value matching the first successful test or null if none pass.
     */
    public Object test(Object x) {
      midas.getResults().put("X",x);
      Object val = null;

      for (int i = 0; (i < equations.length) && (val == null); i++) {
        if (Convert.inLineTest(midas,equations[i])) {
          val = values[i];
        }
      }
      return val;
    }
  }

  /** Gets a color from a given colormap.
      @param cmap    The color map (Color[] or Table or null).
      @param value   The value to lookup in the color map.
      @return The color or null if not found in the colormap.
   */
  private static Color getColorInColormap(Object cmap, String value) {
    Color color = null;

    if (cmap == null) {
      // skip

    } else if ((cmap instanceof Color[]) && StringUtil.isNumber(value)) {
      Color[] map   = (Color[])cmap;
      int     index = Math.abs(Convert.s2l(value));

      color = map[index % (map.length-1) + 1]; // Note: skip cmap[0]

    } else if (cmap instanceof Table) {
      Table tbl = (Table)cmap;
      Object clr = tbl.get(value);

      if (clr instanceof Color) {
        color = (Color)clr;
      } else if (clr != null) {
        color = MColor.getColor(clr.toString());
        tbl.put(value, color);
      }
    } else {
      Shell.warning("MJList: Internal Error: Unknown colormap type: "+cmap.getClass());
    }

    return color;
  }

  /** Gets the color for a given row/column.
      @param row  The row to get the color for (view-based-index).
      @param col  The column (view-based-index) containing the color value
      @return The color (or null if not found).
   */
  private Color getColor(int row, String col) {
    long   column = getColumnIndex(col);
    Color  color  = null;
    String val    = null;

    if ((row >= 0) && (row < getNumberOfRows())) {
      if (column >= 0) {
        long   rowNum = getRealRowIndex(row);
        Object obj    = getLargeTableModel().getValueAt(rowNum, column);
        val = Convert.o2s(obj);
      }
      else {
        Shell.warning("Column name "+col+" not found in MJList with column names "+Arrays.toString(getColumnNames())+
            "\n      Using direct calls to openFile(), rather than the OPENFILE message, can create a timing conflict"
            + " with the AWT thread and trigger this warning.");
      }
    } else {
      Shell.warning("Invalid row number, row="+row);
    }

    if (val != null) {
      Object mc = mcolorTbl.get(col);

      if (mc == null) { mc = mcolorTbl.get("_GLOBAL_"); }

      if (mc != null) {
        if (mc instanceof Table) {
          Table       ctbl  = (Table)mc;
          Object      clr   = null;
          EquationSet eqSet = (EquationSet)ctbl.get("_EQUATIONSET_");

          // Order is important, as per the LIST2 explain file the order is...
          //   (1) Any equations
          //   (2) Exact matches
          //   (3) _DEFAULT_
          if (eqSet != null) clr = eqSet.test(val);
          if (clr   == null) clr = ctbl.getO(val);
          if (clr   == null) clr = ctbl.getO("_DEFAULT_");

          if ((clr instanceof Color) || (clr == null)) {
            color = (Color)clr;
          } else if (clr.toString().startsWith("FIELD_")) {
            color = getColor(row, clr.toString().substring(6));
          }
        } else if (StringUtil.isNumber(val) && (mc instanceof Color[])) {
          Color[] cmap = (Color[])mc;
          int index = Math.min(cmap.length-1, Math.max(0, Convert.s2l(val)));
          color = cmap[index]; // Assume an X-Midas color.
        }
      } else if (StringUtil.isNumber(val) && !val.startsWith("#") && !val.startsWith("0x")) {
        color = getColor(val,null); // Assume an X-Midas color.

      } else {
        color = MColor.getColor(val);
      }
    }
    return color;
  }

  /** Gets the named color. This checks for color map index values too.
      @param cname  The color name.
      @param field  The field name (null=UseDefaultColorMap).
      @return The color.
   */
  private Color getColor(String cname, String field) {
    Color color = null;

    if (StringUtil.isNumber(cname) && !cname.startsWith("#") && !StringUtil.isHexInteger(cname)) {
      int    index = Convert.s2l(cname);
      Object cmap  = (field==null)? null: mcolorTbl.get(field);

      if (cmap == null) { cmap = mcolorTbl.get("_GLOBAL_"); }

      if (cmap != null) {
        color = getColorInColormap(cmap, cname);
      } else {
        color = MColor.getColorByIndex(index);     // Assume an indexed color.
      }
    }

    if (color == null) {
      color = MColor.getColor(cname);
    }

    return color;
  }

  /** Gets the background/shade color to use for a given row.
      @param row   The row number (model index).
      @param color The normal background color for cell in that row.
      @return The background color to use.
   */
  public Color getShadeColor(int row, Color color) {
    if ((shadeInterval <= 0) || (shadeColor == null)) {
      /* No Shading, do nothing. */
    } else {
      final int   index   = getViewRowIndex(row);
      final Color bgColor = getBackground();

      if ((index % shadeInterval) == (shadeInterval - 1)) {
        if ((color == null) || color.equals(bgColor)) {
          color = shadeColor;
        } else {
          /* Compute the shade color. */
          color = new Color(shadeColor.getRGB() ^ bgColor.getRGB() ^ color.getRGB());
        }
      } else {
        /* No Shading on this row. */
      }
    }

    return color;
  }

  public void setSort (String column) {
    getSortingModel().setSortingStatus(column);
  }

  /** Get sorting status
   *  @return sorting status
   */
  public String getSort() {
    return getSortingModel().getSortingStatus();
  }

  /** enables of disables sorting
   *  @param enabled true to enable
   */
  public void setSortingEnabled(boolean enabled) {
    getSortingModel().setSortingEnabled(enabled);
  }

  /** check if sorting is enabled
   *  @return true if sorting is enabled
   */
  public boolean getSortingEnabled() {
    return getSortingModel().getSortingEnabled();
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  //  The ListMessageListener
  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** Defines an interface that can be used as an alternate message destination. */
  public static interface ListMessageListener {
    /** Outputs an error message.
        @param msg The message.
     */
    public void showErrorMsg(String msg);

    /** Outputs an warning message.
        @param msg The message.
     */
    public void showWarningMsg(String msg);

    /** Outputs an deprecation message.
        @param msg The message.
     */
    public void showDeprecatedMsg(String msg);
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  //  The TableModel
  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** NeXtMidas Table Model */
  private static class NmTableModel extends AbstractTableModel implements LargeTableModel {
    /** NOTE: this should be changed if the structure/fields of this class changes. */
    private static final long serialVersionUID = 2012021040311L; // for Serializable interface

    private final MJList table;  // The table.

    /** Creates a new instance.
        @param table The table.
     */
    private NmTableModel(MJList table) {
      this.table = table;
    }

    /** Gets the file in use. */
    private ListFile getListFile() {
      return table.listFile;
    }

    public int getRowCount() {
      int rowCount = 0;

      if (getListFile() != null) {
        /* If at the end of the file, the next line will give a number smaller than page size. */
        long lastPage = ((long)getListFile().getNumberOfRows() - table.getTopRowIndex());

        rowCount = (int)Math.min((long)table.getNumberOfVisibleRows(), lastPage);
      }

      return rowCount;
    }

    public int getColumnCount() {
      return (getListFile()!=null)? getListFile().getRecordDefCount(): 0;
    }

    public String getColumnName(int col) {
      return table.columnNames[col];
    }

    public Class<?> getColumnClass(int col) {
      Class<?> clazz = Object.class;
      String fmt = table.columnFormats[col];

           if (Data.isString((byte)fmt.charAt(1))) { clazz = String.class;   }
      else if (fmt.charAt(0) == Data.SCALAR) { clazz = Number.class;   }
      else if (fmt.charAt(0) == '1') { clazz = Number.class;   }
      else                           { clazz = Number[].class; }

      return clazz;
    }

    public boolean isCellEditable(int rowIndex, int columnIndex) {
      return isCellEditable(table.getRealRowIndex(rowIndex), table.getRealColIndex(columnIndex));
    }

    public Object getValueAt(int rowIndex, int columnIndex) {
      return getValueAt(table.getRealRowIndex(rowIndex), table.getRealColIndex(columnIndex));
    }

    public void setValueAt(Object val, int rowIndex, int columnIndex) {
      setValueAt(val, table.getRealRowIndex(rowIndex), table.getRealColIndex(columnIndex));
    }

    public boolean isCellEditable(long row, long col) {
      /* Allow edits if the file is OUTPUT or I/O and the file is not write protected. */
      return (!getListFile().getProtected()) && table.editable; //
    }

    public Object getValueAt(long row, long col) {
      Object value = null;

      if (getListFile() != null) {
        String name = table.columnNames[(int)col];
        Table  tbl  = getListFile().getDataTable((double)row);

        if (tbl != null) {
          value = tbl.getKey(name);
        }
      }

      return value;
    }

    public void setValueAt(Object val, long row, long col) {
      if (getListFile() != null) {
        String name = table.columnNames[(int)col];

        if (isCellEditable(row, col)) {
          getListFile().setData((double)row, name, val);

        } else {
          table.showErrorMsg("Can not edit cell at ("+row+","+col+"), cell is not editable.");
        }
      }
    }

    public void fireNeedsRedraw(long top, long btm) {
      int[] rows = table.getModelRowsVisible(top, btm);

      if (rows != null) {
        fireTableRowsUpdated(rows[0], rows[1]);
        table.getRowHeader().getRowHeaderModel().fireTableRowsUpdated(rows[0], rows[1]);
      }
    }

    public void fireRowsInserted(long top, long btm) {
      long[] range = table.getRowsVisible(top, -1L);

      if (range != null) {
        int[] rows = table.getModelRowsVisible(range[0], range[0]+(btm-top));

        fireTableRowsInserted(rows[0], rows[1]);
        table.getRowHeader().getRowHeaderModel().fireTableRowsInserted(rows[0], rows[1]);
      }
      table.getVertScrollBar().revalidate();
    }

    public void fireRowsDeleted(long top, long btm) {
      long[] range = table.getRowsVisible(top, -1L);

      if (range != null) {
        int[] rows = table.getModelRowsVisible(range[0], range[0]+(btm-top));

        fireTableRowsDeleted(rows[0], rows[1]);
        table.getVertScrollBar().revalidate();
        table.getRowHeader().getRowHeaderModel().fireTableRowsDeleted(rows[0], rows[1]);
      }
      table.getVertScrollBar().revalidate();
    }

    public void fireStructureChanged() {
      fireTableStructureChanged();
      table.getRowHeader().revalidate();
      table.getVertScrollBar().revalidate();
      table.getRowHeader().getRowHeaderModel().fireTableStructureChanged();
    }
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // The underlying data model
  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** The column model to use. */
  private class NmColumnModel extends DefaultTableColumnModel {
    /** NOTE: this should be changed if the structure/fields of this class changes. */
    private static final long serialVersionUID = 2012021050311L; // for Serializable interface

    private Vector<TableColumn> hiddenColumns = new Vector<>(); // The columns that are hidden.

    /** Removes all of the columns. */
    public void removeAllColumns() {
      tableColumns.clear();
      hiddenColumns.clear();
    }

    public void moveColumn(int columnIndex, int newIndex) {
      if (colDrag) {
        super.moveColumn(columnIndex, newIndex);
        if (columnIndex != newIndex) {
//System.out.println("moveColumn("+columnIndex+", "+newIndex+") -> "+getColumnOrder());
        }
      }
    }

    /** Returns a string of the visible column names in order. */
    public String getColumnOrder() {
      StringBuilder sb = new StringBuilder();

      for (int i = 0; i < tableColumns.size(); i++) {
        if (i != 0) { sb.append(","); }
        sb.append(getColumnName(i));
      }
      return sb.toString();
    }

    /** Gets the name of the column matching the given index. */
    private String getColumnName(int index) {
      TableColumn column  = tableColumns.get(index);
      int         colNum  = column.getModelIndex();

      return columnNames[colNum];
    }

    /** Finds a column index.
        @param name The name of the column.
        @return The column index or -1 if not found.
     */
    public int findColumnIndex(String name) {
      int index = -1;

      for (int i = 0; (index == -1) && (i < tableColumns.size()); i++) {
        if (getColumnName(i).equals(name)) {
          index = i;
        }
      }

      // Check for the "_<index>_" syntax
      if ((index == -1) && name.startsWith("_") && name.endsWith("_")) {
        String val = name.substring(1, name.length()-1);

        if (StringUtil.isNumber(val)) {
          int num = Convert.o2l(val, null, -1);

          if ((num >= 0) && (num < tableColumns.size())) {
            index = num;
          }
        }
      }

      return index;
    }

    /** Finds a column.
        @param index The column index in the model.
        @return The column.
     */
    public TableColumn findColumn (int index) {
      TableColumn column = null;

      for (int i = 0; (column == null) && (i < tableColumns.size()); i++) {
        TableColumn col = tableColumns.get(i);

        if (col.getModelIndex() == index) {
          column = col;
        }
      }

      for (int i = 0; (column == null) && (i < hiddenColumns.size()); i++) {
        TableColumn col = hiddenColumns.get(i);

        if (col.getModelIndex() == index) {
          column = col;
        }
      }

      return column;
    }

    /** Sets a column to be visible or invisible.
        @param column  The column index in the model.
        @param visible Indicates if the given column should be visible.
     */
    public void setColumnVisible(int column, boolean visible) {
      if (visible) {
        for (TableColumn col : hiddenColumns) {
          if (col.getModelIndex() == column) {
            hiddenColumns.remove(col);
            addColumn(col);
            break;
          }
        }
      } else {
        for (TableColumn col : tableColumns) {
          if (col.getModelIndex() == column) {
            hiddenColumns.add(col);
            removeColumn(col);
            break;
          }
        }
      }
    }
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** The renderer for rendering the cells using a specified format. */
  private static class NmTableCellRenderer extends DefaultTableCellRenderer {
    /** NOTE: this should be changed if the structure/fields of this class changes. */
    private static final long serialVersionUID = 2012021060311L; // for Serializable interface

    private static final Border noFocusBorder = new EmptyBorder(1, 1, 1, 1);
    private static final Border focusBorder   = UIManager.getBorder("Table.focusCellHighlightBorder");

    // JC4: The foreground/background color needs to be handled in a special manner since the
    //      superclass overrides the setBackground(Color) method and makes it have a double-
    //      meaning.
    private java.text.Format format;        // The format to use.
    private Color  normalfgcolor; // The normal foreground color to used.
    private Color  normalbgcolor; // The normal background color to used.
    private Color  fgcolor;       // The foreground color being used.
    private Color  bgcolor;       // The background color being used.
    private String fgColorCol;    // Name of the column with foreground color override information.
    private String bgColorCol;    // Name of the column with background color override information.

    /** Creates a new instance. */
    public NmTableCellRenderer() {
      this((java.text.Format)null);
    }

    /** Creates a new instance using center horizontal alignment.
        @param format The format to use.
     */
    public NmTableCellRenderer(java.text.Format format) {
      this(format, CENTER);
    }

    /** Creates a new instance.
        @param format         The format to use.
        @param horizAlignment The horizontal alignment.
     */
    public NmTableCellRenderer(java.text.Format format, int horizAlignment) {
      super();
      this.format = format;
      this.setHorizontalAlignment(horizAlignment);
    }

    public Color getForeground() { return (fgcolor != null)? fgcolor: super.getForeground(); }
    public Color getBackground() { return (bgcolor != null)? bgcolor: super.getBackground(); }

    public void  setForeground(Color color) { fgcolor = color; normalfgcolor = color; }
    public void  setBackground(Color color) { bgcolor = color; normalbgcolor = color; }

    /** Sets the name of the column with foreground color override information (null=NoOverride). */
    public void setFgOverrideCol(String col) { fgColorCol = col; }

    /** Sets the name of the column with background color override information (null=NoOverride). */
    public void setBgOverrideCol(String col) { bgColorCol = col; }

    protected void setValue(Object value) {
      super.setValue(format(value));
    }

    /** Formats the given object with the standard formatter. */
    public String format(Object val) {
      String str = null;

           if (val    == null) { str = "";                 }
      else if (format == null) { str = val.toString();     }
      else                     {
        str = format.format(val); }
        if (str.equals("()")) str = val.toString(); // fix list formatting when format does not work
      return str;
    }


    public java.awt.Component getTableCellRendererComponent(JTable table, Object val, boolean sel,
                                                            boolean focus, int row, int col) {
      java.awt.Component comp = super.getTableCellRendererComponent(table,val,sel,focus,row,col);

      if (comp != this) {
        /* This should NEVER happen... but just in case. */
        ((MJList)table).showWarningMsg("WARNING: Internal error in rendering cells.");
      }

      /* Set the colors */
      Color defFgColor = getRowColor(table, row, this.normalfgcolor, fgColorCol);
      Color defBgColor = getRowColor(table, row, this.normalbgcolor, bgColorCol);

      this.fgcolor = getEnumColor(format, val, defFgColor);
      this.bgcolor = getShadeColor(table, row, defBgColor, sel);
      if (focus) { setBorder(focusBorder);   }
      else       { setBorder(noFocusBorder); }

      return this;
    }

    /** Get updated cell color. Used only by getTableCellRendererComponent(..). */
    private Color getRowColor(JTable table, int row, Color defColor, String colorCol) {
      Color color = defColor;

      if ((colorCol != null) && (table instanceof MJList)) { // Legacy color by value
        MJList  mjl = (MJList)table;
        color = mjl.getColor(row,colorCol);
      }
      return color;
    }

    /** Get enumerated FG color. Used only by getTableCellRendererComponent(..). */
    private Color getEnumColor(java.text.Format format, Object val, Color defColor) {
      Color color = defColor;

      if (format instanceof MFormat) { // Possible color by enumeration
        color = ((MFormat)format).getColor(val);
        if (color == null) { color = defColor; }
      }
      return color;
    }

    /** Get BG color for shade/highlight. Used only by getTableCellRendererComponent(..). */
    private Color getShadeColor(JTable table, int row, Color defColor, boolean sel) {
      Color color = defColor;

      if (sel) {
        color = table.getSelectionBackground();
      } else if (table instanceof MJList) { // Legacy color by value
        MJList  mjl = (MJList)table;
        color = mjl.getShadeColor(row, defColor);
      }
      return color;
    }

    /** Sets the format to use.
        @param format The format to use.
     */
    public void setFormat(java.text.Format format) {
      this.format = format;
    }
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** The renderer for rendering the cells using a checkbox. */
  private static class NmCheckboxRenderer extends JCheckBox implements TableCellRenderer {
    /** NOTE: this should be changed if the structure/fields of this class changes. */
    private static final long serialVersionUID = 2012021070311L; // for Serializable interface

    private static final Border noFocusBorder = new EmptyBorder(1, 1, 1, 1);
    private static final Border focusBorder   = UIManager.getBorder("Table.focusCellHighlightBorder");
    private transient Color  normalfgcolor; // The normal foreground color to used.
    private transient Color  normalbgcolor; // The normal background color to used.
    private transient Color  fgcolor;       // The foreground color being used.
    private transient Color  bgcolor;       // The background color being used.

    /** Creates a new instance. */
    public NmCheckboxRenderer() {
      super();
      setHorizontalAlignment(CENTER);
      setVerticalAlignment(CENTER);
      setIcon(new Symbol.CheckBox(false));
      setSelectedIcon(new Symbol.CheckBox(true));
    }

    /** Indicates if a given object represents a true value. */
    private static boolean isTrue(Object val) {
      String str = (val == null)? "FALSE": val.toString();
      boolean value = false;

      if (StringUtil.isNumber(str)) {
        value = (Convert.s2d(str) != 0);
      } else {
        value = StringUtil.isTrue(str);
      }

      return value;
    }

    public Color getForeground() { return (fgcolor != null)? fgcolor: super.getForeground(); }
    public Color getBackground() { return (bgcolor != null)? bgcolor: super.getBackground(); }

    public void  setForeground(Color color) { fgcolor = color; normalfgcolor = color; }
    public void  setBackground(Color color) { bgcolor = color; normalbgcolor = color; }

    public java.awt.Component getTableCellRendererComponent(JTable table, Object val, boolean sel,
                                                            boolean focus, int row, int col) {
      setSelected(isTrue(val));

      /* Set the colors *//* Set the colors */
      this.fgcolor = this.normalfgcolor;

      if (sel) {
        this.bgcolor = table.getSelectionBackground();
      } else {
        this.bgcolor = ((MJList)table).getShadeColor(row, this.normalbgcolor);
      }

      if (focus) {
        setBorder(focusBorder);
      } else {
        setBorder(noFocusBorder);
      }

      return this;
    }
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // THE SELECTION MODELS
  //////////////////////////////////////////////////////////////////////////////////////////////////

  /** The selection model that handles row/column/cell selections for a large JTable.
      @see LargeJTable.LargeTableSelectionModel
      @since neXtMidas 3.1.0
   */
  public static class NMTableSelectionModel extends LargeTableSelectionModel {
    protected boolean        prevAndCurr;   // Show previous and current selections?
    protected boolean        autoSelect;    // Indicates if autoselect is on or off.
    protected MessageHandler msgHandler;    // The message handler to send select messages to.
    protected long           lastRow = -1L; // The last selected row.    (-1 = not set)
    protected long           lastCol = -1L; // The last selected col.    (-1 = not set)
    protected boolean        legacy = false;// whether or not the /legacy switch is set

    /** Creates a new instance.
        @param table      The table this is used for.
        @param msgHandler The message handler to send select/deselect messages to (usually LIST2).
        @param settings   Settings for the table ('AUTOSELECT','PREVANDCURR','LEGACY')
     */
    public NMTableSelectionModel(MJList table, MessageHandler msgHandler, Table settings) {
      super(table);
      this.msgHandler  = msgHandler;
      if (settings.containsKey("AUTOSELECT"))  autoSelect  = settings.getState("AUTOSELECT");
      if (settings.containsKey("PREVANDCURR")) prevAndCurr = settings.getState("PREVANDCURR");
      if (settings.containsKey("LEGACY"))      legacy      = settings.getState("LEGACY");
    }

    /** Gets the current table. */
    private MJList getMJList() {
      return (MJList)getTable();
    }

    /** Create a Message to be returned back to the message handler (usually the macro).
        The Message Data section will contain a Table corresponding to the action
        given in the Name of the Message. The Table will also contain the currently
        selected cells and the previously selected cells.
        @param row Row of current selection
        @param col Column od current selection
        @param type The type of Action (ANCHOR, CLICK, SHIFT_CLICK, CTRL_CLICK, DESELECT)
        @param prev Table of previous selection cells
        @return A new Message
        @since NeXtMidas 2.8.0 (Moved from {@link #changeCellSelection(long, long, int, boolean)}
     */
    private Message createReturnMessage (long row, long col, int type, Table prev) {
      String name     = null;                                               // Message name
      Table  tbl      = new Table();                                        // Message data (Table)
      Table  curr     = null;                                               // Current selection
      Object value    = null;                                               // Selected value
      String selType  = null;                                               // Select type
      long   rowNum   = getMJList().getSortingModel().toReal(row);          // Adjust for sorting
      long   rowIndex = (row < 0L)? -1L: getMJList().getMacroRowIndex(rowNum);
      String colName  = (col < 0L)?  "": getMJList().getColumnName((int)col);
      long   colIndex = (col < 0L)? -1L: getMJList().getMacroColIndex(colName);

//System.out.println("changeCellSelection: row="+row+" col="+col+"="+colName+" colIndex="+colIndex+" type="+type+" msg="+msg);

      if ((col < 0) && (row < 0)) {
        selType = "ALL";
        value   = new Table();
        curr    = new Table();
      } else if (col < 0 || getTable().isCellClickActionRow()) {
        selType = "ROW";
        value   = getMJList().getRow(rowIndex);
        curr    = getMJList().getAllSelectedCells();
      } else if (row < 0 || getTable().isCellClickActionCol()) {
        selType = "COLUMN";
        value   = "value"; // <-- FIX THIS
      } else {
        selType = "CELL";
        value   = getMJList().getRow(rowIndex).get(colName);
        curr    = getMJList().getAllSelectedCells();
      }
      switch (type) {
        case ANCHOR:      name = null;       break;
        case CLICK:       name = "SELECT";   break;
        case SHIFT_CLICK: name = "SELECT";   break;
        case CTRL_CLICK:  name = "SELECT";   break;
        case DESELECT:    name = "DESELECT"; break;
        default:          name = "SELECT";   break;
      }
      if (!prevAndCurr) { // %test_list2_select matches this one... but that may change
        if (type == CLICK)    { prev = null; curr = null; }
        if (type == DESELECT) { curr = null; colName = null; }
      }
      tbl.setKey("ROW",     Long.valueOf(rowIndex));
      tbl.setKey("COLUMN",  Long.valueOf(colIndex));
      tbl.setKey("TYPE",    selType);

      if (colName != null) { tbl.setKey("NAME",    colName); }
      if (value   != null) { tbl.setKey("VALUE",   value);   }
      if (prev    != null) { tbl.setKey("PREVSEL", prev);    }
      if (curr    != null) { tbl.setKey("CURRSEL", curr);    }

      return new Message(name, 0, tbl);
    }

    protected void changeRowSelection (long start, long end, int type, boolean msg) {
      if (msgHandler != null && type != ANCHOR && msg) {
        Table prev = getMJList().getAllSelectedCells();
        super.changeRowSelection(start, end, type, msg);
        msgHandler.processMessage(createReturnMessage(start, -1, type, prev));
      }
      else {
        super.changeRowSelection(start, end, type, msg);
      }
    }

    /** Method for accessing from macro language
     *  @param row  cell row
     *  @param col  cell column
     *  @param type the selection type
     *  @param msg String representation of boolean state
     *  @since NeXtMidas 3.1.0
     */
    public void changeCellSelection (long row, long col, int type, String msg) {
      boolean bmsg = Convert.s2state(msg, true);
      changeCellSelection(row,col,type,bmsg);
    }

    @Override
    protected void changeCellSelection (long row, long col, int type, boolean msg) {
      if ((type == DESELECT) && (row < 0)) { row = lastRow; }
      if ((type == DESELECT) && (col < 0)) { col = lastCol; }
      lastRow = row;
      lastCol = col;

      if ((msgHandler == null) || (type == ANCHOR) || !msg) {
        super.changeCellSelection(row, col, type, msg); // <-- THIS CHANGES SELECTION!!!
      } else {
        Table  prev     = getMJList().getAllSelectedCells();                  // Previous selection
        super.changeCellSelection(row, col, type, msg); // <-- THIS CHANGES SELECTION!!!
        msgHandler.processMessage(createReturnMessage(row, col, type, prev));
      }
    }

    /** Changes the selection. This is a convenience method since the boolean toggle/extend flags
        tend to be confusing.
        @param row  The row.
        @param col  The column.
        @param type The selection type (see {@link #selectionList}).
     */
    protected void changeCellSelection (long row, long col, String type, boolean msg) {
      long realRow = getMJList().getRealRowIndex(row);
      long realCol = getMJList().getRealColIndex(col);
      int  typeID  = Parser.find(selectionList, type, -1);

      if (typeID < 0) {
        getMJList().showErrorMsg("Invalid selection type '"+type+"'.");
      } else {
        super.changeCellSelection(realRow, realCol, typeID, msg);
      }
    }

  }

  /** The selection model that handles row/column/cell selections for a large JTable.
      This class is meant for internal use only. */
  public static class LegacySelectionModel extends NMTableSelectionModel {

    /** Create LegacySelectionModel
     *
     *  @param table       The table this is used for.
     *  @param autoSelect  autoselect setting
     *  @param msgHandler  The message handler to send select/deselect messages to (usually LIST2).
     *  @param prevAndCurr previous and current setting
     */
    public LegacySelectionModel(MJList table, boolean autoSelect, MessageHandler msgHandler, boolean prevAndCurr) {
      super(table,msgHandler,new Table("AUTOSELECT="+autoSelect+",PREVANDCURR="+prevAndCurr+",LEGACY=true"));
    }
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // SPECIALIZED FILE WRAPPERS
  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** Marks a list file that is able to poll its own data. */
  private static interface PollsOwnData extends ListFile {
    /** Polls for new/updated data.
        @param poll If true checks for new data (added to end of file) and changed data in the file.
                    If false checks only for new data.
        @return The number of rows read.
     */
    public double pollData(boolean poll);
  }

  /** Provides in-memory caching for a DataFile to improve performance. */
  private static abstract class AbstractCachedDataFile implements ListFile {
    protected final int         CACHE_SIZE = 128; // The default cache size to use.
    protected final MJList      table;            // The table to send updates to.
    protected final Midas       midas;            // The Midas context to use.
    protected       DataFile    dataFile;         // The data file being wrapped.
    protected       Table[]     recordDef;        // The individual record definitions (null until loaded).
    protected       Table       recordDefs;       // The set of record definitions (null until loaded).


    /** Creates a new instance.
        @param df    The data file being wrapped.
        @param table The table to send updates to.
     */
    protected AbstractCachedDataFile(Midas midas, DataFile df, MJList table) {
      this.dataFile = df;
      this.table    = table;
      this.midas    = midas;
    }

    public boolean  isOpen()          { return (dataFile != null) && dataFile.isOpen(); }
    public boolean  open(int flags)   { return (dataFile != null)? dataFile.open(flags): false; }
    public String   getURL()          { return (dataFile != null)? dataFile.getURL():    null;  }
    public FileName getName()         { return (dataFile != null)? dataFile.getName():   null;  }
    public double   getNumberOfRows() { return (isOpen())? dataFile.getNumberOfRows():   0.0;   }
    public boolean  getProtected()    { return (isOpen())? dataFile.getProtected():      true;  }
    public void     close()           { if (isOpen()) dataFile.close(); }
    public void     update()          { if (isOpen()) dataFile.update(); }

    public int getRecordDefCount() {
      return getRecordDef().length;
    }

    public void setRecordDefs(Table defs) {
      throw new MidasException(getClass().getName()+": Altering recodr definitions not permitted.");
    }

    public Table getRecordDefs() {
      getRecordDef(); // <-- Loads the definitions
      return recordDefs;
    }

    public Table getRecordDef(int i) {
      return (i < getRecordDef().length)? getRecordDef()[i]: null;
    }

    /** Loads all of the individual record definitions. */
    private Table[] getRecordDef() {
      if (!isOpen()) {
        recordDef  = new Table[0];
        recordDefs = new Table();
      } else if ((recordDef != null) && (recordDefs != null)) {
        // nothing to do
      } else {
        recordDef  = new Table[dataFile.getRecordDefCount()];
        recordDefs = dataFile.getRecordDefs();
        for (int i = 0; i < recordDef.length; i++) {
          recordDef[i] = dataFile.getRecordDef(i);
        }
      }
      return recordDef;
    }

    /** Resets the record definitions. */
    protected void resetRecordDef() {
      recordDef  = null;
      recordDefs = null;
    }
  }

  /** Provides in-memory caching for a DataFile to improve performance. */
  private static final class CachedDataFile extends AbstractCachedDataFile implements PollsOwnData {
    private Cache   cache    = new Cache(CACHE_SIZE); // The local data cache.
    private double  lastSize = 0.0;                   // The number of rows at last poll.

    /** Creates a new instance.
        @param df    The data file being wrapped.
        @param table The table to send updates to.
     */
    public CachedDataFile(Midas midas, DataFile df, MJList table) {
      super(midas,df,table);
    }

    public Table getDataTable(double offset) {
      Table tbl = null;

      if (isOpen()) {
        String index = Double.toString(offset);

        tbl = (Table)cache.get(index);
        if (tbl == null) {
          tbl = dataFile.getDataTable(offset);
          cache.put(index, tbl);
        }
      } else {
        midas.warning("MJList: Can not read from file, file is not open.");
      }
      return tbl;
    }

    public int setData (double offset, Table table) {
      if (isOpen()) {
        cache.remove(Double.toString(offset));
        return dataFile.setData(offset, table);
      } else {
        midas.warning("MJList: Can not write to file, file is not open.");
        return 0;
      }
    }

    public void setData (double element, String fieldName, Object obj) {
      if (isOpen()) {
        cache.remove(Double.toString(element));
        dataFile.setData(element, fieldName, obj);
      } else {
        midas.warning("MJList: Can not write to file, file is not open.");
      }
    }

    public void insertData (double offset, Object obj) {
      if (isOpen()) {
        cache.clear();
        dataFile.insertData(offset, obj);
      } else {
        midas.warning("MJList: Can not insert data, file is not open.");
      }
    }

    public void removeData (double offset) {
      removeData(offset, 1L);
    }

    public final void removeData (double offset, long count) {
      if (count != 0) {
        cache.clear();
        if (isOpen()) {
          dataFile.removeData(offset, count);
        } else {
          midas.warning("MJList: Can not remove data, file is not open.");
          cache.clear();
          recordDef = null;
        }
      }
    }

    public double pollData(boolean poll) {
      boolean resetAll = false; // Reset the entire display.

      if (poll) {
        boolean dfIsOpen = isOpen();
        try {
          dataFile = dataFile.reOpen();
        } catch (Exception e) {
          midas.printStackTrace("MJList: Error reopening "+dataFile.getName()+" during poll",e);
        }
        resetAll = dfIsOpen != isOpen();
      }

      double newSize = getNumberOfRows();
      double avail   = newSize - lastSize;

      if (isOpen()) {
        if ((avail > 0.0) && !resetAll) {
          table.getLargeTableModel().fireRowsInserted((long)lastSize, (long)newSize);
        }

        if (poll) {
          Cache oldCache = cache;
          avail = newSize;
          cache = new Cache(CACHE_SIZE);

          /* Refresh the changed rows in the current view. */
          if (!resetAll) {
            for (int i = 0; i < table.getNumberOfVisibleRows(); i++) {
              double offset = (double)i + (double)table.getTopRowIndex();

              if (offset < lastSize) { // Newly-added rows have already been accounted for.
                Table newData = getDataTable(offset);
                Table oldData = (Table)oldCache.get(Double.toString(offset));

                if (!rowEquals(oldData, newData)) {
                  table.getLargeTableModel().fireNeedsRedraw((long)offset, (long)offset);
                }
              }
            }
          }
        }
        lastSize = newSize;
      }
      if (resetAll) {
        table.getLargeTableModel().fireStructureChanged();
      }
      return avail;
    }

    /** Compares two rows.
        @param oldData The old data
        @param newData The new data
        @return true if they are the same, false if not.
     */
    private boolean rowEquals(Table oldData, Table newData) {
      if (oldData == newData) return true;
      boolean  same = (oldData != null && newData != null);
      if (same) {
        String[] keys = newData.getKeys();
        for (int i = 0; same && (i < keys.length); i++) {
          Object newVal = newData.get(keys[i]);
          Object oldVal = oldData.get(keys[i]);

          same = (newVal != null) && (oldVal != null) && newVal.toString().equals(oldVal.toString());
        }
      }
      return same;
    }
  }

  /** Provides in-memory caching for a DataFile to improve performance. */
  private static final class PipedDataFile extends AbstractCachedDataFile implements PollsOwnData {
    private final int        DEFAULT_MAX =  16;           // Default maximum.
    private final int        UPPER_MAX   = 128;           // Upper limit to auto-set maximum.
    private final LinkedList<Table> cache = new LinkedList<>();  // The local data cache.
    private       double     start   = 0.0;               // The index of the first cached element.
    private       int        maxSize = DEFAULT_MAX;       // The maximum size of cache.

    /** Creates a new instance.
        @param df    The data file being wrapped.
        @param table The table to send updates to.
     */
    public PipedDataFile(Midas midas, DataFile df, MJList table) {
      super(midas,df,table);
      this.maxSize = (int)Math.min((double)UPPER_MAX, Math.max(2.0, dataFile.getSize()));
    }

    public boolean getProtected() {
      return true;
    }

    public double getNumberOfRows() {
      return (double)cache.size();
    }

    public Table getDataTable(double offset) {
      double index = offset;
      Table  tbl   = null;

      if ((index >= 0) && (index < cache.size())) {
        tbl = cache.get((int)index);
      }
      return tbl;
    }

    public int  setData (double offset, Table table) { return 0; }
    public void setData (double element, String fieldName, Object obj) { }
    public void insertData (double offset, Object obj) { }
    public void removeData (double offset, long count) { }
    public void removeData (double offset) { removeData(offset, 1L); }

    /** Poll for new data added to the pipe.
        @param poll This is ignored since pipes are only able to read new data.
        @return The number of rows read.
     */
    public double pollData(boolean poll) {
      int    oldLength   = cache.size();
      int    maxViewable = table.getNumberOfVisibleRows();
      double avail       = Math.min(maxViewable/2, Math.floor(dataFile.avail()));

      for (int i = 0; i < avail; i++) {
        Table tbl = dataFile.getDataTable(dataFile.seek()); // <-- Advances seek pointer after read.
        cache.addLast(tbl);

        while(cache.size() > maxSize) {
          cache.removeFirst();
          start++;
        }
      }

      if (avail > 0) {
        table.rowOffset = (long)start;
        table.getLargeTableModel().fireRowsInserted(-1, -1);
      }

      return avail;
    }
  }

  /** If enabled, allows for dragging and dropping selected rows on and off the LIST2.
      <br><br>
      If enabling, this method creates a new instance of {@link ListTransferHandler}
      and sets the underlying JTable dropMode to INSERT_ROWS
      @param enabled true to enable Drag and Drop, false to disable it
      @since NeXtMidas 3.3.0
   */
  public void setDragAndDropEnabled (boolean enabled) {
    if (dragDrop == enabled) return;   // nothing to do
    dragDrop = enabled;
    if (dragDrop) {
      setTransferHandler(new ListTransferHandler());
      setDropMode(DropMode.INSERT_ROWS);
    }
    setDragEnabled(enabled);
  }

  /** Is Drag and Drop enabled
   *  @return true if Drag and Drop is enabled, otherwise false
      @since NeXtMidas 3.3.0
   */
  public boolean isDragAndDropEnabled () {
    return dragDrop;
  }

  /** Class to handle Drag and Drop actions if Drag and Drop are enabled on the LIST2.<br><br>
 <pre>
 Upon initialization (in {@link MJList#setDragAndDropEnabled(boolean)}, has the following properties
    Source Actions    =  COPY_OR_MOVE
    Drag Data Object  =  StringSelection  (Table.toString()) of selected rows
    Drop Action       =  Send incoming Obj as msg.data of DROPDATA msg to list2.processMessage(..)
 </pre>
      @author jwb
      @since NeXtMidas 3.3.0
   */
  public class ListTransferHandler extends TransferHandler {
    /** NOTE: this should be changed if the structure/fields of this class changes. */
    private static final long serialVersionUID = 2013022100330L; // for Serializable interface

    /** default constructor */
    public ListTransferHandler () {}

    public int getSourceActions (JComponent c) {
      return COPY_OR_MOVE;
    }

    protected Transferable createTransferable (JComponent c) {
      if (c instanceof MJList) {
        String ss = ((MJList)c).getAllSelectedRows().toString();
        return new StringSelection(ss);
      } else return null;
    }

    public boolean canImport (TransferHandler.TransferSupport support) {
      return isDragAndDropEnabled();
    }

    public boolean importData (JComponent comp, Transferable t) {
      if (comp instanceof MJList) {
        Object tdata;
        try {
          tdata = t.getTransferData(DataFlavor.stringFlavor);
          Message msg = new Message("DROPDATA",0,tdata);
          ((MJList)comp).getMessageHandler().processMessage(msg);
        } catch (Exception e) {
          //e.printStackTrace();   // for now, just return false
          return false;
        }
      }
      return true;
    }

  } // end inner class ListTransferHandler

}
