package nxm.sys.test.junit.lib;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.HashSet;

import nxm.sys.inc.InternalUseOnly;
import nxm.sys.inc.Units;
import nxm.sys.lib.BaseFile;
import nxm.sys.lib.Convert;
import nxm.sys.lib.Data;
import nxm.sys.lib.DataFile;
import nxm.sys.lib.IOResource;
import nxm.sys.lib.Keywords;
import nxm.sys.lib.Midas;
import nxm.sys.lib.Shell;
import nxm.sys.lib.Table;
import nxm.sys.lib.Time;
import nxm.sys.test.junit.NMTestCase;

/** This will test the DataFile class.
    @author  A. Weisz
    @since 3.5.1
 */
public class DataFileTest extends NMTestCase {

  /** public constructor to allow running this test case directly with JUnit. */
  public DataFileTest (String methodName) {
    this(methodName, null);
  }

  /** This constructor is required by NeXtMidas Unit Testing framework.
      @param methodName The name of the method being called.
      @param midas      The midas context to use when running the tests.
   */
  public DataFileTest (String methodName, Midas midas) {
    super(methodName, midas);
  }

///////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
  private static final String[] test_files = {urlDat+"test1000.prm",
                                              urlDat+"test2000_cf.prm",
                                              urlDat+"test2000_tcprec.prm",
                                              urlDat+"test3000.prm",
                                              urlDat+"test6000.prm",
                                              urlDat+"testxy3000.prm",
                                              urlDat+"sv1.prm",        // type 5000 file
                                              urlDat+"testxy6000.prm",
                                              urlDat+"world.prm"};     // type 1999 file

  private static final String[] test_files_1 = {urlDat+"test1000_eeei.prm",
                                                urlDat+"test3000_eeei.prm"};

  private static final String[] test_files_2 = {urlDat+"test1000_ieee.prm",
                                                urlDat+"test3000_ieee.prm"};

  /** test files with extended header that should be skip under Web Start until Bug 1899 is fixed */
  private static final HashSet<String> skipUntilBug1899IsFixed = new HashSet<>();
  static { // files to skip testing in Web Start, until BUG 1899 is fixed
    // Restoring these test in 3.7.3, as the problem seems to have been fixed
//    skipUntilBug1899IsFixed.add(urlDat+"test2000_tcprec.prm");
//    skipUntilBug1899IsFixed.add(urlDat+"test6000.prm");
//    skipUntilBug1899IsFixed.add(urlDat+"testxy6000.prm");
//    skipUntilBug1899IsFixed.add(urlDat+"test1000_eeei.prm");
//    skipUntilBug1899IsFixed.add(urlDat+"test1000_ieee.prm");
  }

  Midas midasMacro = getMidas().cloneOf();
  {
    midasMacro.setIfJavaBehavior(false);
  }

///////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
  /** Test the equals() and hashCode() methods to ensure the Java equals/hashcode contract is satisfied. */
  public void testEquals() {
    Midas M = getMidasContext();
    boolean runningInWebStart = Shell.isWebStart();

    // Test Self Equivalence and Hash Codes
    for (String testfile : test_files) {
      if (runningInWebStart && skipUntilBug1899IsFixed.contains(testfile)) {
        warn("Skipping file with extended header until Bug 1899 is fixed: " + testfile);
        continue;
      }
      DataFile df1 = new DataFile(M, testfile);
      DataFile df2 = new DataFile(M, testfile);
      df1.open();
      df2.open();
      boolean areEqual = df1.equals(df2);
      int hash1 = df1.hashCode();
      int hash2 = df2.hashCode();
      assertTrue("testHashCode: DataFiles Equal for " + testfile, areEqual);
      assertEquals("testHashCode: DataFile HashCodes Equal for " + testfile, hash1, hash2);
      df1.close();
      df2.close();
    }

    // Test Similar Files
    for (int ii = 0; ii < test_files_1.length; ii++) {
      final String testfile1 = test_files_1[ii];
      final String testfile2 = test_files_2[ii];
      if (runningInWebStart) {
        if (skipUntilBug1899IsFixed.contains(testfile1)) {
          warn("Skipping file with extended header until Bug 1899 is fixed: " + testfile1); continue;
        }
        if (skipUntilBug1899IsFixed.contains(testfile2)) {
          warn("Skipping file with extended header until Bug 1899 is fixed: " + testfile2); continue;
        }
      }
      DataFile df1 = new DataFile(M, testfile1);
      DataFile df2 = new DataFile(M, testfile2);
      df1.open();
      df2.open();
      if (df1.equals(df2)) {
        int hash1 = df1.hashCode();
        int hash2 = df2.hashCode();
        assertEquals("testHashCode: DataFile HashCodes Equal for " + testfile1 + " and " + testfile2, hash1, hash2);
      }

      df1.close();
      df2.close();
    }

    // Test Different Files
    for (int ii = 0; ii < (test_files_1.length - 1); ii++) {
      final String testfile1 = test_files_1[ii];
      final String testfile2 = test_files_1[ii+1];
      if (runningInWebStart) {
        if (skipUntilBug1899IsFixed.contains(testfile1)) {
          warn("Skipping file with extended header until Bug 1899 is fixed: " + testfile1); continue;
        }
        if (skipUntilBug1899IsFixed.contains(testfile2)) {
          warn("Skipping file with extended header until Bug 1899 is fixed: " + testfile2); continue;
        }
      }
      DataFile df1 = new DataFile(M, testfile1);
      DataFile df2 = new DataFile(M, testfile2);
      df1.open();
      df2.open();
      int hash1 = df1.hashCode();
      int hash2 = df2.hashCode();
      if (hash1 != hash2) {
        assertFalse("testHashCode: DataFile Not Equal for " + testfile1 + " and " + testfile2, df1.equals(df2));
      }

      df1.close();
      df2.close();
    }
  }

  public void testJavaVsMidasKeepCase () {
    String[] testCasingFiles = new String[] {
        "ALL_UPPER_CASE_FN_20230417", "MixedCaseFileName.txt",
        "lowercasefn.tmp"
    };
    Midas midasMacro = new Midas();
    midasMacro.setIfJavaBehavior(false);
    check2ArgCase(testCasingFiles,midasMacro);
    checkFlagArgCases(testCasingFiles,midasMacro,0);
    checkFlagArgCases(testCasingFiles,midasMacro,BaseFile.MAINTAINCASE);
    Midas midasJava = new Midas();
    midasMacro.setIfJavaBehavior(true);
    check2ArgCase(testCasingFiles,midasJava);
    checkFlagArgCases(testCasingFiles,midasJava,0);
    checkFlagArgCases(testCasingFiles,midasJava,BaseFile.MAINTAINCASE);
    Midas midasNull = null;
    check2ArgCase(testCasingFiles,midasNull);
    checkFlagArgCases(testCasingFiles,midasNull,0);
    checkFlagArgCases(testCasingFiles,midasNull,BaseFile.MAINTAINCASE);

  }

  private void check2ArgCase (String[] filenames, Midas thismidas) {
    for (String fn : filenames) {
      //System.out.println("DataFileTest check2ArgCase fn:"+fn+" thisMidas:"+thismidas);
      DataFile df = new DataFile(thismidas,fn);
      boolean isJavaUsage = (thismidas!=null) && thismidas.isJavaBehavior();
      String usage = (isJavaUsage) ? "Java" : "Macro";
      String expected = fn;
      String actual   = df.getFileName().toString();
      if (!isJavaUsage && fn.toUpperCase().equals(fn)) expected = expected.toLowerCase();
      assertEquals(usage+" usage DataFile name:"+df.getFileName(),expected,actual);
      df.close();
    }
  }

  /** Midas should make no difference, as should just honor flags
   *
   *  @param filenames array of filenames to check
   *  @param thismidas Midas instance
   *  @param flags     BaseFile flags as int
   */
  private void checkFlagArgCases (String[] filenames, Midas thismidas, int flags) {
    boolean keepCase = (flags&DataFile.MAINTAINCASE)!=0;
    for (String fn : filenames) {
      //System.out.println("\n\nDataFileTest checkFlagArgCases fn:"+fn+" thisMidas:"+thismidas+" keepCase:"+keepCase);
      String expected = fn;
      if (!keepCase && fn.toUpperCase().equals(fn)) expected = expected.toLowerCase();
      DataFile df;
      String actual;

      df = new DataFile(thismidas,fn,flags);
      actual = df.getFileName().toString();
      assertEquals("DataFile(ref,fileName,flags) with MAINTAINCASE="+keepCase+"  name:"+df.getFileName(),expected,actual);
      df.close();

      df = new DataFile(thismidas,fn,"1000","SF",flags);
      actual = df.getFileName().toString();
      assertEquals("DataFile(ref,fileName,types,formats,flags) with MAINTAINCASE="+keepCase+" name:"+df.getFileName(),expected,actual);
      df.close();

      df = new DataFile(thismidas,fn,null,flags);
      actual = df.getFileName().toString();
      assertEquals("DataFile(ref,fileName,hcbin,flags) with MAINTAINCASE="+keepCase+" name:"+df.getFileName(),expected,actual);
      df.close();

      df = new DataFile(thismidas,fn,"1000","SF",flags,1.0,null);
      actual = df.getFileName().toString();
      assertEquals("DataFile(ref,fileName,types,formats,flags,size,hcbin) with MAINTAINCASE="+keepCase+" name:"+df.getFileName(),expected,actual);
      df.close();
    }
  }


  /** Test behavior of DataFile constructor deprecated in 3.5.0 (bug 2851) */
  public void testDeprecatedContructor() {
    DataFile dfNew = null;
    try {
      dfNew = new DataFile(dfNew, "ram:DeprecatedConstructorT1000", "1000", "CF", DataFile.OUTPUT);
      boolean opened = dfNew.open();
      assertTrue("DataFile creation using 5 arg deprecated constructor", opened);
      Keywords kw = dfNew.getKeywordsObject();
      kw.setKey("S:TESTKEY", "This is a test");
      dfNew.close();
      assertTrue("Write to DataFile created using 5 arg deprecated constructor", true);
    }
    catch (Exception e){
      Midas M = getMidasContext();
      if (M != null){
        M.printStackTrace(e);
      }
      fail("test deprecated constructor: DataFile (Object ref,Object name,String types,String formats,int flags)");
    }
 }

  public void testFileName () {
    Midas M = Shell.getMidasContext();
    if (Shell.isGradleBuild()) {
      M.warning("nxm.sys.junit.lib.DataTest skipping testFileName() because fails in Gradle build");
      return;
    }
    String options = getMidasContext().io.getOptions();
    // Input filenames to be used
    String[] inputs = new String[] { "AXABCXX100101Q", "AXABCXX100101Q", "AXABCXX100101Q",
        "axabcxx100101q.tmp", "THISSHOULDBELOWERAFTERCREATION", "THISSHOULDBEUPPERCASE",
        "SHOULDBELOWERWITHEXTENSION", "SHOULDBEUPPERWITHEXTENSION", "THISSHOULDBEUPPERCASE",
        "SHOULDBELOWERWITHEXTENSION", "SHOULDBEUPPERWITHEXTENSION.tmp", "TESTFILETHATWILLREMAINALLCAPSWITHOUTEXTENSION{FLAGS=DONOTADDEXTENSION, FNCASE=KEEPCASE}",
        "TESTFILETHATWILLREMAINALLCAPSWITHEXTENSION{FNCASE=KEEPCASE}", "TESTFILETHATWILLREMAINLOWERCASEWITHOUTEXTENSION{FLAGS=DONOTADDEXTENSION}"};

    // The resulting file name based on the flags or coreio options
    String[] results = new String[] { "AXABCXX100101Q", "AXABCXX100101Q.tmp", "axabcxx100101q",
        "axabcxx100101q.tmp", "thisshouldbeloweraftercreation.tmp", "THISSHOULDBEUPPERCASE",
        "shouldbelowerwithextension", "SHOULDBEUPPERWITHEXTENSION.tmp", "THISSHOULDBEUPPERCASE",
        "shouldbelowerwithextension", "SHOULDBEUPPERWITHEXTENSION.tmp", "TESTFILETHATWILLREMAINALLCAPSWITHOUTEXTENSION", "TESTFILETHATWILLREMAINALLCAPSWITHEXTENSION.tmp", "testfilethatwillremainlowercasewithoutextension"};

    // The possible input flags that can be used.
    int[] flags = new int[] { DataFile.MAINTAINCASE | DataFile.DONOTADDEXTENSION,
        DataFile.MAINTAINCASE, DataFile.DONOTADDEXTENSION, 0, };

    // The possible coreio options that can be used.
    String[] coreIO = new String[] { "|", "+CreateFileMaintainCase|+CreateFileWithoutTmpExtenstion",
        "+CreateFileWithoutTmpExtenstion", "+CreateFileMaintainCase",
        "+CreateFileMaintainCase|+CreateFileWithoutTmpExtenstion", "+CreateFileWithoutTmpExtenstion",
        "+CreateFileMaintainCase" };

    // using flags
    int i = 0;
    // Loop through the inputs using the flags that match the index.
    while(i<flags.length) {
      File file = new File(inputs[i] + "{aux=cwd}") ;
      DataFile df = new DataFile(null, file, flags[i]);
      df.open(DataFile.OUTPUT);
      df.close();
      String fileName = Shell.getAuxPath("cwd")+results[i];
      if (fileName.startsWith("file:")) fileName = fileName.substring(5);
      file = new File(fileName);
      assertTrue("The file " + file.getName() + " exists at " + file.getPath(), file.exists());
      if(!file.delete()) System.out.println("unable to delete file");
      i++;
    }
    while(i<flags.length + coreIO.length) {
      getMidasContext().io.setOptions(coreIO[i-flags.length]);
      File file =  new File(inputs[i]  + "{aux=cwd}");
      DataFile df = new DataFile(midasMacro, file);
      df.open(DataFile.OUTPUT);
      df.close();
      String fileName = Shell.getAuxPath("cwd")+results[i];
      if (fileName.startsWith("file:")) fileName = fileName.substring(5);
      file = new File(fileName);
      assertTrue("The file " + file.getName() + " exists", file.exists());
      getMidasContext().io.setOptions(options);
      if(!file.delete()) System.out.println("unable to delete file");
      i++;
    }

    while(i<results.length) {
      File file = new File(inputs[i]  + "{aux=cwd}");
      DataFile df = new DataFile(midasMacro, file);
      df.open(DataFile.OUTPUT);
      df.close();
      String fileName = Shell.getAuxPath("cwd")+results[i];
      if (fileName.startsWith("file:")) fileName = fileName.substring(5);
      file = new File(fileName);
      assertTrue("The file " + file.getName() + " exists", file.exists());
      if(!file.delete()) System.out.println("unable to delete file");
      i++;
    }
  }

  /**
   * Testing DataFile's setRecMinimum/setRexMaximum packing of double values into its minVal/maxVal
   * field in a Type 6000 file (bug 2957)
   * @since NeXtMidas 3.6.0
   */
  public void testSetRecMinimum(){
    Midas M = getMidasContext();
    String testFileName = urlDat+"test6000.prm";
    boolean runningInWebStart = Shell.isWebStart();
    if (runningInWebStart && skipUntilBug1899IsFixed.contains(testFileName)) {
      warn("Skipping file with extended header until Bug 1899 is fixed: " + testFileName);
      return;
    }

    DataFile df = new DataFile(M, testFileName);
    df.open();

    double expectedZero = 0.0;
    double expectedMin = 19.2;
    double expectedMax = 999.2;
    assertEquals("initial getRecMinimum", expectedZero, df.getRecMinimum(1));
    assertEquals("initial getRecMaximum", expectedZero, df.getRecMaximum(2));
    df.setRecMinimum(1, expectedMin);
    df.setRecMaximum(2, expectedMax);
    assertEquals("setRecMinimum", expectedMin, df.getRecMinimum(1));
    assertEquals("setRecMaximum", expectedMax, df.getRecMaximum(2));
    df.close();
  }

  public void testSetRecMaximum() {} // tested in testSetRecMinimum()
  public void testGetRecMinimum() {} // tested in testSetRecMinimum()
  public void testGetRecMaximum() {} // tested in testSetRecMinimum()

  /**
   * Testing that DataFile's getData(..) and getDataTable() methods return the correct data
   * when num_elts > 1 on type 6000 file (bug 2958)
   * @since NeXtMidas 3.6.0
  */
 public void testGetDataTable() {
   Midas M = getMidasContext();
   String xmTestFileName = urlSysTest+"t6000_xm.prm";
   boolean runningInWebStart = Shell.isWebStart();
   if (runningInWebStart && skipUntilBug1899IsFixed.contains(xmTestFileName)) {
     warn("Skipping file with extended header until Bug 1899 is fixed: " + xmTestFileName);
     return;
   }
   DataFile dataFile = new DataFile(M, xmTestFileName);
   dataFile.open();

   Table expectedGetDataTable0 = new Table();
   short[] subRec2List = {1,2,3};
   short[] subRec4List = {1,1,1,1};
   Data expDataSubRec1 = new Data((short) 1);
   Data expdataSubRec2 = new Data(subRec2List);
   Data expDataSubRec3 = new Data((short) 3);
   Data expDataSubRec4 = new Data(subRec4List);
   expectedGetDataTable0.put("SUBREC1", expDataSubRec1);
   expectedGetDataTable0.put("SUBREC2", expdataSubRec2);
   expectedGetDataTable0.put("SUBREC3", expDataSubRec3);
   expectedGetDataTable0.put("SUBREC4", expDataSubRec4);

   assertEquals("getData(0, SUBREC1)", expDataSubRec1, dataFile.getData(0, "SUBREC1"));
   assertEquals("getData(0, 0)",       expDataSubRec1, dataFile.getData(0, 0));
   assertEquals("getData(0, SUBREC2)", expdataSubRec2, dataFile.getData(0, "SUBREC2"));
   assertEquals("getData(0, 1)",       expdataSubRec2, dataFile.getData(0, 1));
   assertEquals("getData(0, SUBREC3)", expDataSubRec3, dataFile.getData(0, "SUBREC3"));
   assertEquals("getData(0, 2)",       expDataSubRec3, dataFile.getData(0, 2));
   assertEquals("getData(0, SUBREC4)", expDataSubRec4, dataFile.getData(0, "SUBREC4"));
   assertEquals("getData(0, 3)",       expDataSubRec4, dataFile.getData(0, 3));
   assertEquals("getDataTable(0)", expectedGetDataTable0, dataFile.getDataTable(0));

   dataFile.close();
 }

 public void testGetData() {} // tested in testGetDataTable()

 /**
  * Testing that DataFile's getRecordDef(int i) returns the expected values in
  * a type 6000 file. Also testing that getRecordDefs() returns the expected
  * values when setRecordDefs() changed the values in the table. (bug 2960)
 * @since NeXtMidas 3.6.0
 */
public void testSetRecordDefs(){
  Midas M = getMidasContext();
  String testFileName = urlDat+"test6000.prm";
  boolean runningInWebStart = Shell.isWebStart();
  if (runningInWebStart && skipUntilBug1899IsFixed.contains(testFileName)) {
    warn("Skipping file with extended header until Bug 1899 is fixed: " + testFileName);
    return;
  }

  DataFile df = new DataFile(M, testFileName);
  df.open();

  Table getRecordDefInitial = new Table();
  Table setRecordDefInnerTable = new Table();
  Table setRecordDefOutterTable = new Table();
  Data nxmIntegerZero = new Data((short) 0);

  String[] RecordNames = {"F1", "R1", "CC", "FF", "GG"};
  double[] RecordMinVal = {0.0, 1.0, 2.0, 3.0, 4.0};
  double[] RecordMaxVal = {10.0, 11.0, 12.0, 13.0, 14.0};

  getRecordDefInitial.put("NAME", "F1");
  getRecordDefInitial.put("FORMAT", "1A");
  getRecordDefInitial.put("OFFSET", nxmIntegerZero);
  getRecordDefInitial.put("TYPE", "UNDEFINED");
  getRecordDefInitial.put("UNITS", "NONE");
  getRecordDefInitial.put("MINVAL", 0.0);
  getRecordDefInitial.put("MAXVAL", 0.0);
  getRecordDefInitial.put("NUM_ELTS", 1);
  getRecordDefInitial.put("UPREFIX", "000");
  getRecordDefInitial.put("RESERVED", "000");
  assertEquals("initial getRecordDef: ", getRecordDefInitial, df.getRecordDef(0));

  for (int i=0; i<5; i++) {
    setRecordDefInnerTable.put("NAME", RecordNames[i]);
    setRecordDefInnerTable.put("FORMAT", "1A");
    setRecordDefInnerTable.put("OFFSET", nxmIntegerZero);
    setRecordDefInnerTable.put("TYPE", "UNDEFINED");
    setRecordDefInnerTable.put("UNITS", "NONE");
    setRecordDefInnerTable.put("MINVAL", RecordMinVal[i]);
    setRecordDefInnerTable.put("MAXVAL", RecordMaxVal[i]);
    setRecordDefInnerTable.put("NUM_ELTS", 1);
    setRecordDefInnerTable.put("UPREFIX", "111");
    setRecordDefInnerTable.put("RESERVED", "111");
    Table copyofTable = setRecordDefInnerTable.copy();
    setRecordDefOutterTable.put(RecordNames[i], copyofTable);
  }

  df.setRecordDefs(setRecordDefOutterTable);
  assertEquals("setRecordDefs called to change values: ", setRecordDefOutterTable, df.getRecordDefs());

  df.close();
}

 public void testGetRecordDef(int i) {} // tested in testSetRecordDefs()
 public void testGetRecordDefs() {}     // tested in testSetRecordDefs()

 /**
  * Illustrates how to create and setData in Type 1000 and Type 3000 files
  */
 public void testSetData () {
   Midas M = getMidasContext();
   if (Shell.isGradleBuild()) {
     M.warning("nxm.sys.junit.lib.DataFileTest skipping testSetData() because fails in Gradle build");
     return;
   }
   // Create Type 1000 file from WAVEFORM command and then modify file
   // using setData
   String waveCmd = "WAVEFORM tmp_wave_file.tmp FORM=CF ELEM=256 SHAPE=SINE ";
   Shell.runCommand(M, waveCmd);
   DataFile dfWave = new DataFile(M, "tmp_wave_file.tmp");
   dfWave.open(DataFile.INOUT);
   Keywords kw = dfWave.getKeywordsObject();
   kw.setKey("S:TESTKEY", "This is a test");

   // Set Data to different values than sine wave data
   Data newFirstElem = new Data(0.25,0.5);
   Data newSecondElem = new Data(1.25,1.5);
   dfWave.setData(0,newFirstElem);
   dfWave.setData(1,newSecondElem);

   dfWave.close();


   // Create Type 3000 file by copying header information from another 3000 file
   DataFile example3000 = new DataFile(M,urlDat+""+"test3000.prm");
   example3000.open();
   DataFile new3000File = new DataFile(M, "tmp_3000_file.tmp",example3000,DataFile.OUTPUT);
   new3000File.open();
   Keywords kw3000 = new3000File.getKeywordsObject();
   kw3000.setKey("S:3000KEY", "3000 test value");

   new3000File.setSubRecords("(SR2A|2A,SRSL|SL)");
   new3000File.setData (0, "SR2A", "FIRST");
   new3000File.setData (0, "SRSL", 1);
   new3000File.setData (1, "SR2A", "SECOND");
   new3000File.setData (1, "SRSL", 2);
   new3000File.close();
   example3000.close();

   // Read in data from Type 1000 and validate file
   DataFile checkDfWave = new DataFile(M, "tmp_wave_file.tmp");
   checkDfWave.open(DataFile.INPUT);
   Keywords checkKw = checkDfWave.getKeywordsObject();
   String kwValue = checkKw.getKey("TESTKEY").toString();
   assertEquals("setData Keyword set correctly in Type 1000 file: ",
       "This is a test", kwValue);
   Data firstElem = checkDfWave.getData(0);
   Data secondElem = checkDfWave.getData(1);
   assertEquals("setData First Data element set correctly in Type 1000 file: ",
       newFirstElem.toString(),firstElem.toString());
   assertEquals("setData Second Data element set correctly in Type 1000 file: ",
       newSecondElem.toString(), secondElem.toString());
   checkDfWave.close();
   BaseFile.delete("tmp_wave_file.tmp");



   // Read in data from Type 3000 validate file
   DataFile check3000File = new DataFile(M,"tmp_3000_file.tmp");
   check3000File.open(DataFile.INPUT);
   Keywords check3000Kw = check3000File.getKeywordsObject();
   String kw3000Value = check3000Kw.getKey("3000KEY").toString();
   assertEquals("setData Keyword set correctly in Type 3000 file: ",
       "3000 test value", kw3000Value);
   Data row0Col1 = check3000File.getData(0,"SR2A");
   Data row0Col2 = check3000File.getData(0,"SRSL");
   Data row1Col1 = check3000File.getData(1,"SR2A");
   Data row1Col2 = check3000File.getData(1,"SRSL");
   assertEquals("setData Top Left Data element set correctly in Type 3000 file: ",
       "FIRST",row0Col1.toString());
   assertEquals("setData Bottom Left Data element set correctly in Type 3000 file: ",
       "SECOND", row1Col1.toString());
   assertEquals("setData Top Right Data element set correctly in Type 3000 file: ",
       new Data(1),row0Col2);
   assertEquals("setData Bottom Right Data element set correctly in Type 3000 file: ",
       new Data(2), row1Col2);
   check3000File.close();
   BaseFile.delete("tmp_3000_file.tmp");

 }

 /**
  * This is not a test but a file generator. This method can be ignored when it comes to unit testing.
  * If you want to create a type 5000 file, this is a great place to start.
  * @since NeXtMidas 3.8.1
  */
 @InternalUseOnly
 public void create5000FileForUnitsTesting () {
   //Define where the file will be saved. In this case, aux = dat
   DataFile df = new DataFile(getMidasContext(), datDir+"test5000Units.prm", "5000", "NH", 2);
   df.open();

   // These are the units that are being tested. These will be used for column headers.
   String[] units = { "BAUD", "BIT_RATE", "POTENTIAL_DIFF", "CURRENT", "RESISTANCE", "CAPACITANCE", "INDUCTANCE",
       "TEMPERATURE", "PRESSURE", "RELATIVE_POWER", "RELATIVE_POWER_DENSE" };
   // Value to be inserted
   int i = 0;

   // Column definitions.
   Table recDefs = new Table();

   // The row to be added after the file is defined.
   Table row = new Table();
   for (String unit : units) {
     // The definition for the column in question
     Table getRecordDefInitial = new Table();

     getRecordDefInitial.put("NAME", unit);
     getRecordDefInitial.put("FORMAT", "VD");
     getRecordDefInitial.put("TYPE", "GEODETIC");
     getRecordDefInitial.put("UNITS", unit);
     recDefs.put(unit, getRecordDefInitial);
     // Add a value to the row that will go with this column definition.
     row.put(unit, "(" + i + "," + i + "," + i + ")");
     i++;

   }

   // Set the record definition off of the table that we created
   df.setRecordDefs(recDefs);
   // Add a row that we have created that matches the record defs
   df.writeDataTable(row);
   // close/write the file
   df.close();
 }

 /**
  * This test will test the function {@link DataFile#getUnitsName(int)}. It will test that the function returns the correcty unit type (i.e. time (sec)).
  * @since NeXtMidas 3.8.1
  */
  public void testGetUnitsName () {
    for(int i = 1;i< Units.UNITS_INFO.length;i++) {
      if(Units.UNITS_INFO[i][1]!=null) { // If the value is null, no need to test
        assertEquals("The units name should be "+Units.UNITS_INFO[i][1] + " (" + (Units.UNITS_INFO[i][2]!=null?Units.UNITS_INFO[i][2]:"U") +")",Units.UNITS_INFO[i][1] + " (" + (Units.UNITS_INFO[i][2]!=null?Units.UNITS_INFO[i][2]:"U") +")", DataFile.getUnitsName(i));
      }
    }
  }

  /**
   * Test to check whether the function {@link DataFile#getUnitsID(String)} will return the correct integer value.
   * @since NeXtMidas 3.8.1
   */
  public void testGetUnitsID () {
    for(int i =0;i<Units.UNITS_INFO.length;i++) {
      if(Units.UNITS_INFO[i][1]!=null) { // If the value is null, no need to test
        assertEquals("The unit id should be equalt to "+i,i,DataFile.getUnitsID(Units.UNITS_INFO[i][0]));
      }
    }
  }

  /**
   * This will test that the values in the file are matching the values that are defined in {@link Units}
   * @since NeXtMidas 3.8.1
   */
  public void testFileUnitsName() {
    DataFile df = new DataFile(getMidasContext(), urlDat+"test5000Units.prm", "5000", "NH", DataFile.INPUT);
    df.open();
    String[] units = { "BAUD", "BIT_RATE", "POTENTIAL_DIFF", "CURRENT", "RESISTANCE", "CAPACITANCE", "INDUCTANCE",
        "TEMPERATURE", "PRESSURE", "RELATIVE_POWER", "RELATIVE_POWER_DENSE" }; //These are the values added for NXM-3201
    Table t = df.getRecord(0);
    for(int i = 0;i<units.length;i++) {
      assertEquals("(" + i + ".0," + i + ".0," + i + ".0)", t.get(units[i]).toString());
    }
    df.close();
  }

  /**
   * Test to check that the short value of the units is returned when give the id.
   * {@link DataFile#getUnitsNameShort(int)}
   * @since NeXtMidas 3.8.1
   */
  public void testGetUnitsNameShort () {
    for(int i =0;i<Units.UNITS_INFO.length;i++) {
      if(Units.UNITS_INFO[i][2]!=null) { // If the value is null, no need to test
        assertEquals("The unit id should be equalt to "+i,Units.UNITS_INFO[i][2],DataFile.getUnitsNameShort(i));
      }
    }
  }

  /**
   * Test to check that the var name is retrieved correctly by using the int value of the Unit.
   * {@link DataFile#getUnitsVarName(int)}
   * @since NeXtMidas 3.8.1
   */
  public void testGetUnitsVarName() {
    for(int i =0;i<Units.UNITS_INFO.length;i++) {
      if(Units.UNITS_INFO[i][0]!=null) { // If the value is null, no need to test
        assertEquals("The unit id should be equalt to "+i,Units.UNITS_INFO[i][0],DataFile.getUnitsVarName(i));
      }
    }
  }

  /** Test to make sure keywords, including BLUE TC TileLine Keywords can be modified
   *  @since NeXtMidas 3.9.1 for NXM-3185
   */
  public void testModifyKeywords() {

    String nmroot = Shell.getNmRoot();
    boolean isJarFile = nmroot.startsWith("jar:") || Shell.isWebStart() || nmroot.startsWith("classpath:") || Shell.isGradleBuild();
    if (isJarFile) {
      getMidasContext().info("Skipping modifying timeline keywords test, since running from jar");
      return;
    }

    // get url of nxm.sys.test area
    String urlSysTestString = IOResource.getInstance(Shell.getMidasContext(), IOResource.UNKNOWN,
        Shell.getOptPath("SYS", "TEST"), 0, null).getURL();
    if (Shell.isWindows()) {
      urlSysTestString = urlSysTest.replace("file:/", "");
    }
    else {
      urlSysTestString = urlSysTest.replace("file:", "");
    }

    // base names of test files in test area
    String test_file_tc_keywords_orig = "fft_wavechirp_tc.prm";
    String test_file_no_tc_keywords_orig = "fft_wavechirp_no_tc.prm";

    // Copy files over to temporary files in RAM, so the temporary files can be modified
    String test_file_tc_keywords = "fft_wavechirp_tc";
    String test_file_no_tc_keywords = "fft_wavechirp_no_tc";
    Path tcOutFile = null;
    Path noTCOutFile = null;
    try {
      tcOutFile = Files.createTempFile(test_file_tc_keywords, ".tmp");
      noTCOutFile = Files.createTempFile(test_file_no_tc_keywords, ".tmp");
      Files.copy(Paths.get(urlSysTestString+test_file_tc_keywords_orig), tcOutFile, StandardCopyOption.REPLACE_EXISTING);
      Files.copy(Paths.get(urlSysTestString+test_file_no_tc_keywords_orig), noTCOutFile, StandardCopyOption.REPLACE_EXISTING);
    }
    catch (IOException ioe) {
      fail("unable to copy files needed for testModifyKeywords()"+ioe);
    }

    // Run the TC Keywords update and TC Keywords addition test
    testUpdateAndAdditionOfTCKeywords(tcOutFile, noTCOutFile);
  }

  /** Tests the following cases involving updating BLUE high precision TC Keywords
   *  and ensures the resulting file has the proper keywords:<br>
   *  1) Update of a TimeLine entry - note that since the timeline is recreated<br>
   *     from keywords, major updates that involve shifting of numerous values are allowed.<br>
   *  2) Setting timeline keywords of file that does not have any
   *
   *  @param tcOutFile   Path for file containing TC Keywords, where one is going to get modified
   *  @param noTCOutFile Path for file without TC keywords, where TC keywords will be set
   */
  private void testUpdateAndAdditionOfTCKeywords (Path tcOutFile, Path noTCOutFile) {
    Midas M = getMidasContext();
    Table kwdsForUpdate = new Table("{NEWSTR=testString,D:TCSAMPLE_2=2.0,D:TC_WHOLE_2=2.146032E9,D:TC_FRAC_2=0.9}");
    // note original in file is D:TCSAMPLE_2=3.0,D:TC_WHOLE_2=2.146032001E9,D:TC_FRAC_2=0.228808
    try (DataFile dfTCKeywords = new DataFile(M,tcOutFile);
         DataFile dfNoTCKeywords = new DataFile(M,noTCOutFile)) {
      tcOutFile.toFile().deleteOnExit();
      noTCOutFile.toFile().deleteOnExit();

      // Open the file with TC Keywords
      dfTCKeywords.setFlags(BaseFile.INPUT|BaseFile.OUTPUT);
      dfTCKeywords.open();
      // save off TimeCode for next text
      Time origTimeCode = dfTCKeywords.getTime();
      // get this fie's current Keywords object and update some keywords
      Keywords kwTCFile = dfTCKeywords.getKeywordsObject();
      kwTCFile.putAll(kwdsForUpdate);
      // Since this file had a timeline, we need to update it based on the updated keywords
      // (otherwise, the actual timeline will not get updated and the updates will NOT appear in
      // the file when it is closed)
      dfTCKeywords.updateTimeLineFromKeywords();
      // save off the updated extended header keywords in a table (used during assert)
      kwTCFile.setScope("EXT");
      Table origKwds = kwTCFile.makeTable();
      dfTCKeywords.close();

      // Now we are switching to the file without TC Keywords and adding the set of TC Keywords
      dfNoTCKeywords.setFlags(BaseFile.INPUT|BaseFile.OUTPUT);
      dfNoTCKeywords.open();
      // set TimeCode matching TC Keywords
      dfNoTCKeywords.setTimeCode(origTimeCode);
      // put in all the TC Keywords (and others) from the other file
      Keywords kwNoTCFile = dfNoTCKeywords.getKeywordsObject();
      kwNoTCFile.putAll(origKwds);
      //Do not need to call updateTimeLineFromKeywords(), since this file does not have an existing timeline
      dfNoTCKeywords.close();

      // Reopen file that originally contained TC Keywords and make sure update took place
      dfTCKeywords.open();
      assertEquals("DataFile with TC Keywords had TC Keyword modified:",new Data(2.0),dfTCKeywords.getKeyWord("TCSAMPLE_2"));
      dfTCKeywords.close();

      // Reopen file that originally did NOT contain TC Keywords and make sure the TC Keywords were added
      dfNoTCKeywords.open();
      assertEquals("DataFile with No TC Keywords had TC Keywords added:",new Data(0.9),dfNoTCKeywords.getKeyWord("TC_FRAC_2"));
      dfNoTCKeywords.close();
    }

  }

  /**
   * Test the duration of a data file functionality.
   * This will create a temp file using waveform. It will have 256 elements with a delta of 2 so it will have a duration of 512 seconds.
   * @since 3.9.1
   */
  public void testGetTimeDuration () {
    Midas M = getMidasContext();
    if (Shell.isGradleBuild()) {
      M.warning("nxm.sys.junit.lib.DataTest skipping testGetTimeDuration() because fails in Gradle build");
      return;
    }
    int numElements = 256;
    // create a test file and open it
    String waveCmd = "WAVEFORM tmp_wave_file.tmp{aux=ram} FORM=CF ELEM="+numElements+" SHAPE=SINE DELTA=2 ";
    Shell.runCommand(M, waveCmd);
    DataFile df = new DataFile(M, "tmp_wave_file.tmp{aux=ram}");
    df.open();
    assertEquals("The time duration should be "+ numElements, df.getTimeDuration(),numElements*2);
    // close the test file
    df.close();
  }



  public void testIsBlueFile () {
    assertFalse(DataFile.isBlueFile(null, "test.badext"));
    String apennyName = (sysDatFull) ? "apenny" : "apenny.prm";
    assertTrue(DataFile.isBlueFile(null, urlDat+""+apennyName));
  }
  public void testIsBlueFileName () {
    String[] fileNames = new String[] {"test.tmp","test.prm","test.bdif","test.cdif","test.cwdw","test.ncwdw","test.npdw","test.pdw","test.ppdw","test.rpdw","test.sdw","test.spdw","test.wpdw","test.wcwdw"};
    for(String file : fileNames) {
      assertTrue(DataFile.isBlueFileName(file));
    }
    assertFalse(DataFile.isBlueFileName("test.badext"));
  }

  public void testIsBlueFileMagic () throws IOException {
    String[] validMagic   = { "BLUEIEEEIEEE", "BLUEIEEEEEEI", "BLUEEEEIIEEE", "BLUEEEEIEEEI" };
    String[] invalidMagic = { "BLUE--------", "BLUEIEEE----", "BLUE----IEEE", "----IEEEIEEE",
                              "ABCDEFGHIJKL" };
    File     temp         = File.createTempFile("myfile", ".junk");

    try {
      for (String magic : validMagic) {
        byte[] header = new byte[512];
        Convert.packS(header, 0, 12, magic);

        try (FileOutputStream out = new FileOutputStream(temp)) {
          out.write(header);
        }
        assertTrue("isBlueFileMagic(null,"+temp+") with '"+magic+"'", DataFile.isBlueFileMagic(null,temp));
        assertTrue("isBlueFile(null,"+temp+") with '"+magic+"'",      DataFile.isBlueFile(null,temp));
      }

      for (String magic : invalidMagic) {
        byte[] header = new byte[512];
        Convert.packS(header, 0, 12, magic);

        try (FileOutputStream out = new FileOutputStream(temp)) {
          out.write(header);
        }
        assertFalse("isBlueFileMagic(null,"+temp+") with '"+magic+"'", DataFile.isBlueFileMagic(null,temp));
        assertFalse("isBlueFile(null,"+temp+") with '"+magic+"'",      DataFile.isBlueFile(null,temp));
      }
    }
    finally {
      temp.delete();
    }
  }

//  public void testIsCompleteBlueFile () {
//    String waveChirp = "nxm/sys/test/fft_wavechirp_no_tc.prm";
//    File waveChirpFile = new File(waveChirp);
//
//    Path p = FileSystems.getDefault().getPath("nxm", "sys", "test", "fft_wavechirp_no_tc.prm");
//    assertTrue(waveChirpFile + " is a valid blue file", DataFile.isCompleteBlueFile(waveChirpFile));
//    assertTrue(DataFile.isCompleteBlueFile(p));
//    String kmlFile = "nxm/sys/test/kmlTest.kml";
//    assertFalse(kmlFile + " is not a valid blue file", DataFile.isCompleteBlueFile(new File(kmlFile)));
//    assertFalse(kmlFile + " is not a valid blue file",
//        DataFile.isCompleteBlueFile(FileSystems.getDefault().getPath("nxm", "sys", "test", "kmlTest.kml")));
//
//  }
//  public void testReadPartial () {
//    DataFile dfin = new DataFile(getMidas(), "nxm.sys.test.sinewave_incomplete.prm");
//    dfin.open();
//    int xfer = (int) dfin.getSize();
//    String expected = "WARN: DataFile sinewave_incomplete.prm does not have expected size only able to read 64.0 out of 4096 elements" + (char)10;
//    Data data = dfin.getDataBuffer(xfer);
//    ByteArrayOutputStream baos = new ByteArrayOutputStream();
//    OutputStream old = getMidas().getTerminal().getOutputStream();
//    // Steal all the output of the terminal
//    getMidas().getTerminal().setOutputStream(baos);
//    while (dfin.read(data) > 0);
//    getMidas().getTerminal().setOutputStream(old);
//    assertEquals("The message should have been intercepted by the outputstream provided", expected,baos.toString());
//    dfin.close();
//    baos.reset();
//    getMidas().warning("This is a test message");
//    assertEquals("the message should write to the original output stream for Terminal", "", baos.toString());
//  }

//  public void partialReadHelper (DataFile df) {
//    int xfer = (int) df.getSize();
//    Data data = df.getDataBuffer(xfer);
//    while (df.read(data) > 0);
//  }
}
