!******************************************************************************!
! NeXtMidas test multiple lines in a cell in csv file.  Qualifiers should 
! include FLAGS=EMBEDDEDLF in cells contain line feeds. 
! e.g. "0\n1\n2"
!
! Also tests regular CSV files (as NXM-2931 caused a regression)
! 1) that do not have a LF at the end
! 2) that have spaces or tabs within a field (should remain)
!    Note: be VERY careful if editing nxm.sys.test.test_csv_tab_quote_at_end.csv
!    as it intentionally has TABs that should NOT be converted to spaces
!
! Arguments: None
!
! @Issue    NXM-2931 CsvFile doesn't parse multi-line fields
!           NXM-3155 CsvFile:Regression - CSV file no longer recognizes spaces
! @author:  Elizabeth Ryan & Jeanette McCall
!******************************************************************************!
startmacro

call setDatDir

call testMultiLine  ! NXM-2931 - support for multi-line fields
call testSpaceNoLF  ! NXM-3155 - spaces within fields should remain and last row needs to be added even when it does not end with a LF
call testTabQuote   ! NXM-3155 - tabs within fields should remain and last row needs to be added even when it ends with'"' and not a LF

endmacro

!******************************************************************************!
! If an unexpected exception is caught, mark test as failed, unless it is expected
!******************************************************************************!
procedure processException m:emsg
  if expectedException REXISTS then
    assert/text="^reg.main.name: Expected ERROR starts with: ^{expectedException}" expectedException EQSS emsg.data.message
    remove expectedException
  else
    assert/text="^reg.main.name: Caught unexpected exception ^{emsg.data}" "FALSE" isTrue
    if "Trace" SUBS ENV.DEBUG then invoke ,, emsg.data.printStackTrace()
  endif
return

!******************************************************************************!
! Set the dat directory
!******************************************************************************!
procedure setDatDir
  invoke nxmMajor env.nmversion.subString(0,1)
  invoke major java.lang.Integer.valueOf(nxmMajor)
  if major GE 4 then
    invoke s:datDir           nxm.sys.test.DatLocator.getDatDirLocation()
    invoke s:datDirSlash      nxm.sys.test.DatLocator.getDatDirSlashLocation()
    invoke z:sysDatFull       nxm.sys.test.DatLocator.sysDatFull()
    invoke z:sysTestDatExists nxm.sys.test.DatLocator.sysTestDatExists()
  else
    set s:datDir           "nxm.sys.dat"
    set s:datDirSlash      "nxm/sys/dat"
    set z:sysDatFull       true
    set z:sysTestDatExists false
  endif
return

!******************************************************************************!
! Test for special CSV with multi-line fields
!******************************************************************************!
procedure testMultiLine

  ! set qualifier to indicate line feeds are embedded within quoted strings
  set quals {HEADERROWS=1,FLAGS=EMBEDDEDLF}

  CONVERT ^{datDir}.convert_special.csv^{quals} csv2tbl tbl_multi

  assert/text="^reg.main.name: multi-line CSV2TBL row=ROW_1" &
               ^{tbl_multi.ROW_1.X->2} EQS "0\n0\n0" AND &
               ^{tbl_multi.ROW_1.Y} EQS "4\n4"

  assert/text="^reg.main.name: multi-line CSV2TBL row=ROW_2" &
               ^{tbl_multi.ROW_2.X->2} EQS "10" AND &
               ^{tbl_multi.ROW_2.Y} EQS "are, there more"
      
invoke rows tbl_multi.getSize()
assert/text="^reg.main.name: multi-line CSV2TBL size = 9 rows" rows EQ 9

return

!******************************************************************************!
! Test for CSV with spaces within fields and no CR at end of file
!******************************************************************************!
procedure testSpaceNoLF

set quals {HEADERROWS=1}

  CONVERT nxm.sys.test.test_csv_spaces_no_lf_at_end.csv^{quals} csv2tbl tbl_space

  ! Check that space within field remains, but at end of field is not
  assert/text="^reg.main.name: only spaces in middle of fields are kept" &
               ^{tbl_space.ROW_1.NAME} EQS "John Doe" AND &
               ^{tbl_space.ROW_1.AGE} EQ 21

  ! Check that last row has been added to table and is as expected
  assert/text="^reg.main.name: space, no LF at end - ROW_2 as expected" &
               ^{tbl_space.ROW_2.NAME} EQS "Jane Doe" AND &
               ^{tbl_space.ROW_2.AGE} EQ 23

  ! confirm that all the rows got added, even though the last row does not have a LF at the end
  invoke rows tbl_space.getSize()
  assert/text="^reg.main.name: space, no LF at end - got all the rows" rows EQ 2

return


!******************************************************************************!
! Test for CSV with tabs within fields and '"' end of file
!******************************************************************************!
procedure testTabQuote

  ! set qualifier to indicate line feeds are embedded within quoted strings
  set quals {HEADERROWS=1,FLAGS=EMBEDDEDLF}

  ! IMPORTANT: This CSV file has tabs.
  ! If you need to edit it, make sure the tabs ARE NOT converted to spaces
  CONVERT nxm.sys.test.test_csv_tab_quote_at_end.csv^{quals} csv2tbl tbl_tab

  res address "15040 Conference Center Dr,\nChantilly, VA 20151"

  ! check that tabs within the field remain, but those at the beginning or end are removed
  assert/text="^reg.main.name: only tabs in middle of fields are kept" &
               ^{tbl_tab.ROW_1.NAME} EQS "John\tDoe"

  ! check that tabs remain and last row is added to table and is as expected
  assert/text="^reg.main.name: tab, multi-line, quote at end - ROW_2 as expected" &
        ^{tbl_tab.ROW_2.NAME} EQS "Jane\tDoe" AND &
        ^{tbl_tab.ROW_2.AGE} EQ 23 AND &
        ^{tbl_tab.ROW_2.ADDRESS} EQS ^address

  ! confirm that all the rows got added, even though the last row does not have a LF at the end
  invoke rows tbl_tab.getSize()
  assert/text="^reg.main.name: tab, multi-line, quote at end - got all the rows" rows EQ 2

return