!******************************************************************************!
! Confidence macro for NeXtMidas Option trees
!
!  Author: Jay Hallam / J. Czechowski IV / Neon Ngo
!******************************************************************************!
startmacro s:option
call checkAuxPath option

! reset homepath and home aux for testing
homepath/store test option
AUX/store

global g_status
assert/init=g_status

archive on confidence.log

switch "MISS"         S:showMissing  DEF=TRUE    SDEF=TRUE  ! Show missing tests?
switch "BASELINE"     S:baseln       DEF=FALSE   SDEF=TRUE  ! Baseline testing?
switch "INTERACTIVE"  S:interAct     DEF=^baseln SDEF=TRUE  ! Interactive?
switch "WEB"          S:web          DEF=FALSE   SDEF=TRUE  ! Do web tests?
switch "DBG"          S:dbg          DEF=FALSE   SDEF=TRUE  ! Submacro debug?
switch "RESUME"       S:resume       DEF=""      SDEF=""    ! Skip all test macros until reaches this one
switch "STOPAT"       S:stopAtCmd    DEF=""      SDEF=""    ! Skip all test macros after this one
switch "DEBUGCONF"    Z:debugConf    DEF=FALSE   SDEF=TRUE  ! internal use only - for debugging this confidence macro
switch "COREGRAPHICS" S:coregraphics DEF=FALSE   SDEF=TRUE  ! for a quick check of graphics - more extreme tests - since 3.8.1
switch "UNITTEST"     S:unittest     DEF=TRUE    SDEF=TRUE  ! for running unittests- since 3.8.1
switch "ALTTESTENV"   S:alttestenv   DEF=FALSE   SDEF=TRUE
! need to figure this out - do not want a nxm.sys.net dependency here
!invoke statemDefault nxm.sys.net.Rmif.getStateMachineLogicAsDefault()
res statemDefault false
switch "STATEM"       S:statemachine DEF=^statemDefault   SDEF=TRUE ! Statemachine mode for Rmif testing

if baseln isTrue and web isFalse then
  warn "CONFIDENCE: Running tests in /BASELINE mode, add /WEB to include web server tests."
endif

! For historical reasons, make the switch values numerical
set l:interAct  test(interAct)
set l:web       test(web)
set l:dbg       test(dbg)

timer set

if resume eqs "" and unittest then
  ! Run the unit tests
  run "unittest OPT=^option RES=unit_test_res"
else
  INFO "skipping unit tests for OPT=^option"
  res t:unit_test_res {summary="SKIPPED Unit tests",status=0}
endif

! open the config table
fname cfn test test_config tbl ^option
if cfn fexists
  set t:config cfn
else
  set t:config {TESTLIST=}
endif

! init counters
set numTests {total=0,notest=0,broken=0,missing=0,run=0,failed=0,byanother=0}

! run all tests in config list
foreach cmd inlist config.testList
  call runTest cmd
endfor

! run individual command tests
invoke supports nxm.sys.lib.CommandLine.VALID_SUPPORTS
set fn_opt_cmds_list ram:temp_confidence_command_list
menu/f=fn_opt_cmds_list opt=^option sup=^supports
new java.util.HashSet() hashset
new java.util.ArrayList() dupList
foreach cmd inDF fn_opt_cmds_list
  invoke noExistingEntry hashset.add(cmd)
  if noExistingEntry isFalse then
    warn "SKIPPING already executed test case for ^cmd"
    invoke ,, dupList.add(^cmd)
    continue
  endif
  call runTest cmd
endfor
invoke numDups dupList.size()
if numDups gt 0 then
  warn "^numDups duplicate commands found for [^option] option tree. dupList: ^dupList"
endif
erase/all fn_opt_cmds_list

ASSERT/TID="ALL"/TOTAL=TESTALL

! restore original state
HOMEPATH/restore
AUX/restore

set outstr "Ran ^numtests.run macro tests, &
            ^numtests.failed failed. &
            (^numtests.broken known drs, &
            ^numtests.notest N/A, &
            ^numtests.byanother dups"
if showMissing then
  set outstr "^outStr, ^numtests.missing missing)"
else
  set outstr "^outStr)"
endif

say ""
if numtests.failed GT 0 then
  warn "^outstr"
else
  info "^outstr"
endif

! Re-print the unit test info.
if unit_test_res.status LT 0 then
  warn unit_test_res.summary
else
  info unit_test_res.summary
endif

say ""
if numtests.failed GT 0 or unit_test_res.status LT 0 then
  warn "FAILED TESTS"
else
  info "ALL TESTS PASSED"
endif

! Display total run time to run test cases
timer elapse
timer off

endmacro

!******************************************************************************!
procedure runTest s:cmd

  ! check if we want to skip all test macros prior to specified command
  if cmd.startsWith(resume) then
    ! reset resume to blank so that all test macros after it runs
    if resume NEQS "" then set resume ""
  else
    INFO "skipping test: ^cmd"
    return
  endif

  ! check if we want to skip all test macros after specified command
  if cmd.startsWith(stopAtCmd) AND stopAtCmd NEQS "" then
    info "^reg.main.name: stopping at command ^cmd"
    set z:noMoreCommands true
  endif

  if noMoreCommands REXISTS then
    !!!INFO "skipping test: ^cmd"
    return
  endif

calc numTests.total numTests.total+1

if config contains cmd
  set options config.^cmd
else
  set options "NONE"
endif
if options eqs "NOTEST" then
  ! command not appropriate for testing
  calc L:numtests.notest numtests.notest+1

elseif options eqs "BROKEN" then
  ASSERT/TID=^cmd/text="^CMD: Test skipped - known DR." /noop=BROK
  calc L:numtests.broken numtests.broken+1

elseif options eqs "NONE" or &
       options eqs "WEB" then
  fname macroFile TEST TEST_^CMD MM ^option
  if macrofile FEXISTS then
    set doTest "FALSE"
    if options eqs "NONE" then
      set doTest "TRUE"
    elseif options eqs "WEB" AND web then
      set doTest "TRUE"
    endif
    if doTest then
      ! since 3.1.0: We are putting /debug, /interactive, and /baseline at end of argument list so
      !              that they automatically get inherited by (propagated to) sub-macros (BUG 2531)
      run "%test_^cmd/mfn=macrofile  /debug=^dbg /interactive=^interAct /baseline=^baseln /coregraphics=^coregraphics /alttestenv=^alttestenv /statem=^statemachine"
      ASSERT/TID="^cmd"/TOTAL=TEST/stat=status
      if status NEQS "PASS" calc L:numtests.failed numtests.failed+1
      calc L:numtests.run numtests.run+1
    endif
  elseif showMissing then
    ASSERT/TID=^cmd/text="^CMD: No test macro found " /noop=MISS
    calc L:numtests.missing numtests.missing+1
  endif

else   ! not one of the special names
  ASSERT/TID=^cmd/text="^CMD: Tested in ^options test " /noop=DUPL
  calc L:numtests.byanother numtests.byanother+1
endif
return

!******************************************************************************!
! Checks to make sure that the AUX path is correctly setup before running the tests.
!   (1) Write AUX must exist.
!   (2) Write AUX must be writable.
!   (3) Read AUX must include write AUX.
!   (4) Read AUX must include RAM.
!   (5) Read AUX must include DAT* (for SYS, XBC, NXM (NeXtOpt) option trees)
!
!   (*May be renamed to 101 when run from X-Midas)
!******************************************************************************!
procedure checkAuxPath s:option
  set readAuxList "|^{AUX.READ}|"

  ! Can not perform tests without a WRITE aux.
  fname LABEL=wdir AREA=^aux.write OPT=AUX
  if wdir NDEXISTS then
    ! Bug 2053 (allow RAM as Write AUX to test all option trees except SYS)
    if wdir neqs "ram:" then
      warning "CONFIDENCE: Can not perform tests: Write aux ^wdir, does not exist"
      stop
    elseif option eqs "SYS" then
      warning "CONFIDENCE: Can not perform [^option] tests with write AUX set to RAM"
      stop
    endif
  endif

  ! create temporary file to see if we can write to the WRITE aux
  ! as some test cases needs to create temporary files
  set tempfile temp_confidence_test_write_aux.tmp{AUX=^{AUX.WRITE}}
  header/create ^tempfile
  if ^tempfile NFEXISTS then
    warning "CONFIDENCE: Can not perform tests: write path ^wdir, is not writable"
    stop
  else
    erase ^tempfile
  endif

  ! Check to see that RAM is on the read aux.
  if "|^{AUX.WRITE}|" NSUBS readAuxList then
    warning "CONFIDENCE: Can not perform tests: Need to have ^{AUX.WRITE},RAM,DAT on read aux."
    stop
  endif

  ! Check to see that RAM is on the read aux.
  if "|RAM|" NSUBS readAuxList then
    warning "CONFIDENCE: Can not perform tests: Need to have ^{AUX.WRITE},RAM,DAT on read aux."
    stop
  endif

  if ",^{option}," SUBS ",SYS,XBC,NXM," then
    ! Check to see that DAT is on the read aux (DAT may use a number when run from X-Midas)
    if "|DAT|" NSUBS readAuxList then
      if "|101|" NSUBS readAuxList then
        warning "CONFIDENCE: Can not perform tests: Need to have ^{AUX.WRITE},RAM,DAT on read aux."
        stop
      elseif testxy3000.prm nfexists then ! AUX.101 does not points to normal AUX.DAT
        warning "CONFIDENCE: Can not perform tests: Need to have ^{AUX.WRITE},RAM,DAT on read aux."
        stop
      endif
    endif
  endif
return
