! Automated test for NEWRATE for the special case where the
! downsampling is so great that inStep is greater than nTaps
!
! Note: test_newrate_plot_nstep_gt_ntaps is the corresponding
!       interactive test, that requires running X-Midas NEWRATE
!       and comparing results
!
! @issue   NXM-3613
! @author  Jeanette McCall
! @since   NeXtMidas 3.9.4
!************************************************************
startmacro s:func[""]
  switch INTERACTIVE interAct   GET DEF=TRUE  SDEF=TRUE  ! for normal interactive test(s)
  switch BASELINE    baseline   GET DEF=FALSE SDEF=TRUE  ! for defects or more obscure test case(s)
  switch KEEP        keep_files GET DEF=FALSE SDEF=TRUE  !
  switch DEBUG       debug      GET DEF=FALSE SDEF=TRUE  !

  res input_file_6250000hz   newrate_6250000hz_input.tmp

  timex "2020:09:09::10:09:04.22" sec=tsec
  ! values below chosen to somewhat failure scenario
  res numelem    4275775
  res delta      1.6E-7
  res initial_hz 1/delta
  res out_hz     8000
  res out_hz2    8311
  ! test freq to stay within nyquist of output rate
  res l:freq1    1000
  res l:freq2    2000
  res l:freq3    3000


  if func NEQS "" then
    call test_^{func}
  else
    ! test with simple low frequency waveforms
    ! (compare to waveform generated at lower sample rate)
    call test_specified_freq SF SIN 1 delta numelem ^tsec out_hz2 3
    call test_specified_freq SF TRI 1 delta numelem ^tsec out_hz2 0.1
    call test_specified_freq SF SIN 10  delta numelem ^tsec out_hz2 3
    call test_specified_freq SF TRI 10  delta numelem ^tsec out_hz2 0.8

    ! test with some slightly higher frequencies
    call test_specified_freq SF SIN 100 delta numelem ^tsec out_hz2 3

    ! test with other data types
    call test_specified_freq SB SIN 100 delta numelem ^tsec out_hz2 3
    call test_specified_freq SI SIN 100 delta numelem ^tsec out_hz2 3
    call test_specified_freq SL SIN 100 delta numelem ^tsec out_hz2 3
    call test_specified_freq SD SIN 100 delta numelem ^tsec out_hz2 3
    call test_specified_freq CB SIN 100 delta numelem ^tsec out_hz2 3
    call test_specified_freq CI SIN 100 delta numelem ^tsec out_hz2 3
    call test_specified_freq CL SIN 100 delta numelem ^tsec out_hz2 3
    call test_specified_freq CF SIN 100 delta numelem ^tsec out_hz2 2
    call test_specified_freq CD SIN 100 delta numelem ^tsec out_hz2 2

    ! test with a different output rate
    call test_specified_freq SF SIN 100 delta numelem ^tsec out_hz 3
    call test_specified_freq CD SIN 100 delta numelem ^tsec out_hz 2

    ! create tri-frequency waveform with all frequencies within nyquist for new rate
    call create_tri_freq_wave input_file_6250000hz SF SIN delta numelem ^tsec freq1 freq2 freq3

    ! test with integer decimation rate (compare to thin)
    res l:int_decimation 1000
    calc out_rate initial_hz int_decimation /
    res test_msg "^{freq1},^{freq2},^{freq3}hz"
    call test_integer_decimation input_file_6250000hz int_decimation out_rate test_msg 3

    ! test with direct comparison to XM NEWRATE results
    call test_vs_xm input_file_6250000hz initial_hz out_hz2 &
         xm_newrate_^{initial_hz}_to_^{out_hz2}_seg.csv "(~2.230798144422E9:~2.230798144522E9)"

    ! test piped with direct comparison to XM NEWRATE results
    call test_pipe_small_tl_vs_xm input_file_6250000hz initial_hz out_hz2 &
         xm_newrate_^{initial_hz}_to_^{out_hz2}_seg.csv "(~2.230798144422E9:~2.230798144522E9)"

    ! Test use of sync file to adjust inStep
    call test_sync

    ! Test with frequency that is above nyquist.
    ! Both X-Midas and NeXtMidas NEWRATE do not handle that filtering
    ! The frequency will fold into the band
    call create_quad_freq_wave newrate_input_above_nyquist.tmp SF SIN delta numelem ^tsec freq1 freq2 freq3 8811
    res expected {l:HZ1=^freq1,l:HZ2=^freq2,l:HZ3=^freq3,l:HZ4=500}
    call test_output_frequencies newrate_input_above_nyquist.tmp 8311 6 "signal with freq > nyquist" ^expected 0.04
    if KEEP_FILES ISFALSE then erase newrate_input_above_nyquist.tmp

  endif

  if KEEP_FILES ISFALSE then erase input_file_6250000hz

endmacro

!******************************************************************************!
! Main process message for this macro. Should be right after the ENDMACRO.
!******************************************************************************!
procedure processMessage m:msg
  if msg.name eqs "MAIN" then
    assert/text="^reg.main.name: User indicated tests" msg.data eqss "PASS"
    pipe stop
  else
    say "Unhandled message name: ^msg.name from fid: ^msg.fid data: ^msg.data"
  endif
return

!******************************************************************************!
! If an unexpected exception is caught, mark test as failed, unless it is expected
!******************************************************************************!
procedure processException m:emsg
  if expectedException REXISTS then
    assert/text="^reg.main.name: Expected ERROR starts with: ^{expectedException}" expectedException EQSS emsg.data.message
    remove expectedException
  else
    assert/text="^reg.main.name: Caught unexpected exception ^{emsg.data}" "FALSE" isTrue
    if "Trace" SUBS ENV.DEBUG then invoke ,, emsg.data.printStackTrace()
  endif
return

!******************************************************************************!
! Test NEWRATE with input of the specified frequency wave with high sampling rate
! Compares output of NEWRATE to same waveform generated at lower sampling rate
!******************************************************************************!
procedure test_specified_freq S:format S:wshape d:freq_hz d:in_delta d:in_elem d:start_time i:new_rate d:tol

  res  out_delta 1/new_rate
  calc out_elem  in_elem out_delta in_delta / / CEIL
  res  amp       1

  ! create file names and display them
  invoke lc_wshape wshape.toLowerCase()
  res  newrate_input "newrate_^{lc_wshape}_^{freq_hz}hz_in.tmp"
  res  newrate_output "newrate_^{lc_wshape}_^{freq_hz}hz_out.tmp"
  res  rough_expected "^{lc_wshape}_^{freq_hz}hz_^{new_rate}.tmp"

  if debug then
    info "test_^{freq_hz}_freq input  file name: ^newrate_input"
    info "test_^{freq_hz}_freq output file name: ^newrate_output"
    info "test_^{freq_hz}_freq comparison file: ^rough_expected"
  endif 

  waveform newrate_input  format elem=in_elem shape=wshape freq=freq_hz amp=amp delta=in_delta start=start_time
  newrate newrate_input newrate_output new_rate
  waveform rough_expected format elem=out_elem shape=wshape freq=freq_hz amp=amp delta=out_delta start=start_time

  if DEBUG then plot rough_expected|newrate_output
  assert/text="^reg.main.name: ^{freq_hz}hz ^format ^wshape decimation to ^{new_rate}Hz" newrate_output FEQ/D/T rough_expected /TOL=^tol

  if KEEP_FILES ISFALSE then
    erase newrate_input
    erase newrate_output
    erase rough_expected
  endif

return

!******************************************************************************
! Create a waveform composed of 3 frequencies
!******************************************************************************
procedure create_tri_freq_wave fn:fname S:format S:wshape d:tri_delta d:tri_elem d:start_time d:freq_1 d:freq_2 d:freq_3

  res first_freq   ram:newrate_first_freq.tmp
  res second_freq  ram:newrate_second_freq.tmp
  res third_freq   ram:newrate_third_freq.tmp

  waveform first_freq  format elem=tri_elem shape=wshape freq=freq_1 amp=amp delta=tri_delta start=^start_time
  waveform second_freq format elem=tri_elem shape=wshape freq=freq_2 amp=amp delta=tri_delta start=^start_time
  waveform third_freq  format elem=tri_elem shape=wshape freq=freq_3 amp=amp delta=tri_delta start=^start_time
  fcalc fname first_freq second_freq +
  fcalc fname fname      third_freq  +

return

!******************************************************************************
! Create a waveform composed of 4 frequencies
!******************************************************************************
procedure create_quad_freq_wave fn:fname S:format S:wshape d:quad_delta d:Quad_elem d:start_time d:freq_1 d:freq_2 d:freq_3 d:freq_4

  res first_freq   ram:newrate_first_freq.tmp
  res second_freq  ram:newrate_second_freq.tmp
  res third_freq   ram:newrate_third_freq.tmp
  res fourth_freq   ram:newrate_fourth_freq.tmp

  waveform first_freq  format elem=quad_elem shape=wshape freq=freq_1 amp=amp delta=quad_delta start=^start_time
  waveform second_freq format elem=quad_elem shape=wshape freq=freq_2 amp=amp delta=quad_delta start=^start_time
  waveform third_freq  format elem=quad_elem shape=wshape freq=freq_3 amp=amp delta=quad_delta start=^start_time
  waveform fourth_freq  format elem=quad_elem shape=wshape freq=freq_4 amp=amp delta=quad_delta start=^start_time
  fcalc fname first_freq second_freq +
  fcalc fname fname      third_freq  +
  fcalc fname fname      fourth_freq +

return

!*********************************************************************************
! Compare NEWRATE with an integer decimation rate vs the corresponding THIN
!*********************************************************************************
procedure test_integer_decimation fn:input_fname l:decimation l:new_rate S:wave_info d:tol

  res output_newrate newrate_int_decimation_^{decimation}.tmp
  res thin_file      newrate_thin_^{new_rate}.tmp

  newrate input_fname output_newrate new_rate
  thin input_fname thin_file inc=decimation
  assert/text="^reg.main.name: ^wave_info newrate vs thin ^new_rate" output_newrate FEQ/D/T thin_file /TOL=tol

  if KEEP_FILES ISFALSE then
    erase output_newrate
    erase thin_file
  endif

return

!*********************************************************************************
! Compare XM NEWRATE output with NeXtMidas NEWRATE output
!*********************************************************************************
procedure test_vs_xm fn:input_fname l:orig_rate l:new_rate fn:xm_csv_seg_fname S:trimmer

  res nxm_newrate_out          newrate_^{orig_rate}hz_to_^{new_rate}hz.tmp
  res nxm_newrate_out_seg      newrate_^{orig_rate}hz_to_^{new_rate}hz_seg.tmp
  res nxm_newrate_out_csv_seg  newrate_^{orig_rate}hz_to_^{new_rate}hz_seg.csv
  res xm_csv_seg_fpath         "nxm.dsp.test.^{xm_csv_seg_fname}"

  newrate input_fname nxm_newrate_out new_rate
  noop nxm_newrate_out^trimmer nxm_newrate_out_seg 
  file copy/l/f nxm_newrate_out_seg nxm_newrate_out_csv_seg

  assert/text="^reg.main.name: NEWRATE output matches ^{xm_csv_seg_fname}" &
       nxm_newrate_out_csv_seg{HEADERROWS=L:1,COLUMNNAMES="SF:F1"} FEQ/D &
       xm_csv_seg_fpath{HEADERROWS=L:1,COLUMNNAMES="SF:F1"}

  if KEEP_FILES ISFALSE then files/e newrate_^{orig_rate}hz_to_^{new_rate}hz*.*
return

!*********************************************************************************
! Compare XM NEWRATE output with PIPED NeXtMidas NEWRATE output
!*********************************************************************************
procedure test_pipe_small_tl_vs_xm fn:input_fname l:orig_rate l:new_rate fn:xm_csv_seg_fname S:trimmer

  res nxm_newrate_out          newrate_piped_^{orig_rate}hz_to_^{new_rate}hz.tmp
  res nxm_newrate_out_seg      newrate_piped_^{orig_rate}hz_to_^{new_rate}hz_seg.tmp
  res nxm_newrate_out_csv_seg  newrate_piped_^{orig_rate}hz_to_^{new_rate}hz_seg.csv
  res xm_csv_seg_fpath         "nxm.dsp.test.^{xm_csv_seg_fname}"

  pipe on
    noop input_fname _wave /TL=30
    newrate _wave _newrate_out new_rate
    noop _newrate_out nxm_newrate_out

  pipe off

  noop nxm_newrate_out^trimmer nxm_newrate_out_seg
  file copy/l/f nxm_newrate_out_seg nxm_newrate_out_csv_seg

  assert/text="^reg.main.name: NEWRATE piped output matches ^{xm_csv_seg_fname}" &
       nxm_newrate_out_csv_seg{HEADERROWS=L:1,COLUMNNAMES="SF:F1"} FEQ/D &
       xm_csv_seg_fpath{HEADERROWS=L:1,COLUMNNAMES="SF:F1"}

  if KEEP_FILES ISFALSE then files/e newrate_piped_^{orig_rate}hz_to_^{new_rate}hz*.*

return

!*********************************************************************************
! Test using sync
! Major disclaimer: This test checks against X-Midas results with the same sync file
! This scenario uses the new large decimation code and enters the synch "if( ncleft < 1 ) {"
! to adjust inStep. But, this test case was NOT developed by a true DSP person,
! so it may have deficiencies.
!*********************************************************************************

procedure test_sync
  res wave_freq 400

  WAVEFORM,OUT=newrate_wave,FORM=CF,ELEM=1089536,SHAPE=SIN,AMP=1,FREQ=wave_freq,PHASE=0,START=0,DELTA=3.04e-5
  fft/psd/1D newrate_wave   newrate_wave_spec   OVER=0.5 NAVG=16 NFFT=32k
  statistics newrate_wave_spec   AMAX=IN_FREQ

  newrate newrate_wave newrate_resamp 822.368421053  nxm.dsp.test.sample_sync_file.tmp

  fft/psd/1D newrate_resamp  newrate_resamp_spec OVER=0.5 NAVG=16 NFFT=32k
  statistics newrate_resamp_spec AMAX=OUT_FREQ

  ! XM_OUT_FREQ was obtained by effectively running the same newrate command in X-Midas
  ! by copying over newrate_wave and nxm.dsp.test.sample_sync_file.tmp to HOME (X-Midas read dir)
  !
  ! X-Midas> newrate newrate_wave xm_newrate_resamp 822.368421053 sample_sync_file.tmp
  !
  ! and then running the following in a development version of this test case
  !
  ! fft/psd/1D xm_newrate_resamp  xm_newrate_resamp_spec OVER=0.5 NAVG=16 NFFT=32k
  ! statistics xm_newrate_resamp_spec AMAX=XM_OUT_FREQ
  !
  res XM_OUT_FREQ 399.61463526693484

  if DEBUG then
    plot newrate_wave_spec|newrate_resamp_spec|xm_newrate_resamp_spec
  endif
  
  assert/text="NEWRATE Input FREQ reasonably matches expected" wave_freq EQ/T IN_FREQ  /TOL=0.5
  assert/text="NEWRATE Output FREQ matches X-Midas" XM_OUT_FREQ EQ OUT_FREQ

  if KEEP_FILES ISFALSE then
    files/e newrate_wave*
    files/e newrate_resamp*
  endif

return

!*********************************************************************************
! Test that output contains the expected frequencies
!*********************************************************************************
procedure test_output_frequencies  fn:input_fname l:new_rate l:num_peaks S:test_msg t:expected_freqs d:tol

   newrate input_fname newrate_beyond_nyquist.tmp ^new_rate
   fft newrate_beyond_nyquist.tmp fft_newrate_beyond_nyquist.tmp
   peakpick fft_newrate_beyond_nyquist.tmp peakpick_beyond_nyquist_hz.tmp peakpick_beyond_nyquist_val.tmp num_peaks

   if DEBUG then
     plot/bg fft_newrate_beyond_nyquist.tmp
     list2/bg peakpick_beyond_nyquist_hz.tmp
     list2/bg peakpick_beyond_nyquist_val.tmp
   endif

   ! Figure out how many peaks are close in value to the main peak
   ! Note: Peaks are stored largets to smallest
   invoke saveTol  nxm.sys.libm.Tolerance.getTolerance()
   invoke ,, nxm.sys.libm.Tolerance.setTolerance(tol)
   file open tag peakpick_beyond_nyquist_val.tmp
   res max_value tag.getdata(0,F1)
   res l:col_num 0
   res num_equal 1
   calc l:num_loops num_peaks 1 -
   loop ^num_loops
     calc l:col_num col_num 1 +
     res col_name F^col_num
     res current tag.getdata(0,^col_name)
     if current EQ/T max_value then
       calc num_equal Num_equal 1 +
     endif
   ENDLOOP
   file close tag
   invoke ,, nxm.sys.libm.Tolerance.setTolerance(^saveTol)

   ! Create actual table with these frequencies
   file open tag peakpick_beyond_nyquist_hz.tmp
   ! 
   res actual_freqs {}
   loop ^num_equal loop_num
     calc l:col_num loop_num 1 -
     res col_name F^col_num
     res key_name HZ^loop_num
     ! get frequency
     res current tag.getdata(0,^col_name)
     ! round to nearest 10 hz
     calc current current 10 /
     invoke l:hz_approx java.lang.Math.round(current)
     calc l:hz_approx hz_approx 10 *
     ! save value in actual table
     set actual_freqs.^key_name ^hz_approx
   endloop
   file close tag

   ! check results
   table expected_freqs getsize expected_size
   table actual_freqs   getsize actual_size
   assert/text="^reg.main.name: ^test_msg - # of peaks" actual_size EQ expected_size
   foreach freq_value INTABLE/V expected_freqs
     assert/text="^reg.main.name: ^test_msg contains ^freq_value" actual_freqs CONTAINS/V freq_value
   endfor

  if KEEP_FILES ISFALSE then files/e *_beyond_nyquist*

return
