package nxm.sys.lib;

import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.regex.Pattern;

import nxm.sys.inc.InternalUseOnly;
import nxm.sys.inc.Units;
import nxm.sys.lib.Keywords.Iterator;
import nxm.sys.lib.Keywords.Key;

/** Stores a series of time vs index for general use by DataFiles. <br>
    <br>
    The nominal delta between values is typically specified as the x-delta in a {@link DataFile}.
    If the deltas differ more than the given tolerance ({@link #getTolerance()}), a new entry is
    placed in the time line. <br>
    <br>
    This class was initially designed to handle relatively smooth-flowing data that had occasional
    gaps (caused by brief interruptions in the data). This  class ignores minor deviations in the
    timecode (via tolerance) and adds entries to handle the gaps.<br>
    <br>
    Users frequently get into trouble when they use this class for other than its intended purpose
    (e.g. data where the delta is not constant) or, more commonly, use this class with a delta or
    tolerance that do not match the actual data.<br>
    <br>
    If the delta and tolerance do not match the actual data rate users may observe the following
    behavior:
    <pre>
      (1) The TimeLine overflows the max number of entries (see {@link #getAlloc()}
          causing the beginning of the file to have no timecode entries
          (in a PLOT this will often appear in the readout as "t=0").

      (2) The computed time values between TimeLine entries may be wrong.
          This is often seen when time updates are less frequent than the
          data (e.g. timecode is updated only with every Nth frame). This
          can cause several problems:

            (a) The extrapolated values after the last entry in the
                TimeLine are higher/lower than the actual time.

            (b) The extrapolated values in the middle of a TimeLine
                are not monotonically increasing and could exhibit
                points where the TimeLine "goes back in time."

            (c) In a PLOT the readout and features displayed on it
                may appear incorrect. This is a direct result of
                PLOT using extrapolated values that are just plain
                wrong. (Since the readout in a PLOT is based on mouse
                position, it rarely matches one-for-one with frame
                boundaries and ends up using extrapolated values most
                of the time.)

            (d) In the case of a pipe, all "down stream" primitives
                can end up with bad timecode since they are always
                using the last timeline entry or an extrapolation
                from it based on a bad delta.

      (3) If the number of TimeLine entries is allowed to grow (setAlloc(-1)
          in NextMidas 2.5.0 and later), the TimeLine may grow extremely
          large since each frame of data could necessitate a new entry in
          the TimeLine.
    </pre>
    In short, if you are going to use this class, you need to set delta and tolerance correctly.

    @author  Jeff Schoen
    @author  Neon Ngo - added support for high precision times
*/
public class TimeLine extends Object {
  /** The default time line length (TLL) to use. */
  public static final int    DEF_ALLOC     = 128;
  /** The default high precision time line length (TLL) to use. */
  @InternalUseOnly
  public static final int    DEF_HIGH_PRECISION_ALLOC = -4096;
  /** The default time line tolerance (TLT) to use. */
  public static final double DEF_TOLERANCE = 0.1; // Default tolerance
  /** The microsecond precision time line tolerance (TLT) to use. */
  @InternalUseOnly
  public static final double MICROSECOND_TOLERANCE = 1e-6;
  /** The quarter nanosecond precision time line tolerance (TLT). */
  @InternalUseOnly
  public static final double QUARTER_NANOSECOND_TOLERANCE = 250e-12;

  /** Allocation of -1 means grow as needed up to default maximum */
  @InternalUseOnly
  public static final int GROW_AS_NEEDED = -1;

  /** The grow-to ratio for TC time lines. */
  @InternalUseOnly
  public static final double TC_GROW_TO_RATIO = 0.25;

  /** The maximum time line length when allocation is GROW_AS_NEEDED.
   *   @since 3.7.3 for NXM-3041
   */

  /** Regex expression for filtering time-code keywords. The time-code keyword patterns include:
   *  "TCSAMPLE_[one or more digits]"
   *  "TC_FRAC_[one or more digits]"
   *  "TC_WHOLE_[one or more digits]"
   *  "TIMELINE"
   *  "TIMETAG.OFFSET"
   *  "TIMETAG.TIME"
   *  "TIMETAG.UNITS"
   *  "TIME_DELTA"
   *  "TIME_EPOCH"
   *  "TIME_OFFSET"
   *  "TIME_TICKS"
   */
  public static final String TIMELINE_KEYWORD_REGEX = "T((C((SAMPLE_[0-9]+)|(_((FRAC|WHOLE)_[0-9]+))))|(IME(LINE|"
                                                    + "(TAG\\.(OFFSET|(TIME(|\\.UNITS))))|(_(DELTA|EPOCH|OFFSET|TICKS)))))";
  /** timeline keyword pattern */
  public static final Pattern TIMELINE_KEYWORD_PATTERN = Pattern.compile(TIMELINE_KEYWORD_REGEX);

  /** default maximum timeline size - used with grow as needed */
  @InternalUseOnly
  public static final int DEF_MAX_ALLOC  = 1_000_000;

  /** The reporting level for invalid TimeLine entries */
  @InternalUseOnly
  public enum NotifyInvalidEntry {
      /** no invalid Timeline entry */
      NONE,
      /** warning for invalid Timeline entry */
      WARNING,
      /** exception for invalid Timeline entry */
      EXCEPTION
    }

  // Note: In cases where the timeline length is fixed (alloc!=-1) the
  //       entries will "wrap around" the array. 'current' always indicates
  //       the last entry written.

  private int      current;        // Index of last time line entry
  private int      alloc;          // User-set TimeLineLength (TLL), (-1 = Grow as needed up to 1M max)
  private int      size;           // Number of time line entries used
  private double[] offsets;        // Byte offsets into file for time entries (see times).
                                   //   In bytes for compatibility with PIPES where output
                                   //   pipe can use different formats.
  private Time[]   times;          // Time values for time line entries (see offsets)
  private double   dt;             // Time delta (sec/byte)
  private double   tolr;           // Time tolerance (sec)
  private boolean  interp;         // Use interpolation rather than extrapolation.
  private boolean  editingEnabled; // Enable correcting past entries and inserting
                                   //   of missed entries that are within buffer
  private boolean  timelineHasWrapped = false;  // Timeline has reached the end of the buffer and has wrapped - since 3.8.0
  private static int usecPrecisionAllocDefault = DEF_ALLOC;  // allow setting of global default
  private static int highPrecisionAllocDefault = DEF_HIGH_PRECISION_ALLOC;  // allow setting of global default
  private static final int DP_WARN = 9; // number decimal places for times in warnings

  /** timeline keyword input/output format option. @since NeXtMidas 3.3.0 */
  static enum Format {
    /** Use NeXtMidas 3.2.x and prior TIMELINE keyword which has microsecond precision (double values - seconds since J1950 epoch). */
    MICROSECOND_PRECISION,
//    /** (TODO: Not implemented) Use NeXtMidas 3.3.x+ high precision TIMELINE keyword (double values - seconds since HCB.timecode + TC_PREC Keyword epoch + ?XStart?). */
//    HIGH_PRECISION,
    /** Use Platinum BLUE Data Exchange Format Standard timeline keywords for very high precision. */
    PLATINUM_HIGH_PRECISION,
    /** Use high precision BLUE TC (Time Code) keywords as outlined in the JICD 4.2 ICD . */
    BLUE_HIGH_PRECISION,
  };

  /** current time line input/output format to use (defaults to microsecond precision) */
  Format format = Format.MICROSECOND_PRECISION;

  /** Creates a new timeline with default values for length and tolerance.
      @param delta The delta between entries in sec/byte.
      @since NeXtMidas 2.5.0
   */
  public TimeLine (double delta) {
    this(usecPrecisionAllocDefault, delta, DEF_TOLERANCE);
  }

  /** Creates a new timeline with given TimeLine Length (TLL) and TimeLine Tolerance (TLT).
      @param tll   Max number of TimeLine entries (-1=Grow As Needed).
      @param delta The delta between entries in sec/byte.
      @param tlt   Allowable tolerance (smooths out small errors), in sec.
   */
  public TimeLine (int tll, double delta, double tlt) {
    current = -1;
    size = 0;
    setAlloc(tll);
    setDelta(delta);
    setTolerance(tlt);
    setEditingEnabled(false);
  }

  /** Adds a time+offset entry to the end of the timeline. This is intended to
      be used for internal initialization similar to:
      <pre>
        TimeLine tl = new TimeLine(tll, delta, tlt);
        for (...) {
          addTimeOffset(...);
        }
      </pre>
      This function presumes that the tll used is sufficient and that no other
      entries have been added yet. This function is private since it is for
      internal use only and is very different from the normal <code>setTimeAt(..)</code>
      functions and is not intended for user interaction.
      @param time Time entry
      @param offset in bytes
   */
  private void addTimeOffset (Time time, double offset) {
    current++;
    size++;
    times[current]   = time;
    offsets[current] = offset;
  }

  /** Should interpolation be used rather than extrapolation between TiimeLine entries?
      @return true if interpolation is on
      @since NeXtMidas 2.5.0
   */
  public boolean getUseInterpolation () { return interp; }

  /** Indicates if interpolation should be used rather than extrapolation between TiimeLine
      entries. Note that extrapolation is always used after the last entry in the TimeLine.
      This is should only be turned on when both of the following are true:
      <pre>
        o The time delta ({@link #getDelta()}) is likely to be inaccurate.
        o The data is continuous (no gaps).
      </pre>
      @param interp Turns interpolation on (true) or off (false).
      @since NeXtMidas 2.5.0
   */
  public void setUseInterpolation (boolean interp) { this.interp = interp; }

  /** get the time delta between offsets (in sec/byte)
   *  @return time delta (in sec/byte)
   */
  public double getDelta () { return dt; }

  /** Set the time delta between offsets (in sec/byte) (aka rate of change for extrapolation).
      The TimeLine uses a single delta, changing this in the middle of the TimeLine will change
      the output of the {@link #getOffsetAt(double)} and {@link #getTimeAt(double)} methods.
      @param delta The time delta (rate) between bytes (in sec/byte).
      @throws IllegalArgumentException if delta is invalid: infinity, NaN, or zero (since 3.3.0)
   */
  public void setDelta (double delta) {
    if (Double.isInfinite(delta) || Double.isNaN(delta) || delta == 0)
      throw new IllegalArgumentException("Invalid TimeLine delta (sec/byte) = "+delta);
    dt = delta;
  }

  /** get the time tolerance (in seconds)
   *  @return time tolerance
   */
  public double getTolerance () { return tolr; }

  /** set the time tolerance (in seconds)
   *  @param tolerance time tolerance (in seconds)
   */
  public void setTolerance (double tolerance) { tolr = tolerance; }

  /** Return number of elements used in the timeline (see also {@link #getAlloc()}).
   *  @return number of elements used in the timeline
   */
  public int getSize () { return size; }

  /** Return max number of number of entries permitted in the timeline. Since NeXtMidas 2.5.0
      this will return <code>-1</code> if there is no fixed limit (i.e. the TimeLine will grow as
      needed). See {@link #getSize()} for the current number of entries used.
      @return max number of number of entries permitted in the timeline
   */
  public int getAlloc () { return alloc; }

  /** Sets the max number of number of entries permitted in the timeline. Since NeXtMidas 2.5.0
      can be set to <code>-1</code> so that the TimeLine will grow as needed. Care should be taken
      when setting this to <code>-1</code> since the TimeLine could grow extremely large if the
      delta ({@link #getDelta()}) or tolerance ({@link #getTolerance()}) is wrong.
      @param newSize max number of entries (-1 = Grow As Needed).
      @since NeXtMidas 2.5.0
   */
  public void setAlloc (int newSize) {
    if (newSize == GROW_AS_NEEDED) { // Since 2.5.0 alloc=-1 to grow as needed
      alloc = newSize;
      realloc(Math.max(DEF_ALLOC, size));
    } else if (newSize < -DEF_ALLOC) {
      alloc = newSize;
      realloc(Math.max(DEF_ALLOC,Math.max((int)(-newSize*TC_GROW_TO_RATIO), size)));
    } else if (newSize < 1) {
      alloc = 1;
      realloc(1);
      Shell.getSharedMidasContext().deprecate("TimeLine: Since NeXtMidas 2.5.0: Use of alloc="+alloc
                                       +" is deprecated. Use alloc > 0; or alloc=-1 for auto.");
    } else {
      alloc = newSize;
      realloc(alloc);
    }
  }

  /** Sets the default usec-precision TimeLine allocation.
   *  @param allocDefault new usec default allocation
   *  @since NeXtMidas 3.9.4
   */
  @InternalUseOnly
  public static void setUsecPrecisionAllocDefault (int allocDefault) {
    usecPrecisionAllocDefault = getValidDefault(allocDefault);
  }

  private static int getValidDefault(int allocDef) {
    if (allocDef == GROW_AS_NEEDED) return allocDef; // Since 2.5.0 alloc=-1 to grow as needed
    else if (allocDef < -DEF_ALLOC) return allocDef;
    else if (allocDef < 1) return 1;
    else return allocDef;
  }

  /** Gets the current default usec-precision TimeLine allocation.
   *  @return current usec default allocation
   *  @since NeXtMidas 3.9.4
   */
  @InternalUseOnly
  public static int getUsecPrecisionAllocDefault () {
    return usecPrecisionAllocDefault;
  }

  /** Sets the default high precision TimeLine allocation.
   *  @param allocDefault new high precision default allocation
   *  @since NeXtMidas 3.9.4
   */
  @InternalUseOnly
  public static void setHighPrecisionAllocDefault (int allocDefault) {
    highPrecisionAllocDefault = getValidDefault(allocDefault);
  }

  /** Gets the current default tc-precision TimeLine allocation.
   *  @return current tc default allocation
   *  @since NeXtMidas 3.9.4
   */
  @InternalUseOnly
  public static int getHighPrecisionAllocDefault () {
    return highPrecisionAllocDefault;
  }

  /** Returns whether timeline editing is enabled
   *
   *  @return true if timeline editing/correction is enabled
   */
  public boolean isEditingEnabled () {
    return editingEnabled;
  }

  /** Set whether editing of older timeline values is enabled.  When enabled,
   *  missed values can be inserted and incorrect values can be overridden.
   *  This is limited to values that are currently in the timeline buffer.  If
   *  the timeline has overflowed and wrapped, only value not overridden can be
   *  edited. When the timeline has not wrapped, inserting an additional value
   *  into a full timeline is not allowed.  The user should increase the timeline
   *  length before inserting additional values.
   *
   *  @param enabled editing/correction of old timeline values is enabled
   */
  public void setEditingEnabled (boolean enabled) {
    editingEnabled = enabled;
  }

  /** Reallocates the offsets and times arrays. This method is ONLY used internally, everyone
      else should call setAlloc(..).
   */
  private void realloc (int toAlloc) {
    double[] oldOffsets = offsets;
    Time[]   oldTimes   = times;

    if (oldOffsets == null) {
      // Initial setup -- fast, no copy needed
      offsets = new double[toAlloc];
      times   = new Time[toAlloc];
    } else if (oldOffsets.length == toAlloc) {
      // nothing to do
    } else {
      offsets = new double[toAlloc];
      times   = new Time[toAlloc];

      // Be careful to handle wrap-around correctly.
      boolean noWrap = (size < oldOffsets.length) || (current == oldOffsets.length);
      if (noWrap && (offsets.length > oldOffsets.length)) {
        // Be fast for the 90% case
        System.arraycopy(oldOffsets, 0, offsets, 0, oldOffsets.length);
        System.arraycopy(oldTimes,   0, times,   0, oldOffsets.length);
      } else {
        int start = 0;
        if (oldOffsets.length > offsets.length) {
          start = offsets.length - oldOffsets.length; // Decrease size
        }
        for (int i = 1, j = start; i <= oldOffsets.length; i++, j++) {
          int index = (current + i) % oldOffsets.length;

          if (j >= 0) {
            offsets[j] = oldOffsets[index];
            times[j]   = oldTimes[index];
          }
        }
      }
    }
    current = size-1;
  }

  /** Add new entry (up to microseconds precision) if it cannot be described within tolerance by the last entry.
      @param offset file offset (in bytes).
      @param time   time to set (number of J1950 seconds).
      @return true if entry was added (since 3.7.3)
      @see #setTimeAt(double, Time) setTime(double offset, Time time) for a high precision version
   */
  public boolean setTimeAt (double offset, double time) {
    // Since 3.7.3, throw an Exception if the user tries to set an invalid entry - NXM-3040
    return setTimeAt(offset, new Time(time), NotifyInvalidEntry.EXCEPTION);
  }

  /** Add new high precision entry if it cannot be described within tolerance by the last entry.
      @param offset file offset (in bytes).
      @param time   high precision time to set if NOT within tolerance of last entry;
                    (null and zero/negative entries are ignored).
      @return true if entry was added (since 3.7.3)
      @throws MidasException for invalid entries, unless
                             {@link CoreIO.IOOptions#SuppressInvalidTimeLineEntryErrors}
                             is set
      @since NeXtMidas 3.3.0
   */
  public boolean setTimeAt (double offset, Time time) {
    // Since 3.7.3, throw an Exception if the user tries to set an invalid entry - NXM-3040
    return setTimeAt(offset, time, NotifyInvalidEntry.EXCEPTION);
  }

  /**
   * Add new high precision entry if it cannot be described within tolerance by the last entry
   * @param index file offset in elements (zero based)
   * @param dbpe  bytes per element
   * @param time  high precision time to set if NOT within tolerance of last entry;
   *              (null and zero/negative entries are ignored).
   * @return true if entry was added (since 3.7.3)
   * @throws MidasException if entry is an invalid entry
   * @since NeXtMidas 3.7.3
   */
  public boolean setTimeAt (double index, double dbpe, double time) {
    // throw an Exception if the user tries to set an invalid entry - NXM-3040
    return setTimeAt(index, dbpe, new Time(time), NotifyInvalidEntry.EXCEPTION);
  }

  /**
   * Add new high precision entry if it cannot be described within tolerance by the last entry
   * @param index file offset in elements (zero based)
   * @param dbpe  bytes per element
   * @param time  high precision time to set if NOT within tolerance of last entry;
   *              (null and zero/negative entries are ignored).
   * @return true if entry was added (since 3.7.3)
   * @throws MidasException for invalid entries, unless
   *               {@link CoreIO.IOOptions#SuppressInvalidTimeLineEntryErrors}
   *                is set
   * @since NeXtMidas 3.7.3
   */
  public boolean setTimeAt (double index, double dbpe, Time time) {
    // Since 3.7.3, throw an Exception if the user tries to set an invalid entry - NXM-3040
    return setTimeAt(index, dbpe, time, NotifyInvalidEntry.EXCEPTION);
  }

  /**
   * Add new high precision entry if it cannot be described within tolerance by the last entry.
   * @param index file offset in elements (zero based)
   * @param dbpe  bytes per element
   * @param time  high precision time to set if NOT within tolerance of last entry;
   *              (null and zero/negative entries are ignored).
   * @param notifyLevel notification level for invalid timeline entries (none, warning, or exception)
   * @return true if entry was added (since 3.7.3)
   * @since NeXtMidas 3.7.3 for NXM-3040
   */
  @InternalUseOnly
  public boolean setTimeAt (double index, double dbpe, Time time, NotifyInvalidEntry notifyLevel) {
    return setTimeAt(index*dbpe, time, notifyLevel);
  }

  /**
   * Add new high precision entry if it cannot be described within tolerance by the last entry.
   * Note: Editing of earlier timeline entries (excluding the oldest) can be enabled
   * via {@link #setEnableEditing(boolean)}
   *
   * @param offset file offset (in bytes).
   * @param time  high precision time to set if NOT within tolerance of last entry;
   *              (null and zero/negative entries are ignored).
   * @param notifyLevel notification level for invalid timeline entries (none, warning, or exception)
   *                    This can be overridden by CoreIO.SuppressInvalidTimeLineEntryErrors
   * @return true if entry was added (since 3.7.3)
   * @throws MidasException for bad timeline entry if notification level is EXCEPTION
   */
  private boolean setTimeAt (double offset, Time time, NotifyInvalidEntry notifyLevel) {
    // setTimeAt (double offset, Time time) code moved into this private method
    // in 3.7.3 to allow better error reporting for invalid entries
    if (time == null || time.getSec() <= 0) return false;
    int i = current;
    if (i<0) { }                        // first entry
    else if (offset==offsets[i]) i--;   // overwrite last entry
    else {                              // is this sufficiently described by the last entry
      double terr = time.diff(times[i]) - (offset - offsets[i])*dt;
      if (Math.abs(terr)<=tolr) return false; // last entry is OK (within tolerance)

      // Handle conditions where this time set is invalid / probably invalid
      // (increased error handling added in 3.7.3 for NXM-3040)
      boolean disableErrorMsg = Shell.getMidasContext().io.isOptionSet(
          CoreIO.IOOptions.SuppressInvalidTimeLineEntryErrors);
      String errorCondition = null;
      if ((offset < offsets[i]) && (i!=0 || editingEnabled))  {    // editing or inserting earlier entry
        // since 3.7.3 for NXM-3042 timelines can now be edited if an
        // entry was missed or needs to be corrected
        // Since 3.9.1 for NXM-3301, also handle case that happens with repeat pipe inits (i==0)
        if (editingEnabled) {
          return insertEntry(offset, time, notifyLevel);
        }
        else {
          if (!disableErrorMsg) {
            String notification = "";
            if(Shell.getMidasContext().io.isOptionSet(CoreIO.IOOptions.DoublePropagateTimelineKeywordsInPipe)) {
              notification = "\n\t- It is recommended to turn off the CoreIO option DoublePropagateTimelineKeywordsInPipe";
            }
            errorCondition = "ERROR: Adding in a missed TimeLine entry with an " +
                "earlier offset is currently disabled "+
                "- Offset - Current:"+offset+" Previous:"+offsets[i]+"\n" +
                "         Time- Current:"+time.toString(DP_WARN)+" Previous:"+times[i].toString(DP_WARN)+
                "    Index - "+i+"  thread name:"+Thread.currentThread().getName()+
                notification;
            reportInvalidEntry (errorCondition, notifyLevel);
          }
          // Since 3.8.0, go back to old behavior of adding this entry
        }
      }
      else if ((offset > offsets[i]) && (time.diff(times[i]) < 0)) {
        // increasing offset, but decreasing time value
        if (!disableErrorMsg) {
          errorCondition = "ERROR: Invalid TimeLine entry that is at later offset, " +
              "but is back in time "+
              "- Offset - Current:"+offset+" Previous:"+offsets[i]+"\n" +
              "         Time- Current:"+time.toString(DP_WARN)+" Previous:"+times[i].toString(DP_WARN)+
              "    Index - "+i+"  thread name:"+Thread.currentThread().getName();
          reportInvalidEntry (errorCondition, notifyLevel);
        }
      }
      else if (terr < -1e7) {
        // extreme back in time value (over 4 months) is almost certainly invalid
        if (!disableErrorMsg) {
          errorCondition = "Skipping probably invalid timeline entry " +
              "(offset:"+offset+" time:"+time+"), which is more than 4 months" +
              " back in time with respect to expected time. "+
              "- Offset - Current:"+offset+" Previous:"+offsets[i]+"\n" +
              "         Time- Current:"+time.toString(DP_WARN)+" Previous:"+times[i].toString(DP_WARN)+
              "    Index - "+i+"  thread name:"+Thread.currentThread().getName();
          reportInvalidEntry (errorCondition, notifyLevel);

        }
        return false; // skip this entry
      }
    }

    // add new timeline entry
    i = i + 1;
    if (i == offsets.length) {
      // timeline buffer is full: either wrap or grow buffer
      if ((alloc == GROW_AS_NEEDED) && (offsets.length < DEF_MAX_ALLOC)) {
        // double size until reach maximum allowed under grow as needed
        int newLength = Math.min(offsets.length*2,DEF_MAX_ALLOC);
        realloc(newLength); // resize
      }
      else if ((alloc <= -DEF_ALLOC) && (offsets.length < -alloc)) {
        // double size until reach maximum allowed under grow to size
        int newLength = Math.min(offsets.length*2,-alloc);
        realloc(newLength); // resize
      }
      else{
        // Warning added in 3.8.0
        boolean disableErrorMsg = Shell.getMidasContext().io.isOptionSet(
            CoreIO.IOOptions.SuppressInvalidTimeLineEntryErrors);
        timelineHasWrapped = true; // getting ready for first wrap of timeline, which will result in a bad timeline
        if (!timelineHasWrapped && !disableErrorMsg && (notifyLevel != NotifyInvalidEntry.NONE)) {
          Shell.warning("This TimeLine buffer is full - wrapping entries (alloc="+alloc+" i="+i+" thread name:"+Thread.currentThread().getName()+")");
        }
        i = 0;                     // just wrap around and overwrite
      }
    }
    offsets[i]=offset;
    if (times[i] == null) times[i] = new Time(time); // copy into new instance
    else                  times[i].fromTime(time);   // set in existing instance
    size=Math.max(i+1,size);
    current=i;
    return true;
  }

  /**
   * Overwrite older timeline entry or insert missed value.  The new entry be within
   * the range of the existing entries.  Overriding the oldest or newest entries is
   * not allowed (adjacent entries used to check for time value in rage).
   * @param offset offset file offset (in bytes) (offset must be greater than the
   *               smallest offset and smaller than the largest offset to be inserted)
   * @param time   high precision time to set
   * @param notifyLevel
   * @return true in entry was successfully inserted (or ovverwritten)
   */
  private boolean insertEntry (double offset, Time time, NotifyInvalidEntry notifyLevel) {
    String errorCondition = null;
    boolean disableErrorMsg = Shell.getMidasContext().io.isOptionSet(
        CoreIO.IOOptions.SuppressInvalidTimeLineEntryErrors);
    boolean haveWrapped = current < size - 1;
    int minOffsetIdx = haveWrapped ? current + 1 : 0; // oldest entry may be 0 or oldest one that has not been written over
    int maxOffsetIdx = current;  // current always last entry and latest offset/time
    if (offset >= offsets[maxOffsetIdx]) {
      throw new IllegalArgumentException("offset ["+offset+"] >= greatest offset in buffer "
                                          + offsets[maxOffsetIdx]);
    }
    else if (offset <= offsets[minOffsetIdx]) {
      if (!disableErrorMsg) {
        errorCondition = "Invalid Insertion: where offset <= minimum offset in buffer:" +
            " (offset:"+offset+" time:"+time+") where minimum offset is " + offsets[minOffsetIdx];
        reportInvalidEntry (errorCondition, notifyLevel);
      }
      return false;
    }
    else {
      // default to searching from beginning of buffer to last value entered
      int minSearch = 0;
      int maxSearch = current;
      if (haveWrapped && (offset <= offsets[size-1])) {
        // search old entries before last wrap
        minSearch = current;
        maxSearch = size;
      }
      // find the insertion point
      int insertionPt = Arrays.binarySearch(offsets, minSearch, maxSearch, offset);
      // convert to insertion index
      int insertIdx = (insertionPt < 0) ? (Math.abs(insertionPt) - 1) : insertionPt;
      // check times[] before and after this point to see if time is in range
      boolean overwriting = insertIdx == insertionPt;
      int time1Idx = (insertIdx == 0) ? size-1 : insertIdx - 1; // need to handle wrap condition
      // if overwriting [n], then need to check [n-1] and [n+1],
      // if inserting at [n], then need to check [n-1] and [n]
      int time2Idx = (overwriting) ? (insertIdx + 1) : insertIdx;
      if ((time.compareTo(times[time1Idx]) <= 0) || (time.compareTo(times[time2Idx]) >= 0)) {
        if (!disableErrorMsg) {
          errorCondition =
              "Invalid Insertion: time is out of range - not adding entry at index "
              + insertIdx + " for  (offset:" + offset + " time:" + time + ")";
          reportInvalidEntry (errorCondition, notifyLevel);
        }
        return false;
      }
      if (overwriting) {
        // overwrite entry
        times[insertIdx] = time;
      }
      else {
        boolean atEndBuffer = (current == (offsets.length - 1));
        if (!haveWrapped && atEndBuffer) {
          // timeline buffer is full: either wrap or grow buffer
          if ((alloc == GROW_AS_NEEDED) && (offsets.length < DEF_MAX_ALLOC)) {
            // double size until reach maximum allowed under grow as needed
            int newLength = Math.min(offsets.length*2,DEF_MAX_ALLOC);
            realloc(newLength); // resize
          }
          else if ((alloc < -DEF_ALLOC) && (offsets.length < alloc)) {
            // double size until reach maximum allowed under grow to size
            int newLength = Math.min(offsets.length*2,alloc);
            realloc(newLength); // resize
          }
          else {
            if (!disableErrorMsg) {
              errorCondition =
                  "Invalid Insertion: timeline full and is currently an " +
                  "unwrapped valid timeline - please increase size to insert " +
                  "entry (offset:"+offset+" time:"+time+") at index:"+ insertIdx;
              reportInvalidEntry (errorCondition, notifyLevel);
            }
            return false;
          }
        }
        // Inserting at or below current index if have not wrapped or are inserting
        // in portion after last wrap
        boolean insertAtIdxBelowCurrent = insertIdx <= current;
        current++;
        if (insertAtIdxBelowCurrent) {
          if (!haveWrapped) size++; // adding additional timeline element
          timelineInsertShiftRight(offset, time, insertIdx, current);
        }
        else {
          // have to do circular shift
          // shift up elements above insertion point saving last buffer values to be inserting at beginning
          double savedOffset = offsets[size -1];
          Time savedTime = times[size-1];
          timelineInsertShiftRight(offset, time, insertIdx, offsets.length - 1);
          timelineInsertShiftRight(savedOffset, savedTime, 0, current);
        }
      }
      return true;
    }
  }

  /**
   * Shift timeline entries in order to be able to insert this entry in the correct location
   * @param offset    offset (in bytes) of new entry
   * @param time      high precision Time of new entry
   * @param insertIdx index where it needs to be inserted
   * @param lastIdx   new index of last timeline entry
   */
  private void timelineInsertShiftRight(double offset, Time time, int insertIdx, int lastIdx) {
    for (int idx = lastIdx; idx > insertIdx; idx--) {
      offsets[idx] = offsets[idx-1];
      times[idx] = times[idx-1];
    }
    // insert entry
    offsets[insertIdx] = offset;
    times[insertIdx] = time;
  }

  /**
   * Report invalid timeline entries based on the notification level
   * @param errorCondition description of of invalid entry
   * @param notifyLevel    reporting level
   * @since 3.7.3 for NXM-3040
   */
  private void reportInvalidEntry (String errorCondition, NotifyInvalidEntry notifyLevel) {
    if (errorCondition != null) {
      switch (notifyLevel) {
        case WARNING:
          Shell.warning(errorCondition);
          break;
        case EXCEPTION:
          throw new MidasException(errorCondition);
        case NONE:
          break;
      }
    }
  }

  /** Get time (high precision) entry for specified offset in file.
      @param offset file offset (in bytes).
      @param time   {@link Time} object to populate (null to create a new instance)
      @return Time entry for specified offset (zero Time when timeline is empty).
              This is the same as input param when it is not null.
      @since NeXtMidas 3.3.0
   */
  public Time getTimeAt (double offset, Time time) {
    if (time == null) {
      time = new Time();
    }
    int i = current; // Current index
    if (i < 0) {     // is the timeline empty?
      return time.fromJ1950x(0.0, 0.0); // matches previous implementation of returning 0.0;
    }

    double iOffset  = offsets[i]; // i-th offset
    if (offset >= iOffset) {      // is this equal to/later than the last time entry?
      time.fromTime(times[i]);
      return time.addSec( (offset-iOffset)*dt ); // Extrapolation from previous entry using dt
    }

    // go back and look for it
    for (int j=0; j<size; j++) {
      int k = i;       // Last index (for extrapolation)
      if (--i<0) i = size - 1;
      iOffset = offsets[i];
      if (offset >= iOffset) {
        time.fromTime(times[i]);
        if (interp && (j != 0)) {
          // Linear interpolation between two points
          double add = (times[k].diff(time) * (offset-iOffset)) / (offsets[k]-iOffset);
          return time.addSec(add);
        } else {
          // Extrapolation from previous point using dt
          return time.addSec( (offset-iOffset)*dt );
        }
      }
    }
    return time.fromJ1950x(0.0, 0.0); // not found (matches previous implementation of returning 0.0;)
  }

  /** get the time (seconds since J1950) from the current entries
      @param offset file offset (in bytes).
      @return Time entry for specified offset (zero Time when timeline is empty).
      @see #getTimeAt(double,Time) getTime(double offset, Time time) for a high precision version
   */
  public double getTimeAt (double offset) {
    return getTimeAt(offset, new Time()).getSec();
  }

  /** get the file offset (in bytes) for a given time from the current entries.
      @param time time to set (number of J1950 seconds).
      @return file offset for specified time (in bytes).
      @see #getOffsetAt(Time) getOffset(Time) for a high precision version
   */
  public double getOffsetAt (double time) {
    return getOffsetAt(new Time(time));
  }

  /** get the file offset (in bytes) for a given time from the current entries.
      @param time high precision time to find offset (cannot be null).
      @return file offset for specified time (in bytes), 0 if timeline is empty.
      @since NeXtMidas 3.3.0 (extracted originally from getOffsetAt(double))
   */
  public double getOffsetAt (Time time) {
    int i = current; // Current index

    // is the timeline empty?
    if (i<0) return 0.0;

    // is this later than the last time entry ?
    double tdiff = time.diff(times[i]);
    if (tdiff >= 0) return Math.floor(offsets[i] + tdiff/dt);

    // go back and look for it
    for (int j=0; j<size; j++) {
      int k = i;       // Last index (for extrapolation)
      if (--i<0) i=size-1;
      tdiff = time.diff(times[i]);
      // protect from possibly invalid times[i] entry
      if (tdiff>=0 && tdiff<1e7) {
        if (interp && (k != current)) {
          // Linear interpolation between two points
          return Math.floor((((offsets[k]-offsets[i]) * tdiff) / (times[k].diff(times[i]))) + offsets[i]);
        } else {
          // Extrapolation from previous point using dt
          return Math.floor(offsets[i] + tdiff/dt);
        }
      }
    }
    return 0.0;
  }

  /** Used to purge entries from the TimeLine.
   *  @param toOffset first offset to be preserved (purging most recent to oldest)
   */
  public void purge (double toOffset) {
    if (toOffset<=0) return;
    int i = current;
    for (; i>0 && offsets[i]>toOffset; i--) { }
    if (i==0) return;
    size = current+1-i;
    for (int j=0; j<size; j++) {
      offsets[j] = offsets[j+i];
      times[j] = times[j+i];
    }
  }

  /** <b>INTERNAL USE ONLY:</b> Get as Data object for keywords. This is for internal use between
      {@link DataFile} and {@link Keywords}, it is <code>public</code> so that it can be used by custom
      file handlers in other option trees. The {@link Data} object holds <code>"SD"</code> data in
      the following form:
      <pre>
        data[0]     = {@link #getDelta()}
        data[1]     = {@link #getTolerance()}
        data[2*i+2] = offsets[i]
        data[2*i+3] = times[i]

        Note: The times and offsets do not start at i=0. When the TimeLine gets full
              and wraps around the offsets and times begin overwriting at i=0. For
              example:
                                                v
                  times = { 12, 13, 14, 15, 16, 5, 6, 7, 8, 9, 10, 11 }
                                                ^
        Note2: The times are seconds since J1950 epoch (giving microsecond precision)
      </pre>
      This is the same form that is stored in the extended header keywords of a NeXtMidas BLUE file.
      @return timeline as a Data object
   */
  @InternalUseOnly
  public Data toData () {
/*
        Note2: When format == {@link Format#MICROSECOND_PRECISION} then the times are seconds since J1950 epoch.
BEGIN TODO: NOT IMPLEMENTED -->
               Otherwise (format == {@link Format#HIGH_PRECISION}) and the times are seconds since file's start time (timecode + TC_PREC keyword + xstart?) as epoch,
               which allows high precision times: e.g.
                 2^(-40) (~9.09E-13) gives picosecond precision
                   2^(53-40) = 2^13 = 8192 max seconds
                   8192 seconds / 60 seconds/minute ~ 136 minutes ~ 2 hours
                 2^(-30) (~9.31E-10) gives nanosecond precision
                   2^(53-30) = 2^23 = 8388608 max seconds
                   8388608 seconds / 86400 seconds/day ~ 97 days
(picosecond precision for 75 minute duration from epoch ??? 2^13 (8192) seconds / 60 min/sec ~ 69 minute duration <-- TODO double check
-or- nanosecond precision for ~52 day duration from epoch) ->> vs 2^22 / 86400 sec/day ~ 48 day duration <-- TODO double check
END TODO: NOT IMPLEMENTED

      NOTE: This is the non-high precision version (up to microsecond precision)
            for backwards compatibility with NeXtMidas versions prior to 3.3.0.
 */
    int j=0;
    Data data = new Data("SD",size*2+2);
    data.setD( j++, dt );
    data.setD( j++, tolr );
    int oldest = (timelineHasWrapped) ? ((current + 1) % offsets.length) : 0;
    int numEntries = (timelineHasWrapped) ? offsets.length : current + 1;
    for (int i=oldest,n=1; n <= numEntries; n++) {
      data.setD( j++, offsets[i] );
      data.setD( j++, times[i].getSec() );
      i = (i + 1) % offsets.length;
    }
    return data;
  }

  /** <b>Internal use only:</b> Convert this TimeLine to Map that is suitable to store as Keywords
      in BLUE files with high precision time entries (up to picosecond precision).
      This follows the Platinum BLUE Data File Exchange Format Standard rev 4.
      <br>
      Typical Platinum use is with timeEpoch=1994-01-01 and timeDelta=1e-10 for clock ticks.
      @param timeEpoch time epoch to use.
      @param timeDelta time delta (in seconds) of each clock tick (resolution) use time units of {@link Units#COUNTS}.
                       -or- 0.0 for seconds from epoch to use time units of {@link Units#TIME_S}.
      @param dbpe  bytes per element (to convert file offset in bytes to offset in elements).
      @throws IllegalStateException when this timeline does not have any entries
      @return Map of keywords to store the high precision timeline in the extended header of a BLUE file
      @since NeXtMidas 3.3.0
      @see #fromPlatinumKeywords(Keywords, double, double)
   */
  @InternalUseOnly
  Map<String,Object> toPlatinumKeywords (Time timeEpoch, double timeDelta, double dbpe) throws IllegalStateException { // package-private
    if (current < 0) throw new IllegalStateException("TimeLine is empty");

    // Time Epoch is only stores in whole seconds
    // This is taken into account since 3.7.2 for NXM-3040
    Time wsecTimeEpoch = new Time(timeEpoch.getWSec());

    Map<String,Object> map = new LinkedHashMap<>();
    final int units = (timeDelta == 0)? Units.TIME_S : Units.COUNTS;
    Data offsetData = new Data("SX", size);
    Data timeData   = new Data(((units == Units.COUNTS)? "SX" : "SD"), size);
    int oldest = (timelineHasWrapped) ? ((current + 1) % offsets.length) : 0;
    int numEntries = (timelineHasWrapped) ? offsets.length : current + 1;
    for (int i=oldest,j=0; j < numEntries; j++) {  // Since 3.7.2 - <= current since current==0 when has 1 entry
      long elemOffset = (long) (offsets[i] / dbpe); // convert to offset in elements (from bytes) - corrected in NeXtMidas 3.7.2
      offsetData.setX(j, elemOffset);
      if      (units == Units.COUNTS) timeData.setNumber(i, times[i].toTimeTicks(wsecTimeEpoch, timeDelta));
      else if (units == Units.TIME_S) timeData.setNumber(i, times[i].diff(wsecTimeEpoch));
      i = (i + 1) % offsets.length;
    }
    map.put("TIMETAG.OFFSET",     offsetData);              // array of offsets type X (long)
    map.put("TIMETAG.TIME",       timeData);                // array of times type X or D (long or double)
    map.put("TIMETAG.TIME.UNITS", units);                   // type L (integer)
    map.put("TIME_EPOCH",         wsecTimeEpoch.toString(Time.ISO8601, 0)); // string in ISO8601 format
    if (units == Units.COUNTS) {
      map.put("TIME_DELTA", timeDelta);                     // type D (double)
      map.put("TIME_TICKS", 1L/timeDelta);                  // type X (long) (inverse of delta)
    }
    return map;  // Since 3.7.2 for NXM-3010, actually return map rather than null
  }

  /** Try to create high precision timeline from a {@link Keywords} object
      that follows the Platinum BLUE Data File Exchange Format Standard rev 4.
      @param keywords The Keywords from DataFile with Platinum time-line keywords.
      @param dfDelta  The DataFile's time delta (XDELTA/YDELTA) per element.
      @param dbpe  bytes per element (to convert file offset in bytes to offset in elements).
      @return new TimeLine instance from Platinum time-line keywords if successful,
              otherwise null if unable to extract Platinum time-line from DataFile/Keywords.
      @see #toPlatinumKeywords(Time, double, double)
      @since NeXtMidas 3.3.0
   */
  @InternalUseOnly
  static TimeLine fromPlatinumKeywords (Keywords keywords, double dfDelta, double dbpe) {

    // KEYWORD VALUES
    Object  _timeData   = keywords.get("TIMETAG.TIME");
    Object  _offsetData = keywords.get("TIMETAG.OFFSET");
    Object  _timeEpoch  = keywords.get("TIME_EPOCH");
    Double  timeDelta   = keywords.getD("TIME_DELTA",          null); // optional (double)
    Integer unitsKW     = keywords.getUnitsFor("TIMETAG.TIME", null); // optional (integer)

    if ((_timeEpoch  == null) || !(_timeEpoch  instanceof String)) return null; // missing/invalid TIME_EPOCH
    if ((_timeData   == null) || !(_timeData   instanceof Data  )) return null; // missing/invalid TIMETAG.TIME
    if ((_offsetData == null) || !(_offsetData instanceof Data  )) return null; // missing/invalid TIMETAG.OFFSET

    // DERIVED VALUES
    Time   timeEpoch; // set below
    int    units      = (timeDelta == null)? Units.TIME_S : Units.COUNTS;
    double tolerance  = (timeDelta == null)? dfDelta      : timeDelta;
    Data   offsetData = (Data)_offsetData; // array of offsets (usually of type X (long))
    Data   timeData   = (Data)_timeData;   // array of times   (usually of type X (long) or D (double))

    // ERROR CHECKING
    try {
      timeEpoch = Time.toTime(_timeEpoch, Time.ISO8601);
    }
    catch (Exception ex) {
      Shell.printStackTrace("Invalid value in TIME_EPOCH keyword ["+_timeEpoch+"]!", ex);
      return null;
    }
    if (unitsKW != null && unitsKW != units) {
      Shell.warning("Mismatch time units: TIMETAG.TIME.UNIT(S)="+unitsKW+", using time units="+units
                  + " derived from TIME_DELTA keyword");
    }
    if (offsetData.size != timeData.size) {
      Shell.warning("TIMETAG.OFFSET["+offsetData.size+"] and TIMETAG.TIME["+timeData.size
                  + "] MUST have same number of entries (size)");
      return null;
    }
//    else if (offsetData.spa != timeData.spa) {
//      Shell.warning("TIMETAG.OFFSET["+offsetData.spa+"] and TIMETAG.TIME["+timeData.spa+"] MUST have same number of entries (spa)");
//      return null;
//    }

    // Error out if have invalid bytes per element
    if ((timeData.getSize() > 0) && (dbpe <= 0)) throw new MidasException(
        "TimeLine.fromPlatinumKeywords dpbe needs to be positive when data "+
        "elements exist- was:" + dbpe);

    // CREATE & INIT TIMELINE
    TimeLine timeline = new TimeLine(timeData.getSize(), dfDelta / dbpe, tolerance);

    for (int ii=0; ii < timeData.getSize(); ii++) {
      Number td     = timeData.getNumber(ii);
      Time   time   = (units == Units.COUNTS)? new Time(td.longValue(), timeEpoch, timeDelta)
                                             : new Time(timeEpoch).addSec(td.doubleValue());
      double offset = offsetData.getNumber(ii).doubleValue() * dbpe; // convert offset to bytes from elements - Since 3.7.2

      timeline.addTimeOffset(time, offset);

      // Check for back in time timeline entries
      boolean disableErrorMsg = Shell.getMidasContext().io.isOptionSet(
          CoreIO.IOOptions.SuppressInvalidTimeLineEntryErrors);
      if ((ii > 0) && (time.diff(timeline.times[ii-1]) < 0) && !disableErrorMsg) {
        // Since 3.7.2 for NXM-3040 - WARN/ERROR on bad timelimes
        double prevOffset = offsetData.getNumber(ii).doubleValue() * dbpe;
        Shell.warning("ERROR: Invalid TimeLine entry that occurs later in the timeline, " +
            "but is back in time - Offset - Current:"+offset+" Previous:"+prevOffset+"\n" +
            "         Time- Current:"+time.toString(DP_WARN)+" Previous:"+timeline.times[ii-1].toString(DP_WARN)+
            "    Index - "+ii);
      }
    }
    timeline.format = Format.PLATINUM_HIGH_PRECISION;
    return timeline;
  }

  /** Convert this TimeLine to Map that is suitable to store as Keywords in BLUE files with high
   *  precision time entries following the BLUE TC (Time Code) keywords as outlined in the JICD 4.2
   *  ICD. This follows the Platinum BLUE Data File Exchange Format Standard rev 4.
   *  @param dbpe     Bytes per element (to convert file offset in bytes to offset in elements).
   *  @throws IllegalStateException when this timeline does not have any entries
   *  @return Map of keywords to store the high precision timeline in the extended header of a BLUE file
   *  @since NeXtMidas 3.7.2 for NXM-3010
   *  @see #fromBlueTCKeywords
   */
  @InternalUseOnly
  Map<String,Object> toBlueTCKeywords (double dbpe) throws IllegalStateException {
    if (current < 0) throw new IllegalStateException("TimeLine is empty");

    // Error out if have invalid bytes per element
    if (dbpe <= 0) throw new MidasException(
        "TimeLine.toBlueTCKeywords dpbe needs to be positive - was:"+dbpe);

    Map<String,Object> map = new LinkedHashMap<>((current+1)*3);
    // logic corrected in 3.9.4 for NXM-3637, so all entries are written out
    int oldest = (timelineHasWrapped) ? ((current + 1) % offsets.length) : 0;
    int numEntries = (timelineHasWrapped) ? offsets.length : current + 1;
    for (int i=oldest,j=1; j <= numEntries; j++) {
      map.put("TCSAMPLE_"+j, offsets[i]/dbpe + 1);
      map.put("TC_WHOLE_"+j, times[i].getWSec());
      map.put("TC_FRAC_"+j,  times[i].getFSec());
      i = (i + 1) % offsets.length;
    }
    return map;
  }

  /** Try to create high precision time-line from a {@link Keywords} object that follows the
   *  high precision BLUE TC (Time Code) keywords as outlined in the JICD 4.2 ICD.
   *  @param kw         The Keywords from DataFile with Platinum time-line keywords.
   *  @param dfTimecode Time object based on DataFile Timecode and TC_PREC
   *  @param dfDelta    The DataFile's time delta (XDELTA/YDELTA) per element.
   *  @param dbpe       Bytes per element (to convert file offset in bytes to offset in elements).
   *  @return The new TimeLine instance from BLUE TC keywords if successful,
   *          otherwise null if unable to extract the time-line.
   *  @see #toBlueTCKeywords
   *  @since NeXtMidas 3.7.2 for NXM-3010
   */
  @InternalUseOnly
  static TimeLine fromBlueTCKeywords (Keywords kw, Time dfTimecode, double dfDelta, double dbpe) {

    // This method is called to see if there is a Blue TC timeline, so the
    // response should be to return null if creating a BLUE TC timeline is not possible
    if (dbpe <= 0) {
      return null;
    }

    double   tolerance = QUARTER_NANOSECOND_TOLERANCE; // matches SDDS
    int numTCEntries = getNumTCEntries(kw);
    if (numTCEntries < DEF_ALLOC) numTCEntries = highPrecisionAllocDefault;
    TimeLine timeline  = new TimeLine(numTCEntries, dfDelta / dbpe, tolerance);
    // disabling of autocorrect needed for TC keywords as well - 3.9.3 for NXM-3532
    CoreIO coreIO = Shell.getMidasContext().io;
    boolean enableOffset0AutoCorrect =
        !coreIO.isOptionSet(CoreIO.IOOptions.DisableTimeLineAutoCorrectNoOffset0Entry) &&
        !coreIO.isOptionSet(CoreIO.IOOptions.SameTimelineKeywords);
    if (enableOffset0AutoCorrect) timeline.setTimeAt(0, dbpe, dfTimecode);
    boolean  any = false;

    Double offset;
    for (int i = 1; (offset = kw.getD("TCSAMPLE_"+i, null)) != null; i++) {
      Double wsec = kw.getD("TC_WHOLE_"+i, null);
      Double fsec = kw.getD("TC_FRAC_"+i, null);

      if (wsec == null) {
        throw new MidasException("Invalid BLUE TC keywords, missing TC_WHOLE_"+i);
      }
      if (fsec == null) {
        throw new MidasException("Invalid BLUE TC keywords, missing TC_FRAC_"+i);
      }

      any = true;
      // TC offsets are 1 based sample offsets - convert to 0 based offset
      // Warn if there are invalid timeline entries in the file - since 3.7.3 for NXM-3040
      timeline.setTimeAt(offset-1, dbpe, new Time(wsec, fsec), NotifyInvalidEntry.WARNING);
    }
    timeline.format = Format.BLUE_HIGH_PRECISION;
    return (any)? timeline : null;
  }

  /** <b>INTERNAL USE ONLY:</b> Load from a Keywords Data object. This is for internal use between
      {@link DataFile} and {@link Keywords}, it is <code>public</code> so that it can be used by custom
      file handlers in other option trees. See {@link #toData()} for details on the structure
      of the {@link Data} object.
      WARNING: MUST call {@link #setAlloc(int)} with enough space to store time entries from input
               data before calling this (and alloc SHOULD NOT be set to -1) otherwise the time
               entries will be ignored or truncated to size of alloc.
               Since 3.7.3 an additional parameter was added to handle cases where the first
               offset in the file is not for the first element. In this case alloc
               for the timeline needs to be 1 larger than would be needed from the Data alone,
               to allow for an entry being added for the first element.
      @param data Keywords Data object from which to load the TimeLine
      @param startingTime value of DataFile's Timecode + getStart()
   */
  @InternalUseOnly
  public void fromData (Data data, Time startingTime) {
    int j=0;
    size    = Math.min((data.size*data.spa-2)/2, alloc);
    dt      = data.getD(j++);
    tolr    = data.getD(j++);
    current = -1;
    double tmax = 0;
    double time;
    double lastTime = 0;  // Initialize last time to 1950
    int initIdx =0;
    CoreIO coreIO = Shell.getMidasContext().io;
    boolean enableOffset0AutoCorrect =
        !coreIO.isOptionSet(CoreIO.IOOptions.DisableTimeLineAutoCorrectNoOffset0Entry) &&
        !coreIO.isOptionSet(CoreIO.IOOptions.SameTimelineKeywords);
    if (enableOffset0AutoCorrect) { // 3.9.4 back to being on by default
      // Add entry for offset 0 if not contained in keywords
      // and TimeCode is set non-1950 (i.e. not 0 seconds) NXM-3087
      if ((data.getD(j) != 0) && (startingTime.getSec() > 0)) {
        if (offsets.length == size) {
          offsets = new double[size+1];
          times   = new Time[size+1];
        }
        offsets[0] = 0;
        times[0] = startingTime; // set time for first element to Timecode
        lastTime = startingTime.getSec();
        initIdx = 1; // first entry from keyword data will be for offset[1]
        size += initIdx;
      }
    }
    for (int i=0+initIdx; i<size; i++) {
      offsets[i] = data.getD(j++);
      time       = data.getD(j++);
      times[i]   = new Time(time);
      // Check for back in time timeline entries
      boolean disableErrorMsg = Shell.getMidasContext().io.isOptionSet(
          CoreIO.IOOptions.SuppressInvalidTimeLineEntryErrors);
      if ((time < lastTime) && !disableErrorMsg) {
        // Since 3.7.2 for NXM-3040 - WARN/ERROR about bad timelimes
        Shell.warning("ERROR: Invalid TimeLine entry that occurs later in the timeline, " +
            "but is back in time - Offset - Current:"+offsets[i]+" Previous:"+offsets[i-1]+"\n" +
            "         Time- Current:"+times[i].toString(DP_WARN)+" Previous:"+times[i-1].toString(DP_WARN)+
            "    Index - "+i);
      }
      lastTime = time;
      if (time>tmax) { tmax=time; current=i; }
    }
    format = Format.MICROSECOND_PRECISION;
  }

  /**
   * Checks a Keywords Data Object to see if the timeline starts with
   * the first element.  For internal use by {@link DataFile} to allow
   * it to allocate sufficient space for the timeline.
   * @param data  Data object containg the timeline
   * @return true if it is a normal timeline with the first offset being
   *              element 0
   */
  @InternalUseOnly
  static boolean dataObjectTimelineStartsWithFirstElement (Data data) {
    int positionFirstOffset = 2;
    return (data.getD(positionFirstOffset) == 0);
  }

  /** <b>INTERNAL USE ONLY:</b> list entire contents of TimeLine (used by STATUS/T)
   *  @return list entire contents of TimeLine
   */
  @InternalUseOnly
  public String list () {
    return list(3);
  }

  /** <b>INTERNAL USE ONLY:</b> list entire contents of TimeLine (used by STATUS/TIME)
      @param decimalPlaces number of decimal places to show for time.
      @return list entire contents of TimeLine
      @since NeXtMidas 3.3.0
   */
  @InternalUseOnly
  public String list (int decimalPlaces) {
    StringBuilder s = new StringBuilder(80 + size*(56+decimalPlaces)); // header (usually <80) + entries (usually <56 chars each)
    s.append("TimeLine Size=").append(size);
    s.append(" Alloc=").append(alloc);
    s.append(" Delta=").append(dt).append(" (sec/byte)");
    s.append(" Tolr=").append(tolr).append(" (sec)\n");
    for (int i=0; i<size; i++) {
      s.append("Index=").append(i).append(" Off=").append(offsets[i]);
      s.append(" Time=").append(times[i].toString(Time.STD,decimalPlaces));
      if (i == 0 && (offsets[0] != 0.0)) {
        s.append(" <-- Does not start at beginning of file!");
      }
      s.append("\n");
    }
    if ((size == offsets.length) && (offsets[0] != 0.0)) {
      // JC4: 15MAY07 Added this warning since it is a very common mistake.
      s.append("WARN: TimeLine is full, some entries may have been discarded. It is likely that\n");
      s.append("WARN: the Delta and Tolerance values do not match the observed data rate.\n");
    }
    return s.toString();
  }

  @Override
  public String toString () {
    int decimalPlaces = 3;
    // String s = "TimeLine Size="+size+" Of="+alloc;
    StringBuilder sb = new StringBuilder("TimeLine Size="+size+" Alloc="+alloc+" Delta="+dt+"(sec/byte) Tolr="+tolr+" (sec)");
    if (size==0) return sb.toString();
    Time tmin=times[0], tmax=tmin;
    for (int i=1; i<size; i++) {
      if (times[i].compareTo(tmax) > 0) tmax=times[i];
      if (times[i].compareTo(tmin) < 0) tmin=times[i];
    }
    sb.append("\n From=").append(tmin.toString(Time.STD,decimalPlaces));
    sb.append("\n To  =").append(tmax.toString(Time.STD,decimalPlaces));
    return sb.toString();
  }

  /**
   * <b>INTERNAL USE ONLY:</b> Create a Table from the TimeLine entries,
   * where the keys are the String version element offsets (as an int)
   * and the values are the Times as a String
   *
   * Note: This is designed to be used along with
   * nxm.sys.test.TestTimelineCreator#timelineFromTable(double,int,Table)
   *
   * @param decimalPlaces number of decimal places to be represented in String
   *                      version of Time
   * @param bpe           bytes per element
   * @return              Table representing timeline
   * @since NeXtMidas 3.7.2
   */
  @InternalUseOnly("since initial version")
  public Table toTable (int decimalPlaces, double bpe) {
    Table timelineTbl = new Table();
    for (int i=0; i<size; i++) {
      String offset = String.valueOf((int)(offsets[i]/bpe));
      timelineTbl.put(offset, times[i].toString(decimalPlaces));
    }
    return timelineTbl;
  }

  /** <b>INTERNAL USE ONLY:</b> access to read the array of Time objects in the Timeline.
      NOTE: this method is used only by nxm.sys.test.junit.lib.TimeLineTest
      @return {@link #times}, the array of times in the timeline.
      @since NeXtMidas 3.7.3 changed the return type back to public for JUnit test
   */
  @InternalUseOnly("since initial version")
  public Time[] getTimes () {
    return Arrays.copyOf(times, times.length);
  }

  /** <b>INTERNAL USE ONLY:</b> access to read the array of offsets in the Timeline.
  NOTE: this method is used only by nxm.sys.test.junit.lib.TimeLineTest
  @return {@link #offsets}, the array of offsets in the timeline.
  @since NeXtMidas 3.7.3
  */
  @InternalUseOnly("since initial version")
  public double[] getOffsets () {
    return Arrays.copyOf(offsets, offsets.length);
  }

  /** Get number of high precision TC entries in Keywords
   *
   *  @param kw Keywords
   *  @return number of high precision TC entries
   */
  public static int getNumTCEntries (Keywords kw) {
    String origScope = kw.getScope();
    Iterator iter = kw.iterator("EXT");
    String last = "TCSAMPLE_0";
    int numEntries = 0;

    try {
      while (iter.hasNext()) {
        Key k = iter.next();
        String name = k.name;
        if (name.startsWith("TCSAMPLE_")) {
          last = name;
        }
      }
      String lastEntry = last.substring(9);
      numEntries = Integer.valueOf(lastEntry);
    }
    catch (RuntimeException e) {
      // Note that we don't print out the exception since it should already have been printed as a warning
    }

    kw.setScope(origScope);
    return numEntries;
  }


}
