package nxm.net.lib;

import java.util.Arrays;
import java.util.logging.Level;

import nxm.sys.inc.InternalUseOnly;
import nxm.sys.lib.DataFile;
import nxm.sys.lib.ICEPacket;
import nxm.sys.lib.LogUtilities;
import nxm.sys.lib.MidasException;
import nxm.sys.lib.TimeLine;
import nxm.sys.libm.Tolerance;

/** Adds functionality to RTP Packets cache for frame data with ICE Headers<br>
 *  Handles:
 *  <ol>
 *   <li>out-of-order</li>
 *   <li>missing</li>
 *   <li>TimeLine entries based on effective Y-Delta</li>
 *  </ol>
 *  <br>
 *  This is designed to work with DatagramData's read method where:
 *  <ol>
 *    <li>where geticedata is set to true so ICE Header information is saved and accessible</li>
 *    <li>only the data portion of the packet is passed to this classes add methods</li>
 *    <li>extra DatagramData features that trigger restarts or changing of sequence number
 *          (like call-back and fill) are disabled, so the appropriate action can be taken by
 *          this class</li>
 *  </ol>
 *  <br>
 *  Generally <b><i>addPacketToCacheAndDropOld</i></b> is used during the initialization phase
 *  where header information for the output file is still being gleaned from ICE Headers.
 *  It just adds packets to the cache and, if needed, drops really old packets.
 *  <br>
 *  <br>
 *  <b><i>addPacketAndOutputFullFrame</i></b> is the main operational method that, along
 *  with supporting methods, adds the current packet and outputs frames of data
 *  (filling missing data in old frames if necessary)
 *  <br>
 *  <br>
 *  <b>Initialization Phase actions:</b>
 *  <ol>
 *    <li>create RTPICEPacketsCache</li>
 *    <li>customize RTPICEPacketsCache using methods such as:
 *      <ul>
 *      <li><i>setUsePacketSizePattern</i></li>
 *      <li>setMissingFillValue</li>
 *      </ul>
 *    </li>
 *    <li>the calling class should initialize the output pipe header based on
 *        ICE Header information
 *      <ul>
 *      <li><i>calcYDelta</i> can be used to find the Y-Delta value</li>
 *      </ul>
 *    </li>
 *    <li>while initializing add each packet's information to cache via
 *        <b><i>addPacketToCacheAndDropOld</i></b>
 *    </li>
 *  </ol>
 *  <br>
 *  <br>
 *  <b>Operation Phase actions:</b>
 *  <ol>
 *    <li>use <b><i>addPacketAndOutputFullFrame</i></b> to add packet and output
 *        frames as they are completed</li>
 *    <li>if applicable, act on RESTART condition</li>
 *  </ol>
 *
 * @author Jeanette McCall
 * @since  NeXtMidas 3.8.1
 *
 */
public class RTPICEPacketsCache extends RTPPacketsCache{

  /** Default fill for missing values - Good default for PSD frames */
  @InternalUseOnly
  public final static byte FRAME_FILL_DEFAULT = -127; // Good default for PSD frames

  final static int NO_RESTART_NEEDED        = 0x0;
  final static int RESTART_NEEDED_XSTART    = 0x1;
  final static int RESTART_NEEDED_XDELTA    = 0x2;
  final static int RESTART_NEEDED_YDELTA    = 0x3;
  final static int RESTART_NEEDED_FRAMESIZE = 0x4;
  final static Class<?> THIS_CLASS = RTPICEPacketsCache.class;
  /** Default number of frames to wait for missing data/packets before going
   *  ahead and filling missing values and writing out what we have */
  @InternalUseOnly
  public static final int DEFAULT_MAX_MISSING_FRAMES = 3;

  /** ICEPacket header information associated with packet
   *  (null is not valid unsent ICE Packet) */
  private ICEPacket[] associatedICEPackets;

  /** Holds last ICE Packet - used for Y-Delta calculations */
  private ICEPacket previousICEPacket  = null;
  /** Previous Y-Delta - used when checking if Y-Delta change is big enough we need a restart */
  private double    previousYDelta = NOT_INITIALIZED;
  /** Current Y-Delta - Y-Delta from the frame we are writing out */
  private double    currentYDelta  = NOT_INITIALIZED;

  private double xStartToleranceRestart = Tolerance.getTolerance(); // Default to NeXtMidas tolerance
  private double xDeltaToleranceRestart = Tolerance.getTolerance(); // Default to NeXtMidas tolerance
  private double yDeltaToleranceRestart = 0.1; // Reasonable default based on operational code

  // logging information
  /** tracks the amount of packets dropped for logging purposes */
  private long packetsDroppedOrMissingForLogging = 0;
  /** Log every this many dropped packets - modulo value */
  private int logFactor = 50; // Default value
  /** If we have dropped this many packets within one frame, log it
   *  Note: Right now this value is EXTREMELY high, as very few if any
   *  frames will have this many packets.
   */
  private static final int MINIMUM_DROPPED_PACKETS_PER_FRAME_LOG = 10;

  /** A full frame of data is written at a time, even if it requires multiple
   *  packets to fill it. It may contain just data, or if getIceHdrInPipe() is
   *  true, it also contains the ICE header.
   */
  private byte[]   frameToWrite;

  private int packetsPerFrame = NOT_INITIALIZED; // -1 means not initialized
  /** Holds the most recent bytes per packet, where [0] is the packet with the ICE header
   *  Note: Only the indexes 0 through packetsPerFrame-1 are valid
   */
  private int[] bytesDataPerPacket;
  private static int FRAME_START_BYTES_INDEX = 0;

  private boolean usePacketSizePattern = true; // default to using packet data size pattern as well as packets per frame
  /** True if all values in cache are good to go */
  private boolean cacheFullyInitialized = false;;
  /** If this is the first time through writing to pout, need to do the fist frame */
  private boolean firstRun=true;

  /** Recheck the data per packet information each full frame */
  private boolean dataPerPacketsRecheck = false;

  /** maximum frames ahead before assume some packets missing */
  private int maxMissingFrames = DEFAULT_MAX_MISSING_FRAMES;

  /** ICE Packet that has restart information */
  private ICEPacket icePacketRestart;

  /** Calling code should treat the restart as a hard restart, not a soft restart
   *  (i.e. This type of restart is best handled by creating a new ICEPacketCache
   *  and adding the current packet's data and header into the new cache
   */
  private boolean isHardRestart = false;

  /** Really safe maximum ICE Count usedin determining if we have a new stream */
  private static final int MAX_ICE_COUNT_RESTART = 10;

  /** Maximum gap in ICE count number before ICE Count is assumed to be behind rather than ahead */
  private static long MAX_AHEAD_ICE_COUNT = (long) (ICEPacket.MAX_COUNT * 0.5);

  /** Only update YDelta if ICE Packets are frames that are fairly close to each other */
  private static long MAX_ICE_COUNT_DIFF_CALC_YDELTA = 10;

  /** Settable log level for normal log messages */
  private static Level mainLogLevel = Level.FINER;

  /** Extreme detail log messages */
  private static Level detailLogLevel = Level.FINEST;
  private LoggerUtilityMethods logMethods = new LoggerUtilityMethods();
  /** Create an cache holding ICE header information
  *   @param dgd      Datagram Data object holding packet information
  *   @param pipeOut  Output pipe (DataFile)
  */
  public RTPICEPacketsCache (DatagramData dgd, DataFile pipeOut) {
    this(dgd, pipeOut, DEFAULT_PACKET_CACHE_SIZE);
  }

  /** Create an cache holding ICE header information
  *   @param dgd        Datagram Data object holding packet information
  *   @param frameSize  frame size
  *   @param pipeOut    Output pipe (DataFile)
  */
  public RTPICEPacketsCache (DatagramData dgd, int frameSize, DataFile pipeOut) {
    this(dgd, pipeOut, DEFAULT_PACKET_CACHE_SIZE);
    this.frameToWrite = new byte[frameSize];
  }

  /** Create an cache holding ICE header information
   *  @param dgd        Datagram Data object holding packet information
   *  @param pipeOut    Output pipe (DataFile)
   *  @param cacheSize  Size of cache
   */
  public RTPICEPacketsCache (DatagramData dgd, DataFile pipeOut, int cacheSize) {
    this(dgd, pipeOut, cacheSize, FRAME_FILL_DEFAULT);
  }

  /** Create an cache holding ICE header information
   *  @param dgd                 Datagram Data object holding packet information
   *  @param pipeOut             Output pipe (DataFile)
   *  @param cacheSize           Size of cache
   *  @param frameFillValue      The value to pad the missing data
   */
  public RTPICEPacketsCache (DatagramData dgd, DataFile pipeOut, int cacheSize, byte frameFillValue) {
    super(dgd, pipeOut, cacheSize, frameFillValue);

    // array holding valid unsent ICEPacket header information associated with packet
    associatedICEPackets = new ICEPacket[cacheSize];

    // Note this is OVERKILL on size for this, and maybe should be half the size
    bytesDataPerPacket = new int[cacheSize];
    Arrays.fill(bytesDataPerPacket, NOT_INITIALIZED);

    //maxAheadBeforeAssumeSomeMissing = (int)(LARGEST_RANGE_BEFORE_FILL_AS_FRACTION_CACHE * cacheSize);
  }

  /** Add packet to cache and output full frames
   *
   *  Operational diagram skips paths taken when cache initialization is not complete,
   *  as addPacketToCacheAndDropOld usually initializes this information while waiting
   *  on getting the first two ICE Headers to complete header information.<br>
   *  <pre>
   *                     |
   *                     *
   *                   *    *             ************************************
   *                 * stream *    yes    * set that hard restart involving  *
   *               *   change?  *---------*creation of new RTPICEPacketsCache*-- return (restart needed)
   *                 *        *           * with just this ICE Packet added  *
   *                   *    *             * is appropriate action for caller *
   *                     *                ************************************
   *                     | no
   *                     |
   *                     *
   *                   *   *
   *                 * packet *    yes    ***************
   *               *  too old ? *---------* Drop packet *-- return (restart needed
   *                 *        *           ***************           is always false)
   *                   *   *
   *                     *
   *                     | no
   *                     |
   *                     *
   *                   * ? *
   *                 * need  *    yes    *****************************************
   *               * make space*---------* Send out enough frames to make space, *
   *                 * before *          * filling in data where needed          *
   *                   *add*             *****************************************
   *                     *                                   |
   *                     | no                                |
   *                     K-----------------------------------|
   *                     |
   *               **************
   *               * add packet *
   *               *  to cache  *
   *               **************
   *                     |
   *                     *
   *                   *   *             **********************************
   *                 * is the*    yes    * Send out frames until we reach *
   *               * next frame*---------* one that is not full or one    *-- return
   *                 * full ? *          * that is triggering a RESTART   *
   *                   *   *             **********************************
   *                     *
   *                     | no
   *                     |
   *          return (restart needed always false)
   *
   *  </pre>
   *  @param buf  buffer containing data (and possibly ICE Header if iceheaderinpipe is true)
   *  @param len  length in bytes of data in this packet
   *  @return is restart needed before can put out this frame
   */
  @InternalUseOnly
  public boolean addPacketAndOutputFullFrames (byte[] buf, int len) {
    boolean restartNeeded = false;

    int seq = dgd.getSeq();

    int numPktsAheadOfLowest = RTPHeader.computeNumPktsAhead(seq,lowestValidPkt);
    int numPktsAheadOfLastOut = RTPHeader.computeNumPktsAhead(seq,lastPacketOutput);
    int numPktsAheadOfHighest = RTPHeader.computeNumPktsAhead(seq,highestPkt);
    LogUtilities.log(THIS_CLASS, detailLogLevel, "Number of Packet Ahead Calculations for pipe out = {0}" +
        " sequence number = {1}\n   lowest = {2}, ahead lowest = {3}, highest = {4}, ahead highest = {5}"
        , pipeOut.getName(), seq,lowestValidPkt, numPktsAheadOfLowest,
         highestPkt, numPktsAheadOfHighest, lastPacketOutput, numPktsAheadOfLastOut);

    //**************************************************************************
    // HANDLE ANY PRE-PROCESSING NEEDED BEFORE ADDING PACKET TO CACHE
    //
    // - We may need to clear out (possibly write out) old packets that will
    //   interfere with adding this packet.
    // - We may need to just drop this packet and return of it is older than the
    //   ones we have already output
    //**************************************************************************

    //--------------------------------------------------------------------------
    // CHECK FOR STREAM SWITCH
    //--------------------------------------------------------------------------
    if (smartCheckForStreamChange(numPktsAheadOfLastOut, numPktsAheadOfHighest)) {
      return true;
    }

    //--------------------------------------------------------------------------
    // IN NORMAL OPERATION CACHE INITIALIZATION IS USUALLY COMPLETE AT THIS POINT
    // IF CACHE INITIALIZATION IS NOT COMPLETE, THIS PACKET MIGHT HELP INITIALIZE
    //--------------------------------------------------------------------------
    if (!cacheFullyInitialized) initCacheBasedOnPkt (seq, len);

    //--------------------------------------------------------------------------
    // DROP PACKET IF IT IS IN RANGE THAT WE HAVE ALREADY OUTPUT
    //--------------------------------------------------------------------------
    if (numPktsAheadOfLastOut <= 0) {
      logMethods.logDropped();
      return false;
    }

    //--------------------------------------------------------------------------
    // HANDLE CASE WHERE WE NEED TO SEND OUT SOME OF THE CACHED PACKETS BEFORE ADDING
    //
    // We need to just fill and send out old frames if the sequence
    // number on this packet is large enough with respect to the oldest we
    // are still actively caching
    //--------------------------------------------------------------------------
    boolean needFillOldFrames = numPktsAheadOfLastOut >= maxAheadBeforeAssumeSomeMissing;
    if (needFillOldFrames) {
      int numPacketsNeedToClearOut = numPktsAheadOfLastOut - maxAheadBeforeAssumeSomeMissing + 1;
      LogUtilities.log(THIS_CLASS, detailLogLevel, "Need to fill out and send out old frames for pipe out = {0}" +
          " sequence number = {1}\n   numPktsAheadOfLastOut = {2}, maxAheadBeforeAssumeSomeMissing = {3},"+
          " packetsPerFrame = {4}, numPacketsNeedToClearOut = {5}", pipeOut.getName(),
          seq, numPktsAheadOfLastOut,maxAheadBeforeAssumeSomeMissing, packetsPerFrame,
          numPacketsNeedToClearOut);

      if (packetsPerFrame != NOT_INITIALIZED) {
        // We want to output enough frames (clear up enough cache) to add this latest packet
        int numFramesToOutput = (int) Math.ceil(numPacketsNeedToClearOut / (double)packetsPerFrame);
        fillAndSendOutOldFrames(numFramesToOutput);
      }
      else {
        // We do not have enough information to send out old frames, just need to clear
        // (usual to reach here when using addPacketsToCacheAndDropOld for initialization)
        clearPackets(lowestValidPkt,numPacketsNeedToClearOut);
      }
    }

    //**************************************************************************
    // ADD PACKET TO CACHE
    //**************************************************************************
    boolean thisPacketHasICEHeader = addPacketToCache(buf, len);

    //**************************************************************************
    // CHECK FOR NEXT FRAME BEING FULL AND OF SO OUTPUT FRAMES UNTIL WE REACH
    // A FRAME THAT IS NOT FULL
    //**************************************************************************

    // Used to see if we have the next frame (one after last written ready now)
    int seqOfICEFullFrame = NOT_INITIALIZED;

    //--------------------------------------------------------------------------
    // IF THIS IS OUR FIRST TIME IN THIS CODE, WE MAY HAVE A FULL FRAME FROM
    // THE INITIALIZATION PHASE
    //--------------------------------------------------------------------------
    if (firstRun) {
      seqOfICEFullFrame = checkIfHaveFullFrame(true,RTPHeader.incrementAndWrapSeqNum(lastPacketOutput));
      firstRun=false;
    }
    //--------------------------------------------------------------------------
    // IN NORMAL OPERATION CACHE INITIALIZATION IS USUALLY COMPLETE AT THIS POINT:
    // USE PACKET INFORMATION FOR INTITALIZING CACHE AND CHECK IF FULL FRAME
    //--------------------------------------------------------------------------
    else if (!cacheFullyInitialized) {
      if (thisPacketHasICEHeader && (dgd.lastICE.getBytes() <= len)) {
        // 1 packet for frame case
        packetsPerFrame = 1;
        if (usePacketSizePattern) {
          bytesDataPerPacket[0] = dgd.lastICE.getBytes();
        }
        seqOfICEFullFrame = seq;
      }
      else if ((thisPacketHasICEHeader && (numPktsAheadOfHighest < 0)) ||
              (!thisPacketHasICEHeader && (numPktsAheadOfLowest > 0))) {
        seqOfICEFullFrame = checkIfHaveFullFrame(thisPacketHasICEHeader, seq);
      }
      LogUtilities.log(THIS_CLASS, mainLogLevel, "Cache was not fully initialized - pipe out = {0} "+
          "Sequence Number = {1},\n thisPacketHasICEHeader = {2}, seqOfICEFullFrame = {3},"
          +"numPktsAheadOfHighest = {4}, numPktsAheadOfLowest = {5}",
          pipeOut.getName(), seq, thisPacketHasICEHeader, seqOfICEFullFrame,
          numPktsAheadOfHighest, numPktsAheadOfLowest);
    }
    //--------------------------------------------------------------------------
    // IF WE JUST CLEARED THE CACHE OF PARTIAL FRAMES OR
    // THIS PACKET FALLS WITHIN THE NEXT FRAME, THE NEXT FRAME MIGHT BE FULL
    //--------------------------------------------------------------------------
    else if (needFillOldFrames ||
            (RTPHeader.computeNumPktsAhead(seq, lastPacketOutput) <= packetsPerFrame)) {
      int seqNumOfNextPkt = RTPHeader.incrementAndWrapSeqNum(lastPacketOutput);
      if (dataPerPacketsRecheck) {
        seqOfICEFullFrame = checkIfHaveFullFrame(true, seqNumOfNextPkt);
      }
      else {
        // shortcut that can be used if we do not need to keep checking the
        // amount of data in each packet
        if (thisPacketHasICEHeader) {
          updateYDelta(seq%cacheSize);
          restartNeeded = checkIfNeedsYDeltaRestart(seq%cacheSize);
        }
        if (restartNeeded) return restartNeeded;
        seqOfICEFullFrame = isNextFrameComplete() ? seqNumOfNextPkt : NOT_INITIALIZED;
      }
    }

    //--------------------------------------------------------------------------
    // IF WE DO NOT HAVE A FULL FRAME, JUST RETURN, NO RETART IS NEEDED
    //--------------------------------------------------------------------------
    if (seqOfICEFullFrame == NOT_INITIALIZED) {
      LogUtilities.log(THIS_CLASS, detailLogLevel, "Just added packet to cache, but did not have full frame - "+
          "sequence number = {0}",seq);
      return false;
    }

    //--------------------------------------------------------------------------
    // IF THE NEXT FRAME THAT NEEDS TO BE WRITTEN IS FULL WRITE IT OUT
    // (UNLESS IT TRIGGERS A RESTART)
    //--------------------------------------------------------------------------
    // We need to send out first packet.
    restartNeeded = outputFullFrames(seqOfICEFullFrame,restartNeeded);

    return restartNeeded; //  restart needed if header information significantly changed
  }

  /** Check if the input stream changed out from under us.  If so, a hard
   *  restart including creating a new RTPICEPacketsCache with only the current
   *  packet is indicated. Only packets containing ICE Headers can trigger a
   *  restart.<br>
   *  Indicators of input stream change
   *  <li>Significantly higher sequence number, but lower ICE Packet count</li>
   *  <li>Frame size change</li>
   *  <li>XStart or XDelta change that is out of tolerance</li>
   *  <li>Timestamp that is out of range</li>
   *  Note: YDelta changes are more complex to detect and are detected when a
   *  frame is ready to be written.
   *  @param numPktsAheadOfLastOut number of packets ahead of the last packet written out
   *  @param numPktsAheadOfHighest number of packets ahead of the highest in the cache
   *  @return true if restart is needed due to input stream change
   */
  private boolean smartCheckForStreamChange (int numPktsAheadOfLastOut, int numPktsAheadOfHighest) {
    boolean newStreamRestartNeeded = false;
    boolean frameSizeChange = false;
    if (dgd.isValidIceHdrPacket() && (previousICEPacket != null) && cacheFullyInitialized && (lastPacketOutput >= 0)) {

      numPktsAheadOfHighest = (highestPkt >= 0) ? numPktsAheadOfHighest
          : RTPHeader.computeNumPktsAhead(dgd.getSeq(), lastPacketOutput);
      // check for case where sequence number jumps ahead
      long currentICECount = dgd.lastICE.getUnsignedCount();
      long previousIceCount = previousICEPacket.getUnsignedCount();

      // Is it a lower count and has it wrapped to make it lower?
      boolean lowerICECount = (currentICECount < previousIceCount) &&
          (currentICECount <= MAX_ICE_COUNT_RESTART) &&
          ((previousIceCount - currentICECount) < (Integer.MAX_VALUE / 2));

      if ((numPktsAheadOfHighest > cacheSize) && lowerICECount) { // This has to be a restart because the counts are way off
        frameSizeChange = dgd.lastICE.getBytes() != pipeOut.getBPE();
        LogUtilities.log(THIS_CLASS,mainLogLevel, "Stream Change Restart based on larger sequence number with lower ICE count - " +
            "Sequence Number = {0}", dgd.getSeq());
        newStreamRestartNeeded = true;
      }
      else {
        // check for header changes
        frameSizeChange = dgd.lastICE.getBytes() != pipeOut.getBPE();
        double currentXStart = dgd.lastICE.getUserDataD(ICEPacket.XSTART_POSITION_IN_USER_DATA);
        double currentXDelta = dgd.lastICE.getUserDataD(ICEPacket.XDELTA_POSITION_IN_USER_DATA);
        boolean xStartChange = !Tolerance.equalsWithAbsTol(currentXStart, pipeOut.getXStart(), xStartToleranceRestart);
        boolean xDeltaChange = !Tolerance.equalsWithAbsTol(currentXDelta, pipeOut.getXDelta(), xDeltaToleranceRestart);
        double icePTime = dgd.lastICE.getTC().getSec();
        boolean timestampOutOfRange = ((numPktsAheadOfLastOut > 0) && (icePTime < pipeOut.getYStart())) ||
            ((numPktsAheadOfLastOut < 0) && (icePTime > previousICEPacket.getTC().getSec()));
        newStreamRestartNeeded = frameSizeChange || xStartChange || xDeltaChange || timestampOutOfRange;
        if (newStreamRestartNeeded) {
          logStreamChangeRestart(frameSizeChange, xStartChange, xDeltaChange, timestampOutOfRange,dgd.getSeq());
        }
      }
      if (newStreamRestartNeeded) {
        isHardRestart = true;
      }
      return newStreamRestartNeeded;
    }
    else {
      return false; // no restart needed for change in stream
    }
  }

  /** Starting with the full frame referenced by the sequence number of the ICE Header packet,
   *  continue to output frames until reach one that is not full
   *
   *  @param seqOfICEFullFrame sequence number of ICE Header
   *  @param restartNeeded     true if restart needed
   *  @return                  restart needed
   */
  private boolean outputFullFrames (int seqOfICEFullFrame, boolean restartNeeded) {
    while (!restartNeeded && (seqOfICEFullFrame == RTPHeader.incrementAndWrapSeqNum(lastPacketOutput))) {
      // Our next frame is ready
      restartNeeded = fillFullFrame(true);
      if (!restartNeeded) {
        int indexOfIceHdrPacket = seqOfICEFullFrame % cacheSize;
        LogUtilities.log(THIS_CLASS, mainLogLevel,"WRITING FRAME that starts with packet {0}"+
            " to output pipe {1} ", seqOfICEFullFrame, pipeOut.getName());
        pipeOut.setTimeAt(associatedICEPackets[indexOfIceHdrPacket].getTC(), TimeLine.NotifyInvalidEntry.WARNING); // If we have an ice header, we need to set the timecodes
        pipeOut.write(frameToWrite,0,frameToWrite.length);
        previousICEPacket = associatedICEPackets[indexOfIceHdrPacket];
        clearFrameFromCache();
        int nextPktToCheck = RTPHeader.incrementAndWrapSeqNum(lastPacketOutput);
        if (dataPerPacketsRecheck) {
          // go ahead and do full re-check of data per packet
          seqOfICEFullFrame = checkIfHaveFullFrame(true, nextPktToCheck);
        }
        else {
          // short-cut
          seqOfICEFullFrame = isNextFrameComplete() ? nextPktToCheck : NOT_INITIALIZED;
        }
      }
    }
    return restartNeeded;
  }

  /** Add packet to cache
     *  (assumes checks for cache being in good state to accept this packet have
     *  already been done)
     *
     * @param buf contains data for this packet
     *            (should not include headers!)
     * @param len length of actual data in this buffer
     * @return true if packet was added to cache and was ICEHeader
     */
  @InternalUseOnly
  public boolean addPacketToCache (byte[] buf, int len) {
    int seq = dgd.getSeq();

    // Update lowest and/or highest "viable" packet (sequence number),
    // if this packet extends the range
    if ((lowestValidPkt == NOT_INITIALIZED) || (RTPHeader.computeNumPktsAhead(seq, lowestValidPkt) < 0)) {
      lowestValidPkt = seq;
      LogUtilities.log(THIS_CLASS, detailLogLevel,"for output pipe {0} adding packet {1} which is now the lowestValidPkt",
          pipeOut.getName(), seq);
    }
    if (RTPHeader.computeNumPktsAhead(seq, highestPkt) > 0) {
      highestPkt = seq;
      LogUtilities.log(THIS_CLASS, detailLogLevel, "for output pipe {0} adding packet {1} which is now the highestPkt",
          pipeOut.getName(), seq);
    }

    // Save off the data from this read
    // copy data from buf to correct location in store (use modulo for placement)
    int indexForStore = seq % cacheSize;
    storeBuf[indexForStore] = new byte[dgd.getLen()];

    logMethods.logCacheAddInformation(seq); // log information before add
    System.arraycopy(buf, 0, storeBuf[indexForStore], 0, dgd.getLen());
    // indicate this is a viable packet
    viablePackets[indexForStore] = true;
    packetInStore[indexForStore] = seq;

    // save information on whether the packet had an extended ICE Header
    boolean thisPacketHasValidICEHeader = dgd.isValidIceHdrPacket();

    // Log information about packet added
    int currentFrameSize = (frameToWrite == null) ? NOT_INITIALIZED : frameToWrite.length;
    LogUtilities.log(THIS_CLASS, detailLogLevel,"For output pipe {0} with frame size {1},"+
        " added packet with sequence number {2}", pipeOut.getName(),
        currentFrameSize,seq);
    if (thisPacketHasValidICEHeader) {
      LogUtilities.log(THIS_CLASS, detailLogLevel," ICE Header associated with packet: \n {0}",dgd.lastICE);
    }

    if (thisPacketHasValidICEHeader) {
      // Get IceHeader information and store circular buffers
      // Note: dgd.setICEData(true) must always be set when handling out-of-order
      associatedICEPackets[indexForStore] = dgd.lastICE;
    }

    return thisPacketHasValidICEHeader;
  }

  /** Add packet to cache and drops/clears any packets that are too old
   * (this is used in initial startup situations where we need to receive an ICE Header
   *  or 2 to obtain frame size and yDelta information)
   *  @param buf contains data for this packet
   *  @param len This is the buf.length - is it necessary?
   *  @return true if packet was added to cache and was ICEHeader
   */
  @InternalUseOnly
  public boolean addPacketToCacheAndDropOld (byte[] buf, int len) {
    int seq = dgd.getSeq();
    int numPktsAheadOfLowest = RTPHeader.computeNumPktsAhead(seq,lowestValidPkt);
    int numPktsAheadOfHighest = RTPHeader.computeNumPktsAhead(seq,highestPkt);
    LogUtilities.log(THIS_CLASS, detailLogLevel, "Number of Packet Ahead Calculations for pipe out = {0}" +
        " sequence number = {1}\n   lowest = {2}, ahead lowest = {3}, highest = {4}, ahead highest = {5}"
        , pipeOut.getName(), seq,lowestValidPkt, numPktsAheadOfLowest,
        highestPkt, numPktsAheadOfHighest);

    //**************************************************************************
    // IF CACHE INITIALIZATION IS NOT COMPLETE, THIS PACKET MIGHT HELP INITIALIZE
    //**************************************************************************
    if (!cacheFullyInitialized) initCacheBasedOnPkt(seq, len);

    //**************************************************************************
    // IF THIS PACKET IS TOO OLD, DROP IT
    //**************************************************************************
    if (numPktsAheadOfHighest <= -cacheSize) {
      logMethods.logDropped();
      return false;
    }

    //**************************************************************************
    // HANDLE CASE WHERE WE NEED TO CLEAR OUT SOME OF THE CACHED PACKETS BEFORE ADDING
    //**************************************************************************
    int numPacketsToClear = numPktsAheadOfLowest - cacheSize + 1; // If it is greater than or equal to the cache size,
                                                                  // we need to remove packets up to the current one we
                                                                  // are adding ( +1 accounts for the current)
    if (numPacketsToClear > 0) {
      int potentialLowest = clearPackets(lowestValidPkt, numPacketsToClear);

      // recalculate lowest
      int numOfPotentiallyValidPkts = RTPHeader.computeNumPktsAhead(highestPkt,potentialLowest) + 1;
      if (numOfPotentiallyValidPkts == 1) {
        lowestValidPkt = highestPkt;
      }
      else if (numOfPotentiallyValidPkts < 1) {
        lowestValidPkt = highestPkt = NOT_INITIALIZED; // we cleared out all our previous packets
      }
      else {
        // We still might have a lowestValidPacket that is different from our highestPkt
        // (check starting with the first not cleared up to and including the highestPkt
        for (int i = 0; i < numOfPotentiallyValidPkts; i++) {
          int indexToCheck = potentialLowest % cacheSize;
          if (viablePackets[indexToCheck]) {
            lowestValidPkt = potentialLowest;
            break;  // we have found the newest lowestValidPkt;
          }
          potentialLowest = RTPHeader.incrementAndWrapSeqNum(potentialLowest);
        }
      }
    }

    //**************************************************************************
    // NOW ADD PACKET TO CACHE
    //**************************************************************************
    boolean thisPacketHasICEHeader = addPacketToCache(buf, len);
    return thisPacketHasICEHeader;
  }

  /** Fill up and send out old incomplete frames
   *  Note: at least <b>packetsPerFrame</b> MUST be initialized before calling
   *  this method<br>
   *  <br>
   *  The user has three option for how missing data within a frame is filled
   *  <p>
   *  Option 1: Packet Size Pattern - used where there is a pattern to how much
   *  data is in each packet within a frame (i.e. first packet in frame has X
   *  amount of data, second packet of frame always has Y amount of data, etc.).
   *  This also involves a constant number of packets per frame.
   *  <p>
   *  Option 2: Have fixed data amount per packet (subset of option 1)
   *  <p>
   *  Option 3: Packet Per Frame - Does not rely on any pattern to the amount of
   *  data per packet, but only that there will always be a set number of
   *  packets per frame<br>
   *
   *  @param numFramesToOutput number of frames to fill as needed and output
   */
  @InternalUseOnly
  public void fillAndSendOutOldFrames (int numFramesToOutput) {
    // 1) need to effectively start sending with sequence number right after last sent
    // whether it is one with an ICE Header of not
    // 2) need to stop at frame boundary, where next frame contains the
    // packet we just received
    for (int f = 0; f < numFramesToOutput; f++) {
      if (usePacketSizePattern && (bytesDataPerPacket[0] != NOT_INITIALIZED)) {
        // make sure packet size pattern has been selected and the information has been initialized
        createFilledFrameUsingPktSizePattern();
      }
      else { // use only packets per frame
        // Make sure have packets per frame to use this
        createFilledFrameUsingOnlyPktsPerFrame();
      }
      int iceHeaderIndex = RTPHeader.incrementAndWrapSeqNum(lastPacketOutput)%cacheSize;
      LogUtilities.log(THIS_CLASS, mainLogLevel,"WRITING FRAME that starts with packet {0}"+
          " to output pipe {1} ", RTPHeader.incrementAndWrapSeqNum(lastPacketOutput), pipeOut.getName());

      if (associatedICEPackets[iceHeaderIndex]!= null) {
        pipeOut.setTimeAt(associatedICEPackets[iceHeaderIndex].getTC(), TimeLine.NotifyInvalidEntry.WARNING); // If we have an ice header, we need to set the timecodes
        previousICEPacket = associatedICEPackets[iceHeaderIndex];
      }
      pipeOut.write(frameToWrite,0,frameToWrite.length);
      clearFrameFromCache();
    }
  }

  /** Fills frameToWrite with full frame of valid data
   *  @param checkForRestart true if we should check for a restart being needed
   *  @return true restart if is needed before this frame can be output
   */
  @InternalUseOnly
  public boolean fillFullFrame (boolean checkForRestart) {
    int thisICESequenceNum = RTPHeader.incrementAndWrapSeqNum(lastPacketOutput);
    updateYDelta(thisICESequenceNum % cacheSize);
    boolean restartNeeded = checkForRestart && checkIfNeedsYDeltaRestart(thisICESequenceNum % cacheSize);
    if (!restartNeeded) {
      int fillLoc = 0;
      for (int posInFrame = 0; posInFrame < packetsPerFrame; posInFrame++) {
        int seqPkt = RTPHeader.addToSeqNumberWithWrap(thisICESequenceNum,posInFrame);
        int pktIdx = seqPkt % cacheSize;
        // Write either the full amount of data for this frame or the remaining amount of data
        // needed to fill the frame
        int bytesToWrite; // bytes to put in frame buffer
        bytesToWrite = ((fillLoc + storeBuf[pktIdx].length) <= frameToWrite.length) ?
            storeBuf[pktIdx].length : (frameToWrite.length - fillLoc);
        System.arraycopy(storeBuf[pktIdx], 0, frameToWrite, fillLoc, bytesToWrite);
        fillLoc += bytesToWrite;
        if (!cacheFullyInitialized || dataPerPacketsRecheck) {
          updateDataPerPacket(posInFrame, storeBuf[pktIdx].length);
        }
      }
    }
    return restartNeeded;
  }

  /** Fills in missing packets in a frame based on pre-existing pattern
   *  of the amount of data in each packet and the number of packets that
   *  form a frame
   *  <pre>
   *  Example of fill with 5 packets per frame and only 1st, 3rd, and 5th
   *  having valid data.
   *
   *    ***********************************************
   *  1 *  put in valid data from 1st packet
   *    ***********************************************
   *    ***********************************************
   *  2 *  fill in data for missing 2nd packet
   *    ***********************************************
   *    ***********************************************
   *  3 *  put in valid data from 3rd packet
   *    ***********************************************
   *    ***********************************************
   *  4 *  fill in data for missing 4th packet
   *    ***********************************************
   *    ***********************************************
   *  5 *  put in valid data from 5th packet
   *    ***********************************************
   *  </pre>
   */
  @InternalUseOnly
  public void createFilledFrameUsingPktSizePattern () {
    int thisICESequenceNumber = RTPHeader.incrementAndWrapSeqNum(lastPacketOutput);
    int fillLoc = 0;
    int currentCount = 0;
    for (int posInFrame = 0; posInFrame < packetsPerFrame; posInFrame++) {
      int packetIndex = RTPHeader.addToSeqNumberWithWrap(thisICESequenceNumber, posInFrame) % cacheSize;
      // Write either the full amount of data for this frame or the remaining amount of data
      // needed to fill the frame
      int bytesToWrite; // bytes to put in frame buffer
      if (viablePackets[packetIndex]) {
        bytesToWrite = ((fillLoc + storeBuf[packetIndex].length) <= frameToWrite.length) ?
            storeBuf[packetIndex].length : (frameToWrite.length - fillLoc);
        System.arraycopy(storeBuf[packetIndex], 0, frameToWrite, fillLoc, bytesToWrite);
      }
      else {
        currentCount++;
        logMethods.logDropped(); // log if needed. This also increments the number of packets dropped.
        bytesToWrite = ((fillLoc + bytesDataPerPacket[posInFrame]) <= frameToWrite.length) ?
            bytesDataPerPacket[posInFrame] : (frameToWrite.length - fillLoc);

        Arrays.fill(frameToWrite, fillLoc, fillLoc+bytesToWrite, missingFillValue);
      }
      fillLoc += bytesToWrite;
    }
    logMethods.logDroppedInFrame(currentCount);
  }

  /** Fills in missing packets in a frame based on only on pre-existing pattern
   *  of the number of packets that form a frame.
   *  This fills from the first packet until it reaches a missing packet,
   *  and then starts from the last packet and works backwards until it reaches
   *  a missing packet. The middle of the frame is then filled with the fill value.
   *  <pre>
   *  Example of fill with 5 packets per frame and only 1st, 3rd, and 5th
   *  having valid data.
   *
   *    *******************************************************
   *  1 *  put in valid data from 1st packet
   *    *******************************************************
   *    *******************************************************
   *  3 *  fill in data for 2nd, 3rd, and 4th packet
   *    *  (can not take advantage of valid data in 3rd packet)
   *    *******************************************************
   *    *******************************************************
   *  2 *  put in valid data from 5th packet
   *    *******************************************************
   *  </pre>
   */
  @InternalUseOnly
  public void createFilledFrameUsingOnlyPktsPerFrame () {
      int thisICESequenceNumber = RTPHeader.incrementAndWrapSeqNum(lastPacketOutput);
      int lastSequenceNumberForFrame = RTPHeader.addToSeqNumberWithWrap(thisICESequenceNumber, (packetsPerFrame - 1));
      int fillLoc = 0;
      // place in any valid packet data that is at the beginning
      int packetIndex = thisICESequenceNumber%cacheSize;
      int posInFrame = 0;
      while (viablePackets[packetIndex] && posInFrame < packetsPerFrame) {
        int bytesToWrite = ((fillLoc + storeBuf[packetIndex].length) <= frameToWrite.length) ?
            storeBuf[packetIndex].length : (frameToWrite.length - fillLoc);
        System.arraycopy(storeBuf[packetIndex], 0, frameToWrite, fillLoc, bytesToWrite);
        fillLoc += bytesToWrite;
        packetIndex=(packetIndex+1)%cacheSize;
        posInFrame++;
      }
      posInFrame = packetsPerFrame -1;
      int endBeginFill = fillLoc - 1;
      // place in any valid packet data that is at the end
      packetIndex = lastSequenceNumberForFrame % cacheSize;
      fillLoc = frameToWrite.length; //??
      while (viablePackets[packetIndex] && posInFrame >= 0) {
        int previousFillLoc = fillLoc;
        int expectedFillLoc = fillLoc - storeBuf[packetIndex].length;
        fillLoc = (expectedFillLoc > endBeginFill) ? expectedFillLoc : endBeginFill + 1;
        int bytesToWrite = previousFillLoc - fillLoc;
        System.arraycopy(storeBuf[packetIndex], 0, frameToWrite, fillLoc, bytesToWrite);
        posInFrame--;
        packetIndex = decrementAndWrapCacheSize(packetIndex);
      }

      int firstEndFill = fillLoc;
      // use fill value for any data in the middle (even if we have some of the packets)
      int numRemainingToFill = firstEndFill - (endBeginFill + 1);
      if (numRemainingToFill > 0) {
        Arrays.fill(frameToWrite, endBeginFill + 1, endBeginFill + 1 + numRemainingToFill, missingFillValue);
      }
    }

  /** Calculates current YDelta based on current and previous ICE Packet
   *  @param currentICEPacket  current ICE Packet (contains header information for current frame)
   *  @param previousICEPacket previous ICE Packet (contains header information for previous frame)
   *  @param defaultYDelta     used for default when adequate information is not available
   *  @return                  YDelta to use if ICE Packet Pair is not valid for calculating YDelta
   */
  @InternalUseOnly
  public static double calcYDelta (ICEPacket currentICEPacket, ICEPacket previousICEPacket, double defaultYDelta) {
    if ((currentICEPacket == null) || (previousICEPacket == null)) return defaultYDelta; // no information on which to base ydelta change
    double newYDelta;
    // Handle ICE Counts that are wrapping. ICE Count is 4 bytes (unsigned) but was being sorted in an
    // int in ICEPacket, but that has been updated.
    // dgnetsend and Jay's code do not account for this as of 12SEP18
    long countDiff = numIcePacketsAhead(currentICEPacket.getUnsignedCount(), previousICEPacket.getUnsignedCount());
    if ((Math.abs(countDiff) > MAX_ICE_COUNT_DIFF_CALC_YDELTA) || (countDiff == 0)) {
      // same packet / count or really far apart ICE Packets- can not calculate
      newYDelta = defaultYDelta;
    }
    else if (countDiff > 0) {
      newYDelta = currentICEPacket.getTC().diff(previousICEPacket.getTC()) / countDiff;
    }
    else {
      newYDelta = previousICEPacket.getTC().diff(currentICEPacket.getTC()) / -countDiff;
    }

    return newYDelta;
  }

  /** Calculates if this ICE Packet pair can be used for a Y-Delta calculation
   *  1) must not be the same packet
   *  2) need to be packets that are not too far apart from each other to get valid calculation
   *  3) the timestamps on the packets must not be identical
   *  @param currentICE  current ICE Packet
   *  @param previousICE previous ICE Packet
   *  @return if the pair are valid for using for a yDelta calculation
   */
  @InternalUseOnly
  public static boolean isValidICEPacketPairForYDelta (ICEPacket currentICE, ICEPacket previousICE) {
    long currentCount  = currentICE.getUnsignedCount();
    long previousCount = previousICE.getUnsignedCount();
    long numAhead = numIcePacketsAhead (currentCount, previousCount);
    double timeDiffSeconds = currentICE.getTC().diff(previousICE.getTC());
    return ((numAhead != 0) && (Math.abs(numAhead) <= MAX_ICE_COUNT_DIFF_CALC_YDELTA) && (timeDiffSeconds != 0));
  }

  /** Calculates the number of ICE Packets (i.e. frames) ahead current ICE Count is
   *  compared to the previous ICE Count (takes into account wrapping)
   *  @param currentCount  current ICE Packet Count
   *  @param previousCount previous ICE Packet count
   *  @return how many ICE Packets (frames) ahead the current is over the previous
   */
  private static long numIcePacketsAhead (long currentCount, long previousCount) {
    long pktsAhead;
    if (Math.abs(currentCount - previousCount) < MAX_AHEAD_ICE_COUNT) {
      // Assume has NOT wrapped
      pktsAhead = currentCount - previousCount;
    }
    else
      if (currentCount < previousCount) { // assume has wrapped and current is actually larger
        long adjustedCurrent = currentCount + ICEPacket.MAX_COUNT + 1; // get effective value
        pktsAhead = adjustedCurrent - previousCount;
      }
      else {
        long adjustedOld = previousCount + ICEPacket.MAX_COUNT + 1; // get effective value
        pktsAhead = currentCount - adjustedOld;
      }

    return pktsAhead;
  }



  /** Throw Midas Exception in case where not enough data is found to match frame size and BPA
   *  listed in ICE Header
   *  @param seqNumOfIceHdrForThisFrame sequence number of ICE Header for this frame
   *  @param bytesOfDataFound           bytes of data found related to current ICE Frame
   */
  private void exceptionInvalidFrameSize (int seqNumOfIceHdrForThisFrame,
      int bytesOfDataFound) {
    String frameSizeMismatchWarn = "Packets related to ICE Header only had " + bytesOfDataFound +
        " bytes of data,\nwhich is not enough to match initial frame size * BPA of "+
        frameToWrite.length + "\nCurrent ICE Header:\n" +
        associatedICEPackets[seqNumOfIceHdrForThisFrame % cacheSize];
    throw new MidasException(frameSizeMismatchWarn);
  }

  private void logStreamChangeRestart (boolean frameSizeChange, boolean xStartChange,
      boolean xDeltaChange, boolean timestampOutOfRange, int seq) {
    StringBuilder logStr = new StringBuilder(32768);
    logStr.append("Stream Change Detected based on: ");
    if (frameSizeChange) logStr.append(" frame size change,");
    if (xStartChange) logStr.append(" XStart change,");
    if (xDeltaChange) logStr.append(" XDelta change,");
    if (timestampOutOfRange) logStr.append(" out of range timestamp,");
    logStr.append(" based on packet with seq # "+seq);
    LogUtilities.log(THIS_CLASS, mainLogLevel,logStr.toString());
  }

  /** Cache initialization based on packet header (and extended header)
   *   @param seqNum sequence number of current packet
   *   @param len    size in bytes of data in current packet
   */
  private void initCacheBasedOnPkt (int seqNum, int len) {
    // Initialization that can use any packet
    if (dgd.getFill() && (bytesDataPerPacket[0] == NOT_INITIALIZED)) {
      Arrays.fill(bytesDataPerPacket,dgd.getLen());
      usePacketSizePattern = true;
    }

    // Initialization that requires ICE Packet
    boolean isValidICEHeader = (dgd.getLastValidIceHeader() == seqNum);
    if (isValidICEHeader) {
      // If this is our first ICE Header and frameToWrite has not been initialized, initialize it
      if (frameToWrite == null) {
        frameToWrite = new byte[dgd.lastICE.getBytes()];
      }

      // Need to set lastPacketOutput for the initial run. (example, if seq = 4, lastPacketOutput = 3)
      lastPacketOutput = (lastPacketOutput==NOT_INITIALIZED) ? RTPHeader.decrementAndWrapSeqNum(seqNum) : lastPacketOutput;

      // MAKE SURE packets per frame is initialized if we are in /FILL mode
      // (where the amount of data per packet is constant)
      if (dgd.getFill() && (packetsPerFrame == NOT_INITIALIZED)) {
        setPacketsPerFrame(dgd.lastICE.getBytes() / len);
      }
    }
    LogUtilities.log(THIS_CLASS, detailLogLevel, "Tried to initialize cache for pipe output = {0} "+
        "using sequence number = {1},\n with data length = {2}, isValidICEHeader = {3},"
        +" lastPacketOutput = {4}, packetsPerFrame = {5}",
        pipeOut.getName(), seqNum, len, isValidICEHeader, lastPacketOutput,packetsPerFrame);
    // Update the cache initialization state
    updateCacheFullyInitialized ();
  }

  /** Updates if the cache is fully initialized
   *  (i.e. has information needed to properly fill and write frames)
   */
  private void updateCacheFullyInitialized () {
    cacheFullyInitialized = (frameToWrite != null) &&
                          (lastPacketOutput != NOT_INITIALIZED) &&
                          (packetsPerFrame  != NOT_INITIALIZED) &&
                          (!usePacketSizePattern || (bytesDataPerPacket[0] != NOT_INITIALIZED));
  }

  /** Checks if this packet fills out frame of data and gives the sequence
   *  number matching the packet containing the ICE Header for this full frame
   *  Note: This method also handles initialization of packets per frame and
   *  data per packet
   *  @param thisPacketHasICEHeader indicates if the packet being added has an ICE header
   *  @param seq                    sequence number of packet added
   *  @return                       sequence number of packet containing ICE header
   *                                for the full frame resulting from adding this
   *                                packet (of NOT_INITIALIZED if not applicable)
   */
  private int checkIfHaveFullFrame (boolean thisPacketHasICEHeader, int seq) {
    LogUtilities.log(THIS_CLASS, detailLogLevel, "Checking if have full frame for pipe output = {0} "+
        "and current sequence number = {1},\n with data length = {2}, thisPacketHasICEHeader = {3}",
        pipeOut.getName(), seq, thisPacketHasICEHeader, lastPacketOutput, packetsPerFrame);
    // Handle the simple case of 1 packet per frame
    if (packetsPerFrame == 1) {
      return viablePackets[seq % cacheSize] ? seq : NOT_INITIALIZED;
    }

    // Handle the more complex case of multiple packets per frame
    int seqNumOfIceHdrFullFrame = NOT_INITIALIZED;
    // Get Sequence Number for Ice Header Packet belonging to this frame
    int seqNumOfIceHdrForThisFrame = NOT_INITIALIZED;
    if (thisPacketHasICEHeader) {
      seqNumOfIceHdrForThisFrame = seq;
    }
    else {
      // Go backwards through valid packets, wrapping packet # as needed
      for (int pktNum= RTPHeader.decrementAndWrapSeqNum(seq);
          RTPHeader.computeNumPktsAhead(pktNum,lowestValidPkt) >= 0;
          pktNum = RTPHeader.decrementAndWrapSeqNum(pktNum)) {
        if (associatedICEPackets[(pktNum % cacheSize)] != null) {
          seqNumOfIceHdrForThisFrame = pktNum;
          LogUtilities.log(THIS_CLASS, detailLogLevel,"Sequence number of ice header for output pipe {}"+
              " is {1}",pipeOut.getName(), seqNumOfIceHdrForThisFrame);
          break;
        }
      }
    }

    //*********************************************************************
    // IF DO NOT HAVE ICE HEADER FOR FRAME - RETURN
    //*********************************************************************
    if (seqNumOfIceHdrForThisFrame == NOT_INITIALIZED) {
      return NOT_INITIALIZED;
    }

    //*********************************************************************
    // CHECK IF WE HAVE ALL THE DATA FOR THIS FRAME
    //---------------------------------------------------------------------
    // We at least have an ICE header for this frame and there is a chance this packet
    // completes the frame, so check to see if we have all the data for this frame
   //*********************************************************************
    int packetsThisFrame = 1;
    int indexOfIceHdrPacket = seqNumOfIceHdrForThisFrame % cacheSize;
    if (associatedICEPackets[indexOfIceHdrPacket]==null) return NOT_INITIALIZED;
    int dataNeededForFrame = associatedICEPackets[indexOfIceHdrPacket].getBytes();
    LogUtilities.log(THIS_CLASS, detailLogLevel,"ouput pipe {0} dataNeededForFrame is {1}",
        pipeOut.getName(), dataNeededForFrame);
    dataNeededForFrame -= storeBuf[indexOfIceHdrPacket].length;
    if (dataNeededForFrame==0) {
      //we have complete frame with 1 packet!!!!
      setPacketsPerFrame(packetsThisFrame);
      seqNumOfIceHdrFullFrame = seqNumOfIceHdrForThisFrame;
    }
    else {
      LogUtilities.log(THIS_CLASS, detailLogLevel,"ouput pipe {0} dataNeededForFrame "+""
          + "after subtracting header packet is {1}",
          pipeOut.getName(), dataNeededForFrame);
      dataNeededForFrame = checkIfHaveAllData(dataNeededForFrame, packetsThisFrame, seqNumOfIceHdrForThisFrame);
      //*********************************************************************
      // RETURN SEQUENCE NUMBER OF ICE HEADER MATCHING FULL FRAME OF DATA
      //*********************************************************************
      seqNumOfIceHdrFullFrame = (dataNeededForFrame > 0) ? NOT_INITIALIZED : seqNumOfIceHdrForThisFrame;
    }

    return seqNumOfIceHdrFullFrame;
  }

  private boolean isNextFrameComplete () {
    for(int i = 1;i<=packetsPerFrame;i++) {
      if (!viablePackets[RTPHeader.addToSeqNumberWithWrap(lastPacketOutput, i) % cacheSize]) return false;
    }
    return true;
  }

  private int checkIfHaveAllData(int dataNeededForFrame, int packetsThisFrame, int seqNumOfIceHdrForThisFrame) {
    int seqNumOfNextPkt = RTPHeader.incrementAndWrapSeqNum(seqNumOfIceHdrForThisFrame); // first follow-on pkt
    // keep seeing if we have a full frame of data as long as
    // 1) more data is needed to complete frame
    // 2) we still have viable stored packets we have not checked
    // 3) we do not hit a missing packet
    while ((dataNeededForFrame > 0) && (RTPHeader.computeNumPktsAhead(seqNumOfNextPkt, highestPkt) <= 0) &&
           viablePackets[seqNumOfNextPkt % cacheSize]) {
      if (associatedICEPackets[seqNumOfNextPkt % cacheSize] != null) {
        exceptionInvalidFrameSize(seqNumOfIceHdrForThisFrame, frameToWrite.length - dataNeededForFrame);
      }
      packetsThisFrame++; // Need to increase the packet frames count because we have a packet to add to the frame
      dataNeededForFrame -= storeBuf[seqNumOfNextPkt % cacheSize].length;
      LogUtilities.log(THIS_CLASS, detailLogLevel,"ouput pipe {0} dataNeededForFrame while in loop is {1}",
          pipeOut.getName(), dataNeededForFrame);
      if (dataNeededForFrame <= 0) {
        // completed a packet - update packets/frame and bytes/packet information
        // TODO: in future, may want to detect if packetsPerFrame is changing and trigger restart
        setPacketsPerFrame(packetsThisFrame);
        int packetToReference = seqNumOfNextPkt; // start with last packet of frame
        for (int index = packetsPerFrame-1; index >= FRAME_START_BYTES_INDEX; index--) {
          bytesDataPerPacket[index] = storeBuf[packetToReference % cacheSize].length;
          packetToReference = RTPHeader.decrementAndWrapSeqNum(packetToReference); // back up to previous packet that was part of this frame
        }
        if (!cacheFullyInitialized) updateCacheFullyInitialized ();
      }
      seqNumOfNextPkt = RTPHeader.incrementAndWrapSeqNum(seqNumOfNextPkt);
    }
    return dataNeededForFrame;
  }

  /** Clear Packets starting at sequence number specified
   *  @param startingSeqNum    sequence number of first packet to clear
   *  @param numPacketsToClear number of packets to drop/clear
   *                           Note: These packets may or may not have actually arrived
   *  @return                  sequence number of first packet not cleared
   *                           (this packet may or may not have arrived)
   */
  private int clearPackets (int startingSeqNum, int numPacketsToClear) {
    LogUtilities.log(THIS_CLASS, detailLogLevel,"clearing {0} packets for output pipe {1} packets starting with sequence number {2}",
        pipeOut.getName(),numPacketsToClear,startingSeqNum);
    int seqForPktToClear = startingSeqNum;
    for (int clearCnt = 1; clearCnt <= numPacketsToClear; clearCnt++) {
      int index = seqForPktToClear % cacheSize;
      viablePackets[index] = false;
      associatedICEPackets[index] = null;
      logMethods.logDropped(); // We are dropping
      seqForPktToClear = RTPHeader.incrementAndWrapSeqNum(seqForPktToClear);
    }
    return seqForPktToClear;
  }

  boolean setPacketsPerFrame (int packetsPerFrame) {
    if (this.packetsPerFrame!=NOT_INITIALIZED && this.packetsPerFrame!=packetsPerFrame) LogUtilities.log(THIS_CLASS, Level.WARNING, "Changing packets per frame after it has already been set.");

    boolean changed = (this.packetsPerFrame != NOT_INITIALIZED) && (this.packetsPerFrame != packetsPerFrame);
    setMaxAheadBeforeAssumeSomeMissing(packetsPerFrame*maxMissingFrames);
    this.packetsPerFrame = packetsPerFrame;
    return changed;
  }

  /** Clear/Reset cache for packets just sent
   *  - these packets are no longer available for output
   *  - the ICE Packets no longer match frames to be output
   */
  private void clearFrameFromCache () {
    int seqOfICEForFrameJustWritten = RTPHeader.incrementAndWrapSeqNum(lastPacketOutput);
    // update last packet output
    lastPacketOutput = RTPHeader.addToSeqNumberWithWrap(lastPacketOutput, packetsPerFrame);
    for (int seqForPkt = seqOfICEForFrameJustWritten;
        RTPHeader.computeNumPktsAhead(seqForPkt,lastPacketOutput) <= 0;
        seqForPkt=RTPHeader.incrementAndWrapSeqNum(seqForPkt)) {
      int index = seqForPkt % cacheSize;
      viablePackets[index] = false;
      associatedICEPackets[index] = null;
    }

    if (lastPacketOutput==highestPkt) {
      lowestValidPkt=NOT_INITIALIZED;
      highestPkt = NOT_INITIALIZED;
    }
    else {
      int startSequence = RTPHeader.incrementAndWrapSeqNum(lastPacketOutput);
      while(startSequence != RTPHeader.computeNumPktsAhead(RTPHeader.incrementAndWrapSeqNum(lastPacketOutput), cacheSize)) {
        int index = startSequence % cacheSize;
        if (viablePackets[index]) {
          lowestValidPkt = startSequence;
          break;
        }
        startSequence = RTPHeader.incrementAndWrapSeqNum(startSequence);
      }
    }

  }

  /** Data per packet
   *  @param posInFrame      position of packet in group of packets that form a frame
   *  @param dataSizeInBytes bytes of data contained in this packet
   *  @return true if value was updated
   */
  private boolean updateDataPerPacket (int posInFrame, int dataSizeInBytes) {
    boolean updated = false;
    boolean wasUninitialized = bytesDataPerPacket[posInFrame] == NOT_INITIALIZED;
    if (bytesDataPerPacket[posInFrame] != dataSizeInBytes) {
      if (!wasUninitialized) {
        // TODO: A packet size pattern change other than possibly running out
        // of data on the last frame should trigger a hard restart, as it probably means
        // a stream change.  Since we do not want a restart when it is the end of the
        // stream, it is probably best to hope one of the other checks will detect a stream
        // change.
        LogUtilities.log(THIS_CLASS, Level.WARNING,"The pattern of how much data is in each packet within a frame has changed."+
            " Packet in position {0} used to contain {1} bytes, and now contains {2} bytes.",
            posInFrame, bytesDataPerPacket[posInFrame] ,dataSizeInBytes);
      }
      bytesDataPerPacket[posInFrame] = dataSizeInBytes;
      updated = true;
    }
    return updated;
  }

  /**
   * Helper function to wrap the index number
   * @param index
   * @return the updated value
   */
  private int decrementAndWrapCacheSize (int index) {
    if (index < 0)  return index; // -1 not a good index
    if (index == 0) return cacheSize-1; // wrap back around to largest sequence number
    return index - 1; // no wrapping needed
  }

  /** Method that checks if we need to restart based on a YDelta change
   *  @param indexICEHeader index of packet containing ICE Header for this frame
   *  @return true if need a restart
   */
  protected boolean checkIfNeedsYDeltaRestart (int indexICEHeader) {
    ICEPacket thisICEPacket = associatedICEPackets[indexICEHeader];
    if (thisICEPacket == null)return false; // no information on which to base restart
    if ((previousYDelta > -1) && !Tolerance.equalsWithAbsTol(currentYDelta, previousYDelta, yDeltaToleranceRestart)) {
      LogUtilities.log(THIS_CLASS, mainLogLevel, "output pipe {0} RESTARTING due to YDelta change on ICE with count {1}"+
          " - current YDelta = {2}, previous YDelta = {3}", pipeOut.getName(),
              thisICEPacket.getUnsignedCount(), currentYDelta, previousYDelta);
      icePacketRestart = thisICEPacket; // To be used by sourcedg as the first ICEPacket in the process
      resetFrameInformation(false); // Set this instantiation information for restart
      return true;
    }
    return false;
  }

  /** To be called when doing a restart triggered by a frame size.
   *  Many cache related components have to be re-initialized from scratch in this case.
   */
  private void resetFrameInformation(boolean frameSizeChange) {
    Arrays.fill(bytesDataPerPacket, NOT_INITIALIZED); // Bytes per packet will change since frame size has changed
    if (frameSizeChange) frameToWrite=null; // There should be no frame to write now, set it to null and it will be lazily instantiated later.
    cacheFullyInitialized=false; // We are no longer fully initialized
    packetsPerFrame = NOT_INITIALIZED; // The packets per frame may have changed
    firstRun=true; // We are now in first run again. We need to do the first run processing again.
  }

  /** Update YDelta fields based on the ICE Header referenced by this index
   *  This updates current and previous even if there is no change
   *  @param indexICEHeader index of ICE Header that is later (count greater) than previousICEPacket
   * @return true if fields updated
   */
  private boolean updateYDelta (int indexICEHeader) {
    ICEPacket thisICEPacket = associatedICEPackets[indexICEHeader];
    if ((thisICEPacket == null) || (previousICEPacket == null)) return false; // no information on which to base ydelta change
    long countDiff = numIcePacketsAhead(thisICEPacket.getUnsignedCount(), previousICEPacket.getUnsignedCount());
    if (countDiff <= 0) return false; // not updating if this ICE Header is the same or older ICE header
    previousYDelta = currentYDelta;
    if (countDiff > 1) {
      currentYDelta = thisICEPacket.getTC().diff(previousICEPacket.getTC()) / countDiff;
    }
    else if (countDiff == 1){
      currentYDelta = thisICEPacket.getTC().diff(previousICEPacket.getTC());
    }
    return true; // updated
  }

  /** Set initial yDelta values - these should generally be obtained from ICE Header information
   *  If the values are already initialized, do not change them
   *  @param yDelta intial value
   *  @return true if values updated
   */
  @InternalUseOnly
  public boolean setInitialYDelta (double yDelta) {
    if ((previousYDelta == NOT_INITIALIZED) && (currentYDelta == NOT_INITIALIZED)) {
      previousYDelta = yDelta;
      currentYDelta = yDelta;
      return true;
    }
    return false;  // not initializing because already intialized
  }

  /** Are we using a packet size pattern to fill in missing data
   *  @return true if we are using the packet size pattern
   */
  @InternalUseOnly
  public boolean isUsePacketSizePattern () {
    return usePacketSizePattern;
  }

  /** Set if we are using the packet size pattern within a frame to fill in
   *  missing data
   *  @param usePacketSizePattern true if we are to detect the packet size
   *                              pattern within and use it to fill in missing data
   */
  @InternalUseOnly
  public void setUsePacketSizePattern (boolean usePacketSizePattern) {
    this.usePacketSizePattern = usePacketSizePattern;
  }

  /** Get the YDelta change tolerance before a restart is needed
   *  @return yDeltaToleranceRestart YDelta change tolerance before a restart is needed
   */
  @InternalUseOnly
  public double getYDeltaToleranceRestart () {
    return yDeltaToleranceRestart;
  }

  /** Set the YDelta change tolerance before a restart is needed
   *  @param yDeltaToleranceRestart YDelta change tolerance before a restart is needed
   */
  @InternalUseOnly
  public void setYDeltaToleranceRestart (double yDeltaToleranceRestart) {
    this.yDeltaToleranceRestart = yDeltaToleranceRestart;
  }

  /** Get the XDelta change tolerance before a restart is needed
   *  @return XDelta change tolerance before a restart is needed
   */
  @InternalUseOnly
  public double getXDeltaToleranceRestart () {
    return xDeltaToleranceRestart;
  }

  /** Set the XDelta change tolerance before a restart is needed
   *  @param xDeltaToleranceRestart XDelta change tolerance before a restart is needed
   */
  @InternalUseOnly
  public void setXDeltaToleranceRestart (double xDeltaToleranceRestart) {
    this.xDeltaToleranceRestart = xDeltaToleranceRestart;
  }

  /** Get the XStart change tolerance before a restart is needed
   * @return XStart change tolerance before a restart is needed
   */
  @InternalUseOnly
  public double getXStartToleranceRestart () {
    return xStartToleranceRestart;
  }

  /** Set the XStart change tolerance before a restart is needed
   *  @param xStartToleranceRestart XStart change tolerance before a restart is needed
   */
  @InternalUseOnly
  public void setStartToleranceRestart (double xStartToleranceRestart) {
    this.xStartToleranceRestart = xStartToleranceRestart;
  }

  /** Are we re-checking the data per packet information each full frame
   *  @return true if we re-checking the data per packet information each full frame
   */
  @InternalUseOnly
  public boolean isDataPerPacketsRecheck () {
    return dataPerPacketsRecheck;
  }

  /** Set whether to re-check the data per packet information each full frame
   *  @param dataPerPacketsRecheck true to re-check the data per packet
   */
  @InternalUseOnly
  public void setDataPerPacketsRecheck (boolean dataPerPacketsRecheck) {
    this.dataPerPacketsRecheck = dataPerPacketsRecheck;
  }

  /** Set debug level - main log level
   *  @param newLevel new debug level
   */
  @InternalUseOnly
  public static void setDebugLevel(Level newLevel) {
    mainLogLevel = newLevel;
  }

  /** Set level for super detailed log messages
   *  Note: this should almost always stay at FINEST
   *  @param newLevel log level of the super detailed log messages
   */
  @InternalUseOnly
  public static void setDetailLogLevel(Level newLevel) {
    detailLogLevel = newLevel;
  }

  /** Set maximum frames ahead before assume some packets missing
   * @param maxMissingFrames maximum frames ahead before assume some packets missing
   */
  @InternalUseOnly
  public void setMaxMissingFrames (int maxMissingFrames) {
    this.maxMissingFrames = maxMissingFrames;
  }

  /** Set how often to log dropped packets - modulo value
   *  @param logFactor only log drops every this many drops
   */
  @InternalUseOnly
  public void setLogFactor (int logFactor) {
    this.logFactor = logFactor;
  }

  /** Get ICE Packet that has restart information
   *  @return the ICE Packet that has restart information
   */
  @InternalUseOnly
  public ICEPacket getIcePacketRestart () {
    return icePacketRestart;
  }

  /** Set ICE Packet that caused restart
   *  @param icePacketRestart ICE Packet that caused restart
   */
  public void setIcePacketRestart (ICEPacket icePacketRestart) {
    this.icePacketRestart = icePacketRestart;
  }

  /** Do we have a hard restart that indicates caller should
   *  just create a new RTPICEPacketsCache with the current packet
   *  @return true if we have a hard restart
   */
  public boolean isHardRestart () {
    return isHardRestart;
  }

  private class LoggerUtilityMethods{

    /** Log information on cache and where this packet is being stored
     *  (keeping this for future debug options)
     *  @param seq sequence number of packet being added to cache
     */
    private void logCacheAddInformation (int seq) {

      LogUtilities.log(THIS_CLASS, detailLogLevel,()->logCacheHelper(seq));
    }

    private String logCacheHelper(int seq) {
      StringBuilder debugStr = new StringBuilder(32768);
      int indexForStore = seq % cacheSize;
      debugStr.append("RTPICEPacketsCache adding packet: \n");
      debugStr.append("  Sequence          = ").append(seq          ).append('\n');
      debugStr.append("  cacheSize         = ").append(cacheSize    ).append('\n');
      debugStr.append("  indexForStore     = ").append(indexForStore).append('\n');
      debugStr.append("  length of buffers = ").append(dgd.getLen()).append('\n');
      return debugStr.toString();
    }

    /** Log when missing or dropping packet, need to log this if we hit a certain amount of packets lost.
     */
    private void logDropped () {
      packetsDroppedOrMissingForLogging++;
      //log factor used to determine if we need to log or not.
      if (packetsDroppedOrMissingForLogging % logFactor  == 0) {
        LogUtilities.log(THIS_CLASS, Level.WARNING,"Currently dropped or missing {0} packets overall"+
            " on output pipe {1}", packetsDroppedOrMissingForLogging, pipeOut.getName());
      }
    }

    /** Log if we hit a certain amount of packets lost.
     *  @param currentCount Current count of packets dropped, must be greater than 0;
     */
    private void logDroppedInFrame (int currentCount) {
      if (currentCount<=0) return;
      if (currentCount % MINIMUM_DROPPED_PACKETS_PER_FRAME_LOG == 0) { // log by factor of 10 only if it is 10 or higher
        LogUtilities.log(THIS_CLASS, Level.WARNING,"Currently dropped {0} packets in a frame"+
            " on output pipe {1}", currentCount, pipeOut.getName());
      }
    }
  }
}
