package nxm.sys.lib;

import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.SequenceInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import nxm.sys.inc.ListFile;
import nxm.sys.inc.MidasReference;
import nxm.sys.inc.ProvisionalUseOnly;
import nxm.sys.inc.Tablizable;


/** A class to handle Tape Archive (TAR) files. <br>
 *  <br>
 *  Basic TAR header (rarely seen):
 *  <pre>
 *    OFF SIZE  NAME        DESCRIPTION
 *    ==============================================
 *      0 100   name        name of file
 *    100 8     mode        file mode
 *    108 8     uid         owner user ID
 *    116 8     gid         owner group ID
 *    124 12    size        length of file in bytes
 *    136 12    mtime       modify time of file
 *    148 8     chksum      checksum for header
 *    156 1     typeflag    type of file
 *    157 100   linkname    name of linked file
 *    257
 *  </pre>
 *  USTAR Header:
 *  <pre>
 *    OFF SIZE  NAME        DESCRIPTION
 *    ==============================================
 *      0 100   name        name of file              --+
 *    100 8     mode        file mode                   |
 *    108 8     uid         owner user ID               |
 *    116 8     gid         owner group ID              | Same as a
 *    124 12    size        length of file in bytes     | Basic TAR
 *    136 12    mtime       modify time of file         | Header
 *    148 8     chksum      checksum for header         |
 *    156 1     typeflag    type of file                |
 *    157 100   linkname    name of linked file       --+
 *    257 6     magic       USTAR indicator
 *    263 2     version     USTAR version
 *    265 32    uname       owner user name
 *    297 32    gname       owner group name
 *    329 8     devmajor    device major number
 *    337 8     devminor    device minor number
 *    345 155   prefix      prefix for file name
 *    500
 *  </pre>
 *  @author  J. Czechowski IV
 *  @since   NeXtMidas 2.5.0
 */
public class TarFile extends BaseFile implements ListFile {
  /** Length of a standard TAR file header. <code>TAR_HEAD_LEN={@value}</code> */
  private static final int TAR_HEAD_LEN   = 256;

  /** Length of a USTAR file header. <code>USTAR_HEAD_LEN={@value}</code> <br>
   *  Note that only 500 bytes are actually used, however headers are 256-byte aligned.
   */
  private static final int USTAR_HEAD_LEN = 512;

  private static final char LINK_NORMAL_ALT        =  0;   // Link Indicator: Normal (same as LINK_NORMAL)
  private static final char LINK_NORMAL            = '0';  // Link Indicator: Normal
  private static final char LINK_HARD_LINK         = '1';  // Link Indicator: Hard Link (link to file in TAR)
  private static final char LINK_SYMBOLIC_LINK     = '2';  // Link Indicator: Symbolic Link
  private static final char LINK_CHARACTER_SPECIAL = '3';  // Link Indicator: Char Special
  private static final char LINK_BLOCK_SPECIAL     = '4';  // Link Indicator: Block Special
  private static final char LINK_DIRECTORY         = '5';  // Link Indicator: Directory
  private static final char LINK_FIFO              = '6';  // Link Indicator: FIFO
  private static final char LINK_CONTIGUIOUS_FILE  = '7';  // Link Indicator: Contig File (unofficial)

  // These are copied directly from the man page for chmod(2)
  private static final int S_IRWXU = 0000700;    /* RWX mask for owner */
  private static final int S_IRUSR = 0000400;    /* R for owner */
  private static final int S_IWUSR = 0000200;    /* W for owner */
  private static final int S_IXUSR = 0000100;    /* X for owner */

  private static final int S_IRWXG = 0000070;    /* RWX mask for group */
  private static final int S_IRGRP = 0000040;    /* R for group */
  private static final int S_IWGRP = 0000020;    /* W for group */
  private static final int S_IXGRP = 0000010;    /* X for group */

  private static final int S_IRWXO = 0000007;    /* RWX mask for other */
  private static final int S_IROTH = 0000004;    /* R for other */
  private static final int S_IWOTH = 0000002;    /* W for other */
  private static final int S_IXOTH = 0000001;    /* X for other */

  private static final int S_ISUID = 0004000;    /* set user id on execution */
  private static final int S_ISGID = 0002000;    /* set group id on execution */
  private static final int S_ISVTX = 0001000;    /* save swapped text even after use */


  private TarEntry entries[] = null;

  /** Creates a new uninitialized instance. */
  public TarFile () { }

  /** Creates a new instance.
   *  @param ref      The Midas reference to use.
   *  @param filename The file name to use.
   *  @deprecated since NeXtMidas 3.5.0 use {@link #TarFile(MidasReference, Object)}
   */
  @Deprecated
  public TarFile (Object ref, Object filename) {
    if (ref instanceof MidasReference) {
      init((MidasReference) ref, filename);
    } else {
      init(Convert.ref2Midas(ref), filename);
    }
  }

  /** Creates a new instance.
   *  @param ref      The Midas reference to use.
   *  @param filename The file name to use.
   *  @since NeXtMidas 3.5.0
   */
  public TarFile (MidasReference ref, Object filename) {
    init(ref,filename);
  }

  @Override
  public String getMimeType () {
    return "application/x-tar";
  }

  @Override
  public boolean open ()  {
    if (super.open()) {
      List<TarEntry> vect = new ArrayList<>();
      TarEntry entry;
      while ((entry = nextEntry()) != null) {
        vect.add(entry);
      }
      entries = vect.toArray(new TarEntry[vect.size()]);
    }
    return isOpen;
  }

  /** Gets a named TAR entry. Note that multiple entries in the tar are allowed to have the same
      name. When multiple entries have the same name, the LAST one is used (earlier entries are
      assumed to be out-of-date).
   */
  TarEntry getEntry (String name) { // package-private only used by TarResource
    TarEntry entry = null;
    for (int i = entries.length-1; (entry == null) && (i >= 0); i--) {
      if (entries[i].getName().equals(name)) {
        entry = entries[i];
      }
    }
    return entry;
  }

  /** Gets all the TAR entries.
   *  @return The file entries.
   *  @since NeXtMidas 3.3.0
   */
  @Override
  public Object[] getEntries () {
    return entries;  // added for Bug 2574 to allow for getting reference to entries
  }

  /** Reads the next entry from the file. */
  private TarEntry nextEntry () {
    TarEntry entry      = null;
    byte[]   buf        = new byte[USTAR_HEAD_LEN];
    long     fileOffset = (long)seek();
    int      numRead    = read(buf, 0, buf.length);

    if (numRead > 0) {
      entry = new TarEntry(buf,fileOffset);
      if (entry.isBlank()) {
        // Blank entry, assume end of file
        entry = null;
      } else {
        // Seek to start of next entry, note that entries are aligned to 512-byte blocks.
        long seek = (long)seek() + entry.getEntrySize() - buf.length;
        seek(seek);
      }
    }
    return entry;
  }

  @Override
  public String toString () {
    String str = "TarFile ";
    if (io!=null) str += io.getTypeString();
    str += " Resource";
    return str;
  }

  @Override
  public String listHeader () {
    String list =
      "TarFile   :  "+getURL()+"\n"
    + "Size      :  "+(long)getSize()+" bytes\n"
    + "Entries   :  "+entries.length+"\n";
    return list;
  }

  @Override
  public String listElements (double start, int elements, String format, int flags) {
    int i = (int)start;
    return (i < entries.length)? (entries[i].toString(true)+'\n'): null;
  }

  @Override
  public int listElementsPerLine (int lineWidth, String form, int flags) {
    return 1;
  }

  /** Gets the number of entries in the file.
   *  @return The number of entries in the file.
   */
  public int getNumEntries() { return entries.length; }



  //////////////////////////////////////////////////////////////////////////////////////////////////
  // Methods to implement ListFile
  //////////////////////////////////////////////////////////////////////////////////////////////////
  private void notEditable () { throw new UnsupportedOperationException("TarFile is not editable."); }
  @Override public double  getNumberOfRows ()                        { return getNumEntries(); }
  @Override public boolean getProtected ()                           { return true;   } // not editable at this time
  @Override public int     setData    (double d, Table t)            { notEditable(); return 0; }
  @Override public void    setData    (double d, String s, Object o) { notEditable(); }
  @Override public void    insertData (double d, Object o)           { notEditable(); }
  @Override public void    removeData (double d)                     { notEditable(); }
  @Override public void    removeData (double d, long c)             { notEditable(); }
  @Override public void    setRecordDefs (Table t)                   { notEditable(); }

  @Override
  public Table getDataTable (double offset) {
    if (offset < 0) return null;
    if (offset >= entries.length) return null;
    return entries[(int)offset].toTable();
  }

  @Override
  public int getRecordDefCount () {
    return ((entries.length == 0) || (entries[0].isUstar()))? 13 : 11;
  }

  @Override
  public Table getRecordDef (int i) {
    Table t = new Table();
    switch (i) {
      case  0: t.put("NAME", "NAME");                                        break;
      case  1: t.put("NAME", "MODE");                t.put("FORMAT", "SL");  break;
      case  2: t.put("NAME", "MODE_STRING");                                 break;
      case  3: t.put("NAME", "OWNER");               t.put("FORMAT", "SL");  break;
      case  4: t.put("NAME", "GROUP");               t.put("FORMAT", "SL");  break;
      case  5: t.put("NAME", "SIZE");                t.put("FORMAT", "SX");  break;
      case  6: t.put("NAME", "TIME");                t.put("FORMAT", "SD");  break;
      case  7: t.put("NAME", "CHECKSUM");                                    break;
      case  8: t.put("NAME", "LINK_INDICATOR");                              break;
      case  9: t.put("NAME", "LINK_INDICATOR_NAME");                         break;
      case 10: t.put("NAME", "LINKED_FILE");                                 break;
      case 11: t.put("NAME", "OWNER_NAME");                                  break; // USTAR ONLY
      case 12: t.put("NAME", "GROUP_NAME");                                  break; // USTAR ONLY
      // DEVICE
      default: throw new ArrayIndexOutOfBoundsException("Invalid record definition i="+i);
    }
    return t;
  }

  @Override
  public Table getRecordDefs () {
    Table t = new Table();
    for (int i = 0; i < getRecordDefCount(); i++) {
      t.put(Integer.toString(i), getRecordDef(i));
    }
    return t;
  }

  /** Creates an {@link InputStream} that holds the contents for a TAR file composed of the given
   *  entries. This stream can then be used to write the contents to disk or to a socket using the
   *  normal InputStream-to-OutputStream paradigm.
   *  @param entries The entries that make up the TAR file.
   *  @return The {@link InputStream}.
   *  @since NeXtMidas 3.7.0
   */
  @ProvisionalUseOnly("Added in NeXtMidas 3.7.0")
  public static InputStream asTarFileInputStream (TarEntryWithData... entries) {
    return asTarFileInputStream(Arrays.asList(entries));
  }

  /** Creates an {@link InputStream} that holds the contents for a TAR file composed of the given
   *  entries. This stream can then be used to write the contents to disk or to a socket using the
   *  normal InputStream-to-OutputStream paradigm.
   *  @param entries The entries that make up the TAR file.
   *  @return The {@link InputStream}.
   *  @since NeXtMidas 3.7.0
   */
  @ProvisionalUseOnly("Added in NeXtMidas 3.7.0")
  public static InputStream asTarFileInputStream (List<TarEntryWithData> entries) {
    List<InputStream> streams = new ArrayList<>(entries.size());
    for (TarEntryWithData e : entries) {
      e.pushTarEntry(streams);
    }

    // TAR always ends with zero padding equal to 2 headers in length (GNU TAR may pad with additional
    // zeros based on local buffer sizes -- we don't do this)
    streams.add(new PadInputStream(USTAR_HEAD_LEN*2, null));

    return new SequenceInputStream(Collections.enumeration(streams));
  }

  /** Creates a new TAR file. This is a static method since the {@link TarFile} class does not
   *  normally support writing to a TAR file (the mechanics of modifying a TAR file are more
   *  complex than just writing a TAR file all-in-one-go).
   *  @param ref   The MidasReference used when accessing the file data (can be null).
   *  @param fname The name of the file containing the data (can be a {@link java.io.File},
   *               {@link FileName} or {@link String}).
   *  @param entries The entries that make up the TAR file.
   *  @since NeXtMidas 3.7.0
   */
  public static void createTarFile (MidasReference ref, Object fname, TarEntryWithData... entries) {
    createTarFile(ref, fname, Arrays.asList(entries));
  }

  /** Creates a new TAR file. This is a static method since the {@link TarFile} class does not
   *  normally support writing to a TAR file (the mechanics of modifying a TAR file are more
   *  complex than just writing a TAR file all-in-one-go).
   *  @param ref   The MidasReference used when accessing the file data (can be null).
   *  @param fname The name of the file containing the data (can be a {@link java.io.File},
   *               {@link FileName} or {@link String}).
   *  @param entries The entries that make up the TAR file.
   *  @since NeXtMidas 3.7.0
   */
  public static void createTarFile (MidasReference ref, Object fname, List<TarEntryWithData> entries) {
    try {
      BaseFile    bf = new BaseFile(ref, fname); // use BaseFile not TarFile since we will write as bytes
      InputStream in = asTarFileInputStream(entries);
      bf.open(BaseFile.OUTPUT);

      byte[] buffer = new byte[4096];
      while (true) {
        int n = in.read(buffer, 0, buffer.length);
        if (n < 0) break;
        bf.write(buffer, 0, n);
      }
      in.close();
      bf.close();
    }
    catch (IOException e) {
      throw new MidasException("Unable to create TAR file "+fname, e);
    }
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // Inner class defining an entry in the TAR file
  //  - Made public in 3.3.0 for Bug 2574
  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** An entry in a TarFile. */
  public static class TarEntry implements Tablizable { // Tablizable since NeXtMidas 3.7.0, but had methods prior
    final byte[]  buf;
    final int     offset;
          boolean skipChecksum;

    /** Creates a new instance for an existing TAR header.
     *  @param buf The buffer containing the header.
     *  @param off The offset into the buffer.
     */
    public TarEntry (byte[] buf, long off) {
      this.buf          = buf;
      this.offset       = (int)align((isUstar())? off+USTAR_HEAD_LEN : off+TAR_HEAD_LEN);
      this.skipChecksum = false;
    }

    /** Creates a new instance of a USTAR header with the given parameters.
     *  @param name         The file name        (see {@link #setName}).
     *  @param mode         The file permissions (see {@link #setModeString}).
     *  @param owner        The file owner       (see {@link #setOwnerName}).
     *  @param ownerID      The file owner ID    (see {@link #setOwner}).
     *  @param owner        The file owner       (see {@link #setOwnerName}).
     *  @param groupID      The file group ID    (see {@link #setGroup}).
     *  @param time         The file mod time    (see {@link #setTime}).
     *  @param size         The file size        (see {@link #setSize}).
     *  @param skipChecksum Skip checksum computations.
     *  @since NeXtMidas 3.7.0
     */
    @SuppressWarnings("OverridableMethodCallInConstructor")
    TarEntry (String name, String mode, int ownerID, String owner, int groupID, String group,
              Time time, long size, boolean skipChecksum) {
      this.buf          = new byte[USTAR_HEAD_LEN];
      this.offset       = 0;
      this.skipChecksum = true; // skip since we will do a set of numerous fields

      // Set USTAR indicator
      buf[257] = 'u';
      buf[258] = 's';
      buf[259] = 't';
      buf[260] = 'a';
      buf[261] = 'r';
      buf[262] = ' ';
      buf[263] = ' ';
      buf[264] = '\0';

      // Set entries
      setName(name);
      setSize(size);
      setModeString(mode);
      setOwner(ownerID);
      setOwnerName(owner);
      setGroup(groupID);
      setGroupName(group);
      setTime(time);
      setLinkedFile(null);

      // Now set the checksum
      if (!skipChecksum) {
        this.skipChecksum = skipChecksum;
        updateCheckSum();
      }
    }

    /** Aligns an offset value to the next 512-byte blocks. */
    private static long align (long seek) {
      long mod  = seek % 512;
      if (mod != 0) seek = seek + 512 - mod;
      return seek;
    }

    long getOffset () { // package-private, used only by TarFile and TarResource
      return offset;
    }

    long getEntrySize () {
      long size = getSize();

      if (isUstar()) size += USTAR_HEAD_LEN;
      else           size += TAR_HEAD_LEN;

      return align(size);
    }

    /** Indicates if it is a USTAR entry. */
    private boolean isUstar() {
      return (buf[257] == 'u')
          && (buf[258] == 's')
          && (buf[259] == 't')
          && (buf[260] == 'a')
          && (buf[261] == 'r')
          && (buf[262] == ' ')
          && (buf[263] == ' ')
          && (buf[264] == '\0');
    }

    /** Indicates if it is a blank entry. */
    boolean isBlank () {
      boolean blank = true;
      for (int i = 0; blank && (i < buf.length); i++) {
        blank = (buf[i] == 0);
      }
      return blank;
    }

    @Override
    public String toString () {
      return toString(false);
    }

    String toString (boolean pad) {
      // This syntax matches that used by 'tar tvf <file>' except that pad causes the
      // file sizes to be padded out (like 'ls -al') so things line up nicely.
      StringBuilder str  = new StringBuilder(32);
      String        size = Long.toString(getSize());

      if (pad) size = StringUtil.padLeft(size, 9); // <-- JC4: 9 seems to match 'ls -al' on my box

      str.append(getModeString()).append(' ');
      str.append(getOwnerName()).append('/');
      str.append(getGroupName()).append(' ');
      str.append(size).append(' ');
      str.append(getTime().toString(-1)).append(' '); // <-- Time in tar file has no fractional seconds
      str.append(getName());

      if (isLink()) {
        str.append(" -> ").append(getLinkedFile());
      }
      return str.toString();
    }

    @Override
    public Table toTable () {
      Table tbl = new Table();
      tbl.put("NAME",                getName());
      tbl.put("MODE",                getMode());
      tbl.put("MODE_STRING",         getModeString());
      tbl.put("OWNER",               getOwner());
      tbl.put("OWNER_NAME",          getOwnerName());     // Prior to 3.7.0 this was skipped for non-USTAR
      tbl.put("GROUP",               getGroup());
      tbl.put("GROUP_NAME",          getGroupName());     // Prior to 3.7.0 this was skipped for non-USTAR
      tbl.put("TIME",                getTime().getSec()); // Uses FORM=SD (pre 3.7.0)
      tbl.put("TIMESTAMP",           getTime());          // Since NeXtMidas 3.7.0
      tbl.put("CHECKSUM",            getCheckSum());
      tbl.put("LINK_INDICATOR",      getLinkIndicator());
      tbl.put("LINK_INDICATOR_NAME", getLinkIndicatorName());
      tbl.put("LINKED_FILE",         getLinkedFile(""));
      return tbl;
    }

    /** Dumps info to a string */
    String dumpToString (String sp) {
      return sp + "Name           : " + getName() + "\n"
           + sp + "Mode           : " + getModeString() + " (mode=" + getMode() + ")\n"
           + sp + "Owner          : " + getOwnerName() + " (ID=" + getOwner() + ")\n"
           + sp + "Group          : " + getGroupName() + " (ID=" + getGroup() + ")\n"
           + sp + "File Size      : " + getSize() + " bytes\n"
           + sp + "Last Mod Time  : " + getTime() + "\n"
           + sp + "Check Sum      : " + getCheckSum() + "\n"
           + sp + "Link Indicator : " + getLinkIndicatorName() + "\n"
           + sp + "Linked File    : " + getLinkedFile("n/a") + "\n"
           ;
    }

    /** Gets name of archived file.
     *  @return The name of archived file.
     */
    public String getName () {
      String name;
      if (isUstar()) {
        name = unpackS(345, 155) + unpackS(0, 100);
      } else {
        name = unpackS(0, 100);
      }
      return name;
    }

    /** Sets name of archived file.
     *  @param name The name of archived file.
     */
    public void setName (String name) {
      if (name == null) name = "";
      int length = name.length();

      if (!isUstar()) {
        packS(  0, 100, name);
      }
      else if (length > 100) {
        packS(  0, 100, name.substring(length-100));
        packS(345, 155, name.substring(0,length-100));
      }
      else {
        packS(  0, 100, name);
        packS(345, 155, null); // clear out name prefix
      }
      _updateCheckSum();
    }

    /** Gets the file mode (Unix-style file permissions) as a string.
     *  @return The file mode (Unix-style file permissions) as a string.
     */
    public String getModeString () {
      StringBuilder str = new StringBuilder(10);
      int link = getLinkIndicator();
      int mode = getMode();

      switch (link) {
        case LINK_NORMAL_ALT:        str.append('-'); break;
        case LINK_NORMAL:            str.append('-'); break;
        case LINK_HARD_LINK:         str.append('l'); break; // <-- matches behavior of tar
        case LINK_SYMBOLIC_LINK:     str.append('l'); break;
        case LINK_CHARACTER_SPECIAL: str.append('c'); break;
        case LINK_BLOCK_SPECIAL:     str.append('b'); break;
        case LINK_DIRECTORY:         str.append('d'); break;
        case LINK_FIFO:              str.append('p'); break;
        default:                     str.append('?'); break; // <-- LINK_CONTIGUIOUS_FILE, other?
      }

      if ((mode & S_IRUSR) != 0) str.append('r');  else  str.append('-');
      if ((mode & S_IWUSR) != 0) str.append('w');  else  str.append('-');
      if ((mode & S_IXUSR) != 0) str.append('x');  else  str.append('-');

      if ((mode & S_IRGRP) != 0) str.append('r');  else  str.append('-');
      if ((mode & S_IWGRP) != 0) str.append('w');  else  str.append('-');
      if ((mode & S_IXGRP) != 0) str.append('x');  else  str.append('-');

      if ((mode & S_IROTH) != 0) str.append('r');  else  str.append('-');
      if ((mode & S_IWOTH) != 0) str.append('w');  else  str.append('-');
      if ((mode & S_IXOTH) != 0) str.append('x');  else  str.append('-');

      if ((mode & S_ISUID) != 0) {
        if ((mode & S_IXUSR) != 0) str.setCharAt(3, 's');  else  str.setCharAt(3, 'S');
      }
      if ((mode & S_ISGID) != 0) {
        if ((mode & S_IXGRP) != 0) str.setCharAt(6, 's');  else  str.setCharAt(6, 'S');
      }
      if ((mode & S_ISVTX) != 0) {
        if ((mode & S_IXOTH) != 0) str.setCharAt(6, 't');  else  str.setCharAt(6, 'T');
      }

      return (link == LINK_CONTIGUIOUS_FILE)? null : str.toString();
    }

    /** Gets the file mode (Unix-style file permissions) as a string.
     *  @param str The file mode (Unix-style file permissions) as a string (e.g. "-rwxrwxrwx"
     *             or "rwxrwxrwx") or the file mode as a 3-digit octal number. Set this to
     *             null to use the default ("664" permissions).
     *  @since NeXtMidas 3.7.0
     */
    public void setModeString (String str) {
      if (str          == null) { setMode(0664);                    return; }
      if (str.isEmpty()       ) { setMode(0664);                    return; }
      if (str.length() ==    3) { setMode(Integer.parseInt(str,8)); return; }
      if (str.length() ==    9) str = "-"+str; // normal file
      if (str.length() !=   10) throw new MidasException("Illegal file mode: "+str);

      switch (str.charAt(0)) {
        case '-': setLinkIndicator(LINK_NORMAL);            break;
        case 'l': setLinkIndicator(LINK_HARD_LINK);         break; // assume LINK_HARD_LINK vs LINK_SYMBOLIC_LINK
        case 'c': setLinkIndicator(LINK_CHARACTER_SPECIAL); break;
        case 'b': setLinkIndicator(LINK_BLOCK_SPECIAL);     break;
        case 'd': setLinkIndicator(LINK_DIRECTORY);         break;
        case 'p': setLinkIndicator(LINK_FIFO);              break;
        case '?': /* ignore */                              break;
        default: throw new MidasException("Illegal file mode: "+str);
      }

      int mode = 0;
      mode |= modeFlag(str, 1, S_IRUSR, 'r',               0, '-', 0,       '-');
      mode |= modeFlag(str, 2, S_IWUSR, 'w',               0, '-', 0,       '-');
      mode |= modeFlag(str, 3, S_IXUSR, 'x', S_ISUID|S_IXUSR, 's', S_ISUID, 'S');
      mode |= modeFlag(str, 4, S_IRGRP, 'r',               0, '-', 0,       '-');
      mode |= modeFlag(str, 5, S_IWGRP, 'w',               0, '-', 0,       '-');
      mode |= modeFlag(str, 6, S_IXGRP, 'x', S_ISGID|S_IXGRP, 's', S_ISGID, 'S');
      mode |= modeFlag(str, 4, S_IROTH, 'r',               0, '-', 0,       '-');
      mode |= modeFlag(str, 5, S_IWOTH, 'w',               0, '-', 0,       '-');
      mode |= modeFlag(str, 6, S_IXOTH, 'x', S_ISVTX|S_IXOTH, 't', S_ISVTX, 'T');
      setMode(mode);
    }

    /** Utility method used by setModeString(..).
     *  @param str   Input string.
     *  @param off   Current offset into string.
     *  @param flag1 Mode flag (primary).
     *  @param on1   Mode char (primary).
     *  @param flag2 Mode flag (alternate) or 0 if n/a.
     *  @param on2   Mode flag (alternate) or '-' if n/a.
     *  @param flag3 Mode flag (alternate) or 0 if n/a.
     *  @param on3   Mode flag (alternate) or '-' if n/a.
     *  @return The mode flag selected.
     *  @since NeXtMidas 3.7.0
     */
    private int modeFlag (String str, int off, int flag1, char on1,
                          int flag2, char on2, int flag3, char on3) {
      char c = str.charAt(off);
      if (c == '-' ) return 0;
      if (c == on1 ) return flag1;
      if (c == on2 ) return flag2;
      if (c == on3 ) return flag3;
      throw new MidasException("Illegal file mode: "+str);
    }

    /** Gets the file mode (Unix-style file permissions) as an integer.
     *  @return The file mode (Unix-style file permissions) as an integer.
     *  @see #getModeString()
     */
    public int getMode () {
      return unpackL(100, 8);
    }

    /** Sets the file mode (Unix-style file permissions) as an integer.
     *  @param val The file mode (Unix-style file permissions) as an integer.
     */
    public void setMode (int val) {
      packL(100, 8, val);
      _updateCheckSum();
    }

    /** Gets the numeric UID (user ID) of the file.
     *  @return The UID for the owner of the file.
     */
    public int getOwner () {
      return unpackL(108, 8);
    }

    /** Sets the numeric UID (user ID) of the file.
     *  @param val The UID for the owner of the file.
     */
    public void setOwner (int val) {
      packL(108, 8, val);
      _updateCheckSum();
    }

    /** Gets the name of the file owner (USTAR ONLY). For non-USTAR files this returns the string
     *  version of {@link #getOwner()}.
     *  @return The name of the owner of the file.
     */
    public String getOwnerName () {
      return (isUstar())? unpackS(265, 32): Integer.toString(getOwner());
    }

    /** Sets the name of the file owner (USTAR ONLY). If the file is not a USTAR file, any calls to
     *  set are silently ignored.
     *  @param val The name of the owner of the file.
     */
    public void setOwnerName (String val) {
      packS(265, 32, val);
      _updateCheckSum();
    }

    /** Gets the numeric GID (group ID) of the file.
     *  @return The GID for the owner of the file.
     */
    public int getGroup () {
      return unpackL(116, 8);
    }

    /** Sets the numeric GID (group ID) of the file.
     *  @param val The GID for the owner of the file.
     */
    public void setGroup (int val) {
      packL(116, 8, val);
      _updateCheckSum();
    }


    /** Gets the name of the file group (USTAR ONLY). For non-USTAR files this returns the string
     *  version of {@link #getGroup()}.
     *  @return The name of the group of the file.
     */
    public String getGroupName () {
      return (isUstar())? unpackS(297, 32): Integer.toString(getGroup());
    }

    /** Sets the name of the file group (USTAR ONLY). If the file is not a USTAR file, any calls to
     *  set are silently ignored.
     *  @param val The name of the group of the file.
     */
    public void setGroupName (String val) {
      packS(297, 32, val);
      _updateCheckSum();
    }

    /** Gets the number pad bytes inserted into the TAR following the data such that the data
     *  segment takes up 512 bytes in length.
     *  @return The number of pad bytes.
     *  @since NeXtMidas 3.7.0
     */
    int getPadding () {
      long padBytes = (512 - (getSize() % 512)) % 512;
      return (int)padBytes;
    }

    /** Gets the size of the file.
     *  @return The size of the file in octets.
     */
    public long getSize () {
      return unpackL(124, 12);
    }

    /** Sets the size of the file.
     *  @param val The size of the file in octets.
     */
    public void setSize (long val) {
      // prior to 3.7.0 this took in an 'int' and packed the number incorrectly
      packX(124, 12, val);
      _updateCheckSum();
    }

    /** Returns creation time of file in seconds since 1970.
     *  @return Creation time of file in seconds since 1970.
     *  @since NeXtMidas 3.3.0
     *  @see #getTime()
     */
    public long getTime1970 () {
      return unpackX(136, 12);
    }

    /** Returns creation time of file in seconds since 1970.
     *  @param val Creation time of file in seconds since 1970.
     *  @since NeXtMidas 3.7.0
     *  @see #getTime()
     */
    public void setTime1970 (long val) {
      packX(136, 12, val);
      _updateCheckSum();
    }

    /** Gets the file creation time.
     *  @return Creation time of file.
     */
    public Time getTime () {
      double j1970 = unpackX(136, 12);
      return new Time(j1970 + Time.J1970TOJ1950);
    }

    /** Sets the file creation time.
     *  @param t Creation time of file (null or 0 to use current system time).
     */
    public void setTime (Time t) {
      if ((t == null) || (t.isZero())) {
        setTime1970(System.currentTimeMillis() / 1000); // since NeXtMidas 3.7.0
      }
      else {
        long j1970 = (long)(t.getWSec() - Time.J1970TOJ1950);
        setTime1970(j1970);
      }
    }

    /** Gets the header checksum.
     *  @return The header checksum as a string.
     */
    public String getCheckSum () {
      return unpackS(148, 8);
    }

    /** Sets the header checksum manually. Most users should call {@link #updateCheckSum()} to
     *  update the checksum.
     *  @param val The header checksum as a string.
     */
    public void setCheckSum (int val) {
      packL(148, 7, val);
      // NUL and SPACE follow checksum in current (GNU-style) usage
      buf[offset+154] = '\0'; // insert space at end
      buf[offset+155] = ' ';  // insert space at end
      // Note that this does NOT call updateCheckSum();
    }

    /** Recomputes the checksum and sets it in the header if skipChecksum is not set.
     *  @since NeXtMidas 3.7.0
     */
    private void _updateCheckSum () {
      if (!skipChecksum) updateCheckSum();
    }

    /** Recomputes the checksum and sets it in the header.
     *  @since NeXtMidas 3.7.0
     */
    public void updateCheckSum () {
      buf[offset+148] = ' ';
      buf[offset+149] = ' ';
      buf[offset+150] = ' ';
      buf[offset+151] = ' ';
      buf[offset+152] = ' ';
      buf[offset+153] = ' ';
      buf[offset+154] = ' ';
      buf[offset+155] = ' ';
      int len   = (isUstar())? USTAR_HEAD_LEN : TAR_HEAD_LEN;
      int value = 0;
      for (int i = 0; i < len; i++) {
        value += 0xFF & buf[offset+i];
      }
      setCheckSum(value);
    }

    /** Gets the link indicator char.
     *  @return The link indicator char.
     *  @see #getLinkIndicatorName()
     */
    public char getLinkIndicator () {
      return (char)buf[156];
    }

    /** Sets the link indicator char.
     *  @param val The link indicator char (must be a string of length 1).
     *  @since NeXtMidas 3.7.0
     */
    public void setLinkIndicator (String val) {
      if (val.length() != 1) throw new MidasException("Invalid link indicator: '"+val+"'");
      setLinkIndicator(val.charAt(0));
    }

    /** Sets the link indicator char.
     *  @param val The link indicator char.
     */
    public void setLinkIndicator (char val) {
      buf[156] = (byte)val;
      _updateCheckSum();
    }

    /** Gets the link indicator name.
     *  @return The link indicator name.
     */
    public String getLinkIndicatorName () {
      char c = getLinkIndicator();
      switch (c) {
        case LINK_NORMAL_ALT:         return "Normal";
        case LINK_NORMAL:             return "Normal";
        case LINK_HARD_LINK:          return "HardLink";
        case LINK_SYMBOLIC_LINK:      return "SymbolicLink";
        case LINK_CHARACTER_SPECIAL:  return "CharacterSpecial";
        case LINK_BLOCK_SPECIAL:      return "BlockSpecial";
        case LINK_DIRECTORY:          return "Directory";
        case LINK_FIFO:               return "FIFO";
        case LINK_CONTIGUIOUS_FILE:   return "ContiguousFile";
        default: throw new MidasException("Unknown link type '"+c+"'.");
      }
    }

    /** Gets the link indicator name.
     *  @param val The link indicator name.
     *  @since NeXtMidas 3.7.0
     */
    public void setLinkIndicatorName (String val) {
           if (val.equalsIgnoreCase("Normal"          )) setLinkIndicator(LINK_NORMAL);
      else if (val.equalsIgnoreCase("HardLink"        )) setLinkIndicator(LINK_HARD_LINK);
      else if (val.equalsIgnoreCase("SymbolicLink"    )) setLinkIndicator(LINK_SYMBOLIC_LINK);
      else if (val.equalsIgnoreCase("CharacterSpecial")) setLinkIndicator(LINK_CHARACTER_SPECIAL);
      else if (val.equalsIgnoreCase("BlockSpecial"    )) setLinkIndicator(LINK_BLOCK_SPECIAL);
      else if (val.equalsIgnoreCase("Directory"       )) setLinkIndicator(LINK_DIRECTORY);
      else if (val.equalsIgnoreCase("FIFO"            )) setLinkIndicator(LINK_FIFO);
      else if (val.equalsIgnoreCase("ContiguousFile"  )) setLinkIndicator(LINK_CONTIGUIOUS_FILE);
      else throw new MidasException("Unknown link type: '"+val+"'.");
    }

    /** Indicates if the file is a link (hard link or symbolic link).
     *  @return true if a link, false otherwise.
     */
    public boolean isLink () {
      char c = getLinkIndicator();
      return (c == LINK_HARD_LINK) || (c == LINK_SYMBOLIC_LINK);
    }

    /** Gets the name of the linked file.
     *  @return The name of the linked file (null if not a linked file).
     */
    public String getLinkedFile () {
      return getLinkedFile(null);
    }

    /** Gets the name of the linked file.
     *  @param def Default value if not set (should be null or "").
     *  @return The name of the linked file (null if not a linked file).
     *  @since NeXtMidas 3.7.0
     */
    private String getLinkedFile (String def) {
      String str = def;
      char c = getLinkIndicator();

      if ((c == LINK_HARD_LINK) || (c == LINK_SYMBOLIC_LINK)) {
        str = unpackS(157, 100);
      }
      return str;
    }

    /** Sets the name of the linked file.
     *  @param val The name of the linked file (null if not a linked file).
     */
    public void setLinkedFile (String val) {
      // Since NeXtMidas 3.7.0 this updates the link setting as appropriate
      if (val == null) {
        setLinkIndicator(LINK_NORMAL);
      }
      else if (!isLink()) {
        // keep current setting
      }
      else {
        setLinkIndicator(LINK_SYMBOLIC_LINK); // symbolic link by default
      }
      packS(157, 100, val);
      _updateCheckSum();
    }

    /** Unpacks a SL from the byte array. This handles the OCTAL to int conversion. */
    private int unpackL (int off, int len) {
      return (int)unpackX(off, len);
    }

    /** Packs a SL in the byte array. This handles the int to OCTAL conversion. */
    private void packL (int off, int len, int val) {
      packX(off, len, val);
    }

    /** Unpacks a SX from the byte array. This handles the OCTAL to long conversion. */
    private long unpackX (int off, int len) {
      String str = unpackS(off, len);
      try {
        return (str.length() == 0)? 0L : Long.parseLong(str, 8);
      } catch (NumberFormatException e) {
        throw new MidasException("Error converting octal number '"+str+"' to decimal.");
      }
    }

    /** Packs a SX in the byte array. This handles the long to OCTAL conversion. */
    private void packX (int off, int len, long val) {
      String str  = Long.toOctalString(val);
      while (str.length() < len-1) {
        str = "0" + str;
      }
      if (str.length() >= len) {
        throw new IllegalArgumentException("Given value exceeds limits: "+val);
      }
      packS(off, len, str);
    }

    /** Unpacks a string from the byte array. This will trim off any padding characters. */
    private String unpackS (int off, int len) {
      if ((off > 256) && !isUstar()) return null;

      while ((len > 0) && (buf[off+len-1] == 0)) len--;

      char[] chars = new char[len];
      for (int i = 0; i < len; i++) {
        chars[i] = (char)buf[i+off];
      }
      return new String(chars);
    }

    /** Packs a string from the byte array. This will add any required padding characters. */
    private void packS (int off, int len, String str) {
      if (str == null) str = "";
      if ((off > 256) && !isUstar()) {
        throw new IllegalArgumentException("TarFile: Can not set string value "
                                          + "'"+str+"' at off="+off+" tar file "
                                          + "is not in in USTAR format.");
      }
      if (str.length() > len) {
        throw new IllegalArgumentException("TarFile: String value '"+str+"' "
                                         + "exceeds maximum length of "+len+".");
      }
      for (int i = 0; i < len; i++) {
        buf[off++] = (byte)((i < str.length())? str.charAt(i) : '\0');
      }
    }
  }

  //////////////////////////////////////////////////////////////////////////////////////////////////
  // tbd
  //////////////////////////////////////////////////////////////////////////////////////////////////
  /** An entry in a TarFile inclusive of the file data.
   *  @since NeXtMidas 3.7.0
   */
  @ProvisionalUseOnly("Added in NeXtMidas 3.7.0")
  public static class TarEntryWithData extends TarEntry implements Closeable {
    final InputStream data;    // The data content
    final Closeable   toClose; // Additional item to close

    /** Creates a new instance of a USTAR header with the given parameters. This is identical to
     *  <code>new TarEntryWithData(name,"664",0,"root",0,"root",null,ref,fname)</code>. The use of 0 and
     *  "root" as for the user and group IDs is a "best practice" for TAR files being distributed.
     *  @param name  The file name as represented in the TAR file (if null the name of the file
     *               is used). If the name ends with a "/" it will be taken as a directory prefix
     *               to the file name.
     *  @param ref   The MidasReference used when accessing the file data (can be null).
     *  @param fname The name of the file containing the data (can be a {@link java.io.File},
     *               {@link FileName} or {@link String}).
     *  @since NeXtMidas 3.7.0
     */
    public TarEntryWithData (String name, MidasReference ref, Object fname) {
      this(name,"664",0,"root",0,"root",null,ref,fname);
    }

    /** Creates a new instance of a USTAR header with the given parameters. This constructor creates
     *  an entry that is the most similar to what GNU TAR will create on most Linux platforms.
     *  @param name    The file name        (see {@link #setName}).
     *  @param mode    The file permissions (see {@link #setModeString}) (e.g. "664" or "-rw-rw-r--").
     *  @param ownerID The file owner ID    (see {@link #setOwner}).
     *  @param owner   The file owner       (see {@link #setOwnerName}).
     *  @param groupID The file group ID    (see {@link #setGroup}).
     *  @param group   The file group       (see {@link #setGroupName}).
     *  @param time    The file mod time    (see {@link #setTime}).
     *  @param ref     The MidasReference used when accessing the file data (can be null).
     *  @param fname   The name of the file containing the data (can be a {@link java.io.File},
     *                 {@link FileName} or {@link String}).
     *  @since NeXtMidas 3.7.0
     */
    public TarEntryWithData (String name, String mode, int ownerID, String owner, int groupID,
                             String group, Time time, MidasReference ref, Object fname) {
      super(name,mode,ownerID,owner,groupID,group,time,0L,true);
      BaseFile bf = new BaseFile(ref,fname);
      bf.open();

      long len = bf.io.getLength();
      setSize(len);
      if (len == 0) {
        Shell.warning("Zero length file given "+bf.getName());
      }
      if ((name == null) || name.isEmpty()) {
        setName(bf.getName().getBasename());
      }
      else if (name.endsWith("/")) {
        setName(name + bf.getName().getBasename());
      }
      if (time == null) {
        setTime(bf.io.lastModifiedTime());
      }

      this.data         = bf.io.getInputStream();
      this.toClose      = bf;
      this.skipChecksum = false;
      updateCheckSum();
    }

    /** Creates a new instance of a USTAR header with the given parameters.
     *  @param name    The file name        (see {@link #setName}).
     *  @param mode    The file permissions (see {@link #setModeString}) (e.g. "664" or "-rw-rw-r--").
     *  @param ownerID The file owner ID    (see {@link #setOwner}).
     *  @param owner   The file owner       (see {@link #setOwnerName}).
     *  @param groupID The file group ID    (see {@link #setGroup}).
     *  @param group   The file group       (see {@link #setGroupName}).
     *  @param time    The file mod time    (see {@link #setTime}).
     *  @param size    The file size        (see {@link #setSize}). This must match data size.
     *  @param data    The file data.
     *  @since NeXtMidas 3.7.0
     */
    public TarEntryWithData (String name, String mode, int ownerID, String owner, int groupID,
                             String group, Time time, long size, InputStream data) {
      super(name,mode,ownerID,owner,groupID,group,time,size,true);
      this.data    = data;
      this.toClose = null;
    }

    /** Push the contents of this TAR entry to the list of streams provided.
     *  @param streams The list of streams to add to.
     */
    void pushTarEntry (List<InputStream> streams) {
      streams.add(new ByteArrayInputStream(buf, offset, USTAR_HEAD_LEN));
      if (getSize() > 0) streams.add(data);
      streams.add(new PadInputStream(getPadding(), this));
    }

    @Override
    public void close () throws IOException {
      if (data    != null) data.close();
      if (toClose != null) toClose.close();
    }
  }

  /** An input stream that writes out pad bytes. */
  private static class PadInputStream extends InputStream {
    Closeable toClose;
    int count;

    /** Creates a new instance.
     *  @param count   The number of pad bytes to write.
     *  @param toClose Additional item to close when done.
     */
    PadInputStream (int count, Closeable toClose) {
      this.count   = count;
      this.toClose = toClose;
    }
    @Override
    public int read () {
      return (count-- > 0)? 0 : -1;
    }
    @Override
    public void close () throws IOException {
      if (toClose != null) toClose.close();
    }
  }
}
