package nxm.sys.lib;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.Proxy;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;

import nxm.sys.inc.Installation;
import nxm.sys.inc.InternalUseOnly;

/** Resource for interacting with an Http Server. <br>
    <br>
    URLs that are referenced by this class follow the standard URL syntax.  If a
    URL requires basic authentication for access then the url must include the
    username and password in it.  The syntax for this is then.
    <pre>
    http://username:passwd@host/path
    </pre>
    When this occurs, the username and password will be passed to the server to
    support Basic HTTP authentication.

    <h2>Existence verification</h2>
    The existence verification for HttpResources can tuned to fit the particular
    circumstances of a situation.  The existence checking algorithm is
    controlled by the <code>existsFlags</code> property.  This can be set
    explicitly by calling the mutator method or by attaching a qualifier to the
    the url in the <code>setURL</code> or constructor methods.  When a qualifier
    is attached to a url, the 'exists' key will contain the particular
    algorithms that should be used.  There are three different algorithms that
    are used.
    <table>
      <caption style="visibility:hidden">""</caption>
      <tr>
        <td> cgi </td>
        <td> Performs a check on the URL string itself to determine if the URL
             points to a CGI script.  If the URL string contains known patterns
             then the existence is true.
        </td>
      </tr>

      <tr>
        <td> list </td>
        <td> An attempt is made to retrieve a directory listing of the directory
             containing the URL.  This listing is then examined to determine if
             the named file exists.  In some contexts this is a very nice way to
             test for existence.  The server and the directory must support
             server generated listings for this to work properly.
        </td>
      </tr>

      <tr>
        <td> status </td>
        <td> A HEAD request is performed on the URL and the status code of the
             response is examined.  If the status code indicates success then
             the URL is assumed to exist.  This has problems with CGI scripts
             since the script must be executed in order for the status code to
             be generated.
        </td>
      </tr>
    </table>
    The three algorithms can be turned combined.  See the setExistsFlags for
    details on how to form the string or integer to perform this combination in
    Java code.  To perform this in qualifiers, use a + as the separator between
    the strings.
    <pre>
      Examples:
        Turn off all existence checks and simply assume the URL exists:
          http://localhost/foo.prm{exist=none}

        Turn on checking for CGI scripts:
          http://localhost/foo.prm{exist=cgi}

        Turn on CGI and LIST algorithms:
          http://localhost/foo.prm{exist=cgi|list}

        Turn on all algorithms:
          http://localhost/foo.prm{exist=all}
    </pre>

    <pre>
    Examples to specify HTTP timeout (via TIMEOUT and/or CONNECTTIMEOUT qualifiers):
      Specify HTTP connect timeout and read timeout of 10 seconds:
        http://server/foo?param1=a&amp;param2=b{TIMEOUT=10}
       or (since NeXtMidas 2.7.4)
        http://server/foo?param1=a&amp;param2=b{CONNECTTIMEOUT=10,TIMEOUT=10}

      Specify HTTP connect timeout of 5s and read timeout of 10s (since NeXtMidas 2.7.4):
        http://server/foo?param1=a&amp;param2=b{CONNECTTIMEOUT=5,TIMEOUT=10}

      Specify HTTP connect timeout of 5s and infinite read timeout (since NeXtMidas 2.7.4):
        http://server/foo?param1=a&amp;param2=b{CONNECTTIMEOUT=5}
       or
        http://server/foo?param1=a&amp;param2=b{CONNECTTIMEOUT=5,TIMEOUT=0}

    Example to specify HTTP POST request with post data and post data MIME type:
        http://server/foo?param1=a{METHOD="POST",POSTMIME="text",POSTDATA="data in post request"}

    Since NeXtMidas 2.8.3, to use strict HTTP 1.1 (RFC 2616) rules when communicating
    with a HTTP 1.1 server that does not support byte-range (Partial Content) requests,
    set the STRICTHTTP qualifier to TRUE or ON.
        http://server/path/to/file.tmp{STRICTHTTP="TRUE"}

    Since NeXtMidas 3.1.0, the PROXYHOST and PROXYPORT qualifiers can be use to set
    an explicit proxy hostname and proxy port.
        http://server/path/to/file.tmp{PROXYHOST="yourproxyserver",PROXYPORT=80}
    </pre>

    @author Jeff Schoen, J. Czechowski IV, Neon Ngo
 */
public class HttpResource extends IOResource {
  /** Key that specifies the existence algorithm to use.  This is searched for
      in the qualifiers of a URL in setURL.
   */
  public static final String EXIST_QUAL_KEY = "EXIST";

  /** list version of CHECK flags mask bits */
  public static final String existsList = "Status,CGI,List";

  /** Specifies turning off the exists behavior. */
  public static final int CHECK_NONE   = 0x00;

  /** Flag that specifies to check the status in exists(). */
  public static final int CHECK_STATUS = 0x01;

  /** Flag to specify checking whether CGI in exists(). */
  public static final int CHECK_CGI    = 0x02;

  /** Flag to specify checking the file list in exists(). */
  public static final int CHECK_LIST   = 0x04;

  /** Specifies checking all of the exists behavior. */
  public static final int CHECK_ALL    = -1;

  /** Specifies checking all the default exists behavior (same as {@link #CHECK_CGI} and
      {@link #CHECK_STATUS}.
      @since NeXtMidas 2.7.0
   */
  public static final int CHECK_DEF    = CHECK_CGI|CHECK_STATUS;

  /** List of recognized CGI extensions. */
  private static final String[] CGI_EXT  = { ".pl", ".rb", ".py", ".exe", ".csh", ".sh" };
  /** The multi-part boundary to use. */
  private static final String   boundary = "---------------------------7d012941501d8";
  /** is debugging messages enabled? */
  private static boolean debug = false;

  private int              existsFlags = CHECK_DEF; // Existence checks
  private Boolean          existResult = null;      // cache existence result (null, means we haven't checked)
  private DataInputStream  sis;                     // Input stream
  private DataOutputStream sos;                     // Output stream
  private URLConnection    urlCon;                  // Connection to URL
  private URL              url;                     // Applicable URL
  private String           userPasswd;              // username/password to connect with (null = not needed)
  private boolean          isStream;                // Is this a stream?
  private String           method;                  // The HTTP method used (null = automatic)
  private String           postMime;                // Mime type for posted data
  private byte[]           postData;                // Data to post
  private Table            responseHeaders;         // HTTP response header (cached read-only version) from server

  /** Construct a new object that is not initialized to connect to any URL.
      The setURL method must be invoked to set the URL before any connection
      can be established.
   */
  public HttpResource() { }

  /** Construct a new object that will connect to the given URL. This does not perform any
      connection to the server, just initializes the object to point to the URL. This is the
      same as calling <code>new HttpResource()</code> followed by <code>setURL(url)</code>.
      @param url The URL for this object.
   */
  public HttpResource (String url) {
    setURL(url);
  }

  /** Set the URL to the resource that this object is connected to. This does
      not perform any connection to the server, just initializes the object to
      point to the URL.
      @param str The URL for this object (cannot be null)
   */
  public void setURL (String str) {
    // Look for the username/password string and strip it out and save it.
    int i = str.indexOf('@');
    if (i>0) {
      // account for http or https prefix - Since 3.7.3 for NXM-3118
      // (ex: http://username:password@www.my_site.com)
      int passwdLoc = str.indexOf(':') + 3;
      userPasswd = str.substring(passwdLoc,i);
      str = str.substring(0,passwdLoc) + str.substring(i+1);
    }
    isStream = str.indexOf("/RAM/_")>0 || str.indexOf("/ram/_")>0;
    String prev_urlstr = getURL();
    if (!str.equals(prev_urlstr)) { // do below only if different than previous urlstr
      super.setURL(str);
      try {
        url = (new URI(str)).toURL();
        url = createURLWithPort(url); // this will add the default port to either HTTP or HTTPS. Since 3.9.1
      }
      catch (Exception e) { throw new MidasException("HTTP URL: " + str, e); }
      existResult = null; // clear cached existsResult
    }
  }

  /** Gets the HTTP request method that will be used.
      @return The method to be used or  "I/O" for i/o (combines "GET" and "POST").
      @since NeXtMidas 2.7.0
   */
  public String getRequestMethod () {
    return getRequestMethod(dir);
  }

  /** Get request method for given i/o direction. */
  private String getRequestMethod (int dir) {
    if (method != null) return method;

    if (dir <= DIR_INPUT)  return "GET"; // input only
    if (dir >= DIR_OUTPUT) return "PUT"; // output only
    return "GET/PUT";                    // output+input
  }

  /** Sets the HTTP request method.
      @param val The HTTP method to be used ("GET", "PUT", "POST"), "GET/PUT" for in/out (combines
                 "GET" and "PUT") or null if such should be automatically determined based on
                 {@link #getDir()}.
      @since NeXtMidas 2.7.0
   */
  public void setRequestMethod (String val) {
    if (val == null) {
      method = null;
    }
    else if ((dir <= DIR_INPUT) && (val.equals("GET") || val.equals("POST"))) {
      method = val; // input
    }
    else if ((dir >= DIR_OUTPUT) && val.equals("PUT")) {
      method = val; // output
    }
    else if ((dir == DIR_INOUT) && val.equals("GET/PUT")) {
      method = val; // in/out
    }
    else {
      throw new MidasException("Use of METHOD="+val+" is not compatible with "+dirToString(dir));
    }
  }

  /** Set the qualifiers on this resource.
      <code>METHOD - set HTTP request method (see {@link #setRequestMethod(String)})
          EXIST  - existence check (see {@link #setExistsFlags(String)})
          POSTMIME and POSTDATA - only applies when METHOD is "POST" (see {@link #setPostData(String, byte[])})
      </code>
      @param table Table containing the qualifiers to set for this resource.
   */
  public void setQualifiers (Table table) {
    super.setQualifiers(table);
    if (table == null) return;

    String meth  = table.getS("METHOD",method);
    String exist = table.getS("EXIST",null);

    setRequestMethod(meth);
    setExistsFlags(exist);

    if ((meth != null) && meth.equals("POST")) {
      String mime = table.getS("POSTMIME");
      Object obj  = table.get("POSTDATA");
      byte[] data = Convert.o2ba(obj);

      if ((obj != null) && (data == null)) {
        throw new MidasException("Unable to convert "+obj+" to bytes.");
      }
      setPostData(mime, data);
    }
  }

  /** For an "input only" POST request, this sets the associated data. To clear previously-set POST
      data, call this method with both parameters null. This does not set the request method, a
      separate call to <code>setRequestMethod("POST")</code> is required.
      @param mime The MIME type associated with the data.
      @param data The data.
      @throws NullPointerException If mime is set, but data is null or if data is set and mime is null.
      @since NeXtMidas 2.7.0
   */
  public void setPostData (String mime, byte[] data) {
    if ((postMime == null) && (postData != null)) {
      throw new NullPointerException("HttpResource: POST data specified, but no MIME type given.");
    }
    if ((postMime != null) && (postData == null)) {
      throw new NullPointerException("HttpResource: POST MIME type specified, but no data given.");
    }
    postMime = mime;
    postData = data;
  }

  /** For an "input only" POST request, this gets the associated data.
      @return The associated data or null if not applicable.
      @since NeXtMidas 2.7.0
      @see #setPostData
   */
  public byte[] getPostData () {
    return postData;
  }

  /** For an "input only" POST request, this gets the MIME type of the associated data.
      @return The MIME type of the associated data or null if not applicable.
      @since NeXtMidas 2.7.0
      @see #setPostData
   */
  public String getPostMime () {
    return postMime;
  }


  /** Return the current value of the exists flag.
      @return The existence flags (see {@link #existsList}).
   */
  public int getExistsFlags () {
    return existsFlags;
  }

  /** Set the flags that control the behavior of the exists method.
      @param flags The existence flags (see {@link #existsList}).
   */
  public void setExistsFlags (int flags) {
    if (flags == existsFlags) { return; }
    existsFlags = flags;
    existResult = null; // clear cached existsResult
  }

  /** Set the flags that control the behavior of the exists method. The flag string must be a
      concatenation of one or more of the flags as mentioned in the class documentation. The
      flags are separated by the pipe (|) delimiter.
      @param flagstr The existence flags (see {@link #existsList}).
   */
  public void setExistsFlags (String flagstr) {
    int mask = Parser.mask(existsList,flagstr,existsFlags);
    setExistsFlags(mask);
  }

  /** Test for the existence of the underlying data source. This method's job is to delegate to
      one or more of the specific exists methods based upon the instance settings.
      @return true if the file exists (based on the flags set), false otherwise.
   */
  public boolean exists () {
    if (url==null) return false;

    if (existsFlags == CHECK_NONE) return true;

    // if have cached exists result, use it to reduce potential extra network requests (BUG 1873)
    if (existResult != null) return existResult.booleanValue();

    boolean result = false;
    if      (((existsFlags & CHECK_CGI)    != 0) && existsCheckCgi()   ) result = true;
    else if (((existsFlags & CHECK_STATUS) != 0) && existsCheckStatus()) result = true;
    else if (((existsFlags & CHECK_LIST)   != 0) && existsCheckList()  ) result = true;

    existResult = Boolean.valueOf(result); // cache the existence check results
    return result;
  }


  /** Test for the existence of the underlying data source. An attempt to connect to the current
      URL is performed and if the connection is successful then the method will return true.
      @return true if the resource can successfully request data from the URL.
   */
  public boolean existsCheckStatus () {
    try {
      URLConnection conn = openConnection(url,userPasswd,dir,"HEAD");

      if (conn instanceof HttpURLConnection) {
        HttpURLConnection httpConn = (HttpURLConnection)conn;
        httpConn.connect();

        // Check the response code.
        int respCode = httpConn.getResponseCode();
        httpConn.disconnect();

        if (respCode == HttpURLConnection.HTTP_UNAUTHORIZED) { // 401
          M.info("Authorization required for " + url);
        }
        // All successful response codes are in the 200-299 section.
        return ((respCode / 100) == 2);
      }
      else {
        // We have to do this block because in Netscape 4.x the URLConnection
        // returned is NOT an instance of an HttpURLConnection.  We don't know
        // how to manipulate the object that is returned.  We'll assume that
        // its true.
        return true;
      }
    }
    catch (Exception e) {
      M.info(e);
      url = null;
      return false;
    }
  }


  /** Implementation of an exists method that performs a directory listing on
      the path of the url and checks to see if the file exists in that path.
      @return true if the file is listed in the directory listing, false otherwise.
   */
  public boolean existsCheckList () {
    // Break the pathname in the url into the directory and name lists
    // url.getParent() and all that jazz does not exist in the Browsers 1.1.5
    try {
      int j = urlstr.lastIndexOf("/");
      String path = urlstr.substring(0,j+1);
      String name = urlstr.substring(j+1);
      URL    root = (new URI(path)).toURL();

      // Do a directory listing of the path portion of the url and look to see if the file is in it.
      for (String fname : getEntryList(root,userPasswd,null)) {
        return fname.equals(name);
      }
    }
    catch (Exception e) {
      return false;
    }
    return false;
  }

  /** Determine if the current URL is a CGI URL. This is simply a check on
      the string looking for known patterns.
      @return True if the URL is suspected to be a CGI URL, false otherwise.
   */
  public boolean existsCheckCgi () {
    // If the URL has a query string then assume its CGI.
    if (url.toString().indexOf("?") != -1) return true;

    for (int i = 0; i < CGI_EXT.length; i++) {
      if (url.getFile().endsWith(CGI_EXT[i])) return true; // matches known extension
    }
    return false;
  }

  public boolean isStream() { return isStream; }
  public boolean isStreaming() { return (isStream && isOpen); }

  public boolean open () {
    isOpen = false;
    try {
      url = createURLWithPort(url); // To make sure that the default port is defined since 3.9.1
      String surl = url.toString();                      int i = surl.indexOf('?');
      String path = (i < 0)? surl : surl.substring(0,i); int j = path.lastIndexOf('/');
      String name = (j < 0)? path : path.substring(j+1); // file name
      String args = (i < 0)? ""   : surl.substring(i+1); // parameters

      if (dir >= DIR_INOUT) {  // output or in/out
        // NTN: Output to HTTP Resource is via an HTTP POST with a multipart/form-data
        //      just like what web browsers do to submit POST requests to web server.
        //      The parameters are written in the "params" MIME PART (1st).
        //      The file data contents are written in the "filedata" MIME PART (2nd).
        String mime = BaseFile.getInstanceFor(M,name).getMimeType();
        // urlCon = openConnection(surl.substring(0,i), userPasswd, dir, null); // 2.7.0 (defaults to HTTP PUT)
        urlCon = openConnection(url, userPasswd, dir, "POST");                  // 2.6.x & 2.7.1 use HTTP POST (Bug 1810)
        urlCon.setRequestProperty("Content-Type", "multipart/form-data; boundary="+boundary);
        sos = new DataOutputStream(urlCon.getOutputStream());
        if (i>0) {                               // MIME part for request parameters
          sos.writeBytes("--"+boundary+"\r\n");  // send BEGIN boundary for parameters
          sos.writeBytes("Content-Disposition: form-data; name=\"params\"\r\n");
          // sos.writeBytes("Content-Type: application/x-url-encoded\r\n");      // 2.7.0: some old RFCs use this
          sos.writeBytes("Content-Type: application/x-www-form-urlencoded\r\n"); // 2.7.1: this is the default encoding for HTML forms
          sos.writeBytes("\r\n");
          sos.writeBytes(args);
          sos.writeBytes("\r\n");
          sos.writeBytes("--"+boundary+"\r\n"); // send END boundary for parameters
        }
        // Bug 1810: MSZ's perl cgi script gets confuse by the filename=".." sub-attribute
        //           even though most web browsers submit this information in a form submission
        // sos.writeBytes("Content-Disposition: form-data; name=\"filedata\"; filename=\""+url.getPath()+"\"\r\n"); // 2.7.0
        sos.writeBytes("Content-Disposition: form-data; name=\"filedata\"\r\n"); // 2.6.x & 2.7.1 behavior (Bug 1810)
        sos.writeBytes("Content-Type: "+mime+"\r\n");
        sos.writeBytes("\r\n");
      }
      if (dir == DIR_INOUT) { // input part of in/out
        urlCon = openConnection(url, userPasswd, dir, null);
        sis    = new DataInputStream(urlCon.getInputStream());
        length = urlCon.getContentLength();
      }
      else if (dir <= DIR_INPUT) { // input
        String meth = getRequestMethod(dir);

        if (meth.equals("GET")) {
          urlCon = openConnection(url, userPasswd, DIR_INPUT, "GET");
        }
        else if (meth.equals("POST")) {
          urlCon = openConnection(path, userPasswd, DIR_INOUT, "POST");
          urlCon.setRequestProperty("Content-Type", "multipart/form-data; boundary="+boundary);
          sos = new DataOutputStream(urlCon.getOutputStream());
          sos.writeBytes("--"+boundary+"\r\n");
          sos.writeBytes("Content-Disposition: form-data; name=\"params\"\r\n");
          sos.writeBytes("Content-Type: application/x-url-encoded\r\n");
          sos.writeBytes("Content-Length: "+args.length()+"\r\n");
          sos.writeBytes("Content-Transfer-Encoding: 7Bit\r\n");
          sos.writeBytes("\r\n");
          sos.writeBytes(args);
          sos.writeBytes("\r\n");
          if (postData != null) {
            sos.writeBytes("--"+boundary+"\r\n");
            sos.writeBytes("Content-Disposition: form-data; name=\"data\"\r\n");
            sos.writeBytes("Content-Type: "+postMime+"\r\n");
            sos.writeBytes("Content-Length: "+postData.length+"\r\n");
            sos.writeBytes("Content-Transfer-Encoding: binary\r\n"); // use of 7Bit or 8Bit requires SMTP line-length checks
            sos.writeBytes("\r\n");
            sos.write(postData);
            sos.writeBytes("\r\n");
          }
          sos.writeBytes("--"+boundary+"--\r\n");
          sos.writeBytes("\r\n");
          sos.flush();
          sos.close();
          sos = null;
        }
        sis    = new DataInputStream(urlCon.getInputStream()); // Note: the getinputStream is what errors out with large SSL handshake sizes
        length = urlCon.getContentLength();
        // To support HTTP servers that does not support byte-range (BUG 2162)
        checkRangeSupport();
        // Check that Partial Responses are only received when expected and are
        // the range starts with the expected range
        checkRangeOfPartials();
      }
      isOpen = true;
      if (debug) {
        M.type("HttpResource.open() url="+url+" responseHeaders="+urlCon.getHeaderFields());
      }
    }
    catch (Exception e) {
      throw new MidasException("Caught error in HttpResource.open", e);
    }
    return isOpen;
  }

  /** Check if HTTP server supports byte range (Partial Content) - a.k.a random read support.
      If server does not, then try to workaround it by skipping to the desired offset byte position.
      This is required to support HTTP 1.0 servers and HTTP 1.1 servers that DO NOT support
      the Range header in the GET request.
      This should only be called after a connection is established to the server.
      @throws IOException
      @since NeXtMidas 2.8.3
   */
  private void checkRangeSupport () throws IOException {
    if (offset > 0) { // if we submitted a Range header (for partial content) in the request then offset>0
      boolean noRangeSupport = false;
      String statusLine = urlCon.getHeaderField(0); // HTTP Status-Line IS ALWAYS first line
      if (statusLine != null) {
        if (statusLine.startsWith("HTTP/1.1 200")) {
          String server = urlCon.getHeaderField("Server");
          if (server == null) {
            // Since NeXtMidas 2.8.2 and prior does NOT return a "Server" in the response header,
            // this could be coming from a HTTPSERV that supports the Range request, but is not
            // fully compliant with the HTTP 1.1 RFC 2616 spec (e.g. it returns version 1.1 and
            // status code 200 for the Range request, instead it should return status code 206.
            // 2010-09-08 NTN: For backwards compatibility with NeXtMidas 2.8.2 and prior HTTP servers,
            //   we will not workaround HTTP 1.1 servers that do not support the Range request at this time
            //   and does NOT return a "Server" response header
            // Allow STRICTHTTP qualifier or ENV.IOOPTIONS to override so that we can talk to a HTTP 1.1
            //   server without byte Range support (but may have trouble communicating with NeXtMidas 2.8.2
            //   and prior HTTP server)
            Object isStrict = getQualifier("STRICTHTTP");
            if (isStrict != null) {                         // 1. have STRICTHTTP qualifier
              noRangeSupport = Convert.o2bool(isStrict, M);
            } else {                                        // 2. otherwise check ENV.IOOPTIONS for STRICTHTTP
              noRangeSupport = M.io.isOptionSet(CoreIO.IOOptions.StrictHTTP);
            }
          }
          else {
            noRangeSupport = true;
          }
        }
        else if (statusLine.startsWith("HTTP/1.0 200")) {
          // server does NOT support Range request (i.e. it is returning full data contents)
          noRangeSupport = true;
        }
      }
      if (noRangeSupport) {
        // workaround HTTP servers that do not support Range (partial content) request
        long totalSkipped = 0;
        long curSkipped   = 0;
        while ((totalSkipped < offset) && ((curSkipped = sis.skip(offset-totalSkipped)) >= 0)) {
          totalSkipped += curSkipped;
        }
        if (totalSkipped != offset) {
          M.warning("Unable to skip to offset="+offset+" currently at "+totalSkipped+" for url="+url);
        }
        else if (debug) {
          M.type("HttpResource.open() url="+url+" successfully skipped to offset "+offset);
        }
      }
    } // end if (offset > 0)
  }

  /** Check if this response is an HTTP Partial (206) and if so check that
   *  that the start of the range returned matches the current offset.
   *  Checks of responses with 'multipart/byteranges' are not currently
   *  supported.
   * @throws IOException
   * @since 3.7.3 for NXM-3044
   */
  private void checkRangeOfPartials () throws IOException{
    if (urlCon instanceof HttpURLConnection) {
      HttpURLConnection http = (HttpURLConnection)urlCon;
      if (http.getResponseCode() == HttpURLConnection.HTTP_PARTIAL) {
        Table headers = getHeaders(http);
        String range = headers.getS("Content-Range");
        String type = headers.getS("Content-Type");
        if (range == null) {
          // The only valid case for 'Content-Range' not being specified in a
          // partial is for the 'Content-Type' 'multipart/byteranges', where
          // multiple ranges are returned
          if ((type==null) || !type.contains("multipart")) {
            throw new MidasException("Invalid HTTP Patial (206) without range specified.  " +
                "This probably indicates a problem with an HTTP cache returning "+
                "invalid content. - headers:" + getHeaders(http));
          }
        }
        else {
          // ensure the start of the range returned matches 'offset'
          String[] ranges = range.split("[\\D]+");
          int startRange = Integer.valueOf(ranges[1]);
          if (debug) M.type("HttpResource.checkRangeOfPartials () startRange:"+startRange+" offset:"+offset);
          if (startRange != offset) {
            throw new MidasException("Expecting HTTP Partial Content with start:" +
                offset + " but content range is:" + range + " - headers:" +
                getHeaders(http));
          }
        }
      }
    }
  }

  public InputStream  getInputStream()  { return sis; }
  public OutputStream getOutputStream() { return sos; }

  public boolean close() {
    if (!isOpen) return isOpen;
    closeConnection(url,userPasswd,dir);
    // Reset the offset so that the next open call on this instance will
    // start reading from the beginning of the resource.
    offset = 0;
    isOpen = false;
    return isOpen;
  }

  /** Attempts to delete the file on the server. Not all servers support this.
      @return true if successful, false if not successful.
   */
  public boolean delete () {
    try {
      URLConnection conn = openConnection(url, userPasswd, DIR_INPUT, "DELETE");
      if (conn instanceof HttpURLConnection) {
        HttpURLConnection http = (HttpURLConnection)conn;
        return ((http.getResponseCode() / 100) == 2); // 200-series codes indicate success
      }
      return false;
    }
    catch (Exception e) {
      return false;
    }
  }

  /** Get length of resource. In the case of an input stream this will return the value in the
      <code>content-length</code> field of the header (if set), otherwise this will return the
      number of bytes currently available.
      @return The length of the resource in bytes.
   */
  public long getLength () {
    if (length>0) return length;
    else return avail();
  }

  public long avail () {
    if (sis==null || !isOpen) return 0L;
    try {
      long avail = sis.available();
      if (length>0) avail = Math.min(avail,length-offset);
      return avail;
    }
    catch (IOException e) { return 0L; }
  }

  /** Seek to the specified offset in file */
  public boolean seek (long offset) {
    if (isOpen && offset!=this.offset) close();
    this.offset = offset;
    return true;
  }

  /** read data from resource */
  public int read (byte[] buf, int boff, int bytes) {
    if (!isOpen) open();
    int status=0, off=boff;
    while (bytes>0) {
      try { status = sis.read(buf,off,bytes); }
      catch (IOException e) { status = -1; }
      if (status>0) { bytes-=status; off+=status; }
      else break;
    } bytes = off-boff;
    if (status<0 && bytes==0) return status;
    offset += bytes;
    return bytes;
  }

  /** Write data to resource. HttpResource, due to the HTTP protocol, is currently different than
      other resources in the way that writes are handled. In other {@link IOResource} classes the
      write is intended to modify the datafile. With the HttpResource the data given in the write
      method is really passed as the body of the POST request which might not cause writing to the
      remote file to occur. <br>
      <br>
      In order to get the real functionality that is required for seamless integration DAV support
      has to be integrated.
   */
  public int write (byte[] buf, int boff, int bytes) {
    if (!isOpen) open();
    try { sos.write(buf,boff,bytes); }
    catch (IOException e) { return -1; }
    offset += bytes;
    return bytes;
  }

  /** get entry from Container */
  public IOResource getEntry (String name) { return null; }

  /** get List of container entries */
  public String[] getEntryList (FilenameFilter filter) {
    return getEntryList(url, userPasswd, filter);
  }

  /** get List of container entries.
      @param url        The URL to attempt to connect to.
      @param userPasswd The username/password to connect with or null if no authentication is needed.
   */
  private String[] getEntryList (URL url, String userPasswd, FilenameFilter filter) {
    StringBuilder sb = new StringBuilder(1024); // start with a 1024 initial capacity.
    try {
      URLConnection conn = openConnection(url,userPasswd,-1,"GET");

      // Read the file listing into a single string buffer. This may seem inefficient, but it
      // is the only way to ensure that we get all of the href tags not to span multiple lines.
      BufferedReader br = new BufferedReader(new InputStreamReader( conn.getInputStream() ));

      String line;
      while ((line = br.readLine()) != null) {
        sb.append(line);
      }
      br.close();
    }
    catch (IOException e) {
      throw new MidasException("Getting EntryList", e);
    }
    String uc   = sb.toString().toUpperCase(); // Convert to UPPER-CASE to make searching easier

    ArrayList<String> list = new ArrayList<>();
    String pathInURL = url.getPath();
    int pos = -1;
    while ((pos = uc.indexOf("<A ",pos+1)) >= 0) {
      int end = uc.indexOf("</A>", pos);
      if (end < 0) break;

      String link = sb.substring(pos, end+4);
      String href = getHrefTarget(link);
           if (href == null)           { } // not found             -> ignore
      else if (href.length() == 0)     { } // blank                 -> ignore
      else if (href.indexOf('?') >= 0) { } // query                 -> ignore
      else if (href.startsWith(pathInURL)) { // absolute path that may be under this URL path
        // 2010-10-04 NTN: support NeXtMidas HTTPSERV directory listing (version 2.[8|9]?.x and prior)
        // NTN TODO: update directory listing to use relative path (instead of absolute path) in HFiles
        //           so that this workaround is not necessary moving forward.
        String tmphref = href.substring(pathInURL.length()); // remove leading path from url from href
        if ((tmphref.length() > 0) && (tmphref.indexOf('/') < 0)) {
          list.add(tmphref);              // add to list (inside URL path)
        }                                 // else not in this directory -> ignore
      }
      else if (href.indexOf('/') >= 0) {   // not in this directory -> ignore
      }
      else                             { list.add(href); }
    }
    String[] entries = list.toArray(EMPTY_STRING_ARRAY);
    return filterEntries(filter, entries);
  }

  /** Opens the connection to the URL and sets the headers on the request. */
  private URLConnection openConnection (String url, String userPasswd, int ioMode, String meth) throws IOException {
    URL newURL = null;
    try {
      newURL = (new URI(url)).toURL();
    } catch (URISyntaxException e) {
      throw new MalformedURLException("Forming URL via URI failed:"+e.getMessage()+" "+e.getStackTrace());
    }
    return openConnection(newURL, userPasswd, ioMode, meth);
  }

  /** Opens the connection to the URL and sets the headers on the request.
      @param url        The URL to attempt to connect to.
      @param userPasswd The username/password to connect with or null if no authentication is needed.
      @param ioMode     The mode to open the connection in.
      @param meth       The HTTP method to use.
      @return The URLConnection object resulting from opening the connection.
   */
  private URLConnection openConnection (URL url, String userPasswd, int ioMode, String meth) throws IOException {
    if (meth == null) meth = getRequestMethod(ioMode);

    URLConnection conn;
    double timeout        = 0.0;
    double connectTimeout = 0.0;
    String proxyHost      = null;
    int    proxyPort      = -1;

    if (qualTable != null) { // check qualifiers for HttpResource
      proxyHost      = qualTable.getS("PROXYHOST");
      proxyPort      = qualTable.getL("PROXYPORT", 80);
      timeout        = qualTable.getD("TIMEOUT", 0.0); // read timeout
      connectTimeout = qualTable.getD("CONNECTTIMEOUT", timeout);
    }

    // Go ahead and open the connection.  Note that this does not
    // actually establish the connection, it just creates the objects.
    if (proxyHost != null && proxyHost.length() > 0) {
      // use specified proxy host / port for BUG 2466
      InetSocketAddress isa = new InetSocketAddress(proxyHost, proxyPort);
      Proxy proxy = new Proxy(Proxy.Type.HTTP, isa);
      conn = url.openConnection(proxy);
    } else {
      conn = url.openConnection();
    }

    // override default User-Agent from URLConnection (and it subclasses) with our info
    String userAgent = "NeXtMidas/"+Installation.NM_VERSION + " Java/"+Shell.getJavaVersion();
    conn.addRequestProperty("User-Agent", userAgent);   // RFC 2616 14.43 User-Agent (OPTIONAL)

    HttpURLConnection http = null;
    if (conn instanceof HttpURLConnection) {
      http = (HttpURLConnection)conn;
    }

    if (timeout > 0.0) {
      int rtimeout = (int)(timeout * 1000); // convert seconds to ms
      conn.setReadTimeout(rtimeout);
      if (rtimeout != conn.getReadTimeout()) { //some platforms may ignore the timeout settings
        M.warning("This platform is unable to set timeout for "+url+" to "+rtimeout+"ms using "+conn.getReadTimeout()+"ms.");
      }
    }
   if (connectTimeout > 0.0) {
     int ctimeout = (int)(connectTimeout * 1000); // convert seconds to ms
     conn.setConnectTimeout(ctimeout);
      if (ctimeout != conn.getConnectTimeout()) { //some platforms may ignore the timeout settings
        M.warning("This platform is unable to set timeout for "+url+" to "+ctimeout+"ms using "+conn.getConnectTimeout()+"ms.");
      }
    }

    // Set the Authorization header if the user password combination is not null.
    if (userPasswd != null) {
      String encoding = Convert.encodeBase64(userPasswd); // userPassword="user:password"
      conn.setRequestProperty("Authorization", "Basic " + encoding);
    }

    // If the offset is not zero, then we are to add the Range header
    // to the request to request from that particular offset to the
    // end of the resource. See HTTP 1.1 RFC 2616 Section 14.35
    if (offset != 0) {
      String header = "bytes=" + offset + "-";
      conn.setRequestProperty ("Range", header);
    }

    // If we need to write stuff into the body of the request, then we
    // must set various properties of the connection to enable this.
    if (ioMode >= 0) {
      conn.setDoOutput(true);
      conn.setDoInput(true);
      conn.setUseCaches(false);
    }
    if (http != null) {
      http.setRequestMethod(meth);
    }

    if (Shell.getCurrent().parent instanceof Webstart) {
      // Work around an issue in Java Web Start (seen in Java 7 and Java 8)
      // where the Web Start specific cache will cache partial file responses
      // ({@link HttpURLConnection.HTTP_PARTIAL} from web server) as if they
      // were copies of the entire file.  This must be set for *every* request
      // and not just the ones with 'Range' header set since a request with
      // no 'Range' header can result in a previously-cached "Range' request
      // being returned.
      //
      // Note that we do not set URLConnection.setDefaultUseCaches(false) from
      // within the NeXtMidas Webstart class since that can be altered and we
      // want to force useCahes=false in this scenario.
      //
      // I (JC4) don't particularly like this work-around as it only addresses
      // one very specific use-case and doesn't trap other flawed caching
      // mechanisms.  There is an exception thrown in checkRangeSupport that we
      // throw if we get a {@link HttpURLConnection.HTTP_PARTIAL} response
      // where a {@link HttpURLConnection.HTTP_OK} is expected, but that
      // hardly counts. Such a check will NOT detect a scenario where we
      // expect a {@link HttpURLConnection.HTTP_PARTIAL} response with
      // partial content, but the partial content is from the wrong part
      // of the file
      // Since 3.7.2 for NXM-3044
      conn.setUseCaches(false);
    }

    if (debug) {
      M.type("HttpResource.openConnection(); url="+url+" request method="+meth+
          " request range:"+conn.getRequestProperty("Range") +
          " connection headers="+getHeaders(conn));
    }
    return conn;
  }

  /** Closes the connection to the remote URL.
      deprecated Since NeXtMidas 2.7.0: This should be private, use {@link #close()}.
      Since NeXtMidas 3.1.0, this method is now private.
   */
  private void closeConnection (URL url, String userPasswd, int ioMode) {
    try {
      if (sos!=null) {
        sos.writeBytes("\r\n--"+boundary+"--\r\n");
        sos.flush();
        sos.close();
        sis = new DataInputStream(urlCon.getInputStream());
        sis.close();
      }
      if (sis!=null) {
        sis.close();
      }
    }
    catch (IOException e) { M.warning("IO exception in closeURL() call"); }
  }

  /** Given an HTML link tag, get the target HREF.
      @param href An HTML link tag (e.g. "&lt;a href='myfile.txt'&gt;My Text File&lt;/a&gt;"
      @return The HREF target (e.g. "myfile.txt").
   */
  private static String getHrefTarget (String href) {
    // Create a copy of the string that is all upper case.
    String hrefcp = href.toUpperCase();

    int hrefPos = hrefcp.indexOf("HREF=");
    if (hrefPos < 0) return null; // not found

    char quote = href.charAt(hrefPos+5); // either quote (") or single-tic (')
    int  end   = href.indexOf(quote,hrefPos+6);

    return (end < 0)? null : href.substring(hrefPos+6,end);
  }

  /** Get HTTP response headers returned from a web server. HTTP response Status-Line is
      in the "STATUS_LINE" key (this should contain the HTTP version of the server and
      status code). The keys are case-sensitive to match what the server is returning.
      If there is no connection, an empty table is returned.
      @return read-only Table with all the response headers from HTTP server
      @since NeXtMidas 2.9.0
   */
  public Table getResponseHeaders () {
    if (urlCon != null) {
      if (responseHeaders == null) {   // if don't have cached version, create it
        responseHeaders = getHeaders(urlCon);
      }
      return responseHeaders;
    }
    else { // no connection
      return getHeaders(urlCon);  // will create an empty read-only table
    }
  }

  /**
   * Gets the headers associated with a connection
   * @param connThe connection (can be null)
   * @return The headers
   * @since 3.7.2 for NXM-3044
   */
  private static Table getHeaders (URLConnection conn) {
    Table headers = new Table();

    if (conn != null) {
      String statusLine = conn.getHeaderField(0); // HTTP Status-Line IS ALWAYS first line
      headers.put("STATUS_LINE", statusLine);

      for (int ii=1;  ; ii++) {        // iterate through rest of headers
        String val = conn.getHeaderField(ii);
        if (val == null) { break;    } // at last header field, break out of loop
        String key = conn.getHeaderFieldKey(ii);
        if (key == null) { continue; } // null keys are ignored (since Table won't allow it)
        headers.put(key, val);
      } // end for loop
    }
    headers.setFlags("+READONLY"); // make table read-only
    return headers;
  }

  /** @since NeXtMidas 3.3.0 */
  @Override
  public long lastModified () {
    long utcSecs = (urlCon == null)? 0 : urlCon.getLastModified();
    return utcSecs;
  }

  /** enable/disable debugging messages for all instances of this class.
      @param val true to enable debug messages, false to disable it
      @since NeXtMidas 2.8.3
   */
  public static void setDebug (boolean val) {
    debug = val;
  }

  /** is debugging messages enable/disable  for all instances of this class.
      @return true if debug messages are enabled, otherwise false.
      @since NeXtMidas 2.8.3
   */
  public static boolean getDebug () {
    return debug;
  }

  /** Will add the default port to the url field.
   *  HTTP  = 80
   *  HTTPS = 443
   *  @param url The reference url to create a url with the port defined
   *  @return The URL with the default port defined
   *  @since 3.9.1
   */
  public static URL createURLWithPort (URL url) {
    if (url.getPort() > -1) return url;
    URL urlWithPort = null;
    try {
      urlWithPort = new URI(url.getProtocol(),url.getUserInfo(),url.getHost(),url.getDefaultPort(),url.getPath(),url.getQuery(),url.getRef()).toURL();
    }
    catch (MalformedURLException | URISyntaxException e) {
      Midas.log("The url was not formed correctly " + url.toString() + " " + e, Midas.INFO);
    }
    return urlWithPort==null ? url : urlWithPort;
  }

  /** Gets the URL as a URL Object
   *  @return The URL as a URL Object */
  @InternalUseOnly("Testing purposes only")
  public URL getURLObject () {
    return url;
  }
}
