1 files changed, 178 insertions, 104 deletions

diff --git a/lib/url.c b/lib/url.c
index 1af859dd..17279bbe 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -124,7 +124,6 @@ int curl_win32_idn_to_ascii(const char *in, char **out);
 #include "curl_rtmp.h"
 #include "gopher.h"
 #include "http_proxy.h"
-#include "bundles.h"
 #include "conncache.h"
 #include "multihandle.h"
 #include "pipeline.h"
@@ -576,6 +575,18 @@ CURLcode Curl_init_userdefined(struct UserDefined *set)
                      (char *) CURL_DEFAULT_SOCKS5_GSSAPI_SERVICE);
   if(result)
     return result;
+
+  /* set default negotiate proxy service name */
+  result = setstropt(&set->str[STRING_PROXY_SERVICE_NAME],
+                     (char *) CURL_DEFAULT_PROXY_SERVICE_NAME);
+  if(result)
+    return result;
+
+  /* set default negotiate service name */
+  result = setstropt(&set->str[STRING_SERVICE_NAME],
+                     (char *) CURL_DEFAULT_SERVICE_NAME);
+  if(result)
+    return result;
 #endif
 
   /* This is our preferred CA cert bundle/path since install time */
@@ -1473,12 +1484,29 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
                        va_arg(param, char *));
     break;
 
+  case CURLOPT_PROXY_SERVICE_NAME:
+    /*
+     * Set negotiate proxy service name
+     */
+    result = setstropt(&data->set.str[STRING_PROXY_SERVICE_NAME],
+                       va_arg(param, char *));
+    break;
+
   case CURLOPT_SOCKS5_GSSAPI_NEC:
     /*
      * set flag for nec socks5 support
      */
     data->set.socks5_gssapi_nec = (0 != va_arg(param, long))?TRUE:FALSE;
     break;
+
+  case CURLOPT_SERVICE_NAME:
+    /*
+     * Set negotiate service identity
+     */
+    result = setstropt(&data->set.str[STRING_SERVICE_NAME],
+                       va_arg(param, char *));
+    break;
+
 #endif
 
   case CURLOPT_HEADERDATA:
@@ -2158,9 +2186,9 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
 
       data->share->dirty++;
 
-      if(data->share->hostcache) {
+      if(data->share->specifier & (1<< CURL_LOCK_DATA_DNS)) {
         /* use shared host cache */
-        data->dns.hostcache = data->share->hostcache;
+        data->dns.hostcache = &data->share->hostcache;
         data->dns.hostcachetype = HCACHE_SHARED;
       }
 #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES)
@@ -2621,6 +2649,9 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
   case CURLOPT_PATH_AS_IS:
     data->set.path_as_is = (0 != va_arg(param, long))?TRUE:FALSE;
     break;
+  case CURLOPT_PIPEWAIT:
+    data->set.pipewait = (0 != va_arg(param, long))?TRUE:FALSE;
+    break;
   default:
     /* unknown tag and its companion, just ignore: */
     result = CURLE_UNKNOWN_OPTION;
@@ -2754,7 +2785,7 @@ CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection)
   Curl_ssl_close(conn, FIRSTSOCKET);
 
   /* Indicate to all handles on the pipe that we're dead */
-  if(Curl_multi_pipeline_enabled(data->multi)) {
+  if(Curl_pipeline_wanted(data->multi, CURLPIPE_ANY)) {
     signalPipeClose(conn->send_pipe, TRUE);
     signalPipeClose(conn->recv_pipe, TRUE);
   }
@@ -2782,32 +2813,31 @@ static bool SocketIsDead(curl_socket_t sock)
   return ret_val;
 }
 
+/*
+ * IsPipeliningPossible() returns TRUE if the options set would allow
+ * pipelining/multiplexing and the connection is using a HTTP protocol.
+ */
 static bool IsPipeliningPossible(const struct SessionHandle *handle,
                                  const struct connectdata *conn)
 {
-  if((conn->handler->protocol & PROTO_FAMILY_HTTP) &&
-     Curl_multi_pipeline_enabled(handle->multi) &&
-     (handle->set.httpreq == HTTPREQ_GET ||
-      handle->set.httpreq == HTTPREQ_HEAD) &&
-     handle->set.httpversion != CURL_HTTP_VERSION_1_0)
-    return TRUE;
+  /* If a HTTP protocol and pipelining is enabled */
+  if(conn->handler->protocol & PROTO_FAMILY_HTTP) {
+
+    if(Curl_pipeline_wanted(handle->multi, CURLPIPE_HTTP1) &&
+       (handle->set.httpversion != CURL_HTTP_VERSION_1_0) &&
+       (handle->set.httpreq == HTTPREQ_GET ||
+        handle->set.httpreq == HTTPREQ_HEAD))
+      /* didn't ask for HTTP/1.0 and a GET or HEAD */
+      return TRUE;
 
+    if(Curl_pipeline_wanted(handle->multi, CURLPIPE_MULTIPLEX) &&
+       (handle->set.httpversion == CURL_HTTP_VERSION_2_0))
+      /* allows HTTP/2 */
+      return TRUE;
+  }
   return FALSE;
 }
 
-bool Curl_isPipeliningEnabled(const struct SessionHandle *handle)
-{
-  return Curl_multi_pipeline_enabled(handle->multi);
-}
-
-CURLcode Curl_addHandleToPipeline(struct SessionHandle *data,
-                                  struct curl_llist *pipeline)
-{
-  if(!Curl_llist_insert_next(pipeline, pipeline->tail, data))
-    return CURLE_OUT_OF_MEMORY;
-  return CURLE_OK;
-}
-
 int Curl_removeHandleFromPipeline(struct SessionHandle *handle,
                                   struct curl_llist *pipeline)
 {
@@ -2855,15 +2885,14 @@ void Curl_getoff_all_pipelines(struct SessionHandle *data,
                                struct connectdata *conn)
 {
   bool recv_head = (conn->readchannel_inuse &&
-    (gethandleathead(conn->recv_pipe) == data)) ? TRUE : FALSE;
-
+                    Curl_recvpipe_head(data, conn));
   bool send_head = (conn->writechannel_inuse &&
-    (gethandleathead(conn->send_pipe) == data)) ? TRUE : FALSE;
+                    Curl_sendpipe_head(data, conn));
 
   if(Curl_removeHandleFromPipeline(data, conn->recv_pipe) && recv_head)
-    conn->readchannel_inuse = FALSE;
+    Curl_pipeline_leave_read(conn);
   if(Curl_removeHandleFromPipeline(data, conn->send_pipe) && send_head)
-    conn->writechannel_inuse = FALSE;
+    Curl_pipeline_leave_write(conn);
 }
 
 static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke)
@@ -2915,7 +2944,7 @@ find_oldest_idle_connection(struct SessionHandle *data)
 
   now = Curl_tvnow();
 
-  Curl_hash_start_iterate(bc->hash, &iter);
+  Curl_hash_start_iterate(&bc->hash, &iter);
 
   he = Curl_hash_next_element(&iter);
   while(he) {
@@ -3049,6 +3078,13 @@ static void prune_dead_connections(struct SessionHandle *data)
   }
 }
 
+
+static size_t max_pipeline_length(struct Curl_multi *multi)
+{
+  return multi ? multi->max_pipeline_length : 0;
+}
+
+
 /*
  * Given one filled in connection struct (named needle), this function should
  * detect if there already is one that has all the significant details
@@ -3065,7 +3101,8 @@ static bool
 ConnectionExists(struct SessionHandle *data,
                  struct connectdata *needle,
                  struct connectdata **usethis,
-                 bool *force_reuse)
+                 bool *force_reuse,
+                 bool *waitpipe)
 {
   struct connectdata *check;
   struct connectdata *chosen = 0;
@@ -3078,6 +3115,7 @@ ConnectionExists(struct SessionHandle *data,
   struct connectbundle *bundle;
 
   *force_reuse = FALSE;
+  *waitpipe = FALSE;
 
   /* We can't pipe if the site is blacklisted */
   if(canPipeline && Curl_pipeline_site_blacklisted(data, needle)) {
@@ -3088,7 +3126,9 @@ ConnectionExists(struct SessionHandle *data,
      particular host */
   bundle = Curl_conncache_find_bundle(needle, data->state.conn_cache);
   if(bundle) {
-    size_t max_pipe_len = Curl_multi_max_pipeline_length(data->multi);
+    /* Max pipe length is zero (unlimited) for multiplexed connections */
+    size_t max_pipe_len = (bundle->multiuse != BUNDLE_MULTIPLEX)?
+      max_pipeline_length(data->multi):0;
     size_t best_pipe_len = max_pipe_len;
     struct curl_llist_element *curr;
 
@@ -3096,9 +3136,17 @@ ConnectionExists(struct SessionHandle *data,
           needle->host.name, (void *)bundle);
 
     /* We can't pipe if we don't know anything about the server */
-    if(canPipeline && !bundle->server_supports_pipelining) {
-      infof(data, "Server doesn't support pipelining\n");
-      canPipeline = FALSE;
+    if(canPipeline) {
+      if(bundle->multiuse <= BUNDLE_UNKNOWN) {
+        if((bundle->multiuse == BUNDLE_UNKNOWN) && data->set.pipewait) {
+          infof(data, "Server doesn't support multi-use yet, wait\n");
+          *waitpipe = TRUE;
+          return FALSE; /* no re-use */
+        }
+
+        infof(data, "Server doesn't support multi-use (yet)\n");
+        canPipeline = FALSE;
+      }
     }
 
     curr = bundle->conn_list->head;
@@ -3122,16 +3170,19 @@ ConnectionExists(struct SessionHandle *data,
       pipeLen = check->send_pipe->size + check->recv_pipe->size;
 
       if(canPipeline) {
-        /* Make sure the pipe has only GET requests */
-        struct SessionHandle* sh = gethandleathead(check->send_pipe);
-        struct SessionHandle* rh = gethandleathead(check->recv_pipe);
-        if(sh) {
-          if(!IsPipeliningPossible(sh, check))
-            continue;
-        }
-        else if(rh) {
-          if(!IsPipeliningPossible(rh, check))
-            continue;
+
+        if(!check->bits.multiplex) {
+          /* If not multiplexing, make sure the pipe has only GET requests */
+          struct SessionHandle* sh = gethandleathead(check->send_pipe);
+          struct SessionHandle* rh = gethandleathead(check->recv_pipe);
+          if(sh) {
+            if(!IsPipeliningPossible(sh, check))
+              continue;
+          }
+          else if(rh) {
+            if(!IsPipeliningPossible(rh, check))
+              continue;
+          }
         }
       }
       else {
@@ -3310,19 +3361,42 @@ ConnectionExists(struct SessionHandle *data,
           }
 
           /* We can't use the connection if the pipe is full */
-          if(pipeLen >= max_pipe_len)
+          if(max_pipe_len && (pipeLen >= max_pipe_len)) {
+            infof(data, "Pipe is full, skip (%zu)\n", pipeLen);
             continue;
-
+          }
+#ifdef USE_NGHTTP2
+          /* If multiplexed, make sure we don't go over concurrency limit */
+          if(check->bits.multiplex) {
+            /* Multiplexed connections can only be HTTP/2 for now */
+            struct http_conn *httpc = &check->proto.httpc;
+            if(pipeLen >= httpc->settings.max_concurrent_streams) {
+              infof(data, "MAX_CONCURRENT_STREAMS reached, skip (%zu)\n",
+                    pipeLen);
+              continue;
+            }
+          }
+#endif
           /* We can't use the connection if the pipe is penalized */
-          if(Curl_pipeline_penalized(data, check))
+          if(Curl_pipeline_penalized(data, check)) {
+            infof(data, "Penalized, skip\n");
             continue;
+          }
 
-          if(pipeLen < best_pipe_len) {
-            /* This connection has a shorter pipe so far. We'll pick this
-               and continue searching */
+          if(max_pipe_len) {
+            if(pipeLen < best_pipe_len) {
+              /* This connection has a shorter pipe so far. We'll pick this
+                 and continue searching */
+              chosen = check;
+              best_pipe_len = pipeLen;
+              continue;
+            }
+          }
+          else {
+            /* When not pipelining (== multiplexed), we have a match here! */
             chosen = check;
-            best_pipe_len = pipeLen;
-            continue;
+            infof(data, "Multiplexed connection found!\n");
+            break;
           }
         }
         else {
@@ -3374,20 +3448,6 @@ ConnectionDone(struct SessionHandle *data, struct connectdata *conn)
   return (conn_candidate == conn) ? FALSE : TRUE;
 }
 
-/*
- * The given input connection struct pointer is to be stored in the connection
- * cache. If the cache is already full, least interesting existing connection
- * (if any) gets closed.
- *
- * The given connection should be unique. That must've been checked prior to
- * this call.
- */
-static CURLcode ConnectionStore(struct SessionHandle *data,
-                                struct connectdata *conn)
-{
-  return Curl_conncache_add_conn(data->state.conn_cache, conn);
-}
-
 /* after a TCP connection to the proxy has been verified, this function does
    the next magic step.
 
@@ -3763,9 +3823,9 @@ static struct connectdata *allocate_conn(struct SessionHandle *data)
   conn->response_header = NULL;
 #endif
 
-  if(Curl_multi_pipeline_enabled(data->multi) &&
-      !conn->master_buffer) {
-    /* Allocate master_buffer to be used for pipelining */
+  if(Curl_pipeline_wanted(data->multi, CURLPIPE_HTTP1) &&
+     !conn->master_buffer) {
+    /* Allocate master_buffer to be used for HTTP/1 pipelining */
     conn->master_buffer = calloc(BUFSIZE, sizeof (char));
     if(!conn->master_buffer)
       goto error;
@@ -4458,7 +4518,6 @@ static char *detect_proxy(struct connectdata *conn)
  * If this is supposed to use a proxy, we need to figure out the proxy
  * host name, so that we can re-use an existing connection
  * that may exist registered to the same proxy host.
- * proxy will be freed before this function returns.
  */
 static CURLcode parse_proxy(struct SessionHandle *data,
                             struct connectdata *conn, char *proxy)
@@ -5295,8 +5354,9 @@ static CURLcode create_conn(struct SessionHandle *data,
   bool reuse;
   char *proxy = NULL;
   bool prot_missing = FALSE;
-  bool no_connections_available = FALSE;
+  bool connections_available = TRUE;
   bool force_reuse = FALSE;
+  bool waitpipe = FALSE;
   size_t max_host_connections = Curl_multi_max_host_connections(data->multi);
   size_t max_total_connections = Curl_multi_max_total_connections(data->multi);
 
@@ -5505,8 +5565,10 @@ static CURLcode create_conn(struct SessionHandle *data,
       conn->bits.httpproxy = TRUE;
 #endif
     }
-    else
+    else {
       conn->bits.httpproxy = FALSE; /* not a HTTP proxy */
+      conn->bits.tunnel_proxy = FALSE; /* no tunneling if not HTTP */
+    }
     conn->bits.proxy = TRUE;
   }
   else {
@@ -5571,7 +5633,7 @@ static CURLcode create_conn(struct SessionHandle *data,
       conn->data = data;
       conn->bits.tcpconnect[FIRSTSOCKET] = TRUE; /* we are "connected */
 
-      ConnectionStore(data, conn);
+      Curl_conncache_add_conn(data->state.conn_cache, conn);
 
       /*
        * Setup whatever necessary for a resumed transfer
@@ -5636,7 +5698,7 @@ static CURLcode create_conn(struct SessionHandle *data,
   if(data->set.reuse_fresh && !data->state.this_is_a_follow)
     reuse = FALSE;
   else
-    reuse = ConnectionExists(data, conn, &conn_temp, &force_reuse);
+    reuse = ConnectionExists(data, conn, &conn_temp, &force_reuse, &waitpipe);
 
   /* If we found a reusable connection, we may still want to
      open a new connection if we are pipelining. */
@@ -5682,9 +5744,15 @@ static CURLcode create_conn(struct SessionHandle *data,
     /* We have decided that we want a new connection. However, we may not
        be able to do that if we have reached the limit of how many
        connections we are allowed to open. */
-    struct connectbundle *bundle;
+    struct connectbundle *bundle = NULL;
+
+    if(waitpipe)
+      /* There is a connection that *might* become usable for pipelining
+         "soon", and we wait for that */
+      connections_available = FALSE;
+    else
+      bundle = Curl_conncache_find_bundle(conn, data->state.conn_cache);
 
-    bundle = Curl_conncache_find_bundle(conn, data->state.conn_cache);
     if(max_host_connections > 0 && bundle &&
        (bundle->num_connections >= max_host_connections)) {
       struct connectdata *conn_candidate;
@@ -5697,11 +5765,15 @@ static CURLcode create_conn(struct SessionHandle *data,
         conn_candidate->data = data;
         (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE);
       }
-      else
-        no_connections_available = TRUE;
+      else {
+        infof(data, "No more connections allowed to host: %d\n",
+              max_host_connections);
+        connections_available = FALSE;
+      }
     }
 
-    if(max_total_connections > 0 &&
+    if(connections_available &&
+       (max_total_connections > 0) &&
        (data->state.conn_cache->num_connections >= max_total_connections)) {
       struct connectdata *conn_candidate;
 
@@ -5713,12 +5785,13 @@ static CURLcode create_conn(struct SessionHandle *data,
         conn_candidate->data = data;
         (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE);
       }
-      else
-        no_connections_available = TRUE;
+      else {
+        infof(data, "No connections available in cache\n");
+        connections_available = FALSE;
+      }
     }
 
-
-    if(no_connections_available) {
+    if(!connections_available) {
       infof(data, "No connections available.\n");
 
       conn_free(conn);
@@ -5732,7 +5805,7 @@ static CURLcode create_conn(struct SessionHandle *data,
        * This is a brand new connection, so let's store it in the connection
        * cache of ours!
        */
-      ConnectionStore(data, conn);
+      Curl_conncache_add_conn(data->state.conn_cache, conn);
     }
 
 #if defined(USE_NTLM)
@@ -5772,8 +5845,6 @@ static CURLcode create_conn(struct SessionHandle *data,
    * Inherit the proper values from the urldata struct AFTER we have arranged
    * the persistent connection stuff
    */
-  conn->fread_func = data->set.fread_func;
-  conn->fread_in = data->set.in;
   conn->seek_func = data->set.seek_func;
   conn->seek_client = data->set.seek_client;
 
@@ -5931,34 +6002,20 @@ CURLcode Curl_done(struct connectdata **connp,
   conn = *connp;
   data = conn->data;
 
-  if(conn->bits.done)
+  DEBUGF(infof(data, "Curl_done\n"));
+
+  if(data->state.done)
     /* Stop if Curl_done() has already been called */
     return CURLE_OK;
 
   Curl_getoff_all_pipelines(data, conn);
 
-  if((conn->send_pipe->size + conn->recv_pipe->size != 0 &&
-      !data->set.reuse_forbid &&
-      !conn->bits.close))
-    /* Stop if pipeline is not empty and we do not have to close
-       connection. */
-    return CURLE_OK;
-
-  conn->bits.done = TRUE; /* called just now! */
-
   /* Cleanup possible redirect junk */
   free(data->req.newurl);
   data->req.newurl = NULL;
   free(data->req.location);
   data->req.location = NULL;
 
-  Curl_resolver_cancel(conn);
-
-  if(conn->dns_entry) {
-    Curl_resolv_unlock(data, conn->dns_entry); /* done with this */
-    conn->dns_entry = NULL;
-  }
-
   switch(status) {
   case CURLE_ABORTED_BY_CALLBACK:
   case CURLE_READ_ERROR:
@@ -5981,6 +6038,23 @@ CURLcode Curl_done(struct connectdata **connp,
   if(!result && Curl_pgrsDone(conn))
     result = CURLE_ABORTED_BY_CALLBACK;
 
+  if((conn->send_pipe->size + conn->recv_pipe->size != 0 &&
+      !data->set.reuse_forbid &&
+      !conn->bits.close)) {
+    /* Stop if pipeline is not empty and we do not have to close
+       connection. */
+    DEBUGF(infof(data, "Connection still in use, no more Curl_done now!\n"));
+    return CURLE_OK;
+  }
+
+  data->state.done = TRUE; /* called just now! */
+  Curl_resolver_cancel(conn);
+
+  if(conn->dns_entry) {
+    Curl_resolv_unlock(data, conn->dns_entry); /* done with this */
+    conn->dns_entry = NULL;
+  }
+
   /* if the transfer was completed in a paused state there can be buffered
      data left to write and then kill */
   free(data->state.tempwrite);
@@ -6050,7 +6124,7 @@ static CURLcode do_init(struct connectdata *conn)
   struct SessionHandle *data = conn->data;
   struct SingleRequest *k = &data->req;
 
-  conn->bits.done = FALSE; /* Curl_done() is not called yet */
+  data->state.done = FALSE; /* Curl_done() is not called yet */
   conn->bits.do_more = FALSE; /* by default there's no curl_do_more() to use */
   data->state.expect100header = FALSE;