url: use the URL API internally as well

... to make it a truly unified URL parser. Closes #3017

Author: bagder

lib/curl_path.c

@@ -5,7 +5,7 @@
  * | (__| |_| | _ <| |___
  * \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -39,7 +39,7 @@ CURLcode Curl_getworkingpath(struct connectdata *conn,
  char *working_path;
  size_t working_path_len;
  CURLcode result =
- Curl_urldecode(data, data->state.path, 0, &working_path,
+ Curl_urldecode(data, data->state.up.path, 0, &working_path,
  &working_path_len, FALSE);
  if(result)
  return result;

lib/dict.c

@@ -136,7 +136,7 @@ static CURLcode dict_do(struct connectdata *conn, bool *done)
  struct Curl_easy *data = conn->data;
  curl_socket_t sockfd = conn->sock[FIRSTSOCKET];
 
- char *path = data->state.path;
+ char *path = data->state.up.path;
  curl_off_t *bytecount = &data->req.bytecount;
 
  *done = TRUE; /* unconditionally */

lib/easy.c

@@ -1002,10 +1002,6 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
  */
 void curl_easy_reset(struct Curl_easy *data)
 {
- Curl_safefree(data->state.pathbuffer);
-
- data->state.path = NULL;
-
  Curl_free_request_state(data);
 
  /* zero out UserDefined data: */

lib/file.c

@@ -143,7 +143,7 @@ static CURLcode file_connect(struct connectdata *conn, bool *done)
 #endif
  size_t real_path_len;
 
- CURLcode result = Curl_urldecode(data, data->state.path, 0, &real_path,
+ CURLcode result = Curl_urldecode(data, data->state.up.path, 0, &real_path,
  &real_path_len, FALSE);
  if(result)
  return result;
@@ -197,7 +197,7 @@ static CURLcode file_connect(struct connectdata *conn, bool *done)
 
  file->fd = fd;
  if(!data->set.upload && (fd == -1)) {
- failf(data, "Couldn't open file %s", data->state.path);
+ failf(data, "Couldn't open file %s", data->state.up.path);
  file_done(conn, CURLE_FILE_COULDNT_READ_FILE, FALSE);
  return CURLE_FILE_COULDNT_READ_FILE;
  }

lib/ftp.c

@@ -1444,6 +1444,7 @@ static CURLcode ftp_state_list(struct connectdata *conn)
 {
  CURLcode result = CURLE_OK;
  struct Curl_easy *data = conn->data;
+ struct FTP *ftp = data->req.protop;
 
  /* If this output is to be machine-parsed, the NLST command might be better
  to use, since the LIST command output is not specified or standard in any
@@ -1460,7 +1461,7 @@ static CURLcode ftp_state_list(struct connectdata *conn)
  then just do LIST (in that case: nothing to do here)
  */
  char *cmd, *lstArg, *slashPos;
- const char *inpath = data->state.path;
+ const char *inpath = ftp->path;
 
  lstArg = NULL;
  if((data->set.ftp_filemethod == FTPFILE_NOCWD) &&
@@ -3141,7 +3142,7 @@ static CURLcode ftp_done(struct connectdata *conn, CURLcode status,
  int ftpcode;
  CURLcode result = CURLE_OK;
  char *path = NULL;
- const char *path_to_use = data->state.path;
+ const char *path_to_use = ftp->path;
 
  if(!ftp)
  return CURLE_OK;
@@ -3346,7 +3347,7 @@ static CURLcode ftp_done(struct connectdata *conn, CURLcode status,
  /* Send any post-transfer QUOTE strings? */
  if(!status && !result && !premature && data->set.postquote)
  result = ftp_sendquote(conn, data->set.postquote);
-
+ Curl_safefree(ftp->pathalloc);
  return result;
 }
 
@@ -3695,12 +3696,13 @@ static void wc_data_dtor(void *ptr)
 static CURLcode init_wc_data(struct connectdata *conn)
 {
  char *last_slash;
- char *path = conn->data->state.path;
+ struct FTP *ftp = conn->data->req.protop;
+ char *path = ftp->path;
  struct WildcardData *wildcard = &(conn->data->wildcard);
  CURLcode result = CURLE_OK;
  struct ftp_wc *ftpwc = NULL;
 
- last_slash = strrchr(conn->data->state.path, '/');
+ last_slash = strrchr(ftp->path, '/');
  if(last_slash) {
  last_slash++;
  if(last_slash[0] == '\0') {
@@ -3757,7 +3759,7 @@ static CURLcode init_wc_data(struct connectdata *conn)
  goto fail;
  }
 
- wildcard->path = strdup(conn->data->state.path);
+ wildcard->path = strdup(ftp->path);
  if(!wildcard->path) {
  result = CURLE_OUT_OF_MEMORY;
  goto fail;
@@ -3828,16 +3830,15 @@ static CURLcode wc_statemach(struct connectdata *conn)
  /* filelist has at least one file, lets get first one */
  struct ftp_conn *ftpc = &conn->proto.ftpc;
  struct curl_fileinfo *finfo = wildcard->filelist.head->ptr;
+ struct FTP *ftp = conn->data->req.protop;
 
  char *tmp_path = aprintf("%s%s", wildcard->path, finfo->filename);
  if(!tmp_path)
  return CURLE_OUT_OF_MEMORY;
 
- /* switch default "state.pathbuffer" and tmp_path, good to see
- ftp_parse_url_path function to understand this trick */
- Curl_safefree(conn->data->state.pathbuffer);
- conn->data->state.pathbuffer = tmp_path;
- conn->data->state.path = tmp_path;
+ /* switch default ftp->path and tmp_path */
+ free(ftp->pathalloc);
+ ftp->pathalloc = ftp->path = tmp_path;
 
  infof(conn->data, "Wildcard - START of \"%s\"\n", finfo->filename);
  if(conn->data->set.chunk_bgn) {
@@ -4105,7 +4106,7 @@ CURLcode ftp_parse_url_path(struct connectdata *conn)
  struct FTP *ftp = data->req.protop;
  struct ftp_conn *ftpc = &conn->proto.ftpc;
  const char *slash_pos; /* position of the first '/' char in curpos */
- const char *path_to_use = data->state.path;
+ const char *path_to_use = ftp->path;
  const char *cur_pos;
  const char *filename = NULL;
 
@@ -4191,7 +4192,7 @@ CURLcode ftp_parse_url_path(struct connectdata *conn)
  /* parse the URL path into separate path components */
  while((slash_pos = strchr(cur_pos, '/')) != NULL) {
  /* 1 or 0 pointer offset to indicate absolute directory */
- ssize_t absolute_dir = ((cur_pos - data->state.path > 0) &&
+ ssize_t absolute_dir = ((cur_pos - ftp->path > 0) &&
  (ftpc->dirdepth == 0))?1:0;
 
  /* seek out the next path component */
@@ -4268,7 +4269,7 @@ CURLcode ftp_parse_url_path(struct connectdata *conn)
  size_t dlen;
  char *path;
  CURLcode result =
- Curl_urldecode(conn->data, data->state.path, 0, &path, &dlen, TRUE);
+ Curl_urldecode(conn->data, ftp->path, 0, &path, &dlen, TRUE);
  if(result) {
  freedirs(ftpc);
  return result;
@@ -4388,16 +4389,16 @@ static CURLcode ftp_setup_connection(struct connectdata *conn)
  char *type;
  struct FTP *ftp;
 
- conn->data->req.protop = ftp = malloc(sizeof(struct FTP));
+ conn->data->req.protop = ftp = calloc(sizeof(struct FTP), 1);
  if(NULL == ftp)
  return CURLE_OUT_OF_MEMORY;
 
- data->state.path++;  /* don't include the initial slash */
+ ftp->path = &data->state.up.path[1]; /* don't include the initial slash */
  data->state.slash_removed = TRUE; /* we've skipped the slash */
 
  /* FTP URLs support an extension like ";type=<typecode>" that
  * we'll try to get now! */
- type = strstr(data->state.path, ";type=");
+ type = strstr(ftp->path, ";type=");
 
  if(!type)
  type = strstr(conn->host.rawalloc, ";type=");

lib/ftp.h

@@ -105,6 +105,8 @@ struct FTP {
  curl_off_t *bytecountp;
  char *user; /* user name string */
  char *passwd; /* password string */
+ char *path; /* points to the urlpieces struct field */
+ char *pathalloc; /* if non-NULL a pointer to an allocated path */
 
  /* transfer a file/body or not, done as a typedefed enum just to make
  debuggers display the full symbol and not just the numerical value */

lib/gopher.c

@@ -78,7 +78,7 @@ static CURLcode gopher_do(struct connectdata *conn, bool *done)
  curl_socket_t sockfd = conn->sock[FIRSTSOCKET];
 
  curl_off_t *bytecount = &data->req.bytecount;
- char *path = data->state.path;
+ char *path = data->state.up.path;
  char *sel = NULL;
  char *sel_org = NULL;
  ssize_t amount, k;

lib/http.c

@@ -1877,7 +1877,8 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
  struct Curl_easy *data = conn->data;
  CURLcode result = CURLE_OK;
  struct HTTP *http;
- const char *ppath = data->state.path;
+ const char *path = data->state.up.path;
+ const char *query = data->state.up.query;
  bool paste_ftp_userpwd = FALSE;
  char ftp_typecode[sizeof("/;type=?")] = "";
  const char *host = conn->host.name;
@@ -1995,7 +1996,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
  }
 
  /* setup the authentication headers */
- result = Curl_http_output_auth(conn, request, ppath, FALSE);
+ result = Curl_http_output_auth(conn, request, path, FALSE);
  if(result)
  return result;
 
@@ -2223,47 +2224,59 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
  /* The path sent to the proxy is in fact the entire URL. But if the remote
  host is a IDN-name, we must make sure that the request we produce only
  uses the encoded host name! */
+
+ /* and no fragment part */
+ CURLUcode uc;
+ char *url;
+ CURLU *h = curl_url_dup(data->state.uh);
+ if(!h)
+ return CURLE_OUT_OF_MEMORY;
+
  if(conn->host.dispname != conn->host.name) {
- char *url = data->change.url;
- ptr = strstr(url, conn->host.dispname);
- if(ptr) {
- /* This is where the display name starts in the URL, now replace this
- part with the encoded name. TODO: This method of replacing the host
- name is rather crude as I believe there's a slight risk that the
- user has entered a user name or password that contain the host name
- string. */
- size_t currlen = strlen(conn->host.dispname);
- size_t newlen = strlen(conn->host.name);
- size_t urllen = strlen(url);
-
- char *newurl;
-
- newurl = malloc(urllen + newlen - currlen + 1);
- if(newurl) {
- /* copy the part before the host name */
- memcpy(newurl, url, ptr - url);
- /* append the new host name instead of the old */
- memcpy(newurl + (ptr - url), conn->host.name, newlen);
- /* append the piece after the host name */
- memcpy(newurl + newlen + (ptr - url),
- ptr + currlen, /* copy the trailing zero byte too */
- urllen - (ptr-url) - currlen + 1);
- if(data->change.url_alloc) {
- Curl_safefree(data->change.url);
- data->change.url_alloc = FALSE;
- }
- data->change.url = newurl;
- data->change.url_alloc = TRUE;
- }
- else
- return CURLE_OUT_OF_MEMORY;
+ uc = curl_url_set(h, CURLUPART_HOST, conn->host.name, 0);
+ if(uc) {
+ curl_url_cleanup(h);
+ return CURLE_OUT_OF_MEMORY;
  }
  }
- ppath = data->change.url;
- if(checkprefix("ftp://", ppath)) {
+ uc = curl_url_set(h, CURLUPART_FRAGMENT, NULL, 0);
+ if(uc) {
+ curl_url_cleanup(h);
+ return CURLE_OUT_OF_MEMORY;
+ }
+
+ if(strcasecompare("http", data->state.up.scheme)) {
+ /* when getting HTTP, we don't want the userinfo the URL */
+ uc = curl_url_set(h, CURLUPART_USER, NULL, 0);
+ if(uc) {
+ curl_url_cleanup(h);
+ return CURLE_OUT_OF_MEMORY;
+ }
+ uc = curl_url_set(h, CURLUPART_PASSWORD, NULL, 0);
+ if(uc) {
+ curl_url_cleanup(h);
+ return CURLE_OUT_OF_MEMORY;
+ }
+ }
+ /* now extract the new version of the URL */
+ uc = curl_url_get(h, CURLUPART_URL, &url, 0);
+ if(uc) {
+ curl_url_cleanup(h);
+ return CURLE_OUT_OF_MEMORY;
+ }
+
+ if(data->change.url_alloc)
+ free(data->change.url);
+
+ data->change.url = url;
+ data->change.url_alloc = TRUE;
+
+ curl_url_cleanup(h);
+
+ if(strcasecompare("ftp", data->state.up.scheme)) {
  if(data->set.proxy_transfer_mode) {
  /* when doing ftp, append ;type=<a|i> if not present */
- char *type = strstr(ppath, ";type=");
+ char *type = strstr(path, ";type=");
  if(type && type[6] && type[7] == 0) {
  switch(Curl_raw_toupper(type[6])) {
  case 'A':
@@ -2278,7 +2291,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
  char *p = ftp_typecode;
  /* avoid sending invalid URLs like ftp://example.com;type=i if the
  * user specified ftp://example.com without the slash */
- if(!*data->state.path && ppath[strlen(ppath) - 1] != '/') {
+ if(!*data->state.up.path && path[strlen(path) - 1] != '/') {
  *p++ = '/';
  }
  snprintf(p, sizeof(ftp_typecode) - 1, ";type=%c",
@@ -2431,18 +2444,32 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
  if(result)
  return result;
 
- if(data->set.str[STRING_TARGET])
- ppath = data->set.str[STRING_TARGET];
+ if(data->set.str[STRING_TARGET]) {
+ path = data->set.str[STRING_TARGET];
+ query = NULL;
+ }
 
  /* url */
- if(paste_ftp_userpwd)
+ if(conn->bits.httpproxy && !conn->bits.tunnel_proxy) {
+ char *url = data->change.url;
+ result = Curl_add_buffer(&req_buffer, url, strlen(url));
+ if(result)
+ return result;
+ }
+ else if(paste_ftp_userpwd)
  result = Curl_add_bufferf(&req_buffer, "ftp://%s:%s@%s",
  conn->user, conn->passwd,
- ppath + sizeof("ftp://") - 1);
- else
- result = Curl_add_buffer(&req_buffer, ppath, strlen(ppath));
- if(result)
- return result;
+ path + sizeof("ftp://") - 1);
+ else {
+ result = Curl_add_buffer(&req_buffer, path, strlen(path));
+ if(result)
+ return result;
+ if(query) {
+ result = Curl_add_bufferf(&req_buffer, "?%s", query);
+ if(result)
+ return result;
+ }
+ }
 
  result =
  Curl_add_bufferf(&req_buffer,
@@ -2515,7 +2542,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
  co = Curl_cookie_getlist(data->cookies,
  conn->allocptr.cookiehost?
  conn->allocptr.cookiehost:host,
- data->state.path,
+ data->state.up.path,
  (conn->handler->protocol&CURLPROTO_HTTPS)?
  TRUE:FALSE);
  Curl_share_unlock(data, CURL_LOCK_DATA_COOKIE);
@@ -3836,7 +3863,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data,
  here, or else use real peer host name. */
  conn->allocptr.cookiehost?
  conn->allocptr.cookiehost:conn->host.name,
- data->state.path);
+ data->state.up.path);
  Curl_share_unlock(data, CURL_LOCK_DATA_COOKIE);
  }
 #endif