/*************************************************************************** * _ _ ____ _ * Project ___| | | | _ \| | * / __| | | | |_) | | * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * * Copyright (C) Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms * are also available at https://curl.se/docs/copyright.html. * * You may opt to use, copy, modify, merge, publish, distribute and/or sell * copies of the Software, and permit persons to whom the Software is * furnished to do so, under the terms of the COPYING file. * * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY * KIND, either express or implied. * * SPDX-License-Identifier: curl * ***************************************************************************/ #include "tool_setup.h" #ifdef HAVE_FCNTL_H # include #endif #ifdef HAVE_LOCALE_H # include #endif #ifdef HAVE_SYS_SELECT_H # include #elif defined(HAVE_UNISTD_H) # include #endif #ifdef __VMS # include #endif #ifdef __AMIGA__ # include #endif #ifdef HAVE_NETINET_IN_H # include #endif #ifdef HAVE_UV_H /* Hack for Unity mode */ #ifdef HEADER_CURL_MEMDEBUG_H #undef HEADER_CURL_MEMDEBUG_H #undef freeaddrinfo #undef getaddrinfo #endif /* this is for libuv-enabled debug builds only */ #include #endif #include "curlx.h" #include "tool_binmode.h" #include "tool_cfgable.h" #include "tool_cb_dbg.h" #include "tool_cb_hdr.h" #include "tool_cb_prg.h" #include "tool_cb_rea.h" #include "tool_cb_see.h" #include "tool_cb_soc.h" #include "tool_cb_wrt.h" #include "tool_dirhie.h" #include "tool_doswin.h" #include "tool_easysrc.h" #include "tool_filetime.h" #include "tool_getparam.h" #include "tool_helpers.h" #include "tool_findfile.h" #include "tool_libinfo.h" #include "tool_main.h" #include "tool_msgs.h" #include "tool_operate.h" #include "tool_operhlp.h" #include "tool_paramhlp.h" #include "tool_parsecfg.h" #include "tool_setopt.h" #include "tool_sleep.h" #include "tool_urlglob.h" #include "tool_util.h" #include "tool_writeout.h" #include "tool_xattr.h" #include "tool_vms.h" #include "tool_help.h" #include "tool_hugehelp.h" #include "tool_progress.h" #include "tool_ipfs.h" #include "dynbuf.h" #ifdef DEBUGBUILD /* libcurl's debug-only curl_easy_perform_ev() */ CURL_EXTERN CURLcode curl_easy_perform_ev(CURL *easy); #endif #include "memdebug.h" /* keep this as LAST include */ #ifdef CURL_CA_EMBED #ifndef CURL_DECLARED_CURL_CA_EMBED #define CURL_DECLARED_CURL_CA_EMBED extern const unsigned char curl_ca_embed[]; #endif #endif #ifndef O_BINARY /* since O_BINARY as used in bitmasks, setting it to zero makes it usable in source code but yet it does not ruin anything */ # define O_BINARY 0 #endif #ifndef SOL_IP # define SOL_IP IPPROTO_IP #endif #define CURL_CA_CERT_ERRORMSG \ "More details here: https://curl.se/docs/sslcerts.html\n\n" \ "curl failed to verify the legitimacy of the server and therefore " \ "could not\nestablish a secure connection to it. To learn more about " \ "this situation and\nhow to fix it, please visit the webpage mentioned " \ "above.\n" static CURLcode single_transfer(struct GlobalConfig *global, struct OperationConfig *config, CURLSH *share, bool capath_from_env, bool *added, bool *skipped); static CURLcode create_transfer(struct GlobalConfig *global, CURLSH *share, bool *added, bool *skipped); static bool is_fatal_error(CURLcode code) { switch(code) { case CURLE_FAILED_INIT: case CURLE_OUT_OF_MEMORY: case CURLE_UNKNOWN_OPTION: case CURLE_FUNCTION_NOT_FOUND: case CURLE_BAD_FUNCTION_ARGUMENT: /* critical error */ return TRUE; default: break; } /* no error or not critical */ return FALSE; } /* * Check if a given string is a PKCS#11 URI */ static bool is_pkcs11_uri(const char *string) { if(curl_strnequal(string, "pkcs11:", 7)) { return TRUE; } else { return FALSE; } } #ifdef IP_TOS static int get_address_family(curl_socket_t sockfd) { struct sockaddr addr; curl_socklen_t addrlen = sizeof(addr); memset(&addr, 0, sizeof(addr)); if(getsockname(sockfd, (struct sockaddr *)&addr, &addrlen) == 0) return addr.sa_family; return AF_UNSPEC; } #endif #if defined(IP_TOS) || defined(IPV6_TCLASS) || defined(SO_PRIORITY) static int sockopt_callback(void *clientp, curl_socket_t curlfd, curlsocktype purpose) { struct OperationConfig *config = (struct OperationConfig *)clientp; if(purpose != CURLSOCKTYPE_IPCXN) return CURL_SOCKOPT_OK; (void)config; (void)curlfd; #if defined(IP_TOS) || defined(IPV6_TCLASS) if(config->ip_tos > 0) { int tos = (int)config->ip_tos; int result = 0; switch(get_address_family(curlfd)) { case AF_INET: #ifdef IP_TOS result = setsockopt(curlfd, SOL_IP, IP_TOS, (void *)&tos, sizeof(tos)); #endif break; #if defined(IPV6_TCLASS) && defined(AF_INET6) case AF_INET6: result = setsockopt(curlfd, IPPROTO_IPV6, IPV6_TCLASS, (void *)&tos, sizeof(tos)); break; #endif } if(result < 0) { int error = errno; warnf(config->global, "Setting type of service to %d failed with errno %d: %s;\n", tos, error, strerror(error)); } } #endif #ifdef SO_PRIORITY if(config->vlan_priority > 0) { int priority = (int)config->vlan_priority; if(setsockopt(curlfd, SOL_SOCKET, SO_PRIORITY, (void *)&priority, sizeof(priority)) != 0) { int error = errno; warnf(config->global, "VLAN priority %d failed with errno %d: %s;\n", priority, error, strerror(error)); } } #endif return CURL_SOCKOPT_OK; } #endif #ifdef __VMS /* * get_vms_file_size does what it takes to get the real size of the file * * For fixed files, find out the size of the EOF block and adjust. * * For all others, have to read the entire file in, discarding the contents. * Most posted text files will be small, and binary files like zlib archives * and CD/DVD images should be either a STREAM_LF format or a fixed format. * */ static curl_off_t vms_realfilesize(const char *name, const struct_stat *stat_buf) { char buffer[8192]; curl_off_t count; int ret_stat; FILE * file; /* !checksrc! disable FOPENMODE 1 */ file = fopen(name, "r"); /* VMS */ if(!file) { return 0; } count = 0; ret_stat = 1; while(ret_stat > 0) { ret_stat = fread(buffer, 1, sizeof(buffer), file); if(ret_stat) count += ret_stat; } fclose(file); return count; } /* * * VmsSpecialSize checks to see if the stat st_size can be trusted and * if not to call a routine to get the correct size. * */ static curl_off_t VmsSpecialSize(const char *name, const struct_stat *stat_buf) { switch(stat_buf->st_fab_rfm) { case FAB$C_VAR: case FAB$C_VFC: return vms_realfilesize(name, stat_buf); break; default: return stat_buf->st_size; } } #endif /* __VMS */ #define BUFFER_SIZE (100*1024) struct per_transfer *transfers; /* first node */ static struct per_transfer *transfersl; /* last node */ static curl_off_t all_pers; /* add_per_transfer creates a new 'per_transfer' node in the linked list of transfers */ static CURLcode add_per_transfer(struct per_transfer **per) { struct per_transfer *p; p = calloc(1, sizeof(struct per_transfer)); if(!p) return CURLE_OUT_OF_MEMORY; if(!transfers) /* first entry */ transfersl = transfers = p; else { /* make the last node point to the new node */ transfersl->next = p; /* make the new node point back to the formerly last node */ p->prev = transfersl; /* move the last node pointer to the new entry */ transfersl = p; } *per = p; all_xfers++; /* count total number of transfers added */ all_pers++; return CURLE_OK; } /* Remove the specified transfer from the list (and free it), return the next in line */ static struct per_transfer *del_per_transfer(struct per_transfer *per) { struct per_transfer *n; struct per_transfer *p; DEBUGASSERT(transfers); DEBUGASSERT(transfersl); DEBUGASSERT(per); n = per->next; p = per->prev; if(p) p->next = n; else transfers = n; if(n) n->prev = p; else transfersl = p; free(per); all_pers--; return n; } static CURLcode pre_transfer(struct GlobalConfig *global, struct per_transfer *per) { curl_off_t uploadfilesize = -1; struct_stat fileinfo; CURLcode result = CURLE_OK; if(per->uploadfile && !stdin_upload(per->uploadfile)) { /* VMS Note: * * Reading binary from files can be a problem... Only FIXED, VAR * etc WITHOUT implied CC will work. Others need a \n appended to * a line * * - Stat gives a size but this is UNRELIABLE in VMS. E.g. * a fixed file with implied CC needs to have a byte added for every * record processed, this can be derived from Filesize & recordsize * for VARiable record files the records need to be counted! for * every record add 1 for linefeed and subtract 2 for the record * header for VARIABLE header files only the bare record data needs * to be considered with one appended if implied CC */ #ifdef __VMS /* Calculate the real upload size for VMS */ per->infd = -1; if(stat(per->uploadfile, &fileinfo) == 0) { fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo); switch(fileinfo.st_fab_rfm) { case FAB$C_VAR: case FAB$C_VFC: case FAB$C_STMCR: per->infd = open(per->uploadfile, O_RDONLY | O_BINARY); break; default: per->infd = open(per->uploadfile, O_RDONLY | O_BINARY, "rfm=stmlf", "ctx=stm"); } } if(per->infd == -1) #else per->infd = open(per->uploadfile, O_RDONLY | O_BINARY); if((per->infd == -1) || fstat(per->infd, &fileinfo)) #endif { helpf(tool_stderr, "cannot open '%s'", per->uploadfile); if(per->infd != -1) { close(per->infd); per->infd = STDIN_FILENO; } return CURLE_READ_ERROR; } per->infdopen = TRUE; /* we ignore file size for char/block devices, sockets, etc. */ if(S_ISREG(fileinfo.st_mode)) uploadfilesize = fileinfo.st_size; #ifdef DEBUGBUILD /* allow dedicated test cases to override */ { char *ev = getenv("CURL_UPLOAD_SIZE"); if(ev) { int sz = atoi(ev); uploadfilesize = (curl_off_t)sz; } } #endif if(uploadfilesize != -1) { struct OperationConfig *config = per->config; /* for the macro below */ #ifdef CURL_DISABLE_LIBCURL_OPTION (void)config; (void)global; #endif my_setopt(per->curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize); } } per->uploadfilesize = uploadfilesize; per->start = tvnow(); return result; } /* When doing serial transfers, we use a single fixed error area */ static char global_errorbuffer[CURL_ERROR_SIZE]; void single_transfer_cleanup(struct OperationConfig *config) { if(config) { struct State *state = &config->state; /* Free list of remaining URLs */ glob_cleanup(&state->urls); Curl_safefree(state->outfiles); Curl_safefree(state->uploadfile); /* Free list of globbed upload files */ glob_cleanup(&state->inglob); } } /* * Call this after a transfer has completed. */ static CURLcode post_per_transfer(struct GlobalConfig *global, struct per_transfer *per, CURLcode result, bool *retryp, long *delay) /* milliseconds! */ { struct OutStruct *outs = &per->outs; CURL *curl = per->curl; struct OperationConfig *config = per->config; int rc; *retryp = FALSE; *delay = 0; /* for no retry, keep it zero */ if(!curl || !config) return result; if(per->infdopen) close(per->infd); if(per->skip) goto skip; #ifdef __VMS if(is_vms_shell()) { /* VMS DCL shell behavior */ if(global->silent && !global->showerror) vms_show = VMSSTS_HIDE; } else #endif if(!config->synthetic_error && result && (!global->silent || global->showerror)) { const char *msg = per->errorbuffer; fprintf(tool_stderr, "curl: (%d) %s\n", result, (msg && msg[0]) ? msg : curl_easy_strerror(result)); if(result == CURLE_PEER_FAILED_VERIFICATION) fputs(CURL_CA_CERT_ERRORMSG, tool_stderr); } else if(config->failwithbody) { /* if HTTP response >= 400, return error */ long code = 0; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); if(code >= 400) { if(!global->silent || global->showerror) fprintf(tool_stderr, "curl: (%d) The requested URL returned error: %ld\n", CURLE_HTTP_RETURNED_ERROR, code); result = CURLE_HTTP_RETURNED_ERROR; } } /* Set file extended attributes */ if(!result && config->xattr && outs->fopened && outs->stream) { rc = fwrite_xattr(curl, per->url, fileno(outs->stream)); if(rc) warnf(config->global, "Error setting extended attributes on '%s': %s", outs->filename, strerror(errno)); } if(!result && !outs->stream && !outs->bytes) { /* we have received no data despite the transfer was successful ==> force creation of an empty output file (if an output file was specified) */ long cond_unmet = 0L; /* do not create (or even overwrite) the file in case we get no data because of unmet condition */ curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &cond_unmet); if(!cond_unmet && !tool_create_output_file(outs, config)) result = CURLE_WRITE_ERROR; } if(!outs->s_isreg && outs->stream) { /* Dump standard stream buffered data */ rc = fflush(outs->stream); if(!result && rc) { /* something went wrong in the writing process */ result = CURLE_WRITE_ERROR; errorf(global, "Failed writing body"); } } #ifdef _WIN32 /* Discard incomplete UTF-8 sequence buffered from body */ if(outs->utf8seq[0]) memset(outs->utf8seq, 0, sizeof(outs->utf8seq)); #endif /* if retry-max-time is non-zero, make sure we have not exceeded the time */ if(per->retry_remaining && (!config->retry_maxtime || (tvdiff(tvnow(), per->retrystart) < config->retry_maxtime*1000L)) ) { enum { RETRY_NO, RETRY_ALL_ERRORS, RETRY_TIMEOUT, RETRY_CONNREFUSED, RETRY_HTTP, RETRY_FTP, RETRY_LAST /* not used */ } retry = RETRY_NO; long response = 0; if((CURLE_OPERATION_TIMEDOUT == result) || (CURLE_COULDNT_RESOLVE_HOST == result) || (CURLE_COULDNT_RESOLVE_PROXY == result) || (CURLE_FTP_ACCEPT_TIMEOUT == result)) /* retry timeout always */ retry = RETRY_TIMEOUT; else if(config->retry_connrefused && (CURLE_COULDNT_CONNECT == result)) { long oserrno = 0; curl_easy_getinfo(curl, CURLINFO_OS_ERRNO, &oserrno); if(ECONNREFUSED == oserrno) retry = RETRY_CONNREFUSED; } else if((CURLE_OK == result) || ((config->failonerror || config->failwithbody) && (CURLE_HTTP_RETURNED_ERROR == result))) { /* If it returned OK. _or_ failonerror was enabled and it returned due to such an error, check for HTTP transient errors to retry on. */ const char *scheme; curl_easy_getinfo(curl, CURLINFO_SCHEME, &scheme); scheme = proto_token(scheme); if(scheme == proto_http || scheme == proto_https) { /* This was HTTP(S) */ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); switch(response) { case 408: /* Request Timeout */ case 429: /* Too Many Requests (RFC6585) */ case 500: /* Internal Server Error */ case 502: /* Bad Gateway */ case 503: /* Service Unavailable */ case 504: /* Gateway Timeout */ retry = RETRY_HTTP; /* * At this point, we have already written data to the output * file (or terminal). If we write to a file, we must rewind * or close/re-open the file so that the next attempt starts * over from the beginning. * * TODO: similar action for the upload case. We might need * to start over reading from a previous point if we have * uploaded something when this was returned. */ break; } } } /* if CURLE_OK */ else if(result) { const char *scheme; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); curl_easy_getinfo(curl, CURLINFO_SCHEME, &scheme); scheme = proto_token(scheme); if((scheme == proto_ftp || scheme == proto_ftps) && response / 100 == 4) /* * This is typically when the FTP server only allows a certain * amount of users and we are not one of them. All 4xx codes * are transient. */ retry = RETRY_FTP; } if(result && !retry && config->retry_all_errors) retry = RETRY_ALL_ERRORS; if(retry) { long sleeptime = 0; curl_off_t retry_after = 0; static const char * const m[]={ NULL, "(retrying all errors)", ": timeout", ": connection refused", ": HTTP error", ": FTP error" }; sleeptime = per->retry_sleep; if(RETRY_HTTP == retry) { curl_easy_getinfo(curl, CURLINFO_RETRY_AFTER, &retry_after); if(retry_after) { /* store in a 'long', make sure it does not overflow */ if(retry_after > LONG_MAX/1000) sleeptime = LONG_MAX; else if((retry_after * 1000) > sleeptime) sleeptime = (long)retry_after * 1000; /* milliseconds */ /* if adding retry_after seconds to the process would exceed the maximum time allowed for retrying, then exit the retries right away */ if(config->retry_maxtime) { curl_off_t seconds = tvdiff(tvnow(), per->retrystart)/1000; if((CURL_OFF_T_MAX - retry_after < seconds) || (seconds + retry_after > config->retry_maxtime)) { warnf(config->global, "The Retry-After: time would " "make this command line exceed the maximum allowed time " "for retries."); goto noretry; } } } } warnf(config->global, "Problem %s. " "Will retry in %ld seconds. " "%ld retries left.", m[retry], sleeptime/1000L, per->retry_remaining); per->retry_remaining--; if(!config->retry_delay) { per->retry_sleep *= 2; if(per->retry_sleep > RETRY_SLEEP_MAX) per->retry_sleep = RETRY_SLEEP_MAX; } if(outs->bytes && outs->filename && outs->stream) { /* We have written data to an output file, we truncate file */ notef(config->global, "Throwing away %" CURL_FORMAT_CURL_OFF_T " bytes", outs->bytes); fflush(outs->stream); /* truncate file at the position where we started appending */ #ifdef HAVE_FTRUNCATE if(ftruncate(fileno(outs->stream), outs->init)) { /* when truncate fails, we cannot just append as then we will create something strange, bail out */ errorf(config->global, "Failed to truncate file"); return CURLE_WRITE_ERROR; } /* now seek to the end of the file, the position where we just truncated the file in a large file-safe way */ rc = fseek(outs->stream, 0, SEEK_END); #else /* ftruncate is not available, so just reposition the file to the location we would have truncated it. This will not work properly with large files on 32-bit systems, but most of those will have ftruncate. */ rc = fseek(outs->stream, (long)outs->init, SEEK_SET); #endif if(rc) { errorf(config->global, "Failed seeking to end of file"); return CURLE_WRITE_ERROR; } outs->bytes = 0; /* clear for next round */ } *retryp = TRUE; per->num_retries++; *delay = sleeptime; return CURLE_OK; } } /* if retry_remaining */ noretry: if((global->progressmode == CURL_PROGRESS_BAR) && per->progressbar.calls) /* if the custom progress bar has been displayed, we output a newline here */ fputs("\n", per->progressbar.out); /* Close the outs file */ if(outs->fopened && outs->stream) { rc = fclose(outs->stream); if(!result && rc) { /* something went wrong in the writing process */ result = CURLE_WRITE_ERROR; errorf(config->global, "curl: (%d) Failed writing body", result); } if(result && config->rm_partial) { struct_stat st; if(!stat(outs->filename, &st) && S_ISREG(st.st_mode)) { if(!unlink(outs->filename)) notef(global, "Removed output file: %s", outs->filename); else warnf(global, "Failed removing: %s", outs->filename); } else warnf(global, "Skipping removal; not a regular file: %s", outs->filename); } } /* File time can only be set _after_ the file has been closed */ if(!result && config->remote_time && outs->s_isreg && outs->filename) { /* Ask libcurl if we got a remote file time */ curl_off_t filetime = -1; curl_easy_getinfo(curl, CURLINFO_FILETIME_T, &filetime); setfiletime(filetime, outs->filename, global); } skip: /* Write the --write-out data before cleanup but after result is final */ if(config->writeout) ourWriteOut(config, per, result); /* Close function-local opened file descriptors */ if(per->heads.fopened && per->heads.stream) fclose(per->heads.stream); if(per->heads.alloc_filename) Curl_safefree(per->heads.filename); if(per->etag_save.fopened && per->etag_save.stream) fclose(per->etag_save.stream); if(per->etag_save.alloc_filename) Curl_safefree(per->etag_save.filename); curl_easy_cleanup(per->curl); if(outs->alloc_filename) free(outs->filename); free(per->url); free(per->outfile); free(per->uploadfile); if(global->parallel) free(per->errorbuffer); curl_slist_free_all(per->hdrcbdata.headlist); per->hdrcbdata.headlist = NULL; return result; } /* * Possibly rewrite the URL for IPFS and return the protocol token for the * scheme used in the given URL. */ static CURLcode url_proto_and_rewrite(char **url, struct OperationConfig *config, const char **scheme) { CURLcode result = CURLE_OK; CURLU *uh = curl_url(); const char *proto = NULL; *scheme = NULL; DEBUGASSERT(url && *url); if(uh) { char *schemep = NULL; if(!curl_url_set(uh, CURLUPART_URL, *url, CURLU_GUESS_SCHEME | CURLU_NON_SUPPORT_SCHEME) && !curl_url_get(uh, CURLUPART_SCHEME, &schemep, CURLU_DEFAULT_SCHEME)) { #ifdef CURL_DISABLE_IPFS (void)config; #else if(curl_strequal(schemep, proto_ipfs) || curl_strequal(schemep, proto_ipns)) { result = ipfs_url_rewrite(uh, schemep, url, config); /* short-circuit proto_token, we know it is ipfs or ipns */ if(curl_strequal(schemep, proto_ipfs)) proto = proto_ipfs; else if(curl_strequal(schemep, proto_ipns)) proto = proto_ipns; if(result) config->synthetic_error = TRUE; } else #endif /* !CURL_DISABLE_IPFS */ proto = proto_token(schemep); curl_free(schemep); } curl_url_cleanup(uh); } else result = CURLE_OUT_OF_MEMORY; *scheme = proto ? proto : "?"; /* Never match if not found. */ return result; } /* return current SSL backend name, chop off multissl */ static char *ssl_backend(void) { static char ssl_ver[80] = "no ssl"; static bool already = FALSE; if(!already) { /* if there is no existing version */ const char *v = curl_version_info(CURLVERSION_NOW)->ssl_version; if(v) msnprintf(ssl_ver, sizeof(ssl_ver), "%.*s", (int) strcspn(v, " "), v); already = TRUE; } return ssl_ver; } static CURLcode set_cert_types(struct OperationConfig *config) { if(feature_ssl) { /* Check if config->cert is a PKCS#11 URI and set the config->cert_type if * necessary */ if(config->cert && !config->cert_type && is_pkcs11_uri(config->cert)) { config->cert_type = strdup("ENG"); if(!config->cert_type) return CURLE_OUT_OF_MEMORY; } /* Check if config->key is a PKCS#11 URI and set the config->key_type if * necessary */ if(config->key && !config->key_type && is_pkcs11_uri(config->key)) { config->key_type = strdup("ENG"); if(!config->key_type) return CURLE_OUT_OF_MEMORY; } /* Check if config->proxy_cert is a PKCS#11 URI and set the * config->proxy_type if necessary */ if(config->proxy_cert && !config->proxy_cert_type && is_pkcs11_uri(config->proxy_cert)) { config->proxy_cert_type = strdup("ENG"); if(!config->proxy_cert_type) return CURLE_OUT_OF_MEMORY; } /* Check if config->proxy_key is a PKCS#11 URI and set the * config->proxy_key_type if necessary */ if(config->proxy_key && !config->proxy_key_type && is_pkcs11_uri(config->proxy_key)) { config->proxy_key_type = strdup("ENG"); if(!config->proxy_key_type) return CURLE_OUT_OF_MEMORY; } } return CURLE_OK; } static CURLcode config2setopts(struct GlobalConfig *global, struct OperationConfig *config, struct per_transfer *per, bool capath_from_env, CURL *curl, CURLSH *share) { const char *use_proto; CURLcode result = url_proto_and_rewrite(&per->url, config, &use_proto); /* Avoid having this setopt added to the --libcurl source output. */ if(!result) result = curl_easy_setopt(curl, CURLOPT_SHARE, share); if(result) return result; #ifndef DEBUGBUILD /* On most modern OSes, exiting works thoroughly, we will clean everything up via exit(), so do not bother with slow cleanups. Crappy ones might need to skip this. Note: avoid having this setopt added to the --libcurl source output. */ result = curl_easy_setopt(curl, CURLOPT_QUICK_EXIT, 1L); if(result) return result; #endif if(!config->tcp_nodelay) my_setopt(curl, CURLOPT_TCP_NODELAY, 0L); if(config->tcp_fastopen) my_setopt(curl, CURLOPT_TCP_FASTOPEN, 1L); if(config->mptcp) my_setopt(curl, CURLOPT_OPENSOCKETFUNCTION, tool_socket_open_mptcp_cb); /* where to store */ my_setopt(curl, CURLOPT_WRITEDATA, per); my_setopt(curl, CURLOPT_INTERLEAVEDATA, per); /* what call to write */ my_setopt(curl, CURLOPT_WRITEFUNCTION, tool_write_cb); /* Note that if CURLOPT_READFUNCTION is fread (the default), then * lib/telnet.c will Curl_poll() on the input file descriptor * rather than calling the READFUNCTION at regular intervals. * The circumstances in which it is preferable to enable this * behavior, by omitting to set the READFUNCTION & READDATA options, * have not been determined. */ my_setopt(curl, CURLOPT_READDATA, per); /* what call to read */ my_setopt(curl, CURLOPT_READFUNCTION, tool_read_cb); /* in 7.18.0, the CURLOPT_SEEKFUNCTION/DATA pair is taking over what CURLOPT_IOCTLFUNCTION/DATA pair previously provided for seeking */ my_setopt(curl, CURLOPT_SEEKDATA, per); my_setopt(curl, CURLOPT_SEEKFUNCTION, tool_seek_cb); { #ifdef DEBUGBUILD char *env = getenv("CURL_BUFFERSIZE"); if(env) { long size = strtol(env, NULL, 10); if(size) my_setopt(curl, CURLOPT_BUFFERSIZE, size); } else #endif if(config->recvpersecond && (config->recvpersecond < BUFFER_SIZE)) /* use a smaller sized buffer for better sleeps */ my_setopt(curl, CURLOPT_BUFFERSIZE, (long)config->recvpersecond); else my_setopt(curl, CURLOPT_BUFFERSIZE, (long)BUFFER_SIZE); } my_setopt_str(curl, CURLOPT_URL, per->url); my_setopt(curl, CURLOPT_NOPROGRESS, global->noprogress || global->silent ? 1L : 0L); if(config->no_body) my_setopt(curl, CURLOPT_NOBODY, 1L); if(config->oauth_bearer) my_setopt_str(curl, CURLOPT_XOAUTH2_BEARER, config->oauth_bearer); my_setopt_str(curl, CURLOPT_PROXY, config->proxy); if(config->proxy && result) { errorf(global, "proxy support is disabled in this libcurl"); config->synthetic_error = TRUE; return CURLE_NOT_BUILT_IN; } /* new in libcurl 7.5 */ if(config->proxy) my_setopt_enum(curl, CURLOPT_PROXYTYPE, config->proxyver); my_setopt_str(curl, CURLOPT_PROXYUSERPWD, config->proxyuserpwd); /* new in libcurl 7.3 */ my_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, config->proxytunnel ? 1L : 0L); /* new in libcurl 7.52.0 */ if(config->preproxy) my_setopt_str(curl, CURLOPT_PRE_PROXY, config->preproxy); /* new in libcurl 7.10.6 */ if(config->proxyanyauth) my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, (long)CURLAUTH_ANY); else if(config->proxynegotiate) my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, CURLAUTH_GSSNEGOTIATE); else if(config->proxyntlm) my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, CURLAUTH_NTLM); else if(config->proxydigest) my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, CURLAUTH_DIGEST); else if(config->proxybasic) my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, CURLAUTH_BASIC); /* new in libcurl 7.19.4 */ my_setopt_str(curl, CURLOPT_NOPROXY, config->noproxy); my_setopt(curl, CURLOPT_SUPPRESS_CONNECT_HEADERS, config->suppress_connect_headers ? 1L : 0L); my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror ? 1L : 0L); my_setopt(curl, CURLOPT_REQUEST_TARGET, config->request_target); my_setopt(curl, CURLOPT_UPLOAD, per->uploadfile ? 1L : 0L); my_setopt(curl, CURLOPT_DIRLISTONLY, config->dirlistonly ? 1L : 0L); my_setopt(curl, CURLOPT_APPEND, config->ftp_append ? 1L : 0L); if(config->netrc_opt) my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_OPTIONAL); else if(config->netrc || config->netrc_file) my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_REQUIRED); else my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_IGNORED); if(config->netrc_file) my_setopt_str(curl, CURLOPT_NETRC_FILE, config->netrc_file); my_setopt(curl, CURLOPT_TRANSFERTEXT, config->use_ascii ? 1L : 0L); if(config->login_options) my_setopt_str(curl, CURLOPT_LOGIN_OPTIONS, config->login_options); my_setopt_str(curl, CURLOPT_USERPWD, config->userpwd); my_setopt_str(curl, CURLOPT_RANGE, config->range); if(!global->parallel) { per->errorbuffer = global_errorbuffer; my_setopt(curl, CURLOPT_ERRORBUFFER, global_errorbuffer); } my_setopt(curl, CURLOPT_TIMEOUT_MS, config->timeout_ms); switch(config->httpreq) { case TOOL_HTTPREQ_SIMPLEPOST: if(config->resume_from) { errorf(global, "cannot mix --continue-at with --data"); result = CURLE_FAILED_INIT; } else { my_setopt_str(curl, CURLOPT_POSTFIELDS, curlx_dyn_ptr(&config->postdata)); my_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)curlx_dyn_len(&config->postdata)); } break; case TOOL_HTTPREQ_MIMEPOST: /* free previous remainders */ curl_mime_free(config->mimepost); config->mimepost = NULL; if(config->resume_from) { errorf(global, "cannot mix --continue-at with --form"); result = CURLE_FAILED_INIT; } else { result = tool2curlmime(curl, config->mimeroot, &config->mimepost); if(!result) my_setopt_mimepost(curl, CURLOPT_MIMEPOST, config->mimepost); } break; default: break; } if(result) return result; /* new in libcurl 7.81.0 */ if(config->mime_options) my_setopt(curl, CURLOPT_MIME_OPTIONS, config->mime_options); /* new in libcurl 7.10.6 (default is Basic) */ if(config->authtype) my_setopt_bitmask(curl, CURLOPT_HTTPAUTH, (long)config->authtype); my_setopt_slist(curl, CURLOPT_HTTPHEADER, config->headers); if(proto_http || proto_rtsp) { my_setopt_str(curl, CURLOPT_REFERER, config->referer); my_setopt_str(curl, CURLOPT_USERAGENT, config->useragent); } if(proto_http) { long postRedir = 0; my_setopt(curl, CURLOPT_FOLLOWLOCATION, config->followlocation ? 1L : 0L); my_setopt(curl, CURLOPT_UNRESTRICTED_AUTH, config->unrestricted_auth ? 1L : 0L); my_setopt_str(curl, CURLOPT_AWS_SIGV4, config->aws_sigv4); my_setopt(curl, CURLOPT_AUTOREFERER, config->autoreferer ? 1L : 0L); /* new in libcurl 7.36.0 */ if(config->proxyheaders) { my_setopt_slist(curl, CURLOPT_PROXYHEADER, config->proxyheaders); my_setopt(curl, CURLOPT_HEADEROPT, (long)CURLHEADER_SEPARATE); } /* new in libcurl 7.5 */ my_setopt(curl, CURLOPT_MAXREDIRS, config->maxredirs); if(config->httpversion) my_setopt_enum(curl, CURLOPT_HTTP_VERSION, config->httpversion); else if(feature_http2) my_setopt_enum(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); /* curl 7.19.1 (the 301 version existed in 7.18.2), 303 was added in 7.26.0 */ if(config->post301) postRedir |= CURL_REDIR_POST_301; if(config->post302) postRedir |= CURL_REDIR_POST_302; if(config->post303) postRedir |= CURL_REDIR_POST_303; my_setopt(curl, CURLOPT_POSTREDIR, postRedir); /* new in libcurl 7.21.6 */ if(config->encoding) my_setopt_str(curl, CURLOPT_ACCEPT_ENCODING, ""); /* new in libcurl 7.21.6 */ if(config->tr_encoding) my_setopt(curl, CURLOPT_TRANSFER_ENCODING, 1L); /* new in libcurl 7.64.0 */ my_setopt(curl, CURLOPT_HTTP09_ALLOWED, config->http09_allowed ? 1L : 0L); if(result) { errorf(global, "HTTP/0.9 is not supported in this build"); return result; } } /* (proto_http) */ if(proto_ftp) my_setopt_str(curl, CURLOPT_FTPPORT, config->ftpport); my_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, config->low_speed_limit); my_setopt(curl, CURLOPT_LOW_SPEED_TIME, config->low_speed_time); my_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE, config->sendpersecond); my_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, config->recvpersecond); if(config->use_resume) my_setopt(curl, CURLOPT_RESUME_FROM_LARGE, config->resume_from); else my_setopt(curl, CURLOPT_RESUME_FROM_LARGE, CURL_OFF_T_C(0)); my_setopt_str(curl, CURLOPT_KEYPASSWD, config->key_passwd); my_setopt_str(curl, CURLOPT_PROXY_KEYPASSWD, config->proxy_key_passwd); if(use_proto == proto_scp || use_proto == proto_sftp) { /* SSH and SSL private key uses same command-line option */ /* new in libcurl 7.16.1 */ my_setopt_str(curl, CURLOPT_SSH_PRIVATE_KEYFILE, config->key); /* new in libcurl 7.16.1 */ my_setopt_str(curl, CURLOPT_SSH_PUBLIC_KEYFILE, config->pubkey); /* new in libcurl 7.17.1: SSH host key md5 checking allows us to fail if we are not talking to who we think we should */ my_setopt_str(curl, CURLOPT_SSH_HOST_PUBLIC_KEY_MD5, config->hostpubmd5); /* new in libcurl 7.80.0: SSH host key sha256 checking allows us to fail if we are not talking to who we think we should */ my_setopt_str(curl, CURLOPT_SSH_HOST_PUBLIC_KEY_SHA256, config->hostpubsha256); /* new in libcurl 7.56.0 */ if(config->ssh_compression) my_setopt(curl, CURLOPT_SSH_COMPRESSION, 1L); if(!config->insecure_ok) { char *known = findfile(".ssh/known_hosts", FALSE); if(known) { /* new in curl 7.19.6 */ result = res_setopt_str(curl, CURLOPT_SSH_KNOWNHOSTS, known); curl_free(known); if(result == CURLE_UNKNOWN_OPTION) /* libssh2 version older than 1.1.1 */ result = CURLE_OK; if(result) return result; } else warnf(global, "Couldn't find a known_hosts file"); } } if(config->cacert) my_setopt_str(curl, CURLOPT_CAINFO, config->cacert); if(config->proxy_cacert) my_setopt_str(curl, CURLOPT_PROXY_CAINFO, config->proxy_cacert); if(config->capath) { result = res_setopt_str(curl, CURLOPT_CAPATH, config->capath); if(result == CURLE_NOT_BUILT_IN) { warnf(global, "ignoring %s, not supported by libcurl with %s", capath_from_env ? "SSL_CERT_DIR environment variable" : "--capath", ssl_backend()); } else if(result) return result; } /* For the time being if --proxy-capath is not set then we use the --capath value for it, if any. See #1257 */ if(config->proxy_capath || config->capath) { result = res_setopt_str(curl, CURLOPT_PROXY_CAPATH, (config->proxy_capath ? config->proxy_capath : config->capath)); if((result == CURLE_NOT_BUILT_IN) || (result == CURLE_UNKNOWN_OPTION)) { if(config->proxy_capath) { warnf(global, "ignoring %s, not supported by libcurl with %s", config->proxy_capath ? "--proxy-capath" : "--capath", ssl_backend()); } } else if(result) return result; } #ifdef CURL_CA_EMBED if(!config->cacert && !config->capath) { struct curl_blob blob; blob.data = (void *)curl_ca_embed; blob.len = strlen((const char *)curl_ca_embed); blob.flags = CURL_BLOB_NOCOPY; notef(config->global, "Using embedded CA bundle (%zu bytes)", blob.len); result = curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &blob); if(result == CURLE_NOT_BUILT_IN) { warnf(global, "ignoring %s, not supported by libcurl with %s", "embedded CA bundle", ssl_backend()); } } if(!config->proxy_cacert && !config->proxy_capath) { struct curl_blob blob; blob.data = (void *)curl_ca_embed; blob.len = strlen((const char *)curl_ca_embed); blob.flags = CURL_BLOB_NOCOPY; notef(config->global, "Using embedded CA bundle, for proxies (%zu bytes)", blob.len); result = curl_easy_setopt(curl, CURLOPT_PROXY_CAINFO_BLOB, &blob); if(result == CURLE_NOT_BUILT_IN) { warnf(global, "ignoring %s, not supported by libcurl with %s", "embedded CA bundle", ssl_backend()); } } #endif if(config->crlfile) my_setopt_str(curl, CURLOPT_CRLFILE, config->crlfile); if(config->proxy_crlfile) my_setopt_str(curl, CURLOPT_PROXY_CRLFILE, config->proxy_crlfile); else if(config->crlfile) /* CURLOPT_PROXY_CRLFILE default is crlfile */ my_setopt_str(curl, CURLOPT_PROXY_CRLFILE, config->crlfile); if(config->pinnedpubkey) { result = res_setopt_str(curl, CURLOPT_PINNEDPUBLICKEY, config->pinnedpubkey); if(result == CURLE_NOT_BUILT_IN) warnf(global, "ignoring %s, not supported by libcurl with %s", "--pinnedpubkey", ssl_backend()); } if(config->proxy_pinnedpubkey) { result = res_setopt_str(curl, CURLOPT_PROXY_PINNEDPUBLICKEY, config->proxy_pinnedpubkey); if(result == CURLE_NOT_BUILT_IN) warnf(global, "ignoring %s, not supported by libcurl with %s", "--proxy-pinnedpubkey", ssl_backend()); } if(config->ssl_ec_curves) my_setopt_str(curl, CURLOPT_SSL_EC_CURVES, config->ssl_ec_curves); if(config->writeout) my_setopt_str(curl, CURLOPT_CERTINFO, 1L); if(feature_ssl) { my_setopt_str(curl, CURLOPT_SSLCERT, config->cert); my_setopt_str(curl, CURLOPT_PROXY_SSLCERT, config->proxy_cert); my_setopt_str(curl, CURLOPT_SSLCERTTYPE, config->cert_type); my_setopt_str(curl, CURLOPT_PROXY_SSLCERTTYPE, config->proxy_cert_type); my_setopt_str(curl, CURLOPT_SSLKEY, config->key); my_setopt_str(curl, CURLOPT_PROXY_SSLKEY, config->proxy_key); my_setopt_str(curl, CURLOPT_SSLKEYTYPE, config->key_type); my_setopt_str(curl, CURLOPT_PROXY_SSLKEYTYPE, config->proxy_key_type); /* libcurl default is strict verifyhost -> 1L, verifypeer -> 1L */ if(config->insecure_ok) { my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); } if(config->doh_insecure_ok) { my_setopt(curl, CURLOPT_DOH_SSL_VERIFYPEER, 0L); my_setopt(curl, CURLOPT_DOH_SSL_VERIFYHOST, 0L); } if(config->proxy_insecure_ok) { my_setopt(curl, CURLOPT_PROXY_SSL_VERIFYPEER, 0L); my_setopt(curl, CURLOPT_PROXY_SSL_VERIFYHOST, 0L); } if(config->verifystatus) my_setopt(curl, CURLOPT_SSL_VERIFYSTATUS, 1L); if(config->doh_verifystatus) my_setopt(curl, CURLOPT_DOH_SSL_VERIFYSTATUS, 1L); if(config->falsestart) my_setopt(curl, CURLOPT_SSL_FALSESTART, 1L); my_setopt_SSLVERSION(curl, CURLOPT_SSLVERSION, config->ssl_version | config->ssl_version_max); if(config->proxy) my_setopt_SSLVERSION(curl, CURLOPT_PROXY_SSLVERSION, config->proxy_ssl_version); { long mask = (config->ssl_allow_beast ? CURLSSLOPT_ALLOW_BEAST : 0) | (config->ssl_allow_earlydata ? CURLSSLOPT_EARLYDATA : 0) | (config->ssl_no_revoke ? CURLSSLOPT_NO_REVOKE : 0) | (config->ssl_revoke_best_effort ? CURLSSLOPT_REVOKE_BEST_EFFORT : 0) | (config->native_ca_store ? CURLSSLOPT_NATIVE_CA : 0) | (config->ssl_auto_client_cert ? CURLSSLOPT_AUTO_CLIENT_CERT : 0); if(mask) my_setopt_bitmask(curl, CURLOPT_SSL_OPTIONS, mask); } { long mask = (config->proxy_ssl_allow_beast ? CURLSSLOPT_ALLOW_BEAST : 0) | (config->proxy_ssl_auto_client_cert ? CURLSSLOPT_AUTO_CLIENT_CERT : 0) | (config->proxy_native_ca_store ? CURLSSLOPT_NATIVE_CA : 0); if(mask) my_setopt_bitmask(curl, CURLOPT_PROXY_SSL_OPTIONS, mask); } } if(config->path_as_is) my_setopt(curl, CURLOPT_PATH_AS_IS, 1L); if(config->no_body || config->remote_time) { /* no body or use remote time */ my_setopt(curl, CURLOPT_FILETIME, 1L); } my_setopt(curl, CURLOPT_CRLF, config->crlf ? 1L : 0L); my_setopt_slist(curl, CURLOPT_QUOTE, config->quote); my_setopt_slist(curl, CURLOPT_POSTQUOTE, config->postquote); my_setopt_slist(curl, CURLOPT_PREQUOTE, config->prequote); if(config->cookies) { struct curlx_dynbuf cookies; struct curl_slist *cl; /* The maximum size needs to match MAX_NAME in cookie.h */ #define MAX_COOKIE_LINE 8200 curlx_dyn_init(&cookies, MAX_COOKIE_LINE); for(cl = config->cookies; cl; cl = cl->next) { if(cl == config->cookies) result = curlx_dyn_addf(&cookies, "%s", cl->data); else result = curlx_dyn_addf(&cookies, ";%s", cl->data); if(result) { warnf(global, "skipped provided cookie, the cookie header " "would go over %u bytes", MAX_COOKIE_LINE); return result; } } my_setopt_str(curl, CURLOPT_COOKIE, curlx_dyn_ptr(&cookies)); curlx_dyn_free(&cookies); } if(config->cookiefiles) { struct curl_slist *cfl; for(cfl = config->cookiefiles; cfl; cfl = cfl->next) my_setopt_str(curl, CURLOPT_COOKIEFILE, cfl->data); } /* new in libcurl 7.9 */ if(config->cookiejar) my_setopt_str(curl, CURLOPT_COOKIEJAR, config->cookiejar); /* new in libcurl 7.9.7 */ my_setopt(curl, CURLOPT_COOKIESESSION, config->cookiesession ? 1L : 0L); my_setopt_enum(curl, CURLOPT_TIMECONDITION, (long)config->timecond); my_setopt(curl, CURLOPT_TIMEVALUE_LARGE, config->condtime); my_setopt_str(curl, CURLOPT_CUSTOMREQUEST, config->customrequest); customrequest_helper(config, config->httpreq, config->customrequest); my_setopt(curl, CURLOPT_STDERR, tool_stderr); /* three new ones in libcurl 7.3: */ my_setopt_str(curl, CURLOPT_INTERFACE, config->iface); my_setopt_str(curl, CURLOPT_KRBLEVEL, config->krblevel); progressbarinit(&per->progressbar, config); if((global->progressmode == CURL_PROGRESS_BAR) && !global->noprogress && !global->silent) { /* we want the alternative style, then we have to implement it ourselves! */ my_setopt(curl, CURLOPT_XFERINFOFUNCTION, tool_progress_cb); my_setopt(curl, CURLOPT_XFERINFODATA, per); } else if(per->uploadfile && !strcmp(per->uploadfile, ".")) { /* when reading from stdin in non-blocking mode, we use the progress function to unpause a busy read */ my_setopt(curl, CURLOPT_NOPROGRESS, 0L); my_setopt(curl, CURLOPT_XFERINFOFUNCTION, tool_readbusy_cb); my_setopt(curl, CURLOPT_XFERINFODATA, per); } /* new in libcurl 7.24.0: */ if(config->dns_servers) my_setopt_str(curl, CURLOPT_DNS_SERVERS, config->dns_servers); /* new in libcurl 7.33.0: */ if(config->dns_interface) my_setopt_str(curl, CURLOPT_DNS_INTERFACE, config->dns_interface); if(config->dns_ipv4_addr) my_setopt_str(curl, CURLOPT_DNS_LOCAL_IP4, config->dns_ipv4_addr); if(config->dns_ipv6_addr) my_setopt_str(curl, CURLOPT_DNS_LOCAL_IP6, config->dns_ipv6_addr); /* new in libcurl 7.6.2: */ my_setopt_slist(curl, CURLOPT_TELNETOPTIONS, config->telnet_options); /* new in libcurl 7.7: */ my_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, config->connecttimeout_ms); if(config->doh_url) my_setopt_str(curl, CURLOPT_DOH_URL, config->doh_url); if(config->cipher_list) { result = res_setopt_str(curl, CURLOPT_SSL_CIPHER_LIST, config->cipher_list); if(result == CURLE_NOT_BUILT_IN) warnf(global, "ignoring %s, not supported by libcurl with %s", "--ciphers", ssl_backend()); } if(config->proxy_cipher_list) { result = res_setopt_str(curl, CURLOPT_PROXY_SSL_CIPHER_LIST, config->proxy_cipher_list); if(result == CURLE_NOT_BUILT_IN) warnf(global, "ignoring %s, not supported by libcurl with %s", "--proxy-ciphers", ssl_backend()); } if(config->cipher13_list) { result = res_setopt_str(curl, CURLOPT_TLS13_CIPHERS, config->cipher13_list); if(result == CURLE_NOT_BUILT_IN) warnf(global, "ignoring %s, not supported by libcurl with %s", "--tls13-ciphers", ssl_backend()); } if(config->proxy_cipher13_list) { result = res_setopt_str(curl, CURLOPT_PROXY_TLS13_CIPHERS, config->proxy_cipher13_list); if(result == CURLE_NOT_BUILT_IN) warnf(global, "ignoring %s, not supported by libcurl with %s", "--proxy-tls13-ciphers", ssl_backend()); } /* new in libcurl 7.9.2: */ if(config->disable_epsv) /* disable it */ my_setopt(curl, CURLOPT_FTP_USE_EPSV, 0L); /* new in libcurl 7.10.5 */ if(config->disable_eprt) /* disable it */ my_setopt(curl, CURLOPT_FTP_USE_EPRT, 0L); if(global->tracetype != TRACE_NONE) { my_setopt(curl, CURLOPT_DEBUGFUNCTION, tool_debug_cb); my_setopt(curl, CURLOPT_DEBUGDATA, config); my_setopt(curl, CURLOPT_VERBOSE, 1L); } /* new in curl 7.9.3 */ if(config->engine) { result = res_setopt_str(curl, CURLOPT_SSLENGINE, config->engine); if(result) return result; } /* new in curl 7.10.7, extended in 7.19.4. Modified to use CREATE_DIR_RETRY in 7.49.0 */ my_setopt(curl, CURLOPT_FTP_CREATE_MISSING_DIRS, (long)(config->ftp_create_dirs ? CURLFTP_CREATE_DIR_RETRY : CURLFTP_CREATE_DIR_NONE)); /* new in curl 7.10.8 */ if(config->max_filesize) my_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, config->max_filesize); my_setopt(curl, CURLOPT_IPRESOLVE, config->ip_version); /* new in curl 7.15.5 */ if(config->ftp_ssl_reqd) my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_ALL); /* new in curl 7.11.0 */ else if(config->ftp_ssl) my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_TRY); /* new in curl 7.16.0 */ else if(config->ftp_ssl_control) my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_CONTROL); /* new in curl 7.16.1 */ if(config->ftp_ssl_ccc) my_setopt_enum(curl, CURLOPT_FTP_SSL_CCC, (long)config->ftp_ssl_ccc_mode); /* new in curl 7.19.4 */ if(config->socks5_gssapi_nec) my_setopt_str(curl, CURLOPT_SOCKS5_GSSAPI_NEC, 1L); /* new in curl 7.55.0 */ if(config->socks5_auth) my_setopt_bitmask(curl, CURLOPT_SOCKS5_AUTH, (long)config->socks5_auth); /* new in curl 7.43.0 */ if(config->proxy_service_name) my_setopt_str(curl, CURLOPT_PROXY_SERVICE_NAME, config->proxy_service_name); /* new in curl 7.43.0 */ if(config->service_name) my_setopt_str(curl, CURLOPT_SERVICE_NAME, config->service_name); /* curl 7.13.0 */ my_setopt_str(curl, CURLOPT_FTP_ACCOUNT, config->ftp_account); my_setopt(curl, CURLOPT_IGNORE_CONTENT_LENGTH, config->ignorecl ? 1L : 0L); /* curl 7.14.2 */ my_setopt(curl, CURLOPT_FTP_SKIP_PASV_IP, config->ftp_skip_ip ? 1L : 0L); /* curl 7.15.1 */ if(proto_ftp) my_setopt(curl, CURLOPT_FTP_FILEMETHOD, (long)config->ftp_filemethod); /* curl 7.15.2 */ if(config->localport) { my_setopt(curl, CURLOPT_LOCALPORT, config->localport); my_setopt_str(curl, CURLOPT_LOCALPORTRANGE, config->localportrange); } /* curl 7.15.5 */ my_setopt_str(curl, CURLOPT_FTP_ALTERNATIVE_TO_USER, config->ftp_alternative_to_user); /* curl 7.16.0 */ if(config->disable_sessionid) /* disable it */ my_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE, 0L); /* curl 7.16.2 */ if(config->raw) { my_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 0L); my_setopt(curl, CURLOPT_HTTP_TRANSFER_DECODING, 0L); } /* curl 7.17.1 */ if(!config->nokeepalive) { my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L); if(config->alivetime) { my_setopt(curl, CURLOPT_TCP_KEEPIDLE, config->alivetime); my_setopt(curl, CURLOPT_TCP_KEEPINTVL, config->alivetime); } if(config->alivecnt) my_setopt(curl, CURLOPT_TCP_KEEPCNT, config->alivecnt); } else my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 0L); /* curl 7.20.0 */ if(config->tftp_blksize && proto_tftp) my_setopt(curl, CURLOPT_TFTP_BLKSIZE, config->tftp_blksize); if(config->mail_from) my_setopt_str(curl, CURLOPT_MAIL_FROM, config->mail_from); if(config->mail_rcpt) my_setopt_slist(curl, CURLOPT_MAIL_RCPT, config->mail_rcpt); /* curl 7.69.x */ my_setopt(curl, CURLOPT_MAIL_RCPT_ALLOWFAILS, config->mail_rcpt_allowfails ? 1L : 0L); /* curl 7.20.x */ if(config->ftp_pret) my_setopt(curl, CURLOPT_FTP_USE_PRET, 1L); if(config->create_file_mode) my_setopt(curl, CURLOPT_NEW_FILE_PERMS, config->create_file_mode); if(config->proto_present) my_setopt_str(curl, CURLOPT_PROTOCOLS_STR, config->proto_str); if(config->proto_redir_present) my_setopt_str(curl, CURLOPT_REDIR_PROTOCOLS_STR, config->proto_redir_str); my_setopt(curl, CURLOPT_HEADERFUNCTION, tool_header_cb); my_setopt(curl, CURLOPT_HEADERDATA, per); if(config->resolve) /* new in 7.21.3 */ my_setopt_slist(curl, CURLOPT_RESOLVE, config->resolve); if(config->connect_to) /* new in 7.49.0 */ my_setopt_slist(curl, CURLOPT_CONNECT_TO, config->connect_to); /* new in 7.21.4 */ if(feature_tls_srp) { if(config->tls_username) my_setopt_str(curl, CURLOPT_TLSAUTH_USERNAME, config->tls_username); if(config->tls_password) my_setopt_str(curl, CURLOPT_TLSAUTH_PASSWORD, config->tls_password); if(config->tls_authtype) my_setopt_str(curl, CURLOPT_TLSAUTH_TYPE, config->tls_authtype); if(config->proxy_tls_username) my_setopt_str(curl, CURLOPT_PROXY_TLSAUTH_USERNAME, config->proxy_tls_username); if(config->proxy_tls_password) my_setopt_str(curl, CURLOPT_PROXY_TLSAUTH_PASSWORD, config->proxy_tls_password); if(config->proxy_tls_authtype) my_setopt_str(curl, CURLOPT_PROXY_TLSAUTH_TYPE, config->proxy_tls_authtype); } /* new in 7.22.0 */ if(config->gssapi_delegation) my_setopt_str(curl, CURLOPT_GSSAPI_DELEGATION, config->gssapi_delegation); if(config->mail_auth) my_setopt_str(curl, CURLOPT_MAIL_AUTH, config->mail_auth); /* new in 7.66.0 */ if(config->sasl_authzid) my_setopt_str(curl, CURLOPT_SASL_AUTHZID, config->sasl_authzid); /* new in 7.31.0 */ if(config->sasl_ir) my_setopt(curl, CURLOPT_SASL_IR, 1L); if(config->noalpn) { my_setopt(curl, CURLOPT_SSL_ENABLE_ALPN, 0L); } /* new in 7.40.0, abstract support added in 7.53.0 */ if(config->unix_socket_path) { if(config->abstract_unix_socket) { my_setopt_str(curl, CURLOPT_ABSTRACT_UNIX_SOCKET, config->unix_socket_path); } else { my_setopt_str(curl, CURLOPT_UNIX_SOCKET_PATH, config->unix_socket_path); } } /* new in 7.45.0 */ if(config->proto_default) my_setopt_str(curl, CURLOPT_DEFAULT_PROTOCOL, config->proto_default); /* new in 7.47.0 */ if(config->expect100timeout_ms > 0) my_setopt_str(curl, CURLOPT_EXPECT_100_TIMEOUT_MS, config->expect100timeout_ms); /* new in 7.48.0 */ if(config->tftp_no_options && proto_tftp) my_setopt(curl, CURLOPT_TFTP_NO_OPTIONS, 1L); /* new in 7.59.0 */ if(config->happy_eyeballs_timeout_ms != CURL_HET_DEFAULT) my_setopt(curl, CURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS, config->happy_eyeballs_timeout_ms); /* new in 7.60.0 */ if(config->haproxy_protocol) my_setopt(curl, CURLOPT_HAPROXYPROTOCOL, 1L); /* new in 8.2.0 */ if(config->haproxy_clientip) my_setopt_str(curl, CURLOPT_HAPROXY_CLIENT_IP, config->haproxy_clientip); if(config->disallow_username_in_url) my_setopt(curl, CURLOPT_DISALLOW_USERNAME_IN_URL, 1L); if(config->altsvc) my_setopt_str(curl, CURLOPT_ALTSVC, config->altsvc); if(config->hsts) my_setopt_str(curl, CURLOPT_HSTS, config->hsts); if(feature_ech) { /* only if enabled in libcurl */ if(config->ech) /* only if set (optional) */ my_setopt_str(curl, CURLOPT_ECH, config->ech); if(config->ech_public) /* only if set (optional) */ my_setopt_str(curl, CURLOPT_ECH, config->ech_public); if(config->ech_config) /* only if set (optional) */ my_setopt_str(curl, CURLOPT_ECH, config->ech_config); } /* new in 8.9.0 */ if(config->ip_tos > 0 || config->vlan_priority > 0) { #if defined(IP_TOS) || defined(IPV6_TCLASS) || defined(SO_PRIORITY) my_setopt(curl, CURLOPT_SOCKOPTFUNCTION, sockopt_callback); my_setopt(curl, CURLOPT_SOCKOPTDATA, config); #else if(config->ip_tos > 0) { errorf(config->global, "Type of service is not supported in this build."); result = CURLE_NOT_BUILT_IN; } if(config->vlan_priority > 0) { errorf(config->global, "VLAN priority is not supported in this build."); result = CURLE_NOT_BUILT_IN; } #endif } return result; } static CURLcode append2query(struct GlobalConfig *global, struct OperationConfig *config, struct per_transfer *per, const char *q) { CURLcode result = CURLE_OK; CURLU *uh = curl_url(); if(uh) { CURLUcode uerr; uerr = curl_url_set(uh, CURLUPART_URL, per->url, CURLU_GUESS_SCHEME); if(uerr) { result = urlerr_cvt(uerr); errorf(global, "(%d) Could not parse the URL, " "failed to set query", result); config->synthetic_error = TRUE; } else { char *updated = NULL; uerr = curl_url_set(uh, CURLUPART_QUERY, q, CURLU_APPENDQUERY); if(!uerr) uerr = curl_url_get(uh, CURLUPART_URL, &updated, CURLU_GUESS_SCHEME); if(uerr) result = urlerr_cvt(uerr); else { Curl_safefree(per->url); /* free previous URL */ per->url = updated; /* use our new URL instead! */ } } curl_url_cleanup(uh); } return result; } /* create the next (singular) transfer */ static CURLcode single_transfer(struct GlobalConfig *global, struct OperationConfig *config, CURLSH *share, bool capath_from_env, bool *added, bool *skipped) { CURLcode result = CURLE_OK; struct getout *urlnode; bool orig_noprogress = global->noprogress; bool orig_isatty = global->isatty; struct State *state = &config->state; char *httpgetfields = state->httpgetfields; *skipped = *added = FALSE; /* not yet */ if(config->postfields) { if(config->use_httpget) { if(!httpgetfields) { /* Use the postfields data for an HTTP get */ httpgetfields = state->httpgetfields = config->postfields; config->postfields = NULL; if(SetHTTPrequest(config, (config->no_body ? TOOL_HTTPREQ_HEAD : TOOL_HTTPREQ_GET), &config->httpreq)) { result = CURLE_FAILED_INIT; } } } else { if(SetHTTPrequest(config, TOOL_HTTPREQ_SIMPLEPOST, &config->httpreq)) result = CURLE_FAILED_INIT; } if(result) goto fail; } if(!state->urlnode) { /* first time caller, setup things */ state->urlnode = config->url_list; state->infilenum = 1; } result = set_cert_types(config); if(result) goto fail; for(; state->urlnode; state->urlnode = urlnode->next) { static bool warn_more_options = FALSE; curl_off_t urlnum; urlnode = state->urlnode; /* urlnode->url is the full URL or NULL */ if(!urlnode->url) { /* This node has no URL. Free node data without destroying the node itself nor modifying next pointer and continue to next */ urlnode->flags = 0; state->up = 0; if(!warn_more_options) { /* only show this once */ warnf(config->global, "Got more output options than URLs"); warn_more_options = TRUE; } continue; /* next URL please */ } /* save outfile pattern before expansion */ if(urlnode->outfile && !state->outfiles) { state->outfiles = strdup(urlnode->outfile); if(!state->outfiles) { errorf(global, "out of memory"); result = CURLE_OUT_OF_MEMORY; break; } } if(!config->globoff && urlnode->infile && !state->inglob) { /* Unless explicitly shut off */ result = glob_url(&state->inglob, urlnode->infile, &state->infilenum, (!global->silent || global->showerror) ? tool_stderr : NULL); if(result) break; } if(state->up || urlnode->infile) { if(!state->uploadfile) { if(state->inglob) { result = glob_next_url(&state->uploadfile, state->inglob); if(result == CURLE_OUT_OF_MEMORY) errorf(global, "out of memory"); } else if(!state->up) { /* copy the allocated string */ state->uploadfile = urlnode->infile; urlnode->infile = NULL; } } if(result) break; } if(!state->urlnum) { if(!config->globoff) { /* Unless explicitly shut off, we expand '{...}' and '[...]' expressions and return total number of URLs in pattern set */ result = glob_url(&state->urls, urlnode->url, &state->urlnum, (!global->silent || global->showerror) ? tool_stderr : NULL); if(result) break; urlnum = state->urlnum; } else urlnum = 1; /* without globbing, this is a single URL */ } else urlnum = state->urlnum; if(state->up < state->infilenum) { struct per_transfer *per = NULL; struct OutStruct *outs; struct OutStruct *heads; struct OutStruct *etag_save; struct HdrCbData *hdrcbdata = NULL; struct OutStruct etag_first; CURL *curl; /* --etag-save */ memset(&etag_first, 0, sizeof(etag_first)); etag_save = &etag_first; etag_save->stream = stdout; /* --etag-compare */ if(config->etag_compare_file) { char *etag_from_file = NULL; char *header = NULL; ParameterError pe; /* open file for reading: */ FILE *file = fopen(config->etag_compare_file, FOPEN_READTEXT); if(!file && !config->etag_save_file) { errorf(global, "Failed to open %s", config->etag_compare_file); result = CURLE_READ_ERROR; break; } if((PARAM_OK == file2string(&etag_from_file, file)) && etag_from_file) { header = aprintf("If-None-Match: %s", etag_from_file); Curl_safefree(etag_from_file); } else header = aprintf("If-None-Match: \"\""); if(!header) { if(file) fclose(file); errorf(global, "Failed to allocate memory for custom etag header"); result = CURLE_OUT_OF_MEMORY; break; } /* add Etag from file to list of custom headers */ pe = add2list(&config->headers, header); Curl_safefree(header); if(file) fclose(file); if(pe != PARAM_OK) { result = CURLE_OUT_OF_MEMORY; break; } } if(config->etag_save_file) { /* open file for output: */ if(strcmp(config->etag_save_file, "-")) { FILE *newfile = fopen(config->etag_save_file, "ab"); if(!newfile) { warnf(global, "Failed creating file for saving etags: \"%s\". " "Skip this transfer", config->etag_save_file); Curl_safefree(state->outfiles); glob_cleanup(&state->urls); return CURLE_OK; } else { etag_save->filename = config->etag_save_file; etag_save->s_isreg = TRUE; etag_save->fopened = TRUE; etag_save->stream = newfile; } } else { /* always use binary mode for protocol header output */ set_binmode(etag_save->stream); } } curl = curl_easy_init(); if(curl) result = add_per_transfer(&per); else result = CURLE_OUT_OF_MEMORY; if(result) { curl_easy_cleanup(curl); if(etag_save->fopened) fclose(etag_save->stream); break; } per->etag_save = etag_first; /* copy the whole struct */ if(state->uploadfile) { per->uploadfile = strdup(state->uploadfile); if(!per->uploadfile) { curl_easy_cleanup(curl); result = CURLE_OUT_OF_MEMORY; break; } if(SetHTTPrequest(config, TOOL_HTTPREQ_PUT, &config->httpreq)) { Curl_safefree(per->uploadfile); curl_easy_cleanup(curl); result = CURLE_FAILED_INIT; break; } } *added = TRUE; per->config = config; per->curl = curl; per->urlnum = (unsigned int)urlnode->num; /* default headers output stream is stdout */ heads = &per->heads; heads->stream = stdout; /* Single header file for all URLs */ if(config->headerfile) { /* open file for output: */ if(!strcmp(config->headerfile, "%")) { heads->stream = stderr; /* use binary mode for protocol header output */ set_binmode(heads->stream); } else if(strcmp(config->headerfile, "-")) { FILE *newfile; /* * Since every transfer has its own file handle for dumping * the headers, we need to open it in append mode, since transfers * might finish in any order. * The first transfer just clears the file. * TODO: Consider placing the file handle inside the * OperationConfig, so that it does not need to be opened/closed * for every transfer. */ if(config->create_dirs) { result = create_dir_hierarchy(config->headerfile, global); /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */ if(result) break; } if(!per->prev || per->prev->config != config) { newfile = fopen(config->headerfile, "wb"); if(newfile) fclose(newfile); } newfile = fopen(config->headerfile, "ab"); if(!newfile) { errorf(global, "Failed to open %s", config->headerfile); result = CURLE_WRITE_ERROR; break; } else { heads->filename = config->headerfile; heads->s_isreg = TRUE; heads->fopened = TRUE; heads->stream = newfile; } } else { /* always use binary mode for protocol header output */ set_binmode(heads->stream); } } hdrcbdata = &per->hdrcbdata; outs = &per->outs; per->outfile = NULL; per->infdopen = FALSE; per->infd = STDIN_FILENO; /* default output stream is stdout */ outs->stream = stdout; if(state->urls) { result = glob_next_url(&per->url, state->urls); if(result) break; } else if(!state->li) { per->url = strdup(urlnode->url); if(!per->url) { result = CURLE_OUT_OF_MEMORY; break; } } else per->url = NULL; if(!per->url) break; if(state->outfiles) { per->outfile = strdup(state->outfiles); if(!per->outfile) { result = CURLE_OUT_OF_MEMORY; break; } } if(((urlnode->flags&GETOUT_USEREMOTE) || (per->outfile && strcmp("-", per->outfile)))) { /* * We have specified a filename to store the result in, or we have * decided we want to use the remote filename. */ if(!per->outfile) { /* extract the filename from the URL */ result = get_url_file_name(global, &per->outfile, per->url); if(result) { errorf(global, "Failed to extract a filename" " from the URL to use for storage"); break; } } else if(state->urls) { /* fill '#1' ... '#9' terms from URL pattern */ char *storefile = per->outfile; result = glob_match_url(&per->outfile, storefile, state->urls); Curl_safefree(storefile); if(result) { /* bad globbing */ warnf(global, "bad output glob"); break; } if(!*per->outfile) { warnf(global, "output glob produces empty string"); result = CURLE_WRITE_ERROR; break; } } DEBUGASSERT(per->outfile); if(config->output_dir && *config->output_dir) { char *d = aprintf("%s/%s", config->output_dir, per->outfile); if(!d) { result = CURLE_WRITE_ERROR; break; } free(per->outfile); per->outfile = d; } /* Create the directory hierarchy, if not pre-existent to a multiple file output call */ if(config->create_dirs) { result = create_dir_hierarchy(per->outfile, global); /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */ if(result) break; } if(config->skip_existing) { struct_stat fileinfo; if(!stat(per->outfile, &fileinfo)) { /* file is present */ notef(global, "skips transfer, \"%s\" exists locally", per->outfile); per->skip = TRUE; *skipped = TRUE; } } if((urlnode->flags & GETOUT_USEREMOTE) && config->content_disposition) { /* Our header callback MIGHT set the filename */ DEBUGASSERT(!outs->filename); } if(config->resume_from_current) { /* We are told to continue from where we are now. Get the size of the file as it is now and open it for append instead */ struct_stat fileinfo; /* VMS -- Danger, the filesize is only valid for stream files */ if(0 == stat(per->outfile, &fileinfo)) /* set offset to current file size: */ config->resume_from = fileinfo.st_size; else /* let offset be 0 */ config->resume_from = 0; } if(config->resume_from) { #ifdef __VMS /* open file for output, forcing VMS output format into stream mode which is needed for stat() call above to always work. */ FILE *file = fopen(outfile, "ab", "ctx=stm", "rfm=stmlf", "rat=cr", "mrs=0"); #else /* open file for output: */ FILE *file = fopen(per->outfile, "ab"); #endif if(!file) { errorf(global, "cannot open '%s'", per->outfile); result = CURLE_WRITE_ERROR; break; } outs->fopened = TRUE; outs->stream = file; outs->init = config->resume_from; } else { outs->stream = NULL; /* open when needed */ } outs->filename = per->outfile; outs->s_isreg = TRUE; } if(per->uploadfile && !stdin_upload(per->uploadfile)) { /* * We have specified a file to upload and it is not "-". */ result = add_file_name_to_url(per->curl, &per->url, per->uploadfile); if(result) break; } else if(per->uploadfile && stdin_upload(per->uploadfile)) { /* count to see if there are more than one auth bit set in the authtype field */ int authbits = 0; int bitcheck = 0; while(bitcheck < 32) { if(config->authtype & (1UL << bitcheck++)) { authbits++; if(authbits > 1) { /* more than one, we are done! */ break; } } } /* * If the user has also selected --anyauth or --proxy-anyauth * we should warn them. */ if(config->proxyanyauth || (authbits > 1)) { warnf(global, "Using --anyauth or --proxy-anyauth with upload from stdin" " involves a big risk of it not working. Use a temporary" " file or a fixed auth type instead"); } DEBUGASSERT(per->infdopen == FALSE); DEBUGASSERT(per->infd == STDIN_FILENO); set_binmode(stdin); if(!strcmp(per->uploadfile, ".")) { if(curlx_nonblock((curl_socket_t)per->infd, TRUE) < 0) warnf(global, "fcntl failed on fd=%d: %s", per->infd, strerror(errno)); } } if(per->uploadfile && config->resume_from_current) config->resume_from = -1; /* -1 will then force get-it-yourself */ if(output_expected(per->url, per->uploadfile) && outs->stream && isatty(fileno(outs->stream))) /* we send the output to a tty, therefore we switch off the progress meter */ per->noprogress = global->noprogress = global->isatty = TRUE; else { /* progress meter is per download, so restore config values */ per->noprogress = global->noprogress = orig_noprogress; global->isatty = orig_isatty; } if(httpgetfields || config->query) { result = append2query(global, config, per, httpgetfields ? httpgetfields : config->query); if(result) break; } if((!per->outfile || !strcmp(per->outfile, "-")) && !config->use_ascii) { /* We get the output to stdout and we have not got the ASCII/text flag, then set stdout to be binary */ set_binmode(stdout); } /* explicitly passed to stdout means okaying binary gunk */ config->terminal_binary_ok = (per->outfile && !strcmp(per->outfile, "-")); if(config->content_disposition && (urlnode->flags & GETOUT_USEREMOTE)) hdrcbdata->honor_cd_filename = TRUE; else hdrcbdata->honor_cd_filename = FALSE; hdrcbdata->outs = outs; hdrcbdata->heads = heads; hdrcbdata->etag_save = etag_save; hdrcbdata->global = global; hdrcbdata->config = config; result = config2setopts(global, config, per, capath_from_env, curl, share); if(result) break; /* initialize retry vars for loop below */ per->retry_sleep_default = (config->retry_delay) ? config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */ per->retry_remaining = config->req_retry; per->retry_sleep = per->retry_sleep_default; /* ms */ per->retrystart = tvnow(); state->li++; /* Here's looping around each globbed URL */ if(state->li >= urlnum) { state->li = 0; state->urlnum = 0; /* forced reglob of URLs */ glob_cleanup(&state->urls); state->up++; Curl_safefree(state->uploadfile); /* clear it to get the next */ } } else { /* Free this URL node data without destroying the node itself nor modifying next pointer. */ urlnode->flags = 0; glob_cleanup(&state->urls); state->urlnum = 0; Curl_safefree(state->outfiles); Curl_safefree(state->uploadfile); /* Free list of globbed upload files */ glob_cleanup(&state->inglob); state->up = 0; continue; } break; } Curl_safefree(state->outfiles); fail: if(!*added || result) { *added = FALSE; single_transfer_cleanup(config); } return result; } static long all_added; /* number of easy handles currently added */ /* * add_parallel_transfers() sets 'morep' to TRUE if there are more transfers * to add even after this call returns. sets 'addedp' to TRUE if one or more * transfers were added. */ static CURLcode add_parallel_transfers(struct GlobalConfig *global, CURLM *multi, CURLSH *share, bool *morep, bool *addedp) { struct per_transfer *per; CURLcode result = CURLE_OK; CURLMcode mcode; bool sleeping = FALSE; char *errorbuf; *addedp = FALSE; *morep = FALSE; if(all_pers < (global->parallel_max*2)) { bool skipped = FALSE; do { result = create_transfer(global, share, addedp, &skipped); if(result) return result; } while(skipped); } for(per = transfers; per && (all_added < global->parallel_max); per = per->next) { if(per->added || per->skip) /* already added or to be skipped */ continue; if(per->startat && (time(NULL) < per->startat)) { /* this is still delaying */ sleeping = TRUE; continue; } per->added = TRUE; result = pre_transfer(global, per); if(result) return result; errorbuf = malloc(CURL_ERROR_SIZE); if(!errorbuf) return CURLE_OUT_OF_MEMORY; /* parallel connect means that we do not set PIPEWAIT since pipewait will make libcurl prefer multiplexing */ (void)curl_easy_setopt(per->curl, CURLOPT_PIPEWAIT, global->parallel_connect ? 0L : 1L); (void)curl_easy_setopt(per->curl, CURLOPT_PRIVATE, per); (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_cb); (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFODATA, per); (void)curl_easy_setopt(per->curl, CURLOPT_NOPROGRESS, 0L); #ifdef DEBUGBUILD if(getenv("CURL_FORBID_REUSE")) (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L); #endif mcode = curl_multi_add_handle(multi, per->curl); if(mcode) { DEBUGASSERT(mcode == CURLM_OUT_OF_MEMORY); result = CURLE_OUT_OF_MEMORY; } if(!result) { bool getadded = FALSE; bool skipped = FALSE; do { result = create_transfer(global, share, &getadded, &skipped); if(result) break; } while(skipped); } if(result) { free(errorbuf); return result; } errorbuf[0] = 0; (void)curl_easy_setopt(per->curl, CURLOPT_ERRORBUFFER, errorbuf); per->errorbuffer = errorbuf; per->added = TRUE; all_added++; *addedp = TRUE; } *morep = (per || sleeping); return CURLE_OK; } struct parastate { struct GlobalConfig *global; CURLM *multi; CURLSH *share; CURLMcode mcode; CURLcode result; int still_running; struct timeval start; bool more_transfers; bool added_transfers; /* wrapitup is set TRUE after a critical error occurs to end all transfers */ bool wrapitup; /* wrapitup_processed is set TRUE after the per transfer abort flag is set */ bool wrapitup_processed; time_t tick; }; #if defined(DEBUGBUILD) && defined(USE_LIBUV) #define DEBUG_UV 0 /* object to pass to the callbacks */ struct datauv { uv_timer_t timeout; uv_loop_t *loop; struct parastate *s; }; struct contextuv { uv_poll_t poll_handle; curl_socket_t sockfd; struct datauv *uv; }; static CURLcode check_finished(struct parastate *s); static void check_multi_info(struct datauv *uv) { CURLcode result; result = check_finished(uv->s); if(result && !uv->s->result) uv->s->result = result; if(uv->s->more_transfers) { result = add_parallel_transfers(uv->s->global, uv->s->multi, uv->s->share, &uv->s->more_transfers, &uv->s->added_transfers); if(result && !uv->s->result) uv->s->result = result; if(result) uv_stop(uv->loop); } } /* callback from libuv on socket activity */ static void on_uv_socket(uv_poll_t *req, int status, int events) { int flags = 0; struct contextuv *c = (struct contextuv *) req->data; (void)status; if(events & UV_READABLE) flags |= CURL_CSELECT_IN; if(events & UV_WRITABLE) flags |= CURL_CSELECT_OUT; curl_multi_socket_action(c->uv->s->multi, c->sockfd, flags, &c->uv->s->still_running); } /* callback from libuv when timeout expires */ static void on_uv_timeout(uv_timer_t *req) { struct datauv *uv = (struct datauv *) req->data; #if DEBUG_UV fprintf(tool_stderr, "parallel_event: on_uv_timeout\n"); #endif if(uv && uv->s) { curl_multi_socket_action(uv->s->multi, CURL_SOCKET_TIMEOUT, 0, &uv->s->still_running); check_multi_info(uv); } } /* callback from libcurl to update the timeout expiry */ static int cb_timeout(CURLM *multi, long timeout_ms, struct datauv *uv) { (void)multi; #if DEBUG_UV fprintf(tool_stderr, "parallel_event: cb_timeout=%ld\n", timeout_ms); #endif if(timeout_ms < 0) uv_timer_stop(&uv->timeout); else { if(timeout_ms == 0) timeout_ms = 1; /* 0 means call curl_multi_socket_action asap but NOT within the callback itself */ uv_timer_start(&uv->timeout, on_uv_timeout, timeout_ms, 0); /* do not repeat */ } return 0; } static struct contextuv *create_context(curl_socket_t sockfd, struct datauv *uv) { struct contextuv *c; c = (struct contextuv *) malloc(sizeof(*c)); c->sockfd = sockfd; c->uv = uv; uv_poll_init_socket(uv->loop, &c->poll_handle, sockfd); c->poll_handle.data = c; return c; } static void close_cb(uv_handle_t *handle) { struct contextuv *c = (struct contextuv *) handle->data; free(c); } static void destroy_context(struct contextuv *c) { uv_close((uv_handle_t *) &c->poll_handle, close_cb); } /* callback from libcurl to update socket activity to wait for */ static int cb_socket(CURL *easy, curl_socket_t s, int action, struct datauv *uv, void *socketp) { struct contextuv *c; int events = 0; (void)easy; switch(action) { case CURL_POLL_IN: case CURL_POLL_OUT: case CURL_POLL_INOUT: c = socketp ? (struct contextuv *) socketp : create_context(s, uv); curl_multi_assign(uv->s->multi, s, c); if(action != CURL_POLL_IN) events |= UV_WRITABLE; if(action != CURL_POLL_OUT) events |= UV_READABLE; uv_poll_start(&c->poll_handle, events, on_uv_socket); break; case CURL_POLL_REMOVE: if(socketp) { c = (struct contextuv *)socketp; uv_poll_stop(&c->poll_handle); destroy_context(c); curl_multi_assign(uv->s->multi, s, NULL); /* check if we can do more now */ check_multi_info(uv); } break; default: abort(); } return 0; } static CURLcode parallel_event(struct parastate *s) { CURLcode result = CURLE_OK; struct datauv uv = { 0 }; s->result = CURLE_OK; uv.s = s; uv.loop = uv_default_loop(); uv_timer_init(uv.loop, &uv.timeout); uv.timeout.data = &uv; /* setup event callbacks */ curl_multi_setopt(s->multi, CURLMOPT_SOCKETFUNCTION, cb_socket); curl_multi_setopt(s->multi, CURLMOPT_SOCKETDATA, &uv); curl_multi_setopt(s->multi, CURLMOPT_TIMERFUNCTION, cb_timeout); curl_multi_setopt(s->multi, CURLMOPT_TIMERDATA, &uv); /* kickstart the thing */ curl_multi_socket_action(s->multi, CURL_SOCKET_TIMEOUT, 0, &s->still_running); while(!s->mcode && (s->still_running || s->more_transfers)) { #if DEBUG_UV fprintf(tool_stderr, "parallel_event: uv_run(), mcode=%d, %d running, " "%d more\n", s->mcode, uv.s->still_running, s->more_transfers); #endif uv_run(uv.loop, UV_RUN_DEFAULT); #if DEBUG_UV fprintf(tool_stderr, "parallel_event: uv_run() returned\n"); #endif result = check_finished(s); if(result && !s->result) s->result = result; /* early exit called */ if(s->wrapitup) { if(s->still_running && !s->wrapitup_processed) { struct per_transfer *per; for(per = transfers; per; per = per->next) { if(per->added) per->abort = TRUE; } s->wrapitup_processed = TRUE; } break; } if(s->more_transfers) { result = add_parallel_transfers(s->global, s->multi, s->share, &s->more_transfers, &s->added_transfers); if(result && !s->result) s->result = result; } } #if DEBUG_UV fprintf(tool_stderr, "DONE parallel_event -> %d, mcode=%d, %d running, " "%d more\n", s->result, s->mcode, uv.s->still_running, s->more_transfers); #endif return s->result; } #endif static CURLcode check_finished(struct parastate *s) { CURLcode result = CURLE_OK; int rc; CURLMsg *msg; bool checkmore = FALSE; struct GlobalConfig *global = s->global; progress_meter(global, &s->start, FALSE); do { msg = curl_multi_info_read(s->multi, &rc); if(msg) { bool retry; long delay; struct per_transfer *ended; CURL *easy = msg->easy_handle; CURLcode tres = msg->data.result; curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void *)&ended); curl_multi_remove_handle(s->multi, easy); if(ended->abort && (tres == CURLE_ABORTED_BY_CALLBACK) && ended->errorbuffer) { msnprintf(ended->errorbuffer, CURL_ERROR_SIZE, "Transfer aborted due to critical error " "in another transfer"); } tres = post_per_transfer(global, ended, tres, &retry, &delay); progress_finalize(ended); /* before it goes away */ all_added--; /* one fewer added */ checkmore = TRUE; if(retry) { ended->added = FALSE; /* add it again */ /* we delay retries in full integer seconds only */ ended->startat = delay ? time(NULL) + delay/1000 : 0; } else { /* result receives this transfer's error unless the transfer was marked for abort due to a critical error in another transfer */ if(tres && (!ended->abort || !result)) result = tres; if(is_fatal_error(result) || (result && global->fail_early)) s->wrapitup = TRUE; (void)del_per_transfer(ended); } } } while(msg); if(!s->wrapitup) { if(!checkmore) { time_t tock = time(NULL); if(s->tick != tock) { checkmore = TRUE; s->tick = tock; } } if(checkmore) { /* one or more transfers completed, add more! */ CURLcode tres = add_parallel_transfers(global, s->multi, s->share, &s->more_transfers, &s->added_transfers); if(tres) result = tres; if(s->added_transfers) /* we added new ones, make sure the loop does not exit yet */ s->still_running = 1; } if(is_fatal_error(result) || (result && global->fail_early)) s->wrapitup = TRUE; } return result; } static CURLcode parallel_transfers(struct GlobalConfig *global, CURLSH *share) { CURLcode result; struct parastate p; struct parastate *s = &p; s->share = share; s->mcode = CURLM_OK; s->result = CURLE_OK; s->still_running = 1; s->start = tvnow(); s->wrapitup = FALSE; s->wrapitup_processed = FALSE; s->tick = time(NULL); s->global = global; s->multi = curl_multi_init(); if(!s->multi) return CURLE_OUT_OF_MEMORY; result = add_parallel_transfers(global, s->multi, s->share, &s->more_transfers, &s->added_transfers); if(result) { curl_multi_cleanup(s->multi); return result; } #ifdef DEBUGBUILD if(global->test_event_based) #ifdef USE_LIBUV result = parallel_event(s); #else errorf(global, "Testing --parallel event-based requires libuv"); #endif else #endif if(all_added) { while(!s->mcode && (s->still_running || s->more_transfers)) { /* If stopping prematurely (eg due to a --fail-early condition) then signal that any transfers in the multi should abort (via progress callback). */ if(s->wrapitup) { if(!s->still_running) break; if(!s->wrapitup_processed) { struct per_transfer *per; for(per = transfers; per; per = per->next) { if(per->added) per->abort = TRUE; } s->wrapitup_processed = TRUE; } } s->mcode = curl_multi_poll(s->multi, NULL, 0, 1000, NULL); if(!s->mcode) s->mcode = curl_multi_perform(s->multi, &s->still_running); if(!s->mcode) result = check_finished(s); } (void)progress_meter(global, &s->start, TRUE); } /* Make sure to return some kind of error if there was a multi problem */ if(s->mcode) { result = (s->mcode == CURLM_OUT_OF_MEMORY) ? CURLE_OUT_OF_MEMORY : /* The other multi errors should never happen, so return something suitably generic */ CURLE_BAD_FUNCTION_ARGUMENT; } curl_multi_cleanup(s->multi); return result; } static CURLcode serial_transfers(struct GlobalConfig *global, CURLSH *share) { CURLcode returncode = CURLE_OK; CURLcode result = CURLE_OK; struct per_transfer *per; bool added = FALSE; bool skipped = FALSE; result = create_transfer(global, share, &added, &skipped); if(result) return result; if(!added) { errorf(global, "no transfer performed"); return CURLE_READ_ERROR; } for(per = transfers; per;) { bool retry; long delay_ms; bool bailout = FALSE; struct timeval start; start = tvnow(); if(!per->skip) { result = pre_transfer(global, per); if(result) break; if(global->libcurl) { result = easysrc_perform(); if(result) break; } #ifdef DEBUGBUILD if(getenv("CURL_FORBID_REUSE")) (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L); if(global->test_event_based) result = curl_easy_perform_ev(per->curl); else #endif result = curl_easy_perform(per->curl); } returncode = post_per_transfer(global, per, result, &retry, &delay_ms); if(retry) { tool_go_sleep(delay_ms); continue; } /* Bail out upon critical errors or --fail-early */ if(is_fatal_error(returncode) || (returncode && global->fail_early)) bailout = TRUE; else { do { /* setup the next one just before we delete this */ result = create_transfer(global, share, &added, &skipped); if(result) { returncode = result; bailout = TRUE; break; } } while(skipped); } per = del_per_transfer(per); if(bailout) break; if(per && global->ms_per_transfer) { /* how long time did the most recent transfer take in number of milliseconds */ long milli = tvdiff(tvnow(), start); if(milli < global->ms_per_transfer) { notef(global, "Transfer took %ld ms, waits %ldms as set by --rate", milli, global->ms_per_transfer - milli); /* The transfer took less time than wanted. Wait a little. */ tool_go_sleep(global->ms_per_transfer - milli); } } } if(returncode) /* returncode errors have priority */ result = returncode; if(result) single_transfer_cleanup(global->current); return result; } static CURLcode is_using_schannel(int *using) { CURLcode result = CURLE_OK; static int using_schannel = -1; /* -1 = not checked 0 = nope 1 = yes */ if(using_schannel == -1) { CURL *curltls = curl_easy_init(); /* The TLS backend remains, so keep the info */ struct curl_tlssessioninfo *tls_backend_info = NULL; if(!curltls) result = CURLE_OUT_OF_MEMORY; else { result = curl_easy_getinfo(curltls, CURLINFO_TLS_SSL_PTR, &tls_backend_info); if(!result) using_schannel = (tls_backend_info->backend == CURLSSLBACKEND_SCHANNEL); } curl_easy_cleanup(curltls); if(result) return result; } *using = using_schannel; return result; } /* Set the CA cert locations specified in the environment. For Windows if no * environment-specified filename is found then check for CA bundle default * filename curl-ca-bundle.crt in the user's PATH. * * If Schannel is the selected SSL backend then these locations are ignored. * We allow setting CA location for Schannel only when explicitly specified by * the user via CURLOPT_CAINFO / --cacert. */ static CURLcode cacertpaths(struct OperationConfig *config) { CURLcode result = CURLE_OUT_OF_MEMORY; char *env = curl_getenv("CURL_CA_BUNDLE"); if(env) { config->cacert = strdup(env); curl_free(env); if(!config->cacert) goto fail; } else { env = curl_getenv("SSL_CERT_DIR"); if(env) { config->capath = strdup(env); curl_free(env); if(!config->capath) goto fail; } env = curl_getenv("SSL_CERT_FILE"); if(env) { config->cacert = strdup(env); curl_free(env); if(!config->cacert) goto fail; } } #ifdef _WIN32 if(!env) { #if defined(CURL_CA_SEARCH_SAFE) char *cacert = NULL; FILE *cafile = Curl_execpath("curl-ca-bundle.crt", &cacert); if(cafile) { fclose(cafile); config->cacert = strdup(cacert); } #elif !defined(CURL_WINDOWS_UWP) && !defined(CURL_DISABLE_CA_SEARCH) result = FindWin32CACert(config, TEXT("curl-ca-bundle.crt")); if(result) goto fail; #endif } #endif return CURLE_OK; fail: free(config->capath); return result; } /* setup a transfer for the given config */ static CURLcode transfer_per_config(struct GlobalConfig *global, struct OperationConfig *config, CURLSH *share, bool *added, bool *skipped) { CURLcode result = CURLE_OK; bool capath_from_env; *added = FALSE; /* Check we have a url */ if(!config->url_list || !config->url_list->url) { helpf(tool_stderr, "(%d) no URL specified", CURLE_FAILED_INIT); return CURLE_FAILED_INIT; } /* On Windows we cannot set the path to curl-ca-bundle.crt at compile time. * We look for the file in two ways: * 1: look at the environment variable CURL_CA_BUNDLE for a path * 2: if #1 is not found, use the Windows API function SearchPath() * to find it along the app's path (includes app's dir and CWD) * * We support the environment variable thing for non-Windows platforms * too. Just for the sake of it. */ capath_from_env = false; if(feature_ssl && !config->cacert && !config->capath && (!config->insecure_ok || (config->doh_url && !config->doh_insecure_ok))) { int using_schannel = -1; result = is_using_schannel(&using_schannel); /* With the addition of CAINFO support for Schannel, this search could * find a certificate bundle that was previously ignored. To maintain * backward compatibility, only perform this search if not using Schannel. */ if(!result && !using_schannel) result = cacertpaths(config); } if(!result) result = single_transfer(global, config, share, capath_from_env, added, skipped); return result; } /* * 'create_transfer' gets the details and sets up a new transfer if 'added' * returns TRUE. */ static CURLcode create_transfer(struct GlobalConfig *global, CURLSH *share, bool *added, bool *skipped) { CURLcode result = CURLE_OK; *added = FALSE; while(global->current) { result = transfer_per_config(global, global->current, share, added, skipped); if(!result && !*added) { /* when one set is drained, continue to next */ global->current = global->current->next; continue; } break; } return result; } static CURLcode run_all_transfers(struct GlobalConfig *global, CURLSH *share, CURLcode result) { /* Save the values of noprogress and isatty to restore them later on */ bool orig_noprogress = global->noprogress; bool orig_isatty = global->isatty; struct per_transfer *per; /* Time to actually do the transfers */ if(!result) { if(global->parallel) result = parallel_transfers(global, share); else result = serial_transfers(global, share); } /* cleanup if there are any left */ for(per = transfers; per;) { bool retry; long delay; CURLcode result2 = post_per_transfer(global, per, result, &retry, &delay); if(!result) /* do not overwrite the original error */ result = result2; /* Free list of given URLs */ clean_getout(per->config); per = del_per_transfer(per); } /* Reset the global config variables */ global->noprogress = orig_noprogress; global->isatty = orig_isatty; return result; } CURLcode operate(struct GlobalConfig *global, int argc, argv_item_t argv[]) { CURLcode result = CURLE_OK; char *first_arg = argc > 1 ? curlx_convert_tchar_to_UTF8(argv[1]) : NULL; #ifdef HAVE_SETLOCALE /* Override locale for number parsing (only) */ setlocale(LC_ALL, ""); setlocale(LC_NUMERIC, "C"); #endif /* Parse .curlrc if necessary */ if((argc == 1) || (first_arg && strncmp(first_arg, "-q", 2) && strcmp(first_arg, "--disable"))) { parseconfig(NULL, global); /* ignore possible failure */ /* If we had no arguments then make sure a url was specified in .curlrc */ if((argc < 2) && (!global->first->url_list)) { helpf(tool_stderr, NULL); result = CURLE_FAILED_INIT; } } curlx_unicodefree(first_arg); if(!result) { /* Parse the command line arguments */ ParameterError res = parse_args(global, argc, argv); if(res) { result = CURLE_OK; /* Check if we were asked for the help */ if(res == PARAM_HELP_REQUESTED) tool_help(global->help_category); /* Check if we were asked for the manual */ else if(res == PARAM_MANUAL_REQUESTED) hugehelp(); /* Check if we were asked for the version information */ else if(res == PARAM_VERSION_INFO_REQUESTED) tool_version_info(); /* Check if we were asked to list the SSL engines */ else if(res == PARAM_ENGINES_REQUESTED) tool_list_engines(); /* Check if we were asked to dump the embedded CA bundle */ else if(res == PARAM_CA_EMBED_REQUESTED) { #ifdef CURL_CA_EMBED printf("%s", curl_ca_embed); #endif } else if(res == PARAM_LIBCURL_UNSUPPORTED_PROTOCOL) result = CURLE_UNSUPPORTED_PROTOCOL; else if(res == PARAM_READ_ERROR) result = CURLE_READ_ERROR; else result = CURLE_FAILED_INIT; } else { if(global->libcurl) { /* Initialise the libcurl source output */ result = easysrc_init(); } /* Perform the main operations */ if(!result) { size_t count = 0; struct OperationConfig *operation = global->first; CURLSH *share = curl_share_init(); if(!share) { if(global->libcurl) { /* Cleanup the libcurl source output */ easysrc_cleanup(); } result = CURLE_OUT_OF_MEMORY; } if(!result) { curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT); curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_PSL); curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_HSTS); /* Get the required arguments for each operation */ do { result = get_args(operation, count++); operation = operation->next; } while(!result && operation); /* Set the current operation pointer */ global->current = global->first; /* now run! */ result = run_all_transfers(global, share, result); curl_share_cleanup(share); if(global->libcurl) { /* Cleanup the libcurl source output */ easysrc_cleanup(); /* Dump the libcurl code if previously enabled */ dumpeasysrc(global); } } } else errorf(global, "out of memory"); } } varcleanup(global); return result; }