1 /** **************************************************************************
4 * Copyright 2008 Bryan Ischo <bryan@ischo.com>
6 * This file is part of libs3.
8 * libs3 is free software: you can redistribute it and/or modify it under the
9 * terms of the GNU General Public License as published by the Free Software
10 * Foundation, version 3 of the License.
12 * In addition, as a special exception, the copyright holders give
13 * permission to link the code of this library and its programs with the
14 * OpenSSL library, and distribute linked combinations including the two.
16 * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
17 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
21 * You should have received a copy of the GNU General Public License version 3
22 * along with libs3, in a file named COPYING. If not, see
23 * <http://www.gnu.org/licenses/>.
25 ************************************************************************** **/
31 #include <sys/utsname.h>
33 #include "request_context.h"
34 #include "response_headers_handler.h"
38 #define USER_AGENT_SIZE 256
39 #define REQUEST_STACK_SIZE 32
41 static char userAgentG[USER_AGENT_SIZE];
43 static pthread_mutex_t requestStackMutexG;
45 static Request *requestStackG[REQUEST_STACK_SIZE];
47 static int requestStackCountG;
50 typedef struct RequestComputedValues
52 // All x-amz- headers, in normalized form (i.e. NAME: VALUE, no other ws)
53 char *amzHeaders[S3_MAX_METADATA_COUNT + 2]; // + 2 for acl and date
55 // The number of x-amz- headers
58 // Storage for amzHeaders (the +256 is for x-amz-acl and x-amz-date)
59 char amzHeadersRaw[COMPACTED_METADATA_BUFFER_SIZE + 256 + 1];
61 // Canonicalized x-amz- headers
62 string_multibuffer(canonicalizedAmzHeaders,
63 COMPACTED_METADATA_BUFFER_SIZE + 256 + 1);
66 char urlEncodedKey[MAX_URLENCODED_KEY_SIZE + 1];
68 // Canonicalized resource
69 char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE + 1];
71 // Cache-Control header (or empty)
72 char cacheControlHeader[128];
74 // Content-Type header (or empty)
75 char contentTypeHeader[128];
77 // Content-MD5 header (or empty)
80 // Content-Disposition header (or empty)
81 char contentDispositionHeader[128];
83 // Content-Encoding header (or empty)
84 char contentEncodingHeader[128];
86 // Expires header (or empty)
87 char expiresHeader[128];
89 // If-Modified-Since header
90 char ifModifiedSinceHeader[128];
92 // If-Unmodified-Since header
93 char ifUnmodifiedSinceHeader[128];
96 char ifMatchHeader[128];
98 // If-None-Match header
99 char ifNoneMatchHeader[128];
102 char rangeHeader[128];
104 // Authorization header
105 char authorizationHeader[128];
106 } RequestComputedValues;
109 // Called whenever we detect that the request headers have been completely
110 // processed; which happens either when we get our first read/write callback,
111 // or the request is finished being procesed. Returns nonzero on success,
113 static void request_headers_done(Request *request)
115 if (request->propertiesCallbackMade) {
119 request->propertiesCallbackMade = 1;
121 // Get the http response code
122 long httpResponseCode;
123 request->httpResponseCode = 0;
124 if (curl_easy_getinfo(request->curl, CURLINFO_RESPONSE_CODE,
125 &httpResponseCode) != CURLE_OK) {
126 // Not able to get the HTTP response code - error
127 request->status = S3StatusInternalError;
131 request->httpResponseCode = httpResponseCode;
134 response_headers_handler_done(&(request->responseHeadersHandler),
137 // Only make the callback if it was a successful request; otherwise we're
138 // returning information about the error response itself
139 if (request->propertiesCallback &&
140 (request->httpResponseCode >= 200) &&
141 (request->httpResponseCode <= 299)) {
142 request->status = (*(request->propertiesCallback))
143 (&(request->responseHeadersHandler.responseProperties),
144 request->callbackData);
149 static size_t curl_header_func(void *ptr, size_t size, size_t nmemb,
152 Request *request = (Request *) data;
154 int len = size * nmemb;
156 response_headers_handler_add
157 (&(request->responseHeadersHandler), (char *) ptr, len);
163 static size_t curl_read_func(void *ptr, size_t size, size_t nmemb, void *data)
165 Request *request = (Request *) data;
167 int len = size * nmemb;
169 request_headers_done(request);
171 if (request->status != S3StatusOK) {
172 return CURL_READFUNC_ABORT;
175 // If there is no data callback, or the data callback has already returned
176 // contentLength bytes, return 0;
177 if (!request->toS3Callback || !request->toS3CallbackBytesRemaining) {
181 // Don't tell the callback that we are willing to accept more data than we
183 if (len > request->toS3CallbackBytesRemaining) {
184 len = request->toS3CallbackBytesRemaining;
187 // Otherwise, make the data callback
188 int ret = (*(request->toS3Callback))
189 (len, (char *) ptr, request->callbackData);
191 request->status = S3StatusAbortedByCallback;
192 return CURL_READFUNC_ABORT;
195 if (ret > request->toS3CallbackBytesRemaining) {
196 ret = request->toS3CallbackBytesRemaining;
198 request->toS3CallbackBytesRemaining -= ret;
204 static size_t curl_write_func(void *ptr, size_t size, size_t nmemb,
207 Request *request = (Request *) data;
209 int len = size * nmemb;
211 request_headers_done(request);
213 if (request->status != S3StatusOK) {
217 // On HTTP error, we expect to parse an HTTP error response
218 if ((request->httpResponseCode < 200) ||
219 (request->httpResponseCode > 299)) {
220 request->status = error_parser_add
221 (&(request->errorParser), (char *) ptr, len);
223 // If there was a callback registered, make it
224 else if (request->fromS3Callback) {
225 request->status = (*(request->fromS3Callback))
226 (len, (char *) ptr, request->callbackData);
228 // Else, consider this an error - S3 has sent back data when it was not
231 request->status = S3StatusInternalError;
234 return ((request->status == S3StatusOK) ? len : 0);
238 // This function 'normalizes' all x-amz-meta headers provided in
239 // params->requestHeaders, which means it removes all whitespace from
240 // them such that they all look exactly like this:
241 // x-amz-meta-${NAME}: ${VALUE}
242 // It also adds the x-amz-acl, x-amz-copy-source, and x-amz-metadata-directive
243 // headers if necessary, and always adds the x-amz-date header. It copies the
244 // raw string values into params->amzHeadersRaw, and creates an array of
245 // string pointers representing these headers in params->amzHeaders (and also
246 // sets params->amzHeadersCount to be the count of the total number of x-amz-
247 // headers thus created).
248 static S3Status compose_amz_headers(const RequestParams *params,
249 RequestComputedValues *values)
251 const S3PutProperties *properties = params->putProperties;
253 values->amzHeadersCount = 0;
254 values->amzHeadersRaw[0] = 0;
257 // Append a header to amzHeaders, trimming whitespace from the end.
258 // Does NOT trim whitespace from the beginning.
259 #define headers_append(isNewHeader, format, ...) \
262 values->amzHeaders[values->amzHeadersCount++] = \
263 &(values->amzHeadersRaw[len]); \
265 len += snprintf(&(values->amzHeadersRaw[len]), \
266 sizeof(values->amzHeadersRaw) - len, \
267 format, __VA_ARGS__); \
268 if (len >= (int) sizeof(values->amzHeadersRaw)) { \
269 return S3StatusMetaDataHeadersTooLong; \
271 while ((len > 0) && (values->amzHeadersRaw[len - 1] == ' ')) { \
274 values->amzHeadersRaw[len++] = 0; \
277 #define header_name_tolower_copy(str, l) \
279 values->amzHeaders[values->amzHeadersCount++] = \
280 &(values->amzHeadersRaw[len]); \
281 if ((len + l) >= (int) sizeof(values->amzHeadersRaw)) { \
282 return S3StatusMetaDataHeadersTooLong; \
286 if ((*(str) >= 'A') && (*(str) <= 'Z')) { \
287 values->amzHeadersRaw[len++] = 'a' + (*(str) - 'A'); \
290 values->amzHeadersRaw[len++] = *(str); \
296 // Check and copy in the x-amz-meta headers
299 for (i = 0; i < properties->metaDataCount; i++) {
300 const S3NameValue *property = &(properties->metaData[i]);
301 char headerName[S3_MAX_METADATA_SIZE - sizeof(": v")];
302 int l = snprintf(headerName, sizeof(headerName),
303 S3_METADATA_HEADER_NAME_PREFIX "%s",
305 char *hn = headerName;
306 header_name_tolower_copy(hn, l);
308 headers_append(0, ": %s", property->value);
311 // Add the x-amz-acl header, if necessary
312 const char *cannedAclString;
313 switch (params->putProperties->cannedAcl) {
314 case S3CannedAclPrivate:
317 case S3CannedAclPublicRead:
318 cannedAclString = "public-read";
320 case S3CannedAclPublicReadWrite:
321 cannedAclString = "public-read-write";
323 default: // S3CannedAclAuthenticatedRead
324 cannedAclString = "authenticated-read";
327 if (cannedAclString) {
328 headers_append(1, "x-amz-acl: %s", cannedAclString);
332 // Add the x-amz-date header
333 time_t now = time(NULL);
335 strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&now));
336 headers_append(1, "x-amz-date: %s", date);
338 if (params->httpRequestType == HttpRequestTypeCOPY) {
339 // Add the x-amz-copy-source header
340 if (params->copySourceBucketName && params->copySourceBucketName[0] &&
341 params->copySourceKey && params->copySourceKey[0]) {
342 headers_append(1, "x-amz-copy-source: /%s/%s",
343 params->copySourceBucketName,
344 params->copySourceKey);
346 // And the x-amz-metadata-directive header
347 if (params->putProperties) {
348 headers_append(1, "%s", "x-amz-metadata-directive: REPLACE");
356 // Composes the other headers
357 static S3Status compose_standard_headers(const RequestParams *params,
358 RequestComputedValues *values)
361 #define do_put_header(fmt, sourceField, destField, badError, tooLongError) \
363 if (params->putProperties && \
364 params->putProperties-> sourceField && \
365 params->putProperties-> sourceField[0]) { \
366 /* Skip whitespace at beginning of val */ \
367 const char *val = params->putProperties-> sourceField; \
368 while (*val && is_blank(*val)) { \
374 /* Compose header, make sure it all fit */ \
375 int len = snprintf(values-> destField, \
376 sizeof(values-> destField), fmt, val); \
377 if (len >= (int) sizeof(values-> destField)) { \
378 return tooLongError; \
380 /* Now remove the whitespace at the end */ \
381 while (is_blank(values-> destField[len])) { \
384 values-> destField[len] = 0; \
387 values-> destField[0] = 0; \
391 #define do_get_header(fmt, sourceField, destField, badError, tooLongError) \
393 if (params->getConditions && \
394 params->getConditions-> sourceField && \
395 params->getConditions-> sourceField[0]) { \
396 /* Skip whitespace at beginning of val */ \
397 const char *val = params->getConditions-> sourceField; \
398 while (*val && is_blank(*val)) { \
404 /* Compose header, make sure it all fit */ \
405 int len = snprintf(values-> destField, \
406 sizeof(values-> destField), fmt, val); \
407 if (len >= (int) sizeof(values-> destField)) { \
408 return tooLongError; \
410 /* Now remove the whitespace at the end */ \
411 while (is_blank(values-> destField[len])) { \
414 values-> destField[len] = 0; \
417 values-> destField[0] = 0; \
422 do_put_header("Cache-Control: %s", cacheControl, cacheControlHeader,
423 S3StatusBadCacheControl, S3StatusCacheControlTooLong);
426 do_put_header("Content-Type: %s", contentType, contentTypeHeader,
427 S3StatusBadContentType, S3StatusContentTypeTooLong);
430 do_put_header("Content-MD5: %s", md5, md5Header, S3StatusBadMD5,
433 // Content-Disposition
434 do_put_header("Content-Disposition: attachment; filename=\"%s\"",
435 contentDispositionFilename, contentDispositionHeader,
436 S3StatusBadContentDispositionFilename,
437 S3StatusContentDispositionFilenameTooLong);
440 do_put_header("Content-Encoding: %s", contentEncoding,
441 contentEncodingHeader, S3StatusBadContentEncoding,
442 S3StatusContentEncodingTooLong);
445 if (params->putProperties && (params->putProperties->expires >= 0)) {
446 time_t t = (time_t) params->putProperties->expires;
447 strftime(values->expiresHeader, sizeof(values->expiresHeader),
448 "Expires: %a, %d %b %Y %H:%M:%S UTC", gmtime(&t));
451 values->expiresHeader[0] = 0;
455 if (params->getConditions &&
456 (params->getConditions->ifModifiedSince >= 0)) {
457 time_t t = (time_t) params->getConditions->ifModifiedSince;
458 strftime(values->ifModifiedSinceHeader,
459 sizeof(values->ifModifiedSinceHeader),
460 "If-Modified-Since: %a, %d %b %Y %H:%M:%S UTC", gmtime(&t));
463 values->ifModifiedSinceHeader[0] = 0;
466 // If-Unmodified-Since header
467 if (params->getConditions &&
468 (params->getConditions->ifNotModifiedSince >= 0)) {
469 time_t t = (time_t) params->getConditions->ifNotModifiedSince;
470 strftime(values->ifUnmodifiedSinceHeader,
471 sizeof(values->ifUnmodifiedSinceHeader),
472 "If-Unmodified-Since: %a, %d %b %Y %H:%M:%S UTC", gmtime(&t));
475 values->ifUnmodifiedSinceHeader[0] = 0;
479 do_get_header("If-Match: %s", ifMatchETag, ifMatchHeader,
480 S3StatusBadIfMatchETag, S3StatusIfMatchETagTooLong);
482 // If-None-Match header
483 do_get_header("If-None-Match: %s", ifNotMatchETag, ifNoneMatchHeader,
484 S3StatusBadIfNotMatchETag,
485 S3StatusIfNotMatchETagTooLong);
488 if (params->startByte || params->byteCount) {
489 if (params->byteCount) {
490 snprintf(values->rangeHeader, sizeof(values->rangeHeader),
491 "Range: bytes=%llu-%llu",
492 (unsigned long long) params->startByte,
493 (unsigned long long) (params->startByte +
494 params->byteCount - 1));
497 snprintf(values->rangeHeader, sizeof(values->rangeHeader),
498 "Range: bytes=%llu-",
499 (unsigned long long) params->startByte);
503 values->rangeHeader[0] = 0;
510 // URL encodes the params->key value into params->urlEncodedKey
511 static S3Status encode_key(const RequestParams *params,
512 RequestComputedValues *values)
514 return (urlEncode(values->urlEncodedKey, params->key, S3_MAX_KEY_SIZE) ?
515 S3StatusOK : S3StatusUriTooLong);
519 // Simple comparison function for comparing two HTTP header names that are
520 // embedded within an HTTP header line, returning true if header1 comes
521 // before header2 alphabetically, false if not
522 static int headerle(const char *header1, const char *header2)
525 if (*header1 == ':') {
526 return (*header2 == ':');
528 else if (*header2 == ':') {
531 else if (*header2 < *header1) {
534 else if (*header2 > *header1) {
537 header1++, header2++;
542 // Replace this with merge sort eventually, it's the best stable sort. But
543 // since typically the number of elements being sorted is small, it doesn't
544 // matter that much which sort is used, and gnome sort is the world's simplest
545 // stable sort. Added a slight twist to the standard gnome_sort - don't go
546 // forward +1, go forward to the last highest index considered. This saves
547 // all the string comparisons that would be done "going forward", and thus
548 // only does the necessary string comparisons to move values back into their
550 static void header_gnome_sort(const char **headers, int size)
552 int i = 0, last_highest = 0;
555 if ((i == 0) || headerle(headers[i - 1], headers[i])) {
559 const char *tmp = headers[i];
560 headers[i] = headers[i - 1];
567 // Canonicalizes the x-amz- headers into the canonicalizedAmzHeaders buffer
568 static void canonicalize_amz_headers(RequestComputedValues *values)
570 // Make a copy of the headers that will be sorted
571 const char *sortedHeaders[S3_MAX_METADATA_COUNT];
573 memcpy(sortedHeaders, values->amzHeaders,
574 (values->amzHeadersCount * sizeof(sortedHeaders[0])));
577 header_gnome_sort(sortedHeaders, values->amzHeadersCount);
579 // Now copy this sorted list into the buffer, all the while:
580 // - folding repeated headers into single lines, and
581 // - folding multiple lines
582 // - removing the space after the colon
583 int lastHeaderLen = 0, i;
584 char *buffer = values->canonicalizedAmzHeaders;
585 for (i = 0; i < values->amzHeadersCount; i++) {
586 const char *header = sortedHeaders[i];
587 const char *c = header;
588 // If the header names are the same, append the next value
590 !strncmp(header, sortedHeaders[i - 1], lastHeaderLen)) {
591 // Replacing the previous newline with a comma
593 // Skip the header name and space
594 c += (lastHeaderLen + 1);
596 // Else this is a new header
598 // Copy in everything up to the space in the ": "
602 // Save the header len since it's a new header
603 lastHeaderLen = c - header;
607 // Now copy in the value, folding the lines
609 // If c points to a \r\n[whitespace] sequence, then fold
611 if ((*c == '\r') && (*(c + 1) == '\n') && is_blank(*(c + 2))) {
613 while (is_blank(*c)) {
616 // Also, what has most recently been copied into buffer amy
617 // have been whitespace, and since we're folding whitespace
618 // out around this newline sequence, back buffer up over
619 // any whitespace it contains
620 while (is_blank(*(buffer - 1))) {
627 // Finally, add the newline
631 // Terminate the buffer
636 // Canonicalizes the resource into params->canonicalizedResource
637 static void canonicalize_resource(const char *bucketName,
638 const char *subResource,
639 const char *urlEncodedKey,
646 #define append(str) len += sprintf(&(buffer[len]), "%s", str)
648 if (bucketName && bucketName[0]) {
655 if (urlEncodedKey && urlEncodedKey[0]) {
656 append(urlEncodedKey);
659 if (subResource && subResource[0]) {
666 // Convert an HttpRequestType to an HTTP Verb string
667 static const char *http_request_type_to_verb(HttpRequestType requestType)
669 switch (requestType) {
670 case HttpRequestTypeGET:
672 case HttpRequestTypeHEAD:
674 case HttpRequestTypePUT:
675 case HttpRequestTypeCOPY:
677 default: // HttpRequestTypeDELETE
683 // Composes the Authorization header for the request
684 static S3Status compose_auth_header(const RequestParams *params,
685 RequestComputedValues *values)
688 // 17 bytes for HTTP-Verb + \n
689 // 129 bytes for Content-MD5 + \n
690 // 129 bytes for Content-Type + \n
691 // 1 byte for empty Date + \n
692 // CanonicalizedAmzHeaders & CanonicalizedResource
693 char signbuf[17 + 129 + 129 + 1 +
694 (sizeof(values->canonicalizedAmzHeaders) - 1) +
695 (sizeof(values->canonicalizedResource) - 1) + 1];
698 #define signbuf_append(format, ...) \
699 len += snprintf(&(signbuf[len]), sizeof(signbuf) - len, \
703 ("%s\n", http_request_type_to_verb(params->httpRequestType));
705 // For MD5 and Content-Type, use the value in the actual header, because
706 // it's already been trimmed
707 signbuf_append("%s\n", values->md5Header[0] ?
708 &(values->md5Header[sizeof("Content-MD5: ") - 1]) : "");
711 ("%s\n", values->contentTypeHeader[0] ?
712 &(values->contentTypeHeader[sizeof("Content-Type: ") - 1]) : "");
714 signbuf_append("%s", "\n"); // Date - we always use x-amz-date
716 signbuf_append("%s", values->canonicalizedAmzHeaders);
718 signbuf_append("%s", values->canonicalizedResource);
720 // Generate an HMAC-SHA-1 of the signbuf
721 unsigned char hmac[20];
723 HMAC_SHA1(hmac, (unsigned char *) params->bucketContext.secretAccessKey,
724 strlen(params->bucketContext.secretAccessKey),
725 (unsigned char *) signbuf, len);
727 // Now base-64 encode the results
728 char b64[((20 + 1) * 4) / 3];
729 int b64Len = base64Encode(hmac, 20, b64);
731 snprintf(values->authorizationHeader, sizeof(values->authorizationHeader),
732 "Authorization: AWS %s:%.*s", params->bucketContext.accessKeyId,
739 // Compose the URI to use for the request given the request parameters
740 static S3Status compose_uri(char *buffer, int bufferSize,
741 const S3BucketContext *bucketContext,
742 const char *urlEncodedKey,
743 const char *subResource, const char *queryParams)
747 #define uri_append(fmt, ...) \
749 len += snprintf(&(buffer[len]), bufferSize - len, fmt, __VA_ARGS__); \
750 if (len >= bufferSize) { \
751 return S3StatusUriTooLong; \
755 uri_append("http%s://",
756 (bucketContext->protocol == S3ProtocolHTTP) ? "" : "s");
758 if (bucketContext->bucketName &&
759 bucketContext->bucketName[0]) {
760 if (bucketContext->uriStyle == S3UriStyleVirtualHost) {
761 uri_append("%s.s3.amazonaws.com", bucketContext->bucketName);
764 uri_append("s3.amazonaws.com/%s", bucketContext->bucketName);
768 uri_append("%s", "s3.amazonaws.com");
771 uri_append("%s", "/");
773 uri_append("%s", urlEncodedKey);
775 if (subResource && subResource[0]) {
776 uri_append("?%s", subResource);
780 uri_append("%s%s", (subResource && subResource[0]) ? "&" : "?",
788 // Sets up the curl handle given the completely computed RequestParams
789 static S3Status setup_curl(Request *request,
790 const RequestParams *params,
791 const RequestComputedValues *values)
795 #define curl_easy_setopt_safe(opt, val) \
796 if ((status = curl_easy_setopt \
797 (request->curl, opt, val)) != CURLE_OK) { \
798 return S3StatusFailedToInitializeRequest; \
802 // curl_easy_setopt_safe(CURLOPT_VERBOSE, 1);
804 // Set private data to request for the benefit of S3RequestContext
805 curl_easy_setopt_safe(CURLOPT_PRIVATE, request);
807 // Set header callback and data
808 curl_easy_setopt_safe(CURLOPT_HEADERDATA, request);
809 curl_easy_setopt_safe(CURLOPT_HEADERFUNCTION, &curl_header_func);
811 // Set read callback, data, and readSize
812 curl_easy_setopt_safe(CURLOPT_READFUNCTION, &curl_read_func);
813 curl_easy_setopt_safe(CURLOPT_READDATA, request);
815 // Set write callback and data
816 curl_easy_setopt_safe(CURLOPT_WRITEFUNCTION, &curl_write_func);
817 curl_easy_setopt_safe(CURLOPT_WRITEDATA, request);
819 // Ask curl to parse the Last-Modified header. This is easier than
820 // parsing it ourselves.
821 curl_easy_setopt_safe(CURLOPT_FILETIME, 1);
823 // Curl docs suggest that this is necessary for multithreaded code.
824 // However, it also points out that DNS timeouts will not be honored
825 // during DNS lookup, which can be worked around by using the c-ares
826 // library, which we do not do yet.
827 curl_easy_setopt_safe(CURLOPT_NOSIGNAL, 1);
829 // Turn off Curl's built-in progress meter
830 curl_easy_setopt_safe(CURLOPT_NOPROGRESS, 1);
832 // xxx todo - support setting the proxy for Curl to use (can't use https
833 // for proxies though)
835 // xxx todo - support setting the network interface for Curl to use
837 // I think this is useful - we don't need interactive performance, we need
838 // to complete large operations quickly
839 curl_easy_setopt_safe(CURLOPT_TCP_NODELAY, 1);
841 // Don't use Curl's 'netrc' feature
842 curl_easy_setopt_safe(CURLOPT_NETRC, CURL_NETRC_IGNORED);
844 // Don't verify S3's certificate, there are known to be issues with
846 // xxx todo - support an option for verifying the S3 CA (default false)
847 curl_easy_setopt_safe(CURLOPT_SSL_VERIFYPEER, 0);
849 // Follow any redirection directives that S3 sends
850 curl_easy_setopt_safe(CURLOPT_FOLLOWLOCATION, 1);
852 // A safety valve in case S3 goes bananas with redirects
853 curl_easy_setopt_safe(CURLOPT_MAXREDIRS, 10);
855 // Set the User-Agent; maybe Amazon will track these?
856 curl_easy_setopt_safe(CURLOPT_USERAGENT, userAgentG);
858 // Set the low speed limit and time; we abort transfers that stay at
859 // less than 1K per second for more than 15 seconds.
860 // xxx todo - make these configurable
861 // xxx todo - allow configurable max send and receive speed
862 curl_easy_setopt_safe(CURLOPT_LOW_SPEED_LIMIT, 1024);
863 curl_easy_setopt_safe(CURLOPT_LOW_SPEED_TIME, 15);
865 // Append standard headers
866 #define append_standard_header(fieldName) \
867 if (values-> fieldName [0]) { \
868 request->headers = curl_slist_append(request->headers, \
869 values-> fieldName); \
872 // Would use CURLOPT_INFILESIZE_LARGE, but it is buggy in libcurl
873 if (params->httpRequestType == HttpRequestTypePUT) {
875 snprintf(header, sizeof(header), "Content-Length: %llu",
876 (unsigned long long) params->toS3CallbackTotalSize);
877 request->headers = curl_slist_append(request->headers, header);
878 request->headers = curl_slist_append(request->headers,
879 "Transfer-Encoding:");
881 else if (params->httpRequestType == HttpRequestTypeCOPY) {
882 request->headers = curl_slist_append(request->headers,
883 "Transfer-Encoding:");
886 append_standard_header(cacheControlHeader);
887 append_standard_header(contentTypeHeader);
888 append_standard_header(md5Header);
889 append_standard_header(contentDispositionHeader);
890 append_standard_header(contentEncodingHeader);
891 append_standard_header(expiresHeader);
892 append_standard_header(ifModifiedSinceHeader);
893 append_standard_header(ifUnmodifiedSinceHeader);
894 append_standard_header(ifMatchHeader);
895 append_standard_header(ifNoneMatchHeader);
896 append_standard_header(rangeHeader);
897 append_standard_header(authorizationHeader);
899 // Append x-amz- headers
901 for (i = 0; i < values->amzHeadersCount; i++) {
903 curl_slist_append(request->headers, values->amzHeaders[i]);
906 // Set the HTTP headers
907 curl_easy_setopt_safe(CURLOPT_HTTPHEADER, request->headers);
910 curl_easy_setopt_safe(CURLOPT_URL, request->uri);
913 switch (params->httpRequestType) {
914 case HttpRequestTypeHEAD:
915 curl_easy_setopt_safe(CURLOPT_NOBODY, 1);
917 case HttpRequestTypePUT:
918 case HttpRequestTypeCOPY:
919 curl_easy_setopt_safe(CURLOPT_UPLOAD, 1);
921 case HttpRequestTypeDELETE:
922 curl_easy_setopt_safe(CURLOPT_CUSTOMREQUEST, "DELETE");
924 default: // HttpRequestTypeGET
932 static void request_deinitialize(Request *request)
934 if (request->headers) {
935 curl_slist_free_all(request->headers);
938 error_parser_deinitialize(&(request->errorParser));
940 // curl_easy_reset prevents connections from being re-used for some
941 // reason. This makes HTTP Keep-Alive meaningless and is very bad for
942 // performance. But it is necessary to allow curl to work properly.
943 // xxx todo figure out why
944 curl_easy_reset(request->curl);
948 static S3Status request_get(const RequestParams *params,
949 const RequestComputedValues *values,
952 Request *request = 0;
954 // Try to get one from the request stack. We hold the lock for the
955 // shortest time possible here.
956 pthread_mutex_lock(&requestStackMutexG);
958 if (requestStackCountG) {
959 request = requestStackG[--requestStackCountG];
962 pthread_mutex_unlock(&requestStackMutexG);
964 // If we got one, deinitialize it for re-use
966 request_deinitialize(request);
968 // Else there wasn't one available in the request stack, so create one
970 if (!(request = (Request *) malloc(sizeof(Request)))) {
971 return S3StatusOutOfMemory;
973 if (!(request->curl = curl_easy_init())) {
975 return S3StatusFailedToInitializeRequest;
979 // Initialize the request
983 // Request status is initialized to no error, will be updated whenever
985 request->status = S3StatusOK;
989 // Start out with no headers
990 request->headers = 0;
993 if ((status = compose_uri
994 (request->uri, sizeof(request->uri),
995 &(params->bucketContext), values->urlEncodedKey,
996 params->subResource, params->queryParams)) != S3StatusOK) {
997 curl_easy_cleanup(request->curl);
1002 // Set all of the curl handle options
1003 if ((status = setup_curl(request, params, values)) != S3StatusOK) {
1004 curl_easy_cleanup(request->curl);
1009 request->propertiesCallback = params->propertiesCallback;
1011 request->toS3Callback = params->toS3Callback;
1013 request->toS3CallbackBytesRemaining = params->toS3CallbackTotalSize;
1015 request->fromS3Callback = params->fromS3Callback;
1017 request->completeCallback = params->completeCallback;
1019 request->callbackData = params->callbackData;
1021 response_headers_handler_initialize(&(request->responseHeadersHandler));
1023 request->propertiesCallbackMade = 0;
1025 error_parser_initialize(&(request->errorParser));
1027 *reqReturn = request;
1033 static void request_destroy(Request *request)
1035 request_deinitialize(request);
1036 curl_easy_cleanup(request->curl);
1041 static void request_release(Request *request)
1043 pthread_mutex_lock(&requestStackMutexG);
1045 // If the request stack is full, destroy this one
1046 if (requestStackCountG == REQUEST_STACK_SIZE) {
1047 pthread_mutex_unlock(&requestStackMutexG);
1048 request_destroy(request);
1050 // Else put this one at the front of the request stack; we do this because
1051 // we want the most-recently-used curl handle to be re-used on the next
1052 // request, to maximize our chances of re-using a TCP connection before it
1055 requestStackG[requestStackCountG++] = request;
1056 pthread_mutex_unlock(&requestStackMutexG);
1061 S3Status request_api_initialize(const char *userAgentInfo, int flags)
1063 if (curl_global_init(CURL_GLOBAL_ALL &
1064 ~((flags & S3_INIT_WINSOCK) ? 0 : CURL_GLOBAL_WIN32))
1066 return S3StatusInternalError;
1069 pthread_mutex_init(&requestStackMutexG, 0);
1071 requestStackCountG = 0;
1073 if (!userAgentInfo || !*userAgentInfo) {
1074 userAgentInfo = "Unknown";
1078 struct utsname utsn;
1080 strncpy(platform, "Unknown", sizeof(platform));
1081 // Because strncpy doesn't always zero terminate
1082 platform[sizeof(platform) - 1] = 0;
1085 snprintf(platform, sizeof(platform), "%s%s%s", utsn.sysname,
1086 utsn.machine[0] ? " " : "", utsn.machine);
1089 snprintf(userAgentG, sizeof(userAgentG),
1090 "Mozilla/4.0 (Compatible; %s; libs3 %s.%s; %s)",
1091 userAgentInfo, LIBS3_VER_MAJOR, LIBS3_VER_MINOR, platform);
1097 void request_api_deinitialize()
1099 pthread_mutex_destroy(&requestStackMutexG);
1101 while (requestStackCountG--) {
1102 request_destroy(requestStackG[requestStackCountG]);
1107 void request_perform(const RequestParams *params, S3RequestContext *context)
1112 #define return_status(status) \
1113 (*(params->completeCallback))(status, 0, params->callbackData); \
1116 // These will hold the computed values
1117 RequestComputedValues computed;
1119 // Validate the bucket name
1120 if (params->bucketContext.bucketName &&
1121 ((status = S3_validate_bucket_name
1122 (params->bucketContext.bucketName,
1123 params->bucketContext.uriStyle)) != S3StatusOK)) {
1124 return_status(status);
1127 // Compose the amz headers
1128 if ((status = compose_amz_headers(params, &computed)) != S3StatusOK) {
1129 return_status(status);
1132 // Compose standard headers
1133 if ((status = compose_standard_headers
1134 (params, &computed)) != S3StatusOK) {
1135 return_status(status);
1138 // URL encode the key
1139 if ((status = encode_key(params, &computed)) != S3StatusOK) {
1140 return_status(status);
1143 // Compute the canonicalized amz headers
1144 canonicalize_amz_headers(&computed);
1146 // Compute the canonicalized resource
1147 canonicalize_resource(params->bucketContext.bucketName,
1148 params->subResource, computed.urlEncodedKey,
1149 computed.canonicalizedResource);
1151 // Compose Authorization header
1152 if ((status = compose_auth_header(params, &computed)) != S3StatusOK) {
1153 return_status(status);
1156 // Get an initialized Request structure now
1157 if ((status = request_get(params, &computed, &request)) != S3StatusOK) {
1158 return_status(status);
1161 // If a RequestContext was provided, add the request to the curl multi
1163 CURLMcode code = curl_multi_add_handle(context->curlm, request->curl);
1164 if (code == CURLM_OK) {
1165 if (context->requests) {
1166 request->prev = context->requests->prev;
1167 request->next = context->requests;
1168 context->requests->prev->next = request;
1169 context->requests->prev = request;
1172 context->requests = request->next = request->prev = request;
1176 if (request->status == S3StatusOK) {
1177 request->status = (code == CURLM_OUT_OF_MEMORY) ?
1178 S3StatusOutOfMemory : S3StatusInternalError;
1180 request_finish(request);
1183 // Else, perform the request immediately
1185 CURLcode code = curl_easy_perform(request->curl);
1186 if ((code != CURLE_OK) && (request->status == S3StatusOK)) {
1187 request->status = request_curl_code_to_status(code);
1189 // Finish the request, ensuring that all callbacks have been made, and
1190 // also releases the request
1191 request_finish(request);
1196 void request_finish(Request *request)
1198 // If we haven't detected this already, we now know that the headers are
1199 // definitely done being read in
1200 request_headers_done(request);
1202 // If there was no error processing the request, then possibly there was
1203 // an S3 error parsed, which should be converted into the request status
1204 if (request->status == S3StatusOK) {
1205 error_parser_convert_status(&(request->errorParser),
1206 &(request->status));
1207 // If there still was no error recorded, then it is possible that
1208 // there was in fact an error but that there was no error XML
1209 // detailing the error
1210 if ((request->status == S3StatusOK) &&
1211 ((request->httpResponseCode < 200) ||
1212 (request->httpResponseCode > 299))) {
1213 switch (request->httpResponseCode) {
1215 // This happens if the request never got any HTTP response
1216 // headers at all, we call this a ConnectionFailed error
1217 request->status = S3StatusConnectionFailed;
1219 case 100: // Some versions of libcurl erroneously set HTTP
1223 request->status = S3StatusErrorPermanentRedirect;
1226 request->status = S3StatusHttpErrorMovedTemporarily;
1229 request->status = S3StatusHttpErrorBadRequest;
1232 request->status = S3StatusHttpErrorForbidden;
1235 request->status = S3StatusHttpErrorNotFound;
1238 request->status = S3StatusErrorMethodNotAllowed;
1241 request->status = S3StatusHttpErrorConflict;
1244 request->status = S3StatusErrorMissingContentLength;
1247 request->status = S3StatusErrorPreconditionFailed;
1250 request->status = S3StatusErrorInvalidRange;
1253 request->status = S3StatusErrorInternalError;
1256 request->status = S3StatusErrorNotImplemented;
1259 request->status = S3StatusErrorSlowDown;
1262 request->status = S3StatusHttpErrorUnknown;
1268 (*(request->completeCallback))
1269 (request->status, &(request->errorParser.s3ErrorDetails),
1270 request->callbackData);
1272 request_release(request);
1276 S3Status request_curl_code_to_status(CURLcode code)
1279 case CURLE_OUT_OF_MEMORY:
1280 return S3StatusOutOfMemory;
1281 case CURLE_COULDNT_RESOLVE_PROXY:
1282 case CURLE_COULDNT_RESOLVE_HOST:
1283 return S3StatusNameLookupError;
1284 case CURLE_COULDNT_CONNECT:
1285 return S3StatusFailedToConnect;
1286 case CURLE_WRITE_ERROR:
1287 case CURLE_OPERATION_TIMEDOUT:
1288 return S3StatusConnectionFailed;
1289 case CURLE_PARTIAL_FILE:
1291 case CURLE_SSL_CACERT:
1292 return S3StatusServerFailedVerification;
1294 return S3StatusInternalError;
1299 S3Status S3_generate_authenticated_query_string
1300 (char *buffer, const S3BucketContext *bucketContext,
1301 const char *key, int64_t expires, const char *resource)
1303 #define MAX_EXPIRES (((int64_t) 1 << 31) - 1)
1304 // S3 seems to only accept expiration dates up to the number of seconds
1305 // representably by a signed 32-bit integer
1307 expires = MAX_EXPIRES;
1309 else if (expires > MAX_EXPIRES) {
1310 expires = MAX_EXPIRES;
1313 // xxx todo: rework this so that it can be incorporated into shared code
1314 // with request_perform(). It's really unfortunate that this code is not
1315 // shared with request_perform().
1317 // URL encode the key
1318 char urlEncodedKey[S3_MAX_KEY_SIZE * 3];
1320 urlEncode(urlEncodedKey, key, strlen(key));
1323 urlEncodedKey[0] = 0;
1326 // Compute canonicalized resource
1327 char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE];
1328 canonicalize_resource(bucketContext->bucketName, resource, urlEncodedKey,
1329 canonicalizedResource);
1332 // 17 bytes for HTTP-Verb + \n
1333 // 1 byte for empty Content-MD5 + \n
1334 // 1 byte for empty Content-Type + \n
1335 // 20 bytes for Expires + \n
1336 // 0 bytes for CanonicalizedAmzHeaders
1337 // CanonicalizedResource
1338 char signbuf[17 + 1 + 1 + 1 + 20 + sizeof(canonicalizedResource) + 1];
1341 #define signbuf_append(format, ...) \
1342 len += snprintf(&(signbuf[len]), sizeof(signbuf) - len, \
1343 format, __VA_ARGS__)
1345 signbuf_append("%s\n", "GET"); // HTTP-Verb
1346 signbuf_append("%s\n", ""); // Content-MD5
1347 signbuf_append("%s\n", ""); // Content-Type
1348 signbuf_append("%llu\n", (unsigned long long) expires);
1349 signbuf_append("%s", canonicalizedResource);
1351 // Generate an HMAC-SHA-1 of the signbuf
1352 unsigned char hmac[20];
1354 HMAC_SHA1(hmac, (unsigned char *) bucketContext->secretAccessKey,
1355 strlen(bucketContext->secretAccessKey),
1356 (unsigned char *) signbuf, len);
1358 // Now base-64 encode the results
1359 char b64[((20 + 1) * 4) / 3];
1360 int b64Len = base64Encode(hmac, 20, b64);
1362 // Now urlEncode that
1363 char signature[sizeof(b64) * 3];
1364 urlEncode(signature, b64, b64Len);
1366 // Finally, compose the uri, with params:
1367 // ?AWSAccessKeyId=xxx[&Expires=]&Signature=xxx
1368 char queryParams[sizeof("AWSAccessKeyId=") + 20 +
1369 sizeof("&Expires=") + 20 +
1370 sizeof("&Signature=") + sizeof(signature) + 1];
1372 sprintf(queryParams, "AWSAccessKeyId=%s&Expires=%ld&Signature=%s",
1373 bucketContext->accessKeyId, (long) expires, signature);
1375 return compose_uri(buffer, S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE,
1376 bucketContext, urlEncodedKey, resource, queryParams);