--- /dev/null
+/** **************************************************************************
+ * request.c
+ *
+ * Copyright 2008 Bryan Ischo <bryan@ischo.com>
+ *
+ * This file is part of libs3.
+ *
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING. If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ ************************************************************************** **/
+
+#include <ctype.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/utsname.h>
+#include "request.h"
+#include "request_context.h"
+#include "response_headers_handler.h"
+#include "util.h"
+
+
+#define USER_AGENT_SIZE 256
+#define REQUEST_STACK_SIZE 32
+
+static char userAgentG[USER_AGENT_SIZE];
+
+static pthread_mutex_t requestStackMutexG;
+
+static Request *requestStackG[REQUEST_STACK_SIZE];
+
+static int requestStackCountG;
+
+
+typedef struct RequestComputedValues
+{
+ // All x-amz- headers, in normalized form (i.e. NAME: VALUE, no other ws)
+ char *amzHeaders[S3_MAX_METADATA_COUNT + 2]; // + 2 for acl and date
+
+ // The number of x-amz- headers
+ int amzHeadersCount;
+
+ // Storage for amzHeaders (the +256 is for x-amz-acl and x-amz-date)
+ char amzHeadersRaw[COMPACTED_METADATA_BUFFER_SIZE + 256 + 1];
+
+ // Canonicalized x-amz- headers
+ string_multibuffer(canonicalizedAmzHeaders,
+ COMPACTED_METADATA_BUFFER_SIZE + 256 + 1);
+
+ // URL-Encoded key
+ char urlEncodedKey[MAX_URLENCODED_KEY_SIZE + 1];
+
+ // Canonicalized resource
+ char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE + 1];
+
+ // Cache-Control header (or empty)
+ char cacheControlHeader[128];
+
+ // Content-Type header (or empty)
+ char contentTypeHeader[128];
+
+ // Content-MD5 header (or empty)
+ char md5Header[128];
+
+ // Content-Disposition header (or empty)
+ char contentDispositionHeader[128];
+
+ // Content-Encoding header (or empty)
+ char contentEncodingHeader[128];
+
+ // Expires header (or empty)
+ char expiresHeader[128];
+
+ // If-Modified-Since header
+ char ifModifiedSinceHeader[128];
+
+ // If-Unmodified-Since header
+ char ifUnmodifiedSinceHeader[128];
+
+ // If-Match header
+ char ifMatchHeader[128];
+
+ // If-None-Match header
+ char ifNoneMatchHeader[128];
+
+ // Range header
+ char rangeHeader[128];
+
+ // Authorization header
+ char authorizationHeader[128];
+} RequestComputedValues;
+
+
+// Called whenever we detect that the request headers have been completely
+// processed; which happens either when we get our first read/write callback,
+// or the request is finished being procesed. Returns nonzero on success,
+// zero on failure.
+static void request_headers_done(Request *request)
+{
+ if (request->propertiesCallbackMade) {
+ return;
+ }
+
+ request->propertiesCallbackMade = 1;
+
+ // Get the http response code
+ long httpResponseCode;
+ request->httpResponseCode = 0;
+ if (curl_easy_getinfo(request->curl, CURLINFO_RESPONSE_CODE,
+ &httpResponseCode) != CURLE_OK) {
+ // Not able to get the HTTP response code - error
+ request->status = S3StatusInternalError;
+ return;
+ }
+ else {
+ request->httpResponseCode = httpResponseCode;
+ }
+
+ response_headers_handler_done(&(request->responseHeadersHandler),
+ request->curl);
+
+ // Only make the callback if it was a successful request; otherwise we're
+ // returning information about the error response itself
+ if (request->propertiesCallback &&
+ (request->httpResponseCode >= 200) &&
+ (request->httpResponseCode <= 299)) {
+ request->status = (*(request->propertiesCallback))
+ (&(request->responseHeadersHandler.responseProperties),
+ request->callbackData);
+ }
+}
+
+
+static size_t curl_header_func(void *ptr, size_t size, size_t nmemb,
+ void *data)
+{
+ Request *request = (Request *) data;
+
+ int len = size * nmemb;
+
+ response_headers_handler_add
+ (&(request->responseHeadersHandler), (char *) ptr, len);
+
+ return len;
+}
+
+
+static size_t curl_read_func(void *ptr, size_t size, size_t nmemb, void *data)
+{
+ Request *request = (Request *) data;
+
+ int len = size * nmemb;
+
+ request_headers_done(request);
+
+ if (request->status != S3StatusOK) {
+ return CURL_READFUNC_ABORT;
+ }
+
+ // If there is no data callback, or the data callback has already returned
+ // contentLength bytes, return 0;
+ if (!request->toS3Callback || !request->toS3CallbackBytesRemaining) {
+ return 0;
+ }
+
+ // Don't tell the callback that we are willing to accept more data than we
+ // really are
+ if (len > request->toS3CallbackBytesRemaining) {
+ len = request->toS3CallbackBytesRemaining;
+ }
+
+ // Otherwise, make the data callback
+ int ret = (*(request->toS3Callback))
+ (len, (char *) ptr, request->callbackData);
+ if (ret < 0) {
+ request->status = S3StatusAbortedByCallback;
+ return CURL_READFUNC_ABORT;
+ }
+ else {
+ if (ret > request->toS3CallbackBytesRemaining) {
+ ret = request->toS3CallbackBytesRemaining;
+ }
+ request->toS3CallbackBytesRemaining -= ret;
+ return ret;
+ }
+}
+
+
+static size_t curl_write_func(void *ptr, size_t size, size_t nmemb,
+ void *data)
+{
+ Request *request = (Request *) data;
+
+ int len = size * nmemb;
+
+ request_headers_done(request);
+
+ if (request->status != S3StatusOK) {
+ return 0;
+ }
+
+ // On HTTP error, we expect to parse an HTTP error response
+ if ((request->httpResponseCode < 200) ||
+ (request->httpResponseCode > 299)) {
+ request->status = error_parser_add
+ (&(request->errorParser), (char *) ptr, len);
+ }
+ // If there was a callback registered, make it
+ else if (request->fromS3Callback) {
+ request->status = (*(request->fromS3Callback))
+ (len, (char *) ptr, request->callbackData);
+ }
+ // Else, consider this an error - S3 has sent back data when it was not
+ // expected
+ else {
+ request->status = S3StatusInternalError;
+ }
+
+ return ((request->status == S3StatusOK) ? len : 0);
+}
+
+
+// This function 'normalizes' all x-amz-meta headers provided in
+// params->requestHeaders, which means it removes all whitespace from
+// them such that they all look exactly like this:
+// x-amz-meta-${NAME}: ${VALUE}
+// It also adds the x-amz-acl, x-amz-copy-source, and x-amz-metadata-directive
+// headers if necessary, and always adds the x-amz-date header. It copies the
+// raw string values into params->amzHeadersRaw, and creates an array of
+// string pointers representing these headers in params->amzHeaders (and also
+// sets params->amzHeadersCount to be the count of the total number of x-amz-
+// headers thus created).
+static S3Status compose_amz_headers(const RequestParams *params,
+ RequestComputedValues *values)
+{
+ const S3PutProperties *properties = params->putProperties;
+
+ values->amzHeadersCount = 0;
+ values->amzHeadersRaw[0] = 0;
+ int len = 0;
+
+ // Append a header to amzHeaders, trimming whitespace from the end.
+ // Does NOT trim whitespace from the beginning.
+#define headers_append(isNewHeader, format, ...) \
+ do { \
+ if (isNewHeader) { \
+ values->amzHeaders[values->amzHeadersCount++] = \
+ &(values->amzHeadersRaw[len]); \
+ } \
+ len += snprintf(&(values->amzHeadersRaw[len]), \
+ sizeof(values->amzHeadersRaw) - len, \
+ format, __VA_ARGS__); \
+ if (len >= (int) sizeof(values->amzHeadersRaw)) { \
+ return S3StatusMetaDataHeadersTooLong; \
+ } \
+ while ((len > 0) && (values->amzHeadersRaw[len - 1] == ' ')) { \
+ len--; \
+ } \
+ values->amzHeadersRaw[len++] = 0; \
+ } while (0)
+
+#define header_name_tolower_copy(str, l) \
+ do { \
+ values->amzHeaders[values->amzHeadersCount++] = \
+ &(values->amzHeadersRaw[len]); \
+ if ((len + l) >= (int) sizeof(values->amzHeadersRaw)) { \
+ return S3StatusMetaDataHeadersTooLong; \
+ } \
+ int todo = l; \
+ while (todo--) { \
+ if ((*(str) >= 'A') && (*(str) <= 'Z')) { \
+ values->amzHeadersRaw[len++] = 'a' + (*(str) - 'A'); \
+ } \
+ else { \
+ values->amzHeadersRaw[len++] = *(str); \
+ } \
+ (str)++; \
+ } \
+ } while (0)
+
+ // Check and copy in the x-amz-meta headers
+ if (properties) {
+ int i;
+ for (i = 0; i < properties->metaDataCount; i++) {
+ const S3NameValue *property = &(properties->metaData[i]);
+ char headerName[S3_MAX_METADATA_SIZE - sizeof(": v")];
+ int l = snprintf(headerName, sizeof(headerName),
+ S3_METADATA_HEADER_NAME_PREFIX "%s",
+ property->name);
+ char *hn = headerName;
+ header_name_tolower_copy(hn, l);
+ // Copy in the value
+ headers_append(0, ": %s", property->value);
+ }
+
+ // Add the x-amz-acl header, if necessary
+ const char *cannedAclString;
+ switch (params->putProperties->cannedAcl) {
+ case S3CannedAclPrivate:
+ cannedAclString = 0;
+ break;
+ case S3CannedAclPublicRead:
+ cannedAclString = "public-read";
+ break;
+ case S3CannedAclPublicReadWrite:
+ cannedAclString = "public-read-write";
+ break;
+ default: // S3CannedAclAuthenticatedRead
+ cannedAclString = "authenticated-read";
+ break;
+ }
+ if (cannedAclString) {
+ headers_append(1, "x-amz-acl: %s", cannedAclString);
+ }
+ }
+
+ // Add the x-amz-date header
+ time_t now = time(NULL);
+ char date[64];
+ strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&now));
+ headers_append(1, "x-amz-date: %s", date);
+
+ if (params->httpRequestType == HttpRequestTypeCOPY) {
+ // Add the x-amz-copy-source header
+ if (params->copySourceBucketName && params->copySourceBucketName[0] &&
+ params->copySourceKey && params->copySourceKey[0]) {
+ headers_append(1, "x-amz-copy-source: /%s/%s",
+ params->copySourceBucketName,
+ params->copySourceKey);
+ }
+ // And the x-amz-metadata-directive header
+ if (params->putProperties) {
+ headers_append(1, "%s", "x-amz-metadata-directive: REPLACE");
+ }
+ }
+
+ return S3StatusOK;
+}
+
+
+// Composes the other headers
+static S3Status compose_standard_headers(const RequestParams *params,
+ RequestComputedValues *values)
+{
+
+#define do_put_header(fmt, sourceField, destField, badError, tooLongError) \
+ do { \
+ if (params->putProperties && \
+ params->putProperties-> sourceField && \
+ params->putProperties-> sourceField[0]) { \
+ /* Skip whitespace at beginning of val */ \
+ const char *val = params->putProperties-> sourceField; \
+ while (*val && is_blank(*val)) { \
+ val++; \
+ } \
+ if (!*val) { \
+ return badError; \
+ } \
+ /* Compose header, make sure it all fit */ \
+ int len = snprintf(values-> destField, \
+ sizeof(values-> destField), fmt, val); \
+ if (len >= (int) sizeof(values-> destField)) { \
+ return tooLongError; \
+ } \
+ /* Now remove the whitespace at the end */ \
+ while (is_blank(values-> destField[len])) { \
+ len--; \
+ } \
+ values-> destField[len] = 0; \
+ } \
+ else { \
+ values-> destField[0] = 0; \
+ } \
+ } while (0)
+
+#define do_get_header(fmt, sourceField, destField, badError, tooLongError) \
+ do { \
+ if (params->getConditions && \
+ params->getConditions-> sourceField && \
+ params->getConditions-> sourceField[0]) { \
+ /* Skip whitespace at beginning of val */ \
+ const char *val = params->getConditions-> sourceField; \
+ while (*val && is_blank(*val)) { \
+ val++; \
+ } \
+ if (!*val) { \
+ return badError; \
+ } \
+ /* Compose header, make sure it all fit */ \
+ int len = snprintf(values-> destField, \
+ sizeof(values-> destField), fmt, val); \
+ if (len >= (int) sizeof(values-> destField)) { \
+ return tooLongError; \
+ } \
+ /* Now remove the whitespace at the end */ \
+ while (is_blank(values-> destField[len])) { \
+ len--; \
+ } \
+ values-> destField[len] = 0; \
+ } \
+ else { \
+ values-> destField[0] = 0; \
+ } \
+ } while (0)
+
+ // Cache-Control
+ do_put_header("Cache-Control: %s", cacheControl, cacheControlHeader,
+ S3StatusBadCacheControl, S3StatusCacheControlTooLong);
+
+ // ContentType
+ do_put_header("Content-Type: %s", contentType, contentTypeHeader,
+ S3StatusBadContentType, S3StatusContentTypeTooLong);
+
+ // MD5
+ do_put_header("Content-MD5: %s", md5, md5Header, S3StatusBadMD5,
+ S3StatusMD5TooLong);
+
+ // Content-Disposition
+ do_put_header("Content-Disposition: attachment; filename=\"%s\"",
+ contentDispositionFilename, contentDispositionHeader,
+ S3StatusBadContentDispositionFilename,
+ S3StatusContentDispositionFilenameTooLong);
+
+ // ContentEncoding
+ do_put_header("Content-Encoding: %s", contentEncoding,
+ contentEncodingHeader, S3StatusBadContentEncoding,
+ S3StatusContentEncodingTooLong);
+
+ // Expires
+ if (params->putProperties && (params->putProperties->expires >= 0)) {
+ time_t t = (time_t) params->putProperties->expires;
+ strftime(values->expiresHeader, sizeof(values->expiresHeader),
+ "Expires: %a, %d %b %Y %H:%M:%S UTC", gmtime(&t));
+ }
+ else {
+ values->expiresHeader[0] = 0;
+ }
+
+ // If-Modified-Since
+ if (params->getConditions &&
+ (params->getConditions->ifModifiedSince >= 0)) {
+ time_t t = (time_t) params->getConditions->ifModifiedSince;
+ strftime(values->ifModifiedSinceHeader,
+ sizeof(values->ifModifiedSinceHeader),
+ "If-Modified-Since: %a, %d %b %Y %H:%M:%S UTC", gmtime(&t));
+ }
+ else {
+ values->ifModifiedSinceHeader[0] = 0;
+ }
+
+ // If-Unmodified-Since header
+ if (params->getConditions &&
+ (params->getConditions->ifNotModifiedSince >= 0)) {
+ time_t t = (time_t) params->getConditions->ifNotModifiedSince;
+ strftime(values->ifUnmodifiedSinceHeader,
+ sizeof(values->ifUnmodifiedSinceHeader),
+ "If-Unmodified-Since: %a, %d %b %Y %H:%M:%S UTC", gmtime(&t));
+ }
+ else {
+ values->ifUnmodifiedSinceHeader[0] = 0;
+ }
+
+ // If-Match header
+ do_get_header("If-Match: %s", ifMatchETag, ifMatchHeader,
+ S3StatusBadIfMatchETag, S3StatusIfMatchETagTooLong);
+
+ // If-None-Match header
+ do_get_header("If-None-Match: %s", ifNotMatchETag, ifNoneMatchHeader,
+ S3StatusBadIfNotMatchETag,
+ S3StatusIfNotMatchETagTooLong);
+
+ // Range header
+ if (params->startByte || params->byteCount) {
+ if (params->byteCount) {
+ snprintf(values->rangeHeader, sizeof(values->rangeHeader),
+ "Range: bytes=%llu-%llu",
+ (unsigned long long) params->startByte,
+ (unsigned long long) (params->startByte +
+ params->byteCount - 1));
+ }
+ else {
+ snprintf(values->rangeHeader, sizeof(values->rangeHeader),
+ "Range: bytes=%llu-",
+ (unsigned long long) params->startByte);
+ }
+ }
+ else {
+ values->rangeHeader[0] = 0;
+ }
+
+ return S3StatusOK;
+}
+
+
+// URL encodes the params->key value into params->urlEncodedKey
+static S3Status encode_key(const RequestParams *params,
+ RequestComputedValues *values)
+{
+ return (urlEncode(values->urlEncodedKey, params->key, S3_MAX_KEY_SIZE) ?
+ S3StatusOK : S3StatusUriTooLong);
+}
+
+
+// Simple comparison function for comparing two HTTP header names that are
+// embedded within an HTTP header line, returning true if header1 comes
+// before header2 alphabetically, false if not
+static int headerle(const char *header1, const char *header2)
+{
+ while (1) {
+ if (*header1 == ':') {
+ return (*header2 == ':');
+ }
+ else if (*header2 == ':') {
+ return 0;
+ }
+ else if (*header2 < *header1) {
+ return 0;
+ }
+ else if (*header2 > *header1) {
+ return 1;
+ }
+ header1++, header2++;
+ }
+}
+
+
+// Replace this with merge sort eventually, it's the best stable sort. But
+// since typically the number of elements being sorted is small, it doesn't
+// matter that much which sort is used, and gnome sort is the world's simplest
+// stable sort. Added a slight twist to the standard gnome_sort - don't go
+// forward +1, go forward to the last highest index considered. This saves
+// all the string comparisons that would be done "going forward", and thus
+// only does the necessary string comparisons to move values back into their
+// sorted position.
+static void header_gnome_sort(const char **headers, int size)
+{
+ int i = 0, last_highest = 0;
+
+ while (i < size) {
+ if ((i == 0) || headerle(headers[i - 1], headers[i])) {
+ i = ++last_highest;
+ }
+ else {
+ const char *tmp = headers[i];
+ headers[i] = headers[i - 1];
+ headers[--i] = tmp;
+ }
+ }
+}
+
+
+// Canonicalizes the x-amz- headers into the canonicalizedAmzHeaders buffer
+static void canonicalize_amz_headers(RequestComputedValues *values)
+{
+ // Make a copy of the headers that will be sorted
+ const char *sortedHeaders[S3_MAX_METADATA_COUNT];
+
+ memcpy(sortedHeaders, values->amzHeaders,
+ (values->amzHeadersCount * sizeof(sortedHeaders[0])));
+
+ // Now sort these
+ header_gnome_sort(sortedHeaders, values->amzHeadersCount);
+
+ // Now copy this sorted list into the buffer, all the while:
+ // - folding repeated headers into single lines, and
+ // - folding multiple lines
+ // - removing the space after the colon
+ int lastHeaderLen = 0, i;
+ char *buffer = values->canonicalizedAmzHeaders;
+ for (i = 0; i < values->amzHeadersCount; i++) {
+ const char *header = sortedHeaders[i];
+ const char *c = header;
+ // If the header names are the same, append the next value
+ if ((i > 0) &&
+ !strncmp(header, sortedHeaders[i - 1], lastHeaderLen)) {
+ // Replacing the previous newline with a comma
+ *(buffer - 1) = ',';
+ // Skip the header name and space
+ c += (lastHeaderLen + 1);
+ }
+ // Else this is a new header
+ else {
+ // Copy in everything up to the space in the ": "
+ while (*c != ' ') {
+ *buffer++ = *c++;
+ }
+ // Save the header len since it's a new header
+ lastHeaderLen = c - header;
+ // Skip the space
+ c++;
+ }
+ // Now copy in the value, folding the lines
+ while (*c) {
+ // If c points to a \r\n[whitespace] sequence, then fold
+ // this newline out
+ if ((*c == '\r') && (*(c + 1) == '\n') && is_blank(*(c + 2))) {
+ c += 3;
+ while (is_blank(*c)) {
+ c++;
+ }
+ // Also, what has most recently been copied into buffer amy
+ // have been whitespace, and since we're folding whitespace
+ // out around this newline sequence, back buffer up over
+ // any whitespace it contains
+ while (is_blank(*(buffer - 1))) {
+ buffer--;
+ }
+ continue;
+ }
+ *buffer++ = *c++;
+ }
+ // Finally, add the newline
+ *buffer++ = '\n';
+ }
+
+ // Terminate the buffer
+ *buffer = 0;
+}
+
+
+// Canonicalizes the resource into params->canonicalizedResource
+static void canonicalize_resource(const char *bucketName,
+ const char *subResource,
+ const char *urlEncodedKey,
+ char *buffer)
+{
+ int len = 0;
+
+ *buffer = 0;
+
+#define append(str) len += sprintf(&(buffer[len]), "%s", str)
+
+ if (bucketName && bucketName[0]) {
+ buffer[len++] = '/';
+ append(bucketName);
+ }
+
+ append("/");
+
+ if (urlEncodedKey && urlEncodedKey[0]) {
+ append(urlEncodedKey);
+ }
+
+ if (subResource && subResource[0]) {
+ append("?");
+ append(subResource);
+ }
+}
+
+
+// Convert an HttpRequestType to an HTTP Verb string
+static const char *http_request_type_to_verb(HttpRequestType requestType)
+{
+ switch (requestType) {
+ case HttpRequestTypeGET:
+ return "GET";
+ case HttpRequestTypeHEAD:
+ return "HEAD";
+ case HttpRequestTypePUT:
+ case HttpRequestTypeCOPY:
+ return "PUT";
+ default: // HttpRequestTypeDELETE
+ return "DELETE";
+ }
+}
+
+
+// Composes the Authorization header for the request
+static S3Status compose_auth_header(const RequestParams *params,
+ RequestComputedValues *values)
+{
+ // We allow for:
+ // 17 bytes for HTTP-Verb + \n
+ // 129 bytes for Content-MD5 + \n
+ // 129 bytes for Content-Type + \n
+ // 1 byte for empty Date + \n
+ // CanonicalizedAmzHeaders & CanonicalizedResource
+ char signbuf[17 + 129 + 129 + 1 +
+ (sizeof(values->canonicalizedAmzHeaders) - 1) +
+ (sizeof(values->canonicalizedResource) - 1) + 1];
+ int len = 0;
+
+#define signbuf_append(format, ...) \
+ len += snprintf(&(signbuf[len]), sizeof(signbuf) - len, \
+ format, __VA_ARGS__)
+
+ signbuf_append
+ ("%s\n", http_request_type_to_verb(params->httpRequestType));
+
+ // For MD5 and Content-Type, use the value in the actual header, because
+ // it's already been trimmed
+ signbuf_append("%s\n", values->md5Header[0] ?
+ &(values->md5Header[sizeof("Content-MD5: ") - 1]) : "");
+
+ signbuf_append
+ ("%s\n", values->contentTypeHeader[0] ?
+ &(values->contentTypeHeader[sizeof("Content-Type: ") - 1]) : "");
+
+ signbuf_append("%s", "\n"); // Date - we always use x-amz-date
+
+ signbuf_append("%s", values->canonicalizedAmzHeaders);
+
+ signbuf_append("%s", values->canonicalizedResource);
+
+ // Generate an HMAC-SHA-1 of the signbuf
+ unsigned char hmac[20];
+
+ HMAC_SHA1(hmac, (unsigned char *) params->bucketContext.secretAccessKey,
+ strlen(params->bucketContext.secretAccessKey),
+ (unsigned char *) signbuf, len);
+
+ // Now base-64 encode the results
+ char b64[((20 + 1) * 4) / 3];
+ int b64Len = base64Encode(hmac, 20, b64);
+
+ snprintf(values->authorizationHeader, sizeof(values->authorizationHeader),
+ "Authorization: AWS %s:%.*s", params->bucketContext.accessKeyId,
+ b64Len, b64);
+
+ return S3StatusOK;
+}
+
+
+// Compose the URI to use for the request given the request parameters
+static S3Status compose_uri(char *buffer, int bufferSize,
+ const S3BucketContext *bucketContext,
+ const char *urlEncodedKey,
+ const char *subResource, const char *queryParams)
+{
+ int len = 0;
+
+#define uri_append(fmt, ...) \
+ do { \
+ len += snprintf(&(buffer[len]), bufferSize - len, fmt, __VA_ARGS__); \
+ if (len >= bufferSize) { \
+ return S3StatusUriTooLong; \
+ } \
+ } while (0)
+
+ uri_append("http%s://",
+ (bucketContext->protocol == S3ProtocolHTTP) ? "" : "s");
+
+ if (bucketContext->bucketName &&
+ bucketContext->bucketName[0]) {
+ if (bucketContext->uriStyle == S3UriStyleVirtualHost) {
+ uri_append("%s.s3.amazonaws.com", bucketContext->bucketName);
+ }
+ else {
+ uri_append("s3.amazonaws.com/%s", bucketContext->bucketName);
+ }
+ }
+ else {
+ uri_append("%s", "s3.amazonaws.com");
+ }
+
+ uri_append("%s", "/");
+
+ uri_append("%s", urlEncodedKey);
+
+ if (subResource && subResource[0]) {
+ uri_append("?%s", subResource);
+ }
+
+ if (queryParams) {
+ uri_append("%s%s", (subResource && subResource[0]) ? "&" : "?",
+ queryParams);
+ }
+
+ return S3StatusOK;
+}
+
+
+// Sets up the curl handle given the completely computed RequestParams
+static S3Status setup_curl(Request *request,
+ const RequestParams *params,
+ const RequestComputedValues *values)
+{
+ CURLcode status;
+
+#define curl_easy_setopt_safe(opt, val) \
+ if ((status = curl_easy_setopt \
+ (request->curl, opt, val)) != CURLE_OK) { \
+ return S3StatusFailedToInitializeRequest; \
+ }
+
+ // Debugging only
+ // curl_easy_setopt_safe(CURLOPT_VERBOSE, 1);
+
+ // Set private data to request for the benefit of S3RequestContext
+ curl_easy_setopt_safe(CURLOPT_PRIVATE, request);
+
+ // Set header callback and data
+ curl_easy_setopt_safe(CURLOPT_HEADERDATA, request);
+ curl_easy_setopt_safe(CURLOPT_HEADERFUNCTION, &curl_header_func);
+
+ // Set read callback, data, and readSize
+ curl_easy_setopt_safe(CURLOPT_READFUNCTION, &curl_read_func);
+ curl_easy_setopt_safe(CURLOPT_READDATA, request);
+
+ // Set write callback and data
+ curl_easy_setopt_safe(CURLOPT_WRITEFUNCTION, &curl_write_func);
+ curl_easy_setopt_safe(CURLOPT_WRITEDATA, request);
+
+ // Ask curl to parse the Last-Modified header. This is easier than
+ // parsing it ourselves.
+ curl_easy_setopt_safe(CURLOPT_FILETIME, 1);
+
+ // Curl docs suggest that this is necessary for multithreaded code.
+ // However, it also points out that DNS timeouts will not be honored
+ // during DNS lookup, which can be worked around by using the c-ares
+ // library, which we do not do yet.
+ curl_easy_setopt_safe(CURLOPT_NOSIGNAL, 1);
+
+ // Turn off Curl's built-in progress meter
+ curl_easy_setopt_safe(CURLOPT_NOPROGRESS, 1);
+
+ // xxx todo - support setting the proxy for Curl to use (can't use https
+ // for proxies though)
+
+ // xxx todo - support setting the network interface for Curl to use
+
+ // I think this is useful - we don't need interactive performance, we need
+ // to complete large operations quickly
+ curl_easy_setopt_safe(CURLOPT_TCP_NODELAY, 1);
+
+ // Don't use Curl's 'netrc' feature
+ curl_easy_setopt_safe(CURLOPT_NETRC, CURL_NETRC_IGNORED);
+
+ // Don't verify S3's certificate, there are known to be issues with
+ // them sometimes
+ // xxx todo - support an option for verifying the S3 CA (default false)
+ curl_easy_setopt_safe(CURLOPT_SSL_VERIFYPEER, 0);
+
+ // Follow any redirection directives that S3 sends
+ curl_easy_setopt_safe(CURLOPT_FOLLOWLOCATION, 1);
+
+ // A safety valve in case S3 goes bananas with redirects
+ curl_easy_setopt_safe(CURLOPT_MAXREDIRS, 10);
+
+ // Set the User-Agent; maybe Amazon will track these?
+ curl_easy_setopt_safe(CURLOPT_USERAGENT, userAgentG);
+
+ // Set the low speed limit and time; we abort transfers that stay at
+ // less than 1K per second for more than 15 seconds.
+ // xxx todo - make these configurable
+ // xxx todo - allow configurable max send and receive speed
+ curl_easy_setopt_safe(CURLOPT_LOW_SPEED_LIMIT, 1024);
+ curl_easy_setopt_safe(CURLOPT_LOW_SPEED_TIME, 15);
+
+ // Append standard headers
+#define append_standard_header(fieldName) \
+ if (values-> fieldName [0]) { \
+ request->headers = curl_slist_append(request->headers, \
+ values-> fieldName); \
+ }
+
+ // Would use CURLOPT_INFILESIZE_LARGE, but it is buggy in libcurl
+ if (params->httpRequestType == HttpRequestTypePUT) {
+ char header[256];
+ snprintf(header, sizeof(header), "Content-Length: %llu",
+ (unsigned long long) params->toS3CallbackTotalSize);
+ request->headers = curl_slist_append(request->headers, header);
+ request->headers = curl_slist_append(request->headers,
+ "Transfer-Encoding:");
+ }
+ else if (params->httpRequestType == HttpRequestTypeCOPY) {
+ request->headers = curl_slist_append(request->headers,
+ "Transfer-Encoding:");
+ }
+
+ append_standard_header(cacheControlHeader);
+ append_standard_header(contentTypeHeader);
+ append_standard_header(md5Header);
+ append_standard_header(contentDispositionHeader);
+ append_standard_header(contentEncodingHeader);
+ append_standard_header(expiresHeader);
+ append_standard_header(ifModifiedSinceHeader);
+ append_standard_header(ifUnmodifiedSinceHeader);
+ append_standard_header(ifMatchHeader);
+ append_standard_header(ifNoneMatchHeader);
+ append_standard_header(rangeHeader);
+ append_standard_header(authorizationHeader);
+
+ // Append x-amz- headers
+ int i;
+ for (i = 0; i < values->amzHeadersCount; i++) {
+ request->headers =
+ curl_slist_append(request->headers, values->amzHeaders[i]);
+ }
+
+ // Set the HTTP headers
+ curl_easy_setopt_safe(CURLOPT_HTTPHEADER, request->headers);
+
+ // Set URI
+ curl_easy_setopt_safe(CURLOPT_URL, request->uri);
+
+ // Set request type.
+ switch (params->httpRequestType) {
+ case HttpRequestTypeHEAD:
+ curl_easy_setopt_safe(CURLOPT_NOBODY, 1);
+ break;
+ case HttpRequestTypePUT:
+ case HttpRequestTypeCOPY:
+ curl_easy_setopt_safe(CURLOPT_UPLOAD, 1);
+ break;
+ case HttpRequestTypeDELETE:
+ curl_easy_setopt_safe(CURLOPT_CUSTOMREQUEST, "DELETE");
+ break;
+ default: // HttpRequestTypeGET
+ break;
+ }
+
+ return S3StatusOK;
+}
+
+
+static void request_deinitialize(Request *request)
+{
+ if (request->headers) {
+ curl_slist_free_all(request->headers);
+ }
+
+ error_parser_deinitialize(&(request->errorParser));
+
+ // curl_easy_reset prevents connections from being re-used for some
+ // reason. This makes HTTP Keep-Alive meaningless and is very bad for
+ // performance. But it is necessary to allow curl to work properly.
+ // xxx todo figure out why
+ curl_easy_reset(request->curl);
+}
+
+
+static S3Status request_get(const RequestParams *params,
+ const RequestComputedValues *values,
+ Request **reqReturn)
+{
+ Request *request = 0;
+
+ // Try to get one from the request stack. We hold the lock for the
+ // shortest time possible here.
+ pthread_mutex_lock(&requestStackMutexG);
+
+ if (requestStackCountG) {
+ request = requestStackG[--requestStackCountG];
+ }
+
+ pthread_mutex_unlock(&requestStackMutexG);
+
+ // If we got one, deinitialize it for re-use
+ if (request) {
+ request_deinitialize(request);
+ }
+ // Else there wasn't one available in the request stack, so create one
+ else {
+ if (!(request = (Request *) malloc(sizeof(Request)))) {
+ return S3StatusOutOfMemory;
+ }
+ if (!(request->curl = curl_easy_init())) {
+ free(request);
+ return S3StatusFailedToInitializeRequest;
+ }
+ }
+
+ // Initialize the request
+ request->prev = 0;
+ request->next = 0;
+
+ // Request status is initialized to no error, will be updated whenever
+ // an error occurs
+ request->status = S3StatusOK;
+
+ S3Status status;
+
+ // Start out with no headers
+ request->headers = 0;
+
+ // Compute the URL
+ if ((status = compose_uri
+ (request->uri, sizeof(request->uri),
+ &(params->bucketContext), values->urlEncodedKey,
+ params->subResource, params->queryParams)) != S3StatusOK) {
+ curl_easy_cleanup(request->curl);
+ free(request);
+ return status;
+ }
+
+ // Set all of the curl handle options
+ if ((status = setup_curl(request, params, values)) != S3StatusOK) {
+ curl_easy_cleanup(request->curl);
+ free(request);
+ return status;
+ }
+
+ request->propertiesCallback = params->propertiesCallback;
+
+ request->toS3Callback = params->toS3Callback;
+
+ request->toS3CallbackBytesRemaining = params->toS3CallbackTotalSize;
+
+ request->fromS3Callback = params->fromS3Callback;
+
+ request->completeCallback = params->completeCallback;
+
+ request->callbackData = params->callbackData;
+
+ response_headers_handler_initialize(&(request->responseHeadersHandler));
+
+ request->propertiesCallbackMade = 0;
+
+ error_parser_initialize(&(request->errorParser));
+
+ *reqReturn = request;
+
+ return S3StatusOK;
+}
+
+
+static void request_destroy(Request *request)
+{
+ request_deinitialize(request);
+ curl_easy_cleanup(request->curl);
+ free(request);
+}
+
+
+static void request_release(Request *request)
+{
+ pthread_mutex_lock(&requestStackMutexG);
+
+ // If the request stack is full, destroy this one
+ if (requestStackCountG == REQUEST_STACK_SIZE) {
+ pthread_mutex_unlock(&requestStackMutexG);
+ request_destroy(request);
+ }
+ // Else put this one at the front of the request stack; we do this because
+ // we want the most-recently-used curl handle to be re-used on the next
+ // request, to maximize our chances of re-using a TCP connection before it
+ // times out
+ else {
+ requestStackG[requestStackCountG++] = request;
+ pthread_mutex_unlock(&requestStackMutexG);
+ }
+}
+
+
+S3Status request_api_initialize(const char *userAgentInfo, int flags)
+{
+ if (curl_global_init(CURL_GLOBAL_ALL &
+ ~((flags & S3_INIT_WINSOCK) ? 0 : CURL_GLOBAL_WIN32))
+ != CURLE_OK) {
+ return S3StatusInternalError;
+ }
+
+ pthread_mutex_init(&requestStackMutexG, 0);
+
+ requestStackCountG = 0;
+
+ if (!userAgentInfo || !*userAgentInfo) {
+ userAgentInfo = "Unknown";
+ }
+
+ char platform[96];
+ struct utsname utsn;
+ if (uname(&utsn)) {
+ strncpy(platform, "Unknown", sizeof(platform));
+ // Because strncpy doesn't always zero terminate
+ platform[sizeof(platform) - 1] = 0;
+ }
+ else {
+ snprintf(platform, sizeof(platform), "%s%s%s", utsn.sysname,
+ utsn.machine[0] ? " " : "", utsn.machine);
+ }
+
+ snprintf(userAgentG, sizeof(userAgentG),
+ "Mozilla/4.0 (Compatible; %s; libs3 %s.%s; %s)",
+ userAgentInfo, LIBS3_VER_MAJOR, LIBS3_VER_MINOR, platform);
+
+ return S3StatusOK;
+}
+
+
+void request_api_deinitialize()
+{
+ pthread_mutex_destroy(&requestStackMutexG);
+
+ while (requestStackCountG--) {
+ request_destroy(requestStackG[requestStackCountG]);
+ }
+}
+
+
+void request_perform(const RequestParams *params, S3RequestContext *context)
+{
+ Request *request;
+ S3Status status;
+
+#define return_status(status) \
+ (*(params->completeCallback))(status, 0, params->callbackData); \
+ return
+
+ // These will hold the computed values
+ RequestComputedValues computed;
+
+ // Validate the bucket name
+ if (params->bucketContext.bucketName &&
+ ((status = S3_validate_bucket_name
+ (params->bucketContext.bucketName,
+ params->bucketContext.uriStyle)) != S3StatusOK)) {
+ return_status(status);
+ }
+
+ // Compose the amz headers
+ if ((status = compose_amz_headers(params, &computed)) != S3StatusOK) {
+ return_status(status);
+ }
+
+ // Compose standard headers
+ if ((status = compose_standard_headers
+ (params, &computed)) != S3StatusOK) {
+ return_status(status);
+ }
+
+ // URL encode the key
+ if ((status = encode_key(params, &computed)) != S3StatusOK) {
+ return_status(status);
+ }
+
+ // Compute the canonicalized amz headers
+ canonicalize_amz_headers(&computed);
+
+ // Compute the canonicalized resource
+ canonicalize_resource(params->bucketContext.bucketName,
+ params->subResource, computed.urlEncodedKey,
+ computed.canonicalizedResource);
+
+ // Compose Authorization header
+ if ((status = compose_auth_header(params, &computed)) != S3StatusOK) {
+ return_status(status);
+ }
+
+ // Get an initialized Request structure now
+ if ((status = request_get(params, &computed, &request)) != S3StatusOK) {
+ return_status(status);
+ }
+
+ // If a RequestContext was provided, add the request to the curl multi
+ if (context) {
+ CURLMcode code = curl_multi_add_handle(context->curlm, request->curl);
+ if (code == CURLM_OK) {
+ if (context->requests) {
+ request->prev = context->requests->prev;
+ request->next = context->requests;
+ context->requests->prev->next = request;
+ context->requests->prev = request;
+ }
+ else {
+ context->requests = request->next = request->prev = request;
+ }
+ }
+ else {
+ if (request->status == S3StatusOK) {
+ request->status = (code == CURLM_OUT_OF_MEMORY) ?
+ S3StatusOutOfMemory : S3StatusInternalError;
+ }
+ request_finish(request);
+ }
+ }
+ // Else, perform the request immediately
+ else {
+ CURLcode code = curl_easy_perform(request->curl);
+ if ((code != CURLE_OK) && (request->status == S3StatusOK)) {
+ request->status = request_curl_code_to_status(code);
+ }
+ // Finish the request, ensuring that all callbacks have been made, and
+ // also releases the request
+ request_finish(request);
+ }
+}
+
+
+void request_finish(Request *request)
+{
+ // If we haven't detected this already, we now know that the headers are
+ // definitely done being read in
+ request_headers_done(request);
+
+ // If there was no error processing the request, then possibly there was
+ // an S3 error parsed, which should be converted into the request status
+ if (request->status == S3StatusOK) {
+ error_parser_convert_status(&(request->errorParser),
+ &(request->status));
+ // If there still was no error recorded, then it is possible that
+ // there was in fact an error but that there was no error XML
+ // detailing the error
+ if ((request->status == S3StatusOK) &&
+ ((request->httpResponseCode < 200) ||
+ (request->httpResponseCode > 299))) {
+ switch (request->httpResponseCode) {
+ case 0:
+ // This happens if the request never got any HTTP response
+ // headers at all, we call this a ConnectionFailed error
+ request->status = S3StatusConnectionFailed;
+ break;
+ case 100: // Some versions of libcurl erroneously set HTTP
+ // status to this
+ break;
+ case 301:
+ request->status = S3StatusErrorPermanentRedirect;
+ break;
+ case 307:
+ request->status = S3StatusHttpErrorMovedTemporarily;
+ break;
+ case 400:
+ request->status = S3StatusHttpErrorBadRequest;
+ break;
+ case 403:
+ request->status = S3StatusHttpErrorForbidden;
+ break;
+ case 404:
+ request->status = S3StatusHttpErrorNotFound;
+ break;
+ case 405:
+ request->status = S3StatusErrorMethodNotAllowed;
+ break;
+ case 409:
+ request->status = S3StatusHttpErrorConflict;
+ break;
+ case 411:
+ request->status = S3StatusErrorMissingContentLength;
+ break;
+ case 412:
+ request->status = S3StatusErrorPreconditionFailed;
+ break;
+ case 416:
+ request->status = S3StatusErrorInvalidRange;
+ break;
+ case 500:
+ request->status = S3StatusErrorInternalError;
+ break;
+ case 501:
+ request->status = S3StatusErrorNotImplemented;
+ break;
+ case 503:
+ request->status = S3StatusErrorSlowDown;
+ break;
+ default:
+ request->status = S3StatusHttpErrorUnknown;
+ break;
+ }
+ }
+ }
+
+ (*(request->completeCallback))
+ (request->status, &(request->errorParser.s3ErrorDetails),
+ request->callbackData);
+
+ request_release(request);
+}
+
+
+S3Status request_curl_code_to_status(CURLcode code)
+{
+ switch (code) {
+ case CURLE_OUT_OF_MEMORY:
+ return S3StatusOutOfMemory;
+ case CURLE_COULDNT_RESOLVE_PROXY:
+ case CURLE_COULDNT_RESOLVE_HOST:
+ return S3StatusNameLookupError;
+ case CURLE_COULDNT_CONNECT:
+ return S3StatusFailedToConnect;
+ case CURLE_WRITE_ERROR:
+ case CURLE_OPERATION_TIMEDOUT:
+ return S3StatusConnectionFailed;
+ case CURLE_PARTIAL_FILE:
+ return S3StatusOK;
+ case CURLE_SSL_CACERT:
+ return S3StatusServerFailedVerification;
+ default:
+ return S3StatusInternalError;
+ }
+}
+
+
+S3Status S3_generate_authenticated_query_string
+ (char *buffer, const S3BucketContext *bucketContext,
+ const char *key, int64_t expires, const char *resource)
+{
+#define MAX_EXPIRES (((int64_t) 1 << 31) - 1)
+ // S3 seems to only accept expiration dates up to the number of seconds
+ // representably by a signed 32-bit integer
+ if (expires < 0) {
+ expires = MAX_EXPIRES;
+ }
+ else if (expires > MAX_EXPIRES) {
+ expires = MAX_EXPIRES;
+ }
+
+ // xxx todo: rework this so that it can be incorporated into shared code
+ // with request_perform(). It's really unfortunate that this code is not
+ // shared with request_perform().
+
+ // URL encode the key
+ char urlEncodedKey[S3_MAX_KEY_SIZE * 3];
+ if (key) {
+ urlEncode(urlEncodedKey, key, strlen(key));
+ }
+ else {
+ urlEncodedKey[0] = 0;
+ }
+
+ // Compute canonicalized resource
+ char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE];
+ canonicalize_resource(bucketContext->bucketName, resource, urlEncodedKey,
+ canonicalizedResource);
+
+ // We allow for:
+ // 17 bytes for HTTP-Verb + \n
+ // 1 byte for empty Content-MD5 + \n
+ // 1 byte for empty Content-Type + \n
+ // 20 bytes for Expires + \n
+ // 0 bytes for CanonicalizedAmzHeaders
+ // CanonicalizedResource
+ char signbuf[17 + 1 + 1 + 1 + 20 + sizeof(canonicalizedResource) + 1];
+ int len = 0;
+
+#define signbuf_append(format, ...) \
+ len += snprintf(&(signbuf[len]), sizeof(signbuf) - len, \
+ format, __VA_ARGS__)
+
+ signbuf_append("%s\n", "GET"); // HTTP-Verb
+ signbuf_append("%s\n", ""); // Content-MD5
+ signbuf_append("%s\n", ""); // Content-Type
+ signbuf_append("%llu\n", (unsigned long long) expires);
+ signbuf_append("%s", canonicalizedResource);
+
+ // Generate an HMAC-SHA-1 of the signbuf
+ unsigned char hmac[20];
+
+ HMAC_SHA1(hmac, (unsigned char *) bucketContext->secretAccessKey,
+ strlen(bucketContext->secretAccessKey),
+ (unsigned char *) signbuf, len);
+
+ // Now base-64 encode the results
+ char b64[((20 + 1) * 4) / 3];
+ int b64Len = base64Encode(hmac, 20, b64);
+
+ // Now urlEncode that
+ char signature[sizeof(b64) * 3];
+ urlEncode(signature, b64, b64Len);
+
+ // Finally, compose the uri, with params:
+ // ?AWSAccessKeyId=xxx[&Expires=]&Signature=xxx
+ char queryParams[sizeof("AWSAccessKeyId=") + 20 +
+ sizeof("&Expires=") + 20 +
+ sizeof("&Signature=") + sizeof(signature) + 1];
+
+ sprintf(queryParams, "AWSAccessKeyId=%s&Expires=%ld&Signature=%s",
+ bucketContext->accessKeyId, (long) expires, signature);
+
+ return compose_uri(buffer, S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE,
+ bucketContext, urlEncodedKey, resource, queryParams);
+}
projects / bluesky.git / blobdiff