diff --git a/README b/README index f706f59..40ae36e 100644 --- a/README +++ b/README @@ -104,8 +104,9 @@ BUGS/SHORTCOMINGS: containers in cloudfiles. * Directory entries are created as empty files with the content-type "application/directory". - * Cloud Files limits container and object listings to 10,000 items. - cloudfuse won't list more than that many files in a single directory. + * Cloud Files container and object listings no more limited to 10,000 items. + * Hard-link calls are mapped to Swift Copy Object requests: + http://docs.openstack.org/api/openstack-object-storage/1.0/content/copy-object.html AWESOME CONTRIBUTORS: @@ -118,6 +119,8 @@ AWESOME CONTRIBUTORS: * David Brownlee https://github.com/abs0 * Mike Lundy https://github.com/novas0x2a * justinb https://github.com/justinsb + * Am1GO https://github.com/Am1GO + * nashyeung https://github.com/nashyeung Thanks, and I hope you find it useful. diff --git a/cloudfsapi.c b/cloudfsapi.c index 583964d..4b4fa23 100644 --- a/cloudfsapi.c +++ b/cloudfsapi.c @@ -17,20 +17,35 @@ #include #include "cloudfsapi.h" #include "config.h" +#include #define RHEL5_LIBCURL_VERSION 462597 #define RHEL5_CERTIFICATE_FILE "/etc/pki/tls/certs/ca-bundle.crt" -#define REQUEST_RETRIES 4 +#define REQUEST_RETRIES 24 + +// defined by Rackspace +#define MAX_RESULTS_PER_REQUEST 10000 static char storage_url[MAX_URL_SIZE]; static char storage_token[MAX_HEADER_SIZE]; static pthread_mutex_t pool_mut; -static CURL *curl_pool[1024]; +static CURL *curl_pool[4096]; static int curl_pool_count = 0; static int debug = 0; static int verify_ssl = 1; static int rhel5_mode = 0; +static struct statvfs statcache = { + .f_bsize = 1, + .f_frsize = 1, + .f_blocks = INT_MAX, + .f_bfree = INT_MAX, + .f_bavail = INT_MAX, + .f_files = INT_MAX, + .f_ffree = INT_MAX, + .f_favail = INT_MAX, + .f_namemax = INT_MAX +}; #ifdef HAVE_OPENSSL #include @@ -92,6 +107,30 @@ static void add_header(curl_slist **headers, const char *name, *headers = curl_slist_append(*headers, x_header); } +static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *stream) +{ + debugf("Dispatching response headers"); + char *header = (char *)alloca(size * nmemb + 1); + char *head = (char *)alloca(size * nmemb + 1); + char *value = (char *)alloca(size * nmemb + 1); + memcpy(header, (char *)ptr, size * nmemb); + header[size * nmemb] = '\0'; + if (sscanf(header, "%[^:]: %[^\r\n]", head, value) == 2) + { + if (!strncasecmp(head, "x-auth-token", size * nmemb)) + strncpy(storage_token, value, sizeof(storage_token)); + if (!strncasecmp(head, "x-storage-url", size * nmemb)) + strncpy(storage_url, value, sizeof(storage_url)); + if (!strncasecmp(head, "x-account-meta-quota", size * nmemb)) + statcache.f_blocks = strtoul(value, NULL, 10); + if (!strncasecmp(head, "x-account-bytes-used", size * nmemb)) + statcache.f_bfree = statcache.f_bavail = statcache.f_blocks - strtoul(value, NULL, 10); + if (!strncasecmp(head, "x-account-object-count", size * nmemb)) + statcache.f_ffree = statcache.f_bavail = statcache.f_files - strtoul(value, NULL, 10); + } + return size * nmemb; +} + static int send_request(char *method, const char *path, FILE *fp, xmlParserCtxtPtr xmlctx, curl_slist *extra_headers) { @@ -164,7 +203,10 @@ static int send_request(char *method, const char *path, FILE *fp, } } else + { curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_dispatch); + } /* add the headers from extra_headers if any */ curl_slist *extra; for (extra = extra_headers; extra; extra = extra->next) @@ -180,6 +222,8 @@ static int send_request(char *method, const char *path, FILE *fp, return_connection(curl); if (response >= 200 && response < 400) return response; + if (response == 404 && !strcasecmp(method, "DELETE")) + return response; sleep(8 << tries); // backoff if (response == 401 && !cloudfs_connect()) // re-authenticate on 401s return response; @@ -189,23 +233,6 @@ static int send_request(char *method, const char *path, FILE *fp, return response; } -static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *stream) -{ - char *header = (char *)alloca(size * nmemb + 1); - char *head = (char *)alloca(size * nmemb + 1); - char *value = (char *)alloca(size * nmemb + 1); - memcpy(header, (char *)ptr, size * nmemb); - header[size * nmemb] = '\0'; - if (sscanf(header, "%[^:]: %[^\r\n]", head, value) == 2) - { - if (!strncasecmp(head, "x-auth-token", size * nmemb)) - strncpy(storage_token, value, sizeof(storage_token)); - if (!strncasecmp(head, "x-storage-url", size * nmemb)) - strncpy(storage_url, value, sizeof(storage_url)); - } - return size * nmemb; -} - /* * Public interface */ @@ -284,17 +311,16 @@ int cloudfs_object_truncate(const char *path, off_t size) return (response >= 200 && response < 300); } -int cloudfs_list_directory(const char *path, dir_entry **dir_list) +int cloudfs_list_directory_internal(const char *path, dir_entry **dir_list) { char container[MAX_PATH_SIZE * 3] = ""; char object[MAX_PATH_SIZE] = ""; char last_subdir[MAX_PATH_SIZE] = ""; int prefix_length = 0; int response = 0; - int retval = 0; + int retval = -1; int entry_count = 0; - *dir_list = NULL; xmlNode *onode = NULL, *anode = NULL, *text_node = NULL; xmlParserCtxtPtr xmlctx = xmlCreatePushParserCtxt(NULL, NULL, "", 0, NULL); if (!strcmp(path, "") || !strcmp(path, "/")) @@ -325,10 +351,18 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) curl_free(encoded_object); } + if (*dir_list != NULL) { + char *encoded_marker = curl_escape((*dir_list)->marker, 0); + strcat(container, "&marker="); + strcat(container, encoded_marker); + curl_free(encoded_marker); + } + printf("%s\n", container); response = send_request("GET", container, NULL, xmlctx, NULL); xmlParseChunk(xmlctx, "", 0, 1); if (xmlctx->wellFormed && response >= 200 && response < 300) { + retval = 0; xmlNode *root_element = xmlDocGetRootElement(xmlctx->myDoc); for (onode = root_element->children; onode; onode = onode->next) { @@ -363,6 +397,7 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) if (slash && (0 == *(slash + 1))) *slash = 0; + de->marker = strdup(de->name); if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0) de->full_name = NULL; } @@ -377,7 +412,7 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) } if (!strcasecmp((const char *)anode->name, "last_modified")) { - struct tm last_modified; + struct tm last_modified = {0}; strptime(content, "%FT%T", &last_modified); de->last_modified = mktime(&last_modified); } @@ -396,13 +431,13 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) } de->next = *dir_list; *dir_list = de; + retval++; } else { debugf("unknown element: %s", onode->name); } } - retval = 1; } debugf("entry count: %d", entry_count); @@ -412,16 +447,37 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) return retval; } +int cloudfs_list_directory(const char *path, dir_entry **dir_list) +{ + int retval; + *dir_list = NULL; + + do { + retval = cloudfs_list_directory_internal(path, dir_list); + debugf("cloudfs_list_directory_internal retval: %d\n", retval); + } while(retval >= MAX_RESULTS_PER_REQUEST); + + return retval == -1 ? 0 : 1; +} + void cloudfs_free_dir_list(dir_entry *dir_list) { while (dir_list) { dir_entry *de = dir_list; dir_list = dir_list->next; - free(de->name); - free(de->full_name); - free(de->content_type); - free(de); + if (NULL != de){ + free(de->name); + de->name = NULL; + free(de->marker); + de->marker = NULL; + free(de->full_name); + de->full_name = NULL; + free(de->content_type); + de->content_type = NULL; + free(de); + de = NULL; + } } } @@ -430,7 +486,7 @@ int cloudfs_delete_object(const char *path) char *encoded = curl_escape(path, 0); int response = send_request("DELETE", encoded, NULL, NULL, NULL); curl_free(encoded); - return (response >= 200 && response < 300); + return ((response >= 200 && response < 300) || response == 404); } int cloudfs_copy_object(const char *src, const char *dst) @@ -445,6 +501,16 @@ int cloudfs_copy_object(const char *src, const char *dst) return (response >= 200 && response < 300); } +int cloudfs_statfs(const char *path, struct statvfs *stat) +{ + int response = send_request("HEAD", "/", NULL, NULL, NULL); + + debugf("Assigning statvfs values from cache."); + *stat = statcache; + + return (response >= 200 && response < 300); +} + int cloudfs_create_directory(const char *path) { char *encoded = curl_escape(path, 0); diff --git a/cloudfsapi.h b/cloudfsapi.h index b816640..1ba2bef 100644 --- a/cloudfsapi.h +++ b/cloudfsapi.h @@ -21,6 +21,7 @@ typedef struct dir_entry time_t last_modified; int isdir; struct dir_entry *next; + char *marker; } dir_entry; void cloudfs_init(); diff --git a/cloudfuse.c b/cloudfuse.c index cd99528..b325a15 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -113,6 +113,7 @@ static void update_dir_cache(const char *path, off_t size, int isdir) de->size = size; de->isdir = isdir; de->name = strdup(&path[strlen(cw->path)+1]); + de->marker = strdup(de->name); de->full_name = strdup(path); de->content_type = strdup(isdir ? "application/directory" : "application/octet-stream"); de->last_modified = time(NULL); @@ -270,9 +271,9 @@ static int cfs_open(const char *path, struct fuse_file_info *info) { FILE *temp_file = tmpfile(); dir_entry *de = path_info(path); - if (!(info->flags & O_WRONLY)) + if (!(info->flags & O_TRUNC)) { - if (!cloudfs_object_write_fp(path, temp_file)) + if (!cloudfs_object_write_fp(path, temp_file) && !(info->flags & O_CREAT)) { fclose(temp_file); return -ENOENT; @@ -369,16 +370,11 @@ static int cfs_truncate(const char *path, off_t size) static int cfs_statfs(const char *path, struct statvfs *stat) { - stat->f_bsize = 4096; - stat->f_frsize = 4096; - stat->f_blocks = INT_MAX; - stat->f_bfree = stat->f_blocks; - stat->f_bavail = stat->f_blocks; - stat->f_files = INT_MAX; - stat->f_ffree = INT_MAX; - stat->f_favail = INT_MAX; - stat->f_namemax = INT_MAX; - return 0; + if (cloudfs_statfs(path, stat)){ + return 0; + } + else + return -EIO; } static int cfs_chown(const char *path, uid_t uid, gid_t gid) @@ -407,6 +403,22 @@ static int cfs_rename(const char *src, const char *dst) return -EIO; } +static int cfs_link(const char *src, const char *dst) +{ + dir_entry *src_de = path_info(src); + if (!src_de) + return -ENOENT; + if (src_de->isdir) + return -EISDIR; + if (cloudfs_copy_object(src, dst)) + { + /* FIXME this isn't quite right as doesn't preserve last modified */ + update_dir_cache(dst, src_de->size, 0); + return 0; + } + return -EIO; +} + static void *cfs_init(struct fuse_conn_info *conn) { signal(SIGPIPE, SIG_IGN); @@ -536,6 +548,7 @@ int main(int argc, char **argv) .chmod = cfs_chmod, .chown = cfs_chown, .rename = cfs_rename, + .link = cfs_link, .init = cfs_init, };