diff --git a/.gitignore b/.gitignore index a24cbd6..254d20b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,13 @@ config.status # Binary outputs cloudfuse +test/specrunner # Ignore Eclipse files .cproject .project + +# Misc dev files +TAGS +**/*~ + diff --git a/Makefile.in b/Makefile.in index 3159c7f..f017dee 100644 --- a/Makefile.in +++ b/Makefile.in @@ -11,8 +11,13 @@ prefix = @prefix@ exec_prefix = @exec_prefix@ bindir = $(DESTDIR)$(exec_prefix)/bin -SOURCES=cloudfsapi.c cloudfuse.c -HEADERS=cloudfsapi.h +SOURCES=cloudfsapi.c cloudfuse.c headerspec.c +HEADERS=cloudfsapi.h headerspec.h + +TESTS=test/header01 \ + test/header02 \ + test/header03 \ + test/header04 all: cloudfuse @@ -28,6 +33,9 @@ $(bindir): cloudfuse: $(SOURCES) $(HEADERS) $(CC) $(CFLAGS) -o cloudfuse $(SOURCES) $(LIBS) +test/specrunner: $(SOURCES) $(HEADERS) test/specrunner.c + $(CC) $(CFLAGS) -o test/specrunner test/specrunner.c headerspec.c cloudfsapi.c $(LIBS) + clean: /bin/rm -f cloudfuse @@ -52,3 +60,10 @@ Makefile: Makefile.in config.status config.status: configure ./config.status --recheck +tests: test/specrunner + for test in $(TESTS); do \ + $$test || exit 1; \ + done + +etags: + etags $$(ls *.[hc]) /usr/include/fuse/fuse.h /usr/include/fuse/fuse_opt.h /usr/include/curl/curl.h diff --git a/README b/README index f706f59..054d4bf 100644 --- a/README +++ b/README @@ -93,6 +93,40 @@ EXAMPLE: authurl=http://10.10.0.1:5000/v2.0 +HEADER SPECIFICATIONS: + + The option headers_spec option allows to specify headers to send + when a file is written. A common use is to specify the + Content-Type header. Rackspace's CloudFiles, for example, allows + 14 different headers for specifying access controls, content + expiration, and content encoding. Cloudfuse doesn't restrict what + headers can be specified. + + The header value is set based on matching a path against a list of + globs. The syntax: + + HeaderKey: [!]GlobMatch HeaderValue [, [!]GlobMatch HeaderValue...] [; HeaderKey:...] + + Here's an example which sets the Content-Type header: + + Content-Type: /foobucket/*.html text/html, "/*/my dir/*.txt" text/plain; + + If no pattern matches, then the header is omitted. Note that + double quotes can be used to specify a pattern that has spaces. A + match pattern preceded by ! is negated, so that the header is + applied when the path does not match the glob pattern. Multiple + headers can be specified, separated by semicolon (;) More examples + are available in the test directory. + + See 'man 3 fnmatch' for accepted glob patterns. + + The path looks like an absolute path beginning at the mount point. + For example, a file "index.html" in bucket "foo" will result in + the string "/foo/index.html" being matched against the glob + pattern (even if the absolute path on your system is actually + /var/www/foo/index.html). + + BUGS/SHORTCOMINGS: * rename() doesn't work on directories (and probably never will). @@ -118,7 +152,7 @@ AWESOME CONTRIBUTORS: * David Brownlee https://github.com/abs0 * Mike Lundy https://github.com/novas0x2a * justinb https://github.com/justinsb - + * Erik Mackdanz http://mackdanz.net Thanks, and I hope you find it useful. diff --git a/cloudfsapi.c b/cloudfsapi.c index 583964d..77fb3e8 100644 --- a/cloudfsapi.c +++ b/cloudfsapi.c @@ -17,6 +17,7 @@ #include #include "cloudfsapi.h" #include "config.h" +#include "headerspec.h" #define RHEL5_LIBCURL_VERSION 462597 #define RHEL5_CERTIFICATE_FILE "/etc/pki/tls/certs/ca-bundle.crt" @@ -31,6 +32,7 @@ static int curl_pool_count = 0; static int debug = 0; static int verify_ssl = 1; static int rhel5_mode = 0; +static header_spec *hspec; #ifdef HAVE_OPENSSL #include @@ -84,9 +86,10 @@ static void return_connection(CURL *curl) pthread_mutex_unlock(&pool_mut); } -static void add_header(curl_slist **headers, const char *name, - const char *value) +void add_header(curl_slist **headers, const char *name, + const char *value) { + debugf("Adding the header %s: %s",name,value); char x_header[MAX_HEADER_SIZE]; snprintf(x_header, sizeof(x_header), "%s: %s", name, value); *headers = curl_slist_append(*headers, x_header); @@ -249,8 +252,11 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) fflush(fp); rewind(fp); char *encoded = curl_escape(path, 0); - int response = send_request("PUT", encoded, fp, NULL, NULL); + curl_slist *headers = NULL; + add_matching_headers(&headers,hspec,path); + int response = send_request("PUT", encoded, fp, NULL, headers); curl_free(encoded); + curl_slist_free_all(headers); return (response >= 200 && response < 300); } @@ -497,6 +503,11 @@ void cloudfs_set_credentials(char *username, char *tenant, char *password, reconnect_args.use_snet = use_snet; } +void cloudfs_set_header_spec(header_spec *spec) +{ + hspec = spec; +} + int cloudfs_connect() { long response = -1; diff --git a/cloudfsapi.h b/cloudfsapi.h index b816640..c4c4b9c 100644 --- a/cloudfsapi.h +++ b/cloudfsapi.h @@ -3,6 +3,7 @@ #include #include +#include "headerspec.h" #define BUFFER_INITIAL_SIZE 4096 #define MAX_HEADER_SIZE 8192 @@ -38,6 +39,9 @@ off_t cloudfs_file_size(int fd); void cloudfs_debug(int dbg); void cloudfs_verify_ssl(int dbg); void cloudfs_free_dir_list(dir_entry *dir_list); +void cloudfs_set_header_spec(header_spec *spec); +void add_header(curl_slist **headers, const char *name, + const char *value); void debugf(char *fmt, ...); #endif diff --git a/cloudfuse.c b/cloudfuse.c index cd99528..8606ff4 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -16,7 +16,7 @@ #include #include "cloudfsapi.h" #include "config.h" - +#include "headerspec.h" #define OPTION_SIZE 1024 @@ -433,6 +433,7 @@ static struct options { char region[OPTION_SIZE]; char use_snet[OPTION_SIZE]; char verify_ssl[OPTION_SIZE]; + char header_spec[OPTION_SIZE*10]; } options = { .username = "", .password = "", @@ -442,6 +443,7 @@ static struct options { .region = "", .use_snet = "false", .verify_ssl = "true", + .header_spec = "" }; int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs) @@ -454,7 +456,8 @@ int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs sscanf(arg, " authurl = %[^\r\n ]", options.authurl) || sscanf(arg, " region = %[^\r\n ]", options.region) || sscanf(arg, " use_snet = %[^\r\n ]", options.use_snet) || - sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl)) + sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) || + sscanf(arg, " header_spec = %[^\r\n]", options.header_spec)) // Note spaces permitted return 0; if (!strcmp(arg, "-f") || !strcmp(arg, "-d") || !strcmp(arg, "debug")) cloudfs_debug(1); @@ -494,12 +497,21 @@ int main(int argc, char **argv) fprintf(stderr, " use_snet=[True to use Rackspace ServiceNet for connections]\n"); fprintf(stderr, " cache_timeout=[Seconds for directory caching, default 600]\n"); fprintf(stderr, " verify_ssl=[False to disable SSL cert verification]\n"); + fprintf(stderr, " header_spec=[Match specification for writing extra headers, see README]\n"); return 1; } cloudfs_init(); + header_spec *parsed = NULL; + if(!parse_spec(options.header_spec, &parsed)) + { + fprintf(stderr, "Could not parse header spec\n"); + return 1; + } + cloudfs_set_header_spec(parsed); + cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true")); cloudfs_set_credentials(options.username, options.tenant, options.password, @@ -541,5 +553,7 @@ int main(int argc, char **argv) pthread_mutex_init(&dmut, NULL); return fuse_main(args.argc, args.argv, &cfs_oper, &options); + + free_spec(parsed); } diff --git a/headerspec.c b/headerspec.c new file mode 100644 index 0000000..0081503 --- /dev/null +++ b/headerspec.c @@ -0,0 +1,308 @@ +#include +#include +#include +#include "headerspec.h" +#include "cloudfsapi.h" + +static int next_token(const char *spec /* in */, int *scanstart /* in/out */, + int *tokenstart /* out */, int *tokenlen /* out */) +{ + + const char *start = spec + *scanstart; + const char *thisc = start; + + // skip any leading whitespace + while(*thisc==' ' || *thisc=='\t' || *thisc=='\n' || *thisc=='\r') + { + thisc++; + } + + // detect end of input + if(!*thisc) return 0; + + // Handle quote + if(*thisc=='"') + { + thisc++; + *tokenstart = thisc - spec; + while(1) + { + if(*thisc == '"') + { + *tokenlen = (thisc - spec) - *tokenstart; + *scanstart = *tokenstart + *tokenlen + 1; // trim trailing " + return 1; + } + else if(*thisc == 0) // Handle unterminated string + { + *tokenlen = (thisc - spec) - *tokenstart; + *scanstart = *tokenstart + *tokenlen; + return 0; + } + thisc++; + } + } + else if(*thisc==':') + { + *tokenstart = thisc - spec; + *tokenlen = 1; + *scanstart = *tokenstart + *tokenlen; + return 1; + } + else if(*thisc==';') + { + *tokenstart = thisc - spec; + *tokenlen = 1; + *scanstart = *tokenstart + *tokenlen; + return 1; + } + else if(*thisc==',') + { + *tokenstart = thisc - spec; + *tokenlen = 1; + *scanstart = *tokenstart + *tokenlen; + return 1; + } + else if(*thisc=='!') + { + *tokenstart = thisc - spec; + *tokenlen = 1; + *scanstart = *tokenstart + *tokenlen; + return 1; + } + else // An actual token + { + *tokenstart = thisc - spec; + + while(1) + { + thisc++; + if(*thisc==' ' || *thisc=='\t' || *thisc=='\n' || *thisc=='\r' || *thisc=='"' + || *thisc==':' || *thisc==';' || *thisc==',' || *thisc=='!' || *thisc==0) break; + } + + *tokenlen = (thisc - spec) - *tokenstart; + *scanstart = *tokenstart + *tokenlen; + + return 1; + } +} + +static enum +{ + EXPECT_EOF_OR_SEMI_OR_HEADERKEY, + EXPECT_COLON, + EXPECT_NEGATE_OR_MATCH, + EXPECT_MATCH, + EXPECT_HEADER_VALUE, + EXPECT_EOF_OR_COMMA_OR_SEMI +} parse_states; + +int parse_spec(const char *spec, header_spec **output) +{ + + debugf("Entire spec is %s",spec); + + int scanstart = 0; + int tokenstart, tokenlen; + + int state = EXPECT_EOF_OR_SEMI_OR_HEADERKEY; + while(next_token(spec,&scanstart,&tokenstart,&tokenlen)) + { + debugf("Found token at %d len %d",tokenstart,tokenlen); + + char fc = *(spec+tokenstart); // first char of token + + char *headerkey, *pattern, *headervalue; + int isnegated; + header_spec *speclist_tail; + + switch(state) + { + case EXPECT_EOF_OR_SEMI_OR_HEADERKEY: + if(';' == fc) continue; + if(':'==fc || '!'==fc || ','==fc) + { + debugf("In state %d, encountered unexpected token at %d len %d",state,tokenstart,tokenlen); + return 0; + } + headerkey = strndup(spec+tokenstart,tokenlen); + debugf("Header key is %s",headerkey); + + // Create a new tail for the linked list. + speclist_tail = *output; + if(speclist_tail) + { + while(speclist_tail->next) + { + speclist_tail = speclist_tail->next; + } + speclist_tail->next = malloc(sizeof(header_spec)); + speclist_tail = speclist_tail->next; + } + else // special case for first list element + { + *output = malloc(sizeof(header_spec)); + speclist_tail = *output; + } + + debugf("Allocated speclist"); + speclist_tail->header_key = headerkey; + speclist_tail->matches = NULL; + speclist_tail->next = NULL; + + state = EXPECT_COLON; + break; + case EXPECT_COLON: + if(':'!=fc) + { + debugf("In state %d, encountered unexpected token at %d len %d",state,tokenstart,tokenlen); + return 0; + } + state = EXPECT_NEGATE_OR_MATCH; + break; + case EXPECT_NEGATE_OR_MATCH: + isnegated = 0; + if('!'==fc) + { + debugf("Match is negated"); + isnegated = 1; + state = EXPECT_MATCH; + } + else if(':'==fc || ','==fc || ';'==fc) + { + debugf("In state %d, encountered unexpected token at %d len %d",state,tokenstart,tokenlen); + return 0; + } + else + { + pattern = strndup(spec+tokenstart,tokenlen); + debugf("Pattern is %s",pattern); + state = EXPECT_HEADER_VALUE; + } + break; + case EXPECT_MATCH: + if(':'==fc || '!'==fc || ','==fc || ';'==fc) + { + debugf("In state %d, encountered unexpected token at %d len %d",state,tokenstart,tokenlen); + return 0; + } + pattern = strndup(spec+tokenstart,tokenlen); + debugf("Pattern is %s",pattern); + state = EXPECT_HEADER_VALUE; + break; + case EXPECT_HEADER_VALUE: + if(':'==fc || '!'==fc || ','==fc || ';'==fc) + { + debugf("In state %d, encountered unexpected token at %d len %d",state,tokenstart,tokenlen); + return 0; + } + headervalue = strndup(spec+tokenstart,tokenlen); + debugf("Header value is %s",headervalue); + + match_spec *matchlist_tail; + if(speclist_tail->matches) + { + matchlist_tail = speclist_tail->matches; + while(matchlist_tail->next) + { + matchlist_tail = matchlist_tail->next; + } + matchlist_tail->next = malloc(sizeof(match_spec)); + matchlist_tail = matchlist_tail->next; + } + else + { + matchlist_tail = malloc(sizeof(match_spec)); + speclist_tail->matches = matchlist_tail; + } + + matchlist_tail->next = NULL; + matchlist_tail->pattern = pattern; + matchlist_tail->is_positive = !isnegated; + matchlist_tail->header_value = headervalue; + + state = EXPECT_EOF_OR_COMMA_OR_SEMI; + break; + case EXPECT_EOF_OR_COMMA_OR_SEMI: + if(','==fc) + { + state=EXPECT_NEGATE_OR_MATCH; + } + else if(';'==fc) + { + state=EXPECT_EOF_OR_SEMI_OR_HEADERKEY; + } + else + { + debugf("In state %d, encountered unexpected token at %d len %d",state,tokenstart,tokenlen); + return 0; + } + break; + default: + debugf("In unexpected state %d with token at %d len %d",state,tokenstart,tokenlen); + return 0; + } + } + + if(state != EXPECT_EOF_OR_SEMI_OR_HEADERKEY && state != EXPECT_EOF_OR_COMMA_OR_SEMI) + { + debugf("Finished parse in unexpected state %d",state); + return 0; + } + + debugf("Parse completed successfully"); + return 1; +} + +void free_spec(header_spec *spec) +{ + if(!spec) return; + + /* debugf("Freeing spec at %p",spec); */ + free(spec->header_key); + + // free matches; + match_spec *onematch = spec->matches; + while(onematch) + { + /* debugf("Freeing match with value %s",onematch->header_value); */ + free(onematch->pattern); + free(onematch->header_value); + match_spec *nextmatch = onematch->next; + free(onematch); + onematch = nextmatch; + } + + header_spec *next = spec->next; + free(spec); + free_spec(next); +} + +int add_matching_headers(struct curl_slist **headers, header_spec *spec, const char *path) +{ + + header_spec *onespec = spec; + while(onespec) + { + debugf("Testing one spec"); + match_spec *onematch = onespec->matches; + while(onematch) + { + int result = fnmatch(onematch->pattern,path,0); + debugf("Testing one match, path being %s, result was %d",path,result); + if((result==0 && onematch->is_positive) || (result==FNM_NOMATCH && !onematch->is_positive)) + { + add_header(headers,onespec->header_key,onematch->header_value); + break; + } + else if(result!=0 && result != FNM_NOMATCH) + { + debugf("fnmatch error"); + break; + } + onematch = onematch->next; + } + onespec = onespec->next; + } +} diff --git a/headerspec.h b/headerspec.h new file mode 100644 index 0000000..722e7e8 --- /dev/null +++ b/headerspec.h @@ -0,0 +1,27 @@ +#ifndef _HEADERSPEC_H +#define _HEADERSPEC_H + +#include + +typedef struct match_spec +{ + char *pattern; + int is_positive; + char *header_value; + struct match_spec *next; +} match_spec; + +typedef struct header_spec +{ + char *header_key; + match_spec *matches; + struct header_spec *next; +} header_spec; + +int parse_spec(const char *spec, header_spec **output); + +int add_matching_headers(struct curl_slist **headers, header_spec *spec, const char *path); + +void free_spec(header_spec *spec); + +#endif // _HEADERSPEC_H diff --git a/test/header01 b/test/header01 new file mode 100755 index 0000000..80b6112 --- /dev/null +++ b/test/header01 @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e +set -x + +result=$(mktemp) + +test/specrunner \ +'Content-Type: *.jpg image/jpeg, ! *.txt text/html; Content-Disposition: * "attachment; filename=foo.txt"' \ +/bar.html >$result + +grep "Content-Type: text/html" $result +grep "Content-Disposition: attachment; filename=foo.txt" $result + +rm $result + diff --git a/test/header02 b/test/header02 new file mode 100755 index 0000000..68d8c24 --- /dev/null +++ b/test/header02 @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e +set -x + +result=$(mktemp) + +test/specrunner \ +'Content-Type: *.css text/css, /blog/code/*/repository/raw/* text/plain, /blog/code/* text/html' \ +/blog/code/foo >$result + +grep "Content-Type: text/html" $result + +rm $result + + diff --git a/test/header03 b/test/header03 new file mode 100755 index 0000000..80c7c84 --- /dev/null +++ b/test/header03 @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e +set -x + +result=$(mktemp) + +test/specrunner \ +'Content-Type: *.css text/css, /blog/code/*/repository/raw/* text/plain, /blog/code/* text/html; Content-Disposition: * bar' \ +/foo >$result + +grep "Content-Disposition: bar" $result + +rm $result + + diff --git a/test/header04 b/test/header04 new file mode 100755 index 0000000..1cd6f8e --- /dev/null +++ b/test/header04 @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e +set -x + +# Ensure bad parse detected +if test/specrunner 'Content-Disposition: bar' /foo; then + exit 1 +fi + + diff --git a/test/specrunner.c b/test/specrunner.c new file mode 100644 index 0000000..55c07dc --- /dev/null +++ b/test/specrunner.c @@ -0,0 +1,36 @@ +#include +#include +#include "../headerspec.h" +#include "../cloudfsapi.h" + +int main(int argc, char **argv) +{ + if(argc != 3) + { + printf("Usage: %s \n", argv[0]); + exit(1); + } + + char *spec = argv[1]; + char *path = argv[2]; + + header_spec *parsed = NULL; + if(!parse_spec(spec, &parsed)) + { + printf("Bad parse\n"); + return 1; + } + + struct curl_slist *headers = NULL; + add_matching_headers(&headers,parsed,path); + + struct curl_slist *onestr = headers; + while(onestr) + { + printf("%s\n",onestr->data); + onestr = onestr->next; + } + + free_spec(parsed); + return 0; +}