diff --git a/Makefile.am b/Makefile.am index 7fe98fac..0d235544 100644 --- a/Makefile.am +++ b/Makefile.am @@ -6,10 +6,11 @@ endif ACLOCAL_AMFLAGS = -I m4 AM_CPPFLAGS = $(PTHREAD_CFLAGS) -DSYSTEM_LIBINIPARSER=@SYSTEM_LIBINIPARSER@ +AM_CFLAGS = $(STORE_CFLAGS) -STORE_SOURCES = src/store.c src/store_file.c src/store_file_utils.c src/store_memcached.c src/store_rados.c src/store_ro_http_proxy.c src/store_ro_composite.c src/store_null.c -STORE_LDFLAGS = $(LIBMEMCACHED_LDFLAGS) $(LIBRADOS_LDFLAGS) $(LIBCURL) -STORE_CPPFLAGS = +STORE_SOURCES = src/store.c src/store_file.c src/store_file_utils.c src/store_memcached.c src/store_rados.c src/store_ro_http_proxy.c src/store_ro_composite.c src/store_null.c src/store_s3.c +STORE_LDFLAGS = $(LIBMEMCACHED_LDFLAGS) $(LIBRADOS_LDFLAGS) $(LIBCURL) $(LIBS3_LDFLAGS) +STORE_CFLAGS = $(LIBS3_CFLAGS) bin_PROGRAMS = renderd render_expired render_list render_speedtest render_old noinst_PROGRAMS = gen_tile_test @@ -34,8 +35,8 @@ render_old_SOURCES = src/store_file_utils.c src/render_old.c src/sys_utils.c src render_old_LDADD = $(PTHREAD_CFLAGS) #convert_meta_SOURCES = src/dir_utils.c src/store.c src/convert_meta.c gen_tile_test_SOURCES = src/gen_tile_test.cpp src/metatile.cpp src/request_queue.c src/protocol_helper.c src/daemon.c src/daemon_compat.c src/gen_tile.cpp src/sys_utils.c src/cache_expire.c src/parameterize_style.cpp $(STORE_SOURCES) -gen_tile_test_CFLAGS = -DMAIN_ALREADY_DEFINED $(PTHREAD_CFLAGS) -gen_tile_test_CXXFLAGS = $(MAPNIK_CFLAGS) +gen_tile_test_CFLAGS = -DMAIN_ALREADY_DEFINED $(PTHREAD_CFLAGS) $(STORE_CFLAGS) +gen_tile_test_CXXFLAGS = $(MAPNIK_CFLAGS) $(STORE_CFLAGS) gen_tile_test_LDADD = $(PTHREAD_CFLAGS) $(MAPNIK_LDFLAGS) $(STORE_LDFLAGS) -liniparser if !SYSTEM_LIBINIPARSER gen_tile_test_SOURCES += iniparser3.0b/libiniparser.la @@ -48,10 +49,10 @@ test: gen_tile_test ./gen_tile_test all-local: - $(APXS) -c $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_null.c + $(APXS) -c $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_s3.c @srcdir@/src/store_null.c install-mod_tile: mkdir -p $(DESTDIR)`$(APXS) -q LIBEXECDIR` - $(APXS) -S LIBEXECDIR=$(DESTDIR)`$(APXS) -q LIBEXECDIR` -c -i $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_null.c + $(APXS) -S LIBEXECDIR=$(DESTDIR)`$(APXS) -q LIBEXECDIR` -c -i $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_s3.c @srcdir@/src/store_null.c diff --git a/configure.ac b/configure.ac index c6ff10ad..62964bad 100644 --- a/configure.ac +++ b/configure.ac @@ -54,6 +54,46 @@ AC_CHECK_LIB(rados, rados_version, [ AC_SUBST(LIBRADOS_LDFLAGS) ][]) +AC_ARG_WITH([libs3], + [AS_HELP_STRING([--with-libs3[=DIR]],[path to libs3])], + [ + libs3_dir="$withval" + if test "$libs3_dir" != "no"; then + if test "$libs3_dir" != "yes"; then + AC_MSG_CHECKING([for libs3]) + if test -f "$libs3_dir/include/libs3.h"; then + AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3]) + LIBS3_LDFLAGS="-L$libs3_dir/lib -ls3" + AC_SUBST(LIBS3_LDFLAGS) + LIBS3_CFLAGS="-I$libs3_dir/include" + AC_SUBST(LIBS3_CFLAGS) + AC_MSG_RESULT($libs3_dir) + else + AC_MSG_ERROR([no libs3 found at $libs3_dir]) + fi + else + AC_CHECK_LIB(s3, S3_deinitialize, + [ + AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3]) + LIBS3_LDFLAGS='-ls3' + AC_SUBST(LIBS3_LDFLAGS) + ], + [ + AC_MSG_ERROR([no libs3 found]) + ]) + fi + fi + ], + [ + AC_CHECK_LIB(s3, S3_deinitialize, + [ + AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3]) + LIBS3_LDFLAGS='-ls3' + AC_SUBST(LIBS3_LDFLAGS) + ], + []) + ]) + AC_CHECK_FUNCS([bzero gethostbyname gettimeofday inet_ntoa memset mkdir pow select socket strchr strdup strerror strrchr strstr strtol strtoul utime],[],[AC_MSG_ERROR([One of the required functions was not found])]) AC_CHECK_FUNCS([daemon getloadavg],[],[]) diff --git a/includes/metatile.h b/includes/metatile.h index c5c8d569..3cf74afa 100644 --- a/includes/metatile.h +++ b/includes/metatile.h @@ -12,21 +12,22 @@ extern "C" { #define META_MAGIC "META" #define META_MAGIC_COMPRESSED "METZ" - - struct entry { - int offset; - int size; - }; - - struct meta_layout { - char magic[4]; - int count; // METATILE ^ 2 - int x, y, z; // lowest x,y of this metatile, plus z - struct entry index[]; // count entries - // Followed by the tile data - // The index offsets are measured from the start of the file - }; +struct entry { + int offset; + int size; +}; + +struct meta_layout { + char magic[4]; + int count; // METATILE ^ 2 + int x, y, z; // lowest x,y of this metatile, plus z + struct entry index[]; // count entries + // Followed by the tile data + // The index offsets are measured from the start of the file +}; + +#define METATILE_HEADER_LEN (sizeof(struct meta_layout) + METATILE * METATILE * sizeof(struct entry)) #ifdef __cplusplus } diff --git a/includes/store_file.h b/includes/store_file.h index 77caae51..faa44aa6 100644 --- a/includes/store_file.h +++ b/includes/store_file.h @@ -6,9 +6,8 @@ extern "C" { #endif #include "store.h" - - struct storage_backend * init_storage_file(const char * tile_dir); - int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, const char *options, int x, int y, int z); + +struct storage_backend * init_storage_file(const char * tile_dir); #ifdef __cplusplus } diff --git a/includes/store_file_utils.h b/includes/store_file_utils.h index 40e167b1..dfef0fa8 100644 --- a/includes/store_file_utils.h +++ b/includes/store_file_utils.h @@ -27,6 +27,7 @@ int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px /* New meta-tile storage functions */ /* Returns the path to the meta-tile and the offset within the meta-tile */ int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z); +int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, const char *options, int x, int y, int z); #endif #ifdef __cplusplus diff --git a/includes/store_s3.h b/includes/store_s3.h new file mode 100644 index 00000000..362396d0 --- /dev/null +++ b/includes/store_s3.h @@ -0,0 +1,16 @@ +#ifndef STORES3_H +#define STORES3_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "store.h" + +struct storage_backend* init_storage_s3(const char *connection_string); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/mod_tile.conf b/mod_tile.conf index 988e29a1..db5c07fd 100644 --- a/mod_tile.conf +++ b/mod_tile.conf @@ -14,6 +14,7 @@ LoadModule tile_module modules/mod_tile.so # The file based storage uses a simple file path as its storage path ( /path/to/tiledir ) # The RADOS based storage takes a location to the rados config file and a pool name ( rados://poolname/path/to/ceph.conf ) # The memcached based storage currently has no configuration options and always connects to memcached on localhost ( memcached:// ) +# The S3 based storage takes an access key, bucket, and path ( s3://access_key_id:secret_access_key[@host]/bucket_name[/basepath] ) # # The storage path can be overwritten on a style by style basis from the style TileConfigFile ModTileTileDir /var/lib/mod_tile diff --git a/readme.txt b/readme.txt index 0bcf0958..76c5ce6f 100644 --- a/readme.txt +++ b/readme.txt @@ -46,8 +46,8 @@ daemon to render (or re-render) the tile. resources on the server and how out of date they are. 4) Use tile storage other than a plain posix file system. -e.g it can store tiles in a ceph object store, or proxy them -from another tile server. +e.g it can store tiles in a ceph object store, an Amazon S3 bucket, +or proxy them from another tile server. 5) Tile expiry. It estimates when the tile is next likely to be rendered and adds the appropriate HTTP diff --git a/src/daemon.c b/src/daemon.c index 56a5b0b4..28737d83 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -430,8 +430,8 @@ int server_socket_init(renderd_config *sConfig) { addrI.sin6_addr = in6addr_any; addrI.sin6_port = htons(sConfig->ipport); if (bind(fd, (struct sockaddr *) &addrI, sizeof(addrI)) < 0) { - fprintf(stderr, "socket bind failed for: %s:%i\n", - sConfig->iphostname, sConfig->ipport); + fprintf(stderr, "socket bind failed for: %s:%i: %s\n", + sConfig->iphostname, sConfig->ipport, strerror(errno)); close(fd); exit(3); } @@ -453,7 +453,8 @@ int server_socket_init(renderd_config *sConfig) { old = umask(0); // Need daemon socket to be writeable by apache if (bind(fd, (struct sockaddr *) &addrU, sizeof(addrU)) < 0) { - fprintf(stderr, "socket bind failed for: %s\n", sConfig->socketname); + fprintf(stderr, "socket bind failed for: %s: %s\n", + sConfig->socketname, strerror(errno)); close(fd); exit(3); } @@ -932,7 +933,7 @@ int main(int argc, char **argv) if (foreground) { fprintf(stderr, "Running in foreground mode...\n"); } else { - if (daemon(0, 0) != 0) { + if (daemon(1, 1) != 0) { fprintf(stderr, "can't daemonize: %s\n", strerror(errno)); } /* write pid file */ diff --git a/src/gen_tile_test.cpp b/src/gen_tile_test.cpp index 903f8b0f..653fc2fb 100644 --- a/src/gen_tile_test.cpp +++ b/src/gen_tile_test.cpp @@ -58,6 +58,9 @@ #include #endif +#ifdef HAVE_LIBS3 +#include +#endif #define NO_QUEUE_REQUESTS 9 #define NO_TEST_REPEATS 100 @@ -591,14 +594,14 @@ TEST_CASE( "renderd", "tile generation" ) { } SECTION("renderd startup unrecognized option", "should return 1") { - int ret = system("./renderd --doesnotexit"); + int ret = system("./renderd --doesnotexist"); ret = WEXITSTATUS(ret); //CAPTURE( ret ); REQUIRE( ret == 1 ); } SECTION("renderd startup invalid option", "should return 1") { - int ret = system("./renderd -doesnotexit"); + int ret = system("./renderd -doesnotexist"); ret = WEXITSTATUS(ret); //CAPTURE( ret ); REQUIRE( ret == 1 ); @@ -946,6 +949,307 @@ TEST_CASE( "projections", "Test projections" ) { } } +#ifdef HAVE_LIBS3 +S3Status test_s3_properties_callback(const S3ResponseProperties *properties, void *callbackData) +{ + return S3StatusOK; +} + +void test_s3_complete_callback(S3Status status, const S3ErrorDetails *errorDetails, void *callbackData) +{ +} + + +TEST_CASE("storage-backend/s3", "S3 tile storage backend") { + + /* Setting up S3 location for testing */ + char *s3_connection_url; + char *keyid = getenv("S3_ACCESS_KEY_ID"); + REQUIRE(keyid != NULL); + char *accesskey = getenv("S3_SECRET_ACCESS_KEY"); + REQUIRE(accesskey != NULL); + char *bucketname = getenv("S3_BUCKET_NAME"); + REQUIRE(bucketname != NULL); + const char *bucketpath = "mod_tile_test"; + + s3_connection_url = (char*) malloc(1024); + snprintf(s3_connection_url, 1024, "s3://%s:%s/%s/%s", keyid, accesskey, bucketname, bucketpath); + + SECTION("storage-backend/s3/initialise", "should return 1") { + struct storage_backend *store = NULL; + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + store->close_storage(store); + } + + SECTION("storage-backend/s3/stat/non existent", "should return size < 0") { + struct storage_backend *store = NULL; + struct stat_info sinfo; + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + sinfo = store->tile_stat(store, "default", "", 0, 0, 0); + REQUIRE(sinfo.size < 0); + store->close_storage(store); + } + + SECTION("storage-backend/s3/read/non existent", "should return size < 0") { + struct storage_backend *store = NULL; + int size; + char *buf = (char*) malloc(10000); + int compressed; + char *err_msg = (char*) malloc(10000); + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + size = store->tile_read(store, "default", "", 0, 0, 0, buf, 10000, &compressed, err_msg); + REQUIRE(size < 0); + + store->close_storage(store); + free(buf); + free(err_msg); + } + + SECTION("storage-backend/s3/write/full metatile", "should complete") { + struct storage_backend *store = NULL; + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024, 1024, 10); + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + std::string tile_data = "DEADBEEF"; + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + + store->close_storage(store); + } + + SECTION("storage-backend/s3/stat/full metatile", "should complete") { + struct storage_backend *store = NULL; + struct stat_info sinfo; + + time_t before_write, after_write; + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024 + METATILE, 1024, 10); + time(&before_write); + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + std::string tile_data = "DEADBEEF"; + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + sleep(1); + time(&after_write); + + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + sinfo = store->tile_stat(store, "default", "", 1024 + METATILE + yy, 1024 + xx, 10); + REQUIRE(sinfo.size > 0); + REQUIRE(sinfo.expired == 0); + REQUIRE(sinfo.mtime >= before_write); + REQUIRE(sinfo.mtime <= after_write); + } + } + + store->close_storage(store); + } + + SECTION("storage-backend/s3/read/full metatile", "should complete") { + std::cerr << "storage-backend/s3/read/full metatile" << std::endl; + + struct storage_backend *store = NULL; + char *buf; + char *buf_tmp; + char msg[4096]; + int compressed; + int tile_size; + + buf = (char*) malloc(8196); + buf_tmp = (char*) malloc(8196); + + time_t before_write, after_write; + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024 + METATILE, 1024, 10); + time(&before_write); + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + sprintf(buf, "DEADBEEF %i %i", xx, yy); + std::string tile_data(buf); + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + time(&after_write); + + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + tile_size = store->tile_read(store, "default", "", 1024 + METATILE + xx, 1024 + yy, 10, buf, 8195, &compressed, msg); + REQUIRE(tile_size == 12); + sprintf(buf_tmp, "DEADBEEF %i %i", xx, yy); + REQUIRE(memcmp(buf_tmp, buf, 11) == 0); + } + } + + free(buf); + free(buf_tmp); + store->close_storage(store); + } + + SECTION("storage-backend/s3/read/partial metatile", "should return correct data") { + std::cerr << "storage-backend/s3/read/partial metatile" << std::endl; + + struct storage_backend *store = NULL; + char *buf; + char *buf_tmp; + char msg[4096]; + int compressed; + int tile_size; + + buf = (char*) malloc(8196); + buf_tmp = (char*) malloc(8196); + + time_t before_write, after_write; + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024 + 2*METATILE, 1024, 10); + time(&before_write); + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < (METATILE >> 1); xx++) { + sprintf(buf, "DEADBEEF %i %i", xx, yy); + std::string tile_data(buf); + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + time(&after_write); + + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + tile_size = store->tile_read(store, "default", "", 1024 + 2*METATILE + xx, 1024 + yy, 10, buf, 8195, &compressed, msg); + if (xx >= (METATILE >> 1)) { + REQUIRE (tile_size == 0); + } else { + REQUIRE (tile_size == 12); + sprintf(buf_tmp, "DEADBEEF %i %i", xx, yy); + REQUIRE (memcmp(buf_tmp, buf, 11) == 0); + } + } + } + + free(buf); + free(buf_tmp); + store->close_storage(store); + } + + SECTION("storage-backend/s3/delete metatile", "should delete tile from storage") { + struct storage_backend *store = NULL; + struct stat_info sinfo; + char *buf; + char *buf_tmp; + + buf = (char*) malloc(8196); + buf_tmp = (char*) malloc(8196); + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024 + 3*METATILE, 1024, 10); + + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + sprintf(buf, "DEADBEEF %i %i", xx, yy); + std::string tile_data(buf); + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + + sinfo = store->tile_stat(store, "default", "", 1024 + 3*METATILE, 1024, 10); + + REQUIRE(sinfo.size > 0); + + store->metatile_delete(store, "default", 1024 + 3*METATILE, 1024, 10); + + sinfo = store->tile_stat(store, "default", "", 1024 + 3*METATILE, 1024, 10); + + REQUIRE(sinfo.size < 0); + + free(buf); + free(buf_tmp); + store->close_storage(store); + } + + SECTION("storage-backend/s3/expire/expiremetatile", "should expire the tile") { + struct storage_backend *store = NULL; + struct stat_info sinfo; + char *buf; + char *buf_tmp; + + buf = (char*) malloc(8196); + buf_tmp = (char*) malloc(8196); + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024 + 4*METATILE, 1024, 10); + + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + sprintf(buf, "DEADBEEF %i %i", xx, yy); + std::string tile_data(buf); + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + + sinfo = store->tile_stat(store, "default", "", 1024 + 4*METATILE, 1024, 10); + + REQUIRE(sinfo.size > 0); + + store->metatile_expire(store, "default", 1024 + 4*METATILE, 1024, 10); + + sinfo = store->tile_stat(store, "default", "", 1024 + 4*METATILE, 1024, 10); + + REQUIRE(sinfo.size > 0); + REQUIRE(sinfo.expired > 0); + + free(buf); + free(buf_tmp); + store->close_storage(store); + } + + S3BucketContext ctx; + ctx.accessKeyId = keyid; + ctx.secretAccessKey = accesskey; + ctx.bucketName = bucketname; + ctx.hostName = NULL; + ctx.protocol = S3ProtocolHTTPS; + ctx.uriStyle = S3UriStyleVirtualHost; + ctx.securityToken = NULL; + + S3ResponseHandler handler; + handler.completeCallback = &test_s3_complete_callback; + handler.propertiesCallback = &test_s3_properties_callback; + S3_delete_object(&ctx, bucketpath, NULL, &handler, NULL); +} + +#endif + int main (int argc, char* const argv[]) { //std::ios_base::sync_with_stdio(false); diff --git a/src/metatile.cpp b/src/metatile.cpp index c0b55455..1fa3e6d5 100644 --- a/src/metatile.cpp +++ b/src/metatile.cpp @@ -32,7 +32,6 @@ #include "cache_expire.h" #include "request_queue.h" - metaTile::metaTile(const std::string &xmlconfig, const std::string &options, int x, int y, int z): x_(x), y_(y), z_(z), xmlconfig_(xmlconfig), options_(options) { clear(); diff --git a/src/store.c b/src/store.c index f9d511c7..243cbb23 100644 --- a/src/store.c +++ b/src/store.c @@ -21,6 +21,7 @@ #include "store_rados.h" #include "store_ro_http_proxy.h" #include "store_ro_composite.h" +#include "store_s3.h" #include "store_null.h" //TODO: Make this function handle different logging backends, depending on if on compiles it from apache or something else @@ -103,6 +104,11 @@ struct storage_backend * init_storage_backend(const char * options) { store = init_storage_ro_composite(options); return store; } + if (strstr(options, "s3://") == options) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_backend: initialising s3 storage backend at: %s", options); + store = init_storage_s3(options); + return store; + } if (strstr(options,"null://") == options) { log_message(STORE_LOGLVL_DEBUG, "init_storage_backend: initialising null storage backend at: %s", options); store = init_storage_null(); diff --git a/src/store_file.c b/src/store_file.c index 9c824455..f574c2a3 100644 --- a/src/store_file.c +++ b/src/store_file.c @@ -49,8 +49,7 @@ static int file_tile_read(struct storage_backend * store, const char *xmlconfig, char path[PATH_MAX]; int meta_offset, fd; unsigned int pos; - unsigned int header_len = sizeof(struct meta_layout) + METATILE*METATILE*sizeof(struct entry); - struct meta_layout *m = (struct meta_layout *)malloc(header_len); + struct meta_layout *m = (struct meta_layout *)malloc(METATILE_HEADER_LEN); size_t file_offset, tile_size; meta_offset = xyzo_to_meta(path, sizeof(path), store->storage_ctx, xmlconfig, options, x, y, z); @@ -63,8 +62,8 @@ static int file_tile_read(struct storage_backend * store, const char *xmlconfig, } pos = 0; - while (pos < header_len) { - size_t len = header_len - pos; + while (pos < METATILE_HEADER_LEN) { + size_t len = METATILE_HEADER_LEN - pos; int got = read(fd, ((unsigned char *) m) + pos, len); if (got < 0) { snprintf(log_msg,PATH_MAX - 1, "Failed to read complete header for metatile %s Reason: %s\n", path, strerror(errno)); @@ -77,7 +76,7 @@ static int file_tile_read(struct storage_backend * store, const char *xmlconfig, break; } } - if (pos < header_len) { + if (pos < METATILE_HEADER_LEN) { snprintf(log_msg,PATH_MAX - 1, "Meta file %s too small to contain header\n", path); close(fd); free(m); diff --git a/src/store_s3.c b/src/store_s3.c new file mode 100644 index 00000000..44a32352 --- /dev/null +++ b/src/store_s3.c @@ -0,0 +1,661 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBS3 +#include +#endif + +#include "store.h" +#include "store_file_utils.h" +#include "store_s3.h" +#include "metatile.h" +#include "render_config.h" +#include "protocol.h" + +#ifdef HAVE_LIBS3 + +static pthread_mutex_t qLock; +static int store_s3_initialized = 0; + +struct s3_tile_request { + const char *path; + size_t tile_size; + char *tile; + int64_t tile_mod_time; + int tile_expired; + size_t cur_offset; + S3Status result; + const S3ErrorDetails *error_details; +}; + +struct store_s3_ctx { + S3BucketContext* ctx; + const char *basepath; + char *urlcopy; +}; + +static int store_s3_xyz_to_storagekey(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, char *key, size_t keylen) +{ + int offset; + if (options) { + offset = xyzo_to_meta(key, keylen, ((struct store_s3_ctx*) (store->storage_ctx))->basepath, xmlconfig, options, x, y, z); + } else { + offset = xyz_to_meta(key, keylen, ((struct store_s3_ctx*) (store->storage_ctx))->basepath, xmlconfig, x, y, z); + } + + return offset; +} + +static S3Status store_s3_properties_callback(const S3ResponseProperties *properties, void *callbackData) +{ + struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; + + rqst->tile_size = properties->contentLength; + rqst->tile_mod_time = properties->lastModified; + rqst->tile_expired = 0; + const S3NameValue *respMetadata = properties->metaData; + for (int i = 0; i < properties->metaDataCount; i++) { + if (0 == strcmp(respMetadata[i].name, "expired")) { + rqst->tile_expired = atoi(respMetadata[i].value); + } + } + + //log_message(STORE_LOGLVL_DEBUG, "store_s3_properties_callback: got properties for tile %s, length: %ld, content type: %s, expired: %d", rqst->path, rqst->tile_size, properties->contentType, rqst->tile_expired); + + return S3StatusOK; +} + +S3Status store_s3_object_data_callback(int bufferSize, const char *buffer, void *callbackData) +{ + struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; + + if (rqst->cur_offset == 0 && rqst->tile == NULL) { + //log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: allocating %z byte buffer for tile", rqst->tile_size); + rqst->tile = malloc(rqst->tile_size); + if (NULL == rqst->tile) { + log_message(STORE_LOGLVL_ERR, "store_s3_object_data_callback: could not allocate %z byte buffer for tile!", rqst->tile_size); + return S3StatusOutOfMemory; + } + } + + //log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: appending %ld bytes to buffer, new offset %ld", bufferSize, rqst->cur_offset + bufferSize); + memcpy(rqst->tile + rqst->cur_offset, buffer, bufferSize); + rqst->cur_offset += bufferSize; + return S3StatusOK; +} + +int store_s3_put_object_data_callback(int bufferSize, char *buffer, void *callbackData) +{ + struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; + if (rqst->cur_offset == rqst->tile_size) { + // indicate "end of data" + log_message(STORE_LOGLVL_DEBUG, "store_s3_put_object_data_callback: completed put"); + return 0; + } + size_t bytesToWrite = MIN(bufferSize, rqst->tile_size - rqst->cur_offset); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_put_object_data_callback: uploading data, writing %ld bytes to buffer, cur offset %ld, new offset %ld", bytesToWrite, rqst->cur_offset, rqst->cur_offset + bytesToWrite); + memcpy(buffer, rqst->tile + rqst->cur_offset, bytesToWrite); + rqst->cur_offset += bytesToWrite; + return bytesToWrite; +} + +void store_s3_complete_callback(S3Status status, const S3ErrorDetails *errorDetails, void *callbackData) +{ + struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; + //log_message(STORE_LOGLVL_DEBUG, "store_s3_complete_callback: request complete, status %d (%s)", status, S3_get_status_name(status)); + //if (errorDetails && errorDetails->message && (strlen(errorDetails->message) > 0)) { + // log_message(STORE_LOGLVL_DEBUG, " error details: %s", errorDetails->message); + //} + rqst->result = status; + rqst->error_details = errorDetails; +} + +static int store_s3_tile_read(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, char *buf, size_t sz, int *compressed, char *log_msg) +{ + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + char *path = malloc(PATH_MAX); + + //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: fetching tile"); + + int tile_offset = store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: retrieving object %s", path); + + struct S3GetObjectHandler getObjectHandler; + getObjectHandler.responseHandler.propertiesCallback = &store_s3_properties_callback; + getObjectHandler.responseHandler.completeCallback = &store_s3_complete_callback; + getObjectHandler.getObjectDataCallback = &store_s3_object_data_callback; + + struct s3_tile_request request; + request.path = path; + request.cur_offset = 0; + request.tile = NULL; + request.tile_expired = 0; + request.tile_mod_time = 0; + request.tile_size = 0; + + S3_get_object(ctx->ctx, path, NULL, 0, 0, NULL, &getObjectHandler, &request); + + if (request.result != S3StatusOK) { + const char *msg = ""; + if (request.error_details && request.error_details->message) { + msg = request.error_details->message; + } + log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: failed to retrieve object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); + free(path); + path = NULL; + return -1; + } + + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: retrieved metatile %s of size %i", path, request.tile_size); + + free(path); + path = NULL; + + // extract tile from metatile + + if (request.tile_size < METATILE_HEADER_LEN) { + snprintf(log_msg, PATH_MAX - 1, "Meta file %s too small to contain header\n", path); + free(request.tile); + return -3; + } + struct meta_layout *m = (struct meta_layout*) request.tile; + + if (memcmp(m->magic, META_MAGIC, strlen(META_MAGIC))) { + if (memcmp(m->magic, META_MAGIC_COMPRESSED, strlen(META_MAGIC_COMPRESSED))) { + snprintf(log_msg, PATH_MAX - 1, "Meta file %s header magic mismatch\n", path); + free(request.tile); + return -4; + } else { + *compressed = 1; + } + } else { + *compressed = 0; + } + + if (m->count != (METATILE * METATILE)) { + snprintf(log_msg, PATH_MAX - 1, "Meta file %s header bad count %d != %d\n", path, m->count, METATILE * METATILE); + free(request.tile); + return -5; + } + + int buffer_offset = m->index[tile_offset].offset; + int tile_size = m->index[tile_offset].size; + + if (tile_size > sz) { + snprintf(log_msg, PATH_MAX - 1, "tile of length %d too big to fit buffer of length %zd\n", tile_size, sz); + free(request.tile); + return -6; + } + + memcpy(buf, request.tile + buffer_offset, tile_size); + + free(request.tile); + request.tile = NULL; + + return tile_size; +} + +static struct stat_info store_s3_tile_stat(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z) +{ + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + + struct stat_info tile_stat; + tile_stat.size = -1; + tile_stat.expired = 0; + tile_stat.mtime = 0; + tile_stat.atime = 0; + tile_stat.ctime = 0; + + char *path = malloc(PATH_MAX); + if (NULL == path) { + log_message(STORE_LOGLVL_ERR, "store_s3_tile_stat: failed to allocate memory for tile path!"); + return tile_stat; + } + + store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: getting properties for object %s", path); + + struct S3ResponseHandler responseHandler; + responseHandler.propertiesCallback = &store_s3_properties_callback; + responseHandler.completeCallback = &store_s3_complete_callback; + + struct s3_tile_request request; + request.path = path; + request.error_details = NULL; + request.cur_offset = 0; + request.result = S3StatusOK; + request.tile = NULL; + request.tile_expired = 0; + request.tile_mod_time = 0; + request.tile_size = 0; + + S3_head_object(ctx->ctx, path, NULL, &responseHandler, &request); + + if (request.result != S3StatusOK) { + if (request.result == S3StatusHttpErrorNotFound) { + // tile does not exist + //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: tile not found in storage"); + } else { + const char *msg = ""; + if (request.error_details && request.error_details->message) { + msg = request.error_details->message; + } + log_message(STORE_LOGLVL_ERR, "store_s3_tile_stat: failed to retrieve object properties for %s: %d (%s) %s", path, request.result, S3_get_status_name(request.result), msg); + } + free(path); + return tile_stat; + } + + //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: successfully read properties of %s", path); + + tile_stat.size = request.tile_size; + tile_stat.expired = request.tile_expired; + tile_stat.mtime = request.tile_mod_time; + free(path); + return tile_stat; +} + +static char* store_s3_tile_storage_id(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, char *string) +{ + // FIXME: assumes PATH_MAX for length of provided string + store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, string, PATH_MAX); + return string; +} + +static int store_s3_metatile_write(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, const char *buf, int sz) +{ + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + char *path = malloc(PATH_MAX); + store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_write: storing object %s, size %ld", path, sz); + + struct S3PutObjectHandler putObjectHandler; + putObjectHandler.responseHandler.propertiesCallback = &store_s3_properties_callback; + putObjectHandler.responseHandler.completeCallback = &store_s3_complete_callback; + putObjectHandler.putObjectDataCallback = &store_s3_put_object_data_callback; + + struct s3_tile_request request; + request.path = path; + request.tile = (char*) buf; + request.tile_size = sz; + request.cur_offset = 0; + request.tile_expired = 0; + request.result = S3StatusOK; + request.error_details = NULL; + + S3PutProperties props; + props.contentType = "application/octet-stream"; + props.cacheControl = NULL; + props.cannedAcl = S3CannedAclPrivate; // results in no ACL header in POST + props.contentDispositionFilename = NULL; + props.contentEncoding = NULL; + props.expires = -1; + props.md5 = NULL; + props.metaData = NULL; + props.metaDataCount = 0; + props.useServerSideEncryption = 0; + + S3_put_object(ctx->ctx, path, sz, &props, NULL, &putObjectHandler, &request); + free(path); + + if (request.result != S3StatusOK) { + const char *msg = ""; + const char *msg2 = ""; + if (request.error_details) { + if (request.error_details->message) { + msg = request.error_details->message; + } + if (request.error_details->furtherDetails) { + msg2 = request.error_details->furtherDetails; + } + } + log_message(STORE_LOGLVL_ERR, "store_s3_metatile_write: failed to write object: %d(%s)/%s%s", request.result, S3_get_status_name(request.result), msg, msg2); + return -1; + } + + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_write: Wrote object of size %i", sz); + + return sz; +} + +static int store_s3_metatile_delete(struct storage_backend *store, const char *xmlconfig, int x, int y, int z) +{ + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + char *path = malloc(PATH_MAX); + store_s3_xyz_to_storagekey(store, xmlconfig, NULL, x, y, z, path, PATH_MAX); + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_delete: deleting object %s", path); + + struct S3ResponseHandler responseHandler; + responseHandler.propertiesCallback = &store_s3_properties_callback; + responseHandler.completeCallback = &store_s3_complete_callback; + + struct s3_tile_request request; + request.path = path; + request.error_details = NULL; + request.cur_offset = 0; + request.result = S3StatusOK; + request.tile = NULL; + request.tile_expired = 0; + request.tile_mod_time = 0; + request.tile_size = 0; + + S3_delete_object(ctx->ctx, path, NULL, &responseHandler, &request); + free(path); + + if (request.result != S3StatusOK) { + const char *msg = ""; + if (request.error_details && request.error_details->message) { + msg = request.error_details->message; + } + log_message(STORE_LOGLVL_ERR, "store_s3_metatile_delete: failed to delete object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); + return -1; + } + + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_delete: deleted object"); + + return 0; +} + +static int store_s3_metatile_expire(struct storage_backend *store, const char *xmlconfig, int x, int y, int z) +{ + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + char *path = malloc(PATH_MAX); + store_s3_xyz_to_storagekey(store, xmlconfig, NULL, x, y, z, path, PATH_MAX); + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_expire: expiring object %s", path); + + struct S3ResponseHandler responseHandler; + responseHandler.propertiesCallback = &store_s3_properties_callback; + responseHandler.completeCallback = &store_s3_complete_callback; + + struct s3_tile_request request; + request.path = path; + request.error_details = NULL; + request.cur_offset = 0; + request.result = S3StatusOK; + request.tile = NULL; + request.tile_expired = 0; + request.tile_mod_time = 0; + request.tile_size = 0; + + struct S3NameValue expireTag; + expireTag.name = "expired"; + expireTag.value = "1"; + + S3PutProperties props; + props.contentType = "application/octet-stream"; + props.cacheControl = NULL; + props.cannedAcl = S3CannedAclPrivate; // results in no ACL header in POST + props.contentDispositionFilename = NULL; + props.contentEncoding = NULL; + props.expires = -1; + props.md5 = NULL; + props.metaDataCount = 1; + props.metaData = &expireTag; + props.useServerSideEncryption = 0; + + int64_t lastModified; + + S3_copy_object(ctx->ctx, path, ctx->ctx->bucketName, path, &props, &lastModified, 0, NULL, NULL, &responseHandler, &request); + free(path); + + if (request.result != S3StatusOK) { + const char *msg = ""; + if (request.error_details && request.error_details->message) { + msg = request.error_details->message; + } + log_message(STORE_LOGLVL_ERR, "store_s3_metatile_expire: failed to update object: %d (%s)/%s", request.result, S3_get_status_name(request.result), msg); + return -1; + } + + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_expire: updated object metadata"); + + return 0; +} + +static int store_s3_close_storage(struct storage_backend *store) +{ + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + + S3_deinitialize(); + if (NULL != ctx->urlcopy) { + free(ctx->urlcopy); + ctx->urlcopy = NULL; + } + free(ctx); + store->storage_ctx = NULL; + store_s3_initialized = 0; + + return 0; +} + +static char* url_decode(const char *src) +{ + if (NULL == src) { + return NULL; + } + char *dst = (char*) malloc(strlen(src) + 1); + dst[0] = '\0'; + while (*src) { + int c = *src; + if (c == '%' && isxdigit(*(src + 1)) && isxdigit(*(src + 2))) { + char hexdigit[] = + { *(src + 1), *(src + 2), '\0' }; + char decodedchar[2]; + sprintf(decodedchar, "%c", (char) strtol(hexdigit, NULL, 16)); + strncat(dst, decodedchar, 1); + src += 2; + } else { + strncat(dst, src, 1); + } + src++; + } + return dst; +} + +static const char* env_expand(const char *src) +{ + if (strstr(src, "${") == src && strrchr(src, '}') == (src + strlen(src) - 1)) { + char tmp[strlen(src) + 1]; + strcpy(tmp, src); + tmp[strlen(tmp) - 1] = '\0'; + char *val = getenv(tmp + 2); + if (NULL == val) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: environment variable %s not defined when initializing S3 configuration!", tmp + 2); + return NULL; + } + return val; + } + return src; +} +#endif //Have libs3 + +struct storage_backend* init_storage_s3(const char *connection_string) +{ +#ifndef HAVE_LIBS3 + log_message(STORE_LOGLVL_ERR, + "init_storage_s3: Support for libs3 and therefore S3 storage has not been compiled into this program"); + return NULL; +#else + if (strstr(connection_string, "s3://") != connection_string) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: connection string invalid for S3 storage!"); + return NULL; + } + + struct storage_backend *store = malloc(sizeof(struct storage_backend)); + struct store_s3_ctx *ctx = malloc(sizeof(struct store_s3_ctx)); + + S3Status res = S3StatusErrorUnknown; + + if (!store || !ctx) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: failed to allocate memory for context"); + if (store) + free(store); + if (ctx) + free(ctx); + return NULL; + } + + pthread_mutex_lock(&qLock); + if (!store_s3_initialized) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3: global init of libs3"); + res = S3_initialize(NULL, S3_INIT_ALL, NULL); + store_s3_initialized = 1; + } else { + res = S3StatusOK; + } + + pthread_mutex_unlock(&qLock); + if (res != S3StatusOK) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: failed to initialize S3 library: %s", S3_get_status_name(res)); + free(ctx); + free(store); + return NULL; + } + + // parse out the context information from the URL: + // s3://:[@]/[@region][/] + struct S3BucketContext *bctx = ctx->ctx = malloc(sizeof(struct S3BucketContext)); + + ctx->urlcopy = strdup(connection_string); + if (NULL == ctx->urlcopy) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: error allocating memory for connection string!"); + free(ctx); + free(store); + return NULL; + } + + // advance past "s3://" + char *fullurl = &ctx->urlcopy[5]; + bctx->accessKeyId = strsep(&fullurl, ":"); + char *nextSlash = strchr(fullurl, '/'); + char *nextAt = strchr(fullurl, '@'); + if ((nextAt != NULL) && (nextAt < nextSlash)) { + // there's an S3 host name in the URL + bctx->secretAccessKey = strsep(&fullurl, "@"); + bctx->hostName = strsep(&fullurl, "/"); + if (bctx->hostName != NULL && strlen(bctx->hostName) <= 0) { + bctx->hostName = NULL; + } + } else { + bctx->secretAccessKey = strsep(&fullurl, "/"); + bctx->hostName = NULL; + } + + if (strchr(fullurl, '@')) { + // there's a region name with the bucket name + bctx->bucketName = strsep(&fullurl, "@"); + bctx->authRegion = strsep(&fullurl, "/"); + } + else { + bctx->bucketName = strsep(&fullurl, "/"); + bctx->authRegion = NULL; + } + + if (bctx->accessKeyId != NULL && strlen(bctx->accessKeyId) <= 0) { + bctx->accessKeyId = NULL; + } + if (bctx->secretAccessKey != NULL && strlen(bctx->secretAccessKey) <= 0) { + bctx->secretAccessKey = NULL; + } + if (bctx->bucketName != NULL && strlen(bctx->bucketName) <= 0) { + bctx->bucketName = NULL; + } + + if (bctx->accessKeyId == NULL) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: S3 access key ID not provided in connection string!"); + free(ctx); + free(store); + return NULL; + } + + if (bctx->secretAccessKey == NULL) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: S3 secret access key not provided in connection string!"); + free(ctx); + free(store); + return NULL; + } + + if (bctx->bucketName == NULL) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: S3 bucket name not provided in connection string!"); + free(ctx); + free(store); + return NULL; + } + + ctx->basepath = fullurl; + + bctx->accessKeyId = env_expand(bctx->accessKeyId); + if (bctx->accessKeyId == NULL) { + free(ctx); + free(store); + return NULL; + } + bctx->accessKeyId = url_decode(bctx->accessKeyId); + + bctx->secretAccessKey = env_expand(bctx->secretAccessKey); + if (bctx->secretAccessKey == NULL) { + free(ctx); + free(store); + return NULL; + } + bctx->secretAccessKey = url_decode(bctx->secretAccessKey); + + if (bctx->hostName) { + bctx->hostName = env_expand(bctx->hostName); + if (bctx->hostName == NULL) { + free(ctx); + free(store); + return NULL; + } + bctx->hostName = url_decode(bctx->hostName); + } + + bctx->bucketName = env_expand(bctx->bucketName); + if (bctx->bucketName == NULL) { + free(ctx); + free(store); + return NULL; + } + bctx->bucketName = url_decode(bctx->bucketName); + + bctx->protocol = S3ProtocolHTTPS; + bctx->securityToken = NULL; + bctx->uriStyle = S3UriStyleVirtualHost; + + ctx->basepath = env_expand(ctx->basepath); + if (ctx->basepath == NULL) { + free(ctx); + free(store); + return NULL; + } + ctx->basepath = url_decode(ctx->basepath); + + if (bctx->hostName && bctx->authRegion) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, host: %s, region: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->hostName, ctx->ctx->authRegion, ctx->ctx->bucketName, ctx->basepath); + } else if (bctx->hostName) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, host: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->hostName, ctx->ctx->bucketName, ctx->basepath); + } else if (bctx->authRegion) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, region: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->authRegion, ctx->ctx->bucketName, ctx->basepath); + } else { + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->bucketName, ctx->basepath); + } + store->storage_ctx = ctx; + + store->tile_read = &store_s3_tile_read; + store->tile_stat = &store_s3_tile_stat; + store->metatile_write = &store_s3_metatile_write; + store->metatile_delete = &store_s3_metatile_delete; + store->metatile_expire = &store_s3_metatile_expire; + store->tile_storage_id = &store_s3_tile_storage_id; + store->close_storage = &store_s3_close_storage; + + return store; +#endif +}