From a0f8f7ae90ff0b88c73fd7637b25771b3c21a14b Mon Sep 17 00:00:00 2001 From: Hugo Laloge Date: Tue, 11 Aug 2020 08:53:36 +0200 Subject: [PATCH 01/12] Use rusoto for S3 cache --- Cargo.lock | 610 ++++++++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 5 +- src/cache/s3.rs | 105 ++++++--- 3 files changed, 640 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49b118575..27b5948f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,6 +82,17 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-trait" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a265e3abeffdce30b2e26b7a11b222fe37c6067404001b434101457d0385eb92" +dependencies = [ + "proc-macro2", + "quote 1.0.7", + "syn 1.0.54", +] + [[package]] name = "atty" version = "0.2.14" @@ -119,6 +130,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base-x" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" + [[package]] name = "base64" version = "0.9.3" @@ -144,6 +161,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + [[package]] name = "base64" version = "0.13.0" @@ -215,6 +238,15 @@ dependencies = [ "generic-array 0.12.3", ] +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array 0.14.4", +] + [[package]] name = "block-padding" version = "0.1.5" @@ -246,6 +278,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + [[package]] name = "byte-tools" version = "0.3.1" @@ -311,7 +349,8 @@ dependencies = [ "libc", "num-integer", "num-traits 0.2.14", - "time", + "serde", + "time 0.1.44", "winapi 0.3.9", ] @@ -365,7 +404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d6364d028778d0d98b6014fa5882da377cd10d3492b7734d266a428e9b1fca" dependencies = [ "log 0.4.11", - "md5", + "md5 0.3.8", ] [[package]] @@ -380,7 +419,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5" dependencies = [ - "time", + "time 0.1.44", "url 1.7.2", ] @@ -397,7 +436,7 @@ dependencies = [ "publicsuffix", "serde", "serde_json", - "time", + "time 0.1.44", "try_from", "url 1.7.2", ] @@ -424,6 +463,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384f8c53175c890920b6e0127b730709d2a173ca6c4dfdc81618ac9b46f648fe" +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + [[package]] name = "crc32fast" version = "1.2.1" @@ -603,6 +648,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "dirs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" +dependencies = [ + "cfg-if 0.1.10", + "dirs-sys", +] + [[package]] name = "dirs-sys" version = "0.3.5" @@ -614,6 +669,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + [[package]] name = "doc-comment" version = "0.3.3" @@ -883,7 +944,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr 2.3.4", - "pin-project", + "pin-project 1.0.2", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -951,7 +1012,7 @@ dependencies = [ "bytes 0.4.12", "fnv", "futures 0.1.30", - "http", + "http 0.1.21", "indexmap", "log 0.4.11", "slab", @@ -959,6 +1020,26 @@ dependencies = [ "tokio-io", ] +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.1", + "indexmap", + "slab", + "tokio 0.2.23", + "tokio-util 0.3.1", + "tracing", + "tracing-futures", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -974,6 +1055,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + [[package]] name = "hmac" version = "0.7.1" @@ -984,6 +1071,16 @@ dependencies = [ "digest 0.8.1", ] +[[package]] +name = "hmac" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" +dependencies = [ + "crypto-mac 0.8.0", + "digest 0.9.0", +] + [[package]] name = "http" version = "0.1.21" @@ -995,6 +1092,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +dependencies = [ + "bytes 0.5.6", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.1.0" @@ -1003,10 +1111,20 @@ checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" dependencies = [ "bytes 0.4.12", "futures 0.1.30", - "http", + "http 0.1.21", "tokio-buf", ] +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes 0.5.6", + "http 0.2.1", +] + [[package]] name = "httparse" version = "1.3.4" @@ -1031,16 +1149,16 @@ dependencies = [ "bytes 0.4.12", "futures 0.1.30", "futures-cpupool", - "h2", - "http", - "http-body", + "h2 0.1.26", + "http 0.1.21", + "http-body 0.1.0", "httparse", "iovec", "itoa", "log 0.4.11", "net2", "rustc_version", - "time", + "time 0.1.44", "tokio 0.1.22", "tokio-buf", "tokio-executor", @@ -1049,7 +1167,31 @@ dependencies = [ "tokio-tcp", "tokio-threadpool", "tokio-timer", - "want", + "want 0.2.0", +] + +[[package]] +name = "hyper" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e68a8dd9716185d9e64ea473ea6ef63529252e3e27623295a0378a19665d5eb" +dependencies = [ + "bytes 0.5.6", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.2.7", + "http 0.2.1", + "http-body 0.3.1", + "httparse", + "itoa", + "pin-project 0.4.27", + "socket2", + "time 0.1.44", + "tokio 0.2.23", + "tower-service 0.3.0", + "tracing", + "want 0.3.0", ] [[package]] @@ -1060,11 +1202,24 @@ checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" dependencies = [ "bytes 0.4.12", "futures 0.1.30", - "hyper", + "hyper 0.12.35", "native-tls", "tokio-io", ] +[[package]] +name = "hyper-tls" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +dependencies = [ + "bytes 0.5.6", + "hyper 0.13.7", + "native-tls", + "tokio 0.2.23", + "tokio-tls", +] + [[package]] name = "hyperx" version = "0.12.0" @@ -1073,13 +1228,13 @@ checksum = "78e2d2253d7a17929560fc3adf48c48fc924c94fa4507e037a60e6bc55c0eda6" dependencies = [ "base64 0.9.3", "bytes 0.4.12", - "http", + "http 0.1.21", "httparse", "language-tags", "log 0.4.11", "mime 0.3.16", "percent-encoding 1.0.1", - "time", + "time 0.1.44", "unicase 2.6.0", ] @@ -1275,9 +1430,9 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a18af3dcaf2b0219366cdb4e2af65a6101457b415c3d1a5c71dd9c2b7c77b9c8" dependencies = [ - "block-buffer", + "block-buffer 0.7.3", "digest 0.8.1", - "opaque-debug", + "opaque-debug 0.2.3", ] [[package]] @@ -1286,6 +1441,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79c56d6a0b07f9e19282511c83fc5b086364cbae4ba8c7d5f190c3d9b0425a48" +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memcached-rs" version = "0.4.2" @@ -1579,6 +1740,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "openssl" version = "0.10.30" @@ -1689,13 +1856,33 @@ dependencies = [ "unicase 1.4.2", ] +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + [[package]] name = "pin-project" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" dependencies = [ - "pin-project-internal", + "pin-project-internal 1.0.2", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2", + "quote 1.0.7", + "syn 1.0.54", ] [[package]] @@ -2040,7 +2227,7 @@ dependencies = [ "pin-project-lite 0.1.11", "sha1", "tokio 0.2.23", - "tokio-util", + "tokio-util 0.2.0", "url 2.2.0", ] @@ -2101,9 +2288,9 @@ dependencies = [ "encoding_rs", "flate2", "futures 0.1.30", - "http", - "hyper", - "hyper-tls", + "http 0.1.21", + "hyper 0.12.35", + "hyper-tls 0.3.2", "log 0.4.11", "mime 0.3.16", "mime_guess 2.0.3", @@ -2111,7 +2298,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "time", + "time 0.1.44", "tokio 0.1.22", "tokio-executor", "tokio-io", @@ -2163,11 +2350,97 @@ dependencies = [ "sha1", "term", "threadpool", - "time", + "time 0.1.44", "tiny_http", "url 1.7.2", ] +[[package]] +name = "rusoto_core" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e977941ee0658df96fca7291ecc6fc9a754600b21ad84b959eb1dbbc9d5abcc7" +dependencies = [ + "async-trait", + "base64 0.12.3", + "bytes 0.5.6", + "crc32fast", + "futures 0.3.8", + "http 0.2.1", + "hyper 0.13.7", + "hyper-tls 0.4.3", + "lazy_static", + "log 0.4.11", + "md5 0.7.0", + "percent-encoding 2.1.0", + "pin-project 0.4.27", + "rusoto_credential", + "rusoto_signature", + "rustc_version", + "serde", + "serde_json", + "tokio 0.2.23", + "xml-rs", +] + +[[package]] +name = "rusoto_credential" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac05563f83489b19b4d413607a30821ab08bbd9007d14fa05618da3ef09d8b" +dependencies = [ + "async-trait", + "chrono", + "dirs 2.0.2", + "futures 0.3.8", + "hyper 0.13.7", + "pin-project 0.4.27", + "regex", + "serde", + "serde_json", + "shlex", + "tokio 0.2.23", + "zeroize", +] + +[[package]] +name = "rusoto_s3" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1146e37a7c1df56471ea67825fe09bbbd37984b5f6e201d8b2e0be4ee15643d8" +dependencies = [ + "async-trait", + "bytes 0.5.6", + "futures 0.3.8", + "rusoto_core", + "xml-rs", +] + +[[package]] +name = "rusoto_signature" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a740a88dde8ded81b6f2cff9cd5e054a5a2e38a38397260f7acdd2c85d17dd" +dependencies = [ + "base64 0.12.3", + "bytes 0.5.6", + "futures 0.3.8", + "hex", + "hmac 0.8.1", + "http 0.2.1", + "hyper 0.13.7", + "log 0.4.11", + "md5 0.7.0", + "percent-encoding 2.1.0", + "pin-project 0.4.27", + "rusoto_credential", + "rustc_version", + "serde", + "sha2 0.9.1", + "time 0.2.16", + "tokio 0.2.23", +] + [[package]] name = "rust-argon2" version = "0.8.3" @@ -2246,9 +2519,9 @@ dependencies = [ "flate2", "futures 0.1.30", "futures 0.3.8", - "hmac", - "http", - "hyper", + "hmac 0.7.1", + "http 0.1.21", + "hyper 0.12.35", "hyperx", "itertools", "jobserver", @@ -2273,17 +2546,20 @@ dependencies = [ "retry", "ring", "rouille", + "rusoto_core", + "rusoto_s3", "selenium-rs", "serde", "serde_derive", "serde_json", "sha-1", - "sha2", + "sha2 0.8.2", "strip-ansi-escapes", "syslog", "tar", "tempfile", - "time", + "time 0.1.44", + "tokio 0.2.23", "tokio-compat", "tokio-io", "tokio-named-pipes", @@ -2423,10 +2699,10 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" dependencies = [ - "block-buffer", + "block-buffer 0.7.3", "digest 0.8.1", "fake-simd", - "opaque-debug", + "opaque-debug 0.2.3", ] [[package]] @@ -2441,12 +2717,31 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" dependencies = [ - "block-buffer", + "block-buffer 0.7.3", "digest 0.8.1", "fake-simd", - "opaque-debug", + "opaque-debug 0.2.3", ] +[[package]] +name = "sha2" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 0.1.10", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "shlex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" + [[package]] name = "signal-hook-registry" version = "1.2.2" @@ -2505,6 +2800,64 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "standback" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0437cfb83762844799a60e1e3b489d5ceb6a650fbacb86437badc1b6d87b246" +dependencies = [ + "version_check 0.9.2", +] + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote 1.0.7", + "serde", + "serde_derive", + "syn 1.0.54", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote 1.0.7", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn 1.0.54", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + [[package]] name = "string" version = "0.2.1" @@ -2593,7 +2946,7 @@ dependencies = [ "error-chain", "libc", "log 0.4.11", - "time", + "time 0.1.44", ] [[package]] @@ -2639,7 +2992,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" dependencies = [ "byteorder", - "dirs", + "dirs 1.0.5", "winapi 0.3.9", ] @@ -2710,6 +3063,44 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "time" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a51cadc5b1eec673a685ff7c33192ff7b7603d0b75446fb354939ee615acb15" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check 0.9.2", + "winapi 0.3.9", +] + +[[package]] +name = "time-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9b6e9f095bc105e183e3cd493d72579be3181ad4004fceb01adbe9eecab2d" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote 1.0.7", + "standback", + "syn 1.0.54", +] + [[package]] name = "tiny_http" version = "0.6.2" @@ -2776,10 +3167,14 @@ dependencies = [ "libc", "memchr 2.3.4", "mio", + "mio-named-pipes", "mio-uds", "num_cpus", "pin-project-lite 0.1.11", + "signal-hook-registry", "slab", + "tokio-macros", + "winapi 0.3.9", ] [[package]] @@ -2863,6 +3258,17 @@ dependencies = [ "log 0.4.11", ] +[[package]] +name = "tokio-macros" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" +dependencies = [ + "proc-macro2", + "quote 1.0.7", + "syn 1.0.54", +] + [[package]] name = "tokio-named-pipes" version = "0.1.0" @@ -3008,6 +3414,16 @@ dependencies = [ "tokio-executor", ] +[[package]] +name = "tokio-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" +dependencies = [ + "native-tls", + "tokio 0.2.23", +] + [[package]] name = "tokio-udp" version = "0.1.6" @@ -3055,6 +3471,20 @@ dependencies = [ "tokio 0.2.23", ] +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log 0.4.11", + "pin-project-lite 0.1.11", + "tokio 0.2.23", +] + [[package]] name = "toml" version = "0.5.7" @@ -3077,7 +3507,7 @@ dependencies = [ "tower-limit", "tower-load-shed", "tower-retry", - "tower-service", + "tower-service 0.2.0", "tower-timeout", "tower-util", ] @@ -3092,7 +3522,7 @@ dependencies = [ "tokio-executor", "tokio-sync", "tower-layer", - "tower-service", + "tower-service 0.2.0", "tracing", ] @@ -3103,7 +3533,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73a7632286f78164d65d18fd0e570307acde9362489aa5c8c53e6315cc2bde47" dependencies = [ "futures 0.1.30", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3113,7 +3543,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ddf07e10c07dcc8f41da6de036dc66def1a85b70eb8a385159e3908bb258328" dependencies = [ "futures 0.1.30", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3126,7 +3556,7 @@ dependencies = [ "tokio-sync", "tokio-timer", "tower-layer", - "tower-service", + "tower-service 0.2.0", "tracing", ] @@ -3138,7 +3568,7 @@ checksum = "04fbaf5bfb63d84204db87b9b2aeec61549613f2bbb8706dcc36f5f3ea8cd769" dependencies = [ "futures 0.1.30", "tower-layer", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3150,7 +3580,7 @@ dependencies = [ "futures 0.1.30", "tokio-timer", "tower-layer", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3162,6 +3592,12 @@ dependencies = [ "futures 0.1.30", ] +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + [[package]] name = "tower-timeout" version = "0.1.1" @@ -3171,7 +3607,7 @@ dependencies = [ "futures 0.1.30", "tokio-timer", "tower-layer", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3183,7 +3619,7 @@ dependencies = [ "futures 0.1.30", "tokio-io", "tower-layer", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3219,6 +3655,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + [[package]] name = "treeline" version = "0.1.0" @@ -3448,6 +3894,16 @@ dependencies = [ "try-lock", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log 0.4.11", + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -3460,6 +3916,60 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasm-bindgen" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +dependencies = [ + "bumpalo", + "lazy_static", + "log 0.4.11", + "proc-macro2", + "quote 1.0.7", + "syn 1.0.54", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +dependencies = [ + "quote 1.0.7", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +dependencies = [ + "proc-macro2", + "quote 1.0.7", + "syn 1.0.54", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" + [[package]] name = "which" version = "4.0.2" @@ -3541,6 +4051,18 @@ dependencies = [ "libc", ] +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + +[[package]] +name = "zeroize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" + [[package]] name = "zip" version = "0.5.9" diff --git a/Cargo.toml b/Cargo.toml index 6d11173b7..408c00538 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,8 @@ regex = "1" reqwest = { version = "0.9.11", optional = true } retry = "0.4.0" ring = { version = "0.14.6", optional = true } +rusoto_core = { version = "0.45.0", optional = true } +rusoto_s3 = { version = "0.45.0", optional = true } sha-1 = { version = "0.8", optional = true } sha2 = { version = "0.8", optional = true } serde = "1.0" @@ -69,6 +71,7 @@ strip-ansi-escapes = "0.1" tar = "0.4" tempfile = "3" time = "0.1.35" +tokio_02 = { package = "tokio", version = "0.2", features = ["io-util"], optional = true } tokio-compat = "0.1" tokio-io = "0.1" tokio-process = "0.2" @@ -127,7 +130,7 @@ features = [ default = ["dist-client", "s3"] all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure"] azure = ["chrono", "hyper", "hyperx", "url", "hmac", "md-5", "sha2"] -s3 = ["chrono", "hyper", "hyperx", "reqwest", "simple-s3", "hmac", "sha-1"] +s3 = ["chrono", "hyper", "hyperx", "reqwest", "rusoto_core", "rusoto_s3", "simple-s3", "tokio_02", "hmac", "sha-1"] simple-s3 = [] gcs = ["chrono", "hyper", "hyperx", "reqwest", "ring", "untrusted", "url"] memcached = ["memcached-rs"] diff --git a/src/cache/s3.rs b/src/cache/s3.rs index b08fcee9e..ac5a62e82 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -19,11 +19,16 @@ use crate::simples3::{ use directories::UserDirs; use futures::future; use futures::future::Future; +use futures_03::{compat::Compat as _, future::TryFutureExt as _}; +use rusoto_core::Region; +use rusoto_s3::{GetObjectOutput, GetObjectRequest, PutObjectRequest, S3Client, S3 as _, S3}; use std::io; use std::rc::Rc; use std::time::{Duration, Instant}; +use tokio_02::io::AsyncReadExt as _; use crate::errors::*; +use hyperx::header::CacheDirective; /// A cache that stores entries in Amazon S3. pub struct S3Cache { @@ -33,10 +38,14 @@ pub struct S3Cache { provider: AutoRefreshingProvider, /// Prefix to be used for bucket keys. key_prefix: String, + client: S3Client, + bucket_name: String, } impl S3Cache { /// Create a new `S3Cache` storing data in `bucket`. + /// TODO: Handle custom region + /// TODO: Handle use_ssl pub fn new(bucket: &str, endpoint: &str, use_ssl: bool, key_prefix: &str) -> Result { let user_dirs = UserDirs::new().context("Couldn't get user directories")?; let home = user_dirs.home_dir(); @@ -51,14 +60,50 @@ impl S3Cache { let provider = AutoRefreshingProvider::new(ChainProvider::with_profile_providers(profile_providers)); let ssl_mode = if use_ssl { Ssl::Yes } else { Ssl::No }; + let bucket_name = bucket.to_owned(); let bucket = Rc::new(Bucket::new(bucket, endpoint, ssl_mode)?); + let client = S3Client::new(Region::default()); Ok(S3Cache { bucket, provider, key_prefix: key_prefix.to_owned(), + client, + bucket_name, }) } + async fn get_object(client: S3Client, request: GetObjectRequest) -> Result { + let result = client.get_object(request).await; + match result { + Ok(output) => Self::read_object_output(output).await, + Err(rusoto_core::RusotoError::Service(rusoto_s3::GetObjectError::NoSuchKey(_))) => { + Ok(Cache::Miss) + } + Err(e) => Err(e.into()), + } + } + + async fn read_object_output(output: GetObjectOutput) -> Result { + let body = output.body.context("no HTTP body")?; + let mut body_reader = body.into_async_read(); + let mut body = Vec::new(); + body_reader + .read_to_end(&mut body) + .await + .context("failed to read HTTP body")?; + let hit = CacheRead::from(io::Cursor::new(body))?; + Ok(Cache::Hit(hit)) + } + + async fn put_object(client: S3Client, request: PutObjectRequest) -> Result<()> { + client + .put_object(request) + .await + .map(|_| ()) + .context("failed to put cache entry in s3") + .into() + } + fn normalize_key(&self, key: &str) -> String { format!( "{}{}/{}/{}/{}", @@ -75,30 +120,14 @@ impl Storage for S3Cache { fn get(&self, key: &str) -> SFuture { let key = self.normalize_key(key); - let result_cb = |result| match result { - Ok(data) => { - let hit = CacheRead::from(io::Cursor::new(data))?; - Ok(Cache::Hit(hit)) - } - Err(e) => { - warn!("Got AWS error: {:?}", e); - Ok(Cache::Miss) - } + let client = self.client.clone(); + let request = GetObjectRequest { + bucket: self.bucket_name.clone(), + key, + ..Default::default() }; - let bucket = self.bucket.clone(); - let response = self - .provider - .credentials() - .then(move |credentials| match credentials { - Ok(creds) => bucket.get(&key, Some(&creds)), - Err(e) => { - debug!("Could not load AWS creds: {}", e); - bucket.get(&key, None) - } - }) - .then(result_cb); - Box::new(response) + Box::new(Box::pin(Self::get_object(client, request)).compat()) } fn put(&self, key: &str, entry: CacheWrite) -> SFuture { @@ -108,19 +137,25 @@ impl Storage for S3Cache { Ok(data) => data, Err(e) => return f_err(e), }; - let credentials = self - .provider - .credentials() - .fcontext("failed to get AWS credentials"); - - let bucket = self.bucket.clone(); - let response = credentials.and_then(move |credentials| { - bucket - .put(&key, data, &credentials) - .fcontext("failed to put cache entry in s3") - }); - - Box::new(response.map(move |_| start.elapsed())) + let data_length = data.len(); + + let client = self.client.clone(); + let request = PutObjectRequest { + bucket: self.bucket_name.clone(), + body: Some(data.into()), + // Two weeks + cache_control: Some(CacheDirective::MaxAge(1_296_000).to_string()), + content_length: Some(data_length as i64), + content_type: Some("application/octet-stream".to_owned()), + key, + ..Default::default() + }; + + Box::new( + Box::pin(Self::put_object(client, request)) + .compat() + .then(move |_| future::ok(start.elapsed())), + ) } fn location(&self) -> String { From 420b2b424bec46e2d5a1827a600be84d0de113e5 Mon Sep 17 00:00:00 2001 From: Hugo Laloge Date: Thu, 13 Aug 2020 08:48:24 +0200 Subject: [PATCH 02/12] Remove simples3 module --- Cargo.toml | 7 +- src/cache/s3.rs | 43 +-- src/lib.rs | 2 - src/simples3/credential.rs | 530 ------------------------------------- src/simples3/mod.rs | 19 -- src/simples3/s3.rs | 251 ------------------ 6 files changed, 13 insertions(+), 839 deletions(-) delete mode 100644 src/simples3/credential.rs delete mode 100644 src/simples3/mod.rs delete mode 100644 src/simples3/s3.rs diff --git a/Cargo.toml b/Cargo.toml index 408c00538..ad1e1c19f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,8 +60,8 @@ regex = "1" reqwest = { version = "0.9.11", optional = true } retry = "0.4.0" ring = { version = "0.14.6", optional = true } -rusoto_core = { version = "0.45.0", optional = true } -rusoto_s3 = { version = "0.45.0", optional = true } +rusoto_core = { version = "0.45.0", features = ["native-tls"], optional = true } +rusoto_s3 = { version = "0.45.0", features = ["native-tls"], optional = true } sha-1 = { version = "0.8", optional = true } sha2 = { version = "0.8", optional = true } serde = "1.0" @@ -130,8 +130,7 @@ features = [ default = ["dist-client", "s3"] all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure"] azure = ["chrono", "hyper", "hyperx", "url", "hmac", "md-5", "sha2"] -s3 = ["chrono", "hyper", "hyperx", "reqwest", "rusoto_core", "rusoto_s3", "simple-s3", "tokio_02", "hmac", "sha-1"] -simple-s3 = [] +s3 = ["hyperx", "rusoto_core", "rusoto_s3", "tokio_02"] gcs = ["chrono", "hyper", "hyperx", "reqwest", "ring", "untrusted", "url"] memcached = ["memcached-rs"] # Enable features that require unstable features of Nightly Rust. diff --git a/src/cache/s3.rs b/src/cache/s3.rs index ac5a62e82..6d532abdf 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -13,62 +13,39 @@ // limitations under the License. use crate::cache::{Cache, CacheRead, CacheWrite, Storage}; -use crate::simples3::{ - AutoRefreshingProvider, Bucket, ChainProvider, ProfileProvider, ProvideAwsCredentials, Ssl, -}; -use directories::UserDirs; use futures::future; use futures::future::Future; use futures_03::{compat::Compat as _, future::TryFutureExt as _}; -use rusoto_core::Region; +use hyperx::header::CacheDirective; +use rusoto_core::{self, Region}; use rusoto_s3::{GetObjectOutput, GetObjectRequest, PutObjectRequest, S3Client, S3 as _, S3}; use std::io; -use std::rc::Rc; use std::time::{Duration, Instant}; use tokio_02::io::AsyncReadExt as _; use crate::errors::*; -use hyperx::header::CacheDirective; /// A cache that stores entries in Amazon S3. pub struct S3Cache { - /// The S3 bucket. - bucket: Rc, - /// Credentials provider. - provider: AutoRefreshingProvider, + /// The name of the bucket. + bucket_name: String, + /// The S3 client to be used for the Get and Put requests. + client: S3Client, /// Prefix to be used for bucket keys. key_prefix: String, - client: S3Client, - bucket_name: String, } impl S3Cache { /// Create a new `S3Cache` storing data in `bucket`. /// TODO: Handle custom region + /// TODO: Handle endpoint? /// TODO: Handle use_ssl pub fn new(bucket: &str, endpoint: &str, use_ssl: bool, key_prefix: &str) -> Result { - let user_dirs = UserDirs::new().context("Couldn't get user directories")?; - let home = user_dirs.home_dir(); - - let profile_providers = vec![ - ProfileProvider::with_configuration(home.join(".aws").join("credentials"), "default"), - //TODO: this is hacky, this is where our mac builders store their - // credentials. We should either match what boto does more directly - // or make those builders put their credentials in ~/.aws/credentials - ProfileProvider::with_configuration(home.join(".boto"), "Credentials"), - ]; - let provider = - AutoRefreshingProvider::new(ChainProvider::with_profile_providers(profile_providers)); - let ssl_mode = if use_ssl { Ssl::Yes } else { Ssl::No }; - let bucket_name = bucket.to_owned(); - let bucket = Rc::new(Bucket::new(bucket, endpoint, ssl_mode)?); let client = S3Client::new(Region::default()); Ok(S3Cache { - bucket, - provider, - key_prefix: key_prefix.to_owned(), + bucket_name: bucket.to_owned(), client, - bucket_name, + key_prefix: key_prefix.to_owned(), }) } @@ -159,7 +136,7 @@ impl Storage for S3Cache { } fn location(&self) -> String { - format!("S3, bucket: {}", self.bucket) + format!("S3, bucket: {}", self.bucket_name) } fn current_size(&self) -> SFuture> { diff --git a/src/lib.rs b/src/lib.rs index 48922cc81..6fdaaa7ca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -54,8 +54,6 @@ mod jobserver; mod mock_command; mod protocol; pub mod server; -#[cfg(feature = "simple-s3")] -mod simples3; #[doc(hidden)] pub mod util; diff --git a/src/simples3/credential.rs b/src/simples3/credential.rs deleted file mode 100644 index bc7c3adff..000000000 --- a/src/simples3/credential.rs +++ /dev/null @@ -1,530 +0,0 @@ -// Originally from https://github.com/rusoto/rusoto/blob/master/src/credential.rs -//! Types for loading and managing AWS access credentials for API requests. -#![allow(dead_code)] - -use chrono::{offset, DateTime, Duration}; -use directories::UserDirs; -use futures::future::{self, Shared}; -use futures::{Async, Future, Stream}; -use hyper::client::HttpConnector; -use hyper::{Client, Request}; -use hyperx::header::Connection; -use regex::Regex; -use serde_json::{from_str, Value}; -#[allow(unused_imports, deprecated)] -use std::ascii::AsciiExt; -use std::cell::RefCell; -use std::collections::HashMap; -use std::env::*; -use std::fs::{self, File}; -use std::io::prelude::*; -use std::io::BufReader; -use std::path::{Path, PathBuf}; -use std::time::Duration as StdDuration; -use tokio_timer::Timeout; - -use crate::errors::*; -use crate::util::RequestExt; - -/// AWS API access credentials, including access key, secret key, token (for IAM profiles), and -/// expiration timestamp. -#[derive(Clone, Debug)] -pub struct AwsCredentials { - key: String, - secret: String, - token: Option, - expires_at: DateTime, -} - -impl AwsCredentials { - /// Create a new `AwsCredentials` from a key ID, secret key, optional access token, and expiry - /// time. - pub fn new( - key: K, - secret: S, - token: Option, - expires_at: DateTime, - ) -> AwsCredentials - where - K: Into, - S: Into, - { - AwsCredentials { - key: key.into(), - secret: secret.into(), - token, - expires_at, - } - } - - /// Get a reference to the access key ID. - pub fn aws_access_key_id(&self) -> &str { - &self.key - } - - /// Get a reference to the secret access key. - pub fn aws_secret_access_key(&self) -> &str { - &self.secret - } - - /// Get a reference to the expiry time. - pub fn expires_at(&self) -> &DateTime { - &self.expires_at - } - - /// Get a reference to the access token. - pub fn token(&self) -> &Option { - &self.token - } - - /// Determine whether or not the credentials are expired. - fn credentials_are_expired(&self) -> bool { - // This is a rough hack to hopefully avoid someone requesting creds then sitting on them - // before issuing the request: - self.expires_at < offset::Utc::now() + Duration::seconds(20) - } -} - -/// A trait for types that produce `AwsCredentials`. -pub trait ProvideAwsCredentials { - /// Produce a new `AwsCredentials`. - fn credentials(&self) -> SFuture; -} - -/// Provides AWS credentials from environment variables. -pub struct EnvironmentProvider; - -impl ProvideAwsCredentials for EnvironmentProvider { - fn credentials(&self) -> SFuture { - Box::new(future::result(credentials_from_environment())) - } -} - -fn credentials_from_environment() -> Result { - let env_key = var("AWS_ACCESS_KEY_ID").context("No AWS_ACCESS_KEY_ID in environment")?; - let env_secret = - var("AWS_SECRET_ACCESS_KEY").context("No AWS_SECRET_ACCESS_KEY in environment")?; - - if env_key.is_empty() || env_secret.is_empty() { - bail!( - "Couldn't find either AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY or both in environment." - ) - } - - // Present when using temporary credentials, e.g. on Lambda with IAM roles - let token = match var("AWS_SESSION_TOKEN") { - Ok(val) => { - if val.is_empty() { - None - } else { - Some(val) - } - } - Err(_) => None, - }; - - Ok(AwsCredentials::new( - env_key, - env_secret, - token, - in_ten_minutes(), - )) -} - -/// Provides AWS credentials from a profile in a credentials file. -#[derive(Clone, Debug)] -pub struct ProfileProvider { - credentials: Option, - file_path: PathBuf, - profile: String, -} - -impl ProfileProvider { - /// Create a new `ProfileProvider` for the default credentials file path and profile name. - pub fn new() -> Result { - // Default credentials file location: - // ~/.aws/credentials (Linux/Mac) - // %USERPROFILE%\.aws\credentials (Windows) - let profile_location = UserDirs::new() - .map(|d| d.home_dir().join(".aws").join("credentials")) - .context("Couldn't get user directories")?; - - Ok(ProfileProvider { - credentials: None, - file_path: profile_location, - profile: "default".to_owned(), - }) - } - - /// Create a new `ProfileProvider` for the credentials file at the given path, using - /// the given profile. - pub fn with_configuration(file_path: F, profile: P) -> ProfileProvider - where - F: Into, - P: Into, - { - ProfileProvider { - credentials: None, - file_path: file_path.into(), - profile: profile.into(), - } - } - - /// Get a reference to the credentials file path. - pub fn file_path(&self) -> &Path { - self.file_path.as_ref() - } - - /// Get a reference to the profile name. - pub fn profile(&self) -> &str { - &self.profile - } - - /// Set the credentials file path. - pub fn set_file_path(&mut self, file_path: F) - where - F: Into, - { - self.file_path = file_path.into(); - } - - /// Set the profile name. - pub fn set_profile

(&mut self, profile: P) - where - P: Into, - { - self.profile = profile.into(); - } -} - -impl ProvideAwsCredentials for ProfileProvider { - fn credentials(&self) -> SFuture { - let result = parse_credentials_file(self.file_path()); - let result = result - .and_then(|mut profiles| profiles.remove(self.profile()).context("profile not found")); - Box::new(future::result(result)) - } -} - -fn parse_credentials_file(file_path: &Path) -> Result> { - let metadata = fs::metadata(file_path).context("couldn't stat credentials file")?; - if !metadata.is_file() { - bail!("Couldn't open file."); - } - - let file = File::open(file_path)?; - - let profile_regex = Regex::new(r"^\[([^\]]+)\]$").unwrap(); - let mut profiles: HashMap = HashMap::new(); - let mut access_key: Option = None; - let mut secret_key: Option = None; - let mut profile_name: Option = None; - - let file_lines = BufReader::new(&file); - for line in file_lines.lines() { - let unwrapped_line: String = line.unwrap(); - - // skip comments - if unwrapped_line.starts_with('#') { - continue; - } - - // handle the opening of named profile blocks - if profile_regex.is_match(&unwrapped_line) { - if let (Some(profile_name), Some(access_key), Some(secret_key)) = - (profile_name, access_key, secret_key) - { - let creds = AwsCredentials::new(access_key, secret_key, None, in_ten_minutes()); - profiles.insert(profile_name, creds); - } - - access_key = None; - secret_key = None; - - let caps = profile_regex.captures(&unwrapped_line).unwrap(); - profile_name = Some(caps.get(1).unwrap().as_str().to_string()); - continue; - } - - // otherwise look for key=value pairs we care about - let lower_case_line = unwrapped_line.to_ascii_lowercase().to_string(); - - if lower_case_line.contains("aws_access_key_id") && access_key.is_none() { - let v: Vec<&str> = unwrapped_line.split('=').collect(); - if !v.is_empty() { - access_key = Some(v[1].trim_matches(' ').to_string()); - } - } else if lower_case_line.contains("aws_secret_access_key") && secret_key.is_none() { - let v: Vec<&str> = unwrapped_line.split('=').collect(); - if !v.is_empty() { - secret_key = Some(v[1].trim_matches(' ').to_string()); - } - } - - // we could potentially explode here to indicate that the file is invalid - } - - if let (Some(profile_name), Some(access_key), Some(secret_key)) = - (profile_name, access_key, secret_key) - { - let creds = AwsCredentials::new(access_key, secret_key, None, in_ten_minutes()); - profiles.insert(profile_name, creds); - } - - if profiles.is_empty() { - bail!("No credentials found.") - } - - Ok(profiles) -} - -/// Provides AWS credentials from a resource's IAM role. -pub struct IamProvider { - client: Client, -} - -impl IamProvider { - pub fn new() -> IamProvider { - IamProvider { - client: Client::new(), - } - } - - fn iam_role(&self) -> SFuture { - // First get the IAM role - let address = "http://169.254.169.254/latest/meta-data/iam/security-credentials/"; - let req = Request::get(address) - .set_header(Connection::close()) - .body("".into()) - .unwrap(); - let response = self.client.request(req).and_then(|response| { - response.into_body().fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, hyper::Error>(body) - }) - }); - - Box::new( - response - .then(|res| { - let bytes = res.context("couldn't connect to metadata service")?; - String::from_utf8(bytes) - .context("Didn't get a parsable response body from metadata service") - }) - .map(move |body| { - let mut address = address.to_string(); - address.push_str(&body); - address - }), - ) - } -} - -impl ProvideAwsCredentials for IamProvider { - fn credentials(&self) -> SFuture { - let url = match var("AWS_IAM_CREDENTIALS_URL") { - Ok(url) => f_ok(url), - Err(_) => self.iam_role(), - }; - let url = url.and_then(|url| { - url.parse::() - .with_context(|| format!("failed to parse `{}` as url", url)) - }); - - let client = self.client.clone(); - let response = url.and_then(move |address| { - debug!("Attempting to fetch credentials from {}", address); - let req = Request::get(address) - .set_header(Connection::close()) - .body("".into()) - .unwrap(); - client.request(req).fcontext("failed to send http request") - }); - let body = response.and_then(|response| { - response - .into_body() - .fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, hyper::Error>(body) - }) - .fcontext("failed to read http body") - }); - let body = body - .map_err(|e| anyhow!("Failed to get IAM credentials: {}", e)) - .and_then(|body| String::from_utf8(body).context("failed to read iam role response")); - - let creds = body.and_then(|body| { - let json_object: Value; - match from_str(&body) { - Err(_) => bail!("Couldn't parse metadata response body."), - Ok(val) => json_object = val, - }; - - let access_key; - match json_object.get("AccessKeyId") { - None => bail!("Couldn't find AccessKeyId in response."), - Some(val) => { - access_key = val - .as_str() - .expect("AccessKeyId value was not a string") - .to_owned() - .replace("\"", "") - } - }; - - let secret_key; - match json_object.get("SecretAccessKey") { - None => bail!("Couldn't find SecretAccessKey in response."), - Some(val) => { - secret_key = val - .as_str() - .expect("SecretAccessKey value was not a string") - .to_owned() - .replace("\"", "") - } - }; - - let expiration; - match json_object.get("Expiration") { - None => bail!("Couldn't find Expiration in response."), - Some(val) => { - expiration = val - .as_str() - .expect("Expiration value was not a string") - .to_owned() - .replace("\"", "") - } - }; - - let expiration_time = expiration - .parse() - .context("failed to parse expiration time")?; - - let token_from_response; - match json_object.get("Token") { - None => bail!("Couldn't find Token in response."), - Some(val) => { - token_from_response = val - .as_str() - .expect("Token value was not a string") - .to_owned() - .replace("\"", "") - } - }; - - Ok(AwsCredentials::new( - access_key, - secret_key, - Some(token_from_response), - expiration_time, - )) - }); - - //XXX: this is crappy, but this blocks on non-EC2 machines like - // our mac builders. - let timeout = Timeout::new(creds, StdDuration::from_secs(2)); - - Box::new(timeout.then(|result| match result { - Ok(creds) => Ok(creds), - Err(err) => match err.into_inner() { - None => bail!("took too long to fetch credentials"), - Some(e) => { - warn!("Failed to fetch IAM credentials: {}", e); - Err(e) - } - }, - })) - } -} - -/// Wrapper for ProvideAwsCredentials that caches the credentials returned by the -/// wrapped provider. Each time the credentials are accessed, they are checked to see if -/// they have expired, in which case they are retrieved from the wrapped provider again. -pub struct AutoRefreshingProvider

{ - credentials_provider: P, - cached_credentials: RefCell>>, -} - -impl AutoRefreshingProvider

{ - pub fn new(provider: P) -> AutoRefreshingProvider

{ - AutoRefreshingProvider { - cached_credentials: RefCell::new(provider.credentials().shared()), - credentials_provider: provider, - } - } -} - -impl ProvideAwsCredentials for AutoRefreshingProvider

{ - fn credentials(&self) -> SFuture { - let mut future = self.cached_credentials.borrow_mut(); - if let Ok(Async::Ready(creds)) = future.poll() { - if creds.credentials_are_expired() { - *future = self.credentials_provider.credentials().shared(); - } - } - Box::new(future.clone().then(|result| match result { - Ok(e) => Ok((*e).clone()), - Err(e) => Err(anyhow!(e.to_string())), - })) - } -} - -/// Provides AWS credentials from multiple possible sources using a priority order. -/// -/// The following sources are checked in order for credentials when calling `credentials`: -/// -/// 1. Environment variables: `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` -/// 2. AWS credentials file. Usually located at `~/.aws/credentials`. -/// 3. IAM instance profile. Will only work if running on an EC2 instance with an instance profile/role. -/// -/// If the sources are exhausted without finding credentials, an error is returned. -#[derive(Clone)] -pub struct ChainProvider { - profile_providers: Vec, -} - -impl ProvideAwsCredentials for ChainProvider { - fn credentials(&self) -> SFuture { - let creds = EnvironmentProvider.credentials().map(|c| { - debug!("Using AWS credentials from environment"); - c - }); - let mut creds = Box::new(creds) as SFuture<_>; - for provider in self.profile_providers.iter() { - let alternate = provider.credentials(); - creds = Box::new(creds.or_else(|_| alternate)); - } - Box::new( - creds - .or_else(move |_| { - IamProvider::new().credentials().map(|c| { - debug!("Using AWS credentials from IAM"); - c - }) - }) - .map_err(|_| { - anyhow!( - "Couldn't find AWS credentials in environment, credentials file, or IAM role." - ) - }), - ) - } -} - -impl ChainProvider { - /// Create a new `ChainProvider` using a `ProfileProvider` with the default settings. - pub fn new() -> ChainProvider { - ChainProvider { - profile_providers: ProfileProvider::new().into_iter().collect(), - } - } - - /// Create a new `ChainProvider` using the provided `ProfileProvider`s. - pub fn with_profile_providers(profile_providers: Vec) -> ChainProvider { - ChainProvider { profile_providers } - } -} - -fn in_ten_minutes() -> DateTime { - offset::Utc::now() + Duration::seconds(600) -} diff --git a/src/simples3/mod.rs b/src/simples3/mod.rs deleted file mode 100644 index 2c0205393..000000000 --- a/src/simples3/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2016 Mozilla Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod credential; -mod s3; - -pub use crate::simples3::credential::*; -pub use crate::simples3::s3::*; diff --git a/src/simples3/s3.rs b/src/simples3/s3.rs deleted file mode 100644 index f7e193ae9..000000000 --- a/src/simples3/s3.rs +++ /dev/null @@ -1,251 +0,0 @@ -// Originally from https://github.com/rust-lang/crates.io/blob/master/src/s3/lib.rs -//#![deny(warnings)] - -#[allow(unused_imports, deprecated)] -use std::ascii::AsciiExt; -use std::fmt; - -use crate::simples3::credential::*; -use futures::{Future, Stream}; -use hmac::{Hmac, Mac}; -use hyper::header::HeaderValue; -use hyper::Method; -use hyperx::header; -use reqwest::r#async::{Client, Request}; -use sha1::Sha1; - -use crate::errors::*; -use crate::util::HeadersExt; - -#[derive(Debug, Copy, Clone)] -#[allow(dead_code)] -/// Whether or not to use SSL. -pub enum Ssl { - /// Use SSL. - Yes, - /// Do not use SSL. - No, -} - -fn base_url(endpoint: &str, ssl: Ssl) -> String { - format!( - "{}://{}/", - match ssl { - Ssl::Yes => "https", - Ssl::No => "http", - }, - endpoint - ) -} - -fn hmac(key: &[u8], data: &[u8]) -> Vec { - let mut hmac = Hmac::::new_varkey(key).expect("HMAC can take key of any size"); - hmac.input(data); - hmac.result().code().iter().copied().collect::>() -} - -fn signature(string_to_sign: &str, signing_key: &str) -> String { - let s = hmac(signing_key.as_bytes(), string_to_sign.as_bytes()); - base64::encode_config::>(&s, base64::STANDARD) -} - -/// An S3 bucket. -pub struct Bucket { - name: String, - base_url: String, - client: Client, -} - -impl fmt::Display for Bucket { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Bucket(name={}, base_url={})", self.name, self.base_url) - } -} - -impl Bucket { - pub fn new(name: &str, endpoint: &str, ssl: Ssl) -> Result { - let base_url = base_url(&endpoint, ssl); - Ok(Bucket { - name: name.to_owned(), - base_url, - client: Client::new(), - }) - } - - pub fn get(&self, key: &str, creds: Option<&AwsCredentials>) -> SFuture> { - let url = format!("{}{}", self.base_url, key); - debug!("GET {}", url); - let url2 = url.clone(); - let mut request = Request::new(Method::GET, url.parse().unwrap()); - if let Some(creds) = creds { - let mut canonical_headers = String::new(); - - if let Some(token) = creds.token().as_ref().map(|s| s.as_str()) { - request.headers_mut().insert( - "x-amz-security-token", - HeaderValue::from_str(token).expect("Invalid `x-amz-security-token` header"), - ); - canonical_headers - .push_str(format!("{}:{}\n", "x-amz-security-token", token).as_ref()); - } - let date = time::now_utc().rfc822().to_string(); - let auth = self.auth("GET", &date, key, "", &canonical_headers, "", creds); - request.headers_mut().insert( - "Date", - HeaderValue::from_str(&date).expect("Invalid date header"), - ); - request.headers_mut().insert( - "Authorization", - HeaderValue::from_str(&auth).expect("Invalid authentication"), - ); - } - - Box::new( - self.client - .execute(request) - .fwith_context(move || format!("failed GET: {}", url)) - .and_then(|res| { - if res.status().is_success() { - let content_length = res - .headers() - .get_hyperx::() - .map(|header::ContentLength(len)| len); - Ok((res.into_body(), content_length)) - } else { - Err(BadHttpStatusError(res.status()).into()) - } - }) - .and_then(|(body, content_length)| { - body.fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, reqwest::Error>(body) - }) - .fcontext("failed to read HTTP body") - .and_then(move |bytes| { - if let Some(len) = content_length { - if len != bytes.len() as u64 { - bail!(format!( - "Bad HTTP body size read: {}, expected {}", - bytes.len(), - len - )); - } else { - info!("Read {} bytes from {}", bytes.len(), url2); - } - } - Ok(bytes) - }) - }), - ) - } - - pub fn put(&self, key: &str, content: Vec, creds: &AwsCredentials) -> SFuture<()> { - let url = format!("{}{}", self.base_url, key); - debug!("PUT {}", url); - let mut request = Request::new(Method::PUT, url.parse().unwrap()); - - let content_type = "application/octet-stream"; - let date = time::now_utc().rfc822().to_string(); - let mut canonical_headers = String::new(); - let token = creds.token().as_ref().map(|s| s.as_str()); - // Keep the list of header values sorted! - for (header, maybe_value) in &[("x-amz-security-token", token)] { - if let Some(ref value) = maybe_value { - request.headers_mut().insert( - *header, - HeaderValue::from_str(value) - .unwrap_or_else(|_| panic!("Invalid `{}` header", header)), - ); - canonical_headers - .push_str(format!("{}:{}\n", header.to_ascii_lowercase(), value).as_ref()); - } - } - let auth = self.auth( - "PUT", - &date, - key, - "", - &canonical_headers, - content_type, - creds, - ); - request.headers_mut().insert( - "Date", - HeaderValue::from_str(&date).expect("Invalid date header"), - ); - request - .headers_mut() - .set(header::ContentType(content_type.parse().unwrap())); - request - .headers_mut() - .set(header::ContentLength(content.len() as u64)); - request.headers_mut().set(header::CacheControl(vec![ - // Two weeks - header::CacheDirective::MaxAge(1_296_000), - ])); - request.headers_mut().insert( - "Authorization", - HeaderValue::from_str(&auth).expect("Invalid authentication"), - ); - *request.body_mut() = Some(content.into()); - - Box::new(self.client.execute(request).then(|result| match result { - Ok(res) => { - if res.status().is_success() { - trace!("PUT succeeded"); - Ok(()) - } else { - trace!("PUT failed with HTTP status: {}", res.status()); - Err(BadHttpStatusError(res.status()).into()) - } - } - Err(e) => { - trace!("PUT failed with error: {:?}", e); - Err(e.into()) - } - })) - } - - // http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html - #[allow(clippy::too_many_arguments)] - fn auth( - &self, - verb: &str, - date: &str, - path: &str, - md5: &str, - headers: &str, - content_type: &str, - creds: &AwsCredentials, - ) -> String { - let string = format!( - "{verb}\n{md5}\n{ty}\n{date}\n{headers}{resource}", - verb = verb, - md5 = md5, - ty = content_type, - date = date, - headers = headers, - resource = format!("/{}/{}", self.name, path) - ); - let signature = signature(&string, creds.aws_secret_access_key()); - format!("AWS {}:{}", creds.aws_access_key_id(), signature) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_signature() { - assert_eq!( - signature("/foo/bar\nbar", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), - "mwbstmHPMEJjTe2ksXi5H5f0c8U=" - ); - - assert_eq!( - signature("/bar/foo\nbaz", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), - "F9gZMso3+P+QTEyRKQ6qhZ1YM6o=" - ); - } -} From 8bcafc3dea63a1c5b851c8eba16cb5c77797d7cf Mon Sep 17 00:00:00 2001 From: Hugo Laloge Date: Fri, 14 Aug 2020 21:52:52 +0200 Subject: [PATCH 03/12] Re-add the possibility to specify region and endpoint for S3 cache --- src/cache/cache.rs | 12 ++++++++++-- src/cache/s3.rs | 27 +++++++++++++++++++++------ src/config.rs | 29 ++++++++++------------------- 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 03cbe9376..049f73c99 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -383,9 +383,17 @@ pub fn storage_from_config(config: &Config, pool: &ThreadPool) -> Arc { - debug!("Trying S3Cache({}, {})", c.bucket, c.endpoint); + let region = c.region.as_deref(); + let endpoint = c.endpoint.as_deref(); + let key_prefix = c.key_prefix.as_deref(); + debug!( + "Trying S3Cache({}, {}, {})", + c.bucket, + region.unwrap_or("default region"), + endpoint.unwrap_or("default endpoint") + ); #[cfg(feature = "s3")] - match S3Cache::new(&c.bucket, &c.endpoint, c.use_ssl, &c.key_prefix) { + match S3Cache::new(&c.bucket, region, endpoint, key_prefix.unwrap_or("")) { Ok(s) => { trace!("Using S3Cache"); return Arc::new(s); diff --git a/src/cache/s3.rs b/src/cache/s3.rs index 6d532abdf..c5a8762fa 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -18,8 +18,9 @@ use futures::future::Future; use futures_03::{compat::Compat as _, future::TryFutureExt as _}; use hyperx::header::CacheDirective; use rusoto_core::{self, Region}; -use rusoto_s3::{GetObjectOutput, GetObjectRequest, PutObjectRequest, S3Client, S3 as _, S3}; +use rusoto_s3::{GetObjectOutput, GetObjectRequest, PutObjectRequest, S3Client, S3 as _}; use std::io; +use std::str::FromStr; use std::time::{Duration, Instant}; use tokio_02::io::AsyncReadExt as _; @@ -37,11 +38,25 @@ pub struct S3Cache { impl S3Cache { /// Create a new `S3Cache` storing data in `bucket`. - /// TODO: Handle custom region - /// TODO: Handle endpoint? - /// TODO: Handle use_ssl - pub fn new(bucket: &str, endpoint: &str, use_ssl: bool, key_prefix: &str) -> Result { - let client = S3Client::new(Region::default()); + pub fn new( + bucket: &str, + region: Option<&str>, + endpoint: Option<&str>, + key_prefix: &str, + ) -> Result { + let region = match endpoint { + Some(endpoint) => Region::Custom { + name: region + .map(ToOwned::to_owned) + .unwrap_or(Region::default().name().to_owned()), + endpoint: endpoint.to_owned(), + }, + None => region + .map(FromStr::from_str) + .unwrap_or_else(|| Ok(Region::default()))?, + }; + + let client = S3Client::new(region); Ok(S3Cache { bucket_name: bucket.to_owned(), client, diff --git a/src/config.rs b/src/config.rs index 3e89289e7..2a4f3ffb3 100644 --- a/src/config.rs +++ b/src/config.rs @@ -197,9 +197,12 @@ pub struct RedisCacheConfig { #[serde(deny_unknown_fields)] pub struct S3CacheConfig { pub bucket: String, - pub endpoint: String, - pub use_ssl: bool, - pub key_prefix: String, + #[serde(default)] + pub endpoint: Option, + #[serde(default)] + pub key_prefix: Option, + #[serde(default)] + pub region: Option, } #[derive(Debug, PartialEq, Eq)] @@ -447,32 +450,20 @@ pub struct EnvConfig { fn config_from_env() -> EnvConfig { let s3 = env::var("SCCACHE_BUCKET").ok().map(|bucket| { - let endpoint = match env::var("SCCACHE_ENDPOINT") { - Ok(endpoint) => format!("{}/{}", endpoint, bucket), - _ => match env::var("SCCACHE_REGION") { - Ok(ref region) if region != "us-east-1" => { - format!("{}.s3-{}.amazonaws.com", bucket, region) - } - _ => format!("{}.s3.amazonaws.com", bucket), - }, - }; - let use_ssl = env::var("SCCACHE_S3_USE_SSL") - .ok() - .filter(|value| value != "off") - .is_some(); + let endpoint = env::var("SCCACHE_ENDPOINT").ok(); + let region = env::var("SCCACHE_REGION").ok(); let key_prefix = env::var("SCCACHE_S3_KEY_PREFIX") .ok() .as_ref() .map(|s| s.trim_end_matches('/')) .filter(|s| !s.is_empty()) - .map(|s| s.to_owned() + "/") - .unwrap_or_default(); + .map(|s| s.to_owned() + "/"); S3CacheConfig { bucket, endpoint, - use_ssl, key_prefix, + region, } }); From 197f916e6b8efa996b25b51a6765a441e881aff3 Mon Sep 17 00:00:00 2001 From: Hugo Laloge Date: Wed, 2 Sep 2020 19:47:43 +0200 Subject: [PATCH 04/12] Update README to document S3 configuration --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 17b3a605c..208988652 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,9 @@ If you want to use S3 storage for the sccache cache, you need to set the `SCCACH You can use `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to set the S3 credentials. Alternately, you can set `AWS_IAM_CREDENTIALS_URL` to a URL that returns credentials in the format supported by the [EC2 metadata service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials), and credentials will be fetched from that location as needed. In the absence of either of these options, credentials for the instance's IAM role will be fetched from the EC2 metadata service directly. -If you need to override the default endpoint you can set `SCCACHE_ENDPOINT`. To connect to a minio storage for example you can set `SCCACHE_ENDPOINT=:`. If your endpoint requires TLS, set `SCCACHE_S3_USE_SSL=true`. +You can set the region of your bucket with one of the environment variables `AWS_DEFAULT_REGION`, `AWS_REGION` or `SCCACHE_REGION`. +If you need to override the default endpoint you can set `SCCACHE_ENDPOINT`. To connect to a minio storage for example you can set `SCCACHE_ENDPOINT=:`. +Optionally, the endpoint can start with `http://` or `https://` to force the protocol. By default, HTTPS will be used. You can also define a prefix that will be prepended to the keys of all cache objects created and read within the S3 bucket, effectively creating a scope. To do that use the `SCCACHE_S3_KEY_PREFIX` environment variable. This can be useful when sharing a bucket with another application. From 5d672708073b2ff972d3af8146c10f958dbd022f Mon Sep 17 00:00:00 2001 From: Rex Hoffman Date: Sun, 18 Oct 2020 00:14:36 -0700 Subject: [PATCH 05/12] [s3] support anonymous reads from public buckets --- README.md | 3 +-- src/cache/cache.rs | 13 ++++++++++--- src/cache/s3.rs | 14 ++++++++++++-- src/config.rs | 3 +++ 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 208988652..94714226f 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The default cache size is 10 gigabytes. To change this, set `SCCACHE_CACHE_SIZE` ### S3 If you want to use S3 storage for the sccache cache, you need to set the `SCCACHE_BUCKET` environment variable to the name of the S3 bucket to use. -You can use `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to set the S3 credentials. Alternately, you can set `AWS_IAM_CREDENTIALS_URL` to a URL that returns credentials in the format supported by the [EC2 metadata service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials), and credentials will be fetched from that location as needed. In the absence of either of these options, credentials for the instance's IAM role will be fetched from the EC2 metadata service directly. +You can use `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to set the S3 credentials. Other supported methods are listed in Rusoto's [ChainProvider](https://rusoto.github.io/rusoto/rusoto_credential/struct.ChainProvider.html). To connect to a public bucket anonymously (read only mode), the environment variable `SCCACHE_S3_PUBLIC` must be set to true, to prevent the default behavior of rusoto's [DefaultCredentialsProvider](https://rusoto.github.io/rusoto/rusoto_credential/struct.DefaultCredentialsProvider.html), which is to pass the error of ChainProvider. You can set the region of your bucket with one of the environment variables `AWS_DEFAULT_REGION`, `AWS_REGION` or `SCCACHE_REGION`. If you need to override the default endpoint you can set `SCCACHE_ENDPOINT`. To connect to a minio storage for example you can set `SCCACHE_ENDPOINT=:`. @@ -200,7 +200,6 @@ Optionally, the endpoint can start with `http://` or `https://` to force the pro You can also define a prefix that will be prepended to the keys of all cache objects created and read within the S3 bucket, effectively creating a scope. To do that use the `SCCACHE_S3_KEY_PREFIX` environment variable. This can be useful when sharing a bucket with another application. - ### Redis Set `SCCACHE_REDIS` to a [Redis](https://redis.io/) url in format `redis://[:@][:port][/]` to store the cache in a Redis instance. Redis can be configured as a LRU (least recently used) cache with a fixed maximum cache size. Set `maxmemory` and `maxmemory-policy` according to the [Redis documentation](https://redis.io/topics/lru-cache). The `allkeys-lru` policy which discards the *least recently accessed or modified* key fits well for the sccache use case. diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 049f73c99..d5cbc0791 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -387,13 +387,20 @@ pub fn storage_from_config(config: &Config, pool: &ThreadPool) -> Arc { trace!("Using S3Cache"); return Arc::new(s); diff --git a/src/cache/s3.rs b/src/cache/s3.rs index c5a8762fa..718f60476 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -17,7 +17,7 @@ use futures::future; use futures::future::Future; use futures_03::{compat::Compat as _, future::TryFutureExt as _}; use hyperx::header::CacheDirective; -use rusoto_core::{self, Region}; +use rusoto_core::{self, Client, HttpClient, Region}; use rusoto_s3::{GetObjectOutput, GetObjectRequest, PutObjectRequest, S3Client, S3 as _}; use std::io; use std::str::FromStr; @@ -43,6 +43,7 @@ impl S3Cache { region: Option<&str>, endpoint: Option<&str>, key_prefix: &str, + public: bool, ) -> Result { let region = match endpoint { Some(endpoint) => Region::Custom { @@ -56,7 +57,16 @@ impl S3Cache { .unwrap_or_else(|| Ok(Region::default()))?, }; - let client = S3Client::new(region); + //we could define our own Annonymous StaticProvider here, but we'd have to verify that the DefaultCredentialsProvider would fail first. + let client = if public { + let client = Client::new_not_signing( + HttpClient::new().expect("failed to create request dispatcher"), + ); + S3Client::new_with_client(client, region) + } else { + S3Client::new(region) + }; + Ok(S3Cache { bucket_name: bucket.to_owned(), client, diff --git a/src/config.rs b/src/config.rs index 2a4f3ffb3..d8f2c5ca2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -203,6 +203,7 @@ pub struct S3CacheConfig { pub key_prefix: Option, #[serde(default)] pub region: Option, + pub public: bool, } #[derive(Debug, PartialEq, Eq)] @@ -458,12 +459,14 @@ fn config_from_env() -> EnvConfig { .map(|s| s.trim_end_matches('/')) .filter(|s| !s.is_empty()) .map(|s| s.to_owned() + "/"); + let public = env::var("SCCACHE_S3_PUBLIC").ok().is_some(); S3CacheConfig { bucket, endpoint, key_prefix, region, + public, } }); From 549babdd3866aa60dae01668c42ee00bf1e8c763 Mon Sep 17 00:00:00 2001 From: Rex Hoffman Date: Tue, 8 Dec 2020 11:31:35 -0800 Subject: [PATCH 06/12] remove unneed import --- src/cache/s3.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cache/s3.rs b/src/cache/s3.rs index 718f60476..ee0279f5a 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -15,7 +15,7 @@ use crate::cache::{Cache, CacheRead, CacheWrite, Storage}; use futures::future; use futures::future::Future; -use futures_03::{compat::Compat as _, future::TryFutureExt as _}; +use futures_03::{future::TryFutureExt as _}; use hyperx::header::CacheDirective; use rusoto_core::{self, Client, HttpClient, Region}; use rusoto_s3::{GetObjectOutput, GetObjectRequest, PutObjectRequest, S3Client, S3 as _}; From 210827fa6c654d30ff6c91f861fedeaccfddaa21 Mon Sep 17 00:00:00 2001 From: Tom Elliff Date: Fri, 9 Apr 2021 13:19:05 +0100 Subject: [PATCH 07/12] WIP --- Cargo.lock | 31 ++++++++++++++++++++++++++++++- Cargo.toml | 3 ++- src/cache/s3.rs | 10 +++++++++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27b5948f2..9bffa3ccb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2297,7 +2297,7 @@ dependencies = [ "native-tls", "serde", "serde_json", - "serde_urlencoded", + "serde_urlencoded 0.5.5", "time 0.1.44", "tokio 0.1.22", "tokio-executor", @@ -2441,6 +2441,22 @@ dependencies = [ "tokio 0.2.23", ] +[[package]] +name = "rusoto_sts" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3815b8c0fc1c50caf9e87603f23daadfedb18d854de287b361c69f68dc9d49e0" +dependencies = [ + "async-trait", + "bytes 0.5.6", + "chrono", + "futures 0.3.8", + "rusoto_core", + "serde_urlencoded 0.6.1", + "tempfile", + "xml-rs", +] + [[package]] name = "rust-argon2" version = "0.8.3" @@ -2548,6 +2564,7 @@ dependencies = [ "rouille", "rusoto_core", "rusoto_s3", + "rusoto_sts", "selenium-rs", "serde", "serde_derive", @@ -2693,6 +2710,18 @@ dependencies = [ "url 1.7.2", ] +[[package]] +name = "serde_urlencoded" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +dependencies = [ + "dtoa", + "itoa", + "serde", + "url 2.2.0", +] + [[package]] name = "sha-1" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index ad1e1c19f..dfd73a3a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ retry = "0.4.0" ring = { version = "0.14.6", optional = true } rusoto_core = { version = "0.45.0", features = ["native-tls"], optional = true } rusoto_s3 = { version = "0.45.0", features = ["native-tls"], optional = true } +rusoto_sts = { version = "0.45.0", features = ["native-tls"], optional = true } sha-1 = { version = "0.8", optional = true } sha2 = { version = "0.8", optional = true } serde = "1.0" @@ -130,7 +131,7 @@ features = [ default = ["dist-client", "s3"] all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure"] azure = ["chrono", "hyper", "hyperx", "url", "hmac", "md-5", "sha2"] -s3 = ["hyperx", "rusoto_core", "rusoto_s3", "tokio_02"] +s3 = ["hyperx", "rusoto_core", "rusoto_s3", "rusoto_sts", "tokio_02"] gcs = ["chrono", "hyper", "hyperx", "reqwest", "ring", "untrusted", "url"] memcached = ["memcached-rs"] # Enable features that require unstable features of Nightly Rust. diff --git a/src/cache/s3.rs b/src/cache/s3.rs index ee0279f5a..a60edd8cb 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -25,6 +25,7 @@ use std::time::{Duration, Instant}; use tokio_02::io::AsyncReadExt as _; use crate::errors::*; +use rusoto_core::credential::AutoRefreshingProvider; /// A cache that stores entries in Amazon S3. pub struct S3Cache { @@ -64,7 +65,14 @@ impl S3Cache { ); S3Client::new_with_client(client, region) } else { - S3Client::new(region) + let eks_credential_provider = + AutoRefreshingProvider::new(rusoto_sts::WebIdentityProvider::from_k8s_env()) + .expect("failed to create eks web identity provider"); + S3Client::new_with( + HttpClient::new().expect("failed to create request dispatcher"), + eks_credential_provider, + region, + ) }; Ok(S3Cache { From 4da4bee3f5119e21a8776889822eaaf0930d9f04 Mon Sep 17 00:00:00 2001 From: Tom Elliff Date: Fri, 9 Apr 2021 16:02:20 +0100 Subject: [PATCH 08/12] Cargo fmt --- src/cache/s3.rs | 2 +- tests/harness/mod.rs | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/cache/s3.rs b/src/cache/s3.rs index a60edd8cb..8463e9b33 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -15,7 +15,7 @@ use crate::cache::{Cache, CacheRead, CacheWrite, Storage}; use futures::future; use futures::future::Future; -use futures_03::{future::TryFutureExt as _}; +use futures_03::future::TryFutureExt as _; use hyperx::header::CacheDirective; use rusoto_core::{self, Client, HttpClient, Region}; use rusoto_s3::{GetObjectOutput, GetObjectRequest, PutObjectRequest, S3Client, S3 as _}; diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index 0943f74d6..c3a1244a0 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -294,8 +294,14 @@ impl DistSystem { wait_for( || { let status = self.scheduler_status(); - if matches!(self.scheduler_status(), SchedulerStatusResult { num_servers: 0, num_cpus: _, in_progress: 0 }) - { + if matches!( + self.scheduler_status(), + SchedulerStatusResult { + num_servers: 0, + num_cpus: _, + in_progress: 0 + } + ) { Ok(()) } else { Err(format!("{:?}", status)) @@ -428,8 +434,14 @@ impl DistSystem { wait_for( || { let status = self.scheduler_status(); - if matches!(self.scheduler_status(), SchedulerStatusResult { num_servers: 1, num_cpus: _, in_progress: 0 }) - { + if matches!( + self.scheduler_status(), + SchedulerStatusResult { + num_servers: 1, + num_cpus: _, + in_progress: 0 + } + ) { Ok(()) } else { Err(format!("{:?}", status)) From f491d52635de9894b7d7196deb0d06f51e9a0140 Mon Sep 17 00:00:00 2001 From: Tom Elliff Date: Fri, 9 Apr 2021 16:03:07 +0100 Subject: [PATCH 09/12] Gitignore .idea --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e294c03cf..febc535a6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ target .cargo *.pyc +.idea + # snapcraft artifacts /parts /prime From d2a94d9dd3960a008c176823322426af1b9b42a0 Mon Sep 17 00:00:00 2001 From: Tom Elliff Date: Fri, 9 Apr 2021 16:03:43 +0100 Subject: [PATCH 10/12] Add EKS IRSA to default credential chain --- Cargo.lock | 2 ++ Cargo.toml | 4 +++- src/cache/s3.rs | 36 +++++++++++++++++++++++++++++++++--- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9bffa3ccb..4bc59db70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2518,6 +2518,7 @@ dependencies = [ "anyhow", "ar", "assert_cmd", + "async-trait", "atty", "base64 0.11.0", "bincode 1.3.1", @@ -2563,6 +2564,7 @@ dependencies = [ "ring", "rouille", "rusoto_core", + "rusoto_credential", "rusoto_s3", "rusoto_sts", "selenium-rs", diff --git a/Cargo.toml b/Cargo.toml index dfd73a3a1..f478c0571 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ required-features = ["dist-server"] [dependencies] anyhow = "1.0" ar = { version = "0.8", optional = true } +async-trait = "0.1" atty = "0.2.6" base64 = "0.11.0" bincode = "1" @@ -61,6 +62,7 @@ reqwest = { version = "0.9.11", optional = true } retry = "0.4.0" ring = { version = "0.14.6", optional = true } rusoto_core = { version = "0.45.0", features = ["native-tls"], optional = true } +rusoto_credential = { version = "0.45.0", optional = true } rusoto_s3 = { version = "0.45.0", features = ["native-tls"], optional = true } rusoto_sts = { version = "0.45.0", features = ["native-tls"], optional = true } sha-1 = { version = "0.8", optional = true } @@ -131,7 +133,7 @@ features = [ default = ["dist-client", "s3"] all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure"] azure = ["chrono", "hyper", "hyperx", "url", "hmac", "md-5", "sha2"] -s3 = ["hyperx", "rusoto_core", "rusoto_s3", "rusoto_sts", "tokio_02"] +s3 = ["hyperx", "rusoto_core", "rusoto_s3", "rusoto_sts", "rusoto_credential", "tokio_02"] gcs = ["chrono", "hyper", "hyperx", "reqwest", "ring", "untrusted", "url"] memcached = ["memcached-rs"] # Enable features that require unstable features of Nightly Rust. diff --git a/src/cache/s3.rs b/src/cache/s3.rs index 8463e9b33..e4d43a929 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -13,11 +13,13 @@ // limitations under the License. use crate::cache::{Cache, CacheRead, CacheWrite, Storage}; +use async_trait::async_trait; use futures::future; use futures::future::Future; use futures_03::future::TryFutureExt as _; use hyperx::header::CacheDirective; use rusoto_core::{self, Client, HttpClient, Region}; +use rusoto_credential::{AwsCredentials, CredentialsError, ProvideAwsCredentials}; use rusoto_s3::{GetObjectOutput, GetObjectRequest, PutObjectRequest, S3Client, S3 as _}; use std::io; use std::str::FromStr; @@ -65,14 +67,15 @@ impl S3Cache { ); S3Client::new_with_client(client, region) } else { - let eks_credential_provider = - AutoRefreshingProvider::new(rusoto_sts::WebIdentityProvider::from_k8s_env()) + let default_plus_eks_credential_provider = + AutoRefreshingProvider::new(FancyCredentialsProvider::new()) .expect("failed to create eks web identity provider"); S3Client::new_with( HttpClient::new().expect("failed to create request dispatcher"), - eks_credential_provider, + default_plus_eks_credential_provider, region, ) + // S3Client::new(region) }; Ok(S3Cache { @@ -179,3 +182,30 @@ impl Storage for S3Cache { Box::new(future::ok(None)) } } + +#[derive(Debug, Clone)] +struct FancyCredentialsProvider {} + +impl FancyCredentialsProvider { + fn new() -> Self { + FancyCredentialsProvider {} + } +} + +#[async_trait] +impl ProvideAwsCredentials for FancyCredentialsProvider { + async fn credentials(&self) -> std::result::Result { + rusoto_credential::ChainProvider::new() + .credentials() + .or_else(|err| async move { + debug!( + "Failed to get S3 credentials from the default chain provider {:?}", + err + ); + rusoto_sts::WebIdentityProvider::from_k8s_env() + .credentials() + .await + }) + .await + } +} From cbe83bb614629fcb0e5e68a91a432c0f93e47077 Mon Sep 17 00:00:00 2001 From: Tom Elliff Date: Fri, 9 Apr 2021 16:34:58 +0100 Subject: [PATCH 11/12] Use EKS IRSA credentials before IMDS credentials --- Cargo.lock | 1 - Cargo.toml | 3 +-- src/cache/s3.rs | 57 ++++++++++++++++++++++++++++++------------------- 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4bc59db70..258579f35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2564,7 +2564,6 @@ dependencies = [ "ring", "rouille", "rusoto_core", - "rusoto_credential", "rusoto_s3", "rusoto_sts", "selenium-rs", diff --git a/Cargo.toml b/Cargo.toml index f478c0571..f6302b04a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,7 +62,6 @@ reqwest = { version = "0.9.11", optional = true } retry = "0.4.0" ring = { version = "0.14.6", optional = true } rusoto_core = { version = "0.45.0", features = ["native-tls"], optional = true } -rusoto_credential = { version = "0.45.0", optional = true } rusoto_s3 = { version = "0.45.0", features = ["native-tls"], optional = true } rusoto_sts = { version = "0.45.0", features = ["native-tls"], optional = true } sha-1 = { version = "0.8", optional = true } @@ -133,7 +132,7 @@ features = [ default = ["dist-client", "s3"] all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure"] azure = ["chrono", "hyper", "hyperx", "url", "hmac", "md-5", "sha2"] -s3 = ["hyperx", "rusoto_core", "rusoto_s3", "rusoto_sts", "rusoto_credential", "tokio_02"] +s3 = ["hyperx", "rusoto_core", "rusoto_s3", "rusoto_sts", "tokio_02"] gcs = ["chrono", "hyper", "hyperx", "reqwest", "ring", "untrusted", "url"] memcached = ["memcached-rs"] # Enable features that require unstable features of Nightly Rust. diff --git a/src/cache/s3.rs b/src/cache/s3.rs index e4d43a929..d9d08eaa6 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -19,7 +19,6 @@ use futures::future::Future; use futures_03::future::TryFutureExt as _; use hyperx::header::CacheDirective; use rusoto_core::{self, Client, HttpClient, Region}; -use rusoto_credential::{AwsCredentials, CredentialsError, ProvideAwsCredentials}; use rusoto_s3::{GetObjectOutput, GetObjectRequest, PutObjectRequest, S3Client, S3 as _}; use std::io; use std::str::FromStr; @@ -27,7 +26,10 @@ use std::time::{Duration, Instant}; use tokio_02::io::AsyncReadExt as _; use crate::errors::*; -use rusoto_core::credential::AutoRefreshingProvider; +use rusoto_core::credential::{ + AutoRefreshingProvider, AwsCredentials, CredentialsError, EnvironmentProvider, + ProvideAwsCredentials, +}; /// A cache that stores entries in Amazon S3. pub struct S3Cache { @@ -68,7 +70,7 @@ impl S3Cache { S3Client::new_with_client(client, region) } else { let default_plus_eks_credential_provider = - AutoRefreshingProvider::new(FancyCredentialsProvider::new()) + AutoRefreshingProvider::new(FancyCredentialsProvider::default()) .expect("failed to create eks web identity provider"); S3Client::new_with( HttpClient::new().expect("failed to create request dispatcher"), @@ -183,29 +185,40 @@ impl Storage for S3Cache { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] struct FancyCredentialsProvider {} -impl FancyCredentialsProvider { - fn new() -> Self { - FancyCredentialsProvider {} - } -} - #[async_trait] impl ProvideAwsCredentials for FancyCredentialsProvider { async fn credentials(&self) -> std::result::Result { - rusoto_credential::ChainProvider::new() - .credentials() - .or_else(|err| async move { - debug!( - "Failed to get S3 credentials from the default chain provider {:?}", - err - ); - rusoto_sts::WebIdentityProvider::from_k8s_env() - .credentials() - .await - }) - .await + chain_provider_credentials().await + } +} + +async fn chain_provider_credentials() -> std::result::Result { + use rusoto_core::credential::{ContainerProvider, InstanceMetadataProvider, ProfileProvider}; + + if let Ok(creds) = EnvironmentProvider::default().credentials().await { + return Ok(creds); + } + if let Ok(creds) = rusoto_sts::WebIdentityProvider::from_k8s_env() + .credentials() + .await + { + return Ok(creds); + } + if let Ok(ref profile_provider) = ProfileProvider::new() { + if let Ok(creds) = profile_provider.credentials().await { + return Ok(creds); + } + } + if let Ok(creds) = ContainerProvider::new().credentials().await { + return Ok(creds); + } + if let Ok(creds) = InstanceMetadataProvider::new().credentials().await { + return Ok(creds); } + Err(CredentialsError::new( + "Couldn't find AWS credentials in environment, credentials file, or IAM role.", + )) } From e74532a487d911b9aa06a1c4aab80e4109632aa2 Mon Sep 17 00:00:00 2001 From: Tom Elliff Date: Fri, 9 Apr 2021 16:58:54 +0100 Subject: [PATCH 12/12] Tidy --- src/cache/s3.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/cache/s3.rs b/src/cache/s3.rs index d9d08eaa6..d305247b2 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -69,15 +69,14 @@ impl S3Cache { ); S3Client::new_with_client(client, region) } else { - let default_plus_eks_credential_provider = - AutoRefreshingProvider::new(FancyCredentialsProvider::default()) - .expect("failed to create eks web identity provider"); + let credential_provider = + AutoRefreshingProvider::new(ChainCredentialProvider::default()) + .expect("failed to create a credential provider"); S3Client::new_with( HttpClient::new().expect("failed to create request dispatcher"), - default_plus_eks_credential_provider, + credential_provider, region, ) - // S3Client::new(region) }; Ok(S3Cache { @@ -186,15 +185,17 @@ impl Storage for S3Cache { } #[derive(Debug, Clone, Default)] -struct FancyCredentialsProvider {} +struct ChainCredentialProvider {} #[async_trait] -impl ProvideAwsCredentials for FancyCredentialsProvider { +impl ProvideAwsCredentials for ChainCredentialProvider { async fn credentials(&self) -> std::result::Result { chain_provider_credentials().await } } +/// This is similar to the default chain provider in rusoto_credential lib.rs but adds the +/// WebIdentityProvider after the EnvironmentProvider to inject credentials from the EKS IRSA method async fn chain_provider_credentials() -> std::result::Result { use rusoto_core::credential::{ContainerProvider, InstanceMetadataProvider, ProfileProvider};