diff --git a/.gitignore b/.gitignore index e294c03cf..febc535a6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ target .cargo *.pyc +.idea + # snapcraft artifacts /parts /prime diff --git a/Cargo.lock b/Cargo.lock index 49b118575..258579f35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,6 +82,17 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-trait" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a265e3abeffdce30b2e26b7a11b222fe37c6067404001b434101457d0385eb92" +dependencies = [ + "proc-macro2", + "quote 1.0.7", + "syn 1.0.54", +] + [[package]] name = "atty" version = "0.2.14" @@ -119,6 +130,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base-x" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" + [[package]] name = "base64" version = "0.9.3" @@ -144,6 +161,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + [[package]] name = "base64" version = "0.13.0" @@ -215,6 +238,15 @@ dependencies = [ "generic-array 0.12.3", ] +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array 0.14.4", +] + [[package]] name = "block-padding" version = "0.1.5" @@ -246,6 +278,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + [[package]] name = "byte-tools" version = "0.3.1" @@ -311,7 +349,8 @@ dependencies = [ "libc", "num-integer", "num-traits 0.2.14", - "time", + "serde", + "time 0.1.44", "winapi 0.3.9", ] @@ -365,7 +404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d6364d028778d0d98b6014fa5882da377cd10d3492b7734d266a428e9b1fca" dependencies = [ "log 0.4.11", - "md5", + "md5 0.3.8", ] [[package]] @@ -380,7 +419,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5" dependencies = [ - "time", + "time 0.1.44", "url 1.7.2", ] @@ -397,7 +436,7 @@ dependencies = [ "publicsuffix", "serde", "serde_json", - "time", + "time 0.1.44", "try_from", "url 1.7.2", ] @@ -424,6 +463,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384f8c53175c890920b6e0127b730709d2a173ca6c4dfdc81618ac9b46f648fe" +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + [[package]] name = "crc32fast" version = "1.2.1" @@ -603,6 +648,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "dirs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" +dependencies = [ + "cfg-if 0.1.10", + "dirs-sys", +] + [[package]] name = "dirs-sys" version = "0.3.5" @@ -614,6 +669,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + [[package]] name = "doc-comment" version = "0.3.3" @@ -883,7 +944,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr 2.3.4", - "pin-project", + "pin-project 1.0.2", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -951,7 +1012,7 @@ dependencies = [ "bytes 0.4.12", "fnv", "futures 0.1.30", - "http", + "http 0.1.21", "indexmap", "log 0.4.11", "slab", @@ -959,6 +1020,26 @@ dependencies = [ "tokio-io", ] +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.1", + "indexmap", + "slab", + "tokio 0.2.23", + "tokio-util 0.3.1", + "tracing", + "tracing-futures", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -974,6 +1055,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + [[package]] name = "hmac" version = "0.7.1" @@ -984,6 +1071,16 @@ dependencies = [ "digest 0.8.1", ] +[[package]] +name = "hmac" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" +dependencies = [ + "crypto-mac 0.8.0", + "digest 0.9.0", +] + [[package]] name = "http" version = "0.1.21" @@ -995,6 +1092,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +dependencies = [ + "bytes 0.5.6", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.1.0" @@ -1003,10 +1111,20 @@ checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" dependencies = [ "bytes 0.4.12", "futures 0.1.30", - "http", + "http 0.1.21", "tokio-buf", ] +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes 0.5.6", + "http 0.2.1", +] + [[package]] name = "httparse" version = "1.3.4" @@ -1031,16 +1149,16 @@ dependencies = [ "bytes 0.4.12", "futures 0.1.30", "futures-cpupool", - "h2", - "http", - "http-body", + "h2 0.1.26", + "http 0.1.21", + "http-body 0.1.0", "httparse", "iovec", "itoa", "log 0.4.11", "net2", "rustc_version", - "time", + "time 0.1.44", "tokio 0.1.22", "tokio-buf", "tokio-executor", @@ -1049,7 +1167,31 @@ dependencies = [ "tokio-tcp", "tokio-threadpool", "tokio-timer", - "want", + "want 0.2.0", +] + +[[package]] +name = "hyper" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e68a8dd9716185d9e64ea473ea6ef63529252e3e27623295a0378a19665d5eb" +dependencies = [ + "bytes 0.5.6", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.2.7", + "http 0.2.1", + "http-body 0.3.1", + "httparse", + "itoa", + "pin-project 0.4.27", + "socket2", + "time 0.1.44", + "tokio 0.2.23", + "tower-service 0.3.0", + "tracing", + "want 0.3.0", ] [[package]] @@ -1060,11 +1202,24 @@ checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" dependencies = [ "bytes 0.4.12", "futures 0.1.30", - "hyper", + "hyper 0.12.35", "native-tls", "tokio-io", ] +[[package]] +name = "hyper-tls" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +dependencies = [ + "bytes 0.5.6", + "hyper 0.13.7", + "native-tls", + "tokio 0.2.23", + "tokio-tls", +] + [[package]] name = "hyperx" version = "0.12.0" @@ -1073,13 +1228,13 @@ checksum = "78e2d2253d7a17929560fc3adf48c48fc924c94fa4507e037a60e6bc55c0eda6" dependencies = [ "base64 0.9.3", "bytes 0.4.12", - "http", + "http 0.1.21", "httparse", "language-tags", "log 0.4.11", "mime 0.3.16", "percent-encoding 1.0.1", - "time", + "time 0.1.44", "unicase 2.6.0", ] @@ -1275,9 +1430,9 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a18af3dcaf2b0219366cdb4e2af65a6101457b415c3d1a5c71dd9c2b7c77b9c8" dependencies = [ - "block-buffer", + "block-buffer 0.7.3", "digest 0.8.1", - "opaque-debug", + "opaque-debug 0.2.3", ] [[package]] @@ -1286,6 +1441,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79c56d6a0b07f9e19282511c83fc5b086364cbae4ba8c7d5f190c3d9b0425a48" +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memcached-rs" version = "0.4.2" @@ -1579,6 +1740,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "openssl" version = "0.10.30" @@ -1689,13 +1856,33 @@ dependencies = [ "unicase 1.4.2", ] +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + [[package]] name = "pin-project" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" dependencies = [ - "pin-project-internal", + "pin-project-internal 1.0.2", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2", + "quote 1.0.7", + "syn 1.0.54", ] [[package]] @@ -2040,7 +2227,7 @@ dependencies = [ "pin-project-lite 0.1.11", "sha1", "tokio 0.2.23", - "tokio-util", + "tokio-util 0.2.0", "url 2.2.0", ] @@ -2101,17 +2288,17 @@ dependencies = [ "encoding_rs", "flate2", "futures 0.1.30", - "http", - "hyper", - "hyper-tls", + "http 0.1.21", + "hyper 0.12.35", + "hyper-tls 0.3.2", "log 0.4.11", "mime 0.3.16", "mime_guess 2.0.3", "native-tls", "serde", "serde_json", - "serde_urlencoded", - "time", + "serde_urlencoded 0.5.5", + "time 0.1.44", "tokio 0.1.22", "tokio-executor", "tokio-io", @@ -2163,11 +2350,113 @@ dependencies = [ "sha1", "term", "threadpool", - "time", + "time 0.1.44", "tiny_http", "url 1.7.2", ] +[[package]] +name = "rusoto_core" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e977941ee0658df96fca7291ecc6fc9a754600b21ad84b959eb1dbbc9d5abcc7" +dependencies = [ + "async-trait", + "base64 0.12.3", + "bytes 0.5.6", + "crc32fast", + "futures 0.3.8", + "http 0.2.1", + "hyper 0.13.7", + "hyper-tls 0.4.3", + "lazy_static", + "log 0.4.11", + "md5 0.7.0", + "percent-encoding 2.1.0", + "pin-project 0.4.27", + "rusoto_credential", + "rusoto_signature", + "rustc_version", + "serde", + "serde_json", + "tokio 0.2.23", + "xml-rs", +] + +[[package]] +name = "rusoto_credential" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac05563f83489b19b4d413607a30821ab08bbd9007d14fa05618da3ef09d8b" +dependencies = [ + "async-trait", + "chrono", + "dirs 2.0.2", + "futures 0.3.8", + "hyper 0.13.7", + "pin-project 0.4.27", + "regex", + "serde", + "serde_json", + "shlex", + "tokio 0.2.23", + "zeroize", +] + +[[package]] +name = "rusoto_s3" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1146e37a7c1df56471ea67825fe09bbbd37984b5f6e201d8b2e0be4ee15643d8" +dependencies = [ + "async-trait", + "bytes 0.5.6", + "futures 0.3.8", + "rusoto_core", + "xml-rs", +] + +[[package]] +name = "rusoto_signature" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a740a88dde8ded81b6f2cff9cd5e054a5a2e38a38397260f7acdd2c85d17dd" +dependencies = [ + "base64 0.12.3", + "bytes 0.5.6", + "futures 0.3.8", + "hex", + "hmac 0.8.1", + "http 0.2.1", + "hyper 0.13.7", + "log 0.4.11", + "md5 0.7.0", + "percent-encoding 2.1.0", + "pin-project 0.4.27", + "rusoto_credential", + "rustc_version", + "serde", + "sha2 0.9.1", + "time 0.2.16", + "tokio 0.2.23", +] + +[[package]] +name = "rusoto_sts" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3815b8c0fc1c50caf9e87603f23daadfedb18d854de287b361c69f68dc9d49e0" +dependencies = [ + "async-trait", + "bytes 0.5.6", + "chrono", + "futures 0.3.8", + "rusoto_core", + "serde_urlencoded 0.6.1", + "tempfile", + "xml-rs", +] + [[package]] name = "rust-argon2" version = "0.8.3" @@ -2229,6 +2518,7 @@ dependencies = [ "anyhow", "ar", "assert_cmd", + "async-trait", "atty", "base64 0.11.0", "bincode 1.3.1", @@ -2246,9 +2536,9 @@ dependencies = [ "flate2", "futures 0.1.30", "futures 0.3.8", - "hmac", - "http", - "hyper", + "hmac 0.7.1", + "http 0.1.21", + "hyper 0.12.35", "hyperx", "itertools", "jobserver", @@ -2273,17 +2563,21 @@ dependencies = [ "retry", "ring", "rouille", + "rusoto_core", + "rusoto_s3", + "rusoto_sts", "selenium-rs", "serde", "serde_derive", "serde_json", "sha-1", - "sha2", + "sha2 0.8.2", "strip-ansi-escapes", "syslog", "tar", "tempfile", - "time", + "time 0.1.44", + "tokio 0.2.23", "tokio-compat", "tokio-io", "tokio-named-pipes", @@ -2417,16 +2711,28 @@ dependencies = [ "url 1.7.2", ] +[[package]] +name = "serde_urlencoded" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +dependencies = [ + "dtoa", + "itoa", + "serde", + "url 2.2.0", +] + [[package]] name = "sha-1" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" dependencies = [ - "block-buffer", + "block-buffer 0.7.3", "digest 0.8.1", "fake-simd", - "opaque-debug", + "opaque-debug 0.2.3", ] [[package]] @@ -2441,12 +2747,31 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" dependencies = [ - "block-buffer", + "block-buffer 0.7.3", "digest 0.8.1", "fake-simd", - "opaque-debug", + "opaque-debug 0.2.3", +] + +[[package]] +name = "sha2" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 0.1.10", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", ] +[[package]] +name = "shlex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" + [[package]] name = "signal-hook-registry" version = "1.2.2" @@ -2505,6 +2830,64 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "standback" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0437cfb83762844799a60e1e3b489d5ceb6a650fbacb86437badc1b6d87b246" +dependencies = [ + "version_check 0.9.2", +] + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote 1.0.7", + "serde", + "serde_derive", + "syn 1.0.54", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote 1.0.7", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn 1.0.54", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + [[package]] name = "string" version = "0.2.1" @@ -2593,7 +2976,7 @@ dependencies = [ "error-chain", "libc", "log 0.4.11", - "time", + "time 0.1.44", ] [[package]] @@ -2639,7 +3022,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" dependencies = [ "byteorder", - "dirs", + "dirs 1.0.5", "winapi 0.3.9", ] @@ -2710,6 +3093,44 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "time" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a51cadc5b1eec673a685ff7c33192ff7b7603d0b75446fb354939ee615acb15" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check 0.9.2", + "winapi 0.3.9", +] + +[[package]] +name = "time-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9b6e9f095bc105e183e3cd493d72579be3181ad4004fceb01adbe9eecab2d" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote 1.0.7", + "standback", + "syn 1.0.54", +] + [[package]] name = "tiny_http" version = "0.6.2" @@ -2776,10 +3197,14 @@ dependencies = [ "libc", "memchr 2.3.4", "mio", + "mio-named-pipes", "mio-uds", "num_cpus", "pin-project-lite 0.1.11", + "signal-hook-registry", "slab", + "tokio-macros", + "winapi 0.3.9", ] [[package]] @@ -2863,6 +3288,17 @@ dependencies = [ "log 0.4.11", ] +[[package]] +name = "tokio-macros" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" +dependencies = [ + "proc-macro2", + "quote 1.0.7", + "syn 1.0.54", +] + [[package]] name = "tokio-named-pipes" version = "0.1.0" @@ -3008,6 +3444,16 @@ dependencies = [ "tokio-executor", ] +[[package]] +name = "tokio-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" +dependencies = [ + "native-tls", + "tokio 0.2.23", +] + [[package]] name = "tokio-udp" version = "0.1.6" @@ -3055,6 +3501,20 @@ dependencies = [ "tokio 0.2.23", ] +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log 0.4.11", + "pin-project-lite 0.1.11", + "tokio 0.2.23", +] + [[package]] name = "toml" version = "0.5.7" @@ -3077,7 +3537,7 @@ dependencies = [ "tower-limit", "tower-load-shed", "tower-retry", - "tower-service", + "tower-service 0.2.0", "tower-timeout", "tower-util", ] @@ -3092,7 +3552,7 @@ dependencies = [ "tokio-executor", "tokio-sync", "tower-layer", - "tower-service", + "tower-service 0.2.0", "tracing", ] @@ -3103,7 +3563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73a7632286f78164d65d18fd0e570307acde9362489aa5c8c53e6315cc2bde47" dependencies = [ "futures 0.1.30", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3113,7 +3573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ddf07e10c07dcc8f41da6de036dc66def1a85b70eb8a385159e3908bb258328" dependencies = [ "futures 0.1.30", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3126,7 +3586,7 @@ dependencies = [ "tokio-sync", "tokio-timer", "tower-layer", - "tower-service", + "tower-service 0.2.0", "tracing", ] @@ -3138,7 +3598,7 @@ checksum = "04fbaf5bfb63d84204db87b9b2aeec61549613f2bbb8706dcc36f5f3ea8cd769" dependencies = [ "futures 0.1.30", "tower-layer", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3150,7 +3610,7 @@ dependencies = [ "futures 0.1.30", "tokio-timer", "tower-layer", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3162,6 +3622,12 @@ dependencies = [ "futures 0.1.30", ] +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + [[package]] name = "tower-timeout" version = "0.1.1" @@ -3171,7 +3637,7 @@ dependencies = [ "futures 0.1.30", "tokio-timer", "tower-layer", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3183,7 +3649,7 @@ dependencies = [ "futures 0.1.30", "tokio-io", "tower-layer", - "tower-service", + "tower-service 0.2.0", ] [[package]] @@ -3219,6 +3685,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + [[package]] name = "treeline" version = "0.1.0" @@ -3448,6 +3924,16 @@ dependencies = [ "try-lock", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log 0.4.11", + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -3460,6 +3946,60 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasm-bindgen" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +dependencies = [ + "bumpalo", + "lazy_static", + "log 0.4.11", + "proc-macro2", + "quote 1.0.7", + "syn 1.0.54", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +dependencies = [ + "quote 1.0.7", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +dependencies = [ + "proc-macro2", + "quote 1.0.7", + "syn 1.0.54", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" + [[package]] name = "which" version = "4.0.2" @@ -3541,6 +4081,18 @@ dependencies = [ "libc", ] +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + +[[package]] +name = "zeroize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" + [[package]] name = "zip" version = "0.5.9" diff --git a/Cargo.toml b/Cargo.toml index 6d11173b7..f6302b04a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ required-features = ["dist-server"] [dependencies] anyhow = "1.0" ar = { version = "0.8", optional = true } +async-trait = "0.1" atty = "0.2.6" base64 = "0.11.0" bincode = "1" @@ -60,6 +61,9 @@ regex = "1" reqwest = { version = "0.9.11", optional = true } retry = "0.4.0" ring = { version = "0.14.6", optional = true } +rusoto_core = { version = "0.45.0", features = ["native-tls"], optional = true } +rusoto_s3 = { version = "0.45.0", features = ["native-tls"], optional = true } +rusoto_sts = { version = "0.45.0", features = ["native-tls"], optional = true } sha-1 = { version = "0.8", optional = true } sha2 = { version = "0.8", optional = true } serde = "1.0" @@ -69,6 +73,7 @@ strip-ansi-escapes = "0.1" tar = "0.4" tempfile = "3" time = "0.1.35" +tokio_02 = { package = "tokio", version = "0.2", features = ["io-util"], optional = true } tokio-compat = "0.1" tokio-io = "0.1" tokio-process = "0.2" @@ -127,8 +132,7 @@ features = [ default = ["dist-client", "s3"] all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure"] azure = ["chrono", "hyper", "hyperx", "url", "hmac", "md-5", "sha2"] -s3 = ["chrono", "hyper", "hyperx", "reqwest", "simple-s3", "hmac", "sha-1"] -simple-s3 = [] +s3 = ["hyperx", "rusoto_core", "rusoto_s3", "rusoto_sts", "tokio_02"] gcs = ["chrono", "hyper", "hyperx", "reqwest", "ring", "untrusted", "url"] memcached = ["memcached-rs"] # Enable features that require unstable features of Nightly Rust. diff --git a/README.md b/README.md index 17b3a605c..94714226f 100644 --- a/README.md +++ b/README.md @@ -192,13 +192,14 @@ The default cache size is 10 gigabytes. To change this, set `SCCACHE_CACHE_SIZE` ### S3 If you want to use S3 storage for the sccache cache, you need to set the `SCCACHE_BUCKET` environment variable to the name of the S3 bucket to use. -You can use `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to set the S3 credentials. Alternately, you can set `AWS_IAM_CREDENTIALS_URL` to a URL that returns credentials in the format supported by the [EC2 metadata service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials), and credentials will be fetched from that location as needed. In the absence of either of these options, credentials for the instance's IAM role will be fetched from the EC2 metadata service directly. +You can use `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to set the S3 credentials. Other supported methods are listed in Rusoto's [ChainProvider](https://rusoto.github.io/rusoto/rusoto_credential/struct.ChainProvider.html). To connect to a public bucket anonymously (read only mode), the environment variable `SCCACHE_S3_PUBLIC` must be set to true, to prevent the default behavior of rusoto's [DefaultCredentialsProvider](https://rusoto.github.io/rusoto/rusoto_credential/struct.DefaultCredentialsProvider.html), which is to pass the error of ChainProvider. -If you need to override the default endpoint you can set `SCCACHE_ENDPOINT`. To connect to a minio storage for example you can set `SCCACHE_ENDPOINT=:`. If your endpoint requires TLS, set `SCCACHE_S3_USE_SSL=true`. +You can set the region of your bucket with one of the environment variables `AWS_DEFAULT_REGION`, `AWS_REGION` or `SCCACHE_REGION`. +If you need to override the default endpoint you can set `SCCACHE_ENDPOINT`. To connect to a minio storage for example you can set `SCCACHE_ENDPOINT=:`. +Optionally, the endpoint can start with `http://` or `https://` to force the protocol. By default, HTTPS will be used. You can also define a prefix that will be prepended to the keys of all cache objects created and read within the S3 bucket, effectively creating a scope. To do that use the `SCCACHE_S3_KEY_PREFIX` environment variable. This can be useful when sharing a bucket with another application. - ### Redis Set `SCCACHE_REDIS` to a [Redis](https://redis.io/) url in format `redis://[:@][:port][/]` to store the cache in a Redis instance. Redis can be configured as a LRU (least recently used) cache with a fixed maximum cache size. Set `maxmemory` and `maxmemory-policy` according to the [Redis documentation](https://redis.io/topics/lru-cache). The `allkeys-lru` policy which discards the *least recently accessed or modified* key fits well for the sccache use case. diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 03cbe9376..d5cbc0791 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -383,9 +383,24 @@ pub fn storage_from_config(config: &Config, pool: &ThreadPool) -> Arc { - debug!("Trying S3Cache({}, {})", c.bucket, c.endpoint); + let region = c.region.as_deref(); + let endpoint = c.endpoint.as_deref(); + let key_prefix = c.key_prefix.as_deref(); + debug!( + "Trying S3Cache({}, {}, {}, Anonymous {})", + c.bucket, + region.unwrap_or("default region"), + endpoint.unwrap_or("default endpoint"), + c.public, + ); #[cfg(feature = "s3")] - match S3Cache::new(&c.bucket, &c.endpoint, c.use_ssl, &c.key_prefix) { + match S3Cache::new( + &c.bucket, + region, + endpoint, + key_prefix.unwrap_or(""), + c.public, + ) { Ok(s) => { trace!("Using S3Cache"); return Arc::new(s); diff --git a/src/cache/s3.rs b/src/cache/s3.rs index b08fcee9e..d305247b2 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -13,52 +13,111 @@ // limitations under the License. use crate::cache::{Cache, CacheRead, CacheWrite, Storage}; -use crate::simples3::{ - AutoRefreshingProvider, Bucket, ChainProvider, ProfileProvider, ProvideAwsCredentials, Ssl, -}; -use directories::UserDirs; +use async_trait::async_trait; use futures::future; use futures::future::Future; +use futures_03::future::TryFutureExt as _; +use hyperx::header::CacheDirective; +use rusoto_core::{self, Client, HttpClient, Region}; +use rusoto_s3::{GetObjectOutput, GetObjectRequest, PutObjectRequest, S3Client, S3 as _}; use std::io; -use std::rc::Rc; +use std::str::FromStr; use std::time::{Duration, Instant}; +use tokio_02::io::AsyncReadExt as _; use crate::errors::*; +use rusoto_core::credential::{ + AutoRefreshingProvider, AwsCredentials, CredentialsError, EnvironmentProvider, + ProvideAwsCredentials, +}; /// A cache that stores entries in Amazon S3. pub struct S3Cache { - /// The S3 bucket. - bucket: Rc, - /// Credentials provider. - provider: AutoRefreshingProvider, + /// The name of the bucket. + bucket_name: String, + /// The S3 client to be used for the Get and Put requests. + client: S3Client, /// Prefix to be used for bucket keys. key_prefix: String, } impl S3Cache { /// Create a new `S3Cache` storing data in `bucket`. - pub fn new(bucket: &str, endpoint: &str, use_ssl: bool, key_prefix: &str) -> Result { - let user_dirs = UserDirs::new().context("Couldn't get user directories")?; - let home = user_dirs.home_dir(); - - let profile_providers = vec![ - ProfileProvider::with_configuration(home.join(".aws").join("credentials"), "default"), - //TODO: this is hacky, this is where our mac builders store their - // credentials. We should either match what boto does more directly - // or make those builders put their credentials in ~/.aws/credentials - ProfileProvider::with_configuration(home.join(".boto"), "Credentials"), - ]; - let provider = - AutoRefreshingProvider::new(ChainProvider::with_profile_providers(profile_providers)); - let ssl_mode = if use_ssl { Ssl::Yes } else { Ssl::No }; - let bucket = Rc::new(Bucket::new(bucket, endpoint, ssl_mode)?); + pub fn new( + bucket: &str, + region: Option<&str>, + endpoint: Option<&str>, + key_prefix: &str, + public: bool, + ) -> Result { + let region = match endpoint { + Some(endpoint) => Region::Custom { + name: region + .map(ToOwned::to_owned) + .unwrap_or(Region::default().name().to_owned()), + endpoint: endpoint.to_owned(), + }, + None => region + .map(FromStr::from_str) + .unwrap_or_else(|| Ok(Region::default()))?, + }; + + //we could define our own Annonymous StaticProvider here, but we'd have to verify that the DefaultCredentialsProvider would fail first. + let client = if public { + let client = Client::new_not_signing( + HttpClient::new().expect("failed to create request dispatcher"), + ); + S3Client::new_with_client(client, region) + } else { + let credential_provider = + AutoRefreshingProvider::new(ChainCredentialProvider::default()) + .expect("failed to create a credential provider"); + S3Client::new_with( + HttpClient::new().expect("failed to create request dispatcher"), + credential_provider, + region, + ) + }; + Ok(S3Cache { - bucket, - provider, + bucket_name: bucket.to_owned(), + client, key_prefix: key_prefix.to_owned(), }) } + async fn get_object(client: S3Client, request: GetObjectRequest) -> Result { + let result = client.get_object(request).await; + match result { + Ok(output) => Self::read_object_output(output).await, + Err(rusoto_core::RusotoError::Service(rusoto_s3::GetObjectError::NoSuchKey(_))) => { + Ok(Cache::Miss) + } + Err(e) => Err(e.into()), + } + } + + async fn read_object_output(output: GetObjectOutput) -> Result { + let body = output.body.context("no HTTP body")?; + let mut body_reader = body.into_async_read(); + let mut body = Vec::new(); + body_reader + .read_to_end(&mut body) + .await + .context("failed to read HTTP body")?; + let hit = CacheRead::from(io::Cursor::new(body))?; + Ok(Cache::Hit(hit)) + } + + async fn put_object(client: S3Client, request: PutObjectRequest) -> Result<()> { + client + .put_object(request) + .await + .map(|_| ()) + .context("failed to put cache entry in s3") + .into() + } + fn normalize_key(&self, key: &str) -> String { format!( "{}{}/{}/{}/{}", @@ -75,30 +134,14 @@ impl Storage for S3Cache { fn get(&self, key: &str) -> SFuture { let key = self.normalize_key(key); - let result_cb = |result| match result { - Ok(data) => { - let hit = CacheRead::from(io::Cursor::new(data))?; - Ok(Cache::Hit(hit)) - } - Err(e) => { - warn!("Got AWS error: {:?}", e); - Ok(Cache::Miss) - } + let client = self.client.clone(); + let request = GetObjectRequest { + bucket: self.bucket_name.clone(), + key, + ..Default::default() }; - let bucket = self.bucket.clone(); - let response = self - .provider - .credentials() - .then(move |credentials| match credentials { - Ok(creds) => bucket.get(&key, Some(&creds)), - Err(e) => { - debug!("Could not load AWS creds: {}", e); - bucket.get(&key, None) - } - }) - .then(result_cb); - Box::new(response) + Box::new(Box::pin(Self::get_object(client, request)).compat()) } fn put(&self, key: &str, entry: CacheWrite) -> SFuture { @@ -108,23 +151,29 @@ impl Storage for S3Cache { Ok(data) => data, Err(e) => return f_err(e), }; - let credentials = self - .provider - .credentials() - .fcontext("failed to get AWS credentials"); + let data_length = data.len(); - let bucket = self.bucket.clone(); - let response = credentials.and_then(move |credentials| { - bucket - .put(&key, data, &credentials) - .fcontext("failed to put cache entry in s3") - }); + let client = self.client.clone(); + let request = PutObjectRequest { + bucket: self.bucket_name.clone(), + body: Some(data.into()), + // Two weeks + cache_control: Some(CacheDirective::MaxAge(1_296_000).to_string()), + content_length: Some(data_length as i64), + content_type: Some("application/octet-stream".to_owned()), + key, + ..Default::default() + }; - Box::new(response.map(move |_| start.elapsed())) + Box::new( + Box::pin(Self::put_object(client, request)) + .compat() + .then(move |_| future::ok(start.elapsed())), + ) } fn location(&self) -> String { - format!("S3, bucket: {}", self.bucket) + format!("S3, bucket: {}", self.bucket_name) } fn current_size(&self) -> SFuture> { @@ -134,3 +183,43 @@ impl Storage for S3Cache { Box::new(future::ok(None)) } } + +#[derive(Debug, Clone, Default)] +struct ChainCredentialProvider {} + +#[async_trait] +impl ProvideAwsCredentials for ChainCredentialProvider { + async fn credentials(&self) -> std::result::Result { + chain_provider_credentials().await + } +} + +/// This is similar to the default chain provider in rusoto_credential lib.rs but adds the +/// WebIdentityProvider after the EnvironmentProvider to inject credentials from the EKS IRSA method +async fn chain_provider_credentials() -> std::result::Result { + use rusoto_core::credential::{ContainerProvider, InstanceMetadataProvider, ProfileProvider}; + + if let Ok(creds) = EnvironmentProvider::default().credentials().await { + return Ok(creds); + } + if let Ok(creds) = rusoto_sts::WebIdentityProvider::from_k8s_env() + .credentials() + .await + { + return Ok(creds); + } + if let Ok(ref profile_provider) = ProfileProvider::new() { + if let Ok(creds) = profile_provider.credentials().await { + return Ok(creds); + } + } + if let Ok(creds) = ContainerProvider::new().credentials().await { + return Ok(creds); + } + if let Ok(creds) = InstanceMetadataProvider::new().credentials().await { + return Ok(creds); + } + Err(CredentialsError::new( + "Couldn't find AWS credentials in environment, credentials file, or IAM role.", + )) +} diff --git a/src/config.rs b/src/config.rs index 3e89289e7..d8f2c5ca2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -197,9 +197,13 @@ pub struct RedisCacheConfig { #[serde(deny_unknown_fields)] pub struct S3CacheConfig { pub bucket: String, - pub endpoint: String, - pub use_ssl: bool, - pub key_prefix: String, + #[serde(default)] + pub endpoint: Option, + #[serde(default)] + pub key_prefix: Option, + #[serde(default)] + pub region: Option, + pub public: bool, } #[derive(Debug, PartialEq, Eq)] @@ -447,32 +451,22 @@ pub struct EnvConfig { fn config_from_env() -> EnvConfig { let s3 = env::var("SCCACHE_BUCKET").ok().map(|bucket| { - let endpoint = match env::var("SCCACHE_ENDPOINT") { - Ok(endpoint) => format!("{}/{}", endpoint, bucket), - _ => match env::var("SCCACHE_REGION") { - Ok(ref region) if region != "us-east-1" => { - format!("{}.s3-{}.amazonaws.com", bucket, region) - } - _ => format!("{}.s3.amazonaws.com", bucket), - }, - }; - let use_ssl = env::var("SCCACHE_S3_USE_SSL") - .ok() - .filter(|value| value != "off") - .is_some(); + let endpoint = env::var("SCCACHE_ENDPOINT").ok(); + let region = env::var("SCCACHE_REGION").ok(); let key_prefix = env::var("SCCACHE_S3_KEY_PREFIX") .ok() .as_ref() .map(|s| s.trim_end_matches('/')) .filter(|s| !s.is_empty()) - .map(|s| s.to_owned() + "/") - .unwrap_or_default(); + .map(|s| s.to_owned() + "/"); + let public = env::var("SCCACHE_S3_PUBLIC").ok().is_some(); S3CacheConfig { bucket, endpoint, - use_ssl, key_prefix, + region, + public, } }); diff --git a/src/lib.rs b/src/lib.rs index 48922cc81..6fdaaa7ca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -54,8 +54,6 @@ mod jobserver; mod mock_command; mod protocol; pub mod server; -#[cfg(feature = "simple-s3")] -mod simples3; #[doc(hidden)] pub mod util; diff --git a/src/simples3/credential.rs b/src/simples3/credential.rs deleted file mode 100644 index bc7c3adff..000000000 --- a/src/simples3/credential.rs +++ /dev/null @@ -1,530 +0,0 @@ -// Originally from https://github.com/rusoto/rusoto/blob/master/src/credential.rs -//! Types for loading and managing AWS access credentials for API requests. -#![allow(dead_code)] - -use chrono::{offset, DateTime, Duration}; -use directories::UserDirs; -use futures::future::{self, Shared}; -use futures::{Async, Future, Stream}; -use hyper::client::HttpConnector; -use hyper::{Client, Request}; -use hyperx::header::Connection; -use regex::Regex; -use serde_json::{from_str, Value}; -#[allow(unused_imports, deprecated)] -use std::ascii::AsciiExt; -use std::cell::RefCell; -use std::collections::HashMap; -use std::env::*; -use std::fs::{self, File}; -use std::io::prelude::*; -use std::io::BufReader; -use std::path::{Path, PathBuf}; -use std::time::Duration as StdDuration; -use tokio_timer::Timeout; - -use crate::errors::*; -use crate::util::RequestExt; - -/// AWS API access credentials, including access key, secret key, token (for IAM profiles), and -/// expiration timestamp. -#[derive(Clone, Debug)] -pub struct AwsCredentials { - key: String, - secret: String, - token: Option, - expires_at: DateTime, -} - -impl AwsCredentials { - /// Create a new `AwsCredentials` from a key ID, secret key, optional access token, and expiry - /// time. - pub fn new( - key: K, - secret: S, - token: Option, - expires_at: DateTime, - ) -> AwsCredentials - where - K: Into, - S: Into, - { - AwsCredentials { - key: key.into(), - secret: secret.into(), - token, - expires_at, - } - } - - /// Get a reference to the access key ID. - pub fn aws_access_key_id(&self) -> &str { - &self.key - } - - /// Get a reference to the secret access key. - pub fn aws_secret_access_key(&self) -> &str { - &self.secret - } - - /// Get a reference to the expiry time. - pub fn expires_at(&self) -> &DateTime { - &self.expires_at - } - - /// Get a reference to the access token. - pub fn token(&self) -> &Option { - &self.token - } - - /// Determine whether or not the credentials are expired. - fn credentials_are_expired(&self) -> bool { - // This is a rough hack to hopefully avoid someone requesting creds then sitting on them - // before issuing the request: - self.expires_at < offset::Utc::now() + Duration::seconds(20) - } -} - -/// A trait for types that produce `AwsCredentials`. -pub trait ProvideAwsCredentials { - /// Produce a new `AwsCredentials`. - fn credentials(&self) -> SFuture; -} - -/// Provides AWS credentials from environment variables. -pub struct EnvironmentProvider; - -impl ProvideAwsCredentials for EnvironmentProvider { - fn credentials(&self) -> SFuture { - Box::new(future::result(credentials_from_environment())) - } -} - -fn credentials_from_environment() -> Result { - let env_key = var("AWS_ACCESS_KEY_ID").context("No AWS_ACCESS_KEY_ID in environment")?; - let env_secret = - var("AWS_SECRET_ACCESS_KEY").context("No AWS_SECRET_ACCESS_KEY in environment")?; - - if env_key.is_empty() || env_secret.is_empty() { - bail!( - "Couldn't find either AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY or both in environment." - ) - } - - // Present when using temporary credentials, e.g. on Lambda with IAM roles - let token = match var("AWS_SESSION_TOKEN") { - Ok(val) => { - if val.is_empty() { - None - } else { - Some(val) - } - } - Err(_) => None, - }; - - Ok(AwsCredentials::new( - env_key, - env_secret, - token, - in_ten_minutes(), - )) -} - -/// Provides AWS credentials from a profile in a credentials file. -#[derive(Clone, Debug)] -pub struct ProfileProvider { - credentials: Option, - file_path: PathBuf, - profile: String, -} - -impl ProfileProvider { - /// Create a new `ProfileProvider` for the default credentials file path and profile name. - pub fn new() -> Result { - // Default credentials file location: - // ~/.aws/credentials (Linux/Mac) - // %USERPROFILE%\.aws\credentials (Windows) - let profile_location = UserDirs::new() - .map(|d| d.home_dir().join(".aws").join("credentials")) - .context("Couldn't get user directories")?; - - Ok(ProfileProvider { - credentials: None, - file_path: profile_location, - profile: "default".to_owned(), - }) - } - - /// Create a new `ProfileProvider` for the credentials file at the given path, using - /// the given profile. - pub fn with_configuration(file_path: F, profile: P) -> ProfileProvider - where - F: Into, - P: Into, - { - ProfileProvider { - credentials: None, - file_path: file_path.into(), - profile: profile.into(), - } - } - - /// Get a reference to the credentials file path. - pub fn file_path(&self) -> &Path { - self.file_path.as_ref() - } - - /// Get a reference to the profile name. - pub fn profile(&self) -> &str { - &self.profile - } - - /// Set the credentials file path. - pub fn set_file_path(&mut self, file_path: F) - where - F: Into, - { - self.file_path = file_path.into(); - } - - /// Set the profile name. - pub fn set_profile

(&mut self, profile: P) - where - P: Into, - { - self.profile = profile.into(); - } -} - -impl ProvideAwsCredentials for ProfileProvider { - fn credentials(&self) -> SFuture { - let result = parse_credentials_file(self.file_path()); - let result = result - .and_then(|mut profiles| profiles.remove(self.profile()).context("profile not found")); - Box::new(future::result(result)) - } -} - -fn parse_credentials_file(file_path: &Path) -> Result> { - let metadata = fs::metadata(file_path).context("couldn't stat credentials file")?; - if !metadata.is_file() { - bail!("Couldn't open file."); - } - - let file = File::open(file_path)?; - - let profile_regex = Regex::new(r"^\[([^\]]+)\]$").unwrap(); - let mut profiles: HashMap = HashMap::new(); - let mut access_key: Option = None; - let mut secret_key: Option = None; - let mut profile_name: Option = None; - - let file_lines = BufReader::new(&file); - for line in file_lines.lines() { - let unwrapped_line: String = line.unwrap(); - - // skip comments - if unwrapped_line.starts_with('#') { - continue; - } - - // handle the opening of named profile blocks - if profile_regex.is_match(&unwrapped_line) { - if let (Some(profile_name), Some(access_key), Some(secret_key)) = - (profile_name, access_key, secret_key) - { - let creds = AwsCredentials::new(access_key, secret_key, None, in_ten_minutes()); - profiles.insert(profile_name, creds); - } - - access_key = None; - secret_key = None; - - let caps = profile_regex.captures(&unwrapped_line).unwrap(); - profile_name = Some(caps.get(1).unwrap().as_str().to_string()); - continue; - } - - // otherwise look for key=value pairs we care about - let lower_case_line = unwrapped_line.to_ascii_lowercase().to_string(); - - if lower_case_line.contains("aws_access_key_id") && access_key.is_none() { - let v: Vec<&str> = unwrapped_line.split('=').collect(); - if !v.is_empty() { - access_key = Some(v[1].trim_matches(' ').to_string()); - } - } else if lower_case_line.contains("aws_secret_access_key") && secret_key.is_none() { - let v: Vec<&str> = unwrapped_line.split('=').collect(); - if !v.is_empty() { - secret_key = Some(v[1].trim_matches(' ').to_string()); - } - } - - // we could potentially explode here to indicate that the file is invalid - } - - if let (Some(profile_name), Some(access_key), Some(secret_key)) = - (profile_name, access_key, secret_key) - { - let creds = AwsCredentials::new(access_key, secret_key, None, in_ten_minutes()); - profiles.insert(profile_name, creds); - } - - if profiles.is_empty() { - bail!("No credentials found.") - } - - Ok(profiles) -} - -/// Provides AWS credentials from a resource's IAM role. -pub struct IamProvider { - client: Client, -} - -impl IamProvider { - pub fn new() -> IamProvider { - IamProvider { - client: Client::new(), - } - } - - fn iam_role(&self) -> SFuture { - // First get the IAM role - let address = "http://169.254.169.254/latest/meta-data/iam/security-credentials/"; - let req = Request::get(address) - .set_header(Connection::close()) - .body("".into()) - .unwrap(); - let response = self.client.request(req).and_then(|response| { - response.into_body().fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, hyper::Error>(body) - }) - }); - - Box::new( - response - .then(|res| { - let bytes = res.context("couldn't connect to metadata service")?; - String::from_utf8(bytes) - .context("Didn't get a parsable response body from metadata service") - }) - .map(move |body| { - let mut address = address.to_string(); - address.push_str(&body); - address - }), - ) - } -} - -impl ProvideAwsCredentials for IamProvider { - fn credentials(&self) -> SFuture { - let url = match var("AWS_IAM_CREDENTIALS_URL") { - Ok(url) => f_ok(url), - Err(_) => self.iam_role(), - }; - let url = url.and_then(|url| { - url.parse::() - .with_context(|| format!("failed to parse `{}` as url", url)) - }); - - let client = self.client.clone(); - let response = url.and_then(move |address| { - debug!("Attempting to fetch credentials from {}", address); - let req = Request::get(address) - .set_header(Connection::close()) - .body("".into()) - .unwrap(); - client.request(req).fcontext("failed to send http request") - }); - let body = response.and_then(|response| { - response - .into_body() - .fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, hyper::Error>(body) - }) - .fcontext("failed to read http body") - }); - let body = body - .map_err(|e| anyhow!("Failed to get IAM credentials: {}", e)) - .and_then(|body| String::from_utf8(body).context("failed to read iam role response")); - - let creds = body.and_then(|body| { - let json_object: Value; - match from_str(&body) { - Err(_) => bail!("Couldn't parse metadata response body."), - Ok(val) => json_object = val, - }; - - let access_key; - match json_object.get("AccessKeyId") { - None => bail!("Couldn't find AccessKeyId in response."), - Some(val) => { - access_key = val - .as_str() - .expect("AccessKeyId value was not a string") - .to_owned() - .replace("\"", "") - } - }; - - let secret_key; - match json_object.get("SecretAccessKey") { - None => bail!("Couldn't find SecretAccessKey in response."), - Some(val) => { - secret_key = val - .as_str() - .expect("SecretAccessKey value was not a string") - .to_owned() - .replace("\"", "") - } - }; - - let expiration; - match json_object.get("Expiration") { - None => bail!("Couldn't find Expiration in response."), - Some(val) => { - expiration = val - .as_str() - .expect("Expiration value was not a string") - .to_owned() - .replace("\"", "") - } - }; - - let expiration_time = expiration - .parse() - .context("failed to parse expiration time")?; - - let token_from_response; - match json_object.get("Token") { - None => bail!("Couldn't find Token in response."), - Some(val) => { - token_from_response = val - .as_str() - .expect("Token value was not a string") - .to_owned() - .replace("\"", "") - } - }; - - Ok(AwsCredentials::new( - access_key, - secret_key, - Some(token_from_response), - expiration_time, - )) - }); - - //XXX: this is crappy, but this blocks on non-EC2 machines like - // our mac builders. - let timeout = Timeout::new(creds, StdDuration::from_secs(2)); - - Box::new(timeout.then(|result| match result { - Ok(creds) => Ok(creds), - Err(err) => match err.into_inner() { - None => bail!("took too long to fetch credentials"), - Some(e) => { - warn!("Failed to fetch IAM credentials: {}", e); - Err(e) - } - }, - })) - } -} - -/// Wrapper for ProvideAwsCredentials that caches the credentials returned by the -/// wrapped provider. Each time the credentials are accessed, they are checked to see if -/// they have expired, in which case they are retrieved from the wrapped provider again. -pub struct AutoRefreshingProvider

{ - credentials_provider: P, - cached_credentials: RefCell>>, -} - -impl AutoRefreshingProvider

{ - pub fn new(provider: P) -> AutoRefreshingProvider

{ - AutoRefreshingProvider { - cached_credentials: RefCell::new(provider.credentials().shared()), - credentials_provider: provider, - } - } -} - -impl ProvideAwsCredentials for AutoRefreshingProvider

{ - fn credentials(&self) -> SFuture { - let mut future = self.cached_credentials.borrow_mut(); - if let Ok(Async::Ready(creds)) = future.poll() { - if creds.credentials_are_expired() { - *future = self.credentials_provider.credentials().shared(); - } - } - Box::new(future.clone().then(|result| match result { - Ok(e) => Ok((*e).clone()), - Err(e) => Err(anyhow!(e.to_string())), - })) - } -} - -/// Provides AWS credentials from multiple possible sources using a priority order. -/// -/// The following sources are checked in order for credentials when calling `credentials`: -/// -/// 1. Environment variables: `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` -/// 2. AWS credentials file. Usually located at `~/.aws/credentials`. -/// 3. IAM instance profile. Will only work if running on an EC2 instance with an instance profile/role. -/// -/// If the sources are exhausted without finding credentials, an error is returned. -#[derive(Clone)] -pub struct ChainProvider { - profile_providers: Vec, -} - -impl ProvideAwsCredentials for ChainProvider { - fn credentials(&self) -> SFuture { - let creds = EnvironmentProvider.credentials().map(|c| { - debug!("Using AWS credentials from environment"); - c - }); - let mut creds = Box::new(creds) as SFuture<_>; - for provider in self.profile_providers.iter() { - let alternate = provider.credentials(); - creds = Box::new(creds.or_else(|_| alternate)); - } - Box::new( - creds - .or_else(move |_| { - IamProvider::new().credentials().map(|c| { - debug!("Using AWS credentials from IAM"); - c - }) - }) - .map_err(|_| { - anyhow!( - "Couldn't find AWS credentials in environment, credentials file, or IAM role." - ) - }), - ) - } -} - -impl ChainProvider { - /// Create a new `ChainProvider` using a `ProfileProvider` with the default settings. - pub fn new() -> ChainProvider { - ChainProvider { - profile_providers: ProfileProvider::new().into_iter().collect(), - } - } - - /// Create a new `ChainProvider` using the provided `ProfileProvider`s. - pub fn with_profile_providers(profile_providers: Vec) -> ChainProvider { - ChainProvider { profile_providers } - } -} - -fn in_ten_minutes() -> DateTime { - offset::Utc::now() + Duration::seconds(600) -} diff --git a/src/simples3/mod.rs b/src/simples3/mod.rs deleted file mode 100644 index 2c0205393..000000000 --- a/src/simples3/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2016 Mozilla Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod credential; -mod s3; - -pub use crate::simples3::credential::*; -pub use crate::simples3::s3::*; diff --git a/src/simples3/s3.rs b/src/simples3/s3.rs deleted file mode 100644 index f7e193ae9..000000000 --- a/src/simples3/s3.rs +++ /dev/null @@ -1,251 +0,0 @@ -// Originally from https://github.com/rust-lang/crates.io/blob/master/src/s3/lib.rs -//#![deny(warnings)] - -#[allow(unused_imports, deprecated)] -use std::ascii::AsciiExt; -use std::fmt; - -use crate::simples3::credential::*; -use futures::{Future, Stream}; -use hmac::{Hmac, Mac}; -use hyper::header::HeaderValue; -use hyper::Method; -use hyperx::header; -use reqwest::r#async::{Client, Request}; -use sha1::Sha1; - -use crate::errors::*; -use crate::util::HeadersExt; - -#[derive(Debug, Copy, Clone)] -#[allow(dead_code)] -/// Whether or not to use SSL. -pub enum Ssl { - /// Use SSL. - Yes, - /// Do not use SSL. - No, -} - -fn base_url(endpoint: &str, ssl: Ssl) -> String { - format!( - "{}://{}/", - match ssl { - Ssl::Yes => "https", - Ssl::No => "http", - }, - endpoint - ) -} - -fn hmac(key: &[u8], data: &[u8]) -> Vec { - let mut hmac = Hmac::::new_varkey(key).expect("HMAC can take key of any size"); - hmac.input(data); - hmac.result().code().iter().copied().collect::>() -} - -fn signature(string_to_sign: &str, signing_key: &str) -> String { - let s = hmac(signing_key.as_bytes(), string_to_sign.as_bytes()); - base64::encode_config::>(&s, base64::STANDARD) -} - -/// An S3 bucket. -pub struct Bucket { - name: String, - base_url: String, - client: Client, -} - -impl fmt::Display for Bucket { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Bucket(name={}, base_url={})", self.name, self.base_url) - } -} - -impl Bucket { - pub fn new(name: &str, endpoint: &str, ssl: Ssl) -> Result { - let base_url = base_url(&endpoint, ssl); - Ok(Bucket { - name: name.to_owned(), - base_url, - client: Client::new(), - }) - } - - pub fn get(&self, key: &str, creds: Option<&AwsCredentials>) -> SFuture> { - let url = format!("{}{}", self.base_url, key); - debug!("GET {}", url); - let url2 = url.clone(); - let mut request = Request::new(Method::GET, url.parse().unwrap()); - if let Some(creds) = creds { - let mut canonical_headers = String::new(); - - if let Some(token) = creds.token().as_ref().map(|s| s.as_str()) { - request.headers_mut().insert( - "x-amz-security-token", - HeaderValue::from_str(token).expect("Invalid `x-amz-security-token` header"), - ); - canonical_headers - .push_str(format!("{}:{}\n", "x-amz-security-token", token).as_ref()); - } - let date = time::now_utc().rfc822().to_string(); - let auth = self.auth("GET", &date, key, "", &canonical_headers, "", creds); - request.headers_mut().insert( - "Date", - HeaderValue::from_str(&date).expect("Invalid date header"), - ); - request.headers_mut().insert( - "Authorization", - HeaderValue::from_str(&auth).expect("Invalid authentication"), - ); - } - - Box::new( - self.client - .execute(request) - .fwith_context(move || format!("failed GET: {}", url)) - .and_then(|res| { - if res.status().is_success() { - let content_length = res - .headers() - .get_hyperx::() - .map(|header::ContentLength(len)| len); - Ok((res.into_body(), content_length)) - } else { - Err(BadHttpStatusError(res.status()).into()) - } - }) - .and_then(|(body, content_length)| { - body.fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, reqwest::Error>(body) - }) - .fcontext("failed to read HTTP body") - .and_then(move |bytes| { - if let Some(len) = content_length { - if len != bytes.len() as u64 { - bail!(format!( - "Bad HTTP body size read: {}, expected {}", - bytes.len(), - len - )); - } else { - info!("Read {} bytes from {}", bytes.len(), url2); - } - } - Ok(bytes) - }) - }), - ) - } - - pub fn put(&self, key: &str, content: Vec, creds: &AwsCredentials) -> SFuture<()> { - let url = format!("{}{}", self.base_url, key); - debug!("PUT {}", url); - let mut request = Request::new(Method::PUT, url.parse().unwrap()); - - let content_type = "application/octet-stream"; - let date = time::now_utc().rfc822().to_string(); - let mut canonical_headers = String::new(); - let token = creds.token().as_ref().map(|s| s.as_str()); - // Keep the list of header values sorted! - for (header, maybe_value) in &[("x-amz-security-token", token)] { - if let Some(ref value) = maybe_value { - request.headers_mut().insert( - *header, - HeaderValue::from_str(value) - .unwrap_or_else(|_| panic!("Invalid `{}` header", header)), - ); - canonical_headers - .push_str(format!("{}:{}\n", header.to_ascii_lowercase(), value).as_ref()); - } - } - let auth = self.auth( - "PUT", - &date, - key, - "", - &canonical_headers, - content_type, - creds, - ); - request.headers_mut().insert( - "Date", - HeaderValue::from_str(&date).expect("Invalid date header"), - ); - request - .headers_mut() - .set(header::ContentType(content_type.parse().unwrap())); - request - .headers_mut() - .set(header::ContentLength(content.len() as u64)); - request.headers_mut().set(header::CacheControl(vec![ - // Two weeks - header::CacheDirective::MaxAge(1_296_000), - ])); - request.headers_mut().insert( - "Authorization", - HeaderValue::from_str(&auth).expect("Invalid authentication"), - ); - *request.body_mut() = Some(content.into()); - - Box::new(self.client.execute(request).then(|result| match result { - Ok(res) => { - if res.status().is_success() { - trace!("PUT succeeded"); - Ok(()) - } else { - trace!("PUT failed with HTTP status: {}", res.status()); - Err(BadHttpStatusError(res.status()).into()) - } - } - Err(e) => { - trace!("PUT failed with error: {:?}", e); - Err(e.into()) - } - })) - } - - // http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html - #[allow(clippy::too_many_arguments)] - fn auth( - &self, - verb: &str, - date: &str, - path: &str, - md5: &str, - headers: &str, - content_type: &str, - creds: &AwsCredentials, - ) -> String { - let string = format!( - "{verb}\n{md5}\n{ty}\n{date}\n{headers}{resource}", - verb = verb, - md5 = md5, - ty = content_type, - date = date, - headers = headers, - resource = format!("/{}/{}", self.name, path) - ); - let signature = signature(&string, creds.aws_secret_access_key()); - format!("AWS {}:{}", creds.aws_access_key_id(), signature) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_signature() { - assert_eq!( - signature("/foo/bar\nbar", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), - "mwbstmHPMEJjTe2ksXi5H5f0c8U=" - ); - - assert_eq!( - signature("/bar/foo\nbaz", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), - "F9gZMso3+P+QTEyRKQ6qhZ1YM6o=" - ); - } -} diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index 0943f74d6..c3a1244a0 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -294,8 +294,14 @@ impl DistSystem { wait_for( || { let status = self.scheduler_status(); - if matches!(self.scheduler_status(), SchedulerStatusResult { num_servers: 0, num_cpus: _, in_progress: 0 }) - { + if matches!( + self.scheduler_status(), + SchedulerStatusResult { + num_servers: 0, + num_cpus: _, + in_progress: 0 + } + ) { Ok(()) } else { Err(format!("{:?}", status)) @@ -428,8 +434,14 @@ impl DistSystem { wait_for( || { let status = self.scheduler_status(); - if matches!(self.scheduler_status(), SchedulerStatusResult { num_servers: 1, num_cpus: _, in_progress: 0 }) - { + if matches!( + self.scheduler_status(), + SchedulerStatusResult { + num_servers: 1, + num_cpus: _, + in_progress: 0 + } + ) { Ok(()) } else { Err(format!("{:?}", status))