From d8977f996365c9fe11a0ab419caaab16c5f0edc8 Mon Sep 17 00:00:00 2001 From: Denis Hovart Date: Thu, 9 Dec 2021 15:14:20 -0300 Subject: [PATCH 1/5] Use sccache with prepolated compilation cache --- Dockerfile | 24 +++++++++++++++++++++++- local-registry/{dummry.rs => dummy.rs} | 0 2 files changed, 23 insertions(+), 1 deletion(-) rename local-registry/{dummry.rs => dummy.rs} (100%) diff --git a/Dockerfile b/Dockerfile index 08ef844..58789c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,19 @@ FROM rust:1.71.0 as build ARG JQ_VERSION=1.6 ARG JQ_URL=https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 +ARG SC_VERSION=v0.2.15 +ARG SC_URL=https://github.com/mozilla/sccache/releases/download/${SC_VERSION}/sccache-${SC_VERSION}-x86_64-unknown-linux-musl.tar.gz + +ENV SCCACHE_DIR=/opt/compilation-cache +ENV RUSTC_WRAPPER=/usr/local/bin/sccache + +# download sccache +RUN curl -L -o sccache.tgz "${SC_URL}" \ + && tar xvzf sccache.tgz --strip-components=1 \ + && cp sccache ${RUSTC_WRAPPER} \ + && chmod +x ${RUSTC_WRAPPER} +RUN mkdir -p ${SCCACHE_DIR} + # install cargo-local-registry dependencies RUN apt-get update && apt-get install -y gcc openssl cmake @@ -12,6 +25,7 @@ RUN mkdir -p /rust-test-runner/src ENV wd /rust-test-runner WORKDIR ${wd} COPY Cargo.* ./ + # for caching, we want to download and build all the dependencies before copying # any of the real source files. We therefore build an empty dummy library, # then remove it. @@ -32,6 +46,9 @@ WORKDIR /local-registry COPY local-registry/* ./ RUN cargo generate-lockfile && cargo local-registry --sync Cargo.lock . +# populate compilation cache +RUN cargo build + # As of Dec 2019, we need to use the nightly toolchain to get JSON test output # tracking issue: https://github.com/rust-lang/rust/issues/49359 @@ -87,12 +104,17 @@ RUN set -eux; \ ################ end-copy-pasta ################ +ENV SCCACHE_DIR=/opt/compilation-cache +ENV RUSTC_WRAPPER=/usr/local/bin/sccache +COPY --from=build ${RUSTC_WRAPPER} ${RUSTC_WRAPPER} +COPY --from=build ${SCCACHE_DIR} ${SCCACHE_DIR} + ENV wd /opt/test-runner RUN mkdir -p ${wd}/bin WORKDIR ${wd} COPY --from=build /rust-test-runner/target/release/transform-output bin COPY --from=build /usr/local/bin/jq /usr/local/bin -# configure local-registry +# configure local registry COPY --from=build /local-registry local-registry/ RUN echo '[source.crates-io]\n\ registry = "https://github.com/rust-lang/crates.io-index"\n\ diff --git a/local-registry/dummry.rs b/local-registry/dummy.rs similarity index 100% rename from local-registry/dummry.rs rename to local-registry/dummy.rs From e8eeea0e0ea44c152c9bfa6c4a5e7b37b26510c8 Mon Sep 17 00:00:00 2001 From: Remo Senekowitsch Date: Sun, 23 Jul 2023 20:49:00 +0200 Subject: [PATCH 2/5] Update sccache version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 58789c2..16c75fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ FROM rust:1.71.0 as build ARG JQ_VERSION=1.6 ARG JQ_URL=https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 -ARG SC_VERSION=v0.2.15 +ARG SC_VERSION=v0.5.4 ARG SC_URL=https://github.com/mozilla/sccache/releases/download/${SC_VERSION}/sccache-${SC_VERSION}-x86_64-unknown-linux-musl.tar.gz ENV SCCACHE_DIR=/opt/compilation-cache From afa0e72d88efd5b11ede213a661b96807085f923 Mon Sep 17 00:00:00 2001 From: Remo Senekowitsch Date: Sun, 23 Jul 2023 20:59:18 +0200 Subject: [PATCH 3/5] Trim the test image Populating the compilation cache would create a larget target folder (1.8 GB) in local-registry/ which would get copied over to the test image accidentally. That made the image balloon to over 3 GB. --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 16c75fc..56ea6f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,6 +48,8 @@ RUN cargo generate-lockfile && cargo local-registry --sync Cargo.lock . # populate compilation cache RUN cargo build +# but remove large target folder +RUN cargo clean # As of Dec 2019, we need to use the nightly toolchain to get JSON test output # tracking issue: https://github.com/rust-lang/rust/issues/49359 From e0a16c635eae650286b814ad9e51ffcdc2829234 Mon Sep 17 00:00:00 2001 From: Remo Senekowitsch Date: Tue, 25 Jul 2023 14:06:06 +0200 Subject: [PATCH 4/5] Extract rust-nightly build stage For sccache to work, its cache must be populated with the same toolchain that will use it later. An additional docker build stage is a simple solution to make sure the build stage uses the same toolchain as the final stage. --- Dockerfile | 117 ++++++++++++++++++++++++++++------------------------- 1 file changed, 63 insertions(+), 54 deletions(-) diff --git a/Dockerfile b/Dockerfile index 56ea6f5..af0d5e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,56 +1,3 @@ -# always build this using the latest stable release -FROM rust:1.71.0 as build - - -ARG JQ_VERSION=1.6 -ARG JQ_URL=https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 - -ARG SC_VERSION=v0.5.4 -ARG SC_URL=https://github.com/mozilla/sccache/releases/download/${SC_VERSION}/sccache-${SC_VERSION}-x86_64-unknown-linux-musl.tar.gz - -ENV SCCACHE_DIR=/opt/compilation-cache -ENV RUSTC_WRAPPER=/usr/local/bin/sccache - -# download sccache -RUN curl -L -o sccache.tgz "${SC_URL}" \ - && tar xvzf sccache.tgz --strip-components=1 \ - && cp sccache ${RUSTC_WRAPPER} \ - && chmod +x ${RUSTC_WRAPPER} -RUN mkdir -p ${SCCACHE_DIR} - -# install cargo-local-registry dependencies -RUN apt-get update && apt-get install -y gcc openssl cmake - -RUN mkdir -p /rust-test-runner/src -ENV wd /rust-test-runner -WORKDIR ${wd} -COPY Cargo.* ./ - -# for caching, we want to download and build all the dependencies before copying -# any of the real source files. We therefore build an empty dummy library, -# then remove it. -RUN echo '// dummy file' > src/lib.rs -RUN cargo build -# now get rid of the stub and copy the real source files -RUN rm src/lib.rs -COPY src/* src/ -# build the executable -RUN cargo build --release -# download jq -RUN curl -L -o /usr/local/bin/jq "${JQ_URL}" \ - && chmod +x /usr/local/bin/jq -# install cargo-local-registry -RUN cargo install cargo-local-registry -# download popular crates to local registry -WORKDIR /local-registry -COPY local-registry/* ./ -RUN cargo generate-lockfile && cargo local-registry --sync Cargo.lock . - -# populate compilation cache -RUN cargo build -# but remove large target folder -RUN cargo clean - # As of Dec 2019, we need to use the nightly toolchain to get JSON test output # tracking issue: https://github.com/rust-lang/rust/issues/49359 @@ -64,7 +11,7 @@ RUN cargo clean ################ start-copy-pasta ################ -FROM debian:bookworm-slim +FROM debian:bookworm-slim as rust-nightly ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ @@ -106,6 +53,68 @@ RUN set -eux; \ ################ end-copy-pasta ################ +# The build stage must use the same compiler version as the test runner itself, +# otherwise sccache will not work. Compilation artifacts from different versions +# of the Rust compiler are incompatible. +FROM rust-nightly as build + +ARG JQ_VERSION=1.6 +ARG JQ_URL=https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 + +ARG SC_VERSION=v0.5.4 +ARG SC_URL=https://github.com/mozilla/sccache/releases/download/${SC_VERSION}/sccache-${SC_VERSION}-x86_64-unknown-linux-musl.tar.gz + +ENV SCCACHE_DIR=/opt/compilation-cache +ENV RUSTC_WRAPPER=/usr/local/bin/sccache + +# install basic things missing in debian-slim +RUN apt-get update && apt-get install -y curl + +# download sccache +RUN curl -L -o sccache.tgz "${SC_URL}" \ + && tar xvzf sccache.tgz --strip-components=1 \ + && cp sccache ${RUSTC_WRAPPER} \ + && chmod +x ${RUSTC_WRAPPER} +RUN mkdir -p ${SCCACHE_DIR} + +# install cargo-local-registry dependencies +RUN apt-get update && apt-get install -y \ + gcc openssl cmake \ + # needed due to use of debian-slim. regular debian has these already. + libssl-dev pkg-config m4 + +RUN mkdir -p /rust-test-runner/src +ENV wd /rust-test-runner +WORKDIR ${wd} +COPY Cargo.* ./ + +# for caching, we want to download and build all the dependencies before copying +# any of the real source files. We therefore build an empty dummy library, +# then remove it. +RUN echo '// dummy file' > src/lib.rs +RUN cargo build +# now get rid of the stub and copy the real source files +RUN rm src/lib.rs +COPY src/* src/ +# build the executable +RUN cargo build --release +# download jq +RUN curl -L -o /usr/local/bin/jq "${JQ_URL}" \ + && chmod +x /usr/local/bin/jq +# install cargo-local-registry +RUN cargo install cargo-local-registry +# download popular crates to local registry +WORKDIR /local-registry +COPY local-registry/* ./ +RUN cargo generate-lockfile && cargo local-registry --sync Cargo.lock . + +# populate compilation cache +RUN cargo build +# but remove large target folder +RUN cargo clean + +FROM rust-nightly + ENV SCCACHE_DIR=/opt/compilation-cache ENV RUSTC_WRAPPER=/usr/local/bin/sccache COPY --from=build ${RUSTC_WRAPPER} ${RUSTC_WRAPPER} From baaaaecdf9497ad94c806773ce541b8c2a7c6e78 Mon Sep 17 00:00:00 2001 From: Remo Senekowitsch Date: Tue, 25 Jul 2023 15:35:01 +0200 Subject: [PATCH 5/5] Fix compilation cache by populating the cache in the same environment as the test runner will be executed in. Most importantly, this means the cache will be populated with the depencencies fetched from the previously generated local registry. This is important because sccache only works for dependencies if they are located at the same absolute path. Otherwise, cache requests to compile a solution are all misses, which even slows the test runner down. See sccache's documentation about known caveats: https://github.com/mozilla/sccache/tree/7074753bfc100ee3e22dc730cf71f3750fc42c1d#known-caveats --- Dockerfile | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index af0d5e8..9d82bbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,7 +81,7 @@ RUN mkdir -p ${SCCACHE_DIR} RUN apt-get update && apt-get install -y \ gcc openssl cmake \ # needed due to use of debian-slim. regular debian has these already. - libssl-dev pkg-config m4 + libssl-dev pkg-config RUN mkdir -p /rust-test-runner/src ENV wd /rust-test-runner @@ -107,18 +107,18 @@ RUN cargo install cargo-local-registry WORKDIR /local-registry COPY local-registry/* ./ RUN cargo generate-lockfile && cargo local-registry --sync Cargo.lock . +# only keep the local registry +RUN rm Cargo.* dummy.rs -# populate compilation cache -RUN cargo build -# but remove large target folder -RUN cargo clean - -FROM rust-nightly +# The environment should be as identical as possible between populating and +# using the compilation cache, so we put this environment in a separate build +# stage that can be shared. +FROM rust-nightly as test-env ENV SCCACHE_DIR=/opt/compilation-cache ENV RUSTC_WRAPPER=/usr/local/bin/sccache COPY --from=build ${RUSTC_WRAPPER} ${RUSTC_WRAPPER} -COPY --from=build ${SCCACHE_DIR} ${SCCACHE_DIR} +RUN mkdir -p ${SCCACHE_DIR} ENV wd /opt/test-runner RUN mkdir -p ${wd}/bin @@ -133,6 +133,16 @@ RUN echo '[source.crates-io]\n\ \n\ [source.local-registry]\n\ local-registry = "/opt/test-runner/local-registry/"\n' >> $CARGO_HOME/config.toml -# set entrypoint + +# populate compilation cache in the same environment as the tests will run +FROM test-env AS cache +RUN apt-get update && apt-get install -y m4 +WORKDIR /local-registry +COPY local-registry/* ./ +RUN cargo build + +# final stage +FROM test-env +COPY --from=cache ${SCCACHE_DIR} ${SCCACHE_DIR} COPY bin/run.sh bin ENTRYPOINT ["bin/run.sh"]