2.4.2 beta backports (#10488)

* version: bump beta

* Сaching through docker volume (#10477)

* _old codebase_ before docker update

* before docker update, testing runnr

* docker update, testing the caching

* distributed job cargo homes

* distributed job cargo homes 2

* distributed job cargo homes 3

* dockerfile with gitlab checkout, audit uses template

* dockerfile gets repo in volume

* change builds_dir

* trying docker cache for repo

* repo cached automatically

* after script is not concatenated

* check sccache non-cacheable reasons nature

* watch cache

* log sccache

* log sccache 2

* debug log sccache

* fix debug log sccache

* fix debug log sccache 2

* debug log cache 3

* debug log cache 3

* trace log all sccache

* test wo cargo cache

* test w removed cargo cache

* report non-cacheable reasons, cargo cache is back and empty

* report non-cacheable reasons, cargo cache is back and empty 2

* report non-cacheable reasons, cargo cache is back and empty 3

* wrap into after_script

* restore CI tags

`qa` -> `linux-docker`

* return to main runners, this will fail until config on runners And Dockerfile won't be updated

* typo fix CI lint

* return to docker tag

* fix win&mac build (#10486)

add CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"

* fix(extract `timestamp_checked_add` as lib) (#10383)

* fix(extract `timestamp_checked_add` as lib)

* fix(whisper types): remove unused `EmptyTopics`

* fix(time-lib): feature-flag to use time-lib or std

This commit adds conditional compilation checks that falls back to `our time-lib` when
`time_checked_add` is not available in the standard library

Note, `time_checked_add` covers both `checked_add` and `checked_sub`

* fix(grumble): use cfg_attr to define rustc feature

Author: soc1c

.gitlab-ci.yml

@@ -9,10 +9,9 @@ variables:
   GIT_STRATEGY:                    fetch
   GIT_SUBMODULE_STRATEGY:          recursive
   CI_SERVER_NAME:                  "GitLab CI"
-  CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"
   CARGO_TARGET:                    x86_64-unknown-linux-gnu
 
-.no_git:                           &no_git #disable git strategy
+.no_git:                           &no_git # disable git strategy
   variables:
     GIT_STRATEGY:                  none
     GIT_SUBMODULE_STRATEGY:        none
@@ -32,12 +31,17 @@ variables:
     paths:
       - artifacts/
 
-.docker-cache-status:                      &docker-cache-status
-  dependencies:                    []
+.docker-cache-status:              &docker-cache-status
+  variables:
+    CARGO_HOME:                    "/cargo/${CI_JOB_NAME}"
   before_script:
+    - SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_error.log RUST_LOG=sccache::server=debug sccache --start-server
     - sccache -s
   after_script:
-    - sccache -s
+    - echo "All crate-types:"
+    - grep 'parse_arguments.*--crate-type' sccache_error.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c
+    - echo "Non-cacheable reasons:"
+    - grep CannotCache sccache_error.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c
   tags:
     - linux-docker
 
@@ -47,50 +51,68 @@ cargo-check 0 3:
   <<:                              *docker-cache-status
   script:
     - time cargo check --target $CARGO_TARGET --locked --no-default-features
+    - sccache -s
 
 cargo-check 1 3:
   stage:                           test
   <<:                              *docker-cache-status
   script:
     - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features
+    - sccache -s
 
 cargo-check 2 3:
   stage:                           test
   <<:                              *docker-cache-status
   script:
     - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio"
+    - sccache -s
 
 cargo-audit:
   stage:                           test
+  <<:                              *docker-cache-status
   script:
     - cargo audit
-  tags:
-    - linux-docker
+    - sccache -s
 
 validate-chainspecs:
   stage:                           test
   <<:                              *docker-cache-status
   script:
     - ./scripts/gitlab/validate-chainspecs.sh
+    - sccache -s
 
 test-cpp:
   stage:                           build
   <<:                              *docker-cache-status
   script:
     - ./scripts/gitlab/test-cpp.sh
+    - sccache -s
 
 test-linux:
   stage:                           build
   <<:                              *docker-cache-status
   script:
     - ./scripts/gitlab/test-linux.sh
+    - sccache -s
+
+build-android:
+  stage:                           build
+  image:                           parity/rust-android:gitlab-ci
+  variables:
+    CARGO_TARGET:                  armv7-linux-androideabi
+  script:
+    - scripts/gitlab/build-linux.sh
+  tags:
+    - linux-docker
+  <<:                              *collect_artifacts
 
 build-linux:                       &build-linux
   stage:                           build
   only:                            *releaseable_branches
   <<:                              *docker-cache-status
   script:
     - scripts/gitlab/build-linux.sh
+    - sccache -s
   <<:                              *collect_artifacts
 
 build-linux-i386:
@@ -116,6 +138,7 @@ build-darwin:
   only:                            *releaseable_branches
   variables:
     CARGO_TARGET:                  x86_64-apple-darwin
+    CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"
     CC:                            gcc
     CXX:                           g++
   script:
@@ -129,6 +152,7 @@ build-windows:
   only:                            *releaseable_branches
   variables:
     CARGO_TARGET:                  x86_64-pc-windows-msvc
+    CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"
   script:
     - sh scripts/gitlab/build-windows.sh
   tags:
@@ -138,7 +162,7 @@ build-windows:
 publish-docker:
   stage:                           publish
   only:                            *releaseable_branches
-  cache: {}
+  cache:                           {}
   dependencies:
     - build-linux
   tags:
@@ -152,7 +176,7 @@ publish-snap:                      &publish-snap
   image:                           snapcore/snapcraft
   variables:
     BUILD_ARCH:                    amd64
-  cache: {}
+  cache:                           {}
   dependencies:
     - build-linux
   tags:
@@ -228,19 +252,9 @@ publish-docs:
     - tags
   except:
     - nightly
-  cache: {}
+  cache:                            {}
   script:
     - scripts/gitlab/publish-docs.sh
   tags:
     - linux-docker
 
-build-android:
-  stage:                           build
-  image:                           parity/rust-android:gitlab-ci
-  variables:
-    CARGO_TARGET:                  armv7-linux-androideabi
-  script:
-    - scripts/gitlab/build-linux.sh
-  tags:
-    - linux-docker
-  <<:                              *collect_artifacts

Cargo.lock

@@ -2,7 +2,7 @@
 description = "Parity Ethereum client"
 name = "parity-ethereum"
 # NOTE Make sure to update util/version/Cargo.toml as well
-version = "2.4.1"
+version = "2.4.2"
 license = "GPL-3.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 
@@ -138,7 +138,8 @@ members = [
 	"util/triehash-ethereum",
 	"util/keccak-hasher",
 	"util/patricia-trie-ethereum",
-	"util/fastmap"
+	"util/fastmap",
+	"util/time-utils"
 ]
 
 [patch.crates-io]

Cargo.toml

@@ -64,6 +64,7 @@ serde = "1.0"
 serde_derive = "1.0"
 stats = { path = "../util/stats" }
 tempdir = {version="0.3", optional = true}
+time-utils = { path = "../util/time-utils" }
 trace-time = "0.1"
 triehash-ethereum = { version = "0.2",  path = "../util/triehash-ethereum" }
 unexpected = { path = "../util/unexpected" }

ethcore/Cargo.toml

@@ -47,6 +47,9 @@ use types::header::{Header, ExtendedHeader};
 use types::ancestry_action::AncestryAction;
 use unexpected::{Mismatch, OutOfBounds};
 
+#[cfg(not(time_checked_add))]
+use time_utils::CheckedSystemTime;
+
 mod finality;
 
 /// `AuthorityRound` params.
@@ -570,8 +573,15 @@ fn verify_timestamp(step: &Step, header_step: u64) -> Result<(), BlockError> {
 			// NOTE This error might be returned only in early stage of verification (Stage 1).
 			// Returning it further won't recover the sync process.
 			trace!(target: "engine", "verify_timestamp: block too early");
-			let oob = oob.map(|n| SystemTime::now() + Duration::from_secs(n));
-			Err(BlockError::TemporarilyInvalid(oob).into())
+
+			let now = SystemTime::now();
+			let found = now.checked_add(Duration::from_secs(oob.found)).ok_or(BlockError::TimestampOverflow)?;
+			let max = oob.max.and_then(|m| now.checked_add(Duration::from_secs(m)));
+			let min = oob.min.and_then(|m| now.checked_add(Duration::from_secs(m)));
+
+			let new_oob = OutOfBounds { min, max, found };
+
+			Err(BlockError::TemporarilyInvalid(new_oob).into())
 		},
 		Ok(_) => Ok(()),
 	}
@@ -607,6 +617,7 @@ fn combine_proofs(signal_number: BlockNumber, set_proof: &[u8], finality_proof:
 	stream.out()
 }
 
+
 fn destructure_proofs(combined: &[u8]) -> Result<(BlockNumber, &[u8], &[u8]), Error> {
 	let rlp = Rlp::new(combined);
 	Ok((
@@ -622,7 +633,7 @@ trait AsMillis {
 
 impl AsMillis for Duration {
 	fn as_millis(&self) -> u64 {
-		self.as_secs()*1_000 + (self.subsec_nanos()/1_000_000) as u64
+		self.as_secs() * 1_000 + (self.subsec_nanos() / 1_000_000) as u64
 	}
 }
 

ethcore/src/engines/authority_round/mod.rs

@@ -15,6 +15,7 @@
 // along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
 
 #![warn(missing_docs, unused_extern_crates)]
+#![cfg_attr(feature = "time_checked_add", feature(time_checked_add))]
 
 //! Ethcore library
 //!
@@ -149,6 +150,9 @@ extern crate fetch;
 #[cfg(all(test, feature = "price-info"))]
 extern crate parity_runtime;
 
+#[cfg(not(time_checked_add))]
+extern crate time_utils;
+
 pub mod block;
 pub mod builtin;
 pub mod client;

ethcore/src/lib.rs

@@ -40,24 +40,8 @@ use types::{BlockNumber, header::Header};
 use types::transaction::SignedTransaction;
 use verification::queue::kind::blocks::Unverified;
 
-
-/// Returns `Ok<SystemTime>` when the result less or equal to `i32::max_value` to prevent `SystemTime` to panic because
-/// it is platform specific, may be i32 or i64.
-///
-/// `Err<BlockError::TimestampOver` otherwise.
-///
-// FIXME: @niklasad1 - remove this when and use `SystemTime::checked_add`
-// when https://github.com/rust-lang/rust/issues/55940 is stabilized.
-fn timestamp_checked_add(sys: SystemTime, d2: Duration) -> Result<SystemTime, BlockError> {
-	let d1 = sys.duration_since(UNIX_EPOCH).map_err(|_| BlockError::TimestampOverflow)?;
-	let total_time = d1.checked_add(d2).ok_or(BlockError::TimestampOverflow)?;
-
-	if total_time.as_secs() <= i32::max_value() as u64 {
-		Ok(sys + d2)
-	} else {
-		Err(BlockError::TimestampOverflow)
-	}
-}
+#[cfg(not(time_checked_add))]
+use time_utils::CheckedSystemTime;
 
 /// Preprocessed block data gathered in `verify_block_unordered` call
 pub struct PreverifiedBlock {
@@ -323,9 +307,11 @@ pub fn verify_header_params(header: &Header, engine: &EthEngine, is_full: bool,
 
 	if is_full {
 		const ACCEPTABLE_DRIFT: Duration = Duration::from_secs(15);
+		// this will resist overflow until `year 2037`
 		let max_time = SystemTime::now() + ACCEPTABLE_DRIFT;
 		let invalid_threshold = max_time + ACCEPTABLE_DRIFT * 9;
-		let timestamp = timestamp_checked_add(UNIX_EPOCH, Duration::from_secs(header.timestamp()))?;
+		let timestamp = UNIX_EPOCH.checked_add(Duration::from_secs(header.timestamp()))
+			.ok_or(BlockError::TimestampOverflow)?;
 
 		if timestamp > invalid_threshold {
 			return Err(From::from(BlockError::InvalidTimestamp(OutOfBounds { max: Some(max_time), min: None, found: timestamp })))
@@ -347,8 +333,11 @@ fn verify_parent(header: &Header, parent: &Header, engine: &EthEngine) -> Result
 	let gas_limit_divisor = engine.params().gas_limit_bound_divisor;
 
 	if !engine.is_timestamp_valid(header.timestamp(), parent.timestamp()) {
-		let min = timestamp_checked_add(SystemTime::now(), Duration::from_secs(parent.timestamp().saturating_add(1)))?;
-		let found = timestamp_checked_add(SystemTime::now(), Duration::from_secs(header.timestamp()))?;
+		let now = SystemTime::now();
+		let min = now.checked_add(Duration::from_secs(parent.timestamp().saturating_add(1)))
+			.ok_or(BlockError::TimestampOverflow)?;
+		let found = now.checked_add(Duration::from_secs(header.timestamp()))
+			.ok_or(BlockError::TimestampOverflow)?;
 		return Err(From::from(BlockError::InvalidTimestamp(OutOfBounds { max: None, min: Some(min), found })))
 	}
 	if header.number() != parent.number() + 1 {
@@ -835,11 +824,4 @@ mod tests {
 		check_fail(unordered_test(&create_test_block_with_data(&header, &bad_transactions, &[]), &engine), TooManyTransactions(keypair.address()));
 		unordered_test(&create_test_block_with_data(&header, &good_transactions, &[]), &engine).unwrap();
 	}
-
-	#[test]
-	fn checked_add_systime_dur() {
-		assert!(timestamp_checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64 + 1, 0)).is_err());
-		assert!(timestamp_checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64, 0)).is_ok());
-		assert!(timestamp_checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64 - 1, 1_000_000_000)).is_ok());
-	}
 }