2.4.1 beta backports (#10471)

* version: bump beta

* Implement parity_versionInfo & parity_setChain on LC; fix parity_setChain (#10312)

* Light client: implement parity_versionInfo RPC

* Light client: implement set_exit_handler & parity_setChain RPC

* parity_setChain RPC: return an error if failed (instead of `true`)

* Implement eth_subscribe('syncing') RPC for full node & light node

* Fix indentation

* Revert commit: Implement eth_subscribe('syncing')

* Revert change to Cr callback function

* CI publish to aws (#10446)

* move publish aws from gitlab.yml to gitlab scripts

* gitlab.yml cleaning
move publish AWS to gitlab scripts
remove dependencies from android build

* CI aws git checkout (#10451)

* Updating the CI system with the publication of releases and binary files on github

Signed-off-by: Denis S. Soldatov aka General-Beck <general.beck@gmail.com>

* move publish aws from gitlab.yml to gitlab scripts

Signed-off-by: Denis S. Soldatov aka General-Beck <general.beck@gmail.com>

* gitlab.yml cleaning
move publish AWS to gitlab scripts
remove dependencies from android build

Signed-off-by: Denis S. Soldatov aka General-Beck <general.beck@gmail.com>

* Revert "Updating the CI system with the publication of releases and binary files on github"

This reverts commit da87e06.

* remove no-git for aws

* microfix

* no need in no_git then

* Revert "CI aws git checkout (#10451)" (#10456)

* Revert "CI aws git checkout (#10451)"

This reverts commit 3e1d731.

* Update .gitlab-ci.yml

revert aws script with small fixes

* Delete publish-aws.sh

* Tests parallelized (#10452)

* tests splitted, phase 1

* typo

* fix wrong launch commands

* typos

* rearrangements

* use `nproc` function for threads

* use nproc for threads

* let theads be auto, build-andriod no more in regular run

* split val chain and cargo check

* renamed some files

* wrong phase

* check rust files before test jobs

* lint error

* rust files modivied var

* test except changes

* add rust_changes except

* lint error

* fixes

* .gitlab-ci.yml can't be excluded

* pipeline shouldn't start

* pipeline must go

* pipeline must go 2

* pipeline must go 3

* pipeline must go 4

* pipeline must go 5

* pipeline must go 6

* pipeline must go 7

* pipeline must not go 1

* pipeline must go 8

* avoid skippng tests yet, reintroducing them after the caching

* test theory

* parallelized cargo check with combusting helicopters

* less uploads

* alias for cargo checks

* nice template

* Ensure static validator set changes are recognized (#10467)

Author: soc1c

.gitlab-ci.yml

@@ -12,20 +12,18 @@ variables:
   CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"
   CARGO_TARGET:                    x86_64-unknown-linux-gnu
 
-.no_git:                           &no_git
+.no_git:                           &no_git #disable git strategy
   variables:
     GIT_STRATEGY:                  none
     GIT_SUBMODULE_STRATEGY:        none
 
-
 .releaseable_branches:             # list of git refs for building GitLab artifacts (think "pre-release binaries")
   only:                            &releaseable_branches
     - stable
     - beta
     - tags
     - schedules
 
-
 .collect_artifacts:                &collect_artifacts
   artifacts:
     name:                          "${CI_JOB_NAME}_${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}"
@@ -34,34 +32,66 @@ variables:
     paths:
       - artifacts/
 
-test-linux:
-  stage:                           test
-  variables:
-    RUN_TESTS:                     all
-  script:
-    - scripts/gitlab/test-all.sh
+.docker-cache-status:                      &docker-cache-status
+  dependencies:                    []
+  before_script:
+    - sccache -s
+  after_script:
     - sccache -s
   tags:
     - linux-docker
 
-test-audit:
+
+cargo-check 0 3:
+  stage:                           test
+  <<:                              *docker-cache-status
+  script:
+    - time cargo check --target $CARGO_TARGET --locked --no-default-features
+
+cargo-check 1 3:
+  stage:                           test
+  <<:                              *docker-cache-status
+  script:
+    - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features
+
+cargo-check 2 3:
+  stage:                           test
+  <<:                              *docker-cache-status
+  script:
+    - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio"
+
+cargo-audit:
   stage:                           test
   script:
-    - set -e
-    - set -u
     - cargo audit
   tags:
     - linux-docker
 
+validate-chainspecs:
+  stage:                           test
+  <<:                              *docker-cache-status
+  script:
+    - ./scripts/gitlab/validate-chainspecs.sh
+
+test-cpp:
+  stage:                           build
+  <<:                              *docker-cache-status
+  script:
+    - ./scripts/gitlab/test-cpp.sh
+
+test-linux:
+  stage:                           build
+  <<:                              *docker-cache-status
+  script:
+    - ./scripts/gitlab/test-linux.sh
+
 build-linux:                       &build-linux
   stage:                           build
   only:                            *releaseable_branches
+  <<:                              *docker-cache-status
   script:
-    - scripts/gitlab/build-unix.sh
-    - sccache -s
+    - scripts/gitlab/build-linux.sh
   <<:                              *collect_artifacts
-  tags:
-    - linux-docker
 
 build-linux-i386:
   <<:                              *build-linux
@@ -89,7 +119,7 @@ build-darwin:
     CC:                            gcc
     CXX:                           g++
   script:
-    - scripts/gitlab/build-unix.sh
+    - scripts/gitlab/build-linux.sh
   tags:
     - rust-osx
   <<:                              *collect_artifacts
@@ -102,7 +132,7 @@ build-windows:
   script:
     - sh scripts/gitlab/build-windows.sh
   tags:
-   - rust-windows
+    - rust-windows
   <<:                              *collect_artifacts
 
 publish-docker:
@@ -178,17 +208,16 @@ publish-awss3-release:
   script:
     - echo "__________Push binaries to AWS S3____________"
     - case "${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" in
-        (beta|stable|nightly)
-          export BUCKET=releases.parity.io/ethereum;
-          ;;
-        (*)
-          export BUCKET=builds-parity;
-          ;;
-      esac
+       (beta|stable|nightly)
+         export BUCKET=releases.parity.io/ethereum;
+         ;;
+       (*)
+         export BUCKET=builds-parity;
+         ;;
+     esac
     - aws s3 sync ./artifacts s3://${BUCKET}/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/
-  after_script:
-    - aws s3 ls s3://${BUCKET}/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/
-        --recursive --human-readable --summarize
+    - echo "__________Read from S3____________"
+    - aws s3 ls s3://${BUCKET}/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}} --recursive --human-readable --summarize
   tags:
     - linux-docker
 
@@ -206,13 +235,12 @@ publish-docs:
     - linux-docker
 
 build-android:
-  stage:                           optional
+  stage:                           build
   image:                           parity/rust-android:gitlab-ci
   variables:
     CARGO_TARGET:                  armv7-linux-androideabi
   script:
-    - scripts/gitlab/build-unix.sh
+    - scripts/gitlab/build-linux.sh
   tags:
     - linux-docker
-  allow_failure:                   true
   <<:                              *collect_artifacts

Cargo.lock

@@ -2,7 +2,7 @@
 description = "Parity Ethereum client"
 name = "parity-ethereum"
 # NOTE Make sure to update util/version/Cargo.toml as well
-version = "2.4.0"
+version = "2.4.1"
 license = "GPL-3.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 

Cargo.toml

@@ -116,6 +116,9 @@ pub trait LightChainClient: Send + Sync {
 	/// Query whether a block is known.
 	fn is_known(&self, hash: &H256) -> bool;
 
+	/// Set the chain via a spec name.
+	fn set_spec_name(&self, new_spec_name: String) -> Result<(), ()>;
+
 	/// Clear the queue.
 	fn clear_queue(&self);
 
@@ -164,6 +167,8 @@ pub struct Client<T> {
 	listeners: RwLock<Vec<Weak<LightChainNotify>>>,
 	fetcher: T,
 	verify_full: bool,
+	/// A closure to call when we want to restart the client
+	exit_handler: Mutex<Option<Box<Fn(String) + 'static + Send>>>,
 }
 
 impl<T: ChainDataFetcher> Client<T> {
@@ -190,6 +195,7 @@ impl<T: ChainDataFetcher> Client<T> {
 			listeners: RwLock::new(vec![]),
 			fetcher,
 			verify_full: config.verify_full,
+			exit_handler: Mutex::new(None),
 		})
 	}
 
@@ -360,6 +366,14 @@ impl<T: ChainDataFetcher> Client<T> {
 		self.chain.heap_size_of_children()
 	}
 
+	/// Set a closure to call when the client wants to be restarted.
+	///
+	/// The parameter passed to the callback is the name of the new chain spec to use after
+	/// the restart.
+	pub fn set_exit_handler<F>(&self, f: F) where F: Fn(String) + 'static + Send {
+		*self.exit_handler.lock() = Some(Box::new(f));
+	}
+
 	/// Get a handle to the verification engine.
 	pub fn engine(&self) -> &Arc<EthEngine> {
 		&self.engine
@@ -563,6 +577,17 @@ impl<T: ChainDataFetcher> LightChainClient for Client<T> {
 		Client::engine(self)
 	}
 
+	fn set_spec_name(&self, new_spec_name: String) -> Result<(), ()> {
+		trace!(target: "mode", "Client::set_spec_name({:?})", new_spec_name);
+		if let Some(ref h) = *self.exit_handler.lock() {
+			(*h)(new_spec_name);
+			Ok(())
+		} else {
+			warn!("Not hypervised; cannot change chain.");
+			Err(())
+		}
+	}
+
 	fn is_known(&self, hash: &H256) -> bool {
 		self.status(hash) == BlockStatus::InChain
 	}

ethcore/light/src/client/mod.rs

@@ -1706,15 +1706,17 @@ impl BlockChainClient for Client {
 		self.config.spec_name.clone()
 	}
 
-	fn set_spec_name(&self, new_spec_name: String) {
+	fn set_spec_name(&self, new_spec_name: String) -> Result<(), ()> {
 		trace!(target: "mode", "Client::set_spec_name({:?})", new_spec_name);
 		if !self.enabled.load(AtomicOrdering::Relaxed) {
-			return;
+			return Err(());
 		}
 		if let Some(ref h) = *self.exit_handler.lock() {
 			(*h)(new_spec_name);
+			Ok(())
 		} else {
 			warn!("Not hypervised; cannot change chain.");
+			Err(())
 		}
 	}
 

ethcore/src/client/client.rs

@@ -863,7 +863,7 @@ impl BlockChainClient for TestBlockChainClient {
 
 	fn spec_name(&self) -> String { "foundation".into() }
 
-	fn set_spec_name(&self, _: String) { unimplemented!(); }
+	fn set_spec_name(&self, _: String) -> Result<(), ()> { unimplemented!(); }
 
 	fn disable(&self) { self.disabled.store(true, AtomicOrder::Relaxed); }
 

ethcore/src/client/test_client.rs

@@ -360,7 +360,7 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra
 	fn spec_name(&self) -> String;
 
 	/// Set the chain via a spec name.
-	fn set_spec_name(&self, spec_name: String);
+	fn set_spec_name(&self, spec_name: String) -> Result<(), ()>;
 
 	/// Disable the client from importing blocks. This cannot be undone in this session and indicates
 	/// that a subsystem has reason to believe this executable incapable of syncing the chain.

ethcore/src/client/traits.rs

@@ -1405,8 +1405,10 @@ impl Engine<EthereumMachine> for AuthorityRound {
 
 		let first = chain_head.number() == 0;
 
-		// apply immediate transitions.
+		// Apply transitions that don't require finality and should be enacted immediately (e.g from chain spec)
 		if let Some(change) = self.validators.is_epoch_end(first, chain_head) {
+			info!(target: "engine", "Immediately applying validator set change signalled at block {}", chain_head.number());
+			self.epoch_manager.lock().note_new_epoch();
 			let change = combine_proofs(chain_head.number(), &change, &[]);
 			return Some(change)
 		}

ethcore/src/engines/authority_round/mod.rs

@@ -634,7 +634,7 @@ impl<C: LightChainClient + 'static> LightDependencies<C> {
 					handler.extend_with(ParityAccounts::to_delegate(ParityAccountsClient::new(&self.accounts)));
 				}
 				Api::ParitySet => handler.extend_with(
-					light::ParitySetClient::new(self.sync.clone(), self.fetch.clone())
+					light::ParitySetClient::new(self.client.clone(), self.sync.clone(), self.fetch.clone())
 						.to_delegate(),
 				),
 				Api::Traces => handler.extend_with(light::TracesClient.to_delegate()),

parity/rpc_apis.rs

@@ -165,7 +165,9 @@ impl ::local_store::NodeInfo for FullNodeInfo {
 type LightClient = ::light::client::Client<::light_helpers::EpochFetch>;
 
 // helper for light execution.
-fn execute_light_impl(cmd: RunCmd, logger: Arc<RotatingLogger>) -> Result<RunningClient, String> {
+fn execute_light_impl<Cr>(cmd: RunCmd, logger: Arc<RotatingLogger>, on_client_rq: Cr) -> Result<RunningClient, String>
+	where Cr: Fn(String) + 'static + Send
+{
 	use light::client as light_client;
 	use sync::{LightSyncParams, LightSync, ManageNetwork};
 	use parking_lot::{Mutex, RwLock};
@@ -367,6 +369,8 @@ fn execute_light_impl(cmd: RunCmd, logger: Arc<RotatingLogger>) -> Result<Runnin
 	service.add_notify(informant.clone());
 	service.register_handler(informant.clone()).map_err(|_| "Unable to register informant handler".to_owned())?;
 
+	client.set_exit_handler(on_client_rq);
+
 	Ok(RunningClient {
 		inner: RunningClientInner::Light {
 			rpc: rpc_direct,
@@ -930,7 +934,7 @@ pub fn execute<Cr, Rr>(cmd: RunCmd, logger: Arc<RotatingLogger>,
 		Rr: Fn() + 'static + Send
 {
 	if cmd.light {
-		execute_light_impl(cmd, logger)
+		execute_light_impl(cmd, logger, on_client_rq)
 	} else {
 		execute_impl(cmd, logger, on_client_rq, on_updater_rq)
 	}