Skip to content

Commit fb66875

Browse files
author
Ethernode.io
authored
sync.js fixes (#1)
* Testing commit to new branch * Correct function variable token * config.js to hold file read/write configs. * forgot require for testing. * fs include for config.js * call web3 * fix * require * test commit ignore * test * test * exported syncChain function * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test new sync memory json file * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test for 0 * test * test * test * test * test * test * Last doub insert test * test * test * New vars * broekn func * rest * block data * limit test * minus dwa * minus jeden * quite stuff * fix brackets * fix * test * quite * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * Patcher test * test * patch test * test * patcher test * test * test * test * fix startBlock config * patcher test * test * test * test * FInal patcher test * Patcher Test #2,000,000 * patch test #2,000,001 * test # two million seven hundred thousand four hundred seventy five * test # two million seven hundred thousand four hundred seventy six * test # two million seven hundred thousand four hundred seventy seve * test # two million seven hundred thousand four hundred seventy eight * test # two million seven hundred thousand four hundred seventy nine * test # two million seven hundred thousand four hundred eighty * test # two million seven hundred thousand four hundred eighty one * test # two million seven hundred thousand four hundred eighty two * test # two million seven hundred thousand four hundred eighty three * Default conf
1 parent 447fb89 commit fb66875

File tree

3 files changed

+142
-126
lines changed

3 files changed

+142
-126
lines changed

‎README.md‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,25 +31,26 @@ Configuration file: `/conf.json`
3131
Basic settings:
3232
```json
3333
{
34-
"nodeAddr": "localhost",
35-
"gethPort": 8545,
36-
"startBlock": 0,
37-
"endBlock": 0,
38-
"lastSynced": 0,
39-
"syncAll": true,
40-
"patch": false
41-
"quiet": false,
34+
"nodeAddr": "localhost",
35+
"gethPort": 8545,
36+
"startBlock": 0,
37+
"endBlock": "latest",
38+
"quiet": true,
39+
"syncAll": true,
40+
"patch": false,
41+
"patchBlocks": 0
4242
}
4343
```
4444

4545
```nodeAddr``` Your node API RPC address.
4646
```gethPort``` Your node API RPC port.
4747
```startBLock``` This is the start block of the blockchain, should always be 0 if you want to sync the whole ETC blockchain.
4848
```endBlock```` This is usually the 'latest'/'newest' block in the blockchain, this value gets updated automatically, and will be used to patch missing blocks if the whole app goes down.
49-
```lastSynced``` If `syncAll` is set to true, the sync will record the last block it has written to the db, this will be automatically updated and used when sync.js prematurely exits.
50-
```syncAll``` If this is set to true at the start of the app, the sync will start syncing all blocks from lastSync, and if lastSync is 0 it will start from whatever the endBlock or latest block in the blockchain is.
51-
```patch``` This is not used yet but it will be directly corolated with the startBlock param.
5249
```quiet``` Prints out the log of what it is doing, Might not work for all messages in this release
50+
```syncAll``` If this is set to true at the start of the app, the sync will start syncing all blocks from lastSync, and if lastSync is 0 it will start from whatever the endBlock or latest block in the blockchain is.
51+
```patch``` If set to true and below value is set, sync will iterated through the # of blocks specified
52+
```patchBlocks``` If `patch` is set to true, the amount of block specified will be check from the latest one.
53+
5354

5455
### Run:
5556
If you run

‎conf.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
"gethPort": 8545,
44
"startBlock": 0,
55
"endBlock": "latest",
6-
"lastSynced": 0,
7-
"quiet": false,
6+
"quiet": true,
87
"syncAll": true,
9-
"patch": false
8+
"patch": false,
9+
"patchBlocks": 0
1010
}

‎tools/sync.js‎

Lines changed: 127 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
Name: Ethereum Blockchain syncer
3-
Version: .0.0.1
3+
Version: .0.0.2
44
This file will start syncing the blockchain from the node address you provide in the conf.json file.
55
Please read the README in the root directory that explains the parameters of this code
66
*/
@@ -19,90 +19,84 @@ var Transaction = mongoose.model( 'Transaction' );
1919
//Just listen for latest blocks and sync from the start of the app.
2020
**/
2121
var listenBlocks = function(config) {
22-
var options = {
23-
logIndex: "Number",
24-
fromBlock: "latest",
25-
};
2622
var newBlocks = web3.eth.filter("latest");
27-
newBlocks.watch(function (error, blockHashOrNumber) {
23+
newBlocks.watch(function (error,latestBlock) {
2824
if(error) {
2925
console.log('Error: ' + error);
30-
} else if (blockHashOrNumber == null) {
26+
} else if (latestBlock == null) {
3127
console.log('Warning: null block hash');
3228
} else {
33-
console.log('Found new block: ' + blockHashOrNumber);
34-
updatedEndBlock(config,blockHashOrNumber);
29+
console.log('Found new block: ' + latestBlock);
3530
if(web3.isConnected()) {
36-
web3.eth.getBlock(blockHashOrNumber, true, function(error, blockData) {
31+
web3.eth.getBlock(latestBlock, true, function(error,blockData) {
3732
if(error) {
38-
console.log('Warning: error on getting block with hash/number: ' + blockHashOrNumber + ': ' + error);
33+
console.log('Warning: error on getting block with hash/number: ' + latestBlock + ': ' + error);
3934
}else if(blockData == null) {
40-
console.log('Warning: null block data received from the block with hash/number: ' + blockHashOrNumber);
35+
console.log('Warning: null block data received from the block with hash/number: ' + latestBlock);
4136
}else{
4237
writeBlockToDB(config, blockData);
4338
writeTransactionsToDB(config, blockData);
44-
return;
4539
}
4640
});
4741
}else{
48-
console.log('Error: Web3 connection time out trying to get block ' + blockHashOrNumber + ' retrying connection now');
49-
return;
42+
console.log('Error: Web3 connection time out trying to get block ' + latestBlock + ' retrying connection now');
43+
listenBlocks(config);
5044
}
5145
}
5246
});
47+
// Starts full sync when set to true in config
48+
if (config.syncAll === true){
49+
console.log('Starting Full Sync');
50+
getOldestBlockDB(config);
51+
}
52+
// Starts full sync when set to true in config
53+
if (config.patch === true){
54+
console.log('Checking for missing blocks');
55+
runPatcher(config);
56+
}
5357
}
5458
/**
5559
If full sync is checked this function will start syncing the block chain from lastSynced param see README
5660
**/
57-
var syncChain = function(config, web3, blockHashOrNumber) {
58-
if(blockHashOrNumber == undefined) {
59-
blockHashOrNumber = config.endBlock
60-
}
61+
var syncChain = function(config,web3,nextBlock){
6162
if(web3.isConnected()) {
62-
web3.eth.getBlock(blockHashOrNumber, true, function(error, blockData) {
63+
web3.eth.getBlock(nextBlock, true, function(error,blockData) {
6364
if(error) {
64-
console.log('Warning: error on getting block with hash/number: ' + blockHashOrNumber + ': ' + error);
65+
console.log('Warning: error on getting block with hash/number: ' + nextBlock + ': ' + error);
66+
getOldestBlockDB();
6567
}else if(blockData == null) {
66-
console.log('Warning: null block data received from the block with hash/number: ' + blockHashOrNumber);
67-
}else{
68-
if(config.lastSynced === 0){
69-
console.log('No last full sync record found, start from block: latest');
70-
writeBlockToDB(config, blockData);
71-
writeTransactionsToDB(config, blockData);
72-
var lastSync = blockData.number;
73-
updateLastSynced(config, lastSync);
74-
}else{
75-
console.log('Found last full sync record: ' + config.lastSynced);
76-
writeBlockToDB(config, blockData);
77-
writeTransactionsToDB(config, blockData);
78-
var lastSync = config.lastSynced - 1;
79-
updateLastSynced(config, lastSync);
80-
}
68+
console.log('Warning: null block data received from the block with hash/number: ' + nextBlock);
69+
getOldestBlockDB();
70+
}else{
71+
writeBlockToDB(config, blockData);
72+
writeTransactionsToDB(config, blockData);
8173
}
8274
});
8375
}else{
84-
console.log('Error: Web3 connection time out trying to get block ' + blockHashOrNumber + ' retrying connection now');
85-
syncChain(config, web3, blockHashOrNumber);
76+
console.log('Error: Web3 connection time out trying to get block ' + nextBlock + ' retrying connection now');
77+
syncChain(config,web3,nextBlock);
8678
}
87-
};
79+
}
8880
/**
8981
Write the whole block object to DB
9082
**/
9183
var writeBlockToDB = function(config, blockData) {
9284
return new Block(blockData).save( function( err, block, count ){
9385
if ( typeof err !== 'undefined' && err ) {
94-
if (err.code == 11000) {
95-
if(!('quiet' in config && config.quiet === true)) {
96-
console.log('Skip: Duplicate key ' + blockData.number.toString() + ': ' + err);
97-
}
98-
} else {
99-
console.log('Error: Aborted due to error on ' + 'block number ' + blockData.number.toString() + ': ' + err);
100-
process.exit(9);
101-
}
102-
} else {
86+
if (err.code == 11000) {
10387
if(!('quiet' in config && config.quiet === true)) {
104-
console.log('DB successfully written for block number ' + blockData.number.toString());
88+
console.log('Skip: Duplicate key ' + blockData.number.toString() + ': ' +err);
10589
}
90+
}else{
91+
console.log('Error: Aborted due to error on ' + 'block number ' + blockData.number.toString() + ': ' + err);
92+
process.exit(9);
93+
}
94+
}else{
95+
console.log('DB successfully written for block number ' + blockData.number.toString() );
96+
// continues sync if flag is still true
97+
if (config.syncAll === true){
98+
getOldestBlockDB(config);
99+
}
106100
}
107101
});
108102
}
@@ -121,98 +115,119 @@ var writeTransactionsToDB = function(config, blockData) {
121115
Transaction.collection.insert(bulkOps, function( err, tx ){
122116
if ( typeof err !== 'undefined' && err ) {
123117
if (err.code == 11000) {
124-
console.log('Skip: Duplicate key ' + err);
125-
} else {
126-
console.log('Error: Aborted due to error: ' + err);
127-
process.exit(9);
128-
}
129-
} else if(!('quiet' in config && config.quiet === true)) {
130-
console.log(blockData.transactions.length.toString() + ' transactions recorded for Block# ' + blockData.number.toString());
118+
if(!('quiet' in config && config.quiet === true)) {
119+
console.log('Skip: Duplicate key ' + err);
120+
}
121+
}else{
122+
console.log('Error: Aborted due to error: ' + err);
123+
getOldestBlockDB();
124+
//process.exit(9);
131125
}
126+
}else{
127+
console.log(blockData.transactions.length.toString() + ' transactions recorded for Block# ' + blockData.number.toString());
128+
// continues sync if flag is still true
129+
if (config.syncAll === true){
130+
getOldestBlockDB(config);
131+
}
132+
}
132133
});
133134
}
134135
}
135-
var checkBlockDBExistsThenWrite = function(config, blockData) {
136-
Block.find({number: blockData.number}, function (err, b) {
137-
if (!b.length){
138-
writeBlockToDB(config, blockData);
139-
writeTransactionsToDB(config, blockData);
140-
}else if(!('quiet' in config && config.quiet === true)) {
141-
console.log('Block number: ' + blockData.number.toString() + ' already exists in DB.');
142-
listenBlocks(config);
143-
}
144-
});
145-
};
146-
var updatedEndBlock = function(config,lastBlock){
147-
var configFile = '../conf.json';
148-
var file = require(configFile);
149-
150-
file.endBlock = lastBlock;
151-
152-
fs.writeFile('conf.json', JSON.stringify(file, null, 2), function (err) {
153-
if (err) return console.log(err);
154-
//console.log('Wirting new Synced Block ' + lastBlock + ' to ' + configFile);
155-
});
156-
};
157136
/**
158-
Take the last block the grabber exited on and update the param 'end' in the config.JSON
137+
//Check oldest block in db and start sync from tehre
159138
**/
160-
var updateLastSynced = function(config,lastSync){
161-
var configFile = '../conf.json';
162-
var file = require(configFile);
163-
164-
file.lastSynced = lastSync;
165-
config.lastSynced = lastSync;
166-
167-
fs.writeFile('conf.json', JSON.stringify(file, null, 2), function (err) {
168-
if (err) return console.log(err);
169-
//console.log('writing block ' + lastSync + ' to ' + configFile);
170-
171-
if (config.lastSynced === config.startBlock){
172-
config.syncAll = false;
173-
file.syncAll = false;
174-
fs.writeFile('conf.json', JSON.stringify(file, null, 2), function (err) {
175-
if (err) return console.log(err);
176-
});
139+
var getOldestBlockDB = function() {
140+
var oldBlockFind = Block.find({}, "number").lean(true).sort('number').limit(1);
141+
oldBlockFind.exec(function (err, docs) {
142+
if(docs.length < 1){
143+
console.log('nothing here starting from latest');
177144
}else{
178-
syncChain(config, web3, config.lastSynced);
145+
var nextBlock = (docs[0].number - 1);
146+
if(nextBlock == config.startBlock){
147+
console.log('Sync Finsihed');
148+
config.syncAll = false;
149+
return;
150+
}else{
151+
syncChain(config,web3,nextBlock);
152+
}
179153
}
180154
});
181155
}
156+
/**
157+
Block Patcher(experimental)
158+
**/
159+
var runPatcher = function(config) {
160+
currentBlock = web3.eth.blockNumber;
161+
patchBlock = currentBlock - config.patchBlocks;
162+
console.log('Starting patching from block: '+patchBlock);
163+
while(config.patchBlocks > 0){
164+
config.patchBlocks--;
165+
patchBlock++;
166+
if(!('quiet' in config && config.quiet === true)) {
167+
console.log('Patching Block: '+patchBlock)
168+
}
169+
web3.eth.getBlock(patchBlock, true, function(error,patchData) {
170+
if(error) {
171+
console.log('Warning: error on getting block with hash/number: ' + patchBlock + ': ' + error);
172+
//getOldestBlockDB();
173+
}else if(patchData == null) {
174+
console.log('Warning: null block data received from the block with hash/number: ' + patchBlock);
175+
//getOldestBlockDB();
176+
}else{
177+
checkBlockDBExistsThenWrite(config,patchData)
178+
}
179+
});
180+
if (config.patchBlocks == 0){
181+
config.patch = false;
182+
console.log('Block Patching Complete')
183+
}
184+
}
185+
}
186+
/**
187+
This will be used for the patcher(experimental)
188+
**/
189+
var checkBlockDBExistsThenWrite = function(config,patchData) {
190+
Block.find({number: patchData.number}, function (err, b) {
191+
if (!b.length){
192+
writeBlockToDB(config,patchData);
193+
writeTransactionsToDB(config,patchData);
194+
}else if(!('quiet' in config && config.quiet === true)) {
195+
console.log('Block number: ' +patchData.number.toString() + ' already exists in DB.');
196+
}
197+
});
198+
};
182199
/**
183200
Start config for node connection and sync
184201
**/
185202
var config = {};
186203
// set the default NODE address to localhost if it's not provided
187204
if (!('nodeAddr' in config) || !(config.nodeAddr)) {
188-
config.nodeAddr = 'localhost'; // default
205+
config.nodeAddr = 'localhost'; // default
189206
}
190207
// set the default geth port if it's not provided
191208
if (!('gethPort' in config) || (typeof config.gethPort) !== 'number') {
192-
config.gethPort = 8545; // default
209+
config.gethPort = 8545; // default
193210
}
194211
// set the default output directory if it's not provided
195212
if (!('output' in config) || (typeof config.output) !== 'string') {
196-
config.output = '.'; // default this directory
213+
config.output = '.'; // default this directory
197214
}
198215
//Look for config.json file if not
199216
try {
200217
var configContents = fs.readFileSync('conf.json');
201218
config = JSON.parse(configContents);
202219
console.log('CONFIG FOUND: Node:'+config.nodeAddr+' | Port:'+config.gethPort);
203-
// Sets address for RPC WEb3 to connect to, usually your node address defaults ot localhost
204-
var web3 = new Web3(new Web3.providers.HttpProvider('http://' + config.nodeAddr + ':' + config.gethPort.toString()));
205-
if (config.syncAll === true){
206-
syncChain(config,web3);
207-
}
208220
}
209221
catch (error) {
210-
if (error.code === 'ENOENT') {
211-
console.log('No config file found. Using default configuration: Node:'+config.nodeAddr+' | Port:'+config.gethPort);
212-
}
213-
else {
214-
throw error;
215-
process.exit(1);
216-
}
222+
if (error.code === 'ENOENT') {
223+
console.log('No config file found. Using default configuration: Node:'+config.nodeAddr+' | Port:'+config.gethPort);
224+
}
225+
else {
226+
throw error;
227+
process.exit(1);
228+
}
217229
}
230+
// Sets address for RPC WEB3 to connect to, usually your node IP address defaults ot localhost
231+
var web3 = new Web3(new Web3.providers.HttpProvider('http://' + config.nodeAddr + ':' + config.gethPort.toString()));
232+
// Start listening for latest blocks
218233
listenBlocks(config);

0 commit comments

Comments
 (0)