From e77b3bbebc4c7c2e21a8dce1611be95ed6080cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s-Combarro=20=27piranna?= Date: Sun, 14 Jan 2018 14:55:03 +0100 Subject: [PATCH 1/7] Updated to `mongodb` 3.0.1 --- index.js | 12 +++++++----- package.json | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index 0edf61a..a635a56 100644 --- a/index.js +++ b/index.js @@ -24,17 +24,19 @@ function bot(config) { var callback = config.callback; var auth = config.username ? `${config.username}:${config.password}@` : ''; - client.connect(`mongodb://${auth}${config.host}/${config.db}`, (err, db) => { + client.connect(`mongodb://${auth}${config.host}/${config.db}`, (err, client) => { if(err) return callback(err); + const db = client.db(config.db) + if(!config.fields || !config.fields.length) { callback(null); - return db.close(); + return client.close(); } // remove empty fields; let fields = config.fields.filter(item => !!item); - if(!fields.length) return db.close(); // fields can be empty + if(!fields.length) return client.close(); // fields can be empty var c = config.collection; var collections = {}; @@ -55,10 +57,10 @@ function bot(config) { var i = 0, l = Object.keys(collections).length - 1; for(let c in collections) { db.collection(c).insertMany(collections[c], (err, ret) => { - if(i++ === l) db.close(); + if(i++ === l) client.close(); if(err) return callback(err); callback(null, ret); }); } }); -}; \ No newline at end of file +}; diff --git a/package.json b/package.json index 64029f3..29b8bc4 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,6 @@ }, "homepage": "https://github.com/sofish/mongoimport", "dependencies": { - "mongodb": "^2.0.48" + "mongodb": "^3.0.1" } } From 3a2bd8c83661071077440fae0ee05b86f2184aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s-Combarro=20=27piranna?= Date: Sun, 14 Jan 2018 19:05:49 +0100 Subject: [PATCH 2/7] Code clean-up (only change, `db` -> `database`) --- index.js | 74 ++++++++++++++++++++++++--------------------------- test/index.js | 16 +++++------ 2 files changed, 43 insertions(+), 47 deletions(-) diff --git a/index.js b/index.js index a635a56..be5f686 100644 --- a/index.js +++ b/index.js @@ -1,57 +1,50 @@ 'use strict'; -const mongodb = require('mongodb'); -const client = mongodb.MongoClient; - -module.exports = bot; - -function bot(config) { - /* Bot that helps to import your data into db - * @param {object} config - * { - * fields: [], // {array} data to import - * db: 'name', // {string} name of db - * collection: 'collection' // {string|function} name of collection, or return a name - * host: 'localhost:27017', // {string} [optional] by default is 27017 - * username: 'sofish', // {string} [optional] - * password: '***' // {string} [optional] - * callback: (err, db) => {} // {function} [optional] - * } - */ - - if(!config.host) config.host = '127.0.0.1:27027'; - if(!config.callback) config.callback = () => {}; - - var callback = config.callback; - var auth = config.username ? `${config.username}:${config.password}@` : ''; - client.connect(`mongodb://${auth}${config.host}/${config.db}`, (err, client) => { +const {MongoClient} = require('mongodb'); + + +/* Helper function to import your data into a MongoDB database + * @param {object} config + * { + * fields: [], // {array} data to import + * database: 'name', // {string} name of database + * collection: 'collection' // {string|function} name of collection, or return a name + * host: 'localhost:27017', // {string} [optional] by default is 27017 + * username: 'sofish', // {string} [optional] + * password: '***' // {string} [optional] + * callback: (err, client) => {} // {function} [optional] + * } + */ +function bot({callback = () => {}, collection, database, fields, +host = '127.0.0.1:27027', password, username}) { + const auth = username ? `${username}:${password}@` : ''; + + MongoClient.connect(`mongodb://${auth}${host}/${database}`, (err, client) => { if(err) return callback(err); - const db = client.db(config.db) + const db = client.db(database) - if(!config.fields || !config.fields.length) { - callback(null); - return client.close(); + if(!(fields && fields.length)) { + client.close(); + return callback(); } // remove empty fields; - let fields = config.fields.filter(item => !!item); - if(!fields.length) return client.close(); // fields can be empty - - var c = config.collection; - var collections = {}; + fields = fields.filter(item => !!item); + if(!fields.length) return client.close(); // fields can be empty // map collection - if(typeof c === 'function') { + const collections = {}; + if(typeof collection === 'function') { fields.forEach(item => { - var name = c(item); + const name = collection(item); if(collections[name]) return collections[name].push(item); collections[name] = [item]; }) - } else if(typeof c === 'string') { - collections[c] = fields; + } else if(typeof collection === 'string') { + collections[collection] = fields; } else { - callback({messsage: 'not matched, no `collection` is specific'}); + return callback({messsage: 'not matched, no `collection` is specific'}); } var i = 0, l = Object.keys(collections).length - 1; @@ -64,3 +57,6 @@ function bot(config) { } }); }; + + +module.exports = bot; diff --git a/test/index.js b/test/index.js index 394886d..e6bafd0 100644 --- a/test/index.js +++ b/test/index.js @@ -1,19 +1,19 @@ 'use strict'; -const mongoimport = require('../'); -var host = '127.0.0.1:27017'; -var db = 'mongoimport'; -var collection = function(field) { +const mongoimport = require('..'); + +const host = '127.0.0.1:27017'; +const database = 'mongoimport'; +const collection = function(field) { return field.name; }; -var fields = [ +const fields = [ { name: 'sofish', createdAt: '1986', isBot: 'guess me', isFun: 'try me' }, { name: 'perf', foo: 'bar' }, { name: 'error', hello: 'world' } ]; -var config = {host, db, collection, callback, fields} -mongoimport(config, callback); +mongoimport({host, database, collection, callback, fields}); function callback(err, ret) { if(err) { @@ -25,4 +25,4 @@ function callback(err, ret) { console.log('✔ %d records inserted', ret.insertedCount); console.log('= done!\n'); -} \ No newline at end of file +} From b813a1829dfb20ebb0aaa4feea606b63f28afff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s-Combarro=20=27piranna?= Date: Sun, 14 Jan 2018 19:37:02 +0100 Subject: [PATCH 3/7] Do the arguments checks in advance --- index.js | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/index.js b/index.js index be5f686..a19f4bf 100644 --- a/index.js +++ b/index.js @@ -18,35 +18,33 @@ const {MongoClient} = require('mongodb'); function bot({callback = () => {}, collection, database, fields, host = '127.0.0.1:27027', password, username}) { const auth = username ? `${username}:${password}@` : ''; + if(!fields) { + return callback(); + } + + // remove empty fields; + fields = fields.filter(item => !!item); + if(!fields.length) return callback(); // fields can be empty + + // map collection + const collections = {}; + if(typeof collection === 'function') { + fields.forEach(item => { + const name = collection(item); + if(collections[name]) return collections[name].push(item); + collections[name] = [item]; + }) + } else if(typeof collection === 'string') { + collections[collection] = fields; + } else { + throw new Error('`collection` is not specified'); + } MongoClient.connect(`mongodb://${auth}${host}/${database}`, (err, client) => { if(err) return callback(err); const db = client.db(database) - if(!(fields && fields.length)) { - client.close(); - return callback(); - } - - // remove empty fields; - fields = fields.filter(item => !!item); - if(!fields.length) return client.close(); // fields can be empty - - // map collection - const collections = {}; - if(typeof collection === 'function') { - fields.forEach(item => { - const name = collection(item); - if(collections[name]) return collections[name].push(item); - collections[name] = [item]; - }) - } else if(typeof collection === 'string') { - collections[collection] = fields; - } else { - return callback({messsage: 'not matched, no `collection` is specific'}); - } - var i = 0, l = Object.keys(collections).length - 1; for(let c in collections) { db.collection(c).insertMany(collections[c], (err, ret) => { From b22f237db2aef27b9ddee296523eae3c1fc53e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s-Combarro=20=27piranna?= Date: Sun, 14 Jan 2018 20:46:41 +0100 Subject: [PATCH 4/7] Use Promise-based API --- index.js | 54 ++++++++++++++++++++++++++++----------------------- package.json | 3 ++- test/index.js | 22 ++++++++++----------- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/index.js b/index.js index a19f4bf..6645379 100644 --- a/index.js +++ b/index.js @@ -1,30 +1,35 @@ 'use strict'; +const {promisify} = require('util'); + const {MongoClient} = require('mongodb'); +require('promise.prototype.finally').shim(); + -/* Helper function to import your data into a MongoDB database +/** + * Helper function to import your data into a MongoDB database + * * @param {object} config * { - * fields: [], // {array} data to import - * database: 'name', // {string} name of database - * collection: 'collection' // {string|function} name of collection, or return a name - * host: 'localhost:27017', // {string} [optional] by default is 27017 - * username: 'sofish', // {string} [optional] - * password: '***' // {string} [optional] - * callback: (err, client) => {} // {function} [optional] + * fields: [], // {array} data to import + * database: 'name', // {string} name of database + * collection: 'collection' // {string|function} name of collection, or return a name + * host: 'localhost:27017', // {string} [optional] by default is 27017 + * username: 'sofish', // {string} [optional] + * password: '***' // {string} [optional] * } */ -function bot({callback = () => {}, collection, database, fields, -host = '127.0.0.1:27027', password, username}) { - const auth = username ? `${username}:${password}@` : ''; +function bot({collection, database, fields, host = '127.0.0.1:27027', password, + username}) +{ if(!fields) { - return callback(); + return; } // remove empty fields; fields = fields.filter(item => !!item); - if(!fields.length) return callback(); // fields can be empty + if(!fields.length) return; // fields can be empty // map collection const collections = {}; @@ -40,19 +45,20 @@ host = '127.0.0.1:27027', password, username}) { throw new Error('`collection` is not specified'); } - MongoClient.connect(`mongodb://${auth}${host}/${database}`, (err, client) => { - if(err) return callback(err); + const auth = username ? `${username}:${password}@` : ''; + + return MongoClient.connect(`mongodb://${auth}${host}/${database}`) + .then(function(client) + { + const promises = Object.keys(collections).map(function(key) + { + const collection = this.collection(key); - const db = client.db(database) + return promisify(collection.insertMany.bind(collection))(collections[key]); + }, + client.db(database)); - var i = 0, l = Object.keys(collections).length - 1; - for(let c in collections) { - db.collection(c).insertMany(collections[c], (err, ret) => { - if(i++ === l) client.close(); - if(err) return callback(err); - callback(null, ret); - }); - } + return Promise.all(promises).finally(client.close.bind(client)); }); }; diff --git a/package.json b/package.json index 29b8bc4..60163d0 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ }, "homepage": "https://github.com/sofish/mongoimport", "dependencies": { - "mongodb": "^3.0.1" + "mongodb": "^3.0.1", + "promise.prototype.finally": "^3.1.0" } } diff --git a/test/index.js b/test/index.js index e6bafd0..4479499 100644 --- a/test/index.js +++ b/test/index.js @@ -13,16 +13,14 @@ const fields = [ { name: 'error', hello: 'world' } ]; -mongoimport({host, database, collection, callback, fields}); - -function callback(err, ret) { - if(err) { - if(err.message.match('ECONNREFUSED')) console.log('✘ make sure you have started mongodb server'); - if(err.message.match('Authentication')) console.log('✘ make sure the username/password pair is matched'); - console.log('= done!\n'); - throw err.message; - } - - console.log('✔ %d records inserted', ret.insertedCount); +mongoimport({host, database, collection, fields}) +.then(function(ret) { + ret.forEach(ret => console.log('✔ %d records inserted', ret.insertedCount)); + console.log('= done!\n'); +}, +function(err) { + if(err.message.match('ECONNREFUSED')) console.log('✘ make sure you have started mongodb server'); + if(err.message.match('Authentication')) console.log('✘ make sure the username/password pair is matched'); console.log('= done!\n'); -} + throw err.message; +}); From d5dc724b2b5547faaf501cf1292b1b43695358b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s-Combarro=20=27piranna?= Date: Tue, 10 Apr 2018 10:58:47 +0200 Subject: [PATCH 5/7] Added option to drop collection content before filling it --- index.js | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 6645379..3354647 100644 --- a/index.js +++ b/index.js @@ -7,6 +7,12 @@ const {MongoClient} = require('mongodb'); require('promise.prototype.finally').shim(); +function ignoreNamespaceNotFound(error) +{ + if(error.codeName !== 'NamespaceNotFound') return Promise.reject(error) +} + + /** * Helper function to import your data into a MongoDB database * @@ -20,8 +26,8 @@ require('promise.prototype.finally').shim(); * password: '***' // {string} [optional] * } */ -function bot({collection, database, fields, host = '127.0.0.1:27027', password, - username}) +function bot({collection, database, drop, fields, host = '127.0.0.1:27027', + password, username}) { if(!fields) { return; @@ -54,7 +60,19 @@ function bot({collection, database, fields, host = '127.0.0.1:27027', password, { const collection = this.collection(key); - return promisify(collection.insertMany.bind(collection))(collections[key]); + const promisedDrop = promisify(collection.drop .bind(collection)) + const promisedInsertMany = promisify(collection.insertMany.bind(collection)) + + function insertMany() + { + return promisedInsertMany(collections[key]); + } + + if(!drop) return insertMany(); + + return promisedDrop() + .catch(ignoreNamespaceNotFound) + .then(insertMany) }, client.db(database)); From e4ebbb172eb42218f3314b7485b3043ee8df3041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s-Combarro=20=27piranna?= Date: Tue, 10 Apr 2018 10:58:58 +0200 Subject: [PATCH 6/7] Code clean-up --- index.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 3354647..a070c90 100644 --- a/index.js +++ b/index.js @@ -40,10 +40,13 @@ function bot({collection, database, drop, fields, host = '127.0.0.1:27027', // map collection const collections = {}; if(typeof collection === 'function') { - fields.forEach(item => { + fields.forEach(function(item) + { const name = collection(item); - if(collections[name]) return collections[name].push(item); - collections[name] = [item]; + + let col = collections[name] + if(!col) collections[name] = col = []; + col.push(item); }) } else if(typeof collection === 'string') { collections[collection] = fields; From 071fc762d56fab2282c97adb1b84a4f03a9b27ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s-Combarro=20=27piranna?= Date: Tue, 10 Apr 2018 10:59:19 +0200 Subject: [PATCH 7/7] Updated `mongodb` dependency --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 60163d0..d2098b7 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ }, "homepage": "https://github.com/sofish/mongoimport", "dependencies": { - "mongodb": "^3.0.1", + "mongodb": "^3.0.6", "promise.prototype.finally": "^3.1.0" } }