From 029db0d7b1abde22daf9f92f317007bf29fa8430 Mon Sep 17 00:00:00 2001 From: Emily Lemmon Date: Mon, 9 Oct 2017 14:30:58 -0500 Subject: [PATCH 1/6] cleanup of branch before start task --- tnAchieves.sqlite | Bin 9216 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tnAchieves.sqlite diff --git a/tnAchieves.sqlite b/tnAchieves.sqlite deleted file mode 100644 index e3b30ad5ff89cfee5dad816ee01e31362634a00d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9216 zcmeHMOK;Oa5cbAtOAaOCLM1q4dq8T{(k7&UiiAXH4UK4$k~k_kRLv%yLKy#{&= z55bXNz;EoG3%ic%SA!5BC=I)^y}PrI+4g>W3x(T-{QY*}Mn0Fz7xI;2v3#dg~GgACg458e(k!{)4-yhrNKbA zKnn|->l^F0ilw_4m+loe?iGu8uNF4e&10fz4!G`};^y7W2VfH2ShIj`8N_w5J;44@ zwN}PAal-<<5AXuFQoNebw;c^PUR&VQJJIAF+rz{GOmN>e3>X3qR=GH8%W3%#mv8;w zmoFO=a2tqbFtl+KOr0U`j1Dqsxu$I2P+Go=zHdwcFJp++5G8@Y|Aqk9e_0wy=mdRX*>OVtx7Xyk zg;EK>pQ!H?+onoXKzkCA-yBDyo9j%EMg+ch3&#COAmaAK`(QVysd%DSt{vlu0+J3uIC z+w2k20#WOp8Io#tk9}mtDNQi=z#xurmQkzL;_!}Vz{`7?*@cxA)bZhWUm7e_d+5ST z53D|@ilFAWKV+|BpHC2M5vMhdfCYdOq!tSeB?xWH^?GjK(R=Zp zU;x{eP#(4#U(Rr(U#l|^L&_t^HY?)@Qi7J~v_l+iywLd6?1g9yRGSjRN5M_CwJn|P zaGZE5R|MY=-y?fK&oXX#z30%UyDV@sUNc_y|Nk?eBzB1;5I8pk-g?=&g>o5vxZ>NT zhDX!BW&&v@X{jdk@J&{n{`;^;9mg?xkAvkd}@C)vQS-%vIl!xK^pJicxc`37ljzs)l#Q)=GDT(t95&ut~ qOI`H%zkhDS^*@u@@%8`QjK5%f;Sm@e%%;!BrtJK0CN1Ao=>Io- Date: Mon, 9 Oct 2017 15:34:07 -0500 Subject: [PATCH 2/6] troubleshoot some express issues and make sure the back end is able to post --- app/factories/upload-factory.js | 35 +++++++++++++++++++++++------ lib/package.json | 1 + messageCtrl.js | 3 ++- nodeApp.js | 14 ++++++++---- package.json | 39 +++++++++++++++++++++++++++++++++ routes.js | 2 +- 6 files changed, 81 insertions(+), 13 deletions(-) create mode 100644 package.json diff --git a/app/factories/upload-factory.js b/app/factories/upload-factory.js index 7b123d4..8f7103e 100644 --- a/app/factories/upload-factory.js +++ b/app/factories/upload-factory.js @@ -1,17 +1,38 @@ 'use strict'; -app.factory('uploadFactory', function ($q) { +app.factory('uploadFactory', function ($q, $http) { //parses data from csv to Json, posts to db, called from uploadCtrl const uploadFile = (data) => { - return $q(() => { - let uploadObj = angular.toJson(data.data); - console.log("uploadObj", uploadObj); + return $q((resolve, reject) => { + let uploadObjStr = angular.toJson(data.data); + let uploadObj = JSON.parse(uploadObjStr); + // STILL NEEDS TO BE DONE -- add upload object to taggedUploadArrObj[1] and add csv file type to obj at taggedUploadArrObj[0] + let taggedUploadArrObj = uploadObj; + console.log("uploadObj", taggedUploadArrObj); + uploadToApi(taggedUploadArrObj) + .then(); + }); }; - return { - uploadFile - }; + function uploadToApi(arrObj){ + return $q((resolve, reject)=>{ + $http({ + method: 'POST', + url: 'http://localhost:3000/uploadfile', + data: arrObj, + headers: {'Content-Type': 'application/json'} + }) + .then((data) => { + console.log("uploaded to Api?", data); + resolve(); + }) + .catch( (err) => { + reject(err); + }); + }); + } + return {uploadFile}; }); \ No newline at end of file diff --git a/lib/package.json b/lib/package.json index 4813e8d..51f1753 100644 --- a/lib/package.json +++ b/lib/package.json @@ -28,6 +28,7 @@ "angular": "^1.6.6", "angular-route": "^1.6.6", "angular-ui-bootstrap": "^2.5.0", + "body-parser": "^1.18.2", "bootstrap": "^4.0.0-beta", "express": "^4.16.1", "grunt": "^1.0.1", diff --git a/messageCtrl.js b/messageCtrl.js index 04ca3c8..1333508 100644 --- a/messageCtrl.js +++ b/messageCtrl.js @@ -20,7 +20,8 @@ let getMessagesByContactId = (req, res, next) => { }; let uploadFileFromWeb = (req, res, next) => { - uploadFileToDB(req.params.file) + console.log("req body", req.body); + uploadFileToDB(req.body) .then( (results) => { //do something with results-- send to front end for usable form, they will just be JSON so far res.status(200).json(results); diff --git a/nodeApp.js b/nodeApp.js index a5ff59e..32b23db 100644 --- a/nodeApp.js +++ b/nodeApp.js @@ -1,8 +1,14 @@ 'use strict'; -let express = require('express'); -let app = express(); -let routes = require('./routes.js'); +const express = require('express'); +const app = express(); +const routes = require('./routes.js'); +const bodyParser = require('body-parser'); +const cors = require('cors'); + + +app.use(bodyParser.json()); +app.use(cors()); app.use('/', routes); @@ -17,7 +23,7 @@ app.use( (err, req, res, next) => { res.status(err.status||500); res.json({ message:"A problem occurred.", - err:err + err: err }) }); diff --git a/package.json b/package.json new file mode 100644 index 0000000..e425e3e --- /dev/null +++ b/package.json @@ -0,0 +1,39 @@ +{ + "name": "tnAchieves-Communication-Archive", + "version": "1.0.0", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "devDependencies": { + "concat": "^1.0.3", + "connect": "^3.6.5", + "grunt-contrib-concat": "^1.0.1", + "grunt-contrib-connect": "^1.0.2", + "grunt-contrib-jshint": "^1.1.0", + "grunt-contrib-sass": "^1.0.0", + "grunt-contrib-watch": "^1.0.0", + "jshint": "^2.9.5", + "jshint-stylish": "^2.2.1", + "matchdep": "^2.0.0", + "nodemon": "^1.12.1", + "sass": "^1.0.0-beta.2", + "sqlite3": "^3.1.13", + "watch": "^1.0.2" + }, + "dependencies": { + "angular": "^1.6.6", + "angular-route": "^1.6.6", + "angular-ui-bootstrap": "^2.5.0", + "body-parser": "^1.18.2", + "bootstrap": "^4.0.0-beta", + "cors": "^2.8.4", + "express": "^4.16.1", + "grunt": "^1.0.1", + "jquery": "^3.2.1", + "popper.js": "^1.12.5" + }, + "description": "" +} diff --git a/routes.js b/routes.js index b032e88..4b2c118 100644 --- a/routes.js +++ b/routes.js @@ -8,7 +8,7 @@ const { sayHello } = require('./helloCtrl.js'); router.get('/messages/:id', getMessagesByContactId); //if this route, get all emails attached to a certain id -router.post('/uploadfile/:file', uploadFileFromWeb); //if this route, get all emails attached to a certain id +router.post('/uploadfile', uploadFileFromWeb); //if this route, get all emails attached to a certain id // router.post('/email/upload', postEmailData); router.get('/hello', sayHello); From d921d73788e01c4c7ae28492e82ac31557aed8ab Mon Sep 17 00:00:00 2001 From: Emily Lemmon Date: Mon, 9 Oct 2017 16:07:29 -0500 Subject: [PATCH 3/6] pass object to DB function - SQLITE errors perhaps to be fixed by converting to bracket notation --- Message.js | 6 ++++-- app/factories/upload-factory.js | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Message.js b/Message.js index 79f229e..e0e000a 100644 --- a/Message.js +++ b/Message.js @@ -298,10 +298,12 @@ module.exports.searchByContactId = (contactId) => { module.exports.uploadFileToDB = (file) => { console.log("file going in", file); + console.log("file.Clicked", file.Clicked); + //try importing it in the way you converted them with bracket notation above! return new Promise( (resolve, reject) => { db.all(`INSERT INTO EmailFiles VALUES (null, - '${file.contactId}', - '${file.clicked}', + '${file.['Contact ID']}', + '${file.Clicked}', '${file.contact}', '${file.contactRecordType}', '${file.dateBounced}', diff --git a/app/factories/upload-factory.js b/app/factories/upload-factory.js index 8f7103e..765b5f2 100644 --- a/app/factories/upload-factory.js +++ b/app/factories/upload-factory.js @@ -10,8 +10,7 @@ app.factory('uploadFactory', function ($q, $http) { // STILL NEEDS TO BE DONE -- add upload object to taggedUploadArrObj[1] and add csv file type to obj at taggedUploadArrObj[0] let taggedUploadArrObj = uploadObj; console.log("uploadObj", taggedUploadArrObj); - uploadToApi(taggedUploadArrObj) - .then(); + uploadToApi(taggedUploadArrObj); }); }; From 32856076205008f4a0c87a2d353afd7dd2491eb3 Mon Sep 17 00:00:00 2001 From: Emily Lemmon Date: Mon, 9 Oct 2017 20:58:32 -0500 Subject: [PATCH 4/6] switched to bracket notation, still getting SQLITE errors --- Message.js | 112 ++++++++++++++++++++++++------------------------- messageCtrl.js | 1 - 2 files changed, 55 insertions(+), 58 deletions(-) diff --git a/Message.js b/Message.js index e0e000a..d67af90 100644 --- a/Message.js +++ b/Message.js @@ -105,7 +105,7 @@ let csvMessageObjs = [ }, ] //Methods for the EMAIL data -//this will not need to be in the production version as the DB will already exist! Just for my build purposes -EL +// //this will not need to be in the production version as the DB will already exist! Just for my build purposes -EL // db.serialize( () => { @@ -135,13 +135,9 @@ let csvMessageObjs = [ // relatedStudentContactId TEXT, // softBounce TEXT NOT NULL, // subjectLine TEXT - - // )`); - - // module.exports.prepEmailData = (data) => { // console.log('printData firing', data["Contact Record Type"]); // data.forEach(d => { @@ -172,31 +168,31 @@ let csvMessageObjs = [ // }; // //POST from upload, iterate over each of the items in the parsed array of objects: -// module.exports.insertEmailsIntoDB = (csvEmailObjs) => { -// csvEmailObjs.forEach( (emailObj) => { -// db.run(`INSERT INTO EmailFiles VALUES (null, -// '${emailObj.clicked}', -// '${emailObj.contact}', -// '${emailObj.contactId}', -// '${emailObj.contactRecordType}', -// '${emailObj.dateBounced}', -// '${emailObj.dateOpened}', -// '${emailObj.dateSent}', -// '${emailObj.dateUnsubscribed}', -// '${emailObj.deleted}', -// '${emailObj.email}', -// '${emailObj.emailName}', -// '${emailObj.fromAddress}', -// '${emailObj.fromName}', -// '${emailObj.hardBounce}', -// ${emailObj.numberOfTotalClicks}, -// ${emailObj.numberOfUniqueClicks}, -// '${emailObj.opened}', -// '${emailObj.relatedStudentContactId}', -// '${emailObj.softBounce}', -// '${emailObj.subjectLine}')`); -// }); -// } +module.exports.insertEmailsIntoDB = (csvEmailObjs) => { + csvEmailObjs.forEach( (emailObj) => { + db.run(`INSERT INTO EmailFiles VALUES (null, + '${emailObj.clicked}', + '${emailObj.contact}', + '${emailObj.contactId}', + '${emailObj.contactRecordType}', + '${emailObj.dateBounced}', + '${emailObj.dateOpened}', + '${emailObj.dateSent}', + '${emailObj.dateUnsubscribed}', + '${emailObj.deleted}', + '${emailObj.email}', + '${emailObj.emailName}', + '${emailObj.fromAddress}', + '${emailObj.fromName}', + '${emailObj.hardBounce}', + ${emailObj.numberOfTotalClicks}, + ${emailObj.numberOfUniqueClicks}, + '${emailObj.opened}', + '${emailObj.relatedStudentContactId}', + '${emailObj.softBounce}', + '${emailObj.subjectLine}')`); + }); +} @@ -281,9 +277,9 @@ module.exports.insertMessagesIntoDB = (csvMessageObjs) => { -//uncommenting this will call this function and input those objects above -// module.exports.insertEmailsIntoDB(csvEmailObjs); - // module.exports.insertMessagesIntoDB(csvMessageObjs); +// // //uncommenting this will call this function and input those objects above + module.exports.insertEmailsIntoDB(csvEmailObjs); +// // // module.exports.insertMessagesIntoDB(csvMessageObjs); // }); module.exports.searchByContactId = (contactId) => { @@ -296,36 +292,38 @@ module.exports.searchByContactId = (contactId) => { }; -module.exports.uploadFileToDB = (file) => { - console.log("file going in", file); - console.log("file.Clicked", file.Clicked); - //try importing it in the way you converted them with bracket notation above! +module.exports.uploadFileToDB = (fileObjsArr) => { + console.log("file going in"); + console.log("file.contactid", fileObjsArr[0]['Contact ID']); return new Promise( (resolve, reject) => { - db.all(`INSERT INTO EmailFiles VALUES (null, - '${file.['Contact ID']}', + fileObjsArr.forEach( (file) => { + + db.all(`INSERT INTO EmailFiles VALUES (null, '${file.Clicked}', - '${file.contact}', - '${file.contactRecordType}', - '${file.dateBounced}', - '${file.dateOpened}', - '${file.dateSent}', - '${file.dateUnsubscribed}', - '${file.deleted}', - '${file.email}', - '${file.emailName}', - '${file.fromAddress}', - '${file.fromName}', - '${file.hardBounce}', - ${file.numberOfTotalClicks}, - ${file.numberOfUniqueClicks}, - '${file.opened}', - '${file.relatedStudentContactId}', - '${file.softBounce}', - '${file.subjectLine}')`, + '${file.Contact}', + '${file['Contact ID']}', + '${file["Contact Record Type"]}', + '${file["Date Bounced"]}', + '${file['Date Opened']}', + '${file["Date Sent"]}', + '${file["Date Unsubscribed"]}', + '${file.Deleted}', + '${file.Email}', + '${file["Email Name"]}', + '${file['From Address']}', + '${file["From Name"]}', + '${file["Hard Bounce"]}', + ${file["Number Of Total Clicks"]}, + ${file["Number Of Unique Clicks"]}, + '${file.Opened}', + '${file["Related Student Contact ID"]}', + '${file["Soft Bounce"]}', + '${file["Subject Line"]}')`, (err, successMessage) => { if (err) return reject(err);//if error, pass on to error handler resolve(successMessage); }); + }) }); }; diff --git a/messageCtrl.js b/messageCtrl.js index 1333508..1eb5b03 100644 --- a/messageCtrl.js +++ b/messageCtrl.js @@ -20,7 +20,6 @@ let getMessagesByContactId = (req, res, next) => { }; let uploadFileFromWeb = (req, res, next) => { - console.log("req body", req.body); uploadFileToDB(req.body) .then( (results) => { //do something with results-- send to front end for usable form, they will just be JSON so far From 82c4ff959b25d122b17bbf8f8e945a9c51b72252 Mon Sep 17 00:00:00 2001 From: Emily Lemmon Date: Sat, 18 Nov 2017 08:56:57 -0600 Subject: [PATCH 5/6] A step in the right direction with sqlite before switching to a different method --- Message.js | 70 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/Message.js b/Message.js index d67af90..7eff039 100644 --- a/Message.js +++ b/Message.js @@ -1,3 +1,5 @@ +import { workers } from 'cluster'; + 'use strict'; const sqlite3 = require('sqlite3').verbose(); @@ -278,7 +280,7 @@ module.exports.insertMessagesIntoDB = (csvMessageObjs) => { // // //uncommenting this will call this function and input those objects above - module.exports.insertEmailsIntoDB(csvEmailObjs); + // module.exports.insertEmailsIntoDB(csvEmailObjs); // // // module.exports.insertMessagesIntoDB(csvMessageObjs); // }); @@ -291,38 +293,56 @@ module.exports.searchByContactId = (contactId) => { }); }; - +//clean up and start building out step by step module.exports.uploadFileToDB = (fileObjsArr) => { console.log("file going in"); - console.log("file.contactid", fileObjsArr[0]['Contact ID']); + console.log("fileObjsArr, fakely 0", fileObjsArr[0]['Contact ID']); return new Promise( (resolve, reject) => { + console.log("fileObjsArr, fakely 1", fileObjsArr[0]['Contact ID']); + + //create empty strings/sub numbers for all values just in case + + //try checking out a new branch and then using the same front end stuff?? + + //try making a whole new back end based on what you have learned with sequelize + // then see if you can use something like bulkinsert... find out how papaparse works and + //if that can be a json file... maybe stdout write it to a file or something + //then you should be able to use that as a 'seeder' ... but is there a way to sequelize it + //straight as a json collection, like instead of importing from a seed data file let {} = require whatever + //just go ahead and straight up import and then bulkinsert that stuff? + fileObjsArr.forEach( (file) => { - - db.all(`INSERT INTO EmailFiles VALUES (null, - '${file.Clicked}', - '${file.Contact}', - '${file['Contact ID']}', - '${file["Contact Record Type"]}', - '${file["Date Bounced"]}', - '${file['Date Opened']}', - '${file["Date Sent"]}', - '${file["Date Unsubscribed"]}', - '${file.Deleted}', - '${file.Email}', - '${file["Email Name"]}', - '${file['From Address']}', - '${file["From Name"]}', - '${file["Hard Bounce"]}', - ${file["Number Of Total Clicks"]}, - ${file["Number Of Unique Clicks"]}, - '${file.Opened}', - '${file["Related Student Contact ID"]}', - '${file["Soft Bounce"]}', - '${file["Subject Line"]}')`, + if (file['Contact ID']) { + + console.log("file file file", file); + db.run(`INSERT INTO EmailFiles (contactId, clicked) VALUES ("${file['Contact ID']}", "${file.Clicked}")`, + // '${file.Clicked}', + // '${file.Contact}', + // '${file['Contact ID']}', + // '${file["Contact Record Type"]}', + // '${file["Date Bounced"]}', + // '${file['Date Opened']}', + // '${file["Date Sent"]}', + // '${file["Date Unsubscribed"]}', + // '${file.Deleted}', + // '${file.Email}', + // '${file["Email Name"]}', + // '${file['From Address']}', + // '${file["From Name"]}', + // '${file["Hard Bounce"]}', + // ${file["Number Of Total Clicks"]}, + // ${file["Number Of Unique Clicks"]}, + // '${file.Opened}', + // '${file["Related Student Contact ID"]}', + // '${file["Soft Bounce"]}', + // '${file["Subject Line"]}' + // ), (err, successMessage) => { if (err) return reject(err);//if error, pass on to error handler resolve(successMessage); }); + //ask JUFE - or someone - how can I resolve this not inside foreach?- or what else to do here + } }) }); }; From b002b0c34b844b3b9cf2ac5974ec10601c03c59d Mon Sep 17 00:00:00 2001 From: Emily Lemmon Date: Sat, 18 Nov 2017 09:01:22 -0600 Subject: [PATCH 6/6] small chages to app.js --- app/factories/upload-factory.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/factories/upload-factory.js b/app/factories/upload-factory.js index 765b5f2..dbee169 100644 --- a/app/factories/upload-factory.js +++ b/app/factories/upload-factory.js @@ -5,6 +5,7 @@ app.factory('uploadFactory', function ($q, $http) { //parses data from csv to Json, posts to db, called from uploadCtrl const uploadFile = (data) => { return $q((resolve, reject) => { + let uploadObjStr = angular.toJson(data.data); let uploadObj = JSON.parse(uploadObjStr); // STILL NEEDS TO BE DONE -- add upload object to taggedUploadArrObj[1] and add csv file type to obj at taggedUploadArrObj[0] @@ -17,10 +18,12 @@ app.factory('uploadFactory', function ($q, $http) { function uploadToApi(arrObj){ + console.log("arrObj", arrObj); return $q((resolve, reject)=>{ $http({ method: 'POST', url: 'http://localhost:3000/uploadfile', + // data: {"fake_json": "This is a small string."}, data: arrObj, headers: {'Content-Type': 'application/json'} })