From 99c7c84cfb10a097fc9dc8f08ab5bb0a5af525f4 Mon Sep 17 00:00:00 2001 From: bill bittner Date: Wed, 12 Jul 2017 15:30:31 -0700 Subject: [PATCH 1/3] associated the File and Request dbs --- config/development.json | 2 +- controllers/statsController.js | 136 +++++++++++++++-------------- helpers/libraries/errorHandlers.js | 8 +- models/file.js | 6 ++ models/index.js | 14 ++- models/{stats.js => request.js} | 37 +++----- routes/api-routes.js | 8 +- routes/home-routes.js | 2 +- routes/serve-routes.js | 12 +-- routes/show-routes.js | 10 +-- routes/sockets-routes.js | 8 +- views/index.handlebars | 1 - 12 files changed, 124 insertions(+), 120 deletions(-) rename models/{stats.js => request.js} (54%) diff --git a/config/development.json b/config/development.json index 44631dd2..32f7e96e 100644 --- a/config/development.json +++ b/config/development.json @@ -9,7 +9,7 @@ }, "Database": { "MySqlConnectionUri": "none", - "DownloadDirectory": "/home/ubuntu/Downloads/" + "DownloadDirectory": "C:\\Users\\Bones\\Downloads\\lbry\\" }, "Logging": { "LogLevel": "silly" diff --git a/controllers/statsController.js b/controllers/statsController.js index 1242a86a..4a8ebcdc 100644 --- a/controllers/statsController.js +++ b/controllers/statsController.js @@ -5,8 +5,7 @@ const db = require('../models'); const googleApiKey = config.get('AnalyticsConfig.GoogleId'); module.exports = { - postToStats (action, url, ipAddress, name, claimId, fileName, fileType, nsfw, result) { - logger.silly(`creating ${action} record for statistics db`); + postToStats (action, url, ipAddress, name, claimId, result) { // make sure the result is a string if (result && (typeof result !== 'string')) { result = result.toString(); @@ -15,22 +14,30 @@ module.exports = { if (ipAddress && (typeof ipAddress !== 'string')) { ipAddress = ipAddress.toString(); } - // create record in the db - db.Stats.create({ - action, - url, - ipAddress, - name, - claimId, - fileName, - fileType, - nsfw, - result, - }) - .then() - .catch(error => { - logger.error('sequelize error', error); - }); + logger.silly(name, claimId); + db.File + .findOne({where: { name, claimId }}) + .then(file => { + // create record in the db + let FileId; + if (file) { + FileId = file.dataValues.id; + } else { + FileId = null; + } + logger.silly('file id:', FileId); + return db.Request + .create({ + action, + url, + ipAddress, + result, + FileId, + }); + }) + .catch(error => { + logger.error('sequelize error', error); + }); }, sendGoogleAnalytics (action, headers, ip, originalUrl) { const visitorId = ip.replace(/\./g, '-'); @@ -64,10 +71,10 @@ module.exports = { }); }, getStatsSummary (startDate) { - logger.debug('retrieving statistics'); + logger.debug('retrieving request records'); const deferred = new Promise((resolve, reject) => { - // get the raw statistics data - db.Stats + // get the raw Requests data + db.Request .findAll({ where: { createdAt: { @@ -138,61 +145,56 @@ module.exports = { return deferred; }, getTrendingClaims (startDate) { - logger.debug('retrieving trending statistics'); + logger.debug('retrieving trending requests'); const deferred = new Promise((resolve, reject) => { - // get the raw statistics data - db.Stats + // get the raw requests data + db.Request .findAll({ where: { createdAt: { gt: startDate, }, - name: { - not: null, - }, - claimId: { - not: null, - }, }, + include: [db.File], }) .then(data => { - let resultHashTable = {}; - let sortableArray = []; - let sortedArray; - // summarise the data - for (let i = 0; i < data.length; i++) { - let key = `${data[i].name}#${data[i].claimId}`; - if (resultHashTable[key] === undefined) { - resultHashTable[key] = { - count : 0, - details: { - name : data[i].name, - claimId : data[i].claimId, - fileName: data[i].fileName, - fileType: data[i].fileType, - nsfw : data[i].nsfw, - }, - }; - } else { - resultHashTable[key]['count'] += 1; - } - } - for (let objKey in resultHashTable) { - if (resultHashTable.hasOwnProperty(objKey)) { - sortableArray.push([ - resultHashTable[objKey]['count'], - resultHashTable[objKey]['details'], - ]); - } - } - sortableArray.sort((a, b) => { - return b[0] - a[0]; - }); - sortedArray = sortableArray.map((a) => { - return a[1]; - }); - // return results - resolve(sortedArray); + // let resultHashTable = {}; + // let sortableArray = []; + // let sortedArray; + // // summarise the data + // for (let i = 0; i < data.length; i++) { + // let key = data[i].fileId; + // if (resultHashTable[key] === undefined) { + // resultHashTable[key] = { + // count : 0, + // details: { + // name : data[i].name, + // claimId : data[i].claimId, + // fileName: data[i].fileName, + // fileType: data[i].fileType, + // nsfw : data[i].nsfw, + // }, + // }; + // } else { + // resultHashTable[key]['count'] += 1; + // } + // } + // for (let objKey in resultHashTable) { + // if (resultHashTable.hasOwnProperty(objKey)) { + // sortableArray.push([ + // resultHashTable[objKey]['count'], + // resultHashTable[objKey]['details'], + // ]); + // } + // } + // sortableArray.sort((a, b) => { + // return b[0] - a[0]; + // }); + // sortedArray = sortableArray.map((a) => { + // return a[1]; + // }); + // // return results + resolve(); }) .catch(error => { logger.error('sequelize error', error); diff --git a/helpers/libraries/errorHandlers.js b/helpers/libraries/errorHandlers.js index a256b048..188a6757 100644 --- a/helpers/libraries/errorHandlers.js +++ b/helpers/libraries/errorHandlers.js @@ -5,16 +5,16 @@ module.exports = { handleRequestError (action, originalUrl, ip, error, res) { logger.error('Request Error >>', error); if (error.response) { - postToStats(action, originalUrl, ip, null, null, null, null, null, error.response.data.error.messsage); + postToStats(action, originalUrl, ip, null, null, error.response.data.error.messsage); res.status(error.response.status).send(error.response.data.error.message); } else if (error.code === 'ECONNREFUSED') { - postToStats(action, originalUrl, ip, null, null, null, null, null, 'Connection refused. The daemon may not be running.'); + postToStats(action, originalUrl, ip, null, null, 'Connection refused. The daemon may not be running.'); res.status(503).send('Connection refused. The daemon may not be running.'); } else if (error.message) { - postToStats(action, originalUrl, ip, null, null, null, null, null, error); + postToStats(action, originalUrl, ip, null, null, error); res.status(400).send(error.message); } else { - postToStats(action, originalUrl, ip, null, null, null, null, null, error); + postToStats(action, originalUrl, ip, null, null, error); res.status(400).send(error); } }, diff --git a/models/file.js b/models/file.js index fc303c93..36fa4937 100644 --- a/models/file.js +++ b/models/file.js @@ -44,5 +44,11 @@ module.exports = (sequelize, { STRING, BOOLEAN, INTEGER }) => { freezeTableName: true, } ); + + File.associate = db => { + console.log('test'); + File.hasMany(db.Request); + }; + return File; }; diff --git a/models/index.js b/models/index.js index 0b64bb69..2eab32b6 100644 --- a/models/index.js +++ b/models/index.js @@ -20,13 +20,19 @@ sequelize logger.error('Sequelize was unable to connect to the database:', err); }); -fs.readdirSync(__dirname).filter(file => file.indexOf('.') !== 0 && file !== basename && file.slice(-3) === '.js').forEach(file => { - const model = sequelize['import'](path.join(__dirname, file)); - db[model.name] = model; -}); +fs + .readdirSync(__dirname) + .filter(file => { + return (file.indexOf('.') !== 0 && file !== basename && file.slice(-3) === '.js'); + }) + .forEach(file => { + const model = sequelize['import'](path.join(__dirname, file)); + db[model.name] = model; + }); Object.keys(db).forEach(modelName => { if (db[modelName].associate) { + logger.verbose('associating', modelName); db[modelName].associate(db); } }); diff --git a/models/stats.js b/models/request.js similarity index 54% rename from models/stats.js rename to models/request.js index a18ee472..0918603d 100644 --- a/models/stats.js +++ b/models/request.js @@ -1,6 +1,6 @@ module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => { - const Stats = sequelize.define( - 'Stats', + const Request = sequelize.define( + 'Request', { action: { type : STRING, @@ -14,26 +14,6 @@ module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => { type : STRING, allowNull: true, }, - name: { - type : STRING, - allowNull: true, - }, - claimId: { - type : STRING, - allowNull: true, - }, - fileName: { - type : STRING, - allowNull: true, - }, - fileType: { - type : STRING, - allowNull: true, - }, - nsfw: { - type : BOOLEAN, - allowNull: true, - }, result: { type : TEXT('long'), allowNull: true, @@ -44,5 +24,16 @@ module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => { freezeTableName: true, } ); - return Stats; + + Request.associate = db => { + console.log('test'); + Request.belongsTo(db.File, { + onDelete : 'cascade', + foreignKey: { + allowNull: true, + }, + }); + }; + + return Request; }; diff --git a/routes/api-routes.js b/routes/api-routes.js index 7c80c058..5fb910dc 100644 --- a/routes/api-routes.js +++ b/routes/api-routes.js @@ -24,7 +24,7 @@ module.exports = app => { lbryApi .getClaimsList(params.name) .then(claimsList => { - postToStats('serve', originalUrl, ip, null, null, null, null, 'success'); + postToStats('serve', originalUrl, ip, null, null, 'success'); res.status(200).json(claimsList); }) .catch(error => { @@ -56,7 +56,7 @@ module.exports = app => { lbryApi .resolveUri(params.uri) .then(resolvedUri => { - postToStats('serve', originalUrl, ip, null, null, null, null, 'success'); + postToStats('serve', originalUrl, ip, null, null, 'success'); res.status(200).json(resolvedUri); }) .catch(error => { @@ -76,7 +76,7 @@ module.exports = app => { try { validateFile(file, name, license, nsfw); } catch (error) { - postToStats('publish', originalUrl, ip, null, null, null, null, error.message); + postToStats('publish', originalUrl, ip, null, null, error.message); logger.debug('rejected >>', error.message); res.status(400).send(error.message); return; @@ -91,7 +91,7 @@ module.exports = app => { publishController .publish(publishParams, fileName, fileType) .then(result => { - postToStats('publish', originalUrl, ip, null, null, null, null, 'success'); + postToStats('publish', originalUrl, ip, null, null, 'success'); res.status(200).json(result); }) .catch(error => { diff --git a/routes/home-routes.js b/routes/home-routes.js index ffe5ec78..d4d51127 100644 --- a/routes/home-routes.js +++ b/routes/home-routes.js @@ -11,7 +11,7 @@ module.exports = app => { app.use('*', ({ originalUrl, ip }, res) => { logger.error(`404 on ${originalUrl}`); // post to stats - postToStats('show', originalUrl, ip, null, null, null, null, null, 'Error: 404'); + postToStats('show', originalUrl, ip, null, null, 'Error: 404'); // send response res.status(404).render('fourOhFour'); }); diff --git a/routes/serve-routes.js b/routes/serve-routes.js index d8608d61..4a6bf307 100644 --- a/routes/serve-routes.js +++ b/routes/serve-routes.js @@ -52,14 +52,14 @@ module.exports = (app) => { if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header const mimetypes = headers['accept'].split(','); if (mimetypes.includes('text/html')) { - postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); + postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); res.status(200).render('showLite', { fileInfo }); } else { - postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); + postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); serveFile(fileInfo, res); } } else { - postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); + postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); serveFile(fileInfo, res); } }) @@ -82,14 +82,14 @@ module.exports = (app) => { if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header const mimetypes = headers['accept'].split(','); if (mimetypes.includes('text/html')) { - postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); + postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); res.status(200).render('showLite', { fileInfo }); } else { - postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); + postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); serveFile(fileInfo, res); } } else { - postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); + postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); serveFile(fileInfo, res); } }) diff --git a/routes/show-routes.js b/routes/show-routes.js index 8f69270a..763822b2 100644 --- a/routes/show-routes.js +++ b/routes/show-routes.js @@ -27,7 +27,7 @@ module.exports = (app) => { startDate.setDate(startDate.getDate() - 1); getStatsSummary(startDate) .then(result => { - postToStats('show', originalUrl, ip, null, null, null, null, null, 'success'); + postToStats('show', originalUrl, ip, null, null, 'success'); res.status(200).render('statistics', result); }) .catch(error => { @@ -39,7 +39,7 @@ module.exports = (app) => { // get and render the content getAllClaims('meme-fodder') .then(orderedFreePublicClaims => { - postToStats('show', originalUrl, ip, null, null, null, null, null, 'success'); + postToStats('show', originalUrl, ip, null, null, 'success'); res.status(200).render('memeFodder', { claims: orderedFreePublicClaims }); }) .catch(error => { @@ -55,7 +55,7 @@ module.exports = (app) => { res.status(307).render('noClaims'); return; } - postToStats('show', originalUrl, ip, null, null, null, null, null, 'success'); + postToStats('show', originalUrl, ip, null, null, 'success'); res.status(200).render('allClaims', { claims: orderedFreePublicClaims }); }) .catch(error => { @@ -73,7 +73,7 @@ module.exports = (app) => { return; } // serve the file or the show route - postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); + postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); res.status(200).render('show', { fileInfo }); }) .catch(error => { @@ -91,7 +91,7 @@ module.exports = (app) => { return; } // serve the show route - postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); + postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); res.status(200).render('show', { fileInfo }); }) .catch(error => { diff --git a/routes/sockets-routes.js b/routes/sockets-routes.js index 614a709d..bb3d23b3 100644 --- a/routes/sockets-routes.js +++ b/routes/sockets-routes.js @@ -27,7 +27,7 @@ module.exports = (app, siofu, hostedContentPath) => { // listener for when file upload encounters an error uploader.on('error', ({ error }) => { logger.error('an error occured while uploading', error); - postToStats('publish', '/', null, null, null, null, null, error); + postToStats('publish', '/', null, null, error); socket.emit('publish-status', error); }); // listener for when file has been uploaded @@ -41,18 +41,18 @@ module.exports = (app, siofu, hostedContentPath) => { publishController .publish(publishParams, file.name, file.meta.type) .then(result => { - postToStats('publish', '/', null, null, null, null, null, 'success'); + postToStats('publish', '/', null, null, 'success'); socket.emit('publish-complete', { name: publishParams.name, result }); }) .catch(error => { error = errorHandlers.handlePublishError(error); - postToStats('publish', '/', null, null, null, null, null, error); + postToStats('publish', '/', null, null, error); socket.emit('publish-failure', error); }); } else { logger.error(`An error occurred in uploading the client's file`); socket.emit('publish-failure', 'File uploaded, but with errors'); - postToStats('publish', '/', null, null, null, null, null, 'File uploaded, but with errors'); + postToStats('publish', '/', null, null, 'File uploaded, but with errors'); // to-do: remove the file if not done automatically } }); diff --git a/views/index.handlebars b/views/index.handlebars index e3b87ed4..2e53b4be 100644 --- a/views/index.handlebars +++ b/views/index.handlebars @@ -4,7 +4,6 @@ {{> publish}} {{> learnMore}} - {{> footer}} From 45ce1c44c6e3c22110a2b2cdfcf91dda48718852 Mon Sep 17 00:00:00 2001 From: bill bittner Date: Wed, 12 Jul 2017 16:57:12 -0700 Subject: [PATCH 2/3] changed model logging --- controllers/statsController.js | 6 ++++++ models/file.js | 1 - models/index.js | 2 +- models/request.js | 1 - 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/controllers/statsController.js b/controllers/statsController.js index 4a8ebcdc..acfcddc5 100644 --- a/controllers/statsController.js +++ b/controllers/statsController.js @@ -154,10 +154,16 @@ module.exports = { createdAt: { gt: startDate, }, + FileId: { + not: null, + }, }, include: [db.File], }) .then(data => { + if (data) { + logger.debug(data[0].File.name); + } // let resultHashTable = {}; // let sortableArray = []; // let sortedArray; diff --git a/models/file.js b/models/file.js index 36fa4937..5c8ff364 100644 --- a/models/file.js +++ b/models/file.js @@ -46,7 +46,6 @@ module.exports = (sequelize, { STRING, BOOLEAN, INTEGER }) => { ); File.associate = db => { - console.log('test'); File.hasMany(db.Request); }; diff --git a/models/index.js b/models/index.js index 2eab32b6..16b5c3db 100644 --- a/models/index.js +++ b/models/index.js @@ -32,7 +32,7 @@ fs Object.keys(db).forEach(modelName => { if (db[modelName].associate) { - logger.verbose('associating', modelName); + logger.info('Associating model:', modelName); db[modelName].associate(db); } }); diff --git a/models/request.js b/models/request.js index 0918603d..846e12c6 100644 --- a/models/request.js +++ b/models/request.js @@ -26,7 +26,6 @@ module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => { ); Request.associate = db => { - console.log('test'); Request.belongsTo(db.File, { onDelete : 'cascade', foreignKey: { From 02d909a73852de588b942be4d5ca0feeb24d18b9 Mon Sep 17 00:00:00 2001 From: bill bittner Date: Wed, 12 Jul 2017 23:37:38 -0700 Subject: [PATCH 3/3] switched trending images controller function to sql query --- config/development.json | 2 +- controllers/statsController.js | 57 +++------------------------------- 2 files changed, 5 insertions(+), 54 deletions(-) diff --git a/config/development.json b/config/development.json index 32f7e96e..81f05f1f 100644 --- a/config/development.json +++ b/config/development.json @@ -9,7 +9,7 @@ }, "Database": { "MySqlConnectionUri": "none", - "DownloadDirectory": "C:\\Users\\Bones\\Downloads\\lbry\\" + "DownloadDirectory": "/home/lbry/Downloads/" }, "Logging": { "LogLevel": "silly" diff --git a/controllers/statsController.js b/controllers/statsController.js index acfcddc5..49eeefc0 100644 --- a/controllers/statsController.js +++ b/controllers/statsController.js @@ -148,59 +148,10 @@ module.exports = { logger.debug('retrieving trending requests'); const deferred = new Promise((resolve, reject) => { // get the raw requests data - db.Request - .findAll({ - where: { - createdAt: { - gt: startDate, - }, - FileId: { - not: null, - }, - }, - include: [db.File], - }) - .then(data => { - if (data) { - logger.debug(data[0].File.name); - } - // let resultHashTable = {}; - // let sortableArray = []; - // let sortedArray; - // // summarise the data - // for (let i = 0; i < data.length; i++) { - // let key = data[i].fileId; - // if (resultHashTable[key] === undefined) { - // resultHashTable[key] = { - // count : 0, - // details: { - // name : data[i].name, - // claimId : data[i].claimId, - // fileName: data[i].fileName, - // fileType: data[i].fileType, - // nsfw : data[i].nsfw, - // }, - // }; - // } else { - // resultHashTable[key]['count'] += 1; - // } - // } - // for (let objKey in resultHashTable) { - // if (resultHashTable.hasOwnProperty(objKey)) { - // sortableArray.push([ - // resultHashTable[objKey]['count'], - // resultHashTable[objKey]['details'], - // ]); - // } - // } - // sortableArray.sort((a, b) => { - // return b[0] - a[0]; - // }); - // sortedArray = sortableArray.map((a) => { - // return a[1]; - // }); - // // return results - resolve(); + db.sequelize + .query('SELECT COUNT(*), file.* FROM request LEFT JOIN file ON request.FileId = file.id WHERE FileId != "null" AND nsfw != 1 GROUP BY FileId ORDER BY COUNT(*) DESC LIMIT 25;', { type: db.sequelize.QueryTypes.SELECT }) + .then(results => { + resolve(results); }) .catch(error => { logger.error('sequelize error', error);