From 3fa69fd75e69e446663ad35eec42596e5ae917f5 Mon Sep 17 00:00:00 2001 From: bill bittner Date: Mon, 10 Jul 2017 17:51:29 -0700 Subject: [PATCH] reworked stats db and added trending route --- controllers/serveController.js | 2 + controllers/statsController.js | 82 ++++++++++++++++++++++++++++-- helpers/libraries/errorHandlers.js | 8 +-- models/stats.js | 10 ++++ routes/api-routes.js | 12 ++--- routes/home-routes.js | 2 +- routes/serve-routes.js | 12 ++--- routes/show-routes.js | 42 ++++++++++----- routes/sockets-routes.js | 8 +-- views/trending.handlebars | 26 ++++++++++ 10 files changed, 165 insertions(+), 39 deletions(-) create mode 100644 views/trending.handlebars diff --git a/controllers/serveController.js b/controllers/serveController.js index b3dfb700..845dfb57 100644 --- a/controllers/serveController.js +++ b/controllers/serveController.js @@ -98,6 +98,8 @@ function getClaimAndHandleResponse (uri, address, height, resolve, reject) { }); // resolve the request resolve({ + name, + claimId : claim_id, fileName: file_name, filePath: download_path, fileType: mime_type, diff --git a/controllers/statsController.js b/controllers/statsController.js index 327636da..dee75b72 100644 --- a/controllers/statsController.js +++ b/controllers/statsController.js @@ -5,7 +5,7 @@ const db = require('../models'); const googleApiKey = config.get('AnalyticsConfig.GoogleId'); module.exports = { - postToStats (action, url, ipAddress, result) { + postToStats (action, url, ipAddress, name, claimId, result) { logger.silly(`creating ${action} record for statistics db`); // make sure the result is a string if (result && (typeof result !== 'string')) { @@ -20,6 +20,8 @@ module.exports = { action, url, ipAddress, + name, + claimId, result, }) .then() @@ -58,12 +60,18 @@ module.exports = { } }); }, - getStatsSummary () { - logger.debug('retrieving site statistics'); + getStatsSummary (startDate) { + logger.debug('retrieving statistics'); const deferred = new Promise((resolve, reject) => { // get the raw statistics data db.Stats - .findAll() + .findAll({ + where: { + createdAt: { + gt: startDate, + }, + }, + }) .then(data => { const resultHashTable = {}; let totalServe = 0; @@ -125,4 +133,70 @@ module.exports = { }); return deferred; }, + getTrendingClaims (startDate) { + logger.debug('retrieving trending statistics'); + const deferred = new Promise((resolve, reject) => { + // get the raw statistics data + db.Stats + .findAll({ + where: { + createdAt: { + gt: startDate, + }, + name: { + not: null, + }, + claimId: { + not: null, + }, + }, + }) + .then(data => { + const resultHashTable = {}; + // summarise the data + for (let i = 0; i < data.length; i++) { + let key = `${data[i].name}#${data[i].claimId}`; + logger.debug(key); + console.log(resultHashTable[key]); + if (resultHashTable[key] === undefined) { + // console.log(resultHashTable[key]); + resultHashTable[key] = { + count : 0, + details: { + name : data[i].name, + claimId: data[i].claimId, + }, + }; + } else { + // console.log(resultHashTable[key]); + resultHashTable[key]['count'] += 1; + } + } + // order the results + let sortableArray = []; + for (let objKey in resultHashTable) { + if (resultHashTable.hasOwnProperty(objKey)) { + sortableArray.push([ + resultHashTable[objKey]['count'], + resultHashTable[objKey]['details'], + ]); + } + } + sortableArray.sort((a, b) => { + return a[0] - b[0]; + }); + const sortedArray = sortableArray.map((a) => { + return a[1]; + }); + // return results + logger.debug(sortedArray); + resolve(sortedArray); + }) + .catch(error => { + logger.error('sequelize error', error); + reject(error); + }); + }); + return deferred; + }, }; diff --git a/helpers/libraries/errorHandlers.js b/helpers/libraries/errorHandlers.js index b7ea5a37..188a6757 100644 --- a/helpers/libraries/errorHandlers.js +++ b/helpers/libraries/errorHandlers.js @@ -5,16 +5,16 @@ module.exports = { handleRequestError (action, originalUrl, ip, error, res) { logger.error('Request Error >>', error); if (error.response) { - postToStats(action, originalUrl, ip, error.response.data.error.messsage); + postToStats(action, originalUrl, ip, null, null, error.response.data.error.messsage); res.status(error.response.status).send(error.response.data.error.message); } else if (error.code === 'ECONNREFUSED') { - postToStats(action, originalUrl, ip, 'Connection refused. The daemon may not be running.'); + postToStats(action, originalUrl, ip, null, null, 'Connection refused. The daemon may not be running.'); res.status(503).send('Connection refused. The daemon may not be running.'); } else if (error.message) { - postToStats(action, originalUrl, ip, error); + postToStats(action, originalUrl, ip, null, null, error); res.status(400).send(error.message); } else { - postToStats(action, originalUrl, ip, error); + postToStats(action, originalUrl, ip, null, null, error); res.status(400).send(error); } }, diff --git a/models/stats.js b/models/stats.js index 931623fe..00fef482 100644 --- a/models/stats.js +++ b/models/stats.js @@ -15,6 +15,16 @@ module.exports = (sequelize, { STRING, TEXT }) => { allowNull: true, default : null, }, + name: { + type : STRING, + allowNull: true, + default : null, + }, + claimId: { + type : STRING, + allowNull: true, + default : null, + }, result: { type : TEXT('long'), allowNull: true, diff --git a/routes/api-routes.js b/routes/api-routes.js index 67c82d95..5f8327de 100644 --- a/routes/api-routes.js +++ b/routes/api-routes.js @@ -11,8 +11,8 @@ const config = require('config'); const hostedContentPath = config.get('Database.PublishUploadPath'); module.exports = app => { - // route to run a claim_list request on the daemon - app.get('/api/streamFile/:name', ({ params, headers }, res) => { + // route to return a file directly + app.get('/api/streamFile/:name', ({ params }, res) => { const filePath = `${hostedContentPath}${params.name}`; res.status(200).sendFile(filePath); }); @@ -24,7 +24,7 @@ module.exports = app => { lbryApi .getClaimsList(params.name) .then(claimsList => { - postToStats('serve', originalUrl, ip, 'success'); + postToStats('serve', originalUrl, ip, null, null, 'success'); res.status(200).json(claimsList); }) .catch(error => { @@ -56,7 +56,7 @@ module.exports = app => { lbryApi .resolveUri(params.uri) .then(resolvedUri => { - postToStats('serve', originalUrl, ip, 'success'); + postToStats('serve', originalUrl, ip, null, null, 'success'); res.status(200).json(resolvedUri); }) .catch(error => { @@ -76,7 +76,7 @@ module.exports = app => { try { validateFile(file, name, license, nsfw); } catch (error) { - postToStats('publish', originalUrl, ip, error.message); + postToStats('publish', originalUrl, ip, null, null, error.message); logger.debug('rejected >>', error.message); res.status(400).send(error.message); return; @@ -91,7 +91,7 @@ module.exports = app => { publishController .publish(publishParams, fileName, fileType) .then(result => { - postToStats('publish', originalUrl, ip, 'success'); + postToStats('publish', originalUrl, ip, null, null, 'success'); res.status(200).json(result); }) .catch(error => { diff --git a/routes/home-routes.js b/routes/home-routes.js index 81fd3219..d4d51127 100644 --- a/routes/home-routes.js +++ b/routes/home-routes.js @@ -11,7 +11,7 @@ module.exports = app => { app.use('*', ({ originalUrl, ip }, res) => { logger.error(`404 on ${originalUrl}`); // post to stats - postToStats('show', originalUrl, ip, 'Error: 404'); + postToStats('show', originalUrl, ip, null, null, 'Error: 404'); // send response res.status(404).render('fourOhFour'); }); diff --git a/routes/serve-routes.js b/routes/serve-routes.js index 8df97494..4a6bf307 100644 --- a/routes/serve-routes.js +++ b/routes/serve-routes.js @@ -52,14 +52,14 @@ module.exports = (app) => { if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header const mimetypes = headers['accept'].split(','); if (mimetypes.includes('text/html')) { - postToStats('show', originalUrl, ip, 'success'); + postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); res.status(200).render('showLite', { fileInfo }); } else { - postToStats('serve', originalUrl, ip, 'success'); + postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); serveFile(fileInfo, res); } } else { - postToStats('serve', originalUrl, ip, 'success'); + postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); serveFile(fileInfo, res); } }) @@ -82,14 +82,14 @@ module.exports = (app) => { if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header const mimetypes = headers['accept'].split(','); if (mimetypes.includes('text/html')) { - postToStats('show', originalUrl, ip, 'success'); + postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); res.status(200).render('showLite', { fileInfo }); } else { - postToStats('serve', originalUrl, ip, 'success'); + postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); serveFile(fileInfo, res); } } else { - postToStats('serve', originalUrl, ip, 'success'); + postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); serveFile(fileInfo, res); } }) diff --git a/routes/show-routes.js b/routes/show-routes.js index d662d6c9..763822b2 100644 --- a/routes/show-routes.js +++ b/routes/show-routes.js @@ -1,6 +1,6 @@ const errorHandlers = require('../helpers/libraries/errorHandlers.js'); const { getClaimByClaimId, getClaimByName, getAllClaims } = require('../controllers/serveController.js'); -const { getStatsSummary, postToStats } = require('../controllers/statsController.js'); +const { postToStats, getStatsSummary, getTrendingClaims } = require('../controllers/statsController.js'); module.exports = (app) => { // route to show 'about' page for spee.ch @@ -8,30 +8,44 @@ module.exports = (app) => { // get and render the content res.status(200).render('about'); }); - // route to show the meme-fodder meme maker - app.get('/meme-fodder/play', ({ ip, originalUrl }, res) => { - // get and render the content - getAllClaims('meme-fodder') - .then(orderedFreePublicClaims => { - postToStats('show', originalUrl, ip, 'success'); - res.status(200).render('memeFodder', { claims: orderedFreePublicClaims }); + // route to display a list of the trending images + app.get('/trending', ({ params, headers }, res) => { + const startDate = new Date(); + startDate.setDate(startDate.getDate() - 1); + getTrendingClaims(startDate) + .then(result => { + res.status(200).render('trending', { trendingAssets: result }); }) .catch(error => { - errorHandlers.handleRequestError('show', originalUrl, ip, error, res); + errorHandlers.handleRequestError(error, res); }); }); // route to show statistics for spee.ch app.get('/stats', ({ ip, originalUrl }, res) => { // get and render the content - getStatsSummary() + const startDate = new Date(); + startDate.setDate(startDate.getDate() - 1); + getStatsSummary(startDate) .then(result => { - postToStats('show', originalUrl, ip, 'success'); + postToStats('show', originalUrl, ip, null, null, 'success'); res.status(200).render('statistics', result); }) .catch(error => { errorHandlers.handleRequestError(error, res); }); }); + // route to show the meme-fodder meme maker + app.get('/meme-fodder/play', ({ ip, originalUrl }, res) => { + // get and render the content + getAllClaims('meme-fodder') + .then(orderedFreePublicClaims => { + postToStats('show', originalUrl, ip, null, null, 'success'); + res.status(200).render('memeFodder', { claims: orderedFreePublicClaims }); + }) + .catch(error => { + errorHandlers.handleRequestError('show', originalUrl, ip, error, res); + }); + }); // route to display all free public claims at a given name app.get('/:name/all', ({ ip, originalUrl, params }, res) => { // get and render the content @@ -41,7 +55,7 @@ module.exports = (app) => { res.status(307).render('noClaims'); return; } - postToStats('show', originalUrl, ip, 'success'); + postToStats('show', originalUrl, ip, null, null, 'success'); res.status(200).render('allClaims', { claims: orderedFreePublicClaims }); }) .catch(error => { @@ -59,7 +73,7 @@ module.exports = (app) => { return; } // serve the file or the show route - postToStats('show', originalUrl, ip, 'success'); + postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); res.status(200).render('show', { fileInfo }); }) .catch(error => { @@ -77,7 +91,7 @@ module.exports = (app) => { return; } // serve the show route - postToStats('show', originalUrl, ip, 'success'); + postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success'); res.status(200).render('show', { fileInfo }); }) .catch(error => { diff --git a/routes/sockets-routes.js b/routes/sockets-routes.js index 0a17e56f..bb3d23b3 100644 --- a/routes/sockets-routes.js +++ b/routes/sockets-routes.js @@ -27,7 +27,7 @@ module.exports = (app, siofu, hostedContentPath) => { // listener for when file upload encounters an error uploader.on('error', ({ error }) => { logger.error('an error occured while uploading', error); - postToStats('publish', '/', null, error); + postToStats('publish', '/', null, null, error); socket.emit('publish-status', error); }); // listener for when file has been uploaded @@ -41,18 +41,18 @@ module.exports = (app, siofu, hostedContentPath) => { publishController .publish(publishParams, file.name, file.meta.type) .then(result => { - postToStats('publish', '/', null, 'success'); + postToStats('publish', '/', null, null, 'success'); socket.emit('publish-complete', { name: publishParams.name, result }); }) .catch(error => { error = errorHandlers.handlePublishError(error); - postToStats('publish', '/', null, error); + postToStats('publish', '/', null, null, error); socket.emit('publish-failure', error); }); } else { logger.error(`An error occurred in uploading the client's file`); socket.emit('publish-failure', 'File uploaded, but with errors'); - postToStats('publish', '/', null, 'File uploaded, but with errors'); + postToStats('publish', '/', null, null, 'File uploaded, but with errors'); // to-do: remove the file if not done automatically } }); diff --git a/views/trending.handlebars b/views/trending.handlebars new file mode 100644 index 00000000..4fb91ac7 --- /dev/null +++ b/views/trending.handlebars @@ -0,0 +1,26 @@ +
+ {{> topBar}} +
+

Trending Images

+ {{#each trendingAssets}} + + + {{/each}} +
+
+ + + + + +