From b3190004f42f4961c107b422d985ef063d63afed Mon Sep 17 00:00:00 2001 From: bill bittner Date: Thu, 29 Jun 2017 14:34:23 -0700 Subject: [PATCH 1/3] changed mysql analytics to 'stats' --- ...lyticsController.js => statsController.js} | 8 ++--- helpers/libraries/errorHandlers.js | 10 +++--- .../{analytics.js => statsHelpers.js} | 6 ++-- models/{analytics.js => stats.js} | 6 ++-- public/assets/css/componentStyle.css | 2 +- routes/analytics-routes.js | 17 ---------- routes/api-routes.js | 16 +++++----- routes/home-routes.js | 4 +-- routes/serve-routes.js | 6 ++-- routes/show-routes.js | 31 +++++++++++++------ routes/sockets-routes.js | 10 +++--- server.js | 1 - ...ytics.handlebars => statistics.handlebars} | 2 +- 13 files changed, 56 insertions(+), 63 deletions(-) rename controllers/{analyticsController.js => statsController.js} (94%) rename helpers/libraries/{analytics.js => statsHelpers.js} (80%) rename models/{analytics.js => stats.js} (86%) delete mode 100644 routes/analytics-routes.js rename views/{analytics.handlebars => statistics.handlebars} (97%) diff --git a/controllers/analyticsController.js b/controllers/statsController.js similarity index 94% rename from controllers/analyticsController.js rename to controllers/statsController.js index 3f702e7d..3aac00df 100644 --- a/controllers/analyticsController.js +++ b/controllers/statsController.js @@ -2,11 +2,11 @@ const logger = require('winston'); const db = require('../models'); module.exports = { - getAnalyticsSummary: () => { - logger.debug('retrieving analytics'); + getStatsSummary: () => { + logger.debug('retrieving site statistics'); const deferred = new Promise((resolve, reject) => { - // get the raw analytics data - db.Analytics + // get the raw statistics data + db.Stats .findAll() .then(data => { const resultHashTable = {}; diff --git a/helpers/libraries/errorHandlers.js b/helpers/libraries/errorHandlers.js index 9e947630..cc057ea5 100644 --- a/helpers/libraries/errorHandlers.js +++ b/helpers/libraries/errorHandlers.js @@ -1,20 +1,20 @@ const logger = require('winston'); -const { postToAnalytics } = require('./analytics'); +const { postToStats } = require('./statsHelpers.js'); module.exports = { handleRequestError (action, originalUrl, ip, error, res) { logger.error('Request Error >>', error); if (error === 'NO_CLAIMS' || error === 'NO_FREE_PUBLIC_CLAIMS') { - postToAnalytics(action, originalUrl, ip, 'success'); + postToStats(action, originalUrl, ip, 'success'); res.status(307).render('noClaims'); } else if (error.response) { - postToAnalytics(action, originalUrl, ip, error.response.data.error.messsage); + postToStats(action, originalUrl, ip, error.response.data.error.messsage); res.status(error.response.status).send(error.response.data.error.message); } else if (error.code === 'ECONNREFUSED') { - postToAnalytics(action, originalUrl, ip, 'Connection refused. The daemon may not be running.'); + postToStats(action, originalUrl, ip, 'Connection refused. The daemon may not be running.'); res.status(503).send('Connection refused. The daemon may not be running.'); } else { - postToAnalytics(action, originalUrl, ip, error); + postToStats(action, originalUrl, ip, error); res.status(400).send(JSON.stringify(error)); } }, diff --git a/helpers/libraries/analytics.js b/helpers/libraries/statsHelpers.js similarity index 80% rename from helpers/libraries/analytics.js rename to helpers/libraries/statsHelpers.js index 84ee2618..f008b081 100644 --- a/helpers/libraries/analytics.js +++ b/helpers/libraries/statsHelpers.js @@ -2,8 +2,8 @@ const db = require('../../models'); const logger = require('winston'); module.exports = { - postToAnalytics: (action, url, ipAddress, result) => { - logger.silly('creating record for analytics'); + postToStats: (action, url, ipAddress, result) => { + logger.silly('creating record for statistics db'); // make sure the result is a string if (result && (typeof result !== 'string')) { result = result.toString(); @@ -13,7 +13,7 @@ module.exports = { ipAddress = ipAddress.toString(); } // create record in the db - db.Analytics.create({ + db.Stats.create({ action, url, ipAddress, diff --git a/models/analytics.js b/models/stats.js similarity index 86% rename from models/analytics.js rename to models/stats.js index e2d5f7e0..931623fe 100644 --- a/models/analytics.js +++ b/models/stats.js @@ -1,6 +1,6 @@ module.exports = (sequelize, { STRING, TEXT }) => { - const Analytics = sequelize.define( - 'Analytics', + const Stats = sequelize.define( + 'Stats', { action: { type : STRING, @@ -25,5 +25,5 @@ module.exports = (sequelize, { STRING, TEXT }) => { freezeTableName: true, } ); - return Analytics; + return Stats; }; diff --git a/public/assets/css/componentStyle.css b/public/assets/css/componentStyle.css index d2d745ba..a371417a 100644 --- a/public/assets/css/componentStyle.css +++ b/public/assets/css/componentStyle.css @@ -65,7 +65,7 @@ canvas { float: left; } -/* analytics */ +/* statistics */ .totals-row { border-top: 1px solid grey; border-bottom: 1px solid grey; diff --git a/routes/analytics-routes.js b/routes/analytics-routes.js deleted file mode 100644 index f5fea1c7..00000000 --- a/routes/analytics-routes.js +++ /dev/null @@ -1,17 +0,0 @@ -const errorHandlers = require('../helpers/libraries/errorHandlers.js'); -const analyticsController = require('../controllers/analyticsController.js'); - -module.exports = (app) => { - // route to show analytics - app.get('/analytics', (req, res) => { - // get and serve content - analyticsController - .getAnalyticsSummary() - .then(result => { - res.status(200).render('analytics', result); - }) - .catch(error => { - errorHandlers.handleRequestError(error, res); - }); - }); -}; diff --git a/routes/api-routes.js b/routes/api-routes.js index 20790dfa..f63d744a 100644 --- a/routes/api-routes.js +++ b/routes/api-routes.js @@ -5,7 +5,7 @@ const publishController = require('../controllers/publishController.js'); const lbryApi = require('../helpers/libraries/lbryApi.js'); const publishHelpers = require('../helpers/libraries/publishHelpers.js'); const errorHandlers = require('../helpers/libraries/errorHandlers.js'); -const { postToAnalytics } = require('../helpers/libraries/analytics'); +const { postToStats } = require('../helpers/libraries/statsHelpers.js'); module.exports = app => { // route to run a claim_list request on the daemon @@ -14,7 +14,7 @@ module.exports = app => { lbryApi .getClaimsList(params.claim) .then(claimsList => { - postToAnalytics('publish', originalUrl, ip, 'success'); + postToStats('publish', originalUrl, ip, 'success'); res.status(200).json(claimsList); }) .catch(error => { @@ -27,7 +27,7 @@ module.exports = app => { lbryApi .resolveUri(params.uri) .then(resolvedUri => { - postToAnalytics('publish', originalUrl, ip, 'success'); + postToStats('publish', originalUrl, ip, 'success'); res.status(200).json(resolvedUri); }) .catch(error => { @@ -40,7 +40,7 @@ module.exports = app => { // validate that a file was provided const file = files.speech || files.null; if (!file) { - postToAnalytics('publish', originalUrl, ip, 'Error: file'); + postToStats('publish', originalUrl, ip, 'Error: file'); res.status(400).send('Error: No file was submitted or the key used was incorrect. Files posted through this route must use a key of "speech" or null'); return; } @@ -48,14 +48,14 @@ module.exports = app => { const name = body.name || file.name.substring(0, file.name.indexOf('.')); const invalidCharacters = /[^\w,-]/.exec(name); if (invalidCharacters) { - postToAnalytics('publish', originalUrl, ip, 'Error: name'); + postToStats('publish', originalUrl, ip, 'Error: name'); res.status(400).send('Error: The name you provided is not allowed. Please use A-Z, a-z, 0-9, "_" and "-" only.'); return; } // validate license const license = body.license || 'No License Provided'; if ((license.indexOf('Public Domain') === -1) && (license.indexOf('Creative Commons') === -1)) { - postToAnalytics('puplish', originalUrl, ip, 'Error: license'); + postToStats('puplish', originalUrl, ip, 'Error: license'); res.status(400).send('Error: Only posts with a license of "Public Domain" or "Creative Commons" are eligible for publishing through spee.ch'); return; } @@ -73,7 +73,7 @@ module.exports = app => { case '1': break; default: - postToAnalytics('publish', originalUrl, ip, 'Error: nsfw'); + postToStats('publish', originalUrl, ip, 'Error: nsfw'); res.status(400).send('Error: NSFW value was not accepted. NSFW must be set to either true, false, "on", or "off"'); return; } @@ -89,7 +89,7 @@ module.exports = app => { publishController .publish(publishParams, fileName, fileType) .then(result => { - postToAnalytics('publish', originalUrl, ip, 'success'); + postToStats('publish', originalUrl, ip, 'success'); res.status(200).json(result); }) .catch(error => { diff --git a/routes/home-routes.js b/routes/home-routes.js index d36d8f6c..6eebf928 100644 --- a/routes/home-routes.js +++ b/routes/home-routes.js @@ -1,5 +1,5 @@ const logger = require('winston'); -const { postToAnalytics } = require('../helpers/libraries/analytics'); +const { postToStats } = require('../helpers/libraries/statsHelpers.js'); module.exports = app => { // route for the home page @@ -11,7 +11,7 @@ module.exports = app => { // a catch-all route if someone visits a page that does not exist app.use('*', ({ originalUrl, ip }, res) => { logger.error(`Get request on ${originalUrl} from ${ip} which was a 404`); - postToAnalytics('post', originalUrl, ip, 'Error: 404'); + postToStats('post', originalUrl, ip, 'Error: 404'); res.status(404).render('fourOhFour'); }); }; diff --git a/routes/serve-routes.js b/routes/serve-routes.js index 16025638..dbabf927 100644 --- a/routes/serve-routes.js +++ b/routes/serve-routes.js @@ -1,7 +1,7 @@ const errorHandlers = require('../helpers/libraries/errorHandlers.js'); const serveController = require('../controllers/serveController.js'); const logger = require('winston'); -const { postToAnalytics } = require('../helpers/libraries/analytics'); +const { postToStats } = require('../helpers/libraries/statsHelpers.js'); function serveFile ({ fileName, fileType, filePath }, res) { logger.info(`serving file ${fileName}`); @@ -42,7 +42,7 @@ module.exports = (app) => { serveController .getClaimByClaimId(params.name, params.claim_id) .then(fileInfo => { - postToAnalytics('serve', originalUrl, ips, 'success'); + postToStats('serve', originalUrl, ips, 'success'); serveFile(fileInfo, res); }) .catch(error => { @@ -58,7 +58,7 @@ module.exports = (app) => { serveController .getClaimByName(params.name) .then(fileInfo => { - postToAnalytics('serve', originalUrl, ips, 'success'); + postToStats('serve', originalUrl, ips, 'success'); serveFile(fileInfo, res); }) .catch(error => { diff --git a/routes/show-routes.js b/routes/show-routes.js index fa6c079f..722daa20 100644 --- a/routes/show-routes.js +++ b/routes/show-routes.js @@ -1,31 +1,42 @@ const errorHandlers = require('../helpers/libraries/errorHandlers.js'); const showController = require('../controllers/showController.js'); -const logger = require('winston'); -const { postToAnalytics } = require('../helpers/libraries/analytics'); +const { postToStats } = require('../helpers/libraries/statsHelpers.js'); +const statsController = require('../controllers/statsController.js'); module.exports = (app) => { - // route to fetch all free public claims + // route to show the meme-fodder meme maker app.get('/meme-fodder/play', ({ originalUrl, ip }, res) => { - logger.debug(`GET request on ${originalUrl} from ${ip}`); - // get and serve content + // get and serve the content showController .getAllClaims('meme-fodder') .then(orderedFreePublicClaims => { - postToAnalytics('show', originalUrl, ip, 'success'); + postToStats('show', originalUrl, ip, 'success'); res.status(200).render('memeFodder', { claims: orderedFreePublicClaims }); }) .catch(error => { errorHandlers.handleRequestError('show', originalUrl, ip, error, res); }); }); - // route to fetch all free public claims + // route to show statistics for spee.ch + app.get('/stats', ({ originalUrl, ip }, res) => { + // get and serve the content + statsController + .getStatsSummary() + .then(result => { + postToStats('show', originalUrl, ip, 'success'); + res.status(200).render('statistics', result); + }) + .catch(error => { + errorHandlers.handleRequestError(error, res); + }); + }); + // route to display all free public claims at a given name app.get('/:name/all', ({ originalUrl, params, ip }, res) => { - logger.debug(`GET request on ${originalUrl} from ${ip}`); - // get and serve content + // get and serve the content showController .getAllClaims(params.name) .then(orderedFreePublicClaims => { - postToAnalytics('show', originalUrl, ip, 'success'); + postToStats('show', originalUrl, ip, 'success'); res.status(200).render('allClaims', { claims: orderedFreePublicClaims }); }) .catch(error => { diff --git a/routes/sockets-routes.js b/routes/sockets-routes.js index b21c5ed9..dfc41b53 100644 --- a/routes/sockets-routes.js +++ b/routes/sockets-routes.js @@ -2,7 +2,7 @@ const logger = require('winston'); const publishController = require('../controllers/publishController.js'); const publishHelpers = require('../helpers/libraries/publishHelpers.js'); const errorHandlers = require('../helpers/libraries/errorHandlers.js'); -const { postToAnalytics } = require('../helpers/libraries/analytics'); +const { postToStats } = require('../helpers/libraries/statsHelpers.js'); module.exports = (app, siofu, hostedContentPath) => { const http = require('http').Server(app); @@ -25,7 +25,7 @@ module.exports = (app, siofu, hostedContentPath) => { // listener for when file upload encounters an error uploader.on('error', ({ error }) => { logger.error('an error occured while uploading', error); - postToAnalytics('publish', '/', null, error); + postToStats('publish', '/', null, error); socket.emit('publish-status', error); }); // listener for when file has been uploaded @@ -39,18 +39,18 @@ module.exports = (app, siofu, hostedContentPath) => { publishController .publish(publishParams, file.name, file.meta.type) .then(result => { - postToAnalytics('publish', '/', null, 'success'); + postToStats('publish', '/', null, 'success'); socket.emit('publish-complete', { name: publishParams.name, result }); }) .catch(error => { error = errorHandlers.handlePublishError(error); - postToAnalytics('publish', '/', null, error); + postToStats('publish', '/', null, error); socket.emit('publish-failure', error); }); } else { logger.error(`An error occurred in uploading the client's file`); socket.emit('publish-failure', 'file uploaded, but with errors'); - postToAnalytics('publish', '/', null, 'file uploaded, but with errors'); + postToStats('publish', '/', null, 'file uploaded, but with errors'); // to-do: remove the file if not done automatically } }); diff --git a/server.js b/server.js index 6d7e9102..0df1158c 100644 --- a/server.js +++ b/server.js @@ -56,7 +56,6 @@ app.set('view engine', 'handlebars'); // require express routes require('./routes/api-routes.js')(app); -require('./routes/analytics-routes.js')(app); require('./routes/show-routes.js')(app); require('./routes/serve-routes.js')(app); require('./routes/home-routes.js')(app); diff --git a/views/analytics.handlebars b/views/statistics.handlebars similarity index 97% rename from views/analytics.handlebars rename to views/statistics.handlebars index dbde1859..f8a440d6 100644 --- a/views/analytics.handlebars +++ b/views/statistics.handlebars @@ -3,7 +3,7 @@ {{> topBar}}
-

Analytics

+

Site Statistics

From 6725345db3a44c396d58a8a315fb4b94efbbbf63 Mon Sep 17 00:00:00 2001 From: bill bittner Date: Thu, 29 Jun 2017 14:48:27 -0700 Subject: [PATCH 2/3] removed the extra ip logs --- routes/api-routes.js | 6 +++--- routes/home-routes.js | 1 - routes/serve-routes.js | 16 ++++++---------- routes/show-routes.js | 4 ++++ 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/routes/api-routes.js b/routes/api-routes.js index f63d744a..3e6cfa91 100644 --- a/routes/api-routes.js +++ b/routes/api-routes.js @@ -10,7 +10,7 @@ const { postToStats } = require('../helpers/libraries/statsHelpers.js'); module.exports = app => { // route to run a claim_list request on the daemon app.get('/api/claim_list/:claim', ({ originalUrl, params, ip }, res) => { - logger.debug(`GET request on ${originalUrl} from ${ip}`); + logger.verbose(`GET request on ${originalUrl} from ${ip}`); lbryApi .getClaimsList(params.claim) .then(claimsList => { @@ -23,7 +23,7 @@ module.exports = app => { }); // route to run a resolve request on the daemon app.get('/api/resolve/:uri', ({ originalUrl, params, ip }, res) => { - logger.debug(`GET request on ${originalUrl} from ${ip}`); + logger.verbose(`GET request on ${originalUrl} from ${ip}`); lbryApi .resolveUri(params.uri) .then(resolvedUri => { @@ -36,7 +36,7 @@ module.exports = app => { }); // route to run a publish request on the daemon app.post('/api/publish', multipartMiddleware, ({ originalUrl, body, files, ip }, res) => { - logger.debug(`POST request on ${originalUrl} from ${ip}`); + logger.verbose(`POST request on ${originalUrl} from ${ip}`); // validate that a file was provided const file = files.speech || files.null; if (!file) { diff --git a/routes/home-routes.js b/routes/home-routes.js index 6eebf928..aa9df045 100644 --- a/routes/home-routes.js +++ b/routes/home-routes.js @@ -5,7 +5,6 @@ module.exports = app => { // route for the home page app.get('/', ({ originalUrl, ip, headers }, res) => { logger.verbose(`GET request on ${originalUrl} from ${ip}`); - logger.debug(`headers ${JSON.stringify(headers)}`); res.status(200).render('index'); }); // a catch-all route if someone visits a page that does not exist diff --git a/routes/serve-routes.js b/routes/serve-routes.js index dbabf927..31cbabb5 100644 --- a/routes/serve-routes.js +++ b/routes/serve-routes.js @@ -34,35 +34,31 @@ function serveFile ({ fileName, fileType, filePath }, res) { module.exports = (app) => { // route to fetch one free public claim - app.get('/:name/:claim_id', ({ originalUrl, params, ip, ips, headers }, res) => { + app.get('/:name/:claim_id', ({ originalUrl, params, ip }, res) => { logger.verbose(`GET request on ${originalUrl} from ${ip}`); - logger.debug(`ips >> ${JSON.stringify(ips)}`); - logger.debug(`headers >> ${JSON.stringify(headers)}`); // begin image-serve processes serveController .getClaimByClaimId(params.name, params.claim_id) .then(fileInfo => { - postToStats('serve', originalUrl, ips, 'success'); + postToStats('serve', originalUrl, ip, 'success'); serveFile(fileInfo, res); }) .catch(error => { - errorHandlers.handleRequestError('serve', originalUrl, ips, error, res); + errorHandlers.handleRequestError('serve', originalUrl, ip, error, res); }); }); // route to fetch one free public claim - app.get('/:name', ({ originalUrl, params, ip, ips, headers }, res) => { + app.get('/:name', ({ originalUrl, params, ip }, res) => { logger.verbose(`GET request on ${originalUrl} from ${ip}`); - logger.debug(`ips >> ${JSON.stringify(ips)}`); - logger.debug(`headers >> ${JSON.stringify(headers)}`); // begin image-serve processes serveController .getClaimByName(params.name) .then(fileInfo => { - postToStats('serve', originalUrl, ips, 'success'); + postToStats('serve', originalUrl, ip, 'success'); serveFile(fileInfo, res); }) .catch(error => { - errorHandlers.handleRequestError('serve', originalUrl, ips, error, res); + errorHandlers.handleRequestError('serve', originalUrl, ip, error, res); }); }); }; diff --git a/routes/show-routes.js b/routes/show-routes.js index 722daa20..7b60bd5f 100644 --- a/routes/show-routes.js +++ b/routes/show-routes.js @@ -1,3 +1,4 @@ +const logger = require('winston'); const errorHandlers = require('../helpers/libraries/errorHandlers.js'); const showController = require('../controllers/showController.js'); const { postToStats } = require('../helpers/libraries/statsHelpers.js'); @@ -6,6 +7,7 @@ const statsController = require('../controllers/statsController.js'); module.exports = (app) => { // route to show the meme-fodder meme maker app.get('/meme-fodder/play', ({ originalUrl, ip }, res) => { + logger.verbose(`POST request on ${originalUrl} from ${ip}`); // get and serve the content showController .getAllClaims('meme-fodder') @@ -19,6 +21,7 @@ module.exports = (app) => { }); // route to show statistics for spee.ch app.get('/stats', ({ originalUrl, ip }, res) => { + logger.verbose(`POST request on ${originalUrl} from ${ip}`); // get and serve the content statsController .getStatsSummary() @@ -32,6 +35,7 @@ module.exports = (app) => { }); // route to display all free public claims at a given name app.get('/:name/all', ({ originalUrl, params, ip }, res) => { + logger.verbose(`POST request on ${originalUrl} from ${ip}`); // get and serve the content showController .getAllClaims(params.name) From 9fcbcd299c3c24c6143d59e26e5bebc8461cfc42 Mon Sep 17 00:00:00 2001 From: bill bittner Date: Thu, 29 Jun 2017 16:10:06 -0700 Subject: [PATCH 3/3] added analytics to serve routes --- routes/serve-routes.js | 22 +++++++++++++++++++++- server.js | 5 +++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/routes/serve-routes.js b/routes/serve-routes.js index 31cbabb5..bc43b1d1 100644 --- a/routes/serve-routes.js +++ b/routes/serve-routes.js @@ -3,6 +3,20 @@ const serveController = require('../controllers/serveController.js'); const logger = require('winston'); const { postToStats } = require('../helpers/libraries/statsHelpers.js'); +function sendGoogleAnalytics (ua, googleApiKey, ip, originalUrl) { + const visitorId = ip.replace(/\./g, '-'); + const visitor = ua(googleApiKey, visitorId, { strictCidFormat: false, https: true }); + visitor.pageview(originalUrl, 'https://spee.ch', 'Serve Route', (err) => { + if (err) { + logger.error('Google Analytics Pageview Error >>', err); + } + }).event('Serve', originalUrl, (err) => { + if (err) { + logger.error('Google Analytics Event Error >>', err); + } + }); +} + function serveFile ({ fileName, fileType, filePath }, res) { logger.info(`serving file ${fileName}`); // set default options @@ -32,9 +46,12 @@ function serveFile ({ fileName, fileType, filePath }, res) { res.status(200).sendFile(filePath, options); } -module.exports = (app) => { +module.exports = (app, ua, googleApiKey) => { // route to fetch one free public claim app.get('/:name/:claim_id', ({ originalUrl, params, ip }, res) => { + // google analytics + sendGoogleAnalytics(ua, googleApiKey, ip, originalUrl); + // logging logger.verbose(`GET request on ${originalUrl} from ${ip}`); // begin image-serve processes serveController @@ -49,6 +66,9 @@ module.exports = (app) => { }); // route to fetch one free public claim app.get('/:name', ({ originalUrl, params, ip }, res) => { + // google analytics + sendGoogleAnalytics(ua, googleApiKey, ip, originalUrl); + // logging logger.verbose(`GET request on ${originalUrl} from ${ip}`); // begin image-serve processes serveController diff --git a/server.js b/server.js index 0df1158c..0e0cda8a 100644 --- a/server.js +++ b/server.js @@ -6,7 +6,8 @@ const expressHandlebars = require('express-handlebars'); const Handlebars = require('handlebars'); const config = require('config'); const winston = require('winston'); - +const ua = require('universal-analytics'); +const googleApiKey = config.get('AnalyticsConfig.GoogleId'); const hostedContentPath = config.get('Database.PublishUploadPath'); // configure logging @@ -57,7 +58,7 @@ app.set('view engine', 'handlebars'); // require express routes require('./routes/api-routes.js')(app); require('./routes/show-routes.js')(app); -require('./routes/serve-routes.js')(app); +require('./routes/serve-routes.js')(app, ua, googleApiKey); require('./routes/home-routes.js')(app); // require socket.io routes
action