From 84c0c37de3ee9f660c4c505893b9a9e3153f1e7d Mon Sep 17 00:00:00 2001 From: bill bittner Date: Tue, 27 Jun 2017 15:53:53 -0700 Subject: [PATCH] replaced server-side google analytics with mysql --- controllers/publishController.js | 13 +++---------- helpers/libraries/analytics.js | 22 ++++++++++++++++++++++ helpers/libraries/publishHelpers.js | 10 ++++++++-- models/usage.js | 27 +++++++++++++++++++++++++++ routes/api-routes.js | 23 +++++++++++++++++------ routes/home-routes.js | 9 ++++++--- routes/serve-routes.js | 20 ++++++++++---------- routes/show-routes.js | 16 +++++++++------- routes/sockets-routes.js | 11 ++++++++--- 9 files changed, 110 insertions(+), 41 deletions(-) create mode 100644 helpers/libraries/analytics.js create mode 100644 models/usage.js diff --git a/controllers/publishController.js b/controllers/publishController.js index 51d5ddfc..e6474276 100644 --- a/controllers/publishController.js +++ b/controllers/publishController.js @@ -1,14 +1,7 @@ -const fs = require('fs'); const logger = require('winston'); -const lbryApi = require('../helpers/libraries/lbryApi.js'); const db = require('../models'); - -function deleteTemporaryFile (filePath) { - fs.unlink(filePath, err => { - if (err) throw err; - logger.debug(`successfully deleted ${filePath}`); - }); -} +const lbryApi = require('../helpers/libraries/lbryApi.js'); +const publishHelpers = require('../helpers/libraries/publishHelpers.js'); function upsert (Model, values, condition) { return Model @@ -59,7 +52,7 @@ module.exports = { .catch(error => { logger.error(`Error publishing ${fileName}`, error); // delete the local file - deleteTemporaryFile(publishParams.file_path); + publishHelpers.deleteTemporaryFile(publishParams.file_path); // reject the promise reject(error); }); diff --git a/helpers/libraries/analytics.js b/helpers/libraries/analytics.js new file mode 100644 index 00000000..62296765 --- /dev/null +++ b/helpers/libraries/analytics.js @@ -0,0 +1,22 @@ +const db = require('../../models'); + +function createAnalyticsRecord (action, url, ipAddress, result) { + db.Usage.create({ + action, + url, + ipAddress, + result, + }); +}; + +module.exports = { + postRequestAnalytics (url, ipAddress, result) { + createAnalyticsRecord('request', url, ipAddress, result); + }, + postPublishAnalytics (url, ipAddress, result) { + createAnalyticsRecord('publish', url, ipAddress, result); + }, + postShowAnalytics (url, ipAddress, result) { + createAnalyticsRecord('show', url, ipAddress, result); + }, +}; diff --git a/helpers/libraries/publishHelpers.js b/helpers/libraries/publishHelpers.js index 60ab5eb3..57a87208 100644 --- a/helpers/libraries/publishHelpers.js +++ b/helpers/libraries/publishHelpers.js @@ -1,10 +1,10 @@ const logger = require('winston'); - const config = require('config'); const walletAddress = config.get('WalletConfig.LbryAddress'); +const fs = require('fs'); module.exports = { - createPublishParams (name, filePath, license, nsfw) { + createPublishParams: (name, filePath, license, nsfw) => { logger.debug(`Creating Publish Parameters for "${name}"`); // ensure nsfw is a boolean if (nsfw === false) { @@ -38,4 +38,10 @@ module.exports = { logger.debug('publishParams:', publishParams); return publishParams; }, + deleteTemporaryFile: (filePath) => { + fs.unlink(filePath, err => { + if (err) throw err; + logger.debug(`successfully deleted ${filePath}`); + }); + }, }; diff --git a/models/usage.js b/models/usage.js new file mode 100644 index 00000000..5c161597 --- /dev/null +++ b/models/usage.js @@ -0,0 +1,27 @@ +module.exports = (sequelize, { STRING }) => { + const Usage = sequelize.define( + 'Usage', + { + action: { + type : STRING, + allowNull: false, + }, + url: { + type : STRING, + allowNull: false, + }, + ipAddress: { + type : STRING, + allowNull: true, + }, + result: { + type : STRING, + allowNull: false, + }, + }, + { + freezeTableName: true, + } + ); + return Usage; +}; diff --git a/routes/api-routes.js b/routes/api-routes.js index eadb3c05..247d581f 100644 --- a/routes/api-routes.js +++ b/routes/api-routes.js @@ -5,38 +5,44 @@ const publishController = require('../controllers/publishController.js'); const lbryApi = require('../helpers/libraries/lbryApi.js'); const publishHelpers = require('../helpers/libraries/publishHelpers.js'); const errorHandlers = require('../helpers/libraries/errorHandlers.js'); +const { postRequestAnalytics, postPublishAnalytics } = require('../helpers/libraries/analytics'); module.exports = app => { // route to run a claim_list request on the daemon - app.get('/api/claim_list/:claim', ({ originalUrl, params }, res) => { - logger.debug(`GET request on ${originalUrl}`); + app.get('/api/claim_list/:claim', ({ originalUrl, params, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); lbryApi .getClaimsList(params.claim) .then(claimsList => { + postRequestAnalytics(originalUrl, ip, 'success'); res.status(200).json(claimsList); }) .catch(error => { + postRequestAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); // route to run a resolve request on the daemon - app.get('/api/resolve/:uri', ({ originalUrl, params }, res) => { - logger.debug(`GET request on ${originalUrl}`); + app.get('/api/resolve/:uri', ({ originalUrl, params, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); lbryApi .resolveUri(params.uri) .then(resolvedUri => { + postRequestAnalytics(originalUrl, ip, 'success'); res.status(200).json(resolvedUri); }) .catch(error => { + postRequestAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); // route to run a publish request on the daemon - app.post('/api/publish', multipartMiddleware, ({ originalUrl, body, files }, res) => { - logger.debug(`POST request on ${originalUrl}`); + app.post('/api/publish', multipartMiddleware, ({ originalUrl, body, files, ip }, res) => { + logger.debug(`POST request on ${originalUrl} from ${ip}`); // validate that a file was provided const file = files.speech || files.null; if (!file) { + postPublishAnalytics(originalUrl, ip, 'Error: file'); res.status(400).send('Error: No file was submitted or the key used was incorrect. Files posted through this route must use a key of "speech" or null'); return; } @@ -44,12 +50,14 @@ module.exports = app => { const name = body.name || file.name.substring(0, file.name.indexOf('.')); const invalidCharacters = /[^\w,-]/.exec(name); if (invalidCharacters) { + postPublishAnalytics(originalUrl, ip, 'Error: name'); res.status(400).send('Error: The name you provided is not allowed. Please use A-Z, a-z, 0-9, "_" and "-" only.'); return; } // validate license const license = body.license || 'No License Provided'; if ((license.indexOf('Public Domain') === -1) && (license.indexOf('Creative Commons') === -1)) { + postPublishAnalytics(originalUrl, ip, 'Error: license'); res.status(400).send('Error: Only posts with a license of "Public Domain" or "Creative Commons" are eligible for publishing through spee.ch'); return; } @@ -67,6 +75,7 @@ module.exports = app => { case '1': break; default: + postPublishAnalytics(originalUrl, ip, 'Error: nsfw'); res.status(400).send('Error: NSFW value was not accepted. NSFW must be set to either true, false, "on", or "off"'); return; } @@ -82,9 +91,11 @@ module.exports = app => { publishController .publish(publishParams, fileName, fileType) .then(result => { + postPublishAnalytics(originalUrl, ip, 'success'); res.status(200).json(result); }) .catch(error => { + postPublishAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); diff --git a/routes/home-routes.js b/routes/home-routes.js index 09c40644..f04e78f5 100644 --- a/routes/home-routes.js +++ b/routes/home-routes.js @@ -1,13 +1,16 @@ const logger = require('winston'); +const { postShowAnalytics } = require('../helpers/libraries/analytics'); module.exports = app => { // route for the home page - app.get('/', (req, res) => { + app.get('/', ({ originalUrl, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); res.status(200).render('index'); }); // a catch-all route if someone visits a page that does not exist - app.use('*', (req, res) => { - logger.error(`Get request on ${req.originalUrl} which was a 404`); + app.use('*', ({ originalUrl, ip }, res) => { + logger.error(`Get request on ${originalUrl} from ${ip} which was a 404`); + postShowAnalytics(originalUrl, ip, 'Error: 404'); res.status(404).render('fourOhFour'); }); }; diff --git a/routes/serve-routes.js b/routes/serve-routes.js index c3d25814..bfce7db1 100644 --- a/routes/serve-routes.js +++ b/routes/serve-routes.js @@ -1,6 +1,7 @@ const errorHandlers = require('../helpers/libraries/errorHandlers.js'); const serveController = require('../controllers/serveController.js'); const logger = require('winston'); +const { postRequestAnalytics } = require('../helpers/libraries/analytics'); function serveFile ({ fileName, fileType, filePath }, res) { logger.info(`serving file ${fileName}`); @@ -31,35 +32,34 @@ function serveFile ({ fileName, fileType, filePath }, res) { res.status(200).sendFile(filePath, options); } -module.exports = (app, ua, googleAnalyticsId) => { +module.exports = (app) => { // route to fetch one free public claim - app.get('/:name/:claim_id', ({ originalUrl, params }, res) => { - logger.debug(`GET request on ${originalUrl}`); - const routeString = `${params.name}/${params.claim_id}`; - // google analytics - ua(googleAnalyticsId, { https: true }).event('Serve Route', '/name/claimId', routeString).send(); + app.get('/:name/:claim_id', ({ originalUrl, params, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); // begin image-serve processes serveController .getClaimByClaimId(params.name, params.claim_id) .then(fileInfo => { + postRequestAnalytics(originalUrl, ip, 'success'); serveFile(fileInfo, res); }) .catch(error => { + postRequestAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); // route to fetch one free public claim - app.get('/:name', ({ originalUrl, params }, res) => { - logger.debug(`GET request on ${originalUrl}`); - // google analytics - ua(googleAnalyticsId, { https: true }).event('Serve Route', '/name', params.name).send(); + app.get('/:name', ({ originalUrl, params, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); // begin image-serve processes serveController .getClaimByName(params.name) .then(fileInfo => { + postRequestAnalytics(originalUrl, ip, 'success'); serveFile(fileInfo, res); }) .catch(error => { + postRequestAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); diff --git a/routes/show-routes.js b/routes/show-routes.js index 6bd16fe4..712622ec 100644 --- a/routes/show-routes.js +++ b/routes/show-routes.js @@ -1,34 +1,36 @@ const errorHandlers = require('../helpers/libraries/errorHandlers.js'); const showController = require('../controllers/showController.js'); const logger = require('winston'); +const { postShowAnalytics } = require('../helpers/libraries/analytics'); module.exports = (app, ua, googleAnalyticsId) => { // route to fetch all free public claims - app.get('/meme-fodder/play', ({ originalUrl }, res) => { - // google analytics - logger.debug(`GET request on ${originalUrl}`); + app.get('/meme-fodder/play', ({ originalUrl, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); // get and serve content showController .getAllClaims('meme-fodder') .then(orderedFreePublicClaims => { + postShowAnalytics(originalUrl, ip, 'success'); res.status(200).render('memeFodder', { claims: orderedFreePublicClaims }); }) .catch(error => { + postShowAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); // route to fetch all free public claims - app.get('/:name/all', ({ originalUrl, params }, res) => { - logger.debug(`GET request on ${originalUrl}`); - // google analytics - ua(googleAnalyticsId, { https: true }).event('Show Routes', '/name/all', `${params.name}/all`).send(); + app.get('/:name/all', ({ originalUrl, params, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); // get and serve content showController .getAllClaims(params.name) .then(orderedFreePublicClaims => { + postShowAnalytics(originalUrl, ip, 'success'); res.status(200).render('allClaims', { claims: orderedFreePublicClaims }); }) .catch(error => { + postShowAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); diff --git a/routes/sockets-routes.js b/routes/sockets-routes.js index af4ecd41..bf357f97 100644 --- a/routes/sockets-routes.js +++ b/routes/sockets-routes.js @@ -2,6 +2,7 @@ const logger = require('winston'); const publishController = require('../controllers/publishController.js'); const publishHelpers = require('../helpers/libraries/publishHelpers.js'); const errorHandlers = require('../helpers/libraries/errorHandlers.js'); +const { postPublishAnalytics } = require('../helpers/libraries/analytics'); module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => { const http = require('http').Server(app); @@ -9,13 +10,11 @@ module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => { io.on('connection', socket => { logger.silly('a user connected via sockets'); - // google analytics - const visitor = ua(googleAnalyticsId, { https: true }); - visitor.event('Publish Route', 'Publish Request').send(); // attach upload listeners const uploader = new siofu(); uploader.dir = hostedContentPath; uploader.listen(socket); + // listener for when file upload starts uploader.on('start', ({ file }) => { logger.info('client started an upload:', file.name); // server side test to make sure file is not a bad file type @@ -23,10 +22,13 @@ module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => { uploader.abort(file.id, socket); } }); + // listener for when file upload encounters an error uploader.on('error', ({ error }) => { logger.error('an error occured while uploading', error); + postPublishAnalytics('spee.ch/', null, error); socket.emit('publish-status', error); }); + // listener for when file has been uploaded uploader.on('saved', ({ file }) => { if (file.success) { logger.debug(`Client successfully uploaded ${file.name}`); @@ -37,14 +39,17 @@ module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => { publishController .publish(publishParams, file.name, file.meta.type) .then(result => { + postPublishAnalytics('spee.ch/', null, 'success'); socket.emit('publish-complete', { name: publishParams.name, result }); }) .catch(error => { + postPublishAnalytics('spee.ch/', null, error); socket.emit('publish-failure', errorHandlers.handlePublishError(error)); }); } else { logger.error(`An error occurred in uploading the client's file`); socket.emit('publish-failure', 'file uploaded, but with errors'); + postPublishAnalytics('spee.ch/', null, 'file uploaded, but with errors'); // to-do: remove the file if not done automatically } });