diff --git a/controllers/analyticsController.js b/controllers/analyticsController.js new file mode 100644 index 00000000..0aee6bc4 --- /dev/null +++ b/controllers/analyticsController.js @@ -0,0 +1,72 @@ +const logger = require('winston'); +const db = require('../models'); + +module.exports = { + getAnalyticsSummary: () => { + logger.debug('retrieving analytics'); + const deferred = new Promise((resolve, reject) => { + // get the raw analytics data + db.Analytics + .findAll() + .then(data => { + const resultHashTable = {}; + let totalRequests = 0; + let totalPublish = 0; + let totalShow = 0; + let totalCount = 0; + let totalSuccess = 0; + let totalFailure = 0; + // sumarise the data + for (let i = 0; i < data.length; i++) { + let key = data[i].action + data[i].url; + totalCount += 1; + switch (data[i].action) { + case 'request': + totalRequests += 1; + break; + case 'publish': + totalPublish += 1; + break; + case 'show': + totalShow += 1; + break; + default: break; + } + if (resultHashTable[key]) { + resultHashTable[key]['count'] += 1; + if (data[i].result === 'success') { + resultHashTable[key]['success'] += 1; + totalSuccess += 1; + } else { + resultHashTable[key]['failure'] += 1; + totalFailure += 1; + } + } else { + resultHashTable[key] = { + action : data[i].action, + url : data[i].url, + count : 1, + success: 0, + failure: 0, + }; + if (data[i].result === 'success') { + resultHashTable[key]['success'] += 1; + totalSuccess += 1; + } else { + resultHashTable[key]['failure'] += 1; + totalFailure += 1; + } + } + } + const percentSuccess = Math.round(totalSuccess / totalCount * 100); + // return results + resolve({ records: resultHashTable, totals: { totalRequests, totalPublish, totalShow, totalCount, totalSuccess, totalFailure }, percentSuccess }); + }) + .catch(error => { + logger.error('sequelize error', error); + reject(error); + }); + }); + return deferred; + }, +}; diff --git a/controllers/publishController.js b/controllers/publishController.js index 51d5ddfc..ccc8967f 100644 --- a/controllers/publishController.js +++ b/controllers/publishController.js @@ -1,14 +1,7 @@ -const fs = require('fs'); const logger = require('winston'); -const lbryApi = require('../helpers/libraries/lbryApi.js'); const db = require('../models'); - -function deleteTemporaryFile (filePath) { - fs.unlink(filePath, err => { - if (err) throw err; - logger.debug(`successfully deleted ${filePath}`); - }); -} +const lbryApi = require('../helpers/libraries/lbryApi.js'); +const publishHelpers = require('../helpers/libraries/publishHelpers.js'); function upsert (Model, values, condition) { return Model @@ -57,9 +50,8 @@ module.exports = { }); }) .catch(error => { - logger.error(`Error publishing ${fileName}`, error); // delete the local file - deleteTemporaryFile(publishParams.file_path); + publishHelpers.deleteTemporaryFile(publishParams.file_path); // reject the promise reject(error); }); diff --git a/helpers/libraries/analytics.js b/helpers/libraries/analytics.js new file mode 100644 index 00000000..1a42a2f5 --- /dev/null +++ b/helpers/libraries/analytics.js @@ -0,0 +1,31 @@ +const db = require('../../models'); +const logger = require('winston'); + +function createAnalyticsRecord (action, url, ipAddress, result) { + logger.silly('creating record for analytics'); + if (result && (typeof result !== 'string')) { + result = result.toString(); + } + db.Analytics.create({ + action, + url, + ipAddress, + result, + }) + .then() + .catch(error => { + logger.error('sequelize error', error); + }); +}; + +module.exports = { + postRequestAnalytics (url, ipAddress, result) { + createAnalyticsRecord('request', url, ipAddress, result); + }, + postPublishAnalytics (url, ipAddress, result) { + createAnalyticsRecord('publish', url, ipAddress, result); + }, + postShowAnalytics (url, ipAddress, result) { + createAnalyticsRecord('show', url, ipAddress, result); + }, +}; diff --git a/helpers/libraries/errorHandlers.js b/helpers/libraries/errorHandlers.js index 172ee912..09a18c81 100644 --- a/helpers/libraries/errorHandlers.js +++ b/helpers/libraries/errorHandlers.js @@ -14,12 +14,14 @@ module.exports = { } }, handlePublishError (error) { - logger.error('Publish Error >>', error); if (error.code === 'ECONNREFUSED') { + logger.error('Publish Error:', 'Connection refused. The daemon may not be running.'); return 'Connection refused. The daemon may not be running.'; } else if (error.response.data.error) { - return error.response.data.error; + logger.error('Publish Error:', error.response.data.error); + return error.response.data.error.message; } else { + logger.error('Unhandled Publish Error:', error); return error; } }, diff --git a/helpers/libraries/publishHelpers.js b/helpers/libraries/publishHelpers.js index 60ab5eb3..57a87208 100644 --- a/helpers/libraries/publishHelpers.js +++ b/helpers/libraries/publishHelpers.js @@ -1,10 +1,10 @@ const logger = require('winston'); - const config = require('config'); const walletAddress = config.get('WalletConfig.LbryAddress'); +const fs = require('fs'); module.exports = { - createPublishParams (name, filePath, license, nsfw) { + createPublishParams: (name, filePath, license, nsfw) => { logger.debug(`Creating Publish Parameters for "${name}"`); // ensure nsfw is a boolean if (nsfw === false) { @@ -38,4 +38,10 @@ module.exports = { logger.debug('publishParams:', publishParams); return publishParams; }, + deleteTemporaryFile: (filePath) => { + fs.unlink(filePath, err => { + if (err) throw err; + logger.debug(`successfully deleted ${filePath}`); + }); + }, }; diff --git a/models/analytics.js b/models/analytics.js new file mode 100644 index 00000000..9bb80378 --- /dev/null +++ b/models/analytics.js @@ -0,0 +1,29 @@ +module.exports = (sequelize, { STRING }) => { + const Analytics = sequelize.define( + 'Analytics', + { + action: { + type : STRING, + allowNull: false, + }, + url: { + type : STRING, + allowNull: false, + }, + ipAddress: { + type : STRING, + allowNull: true, + default : null, + }, + result: { + type : STRING, + allowNull: true, + default : null, + }, + }, + { + freezeTableName: true, + } + ); + return Analytics; +}; diff --git a/public/assets/css/allStyle.css b/public/assets/css/allStyle.css index 81832da5..ea3d6427 100644 --- a/public/assets/css/allStyle.css +++ b/public/assets/css/allStyle.css @@ -80,6 +80,10 @@ h4 { padding: 3px; } +.center-text { + text-align: center; +} + .code { font-family: monospace; color: darkgrey; @@ -87,6 +91,10 @@ h4 { } /* other */ +table { + width: 100%; + text-align: left; +} .stop-float { clear: both; diff --git a/public/assets/css/componentStyle.css b/public/assets/css/componentStyle.css index f5914eac..d2d745ba 100644 --- a/public/assets/css/componentStyle.css +++ b/public/assets/css/componentStyle.css @@ -65,3 +65,10 @@ canvas { float: left; } +/* analytics */ +.totals-row { + border-top: 1px solid grey; + border-bottom: 1px solid grey; + font-weight: bold; +} + diff --git a/public/assets/js/claimPublish.js b/public/assets/js/claimPublish.js index ada4ce40..6391d8f6 100644 --- a/public/assets/js/claimPublish.js +++ b/public/assets/js/claimPublish.js @@ -52,7 +52,6 @@ function updatePublishStatus(msg){ } // process the drop-zone drop function drop_handler(ev) { - console.log("drop"); ev.preventDefault(); // if dropped items aren't files, reject them var dt = ev.dataTransfer; @@ -60,11 +59,7 @@ function drop_handler(ev) { if (dt.items[0].kind == 'file') { var droppedFile = dt.items[0].getAsFile(); previewAndStageFile(droppedFile); - } else { - console.log("no files were found") } - } else { - console.log("no items were found") } } // prevent the browser's default drag behavior @@ -169,7 +164,6 @@ socket.on('publish-complete', function(msg){ try { var successful = document.execCommand('copy'); var msg = successful ? 'successful' : 'unsuccessful'; - console.log('Copying text command was ' + msg); } catch (err) { alert('Oops, unable to copy'); } diff --git a/public/assets/js/generalFunctions.js b/public/assets/js/generalFunctions.js index bb4a2d0c..2151357b 100644 --- a/public/assets/js/generalFunctions.js +++ b/public/assets/js/generalFunctions.js @@ -15,5 +15,4 @@ function toggleSection(event){ masterElement.innerText = "[open]"; masterElement.dataset.open = "false"; } - console.log(status); } \ No newline at end of file diff --git a/public/assets/js/memePublish.js b/public/assets/js/memePublish.js index d0e343b2..6cd50171 100644 --- a/public/assets/js/memePublish.js +++ b/public/assets/js/memePublish.js @@ -33,7 +33,6 @@ function startPublish() { var blob = dataURItoBlob(dataUrl) var fileName = nameInput.value + ".jpg"; //note: need to dynamically grab type var file = new File([blob], fileName, {type: 'image/jpeg', lastModified: Date.now()}); - console.log(file); stageAndPublish(file); }; @@ -144,7 +143,6 @@ socket.on('publish-complete', function(msg){ try { var successful = document.execCommand('copy'); var msg = successful ? 'successful' : 'unsuccessful'; - console.log('Copying text command was ' + msg); } catch (err) { alert('Oops, unable to copy'); } diff --git a/routes/analytics-routes.js b/routes/analytics-routes.js new file mode 100644 index 00000000..f5fea1c7 --- /dev/null +++ b/routes/analytics-routes.js @@ -0,0 +1,17 @@ +const errorHandlers = require('../helpers/libraries/errorHandlers.js'); +const analyticsController = require('../controllers/analyticsController.js'); + +module.exports = (app) => { + // route to show analytics + app.get('/analytics', (req, res) => { + // get and serve content + analyticsController + .getAnalyticsSummary() + .then(result => { + res.status(200).render('analytics', result); + }) + .catch(error => { + errorHandlers.handleRequestError(error, res); + }); + }); +}; diff --git a/routes/api-routes.js b/routes/api-routes.js index eadb3c05..247d581f 100644 --- a/routes/api-routes.js +++ b/routes/api-routes.js @@ -5,38 +5,44 @@ const publishController = require('../controllers/publishController.js'); const lbryApi = require('../helpers/libraries/lbryApi.js'); const publishHelpers = require('../helpers/libraries/publishHelpers.js'); const errorHandlers = require('../helpers/libraries/errorHandlers.js'); +const { postRequestAnalytics, postPublishAnalytics } = require('../helpers/libraries/analytics'); module.exports = app => { // route to run a claim_list request on the daemon - app.get('/api/claim_list/:claim', ({ originalUrl, params }, res) => { - logger.debug(`GET request on ${originalUrl}`); + app.get('/api/claim_list/:claim', ({ originalUrl, params, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); lbryApi .getClaimsList(params.claim) .then(claimsList => { + postRequestAnalytics(originalUrl, ip, 'success'); res.status(200).json(claimsList); }) .catch(error => { + postRequestAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); // route to run a resolve request on the daemon - app.get('/api/resolve/:uri', ({ originalUrl, params }, res) => { - logger.debug(`GET request on ${originalUrl}`); + app.get('/api/resolve/:uri', ({ originalUrl, params, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); lbryApi .resolveUri(params.uri) .then(resolvedUri => { + postRequestAnalytics(originalUrl, ip, 'success'); res.status(200).json(resolvedUri); }) .catch(error => { + postRequestAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); // route to run a publish request on the daemon - app.post('/api/publish', multipartMiddleware, ({ originalUrl, body, files }, res) => { - logger.debug(`POST request on ${originalUrl}`); + app.post('/api/publish', multipartMiddleware, ({ originalUrl, body, files, ip }, res) => { + logger.debug(`POST request on ${originalUrl} from ${ip}`); // validate that a file was provided const file = files.speech || files.null; if (!file) { + postPublishAnalytics(originalUrl, ip, 'Error: file'); res.status(400).send('Error: No file was submitted or the key used was incorrect. Files posted through this route must use a key of "speech" or null'); return; } @@ -44,12 +50,14 @@ module.exports = app => { const name = body.name || file.name.substring(0, file.name.indexOf('.')); const invalidCharacters = /[^\w,-]/.exec(name); if (invalidCharacters) { + postPublishAnalytics(originalUrl, ip, 'Error: name'); res.status(400).send('Error: The name you provided is not allowed. Please use A-Z, a-z, 0-9, "_" and "-" only.'); return; } // validate license const license = body.license || 'No License Provided'; if ((license.indexOf('Public Domain') === -1) && (license.indexOf('Creative Commons') === -1)) { + postPublishAnalytics(originalUrl, ip, 'Error: license'); res.status(400).send('Error: Only posts with a license of "Public Domain" or "Creative Commons" are eligible for publishing through spee.ch'); return; } @@ -67,6 +75,7 @@ module.exports = app => { case '1': break; default: + postPublishAnalytics(originalUrl, ip, 'Error: nsfw'); res.status(400).send('Error: NSFW value was not accepted. NSFW must be set to either true, false, "on", or "off"'); return; } @@ -82,9 +91,11 @@ module.exports = app => { publishController .publish(publishParams, fileName, fileType) .then(result => { + postPublishAnalytics(originalUrl, ip, 'success'); res.status(200).json(result); }) .catch(error => { + postPublishAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); diff --git a/routes/home-routes.js b/routes/home-routes.js index 09c40644..f04e78f5 100644 --- a/routes/home-routes.js +++ b/routes/home-routes.js @@ -1,13 +1,16 @@ const logger = require('winston'); +const { postShowAnalytics } = require('../helpers/libraries/analytics'); module.exports = app => { // route for the home page - app.get('/', (req, res) => { + app.get('/', ({ originalUrl, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); res.status(200).render('index'); }); // a catch-all route if someone visits a page that does not exist - app.use('*', (req, res) => { - logger.error(`Get request on ${req.originalUrl} which was a 404`); + app.use('*', ({ originalUrl, ip }, res) => { + logger.error(`Get request on ${originalUrl} from ${ip} which was a 404`); + postShowAnalytics(originalUrl, ip, 'Error: 404'); res.status(404).render('fourOhFour'); }); }; diff --git a/routes/serve-routes.js b/routes/serve-routes.js index c3d25814..bfce7db1 100644 --- a/routes/serve-routes.js +++ b/routes/serve-routes.js @@ -1,6 +1,7 @@ const errorHandlers = require('../helpers/libraries/errorHandlers.js'); const serveController = require('../controllers/serveController.js'); const logger = require('winston'); +const { postRequestAnalytics } = require('../helpers/libraries/analytics'); function serveFile ({ fileName, fileType, filePath }, res) { logger.info(`serving file ${fileName}`); @@ -31,35 +32,34 @@ function serveFile ({ fileName, fileType, filePath }, res) { res.status(200).sendFile(filePath, options); } -module.exports = (app, ua, googleAnalyticsId) => { +module.exports = (app) => { // route to fetch one free public claim - app.get('/:name/:claim_id', ({ originalUrl, params }, res) => { - logger.debug(`GET request on ${originalUrl}`); - const routeString = `${params.name}/${params.claim_id}`; - // google analytics - ua(googleAnalyticsId, { https: true }).event('Serve Route', '/name/claimId', routeString).send(); + app.get('/:name/:claim_id', ({ originalUrl, params, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); // begin image-serve processes serveController .getClaimByClaimId(params.name, params.claim_id) .then(fileInfo => { + postRequestAnalytics(originalUrl, ip, 'success'); serveFile(fileInfo, res); }) .catch(error => { + postRequestAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); // route to fetch one free public claim - app.get('/:name', ({ originalUrl, params }, res) => { - logger.debug(`GET request on ${originalUrl}`); - // google analytics - ua(googleAnalyticsId, { https: true }).event('Serve Route', '/name', params.name).send(); + app.get('/:name', ({ originalUrl, params, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); // begin image-serve processes serveController .getClaimByName(params.name) .then(fileInfo => { + postRequestAnalytics(originalUrl, ip, 'success'); serveFile(fileInfo, res); }) .catch(error => { + postRequestAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); diff --git a/routes/show-routes.js b/routes/show-routes.js index 6bd16fe4..cb927e30 100644 --- a/routes/show-routes.js +++ b/routes/show-routes.js @@ -1,34 +1,36 @@ const errorHandlers = require('../helpers/libraries/errorHandlers.js'); const showController = require('../controllers/showController.js'); const logger = require('winston'); +const { postShowAnalytics } = require('../helpers/libraries/analytics'); -module.exports = (app, ua, googleAnalyticsId) => { +module.exports = (app) => { // route to fetch all free public claims - app.get('/meme-fodder/play', ({ originalUrl }, res) => { - // google analytics - logger.debug(`GET request on ${originalUrl}`); + app.get('/meme-fodder/play', ({ originalUrl, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); // get and serve content showController .getAllClaims('meme-fodder') .then(orderedFreePublicClaims => { + postShowAnalytics(originalUrl, ip, 'success'); res.status(200).render('memeFodder', { claims: orderedFreePublicClaims }); }) .catch(error => { + postShowAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); // route to fetch all free public claims - app.get('/:name/all', ({ originalUrl, params }, res) => { - logger.debug(`GET request on ${originalUrl}`); - // google analytics - ua(googleAnalyticsId, { https: true }).event('Show Routes', '/name/all', `${params.name}/all`).send(); + app.get('/:name/all', ({ originalUrl, params, ip }, res) => { + logger.debug(`GET request on ${originalUrl} from ${ip}`); // get and serve content showController .getAllClaims(params.name) .then(orderedFreePublicClaims => { + postShowAnalytics(originalUrl, ip, 'success'); res.status(200).render('allClaims', { claims: orderedFreePublicClaims }); }) .catch(error => { + postShowAnalytics(originalUrl, ip, error); errorHandlers.handleRequestError(error, res); }); }); diff --git a/routes/sockets-routes.js b/routes/sockets-routes.js index af4ecd41..569a4f08 100644 --- a/routes/sockets-routes.js +++ b/routes/sockets-routes.js @@ -2,20 +2,19 @@ const logger = require('winston'); const publishController = require('../controllers/publishController.js'); const publishHelpers = require('../helpers/libraries/publishHelpers.js'); const errorHandlers = require('../helpers/libraries/errorHandlers.js'); +const { postPublishAnalytics } = require('../helpers/libraries/analytics'); -module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => { +module.exports = (app, siofu, hostedContentPath) => { const http = require('http').Server(app); const io = require('socket.io')(http); io.on('connection', socket => { logger.silly('a user connected via sockets'); - // google analytics - const visitor = ua(googleAnalyticsId, { https: true }); - visitor.event('Publish Route', 'Publish Request').send(); // attach upload listeners const uploader = new siofu(); uploader.dir = hostedContentPath; uploader.listen(socket); + // listener for when file upload starts uploader.on('start', ({ file }) => { logger.info('client started an upload:', file.name); // server side test to make sure file is not a bad file type @@ -23,10 +22,13 @@ module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => { uploader.abort(file.id, socket); } }); + // listener for when file upload encounters an error uploader.on('error', ({ error }) => { logger.error('an error occured while uploading', error); + postPublishAnalytics('spee.ch/', null, error); socket.emit('publish-status', error); }); + // listener for when file has been uploaded uploader.on('saved', ({ file }) => { if (file.success) { logger.debug(`Client successfully uploaded ${file.name}`); @@ -37,14 +39,18 @@ module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => { publishController .publish(publishParams, file.name, file.meta.type) .then(result => { + postPublishAnalytics('spee.ch/', null, 'success'); socket.emit('publish-complete', { name: publishParams.name, result }); }) .catch(error => { - socket.emit('publish-failure', errorHandlers.handlePublishError(error)); + error = errorHandlers.handlePublishError(error); + postPublishAnalytics('spee.ch/', null, error); + socket.emit('publish-failure', error); }); } else { logger.error(`An error occurred in uploading the client's file`); socket.emit('publish-failure', 'file uploaded, but with errors'); + postPublishAnalytics('spee.ch/', null, 'file uploaded, but with errors'); // to-do: remove the file if not done automatically } }); diff --git a/server.js b/server.js index 8df35748..22c6fca2 100644 --- a/server.js +++ b/server.js @@ -5,10 +5,8 @@ const siofu = require('socketio-file-upload'); const expressHandlebars = require('express-handlebars'); const Handlebars = require('handlebars'); const config = require('config'); -const ua = require('universal-analytics'); const winston = require('winston'); -const googleAnalyticsId = config.get('AnalyticsConfig.GoogleId'); const hostedContentPath = config.get('Database.PublishUploadPath'); // configure logging @@ -57,17 +55,18 @@ app.set('view engine', 'handlebars'); // require express routes require('./routes/api-routes.js')(app); -require('./routes/show-routes.js')(app, ua, googleAnalyticsId); -require('./routes/serve-routes.js')(app, ua, googleAnalyticsId); +require('./routes/analytics-routes.js')(app); +require('./routes/show-routes.js')(app); +require('./routes/serve-routes.js')(app); require('./routes/home-routes.js')(app); // require socket.io routes -const http = require('./routes/sockets-routes.js')(app, siofu, hostedContentPath, ua, googleAnalyticsId); +const http = require('./routes/sockets-routes.js')(app, siofu, hostedContentPath); // sync sequelize // wrap the server in socket.io to intercept incoming sockets requests // start server -db.sequelize.sync({}).then(() => { +db.sequelize.sync().then(() => { http.listen(PORT, () => { winston.info(`Server is listening on PORT ${PORT}`); }); diff --git a/views/analytics.handlebars b/views/analytics.handlebars new file mode 100644 index 00000000..b7a4acc6 --- /dev/null +++ b/views/analytics.handlebars @@ -0,0 +1,37 @@ +
+
+ {{> topBar}} +
+
+

Analytics

+ + + + + + + + + {{#each records}} + + + + + + + + {{/each}} + + + + + + + +
actionurlcountsuccessfailure
{{ this.action }}{{ this.url }}{{ this.count }}{{ this.success }}{{ this.failure }}
{{ totals.totalCount }}{{ totals.totalSuccess }}{{ totals.totalFailure }}
+

Requests: {{ totals.totalRequests }}

+

Publish: {{ totals.totalPublish }}

+

Show: {{ totals.totalShow }}

+

Percent Success: {{ percentSuccess}}%

+
+
\ No newline at end of file