Merge pull request #43 from lbryio/analytics

Analytics Update

controllers/analyticsController.js

@@ -0,0 +1,72 @@
+const logger = require('winston');
+const db = require('../models');
+
+module.exports = {
+  getAnalyticsSummary: () => {
+    logger.debug('retrieving analytics');
+    const deferred = new Promise((resolve, reject) => {
+      // get the raw analytics data
+      db.Analytics
+        .findAll()
+        .then(data => {
+          const resultHashTable = {};
+          let totalRequests = 0;
+          let totalPublish = 0;
+          let totalShow = 0;
+          let totalCount = 0;
+          let totalSuccess = 0;
+          let totalFailure = 0;
+          // sumarise the data
+          for (let i = 0; i < data.length; i++) {
+            let key = data[i].action + data[i].url;
+            totalCount += 1;
+            switch (data[i].action) {
+              case 'request':
+                totalRequests += 1;
+                break;
+              case 'publish':
+                totalPublish += 1;
+                break;
+              case 'show':
+                totalShow += 1;
+                break;
+              default: break;
+            }
+            if (resultHashTable[key]) {
+              resultHashTable[key]['count'] += 1;
+              if (data[i].result === 'success') {
+                resultHashTable[key]['success'] += 1;
+                totalSuccess += 1;
+              } else {
+                resultHashTable[key]['failure'] += 1;
+                totalFailure += 1;
+              }
+            } else {
+              resultHashTable[key] = {
+                action : data[i].action,
+                url    : data[i].url,
+                count  : 1,
+                success: 0,
+                failure: 0,
+              };
+              if (data[i].result === 'success') {
+                resultHashTable[key]['success'] += 1;
+                totalSuccess += 1;
+              } else {
+                resultHashTable[key]['failure'] += 1;
+                totalFailure += 1;
+              }
+            }
+          }
+          const percentSuccess = Math.round(totalSuccess / totalCount * 100);
+          // return results
+          resolve({ records: resultHashTable, totals: { totalRequests, totalPublish, totalShow, totalCount, totalSuccess, totalFailure }, percentSuccess });
+        })
+        .catch(error => {
+          logger.error('sequelize error', error);
+          reject(error);
+        });
+    });
+    return deferred;
+  },
+};

controllers/publishController.js

@@ -1,14 +1,7 @@
-const fs = require('fs');
 const logger = require('winston');
-const lbryApi = require('../helpers/libraries/lbryApi.js');
 const db = require('../models');
-
-function deleteTemporaryFile (filePath) {
-  fs.unlink(filePath, err => {
-    if (err) throw err;
-    logger.debug(`successfully deleted ${filePath}`);
-  });
-}
+const lbryApi = require('../helpers/libraries/lbryApi.js');
+const publishHelpers = require('../helpers/libraries/publishHelpers.js');
 
 function upsert (Model, values, condition) {
   return Model
@@ -57,9 +50,8 @@ module.exports = {
           });
         })
         .catch(error => {
-          logger.error(`Error publishing ${fileName}`, error);
           // delete the local file
-          deleteTemporaryFile(publishParams.file_path);
+          publishHelpers.deleteTemporaryFile(publishParams.file_path);
           // reject the promise
           reject(error);
         });

helpers/libraries/analytics.js

@@ -0,0 +1,31 @@
+const db = require('../../models');
+const logger = require('winston');
+
+function createAnalyticsRecord (action, url, ipAddress, result) {
+  logger.silly('creating record for analytics');
+  if (result && (typeof result !== 'string')) {
+    result = result.toString();
+  }
+  db.Analytics.create({
+    action,
+    url,
+    ipAddress,
+    result,
+  })
+  .then()
+  .catch(error => {
+    logger.error('sequelize error', error);
+  });
+};
+
+module.exports = {
+  postRequestAnalytics (url, ipAddress, result) {
+    createAnalyticsRecord('request', url, ipAddress, result);
+  },
+  postPublishAnalytics (url, ipAddress, result) {
+    createAnalyticsRecord('publish', url, ipAddress, result);
+  },
+  postShowAnalytics (url, ipAddress, result) {
+    createAnalyticsRecord('show', url, ipAddress, result);
+  },
+};

helpers/libraries/errorHandlers.js

@@ -14,12 +14,14 @@ module.exports = {
     }
   },
   handlePublishError (error) {
-    logger.error('Publish Error >>', error);
     if (error.code === 'ECONNREFUSED') {
+      logger.error('Publish Error:', 'Connection refused.  The daemon may not be running.');
       return 'Connection refused.  The daemon may not be running.';
     } else if (error.response.data.error) {
-      return error.response.data.error;
+      logger.error('Publish Error:', error.response.data.error);
+      return error.response.data.error.message;
     } else {
+      logger.error('Unhandled Publish Error:', error);
       return error;
     }
   },

helpers/libraries/publishHelpers.js

@@ -1,10 +1,10 @@
 const logger = require('winston');
-
 const config = require('config');
 const walletAddress = config.get('WalletConfig.LbryAddress');
+const fs = require('fs');
 
 module.exports = {
-  createPublishParams (name, filePath, license, nsfw) {
+  createPublishParams: (name, filePath, license, nsfw) => {
     logger.debug(`Creating Publish Parameters for "${name}"`);
     // ensure nsfw is a boolean
     if (nsfw === false) {
@@ -38,4 +38,10 @@ module.exports = {
     logger.debug('publishParams:', publishParams);
     return publishParams;
   },
+  deleteTemporaryFile: (filePath) => {
+    fs.unlink(filePath, err => {
+      if (err) throw err;
+      logger.debug(`successfully deleted ${filePath}`);
+    });
+  },
 };

models/analytics.js

@@ -0,0 +1,29 @@
+module.exports = (sequelize, { STRING }) => {
+  const Analytics = sequelize.define(
+    'Analytics',
+    {
+      action: {
+        type     : STRING,
+        allowNull: false,
+      },
+      url: {
+        type     : STRING,
+        allowNull: false,
+      },
+      ipAddress: {
+        type     : STRING,
+        allowNull: true,
+        default  : null,
+      },
+      result: {
+        type     : STRING,
+        allowNull: true,
+        default  : null,
+      },
+    },
+    {
+      freezeTableName: true,
+    }
+  );
+  return Analytics;
+};

public/assets/css/allStyle.css

@@ -80,6 +80,10 @@ h4 {
 	padding: 3px;
 }
 
+.center-text {
+	text-align: center;
+}
+
 .code {
 	font-family: monospace;
 	color: darkgrey;
@@ -87,6 +91,10 @@ h4 {
 }
 
 /* other */
+table {
+	width: 100%;
+	text-align: left;
+}
 
 .stop-float {
 	clear: both;

public/assets/css/componentStyle.css

@@ -65,3 +65,10 @@ canvas {
 	float: left;
 }
 
+/* analytics */
+.totals-row {
+	border-top: 1px solid grey;
+	border-bottom: 1px solid grey;
+	font-weight: bold; 
+}
+

public/assets/js/claimPublish.js

@@ -52,7 +52,6 @@ function updatePublishStatus(msg){
 }
 // process the drop-zone drop
 function drop_handler(ev) {
-	console.log("drop");
 	ev.preventDefault();
 	// if dropped items aren't files, reject them
 	var dt = ev.dataTransfer;
@@ -60,11 +59,7 @@ function drop_handler(ev) {
 		if (dt.items[0].kind == 'file') {
 			var droppedFile = dt.items[0].getAsFile();
 			previewAndStageFile(droppedFile);
-		} else {
-			console.log("no files were found")
 		}
-	} else {
-		console.log("no items were found")
 	}
 }
 // prevent the browser's default drag behavior
@@ -169,7 +164,6 @@ socket.on('publish-complete', function(msg){
 		try {
 			var successful = document.execCommand('copy');
 			var msg = successful ? 'successful' : 'unsuccessful';
-			console.log('Copying text command was ' + msg);
 		} catch (err) {
 			alert('Oops, unable to copy');
 		}

public/assets/js/generalFunctions.js

@@ -15,5 +15,4 @@ function toggleSection(event){
 		masterElement.innerText = "[open]";
 		masterElement.dataset.open = "false";
 	}
-	console.log(status);
 }

public/assets/js/memePublish.js

@@ -33,7 +33,6 @@ function startPublish() {
 	var blob = dataURItoBlob(dataUrl)
 	var fileName = nameInput.value + ".jpg";  //note: need to dynamically grab type
 	var file = new File([blob], fileName, {type: 'image/jpeg', lastModified: Date.now()});
-	console.log(file);
 	stageAndPublish(file); 
 };
 
@@ -144,7 +143,6 @@ socket.on('publish-complete', function(msg){
 		try {
 			var successful = document.execCommand('copy');
 			var msg = successful ? 'successful' : 'unsuccessful';
-			console.log('Copying text command was ' + msg);
 		} catch (err) {
 			alert('Oops, unable to copy');
 		}

routes/analytics-routes.js

@@ -0,0 +1,17 @@
+const errorHandlers = require('../helpers/libraries/errorHandlers.js');
+const analyticsController = require('../controllers/analyticsController.js');
+
+module.exports = (app) => {
+  // route to show analytics
+  app.get('/analytics', (req, res) => {
+    // get and serve content
+    analyticsController
+      .getAnalyticsSummary()
+      .then(result => {
+        res.status(200).render('analytics', result);
+      })
+      .catch(error => {
+        errorHandlers.handleRequestError(error, res);
+      });
+  });
+};

routes/api-routes.js

@@ -5,38 +5,44 @@ const publishController = require('../controllers/publishController.js');
 const lbryApi = require('../helpers/libraries/lbryApi.js');
 const publishHelpers = require('../helpers/libraries/publishHelpers.js');
 const errorHandlers = require('../helpers/libraries/errorHandlers.js');
+const { postRequestAnalytics, postPublishAnalytics } = require('../helpers/libraries/analytics');
 
 module.exports = app => {
   // route to run a claim_list request on the daemon
-  app.get('/api/claim_list/:claim', ({ originalUrl, params }, res) => {
-    logger.debug(`GET request on ${originalUrl}`);
+  app.get('/api/claim_list/:claim', ({ originalUrl, params, ip }, res) => {
+    logger.debug(`GET request on ${originalUrl} from ${ip}`);
     lbryApi
       .getClaimsList(params.claim)
       .then(claimsList => {
+        postRequestAnalytics(originalUrl, ip, 'success');
         res.status(200).json(claimsList);
       })
       .catch(error => {
+        postRequestAnalytics(originalUrl, ip, error);
         errorHandlers.handleRequestError(error, res);
       });
   });
   // route to run a resolve request on the daemon
-  app.get('/api/resolve/:uri', ({ originalUrl, params }, res) => {
-    logger.debug(`GET request on ${originalUrl}`);
+  app.get('/api/resolve/:uri', ({ originalUrl, params, ip }, res) => {
+    logger.debug(`GET request on ${originalUrl} from ${ip}`);
     lbryApi
       .resolveUri(params.uri)
       .then(resolvedUri => {
+        postRequestAnalytics(originalUrl, ip, 'success');
         res.status(200).json(resolvedUri);
       })
       .catch(error => {
+        postRequestAnalytics(originalUrl, ip, error);
         errorHandlers.handleRequestError(error, res);
       });
   });
   // route to run a publish request on the daemon
-  app.post('/api/publish', multipartMiddleware, ({ originalUrl, body, files }, res) => {
-    logger.debug(`POST request on ${originalUrl}`);
+  app.post('/api/publish', multipartMiddleware, ({ originalUrl, body, files, ip }, res) => {
+    logger.debug(`POST request on ${originalUrl} from ${ip}`);
     // validate that a file was provided
     const file = files.speech || files.null;
     if (!file) {
+      postPublishAnalytics(originalUrl, ip, 'Error: file');
       res.status(400).send('Error: No file was submitted or the key used was incorrect.  Files posted through this route must use a key of "speech" or null');
       return;
     }
@@ -44,12 +50,14 @@ module.exports = app => {
     const name = body.name || file.name.substring(0, file.name.indexOf('.'));
     const invalidCharacters = /[^\w,-]/.exec(name);
     if (invalidCharacters) {
+      postPublishAnalytics(originalUrl, ip, 'Error: name');
       res.status(400).send('Error: The name you provided is not allowed.  Please use A-Z, a-z, 0-9, "_" and "-" only.');
       return;
     }
     // validate license
     const license = body.license || 'No License Provided';
     if ((license.indexOf('Public Domain') === -1) && (license.indexOf('Creative Commons') === -1)) {
+      postPublishAnalytics(originalUrl, ip, 'Error: license');
       res.status(400).send('Error: Only posts with a license of "Public Domain" or "Creative Commons" are eligible for publishing through spee.ch');
       return;
     }
@@ -67,6 +75,7 @@ module.exports = app => {
       case '1':
         break;
       default:
+        postPublishAnalytics(originalUrl, ip, 'Error: nsfw');
         res.status(400).send('Error: NSFW value was not accepted.  NSFW must be set to either true, false, "on", or "off"');
         return;
     }
@@ -82,9 +91,11 @@ module.exports = app => {
     publishController
       .publish(publishParams, fileName, fileType)
       .then(result => {
+        postPublishAnalytics(originalUrl, ip, 'success');
         res.status(200).json(result);
       })
       .catch(error => {
+        postPublishAnalytics(originalUrl, ip, error);
         errorHandlers.handleRequestError(error, res);
       });
   });

routes/home-routes.js

@@ -1,13 +1,16 @@
 const logger = require('winston');
+const { postShowAnalytics } = require('../helpers/libraries/analytics');
 
 module.exports = app => {
   // route for the home page
-  app.get('/', (req, res) => {
+  app.get('/', ({ originalUrl, ip }, res) => {
+    logger.debug(`GET request on ${originalUrl} from ${ip}`);
     res.status(200).render('index');
   });
   // a catch-all route if someone visits a page that does not exist
-  app.use('*', (req, res) => {
-    logger.error(`Get request on ${req.originalUrl} which was a 404`);
+  app.use('*', ({ originalUrl, ip }, res) => {
+    logger.error(`Get request on ${originalUrl} from ${ip} which was a 404`);
+    postShowAnalytics(originalUrl, ip, 'Error: 404');
     res.status(404).render('fourOhFour');
   });
 };

routes/serve-routes.js

@@ -1,6 +1,7 @@
 const errorHandlers = require('../helpers/libraries/errorHandlers.js');
 const serveController = require('../controllers/serveController.js');
 const logger = require('winston');
+const { postRequestAnalytics } = require('../helpers/libraries/analytics');
 
 function serveFile ({ fileName, fileType, filePath }, res) {
   logger.info(`serving file ${fileName}`);
@@ -31,35 +32,34 @@ function serveFile ({ fileName, fileType, filePath }, res) {
   res.status(200).sendFile(filePath, options);
 }
 
-module.exports = (app, ua, googleAnalyticsId) => {
+module.exports = (app) => {
   // route to fetch one free public claim
-  app.get('/:name/:claim_id', ({ originalUrl, params }, res) => {
-    logger.debug(`GET request on ${originalUrl}`);
-    const routeString = `${params.name}/${params.claim_id}`;
-    // google analytics
-    ua(googleAnalyticsId, { https: true }).event('Serve Route', '/name/claimId', routeString).send();
+  app.get('/:name/:claim_id', ({ originalUrl, params, ip }, res) => {
+    logger.debug(`GET request on ${originalUrl} from ${ip}`);
     // begin image-serve processes
     serveController
       .getClaimByClaimId(params.name, params.claim_id)
       .then(fileInfo => {
+        postRequestAnalytics(originalUrl, ip, 'success');
         serveFile(fileInfo, res);
       })
       .catch(error => {
+        postRequestAnalytics(originalUrl, ip, error);
         errorHandlers.handleRequestError(error, res);
       });
   });
   // route to fetch one free public claim
-  app.get('/:name', ({ originalUrl, params }, res) => {
-    logger.debug(`GET request on ${originalUrl}`);
-    // google analytics
-    ua(googleAnalyticsId, { https: true }).event('Serve Route', '/name', params.name).send();
+  app.get('/:name', ({ originalUrl, params, ip }, res) => {
+    logger.debug(`GET request on ${originalUrl} from ${ip}`);
     // begin image-serve processes
     serveController
       .getClaimByName(params.name)
       .then(fileInfo => {
+        postRequestAnalytics(originalUrl, ip, 'success');
         serveFile(fileInfo, res);
       })
       .catch(error => {
+        postRequestAnalytics(originalUrl, ip, error);
         errorHandlers.handleRequestError(error, res);
       });
   });

routes/show-routes.js

@@ -1,34 +1,36 @@
 const errorHandlers = require('../helpers/libraries/errorHandlers.js');
 const showController = require('../controllers/showController.js');
 const logger = require('winston');
+const { postShowAnalytics } = require('../helpers/libraries/analytics');
 
-module.exports = (app, ua, googleAnalyticsId) => {
+module.exports = (app) => {
   // route to fetch all free public claims
-  app.get('/meme-fodder/play', ({ originalUrl }, res) => {
-    // google analytics
-    logger.debug(`GET request on ${originalUrl}`);
+  app.get('/meme-fodder/play', ({ originalUrl, ip }, res) => {
+    logger.debug(`GET request on ${originalUrl} from ${ip}`);
     // get and serve content
     showController
       .getAllClaims('meme-fodder')
       .then(orderedFreePublicClaims => {
+        postShowAnalytics(originalUrl, ip, 'success');
         res.status(200).render('memeFodder', { claims: orderedFreePublicClaims });
       })
       .catch(error => {
+        postShowAnalytics(originalUrl, ip, error);
         errorHandlers.handleRequestError(error, res);
       });
   });
   // route to fetch all free public claims
-  app.get('/:name/all', ({ originalUrl, params }, res) => {
-    logger.debug(`GET request on ${originalUrl}`);
-    // google analytics
-    ua(googleAnalyticsId, { https: true }).event('Show Routes', '/name/all', `${params.name}/all`).send();
+  app.get('/:name/all', ({ originalUrl, params, ip }, res) => {
+    logger.debug(`GET request on ${originalUrl} from ${ip}`);
     // get and serve content
     showController
       .getAllClaims(params.name)
       .then(orderedFreePublicClaims => {
+        postShowAnalytics(originalUrl, ip, 'success');
         res.status(200).render('allClaims', { claims: orderedFreePublicClaims });
       })
       .catch(error => {
+        postShowAnalytics(originalUrl, ip, error);
         errorHandlers.handleRequestError(error, res);
       });
   });

routes/sockets-routes.js

@@ -2,20 +2,19 @@ const logger = require('winston');
 const publishController = require('../controllers/publishController.js');
 const publishHelpers = require('../helpers/libraries/publishHelpers.js');
 const errorHandlers = require('../helpers/libraries/errorHandlers.js');
+const { postPublishAnalytics } = require('../helpers/libraries/analytics');
 
-module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => {
+module.exports = (app, siofu, hostedContentPath) => {
   const http = require('http').Server(app);
   const io = require('socket.io')(http);
 
   io.on('connection', socket => {
     logger.silly('a user connected via sockets');
-    // google analytics
-    const visitor = ua(googleAnalyticsId, { https: true });
-    visitor.event('Publish Route', 'Publish Request').send();
     // attach upload listeners
     const uploader = new siofu();
     uploader.dir = hostedContentPath;
     uploader.listen(socket);
+    // listener for when file upload starts
     uploader.on('start', ({ file }) => {
       logger.info('client started an upload:', file.name);
       // server side test to make sure file is not a bad file type
@@ -23,10 +22,13 @@ module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => {
         uploader.abort(file.id, socket);
       }
     });
+    // listener for when file upload encounters an error
     uploader.on('error', ({ error }) => {
       logger.error('an error occured while uploading', error);
+      postPublishAnalytics('spee.ch/', null, error);
       socket.emit('publish-status', error);
     });
+    // listener for when file has been uploaded
     uploader.on('saved', ({ file }) => {
       if (file.success) {
         logger.debug(`Client successfully uploaded ${file.name}`);
@@ -37,14 +39,18 @@ module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => {
         publishController
           .publish(publishParams, file.name, file.meta.type)
           .then(result => {
+            postPublishAnalytics('spee.ch/', null, 'success');
             socket.emit('publish-complete', { name: publishParams.name, result });
           })
           .catch(error => {
-            socket.emit('publish-failure', errorHandlers.handlePublishError(error));
+            error = errorHandlers.handlePublishError(error);
+            postPublishAnalytics('spee.ch/', null, error);
+            socket.emit('publish-failure', error);
           });
       } else {
         logger.error(`An error occurred in uploading the client's file`);
         socket.emit('publish-failure', 'file uploaded, but with errors');
+        postPublishAnalytics('spee.ch/', null, 'file uploaded, but with errors');
         // to-do: remove the file if not done automatically
       }
     });

server.js

@@ -5,10 +5,8 @@ const siofu = require('socketio-file-upload');
 const expressHandlebars = require('express-handlebars');
 const Handlebars = require('handlebars');
 const config = require('config');
-const ua = require('universal-analytics');
 const winston = require('winston');
 
-const googleAnalyticsId = config.get('AnalyticsConfig.GoogleId');
 const hostedContentPath = config.get('Database.PublishUploadPath');
 
 // configure logging
@@ -57,17 +55,18 @@ app.set('view engine', 'handlebars');
 
 // require express routes
 require('./routes/api-routes.js')(app);
-require('./routes/show-routes.js')(app, ua, googleAnalyticsId);
-require('./routes/serve-routes.js')(app, ua, googleAnalyticsId);
+require('./routes/analytics-routes.js')(app);
+require('./routes/show-routes.js')(app);
+require('./routes/serve-routes.js')(app);
 require('./routes/home-routes.js')(app);
 
 // require socket.io routes
-const http = require('./routes/sockets-routes.js')(app, siofu, hostedContentPath, ua, googleAnalyticsId);
+const http = require('./routes/sockets-routes.js')(app, siofu, hostedContentPath);
 
 // sync sequelize
 // wrap the server in socket.io to intercept incoming sockets requests
 // start server
-db.sequelize.sync({}).then(() => {
+db.sequelize.sync().then(() => {
   http.listen(PORT, () => {
     winston.info(`Server is listening on PORT ${PORT}`);
   });

views/analytics.handlebars

@@ -0,0 +1,37 @@
+<div class="wrapper">
+	<div class="top-bar">
+		{{> topBar}}
+	</div>
+	<div>
+		<h3>Analytics</h3>
+		<table>
+			<tr>
+				<th>action</th>
+				<th>url</th> 
+				<th class="center-text">count</th>
+				<th class="center-text">success</th>
+				<th class="center-text">failure</th>
+			</tr>
+			{{#each records}}
+			<tr>
+				<td>{{ this.action }}</td>
+				<td>{{ this.url }}</td> 
+				<td class="center-text">{{ this.count }}</td>
+				<td class="center-text">{{ this.success }}</td>
+				<td class="center-text">{{ this.failure }}</td>
+			</tr>
+			{{/each}}
+			<tr>
+				<td></td>
+				<td></td> 
+				<td class="totals-row center-text">{{ totals.totalCount }}</td>
+				<td class="totals-row center-text">{{ totals.totalSuccess }}</td>
+				<td class="totals-row center-text">{{ totals.totalFailure }}</td>
+			</tr>
+		</table>
+		<p>Requests: {{ totals.totalRequests }}</p>
+		<p>Publish: {{ totals.totalPublish }}</p>
+		<p>Show: {{ totals.totalShow }}</p>
+		<p>Percent Success: {{ percentSuccess}}%</p>
+	</div>
+</div>