LBRYCommunity/spee.ch
1
0
Fork 0
Code Issues 95 Pull requests 2 Projects Releases 4 Packages Wiki Activity

295 - server-side GA events for tracking serve requests #351

Merged
bones7242 merged 2 commits from 295-google-analytics into master 2018-02-16 03:02:36 +01:00
Conversation 1 Commits 2 Files changed 6 +185 -202
6 changed files with 185 additions and 202 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
helpers/googleAnalytics.js Normal file
View file

@ -0,0 +1,62 @@
const logger = require('winston');
const ua = require('universal-analytics');
const config = require('../config/speechConfig.js');
const googleApiKey = config.analytics.googleId;
function createServeEventParams (headers, ip, originalUrl) {
return {
eventCategory : 'client requests',
eventAction : 'serve request',
eventLabel : originalUrl,
ipOverride : ip,
userAgentOverride: headers['user-agent'],
};
};
function createPublishTimingEventParams (label, startTime, endTime, ip, headers) {
const durration = endTime - startTime;
return {
userTimingCategory : 'lbrynet',
userTimingVariableName: 'publish',
userTimingTime : durration,
userTimingLabel : label,
uip : ip,
userAgentOverride : headers['user-agent'],
};
};
function sendGoogleAnalyticsEvent (ip, params) {
const visitorId = ip.replace(/\./g, '-');
const visitor = ua(googleApiKey, visitorId, { strictCidFormat: false, https: true });
visitor.event(params, (err) => {
if (err) {
logger.error('Google Analytics Event Error >>', err);
}
});
};
function sendGoogleAnalyticsTiming (ip, params) {
const visitorId = ip.replace(/\./g, '-');
const visitor = ua(googleApiKey, visitorId, { strictCidFormat: false, https: true });
visitor.timing(params, (err) => {
if (err) {
logger.error('Google Analytics Event Error >>', err);
}
logger.debug(`Timing event successfully sent to google analytics`);
});
};
module.exports = {
sendGAServeEvent (headers, ip, originalUrl) {
const params = createServeEventParams(headers, ip, originalUrl);
sendGoogleAnalyticsEvent(ip, params);
},
sendGAAnonymousPublishTiming (headers, ip, originalUrl, startTime, endTime) {
const params = createPublishTimingEventParams('anonymous', startTime, endTime, ip, headers);
sendGoogleAnalyticsTiming(ip, params);
},
sendGAChannelPublishTiming (headers, ip, originalUrl, startTime, endTime) {
const params = createPublishTimingEventParams('anonymous', startTime, endTime, ip, headers);
sendGoogleAnalyticsTiming(ip, params);
},
};

8
helpers/publishHelpers.js
View file

@ -1,4 +1,3 @@
const constants = require('../constants');
const logger = require('winston');
const fs = require('fs');
const { site, wallet } = require('../config/speechConfig.js');
@ -177,11 +176,4 @@ module.exports = {
nsfw,
};
},
returnPublishTimingActionType (channelName) {
if (channelName) {
return constants.PUBLISH_IN_CHANNEL_CLAIM;
} else {
return constants.PUBLISH_ANONYMOUS_CLAIM;
}
},
};

112
helpers/serveHelpers.js
View file

@ -1,25 +1,109 @@
const logger = require('winston');
const { getClaimId, getLocalFileRecord } = require('../controllers/serveController.js');
const { handleErrorResponse } = require('../helpers/errorHandlers.js');
module.exports = {
serveFile ({ filePath, fileType }, claimId, name, res) {
const SERVE = 'SERVE';
const SHOW = 'SHOW';
const NO_FILE = 'NO_FILE';
const NO_CHANNEL = 'NO_CHANNEL';
const NO_CLAIM = 'NO_CLAIM';
function clientAcceptsHtml ({accept}) {
return accept && accept.match(/text\/html/);
};
function requestIsFromBrowser (headers) {
return headers['user-agent'] && headers['user-agent'].match(/Mozilla/);
};
function clientWantsAsset ({accept, range}) {
const imageIsWanted = accept && accept.match(/image\/.*/) && !accept.match(/text\/html/) && !accept.match(/text\/\*/);
const videoIsWanted = accept && range;
return imageIsWanted || videoIsWanted;
};
function isValidClaimId (claimId) {
return ((claimId.length === 40) && !/[^A-Za-z0-9]/g.test(claimId));
};
function isValidShortId (claimId) {
return claimId.length === 1; // it should really evaluate the short url itself
};
function isValidShortIdOrClaimId (input) {
return (isValidClaimId(input) || isValidShortId(input));
};
function serveAssetToClient (claimId, name, res) {
return getLocalFileRecord(claimId, name)
.then(fileRecord => {
// check that a local record was found
if (fileRecord === NO_FILE) {
return res.status(307).redirect(`/api/claim/get/${name}/${claimId}`);
}
// serve the file
const {filePath, fileType} = fileRecord;
logger.verbose(`serving file: ${filePath}`);
// set response options
const headerContentType = fileType || 'image/jpeg';
const options = {
const sendFileOptions = {
headers: {
'X-Content-Type-Options': 'nosniff',
'Content-Type' : headerContentType,
'Content-Type' : fileType || 'image/jpeg',
},
};
// send the file
res.status(200).sendFile(filePath, options);
res.status(200).sendFile(filePath, sendFileOptions);
})
.catch(error => {
throw error;
});
};
module.exports = {
getClaimIdAndServeAsset (channelName, channelClaimId, claimName, claimId, originalUrl, ip, res) {
// get the claim Id and then serve the asset
getClaimId(channelName, channelClaimId, claimName, claimId)
.then(fullClaimId => {
if (fullClaimId === NO_CLAIM) {
return res.status(404).json({success: false, message: 'no claim id could be found'});
} else if (fullClaimId === NO_CHANNEL) {
return res.status(404).json({success: false, message: 'no channel id could be found'});
}
serveAssetToClient(fullClaimId, claimName, res);
// postToStats(responseType, originalUrl, ip, claimName, fullClaimId, 'success');
})
.catch(error => {
handleErrorResponse(originalUrl, ip, error, res);
// postToStats(responseType, originalUrl, ip, claimName, fullClaimId, 'fail');
});
},
showFile (claimInfo, shortId, res) {
logger.verbose(`showing claim: ${claimInfo.name}#${claimInfo.claimId}`);
res.status(200).render('index');
determineResponseType (hasFileExtension, headers) {
let responseType;
if (hasFileExtension) {
responseType = SERVE; // assume a serve request if file extension is present
if (clientAcceptsHtml(headers)) { // if the request comes from a browser, change it to a show request
responseType = SHOW;
}
} else {
responseType = SHOW;
if (clientWantsAsset(headers) && requestIsFromBrowser(headers)) { // this is in case someone embeds a show url
logger.debug('Show request came from browser but wants an image/video. Changing response to serve...');
responseType = SERVE;
}
}
return responseType;
},
showFileLite (claimInfo, shortId, res) {
logger.verbose(`showlite claim: ${claimInfo.name}#${claimInfo.claimId}`);
res.status(200).render('index');
flipClaimNameAndIdForBackwardsCompatibility (identifier, name) {
// this is a patch for backwards compatability with '/name/claim_id' url format
if (isValidShortIdOrClaimId(name) && !isValidShortIdOrClaimId(identifier)) {
const tempName = name;
name = identifier;
identifier = tempName;
}
return [identifier, name];
},
logRequestData (responseType, claimName, channelName, claimId) {
logger.debug('responseType ===', responseType);
logger.debug('claim name === ', claimName);
logger.debug('channel name ===', channelName);
logger.debug('claim id ===', claimId);
},
};

57
helpers/statsHelpers.js
View file

@ -1,22 +1,7 @@
const constants = require('../constants');
const logger = require('winston');
const ua = require('universal-analytics');
const config = require('../config/speechConfig.js');
const googleApiKey = config.analytics.googleId;
const db = require('../models');
module.exports = {
createPublishTimingEventParams (publishDurration, ip, headers, label) {
return {
userTimingCategory : 'lbrynet',
userTimingVariableName: 'publish',
userTimingTime : publishDurration,
userTimingLabel : label,
uip : ip,
ua : headers['user-agent'],
ul : headers['accept-language'],
};
},
postToStats (action, url, ipAddress, name, claimId, result) {
logger.debug('action:', action);
// make sure the result is a string
@ -50,46 +35,4 @@ module.exports = {
logger.error('Sequelize error >>', error);
});
},
sendGoogleAnalyticsEvent (action, headers, ip, originalUrl) {
const visitorId = ip.replace(/\./g, '-');
const visitor = ua(googleApiKey, visitorId, { strictCidFormat: false, https: true });
let params;
switch (action) {
case 'SERVE':
params = {
ec : 'serve',
ea : originalUrl,
uip: ip,
ua : headers['user-agent'],
ul : headers['accept-language'],
};
break;
default: break;
}
visitor.event(params, (err) => {
if (err) {
logger.error('Google Analytics Event Error >>', err);
}
});
},
sendGoogleAnalyticsTiming (action, headers, ip, originalUrl, startTime, endTime) {
const visitorId = ip.replace(/\./g, '-');
const visitor = ua(googleApiKey, visitorId, { strictCidFormat: false, https: true });
const durration = endTime - startTime;
let params;
switch (action) {
case constants.PUBLISH_ANONYMOUS_CLAIM:
case constants.PUBLISH_IN_CHANNEL_CLAIM:
logger.verbose(`${action} completed successfully in ${durration}ms`);
params = module.exports.createPublishTimingEventParams(durration, ip, headers, action);
break;
default: break;
}
visitor.timing(params, (err) => {
if (err) {
logger.error('Google Analytics Event Error >>', err);
}
logger.debug(`${action} timing event successfully sent to google analytics`);
});
},
};

21
routes/api-routes.js
View file

@ -5,9 +5,9 @@ const multipartMiddleware = multipart({uploadDir: files.uploadDirectory});
const db = require('../models');
const { checkClaimNameAvailability, checkChannelAvailability, publish } = require('../controllers/publishController.js');
const { getClaimList, resolveUri, getClaim } = require('../helpers/lbryApi.js');
const { createPublishParams, parsePublishApiRequestBody, parsePublishApiRequestFiles, parsePublishApiChannel, addGetResultsToFileData, createFileData, returnPublishTimingActionType } = require('../helpers/publishHelpers.js');
const { createPublishParams, parsePublishApiRequestBody, parsePublishApiRequestFiles, parsePublishApiChannel, addGetResultsToFileData, createFileData } = require('../helpers/publishHelpers.js');
const errorHandlers = require('../helpers/errorHandlers.js');
const { sendGoogleAnalyticsTiming } = require('../helpers/statsHelpers.js');
const { sendGAAnonymousPublishTiming, sendGAChannelPublishTiming } = require('../helpers/googleAnalytics.js');
const { authenticateIfNoUserToken } = require('../auth/authentication.js');
const { getChannelData, getChannelClaims, getClaimId } = require('../controllers/serveController.js');
@ -134,12 +134,10 @@ module.exports = (app) => {
app.post('/api/claim/publish', multipartMiddleware, ({ body, files, headers, ip, originalUrl, user }, res) => {
logger.debug('api/claim-publish body:', body);
logger.debug('api/claim-publish files:', files);
// record the start time of the request and create variable for storing the action type
const publishStartTime = Date.now();
logger.debug('publish request started @', publishStartTime);
let timingActionType;
// define variables
let name, fileName, filePath, fileType, nsfw, license, title, description, thumbnail, channelName, channelPassword;
// record the start time of the request
const publishStartTime = Date.now();
// validate the body and files of the request
try {
// validateApiPublishRequest(body, files);
@ -166,8 +164,6 @@ module.exports = (app) => {
return createPublishParams(filePath, name, title, description, license, nsfw, thumbnail, channelName);
})
.then(publishParams => {
// set the timing event type for reporting
timingActionType = returnPublishTimingActionType(publishParams.channel_name);
// publish the asset
return publish(publishParams, fileName, fileType);
})
@ -182,10 +178,13 @@ module.exports = (app) => {
lbryTx : result,
},
});
// log the publish end time
// record the publish end time and send to google analytics
const publishEndTime = Date.now();
logger.debug('publish request completed @', publishEndTime);
sendGoogleAnalyticsTiming(timingActionType, headers, ip, originalUrl, publishStartTime, publishEndTime);
if (channelName) {
sendGAChannelPublishTiming(headers, ip, originalUrl, publishStartTime, publishEndTime);
} else {
sendGAAnonymousPublishTiming(headers, ip, originalUrl, publishStartTime, publishEndTime);
}
})
.catch(error => {
errorHandlers.handleErrorResponse(originalUrl, ip, error, res);

119
routes/serve-routes.js
View file

@ -1,88 +1,9 @@
const logger = require('winston');
const { getClaimId, getLocalFileRecord } = require('../controllers/serveController.js');
const serveHelpers = require('../helpers/serveHelpers.js');
const { handleErrorResponse } = require('../helpers/errorHandlers.js');
const { sendGAServeEvent } = require('../helpers/googleAnalytics');
const { determineResponseType, flipClaimNameAndIdForBackwardsCompatibility, logRequestData, getClaimIdAndServeAsset } = require('../helpers/serveHelpers.js');
const lbryUri = require('../helpers/lbryUri.js');
const SERVE = 'SERVE';
const SHOW = 'SHOW';
const NO_CHANNEL = 'NO_CHANNEL';
const NO_CLAIM = 'NO_CLAIM';
const NO_FILE = 'NO_FILE';
function isValidClaimId (claimId) {
return ((claimId.length === 40) && !/[^A-Za-z0-9]/g.test(claimId));
}
function isValidShortId (claimId) {
return claimId.length === 1; // it should really evaluate the short url itself
}
function isValidShortIdOrClaimId (input) {
return (isValidClaimId(input) || isValidShortId(input));
}
function clientAcceptsHtml ({accept}) {
return accept && accept.match(/text\/html/);
}
function requestIsFromBrowser (headers) {
return headers['user-agent'] && headers['user-agent'].match(/Mozilla/);
};
function clientWantsAsset ({accept, range}) {
const imageIsWanted = accept && accept.match(/image\/.*/) && !accept.match(/text\/html/) && !accept.match(/text\/\*/);
const videoIsWanted = accept && range;
return imageIsWanted || videoIsWanted;
}
function determineResponseType (hasFileExtension, headers) {
let responseType;
if (hasFileExtension) {
responseType = SERVE; // assume a serve request if file extension is present
if (clientAcceptsHtml(headers)) { // if the request comes from a browser, change it to a show request
responseType = SHOW;
}
} else {
responseType = SHOW;
if (clientWantsAsset(headers) && requestIsFromBrowser(headers)) { // this is in case someone embeds a show url
logger.debug('Show request came from browser but wants an image/video. Changing response to serve...');
responseType = SERVE;
}
}
return responseType;
}
function serveAssetToClient (claimId, name, res) {
return getLocalFileRecord(claimId, name)
.then(fileInfo => {
// logger.debug('fileInfo:', fileInfo);
if (fileInfo === NO_FILE) {
return res.status(307).redirect(`/api/claim/get/${name}/${claimId}`);
}
return serveHelpers.serveFile(fileInfo, claimId, name, res);
})
.catch(error => {
throw error;
});
}
function flipClaimNameAndIdForBackwardsCompatibility (identifier, name) {
// this is a patch for backwards compatability with '/name/claim_id' url format
if (isValidShortIdOrClaimId(name) && !isValidShortIdOrClaimId(identifier)) {
const tempName = name;
name = identifier;
identifier = tempName;
}
return [identifier, name];
}
function logRequestData (responseType, claimName, channelName, claimId) {
logger.debug('responseType ===', responseType);
logger.debug('claim name === ', claimName);
logger.debug('channel name ===', channelName);
logger.debug('claim id ===', claimId);
}
module.exports = (app) => {
// route to serve a specific asset using the channel or claim id
@ -98,6 +19,9 @@ module.exports = (app) => {
if (responseType !== SERVE) {
return res.status(200).render('index');
}
// handle serve request
// send google analytics
sendGAServeEvent(headers, ip, originalUrl);
// parse the claim
let claimName;
try {
@ -118,20 +42,7 @@ module.exports = (app) => {
// log the request data for debugging
logRequestData(responseType, claimName, channelName, claimId);
// get the claim Id and then serve the asset
getClaimId(channelName, channelClaimId, claimName, claimId)
.then(fullClaimId => {
if (fullClaimId === NO_CLAIM) {
return res.status(404).json({success: false, message: 'no claim id could be found'});
} else if (fullClaimId === NO_CHANNEL) {
return res.status(404).json({success: false, message: 'no channel id could be found'});
}
serveAssetToClient(fullClaimId, claimName, res);
// postToStats(responseType, originalUrl, ip, claimName, fullClaimId, 'success');
})
.catch(error => {
handleErrorResponse(originalUrl, ip, error, res);
// postToStats(responseType, originalUrl, ip, claimName, fullClaimId, 'fail');
});
getClaimIdAndServeAsset(channelName, channelClaimId, claimName, claimId, originalUrl, ip, res);
});
// route to serve the winning asset at a claim or a channel page
app.get('/:claim', ({ headers, ip, originalUrl, params, query }, res) => {
@ -146,6 +57,9 @@ module.exports = (app) => {
if (responseType !== SERVE) {
return res.status(200).render('index');
}
// handle serve request
// send google analytics
sendGAServeEvent(headers, ip, originalUrl);
// parse the claim
let claimName;
try {
@ -156,17 +70,6 @@ module.exports = (app) => {
// log the request data for debugging
logRequestData(responseType, claimName, null, null);
// get the claim Id and then serve the asset
getClaimId(null, null, claimName, null)
.then(fullClaimId => {
if (fullClaimId === NO_CLAIM) {
return res.status(404).json({success: false, message: 'no claim id could be found'});
}
serveAssetToClient(fullClaimId, claimName, res);
// postToStats(responseType, originalUrl, ip, claimName, fullClaimId, 'success');
})
.catch(error => {
handleErrorResponse(originalUrl, ip, error, res);
// postToStats(responseType, originalUrl, ip, claimName, fullClaimId, 'fail');
});
getClaimIdAndServeAsset(null, null, claimName, null, originalUrl, ip, res);
});
};