LBRYCommunity/spee.ch
1
0
Fork 0
Code Issues 95 Pull requests 2 Projects Releases 4 Packages Wiki Activity

Merge pull request #78 from lbryio/trending-images-2

Browse source 
Trending images 2
This commit is contained in:
Bill Bittner 2017-07-12 23:50:45 -07:00 • committed by GitHub
commit 8c7346da5d
12 changed files with 87 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
config/development.json
View file

@ -9,7 +9,7 @@
}, },
"Database": { "Database": {
"MySqlConnectionUri": "none", "MySqlConnectionUri": "none",
"DownloadDirectory": "/home/ubuntu/Downloads/" "DownloadDirectory": "/home/lbry/Downloads/"
}, },
"Logging": { "Logging": {
"LogLevel": "silly" "LogLevel": "silly"

109
controllers/statsController.js
View file

@ -5,8 +5,7 @@ const db = require('../models');
const googleApiKey = config.get('AnalyticsConfig.GoogleId'); const googleApiKey = config.get('AnalyticsConfig.GoogleId');
module.exports = { module.exports = {
postToStats (action, url, ipAddress, name, claimId, fileName, fileType, nsfw, result) { postToStats (action, url, ipAddress, name, claimId, result) {
logger.silly(`creating ${action} record for statistics db`);
// make sure the result is a string // make sure the result is a string
if (result && (typeof result !== 'string')) { if (result && (typeof result !== 'string')) {
result = result.toString(); result = result.toString();
@ -15,22 +14,30 @@ module.exports = {
if (ipAddress && (typeof ipAddress !== 'string')) { if (ipAddress && (typeof ipAddress !== 'string')) {
ipAddress = ipAddress.toString(); ipAddress = ipAddress.toString();
} }
// create record in the db logger.silly(name, claimId);
db.Stats.create({ db.File
action, .findOne({where: { name, claimId }})
url, .then(file => {
ipAddress, // create record in the db
name, let FileId;
claimId, if (file) {
fileName, FileId = file.dataValues.id;
fileType, } else {
nsfw, FileId = null;
result, }
}) logger.silly('file id:', FileId);
.then() return db.Request
.catch(error => { .create({
logger.error('sequelize error', error); action,
}); url,
ipAddress,
result,
FileId,
});
})
.catch(error => {
logger.error('sequelize error', error);
});
}, },
sendGoogleAnalytics (action, headers, ip, originalUrl) { sendGoogleAnalytics (action, headers, ip, originalUrl) {
const visitorId = ip.replace(/\./g, '-'); const visitorId = ip.replace(/\./g, '-');
@ -64,10 +71,10 @@ module.exports = {
}); });
}, },
getStatsSummary (startDate) { getStatsSummary (startDate) {
logger.debug('retrieving statistics'); logger.debug('retrieving request records');
const deferred = new Promise((resolve, reject) => { const deferred = new Promise((resolve, reject) => {
// get the raw statistics data // get the raw Requests data
db.Stats db.Request
.findAll({ .findAll({
where: { where: {
createdAt: { createdAt: {
@ -138,61 +145,13 @@ module.exports = {
return deferred; return deferred;
}, },
getTrendingClaims (startDate) { getTrendingClaims (startDate) {
logger.debug('retrieving trending statistics'); logger.debug('retrieving trending requests');
const deferred = new Promise((resolve, reject) => { const deferred = new Promise((resolve, reject) => {
// get the raw statistics data // get the raw requests data
db.Stats db.sequelize
.findAll({ .query('SELECT COUNT(*), file.* FROM request LEFT JOIN file ON request.FileId = file.id WHERE FileId != "null" AND nsfw != 1 GROUP BY FileId ORDER BY COUNT(*) DESC LIMIT 25;', { type: db.sequelize.QueryTypes.SELECT })
where: { .then(results => {
createdAt: { resolve(results);
gt: startDate,
},
name: {
not: null,
},
claimId: {
not: null,
},
},
})
.then(data => {
let resultHashTable = {};
let sortableArray = [];
let sortedArray;
// summarise the data
for (let i = 0; i < data.length; i++) {
let key = `${data[i].name}#${data[i].claimId}`;
if (resultHashTable[key] === undefined) {
resultHashTable[key] = {
count : 0,
details: {
name : data[i].name,
claimId : data[i].claimId,
fileName: data[i].fileName,
fileType: data[i].fileType,
nsfw : data[i].nsfw,
},
};
} else {
resultHashTable[key]['count'] += 1;
}
}
for (let objKey in resultHashTable) {
if (resultHashTable.hasOwnProperty(objKey)) {
sortableArray.push([
resultHashTable[objKey]['count'],
resultHashTable[objKey]['details'],
]);
}
}
sortableArray.sort((a, b) => {
return b[0] - a[0];
});
sortedArray = sortableArray.map((a) => {
return a[1];
});
// return results
resolve(sortedArray);
}) })
.catch(error => { .catch(error => {
logger.error('sequelize error', error); logger.error('sequelize error', error);

8
helpers/libraries/errorHandlers.js
View file

@ -5,16 +5,16 @@ module.exports = {
handleRequestError (action, originalUrl, ip, error, res) { handleRequestError (action, originalUrl, ip, error, res) {
logger.error('Request Error >>', error); logger.error('Request Error >>', error);
if (error.response) { if (error.response) {
postToStats(action, originalUrl, ip, null, null, null, null, null, error.response.data.error.messsage); postToStats(action, originalUrl, ip, null, null, error.response.data.error.messsage);
res.status(error.response.status).send(error.response.data.error.message); res.status(error.response.status).send(error.response.data.error.message);
} else if (error.code === 'ECONNREFUSED') { } else if (error.code === 'ECONNREFUSED') {
postToStats(action, originalUrl, ip, null, null, null, null, null, 'Connection refused. The daemon may not be running.'); postToStats(action, originalUrl, ip, null, null, 'Connection refused. The daemon may not be running.');
res.status(503).send('Connection refused. The daemon may not be running.'); res.status(503).send('Connection refused. The daemon may not be running.');
} else if (error.message) { } else if (error.message) {
postToStats(action, originalUrl, ip, null, null, null, null, null, error); postToStats(action, originalUrl, ip, null, null, error);
res.status(400).send(error.message); res.status(400).send(error.message);
} else { } else {
postToStats(action, originalUrl, ip, null, null, null, null, null, error); postToStats(action, originalUrl, ip, null, null, error);
res.status(400).send(error); res.status(400).send(error);
} }
}, },

5
models/file.js
View file

@ -44,5 +44,10 @@ module.exports = (sequelize, { STRING, BOOLEAN, INTEGER }) => {
freezeTableName: true, freezeTableName: true,
} }
); );
File.associate = db => {
File.hasMany(db.Request);
};
return File; return File;
}; };

14
models/index.js
View file

@ -20,13 +20,19 @@ sequelize
logger.error('Sequelize was unable to connect to the database:', err); logger.error('Sequelize was unable to connect to the database:', err);
}); });
fs.readdirSync(__dirname).filter(file => file.indexOf('.') !== 0 && file !== basename && file.slice(-3) === '.js').forEach(file => { fs
const model = sequelize['import'](path.join(__dirname, file)); .readdirSync(__dirname)
db[model.name] = model; .filter(file => {
}); return (file.indexOf('.') !== 0 && file !== basename && file.slice(-3) === '.js');
})
.forEach(file => {
const model = sequelize['import'](path.join(__dirname, file));
db[model.name] = model;
});
Object.keys(db).forEach(modelName => { Object.keys(db).forEach(modelName => {
if (db[modelName].associate) { if (db[modelName].associate) {
logger.info('Associating model:', modelName);
db[modelName].associate(db); db[modelName].associate(db);
} }
}); });

36
models/stats.js → models/request.js
View file

@ -1,6 +1,6 @@
module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => { module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => {
const Stats = sequelize.define( const Request = sequelize.define(
'Stats', 'Request',
{ {
action: { action: {
type : STRING, type : STRING,
@ -14,26 +14,6 @@ module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => {
type : STRING, type : STRING,
allowNull: true, allowNull: true,
}, },
name: {
type : STRING,
allowNull: true,
},
claimId: {
type : STRING,
allowNull: true,
},
fileName: {
type : STRING,
allowNull: true,
},
fileType: {
type : STRING,
allowNull: true,
},
nsfw: {
type : BOOLEAN,
allowNull: true,
},
result: { result: {
type : TEXT('long'), type : TEXT('long'),
allowNull: true, allowNull: true,
@ -44,5 +24,15 @@ module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => {
freezeTableName: true, freezeTableName: true,
} }
); );
return Stats;
Request.associate = db => {
Request.belongsTo(db.File, {
onDelete : 'cascade',
foreignKey: {
allowNull: true,
},
});
};
return Request;
}; };

8
routes/api-routes.js
View file

@ -24,7 +24,7 @@ module.exports = app => {
lbryApi lbryApi
.getClaimsList(params.name) .getClaimsList(params.name)
.then(claimsList => { .then(claimsList => {
postToStats('serve', originalUrl, ip, null, null, null, null, 'success'); postToStats('serve', originalUrl, ip, null, null, 'success');
res.status(200).json(claimsList); res.status(200).json(claimsList);
}) })
.catch(error => { .catch(error => {
@ -56,7 +56,7 @@ module.exports = app => {
lbryApi lbryApi
.resolveUri(params.uri) .resolveUri(params.uri)
.then(resolvedUri => { .then(resolvedUri => {
postToStats('serve', originalUrl, ip, null, null, null, null, 'success'); postToStats('serve', originalUrl, ip, null, null, 'success');
res.status(200).json(resolvedUri); res.status(200).json(resolvedUri);
}) })
.catch(error => { .catch(error => {
@ -76,7 +76,7 @@ module.exports = app => {
try { try {
validateFile(file, name, license, nsfw); validateFile(file, name, license, nsfw);
} catch (error) { } catch (error) {
postToStats('publish', originalUrl, ip, null, null, null, null, error.message); postToStats('publish', originalUrl, ip, null, null, error.message);
logger.debug('rejected >>', error.message); logger.debug('rejected >>', error.message);
res.status(400).send(error.message); res.status(400).send(error.message);
return; return;
@ -91,7 +91,7 @@ module.exports = app => {
publishController publishController
.publish(publishParams, fileName, fileType) .publish(publishParams, fileName, fileType)
.then(result => { .then(result => {
postToStats('publish', originalUrl, ip, null, null, null, null, 'success'); postToStats('publish', originalUrl, ip, null, null, 'success');
res.status(200).json(result); res.status(200).json(result);
}) })
.catch(error => { .catch(error => {

2
routes/home-routes.js
View file

@ -11,7 +11,7 @@ module.exports = app => {
app.use('*', ({ originalUrl, ip }, res) => { app.use('*', ({ originalUrl, ip }, res) => {
logger.error(`404 on ${originalUrl}`); logger.error(`404 on ${originalUrl}`);
// post to stats // post to stats
postToStats('show', originalUrl, ip, null, null, null, null, null, 'Error: 404'); postToStats('show', originalUrl, ip, null, null, 'Error: 404');
// send response // send response
res.status(404).render('fourOhFour'); res.status(404).render('fourOhFour');
}); });

12
routes/serve-routes.js
View file

@ -52,14 +52,14 @@ module.exports = (app) => {
if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header
const mimetypes = headers['accept'].split(','); const mimetypes = headers['accept'].split(',');
if (mimetypes.includes('text/html')) { if (mimetypes.includes('text/html')) {
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('showLite', { fileInfo }); res.status(200).render('showLite', { fileInfo });
} else { } else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res); serveFile(fileInfo, res);
} }
} else { } else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res); serveFile(fileInfo, res);
} }
}) })
@ -82,14 +82,14 @@ module.exports = (app) => {
if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header
const mimetypes = headers['accept'].split(','); const mimetypes = headers['accept'].split(',');
if (mimetypes.includes('text/html')) { if (mimetypes.includes('text/html')) {
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('showLite', { fileInfo }); res.status(200).render('showLite', { fileInfo });
} else { } else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res); serveFile(fileInfo, res);
} }
} else { } else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res); serveFile(fileInfo, res);
} }
}) })

10
routes/show-routes.js
View file

@ -27,7 +27,7 @@ module.exports = (app) => {
startDate.setDate(startDate.getDate() - 1); startDate.setDate(startDate.getDate() - 1);
getStatsSummary(startDate) getStatsSummary(startDate)
.then(result => { .then(result => {
postToStats('show', originalUrl, ip, null, null, null, null, null, 'success'); postToStats('show', originalUrl, ip, null, null, 'success');
res.status(200).render('statistics', result); res.status(200).render('statistics', result);
}) })
.catch(error => { .catch(error => {
@ -39,7 +39,7 @@ module.exports = (app) => {
// get and render the content // get and render the content
getAllClaims('meme-fodder') getAllClaims('meme-fodder')
.then(orderedFreePublicClaims => { .then(orderedFreePublicClaims => {
postToStats('show', originalUrl, ip, null, null, null, null, null, 'success'); postToStats('show', originalUrl, ip, null, null, 'success');
res.status(200).render('memeFodder', { claims: orderedFreePublicClaims }); res.status(200).render('memeFodder', { claims: orderedFreePublicClaims });
}) })
.catch(error => { .catch(error => {
@ -55,7 +55,7 @@ module.exports = (app) => {
res.status(307).render('noClaims'); res.status(307).render('noClaims');
return; return;
} }
postToStats('show', originalUrl, ip, null, null, null, null, null, 'success'); postToStats('show', originalUrl, ip, null, null, 'success');
res.status(200).render('allClaims', { claims: orderedFreePublicClaims }); res.status(200).render('allClaims', { claims: orderedFreePublicClaims });
}) })
.catch(error => { .catch(error => {
@ -73,7 +73,7 @@ module.exports = (app) => {
return; return;
} }
// serve the file or the show route // serve the file or the show route
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('show', { fileInfo }); res.status(200).render('show', { fileInfo });
}) })
.catch(error => { .catch(error => {
@ -91,7 +91,7 @@ module.exports = (app) => {
return; return;
} }
// serve the show route // serve the show route
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('show', { fileInfo }); res.status(200).render('show', { fileInfo });
}) })
.catch(error => { .catch(error => {

8
routes/sockets-routes.js
View file

@ -27,7 +27,7 @@ module.exports = (app, siofu, hostedContentPath) => {
// listener for when file upload encounters an error // listener for when file upload encounters an error
uploader.on('error', ({ error }) => { uploader.on('error', ({ error }) => {
logger.error('an error occured while uploading', error); logger.error('an error occured while uploading', error);
postToStats('publish', '/', null, null, null, null, null, error); postToStats('publish', '/', null, null, error);
socket.emit('publish-status', error); socket.emit('publish-status', error);
}); });
// listener for when file has been uploaded // listener for when file has been uploaded
@ -41,18 +41,18 @@ module.exports = (app, siofu, hostedContentPath) => {
publishController publishController
.publish(publishParams, file.name, file.meta.type) .publish(publishParams, file.name, file.meta.type)
.then(result => { .then(result => {
postToStats('publish', '/', null, null, null, null, null, 'success'); postToStats('publish', '/', null, null, 'success');
socket.emit('publish-complete', { name: publishParams.name, result }); socket.emit('publish-complete', { name: publishParams.name, result });
}) })
.catch(error => { .catch(error => {
error = errorHandlers.handlePublishError(error); error = errorHandlers.handlePublishError(error);
postToStats('publish', '/', null, null, null, null, null, error); postToStats('publish', '/', null, null, error);
socket.emit('publish-failure', error); socket.emit('publish-failure', error);
}); });
} else { } else {
logger.error(`An error occurred in uploading the client's file`); logger.error(`An error occurred in uploading the client's file`);
socket.emit('publish-failure', 'File uploaded, but with errors'); socket.emit('publish-failure', 'File uploaded, but with errors');
postToStats('publish', '/', null, null, null, null, null, 'File uploaded, but with errors'); postToStats('publish', '/', null, null, 'File uploaded, but with errors');
// to-do: remove the file if not done automatically // to-do: remove the file if not done automatically
} }
}); });

1
views/index.handlebars
View file

@ -4,7 +4,6 @@
{{> publish}} {{> publish}}
{{> learnMore}} {{> learnMore}}
</div> </div>
{{> footer}}
</div> </div>
<script src="/socket.io/socket.io.js"></script> <script src="/socket.io/socket.io.js"></script>