LBRYCommunity/spee.ch
1
0
Fork 0
Code Issues 95 Pull requests 2 Projects Releases 4 Packages Wiki Activity

associated the File and Request dbs

Browse source
This commit is contained in:
bill bittner 2017-07-12 15:30:31 -07:00
parent 9df408690a
commit 99c7c84cfb
12 changed files with 124 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
config/development.json
View file

@ -9,7 +9,7 @@
}, },
"Database": { "Database": {
"MySqlConnectionUri": "none", "MySqlConnectionUri": "none",
"DownloadDirectory": "/home/ubuntu/Downloads/" "DownloadDirectory": "C:\\Users\\Bones\\Downloads\\lbry\\"
}, },
"Logging": { "Logging": {
"LogLevel": "silly" "LogLevel": "silly"

136
controllers/statsController.js
View file

@ -5,8 +5,7 @@ const db = require('../models');
const googleApiKey = config.get('AnalyticsConfig.GoogleId'); const googleApiKey = config.get('AnalyticsConfig.GoogleId');
module.exports = { module.exports = {
postToStats (action, url, ipAddress, name, claimId, fileName, fileType, nsfw, result) { postToStats (action, url, ipAddress, name, claimId, result) {
logger.silly(`creating ${action} record for statistics db`);
// make sure the result is a string // make sure the result is a string
if (result && (typeof result !== 'string')) { if (result && (typeof result !== 'string')) {
result = result.toString(); result = result.toString();
@ -15,22 +14,30 @@ module.exports = {
if (ipAddress && (typeof ipAddress !== 'string')) { if (ipAddress && (typeof ipAddress !== 'string')) {
ipAddress = ipAddress.toString(); ipAddress = ipAddress.toString();
} }
// create record in the db logger.silly(name, claimId);
db.Stats.create({ db.File
action, .findOne({where: { name, claimId }})
url, .then(file => {
ipAddress, // create record in the db
name, let FileId;
claimId, if (file) {
fileName, FileId = file.dataValues.id;
fileType, } else {
nsfw, FileId = null;
result, }
}) logger.silly('file id:', FileId);
.then() return db.Request
.catch(error => { .create({
logger.error('sequelize error', error); action,
}); url,
ipAddress,
result,
FileId,
});
})
.catch(error => {
logger.error('sequelize error', error);
});
}, },
sendGoogleAnalytics (action, headers, ip, originalUrl) { sendGoogleAnalytics (action, headers, ip, originalUrl) {
const visitorId = ip.replace(/\./g, '-'); const visitorId = ip.replace(/\./g, '-');
@ -64,10 +71,10 @@ module.exports = {
}); });
}, },
getStatsSummary (startDate) { getStatsSummary (startDate) {
logger.debug('retrieving statistics'); logger.debug('retrieving request records');
const deferred = new Promise((resolve, reject) => { const deferred = new Promise((resolve, reject) => {
// get the raw statistics data // get the raw Requests data
db.Stats db.Request
.findAll({ .findAll({
where: { where: {
createdAt: { createdAt: {
@ -138,61 +145,56 @@ module.exports = {
return deferred; return deferred;
}, },
getTrendingClaims (startDate) { getTrendingClaims (startDate) {
logger.debug('retrieving trending statistics'); logger.debug('retrieving trending requests');
const deferred = new Promise((resolve, reject) => { const deferred = new Promise((resolve, reject) => {
// get the raw statistics data // get the raw requests data
db.Stats db.Request
.findAll({ .findAll({
where: { where: {
createdAt: { createdAt: {
gt: startDate, gt: startDate,
}, },
name: {
not: null,
},
claimId: {
not: null,
},
}, },
include: [db.File],
}) })
.then(data => { .then(data => {
let resultHashTable = {}; // let resultHashTable = {};
let sortableArray = []; // let sortableArray = [];
let sortedArray; // let sortedArray;
// summarise the data // // summarise the data
for (let i = 0; i < data.length; i++) { // for (let i = 0; i < data.length; i++) {
let key = `${data[i].name}#${data[i].claimId}`; // let key = data[i].fileId;
if (resultHashTable[key] === undefined) { // if (resultHashTable[key] === undefined) {
resultHashTable[key] = { // resultHashTable[key] = {
count : 0, // count : 0,
details: { // details: {
name : data[i].name, // name : data[i].name,
claimId : data[i].claimId, // claimId : data[i].claimId,
fileName: data[i].fileName, // fileName: data[i].fileName,
fileType: data[i].fileType, // fileType: data[i].fileType,
nsfw : data[i].nsfw, // nsfw : data[i].nsfw,
}, // },
}; // };
} else { // } else {
resultHashTable[key]['count'] += 1; // resultHashTable[key]['count'] += 1;
} // }
} // }
for (let objKey in resultHashTable) { // for (let objKey in resultHashTable) {
if (resultHashTable.hasOwnProperty(objKey)) { // if (resultHashTable.hasOwnProperty(objKey)) {
sortableArray.push([ // sortableArray.push([
resultHashTable[objKey]['count'], // resultHashTable[objKey]['count'],
resultHashTable[objKey]['details'], // resultHashTable[objKey]['details'],
]); // ]);
} // }
} // }
sortableArray.sort((a, b) => { // sortableArray.sort((a, b) => {
return b[0] - a[0]; // return b[0] - a[0];
}); // });
sortedArray = sortableArray.map((a) => { // sortedArray = sortableArray.map((a) => {
return a[1]; // return a[1];
}); // });
// return results // // return results
resolve(sortedArray); resolve();
}) })
.catch(error => { .catch(error => {
logger.error('sequelize error', error); logger.error('sequelize error', error);

8
helpers/libraries/errorHandlers.js
View file

@ -5,16 +5,16 @@ module.exports = {
handleRequestError (action, originalUrl, ip, error, res) { handleRequestError (action, originalUrl, ip, error, res) {
logger.error('Request Error >>', error); logger.error('Request Error >>', error);
if (error.response) { if (error.response) {
postToStats(action, originalUrl, ip, null, null, null, null, null, error.response.data.error.messsage); postToStats(action, originalUrl, ip, null, null, error.response.data.error.messsage);
res.status(error.response.status).send(error.response.data.error.message); res.status(error.response.status).send(error.response.data.error.message);
} else if (error.code === 'ECONNREFUSED') { } else if (error.code === 'ECONNREFUSED') {
postToStats(action, originalUrl, ip, null, null, null, null, null, 'Connection refused. The daemon may not be running.'); postToStats(action, originalUrl, ip, null, null, 'Connection refused. The daemon may not be running.');
res.status(503).send('Connection refused. The daemon may not be running.'); res.status(503).send('Connection refused. The daemon may not be running.');
} else if (error.message) { } else if (error.message) {
postToStats(action, originalUrl, ip, null, null, null, null, null, error); postToStats(action, originalUrl, ip, null, null, error);
res.status(400).send(error.message); res.status(400).send(error.message);
} else { } else {
postToStats(action, originalUrl, ip, null, null, null, null, null, error); postToStats(action, originalUrl, ip, null, null, error);
res.status(400).send(error); res.status(400).send(error);
} }
}, },

6
models/file.js
View file

@ -44,5 +44,11 @@ module.exports = (sequelize, { STRING, BOOLEAN, INTEGER }) => {
freezeTableName: true, freezeTableName: true,
} }
); );
File.associate = db => {
console.log('test');
File.hasMany(db.Request);
};
return File; return File;
}; };

14
models/index.js
View file

@ -20,13 +20,19 @@ sequelize
logger.error('Sequelize was unable to connect to the database:', err); logger.error('Sequelize was unable to connect to the database:', err);
}); });
fs.readdirSync(__dirname).filter(file => file.indexOf('.') !== 0 && file !== basename && file.slice(-3) === '.js').forEach(file => { fs
const model = sequelize['import'](path.join(__dirname, file)); .readdirSync(__dirname)
db[model.name] = model; .filter(file => {
}); return (file.indexOf('.') !== 0 && file !== basename && file.slice(-3) === '.js');
})
.forEach(file => {
const model = sequelize['import'](path.join(__dirname, file));
db[model.name] = model;
});
Object.keys(db).forEach(modelName => { Object.keys(db).forEach(modelName => {
if (db[modelName].associate) { if (db[modelName].associate) {
logger.verbose('associating', modelName);
db[modelName].associate(db); db[modelName].associate(db);
} }
}); });

37
models/stats.js → models/request.js
View file

@ -1,6 +1,6 @@
module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => { module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => {
const Stats = sequelize.define( const Request = sequelize.define(
'Stats', 'Request',
{ {
action: { action: {
type : STRING, type : STRING,
@ -14,26 +14,6 @@ module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => {
type : STRING, type : STRING,
allowNull: true, allowNull: true,
}, },
name: {
type : STRING,
allowNull: true,
},
claimId: {
type : STRING,
allowNull: true,
},
fileName: {
type : STRING,
allowNull: true,
},
fileType: {
type : STRING,
allowNull: true,
},
nsfw: {
type : BOOLEAN,
allowNull: true,
},
result: { result: {
type : TEXT('long'), type : TEXT('long'),
allowNull: true, allowNull: true,
@ -44,5 +24,16 @@ module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => {
freezeTableName: true, freezeTableName: true,
} }
); );
return Stats;
Request.associate = db => {
console.log('test');
Request.belongsTo(db.File, {
onDelete : 'cascade',
foreignKey: {
allowNull: true,
},
});
};
return Request;
}; };

8
routes/api-routes.js
View file

@ -24,7 +24,7 @@ module.exports = app => {
lbryApi lbryApi
.getClaimsList(params.name) .getClaimsList(params.name)
.then(claimsList => { .then(claimsList => {
postToStats('serve', originalUrl, ip, null, null, null, null, 'success'); postToStats('serve', originalUrl, ip, null, null, 'success');
res.status(200).json(claimsList); res.status(200).json(claimsList);
}) })
.catch(error => { .catch(error => {
@ -56,7 +56,7 @@ module.exports = app => {
lbryApi lbryApi
.resolveUri(params.uri) .resolveUri(params.uri)
.then(resolvedUri => { .then(resolvedUri => {
postToStats('serve', originalUrl, ip, null, null, null, null, 'success'); postToStats('serve', originalUrl, ip, null, null, 'success');
res.status(200).json(resolvedUri); res.status(200).json(resolvedUri);
}) })
.catch(error => { .catch(error => {
@ -76,7 +76,7 @@ module.exports = app => {
try { try {
validateFile(file, name, license, nsfw); validateFile(file, name, license, nsfw);
} catch (error) { } catch (error) {
postToStats('publish', originalUrl, ip, null, null, null, null, error.message); postToStats('publish', originalUrl, ip, null, null, error.message);
logger.debug('rejected >>', error.message); logger.debug('rejected >>', error.message);
res.status(400).send(error.message); res.status(400).send(error.message);
return; return;
@ -91,7 +91,7 @@ module.exports = app => {
publishController publishController
.publish(publishParams, fileName, fileType) .publish(publishParams, fileName, fileType)
.then(result => { .then(result => {
postToStats('publish', originalUrl, ip, null, null, null, null, 'success'); postToStats('publish', originalUrl, ip, null, null, 'success');
res.status(200).json(result); res.status(200).json(result);
}) })
.catch(error => { .catch(error => {

2
routes/home-routes.js
View file

@ -11,7 +11,7 @@ module.exports = app => {
app.use('*', ({ originalUrl, ip }, res) => { app.use('*', ({ originalUrl, ip }, res) => {
logger.error(`404 on ${originalUrl}`); logger.error(`404 on ${originalUrl}`);
// post to stats // post to stats
postToStats('show', originalUrl, ip, null, null, null, null, null, 'Error: 404'); postToStats('show', originalUrl, ip, null, null, 'Error: 404');
// send response // send response
res.status(404).render('fourOhFour'); res.status(404).render('fourOhFour');
}); });

12
routes/serve-routes.js
View file

@ -52,14 +52,14 @@ module.exports = (app) => {
if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header
const mimetypes = headers['accept'].split(','); const mimetypes = headers['accept'].split(',');
if (mimetypes.includes('text/html')) { if (mimetypes.includes('text/html')) {
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('showLite', { fileInfo }); res.status(200).render('showLite', { fileInfo });
} else { } else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res); serveFile(fileInfo, res);
} }
} else { } else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res); serveFile(fileInfo, res);
} }
}) })
@ -82,14 +82,14 @@ module.exports = (app) => {
if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header
const mimetypes = headers['accept'].split(','); const mimetypes = headers['accept'].split(',');
if (mimetypes.includes('text/html')) { if (mimetypes.includes('text/html')) {
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('showLite', { fileInfo }); res.status(200).render('showLite', { fileInfo });
} else { } else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res); serveFile(fileInfo, res);
} }
} else { } else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res); serveFile(fileInfo, res);
} }
}) })

10
routes/show-routes.js
View file

@ -27,7 +27,7 @@ module.exports = (app) => {
startDate.setDate(startDate.getDate() - 1); startDate.setDate(startDate.getDate() - 1);
getStatsSummary(startDate) getStatsSummary(startDate)
.then(result => { .then(result => {
postToStats('show', originalUrl, ip, null, null, null, null, null, 'success'); postToStats('show', originalUrl, ip, null, null, 'success');
res.status(200).render('statistics', result); res.status(200).render('statistics', result);
}) })
.catch(error => { .catch(error => {
@ -39,7 +39,7 @@ module.exports = (app) => {
// get and render the content // get and render the content
getAllClaims('meme-fodder') getAllClaims('meme-fodder')
.then(orderedFreePublicClaims => { .then(orderedFreePublicClaims => {
postToStats('show', originalUrl, ip, null, null, null, null, null, 'success'); postToStats('show', originalUrl, ip, null, null, 'success');
res.status(200).render('memeFodder', { claims: orderedFreePublicClaims }); res.status(200).render('memeFodder', { claims: orderedFreePublicClaims });
}) })
.catch(error => { .catch(error => {
@ -55,7 +55,7 @@ module.exports = (app) => {
res.status(307).render('noClaims'); res.status(307).render('noClaims');
return; return;
} }
postToStats('show', originalUrl, ip, null, null, null, null, null, 'success'); postToStats('show', originalUrl, ip, null, null, 'success');
res.status(200).render('allClaims', { claims: orderedFreePublicClaims }); res.status(200).render('allClaims', { claims: orderedFreePublicClaims });
}) })
.catch(error => { .catch(error => {
@ -73,7 +73,7 @@ module.exports = (app) => {
return; return;
} }
// serve the file or the show route // serve the file or the show route
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('show', { fileInfo }); res.status(200).render('show', { fileInfo });
}) })
.catch(error => { .catch(error => {
@ -91,7 +91,7 @@ module.exports = (app) => {
return; return;
} }
// serve the show route // serve the show route
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success'); postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('show', { fileInfo }); res.status(200).render('show', { fileInfo });
}) })
.catch(error => { .catch(error => {

8
routes/sockets-routes.js
View file

@ -27,7 +27,7 @@ module.exports = (app, siofu, hostedContentPath) => {
// listener for when file upload encounters an error // listener for when file upload encounters an error
uploader.on('error', ({ error }) => { uploader.on('error', ({ error }) => {
logger.error('an error occured while uploading', error); logger.error('an error occured while uploading', error);
postToStats('publish', '/', null, null, null, null, null, error); postToStats('publish', '/', null, null, error);
socket.emit('publish-status', error); socket.emit('publish-status', error);
}); });
// listener for when file has been uploaded // listener for when file has been uploaded
@ -41,18 +41,18 @@ module.exports = (app, siofu, hostedContentPath) => {
publishController publishController
.publish(publishParams, file.name, file.meta.type) .publish(publishParams, file.name, file.meta.type)
.then(result => { .then(result => {
postToStats('publish', '/', null, null, null, null, null, 'success'); postToStats('publish', '/', null, null, 'success');
socket.emit('publish-complete', { name: publishParams.name, result }); socket.emit('publish-complete', { name: publishParams.name, result });
}) })
.catch(error => { .catch(error => {
error = errorHandlers.handlePublishError(error); error = errorHandlers.handlePublishError(error);
postToStats('publish', '/', null, null, null, null, null, error); postToStats('publish', '/', null, null, error);
socket.emit('publish-failure', error); socket.emit('publish-failure', error);
}); });
} else { } else {
logger.error(`An error occurred in uploading the client's file`); logger.error(`An error occurred in uploading the client's file`);
socket.emit('publish-failure', 'File uploaded, but with errors'); socket.emit('publish-failure', 'File uploaded, but with errors');
postToStats('publish', '/', null, null, null, null, null, 'File uploaded, but with errors'); postToStats('publish', '/', null, null, 'File uploaded, but with errors');
// to-do: remove the file if not done automatically // to-do: remove the file if not done automatically
} }
}); });

1
views/index.handlebars
View file

@ -4,7 +4,6 @@
{{> publish}} {{> publish}}
{{> learnMore}} {{> learnMore}}
</div> </div>
{{> footer}}
</div> </div>
<script src="/socket.io/socket.io.js"></script> <script src="/socket.io/socket.io.js"></script>