LBRYCommunity/spee.ch
1
0
Fork 0
Code Issues 95 Pull requests 2 Projects Releases 4 Packages Wiki Activity

associated the File and Request dbs

Browse source
This commit is contained in:
bill bittner 2017-07-12 15:30:31 -07:00
parent 9df408690a
commit 99c7c84cfb
12 changed files with 124 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
config/development.json
View file

@ -9,7 +9,7 @@
},
"Database": {
"MySqlConnectionUri": "none",
"DownloadDirectory": "/home/ubuntu/Downloads/"
"DownloadDirectory": "C:\\Users\\Bones\\Downloads\\lbry\\"
},
"Logging": {
"LogLevel": "silly"

136
controllers/statsController.js
View file

@ -5,8 +5,7 @@ const db = require('../models');
const googleApiKey = config.get('AnalyticsConfig.GoogleId');
module.exports = {
postToStats (action, url, ipAddress, name, claimId, fileName, fileType, nsfw, result) {
logger.silly(`creating ${action} record for statistics db`);
postToStats (action, url, ipAddress, name, claimId, result) {
// make sure the result is a string
if (result && (typeof result !== 'string')) {
result = result.toString();
@ -15,22 +14,30 @@ module.exports = {
if (ipAddress && (typeof ipAddress !== 'string')) {
ipAddress = ipAddress.toString();
}
// create record in the db
db.Stats.create({
action,
url,
ipAddress,
name,
claimId,
fileName,
fileType,
nsfw,
result,
})
.then()
.catch(error => {
logger.error('sequelize error', error);
});
logger.silly(name, claimId);
db.File
.findOne({where: { name, claimId }})
.then(file => {
// create record in the db
let FileId;
if (file) {
FileId = file.dataValues.id;
} else {
FileId = null;
}
logger.silly('file id:', FileId);
return db.Request
.create({
action,
url,
ipAddress,
result,
FileId,
});
})
.catch(error => {
logger.error('sequelize error', error);
});
},
sendGoogleAnalytics (action, headers, ip, originalUrl) {
const visitorId = ip.replace(/\./g, '-');
@ -64,10 +71,10 @@ module.exports = {
});
},
getStatsSummary (startDate) {
logger.debug('retrieving statistics');
logger.debug('retrieving request records');
const deferred = new Promise((resolve, reject) => {
// get the raw statistics data
db.Stats
// get the raw Requests data
db.Request
.findAll({
where: {
createdAt: {
@ -138,61 +145,56 @@ module.exports = {
return deferred;
},
getTrendingClaims (startDate) {
logger.debug('retrieving trending statistics');
logger.debug('retrieving trending requests');
const deferred = new Promise((resolve, reject) => {
// get the raw statistics data
db.Stats
// get the raw requests data
db.Request
.findAll({
where: {
createdAt: {
gt: startDate,
},
name: {
not: null,
},
claimId: {
not: null,
},
},
include: [db.File],
})
.then(data => {
let resultHashTable = {};
let sortableArray = [];
let sortedArray;
// summarise the data
for (let i = 0; i < data.length; i++) {
let key = `${data[i].name}#${data[i].claimId}`;
if (resultHashTable[key] === undefined) {
resultHashTable[key] = {
count : 0,
details: {
name : data[i].name,
claimId : data[i].claimId,
fileName: data[i].fileName,
fileType: data[i].fileType,
nsfw : data[i].nsfw,
},
};
} else {
resultHashTable[key]['count'] += 1;
}
}
for (let objKey in resultHashTable) {
if (resultHashTable.hasOwnProperty(objKey)) {
sortableArray.push([
resultHashTable[objKey]['count'],
resultHashTable[objKey]['details'],
]);
}
}
sortableArray.sort((a, b) => {
return b[0] - a[0];
});
sortedArray = sortableArray.map((a) => {
return a[1];
});
// return results
resolve(sortedArray);
// let resultHashTable = {};
// let sortableArray = [];
// let sortedArray;
// // summarise the data
// for (let i = 0; i < data.length; i++) {
// let key = data[i].fileId;
// if (resultHashTable[key] === undefined) {
// resultHashTable[key] = {
// count : 0,
// details: {
// name : data[i].name,
// claimId : data[i].claimId,
// fileName: data[i].fileName,
// fileType: data[i].fileType,
// nsfw : data[i].nsfw,
// },
// };
// } else {
// resultHashTable[key]['count'] += 1;
// }
// }
// for (let objKey in resultHashTable) {
// if (resultHashTable.hasOwnProperty(objKey)) {
// sortableArray.push([
// resultHashTable[objKey]['count'],
// resultHashTable[objKey]['details'],
// ]);
// }
// }
// sortableArray.sort((a, b) => {
// return b[0] - a[0];
// });
// sortedArray = sortableArray.map((a) => {
// return a[1];
// });
// // return results
resolve();
})
.catch(error => {
logger.error('sequelize error', error);

8
helpers/libraries/errorHandlers.js
View file

@ -5,16 +5,16 @@ module.exports = {
handleRequestError (action, originalUrl, ip, error, res) {
logger.error('Request Error >>', error);
if (error.response) {
postToStats(action, originalUrl, ip, null, null, null, null, null, error.response.data.error.messsage);
postToStats(action, originalUrl, ip, null, null, error.response.data.error.messsage);
res.status(error.response.status).send(error.response.data.error.message);
} else if (error.code === 'ECONNREFUSED') {
postToStats(action, originalUrl, ip, null, null, null, null, null, 'Connection refused. The daemon may not be running.');
postToStats(action, originalUrl, ip, null, null, 'Connection refused. The daemon may not be running.');
res.status(503).send('Connection refused. The daemon may not be running.');
} else if (error.message) {
postToStats(action, originalUrl, ip, null, null, null, null, null, error);
postToStats(action, originalUrl, ip, null, null, error);
res.status(400).send(error.message);
} else {
postToStats(action, originalUrl, ip, null, null, null, null, null, error);
postToStats(action, originalUrl, ip, null, null, error);
res.status(400).send(error);
}
},

6
models/file.js
View file

@ -44,5 +44,11 @@ module.exports = (sequelize, { STRING, BOOLEAN, INTEGER }) => {
freezeTableName: true,
}
);
File.associate = db => {
console.log('test');
File.hasMany(db.Request);
};
return File;
};

14
models/index.js
View file

@ -20,13 +20,19 @@ sequelize
logger.error('Sequelize was unable to connect to the database:', err);
});
fs.readdirSync(__dirname).filter(file => file.indexOf('.') !== 0 && file !== basename && file.slice(-3) === '.js').forEach(file => {
const model = sequelize['import'](path.join(__dirname, file));
db[model.name] = model;
});
fs
.readdirSync(__dirname)
.filter(file => {
return (file.indexOf('.') !== 0 && file !== basename && file.slice(-3) === '.js');
})
.forEach(file => {
const model = sequelize['import'](path.join(__dirname, file));
db[model.name] = model;
});
Object.keys(db).forEach(modelName => {
if (db[modelName].associate) {
logger.verbose('associating', modelName);
db[modelName].associate(db);
}
});

37
models/stats.js → models/request.js
View file

@ -1,6 +1,6 @@
module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => {
const Stats = sequelize.define(
'Stats',
const Request = sequelize.define(
'Request',
{
action: {
type : STRING,
@ -14,26 +14,6 @@ module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => {
type : STRING,
allowNull: true,
},
name: {
type : STRING,
allowNull: true,
},
claimId: {
type : STRING,
allowNull: true,
},
fileName: {
type : STRING,
allowNull: true,
},
fileType: {
type : STRING,
allowNull: true,
},
nsfw: {
type : BOOLEAN,
allowNull: true,
},
result: {
type : TEXT('long'),
allowNull: true,
@ -44,5 +24,16 @@ module.exports = (sequelize, { STRING, BOOLEAN, TEXT }) => {
freezeTableName: true,
}
);
return Stats;
Request.associate = db => {
console.log('test');
Request.belongsTo(db.File, {
onDelete : 'cascade',
foreignKey: {
allowNull: true,
},
});
};
return Request;
};

8
routes/api-routes.js
View file

@ -24,7 +24,7 @@ module.exports = app => {
lbryApi
.getClaimsList(params.name)
.then(claimsList => {
postToStats('serve', originalUrl, ip, null, null, null, null, 'success');
postToStats('serve', originalUrl, ip, null, null, 'success');
res.status(200).json(claimsList);
})
.catch(error => {
@ -56,7 +56,7 @@ module.exports = app => {
lbryApi
.resolveUri(params.uri)
.then(resolvedUri => {
postToStats('serve', originalUrl, ip, null, null, null, null, 'success');
postToStats('serve', originalUrl, ip, null, null, 'success');
res.status(200).json(resolvedUri);
})
.catch(error => {
@ -76,7 +76,7 @@ module.exports = app => {
try {
validateFile(file, name, license, nsfw);
} catch (error) {
postToStats('publish', originalUrl, ip, null, null, null, null, error.message);
postToStats('publish', originalUrl, ip, null, null, error.message);
logger.debug('rejected >>', error.message);
res.status(400).send(error.message);
return;
@ -91,7 +91,7 @@ module.exports = app => {
publishController
.publish(publishParams, fileName, fileType)
.then(result => {
postToStats('publish', originalUrl, ip, null, null, null, null, 'success');
postToStats('publish', originalUrl, ip, null, null, 'success');
res.status(200).json(result);
})
.catch(error => {

2
routes/home-routes.js
View file

@ -11,7 +11,7 @@ module.exports = app => {
app.use('*', ({ originalUrl, ip }, res) => {
logger.error(`404 on ${originalUrl}`);
// post to stats
postToStats('show', originalUrl, ip, null, null, null, null, null, 'Error: 404');
postToStats('show', originalUrl, ip, null, null, 'Error: 404');
// send response
res.status(404).render('fourOhFour');
});

12
routes/serve-routes.js
View file

@ -52,14 +52,14 @@ module.exports = (app) => {
if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header
const mimetypes = headers['accept'].split(',');
if (mimetypes.includes('text/html')) {
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success');
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('showLite', { fileInfo });
} else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success');
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res);
}
} else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success');
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res);
}
})
@ -82,14 +82,14 @@ module.exports = (app) => {
if (headers['accept']) { // note: added b/c some requests errored out due to no accept param in header
const mimetypes = headers['accept'].split(',');
if (mimetypes.includes('text/html')) {
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success');
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('showLite', { fileInfo });
} else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success');
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res);
}
} else {
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success');
postToStats('serve', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
serveFile(fileInfo, res);
}
})

10
routes/show-routes.js
View file

@ -27,7 +27,7 @@ module.exports = (app) => {
startDate.setDate(startDate.getDate() - 1);
getStatsSummary(startDate)
.then(result => {
postToStats('show', originalUrl, ip, null, null, null, null, null, 'success');
postToStats('show', originalUrl, ip, null, null, 'success');
res.status(200).render('statistics', result);
})
.catch(error => {
@ -39,7 +39,7 @@ module.exports = (app) => {
// get and render the content
getAllClaims('meme-fodder')
.then(orderedFreePublicClaims => {
postToStats('show', originalUrl, ip, null, null, null, null, null, 'success');
postToStats('show', originalUrl, ip, null, null, 'success');
res.status(200).render('memeFodder', { claims: orderedFreePublicClaims });
})
.catch(error => {
@ -55,7 +55,7 @@ module.exports = (app) => {
res.status(307).render('noClaims');
return;
}
postToStats('show', originalUrl, ip, null, null, null, null, null, 'success');
postToStats('show', originalUrl, ip, null, null, 'success');
res.status(200).render('allClaims', { claims: orderedFreePublicClaims });
})
.catch(error => {
@ -73,7 +73,7 @@ module.exports = (app) => {
return;
}
// serve the file or the show route
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success');
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('show', { fileInfo });
})
.catch(error => {
@ -91,7 +91,7 @@ module.exports = (app) => {
return;
}
// serve the show route
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, fileInfo.fileName, fileInfo.fileType, fileInfo.nsfw, 'success');
postToStats('show', originalUrl, ip, fileInfo.name, fileInfo.claimId, 'success');
res.status(200).render('show', { fileInfo });
})
.catch(error => {

8
routes/sockets-routes.js
View file

@ -27,7 +27,7 @@ module.exports = (app, siofu, hostedContentPath) => {
// listener for when file upload encounters an error
uploader.on('error', ({ error }) => {
logger.error('an error occured while uploading', error);
postToStats('publish', '/', null, null, null, null, null, error);
postToStats('publish', '/', null, null, error);
socket.emit('publish-status', error);
});
// listener for when file has been uploaded
@ -41,18 +41,18 @@ module.exports = (app, siofu, hostedContentPath) => {
publishController
.publish(publishParams, file.name, file.meta.type)
.then(result => {
postToStats('publish', '/', null, null, null, null, null, 'success');
postToStats('publish', '/', null, null, 'success');
socket.emit('publish-complete', { name: publishParams.name, result });
})
.catch(error => {
error = errorHandlers.handlePublishError(error);
postToStats('publish', '/', null, null, null, null, null, error);
postToStats('publish', '/', null, null, error);
socket.emit('publish-failure', error);
});
} else {
logger.error(`An error occurred in uploading the client's file`);
socket.emit('publish-failure', 'File uploaded, but with errors');
postToStats('publish', '/', null, null, null, null, null, 'File uploaded, but with errors');
postToStats('publish', '/', null, null, 'File uploaded, but with errors');
// to-do: remove the file if not done automatically
}
});

1
views/index.handlebars
View file

@ -4,7 +4,6 @@
{{> publish}}
{{> learnMore}}
</div>
{{> footer}}
</div>
<script src="/socket.io/socket.io.js"></script>