LBRYCommunity/spee.ch
1
0
Fork 0
Code Issues 95 Pull requests 2 Projects Releases 4 Packages Wiki Activity

Merge pull request #35 from lbryio/logging

Browse source 
Logging
This commit is contained in:
Bill Bittner 2017-06-19 20:24:31 -07:00 committed by GitHub
commit bc37532168
20 changed files with 159 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
config/default.json
View file

@ -8,5 +8,9 @@
"Database": {
"MySqlConnectionUri": "none",
"PublishUploadPath": "none"
},
"Logging": {
"LogLevel": "none",
"LogDirectory": "none"
}
}

4
config/development.json
View file

@ -8,5 +8,9 @@
"Database": {
"MySqlConnectionUri": "none",
"PublishUploadPath": "C:\\lbry\\speech\\hosted_content"
},
"Logging": {
"LogLevel": "debug",
"LogDirectory": "C:\\lbry\\speech\\logs\\"
}
}

4
config/production.json
View file

@ -8,5 +8,9 @@
"Database": {
"MySqlConnectionUri": "none",
"PublishUploadPath": "/home/lbry/Downloads/"
},
"Logging": {
"LogLevel": "debug",
"LogDirectory": "/home/lbry/Logs"
}
}

4
config/test.json
View file

@ -8,5 +8,9 @@
"Database": {
"MySqlConnectionUri": "none",
"PublishUploadPath": "/home/ubuntu/Downloads/"
},
"Logging": {
"LogLevel": "debug",
"LogDirectory": "/home/ubuntu/Logs"
}
}

11
controllers/publishController.js
View file

@ -1,12 +1,12 @@
const fs = require('fs');
const logger = require('winston');
const lbryApi = require('../helpers/libraries/lbryApi.js');
const config = require('config');
const walledAddress = config.get('WalletConfig.lbryAddress');
const errorHandlers = require('../helpers/libraries/errorHandlers.js');
const walledAddress = config.get('WalletConfig.lbryAddress');
function createPublishParams (claim, filePath, license, nsfw) {
console.log('nsfw:', nsfw, typeof nsfw);
logger.debug(`Creating Publish Parameters for "${claim}"`);
const publishParams = {
name : claim,
file_path: filePath,
@ -28,7 +28,7 @@ function createPublishParams (claim, filePath, license, nsfw) {
function deleteTemporaryFile (filePath) {
fs.unlink(filePath, err => {
if (err) throw err;
console.log(`successfully deleted ${filePath}`);
logger.debug(`successfully deleted ${filePath}`);
});
}
@ -44,13 +44,12 @@ module.exports = {
lbryApi
.publishClaim(publishParams, fileName, fileType)
.then(result => {
logger.info(`Successfully published ${fileName}`);
visitor.event('Publish Route', 'Publish Success', filePath).send();
console.log('publish promise success. Tx info:', result);
socket.emit('publish-complete', { name: claim, result });
})
.catch(error => {
visitor.event('Publish Route', 'Publish Failure', filePath).send();
console.log('error:', error);
socket.emit('publish-failure', errorHandlers.handlePublishError(error));
deleteTemporaryFile(filePath);
});

34
controllers/serveController.js
View file

@ -1,6 +1,6 @@
const lbryApi = require('../helpers/libraries/lbryApi.js');
const db = require('../models');
const logger = require('winston');
const getAllFreePublicClaims = require('../helpers/functions/getAllFreePublicClaims.js');
const isFreePublicClaim = require('../helpers/functions/isFreePublicClaim.js');
@ -9,9 +9,9 @@ function getClaimAndHandleResponse (claimUri, resolve, reject) {
.getClaim(claimUri)
.then(({ file_name, download_path, mime_type }) => {
resolve({
file_name,
file_path: download_path,
file_type: mime_type,
fileName: file_name,
filePath: download_path,
fileType: mime_type,
});
})
.catch(error => {
@ -22,34 +22,31 @@ function getClaimAndHandleResponse (claimUri, resolve, reject) {
module.exports = {
getClaimByName (claimName) {
const deferred = new Promise((resolve, reject) => {
console.log('>> lbryHelpers >> getClaim BasedOnNameOnly:', claimName);
// get all free public claims
getAllFreePublicClaims(claimName)
.then(freePublicClaimList => {
const claimId = freePublicClaimList[0].claim_id;
const name = freePublicClaimList[0].name;
const freePublicClaimOutpoint = `${freePublicClaimList[0].txid}:${freePublicClaimList[0].nout}`;
const freePublicClaimUri = name + '#' + claimId;
console.log('>> Decided on public claim id:', claimId);
const freePublicClaimUri = `${name}#${claimId}`;
// check to see if the file is available locally
db.File
.findOne({ where: { name: name, claim_id: claimId } })
.findOne({ where: { name: name, claimId: claimId } })
.then(claim => {
// if a matching claim is found locally...
if (claim) {
console.log('>> A matching claim_id was found locally');
logger.debug(`A mysql record was found for ${claimId}`);
// if the outpoint's match return it
if (claim.dataValues.outpoint === freePublicClaimOutpoint) {
console.log('>> Local outpoint matched');
logger.debug(`local outpoint matched for ${claimId}`);
resolve(claim.dataValues);
// if the outpoint's don't match, fetch updated claim
} else {
console.log('>> local outpoint did not match');
logger.debug(`local outpoint did not match for ${claimId}`);
getClaimAndHandleResponse(freePublicClaimUri, resolve, reject);
}
// ... otherwise use daemon to retrieve it
} else {
// 'get' the claim
getClaimAndHandleResponse(freePublicClaimUri, resolve, reject);
}
})
@ -66,35 +63,30 @@ module.exports = {
getClaimByClaimId (claimName, claimId) {
const deferred = new Promise((resolve, reject) => {
const uri = `${claimName}#${claimId}`;
console.log('>> lbryHelpers >> getClaimBasedOnUri:', uri);
// resolve the Uri
lbryApi
.resolveUri(uri) // note: use 'spread' and make parallel with db.File.findOne()
.then(result => {
// note should just be 'result' returned.
// get the outpoint
const resolvedOutpoint = `${result[uri].claim.txid}:${result[uri].claim.nout}`;
// check locally for the claim
db.File
.findOne({ where: { claim_id: claimId } })
.findOne({ where: { name: claimName, claimId: claimId } })
.then(claim => {
// if a found locally...
if (claim) {
console.log('>> A matching claim_id was found locally');
logger.debug(`A mysql record was found for ${claimId}`);
// if the outpoint's match return it
if (claim.dataValues.outpoint === resolvedOutpoint) {
console.log('>> Local outpoint matched');
logger.debug(`local outpoint matched for ${claimId}`);
resolve(claim.dataValues);
// if the outpoint's don't match, fetch updated claim
} else {
console.log('>> Local outpoint did not match');
logger.debug(`local outpoint did not match for ${claimId}`);
getClaimAndHandleResponse(uri, resolve, reject);
}
// ... otherwise use daemon to retrieve it
} else {
// check to make sure it is free and public (note: no need for another resolve?)
if (isFreePublicClaim(result[uri].claim)) {
// 'get' the claim
getClaimAndHandleResponse(uri, resolve, reject);
} else {
reject('NO_FREE_PUBLIC_CLAIMS');

21
helpers/functions/getAllFreePublicClaims.js
View file

@ -1,8 +1,9 @@
const isFreePublicClaim = require('./isFreePublicClaim.js');
const lbryApi = require('../libraries/lbryApi.js');
const logger = require('winston');
function filterForFreePublicClaims (claimsListArray) {
// console.log("claims list:", claimsListArray)
logger.debug('filtering claims list for free, public claims');
if (!claimsListArray) {
return null;
}
@ -16,12 +17,12 @@ function filterForFreePublicClaims (claimsListArray) {
}
function orderTopClaims (claimsListArray) {
console.log('>> orderTopClaims');
claimsListArray.sort(({ amountA, heightA }, { amountB, heightB }) => {
if (amountA === amountB) {
return heightA > heightB;
logger.debug('ordering the top claims');
claimsListArray.sort((a, b) => {
if (a.amount === b.amount) {
return a.height < b.height;
} else {
return amountA < amountB;
return a.amount < b.amount;
}
});
return claimsListArray;
@ -34,11 +35,11 @@ module.exports = claimName => {
.getClaimsList(claimName)
.then(({ claims }) => {
const claimsList = claims;
console.log('>> Number of claims:', claimsList.length);
logger.debug(`Number of claims: ${claimsList.length}`);
// return early if no claims were found
if (claimsList.length === 0) {
reject('NO_CLAIMS');
console.log('exiting due to lack of claims');
logger.debug('exiting due to lack of claims');
return;
}
// filter the claims to return only free, public claims
@ -46,7 +47,7 @@ module.exports = claimName => {
// return early if no free, public claims were found
if (!freePublicClaims || freePublicClaims.length === 0) {
reject('NO_FREE_PUBLIC_CLAIMS');
console.log('exiting due to lack of free or public claims');
logger.debug('exiting due to lack of free or public claims');
return;
}
// order the claims
@ -55,7 +56,7 @@ module.exports = claimName => {
resolve(orderedPublicClaims);
})
.catch(error => {
console.log(">> 'claim_list' error");
logger.error('error received from lbryApi.getClaimsList', error);
reject(error);
});
});

4
helpers/functions/isFreePublicClaim.js
View file

@ -1,5 +1,7 @@
const logger = require('winston');
module.exports = ({ value }) => {
console.log('>> check: isFreePublicClaim?');
logger.debug('checking isFreePublicClaim ?');
if (
(value.stream.metadata.license.indexOf('Public Domain') !== -1 || value.stream.metadata.license.indexOf('Creative Commons') !== -1) &&
(!value.stream.metadata.fee || value.stream.metadata.fee.amount === 0)

4
helpers/libraries/errorHandlers.js
View file

@ -1,5 +1,8 @@
const logger = require('winston');
module.exports = {
handleRequestError (error, res) {
logger.error('Request Error,', error);
if (error === 'NO_CLAIMS' || error === 'NO_FREE_PUBLIC_CLAIMS') {
res.status(307).render('noClaims');
} else if (error.response) {
@ -11,6 +14,7 @@ module.exports = {
}
},
handlePublishError (error) {
logger.error('Publish Error,', error);
if (error.code === 'ECONNREFUSED') {
return 'Connection refused. The daemon may not be running.';
} else if (error.response.data.error) {

60
helpers/libraries/lbryApi.js
View file

@ -1,51 +1,49 @@
const axios = require('axios');
const db = require('../../models');
const logger = require('winston');
function createFilesRecord (name, claimId, outpoint, fileName, filePath, fileType, nsfw) {
db.File
.create({ name, claimId, outpoint, fileName, filePath, fileType, nsfw })
.catch(error => {
logger.error(`Sequelize File.create error`, error);
});
}
module.exports = {
publishClaim (publishParams, fileName, fileType) {
logger.debug(`Publishing claim for "${fileName}"`);
const deferred = new Promise((resolve, reject) => {
console.log('>> lbryApi >> publishClaim:', publishParams);
axios
.post('http://localhost:5279/lbryapi', {
method: 'publish',
params: publishParams,
})
.then(response => {
console.log(">> 'publish' success", response);
const result = response.data.result;
db.File
.create({
name : publishParams.name,
claim_id : result.claim_id,
outpoint : `${result.txid}:${result.nout}`,
file_name: fileName,
file_path: publishParams.file_path,
file_type: fileType,
nsfw : publishParams.metadata.nsfw,
})
.catch(error => {
console.log('An error occurred when writing to the MySQL database:', error);
});
createFilesRecord(
publishParams.name, result.claim_id, `${result.txid}:${result.nout}`, fileName, publishParams.file_path, fileType, publishParams.metadata.nsfw);
resolve(result);
})
.catch(error => {
console.log(">> 'publish' error");
reject(error);
});
});
return deferred;
},
getClaim (uri) {
logger.debug(`Getting Claim for "${uri}"`);
const deferred = new Promise((resolve, reject) => {
console.log('>> lbryApi >> getClaim:', uri);
axios
.post('http://localhost:5279/lbryapi', {
method: 'get',
params: { uri, timeout: 20 },
params: { uri, timeout: 30 },
})
.then(({ data }) => {
console.log(">> 'get' success");
// check to make sure the daemon didn't just time out
if (!data.result) {
reject(JSON.stringify(data));
}
if (data.result.error) {
reject(data.result.error);
}
@ -54,61 +52,45 @@ module.exports = {
*/
// save a record of the file to the Files table
const result = data.result;
db.File
.create({
name : result.name,
claim_id : result.claim_id,
outpoint : result.outpoint,
file_name: result.file_name,
file_path: result.download_path,
file_type: result.mime_type,
nsfw : result.metadata.stream.metadata.nsfw,
})
.catch(error => {
console.log('An error occurred when writing to the MySQL database:', error);
});
createFilesRecord(
result.name, result.claim_id, result.outpoint, result.file_name, result.download_path, result.mime_type, result.metadata.stream.metadata.nsfw);
resolve(result);
})
.catch(error => {
console.log(">> 'get' error");
reject(error);
});
});
return deferred;
},
getClaimsList (claimName) {
logger.debug(`Getting Claim List for "${claimName}"`);
const deferred = new Promise((resolve, reject) => {
console.log('>> lbryApi >> getClaimList:', claimName);
axios
.post('http://localhost:5279/lbryapi', {
method: 'claim_list',
params: { name: claimName },
})
.then(({ data }) => {
console.log(">> 'claim_list' success");
resolve(data.result);
})
.catch(error => {
console.log(">> 'claim_list' error");
reject(error);
});
});
return deferred;
},
resolveUri (uri) {
logger.debug(`Resolving URI for "${uri}"`);
const deferred = new Promise((resolve, reject) => {
console.log('>> lbryApi >> resolveUri:', uri);
axios
.post('http://localhost:5279/lbryapi', {
method: 'resolve',
params: { uri },
})
.then(({ data }) => {
console.log(">> 'resolve' success");
resolve(data.result);
})
.catch(error => {
console.log(">> 'resolve' error");
reject(error);
});
});

35
helpers/logging/loggerSetup.js Normal file
View file

@ -0,0 +1,35 @@
const fs = require('fs');
module.exports = (winston, logLevel, logDir) => {
if (!fs.existsSync(logDir)) {
fs.mkdirSync(logDir);
}
winston.configure({
transports: [
new (winston.transports.Console)({
level : logLevel,
timestamp : false,
colorize : true,
prettyPrint: true,
}),
new (winston.transports.File)({
filename : `${logDir}speechLogs.log`,
level : logLevel,
json : false,
timestamp : true,
colorize : true,
prettyPrint : true,
handleExceptions : true,
humanReadableUnhandledException: true,
}),
],
});
winston.error('Level 0');
winston.warn('Level 1');
winston.info('Level 2');
winston.verbose('Level 3');
winston.debug('Level 4');
winston.silly('Level 5');
};

8
models/file.js
View file

@ -6,7 +6,7 @@ module.exports = (sequelize, { STRING, BOOLEAN }) => {
type : STRING,
allowNull: false,
},
claim_id: {
claimId: {
type : STRING,
allowNull: false,
},
@ -14,15 +14,15 @@ module.exports = (sequelize, { STRING, BOOLEAN }) => {
type : STRING,
allowNull: false,
},
file_name: {
fileName: {
type : STRING,
allowNull: false,
},
file_path: {
filePath: {
type : STRING,
allowNull: false,
},
file_type: {
fileType: {
type: STRING,
},
nsfw: {

9
models/index.js
View file

@ -4,17 +4,20 @@ const Sequelize = require('sequelize');
const basename = path.basename(module.filename);
const config = require('config');
const db = {};
const logger = require('winston');
const connectionUri = config.get('Database.MySqlConnectionUri');
const sequelize = new Sequelize(connectionUri);
const sequelize = new Sequelize(connectionUri, {
logging: false,
});
sequelize
.authenticate()
.then(() => {
console.log('Connection has been established successfully.');
logger.info('Sequelize has has been established mysql connection successfully.');
})
.catch(err => {
console.error('Unable to connect to the database:', err);
logger.error('Sequelize was nable to connect to the database:', err);
});
fs.readdirSync(__dirname).filter(file => file.indexOf('.') !== 0 && file !== basename && file.slice(-3) === '.js').forEach(file => {

3
package.json
View file

@ -36,7 +36,8 @@
"sequelize": "^4.1.0",
"socket.io": "^2.0.1",
"socketio-file-upload": "^0.6.0",
"universal-analytics": "^0.4.13"
"universal-analytics": "^0.4.13",
"winston": "^2.3.1"
},
"devDependencies": {
"eslint": "3.19.0",

10
routes/api-routes.js
View file

@ -1,26 +1,26 @@
const errorHandlers = require('../helpers/libraries/errorHandlers.js');
const lbryApi = require('../helpers/libraries/lbryApi.js');
const logger = require('winston');
module.exports = app => {
// route to run a claim_list request on the daemon
app.get('/api/claim_list/:claim', ({ params }, res) => {
app.get('/api/claim_list/:claim', ({ originalUrl, params }, res) => {
logger.debug(`GET request on ${originalUrl}`);
lbryApi
.getClaimsList(params.claim)
.then(claimsList => {
console.log('/api/claim_list/:claim success.');
res.status(200).json(claimsList);
})
.catch(error => {
console.log('/api/claim_list/:name error:', error);
errorHandlers.handleRequestError(error, res);
});
});
// route to run a resolve request on the daemon
app.get('/api/resolve/:uri', ({ params }, res) => {
app.get('/api/resolve/:uri', ({ originalUrl, params }, res) => {
logger.debug(`GET request on ${originalUrl}`);
lbryApi
.resolveUri(params.uri)
.then(resolvedUri => {
console.log('/api/resolve/:claim success.');
res.status(200).json(resolvedUri);
})
.catch(error => {

3
routes/home-routes.js
View file

@ -1,3 +1,5 @@
const logger = require('winston');
module.exports = app => {
// route for the home page
app.get('/', (req, res) => {
@ -5,6 +7,7 @@ module.exports = app => {
});
// a catch-all route if someone visits a page that does not exist
app.use('*', (req, res) => {
logger.error(`Get request on ${req.originalUrl} which was 404`);
res.status(404).render('fourOhFour');
});
};

24
routes/serve-routes.js
View file

@ -1,17 +1,19 @@
const errorHandlers = require('../helpers/libraries/errorHandlers.js');
const serveController = require('../controllers/serveController.js');
const logger = require('winston');
function serveFile ({ file_type, file_path }, res) {
function serveFile ({ fileName, fileType, filePath }, res) {
logger.info(`serving file ${fileName}`);
// set default options
const options = {
headers: {
'X-Content-Type-Options': 'nosniff',
'Content-Type' : file_type,
'Content-Type' : fileType,
},
};
// adjust default options as needed
// eslint-disable-next-line camelcase
switch (file_type) {
switch (fileType) {
case 'image/jpeg':
break;
case 'image/gif':
@ -21,47 +23,43 @@ function serveFile ({ file_type, file_path }, res) {
case 'video/mp4':
break;
default:
console.log('sending unknown file type as .jpeg');
logger.warn('sending unknown file type as .jpeg');
options['headers']['Content-Type'] = 'image/jpeg';
break;
}
// send file
res.status(200).sendFile(file_path, options);
res.status(200).sendFile(filePath, options);
}
module.exports = (app, ua, googleAnalyticsId) => {
// route to fetch one free public claim
app.get('/:name/:claim_id', ({ params }, res) => {
app.get('/:name/:claim_id', ({ originalUrl, params }, res) => {
logger.debug(`GET request on ${originalUrl}`);
const routeString = `${params.name}/${params.claim_id}`;
// google analytics
ua(googleAnalyticsId, { https: true }).event('Serve Route', '/name/claimId', routeString).send();
// begin image-serve processes
console.log(`>> GET request on /${routeString}`);
serveController
.getClaimByClaimId(params.name, params.claim_id)
.then(fileInfo => {
console.log('/:name/:claim_id success.', fileInfo.file_name);
serveFile(fileInfo, res);
})
.catch(error => {
console.log('/:name/:claim_id error:', error);
errorHandlers.handleRequestError(error, res);
});
});
// route to fetch one free public claim
app.get('/:name', ({ params }, res) => {
app.get('/:name', ({ originalUrl, params }, res) => {
logger.debug(`GET request on ${originalUrl}`);
// google analytics
ua(googleAnalyticsId, { https: true }).event('Serve Route', '/name', params.name).send();
// begin image-serve processes
console.log(`>> GET request on /${params.name}`);
serveController
.getClaimByName(params.name)
.then(fileInfo => {
console.log('/:name success.', fileInfo.file_name);
serveFile(fileInfo, res);
})
.catch(error => {
console.log('/:name error:', error);
errorHandlers.handleRequestError(error, res);
});
});

7
routes/show-routes.js
View file

@ -1,21 +1,20 @@
const errorHandlers = require('../helpers/libraries/errorHandlers.js');
const showController = require('../controllers/showController.js');
const logger = require('winston');
module.exports = (app, ua, googleAnalyticsId) => {
// route to fetch all free public claims
app.get('/:name/all', ({ params }, res) => {
console.log(`>> GET request on /${params.name}/all`);
app.get('/:name/all', ({ originalUrl, params }, res) => {
logger.debug(`GET request on ${originalUrl}`);
// google analytics
ua(googleAnalyticsId, { https: true }).event('Show Routes', '/name/all', `${params.name}/all`).send();
// fetch all free public claims
showController
.getAllClaims(params.name)
.then(orderedFreePublicClaims => {
console.log('/:name/all success.');
res.status(200).render('allClaims', { claims: orderedFreePublicClaims });
})
.catch(error => {
console.log('/:name/all error:', error);
errorHandlers.handleRequestError(error, res);
});
});

13
routes/sockets-routes.js
View file

@ -1,11 +1,12 @@
const publishController = require('../controllers/publishController.js');
const logger = require('winston');
module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => {
const http = require('http').Server(app);
const io = require('socket.io')(http);
io.on('connection', socket => {
console.log('a user connected via sockets');
logger.silly('a user connected via sockets');
// create visitor record
const visitor = ua(googleAnalyticsId, { https: true });
// attach upload listeners
@ -13,28 +14,30 @@ module.exports = (app, siofu, hostedContentPath, ua, googleAnalyticsId) => {
uploader.dir = hostedContentPath;
uploader.listen(socket);
uploader.on('start', ({ file }) => {
logger.info('client started an upload:', file.name);
// server side test to make sure file is not a bad file type
if (/\.exe$/.test(file.name)) {
uploader.abort(file.id, socket);
}
});
uploader.on('error', ({ error }) => {
console.log('an error occured while uploading', error);
logger.error('an error occured while uploading', error);
socket.emit('publish-status', error);
});
uploader.on('saved', ({ file }) => {
console.log('uploaded ', file.name);
if (file.success) {
logger.debug(`Client successfully uploaded ${file.name}`);
socket.emit('publish-status', 'file upload successfully completed');
publishController.publish(file.meta.name, file.name, file.pathName, file.meta.type, file.meta.license, file.meta.nsfw, socket, visitor);
} else {
logger.error(`An error occurred in uploading the client's file`);
socket.emit('publish-failure', 'file uploaded, but with errors');
// to-do: remove the file
// to-do: remove the file if not done automatically
}
});
// handle disconnect
socket.on('disconnect', () => {
console.log('user disconnected');
logger.silly('a user disconnected via sockets');
});
});

8
server.js
View file

@ -6,10 +6,16 @@ const expressHandlebars = require('express-handlebars');
const Handlebars = require('handlebars');
const config = require('config');
const ua = require('universal-analytics');
const winston = require('winston');
const googleAnalyticsId = config.get('AnalyticsConfig.googleId');
const hostedContentPath = config.get('Database.PublishUploadPath');
// configure logging
const logLevel = config.get('Logging.LogLevel');
const logDir = config.get('Logging.LogDirectory');
require('./helpers/logging/loggerSetup.js')(winston, logLevel, logDir);
// set port
const PORT = 3000;
// initialize express app
@ -63,6 +69,6 @@ const http = require('./routes/sockets-routes.js')(app, siofu, hostedContentPath
// start server
db.sequelize.sync({}).then(() => {
http.listen(PORT, () => {
console.log(`Listening on PORT ${PORT}`);
winston.info(`Server is listening on PORT ${PORT}`);
});
});