From 8d4ae9ce3c1cf245b4ee6b7f98e82a04fcc56518 Mon Sep 17 00:00:00 2001 From: bill bittner Date: Mon, 17 Jul 2017 13:16:11 -0700 Subject: [PATCH 1/6] chained promises and implemented short claimId check --- config/development.json | 2 +- controllers/serveController.js | 80 +++++++++++++++------ controllers/statsController.js | 2 +- helpers/functions/getAllFreePublicClaims.js | 1 - helpers/libraries/serveHelpers.js | 68 ++++++++++++++++++ routes/serve-routes.js | 43 ++--------- 6 files changed, 136 insertions(+), 60 deletions(-) create mode 100644 helpers/libraries/serveHelpers.js diff --git a/config/development.json b/config/development.json index 34c4c849..e91061d3 100644 --- a/config/development.json +++ b/config/development.json @@ -7,7 +7,7 @@ }, "Database": { "MySqlConnectionUri": "none", - "DownloadDirectory": "/home/lbry/Downloads/" + "DownloadDirectory": "C:\\Users\\Bones\\Downloads\\lbry\\" }, "Logging": { "LogLevel": "silly" diff --git a/controllers/serveController.js b/controllers/serveController.js index 845dfb57..0766233c 100644 --- a/controllers/serveController.js +++ b/controllers/serveController.js @@ -3,9 +3,10 @@ const db = require('../models'); const logger = require('winston'); const getAllFreePublicClaims = require('../helpers/functions/getAllFreePublicClaims.js'); const isFreePublicClaim = require('../helpers/functions/isFreePublicClaim.js'); +const { validateClaimId } = require('../helpers/libraries/serveHelpers.js'); function updateFileIfNeeded (uri, claimName, claimId, localOutpoint, localHeight) { - logger.debug(`A mysql record was found for ${claimName}:${claimId}. Initiating resolve to check outpoint.`); + logger.debug(`Initiating resolve to check outpoint for ${claimName}:${claimId}.`); // 1. resolve claim lbryApi .resolveUri(uri) @@ -34,7 +35,7 @@ function updateFileIfNeeded (uri, claimName, claimId, localOutpoint, localHeight } }) .catch(error => { - logger.error(`error resolving "${uri}" >> `, error); + logger.error(error); }); } @@ -110,6 +111,41 @@ function getClaimAndHandleResponse (uri, address, height, resolve, reject) { }); } +function getClaimAndReturnResponse (uri, address, height) { + const deferred = new Promise((resolve, reject) => { + lbryApi + .getClaim(uri) + .then(({ name, claim_id, outpoint, file_name, download_path, mime_type, metadata }) => { + // create entry in the db + logger.silly(`creating new File record`); + db.File + .create({ + name, + claimId : claim_id, + address, // note: passed as an arguent, not from this 'get' call + outpoint, + height, // note: passed as an arguent, not from this 'get' call + fileName: file_name, + filePath: download_path, + fileType: mime_type, + nsfw : metadata.stream.metadata.nsfw, + }) + .then(result => { + logger.debug('successfully created File record'); + resolve(result); // note: result.dataValues ? + }) + .catch(error => { + logger.error('sequelize create error', error); + reject(error); + }); + }) + .catch(error => { + reject(error); + }); + }); + return deferred; +} + module.exports = { getClaimByName (claimName) { const deferred = new Promise((resolve, reject) => { @@ -155,20 +191,23 @@ module.exports = { }, getClaimByClaimId (name, claimId) { const deferred = new Promise((resolve, reject) => { - const uri = `${name}#${claimId}`; - // 1. check locally for the claim - db.File - .findOne({ where: { name, claimId } }) - .then(claim => { - // 2. if a match is found locally, serve it - if (claim) { - // serve the file - resolve(claim.dataValues); - // trigger an update if needed - updateFileIfNeeded(uri, name, claimId, claim.dataValues.outpoint, claim.dataValues.outpoint); - // 2. otherwise use daemon to retrieve it - } else { - // 3. resolve the Uri + let uri; + validateClaimId(name, claimId) // 1. validate the claim id & retrieve the full claim id if needed + .then(validClaimId => { // 2. check locally for the claim + logger.debug('valid claim id:', validClaimId); + uri = `${name}#${validClaimId}`; + return db.File.findOne({ where: { name, claimId: validClaimId } }); + }) + .then(({ dataValues }) => { + if (dataValues) { // 3. if a match is found locally, serve that claim + logger.debug('Result found in File table:', dataValues); + // return the data for the file to be served + resolve(dataValues); // break out of the chain??? + // update the file, as needed + updateFileIfNeeded(uri, name, claimId, dataValues.outpoint, dataValues.outpoint); + // 3. if a match was not found use the daemon to retrieve the claim & return the db data once it is created + } else { // 4. resolve the Uri + logger.debug('No result found in File table,'); lbryApi .resolveUri(uri) .then(result => { @@ -176,14 +215,13 @@ module.exports = { if (!result.claim) { logger.debug('resolve did not return a claim'); resolve(null); - return; } - // 4. check to see if the claim is free & public + // check to see if the claim is free & public if (isFreePublicClaim(result.claim)) { - // 5. get claim and serve - getClaimAndHandleResponse(uri, result.claim.address, result.claim.height, resolve, reject); + // get claim and serve + resolve(getClaimAndReturnResponse(uri, result.claim.address, result.claim.height)); } else { - reject(null); + resolve(null); } }) .catch(error => { diff --git a/controllers/statsController.js b/controllers/statsController.js index 961d7b5a..e13f0d03 100644 --- a/controllers/statsController.js +++ b/controllers/statsController.js @@ -36,7 +36,7 @@ module.exports = { }); }) .catch(error => { - logger.error('sequelize error', error); + logger.error('Sequelize error', error); }); }, sendGoogleAnalytics (action, headers, ip, originalUrl) { diff --git a/helpers/functions/getAllFreePublicClaims.js b/helpers/functions/getAllFreePublicClaims.js index 4a902691..e5f6f8c7 100644 --- a/helpers/functions/getAllFreePublicClaims.js +++ b/helpers/functions/getAllFreePublicClaims.js @@ -55,7 +55,6 @@ module.exports = (claimName) => { resolve(orderedPublicClaims); }) .catch(error => { - logger.error('error received from lbryApi.getClaimsList', error); reject(error); }); }); diff --git a/helpers/libraries/serveHelpers.js b/helpers/libraries/serveHelpers.js new file mode 100644 index 00000000..dfaa663d --- /dev/null +++ b/helpers/libraries/serveHelpers.js @@ -0,0 +1,68 @@ +const logger = require('winston'); +const db = require('../../models'); + +module.exports = { + serveFile ({ fileName, fileType, filePath }, res) { + logger.info(`serving file ${fileName}`); + // set default options + let options = { + headers: { + 'X-Content-Type-Options': 'nosniff', + 'Content-Type' : fileType, + }, + }; + // adjust default options as needed + switch (fileType) { + case 'image/jpeg': + break; + case 'image/gif': + break; + case 'image/png': + break; + case 'video/mp4': + break; + default: + logger.warn('sending file with unknown type as .jpeg'); + options['headers']['Content-Type'] = 'image/jpeg'; + break; + } + // send the file + res.status(200).sendFile(filePath, options); + }, + validateClaimId (name, claimId) { + const deferred = new Promise((resolve, reject) => { + logger.debug('claim id length:', claimId.length); + // make sure the claim id is 40 characters + if (claimId.length === 40) { + logger.debug('Claim Id length is valid.'); + resolve(claimId); + // if the claim id is shorter than 40, check the db for the full claim id + } else if (claimId.length === 1) { + logger.debug(`Finding claim id for "${name}" "${claimId}"`); + db.File + .findOne({ + where: { + name, + claimId: { $like: `${claimId}%` }, + }, + }) + .then(file => { + // if no results were found, throw an error + if (!file) { + reject(new Error('That is not a valid short URL.')); + } + // if a result was found, resolve with the full claim id + logger.debug('Full claim id:', file.dataValues.claimId); + resolve(file.dataValues.claimId); + }) + .catch(error => { + reject(error); + }); + } else { + logger.error('The Claim Id was neither 40 nor 1 character in length'); + reject(new Error('That Claim Id is not valid.')); + } + }); + return deferred; + }, +}; diff --git a/routes/serve-routes.js b/routes/serve-routes.js index 4a6bf307..552c399f 100644 --- a/routes/serve-routes.js +++ b/routes/serve-routes.js @@ -2,47 +2,17 @@ const logger = require('winston'); const { getClaimByClaimId, getClaimByName } = require('../controllers/serveController.js'); const { postToStats, sendGoogleAnalytics } = require('../controllers/statsController.js'); const errorHandlers = require('../helpers/libraries/errorHandlers.js'); - -function serveFile ({ fileName, fileType, filePath }, res) { - logger.info(`serving file ${fileName}`); - // set default options - let options = { - headers: { - 'X-Content-Type-Options': 'nosniff', - 'Content-Type' : fileType, - }, - }; - // adjust default options as needed - switch (fileType) { - case 'image/jpeg': - break; - case 'image/gif': - break; - case 'image/png': - break; - case 'video/mp4': - break; - default: - logger.warn('sending file with unknown type as .jpeg'); - options['headers']['Content-Type'] = 'image/jpeg'; - break; - } - // send file - res.status(200).sendFile(filePath, options); -} - -function sendAnalyticsAndLog (headers, ip, originalUrl) { - // google analytics - sendGoogleAnalytics('serve', headers, ip, originalUrl); -} +const { serveFile } = require('../helpers/libraries/serveHelpers.js'); module.exports = (app) => { // route to serve a specific asset app.get('/:name/:claim_id', ({ headers, ip, originalUrl, params }, res) => { - sendAnalyticsAndLog(headers, ip, originalUrl); + // google analytics + sendGoogleAnalytics('serve', headers, ip, originalUrl); // begin image-serve processes getClaimByClaimId(params.name, params.claim_id) .then(fileInfo => { + logger.debug('file info:', fileInfo); // check to make sure a file was found if (!fileInfo) { res.status(307).render('noClaims'); @@ -67,9 +37,10 @@ module.exports = (app) => { errorHandlers.handleRequestError('serve', originalUrl, ip, error, res); }); }); - // route to serve the winning claim + // route to serve the winning asset at a claim app.get('/:name', ({ headers, ip, originalUrl, params }, res) => { - sendAnalyticsAndLog(headers, ip, originalUrl); + // google analytics + sendGoogleAnalytics('serve', headers, ip, originalUrl); // begin image-serve processes getClaimByName(params.name) .then(fileInfo => { From bf516c07c2a8b99fd8c48d4abb108ae04309bfec Mon Sep 17 00:00:00 2001 From: bill bittner Date: Mon, 17 Jul 2017 13:20:57 -0700 Subject: [PATCH 2/6] added 'details' link on show-lite route --- views/showLite.handlebars | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/views/showLite.handlebars b/views/showLite.handlebars index c3477088..cbbe5287 100644 --- a/views/showLite.handlebars +++ b/views/showLite.handlebars @@ -5,7 +5,11 @@ {{!--fallback--}} Your browser does not support the video element. + {{else}} {{fileInfo.fileName}} {{/ifConditional}} +
+ Details +
\ No newline at end of file From af8242a3c6770f5f21acac82a763144e0cde4111 Mon Sep 17 00:00:00 2001 From: bill bittner Date: Mon, 17 Jul 2017 13:34:21 -0700 Subject: [PATCH 3/6] tested with daemon and fixed handle of invalid uri --- controllers/serveController.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/controllers/serveController.js b/controllers/serveController.js index 0766233c..7c38d9ad 100644 --- a/controllers/serveController.js +++ b/controllers/serveController.js @@ -198,26 +198,25 @@ module.exports = { uri = `${name}#${validClaimId}`; return db.File.findOne({ where: { name, claimId: validClaimId } }); }) - .then(({ dataValues }) => { - if (dataValues) { // 3. if a match is found locally, serve that claim - logger.debug('Result found in File table:', dataValues); + .then(result => { + // 3. if a match is found locally, serve that claim + if (result) { + logger.debug('Result found in File table:', result.dataValues); // return the data for the file to be served - resolve(dataValues); // break out of the chain??? + resolve(result.dataValues); // update the file, as needed - updateFileIfNeeded(uri, name, claimId, dataValues.outpoint, dataValues.outpoint); + updateFileIfNeeded(uri, name, claimId, result.dataValues.outpoint, result.dataValues.outpoint); // 3. if a match was not found use the daemon to retrieve the claim & return the db data once it is created - } else { // 4. resolve the Uri + } else { logger.debug('No result found in File table,'); lbryApi .resolveUri(uri) .then(result => { - // check to make sure the result is a claim - if (!result.claim) { + if (!result.claim) { // check to make sure the result is a claim logger.debug('resolve did not return a claim'); resolve(null); } - // check to see if the claim is free & public - if (isFreePublicClaim(result.claim)) { + if (isFreePublicClaim(result.claim)) { // check to see if the claim is free & public // get claim and serve resolve(getClaimAndReturnResponse(uri, result.claim.address, result.claim.height)); } else { From 13471edda4b78bfa3942fc837a4e7720edb79a71 Mon Sep 17 00:00:00 2001 From: bill bittner Date: Mon, 17 Jul 2017 13:54:50 -0700 Subject: [PATCH 4/6] updated show route with short url --- controllers/serveController.js | 2 +- server.js | 15 +++++--- views/partials/assetInfo.handlebars | 60 ++++++++++++++--------------- views/show.handlebars | 5 --- 4 files changed, 40 insertions(+), 42 deletions(-) diff --git a/controllers/serveController.js b/controllers/serveController.js index 7c38d9ad..a390a32c 100644 --- a/controllers/serveController.js +++ b/controllers/serveController.js @@ -208,7 +208,7 @@ module.exports = { updateFileIfNeeded(uri, name, claimId, result.dataValues.outpoint, result.dataValues.outpoint); // 3. if a match was not found use the daemon to retrieve the claim & return the db data once it is created } else { - logger.debug('No result found in File table,'); + logger.debug('No result found in File table'); lbryApi .resolveUri(uri) .then(result => { diff --git a/server.js b/server.js index 48ed5079..0a8e2adf 100644 --- a/server.js +++ b/server.js @@ -5,13 +5,13 @@ const siofu = require('socketio-file-upload'); const expressHandlebars = require('express-handlebars'); const Handlebars = require('handlebars'); const config = require('config'); -const winston = require('winston'); +const logger = require('winston'); const hostedContentPath = config.get('Database.DownloadDirectory'); // configure logging const logLevel = config.get('Logging.LogLevel'); -require('./config/loggerSetup.js')(winston, logLevel); +require('./config/loggerSetup.js')(logger, logLevel); // set port const PORT = 3000; @@ -29,7 +29,7 @@ app.use(bodyParser.json()); // for parsing application/json app.use(bodyParser.urlencoded({ extended: true })); // for parsing application/x-www-form-urlencoded app.use(siofu.router); app.use((req, res, next) => { // logging middleware - winston.verbose(`Request on ${req.originalUrl} from ${req.ip}`); + logger.verbose(`Request on ${req.originalUrl} from ${req.ip}`); next(); }); @@ -74,6 +74,9 @@ const hbs = expressHandlebars.create({ return options.inverse(this); } }, + firstCharacter (word) { + return word.substring(0, 1); + }, }, }); app.engine('handlebars', hbs.engine); @@ -94,10 +97,10 @@ const server = require('./routes/sockets-routes.js')(app, siofu, hostedContentPa db.sequelize.sync() .then(() => { server.listen(PORT, () => { - winston.info('Trusting proxy?', app.get('trust proxy')); - winston.info(`Server is listening on PORT ${PORT}`); + logger.info('Trusting proxy?', app.get('trust proxy')); + logger.info(`Server is listening on PORT ${PORT}`); }); }) .catch((error) => { - winston.log('Error syncing sequelize db:', error); + logger.log('Error syncing sequelize db:', error); }); diff --git a/views/partials/assetInfo.handlebars b/views/partials/assetInfo.handlebars index 7ad1bace..b33a9026 100644 --- a/views/partials/assetInfo.handlebars +++ b/views/partials/assetInfo.handlebars @@ -4,45 +4,45 @@

Links

- {{!--direct link to asset--}} - Direct Link - -
- - -
-
+ {{!--short direct link to asset--}} + + {{!-- link to show route for asset--}} + + {{!-- html text for embedding asset--}} + {{!--markdown text using asset--}} {{#ifConditional fileInfo.fileType '===' 'video/mp4'}} {{else}} + {{/ifConditional}} - {{!-- html text for embedding asset--}} - Embed HTML - -
- {{#ifConditional fileInfo.fileType '===' 'video/mp4'}} - - {{else}} - - {{/ifConditional}} - -
-
- {{!-- link to show route for asset--}} - Details Link - -
- - -
-

Metadata

diff --git a/views/show.handlebars b/views/show.handlebars index 2e432715..1a841511 100644 --- a/views/show.handlebars +++ b/views/show.handlebars @@ -10,11 +10,6 @@