Merge pull request #931 from lbryio/master

cuts staging from master

README.md

@@ -118,7 +118,7 @@ Instructions are coming at [lbry-docker] to install your own chainquery instance
 ## Settings
 
 There are a number of settings available for customizing the behavior of your installation.  
-_INSERT LINK TO SETTINGS.MD_
+[Here](https://github.com/lbryio/spee.ch/blob/master/docs/settings.md) is some documentation on them.
 
 ## API
 

cli/defaults/siteConfig.json

@@ -46,10 +46,7 @@
       "customByContentType": {
         "application/octet-stream": 50000000
       }
-    },
+    }
-    "maxSizeImage": 10000000,
-    "maxSizeGif": 50000000,
-    "maxSizeVideo": 50000000
   },
   "serving": {
     "markdownSettings": {
@@ -92,18 +89,18 @@
       "disallowedTypesExample": ["image", "html"]
     },
     "customFileExtensions": {
-      "application/x-troff-man": ".man",
+      "application/x-troff-man": "man",
-      "application/x-troff-me": ".me",
+      "application/x-troff-me": "me",
-      "application/x-mif": ".mif",
+      "application/x-mif": "mif",
-      "application/x-troff-ms": ".ms",
+      "application/x-troff-ms": "ms",
-      "application/x-troff": ".roff",
+      "application/x-troff": "roff",
-      "application/x-python-code": ".pyc",
+      "application/x-python-code": "pyc",
-      "text/x-python": ".py",
+      "text/x-python": "py",
-      "application/x-pn-realaudio": ".ram",
+      "application/x-pn-realaudio": "ram",
-      "application/x-sgml": ".sgm",
+      "application/x-sgml": "sgm",
-      "model/stl": ".stl",
+      "model/stl": "stl",
-      "image/pict": ".pct",
+      "image/pict": "pct",
-      "text/xul": ".xul",
+      "text/xul": "xul",
       "text/x-go": "go"
     }
   },

client/scss/_asset-preview.scss

@@ -8,7 +8,7 @@
   height: 280px;
   &:hover {
     border: 1px solid $highlight-border-color;
-    color: #000000;
+    color: $primary-color;
   }
 }
 

client/scss/_input.scss

@@ -8,6 +8,7 @@ input {
   border: 0;
   background-color: $background-color;
   display: inline-block;
+  color: $text-color
 }
 
 .input-slider {

client/scss/_link.scss

@@ -9,7 +9,6 @@ a, a:visited {
 
 .link--nav {
   color: $text-color;
-  border-bottom: 2px solid white;
   &:hover {
     color: $primary-color;
   }

client/scss/_nav-bar.scss

@@ -1,15 +1,24 @@
 .nav-bar {
   box-sizing: border-box;
   padding: $thin-padding $primary-padding;
-  background: $base-color;
+  background: $chrome-color;
   flex: 0 1 auto;
   width: 100%;
   border-bottom: $subtle-border;
+  color: $primary-color;
 
   @media (max-width: $break-point-mobile) {
     margin-left: 15px;
     margin-right: 15px;
   }
+  input {
+    background: $chrome-color;
+
+  }
+  select {
+    background: $chrome-color;
+    color: $text-color;
+  }
 }
 
 .nav-bar-link {

client/scss/_variables.scss

@@ -1,7 +1,7 @@
 //backgrounds
 $base-color: white; //default white
 $card-color: white; //default white
-$chrome-color: lightgray; //default white (navbar)
+$chrome-color: white; //default white (navbar)
 $blockquote-background: #EEEEFF;
 $background-color: $base-color;
 
@@ -19,7 +19,7 @@ $blockquote-text: $text-color;
 $grey: #9095A5;
 $help-color: $grey;
 $subtle-border-color: #DDD;
-$highlight-border-color: #333;
+$highlight-border-color: #777;
 $shadow-color: rgba(169, 173, 186, 0.2);
 $subtle-border: 1px dashed $subtle-border-color;
 $grey-border: $subtle-border-color; //factor this out for all customers

client/src/containers/AssetInfo/view.jsx

@@ -151,7 +151,7 @@ class AssetInfo extends React.Component {
           <a
             className={'link--primary'}
             href={`${assetCanonicalUrl}.${fileExt}`}
-            download={name}
+            download={`${name}.${fileExt}`}
           >
             Download
           </a>

docs/settings.md

@@ -39,9 +39,17 @@ PUBLISHING:
     "publishingChannelWhitelist": [],
     "channelClaimBidAmount": "0.1", - When creating a channel, how much you deposit to control the name
     "fileClaimBidAmount": "0.01", - When publishing content, how much you deposit to control the name
-    "maxSizeImage": 10000000, - You may not want people uploading 50GB files. 1000000 = 1MB
+    "fileSizeLimits": {
-    "maxSizeGif": 50000000,
+      "image": 50000000,
-    "maxSizeVideo": 50000000
+      "video": 50000000,
+      "audio": 50000000,
+      "text": 10000000,
+      "model": 50000000,
+      "application": 500000000,
+      "customByContentType": {
+        "application/octet-stream": 50000000
+      }
+    }
     
 SERVING:
 
@@ -80,9 +88,6 @@ SERVING:
         "html", - potentially DANGEROUS, intended for `serveOnlyApproved = true` environments, includes iframes, divs.
         "parsedHtml"
       ],
-      "disallowedTypesMain": [], - not implemented
-      "disallowedTypesDescriptions": ["image", "html"], - not implemented
-      "disallowedTypesExample": ["image", "html"] - not implemented
     },
     "customFileExtensions": { - suggest a file extension for experimental content types you may be publishing
       "application/example-type": "example"

docs/ubuntuinstall.md

@@ -163,7 +163,7 @@ Log in as username@domainname or username@ip_address
 
   `mysql -u root -p` and then entering your_mysql_password should give you the mysql> shell
 
-# 5 Get Lbrynet Daemon
+# 5 Get Lbrynet SDK Daemon
 
 ## Start tmux
 
@@ -174,11 +174,34 @@ tmux allows you to run multiple things in different sessions. Useful for manuall
   * `tmux`, reenters tmux, then
     * `Ctrl+b`, `(` goes back to through sessions
 
-## Get the daemon
+## Get the SDK
   `wget -O ~/latest_daemon.zip https://lbry.io/get/lbrynet.linux.zip`
 
   `unzip -o -u ~/latest_daemon.zip`
 
+## Customize SDK settings
+
+  These settings will prevent you and your users from spending your server's LBC on paid content. Full documentation is [here](https://lbry.tech/resources/daemon-settings).
+  
+  ~$
+  `mkdir .lbrynet`
+  
+  `cd .lbrynet`
+  
+  `nano daemon_settings.yml`
+  
+   copy and paste in the following code (Ctrl+Shift V)
+
+  ```
+  run_reflector_server: false
+  disable_max_key_fee: false
+  max_key_fee: {amount: 0, currency: LBC}
+  use_upnp: false
+  auto_re_reflect_interval: 0
+ ```
+ 
+ `CONTROL+O` then `CONTROL+X` to save and exit
+ 
 ## Start the daemon
    `./lbrynet start`
 

package.json

@@ -46,6 +46,7 @@
     "express-http-context": "^1.2.0",
     "generate-password": "^1.4.1",
     "get-video-dimensions": "^1.0.0",
+    "gm": "^1.23.1",
     "helmet": "^3.15.0",
     "image-size": "^0.6.3",
     "inquirer": "^5.2.0",

server/controllers/api/claim/update/index.js

@@ -1,6 +1,9 @@
 const logger = require('winston');
 const db = require('server/models');
-const { details, publishing: { disabled, disabledMessage, primaryClaimAddress } } = require('@config/siteConfig');
+const {
+  details,
+  publishing: { disabled, disabledMessage, primaryClaimAddress },
+} = require('@config/siteConfig');
 const { resolveUri } = require('server/lbrynet');
 const { sendGATimingEvent } = require('../../../../utils/googleAnalytics.js');
 const { handleErrorResponse } = require('../../../utils/errorHandlers.js');
@@ -77,7 +80,15 @@ const claimUpdate = ({ body, files, headers, ip, originalUrl, user, tor }, res)
 
   try {
     ({ name, nsfw, license, title, description, thumbnail } = parsePublishApiRequestBody(body));
-    ({fileName, filePath, fileExtension, fileType, thumbnailFileName, thumbnailFilePath, thumbnailFileType} = parsePublishApiRequestFiles(files, true));
+    ({
+      fileName,
+      filePath,
+      fileExtension,
+      fileType,
+      thumbnailFileName,
+      thumbnailFilePath,
+      thumbnailFileType,
+    } = parsePublishApiRequestFiles(files, true));
     ({ channelName, channelId, channelPassword } = body);
   } catch (error) {
     return res.status(400).json({ success: false, message: error.message });
@@ -89,7 +100,9 @@ const claimUpdate = ({ body, files, headers, ip, originalUrl, user, tor }, res)
       if (!channelId) {
         channelId = channelClaimId;
       }
-      return chainquery.claim.queries.resolveClaimInChannel(name, channelClaimId).then(claim => claim.dataValues);
+      return chainquery.claim.queries
+        .resolveClaimInChannel(name, channelClaimId)
+        .then(claim => claim.dataValues);
     })
     .then(claim => {
       claimRecord = claim;
@@ -107,14 +120,18 @@ const claimUpdate = ({ body, files, headers, ip, originalUrl, user, tor }, res)
       return [null, null];
     })
     .then(([fileResult, resolution]) => {
-      metadata = Object.assign({}, {
+      metadata = Object.assign(
+        {},
+        {
           title: claimRecord.title,
           description: claimRecord.description,
           nsfw: claimRecord.nsfw,
           license: claimRecord.license,
           language: 'en',
           author: details.title,
-      }, updateMetadata({title, description, nsfw, license}));
+        },
+        updateMetadata({ title, description, nsfw, license })
+      );
       const publishParams = {
         name,
         bid: '0.01',
@@ -128,19 +145,24 @@ const claimUpdate = ({ body, files, headers, ip, originalUrl, user, tor }, res)
         if (thumbnailUpdate) {
           // publish new thumbnail
           const newThumbnailName = `${name}-${rando()}`;
-          const newThumbnailParams = createThumbnailPublishParams(filePath, newThumbnailName, license, nsfw);
+          const newThumbnailParams = createThumbnailPublishParams(
+            filePath,
+            newThumbnailName,
+            license,
+            nsfw
+          );
           newThumbnailParams['file_path'] = filePath;
           publish(newThumbnailParams, fileName, fileType);
 
-          publishParams['sources'] = resolution.claim.value.stream.source;
+          publishParams['thumbnail'] = `${details.host}/${newThumbnailParams.channel_name}:${
-          publishParams['thumbnail'] = `${details.host}/${newThumbnailParams.channel_name}:${newThumbnailParams.channel_id}/${newThumbnailName}-thumb.jpg`;
+            newThumbnailParams.channel_id
+          }/${newThumbnailName}-thumb.jpg`;
         } else {
           publishParams['file_path'] = filePath;
         }
       } else {
         fileName = fileResult.fileName;
         fileType = fileResult.fileType;
-        publishParams['sources'] = resolution.claim.value.stream.source;
         publishParams['thumbnail'] = claimRecord.thumbnail_url;
       }
 
@@ -151,9 +173,14 @@ const claimUpdate = ({ body, files, headers, ip, originalUrl, user, tor }, res)
       publishResult = result;
 
       if (channelName) {
-        return chainquery.claim.queries.getShortClaimIdFromLongClaimId(result.certificateId, channelName);
+        return chainquery.claim.queries.getShortClaimIdFromLongClaimId(
+          result.certificateId,
+          channelName
+        );
       } else {
-        return chainquery.claim.queries.getShortClaimIdFromLongClaimId(result.claimId, name, result).catch(() => {
+        return chainquery.claim.queries
+          .getShortClaimIdFromLongClaimId(result.claimId, name, result)
+          .catch(() => {
             return result.claimId.slice(0, 1);
           });
       }
@@ -161,7 +188,9 @@ const claimUpdate = ({ body, files, headers, ip, originalUrl, user, tor }, res)
     .then(shortId => {
       let canonicalUrl;
       if (channelName) {
-        canonicalUrl = createCanonicalLink({ asset: { ...publishResult, channelShortId: shortId } });
+        canonicalUrl = createCanonicalLink({
+          asset: { ...publishResult, channelShortId: shortId },
+        });
       } else {
         canonicalUrl = createCanonicalLink({ asset: { ...publishResult, shortId } });
       }

server/controllers/assets/utils/getClaimIdAndServeAsset.js

@@ -15,9 +15,20 @@ const BLOCKED_CLAIM = 'BLOCKED_CLAIM';
 const NO_FILE = 'NO_FILE';
 const CONTENT_UNAVAILABLE = 'CONTENT_UNAVAILABLE';
 
-const { publishing: { serveOnlyApproved, approvedChannels } } = require('@config/siteConfig');
+const {
+  publishing: { serveOnlyApproved, approvedChannels },
+} = require('@config/siteConfig');
 
-const getClaimIdAndServeAsset = (channelName, channelClaimId, claimName, claimId, originalUrl, ip, res, headers) => {
+const getClaimIdAndServeAsset = (
+  channelName,
+  channelClaimId,
+  claimName,
+  claimId,
+  originalUrl,
+  ip,
+  res,
+  headers
+) => {
   getClaimId(channelName, channelClaimId, claimName, claimId)
     .then(fullClaimId => {
       claimId = fullClaimId;
@@ -39,11 +50,19 @@ const getClaimIdAndServeAsset = (channelName, channelClaimId, claimName, claimId
     .then(claim => {
       let claimDataValues = claim.dataValues;
 
-      if (serveOnlyApproved && !isApprovedChannel({ longId: claimDataValues.publisher_id || claimDataValues.certificateId }, approvedChannels)) {
+      if (
+        serveOnlyApproved &&
+        !isApprovedChannel(
+          { longId: claimDataValues.publisher_id || claimDataValues.certificateId },
+          approvedChannels
+        )
+      ) {
         throw new Error(CONTENT_UNAVAILABLE);
       }
 
-      let outpoint = claimDataValues.outpoint || `${claimDataValues.transaction_hash_id}:${claimDataValues.vout}`;
+      let outpoint =
+        claimDataValues.outpoint ||
+        `${claimDataValues.transaction_hash_id}:${claimDataValues.vout}`;
       logger.debug('Outpoint:', outpoint);
       return db.Blocked.isNotBlocked(outpoint).then(() => {
         // If content was found, is approved, and not blocked - log a view.
@@ -70,7 +89,7 @@ const getClaimIdAndServeAsset = (channelName, channelClaimId, claimName, claimId
       if (!fileRecord) {
         throw NO_FILE;
       }
-      serveFile(fileRecord.dataValues, res);
+      serveFile(fileRecord.dataValues, res, originalUrl);
     })
     .catch(error => {
       if (error === NO_CLAIM) {
@@ -98,7 +117,8 @@ const getClaimIdAndServeAsset = (channelName, channelClaimId, claimName, claimId
         logger.debug('claim was blocked');
         return res.status(451).json({
           success: false,
-          message: 'In response to a complaint we received under the US Digital Millennium Copyright Act, we have blocked access to this content from our applications. For more details, see https://lbry.io/faq/dmca',
+          message:
+            'In response to a complaint we received under the US Digital Millennium Copyright Act, we have blocked access to this content from our applications. For more details, see https://lbry.io/faq/dmca',
         });
       }
       if (error === NO_FILE) {

server/controllers/assets/utils/serveFile.js

@@ -1,9 +1,28 @@
 const logger = require('winston');
+const transformImage = require('./transformImage');
+const serveFile = async ({ filePath, fileType }, res, originalUrl) => {
+  const queryObject = {};
+  // TODO: replace quick/dirty try catch with better practice
+  try {
+    originalUrl
+      .split('?')[1]
+      .split('&')
+      .map(pair => {
+        if (pair.includes('=')) {
+          let parr = pair.split('=');
+          queryObject[parr[0]] = parr[1];
+        } else queryObject[pair] = true;
+      });
+  } catch (e) {}
 
-const serveFile = ({ filePath, fileType }, res) => {
   if (!fileType) {
     logger.error(`no fileType provided for ${filePath}`);
   }
+
+  let mediaType = fileType ? fileType.substr(0, fileType.indexOf('/')) : '';
+  const transform =
+    mediaType === 'image' && queryObject.hasOwnProperty('h') && queryObject.hasOwnProperty('w');
+
   const sendFileOptions = {
     headers: {
       'X-Content-Type-Options': 'nosniff',
@@ -13,7 +32,15 @@ const serveFile = ({ filePath, fileType }, res) => {
     },
   };
   logger.debug(`fileOptions for ${filePath}:`, sendFileOptions);
+  if (transform) {
+    logger.debug(`transforming and sending file`);
+
+    let xformed = await transformImage(filePath, queryObject);
+    res.status(200).set(sendFileOptions.headers);
+    res.end(xformed, 'binary');
+  } else {
     res.status(200).sendFile(filePath, sendFileOptions);
+  }
 };
 
 module.exports = serveFile;

server/controllers/assets/utils/transformImage.js

@@ -0,0 +1,76 @@
+const gm = require('gm');
+const logger = require('winston');
+const imageMagick = gm.subClass({ imageMagick: true });
+const { getImageHeightAndWidth } = require('../../../utils/imageProcessing');
+
+module.exports = function transformImage(path, queryObj) {
+  return new Promise((resolve, reject) => {
+    let { h: cHeight = null } = queryObj;
+    let { w: cWidth = null } = queryObj;
+    let { t: transform = null } = queryObj;
+    let { x: xOrigin = null } = queryObj;
+    let { y: yOrigin = null } = queryObj;
+    let oHeight,
+      oWidth = null;
+    try {
+      getImageHeightAndWidth(path).then(hwarr => {
+        oHeight = hwarr[0];
+        oWidth = hwarr[1];
+        // conditional logic here
+        if (transform === 'crop') {
+          resolve(_cropCenter(path, cWidth, cHeight, oWidth, oHeight));
+        } else if (transform === 'stretch') {
+          imageMagick(path)
+            .resize(cWidth, cHeight, '!')
+            .toBuffer(null, (err, buf) => {
+              resolve(buf);
+            });
+        } else {
+          // resize scaled
+          imageMagick(path)
+            .resize(cWidth, cHeight)
+            .toBuffer(null, (err, buf) => {
+              resolve(buf);
+            });
+        }
+      });
+    } catch (e) {
+      logger.error(e);
+      reject(e);
+    }
+  });
+};
+
+function _cropCenter(path, cropWidth, cropHeight, originalWidth, originalHeight) {
+  let oAspect = originalWidth / originalHeight;
+  let cAspect = cropWidth / cropHeight;
+  let resizeX,
+    resizeY,
+    xpoint,
+    ypoint = null;
+
+  if (oAspect >= cAspect) {
+    // if crop is narrower aspect than original
+    resizeY = cropHeight;
+    xpoint = (oAspect * cropHeight) / 2 - cropWidth / 2;
+    ypoint = 0;
+  } else {
+    // if crop is wider aspect than original
+    resizeX = cropWidth;
+    xpoint = 0;
+    ypoint = cropWidth / oAspect / 2 - cropHeight / 2;
+  }
+  return new Promise((resolve, reject) => {
+    try {
+      imageMagick(path)
+        .resize(resizeX, resizeY)
+        .crop(cropWidth, cropHeight, xpoint, ypoint)
+        .toBuffer(null, (err, buf) => {
+          resolve(buf);
+        });
+    } catch (e) {
+      logger.error(e);
+      reject(e);
+    }
+  });
+}

server/lbrynet/index.js

@@ -93,7 +93,7 @@ module.exports = {
       axios
         .post(lbrynetUri, {
           method: 'resolve',
-          params: { uri },
+          params: { urls: uri },
         })
         .then(({ data }) => {
           sendGATimingEvent('lbrynet', 'resolveUri', 'RESOLVE', gaStartTime, Date.now());