Merge pull request #851 from jessopb/dmca

makes enhancements to dmca system, fixes ffprobe

cli/defaults/siteConfig.json

@@ -17,7 +17,8 @@
     "ipAddress": "",
     "host": "https://www.example.com",
     "description": "A decentralized hosting platform built on LBRY",
-    "twitter": false
+    "twitter": false,
+    "blockListEndpoint": "https://api.lbry.io/file/list_blocked"
   },
   "publishing": {
     "primaryClaimAddress": null,

client/scss/_asset-preview.scss

@@ -2,6 +2,16 @@
   position: relative;
 }
 
+.asset-preview__blocked {
+  box-sizing: border-box;
+  background: black;
+  color: white;
+  height: 80%;
+  padding: 5px;
+  //remove margin-bottom after mystery 5px on wrapper is gone.
+  margin-bottom: 5px;
+}
+
 .asset-preview__image {
   width  : 100%;
   padding: 0;

client/src/components/AssetPreview/index.jsx

@@ -2,51 +2,59 @@ import React from 'react';
 import { Link } from 'react-router-dom';
 import createCanonicalLink from '../../../../utils/createCanonicalLink';
 
-const ClaimPending = () => {
-  return (
-    <div className='claim-pending'>PENDING</div>
-  );
-};
-
 const AssetPreview = ({ defaultThumbnail, claimData }) => {
-  const {name, fileExt, contentType, thumbnail, title, pending} = claimData;
+  const {name, fileExt, contentType, thumbnail, title, blocked} = claimData;
   const showUrl = createCanonicalLink({asset: {...claimData}});
   const embedUrl = `${showUrl}.${fileExt}`;
-  switch (contentType) {
+
-    case 'image/jpeg':
+  /*
-    case 'image/jpg':
+  This blocked section shouldn't be necessary after pagination is reworked,
-    case 'image/png':
+  though it might be useful for channel_mine situations.
-    case 'image/gif':
+  */
-      return (
+
-        <Link to={showUrl} className='asset-preview'>
+  if (blocked) {
-          <div>
+    return (
+      <div className='asset-preview'>
+        <div className='asset-preview__blocked'>
+          <h3>Error 451</h3>
+          <p>This content is blocked for legal reasons.</p>
+        </div>
+        <h3 className='asset-preview__title'>Blocked Content</h3>
+      </div>
+    );
+  } else {
+    switch (contentType) {
+      case 'image/jpeg':
+      case 'image/jpg':
+      case 'image/png':
+      case 'image/gif':
+        return (
+          <Link to={showUrl} className='asset-preview'>
             <img
               className={'asset-preview__image'}
               src={embedUrl}
               alt={name}
             />
             <h3 className='asset-preview__title'>{title}</h3>
-          </div>
+          </Link>
-        </Link>
+        );
-      );
+      case 'video/mp4':
-    case 'video/mp4':
+        return (
-      return (
+          <Link to={showUrl} className='asset-preview'>
-        <Link to={showUrl} className='asset-preview'>
-          <div>
             <div className='asset-preview__play-wrapper'>
               <img
                 className={'asset-preview__video'}
                 src={thumbnail || defaultThumbnail}
                 alt={name}
               />
-              <div className='asset-preview__play-overlay'></div>
+              <div className='asset-preview__play-overlay' />
             </div>
             <h3 className='asset-preview__title'>{title}</h3>
-          </div>
+          </Link>
-        </Link>
+        );
-      );
+      default:
-    default:
+        return null;
-      return null;
+    }
   }
 };
 

client/src/containers/AssetBlocked/index.js

@@ -0,0 +1,13 @@
+import { connect } from 'react-redux';
+import View from './view';
+import { selectAsset } from '../../selectors/show';
+
+const mapStateToProps = (props) => {
+  const {show} = props;
+  const asset = selectAsset(show);
+  return {
+    asset,
+  };
+};
+
+export default connect(mapStateToProps, null)(View);

client/src/containers/AssetBlocked/view.jsx

@@ -0,0 +1,58 @@
+import React from 'react';
+import createCanonicalLink from '../../../../utils/createCanonicalLink';
+import HorizontalSplit from '@components/HorizontalSplit';
+/*
+This component shouldn't be necessary after pagination is reworked,
+though it might be useful for channel_mine situations.
+*/
+class BlockedLeft extends React.PureComponent {
+  render () {
+    return (
+      <React.Fragment>
+        <img className='asset-image' src={'https://upload.wikimedia.org/wikipedia/commons/archive/a/af/20120315000030%21OR_451.svg'} alt={'451 image'} />
+      </React.Fragment>
+    );
+  }
+}
+
+class BlockedRight extends React.PureComponent {
+  render () {
+    return (
+      <React.Fragment>
+        <p>In response to a complaint we received under the US Digital Millennium Copyright Act, we have blocked access to this content from our applications.</p>
+        <p><a href={'https://lbry.io/faq/dmca'} >Click here</a> for more information.</p>
+      </React.Fragment>
+    );
+  }
+}
+
+class AssetBlocked extends React.Component {
+  componentDidMount () {
+    /*
+    This function and fetch exists to send the browser the appropriate 451 error.
+     */
+    const { asset } = this.props;
+    const { claimData: { contentType, outpoint } } = asset;
+    let fileExt;
+    if (typeof contentType === 'string') {
+      fileExt = contentType.split('/')[1] || 'jpg';
+    }
+    const sourceUrl = `${createCanonicalLink({ asset: asset.claimData })}.${fileExt}?${outpoint}`;
+    fetch(sourceUrl)
+      .catch();
+  }
+
+  render () {
+    return (
+      <div>
+        <HorizontalSplit
+          collapseOnMobile
+          leftSide={<BlockedLeft />}
+          rightSide={<BlockedRight />}
+        />
+      </div>
+    );
+  }
+};
+
+export default AssetBlocked;

client/src/pages/ShowAssetDetails/view.jsx

@@ -2,6 +2,7 @@ import React from 'react';
 import PageLayout from '@components/PageLayout';
 import * as Icon from 'react-feather';
 import AssetDisplay from '@containers/AssetDisplay';
+import AssetBlocked from '@containers/AssetBlocked';
 import AssetInfo from '@containers/AssetInfo';
 import ErrorPage from '@pages/ErrorPage';
 import AssetTitle from '@containers/AssetTitle';
@@ -29,23 +30,32 @@ class ShowAssetDetails extends React.Component {
   render () {
     const { asset } = this.props;
     if (asset) {
-      const { claimData: { name } } = asset;
+      const { claimData: { name, blocked } } = asset;
-      return (
+      if (!blocked) {
-        <PageLayout
+        return (
-          pageTitle={`${name} - details`}
+          <PageLayout
-          asset={asset}
+            pageTitle={`${name} - details`}
-        >
+            asset={asset}
-          <div className="asset-main">
+          >
-            <AssetDisplay />
+            <div className="asset-main">
-            <AssetTitle />
+              <AssetDisplay />
-            
+              <AssetTitle />
-            <button className='collapse-button' onClick={this.collapse}>
+              <button className='collapse-button' onClick={this.collapse}>
-              {this.state.closed ? <Icon.PlusCircle className='plus-icon' /> : <Icon.MinusCircle />}
+                {this.state.closed ? <Icon.PlusCircle className='plus-icon' /> : <Icon.MinusCircle />}
-            </button>
+              </button>
-          </div>
+            </div>
-          {!this.state.closed && <AssetInfo />}
+            {!this.state.closed && <AssetInfo />}
-        </PageLayout>
+          </PageLayout>
-      );
+        );
+      } else {
+        return (
+          <PageLayout>
+            <div className="asset-main">
+              <AssetBlocked />
+            </div>
+          </PageLayout>
+        );
+      }
     }
     return (
       <ErrorPage error={'loading asset data...'} />

server/controllers/api/claim/get/index.js

@@ -5,6 +5,7 @@ const getClaimData = require('server/utils/getClaimData');
 const chainquery = require('chainquery').default;
 const db = require('../../../../models');
 const waitOn = require('wait-on');
+const logger = require('winston');
 
 /*
 
@@ -17,32 +18,35 @@ const claimGet = async ({ ip, originalUrl, params }, res) => {
   const claimId = params.claimId;
 
   try {
-    let claimData = await chainquery.claim.queries.resolveClaim(name, claimId).catch(() => {});
+    let claimInfo = await chainquery.claim.queries.resolveClaim(name, claimId).catch(() => {});
-    if (!claimData) {
+    if (claimInfo) {
-      claimData = await db.Claim.resolveClaim(name, claimId);
+      logger.info('claim/get: claim resolved in chainquery');
     }
-
+    if (!claimInfo) {
-    if (!claimData) {
+      claimInfo = await db.Claim.resolveClaim(name, claimId);
-      throw new Error('No matching uri found in Claim table');
+    }
+    if (!claimInfo) {
+      throw new Error('claim/get: resolveClaim: No matching uri found in Claim table');
     }
-
     let lbrynetResult = await getClaim(`${name}#${claimId}`);
     if (!lbrynetResult) {
-      throw new Error(`Unable to Get ${name}#${claimId}`);
+      throw new Error(`claim/get: getClaim Unable to Get ${name}#${claimId}`);
+    }
+    const claimData = await getClaimData(claimInfo);
+    if (!claimData) {
+      throw new Error('claim/get: getClaimData failed to get file blobs');
+    }
+    await waitOn({
+      resources: [ lbrynetResult.download_path ],
+      timeout  : 10000, // 10 seconds
+      window   : 500,
+    });
+    const fileData = await createFileRecordDataAfterGet(claimData, lbrynetResult);
+    if (!fileData) {
+      throw new Error('claim/get: createFileRecordDataAfterGet failed to create file in time');
     }
-
-    let fileData = await createFileRecordDataAfterGet(await getClaimData(claimData), lbrynetResult);
     const upsertCriteria = { name, claimId };
     await db.upsert(db.File, fileData, upsertCriteria, 'File');
-
-    try {
-      await waitOn({
-        resources: [ lbrynetResult.file_name ],
-        delay    : 500,
-        timeout  : 10000, // 10 seconds
-      });
-    } catch (e) {}
-
     const { message, completed } = lbrynetResult;
     res.status(200).json({
       success: true,
@@ -53,5 +57,4 @@ const claimGet = async ({ ip, originalUrl, params }, res) => {
     handleErrorResponse(originalUrl, ip, error, res);
   }
 };
-
 module.exports = claimGet;

server/index.js

@@ -16,6 +16,7 @@ const createDatabaseIfNotExists = require('./models/utils/createDatabaseIfNotExi
 const { getWalletBalance } = require('./lbrynet/index');
 const configureLogging = require('./utils/configureLogging');
 const configureSlack = require('./utils/configureSlack');
+const { setupBlockList } = require('./utils/blockList');
 const speechPassport = require('./speechPassport');
 const processTrending = require('./utils/processTrending');
 
@@ -25,7 +26,7 @@ const {
 } = require('./middleware/logMetricsMiddleware');
 
 const {
-  details: { port: PORT },
+  details: { port: PORT, blockListEndpoint },
   startup: {
     performChecks,
     performUpdates,
@@ -34,6 +35,9 @@ const {
 
 const { sessionKey } = require('@private/authConfig.json');
 
+// configure.js doesn't handle new keys in config.json files yet. Make sure it doens't break.
+let bLE;
+
 function Server () {
   this.initialize = () => {
     // configure logging
@@ -166,19 +170,37 @@ function Server () {
         logger.info('Starting LBC balance:', walletBalance);
       });
   };
+
   this.performUpdates = () => {
     if (!performUpdates) {
       return;
     }
+    if (blockListEndpoint) {
+      bLE = blockListEndpoint;
+    } else if (!blockListEndpoint) {
+      if (typeof (blockListEndpoint) !== 'string') {
+        logger.warn('blockListEndpoint is null due to outdated siteConfig file. \n' +
+          'Continuing with default LBRY blocklist api endpoint. \n ' +
+          '(Specify /"blockListEndpoint" : ""/ to disable.')
+        bLE = 'https://api.lbry.io/file/list_blocked';
+      }
+    }
     logger.info(`Peforming updates...`);
-    return Promise.all([
+    if (!bLE) {
-      db.Blocked.refreshTable(),
+      logger.info('Configured for no Block List')
-      db.Tor.refreshTable(),
+      db.Tor.refreshTable().then( (updatedTorList) => {
-    ])
-      .then(([updatedBlockedList, updatedTorList]) => {
-        logger.info('Blocked list updated, length:', updatedBlockedList.length);
         logger.info('Tor list updated, length:', updatedTorList.length);
       });
+    } else {
+
+      return Promise.all([
+        db.Blocked.refreshTable(bLE),
+        db.Tor.refreshTable()])
+        .then(([updatedBlockedList, updatedTorList]) => {
+          logger.info('Blocked list updated, length:', updatedBlockedList.length);
+          logger.info('Tor list updated, length:', updatedTorList.length);
+        })
+    }
   };
   this.start = () => {
     this.initialize();
@@ -194,6 +216,9 @@ function Server () {
           this.performUpdates(),
         ]);
       })
+      .then(() => {
+        return setupBlockList();
+      })
       .then(() => {
         logger.info('Spee.ch startup is complete');
 

server/models/blocked.js

@@ -1,5 +1,4 @@
 const logger = require('winston');
-
 const BLOCKED_CLAIM = 'BLOCKED_CLAIM';
 
 module.exports = (sequelize, { STRING }) => {
@@ -16,6 +15,14 @@ module.exports = (sequelize, { STRING }) => {
     }
   );
 
+  Blocked.getBlockList = function () {
+    logger.debug('returning full block list');
+    return new Promise((resolve, reject) => {
+      this.findAll()
+        .then(list => { return resolve(list) });
+    });
+  };
+
   Blocked.isNotBlocked = function (outpoint) {
     logger.debug(`checking to see if ${outpoint} is not blocked`);
     return new Promise((resolve, reject) => {
@@ -37,9 +44,10 @@ module.exports = (sequelize, { STRING }) => {
     });
   };
 
-  Blocked.refreshTable = function () {
+  Blocked.refreshTable = function (blockEndpoint) {
     let blockedList = [];
-    return fetch('https://api.lbry.io/file/list_blocked')
+
+    return fetch(blockEndpoint)
       .then(response => {
         return response.json();
       })
@@ -50,9 +58,7 @@ module.exports = (sequelize, { STRING }) => {
         if (!jsonResponse.data.outpoints) {
           throw new Error('no outpoints in list_blocked response');
         }
-        return jsonResponse.data.outpoints;
+        let outpoints = jsonResponse.data.outpoints;
-      })
-      .then(outpoints => {
         logger.debug('total outpoints:', outpoints.length);
         // prep the records
         for (let i = 0; i < outpoints.length; i++) {

server/utils/blockList.js

@@ -0,0 +1,26 @@
+const logger = require('winston');
+const db = require('../models');
+
+let blockList = new Set();
+
+const setupBlockList = (intervalInSeconds = 60) => {
+  const fetchList = () => {
+    return new Promise((resolve, reject) => {
+      db.Blocked.getBlockList()
+        .then((result) => {
+          blockList.clear();
+          if (result.length > 0) {
+            result.map((item) => { blockList.add(item.dataValues.outpoint) });
+            resolve();
+          } else reject();
+        })
+        .catch(e => { console.error('list was empty', e) });
+    });
+  };
+  setInterval(() => { fetchList() }, intervalInSeconds * 1000);
+  return fetchList();
+};
+module.exports = {
+  isBlocked: (outpoint) => { return blockList.has(outpoint) },
+  setupBlockList,
+};

server/utils/getClaimData.js

@@ -1,6 +1,7 @@
 const { details: { host } } = require('@config/siteConfig');
 const chainquery = require('chainquery').default;
 const { getClaim } = require('server/lbrynet');
+const { isBlocked } = require('./blockList');
 
 module.exports = async (data, chName = null, chShortId = null) => {
   // TODO: Refactor getching the channel name out; requires invasive changes.
@@ -9,6 +10,11 @@ module.exports = async (data, chName = null, chShortId = null) => {
   let lbrynetFileExt = null;
   let channelShortId = chShortId;
   let channelName = chName;
+  let blocked;
+  const outPoint = `${data.transaction_hash_id}:${data.vout}`;
+  if (isBlocked(outPoint)) {
+    blocked = true;
+  }
 
   if (!chName && certificateId && !channelName) {
     channelName = await chainquery.claim.queries.getClaimChannelName(certificateId).catch(() => {
@@ -38,8 +44,9 @@ module.exports = async (data, chName = null, chShortId = null) => {
     fileExt    : data.generated_extension || data.fileExt || lbrynetFileExt,
     description: data.description,
     thumbnail  : data.generated_thumbnail || data.thumbnail_url || data.thumbnail,
-    outpoint   : `${data.transaction_hash_id}:${data.vout}` || data.outpoint,
+    outpoint   : outPoint || data.outpoint,
     host,
     pending    : Boolean(data.height === 0),
+    blocked    : blocked,
   });
 };