From 758e44eb66b0f4300f82537bdf2caacc31732b01 Mon Sep 17 00:00:00 2001 From: jessop Date: Mon, 31 Dec 2018 21:47:26 -0500 Subject: [PATCH 1/4] eslint now ignores server/bundle --- .eslintignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintignore b/.eslintignore index a9b5af60..20deeba0 100644 --- a/.eslintignore +++ b/.eslintignore @@ -2,5 +2,6 @@ client/build node_modules/ public/bundle server/render/build +server/bundle test/ server/chainquery From 43648f924cbc1e6c07b902496f0035f47dec9901 Mon Sep 17 00:00:00 2001 From: YULIUS KURNIAWAN KRISTIANTO Date: Thu, 3 Jan 2019 03:31:00 +0700 Subject: [PATCH 2/4] Update LICENSE add 2019 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index a0551165..bb7bb074 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2017-2018 LBRY Inc. +Copyright (c) 2017-2019 LBRY Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, From cef94e5743fb57b87b7ae7158bafc90b5742aa16 Mon Sep 17 00:00:00 2001 From: jessop Date: Mon, 7 Jan 2019 17:44:34 -0500 Subject: [PATCH 3/4] enhances dmca system --- cli/defaults/siteConfig.json | 3 +- client/scss/_asset-preview.scss | 10 +++ client/src/components/AssetPreview/index.jsx | 64 +++++++++++--------- client/src/containers/AssetBlocked/index.js | 13 ++++ client/src/containers/AssetBlocked/view.jsx | 58 ++++++++++++++++++ client/src/pages/ShowAssetDetails/view.jsx | 44 ++++++++------ server/index.js | 39 +++++++++--- server/models/blocked.js | 18 ++++-- server/utils/blockList.js | 26 ++++++++ server/utils/getClaimData.js | 9 ++- 10 files changed, 224 insertions(+), 60 deletions(-) create mode 100644 client/src/containers/AssetBlocked/index.js create mode 100644 client/src/containers/AssetBlocked/view.jsx create mode 100644 server/utils/blockList.js diff --git a/cli/defaults/siteConfig.json b/cli/defaults/siteConfig.json index 3b8395c4..de2eca7b 100644 --- a/cli/defaults/siteConfig.json +++ b/cli/defaults/siteConfig.json @@ -17,7 +17,8 @@ "ipAddress": "", "host": "https://www.example.com", "description": "A decentralized hosting platform built on LBRY", - "twitter": false + "twitter": false, + "blockListEndpoint": "https://api.lbry.io/file/list_blocked" }, "publishing": { "primaryClaimAddress": null, diff --git a/client/scss/_asset-preview.scss b/client/scss/_asset-preview.scss index a05c2a2c..02a02ae5 100644 --- a/client/scss/_asset-preview.scss +++ b/client/scss/_asset-preview.scss @@ -2,6 +2,16 @@ position: relative; } +.asset-preview__blocked { + box-sizing: border-box; + background: black; + color: white; + height: 80%; + padding: 5px; + //remove margin-bottom after mystery 5px on wrapper is gone. + margin-bottom: 5px; +} + .asset-preview__image { width : 100%; padding: 0; diff --git a/client/src/components/AssetPreview/index.jsx b/client/src/components/AssetPreview/index.jsx index 5599709e..2d452b36 100644 --- a/client/src/components/AssetPreview/index.jsx +++ b/client/src/components/AssetPreview/index.jsx @@ -2,51 +2,59 @@ import React from 'react'; import { Link } from 'react-router-dom'; import createCanonicalLink from '../../../../utils/createCanonicalLink'; -const ClaimPending = () => { - return ( -
PENDING
- ); -}; - const AssetPreview = ({ defaultThumbnail, claimData }) => { - const {name, fileExt, contentType, thumbnail, title, pending} = claimData; + const {name, fileExt, contentType, thumbnail, title, blocked} = claimData; const showUrl = createCanonicalLink({asset: {...claimData}}); const embedUrl = `${showUrl}.${fileExt}`; - switch (contentType) { - case 'image/jpeg': - case 'image/jpg': - case 'image/png': - case 'image/gif': - return ( - -
+ + /* + This blocked section shouldn't be necessary after pagination is reworked, + though it might be useful for channel_mine situations. + */ + + if (blocked) { + return ( +
+
+

Error 451

+

This content is blocked for legal reasons.

+
+

Blocked Content

+
+ ); + } else { + switch (contentType) { + case 'image/jpeg': + case 'image/jpg': + case 'image/png': + case 'image/gif': + return ( + {name}

{title}

-
- - ); - case 'video/mp4': - return ( - -
+ + ); + case 'video/mp4': + return ( +
{name} -
+

{title}

-
- - ); - default: - return null; + + ); + default: + return null; + } } }; diff --git a/client/src/containers/AssetBlocked/index.js b/client/src/containers/AssetBlocked/index.js new file mode 100644 index 00000000..933762d8 --- /dev/null +++ b/client/src/containers/AssetBlocked/index.js @@ -0,0 +1,13 @@ +import { connect } from 'react-redux'; +import View from './view'; +import { selectAsset } from '../../selectors/show'; + +const mapStateToProps = (props) => { + const {show} = props; + const asset = selectAsset(show); + return { + asset, + }; +}; + +export default connect(mapStateToProps, null)(View); diff --git a/client/src/containers/AssetBlocked/view.jsx b/client/src/containers/AssetBlocked/view.jsx new file mode 100644 index 00000000..de79a14a --- /dev/null +++ b/client/src/containers/AssetBlocked/view.jsx @@ -0,0 +1,58 @@ +import React from 'react'; +import createCanonicalLink from '../../../../utils/createCanonicalLink'; +import HorizontalSplit from '@components/HorizontalSplit'; +/* +This component shouldn't be necessary after pagination is reworked, +though it might be useful for channel_mine situations. +*/ +class BlockedLeft extends React.PureComponent { + render () { + return ( + + {'451 + + ); + } +} + +class BlockedRight extends React.PureComponent { + render () { + return ( + +

In response to a complaint we received under the US Digital Millennium Copyright Act, we have blocked access to this content from our applications.

+

Click here for more information.

+ + ); + } +} + +class AssetBlocked extends React.Component { + componentDidMount () { + /* + This function and fetch exists to send the browser the appropriate 451 error. + */ + const { asset } = this.props; + const { claimData: { contentType, outpoint } } = asset; + let fileExt; + if (typeof contentType === 'string') { + fileExt = contentType.split('/')[1] || 'jpg'; + } + const sourceUrl = `${createCanonicalLink({ asset: asset.claimData })}.${fileExt}?${outpoint}`; + fetch(sourceUrl) + .catch(); + } + + render () { + return ( +
+ } + rightSide={} + /> +
+ ); + } +}; + +export default AssetBlocked; diff --git a/client/src/pages/ShowAssetDetails/view.jsx b/client/src/pages/ShowAssetDetails/view.jsx index 7798531d..71051454 100644 --- a/client/src/pages/ShowAssetDetails/view.jsx +++ b/client/src/pages/ShowAssetDetails/view.jsx @@ -2,6 +2,7 @@ import React from 'react'; import PageLayout from '@components/PageLayout'; import * as Icon from 'react-feather'; import AssetDisplay from '@containers/AssetDisplay'; +import AssetBlocked from '@containers/AssetBlocked'; import AssetInfo from '@containers/AssetInfo'; import ErrorPage from '@pages/ErrorPage'; import AssetTitle from '@containers/AssetTitle'; @@ -29,23 +30,32 @@ class ShowAssetDetails extends React.Component { render () { const { asset } = this.props; if (asset) { - const { claimData: { name } } = asset; - return ( - -
- - - - -
- {!this.state.closed && } - - ); + const { claimData: { name, blocked } } = asset; + if (!blocked) { + return ( + +
+ + + +
+ {!this.state.closed && } + + ); + } else { + return ( + +
+ +
+ + ); + } } return ( diff --git a/server/index.js b/server/index.js index 4d32f4b4..0ec835cf 100644 --- a/server/index.js +++ b/server/index.js @@ -16,6 +16,7 @@ const createDatabaseIfNotExists = require('./models/utils/createDatabaseIfNotExi const { getWalletBalance } = require('./lbrynet/index'); const configureLogging = require('./utils/configureLogging'); const configureSlack = require('./utils/configureSlack'); +const { setupBlockList } = require('./utils/blockList'); const speechPassport = require('./speechPassport'); const processTrending = require('./utils/processTrending'); @@ -25,7 +26,7 @@ const { } = require('./middleware/logMetricsMiddleware'); const { - details: { port: PORT }, + details: { port: PORT, blockListEndpoint }, startup: { performChecks, performUpdates, @@ -34,6 +35,9 @@ const { const { sessionKey } = require('@private/authConfig.json'); +// configure.js doesn't handle new keys in config.json files yet. Make sure it doens't break. +let bLE; + function Server () { this.initialize = () => { // configure logging @@ -166,19 +170,37 @@ function Server () { logger.info('Starting LBC balance:', walletBalance); }); }; + this.performUpdates = () => { if (!performUpdates) { return; } + if (blockListEndpoint) { + bLE = blockListEndpoint; + } else if (!blockListEndpoint) { + if (typeof (blockListEndpoint) !== 'string') { + logger.warn('blockListEndpoint is null due to outdated siteConfig file. \n' + + 'Continuing with default LBRY blocklist api endpoint. \n ' + + '(Specify /"blockListEndpoint" : ""/ to disable.') + bLE = 'https://api.lbry.io/file/list_blocked'; + } + } logger.info(`Peforming updates...`); - return Promise.all([ - db.Blocked.refreshTable(), - db.Tor.refreshTable(), - ]) - .then(([updatedBlockedList, updatedTorList]) => { - logger.info('Blocked list updated, length:', updatedBlockedList.length); + if (!bLE) { + logger.info('Configured for no Block List') + db.Tor.refreshTable().then( (updatedTorList) => { logger.info('Tor list updated, length:', updatedTorList.length); }); + } else { + + return Promise.all([ + db.Blocked.refreshTable(bLE), + db.Tor.refreshTable()]) + .then(([updatedBlockedList, updatedTorList]) => { + logger.info('Blocked list updated, length:', updatedBlockedList.length); + logger.info('Tor list updated, length:', updatedTorList.length); + }) + } }; this.start = () => { this.initialize(); @@ -194,6 +216,9 @@ function Server () { this.performUpdates(), ]); }) + .then(() => { + return setupBlockList(); + }) .then(() => { logger.info('Spee.ch startup is complete'); diff --git a/server/models/blocked.js b/server/models/blocked.js index 5e413889..fff43d51 100644 --- a/server/models/blocked.js +++ b/server/models/blocked.js @@ -1,5 +1,4 @@ const logger = require('winston'); - const BLOCKED_CLAIM = 'BLOCKED_CLAIM'; module.exports = (sequelize, { STRING }) => { @@ -16,6 +15,14 @@ module.exports = (sequelize, { STRING }) => { } ); + Blocked.getBlockList = function () { + logger.debug('returning full block list'); + return new Promise((resolve, reject) => { + this.findAll() + .then(list => { return resolve(list) }); + }); + }; + Blocked.isNotBlocked = function (outpoint) { logger.debug(`checking to see if ${outpoint} is not blocked`); return new Promise((resolve, reject) => { @@ -37,9 +44,10 @@ module.exports = (sequelize, { STRING }) => { }); }; - Blocked.refreshTable = function () { + Blocked.refreshTable = function (blockEndpoint) { let blockedList = []; - return fetch('https://api.lbry.io/file/list_blocked') + + return fetch(blockEndpoint) .then(response => { return response.json(); }) @@ -50,9 +58,7 @@ module.exports = (sequelize, { STRING }) => { if (!jsonResponse.data.outpoints) { throw new Error('no outpoints in list_blocked response'); } - return jsonResponse.data.outpoints; - }) - .then(outpoints => { + let outpoints = jsonResponse.data.outpoints; logger.debug('total outpoints:', outpoints.length); // prep the records for (let i = 0; i < outpoints.length; i++) { diff --git a/server/utils/blockList.js b/server/utils/blockList.js new file mode 100644 index 00000000..0c7637c2 --- /dev/null +++ b/server/utils/blockList.js @@ -0,0 +1,26 @@ +const logger = require('winston'); +const db = require('../models'); + +let blockList = new Set(); + +const setupBlockList = (intervalInSeconds = 60) => { + const fetchList = () => { + return new Promise((resolve, reject) => { + db.Blocked.getBlockList() + .then((result) => { + blockList.clear(); + if (result.length > 0) { + result.map((item) => { blockList.add(item.dataValues.outpoint) }); + resolve(); + } else reject(); + }) + .catch(e => { console.error('list was empty', e) }); + }); + }; + setInterval(() => { fetchList() }, intervalInSeconds * 1000); + return fetchList(); +}; +module.exports = { + isBlocked: (outpoint) => { return blockList.has(outpoint) }, + setupBlockList, +}; diff --git a/server/utils/getClaimData.js b/server/utils/getClaimData.js index b898d683..86385a96 100644 --- a/server/utils/getClaimData.js +++ b/server/utils/getClaimData.js @@ -1,6 +1,7 @@ const { details: { host } } = require('@config/siteConfig'); const chainquery = require('chainquery').default; const { getClaim } = require('server/lbrynet'); +const { isBlocked } = require('./blockList'); module.exports = async (data, chName = null, chShortId = null) => { // TODO: Refactor getching the channel name out; requires invasive changes. @@ -9,6 +10,11 @@ module.exports = async (data, chName = null, chShortId = null) => { let lbrynetFileExt = null; let channelShortId = chShortId; let channelName = chName; + let blocked; + const outPoint = `${data.transaction_hash_id}:${data.vout}`; + if (isBlocked(outPoint)) { + blocked = true; + } if (!chName && certificateId && !channelName) { channelName = await chainquery.claim.queries.getClaimChannelName(certificateId).catch(() => { @@ -38,8 +44,9 @@ module.exports = async (data, chName = null, chShortId = null) => { fileExt : data.generated_extension || data.fileExt || lbrynetFileExt, description: data.description, thumbnail : data.generated_thumbnail || data.thumbnail_url || data.thumbnail, - outpoint : `${data.transaction_hash_id}:${data.vout}` || data.outpoint, + outpoint : outPoint || data.outpoint, host, pending : Boolean(data.height === 0), + blocked : blocked, }); }; From d8f3554bdd576164353cf4d46c3521f8a8716da1 Mon Sep 17 00:00:00 2001 From: jessop Date: Mon, 7 Jan 2019 17:47:56 -0500 Subject: [PATCH 4/4] fixes ffprobe --- server/controllers/api/claim/get/index.js | 43 ++++++++++++----------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/server/controllers/api/claim/get/index.js b/server/controllers/api/claim/get/index.js index 250796fe..94914810 100644 --- a/server/controllers/api/claim/get/index.js +++ b/server/controllers/api/claim/get/index.js @@ -5,6 +5,7 @@ const getClaimData = require('server/utils/getClaimData'); const chainquery = require('chainquery').default; const db = require('../../../../models'); const waitOn = require('wait-on'); +const logger = require('winston'); /* @@ -17,32 +18,35 @@ const claimGet = async ({ ip, originalUrl, params }, res) => { const claimId = params.claimId; try { - let claimData = await chainquery.claim.queries.resolveClaim(name, claimId).catch(() => {}); - if (!claimData) { - claimData = await db.Claim.resolveClaim(name, claimId); + let claimInfo = await chainquery.claim.queries.resolveClaim(name, claimId).catch(() => {}); + if (claimInfo) { + logger.info('claim/get: claim resolved in chainquery'); } - - if (!claimData) { - throw new Error('No matching uri found in Claim table'); + if (!claimInfo) { + claimInfo = await db.Claim.resolveClaim(name, claimId); + } + if (!claimInfo) { + throw new Error('claim/get: resolveClaim: No matching uri found in Claim table'); } - let lbrynetResult = await getClaim(`${name}#${claimId}`); if (!lbrynetResult) { - throw new Error(`Unable to Get ${name}#${claimId}`); + throw new Error(`claim/get: getClaim Unable to Get ${name}#${claimId}`); + } + const claimData = await getClaimData(claimInfo); + if (!claimData) { + throw new Error('claim/get: getClaimData failed to get file blobs'); + } + await waitOn({ + resources: [ lbrynetResult.download_path ], + timeout : 10000, // 10 seconds + window : 500, + }); + const fileData = await createFileRecordDataAfterGet(claimData, lbrynetResult); + if (!fileData) { + throw new Error('claim/get: createFileRecordDataAfterGet failed to create file in time'); } - - let fileData = await createFileRecordDataAfterGet(await getClaimData(claimData), lbrynetResult); const upsertCriteria = { name, claimId }; await db.upsert(db.File, fileData, upsertCriteria, 'File'); - - try { - await waitOn({ - resources: [ lbrynetResult.file_name ], - delay : 500, - timeout : 10000, // 10 seconds - }); - } catch (e) {} - const { message, completed } = lbrynetResult; res.status(200).json({ success: true, @@ -53,5 +57,4 @@ const claimGet = async ({ ip, originalUrl, params }, res) => { handleErrorResponse(originalUrl, ip, error, res); } }; - module.exports = claimGet;