From 49046c9d25a08aee5ed0109930aa477361c37955 Mon Sep 17 00:00:00 2001 From: infinite-persistence Date: Wed, 30 Jun 2021 14:02:09 +0800 Subject: [PATCH 1/2] RSS feed for channels ## Issue 3779 RSS feed for channels ## Initial implementation details - RSS only (not atom) - Grabs latest 10 entries (Beamer have concerns) ## Credit Referenced the community version mentioned in 3779 --- package.json | 1 + ui/component/claimMenuList/view.jsx | 18 +++++++- ui/util/url.js | 6 +++ web/src/chainquery.js | 20 ++++++++- web/src/routes.js | 7 ++++ web/src/rss.js | 64 +++++++++++++++++++++++++++++ yarn.lock | 14 +++++++ 7 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 web/src/rss.js diff --git a/package.json b/package.json index f6addf81b..9d19ade6e 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "electron-notarize": "^1.0.0", "electron-updater": "^4.2.4", "express": "^4.17.1", + "feed": "^4.2.2", "if-env": "^1.0.4", "react-datetime-picker": "^3.2.1", "react-top-loading-bar": "^2.0.1", diff --git a/ui/component/claimMenuList/view.jsx b/ui/component/claimMenuList/view.jsx index ac3623318..e5219e1ee 100644 --- a/ui/component/claimMenuList/view.jsx +++ b/ui/component/claimMenuList/view.jsx @@ -7,7 +7,7 @@ import React from 'react'; import classnames from 'classnames'; import { Menu, MenuButton, MenuList, MenuItem } from '@reach/menu-button'; import Icon from 'component/common/icon'; -import { generateShareUrl } from 'util/url'; +import { generateShareUrl, generateRssUrl } from 'util/url'; import { useHistory } from 'react-router'; import { buildURI, parseURI, COLLECTIONS_CONSTS } from 'lbry-redux'; @@ -103,6 +103,7 @@ function ClaimMenuList(props: Props) { } const shareUrl: string = generateShareUrl(SHARE_DOMAIN, uri); + const rssUrl: string = generateRssUrl(SHARE_DOMAIN, uri); const isCollectionClaim = claim && claim.value_type === 'collection'; // $FlowFixMe const isPlayable = @@ -182,6 +183,10 @@ function ClaimMenuList(props: Props) { } } + function handleCopyRssLink() { + navigator.clipboard.writeText(rssUrl); + } + function handleCopyLink() { navigator.clipboard.writeText(shareUrl); } @@ -350,7 +355,18 @@ function ClaimMenuList(props: Props) { )} )} +
+ + {isChannelPage && ( + +
+ + {__('Copy RSS URL')} +
+ + )} +
diff --git a/ui/util/url.js b/ui/util/url.js index 7c7c60a79..ef950d5de 100644 --- a/ui/util/url.js +++ b/ui/util/url.js @@ -147,3 +147,9 @@ export const generateShareUrl = ( const url = `${domain}/${lbryWebUrl}` + (urlParamsString === '' ? '' : `?${urlParamsString}`); return url; }; + +export const generateRssUrl = (domain, lbryUrl) => { + const { channelName, channelClaimId } = parseURI(lbryUrl); + const url = `${domain}/$/rss/@${channelName}/${channelClaimId}`; + return url; +}; diff --git a/web/src/chainquery.js b/web/src/chainquery.js index 7b745a070..686005dfc 100644 --- a/web/src/chainquery.js +++ b/web/src/chainquery.js @@ -9,7 +9,7 @@ const pool = mysql.createPool({ }); function queryPool(sql, params) { - return new Promise(resolve => { + return new Promise((resolve) => { pool.query(sql, params, (error, rows) => { if (error) { console.log('error', error); // eslint-disable-line @@ -58,3 +58,21 @@ module.exports.getClaim = async function getClaim(claimName, claimId, channelNam return queryPool(sql, params); }; + +module.exports.getChannelClaim = async function getChannelClaim(channelClaimId) { + const params = []; + const select = ['claim_id', 'name', 'title', 'thumbnail_url', 'description'].join(', '); + + const sql = `SELECT ${select} FROM claim WHERE claim_id = "${channelClaimId}"`; + return queryPool(sql, params); +}; + +module.exports.getClaimsFromChannel = async function getClaimsFromChannel(channelClaimId, count = 10) { + const params = []; + const select = ['claim_id', 'name', 'title', 'thumbnail_url', 'description', 'created_at'].join(', '); + const sort = 'ORDER BY created_at DESC'; + const limit = `LIMIT ${count}`; + + const sql = `SELECT ${select} FROM claim WHERE publisher_id = "${channelClaimId}" ${sort} ${limit}`; + return queryPool(sql, params); +}; diff --git a/web/src/routes.js b/web/src/routes.js index 2efbeb601..8284ef9dd 100644 --- a/web/src/routes.js +++ b/web/src/routes.js @@ -1,4 +1,5 @@ const { getHtml } = require('./html'); +const { getRss } = require('./rss'); const { generateStreamUrl } = require('../../ui/util/web'); const fetch = require('node-fetch'); const Router = require('@koa/router'); @@ -42,6 +43,12 @@ router.get(`/$/stream/:claimName/:claimId`, async (ctx) => { ctx.redirect(streamUrl); }); +router.get(`/$/rss/:claimName/:claimId`, async (ctx) => { + const xml = await getRss(ctx); + ctx.set('Content-Type', 'application/rss+xml'); + ctx.body = xml; +}); + router.get('*', async (ctx) => { const html = await getHtml(ctx); ctx.body = html; diff --git a/web/src/rss.js b/web/src/rss.js new file mode 100644 index 000000000..5c60893fa --- /dev/null +++ b/web/src/rss.js @@ -0,0 +1,64 @@ +const { URL, SITE_NAME } = require('../../config.js'); +const { getChannelClaim, getClaimsFromChannel } = require('./chainquery'); +const Feed = require('feed').Feed; + +async function getChannelClaimFromChainquery(claimId) { + const rows = await getChannelClaim(claimId); + if (rows && rows.length) { + const claim = rows[0]; + return claim; + } + + return undefined; +} + +async function getFeed(channelClaim) { + const replaceLineFeeds = (str) => str.replace(/(?:\r\n|\r|\n)/g, '
'); + + const options = { + title: channelClaim.title + ' on ' + SITE_NAME, + description: channelClaim.description ? replaceLineFeeds(channelClaim.description) : '', + link: `${URL}/${channelClaim.name}:${channelClaim.claim_id}`, + favicon: URL + '/public/favicon.png', + generator: SITE_NAME + ' RSS Feed', + image: channelClaim.thumbnail_url, + author: { + name: channelClaim.name, + link: URL + '/' + channelClaim.name + ':' + channelClaim.claim_id, + }, + }; + + const feed = new Feed(options); + + const latestClaims = await getClaimsFromChannel(channelClaim.claim_id, 10); + + latestClaims.forEach((c) => { + feed.addItem({ + guid: c.claim_id, + id: c.claim_id, + title: c.title, + description: c.description ? replaceLineFeeds(c.description) : '', + image: c.thumbnail_url, + link: URL + '/' + c.name + ':' + c.claim_id, + date: new Date(c.created_at), + }); + }); + + return feed; +} + +async function getRss(ctx) { + if (!ctx.params.claimName || !ctx.params.claimId) { + return 'Invalid URL'; + } + + const channelClaim = await getChannelClaimFromChainquery(ctx.params.claimId); + if (channelClaim) { + const feed = await getFeed(channelClaim); + return feed.rss2(); + } + + return 'Invalid channel'; +} + +module.exports = { getRss }; diff --git a/yarn.lock b/yarn.lock index 1199d4534..23a0071b8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5220,6 +5220,13 @@ fd-slicer@~1.1.0: dependencies: pend "~1.2.0" +feed@^4.2.2: + version "4.2.2" + resolved "https://registry.yarnpkg.com/feed/-/feed-4.2.2.tgz#865783ef6ed12579e2c44bbef3c9113bc4956a7e" + integrity sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ== + dependencies: + xml-js "^1.6.11" + figgy-pudding@^3.5.1: version "3.5.2" resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e" @@ -12512,6 +12519,13 @@ xdg-basedir@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13" +xml-js@^1.6.11: + version "1.6.11" + resolved "https://registry.yarnpkg.com/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9" + integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g== + dependencies: + sax "^1.2.4" + xmlbuilder@^10.0.0: version "10.1.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0" -- 2.45.3 From c84d820b0914384ed85dd0b0603c66988a14c0c4 Mon Sep 17 00:00:00 2001 From: infinite-persistence Date: Thu, 1 Jul 2021 09:24:02 +0800 Subject: [PATCH 2/2] RSS: Switch from ChainQuery to SDK ## Issue 3779 RSS feed for channels --- web/src/chainquery.js | 18 ---------- web/src/rss.js | 76 ++++++++++++++++++++++++++++++------------- 2 files changed, 54 insertions(+), 40 deletions(-) diff --git a/web/src/chainquery.js b/web/src/chainquery.js index 686005dfc..bc2fd4a87 100644 --- a/web/src/chainquery.js +++ b/web/src/chainquery.js @@ -58,21 +58,3 @@ module.exports.getClaim = async function getClaim(claimName, claimId, channelNam return queryPool(sql, params); }; - -module.exports.getChannelClaim = async function getChannelClaim(channelClaimId) { - const params = []; - const select = ['claim_id', 'name', 'title', 'thumbnail_url', 'description'].join(', '); - - const sql = `SELECT ${select} FROM claim WHERE claim_id = "${channelClaimId}"`; - return queryPool(sql, params); -}; - -module.exports.getClaimsFromChannel = async function getClaimsFromChannel(channelClaimId, count = 10) { - const params = []; - const select = ['claim_id', 'name', 'title', 'thumbnail_url', 'description', 'created_at'].join(', '); - const sort = 'ORDER BY created_at DESC'; - const limit = `LIMIT ${count}`; - - const sql = `SELECT ${select} FROM claim WHERE publisher_id = "${channelClaimId}" ${sort} ${limit}`; - return queryPool(sql, params); -}; diff --git a/web/src/rss.js b/web/src/rss.js index 5c60893fa..0c25e0c13 100644 --- a/web/src/rss.js +++ b/web/src/rss.js @@ -1,27 +1,56 @@ -const { URL, SITE_NAME } = require('../../config.js'); -const { getChannelClaim, getClaimsFromChannel } = require('./chainquery'); +const { URL, SITE_NAME, LBRY_WEB_API } = require('../../config.js'); +const { Lbry } = require('lbry-redux'); const Feed = require('feed').Feed; -async function getChannelClaimFromChainquery(claimId) { - const rows = await getChannelClaim(claimId); - if (rows && rows.length) { - const claim = rows[0]; - return claim; - } +const SDK_API_PATH = `${LBRY_WEB_API}/api/v1`; +const proxyURL = `${SDK_API_PATH}/proxy`; +Lbry.setDaemonConnectionString(proxyURL); - return undefined; +async function doClaimSearch(options) { + let results; + try { + results = await Lbry.claim_search(options); + } catch {} + return results ? results.items : undefined; +} + +async function getChannelClaim(claimId) { + const options = { + claim_ids: [claimId], + page_size: 1, + no_totals: true, + }; + + const claims = await doClaimSearch(options); + return claims ? claims[0] : undefined; +} + +async function getClaimsFromChannel(claimId, count) { + const options = { + channel_ids: [claimId], + page_size: count, + has_source: true, + claim_type: 'stream', + order_by: ['creation_timestamp'], + no_totals: true, + }; + + return await doClaimSearch(options); } async function getFeed(channelClaim) { const replaceLineFeeds = (str) => str.replace(/(?:\r\n|\r|\n)/g, '
'); + const value = channelClaim.value; + const title = value ? value.title : channelClaim.name; + const options = { - title: channelClaim.title + ' on ' + SITE_NAME, - description: channelClaim.description ? replaceLineFeeds(channelClaim.description) : '', + title: title + ' on ' + SITE_NAME, + description: value ? replaceLineFeeds(value.description) : '', link: `${URL}/${channelClaim.name}:${channelClaim.claim_id}`, favicon: URL + '/public/favicon.png', generator: SITE_NAME + ' RSS Feed', - image: channelClaim.thumbnail_url, + image: value ? value.thumbnail.url : '', author: { name: channelClaim.name, link: URL + '/' + channelClaim.name + ':' + channelClaim.claim_id, @@ -30,17 +59,20 @@ async function getFeed(channelClaim) { const feed = new Feed(options); - const latestClaims = await getClaimsFromChannel(channelClaim.claim_id, 10); + const latestClaims = await getClaimsFromChannel(channelClaim.claim_id, 50); latestClaims.forEach((c) => { + const meta = c.meta; + const value = c.value; + feed.addItem({ guid: c.claim_id, id: c.claim_id, - title: c.title, - description: c.description ? replaceLineFeeds(c.description) : '', - image: c.thumbnail_url, + title: value ? value.title : c.name, + description: value ? replaceLineFeeds(value.description) : '', + image: value ? value.thumbnail.url : '', link: URL + '/' + c.name + ':' + c.claim_id, - date: new Date(c.created_at), + date: new Date(meta ? meta.creation_timestamp * 1000 : null), }); }); @@ -52,13 +84,13 @@ async function getRss(ctx) { return 'Invalid URL'; } - const channelClaim = await getChannelClaimFromChainquery(ctx.params.claimId); - if (channelClaim) { - const feed = await getFeed(channelClaim); - return feed.rss2(); + const channelClaim = await getChannelClaim(ctx.params.claimId); + if (typeof channelClaim === 'string' || !channelClaim) { + return channelClaim; } - return 'Invalid channel'; + const feed = await getFeed(channelClaim); + return feed.rss2(); } module.exports = { getRss }; -- 2.45.3