RSS: podcast support

## Issue
`https://github.com/lbryio/lbry-desktop/issues/6369#issuecomment-882081892`

## Changes
- Replace 'feed' with 'node-rss' for itunes spec support.
- Replace content type from 'rss+xml' to 'xml' so that the browser will display it nicely using the document tree without us having to re-format it ('node-rss' does not). Seems like all feeds that I found does it this way.
- There is no need to escape characters now that 'node-rss' does it. Nice.
This commit is contained in:
infinite-persistence 2021-08-02 11:42:31 +08:00 committed by Thomas Zarebczan
parent 60f4cca007
commit 0af1dfe282
4 changed files with 230 additions and 92 deletions

View file

@ -54,12 +54,12 @@
"electron-notarize": "^1.0.0",
"electron-updater": "^4.2.4",
"express": "^4.17.1",
"feed": "^4.2.2",
"if-env": "^1.0.4",
"react-datetime-picker": "^3.2.1",
"react-plastic": "^1.1.1",
"react-top-loading-bar": "^2.0.1",
"remove-markdown": "^0.3.0",
"rss": "^1.2.2",
"source-map-explorer": "^2.5.2",
"tempy": "^0.6.0",
"videojs-contrib-ads": "^6.9.0",

View file

@ -19,9 +19,11 @@ function getStreamUrl(ctx) {
}
const rssMiddleware = async (ctx) => {
const xml = await getRss(ctx);
ctx.set('Content-Type', 'application/rss+xml');
ctx.body = xml;
const rss = await getRss(ctx);
if (rss.startsWith('<?xml')) {
ctx.set('Content-Type', 'application/xml');
}
ctx.body = rss;
};
router.get(`/$/api/content/v1/get`, async (ctx) => {

View file

@ -1,7 +1,7 @@
const { generateDownloadUrl } = require('../../ui/util/web');
const { URL, SITE_NAME, LBRY_WEB_API, FAVICON } = require('../../config.js');
const { URL, SITE_NAME, LBRY_WEB_API } = require('../../config.js');
const { Lbry } = require('lbry-redux');
const Feed = require('feed').Feed;
const Rss = require('rss');
const SDK_API_PATH = `${LBRY_WEB_API}/api/v1`;
const proxyURL = `${SDK_API_PATH}/proxy`;
@ -9,6 +9,10 @@ Lbry.setDaemonConnectionString(proxyURL);
const NUM_ENTRIES = 500;
// ****************************************************************************
// Fetch claim info
// ****************************************************************************
async function doClaimSearch(options) {
let results;
try {
@ -19,15 +23,21 @@ async function doClaimSearch(options) {
async function getChannelClaim(name, claimId) {
let claim;
let error;
try {
const url = `lbry://${name}#${claimId}`;
const response = await Lbry.resolve({ urls: [url] });
if (response && response[url] && !response[url].error) {
claim = response && response[url];
}
} catch {}
return claim || 'The RSS URL is invalid or is not associated with any channel.';
if (!claim) {
error = 'The RSS URL is invalid or is not associated with any channel.';
}
return { claim, error };
}
async function getClaimsFromChannel(claimId, count) {
@ -43,103 +53,218 @@ async function getClaimsFromChannel(claimId, count) {
return await doClaimSearch(options);
}
async function getFeed(channelClaim, feedLink) {
const replaceLineFeeds = (str) => str.replace(/(?:\r\n|\r|\n)/g, '<br />');
// ****************************************************************************
// Helpers
// ****************************************************************************
const fmtDescription = (description) => replaceLineFeeds(description);
const generateEnclosureForClaimContent = (claim) => {
const value = claim.value;
if (!value || !value.stream_type) {
return undefined;
}
const sanitizeThumbsUrl = (url) => {
if (typeof url === 'string' && url.startsWith('https://')) {
return encodeURI(url).replace(/&/g, '%26');
}
return '';
};
switch (value.stream_type) {
case 'video':
case 'audio':
case 'image':
case 'document':
case 'software':
return {
url: generateDownloadUrl(claim.name, claim.claim_id),
type: (value.source && value.source.media_type) || undefined,
size: (value.source && value.source.size) || 0, // Per spec, 0 is a valid fallback.
};
const getEnclosure = (claim) => {
const value = claim.value;
if (!value || !value.stream_type || !value.source || !value.source.media_type) {
default:
return undefined;
}
};
const getLanguageValue = (claim) => {
if (claim && claim.value && claim.value.languages && claim.value.languages.length > 0) {
return claim.value.languages[0];
}
return 'en';
};
const replaceLineFeeds = (str) => str.replace(/(?:\r\n|\r|\n)/g, '<br />');
const isEmailRoughlyValid = (email) => /^\S+@\S+$/.test(email);
/**
* 'itunes:owner' is required by castfeedvalidator (w3c allows omission), and
* both name and email must be defined. The email must also be a "valid" one.
*
* Use a fallback email when the creator did not specify one. The email will not
* be shown to the user; it is just used for administrative purposes.
*
* @param claim
* @returns any
*/
const generateItunesOwnerElement = (claim) => {
let name = '---';
let email = 'no-reply@odysee.com';
if (claim && claim.value) {
name = claim.name;
if (isEmailRoughlyValid(claim.value.email)) {
email = claim.value.email;
}
}
switch (value.stream_type) {
case 'video':
case 'audio':
case 'image':
case 'document':
case 'software':
return {
url: encodeURI(generateDownloadUrl(claim.name, claim.claim_id)),
type: value.source.media_type,
length: value.source.size || 0, // Per spec, 0 is a valid fallback.
};
return {
'itunes:owner': [{ 'itunes:name': name }, { 'itunes:email': email }],
};
};
default:
return undefined;
const generateItunesExplicitElement = (claim) => {
const tags = (claim && claim.value && claim.tags) || [];
return { 'itunes:explicit': tags.includes('mature') ? 'yes' : 'no' };
};
const getItunesCategory = (claim) => {
const itunesCategories = [
'Arts',
'Business',
'Comedy',
'Education',
'Fiction',
'Government',
'History',
'Health & Fitness',
'Kids & Family',
'Leisure',
'Music',
'News',
'Religion & Spirituality',
'Science',
'Society & Culture',
'Sports',
'Technology',
'True Crime',
'TV & Film',
];
const tags = (claim && claim.value && claim.tags) || [];
for (let i = 0; i < tags.length; ++i) {
const tag = tags[i];
if (itunesCategories.includes(tag)) {
// "Note: Although you can specify more than one category and subcategory
// in your RSS feed, Apple Podcasts only recognizes the first category and
// subcategory."
// --> The only parse the first found tag.
return tag.replace('&', '&amp;');
}
};
}
const value = channelClaim.value;
const title = value ? value.title : channelClaim.name;
// itunes will not accept any other categories, and the element is required
// to pass castfeedvalidator. So, fallback to 'Leisure' (closes to "General")
// if the creator did not specify a tag.
return 'Leisure';
};
const options = {
favicon: FAVICON || URL + '/public/favicon.png',
generator: SITE_NAME + ' RSS Feed',
title: title + ' on ' + SITE_NAME,
description: fmtDescription(value && value.description ? value.description : ''),
link: encodeURI(`${URL}/${channelClaim.name}:${channelClaim.claim_id}`),
image: sanitizeThumbsUrl(value && value.thumbnail ? value.thumbnail.url : ''),
feedLinks: {
rss: encodeURI(feedLink),
},
author: {
name: encodeURI(channelClaim.name),
link: encodeURI(URL + '/' + channelClaim.name + ':' + channelClaim.claim_id),
},
};
const generateItunesDurationElement = (claim) => {
let duration;
if (claim && claim.value) {
if (claim.value.video) {
duration = claim.value.video.duration;
} else if (claim.value.audio) {
duration = claim.value.audio.duration;
}
}
const feed = new Feed(options);
const latestClaims = await getClaimsFromChannel(channelClaim.claim_id, NUM_ENTRIES);
if (duration) {
return { 'itunes:duration': `${duration}` };
}
};
latestClaims.forEach((c) => {
const meta = c.meta;
const value = c.value;
const generateItunesImageElement = (claim) => {
const thumbnailUrl = (claim && claim.value && claim.value.thumbnail && claim.value.thumbnail.url) || '';
if (thumbnailUrl) {
return {
'itunes:image': { _attr: { href: thumbnailUrl } },
};
}
};
const title = value && value.title ? value.title : c.name;
const thumbnailUrl = value && value.thumbnail ? value.thumbnail.url : '';
const thumbnailHtml = thumbnailUrl ? `<p><img src="${thumbnailUrl}" alt="thumbnail" title="${title}" /></p>` : '';
const getFormattedDescription = (claim) => {
return replaceLineFeeds((claim && claim.value && claim.value.description) || '');
};
feed.addItem({
id: c.claim_id,
guid: encodeURI(URL + '/' + c.name + ':' + c.claim_id),
title: value && value.title ? value.title : c.name,
description: thumbnailHtml + fmtDescription(value && value.description ? value.description : ''),
link: encodeURI(URL + '/' + c.name + ':' + c.claim_id),
date: new Date(meta ? meta.creation_timestamp * 1000 : null),
enclosure: getEnclosure(c),
// ****************************************************************************
// Generate
// ****************************************************************************
function generateFeed(feedLink, channelClaim, claimsInChannel) {
// --- Channel ---
const feed = new Rss({
title: ((channelClaim.value && channelClaim.value.title) || channelClaim.name) + ' on ' + SITE_NAME,
description: getFormattedDescription(channelClaim),
feed_url: feedLink,
site_url: URL,
image_url: (channelClaim.value && channelClaim.value.thumbnail && channelClaim.value.thumbnail.url) || undefined,
language: getLanguageValue(channelClaim),
custom_namespaces: { itunes: 'http://www.itunes.com/dtds/podcast-1.0.dtd' },
custom_elements: [
{ 'itunes:author': channelClaim.name },
{
'itunes:category': [
{
_attr: {
text: getItunesCategory(channelClaim),
},
},
],
},
generateItunesImageElement(channelClaim),
generateItunesOwnerElement(channelClaim),
generateItunesExplicitElement(channelClaim),
],
});
// --- Content ---
claimsInChannel.forEach((c) => {
const title = (c.value && c.value.title) || c.name;
const thumbnailUrl = (c.value && c.value.thumbnail && c.value.thumbnail.url) || '';
const thumbnailHtml = thumbnailUrl
? `<p><img src="${thumbnailUrl}" width="480" alt="thumbnail" title="${title}" /></p>`
: '';
const description = thumbnailHtml + getFormattedDescription(c);
feed.item({
title: title,
description: description,
url: `${URL}/${c.name}:${c.claim_id}`,
guid: undefined, // defaults to 'url'
author: undefined, // defaults feed author property
date: new Date(c.meta ? c.meta.creation_timestamp * 1000 : null),
enclosure: generateEnclosureForClaimContent(c),
custom_elements: [
{ 'itunes:title': title },
{ 'itunes:author': channelClaim.name },
generateItunesImageElement(c),
generateItunesDurationElement(c),
generateItunesExplicitElement(c),
],
});
});
return feed;
}
function postProcess(feed) {
// Handle 'Feed' creating an invalid MIME type when trying to guess
// from 'https://thumbnails.lbry.com/UCgQ8eREJzR1dO' style of URLs.
return feed.replace(/type="image\/\/.*"\/>/g, 'type="image/*"/>');
}
async function getRss(ctx) {
if (!ctx.params.claimName || !ctx.params.claimId) {
return 'Invalid URL';
}
const channelClaim = await getChannelClaim(ctx.params.claimName, ctx.params.claimId);
if (typeof channelClaim === 'string' || !channelClaim) {
return channelClaim;
const { claim: channelClaim, error } = await getChannelClaim(ctx.params.claimName, ctx.params.claimId);
if (error) {
return error;
}
const feed = await getFeed(channelClaim, `${URL}${ctx.request.url}`);
return postProcess(feed.rss2());
const latestClaimsInChannel = await getClaimsFromChannel(channelClaim.claim_id, NUM_ENTRIES);
const feed = generateFeed(`${URL}${ctx.request.url}`, channelClaim, latestClaimsInChannel);
return feed.xml();
}
module.exports = { getRss };

View file

@ -7205,13 +7205,6 @@ fd-slicer@~1.1.0:
dependencies:
pend "~1.2.0"
feed@^4.2.2:
version "4.2.2"
resolved "https://registry.yarnpkg.com/feed/-/feed-4.2.2.tgz#865783ef6ed12579e2c44bbef3c9113bc4956a7e"
integrity sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ==
dependencies:
xml-js "^1.6.11"
figgy-pudding@^3.5.1:
version "3.5.2"
resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e"
@ -10908,6 +10901,18 @@ mime-db@1.48.0:
resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.48.0.tgz#e35b31045dd7eada3aaad537ed88a33afbef2d1d"
integrity sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ==
mime-db@~1.25.0:
version "1.25.0"
resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.25.0.tgz#c18dbd7c73a5dbf6f44a024dc0d165a1e7b1c392"
integrity sha1-wY29fHOl2/b0SgJNwNFloeexw5I=
mime-types@2.1.13:
version "2.1.13"
resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.13.tgz#e07aaa9c6c6b9a7ca3012c69003ad25a39e92a88"
integrity sha1-4HqqnGxrmnyjASxpADrSWjnpKog=
dependencies:
mime-db "~1.25.0"
mime-types@^2.1.12, mime-types@~2.1.19:
version "2.1.31"
resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.31.tgz#a00d76b74317c61f9c2db2218b8e9f8e9c5c9e6b"
@ -14307,6 +14312,14 @@ roarr@^2.15.3:
semver-compare "^1.0.0"
sprintf-js "^1.1.2"
rss@^1.2.2:
version "1.2.2"
resolved "https://registry.yarnpkg.com/rss/-/rss-1.2.2.tgz#50a1698876138133a74f9a05d2bdc8db8d27a921"
integrity sha1-UKFpiHYTgTOnT5oF0r3I240nqSE=
dependencies:
mime-types "2.1.13"
xml "1.0.1"
rtlcss@2.5.0:
version "2.5.0"
resolved "https://registry.yarnpkg.com/rtlcss/-/rtlcss-2.5.0.tgz#455549e49113f9e1cf83169a44de526c816de8a4"
@ -17208,18 +17221,16 @@ xdg-basedir@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13"
xml-js@^1.6.11:
version "1.6.11"
resolved "https://registry.yarnpkg.com/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9"
integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==
dependencies:
sax "^1.2.4"
xml-name-validator@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-2.0.1.tgz#4d8b8f1eccd3419aa362061becef515e1e559635"
integrity sha1-TYuPHszTQZqjYgYb7O9RXh5VljU=
xml@1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
xmlbuilder@^10.0.0:
version "10.1.1"
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0"