lbry-desktop/ui/redux/actions/collections.js
infinite-persistence e4658bb044
Fix missing claims in large collection (#726)
## Issue
A huge list like http://localhost:9090/$/list/d91815c1bc8c80a1f354284a8c8e92d84d5f07a6 (193 items) often produces fewer results in the final rendered list.

## Cause
The same list of IDs was passed into `claim_search`, and we just increment the `page`. However, it seems like `claim_search` does not honor the order between each call, and some results from the previous page took the place of the results in the next page.  The total is the same, but there are now duplicates.

## Fix
When batching, slice the ID list ourselves to bypass the problem for now.
2022-01-18 09:33:01 -05:00

376 lines
12 KiB
JavaScript

// @flow
import * as ACTIONS from 'constants/action_types';
import { v4 as uuid } from 'uuid';
import Lbry from 'lbry';
import { doClaimSearch, doAbandonClaim } from 'redux/actions/claims';
import { makeSelectClaimForClaimId } from 'redux/selectors/claims';
import {
makeSelectCollectionForId,
// makeSelectPublishedCollectionForId, // for "save" or "copy" action
makeSelectPublishedCollectionForId,
makeSelectUnpublishedCollectionForId,
makeSelectEditedCollectionForId,
} from 'redux/selectors/collections';
import * as COLS from 'constants/collections';
const getTimestamp = () => {
return Math.floor(Date.now() / 1000);
};
const FETCH_BATCH_SIZE = 50;
export const doLocalCollectionCreate = (
name: string,
collectionItems: Array<string>,
type: string,
sourceId: string
) => (dispatch: Dispatch) => {
return dispatch({
type: ACTIONS.COLLECTION_NEW,
data: {
entry: {
id: uuid(), // start with a uuid, this becomes a claimId after publish
name: name,
updatedAt: getTimestamp(),
items: collectionItems || [],
sourceId: sourceId,
type: type,
},
},
});
};
export const doCollectionDelete = (id: string, colKey: ?string = undefined) => (
dispatch: Dispatch,
getState: GetState
) => {
const state = getState();
const claim = makeSelectClaimForClaimId(id)(state);
const collectionDelete = () =>
dispatch({
type: ACTIONS.COLLECTION_DELETE,
data: {
id: id,
collectionKey: colKey,
},
});
if (claim && !colKey) {
// could support "abandon collection claim, but keep private collection" later
return dispatch(doAbandonClaim(claim, collectionDelete));
}
return collectionDelete();
};
// Given a collection, save its collectionId to be resolved and displayed in Library
// export const doCollectionSave = (
// id: string,
// ) => (dispatch: Dispatch) => {
// return dispatch({
// type: ACTIONS.COLLECTION_SAVE,
// data: {
// id: id,
// },
// });
// };
// Given a collection and name, copy it to a local private collection with a name
// export const doCollectionCopy = (
// id: string,
// ) => (dispatch: Dispatch) => {
// return dispatch({
// type: ACTIONS.COLLECTION_COPY,
// data: {
// id: id,
// },
// });
// };
export const doFetchItemsInCollections = (
resolveItemsOptions: {
collectionIds: Array<string>,
pageSize?: number,
},
resolveStartedCallback?: () => void
) => async (dispatch: Dispatch, getState: GetState) => {
/*
1) make sure all the collection claims are loaded into claims reducer, search/resolve if necessary.
2) get the item claims for each
3) format and make sure they're in the order as in the claim
4) Build the collection objects and update collections reducer
5) Update redux claims reducer
*/
let state = getState();
const { collectionIds, pageSize } = resolveItemsOptions;
dispatch({
type: ACTIONS.COLLECTION_ITEMS_RESOLVE_STARTED,
data: { ids: collectionIds },
});
if (resolveStartedCallback) resolveStartedCallback();
const collectionIdsToSearch = collectionIds.filter((claimId) => !state.claims.byId[claimId]);
if (collectionIdsToSearch.length) {
await dispatch(doClaimSearch({ claim_ids: collectionIdsToSearch, page: 1, page_size: 9999 }));
}
const stateAfterClaimSearch = getState();
async function fetchItemsForCollectionClaim(claim: CollectionClaim, pageSize?: number) {
const totalItems = claim.value.claims && claim.value.claims.length;
const claimId = claim.claim_id;
const itemOrder = claim.value.claims;
const sortResults = (items: Array<Claim>, claimList) => {
const newItems: Array<Claim> = [];
claimList.forEach((id) => {
const index = items.findIndex((i) => i.claim_id === id);
if (index >= 0) {
newItems.push(items[index]);
}
});
/*
This will return newItems[] of length less than total_items below
if one or more of the claims has been abandoned. That's ok for now.
*/
return newItems;
};
const mergeBatches = (
arrayOfResults: Array<{ items: Array<Claim>, total_items: number }>,
claimList: Array<string>
) => {
const mergedResults: { items: Array<Claim>, total_items: number } = {
items: [],
total_items: 0,
};
arrayOfResults.forEach((result) => {
mergedResults.items = mergedResults.items.concat(result.items);
mergedResults.total_items = result.total_items;
});
mergedResults.items = sortResults(mergedResults.items, claimList);
return mergedResults;
};
try {
const batchSize = pageSize || FETCH_BATCH_SIZE;
const batches: Array<Promise<any>> = [];
for (let i = 0; i < Math.ceil(totalItems / batchSize); i++) {
batches[i] = Lbry.claim_search({
claim_ids: claim.value.claims.slice(i * batchSize, (i + 1) * batchSize),
page: 1,
page_size: batchSize,
no_totals: true,
});
}
const itemsInBatches = await Promise.all(batches);
const result = mergeBatches(itemsInBatches, itemOrder);
// $FlowFixMe
const itemsById: { claimId: string, items?: ?Array<GenericClaim> } = { claimId: claimId };
if (result.items) {
itemsById.items = result.items;
} else {
itemsById.items = null;
}
return itemsById;
} catch (e) {
return {
claimId: claimId,
items: null,
};
}
}
function formatForClaimActions(resultClaimsByUri) {
const formattedClaims = {};
Object.entries(resultClaimsByUri).forEach(([uri, uriResolveInfo]) => {
// Flow has terrible Object.entries support
// https://github.com/facebook/flow/issues/2221
if (uriResolveInfo) {
let result = {};
if (uriResolveInfo.value_type === 'channel') {
result.channel = uriResolveInfo;
// $FlowFixMe
result.claimsInChannel = uriResolveInfo.meta.claims_in_channel;
// ALSO SKIP COLLECTIONS
} else if (uriResolveInfo.value_type === 'collection') {
result.collection = uriResolveInfo;
} else {
result.stream = uriResolveInfo;
if (uriResolveInfo.signing_channel) {
result.channel = uriResolveInfo.signing_channel;
result.claimsInChannel =
(uriResolveInfo.signing_channel.meta && uriResolveInfo.signing_channel.meta.claims_in_channel) || 0;
}
}
// $FlowFixMe
formattedClaims[uri] = result;
}
});
return formattedClaims;
}
const invalidCollectionIds = [];
const promisedCollectionItemFetches = [];
collectionIds.forEach((collectionId) => {
const claim = makeSelectClaimForClaimId(collectionId)(stateAfterClaimSearch);
if (!claim) {
invalidCollectionIds.push(collectionId);
} else {
promisedCollectionItemFetches.push(fetchItemsForCollectionClaim(claim, pageSize));
}
});
// $FlowFixMe
const collectionItemsById: Array<{
claimId: string,
items: ?Array<GenericClaim>,
}> = await Promise.all(promisedCollectionItemFetches);
const newCollectionObjectsById = {};
const resolvedItemsByUrl = {};
collectionItemsById.forEach((entry) => {
// $FlowFixMe
const collectionItems: Array<any> = entry.items;
const collectionId = entry.claimId;
if (collectionItems) {
const claim = makeSelectClaimForClaimId(collectionId)(stateAfterClaimSearch);
const editedCollection = makeSelectEditedCollectionForId(collectionId)(stateAfterClaimSearch);
const { name, timestamp, value } = claim || {};
const { title } = value;
const valueTypes = new Set();
const streamTypes = new Set();
let newItems = [];
let isPlaylist;
if (collectionItems) {
collectionItems.forEach((collectionItem) => {
newItems.push(collectionItem.permanent_url);
valueTypes.add(collectionItem.value_type);
if (collectionItem.value.stream_type) {
streamTypes.add(collectionItem.value.stream_type);
}
resolvedItemsByUrl[collectionItem.canonical_url] = collectionItem;
});
isPlaylist =
valueTypes.size === 1 &&
valueTypes.has('stream') &&
((streamTypes.size === 1 && (streamTypes.has('audio') || streamTypes.has('video'))) ||
(streamTypes.size === 2 && streamTypes.has('audio') && streamTypes.has('video')));
}
newCollectionObjectsById[collectionId] = {
items: newItems,
id: collectionId,
name: title || name,
itemCount: claim.value.claims.length,
type: isPlaylist ? 'playlist' : 'collection',
updatedAt: timestamp,
};
if (editedCollection && timestamp > editedCollection['updatedAt']) {
dispatch({
type: ACTIONS.COLLECTION_DELETE,
data: {
id: collectionId,
collectionKey: 'edited',
},
});
}
} else {
invalidCollectionIds.push(collectionId);
}
});
const formattedClaimsByUri = formatForClaimActions(collectionItemsById);
dispatch({
type: ACTIONS.RESOLVE_URIS_COMPLETED,
data: { resolveInfo: formattedClaimsByUri },
});
dispatch({
type: ACTIONS.COLLECTION_ITEMS_RESOLVE_COMPLETED,
data: {
resolvedCollections: newCollectionObjectsById,
failedCollectionIds: invalidCollectionIds,
},
});
};
export const doFetchItemsInCollection = (options: { collectionId: string, pageSize?: number }, cb?: () => void) => {
const { collectionId, pageSize } = options;
const newOptions: { collectionIds: Array<string>, pageSize?: number } = {
collectionIds: [collectionId],
};
if (pageSize) newOptions.pageSize = pageSize;
return doFetchItemsInCollections(newOptions, cb);
};
export const doCollectionEdit = (collectionId: string, params: CollectionEditParams) => async (
dispatch: Dispatch,
getState: GetState
) => {
const state = getState();
const collection: Collection = makeSelectCollectionForId(collectionId)(state);
if (!collection) return dispatch({ type: ACTIONS.COLLECTION_ERROR, data: { message: 'collection does not exist' } });
const editedCollection: Collection = makeSelectEditedCollectionForId(collectionId)(state);
const unpublishedCollection: Collection = makeSelectUnpublishedCollectionForId(collectionId)(state);
const publishedCollection: Collection = makeSelectPublishedCollectionForId(collectionId)(state); // needs to be published only
const { uris, order, remove, type } = params;
const collectionType = type || collection.type;
const currentUrls = collection.items ? collection.items.concat() : [];
let newItems = currentUrls;
// Passed uris to add/remove:
if (uris) {
if (remove) {
// Filters (removes) the passed uris from the current list items
newItems = currentUrls.filter((url) => url && !uris.includes(url));
} else {
// Pushes (adds to the end) the passed uris to the current list items
uris.forEach((url) => newItems.push(url));
}
}
// Passed an ordering to change: (doesn't need the uris here since
// the items are already on the list)
if (order) {
const [movedItem] = currentUrls.splice(order.from, 1);
currentUrls.splice(order.to, 0, movedItem);
}
// Delete 'edited' if newItems are the same as publishedItems
if (editedCollection && newItems && publishedCollection.items.join(',') === newItems.join(',')) {
dispatch({ type: ACTIONS.COLLECTION_DELETE, data: { id: collectionId, collectionKey: 'edited' } });
} else {
dispatch({
type: ACTIONS.COLLECTION_EDIT,
data: {
id: collectionId,
collectionKey:
((editedCollection || publishedCollection) && 'edited') ||
(COLS.BUILTIN_LISTS.includes(collectionId) && 'builtin') ||
(unpublishedCollection && 'unpublished'),
collection: {
items: newItems,
id: collectionId,
name: params.name || collection.name,
updatedAt: getTimestamp(),
type: collectionType,
},
},
});
}
return true;
};