Fill in remaining Recsys fields

## Issue
6366 Recsys Evaluation Telemetry

The recommended list from lighthouse is obtained from `makeSelectRecommendedContentForUri`. This list is further tweaked by the GUI (e.g. move autoplay next item to top, remove blocked content, etc.). Recsys wants the final recommendation list and the clicked index (in exact order), so we need pass these info to the videojs recsys plugin somehow. Also, Recsys wants a recommendation list ID as well as the parent (referrer) ID, we so need to track the clicks and navigation.

## General Approach
- It seems easiest to just spew back the final (displayed) list and all the required info to Redux, and the recsys plugin (or anyone else in the future) can grab it.
- Try to touch few files as possible. The dirty work should all reside in `<RecommendedContent>` only.

## Changes
- `ClaimPreview`: add optional parameters to store an ID of the container that it is in (for this case, it is `ClaimList`) as well as the index within the container.
  - When clicked, we store the container ID in the navigation history `state` object.
    - For general cases, anyone can check this state from  `history.location.state` to know which container referred/navigated to the current page. For the recsys use case, we can use this as the `parentUUID`.
- `ClaimList`: just relay `onClick` and set IDs.
- `RecommendedContent`: now handles the uuid generation (for both parent and child) and stores the data in Redux.

ui/component/claimList/view.jsx

@@ -47,6 +47,7 @@ type Props = {
   searchOptions?: any,
   collectionId?: string,
   showNoSourceClaims?: boolean,
+  onClick?: (e: any, index?: number) => void,
 };
 
 export default function ClaimList(props: Props) {
@@ -55,6 +56,7 @@ export default function ClaimList(props: Props) {
     uris,
     headerAltControls,
     loading,
+    id,
     persistedStorageKey,
     empty,
     defaultSort,
@@ -80,6 +82,7 @@ export default function ClaimList(props: Props) {
     searchOptions,
     collectionId,
     showNoSourceClaims,
+    onClick,
   } = props;
 
   const [currentSort, setCurrentSort] = usePersistedState(persistedStorageKey, SORT_NEW);
@@ -107,6 +110,12 @@ export default function ClaimList(props: Props) {
     setCurrentSort(currentSort === SORT_NEW ? SORT_OLD : SORT_NEW);
   }
 
+  function handleClaimClicked(e, index) {
+    if (onClick) {
+      onClick(e, index);
+    }
+  }
+
   useEffect(() => {
     const handleScroll = debounce((e) => {
       if (page && pageSize && onScrollBottom) {
@@ -191,6 +200,8 @@ export default function ClaimList(props: Props) {
               {injectedItem && index === 4 && <li>{injectedItem}</li>}
               <ClaimPreview
                 uri={uri}
+                containerId={id}
+                indexInContainer={index}
                 type={type}
                 active={activeUri && uri === activeUri}
                 hideMenu={hideMenu}
@@ -209,6 +220,7 @@ export default function ClaimList(props: Props) {
                   return claim.name.length === 24 && !claim.name.includes(' ') && claim.value.author === 'Spee.ch';
                 }}
                 live={resolveLive(index)}
+                onClick={handleClaimClicked}
               />
             </React.Fragment>
           ))}

ui/component/claimPreview/view.jsx

@@ -29,6 +29,7 @@ import ClaimPreviewNoContent from './claim-preview-no-content';
 import { ENABLE_NO_SOURCE_CLAIMS } from 'config';
 import Button from 'component/button';
 import * as ICONS from 'constants/icons';
+import { CONTAINER_ID } from 'constants/navigation';
 
 const AbandonedChannelPreview = lazyImport(() =>
   import('component/abandonedChannelPreview' /* webpackChunkName: "abandonedChannelPreview" */)
@@ -45,7 +46,7 @@ type Props = {
   reflectingProgress?: any, // fxme
   resolveUri: (string) => void,
   isResolvingUri: boolean,
-  history: { push: (string) => void },
+  history: { push: (string | any) => void },
   title: string,
   nsfw: boolean,
   placeholder: string,
@@ -65,7 +66,7 @@ type Props = {
   actions: boolean | Node | string | number,
   properties: boolean | Node | string | number | ((Claim) => Node),
   empty?: Node,
-  onClick?: (any) => any,
+  onClick?: (e: any, index?: number) => any,
   streamingUrl: ?string,
   getFile: (string) => void,
   customShouldHide?: (Claim) => boolean,
@@ -88,6 +89,8 @@ type Props = {
   disableNavigation?: boolean,
   mediaDuration?: string,
   date?: any,
+  containerId?: string, // ID or name of the container (e.g. ClaimList, HOC, etc.) that this is in.
+  indexInContainer?: number, // The index order of this component within 'containerId'.
 };
 
 const ClaimPreview = forwardRef<any, {}>((props: Props, ref: any) => {
@@ -149,6 +152,8 @@ const ClaimPreview = forwardRef<any, {}>((props: Props, ref: any) => {
     isCollectionMine,
     collectionUris,
     disableNavigation,
+    containerId,
+    indexInContainer,
   } = props;
   const isCollection = claim && claim.value_type === 'collection';
   const collectionClaimId = isCollection && claim && claim.claim_id;
@@ -202,9 +207,21 @@ const ClaimPreview = forwardRef<any, {}>((props: Props, ref: any) => {
     collectionParams.set(COLLECTIONS_CONSTS.COLLECTION_ID, listId);
     navigateUrl = navigateUrl + `?` + collectionParams.toString();
   }
+
+  const handleNavLinkClick = (e) => {
+    if (onClick) {
+      onClick(e, indexInContainer);
+    }
+    e.stopPropagation();
+  };
+
   const navLinkProps = {
-    to: navigateUrl,
-    onClick: (e) => e.stopPropagation(),
+    to: {
+      pathname: navigateUrl,
+      state: containerId ? { [CONTAINER_ID]: containerId } : undefined,
+    },
+    onClick: (e) => handleNavLinkClick(e),
+    onAuxClick: (e) => handleNavLinkClick(e),
   };
 
   // do not block abandoned and nsfw claims if showUserBlocked is passed
@@ -250,11 +267,14 @@ const ClaimPreview = forwardRef<any, {}>((props: Props, ref: any) => {
 
   function handleOnClick(e) {
     if (onClick) {
-      onClick(e);
+      onClick(e, indexInContainer);
     }
 
     if (claim && !pending && !disableNavigation) {
-      history.push(navigateUrl);
+      history.push({
+        pathname: navigateUrl,
+        state: containerId ? { [CONTAINER_ID]: containerId } : undefined,
+      });
     }
   }
 

ui/component/recommendedContent/index.js

@@ -1,5 +1,6 @@
 import { connect } from 'react-redux';
 import { makeSelectClaimIsNsfw, makeSelectClaimForUri } from 'lbry-redux';
+import { doRecommendationUpdate, doRecommendationClicked } from 'redux/actions/content';
 import { doFetchRecommendedContent } from 'redux/actions/search';
 import { makeSelectRecommendedContentForUri, selectIsSearching } from 'redux/selectors/search';
 import { selectUserVerifiedEmail } from 'redux/selectors/user';
@@ -17,6 +18,9 @@ const select = (state, props) => ({
 
 const perform = (dispatch) => ({
   doFetchRecommendedContent: (uri, mature) => dispatch(doFetchRecommendedContent(uri, mature)),
+  doRecommendationUpdate: (claimId, urls, id, parentId) =>
+    dispatch(doRecommendationUpdate(claimId, urls, id, parentId)),
+  doRecommendationClicked: (claimId, index) => dispatch(doRecommendationClicked(claimId, index)),
 });
 
 export default connect(select, perform)(RecommendedContent);

ui/component/recommendedContent/view.jsx

@@ -1,6 +1,8 @@
 // @flow
 import { SHOW_ADS } from 'config';
 import React from 'react';
+import { useHistory } from 'react-router-dom';
+import { v4 as uuidv4 } from 'uuid';
 import ClaimList from 'component/claimList';
 import ClaimListDiscover from 'component/claimListDiscover';
 import Ads from 'web/component/ads';
@@ -8,6 +10,7 @@ import Card from 'component/common/card';
 import { useIsMobile, useIsMediumScreen } from 'effects/use-screensize';
 import Button from 'component/button';
 import classnames from 'classnames';
+import { CONTAINER_ID } from 'constants/navigation';
 
 const VIEW_ALL_RELATED = 'view_all_related';
 const VIEW_MORE_FROM = 'view_more_from';
@@ -21,6 +24,8 @@ type Props = {
   mature: boolean,
   isAuthenticated: boolean,
   claim: ?StreamClaim,
+  doRecommendationUpdate: (claimId: string, urls: Array<string>, id: string, parentId: string) => void,
+  doRecommendationClicked: (claimId: string, index: number) => void,
 };
 
 export default React.memo<Props>(function RecommendedContent(props: Props) {
@@ -33,33 +38,70 @@ export default React.memo<Props>(function RecommendedContent(props: Props) {
     isSearching,
     isAuthenticated,
     claim,
+    doRecommendationUpdate,
+    doRecommendationClicked,
   } = props;
   const [viewMode, setViewMode] = React.useState(VIEW_ALL_RELATED);
+  const [recommendationId, setRecommendationId] = React.useState('');
+  const [recommendationUrls, setRecommendationUrls] = React.useState();
+  const history = useHistory();
   const signingChannel = claim && claim.signing_channel;
   const channelName = signingChannel ? signingChannel.name : null;
   const isMobile = useIsMobile();
   const isMedium = useIsMediumScreen();
 
-  function reorderList(recommendedContent) {
-    let newList = recommendedContent;
-    if (newList) {
-      const index = newList.indexOf(nextRecommendedUri);
-      if (index === -1) {
-        // This would be weird. Shouldn't happen since it is derived from the same list.
-      } else if (index !== 0) {
-        // Swap the "next" item to the top of the list
-        const a = newList[0];
-        newList[0] = nextRecommendedUri;
-        newList[index] = a;
+  React.useEffect(() => {
+    function moveAutoplayNextItemToTop(recommendedContent) {
+      let newList = recommendedContent;
+      if (newList) {
+        const index = newList.indexOf(nextRecommendedUri);
+        if (index > 0) {
+          const a = newList[0];
+          newList[0] = nextRecommendedUri;
+          newList[index] = a;
+        }
+      }
+      return newList;
+    }
+
+    function listEq(prev, next) {
+      if (prev && next) {
+        return prev.length === next.length && prev.every((value, index) => value === next[index]);
+      } else {
+        return prev === next;
       }
     }
-    return newList;
-  }
+
+    const newRecommendationUrls = moveAutoplayNextItemToTop(recommendedContent);
+
+    if (claim && !listEq(recommendationUrls, newRecommendationUrls)) {
+      const parentId = (history.location.state && history.location.state[CONTAINER_ID]) || '';
+      const id = uuidv4();
+      setRecommendationId(id);
+      setRecommendationUrls(newRecommendationUrls);
+
+      doRecommendationUpdate(claim.claim_id, newRecommendationUrls, id, parentId);
+    }
+  }, [
+    recommendedContent,
+    nextRecommendedUri,
+    recommendationUrls,
+    setRecommendationUrls,
+    claim,
+    doRecommendationUpdate,
+    history.location.state,
+  ]);
 
   React.useEffect(() => {
     doFetchRecommendedContent(uri, mature);
   }, [uri, mature, doFetchRecommendedContent]);
 
+  function handleRecommendationClicked(e: any, index: number) {
+    if (claim) {
+      doRecommendationClicked(claim.claim_id, index);
+    }
+  }
+
   return (
     <Card
       isBodyList
@@ -91,12 +133,14 @@ export default React.memo<Props>(function RecommendedContent(props: Props) {
         <div>
           {viewMode === VIEW_ALL_RELATED && (
             <ClaimList
+              id={recommendationId}
               type="small"
               loading={isSearching}
-              uris={reorderList(recommendedContent)}
+              uris={recommendationUrls}
               hideMenu={isMobile}
               injectedItem={SHOW_ADS && IS_WEB && !isAuthenticated && <Ads small type={'video'} />}
               empty={__('No related content found')}
+              onClick={handleRecommendationClicked}
             />
           )}
           {viewMode === VIEW_MORE_FROM && signingChannel && (

ui/component/viewers/videoViewer/internal/plugins/videojs-recsys/plugin.js

@@ -1,6 +1,12 @@
 // Created by xander on 6/21/2021
 import videojs from 'video.js';
-import { v4 as uuidV4 } from 'uuid';
+import {
+  makeSelectRecommendationId,
+  makeSelectRecommendationParentId,
+  makeSelectRecommendedClaimIds,
+  makeSelectRecommendationClicks,
+} from 'redux/selectors/content';
+
 const VERSION = '0.0.1';
 
 const recsysEndpoint = 'https://clickstream.odysee.com/log/video/view';
@@ -17,23 +23,28 @@ const RecsysData = {
 };
 
 function createRecsys(claimId, userId, events, loadedAt, isEmbed) {
-  // TODO: use a UUID generator
-  const uuid = uuidV4();
   const pageLoadedAt = loadedAt;
   const pageExitedAt = Date.now();
-  return {
-    uuid: uuid,
-    parentUuid: null,
-    uid: userId,
-    claimId: claimId,
-    pageLoadedAt: pageLoadedAt,
-    pageExitedAt: pageExitedAt,
-    recsysId: recsysId,
-    recClaimIds: null,
-    recClickedVideoIdx: null,
-    events: events,
-    isEmbed: isEmbed,
-  };
+
+  if (window.store) {
+    const state = window.store.getState();
+
+    return {
+      uuid: makeSelectRecommendationId(claimId)(state),
+      parentUuid: makeSelectRecommendationParentId(claimId)(state),
+      uid: userId,
+      claimId: claimId,
+      pageLoadedAt: pageLoadedAt,
+      pageExitedAt: pageExitedAt,
+      recsysId: recsysId,
+      recClaimIds: makeSelectRecommendedClaimIds(claimId)(state),
+      recClickedVideoIdx: makeSelectRecommendationClicks(claimId)(state),
+      events: events,
+      isEmbed: isEmbed,
+    };
+  }
+
+  return undefined;
 }
 
 function newRecsysEvent(eventType, offset, arg) {
@@ -130,7 +141,10 @@ class RecsysPlugin extends Component {
       this.loadedAt,
       false
     );
-    sendRecsysEvents(event);
+
+    if (event) {
+      sendRecsysEvents(event);
+    }
   }
 
   onPlay(event) {
@@ -226,7 +240,7 @@ const onPlayerReady = (player, options) => {
  * @function plugin
  * @param    {Object} [options={}]
  */
-const plugin = function(options) {
+const plugin = function (options) {
   this.ready(() => {
     onPlayerReady(this, videojs.mergeOptions(defaults, options));
   });

ui/constants/action_types.js

@@ -98,6 +98,8 @@ export const CLEAR_CONTENT_POSITION = 'CLEAR_CONTENT_POSITION';
 export const SET_CONTENT_LAST_VIEWED = 'SET_CONTENT_LAST_VIEWED';
 export const CLEAR_CONTENT_HISTORY_URI = 'CLEAR_CONTENT_HISTORY_URI';
 export const CLEAR_CONTENT_HISTORY_ALL = 'CLEAR_CONTENT_HISTORY_ALL';
+export const RECOMMENDATION_UPDATED = 'RECOMMENDATION_UPDATED';
+export const RECOMMENDATION_CLICKED = 'RECOMMENDATION_CLICKED';
 
 // Files
 export const FILE_LIST_STARTED = 'FILE_LIST_STARTED';

ui/constants/navigation.js

@@ -0,0 +1 @@
+export const CONTAINER_ID = 'CONTAINER_ID';

ui/redux/actions/content.js

@@ -43,7 +43,7 @@ export function doUpdateLoadStatus(uri: string, outpoint: string) {
       full_status: true,
       page: 1,
       page_size: 1,
-    }).then(result => {
+    }).then((result) => {
       const { items: fileInfos } = result;
       const fileInfo = fileInfos[0];
       if (!fileInfo || fileInfo.written_bytes === 0) {
@@ -261,3 +261,21 @@ export function doClearContentHistoryAll() {
     dispatch({ type: ACTIONS.CLEAR_CONTENT_HISTORY_ALL });
   };
 }
+
+export const doRecommendationUpdate = (claimId: string, urls: Array<string>, id: string, parentId: string) => (
+  dispatch: Dispatch
+) => {
+  dispatch({
+    type: ACTIONS.RECOMMENDATION_UPDATED,
+    data: { claimId, urls, id, parentId },
+  });
+};
+
+export const doRecommendationClicked = (claimId: string, index: number) => (dispatch: Dispatch) => {
+  if (index !== undefined && index !== null) {
+    dispatch({
+      type: ACTIONS.RECOMMENDATION_CLICKED,
+      data: { claimId, index },
+    });
+  }
+};

ui/redux/reducers/content.js

@@ -7,6 +7,10 @@ const defaultState = {
   channelClaimCounts: {},
   positions: {},
   history: [],
+  recommendationId: {}, // { "claimId": "recommendationId" }
+  recommendationParentId: {}, // { "claimId": "referrerId" }
+  recommendationUrls: {}, // { "claimId": [lbryUrls...] }
+  recommendationClicks: {}, // { "claimId": [clicked indices...] }
 };
 
 reducers[ACTIONS.SET_PRIMARY_URI] = (state, action) =>
@@ -73,7 +77,7 @@ reducers[ACTIONS.SET_CONTENT_LAST_VIEWED] = (state, action) => {
   const { uri, lastViewed } = action.data;
   const { history } = state;
   const historyObj = { uri, lastViewed };
-  const index = history.findIndex(i => i.uri === uri);
+  const index = history.findIndex((i) => i.uri === uri);
   const newHistory =
     index === -1
       ? [historyObj].concat(history)
@@ -84,7 +88,7 @@ reducers[ACTIONS.SET_CONTENT_LAST_VIEWED] = (state, action) => {
 reducers[ACTIONS.CLEAR_CONTENT_HISTORY_URI] = (state, action) => {
   const { uri } = action.data;
   const { history } = state;
-  const index = history.findIndex(i => i.uri === uri);
+  const index = history.findIndex((i) => i.uri === uri);
   return index === -1
     ? state
     : {
@@ -93,7 +97,44 @@ reducers[ACTIONS.CLEAR_CONTENT_HISTORY_URI] = (state, action) => {
       };
 };
 
-reducers[ACTIONS.CLEAR_CONTENT_HISTORY_ALL] = state => ({ ...state, history: [] });
+reducers[ACTIONS.CLEAR_CONTENT_HISTORY_ALL] = (state) => ({ ...state, history: [] });
+
+reducers[ACTIONS.RECOMMENDATION_UPDATED] = (state, action) => {
+  const { claimId, urls, id, parentId } = action.data;
+  const recommendationId = Object.assign({}, state.recommendationId);
+  const recommendationParentId = Object.assign({}, state.recommendationParentId);
+  const recommendationUrls = Object.assign({}, state.recommendationUrls);
+  const recommendationClicks = Object.assign({}, state.recommendationClicks);
+
+  if (urls && urls.length > 0) {
+    recommendationId[claimId] = id;
+    recommendationParentId[claimId] = parentId;
+    recommendationUrls[claimId] = urls;
+    recommendationClicks[claimId] = [];
+  } else {
+    delete recommendationId[claimId];
+    delete recommendationParentId[claimId];
+    delete recommendationUrls[claimId];
+    delete recommendationClicks[claimId];
+  }
+
+  return { ...state, recommendationId, recommendationParentId, recommendationUrls, recommendationClicks };
+};
+
+reducers[ACTIONS.RECOMMENDATION_CLICKED] = (state, action) => {
+  const { claimId, index } = action.data;
+  const recommendationClicks = Object.assign({}, state.recommendationClicks);
+
+  if (state.recommendationUrls[claimId] && index >= 0 && index < state.recommendationUrls[claimId].length) {
+    if (recommendationClicks[claimId]) {
+      recommendationClicks[claimId].push(index);
+    } else {
+      recommendationClicks[claimId] = [index];
+    }
+  }
+
+  return { ...state, recommendationClicks };
+};
 
 // reducers[LBRY_REDUX_ACTIONS.PURCHASE_URI_FAILED] = (state, action) => {
 //   return {

ui/redux/selectors/content.js

@@ -13,6 +13,7 @@ import {
   makeSelectFileNameForUri,
   normalizeURI,
   selectMyActiveClaims,
+  selectClaimIdsByUri,
 } from 'lbry-redux';
 import { makeSelectRecommendedContentForUri } from 'redux/selectors/search';
 import { selectMutedChannels } from 'redux/selectors/blocked';
@@ -247,19 +248,37 @@ export const makeSelectInsufficientCreditsForUri = (uri: string) =>
   );
 
 export const makeSelectSigningIsMine = (rawUri: string) => {
-    let uri;
+  let uri;
+  try {
+    uri = normalizeURI(rawUri);
+  } catch (e) {}
+
+  return createSelector(selectClaimsByUri, selectMyActiveClaims, (claims, myClaims) => {
     try {
-      uri = normalizeURI(rawUri);
-    } catch (e) { }
+      parseURI(uri);
+    } catch (e) {
+      return false;
+    }
+    const signingChannel = claims && claims[uri] && (claims[uri].signing_channel || claims[uri]);
 
-    return createSelector(selectClaimsByUri, selectMyActiveClaims, (claims, myClaims) => {
-      try {
-        parseURI(uri);
-      } catch (e) {
-        return false;
-      }
-      const signingChannel = claims && claims[uri] && (claims[uri].signing_channel || claims[uri]);
+    return signingChannel && myClaims.has(signingChannel.claim_id);
+  });
+};
 
-      return signingChannel && myClaims.has(signingChannel.claim_id);
-    });
-  };
+export const makeSelectRecommendationId = (claimId: string) =>
+  createSelector(selectState, (state) => state.recommendationId[claimId]);
+
+export const makeSelectRecommendationParentId = (claimId: string) =>
+  createSelector(selectState, (state) => state.recommendationParentId[claimId]);
+
+export const makeSelectRecommendedClaimIds = (claimId: string) =>
+  createSelector(selectState, selectClaimIdsByUri, (state, claimIdsByUri) => {
+    const recommendationUrls = state.recommendationUrls[claimId];
+    if (recommendationUrls) {
+      return recommendationUrls.map((url) => claimIdsByUri[url]);
+    }
+    return undefined;
+  });
+
+export const makeSelectRecommendationClicks = (claimId: string) =>
+  createSelector(selectState, (state) => state.recommendationClicks[claimId]);