fix large file uploads

electron/index.js

@@ -24,6 +24,8 @@ const mime = require('mime');
 const remote = require('@electron/remote/main');
 const os = require('os');
 const sudo = require('sudo-prompt');
+const probe = require('ffmpeg-probe');
+const MAX_IPC_SEND_BUFFER_SIZE = 500000000; // large files crash when serialized for ipc message
 
 remote.initialize();
 const filePath = path.join(process.resourcesPath, 'static', 'upgradeDisabled');
@@ -353,6 +355,43 @@ ipcMain.handle('get-file-from-path', (event, path, readContents = true) => {
   });
 });
 
+ipcMain.handle('get-file-details-from-path', async (event, path) => {
+    const isFfMp4 = (ffprobeResults) => {
+      return ffprobeResults &&
+      ffprobeResults.format &&
+      ffprobeResults.format.format_name &&
+      ffprobeResults.format.format_name.includes('mp4');
+    };
+    const folders = path.split(/[\\/]/);
+    const name = folders[folders.length - 1];
+    let duration = 0, size = 0, mimeType;
+    try {
+      await fs.promises.stat(path);
+      let ffprobeResults;
+      try {
+        ffprobeResults = await probe(path);
+        duration = ffprobeResults.format.duration;
+        size = ffprobeResults.format.size;
+      } catch (e) {
+      }
+      let fileReadResult;
+      if (size < MAX_IPC_SEND_BUFFER_SIZE) {
+        try {
+          fileReadResult = await fs.promises.readFile(path);
+        } catch (e) {
+
+        }
+      }
+      // TODO: use mmmagic to inspect file and get mime type
+      mimeType = isFfMp4(ffprobeResults) ? 'video/mp4' : mime.getType(name);
+      const fileData = {name, mime: mimeType || undefined, path, duration: duration, size, buffer: fileReadResult };
+      return fileData;
+    } catch (e) {
+      // no stat
+      return { error: 'no file' };
+    }
+});
+
 ipcMain.on('get-disk-space', async (event) => {
   try {
     const { data_dir } = await Lbry.settings_get();

flow-typed/file-data.js

@@ -0,0 +1,10 @@
+// @flow
+
+declare type FileData = {
+  file?: Blob,
+  path: string,
+  duration?: number,
+  size?: number,
+  mimeType: string,
+  error?: string,
+}

package.json

@@ -51,6 +51,7 @@
     "electron-notarize": "^1.0.0",
     "electron-updater": "^4.2.4",
     "express": "^4.17.1",
+    "ffmpeg-probe": "^1.0.6",
     "humanize-duration": "^3.27.0",
     "match-sorter": "^6.3.0",
     "mime": "^3.0.0",

static/app-strings.json

@@ -2321,5 +2321,6 @@
   "Autoplay Next is on.": "Autoplay Next is on.",
   "This will be visible in a few minutes after you submit this form.": "This will be visible in a few minutes after you submit this form.",
   "Anon --[used in <%anonymous% Reposted>]--": "Anon",
+  "Your update is now pending. It will take a few minutes to appear for other users.": "Your update is now pending. It will take a few minutes to appear for other users.",
   "--end--": "--end--"
 }

ui/component/publishFile/view.jsx

@@ -14,7 +14,7 @@ import I18nMessage from 'component/i18nMessage';
 import usePersistedState from 'effects/use-persisted-state';
 import * as PUBLISH_MODES from 'constants/publish_types';
 import PublishName from 'component/publishName';
-
+import path from 'path';
 type Props = {
   uri: ?string,
   mode: ?string,
@@ -99,18 +99,27 @@ function PublishFile(props: Props) {
     if (!filePath) {
       return;
     }
-    async function readSelectedFile() {
+    async function readSelectedFileDetails() {
       // Read the file to get the file's duration (if possible)
       // and offer transcoding it.
-      const readFileContents = true;
-      const result = await ipcRenderer.invoke('get-file-from-path', filePath, readFileContents);
-      const file = new File([result.buffer], result.name, {
-        type: result.mime,
-      });
-      const fileWithPath = { file, path: result.path };
-      processSelectedFile(fileWithPath);
+      const result = await ipcRenderer.invoke('get-file-details-from-path', filePath);
+      let file;
+      if (result.buffer) {
+        file = new File([result.buffer], result.name, {
+          type: result.mime,
+        });
+      }
+      const fileData: FileData = {
+        path: result.path,
+        name: result.name,
+        mimeType: result.mime || 'application/octet-stream',
+        size: result.size,
+        duration: result.duration,
+        file: file,
+      };
+      processSelectedFile(fileData);
     }
-    readSelectedFile();
+    readSelectedFileDetails();
   }, [filePath]);
 
   useEffect(() => {
@@ -219,11 +228,11 @@ function PublishFile(props: Props) {
     }
   }
 
-  function processSelectedFile(fileWithPath: FileWithPath, clearName = true) {
+  function processSelectedFile(fileData: FileData, clearName = true) {
     window.URL = window.URL || window.webkitURL;
 
     // select file, start to select a new one, then cancel
-    if (!fileWithPath) {
+    if (!fileData || fileData.error) {
       if (isStillEditing || !clearName) {
         updatePublishForm({ filePath: '' });
       } else {
@@ -233,8 +242,11 @@ function PublishFile(props: Props) {
     }
 
     // if video, extract duration so we can warn about bitrate if (typeof file !== 'string')
-    const file = fileWithPath.file;
-    const contentType = file.type && file.type.split('/');
+    const file = fileData.file;
+    // Check to see if it's a video and if mp4
+    const contentType = fileData.mimeType && fileData.mimeType.split('/'); // get this from electron side
+    const duration = fileData.duration;
+    const size = fileData.size;
     const isVideo = contentType && contentType[0] === 'video';
     const isMp4 = contentType && contentType[1] === 'mp4';
 
@@ -243,33 +255,24 @@ function PublishFile(props: Props) {
     if (contentType && contentType[0] === 'text') {
       isTextPost = contentType[1] === 'plain' || contentType[1] === 'markdown';
       setCurrentFileType(contentType.join('/'));
-    } else if (file.name) {
+    } else if (path.parse(fileData.path).ext) {
       // If user's machine is missing a valid content type registration
       // for markdown content: text/markdown, file extension will be used instead
-      const extension = file.name.split('.').pop();
+      const extension = path.parse(fileData.path).ext;
       isTextPost = MARKDOWN_FILE_EXTENSIONS.includes(extension);
     }
 
     if (isVideo) {
       if (isMp4) {
-        const video = document.createElement('video');
-        video.preload = 'metadata';
-        video.onloadedmetadata = () => {
-          updateFileInfo(video.duration, file.size, isVideo);
-          window.URL.revokeObjectURL(video.src);
-        };
-        video.onerror = () => {
-          updateFileInfo(0, file.size, isVideo);
-        };
-        video.src = window.URL.createObjectURL(file);
+        updateFileInfo(duration || 0, size, isVideo);
       } else {
-        updateFileInfo(0, file.size, isVideo);
+        updateFileInfo(duration || 0, size, isVideo);
       }
     } else {
-      updateFileInfo(0, file.size, isVideo);
+      updateFileInfo(0, size, isVideo);
     }
 
-    if (isTextPost) {
+    if (isTextPost && file) {
       // Create reader
       const reader = new FileReader();
       // Handler for file reader
@@ -283,7 +286,7 @@ function PublishFile(props: Props) {
 
     // Strip off extension and replace invalid characters
     if (!isStillEditing) {
-      const fileWithoutExtension = name || (file.name && file.name.substring(0, file.name.lastIndexOf('.'))) || '';
+      const fileWithoutExtension = path.parse(fileData.path).name;
       updatePublishForm({ name: parseName(fileWithoutExtension) });
     }
   }

webpack.base.config.js

@@ -95,7 +95,7 @@ let baseConfig = {
   },
 
   plugins: [
-    new webpack.IgnorePlugin(/^\.\/locale$/, /moment$/),
+    new webpack.IgnorePlugin({resourceRegExp: /^\.\/locale$/, contextRegExp: /moment$/}),
     new webpack.EnvironmentPlugin(['NODE_ENV']),
     new DefinePlugin({
       __static: `"${path.join(__dirname, 'static').replace(/\\/g, '\\\\')}"`,

yarn.lock

@@ -7994,6 +7994,21 @@ __metadata:
   languageName: node
   linkType: hard
 
+"execa@npm:^0.10.0":
+  version: 0.10.0
+  resolution: "execa@npm:0.10.0"
+  dependencies:
+    cross-spawn: ^6.0.0
+    get-stream: ^3.0.0
+    is-stream: ^1.1.0
+    npm-run-path: ^2.0.0
+    p-finally: ^1.0.0
+    signal-exit: ^3.0.0
+    strip-eof: ^1.0.0
+  checksum: da132af2b209e69d79f91751ac6d15ddbb8d9414f9e5f7a53405232679a3dca00fe11eb14e0cd5c2c374a749061410a7717fcc3094f6dd779cf4d259faa58d9a
+  languageName: node
+  linkType: hard
+
 "execa@npm:^0.7.0":
   version: 0.7.0
   resolution: "execa@npm:0.7.0"
@@ -8308,6 +8323,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"ffmpeg-probe@npm:^1.0.6":
+  version: 1.0.6
+  resolution: "ffmpeg-probe@npm:1.0.6"
+  dependencies:
+    execa: ^0.10.0
+  checksum: fe649b2ca41bd48b521d7cc5741663d4c608d7bc596033ee9c76d4c3f5e739881a4d421bdcfa3ea60e28301eae7a85b72cd74d6266e661bccf9aea6578fcfe3c
+  languageName: node
+  linkType: hard
+
 "figgy-pudding@npm:^3.5.1":
   version: 3.5.2
   resolution: "figgy-pudding@npm:3.5.2"
@@ -11450,6 +11474,7 @@ __metadata:
     eslint-plugin-react-hooks: ^1.6.0
     eslint-plugin-standard: ^4.0.1
     express: ^4.17.1
+    ffmpeg-probe: ^1.0.6
     file-loader: ^4.2.0
     flow-bin: ^0.97.0
     flow-typed: ^3.7.0
@@ -17641,7 +17666,20 @@ __metadata:
   languageName: node
   linkType: hard
 
-"terser@npm:^4.1.2, terser@npm:^4.6.12, terser@npm:^4.6.3":
+"terser@npm:^4.1.2":
+  version: 4.8.1
+  resolution: "terser@npm:4.8.1"
+  dependencies:
+    commander: ^2.20.0
+    source-map: ~0.6.1
+    source-map-support: ~0.5.12
+  bin:
+    terser: bin/terser
+  checksum: b342819bf7e82283059aaa3f22bb74deb1862d07573ba5a8947882190ad525fd9b44a15074986be083fd379c58b9a879457a330b66dcdb77b485c44267f9a55a
+  languageName: node
+  linkType: hard
+
+"terser@npm:^4.6.12, terser@npm:^4.6.3":
   version: 4.8.0
   resolution: "terser@npm:4.8.0"
   dependencies: