fix large file uploads (#7736)

* fix large file uploads * changelog * update github action xcode version * update electronbuilder for macos * try use_hard_links=false * no USE_HARD_LINKS * upgrate electron-builder 23_3_3 * revert to electron-builder 22_10_5 electron-builder/issues/6124 says regressions happen after this version. * try mac install homebrew, python2 * typo and ln /usr/bin/python * oops * try sudo * try PYTHON_PATH * comment github action mac python hack
2022-11-17 13:20:01 -05:00 · 2022-11-17 13:20:01 -05:00 · 88ac250fee
commit 88ac250fee
parent 0a5e9e87ed
9 changed files with 184 additions and 51 deletions
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@ -38,7 +38,22 @@ jobs:
      - uses: maxim-lobanov/setup-xcode@v1
        if: startsWith(runner.os, 'mac')
        with:
-          xcode-version: '12.4.0'
+          xcode-version: '13.1.0'
+      # This is gonna be hacky.
+      # Github made us upgrade xcode, which would force an upgrade of electron-builder to fix mac.
+      # But there were bugs with copyfiles / extraFiles that kept seeing duplicates erroring on ln.
+      # A flag USE_HARD_LINKS=false in electron-builder.json was suggested in comments, but that broke windows builds.
+      # So for now we'll install python2 on mac and make sure it can find it.
+      # Remove this after successfully upgrading electron-builder.
+      # HACK part 1
+      - uses: Homebrew/actions/setup-homebrew@master
+        if: startsWith(runner.os, 'mac')
+      # HACK part 2
+      - name: Install Python2
+        if: startsWith(runner.os, 'mac')
+        run: |
+          /bin/bash -c "$(curl -fsSL https://github.com/alfredapp/dependency-scripts/raw/main/scripts/install-python2.sh)"
+          echo "PYTHON_PATH=/usr/local/bin/python" >> $GITHUB_ENV

      - name: Download blockchain headers
        run: |
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -2,6 +2,10 @@
 All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

+## [unreleased] - [2022-11-10]
+ 
+### Fixed
+ - Selecting a large file in publish no longer crashes ([#7736](https://github.com/lbryio/lbry-desktop/pull/7736))
 
 ## [0.53.7] - [2022-11-10]

@ -15,7 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Minor layout fixes _community pr!_ ([#7709](https://github.com/lbryio/lbry-desktop/pull/7709))
 - Comment section buttons layout ([#7716](https://github.com/lbryio/lbry-desktop/pull/7716))
 
- ### Changed
+### Changed
 - Removed watchman and its errors ([#7710](https://github.com/lbryio/lbry-desktop/pull/7710))
 - Updated lbrynet to [0.112.0](https://github.com/lbryio/lbry-sdk/releases/tag/v0.112.0)
  
--- a/electron/index.js
+++ b/electron/index.js
@ -24,6 +24,8 @@ const mime = require('mime');
 const remote = require('@electron/remote/main');
 const os = require('os');
 const sudo = require('sudo-prompt');
+const probe = require('ffmpeg-probe');
+const MAX_IPC_SEND_BUFFER_SIZE = 500000000; // large files crash when serialized for ipc message

 remote.initialize();
 const filePath = path.join(process.resourcesPath, 'static', 'upgradeDisabled');
@ -353,6 +355,43 @@ ipcMain.handle('get-file-from-path', (event, path, readContents = true) => {
  });
 });

+ipcMain.handle('get-file-details-from-path', async (event, path) => {
+    const isFfMp4 = (ffprobeResults) => {
+      return ffprobeResults &&
+      ffprobeResults.format &&
+      ffprobeResults.format.format_name &&
+      ffprobeResults.format.format_name.includes('mp4');
+    };
+    const folders = path.split(/[\\/]/);
+    const name = folders[folders.length - 1];
+    let duration = 0, size = 0, mimeType;
+    try {
+      await fs.promises.stat(path);
+      let ffprobeResults;
+      try {
+        ffprobeResults = await probe(path);
+        duration = ffprobeResults.format.duration;
+        size = ffprobeResults.format.size;
+      } catch (e) {
+      }
+      let fileReadResult;
+      if (size < MAX_IPC_SEND_BUFFER_SIZE) {
+        try {
+          fileReadResult = await fs.promises.readFile(path);
+        } catch (e) {
+
+        }
+      }
+      // TODO: use mmmagic to inspect file and get mime type
+      mimeType = isFfMp4(ffprobeResults) ? 'video/mp4' : mime.getType(name);
+      const fileData = {name, mime: mimeType || undefined, path, duration: duration, size, buffer: fileReadResult };
+      return fileData;
+    } catch (e) {
+      // no stat
+      return { error: 'no file' };
+    }
+});
+
 ipcMain.on('get-disk-space', async (event) => {
  try {
    const { data_dir } = await Lbry.settings_get();
--- a/flow-typed/file-data.js
+++ b/flow-typed/file-data.js
@ -0,0 +1,10 @@
+// @flow
+
+declare type FileData = {
+  file?: Blob,
+  path: string,
+  duration?: number,
+  size?: number,
+  mimeType: string,
+  error?: string,
+}
--- a/package.json
+++ b/package.json
@ -51,6 +51,7 @@
    "electron-notarize": "^1.0.0",
    "electron-updater": "^4.2.4",
    "express": "^4.17.1",
+    "ffmpeg-probe": "^1.0.6",
    "humanize-duration": "^3.27.0",
    "match-sorter": "^6.3.0",
    "mime": "^3.0.0",
--- a/static/app-strings.json
+++ b/static/app-strings.json
@ -2321,5 +2321,6 @@
  "Autoplay Next is on.": "Autoplay Next is on.",
  "This will be visible in a few minutes after you submit this form.": "This will be visible in a few minutes after you submit this form.",
  "Anon --[used in <%anonymous% Reposted>]--": "Anon",
+  "Your update is now pending. It will take a few minutes to appear for other users.": "Your update is now pending. It will take a few minutes to appear for other users.",
  "--end--": "--end--"
 }
--- a/ui/component/publishFile/view.jsx
+++ b/ui/component/publishFile/view.jsx
@ -14,7 +14,7 @@ import I18nMessage from 'component/i18nMessage';
 import usePersistedState from 'effects/use-persisted-state';
 import * as PUBLISH_MODES from 'constants/publish_types';
 import PublishName from 'component/publishName';
-
+import path from 'path';
 type Props = {
  uri: ?string,
  mode: ?string,
@ -99,18 +99,27 @@ function PublishFile(props: Props) {
    if (!filePath) {
      return;
    }
-    async function readSelectedFile() {
+    async function readSelectedFileDetails() {
      // Read the file to get the file's duration (if possible)
      // and offer transcoding it.
-      const readFileContents = true;
-      const result = await ipcRenderer.invoke('get-file-from-path', filePath, readFileContents);
-      const file = new File([result.buffer], result.name, {
+      const result = await ipcRenderer.invoke('get-file-details-from-path', filePath);
+      let file;
+      if (result.buffer) {
+        file = new File([result.buffer], result.name, {
          type: result.mime,
        });
-      const fileWithPath = { file, path: result.path };
-      processSelectedFile(fileWithPath);
      }
-    readSelectedFile();
+      const fileData: FileData = {
+        path: result.path,
+        name: result.name,
+        mimeType: result.mime || 'application/octet-stream',
+        size: result.size,
+        duration: result.duration,
+        file: file,
+      };
+      processSelectedFile(fileData);
+    }
+    readSelectedFileDetails();
  }, [filePath]);

  useEffect(() => {
@ -219,11 +228,11 @@ function PublishFile(props: Props) {
    }
  }

-  function processSelectedFile(fileWithPath: FileWithPath, clearName = true) {
+  function processSelectedFile(fileData: FileData, clearName = true) {
    window.URL = window.URL || window.webkitURL;

    // select file, start to select a new one, then cancel
-    if (!fileWithPath) {
+    if (!fileData || fileData.error) {
      if (isStillEditing || !clearName) {
        updatePublishForm({ filePath: '' });
      } else {
@ -233,8 +242,11 @@ function PublishFile(props: Props) {
    }

    // if video, extract duration so we can warn about bitrate if (typeof file !== 'string')
-    const file = fileWithPath.file;
-    const contentType = file.type && file.type.split('/');
+    const file = fileData.file;
+    // Check to see if it's a video and if mp4
+    const contentType = fileData.mimeType && fileData.mimeType.split('/'); // get this from electron side
+    const duration = fileData.duration;
+    const size = fileData.size;
    const isVideo = contentType && contentType[0] === 'video';
    const isMp4 = contentType && contentType[1] === 'mp4';

@ -243,33 +255,24 @@ function PublishFile(props: Props) {
    if (contentType && contentType[0] === 'text') {
      isTextPost = contentType[1] === 'plain' || contentType[1] === 'markdown';
      setCurrentFileType(contentType.join('/'));
-    } else if (file.name) {
+    } else if (path.parse(fileData.path).ext) {
      // If user's machine is missing a valid content type registration
      // for markdown content: text/markdown, file extension will be used instead
-      const extension = file.name.split('.').pop();
+      const extension = path.parse(fileData.path).ext;
      isTextPost = MARKDOWN_FILE_EXTENSIONS.includes(extension);
    }

    if (isVideo) {
      if (isMp4) {
-        const video = document.createElement('video');
-        video.preload = 'metadata';
-        video.onloadedmetadata = () => {
-          updateFileInfo(video.duration, file.size, isVideo);
-          window.URL.revokeObjectURL(video.src);
-        };
-        video.onerror = () => {
-          updateFileInfo(0, file.size, isVideo);
-        };
-        video.src = window.URL.createObjectURL(file);
+        updateFileInfo(duration || 0, size, isVideo);
      } else {
-        updateFileInfo(0, file.size, isVideo);
+        updateFileInfo(duration || 0, size, isVideo);
      }
    } else {
-      updateFileInfo(0, file.size, isVideo);
+      updateFileInfo(0, size, isVideo);
    }

-    if (isTextPost) {
+    if (isTextPost && file) {
      // Create reader
      const reader = new FileReader();
      // Handler for file reader
@ -283,7 +286,7 @@ function PublishFile(props: Props) {

    // Strip off extension and replace invalid characters
    if (!isStillEditing) {
-      const fileWithoutExtension = name || (file.name && file.name.substring(0, file.name.lastIndexOf('.'))) || '';
+      const fileWithoutExtension = path.parse(fileData.path).name;
      updatePublishForm({ name: parseName(fileWithoutExtension) });
    }
  }
--- a/webpack.base.config.js
+++ b/webpack.base.config.js
@ -95,7 +95,7 @@ let baseConfig = {
  },

  plugins: [
-    new webpack.IgnorePlugin(/^\.\/locale$/, /moment$/),
+    new webpack.IgnorePlugin({resourceRegExp: /^\.\/locale$/, contextRegExp: /moment$/}),
    new webpack.EnvironmentPlugin(['NODE_ENV']),
    new DefinePlugin({
      __static: `"${path.join(__dirname, 'static').replace(/\\/g, '\\\\')}"`,
--- a/yarn.lock
+++ b/yarn.lock
@ -3857,8 +3857,8 @@ __metadata:
  linkType: hard

 "asar@npm:^3.0.3":
-  version: 3.1.0
-  resolution: "asar@npm:3.1.0"
+  version: 3.2.0
+  resolution: "asar@npm:3.2.0"
  dependencies:
    "@types/glob": ^7.1.1
    chromium-pickle-js: ^0.2.0
@ -3870,7 +3870,7 @@ __metadata:
      optional: true
  bin:
    asar: bin/asar.js
-  checksum: facc80845639fa4f9e1d1aa40b96adbd1e8b6fee0725d287e8c8e30a69b235cd5b7131b7b09ff700da06c919dd0595b373e372c55722808f983fdb71ef0d5399
+  checksum: f7d30b45970b053252ac124230bf319459d0728d7f6dedbe2f765cd2a83792d5a716d2c3f2861ceda69372b401f335e1f46460335169eadd0e91a0904a4f5a15
  languageName: node
  linkType: hard

@ -5315,6 +5315,17 @@ __metadata:
  languageName: node
  linkType: hard

+"cliui@npm:^8.0.1":
+  version: 8.0.1
+  resolution: "cliui@npm:8.0.1"
+  dependencies:
+    string-width: ^4.2.0
+    strip-ansi: ^6.0.1
+    wrap-ansi: ^7.0.0
+  checksum: 79648b3b0045f2e285b76fb2e24e207c6db44323581e421c3acbd0e86454cba1b37aea976ab50195a49e7384b871e6dfb2247ad7dec53c02454ac6497394cb56
+  languageName: node
+  linkType: hard
+
 "clone-deep@npm:^4.0.1":
  version: 4.0.1
  resolution: "clone-deep@npm:4.0.1"
@ -7994,6 +8005,21 @@ __metadata:
  languageName: node
  linkType: hard

+"execa@npm:^0.10.0":
+  version: 0.10.0
+  resolution: "execa@npm:0.10.0"
+  dependencies:
+    cross-spawn: ^6.0.0
+    get-stream: ^3.0.0
+    is-stream: ^1.1.0
+    npm-run-path: ^2.0.0
+    p-finally: ^1.0.0
+    signal-exit: ^3.0.0
+    strip-eof: ^1.0.0
+  checksum: da132af2b209e69d79f91751ac6d15ddbb8d9414f9e5f7a53405232679a3dca00fe11eb14e0cd5c2c374a749061410a7717fcc3094f6dd779cf4d259faa58d9a
+  languageName: node
+  linkType: hard
+
 "execa@npm:^0.7.0":
  version: 0.7.0
  resolution: "execa@npm:0.7.0"
@ -8308,6 +8334,15 @@ __metadata:
  languageName: node
  linkType: hard

+"ffmpeg-probe@npm:^1.0.6":
+  version: 1.0.6
+  resolution: "ffmpeg-probe@npm:1.0.6"
+  dependencies:
+    execa: ^0.10.0
+  checksum: fe649b2ca41bd48b521d7cc5741663d4c608d7bc596033ee9c76d4c3f5e739881a4d421bdcfa3ea60e28301eae7a85b72cd74d6266e661bccf9aea6578fcfe3c
+  languageName: node
+  linkType: hard
+
 "figgy-pudding@npm:^3.5.1":
  version: 3.5.2
  resolution: "figgy-pudding@npm:3.5.2"
@ -9074,11 +9109,11 @@ __metadata:
  linkType: hard

 "global-dirs@npm:^3.0.0":
-  version: 3.0.0
-  resolution: "global-dirs@npm:3.0.0"
+  version: 3.0.1
+  resolution: "global-dirs@npm:3.0.1"
  dependencies:
    ini: 2.0.0
-  checksum: 953c17cf14bf6ee0e2100ae82a0d779934eed8a3ec5c94a7a4f37c5b3b592c31ea015fb9a15cf32484de13c79f4a814f3015152f3e1d65976cfbe47c1bfe4a88
+  checksum: 70147b80261601fd40ac02a104581432325c1c47329706acd773f3a6ce99bb36d1d996038c85ccacd482ad22258ec233c586b6a91535b1a116b89663d49d6438
  languageName: node
  linkType: hard

@ -11450,6 +11485,7 @@ __metadata:
    eslint-plugin-react-hooks: ^1.6.0
    eslint-plugin-standard: ^4.0.1
    express: ^4.17.1
+    ffmpeg-probe: ^1.0.6
    file-loader: ^4.2.0
    flow-bin: ^0.97.0
    flow-typed: ^3.7.0
@ -16390,7 +16426,7 @@ __metadata:
  languageName: node
  linkType: hard

-"semver@npm:^7.1.3, semver@npm:^7.2.1, semver@npm:^7.3.2, semver@npm:^7.3.4, semver@npm:^7.3.5":
+"semver@npm:^7.1.3, semver@npm:^7.2.1, semver@npm:^7.3.2, semver@npm:^7.3.5":
  version: 7.3.7
  resolution: "semver@npm:7.3.7"
  dependencies:
@ -16401,6 +16437,17 @@ __metadata:
  languageName: node
  linkType: hard

+"semver@npm:^7.3.4":
+  version: 7.3.8
+  resolution: "semver@npm:7.3.8"
+  dependencies:
+    lru-cache: ^6.0.0
+  bin:
+    semver: bin/semver.js
+  checksum: ba9c7cbbf2b7884696523450a61fee1a09930d888b7a8d7579025ad93d459b2d1949ee5bbfeb188b2be5f4ac163544c5e98491ad6152df34154feebc2cc337c1
+  languageName: node
+  linkType: hard
+
 "send@npm:0.18.0":
  version: 0.18.0
  resolution: "send@npm:0.18.0"
@ -17641,7 +17688,20 @@ __metadata:
  languageName: node
  linkType: hard

-"terser@npm:^4.1.2, terser@npm:^4.6.12, terser@npm:^4.6.3":
+"terser@npm:^4.1.2":
+  version: 4.8.1
+  resolution: "terser@npm:4.8.1"
+  dependencies:
+    commander: ^2.20.0
+    source-map: ~0.6.1
+    source-map-support: ~0.5.12
+  bin:
+    terser: bin/terser
+  checksum: b342819bf7e82283059aaa3f22bb74deb1862d07573ba5a8947882190ad525fd9b44a15074986be083fd379c58b9a879457a330b66dcdb77b485c44267f9a55a
+  languageName: node
+  linkType: hard
+
+"terser@npm:^4.6.12, terser@npm:^4.6.3":
  version: 4.8.0
  resolution: "terser@npm:4.8.0"
  dependencies:
@ -19501,10 +19561,10 @@ __metadata:
  languageName: node
  linkType: hard

-"yargs-parser@npm:^21.0.0":
-  version: 21.0.1
-  resolution: "yargs-parser@npm:21.0.1"
-  checksum: c3ea2ed12cad0377ce3096b3f138df8267edf7b1aa7d710cd502fe16af417bafe4443dd71b28158c22fcd1be5dfd0e86319597e47badf42ff83815485887323a
+"yargs-parser@npm:^21.1.1":
+  version: 21.1.1
+  resolution: "yargs-parser@npm:21.1.1"
+  checksum: ed2d96a616a9e3e1cc7d204c62ecc61f7aaab633dcbfab2c6df50f7f87b393993fe6640d017759fe112d0cb1e0119f2b4150a87305cc873fd90831c6a58ccf1c
  languageName: node
  linkType: hard

@ -19561,17 +19621,17 @@ __metadata:
  linkType: hard

 "yargs@npm:^17.0.1":
-  version: 17.5.1
-  resolution: "yargs@npm:17.5.1"
+  version: 17.6.2
+  resolution: "yargs@npm:17.6.2"
  dependencies:
-    cliui: ^7.0.2
+    cliui: ^8.0.1
    escalade: ^3.1.1
    get-caller-file: ^2.0.5
    require-directory: ^2.1.1
    string-width: ^4.2.3
    y18n: ^5.0.5
-    yargs-parser: ^21.0.0
-  checksum: 00d58a2c052937fa044834313f07910fd0a115dec5ee35919e857eeee3736b21a4eafa8264535800ba8bac312991ce785ecb8a51f4d2cc8c4676d865af1cfbde
+    yargs-parser: ^21.1.1
+  checksum: 47da1b0d854fa16d45a3ded57b716b013b2179022352a5f7467409da5a04a1eef5b3b3d97a2dfc13e8bbe5f2ffc0afe3bc6a4a72f8254e60f5a4bd7947138643
  languageName: node
  linkType: hard