diff --git a/.gitignore b/.gitignore index 1008c14..bf23b05 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ LBRY\ 3D\ Viewer/Assets/Editor/LBRY/lbry-format/*.meta -LBRY\ 3D\ Viewer/Assets/Editor/LBRY/lbry-format/package-lock.json -LBRY\ 3D\ Viewer/Assets/Editor/LBRY/lbry-format/node_modules +#LBRY\ 3D\ Viewer/Assets/Editor/LBRY/lbry-format~/ LBRY\ 3D\ Viewer/LBRY/ LBRY\ 3D\ Viewer/Logs/ diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/EditorSDK.cs b/LBRY 3D Viewer/Assets/Editor/LBRY/EditorSDK.cs index d7a6c52..93d7a4b 100644 --- a/LBRY 3D Viewer/Assets/Editor/LBRY/EditorSDK.cs +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/EditorSDK.cs @@ -66,10 +66,19 @@ public class LBRY var packageScript = "\"" + Path.Combine(Path.GetDirectoryName(unityEditorLbryPath), "package.js") + "\""; UnityEngine.Debug.Log("packageScript: " + packageScript); + var lbryFormatDir = Path.Combine(Path.GetDirectoryName(unityEditorLbryPath), "lbry-format~/"); + + // Copy `lbry-format` to `lbry-format~` (Unity special folder) + if(!Directory.Exists(lbryFormatDir)) { + var srcLbryFormatDir = Path.Combine(Path.GetDirectoryName(unityEditorLbryPath), "lbry-format/"); + UnityEngine.Debug.LogWarning("Creating `lbry-format~` (Unity ignored)"); + FileUtil.CopyFileOrDirectory(srcLbryFormatDir, lbryFormatDir); + } + + // Install `lbry-format` dependencies var lbryFormatModulesDir = Path.Combine(Path.GetDirectoryName(unityEditorLbryPath), "lbry-format~/node_modules/"); if(!Directory.Exists(lbryFormatModulesDir)) { UnityEngine.Debug.LogWarning("'lbry-format' modules missing, running `npm i`"); - var lbryFormatDir = Path.Combine(Path.GetDirectoryName(unityEditorLbryPath), "lbry-format~/"); UnityEngine.Debug.Log(Path.GetDirectoryName(lbryFormatDir)); ExecProcessShell(Path.GetDirectoryName(lbryFormatDir), "npm", "i"); } diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format.meta b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format.meta new file mode 100644 index 0000000..e2b54eb --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: e7d0f7cce256e4c4b8d4ba853e3dd43b +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/.gitignore b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/.gitignore new file mode 100644 index 0000000..ad46b30 --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/.gitignore @@ -0,0 +1,61 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# TypeScript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + +# next.js build output +.next diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/LICENSE b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/LICENSE new file mode 100644 index 0000000..e129ec1 --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 LBRY + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/README.md b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/README.md new file mode 100644 index 0000000..6b225b5 --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/README.md @@ -0,0 +1,32 @@ +**PROPOSAL STAGE** + +# LBRY File Format + +The LBRY format is a multimedia-format designed to promote interoperability and transmission of complex digital media. + +## Name + + - Name Format: `*.lbry` + - Character Set: `UTF-8` + - Extension: `lbry` + +## Metadata + + - Media Type: `application/x-lbry` + +## Container + + - Compression: `Zstandard` + - Archival: `tar` + +## Contents + +### Descriptor + + - Format: `JSON`, `UTF-8` + +# lbry-format + +## Requirements + + - Node.js v11.6.0 (see https://nodejs.org/docs/v11.6.0/api/esm.html) diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/index.js b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/index.js new file mode 100644 index 0000000..dd49362 --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/index.js @@ -0,0 +1,193 @@ +const fs = require('fs'); +const path = require('path'); +const tar = require('tar-stream'); +const tarPack = tar.pack(); +const ZstdCodec = require('zstd-codec').ZstdCodec; +const util = require('util'); + +const COMPRESSION_LEVEL = 5; + +// async readdir +const readdir = async (path, options) => { + return new Promise((resolve) => { + fs.readdir(path, options, (err, files) => { + if(err) { + throw err; + } + resolve(files); + }) + }) +}; + +// async readFile +const readFile = util.promisify(fs.readFile); + +function generateFirstEntry(options) { + return '{}'; +} + +function writeFirstEntry(options, tarPack) { + tarPack.entry({ name: '.' }, generateFirstEntry(options), (err) => { + if(err) { + throw err; + } + }); +} + +function getFileReadStream(options) { + const fileName = options.fileName || 'package.lbry'; + return fs.createReadStream(fileName); +} + +function getFileWriteStream(options) { + const fileName = options.fileName || 'package.lbry'; + return fs.createWriteStream(fileName); +} + +async function getZstd() { + return new Promise((resolve, reject) => { + try { + ZstdCodec.run(zstd => { + const Streaming = new zstd.Streaming(); + resolve(Streaming); + }); + } catch(e) { + reject(e); + } + }) +} + +async function walkAndRun(runCommand, dir, root) { + let files = await readdir(dir); + + for(let file of files) { + const currentPath = path.join(dir, file); + + if (fs.statSync(currentPath).isDirectory()) { + walkAndRun(runCommand, currentPath); + } else { + runCommand(currentPath); + } + } +}; + +async function packDirectory(directory, options = {}) { + const packRoot = directory; + const fileWriteStream = getFileWriteStream(options); + const zstd = await getZstd(); + + tarPack.pipe(fileWriteStream); + writeFirstEntry(options, tarPack); + + walkAndRun(async (file) => { + let contents = await readFile(file); + + contents = new Uint8Array(contents); + + // Must be chunked to avoid issues with fixed memory limits. + const chunkIterator = (() => { + const chunkSize = 4096; + let position = 0; + + const iterator = { + next: function() { + const endIndex = position + chunkSize; + const result = { + value: contents.slice(position, endIndex), + done: endIndex >= contents.length, + }; + + position = endIndex; + return result; + }, + [Symbol.iterator]: function() { return this } + }; + return iterator; + })(); + + contents = zstd.compressChunks(chunkIterator, contents.length, COMPRESSION_LEVEL); + + let name = path.relative(packRoot, file); + + if(/^\.\//.test(name)) { + name = name.slice(2); + } + + const entry = tarPack.entry({ name, size: contents.length }, (err) => { + if(err) { + throw err; + } + }); + + entry.end(contents); + }, directory, packRoot); +}; + +async function unpackDirectory(directory, options = {}) { + if(!fs.existsSync(directory)) { + fs.mkdirSync(directory); + } + + const fileReadStream = getFileReadStream(options); + const zstd = await getZstd(); + + const extract = tar.extract(); + + extract.on('entry', async (header, fileStream, next) => { + fileStream.on('end', () => { + next(); + }) + + if(!/^\./.test(header.name)) { + const writePath = path.join(directory, header.name); + await fs.promises.mkdir(path.dirname(writePath), { recursive: true }); + var fileWriteStream = fs.createWriteStream(writePath); + fileStream.pipe(fileWriteStream); + } else { + // Just drain it + fileStream.resume(); + } + }); + + extract.on('finish', () => { + // all entries read + }); + + fileReadStream.pipe(extract); +} + +/* +// DO NOT USE until converted to use `compressChunks` +async function packPaths(root, pathsArray, options = {}) { + const fileWriteStream = getFileWriteStream(options); + const zstd = await getZstd(); + + tarPack.pipe(fileWriteStream); + writeFirstEntry(options, tarPack); + + for(let name of pathsArray) { + let contents = await readFile(path.join(root, name)); + + contents = new Uint8Array(contents); + contents = zstd.compress(contents, COMPRESSION_LEVEL); + + if(/^\.\//.test(name)) { + name = name.slice(2); + } + + const entry = tarPack.entry({ name, size: contents.length }, (err) => { + if(err) { + throw err; + } + }); + + entry.end(contents); + } + tarPack.finalize(); +} +*/ + +module.exports = { + packDirectory, + unpackDirectory, +} diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/package-lock.json b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/package-lock.json new file mode 100644 index 0000000..a181634 --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/package-lock.json @@ -0,0 +1,143 @@ +{ + "name": "lbry-format", + "version": "0.1.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "bl": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", + "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", + "requires": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, + "buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "requires": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==" + }, + "buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=" + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "end-of-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", + "requires": { + "once": "^1.4.0" + } + }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "tar-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", + "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "requires": { + "bl": "^1.0.0", + "buffer-alloc": "^1.2.0", + "end-of-stream": "^1.0.0", + "fs-constants": "^1.0.0", + "readable-stream": "^2.3.0", + "to-buffer": "^1.1.1", + "xtend": "^4.0.0" + } + }, + "to-buffer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", + "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==" + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "xtend": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" + }, + "zstd-codec": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.1.tgz", + "integrity": "sha512-9u1cihqoDPaq9Tbxneom8rleCSfz1hxXzz+RmydjOVYnlqkzoMqOpAdJ06Y/RgctUfdyuazXT2uyZMhlPb/x0Q==" + } + } +} diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/package.json b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/package.json new file mode 100644 index 0000000..cb4c656 --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format/package.json @@ -0,0 +1,15 @@ +{ + "name": "lbry-format", + "version": "0.1.0", + "description": "LBRY format toolkit", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "MIT", + "dependencies": { + "tar-stream": "^1.6.2", + "zstd-codec": "^0.1.1" + } +} diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/LICENSE.meta b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/LICENSE.meta new file mode 100644 index 0000000..8b1a712 --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/LICENSE.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: f357a175befb70a4aa717a35f7741bce +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/README.md.meta b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/README.md.meta new file mode 100644 index 0000000..278ab8a --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/README.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: fcabe74365e40da4d8729bd8f05ef840 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/index.js.meta b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/index.js.meta new file mode 100644 index 0000000..0047a6b --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/index.js.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 15bacabb4818f2c4aa0fd9bff3df0493 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/package-lock.json.meta b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/package-lock.json.meta new file mode 100644 index 0000000..c22506e --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/package-lock.json.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 708ff209a6af8b744b7322c353e15106 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/package.json.meta b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/package.json.meta new file mode 100644 index 0000000..87b67c8 --- /dev/null +++ b/LBRY 3D Viewer/Assets/Editor/LBRY/lbry-format~/package.json.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: ef9d146443d8d9a4d8290a4c6e4019ae +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: