Add pack/unpack functionality

README.md

@@ -23,4 +23,10 @@ The LBRY format is a multimedia-format designed to promote interoperability and
 
 ### Descriptor
 
- - Format: `JSON`, `UTF-8`
+ - Format: `JSON`, `UTF-8`
+
+# lbry-format
+
+## Requirements
+
+ - Node.js v11.6.0 (see https://nodejs.org/docs/v11.6.0/api/esm.html)

index.js

@@ -0,0 +1,193 @@
+const fs = require('fs');
+const path = require('path');
+const tar = require('tar-stream');
+const tarPack = tar.pack();
+const ZstdCodec = require('zstd-codec').ZstdCodec;
+const util = require('util');
+
+const COMPRESSION_LEVEL = 5;
+
+// async readdir
+const readdir = async (path, options) => {
+  return new Promise((resolve) => {
+    fs.readdir(path, options, (err, files) => {
+      if(err) {
+        throw err;
+      }
+      resolve(files);
+    })
+  })
+};
+
+// async readFile
+const readFile = util.promisify(fs.readFile);
+
+function generateFirstEntry(options) {
+  return '{}';
+}
+
+function writeFirstEntry(options, tarPack) {
+  tarPack.entry({ name: '.' }, generateFirstEntry(options), (err) => {
+    if(err) {
+      throw err;
+    }
+  });
+}
+
+function getFileReadStream(options) {
+  const fileName = options.fileName || 'package.lbry';
+  return fs.createReadStream(fileName);
+}
+
+function getFileWriteStream(options) {
+  const fileName = options.fileName || 'package.lbry';
+  return fs.createWriteStream(fileName);
+}
+
+async function getZstd() {
+  return new Promise((resolve, reject) => {
+    try {
+      ZstdCodec.run(zstd => {
+        const Streaming = new zstd.Streaming();
+        resolve(Streaming);
+      });
+    } catch(e) {
+      reject(e);
+    }
+  })
+}
+
+async function walkAndRun(runCommand, dir, root) {
+  let files = await readdir(dir);
+
+  for(let file of files) {
+    const currentPath = path.join(dir, file);
+
+    if (fs.statSync(currentPath).isDirectory()) {
+      walkAndRun(runCommand, currentPath);
+    } else {
+      runCommand(currentPath);
+    }
+  }
+};
+
+async function packDirectory(directory, options = {}) {
+  const packRoot = directory;
+  const fileWriteStream = getFileWriteStream(options);
+  const zstd = await getZstd();
+
+  tarPack.pipe(fileWriteStream);
+  writeFirstEntry(options, tarPack);
+
+  walkAndRun(async (file) => {
+    let contents = await readFile(file);
+
+    contents = new Uint8Array(contents);
+
+    // Must be chunked to avoid issues with fixed memory limits.
+    const chunkIterator = (() => {
+      const chunkSize = 4096;
+      let position = 0;
+
+      const iterator = {
+        next: function() {
+          const endIndex = position + chunkSize;
+          const result = {
+            value: contents.slice(position, endIndex),
+            done: endIndex >= contents.length,
+          };
+
+          position = endIndex;
+          return result;
+        },
+        [Symbol.iterator]: function() { return this }
+      };
+      return iterator;
+    })();
+
+    contents = zstd.compressChunks(chunkIterator, contents.length, COMPRESSION_LEVEL);
+
+    let name = path.relative(packRoot, file);
+
+    if(/^\.\//.test(name)) {
+      name = name.slice(2);
+    }
+
+    const entry = tarPack.entry({ name, size: contents.length }, (err) => {
+      if(err) {
+        throw err;
+      }
+    });
+
+    entry.end(contents);
+  }, directory, packRoot);
+};
+
+async function unpackDirectory(directory, options = {}) {
+  if(!fs.existsSync(directory)) {
+    fs.mkdirSync(directory);
+  }
+
+  const fileReadStream = getFileReadStream(options);
+  const zstd = await getZstd();
+
+  const extract = tar.extract();
+
+  extract.on('entry', async (header, fileStream, next) => {
+    fileStream.on('end', () => {
+      next();
+    })
+
+    if(!/^\./.test(header.name)) {
+      const writePath = path.join(directory, header.name);
+      await fs.promises.mkdir(path.dirname(writePath), { recursive: true });
+      var fileWriteStream = fs.createWriteStream(writePath);
+      fileStream.pipe(fileWriteStream);
+    } else {
+      // Just drain it
+      fileStream.resume();
+    }
+  });
+
+  extract.on('finish', () => {
+    // all entries read
+  });
+
+  fileReadStream.pipe(extract);
+}
+
+/*
+// DO NOT USE until converted to use `compressChunks`
+async function packPaths(root, pathsArray, options = {}) {
+  const fileWriteStream = getFileWriteStream(options);
+  const zstd = await getZstd();
+
+  tarPack.pipe(fileWriteStream);
+  writeFirstEntry(options, tarPack);
+
+  for(let name of pathsArray) {
+    let contents = await readFile(path.join(root, name));
+
+    contents = new Uint8Array(contents);
+    contents = zstd.compress(contents, COMPRESSION_LEVEL);
+
+    if(/^\.\//.test(name)) {
+      name = name.slice(2);
+    }
+
+    const entry = tarPack.entry({ name, size: contents.length }, (err) => {
+      if(err) {
+        throw err;
+      }
+    });
+
+    entry.end(contents);
+  }
+  tarPack.finalize();
+}
+*/
+
+module.exports = {
+  packDirectory,
+  unpackDirectory,
+}

package-lock.json

@@ -4,6 +4,136 @@
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
+    "bl": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz",
+      "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==",
+      "requires": {
+        "readable-stream": "^2.3.5",
+        "safe-buffer": "^5.1.1"
+      }
+    },
+    "buffer-alloc": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz",
+      "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==",
+      "requires": {
+        "buffer-alloc-unsafe": "^1.1.0",
+        "buffer-fill": "^1.0.0"
+      }
+    },
+    "buffer-alloc-unsafe": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz",
+      "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg=="
+    },
+    "buffer-fill": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
+      "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw="
+    },
+    "core-util-is": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+    },
+    "end-of-stream": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
+      "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
+      "requires": {
+        "once": "^1.4.0"
+      }
+    },
+    "fs-constants": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
+    },
+    "inherits": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
+    },
+    "isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+    },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "requires": {
+        "wrappy": "1"
+      }
+    },
+    "process-nextick-args": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
+      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
+    },
+    "readable-stream": {
+      "version": "2.3.6",
+      "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
+      "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
+      "requires": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+    },
+    "string_decoder": {
+      "version": "1.1.1",
+      "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "requires": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
+    "tar-stream": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz",
+      "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==",
+      "requires": {
+        "bl": "^1.0.0",
+        "buffer-alloc": "^1.2.0",
+        "end-of-stream": "^1.0.0",
+        "fs-constants": "^1.0.0",
+        "readable-stream": "^2.3.0",
+        "to-buffer": "^1.1.1",
+        "xtend": "^4.0.0"
+      }
+    },
+    "to-buffer": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz",
+      "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg=="
+    },
+    "util-deprecate": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
+    },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
+    },
+    "xtend": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
+      "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
+    },
     "zstd-codec": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.1.tgz",

package.json

@@ -9,6 +9,7 @@
   "author": "",
   "license": "MIT",
   "dependencies": {
+    "tar-stream": "^1.6.2",
     "zstd-codec": "^0.1.1"
   }
 }