Add pack/unpack functionality

This commit is contained in:
Shawn 2019-01-06 15:41:49 -06:00
parent a4a1149fbb
commit 9587051289
4 changed files with 331 additions and 1 deletions

View file

@ -24,3 +24,9 @@ The LBRY format is a multimedia-format designed to promote interoperability and
### Descriptor
- Format: `JSON`, `UTF-8`
# lbry-format
## Requirements
- Node.js v11.6.0 (see https://nodejs.org/docs/v11.6.0/api/esm.html)

193
index.js Normal file
View file

@ -0,0 +1,193 @@
const fs = require('fs');
const path = require('path');
const tar = require('tar-stream');
const tarPack = tar.pack();
const ZstdCodec = require('zstd-codec').ZstdCodec;
const util = require('util');
const COMPRESSION_LEVEL = 5;
// async readdir
const readdir = async (path, options) => {
return new Promise((resolve) => {
fs.readdir(path, options, (err, files) => {
if(err) {
throw err;
}
resolve(files);
})
})
};
// async readFile
const readFile = util.promisify(fs.readFile);
function generateFirstEntry(options) {
return '{}';
}
function writeFirstEntry(options, tarPack) {
tarPack.entry({ name: '.' }, generateFirstEntry(options), (err) => {
if(err) {
throw err;
}
});
}
function getFileReadStream(options) {
const fileName = options.fileName || 'package.lbry';
return fs.createReadStream(fileName);
}
function getFileWriteStream(options) {
const fileName = options.fileName || 'package.lbry';
return fs.createWriteStream(fileName);
}
async function getZstd() {
return new Promise((resolve, reject) => {
try {
ZstdCodec.run(zstd => {
const Streaming = new zstd.Streaming();
resolve(Streaming);
});
} catch(e) {
reject(e);
}
})
}
async function walkAndRun(runCommand, dir, root) {
let files = await readdir(dir);
for(let file of files) {
const currentPath = path.join(dir, file);
if (fs.statSync(currentPath).isDirectory()) {
walkAndRun(runCommand, currentPath);
} else {
runCommand(currentPath);
}
}
};
async function packDirectory(directory, options = {}) {
const packRoot = directory;
const fileWriteStream = getFileWriteStream(options);
const zstd = await getZstd();
tarPack.pipe(fileWriteStream);
writeFirstEntry(options, tarPack);
walkAndRun(async (file) => {
let contents = await readFile(file);
contents = new Uint8Array(contents);
// Must be chunked to avoid issues with fixed memory limits.
const chunkIterator = (() => {
const chunkSize = 4096;
let position = 0;
const iterator = {
next: function() {
const endIndex = position + chunkSize;
const result = {
value: contents.slice(position, endIndex),
done: endIndex >= contents.length,
};
position = endIndex;
return result;
},
[Symbol.iterator]: function() { return this }
};
return iterator;
})();
contents = zstd.compressChunks(chunkIterator, contents.length, COMPRESSION_LEVEL);
let name = path.relative(packRoot, file);
if(/^\.\//.test(name)) {
name = name.slice(2);
}
const entry = tarPack.entry({ name, size: contents.length }, (err) => {
if(err) {
throw err;
}
});
entry.end(contents);
}, directory, packRoot);
};
async function unpackDirectory(directory, options = {}) {
if(!fs.existsSync(directory)) {
fs.mkdirSync(directory);
}
const fileReadStream = getFileReadStream(options);
const zstd = await getZstd();
const extract = tar.extract();
extract.on('entry', async (header, fileStream, next) => {
fileStream.on('end', () => {
next();
})
if(!/^\./.test(header.name)) {
const writePath = path.join(directory, header.name);
await fs.promises.mkdir(path.dirname(writePath), { recursive: true });
var fileWriteStream = fs.createWriteStream(writePath);
fileStream.pipe(fileWriteStream);
} else {
// Just drain it
fileStream.resume();
}
});
extract.on('finish', () => {
// all entries read
});
fileReadStream.pipe(extract);
}
/*
// DO NOT USE until converted to use `compressChunks`
async function packPaths(root, pathsArray, options = {}) {
const fileWriteStream = getFileWriteStream(options);
const zstd = await getZstd();
tarPack.pipe(fileWriteStream);
writeFirstEntry(options, tarPack);
for(let name of pathsArray) {
let contents = await readFile(path.join(root, name));
contents = new Uint8Array(contents);
contents = zstd.compress(contents, COMPRESSION_LEVEL);
if(/^\.\//.test(name)) {
name = name.slice(2);
}
const entry = tarPack.entry({ name, size: contents.length }, (err) => {
if(err) {
throw err;
}
});
entry.end(contents);
}
tarPack.finalize();
}
*/
module.exports = {
packDirectory,
unpackDirectory,
}

130
package-lock.json generated
View file

@ -4,6 +4,136 @@
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"bl": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz",
"integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==",
"requires": {
"readable-stream": "^2.3.5",
"safe-buffer": "^5.1.1"
}
},
"buffer-alloc": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz",
"integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==",
"requires": {
"buffer-alloc-unsafe": "^1.1.0",
"buffer-fill": "^1.0.0"
}
},
"buffer-alloc-unsafe": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz",
"integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg=="
},
"buffer-fill": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
"integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw="
},
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"end-of-stream": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
"integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
"requires": {
"once": "^1.4.0"
}
},
"fs-constants": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
},
"inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
},
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"requires": {
"wrappy": "1"
}
},
"process-nextick-args": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
"integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
},
"readable-stream": {
"version": "2.3.6",
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"string_decoder": {
"version": "1.1.1",
"resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
},
"tar-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz",
"integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==",
"requires": {
"bl": "^1.0.0",
"buffer-alloc": "^1.2.0",
"end-of-stream": "^1.0.0",
"fs-constants": "^1.0.0",
"readable-stream": "^2.3.0",
"to-buffer": "^1.1.1",
"xtend": "^4.0.0"
}
},
"to-buffer": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz",
"integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg=="
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"xtend": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
"integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
},
"zstd-codec": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.1.tgz",

View file

@ -9,6 +9,7 @@
"author": "",
"license": "MIT",
"dependencies": {
"tar-stream": "^1.6.2",
"zstd-codec": "^0.1.1"
}
}