Add pack/unpack functionality
This commit is contained in:
Shawn
parent
a4a1149fbb
commit
9587051289
4 changed files with 331 additions and 1 deletions
|
|
@ -23,4 +23,10 @@ The LBRY format is a multimedia-format designed to promote interoperability and
|
|||
|
||||
### Descriptor
|
||||
|
||||
- Format: `JSON`, `UTF-8`
|
||||
- Format: `JSON`, `UTF-8`
|
||||
|
||||
# lbry-format
|
||||
|
||||
## Requirements
|
||||
|
||||
- Node.js v11.6.0 (see https://nodejs.org/docs/v11.6.0/api/esm.html)
|
||||
|
|
|
|||
193
index.js
Normal file
193
index.js
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const tar = require('tar-stream');
|
||||
const tarPack = tar.pack();
|
||||
const ZstdCodec = require('zstd-codec').ZstdCodec;
|
||||
const util = require('util');
|
||||
|
||||
const COMPRESSION_LEVEL = 5;
|
||||
|
||||
// async readdir
|
||||
const readdir = async (path, options) => {
|
||||
return new Promise((resolve) => {
|
||||
fs.readdir(path, options, (err, files) => {
|
||||
if(err) {
|
||||
throw err;
|
||||
}
|
||||
resolve(files);
|
||||
})
|
||||
})
|
||||
};
|
||||
|
||||
// async readFile
|
||||
const readFile = util.promisify(fs.readFile);
|
||||
|
||||
function generateFirstEntry(options) {
|
||||
return '{}';
|
||||
}
|
||||
|
||||
function writeFirstEntry(options, tarPack) {
|
||||
tarPack.entry({ name: '.' }, generateFirstEntry(options), (err) => {
|
||||
if(err) {
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function getFileReadStream(options) {
|
||||
const fileName = options.fileName || 'package.lbry';
|
||||
return fs.createReadStream(fileName);
|
||||
}
|
||||
|
||||
function getFileWriteStream(options) {
|
||||
const fileName = options.fileName || 'package.lbry';
|
||||
return fs.createWriteStream(fileName);
|
||||
}
|
||||
|
||||
async function getZstd() {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
ZstdCodec.run(zstd => {
|
||||
const Streaming = new zstd.Streaming();
|
||||
resolve(Streaming);
|
||||
});
|
||||
} catch(e) {
|
||||
reject(e);
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function walkAndRun(runCommand, dir, root) {
|
||||
let files = await readdir(dir);
|
||||
|
||||
for(let file of files) {
|
||||
const currentPath = path.join(dir, file);
|
||||
|
||||
if (fs.statSync(currentPath).isDirectory()) {
|
||||
walkAndRun(runCommand, currentPath);
|
||||
} else {
|
||||
runCommand(currentPath);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
async function packDirectory(directory, options = {}) {
|
||||
const packRoot = directory;
|
||||
const fileWriteStream = getFileWriteStream(options);
|
||||
const zstd = await getZstd();
|
||||
|
||||
tarPack.pipe(fileWriteStream);
|
||||
writeFirstEntry(options, tarPack);
|
||||
|
||||
walkAndRun(async (file) => {
|
||||
let contents = await readFile(file);
|
||||
|
||||
contents = new Uint8Array(contents);
|
||||
|
||||
// Must be chunked to avoid issues with fixed memory limits.
|
||||
const chunkIterator = (() => {
|
||||
const chunkSize = 4096;
|
||||
let position = 0;
|
||||
|
||||
const iterator = {
|
||||
next: function() {
|
||||
const endIndex = position + chunkSize;
|
||||
const result = {
|
||||
value: contents.slice(position, endIndex),
|
||||
done: endIndex >= contents.length,
|
||||
};
|
||||
|
||||
position = endIndex;
|
||||
return result;
|
||||
},
|
||||
[Symbol.iterator]: function() { return this }
|
||||
};
|
||||
return iterator;
|
||||
})();
|
||||
|
||||
contents = zstd.compressChunks(chunkIterator, contents.length, COMPRESSION_LEVEL);
|
||||
|
||||
let name = path.relative(packRoot, file);
|
||||
|
||||
if(/^\.\//.test(name)) {
|
||||
name = name.slice(2);
|
||||
}
|
||||
|
||||
const entry = tarPack.entry({ name, size: contents.length }, (err) => {
|
||||
if(err) {
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
entry.end(contents);
|
||||
}, directory, packRoot);
|
||||
};
|
||||
|
||||
async function unpackDirectory(directory, options = {}) {
|
||||
if(!fs.existsSync(directory)) {
|
||||
fs.mkdirSync(directory);
|
||||
}
|
||||
|
||||
const fileReadStream = getFileReadStream(options);
|
||||
const zstd = await getZstd();
|
||||
|
||||
const extract = tar.extract();
|
||||
|
||||
extract.on('entry', async (header, fileStream, next) => {
|
||||
fileStream.on('end', () => {
|
||||
next();
|
||||
})
|
||||
|
||||
if(!/^\./.test(header.name)) {
|
||||
const writePath = path.join(directory, header.name);
|
||||
await fs.promises.mkdir(path.dirname(writePath), { recursive: true });
|
||||
var fileWriteStream = fs.createWriteStream(writePath);
|
||||
fileStream.pipe(fileWriteStream);
|
||||
} else {
|
||||
// Just drain it
|
||||
fileStream.resume();
|
||||
}
|
||||
});
|
||||
|
||||
extract.on('finish', () => {
|
||||
// all entries read
|
||||
});
|
||||
|
||||
fileReadStream.pipe(extract);
|
||||
}
|
||||
|
||||
/*
|
||||
// DO NOT USE until converted to use `compressChunks`
|
||||
async function packPaths(root, pathsArray, options = {}) {
|
||||
const fileWriteStream = getFileWriteStream(options);
|
||||
const zstd = await getZstd();
|
||||
|
||||
tarPack.pipe(fileWriteStream);
|
||||
writeFirstEntry(options, tarPack);
|
||||
|
||||
for(let name of pathsArray) {
|
||||
let contents = await readFile(path.join(root, name));
|
||||
|
||||
contents = new Uint8Array(contents);
|
||||
contents = zstd.compress(contents, COMPRESSION_LEVEL);
|
||||
|
||||
if(/^\.\//.test(name)) {
|
||||
name = name.slice(2);
|
||||
}
|
||||
|
||||
const entry = tarPack.entry({ name, size: contents.length }, (err) => {
|
||||
if(err) {
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
entry.end(contents);
|
||||
}
|
||||
tarPack.finalize();
|
||||
}
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
packDirectory,
|
||||
unpackDirectory,
|
||||
}
|
||||
130
package-lock.json
generated
130
package-lock.json
generated
|
|
@ -4,6 +4,136 @@
|
|||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"bl": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz",
|
||||
"integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==",
|
||||
"requires": {
|
||||
"readable-stream": "^2.3.5",
|
||||
"safe-buffer": "^5.1.1"
|
||||
}
|
||||
},
|
||||
"buffer-alloc": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz",
|
||||
"integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==",
|
||||
"requires": {
|
||||
"buffer-alloc-unsafe": "^1.1.0",
|
||||
"buffer-fill": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"buffer-alloc-unsafe": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz",
|
||||
"integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg=="
|
||||
},
|
||||
"buffer-fill": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
|
||||
"integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw="
|
||||
},
|
||||
"core-util-is": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
|
||||
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
|
||||
},
|
||||
"end-of-stream": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
|
||||
"integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
|
||||
"requires": {
|
||||
"once": "^1.4.0"
|
||||
}
|
||||
},
|
||||
"fs-constants": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
|
||||
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
|
||||
},
|
||||
"inherits": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
|
||||
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
|
||||
},
|
||||
"isarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
|
||||
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
|
||||
},
|
||||
"once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
|
||||
"requires": {
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"process-nextick-args": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
|
||||
"integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "2.3.6",
|
||||
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
|
||||
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
|
||||
"requires": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.3",
|
||||
"isarray": "~1.0.0",
|
||||
"process-nextick-args": "~2.0.0",
|
||||
"safe-buffer": "~5.1.1",
|
||||
"string_decoder": "~1.1.1",
|
||||
"util-deprecate": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
|
||||
"requires": {
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
},
|
||||
"tar-stream": {
|
||||
"version": "1.6.2",
|
||||
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz",
|
||||
"integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==",
|
||||
"requires": {
|
||||
"bl": "^1.0.0",
|
||||
"buffer-alloc": "^1.2.0",
|
||||
"end-of-stream": "^1.0.0",
|
||||
"fs-constants": "^1.0.0",
|
||||
"readable-stream": "^2.3.0",
|
||||
"to-buffer": "^1.1.1",
|
||||
"xtend": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"to-buffer": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz",
|
||||
"integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg=="
|
||||
},
|
||||
"util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
|
||||
},
|
||||
"wrappy": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
|
||||
},
|
||||
"xtend": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
|
||||
"integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
|
||||
},
|
||||
"zstd-codec": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.1.tgz",
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
"author": "",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tar-stream": "^1.6.2",
|
||||
"zstd-codec": "^0.1.1"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue