diff --git a/examples/simple-pack.js b/examples/simple-pack.js new file mode 100644 index 0000000..ef9a614 --- /dev/null +++ b/examples/simple-pack.js @@ -0,0 +1,6 @@ +const lbryFormat = require('../index'); + +// should be run from package root directory +lbryFormat.packDirectory('./examples/src', { + fileName: './examples/simple-pack.lbry', +}); diff --git a/examples/src/img_mountains_wide.jpg b/examples/src/img_mountains_wide.jpg new file mode 100644 index 0000000..f93f45c Binary files /dev/null and b/examples/src/img_mountains_wide.jpg differ diff --git a/examples/src/img_nature_wide.jpg b/examples/src/img_nature_wide.jpg new file mode 100644 index 0000000..b3f3d4c Binary files /dev/null and b/examples/src/img_nature_wide.jpg differ diff --git a/examples/src/polar/img_lights_wide.jpg b/examples/src/polar/img_lights_wide.jpg new file mode 100644 index 0000000..b08788c Binary files /dev/null and b/examples/src/polar/img_lights_wide.jpg differ diff --git a/examples/src/polar/img_snow_wide.jpg b/examples/src/polar/img_snow_wide.jpg new file mode 100644 index 0000000..bab2280 Binary files /dev/null and b/examples/src/polar/img_snow_wide.jpg differ diff --git a/examples/template-pack.js b/examples/template-pack.js new file mode 100644 index 0000000..29b6a59 --- /dev/null +++ b/examples/template-pack.js @@ -0,0 +1,7 @@ +const lbryFormat = require('../index'); + +// should be run from package root directory +lbryFormat.packDirectory('./examples/src', { + fileName: './examples/template-pack.lbry', + useTemplate: true, +}); diff --git a/index.js b/index.js index 037aed2..e6baddc 100644 --- a/index.js +++ b/index.js @@ -1,17 +1,33 @@ -const fs = require('fs'); -const path = require('path'); -const tar = require('tar-stream'); +const fs = require("fs"); +const path = require("path"); +const tar = require("tar-stream"); const tarPack = tar.pack(); -const ZstdCodec = require('zstd-codec').ZstdCodec; -const util = require('util'); +const ZstdCodec = require("zstd-codec").ZstdCodec; +const util = require("util"); const COMPRESSION_LEVEL = 5; +const SUPPORTED_FORMATS = [ + [/\.(jpg|png|gif|svg|bmp)$/i, "images"], + [/\.(mp4|m4v|mkv|webm|flv|f4v|ogv)$/i, "videos"], + [/\.(mp3|m4a|aac|wav|flac|ogg|opus)$/i, "audios"] +]; + +function getMediaType(fileName) { + const res = SUPPORTED_FORMATS.reduce((ret, testpair) => { + const [regex, mediaType] = testpair; + + return regex.test(ret) ? mediaType : ret; + }, fileName); + + return res !== fileName ? res : "others"; +} + function mkdirSyncRecursive(dir) { let segments = dir.split(path.sep); for (let i = 1; i <= segments.length; i++) { - let segment = segments.slice(0, i).join('/'); + let segment = segments.slice(0, i).join("/"); if (segment.length > 0 && !fs.existsSync(segment)) { fs.mkdirSync(segment); } @@ -20,38 +36,38 @@ function mkdirSyncRecursive(dir) { // async readdir const readdir = async (path, options) => { - return new Promise((resolve) => { + return new Promise(resolve => { fs.readdir(path, options, (err, files) => { - if(err) { + if (err) { throw err; } resolve(files); - }) - }) + }); + }); }; // async readFile const readFile = util.promisify(fs.readFile); function generateFirstEntry(options) { - return '{}'; + return "{}"; } function writeFirstEntry(options, tarPack) { - tarPack.entry({ name: '.' }, generateFirstEntry(options), (err) => { - if(err) { + tarPack.entry({ name: "." }, generateFirstEntry(options), err => { + if (err) { throw err; } }); } function getFileReadStream(options) { - const fileName = options.fileName || 'package.lbry'; + const fileName = options.fileName || "package.lbry"; return fs.createReadStream(fileName); } function getFileWriteStream(options) { - const fileName = options.fileName || 'package.lbry'; + const fileName = options.fileName || "package.lbry"; return fs.createWriteStream(fileName); } @@ -62,34 +78,69 @@ async function getZstd() { const Streaming = new zstd.Streaming(); resolve(Streaming); }); - } catch(e) { + } catch (e) { reject(e); } - }) + }); +} + +async function catagoriseFilesIndex(runCommand, root, index) { + const files = { images: "", audios: "", videos: "", others: "" }; + + for (let file of index) { + const mediaType = getMediaType(path.basename(file)); + files[mediaType] += `"${file}",`; + } + + let contents = ""; + for (let mediaType in files) { + contents += `${mediaType}=[${files[mediaType]}];\n`; + } + + const bufferedContents = Buffer.from(contents); + const relativeFilePath = path.join(root, "files.js"); + await runCommand(relativeFilePath, bufferedContents); +} + +async function processTemplate(runCommand, root, userDefinedTemplate) { + const template = userDefinedTemplate + ? userDefinedTemplate + : require("./templates").default; + + for (const resource of template.resources) { + if (resource.type === "static") { + const bufferedContents = Buffer.from(resource.contents); + const relativeFilePath = path.join(root, resource.name); + await runCommand(relativeFilePath, bufferedContents); + } + } } async function walkAndRun(runCommand, dir, root) { let files = await readdir(dir); - for(let file of files) { + for (let file of files) { const currentPath = path.join(dir, file); if (fs.statSync(currentPath).isDirectory()) { await walkAndRun(runCommand, currentPath); } else { - await runCommand(currentPath); + const fileContents = await readFile(path.normalize(currentPath)); + await runCommand(currentPath, fileContents); } } -}; +} async function writeStream(stream, data) { - return new Promise((resolve) => { + return new Promise(resolve => { stream.write(data); - stream.end(resolve) + stream.end(resolve); }); } async function packDirectory(directory, options = {}) { + let filesIndex = []; + const zstd = await getZstd(); const packRoot = directory; const fileWriteStream = getFileWriteStream(options); @@ -97,53 +148,58 @@ async function packDirectory(directory, options = {}) { tarPack.pipe(fileWriteStream); writeFirstEntry(options, tarPack); - await walkAndRun(async (file) => { - contents = await readFile(path.normalize(file)); + const makeChunkIterator = contents => { + const chunkSize = 2048; + let position = 0; - // Must be chunked to avoid issues with fixed memory limits. - const chunkIterator = (() => { - const chunkSize = 2048; - let position = 0; + const iterator = { + next: function() { + const endIndex = position + chunkSize; + const result = { + value: contents.slice(position, endIndex), + done: position >= contents.length + }; - const iterator = { - next: function() { - const endIndex = position + chunkSize; - const result = { - value: contents.slice(position, endIndex), - done: position >= contents.length, - }; + position = endIndex; + return result; + }, + [Symbol.iterator]: function() { + return this; + } + }; + return iterator; + }; - position = endIndex; - return result; - }, - [Symbol.iterator]: function() { return this } - }; - return iterator; - })(); - - contents = zstd.compressChunks(chunkIterator, contents.length, COMPRESSION_LEVEL); - - let name = path.relative(packRoot, file).replace('\\', '/'); - - if(/^\.\//.test(name)) { + const createEntry = async (file, contents) => { + const chunkIterator = makeChunkIterator(contents); + contents = zstd.compressChunks( + chunkIterator, + contents.length, + COMPRESSION_LEVEL + ); + let name = path.relative(packRoot, file).replace("\\", "/"); + if (/^\.\//.test(name)) { name = name.slice(2); } - - const entry = tarPack.entry({ name, size: contents.length }, (err) => { - if(err) { + const entry = tarPack.entry({ name, size: contents.length }, err => { + if (err) { throw err; } }); - await writeStream(entry, contents); - entry.end(); - }, directory, packRoot); + filesIndex.push(name); + }; + + await walkAndRun(createEntry, directory, packRoot); + if (options.useTemplate === true) { + await processTemplate(createEntry, packRoot); + await catagoriseFilesIndex(createEntry, packRoot, filesIndex); + } tarPack.finalize(); -}; +} - -function strToBuffer (string) { +function strToBuffer(string) { let arrayBuffer = new ArrayBuffer(string.length * 1); let newUint = new Uint8Array(arrayBuffer); newUint.forEach((_, i) => { @@ -153,18 +209,17 @@ function strToBuffer (string) { } function streamToBuffer(stream) { - const chunks = [] + const chunks = []; return new Promise((resolve, reject) => { - stream.on('data', chunk => chunks.push(chunk)) - stream.on('error', reject) - stream.on('end', () => resolve(Buffer.concat(chunks))) - }) + stream.on("data", chunk => chunks.push(chunk)); + stream.on("error", reject); + stream.on("end", () => resolve(Buffer.concat(chunks))); + }); } - async function unpackDirectory(directory, options = {}) { - return new Promise(async (resolve) => { - if(!fs.existsSync(directory)) { + return new Promise(async resolve => { + if (!fs.existsSync(directory)) { mkdirSyncRecursive(directory); } @@ -173,7 +228,7 @@ async function unpackDirectory(directory, options = {}) { const extract = tar.extract(); - extract.on('entry', async (header, fileStream, next) => { + extract.on("entry", async (header, fileStream, next) => { let contents = await streamToBuffer(fileStream); contents = new Uint8Array(contents); @@ -187,20 +242,22 @@ async function unpackDirectory(directory, options = {}) { const endIndex = position + chunkSize; const result = { value: contents.slice(position, endIndex), - done: position >= contents.length, + done: position >= contents.length }; position = endIndex; return result; }, - [Symbol.iterator]: function() { return this } + [Symbol.iterator]: function() { + return this; + } }; return iterator; })(); contents = zstd.decompressChunks(chunkIterator); - if(!/^\./.test(header.name)) { + if (!/^\./.test(header.name)) { const writePath = path.join(directory, header.name); try { @@ -219,7 +276,7 @@ async function unpackDirectory(directory, options = {}) { } }); - extract.on('finish', () => { + extract.on("finish", () => { resolve(true); }); @@ -260,5 +317,5 @@ async function packPaths(root, pathsArray, options = {}) { module.exports = { packDirectory, - unpackDirectory, -} + unpackDirectory +}; diff --git a/lbry-format.js b/lbry-format.js index 5d0ef6b..c68e153 100644 --- a/lbry-format.js +++ b/lbry-format.js @@ -1,51 +1,65 @@ #!/usr/bin/env node -const { - packDirectory, - unpackDirectory, -} = require('lbry-format'); -const path = require('path'); +const { packDirectory, unpackDirectory } = require("lbry-format"); +const path = require("path"); -require('yargs') - .scriptName('lbry-pack') - .usage('$0 [args]') - .command('pack [directory] [file]', 'Pack a directory', (yargs) => { - yargs.positional('directory', { - default: './src', - describe: 'The source directory to pack' - }).positional('file', { - describe: 'Output file', - default: './package.lbry', - }) - }, function (argv) { - console.log(`Packing ${argv.directory} to ${argv.file}`); +require("yargs") + .scriptName("lbry-pack") + .usage("$0 [args]") + .command( + "pack [directory] [file] [-t]", + "Pack a directory", + yargs => { + yargs + .positional("directory", { + default: "./src", + describe: "The source directory to pack" + }) + .positional("file", { + describe: "Output file", + default: "./package.lbry" + }) + .option("t", { + alias: "template", + describe: "Use web template" + }); + }, + function(argv) { + console.log(`Packing ${argv.directory} to ${argv.file}`); - const resolvedDirectory = path.resolve(argv.directory); - const resolvedfile = path.resolve(argv.file); + const resolvedDirectory = path.resolve(argv.directory); + const resolvedfile = path.resolve(argv.file); - packDirectory(resolvedDirectory, { - fileName: resolvedfile, - }); - }) - .command('unpack [directory] [file]', 'Unpack a file', (yargs) => { - yargs.positional('destination', { - type: 'string', - default: './src', - describe: 'The folder destination to unpack to' - }).positional('file', { - describe: 'Input filename', - default: 'package.lbry', - }) - }, function (argv) { - console.log(`Packing ${argv.directory} to ${argv.file}`); + packDirectory(resolvedDirectory, { + fileName: resolvedfile, + useTemplate: argv.template + }); + } + ) + .command( + "unpack [directory] [file]", + "Unpack a file", + yargs => { + yargs + .positional("destination", { + type: "string", + default: "./src", + describe: "The folder destination to unpack to" + }) + .positional("file", { + describe: "Input filename", + default: "package.lbry" + }); + }, + function(argv) { + console.log(`Packing ${argv.directory} to ${argv.file}`); + const resolvedDirectory = path.resolve(argv.directory); + const resolvedfile = path.resolve(argv.file); - const resolvedDirectory = path.resolve(argv.directory); - const resolvedfile = path.resolve(argv.file); - - unpackDirectory(resolvedDirectory, { - fileName: resolvedfile, - }); - }) + unpackDirectory(resolvedDirectory, { + fileName: resolvedfile + }); + } + ) .help() - .demandCommand() - .argv + .demandCommand().argv; diff --git a/templates.js b/templates.js new file mode 100644 index 0000000..0beb13b --- /dev/null +++ b/templates.js @@ -0,0 +1,25 @@ +module.exports = { + default: { + resources: [ + { + type: "static", + name: "index.html", + contents: + 'Gallery Slideshow' + }, + { + type: "static", + name: "index.css", + contents: + "body,html{width:100%;height:100%;display:flex;background-color:#001}*{margin:0;padding:0}.slideshow-container{margin:auto;background-color:#000;display:flex;height:80%;width:80%;justify-content:space-between;align-items:center;border-radius:3px 3px 3px 3px}.mySlides{order:1;margin:auto;max-width:100%;max-height:100%}.next,.prev{order:0;cursor:pointer;width:auto;padding:16px;color:#fff;font-weight:700;font-size:18px;transition:.6s ease;border-radius:0 3px 3px 0;user-select:none}.next{order:2;border-radius:3px 0 0 3px}.next:hover,.prev:hover{background-color:#eee;color:#000}.fade{-webkit-animation-name:fade;-webkit-animation-duration:1s;animation-name:fade;animation-duration:1s}@-webkit-keyframes fade{from{opacity:.4}to{opacity:1}}@keyframes fade{from{opacity:.4}to{opacity:1}}@media only screen and (max-width:300px){.next,.prev,.text{font-size:11px}}" + }, + { + type: "static", + name: "index.js", + contents: + 'const container=document.getElementById("container");for(let e=0;en.length&&(slideIndex=1),e<1&&(slideIndex=n.length);for(let e=0;e