LBRYCommunity/lbry-format
1
0
Fork 0
Code Issues 1 Pull requests 7 Packages Projects Releases Wiki Activity

add option to pack template

Browse source
This commit is contained in:
zxawry 2019-06-23 20:58:00 +01:00
parent 9fd036d494
commit 896b3bae3c
No known key found for this signature in database
GPG key ID: 70F5D1B4F51F051A
3 changed files with 212 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

201
index.js
View file

@ -1,17 +1,33 @@
const fs = require('fs');
const path = require('path');
const tar = require('tar-stream');
const fs = require("fs");
const path = require("path");
const tar = require("tar-stream");
const tarPack = tar.pack();
const ZstdCodec = require('zstd-codec').ZstdCodec;
const util = require('util');
const ZstdCodec = require("zstd-codec").ZstdCodec;
const util = require("util");
const COMPRESSION_LEVEL = 5;
const SUPPORTED_FORMATS = [
[/\.(jpg|png|gif|svg|bmp)$/i, "images"],
[/\.(mp4|m4v|mkv|webm|flv|f4v|ogv)$/i, "videos"],
[/\.(mp3|m4a|aac|wav|flac|ogg|opus)$/i, "audios"]
];
function getMediaType(fileName) {
const res = SUPPORTED_FORMATS.reduce((ret, testpair) => {
const [regex, mediaType] = testpair;
return regex.test(ret) ? mediaType : ret;
}, fileName);
return res !== fileName ? res : "others";
}
function mkdirSyncRecursive(dir) {
let segments = dir.split(path.sep);
for (let i = 1; i <= segments.length; i++) {
let segment = segments.slice(0, i).join('/');
let segment = segments.slice(0, i).join("/");
if (segment.length > 0 && !fs.existsSync(segment)) {
fs.mkdirSync(segment);
}
@ -20,38 +36,38 @@ function mkdirSyncRecursive(dir) {
// async readdir
const readdir = async (path, options) => {
return new Promise((resolve) => {
return new Promise(resolve => {
fs.readdir(path, options, (err, files) => {
if(err) {
if (err) {
throw err;
}
resolve(files);
})
})
});
});
};
// async readFile
const readFile = util.promisify(fs.readFile);
function generateFirstEntry(options) {
return '{}';
return "{}";
}
function writeFirstEntry(options, tarPack) {
tarPack.entry({ name: '.' }, generateFirstEntry(options), (err) => {
if(err) {
tarPack.entry({ name: "." }, generateFirstEntry(options), err => {
if (err) {
throw err;
}
});
}
function getFileReadStream(options) {
const fileName = options.fileName || 'package.lbry';
const fileName = options.fileName || "package.lbry";
return fs.createReadStream(fileName);
}
function getFileWriteStream(options) {
const fileName = options.fileName || 'package.lbry';
const fileName = options.fileName || "package.lbry";
return fs.createWriteStream(fileName);
}
@ -62,34 +78,69 @@ async function getZstd() {
const Streaming = new zstd.Streaming();
resolve(Streaming);
});
} catch(e) {
} catch (e) {
reject(e);
}
})
});
}
async function catagoriseFilesIndex(runCommand, root, index) {
const files = { images: "", audios: "", videos: "", others: "" };
for (let file of index) {
const mediaType = getMediaType(path.basename(file));
files[mediaType] += `"${file}",`;
}
let contents = "";
for (let mediaType in files) {
contents += `${mediaType}=[${files[mediaType]}];\n`;
}
const bufferedContents = Buffer.from(contents);
const relativeFilePath = path.join(root, "files.js");
await runCommand(relativeFilePath, bufferedContents);
}
async function processTemplate(runCommand, root, userDefinedTemplate) {
const template = userDefinedTemplate
? userDefinedTemplate
: require("./templates").default;
for (const resource of template.resources) {
if (resource.type === "static") {
const bufferedContents = Buffer.from(resource.contents);
const relativeFilePath = path.join(root, resource.name);
await runCommand(relativeFilePath, bufferedContents);
}
}
}
async function walkAndRun(runCommand, dir, root) {
let files = await readdir(dir);
for(let file of files) {
for (let file of files) {
const currentPath = path.join(dir, file);
if (fs.statSync(currentPath).isDirectory()) {
await walkAndRun(runCommand, currentPath);
} else {
await runCommand(currentPath);
const fileContents = await readFile(path.normalize(currentPath));
await runCommand(currentPath, fileContents);
}
}
};
}
async function writeStream(stream, data) {
return new Promise((resolve) => {
return new Promise(resolve => {
stream.write(data);
stream.end(resolve)
stream.end(resolve);
});
}
async function packDirectory(directory, options = {}) {
let filesIndex = [];
const zstd = await getZstd();
const packRoot = directory;
const fileWriteStream = getFileWriteStream(options);
@ -97,53 +148,58 @@ async function packDirectory(directory, options = {}) {
tarPack.pipe(fileWriteStream);
writeFirstEntry(options, tarPack);
await walkAndRun(async (file) => {
contents = await readFile(path.normalize(file));
const makeChunkIterator = contents => {
const chunkSize = 2048;
let position = 0;
// Must be chunked to avoid issues with fixed memory limits.
const chunkIterator = (() => {
const chunkSize = 2048;
let position = 0;
const iterator = {
next: function() {
const endIndex = position + chunkSize;
const result = {
value: contents.slice(position, endIndex),
done: position >= contents.length
};
const iterator = {
next: function() {
const endIndex = position + chunkSize;
const result = {
value: contents.slice(position, endIndex),
done: position >= contents.length,
};
position = endIndex;
return result;
},
[Symbol.iterator]: function() {
return this;
}
};
return iterator;
};
position = endIndex;
return result;
},
[Symbol.iterator]: function() { return this }
};
return iterator;
})();
contents = zstd.compressChunks(chunkIterator, contents.length, COMPRESSION_LEVEL);
let name = path.relative(packRoot, file).replace('\\', '/');
if(/^\.\//.test(name)) {
const createEntry = async (file, contents) => {
const chunkIterator = makeChunkIterator(contents);
contents = zstd.compressChunks(
chunkIterator,
contents.length,
COMPRESSION_LEVEL
);
let name = path.relative(packRoot, file).replace("\\", "/");
if (/^\.\//.test(name)) {
name = name.slice(2);
}
const entry = tarPack.entry({ name, size: contents.length }, (err) => {
if(err) {
const entry = tarPack.entry({ name, size: contents.length }, err => {
if (err) {
throw err;
}
});
await writeStream(entry, contents);
entry.end();
}, directory, packRoot);
filesIndex.push(name);
};
await walkAndRun(createEntry, directory, packRoot);
if (options.useTemplate === true) {
await processTemplate(createEntry, packRoot);
await catagoriseFilesIndex(createEntry, packRoot, filesIndex);
}
tarPack.finalize();
};
}
function strToBuffer (string) {
function strToBuffer(string) {
let arrayBuffer = new ArrayBuffer(string.length * 1);
let newUint = new Uint8Array(arrayBuffer);
newUint.forEach((_, i) => {
@ -153,18 +209,17 @@ function strToBuffer (string) {
}
function streamToBuffer(stream) {
const chunks = []
const chunks = [];
return new Promise((resolve, reject) => {
stream.on('data', chunk => chunks.push(chunk))
stream.on('error', reject)
stream.on('end', () => resolve(Buffer.concat(chunks)))
})
stream.on("data", chunk => chunks.push(chunk));
stream.on("error", reject);
stream.on("end", () => resolve(Buffer.concat(chunks)));
});
}
async function unpackDirectory(directory, options = {}) {
return new Promise(async (resolve) => {
if(!fs.existsSync(directory)) {
return new Promise(async resolve => {
if (!fs.existsSync(directory)) {
mkdirSyncRecursive(directory);
}
@ -173,7 +228,7 @@ async function unpackDirectory(directory, options = {}) {
const extract = tar.extract();
extract.on('entry', async (header, fileStream, next) => {
extract.on("entry", async (header, fileStream, next) => {
let contents = await streamToBuffer(fileStream);
contents = new Uint8Array(contents);
@ -187,20 +242,22 @@ async function unpackDirectory(directory, options = {}) {
const endIndex = position + chunkSize;
const result = {
value: contents.slice(position, endIndex),
done: position >= contents.length,
done: position >= contents.length
};
position = endIndex;
return result;
},
[Symbol.iterator]: function() { return this }
[Symbol.iterator]: function() {
return this;
}
};
return iterator;
})();
contents = zstd.decompressChunks(chunkIterator);
if(!/^\./.test(header.name)) {
if (!/^\./.test(header.name)) {
const writePath = path.join(directory, header.name);
try {
@ -219,7 +276,7 @@ async function unpackDirectory(directory, options = {}) {
}
});
extract.on('finish', () => {
extract.on("finish", () => {
resolve(true);
});
@ -260,5 +317,5 @@ async function packPaths(root, pathsArray, options = {}) {
module.exports = {
packDirectory,
unpackDirectory,
}
unpackDirectory
};

102
lbry-format.js
View file

@ -1,51 +1,65 @@
#!/usr/bin/env node
const {
packDirectory,
unpackDirectory,
} = require('lbry-format');
const path = require('path');
const { packDirectory, unpackDirectory } = require("lbry-format");
const path = require("path");
require('yargs')
.scriptName('lbry-pack')
.usage('$0 <cmd> [args]')
.command('pack [directory] [file]', 'Pack a directory', (yargs) => {
yargs.positional('directory', {
default: './src',
describe: 'The source directory to pack'
}).positional('file', {
describe: 'Output file',
default: './package.lbry',
})
}, function (argv) {
console.log(`Packing ${argv.directory} to ${argv.file}`);
require("yargs")
.scriptName("lbry-pack")
.usage("$0 <cmd> [args]")
.command(
"pack [directory] [file] [-t]",
"Pack a directory",
yargs => {
yargs
.positional("directory", {
default: "./src",
describe: "The source directory to pack"
})
.positional("file", {
describe: "Output file",
default: "./package.lbry"
})
.option("t", {
alias: "template",
describe: "Use web template"
});
},
function(argv) {
console.log(`Packing ${argv.directory} to ${argv.file}`);
const resolvedDirectory = path.resolve(argv.directory);
const resolvedfile = path.resolve(argv.file);
const resolvedDirectory = path.resolve(argv.directory);
const resolvedfile = path.resolve(argv.file);
packDirectory(resolvedDirectory, {
fileName: resolvedfile,
});
})
.command('unpack [directory] [file]', 'Unpack a file', (yargs) => {
yargs.positional('destination', {
type: 'string',
default: './src',
describe: 'The folder destination to unpack to'
}).positional('file', {
describe: 'Input filename',
default: 'package.lbry',
})
}, function (argv) {
console.log(`Packing ${argv.directory} to ${argv.file}`);
packDirectory(resolvedDirectory, {
fileName: resolvedfile,
useTemplate: argv.template
});
}
)
.command(
"unpack [directory] [file]",
"Unpack a file",
yargs => {
yargs
.positional("destination", {
type: "string",
default: "./src",
describe: "The folder destination to unpack to"
})
.positional("file", {
describe: "Input filename",
default: "package.lbry"
});
},
function(argv) {
console.log(`Packing ${argv.directory} to ${argv.file}`);
const resolvedDirectory = path.resolve(argv.directory);
const resolvedfile = path.resolve(argv.file);
const resolvedDirectory = path.resolve(argv.directory);
const resolvedfile = path.resolve(argv.file);
unpackDirectory(resolvedDirectory, {
fileName: resolvedfile,
});
})
unpackDirectory(resolvedDirectory, {
fileName: resolvedfile
});
}
)
.help()
.demandCommand()
.argv
.demandCommand().argv;

25
templates.js Normal file
View file

@ -0,0 +1,25 @@
module.exports = {
default: {
resources: [
{
type: "static",
name: "index.html",
contents:
'<!doctype html><title>Gallary Slideshow</title><meta name=viewport content="width=device-width,initial-scale=1" charset=utf-8><link rel=stylesheet href=index.css><script src=files.js charset=utf-8></script><div id=container class=slideshow-container><a class=prev onclick=plusSlides(-1)>&#10094;</a> <a class=next onclick=plusSlides(1)>&#10095;</a></div><script src=index.js charset=utf-8></script>'
},
{
type: "static",
name: "index.css",
contents:
"body,html{width:100%;height:100%;display:flex;background-color:#001}*{margin:0;padding:0}.slideshow-container{margin:auto;background-color:#000;display:flex;height:80%;width:80%;justify-content:space-between;align-items:center;border-radius:3px 3px 3px 3px}.mySlides{order:1;margin:auto;max-width:100%;max-height:100%}.next,.prev{order:0;cursor:pointer;width:auto;padding:16px;color:#fff;font-weight:700;font-size:18px;transition:.6s ease;border-radius:0 3px 3px 0;user-select:none}.next{order:2;border-radius:3px 0 0 3px}.next:hover,.prev:hover{background-color:#eee;color:#000}.fade{-webkit-animation-name:fade;-webkit-animation-duration:1s;animation-name:fade;animation-duration:1s}@-webkit-keyframes fade{from{opacity:.4}to{opacity:1}}@keyframes fade{from{opacity:.4}to{opacity:1}}@media only screen and (max-width:300px){.next,.prev,.text{font-size:11px}}"
},
{
type: "static",
name: "index.js",
contents:
'const container=document.getElementById("container");for(let e=0;e<images.length;e++){const n=document.createElement("img");n.className="mySlides fade",n.src=images[e],container.appendChild(n)}let slideIndex=1;function plusSlides(e){showSlides(slideIndex+=e)}function showSlides(e){const n=document.getElementsByClassName("mySlides");e>n.length&&(slideIndex=1),e<1&&(slideIndex=n.length);for(let e=0;e<n.length;e++)n[e].style.display="none";n[slideIndex-1].style.display="block"}showSlides(slideIndex);'
}
],
title: "Gallary Slideshow"
}
};