agtic/keycloak_theme
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(cache): fix download caches

Browse Source 
* also fix npm config running 4 times in the worst case
* factor out unzip methods
* factor and enhance trimindent
* factor out more utils
* restore windows build, which failed cause generate-i18n-messages did not write any files
This commit is contained in:
Waldemar Reusch 2023-03-29 09:54:29 +02:00
parent 0ba2f37004
commit 3a6fe1b374
7 changed files with 339 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scripts/generate-i18n-messages.ts
View File

@ -16,7 +16,7 @@ const propertiesParser = require("properties-parser");
const { isSilent } = getCliOptions(process.argv.slice(2));
const logger = getLogger({ isSilent });
(async () => {
async function main() {
const keycloakVersion = "21.0.1";
const tmpDirPath = pathJoin(getProjectRoot(), "tmp_xImOef9dOd44");
@ -37,7 +37,9 @@ const logger = getLogger({ isSilent });
const baseThemeDirPath = pathJoin(tmpDirPath, "base");
crawl(baseThemeDirPath).forEach(filePath => {
const match = filePath.match(/^([^/]+)\/messages\/messages_([^.]+)\.properties$/);
const match =
filePath.match(/^([^/]+)\/messages\/messages_([^.]+)\.properties$/) ||
filePath.match(/^([^\\]+)\\messages\\messages_([^.]+)\.properties$/);
if (match === null) {
return;
@ -114,4 +116,8 @@ const logger = getLogger({ isSilent });
)
);
});
})();
}
if (require.main === module) {
main().catch(e => console.error(e));
}

14
src/bin/download-builtin-keycloak-theme.ts
View File

@ -8,23 +8,20 @@ import { getCliOptions } from "./tools/cliOptions";
import { getLogger } from "./tools/logger";
export async function downloadBuiltinKeycloakTheme(params: { keycloakVersion: string; destDirPath: string; isSilent: boolean }) {
const { keycloakVersion, destDirPath, isSilent } = params;
const { keycloakVersion, destDirPath } = params;
await Promise.all(
["", "-community"].map(ext =>
downloadAndUnzip({
"destDirPath": destDirPath,
"url": `https://github.com/keycloak/keycloak/archive/refs/tags/${keycloakVersion}.zip`,
"pathOfDirToExtractInArchive": `keycloak-${keycloakVersion}/themes/src/main/resources${ext}/theme`,
"cacheDirPath": pathJoin(keycloakThemeBuildingDirPath, ".cache"),
isSilent
"pathOfDirToExtractInArchive": `keycloak-${keycloakVersion}/themes/src/main/resources${ext}/theme`
})
)
);
}
if (require.main === module) {
(async () => {
async function main() {
const { isSilent } = getCliOptions(process.argv.slice(2));
const logger = getLogger({ isSilent });
const { keycloakVersion } = await promptKeycloakVersion();
@ -38,5 +35,8 @@ if (require.main === module) {
destDirPath,
isSilent
});
})();
}
if (require.main === module) {
main().catch(e => console.error(e));
}

290
src/bin/tools/downloadAndUnzip.ts
View File

@ -1,15 +1,13 @@
import { dirname as pathDirname, basename as pathBasename, join as pathJoin, join } from "path";
import { createReadStream, createWriteStream } from "fs";
import { stat, mkdir, unlink, writeFile } from "fs/promises";
import { transformCodebase } from "./transformCodebase";
import { createHash } from "crypto";
import fetch from "make-fetch-happen";
import { createInflateRaw } from "zlib";
import type { Readable } from "stream";
import { homedir } from "os";
import { FetchOptions } from "make-fetch-happen";
import { exec as execCallback } from "child_process";
import { createHash } from "crypto";
import { mkdir, stat, writeFile } from "fs/promises";
import fetch, { type FetchOptions } from "make-fetch-happen";
import { dirname as pathDirname, join as pathJoin } from "path";
import { assert } from "tsafe";
import { promisify } from "util";
import { getProjectRoot } from "./getProjectRoot";
import { transformCodebase } from "./transformCodebase";
import { unzip } from "./unzip";
const exec = promisify(execCallback);
@ -17,25 +15,27 @@ function hash(s: string) {
return createHash("sha256").update(s).digest("hex");
}
async function maybeStat(path: string) {
async function exists(path: string) {
try {
return await stat(path);
await stat(path);
return true;
} catch (error) {
if ((error as Error & { code: string }).code === "ENOENT") return undefined;
if ((error as Error & { code: string }).code === "ENOENT") return false;
throw error;
}
}
/**
* Get an npm configuration value as string, undefined if not set.
*
* @param key
* @returns string or undefined
* Get npm configuration as map
*/
async function getNmpConfig(key: string): Promise<string | undefined> {
const { stdout } = await exec(`npm config get ${key}`);
const value = stdout.trim();
return value && value !== "null" ? value : undefined;
async function getNmpConfig(): Promise<Record<string, string>> {
const { stdout } = await exec("npm config get", { encoding: "utf8" });
return stdout
.split("\n")
.filter(line => !line.startsWith(";"))
.map(line => line.trim())
.map(line => line.split("=", 2))
.reduce((cfg, [key, value]) => ({ ...cfg, [key]: value }), {});
}
/**
@ -45,233 +45,49 @@ async function getNmpConfig(key: string): Promise<string | undefined> {
* @returns proxy configuration
*/
async function getNpmProxyConfig(): Promise<Pick<FetchOptions, "proxy" | "noProxy">> {
const proxy = (await getNmpConfig("https-proxy")) ?? (await getNmpConfig("proxy"));
const noProxy = (await getNmpConfig("noproxy")) ?? (await getNmpConfig("no-proxy"));
const cfg = await getNmpConfig();
const proxy = cfg["https-proxy"] ?? cfg["proxy"];
const noProxy = cfg["noproxy"] ?? cfg["no-proxy"];
return { proxy, noProxy };
}
/**
* Download a file from `url` to `dir`. Will try to avoid downloading existing
* files by using the cache directory ~/.keycloakify/cache
*
* If the target directory does not exist, it will be created.
*
* If the target file exists, it will be overwritten.
*
* We use make-fetch-happen's internal file cache here, so we don't need to
* worry about redownloading the same file over and over. Unfortunately, that
* cache does not have a single file per entry, but bundles and indexes them,
* so we still need to write the contents to the target directory (possibly
* over and over), cause the current unzip implementation wants random access.
*
* @param url download url
* @param dir target directory
* @param filename target filename
* @returns promise for the full path of the downloaded file
*/
async function download(url: string, dir: string, filename: string): Promise<string> {
const proxyOpts = await getNpmProxyConfig();
const cacheRoot = process.env.XDG_CACHE_HOME ?? homedir();
const cachePath = join(cacheRoot, ".keycloakify/cache");
const opts: FetchOptions = { cachePath, cache: 'force-cache', ...proxyOpts };
const response = await fetch(url, opts);
const filepath = pathJoin(dir, filename);
await mkdir(dir, { recursive: true });
await writeFile(filepath, response.body);
return filepath;
}
/**
* @typedef
* @type MultiError = Error & { cause: Error[] }
*/
/**
* Extract the archive `zipFile` into the directory `dir`. If `archiveDir` is given,
* only that directory will be extracted, stripping the given path components.
*
* If dir does not exist, it will be created.
*
* If any archive file exists, it will be overwritten.
*
* Will unzip using all available nodejs worker threads.
*
* Will try to clean up extracted files on failure.
*
* If unpacking fails, will either throw an regular error, or
* possibly an `MultiError`, which contains a `cause` field with
* a number of root cause errors.
*
* Warning this method is not optimized for continuous reading of the zip
* archive, but is a trade-off between simplicity and allowing extraction
* of a single directory from the archive.
*
* @param zipFile the file to unzip
* @param dir the target directory
* @param archiveDir if given, unpack only files from this archive directory
* @throws {MultiError} error
* @returns Promise for a list of full file paths pointing to actually extracted files
*/
async function unzip(zipFile: string, dir: string, archiveDir?: string): Promise<string[]> {
await mkdir(dir, { recursive: true });
const promises: Promise<string>[] = [];
// Iterate over all files in the zip, skip files which are not in archiveDir,
// if given.
for await (const record of iterateZipArchive(zipFile)) {
const { path: recordPath, createReadStream: createRecordReadStream } = record;
const filePath = pathJoin(dir, recordPath);
const parent = pathDirname(filePath);
if (archiveDir && !recordPath.startsWith(archiveDir)) continue;
promises.push(
new Promise<string>(async (resolve, reject) => {
await mkdir(parent, { recursive: true });
// Pull the file out of the archive, write it to the target directory
const input = createRecordReadStream();
const output = createWriteStream(filePath);
output.setMaxListeners(Infinity);
output.on("error", e => reject(Object.assign(e, { filePath })));
output.on("finish", () => resolve(filePath));
input.pipe(output);
})
);
}
// Wait until _all_ files are either extracted or failed
const results = await Promise.allSettled(promises);
const success = results.filter(r => r.status === "fulfilled").map(r => (r as PromiseFulfilledResult<string>).value);
const failure = results.filter(r => r.status === "rejected").map(r => (r as PromiseRejectedResult).reason);
// If any extraction failed, try to clean up, then throw a MultiError,
// which has a `cause` field, containing a list of root cause errors.
if (failure.length) {
await Promise.all(success.map(path => unlink(path)));
await Promise.all(failure.map(e => e && e.path && unlink(e.path as string)));
const e = new Error("Failed to extract: " + failure.map(e => e.message).join(";"));
(e as any).cause = failure;
throw e;
}
return success;
}
/**
*
* @param file file to read
* @param start first byte to read
* @param end last byte to read
* @returns Promise of a buffer of read bytes
*/
async function readFileChunk(file: string, start: number, end: number): Promise<Buffer> {
const chunks: Buffer[] = [];
return new Promise((resolve, reject) => {
const stream = createReadStream(file, { start, end });
stream.setMaxListeners(Infinity);
stream.on("error", e => reject(e));
stream.on("end", () => resolve(Buffer.concat(chunks)));
stream.on("data", chunk => chunks.push(chunk as Buffer));
});
}
type ZipRecord = {
path: string;
createReadStream: () => Readable;
compressionMethod: "deflate" | undefined;
};
type ZipRecordGenerator = AsyncGenerator<ZipRecord, void, unknown>;
/**
* Iterate over all records of a zipfile, and yield a ZipRecord.
* Use `record.createReadStream()` to actually read the file.
*
* Warning this method will only work with single-disk zip files.
* Warning this method may fail if the zip archive has an crazy amount
* of files and the central directory is not fully contained within the
* last 65k bytes of the zip file.
*
* @param zipFile
* @returns AsyncGenerator which will yield ZipRecords
*/
async function* iterateZipArchive(zipFile: string): ZipRecordGenerator {
// Need to know zip file size before we can do anything else
const { size } = await stat(zipFile);
const chunkSize = 65_535 + 22 + 1; // max comment size + end header size + wiggle
// Read last ~65k bytes. Zip files have an comment up to 65_535 bytes at the very end,
// before that comes the zip central directory end header.
let chunk = await readFileChunk(zipFile, size - chunkSize, size);
const unread = size - chunk.length;
let i = chunk.length - 4;
let found = false;
// Find central directory end header, reading backwards from the end
while (!found && i-- > 0) if (chunk[i] === 0x50 && chunk.readUInt32LE(i) === 0x06054b50) found = true;
if (!found) throw new Error("Not a zip file");
// This method will fail on a multi-disk zip, so bail early.
if (chunk.readUInt16LE(i + 4) !== 0) throw new Error("Multi-disk zip not supported");
let nFiles = chunk.readUint16LE(i + 10);
// Get the position of the central directory
const directorySize = chunk.readUint32LE(i + 12);
const directoryOffset = chunk.readUint32LE(i + 16);
if (directoryOffset === 0xffff_ffff) throw new Error("zip64 not supported");
if (directoryOffset > size) throw new Error(`Central directory offset ${directoryOffset} is outside file`);
i = directoryOffset - unread;
// If i < 0, it means that the central directory is not contained within `chunk`
if (i < 0) {
chunk = await readFileChunk(zipFile, directoryOffset, directoryOffset + directorySize);
i = 0;
}
// Now iterate the central directory records, yield an `ZipRecord` for every entry
while (nFiles-- > 0) {
// Check for marker bytes
if (chunk.readUInt32LE(i) !== 0x02014b50) throw new Error("No central directory record at position " + (unread + i));
const compressionMethod = ({ 8: "deflate" } as const)[chunk.readUint16LE(i + 10)];
const compressedFileSize = chunk.readUint32LE(i + 20);
const filenameLength = chunk.readUint16LE(i + 28);
const extraLength = chunk.readUint16LE(i + 30);
const commentLength = chunk.readUint16LE(i + 32);
// Start of the actual content byte stream is after the 'local' record header,
// which is 30 bytes long plus filename and extra field
const start = chunk.readUint32LE(i + 42) + 30 + filenameLength + extraLength;
const end = start + compressedFileSize;
const filename = chunk.slice(i + 46, i + 46 + filenameLength).toString("utf-8");
const createRecordReadStream = () => {
const input = createReadStream(zipFile, { start, end });
if (compressionMethod === "deflate") {
const inflate = createInflateRaw();
input.pipe(inflate);
return inflate;
}
return input;
};
if (end > start) yield { path: filename, createReadStream: createRecordReadStream, compressionMethod };
// advance pointer to next central directory entry
i += 46 + filenameLength + extraLength + commentLength;
}
}
export async function downloadAndUnzip({
url,
destDirPath,
pathOfDirToExtractInArchive,
cacheDirPath
pathOfDirToExtractInArchive
}: {
isSilent: boolean;
url: string;
destDirPath: string;
pathOfDirToExtractInArchive?: string;
cacheDirPath: string;
}) {
const downloadHash = hash(JSON.stringify({ url, pathOfDirToExtractInArchive })).substring(0, 15);
const extractDirPath = pathJoin(cacheDirPath, `_${downloadHash}`);
const downloadHash = hash(JSON.stringify({ url })).substring(0, 15);
const projectRoot = getProjectRoot();
const cacheRoot = process.env.XDG_CACHE_HOME ?? `${projectRoot}/node_modules/.cache`;
const zipFilePath = pathJoin(cacheRoot, "keycloakify", "zip", `_${downloadHash}.zip`);
const extractDirPath = pathJoin(cacheRoot, "keycloakify", "unzip", `_${downloadHash}`);
const filename = pathBasename(url);
const zipFilepath = await download(url, cacheDirPath, filename);
const zipMtime = (await stat(zipFilepath)).mtimeMs;
const unzipMtime = (await maybeStat(extractDirPath))?.mtimeMs;
if (!unzipMtime || zipMtime > unzipMtime) await unzip(zipFilepath, extractDirPath, pathOfDirToExtractInArchive);
const srcDirPath = pathOfDirToExtractInArchive === undefined ? extractDirPath : pathJoin(extractDirPath, pathOfDirToExtractInArchive);
transformCodebase({ srcDirPath, destDirPath });
if (!(await exists(zipFilePath))) {
const proxyOpts = await getNpmProxyConfig();
const response = await fetch(url, proxyOpts);
await mkdir(pathDirname(zipFilePath), { recursive: true });
/**
* The correct way to fix this is to upgrade node-fetch beyond 3.2.5
* (see https://github.com/node-fetch/node-fetch/issues/1295#issuecomment-1144061991.)
* Unfortunately, octokit (a dependency of keycloakify) also uses node-fetch, and
* does not support node-fetch 3.x. So we stick around with this band-aid until
* octokit upgrades.
*/
response.body?.setMaxListeners(Number.MAX_VALUE);
assert(typeof response.body !== "undefined" && response.body != null);
await writeFile(zipFilePath, response.body);
}
await unzip(zipFilePath, extractDirPath, pathOfDirToExtractInArchive);
transformCodebase({
"srcDirPath": extractDirPath,
"destDirPath": destDirPath
});
}

26
src/bin/tools/jar.ts
View File

@ -3,12 +3,9 @@ import { dirname, relative, sep } from "path";
import { createWriteStream } from "fs";
import walk from "./walk";
import type { ZipSource } from "./zip";
import zip from "./zip";
import zip, { type ZipSource } from "./zip";
import { mkdir } from "fs/promises";
/** Trim leading whitespace from every line */
const trimIndent = (s: string) => s.replace(/(\n)\s+/g, "$1");
import trimIndent from "./trimIndent";
type JarArgs = {
rootPath: string;
@ -26,28 +23,23 @@ type JarArgs = {
export default async function jar({ groupId, artifactId, version, rootPath, targetPath }: JarArgs) {
const manifest: ZipSource = {
path: "META-INF/MANIFEST.MF",
data: Buffer.from(
trimIndent(
`Manifest-Version: 1.0
data: Buffer.from(trimIndent`
Manifest-Version: 1.0
Archiver-Version: Plexus Archiver
Created-By: Keycloakify
Built-By: unknown
Build-Jdk: 19.0.0`
)
)
Build-Jdk: 19.0.0
`)
};
const pomProps: ZipSource = {
path: `META-INF/maven/${groupId}/${artifactId}/pom.properties`,
data: Buffer.from(
trimIndent(
`# Generated by keycloakify
data: Buffer.from(trimIndent`# Generated by keycloakify
# ${new Date()}
artifactId=${artifactId}
groupId=${groupId}
version=${version}`
)
)
version=${version}
`)
};
/**

11
src/bin/tools/partitionPromiseSettledResults.ts Normal file
View File

@ -0,0 +1,11 @@
export type PromiseSettledAndPartitioned<T> = [T[], any[]];
export function partitionPromiseSettledResults<T>() {
return [
([successes, failures]: PromiseSettledAndPartitioned<T>, item: PromiseSettledResult<T>) =>
item.status === "rejected"
? ([successes, [item.reason, ...failures]] as PromiseSettledAndPartitioned<T>)
: ([[item.value, ...successes], failures] as PromiseSettledAndPartitioned<T>),
[[], []] as PromiseSettledAndPartitioned<T>
] as const;
}

51
src/bin/tools/trimIndent.ts Normal file
View File

@ -0,0 +1,51 @@
/**
* Concatenate the string fragments and interpolated values
* to get a single string.
*/
function populateTemplate(strings: TemplateStringsArray, ...args: any[]) {
const chunks = [];
for (let i = 0; i < strings.length; i++) {
let lastStringLineLength = 0;
if (strings[i]) {
chunks.push(strings[i]);
// remember last indent of the string portion
lastStringLineLength = strings[i].split("\n").at(-1)?.length ?? 0;
}
if (args[i]) {
// if the interpolation value has newlines, indent the interpolation values
// using the last known string indent
chunks.push(args[i].replace(/([\r?\n])/g, "$1" + " ".repeat(lastStringLineLength)));
}
}
return chunks.join("");
}
function trimIndentPrivate(removeEmptyLeadingAndTrailingLines: boolean, strings: TemplateStringsArray, ...args: any[]) {
// Remove initial and final newlines
let string = populateTemplate(strings, ...args);
if (removeEmptyLeadingAndTrailingLines) string = string.replace(/^[\r\n]/, "").replace(/[^\S\r\n]*[\r\n]$/, "");
const dents = string.match(/^([ \t])+/gm)?.map(s => s.length) ?? [];
// No dents? no change required
if (!dents || dents.length == 0) return string;
const minDent = Math.min(...dents);
// The min indentation is 0, no change needed
if (!minDent) return string;
const dedented = string.replace(new RegExp(`^${" ".repeat(minDent)}`, "gm"), "");
return dedented;
}
/**
* Shift all lines left by the *smallest* indentation level,
* and remove initial newline and all trailing spaces.
*/
export default function trimIndent(strings: TemplateStringsArray, ...args: any[]) {
return trimIndentPrivate(true, strings, ...args);
}
/**
* Shift all lines left by the *smallest* indentation level,
* and _keep_ initial newline and all trailing spaces.
*/
trimIndent.keepLeadingAndTrailingNewlines = function (strings: TemplateStringsArray, ...args: any[]) {
return trimIndentPrivate(false, strings, ...args);
};

184
src/bin/tools/unzip.ts Normal file
View File

@ -0,0 +1,184 @@
import { createReadStream, createWriteStream } from "fs";
import { mkdir, stat, unlink } from "fs/promises";
import { dirname as pathDirname, join as pathJoin, relative as pathRelative } from "path";
import { type Readable } from "stream";
import { createInflateRaw } from "zlib";
import { partitionPromiseSettledResults } from "./partitionPromiseSettledResults";
export type MultiError = Error & { cause: Error[] };
/**
* Extract the archive `zipFile` into the directory `dir`. If `archiveDir` is given,
* only that directory will be extracted, stripping the given path components.
*
* If dir does not exist, it will be created.
*
* If any archive file exists, it will be overwritten.
*
* Will unzip using all available nodejs worker threads.
*
* Will try to clean up extracted files on failure.
*
* If unpacking fails, will either throw an regular error, or
* possibly an `MultiError`, which contains a `cause` field with
* a number of root cause errors.
*
* Warning this method is not optimized for continuous reading of the zip
* archive, but is a trade-off between simplicity and allowing extraction
* of a single directory from the archive.
*
* @param zipFilePath the file to unzip
* @param extractDirPath the target directory
* @param pathOfDirToExtractInArchive if given, unpack only files from this archive directory
* @throws {MultiError} error
* @returns Promise for a list of full file paths pointing to actually extracted files
*/
export async function unzip(zipFilePath: string, extractDirPath: string, pathOfDirToExtractInArchive?: string): Promise<string[]> {
const dirsCreated: (string | undefined)[] = [];
dirsCreated.push(await mkdir(extractDirPath, { recursive: true }));
const promises: Promise<string>[] = [];
// Iterate over all files in the zip, skip files which are not in archiveDir,
// if given.
for await (const record of iterateZipArchive(zipFilePath)) {
const { path: recordPath, createReadStream: createRecordReadStream } = record;
if (pathOfDirToExtractInArchive && !recordPath.startsWith(pathOfDirToExtractInArchive)) {
continue;
}
const relativePath = pathOfDirToExtractInArchive ? pathRelative(pathOfDirToExtractInArchive, recordPath) : recordPath;
const filePath = pathJoin(extractDirPath, relativePath);
const parent = pathDirname(filePath);
promises.push(
new Promise<string>(async (resolve, reject) => {
if (!dirsCreated.includes(parent)) dirsCreated.push(await mkdir(parent, { recursive: true }));
// Pull the file out of the archive, write it to the target directory
const output = createWriteStream(filePath);
output.on("error", e => reject(Object.assign(e, { filePath })));
output.on("finish", () => resolve(filePath));
createRecordReadStream().pipe(output);
})
);
}
// Wait until _all_ files are either extracted or failed
const [success, failure] = (await Promise.allSettled(promises)).reduce(...partitionPromiseSettledResults<string>());
// If any extraction failed, try to clean up, then throw a MultiError,
// which has a `cause` field, containing a list of root cause errors.
if (failure.length) {
await Promise.all([
...success.map(path => unlink(path).catch(_unused => undefined)),
...failure.map(e => e && e.path && unlink(e.path as string).catch(_unused => undefined))
]);
await Promise.all(dirsCreated.filter(Boolean).sort(sortByFolderDepth("desc")));
const e = new Error("Failed to extract: " + failure.map(e => e.message).join(";"));
(e as any).cause = failure;
throw e;
}
return success;
}
function depth(dir: string) {
return dir.match(/\//g)?.length ?? 0;
}
function sortByFolderDepth(order: "asc" | "desc") {
const ord = order === "asc" ? 1 : -1;
return (a: string | undefined, b: string | undefined) => ord * depth(a ?? "") + -ord * depth(b ?? "");
}
/**
*
* @param file file to read
* @param start first byte to read
* @param end last byte to read
* @returns Promise of a buffer of read bytes
*/
async function readFileChunk(file: string, start: number, end: number): Promise<Buffer> {
const chunks: Buffer[] = [];
return new Promise((resolve, reject) => {
const stream = createReadStream(file, { start, end });
stream.setMaxListeners(Infinity);
stream.on("error", e => reject(e));
stream.on("end", () => resolve(Buffer.concat(chunks)));
stream.on("data", chunk => chunks.push(chunk as Buffer));
});
}
type ZipRecord = {
path: string;
createReadStream: () => Readable;
compressionMethod: "deflate" | undefined;
};
type ZipRecordGenerator = AsyncGenerator<ZipRecord, void, unknown>;
/**
* Iterate over all records of a zipfile, and yield a ZipRecord.
* Use `record.createReadStream()` to actually read the file.
*
* Warning this method will only work with single-disk zip files.
* Warning this method may fail if the zip archive has an crazy amount
* of files and the central directory is not fully contained within the
* last 65k bytes of the zip file.
*
* @param zipFile
* @returns AsyncGenerator which will yield ZipRecords
*/
async function* iterateZipArchive(zipFile: string): ZipRecordGenerator {
// Need to know zip file size before we can do anything else
const { size } = await stat(zipFile);
const chunkSize = 65_535 + 22 + 1; // max comment size + end header size + wiggle
// Read last ~65k bytes. Zip files have an comment up to 65_535 bytes at the very end,
// before that comes the zip central directory end header.
let chunk = await readFileChunk(zipFile, size - chunkSize, size);
const unread = size - chunk.length;
let i = chunk.length - 4;
let found = false;
// Find central directory end header, reading backwards from the end
while (!found && i-- > 0) if (chunk[i] === 0x50 && chunk.readUInt32LE(i) === 0x06054b50) found = true;
if (!found) throw new Error("Not a zip file");
// This method will fail on a multi-disk zip, so bail early.
if (chunk.readUInt16LE(i + 4) !== 0) throw new Error("Multi-disk zip not supported");
let nFiles = chunk.readUint16LE(i + 10);
// Get the position of the central directory
const directorySize = chunk.readUint32LE(i + 12);
const directoryOffset = chunk.readUint32LE(i + 16);
if (directoryOffset === 0xffff_ffff) throw new Error("zip64 not supported");
if (directoryOffset > size) throw new Error(`Central directory offset ${directoryOffset} is outside file`);
i = directoryOffset - unread;
// If i < 0, it means that the central directory is not contained within `chunk`
if (i < 0) {
chunk = await readFileChunk(zipFile, directoryOffset, directoryOffset + directorySize);
i = 0;
}
// Now iterate the central directory records, yield an `ZipRecord` for every entry
while (nFiles-- > 0) {
// Check for marker bytes
if (chunk.readUInt32LE(i) !== 0x02014b50) throw new Error("No central directory record at position " + (unread + i));
const compressionMethod = ({ 8: "deflate" } as const)[chunk.readUint16LE(i + 10)];
const compressedFileSize = chunk.readUint32LE(i + 20);
const filenameLength = chunk.readUint16LE(i + 28);
const extraLength = chunk.readUint16LE(i + 30);
const commentLength = chunk.readUint16LE(i + 32);
// Start of the actual content byte stream is after the 'local' record header,
// which is 30 bytes long plus filename and extra field
const start = chunk.readUint32LE(i + 42) + 30 + filenameLength + extraLength;
const end = start + compressedFileSize;
const filename = chunk.slice(i + 46, i + 46 + filenameLength).toString("utf-8");
const createRecordReadStream = () => {
const input = createReadStream(zipFile, { start, end });
if (compressionMethod === "deflate") {
const inflate = createInflateRaw();
input.pipe(inflate);
return inflate;
}
return input;
};
if (end > start) yield { path: filename, createReadStream: createRecordReadStream, compressionMethod };
// advance pointer to next central directory entry
i += 46 + filenameLength + extraLength + commentLength;
}
}