From cd68b07e19b185adccf8b791764d64a529280e36 Mon Sep 17 00:00:00 2001 From: garronej Date: Mon, 21 Aug 2023 04:26:58 +0200 Subject: [PATCH] Build keycloak static assets and improve cache mechanism to keep build time in check https://github.com/xgp/keycloak-account-v1/issues/3 --- src/bin/copy-keycloak-resources-to-public.ts | 1 + src/bin/download-builtin-keycloak-theme.ts | 137 +++++++++--------- src/bin/initialize-email-theme.ts | 7 +- .../downloadKeycloakStaticResources.ts | 4 +- .../generateTheme/generateTheme.ts | 13 +- src/bin/keycloakify/keycloakify.ts | 1 + src/bin/tools/downloadAndUnzip.ts | 45 ++++-- src/bin/tools/unzip.ts | 65 ++++++++- 8 files changed, 182 insertions(+), 91 deletions(-) diff --git a/src/bin/copy-keycloak-resources-to-public.ts b/src/bin/copy-keycloak-resources-to-public.ts index f0049742..b773ef57 100644 --- a/src/bin/copy-keycloak-resources-to-public.ts +++ b/src/bin/copy-keycloak-resources-to-public.ts @@ -24,6 +24,7 @@ import * as fs from "fs"; for (const themeType of themeTypes) { await downloadKeycloakStaticResources({ + projectDirPath, "isSilent": false, "keycloakVersion": buildOptions.keycloakVersionDefaultAssets, "themeType": themeType, diff --git a/src/bin/download-builtin-keycloak-theme.ts b/src/bin/download-builtin-keycloak-theme.ts index b8c73539..58f198b6 100644 --- a/src/bin/download-builtin-keycloak-theme.ts +++ b/src/bin/download-builtin-keycloak-theme.ts @@ -7,79 +7,79 @@ import { readBuildOptions } from "./keycloakify/BuildOptions"; import * as child_process from "child_process"; import * as fs from "fs"; -export async function downloadBuiltinKeycloakTheme(params: { keycloakVersion: string; destDirPath: string; isSilent: boolean }) { - const { keycloakVersion, destDirPath } = params; +export async function downloadBuiltinKeycloakTheme(params: { + projectDirPath: string; + keycloakVersion: string; + destDirPath: string; + isSilent: boolean; +}) { + const { projectDirPath, keycloakVersion, destDirPath } = params; - await Promise.all( - ["", "-community"].map(ext => - downloadAndUnzip({ - "destDirPath": destDirPath, - "url": `https://github.com/keycloak/keycloak/archive/refs/tags/${keycloakVersion}.zip`, - "pathOfDirToExtractInArchive": `keycloak-${keycloakVersion}/themes/src/main/resources${ext}/theme` - }) - ) - ); + const start = Date.now(); - install_common_node_modules: { - const commonResourcesDirPath = pathJoin(destDirPath, "keycloak", "common", "resources"); + console.log("Downloading Keycloak theme...", { keycloakVersion }); - if (!fs.existsSync(commonResourcesDirPath)) { - break install_common_node_modules; + await downloadAndUnzip({ + projectDirPath, + destDirPath, + "url": `https://github.com/keycloak/keycloak/archive/refs/tags/${keycloakVersion}.zip`, + "specificDirsToExtract": ["", "-community"].map(ext => `keycloak-${keycloakVersion}/themes/src/main/resources${ext}/theme`), + "preCacheTransform": { + "actionCacheId": "Build Keycloak resources", + "action": async ({ destDirPath }) => { + install_common_node_modules: { + const commonResourcesDirPath = pathJoin(destDirPath, "keycloak", "common", "resources"); + + if (!fs.existsSync(commonResourcesDirPath)) { + break install_common_node_modules; + } + + if (!fs.existsSync(pathJoin(commonResourcesDirPath, "package.json"))) { + break install_common_node_modules; + } + + if (fs.existsSync(pathJoin(commonResourcesDirPath, "node_modules"))) { + break install_common_node_modules; + } + + child_process.execSync("npm install --omit=dev", { + "cwd": commonResourcesDirPath, + "stdio": "ignore" + }); + } + + install_and_move_to_common_resources_generated_in_keycloak_v2: { + const accountV2DirSrcDirPath = pathJoin(destDirPath, "keycloak.v2", "account", "src"); + + if (!fs.existsSync(accountV2DirSrcDirPath)) { + break install_and_move_to_common_resources_generated_in_keycloak_v2; + } + + child_process.execSync("npm install", { "cwd": accountV2DirSrcDirPath, "stdio": "ignore" }); + + const packageJsonFilePath = pathJoin(accountV2DirSrcDirPath, "package.json"); + + const packageJsonRaw = fs.readFileSync(packageJsonFilePath); + + const parsedPackageJson = JSON.parse(packageJsonRaw.toString("utf8")); + + parsedPackageJson.scripts.build = parsedPackageJson.scripts.build + .replace("npm run check-types", "true") + .replace("npm run babel", "true"); + + fs.writeFileSync(packageJsonFilePath, Buffer.from(JSON.stringify(parsedPackageJson, null, 2), "utf8")); + + child_process.execSync("npm run build", { "cwd": accountV2DirSrcDirPath, "stdio": "ignore" }); + + fs.writeFileSync(packageJsonFilePath, packageJsonRaw); + + fs.rmSync(pathJoin(accountV2DirSrcDirPath, "node_modules"), { "recursive": true }); + } + } } + }); - if (!fs.existsSync(pathJoin(commonResourcesDirPath, "package.json"))) { - break install_common_node_modules; - } - - if (fs.existsSync(pathJoin(commonResourcesDirPath, "node_modules"))) { - break install_common_node_modules; - } - - console.log("npm install --omit=dev start", { keycloakVersion }); - - const start = Date.now(); - - child_process.execSync("npm install --omit=dev", { - "cwd": commonResourcesDirPath, - "stdio": "ignore" - }); - - console.log("npm install --omit=dev end", { keycloakVersion, "time": Date.now() - start }); - } - - install_and_move_to_common_resources_generated_in_keycloak_v2: { - const accountV2DirSrcDirPath = pathJoin(destDirPath, "keycloak.v2", "account", "src"); - - if (!fs.existsSync(accountV2DirSrcDirPath)) { - break install_and_move_to_common_resources_generated_in_keycloak_v2; - } - - console.log("npm install start", { keycloakVersion }); - const startInstall = Date.now(); - - child_process.execSync("npm install", { "cwd": accountV2DirSrcDirPath, "stdio": "ignore" }); - - console.log("npm install end", { keycloakVersion, "time": Date.now() - startInstall }); - - const packageJsonFilePath = pathJoin(accountV2DirSrcDirPath, "package.json"); - - const packageJsonRaw = fs.readFileSync(packageJsonFilePath); - - const parsedPackageJson = JSON.parse(packageJsonRaw.toString("utf8")); - - parsedPackageJson.scripts.build = parsedPackageJson.scripts.build.replace("npm run check-types", "true").replace("npm run babel", "true"); - - fs.writeFileSync(packageJsonFilePath, Buffer.from(JSON.stringify(parsedPackageJson, null, 2), "utf8")); - - console.log("npm run build start", { keycloakVersion }); - const start = Date.now(); - - child_process.execSync("npm run build", { "cwd": accountV2DirSrcDirPath, "stdio": "ignore" }); - - console.log("npm run build end", { keycloakVersion, "time": Date.now() - start }); - - fs.writeFileSync(packageJsonFilePath, packageJsonRaw); - } + console.log("Downloaded Keycloak theme in", Date.now() - start, "ms"); } async function main() { @@ -96,6 +96,7 @@ async function main() { logger.log(`Downloading builtins theme of Keycloak ${keycloakVersion} here ${destDirPath}`); await downloadBuiltinKeycloakTheme({ + "projectDirPath": process.cwd(), keycloakVersion, destDirPath, "isSilent": buildOptions.isSilent diff --git a/src/bin/initialize-email-theme.ts b/src/bin/initialize-email-theme.ts index 04388049..18c7803f 100644 --- a/src/bin/initialize-email-theme.ts +++ b/src/bin/initialize-email-theme.ts @@ -10,15 +10,17 @@ import { getLogger } from "./tools/logger"; import { getThemeSrcDirPath } from "./getSrcDirPath"; export async function main() { + const projectDirPath = process.cwd(); + const { isSilent } = readBuildOptions({ - "projectDirPath": process.cwd(), + projectDirPath, "processArgv": process.argv.slice(2) }); const logger = getLogger({ isSilent }); const { themeSrcDirPath } = getThemeSrcDirPath({ - "projectDirPath": process.cwd() + projectDirPath }); const emailThemeSrcDirPath = pathJoin(themeSrcDirPath, "email"); @@ -34,6 +36,7 @@ export async function main() { const builtinKeycloakThemeTmpDirPath = pathJoin(emailThemeSrcDirPath, "..", "tmp_xIdP3_builtin_keycloak_theme"); await downloadBuiltinKeycloakTheme({ + projectDirPath, keycloakVersion, "destDirPath": builtinKeycloakThemeTmpDirPath, isSilent diff --git a/src/bin/keycloakify/generateTheme/downloadKeycloakStaticResources.ts b/src/bin/keycloakify/generateTheme/downloadKeycloakStaticResources.ts index df23fd58..b781f26b 100644 --- a/src/bin/keycloakify/generateTheme/downloadKeycloakStaticResources.ts +++ b/src/bin/keycloakify/generateTheme/downloadKeycloakStaticResources.ts @@ -13,13 +13,14 @@ import * as crypto from "crypto"; export async function downloadKeycloakStaticResources( // prettier-ignore params: { + projectDirPath: string; themeType: ThemeType; themeDirPath: string; isSilent: boolean; keycloakVersion: string; } ) { - const { themeType, isSilent, themeDirPath, keycloakVersion } = params; + const { projectDirPath, themeType, isSilent, themeDirPath, keycloakVersion } = params; const tmpDirPath = pathJoin( themeDirPath, @@ -28,6 +29,7 @@ export async function downloadKeycloakStaticResources( ); await downloadBuiltinKeycloakTheme({ + projectDirPath, keycloakVersion, "destDirPath": tmpDirPath, isSilent diff --git a/src/bin/keycloakify/generateTheme/generateTheme.ts b/src/bin/keycloakify/generateTheme/generateTheme.ts index 07a5cc74..687c7754 100644 --- a/src/bin/keycloakify/generateTheme/generateTheme.ts +++ b/src/bin/keycloakify/generateTheme/generateTheme.ts @@ -51,6 +51,7 @@ export namespace BuildOptionsLike { assert(); export async function generateTheme(params: { + projectDirPath: string; reactAppBuildDirPath: string; keycloakThemeBuildingDirPath: string; themeSrcDirPath: string; @@ -58,7 +59,15 @@ export async function generateTheme(params: { buildOptions: BuildOptionsLike; keycloakifyVersion: string; }): Promise { - const { reactAppBuildDirPath, keycloakThemeBuildingDirPath, themeSrcDirPath, keycloakifySrcDirPath, buildOptions, keycloakifyVersion } = params; + const { + projectDirPath, + reactAppBuildDirPath, + keycloakThemeBuildingDirPath, + themeSrcDirPath, + keycloakifySrcDirPath, + buildOptions, + keycloakifyVersion + } = params; const getThemeDirPath = (themeType: ThemeType | "email") => pathJoin(keycloakThemeBuildingDirPath, "src", "main", "resources", "theme", buildOptions.themeName, themeType); @@ -197,6 +206,7 @@ export async function generateTheme(params: { } await downloadKeycloakStaticResources({ + projectDirPath, "isSilent": buildOptions.isSilent, "keycloakVersion": buildOptions.keycloakVersionDefaultAssets, "themeDirPath": keycloakDirInPublicDir, @@ -222,6 +232,7 @@ export async function generateTheme(params: { } await downloadKeycloakStaticResources({ + projectDirPath, "isSilent": buildOptions.isSilent, "keycloakVersion": buildOptions.keycloakVersionDefaultAssets, themeDirPath, diff --git a/src/bin/keycloakify/keycloakify.ts b/src/bin/keycloakify/keycloakify.ts index 4be68f12..0ee0aba4 100644 --- a/src/bin/keycloakify/keycloakify.ts +++ b/src/bin/keycloakify/keycloakify.ts @@ -30,6 +30,7 @@ export async function main() { for (const themeName of [buildOptions.themeName, ...buildOptions.extraThemeNames]) { await generateTheme({ + projectDirPath, "keycloakThemeBuildingDirPath": buildOptions.keycloakifyBuildDirPath, themeSrcDirPath, "keycloakifySrcDirPath": pathJoin(keycloakifyDirPath, "src"), diff --git a/src/bin/tools/downloadAndUnzip.ts b/src/bin/tools/downloadAndUnzip.ts index cd6fb18a..60643aff 100644 --- a/src/bin/tools/downloadAndUnzip.ts +++ b/src/bin/tools/downloadAndUnzip.ts @@ -1,13 +1,12 @@ import { exec as execCallback } from "child_process"; import { createHash } from "crypto"; -import { mkdir, readFile, stat, writeFile } from "fs/promises"; +import { mkdir, readFile, stat, writeFile, unlink, rm } from "fs/promises"; import fetch, { type FetchOptions } from "make-fetch-happen"; import { dirname as pathDirname, join as pathJoin, resolve as pathResolve, sep as pathSep } from "path"; import { assert } from "tsafe/assert"; import { promisify } from "util"; -import { getProjectRoot } from "./getProjectRoot"; import { transformCodebase } from "./transformCodebase"; -import { unzip } from "./unzip"; +import { unzip, zip } from "./unzip"; const exec = promisify(execCallback); @@ -113,14 +112,24 @@ async function getFetchOptions(): Promise Promise; + }; +}) { + const { projectDirPath, url, destDirPath, specificDirsToExtract, preCacheTransform } = params; - const downloadHash = hash(JSON.stringify({ url })).substring(0, 15); - const projectRoot = getProjectRoot(); - const cacheRoot = process.env.XDG_CACHE_HOME ?? pathJoin(projectRoot, "node_modules", ".cache"); - const zipFilePath = pathJoin(cacheRoot, "keycloakify", "zip", `_${downloadHash}.zip`); - const extractDirPath = pathJoin(cacheRoot, "keycloakify", "unzip", `_${downloadHash}`); + const downloadHash = hash( + JSON.stringify({ url }) + (preCacheTransform === undefined ? "" : `${preCacheTransform.actionCacheId}${preCacheTransform.action.toString()}`) + ).substring(0, 15); + const cacheRoot = pathJoin(process.env.XDG_CACHE_HOME ?? pathJoin(projectDirPath, "node_modules", ".cache"), "keycloakify"); + const zipFilePath = pathJoin(cacheRoot, `_${downloadHash}.zip`); + const extractDirPath = pathJoin(cacheRoot, `tmp_unzip_${downloadHash}`); if (!(await exists(zipFilePath))) { const opts = await getFetchOptions(); @@ -136,9 +145,23 @@ export async function downloadAndUnzip(params: { url: string; destDirPath: strin response.body?.setMaxListeners(Number.MAX_VALUE); assert(typeof response.body !== "undefined" && response.body != null); await writeFile(zipFilePath, response.body); + + if (specificDirsToExtract !== undefined || preCacheTransform !== undefined) { + await unzip(zipFilePath, extractDirPath, specificDirsToExtract); + + await preCacheTransform?.action({ + "destDirPath": extractDirPath + }); + + await unlink(zipFilePath); + + await zip(extractDirPath, zipFilePath); + + await rm(extractDirPath, { "recursive": true }); + } } - await unzip(zipFilePath, extractDirPath, pathOfDirToExtractInArchive); + await unzip(zipFilePath, extractDirPath); transformCodebase({ "srcDirPath": extractDirPath, diff --git a/src/bin/tools/unzip.ts b/src/bin/tools/unzip.ts index 3180ad2a..eee80721 100644 --- a/src/bin/tools/unzip.ts +++ b/src/bin/tools/unzip.ts @@ -2,6 +2,7 @@ import fsp from "node:fs/promises"; import fs from "fs"; import path from "node:path"; import yauzl from "yauzl"; +import yazl from "yazl"; import stream from "node:stream"; import { promisify } from "node:util"; @@ -19,11 +20,16 @@ async function pathExists(path: string) { } } -export async function unzip(file: string, targetFolder: string, unzipSubPath?: string) { - // add trailing slash to unzipSubPath and targetFolder - if (unzipSubPath && (!unzipSubPath.endsWith("/") || !unzipSubPath.endsWith("\\"))) { - unzipSubPath += "/"; - } +// Handlings of non posix path is not implemented correctly +// it work by coincidence. Don't have the time to fix but it should be fixed. +export async function unzip(file: string, targetFolder: string, specificDirsToExtract?: string[]) { + specificDirsToExtract = specificDirsToExtract?.map(dirPath => { + if (!dirPath.endsWith("/") || !dirPath.endsWith("\\")) { + dirPath += "/"; + } + + return dirPath; + }); if (!targetFolder.endsWith("/") || !targetFolder.endsWith("\\")) { targetFolder += "/"; @@ -42,15 +48,17 @@ export async function unzip(file: string, targetFolder: string, unzipSubPath?: s zipfile.readEntry(); zipfile.on("entry", async entry => { - if (unzipSubPath) { + if (specificDirsToExtract !== undefined) { + const dirPath = specificDirsToExtract.find(dirPath => entry.fileName.startsWith(dirPath)); + // Skip files outside of the unzipSubPath - if (!entry.fileName.startsWith(unzipSubPath)) { + if (dirPath === undefined) { zipfile.readEntry(); return; } // Remove the unzipSubPath from the file name - entry.fileName = entry.fileName.substring(unzipSubPath.length); + entry.fileName = entry.fileName.substring(dirPath.length); } const target = path.join(targetFolder, entry.fileName); @@ -77,6 +85,8 @@ export async function unzip(file: string, targetFolder: string, unzipSubPath?: s return; } + await fsp.mkdir(path.dirname(target), { "recursive": true }); + await pipeline(readStream, fs.createWriteStream(target)); zipfile.readEntry(); @@ -90,3 +100,42 @@ export async function unzip(file: string, targetFolder: string, unzipSubPath?: s }); }); } + +// NOTE: This code was directly copied from ChatGPT and appears to function as expected. +// However, confidence in its complete accuracy and robustness is limited. +export async function zip(sourceFolder: string, targetZip: string) { + return new Promise(async (resolve, reject) => { + const zipfile = new yazl.ZipFile(); + const files: string[] = []; + + // Recursive function to explore directories and their subdirectories + async function exploreDir(dir: string) { + const dirContent = await fsp.readdir(dir); + for (const file of dirContent) { + const filePath = path.join(dir, file); + const stat = await fsp.stat(filePath); + if (stat.isDirectory()) { + await exploreDir(filePath); + } else if (stat.isFile()) { + files.push(filePath); + } + } + } + + // Collecting all files to be zipped + await exploreDir(sourceFolder); + + // Adding files to zip + for (const file of files) { + const relativePath = path.relative(sourceFolder, file); + zipfile.addFile(file, relativePath); + } + + zipfile.outputStream + .pipe(fs.createWriteStream(targetZip)) + .on("close", () => resolve()) + .on("error", err => reject(err)); // Listen to error events + + zipfile.end(); + }); +}