diff --git a/package.json b/package.json index d727a3e4..36d14ccc 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "url": "git://github.com/keycloakify/keycloakify.git" }, "scripts": { - "prepare": "ts-node --skipProject scripts/generate-i18n-messages.ts && patch-package", + "prepare": "patch-package && ts-node --skipProject scripts/generate-i18n-messages.ts", "build": "ts-node --skipProject scripts/build.ts", "storybook": "yarn build && yarn copy-keycloak-resources-to-storybook-static && start-storybook -p 6006", "link-in-starter": "ts-node --skipProject scripts/link-in-starter.ts", diff --git a/scripts/generate-i18n-messages.ts b/scripts/generate-i18n-messages.ts index 89bb590e..0ca74c16 100644 --- a/scripts/generate-i18n-messages.ts +++ b/scripts/generate-i18n-messages.ts @@ -22,20 +22,8 @@ async function main() { const thisCodebaseRootDirPath = getThisCodebaseRootDirPath(); - const tmpDirPath = pathJoin(thisCodebaseRootDirPath, "tmp_xImOef9dOd44"); - - rmSync(tmpDirPath, { recursive: true, force: true }); - - fs.mkdirSync(tmpDirPath); - - fs.writeFileSync( - pathJoin(tmpDirPath, ".gitignore"), - Buffer.from("/*\n!.gitignore\n", "utf8") - ); - - await downloadKeycloakDefaultTheme({ + const { defaultThemeDirPath } = await downloadKeycloakDefaultTheme({ keycloakVersion, - destDirPath: tmpDirPath, buildOptions: { cacheDirPath: pathJoin( thisCodebaseRootDirPath, @@ -52,7 +40,7 @@ async function main() { const record: { [typeOfPage: string]: { [language: string]: Dictionary } } = {}; { - const baseThemeDirPath = pathJoin(tmpDirPath, "base"); + const baseThemeDirPath = pathJoin(defaultThemeDirPath, "base"); const re = new RegExp( `^([^\\${pathSep}]+)\\${pathSep}messages\\${pathSep}messages_([^.]+).properties$` ); @@ -84,8 +72,6 @@ async function main() { }); } - rmSync(tmpDirPath, { recursive: true }); - Object.keys(record).forEach(themeType => { const recordForPageType = record[themeType]; diff --git a/src/bin/download-keycloak-default-theme.ts b/src/bin/download-keycloak-default-theme.ts index 8d787b2c..7e4b3c91 100644 --- a/src/bin/download-keycloak-default-theme.ts +++ b/src/bin/download-keycloak-default-theme.ts @@ -2,6 +2,7 @@ import { join as pathJoin, relative as pathRelative, sep as pathSep } from "path import { promptKeycloakVersion } from "./shared/promptKeycloakVersion"; import { readBuildOptions } from "./shared/buildOptions"; import { downloadKeycloakDefaultTheme } from "./shared/downloadKeycloakDefaultTheme"; +import { transformCodebase } from "./tools/transformCodebase"; import type { CliCommandOptions } from "./main"; import chalk from "chalk"; @@ -48,11 +49,15 @@ export async function command(params: { cliCommandOptions: CliCommandOptions }) ].join("\n") ); - await downloadKeycloakDefaultTheme({ + const { defaultThemeDirPath } = await downloadKeycloakDefaultTheme({ keycloakVersion, - destDirPath, buildOptions }); + transformCodebase({ + srcDirPath: defaultThemeDirPath, + destDirPath + }); + console.log(chalk.green(`✓ done`)); } diff --git a/src/bin/initialize-email-theme.ts b/src/bin/initialize-email-theme.ts index d915e2b0..fb681958 100644 --- a/src/bin/initialize-email-theme.ts +++ b/src/bin/initialize-email-theme.ts @@ -5,7 +5,6 @@ import { promptKeycloakVersion } from "./shared/promptKeycloakVersion"; import { readBuildOptions } from "./shared/buildOptions"; import * as fs from "fs"; import { getThemeSrcDirPath } from "./shared/getThemeSrcDirPath"; -import { rmSync } from "./tools/fs.rmSync"; import type { CliCommandOptions } from "./main"; export async function command(params: { cliCommandOptions: CliCommandOptions }) { @@ -38,24 +37,13 @@ export async function command(params: { cliCommandOptions: CliCommandOptions }) cacheDirPath: buildOptions.cacheDirPath }); - const builtinKeycloakThemeTmpDirPath = pathJoin( - buildOptions.cacheDirPath, - "initialize-email-theme_tmp" - ); - - rmSync(builtinKeycloakThemeTmpDirPath, { - recursive: true, - force: true - }); - - await downloadKeycloakDefaultTheme({ + const { defaultThemeDirPath } = await downloadKeycloakDefaultTheme({ keycloakVersion, - destDirPath: builtinKeycloakThemeTmpDirPath, buildOptions }); transformCodebase({ - srcDirPath: pathJoin(builtinKeycloakThemeTmpDirPath, "base", "email"), + srcDirPath: pathJoin(defaultThemeDirPath, "base", "email"), destDirPath: emailThemeSrcDirPath }); @@ -78,6 +66,4 @@ export async function command(params: { cliCommandOptions: CliCommandOptions }) )}\` directory have been created.` ); console.log("You can delete any file you don't modify."); - - rmSync(builtinKeycloakThemeTmpDirPath, { recursive: true }); } diff --git a/src/bin/keycloakify/generateSrcMainResources/bringInAccountV1.ts b/src/bin/keycloakify/generateSrcMainResources/bringInAccountV1.ts index 687f6437..65ef3923 100644 --- a/src/bin/keycloakify/generateSrcMainResources/bringInAccountV1.ts +++ b/src/bin/keycloakify/generateSrcMainResources/bringInAccountV1.ts @@ -9,7 +9,6 @@ import { } from "../../shared/constants"; import { downloadKeycloakDefaultTheme } from "../../shared/downloadKeycloakDefaultTheme"; import { transformCodebase } from "../../tools/transformCodebase"; -import { rmSync } from "../../tools/fs.rmSync"; type BuildOptionsLike = { cacheDirPath: string; @@ -22,13 +21,7 @@ assert(); export async function bringInAccountV1(params: { buildOptions: BuildOptionsLike }) { const { buildOptions } = params; - const builtinKeycloakThemeTmpDirPath = pathJoin( - buildOptions.cacheDirPath, - "bringInAccountV1_tmp" - ); - - await downloadKeycloakDefaultTheme({ - destDirPath: builtinKeycloakThemeTmpDirPath, + const { defaultThemeDirPath } = await downloadKeycloakDefaultTheme({ keycloakVersion: lastKeycloakVersionWithAccountV1, buildOptions }); @@ -44,32 +37,20 @@ export async function bringInAccountV1(params: { buildOptions: BuildOptionsLike ); transformCodebase({ - srcDirPath: pathJoin(builtinKeycloakThemeTmpDirPath, "base", "account"), + srcDirPath: pathJoin(defaultThemeDirPath, "base", "account"), destDirPath: accountV1DirPath }); transformCodebase({ - srcDirPath: pathJoin( - builtinKeycloakThemeTmpDirPath, - "keycloak", - "account", - "resources" - ), + srcDirPath: pathJoin(defaultThemeDirPath, "keycloak", "account", "resources"), destDirPath: pathJoin(accountV1DirPath, "resources") }); transformCodebase({ - srcDirPath: pathJoin( - builtinKeycloakThemeTmpDirPath, - "keycloak", - "common", - "resources" - ), + srcDirPath: pathJoin(defaultThemeDirPath, "keycloak", "common", "resources"), destDirPath: pathJoin(accountV1DirPath, "resources", resources_common) }); - rmSync(builtinKeycloakThemeTmpDirPath, { recursive: true }); - fs.writeFileSync( pathJoin(accountV1DirPath, "theme.properties"), Buffer.from( diff --git a/src/bin/shared/downloadAndUnzip.ts b/src/bin/shared/downloadAndUnzip.ts deleted file mode 100644 index cc1a6ea3..00000000 --- a/src/bin/shared/downloadAndUnzip.ts +++ /dev/null @@ -1,224 +0,0 @@ -import { createHash } from "crypto"; -import { mkdir, writeFile, unlink } from "fs/promises"; -import fetch from "make-fetch-happen"; -import { dirname as pathDirname, join as pathJoin, basename as pathBasename } from "path"; -import { assert } from "tsafe/assert"; -import { transformCodebase } from "../tools/transformCodebase"; -import { unzip, zip } from "../tools/unzip"; -import { rm } from "../tools/fs.rm"; -import * as child_process from "child_process"; -import { existsAsync } from "../tools/fs.existsAsync"; -import type { BuildOptions } from "./buildOptions"; -import { getProxyFetchOptions } from "../tools/fetchProxyOptions"; - -export type BuildOptionsLike = { - cacheDirPath: string; - npmWorkspaceRootDirPath: string; -}; - -assert(); - -export async function downloadAndUnzip(params: { - url: string; - destDirPath: string; - specificDirsToExtract?: string[]; - preCacheTransform?: { - actionCacheId: string; - action: (params: { destDirPath: string }) => Promise; - }; - buildOptions: BuildOptionsLike; -}) { - const { url, destDirPath, specificDirsToExtract, preCacheTransform, buildOptions } = - params; - - const { extractDirPath, zipFilePath } = (() => { - const zipFileBasenameWithoutExt = generateFileNameFromURL({ - url, - preCacheTransform: - preCacheTransform === undefined - ? undefined - : { - actionCacheId: preCacheTransform.actionCacheId, - actionFootprint: preCacheTransform.action.toString() - } - }); - - const zipFilePath = pathJoin( - buildOptions.cacheDirPath, - `${zipFileBasenameWithoutExt}.zip` - ); - const extractDirPath = pathJoin( - buildOptions.cacheDirPath, - `tmp_unzip_${zipFileBasenameWithoutExt}` - ); - - return { zipFilePath, extractDirPath }; - })(); - - download_zip_and_transform: { - if (await existsAsync(zipFilePath)) { - break download_zip_and_transform; - } - - const { response, isFromRemoteCache } = await (async () => { - const proxyFetchOptions = await getProxyFetchOptions({ - npmWorkspaceRootDirPath: buildOptions.npmWorkspaceRootDirPath - }); - - const response = await fetch( - `https://github.com/keycloakify/keycloakify/releases/download/v0.0.1/${pathBasename( - zipFilePath - )}`, - proxyFetchOptions - ); - - if (response.status === 200) { - return { - response, - isFromRemoteCache: true - }; - } - - return { - response: await fetch(url, proxyFetchOptions), - isFromRemoteCache: false - }; - })(); - - await mkdir(pathDirname(zipFilePath), { recursive: true }); - - /** - * The correct way to fix this is to upgrade node-fetch beyond 3.2.5 - * (see https://github.com/node-fetch/node-fetch/issues/1295#issuecomment-1144061991.) - * Unfortunately, octokit (a dependency of keycloakify) also uses node-fetch, and - * does not support node-fetch 3.x. So we stick around with this band-aid until - * octokit upgrades. - */ - response.body?.setMaxListeners(Number.MAX_VALUE); - assert(typeof response.body !== "undefined" && response.body != null); - - await writeFile(zipFilePath, response.body); - - if (isFromRemoteCache) { - break download_zip_and_transform; - } - - if (specificDirsToExtract === undefined && preCacheTransform === undefined) { - break download_zip_and_transform; - } - - await unzip(zipFilePath, extractDirPath, specificDirsToExtract); - - try { - await preCacheTransform?.action({ - destDirPath: extractDirPath - }); - } catch (error) { - await Promise.all([ - rm(extractDirPath, { recursive: true }), - unlink(zipFilePath) - ]); - - throw error; - } - - await unlink(zipFilePath); - - await zip(extractDirPath, zipFilePath); - - await rm(extractDirPath, { recursive: true }); - - upload_to_remote_cache_if_admin: { - const githubToken = - process.env["KEYCLOAKIFY_ADMIN_GITHUB_PERSONAL_ACCESS_TOKEN"]; - - if (!githubToken) { - break upload_to_remote_cache_if_admin; - } - - console.log("uploading to remote cache"); - - try { - child_process.execSync(`which putasset`); - } catch { - child_process.execSync(`npm install -g putasset`); - } - - try { - child_process.execFileSync("putasset", [ - "--owner", - "keycloakify", - "--repo", - "keycloakify", - "--tag", - "v0.0.1", - "--filename", - zipFilePath, - "--token", - githubToken - ]); - } catch { - console.log( - "upload failed, asset probably already exists in remote cache" - ); - } - } - } - - await unzip(zipFilePath, extractDirPath); - - transformCodebase({ - srcDirPath: extractDirPath, - destDirPath: destDirPath - }); - - await rm(extractDirPath, { recursive: true }); -} - -function generateFileNameFromURL(params: { - url: string; - preCacheTransform: - | { - actionCacheId: string; - actionFootprint: string; - } - | undefined; -}): string { - const { preCacheTransform } = params; - - // Parse the URL - const url = new URL(params.url); - - // Extract pathname and remove leading slashes - let fileName = url.pathname.replace(/^\//, "").replace(/\//g, "_"); - - // Optionally, add query parameters replacing special characters - if (url.search) { - fileName += url.search.replace(/[&=?]/g, "-"); - } - - // Replace any characters that are not valid in filenames - fileName = fileName.replace(/[^a-zA-Z0-9-_]/g, ""); - - // Trim or pad the fileName to a specific length - fileName = fileName.substring(0, 50); - - add_pre_cache_transform: { - if (preCacheTransform === undefined) { - break add_pre_cache_transform; - } - - // Sanitize actionCacheId the same way as other components - const sanitizedActionCacheId = preCacheTransform.actionCacheId.replace( - /[^a-zA-Z0-9-_]/g, - "_" - ); - - fileName += `_${sanitizedActionCacheId}_${createHash("sha256") - .update(preCacheTransform.actionFootprint) - .digest("hex") - .substring(0, 5)}`; - } - - return fileName; -} diff --git a/src/bin/shared/downloadKeycloakDefaultTheme.ts b/src/bin/shared/downloadKeycloakDefaultTheme.ts index 04113fda..7770a616 100644 --- a/src/bin/shared/downloadKeycloakDefaultTheme.ts +++ b/src/bin/shared/downloadKeycloakDefaultTheme.ts @@ -1,12 +1,9 @@ -import { join as pathJoin } from "path"; -import { downloadAndUnzip } from "./downloadAndUnzip"; +import { join as pathJoin, relative as pathRelative } from "path"; import { type BuildOptions } from "./buildOptions"; import { assert } from "tsafe/assert"; -import * as child_process from "child_process"; -import * as fs from "fs"; -import { rmSync } from "../tools/fs.rmSync"; import { lastKeycloakVersionWithAccountV1 } from "./constants"; -import { transformCodebase } from "../tools/transformCodebase"; +import { downloadAndExtractArchive } from "../tools/downloadAndExtractArchive"; +import { isInside } from "../tools/isInside"; export type BuildOptionsLike = { cacheDirPath: string; @@ -17,363 +14,101 @@ assert(); export async function downloadKeycloakDefaultTheme(params: { keycloakVersion: string; - destDirPath: string; buildOptions: BuildOptionsLike; -}) { - const { keycloakVersion, destDirPath, buildOptions } = params; +}): Promise<{ defaultThemeDirPath: string }> { + const { keycloakVersion, buildOptions } = params; - await downloadAndUnzip({ - destDirPath, - url: `https://github.com/keycloak/keycloak/archive/refs/tags/${keycloakVersion}.zip`, - specificDirsToExtract: ["", "-community"].map( - ext => `keycloak-${keycloakVersion}/themes/src/main/resources${ext}/theme` - ), - buildOptions, - preCacheTransform: { - actionCacheId: "npm install and build", - action: async ({ destDirPath }) => { - install_common_node_modules: { - const commonResourcesDirPath = pathJoin( - destDirPath, - "keycloak", - "common", - "resources" - ); + const { extractedDirPath } = await downloadAndExtractArchive({ + url: `https://repo1.maven.org/maven2/org/keycloak/keycloak-themes/${keycloakVersion}/keycloak-themes-${keycloakVersion}.jar`, + cacheDirPath: buildOptions.cacheDirPath, + npmWorkspaceRootDirPath: buildOptions.npmWorkspaceRootDirPath, + uniqueIdOfOnOnArchiveFile: "downloadKeycloakDefaultTheme", + onArchiveFile: async params => { + if (!isInside({ dirPath: "theme", filePath: params.fileRelativePath })) { + return; + } - if (!fs.existsSync(commonResourcesDirPath)) { - break install_common_node_modules; - } + const { readFile, writeFile } = params; - if ( - !fs.existsSync(pathJoin(commonResourcesDirPath, "package.json")) - ) { - break install_common_node_modules; - } + const fileRelativePath = pathRelative("theme", params.fileRelativePath); - if (fs.existsSync(pathJoin(commonResourcesDirPath, "node_modules"))) { - break install_common_node_modules; - } - - child_process.execSync("npm install --omit=dev", { - cwd: commonResourcesDirPath, - stdio: "ignore" - }); + skip_keycloak_v2: { + if ( + !isInside({ + dirPath: pathJoin("keycloak.v2"), + filePath: fileRelativePath + }) + ) { + break skip_keycloak_v2; } - repatriate_common_resources_from_base_login_theme: { - const baseLoginThemeResourceDir = pathJoin( - destDirPath, - "base", - "login", - "resources" - ); + return; + } - if (!fs.existsSync(baseLoginThemeResourceDir)) { - break repatriate_common_resources_from_base_login_theme; + last_account_v1_transformations: { + if (lastKeycloakVersionWithAccountV1 !== keycloakVersion) { + break last_account_v1_transformations; + } + + patch_account_css: { + if ( + fileRelativePath !== + pathJoin("keycloak", "account", "resources", "css", "account.css") + ) { + break patch_account_css; } - transformCodebase({ - srcDirPath: baseLoginThemeResourceDir, - destDirPath: pathJoin( - destDirPath, - "keycloak", - "login", - "resources" + await writeFile({ + fileRelativePath, + modifiedData: Buffer.from( + (await readFile()) + .toString("utf8") + .replace("top: -34px;", "top: -34px !important;"), + "utf8" ) }); + + return; } - install_and_move_to_common_resources_generated_in_keycloak_v2: { - if ( - !fs - .readFileSync( - pathJoin( - destDirPath, - "keycloak", - "login", - "theme.properties" - ) - ) - .toString("utf8") - .includes("web_modules") - ) { - break install_and_move_to_common_resources_generated_in_keycloak_v2; - } - - const accountV2DirSrcDirPath = pathJoin( - destDirPath, - "keycloak.v2", - "account", - "src" - ); - - if (!fs.existsSync(accountV2DirSrcDirPath)) { - break install_and_move_to_common_resources_generated_in_keycloak_v2; - } - - const packageManager = fs.existsSync( - pathJoin(accountV2DirSrcDirPath, "pnpm-lock.yaml") - ) - ? "pnpm" - : "npm"; - - if (packageManager === "pnpm") { - try { - child_process.execSync(`which pnpm`); - } catch { - console.log(`Installing pnpm globally`); - child_process.execSync(`npm install -g pnpm`); - } - } - - child_process.execSync(`${packageManager} install`, { - cwd: accountV2DirSrcDirPath, - stdio: "ignore" - }); - - const packageJsonFilePath = pathJoin( - accountV2DirSrcDirPath, - "package.json" - ); - - const packageJsonRaw = fs.readFileSync(packageJsonFilePath); - - const parsedPackageJson = JSON.parse(packageJsonRaw.toString("utf8")); - - parsedPackageJson.scripts.build = parsedPackageJson.scripts.build - .replace(`${packageManager} run check-types`, "true") - .replace(`${packageManager} run babel`, "true"); - - fs.writeFileSync( - packageJsonFilePath, - Buffer.from(JSON.stringify(parsedPackageJson, null, 2), "utf8") - ); - - child_process.execSync(`${packageManager} run build`, { - cwd: accountV2DirSrcDirPath, - stdio: "ignore" - }); - - fs.writeFileSync(packageJsonFilePath, packageJsonRaw); - - fs.rmSync(pathJoin(accountV2DirSrcDirPath, "node_modules"), { - recursive: true - }); - } - - remove_keycloak_v2: { - const keycloakV2DirPath = pathJoin(destDirPath, "keycloak.v2"); - - if (!fs.existsSync(keycloakV2DirPath)) { - break remove_keycloak_v2; - } - - rmSync(keycloakV2DirPath, { recursive: true }); - } - - // Note, this is an optimization for reducing the size of the jar - remove_unused_node_modules: { - const nodeModuleDirPath = pathJoin( - destDirPath, + skip_unused_node_modules: { + const dirPath = pathJoin( "keycloak", "common", "resources", "node_modules" ); - if (!fs.existsSync(nodeModuleDirPath)) { - break remove_unused_node_modules; + if (!isInside({ dirPath, filePath: fileRelativePath })) { + break skip_unused_node_modules; } - const toDeletePerfixes = [ - "angular", - "bootstrap", - "rcue", - "font-awesome", - "ng-file-upload", - pathJoin("patternfly", "dist", "sass"), - pathJoin("patternfly", "dist", "less"), - pathJoin("patternfly", "dist", "js"), - "d3", - pathJoin("jquery", "src"), - "c3", - "core-js", - "eonasdan-bootstrap-datetimepicker", - "moment", - "react", - "patternfly-bootstrap-treeview", - "popper.js", - "tippy.js", - "jquery-match-height", - "google-code-prettify", - "patternfly-bootstrap-combobox", - "focus-trap", - "tabbable", - "scheduler", - "@types", - "datatables.net", - "datatables.net-colreorder", - "tslib", - "prop-types", - "file-selector", - "datatables.net-colreorder-bs", - "object-assign", - "warning", - "js-tokens", - "loose-envify", - "prop-types-extra", - "attr-accept", - "datatables.net-select", - "drmonty-datatables-colvis", - "datatables.net-bs", - pathJoin("@patternfly", "react"), - pathJoin("@patternfly", "patternfly", "docs") + const toKeepPrefixes = [ + ...[ + "patternfly.min.css", + "patternfly-additions.min.css", + "patternfly-additions.min.css" + ].map(fileBasename => + pathJoin(dirPath, "patternfly", "dist", "css", fileBasename) + ), + pathJoin(dirPath, "patternfly", "dist", "fonts") ]; - transformCodebase({ - srcDirPath: nodeModuleDirPath, - destDirPath: nodeModuleDirPath, - transformSourceCode: ({ sourceCode, fileRelativePath }) => { - if (fileRelativePath.endsWith(".map")) { - return undefined; - } - - if ( - toDeletePerfixes.find(prefix => - fileRelativePath.startsWith(prefix) - ) !== undefined - ) { - return undefined; - } - - if ( - fileRelativePath.startsWith( - pathJoin("patternfly", "dist", "fonts") - ) - ) { - if ( - !fileRelativePath.endsWith(".woff2") && - !fileRelativePath.endsWith(".woff") && - !fileRelativePath.endsWith(".ttf") - ) { - return undefined; - } - } - - return { modifiedSourceCode: sourceCode }; - } - }); - } - - // Just like node_modules - remove_unused_lib: { - const libDirPath = pathJoin( - destDirPath, - "keycloak", - "common", - "resources", - "lib" - ); - - if (!fs.existsSync(libDirPath)) { - break remove_unused_lib; + if ( + toKeepPrefixes.find(prefix => + fileRelativePath.startsWith(prefix) + ) !== undefined + ) { + break skip_unused_node_modules; } - const toDeletePerfixes = [ - "ui-ace", - "filesaver", - "fileupload", - "angular", - "ui-ace" - ]; - - transformCodebase({ - srcDirPath: libDirPath, - destDirPath: libDirPath, - transformSourceCode: ({ sourceCode, fileRelativePath }) => { - if (fileRelativePath.endsWith(".map")) { - return undefined; - } - - if ( - toDeletePerfixes.find(prefix => - fileRelativePath.startsWith(prefix) - ) !== undefined - ) { - return undefined; - } - - return { modifiedSourceCode: sourceCode }; - } - }); - } - - last_account_v1_transformations: { - if (lastKeycloakVersionWithAccountV1 !== keycloakVersion) { - break last_account_v1_transformations; - } - - { - const accountCssFilePath = pathJoin( - destDirPath, - "keycloak", - "account", - "resources", - "css", - "account.css" - ); - - fs.writeFileSync( - accountCssFilePath, - Buffer.from( - fs - .readFileSync(accountCssFilePath) - .toString("utf8") - .replace("top: -34px;", "top: -34px !important;"), - "utf8" - ) - ); - } - - // Note, this is an optimization for reducing the size of the jar, - // For this version we know exactly which resources are used. - { - const nodeModulesDirPath = pathJoin( - destDirPath, - "keycloak", - "common", - "resources", - "node_modules" - ); - - const toKeepPrefixes = [ - ...[ - "patternfly.min.css", - "patternfly-additions.min.css", - "patternfly-additions.min.css" - ].map(fileBasename => - pathJoin("patternfly", "dist", "css", fileBasename) - ), - pathJoin("patternfly", "dist", "fonts") - ]; - - transformCodebase({ - srcDirPath: nodeModulesDirPath, - destDirPath: nodeModulesDirPath, - transformSourceCode: ({ sourceCode, fileRelativePath }) => { - if ( - toKeepPrefixes.find(prefix => - fileRelativePath.startsWith(prefix) - ) === undefined - ) { - return undefined; - } - return { modifiedSourceCode: sourceCode }; - } - }); - } + return; } } + + await writeFile({ fileRelativePath }); } }); + + return { defaultThemeDirPath: extractedDirPath }; } diff --git a/src/bin/shared/downloadKeycloakStaticResources.ts b/src/bin/shared/downloadKeycloakStaticResources.ts index 4aa57f36..4044907e 100644 --- a/src/bin/shared/downloadKeycloakStaticResources.ts +++ b/src/bin/shared/downloadKeycloakStaticResources.ts @@ -1,16 +1,15 @@ import { transformCodebase } from "../tools/transformCodebase"; import { join as pathJoin } from "path"; -import { downloadKeycloakDefaultTheme } from "./downloadKeycloakDefaultTheme"; +import { + downloadKeycloakDefaultTheme, + type BuildOptionsLike as BuildOptionsLike_downloadKeycloakDefaultTheme +} from "./downloadKeycloakDefaultTheme"; import { resources_common, type ThemeType } from "./constants"; import type { BuildOptions } from "./buildOptions"; import { assert } from "tsafe/assert"; -import * as crypto from "crypto"; -import { rmSync } from "../tools/fs.rmSync"; +import { existsAsync } from "../tools/fs.existsAsync"; -export type BuildOptionsLike = { - cacheDirPath: string; - npmWorkspaceRootDirPath: string; -}; +export type BuildOptionsLike = BuildOptionsLike_downloadKeycloakDefaultTheme & {}; assert(); @@ -22,32 +21,33 @@ export async function downloadKeycloakStaticResources(params: { }) { const { themeType, themeDirPath, keycloakVersion, buildOptions } = params; - const tmpDirPath = pathJoin( - buildOptions.cacheDirPath, - `downloadKeycloakStaticResources_tmp_${crypto - .createHash("sha256") - .update(`${themeType}-${keycloakVersion}`) - .digest("hex") - .slice(0, 8)}` - ); - - await downloadKeycloakDefaultTheme({ + const { defaultThemeDirPath } = await downloadKeycloakDefaultTheme({ keycloakVersion, - destDirPath: tmpDirPath, buildOptions }); - const resourcesPath = pathJoin(themeDirPath, themeType, "resources"); + const resourcesDirPath = pathJoin(themeDirPath, themeType, "resources"); + + repatriate_base_resources: { + const srcDirPath = pathJoin(defaultThemeDirPath, "base", themeType, "resources"); + + if (!(await existsAsync(srcDirPath))) { + break repatriate_base_resources; + } + + transformCodebase({ + srcDirPath, + destDirPath: resourcesDirPath + }); + } transformCodebase({ - srcDirPath: pathJoin(tmpDirPath, "keycloak", themeType, "resources"), - destDirPath: resourcesPath + srcDirPath: pathJoin(defaultThemeDirPath, "keycloak", themeType, "resources"), + destDirPath: resourcesDirPath }); transformCodebase({ - srcDirPath: pathJoin(tmpDirPath, "keycloak", "common", "resources"), - destDirPath: pathJoin(resourcesPath, resources_common) + srcDirPath: pathJoin(defaultThemeDirPath, "keycloak", "common", "resources"), + destDirPath: pathJoin(resourcesDirPath, resources_common) }); - - rmSync(tmpDirPath, { recursive: true }); } diff --git a/src/bin/tools/downloadAndExtractArchive/downloadAndExtractArchive.ts b/src/bin/tools/downloadAndExtractArchive/downloadAndExtractArchive.ts new file mode 100644 index 00000000..209bd7aa --- /dev/null +++ b/src/bin/tools/downloadAndExtractArchive/downloadAndExtractArchive.ts @@ -0,0 +1,101 @@ +import fetch from "make-fetch-happen"; +import { mkdir, unlink, writeFile, readdir } from "fs/promises"; +import { dirname as pathDirname, join as pathJoin } from "path"; +import { assert } from "tsafe/assert"; +import { extractArchive } from "../extractArchive"; +import { existsAsync } from "../fs.existsAsync"; +import { getProxyFetchOptions } from "./fetchProxyOptions"; +import * as crypto from "crypto"; + +export async function downloadAndExtractArchive(params: { + url: string; + uniqueIdOfOnOnArchiveFile: string; + onArchiveFile: (params: { + fileRelativePath: string; + readFile: () => Promise; + writeFile: (params: { + fileRelativePath: string; + modifiedData?: Buffer; + }) => Promise; + }) => Promise; + cacheDirPath: string; + npmWorkspaceRootDirPath: string; +}): Promise<{ extractedDirPath: string }> { + const { + url, + uniqueIdOfOnOnArchiveFile, + onArchiveFile, + cacheDirPath, + npmWorkspaceRootDirPath + } = params; + + const archiveFileBasename = url.split("?")[0].split("/").reverse()[0]; + + const archiveFilePath = pathJoin(cacheDirPath, archiveFileBasename); + + download: { + if (await existsAsync(archiveFilePath)) { + break download; + } + + await mkdir(pathDirname(archiveFilePath), { recursive: true }); + + const response = await fetch( + url, + await getProxyFetchOptions({ npmWorkspaceRootDirPath }) + ); + + response.body?.setMaxListeners(Number.MAX_VALUE); + assert(typeof response.body !== "undefined" && response.body != null); + + await writeFile(archiveFilePath, response.body); + } + + const extractDirBasename = `${archiveFileBasename.split(".")[0]}_${uniqueIdOfOnOnArchiveFile}_${crypto + .createHash("sha256") + .update(onArchiveFile.toString()) + .digest("hex") + .substring(0, 5)}`; + + await Promise.all( + (await readdir(cacheDirPath)) + .filter( + (() => { + const prefix = extractDirBasename + .split("_") + .reverse() + .slice(1) + .reverse() + .join("_"); + + return basename => + basename !== extractDirBasename && basename.startsWith(prefix); + })() + ) + .map(basename => unlink(pathJoin(cacheDirPath, basename))) + ); + + const extractedDirPath = pathJoin(cacheDirPath, extractDirBasename); + + extract_and_transform: { + if (await existsAsync(extractedDirPath)) { + break extract_and_transform; + } + + await extractArchive({ + archiveFilePath, + onArchiveFile: async ({ relativeFilePathInArchive, readFile, writeFile }) => + onArchiveFile({ + fileRelativePath: relativeFilePathInArchive, + readFile, + writeFile: ({ fileRelativePath, modifiedData }) => + writeFile({ + filePath: pathJoin(extractedDirPath, fileRelativePath), + modifiedData + }) + }) + }); + } + + return { extractedDirPath }; +} diff --git a/src/bin/tools/downloadAndExtractArchive/fetchProxyOptions.ts b/src/bin/tools/downloadAndExtractArchive/fetchProxyOptions.ts new file mode 100644 index 00000000..3a903d6b --- /dev/null +++ b/src/bin/tools/downloadAndExtractArchive/fetchProxyOptions.ts @@ -0,0 +1,96 @@ +import { exec as execCallback } from "child_process"; +import { readFile } from "fs/promises"; +import { type FetchOptions } from "make-fetch-happen"; +import { promisify } from "util"; + +function ensureArray(arg0: T | T[]) { + return Array.isArray(arg0) ? arg0 : typeof arg0 === "undefined" ? [] : [arg0]; +} + +function ensureSingleOrNone(arg0: T | T[]) { + if (!Array.isArray(arg0)) return arg0; + if (arg0.length === 0) return undefined; + if (arg0.length === 1) return arg0[0]; + throw new Error( + "Illegal configuration, expected a single value but found multiple: " + + arg0.map(String).join(", ") + ); +} + +type NPMConfig = Record; + +/** + * Get npm configuration as map + */ +async function getNmpConfig(params: { npmWorkspaceRootDirPath: string }) { + const { npmWorkspaceRootDirPath } = params; + + const exec = promisify(execCallback); + + const stdout = await exec("npm config get", { + encoding: "utf8", + cwd: npmWorkspaceRootDirPath + }).then(({ stdout }) => stdout); + + const npmConfigReducer = (cfg: NPMConfig, [key, value]: [string, string]) => + key in cfg + ? { ...cfg, [key]: [...ensureArray(cfg[key]), value] } + : { ...cfg, [key]: value }; + + return stdout + .split("\n") + .filter(line => !line.startsWith(";")) + .map(line => line.trim()) + .map(line => line.split("=", 2) as [string, string]) + .reduce(npmConfigReducer, {} as NPMConfig); +} + +export type ProxyFetchOptions = Pick< + FetchOptions, + "proxy" | "noProxy" | "strictSSL" | "cert" | "ca" +>; + +export async function getProxyFetchOptions(params: { + npmWorkspaceRootDirPath: string; +}): Promise { + const { npmWorkspaceRootDirPath } = params; + + const cfg = await getNmpConfig({ npmWorkspaceRootDirPath }); + + const proxy = ensureSingleOrNone(cfg["https-proxy"] ?? cfg["proxy"]); + const noProxy = cfg["noproxy"] ?? cfg["no-proxy"]; + + function maybeBoolean(arg0: string | undefined) { + return typeof arg0 === "undefined" ? undefined : Boolean(arg0); + } + + const strictSSL = maybeBoolean(ensureSingleOrNone(cfg["strict-ssl"])); + const cert = cfg["cert"]; + const ca = ensureArray(cfg["ca"] ?? cfg["ca[]"]); + const cafile = ensureSingleOrNone(cfg["cafile"]); + + if (typeof cafile !== "undefined" && cafile !== "null") { + ca.push( + ...(await (async () => { + function chunks(arr: T[], size: number = 2) { + return arr + .map((_, i) => i % size == 0 && arr.slice(i, i + size)) + .filter(Boolean) as T[][]; + } + + const cafileContent = await readFile(cafile, "utf-8"); + return chunks(cafileContent.split(/(-----END CERTIFICATE-----)/), 2).map( + ca => ca.join("").replace(/^\n/, "").replace(/\n/g, "\\n") + ); + })()) + ); + } + + return { + proxy, + noProxy, + strictSSL, + cert, + ca: ca.length === 0 ? undefined : ca + }; +} diff --git a/src/bin/tools/downloadAndExtractArchive/index.ts b/src/bin/tools/downloadAndExtractArchive/index.ts new file mode 100644 index 00000000..2590e230 --- /dev/null +++ b/src/bin/tools/downloadAndExtractArchive/index.ts @@ -0,0 +1 @@ +export * from "./downloadAndExtractArchive"; diff --git a/src/bin/tools/extractArchive.ts b/src/bin/tools/extractArchive.ts new file mode 100644 index 00000000..b73043e6 --- /dev/null +++ b/src/bin/tools/extractArchive.ts @@ -0,0 +1,125 @@ +import fs from "fs/promises"; +import fsSync from "fs"; +import yauzl from "yauzl"; +import stream from "stream"; +import { Deferred } from "evt/tools/Deferred"; +import { dirname as pathDirname, sep as pathSep } from "path"; + +export async function extractArchive(params: { + archiveFilePath: string; + onArchiveFile: (params: { + relativeFilePathInArchive: string; + readFile: () => Promise; + writeFile: (params: { filePath: string; modifiedData?: Buffer }) => Promise; + }) => Promise; +}) { + const { archiveFilePath, onArchiveFile } = params; + + const zipFile = await new Promise((resolve, reject) => { + yauzl.open(archiveFilePath, { lazyEntries: true }, async (error, zipFile) => { + if (error !== null) { + reject(error); + return; + } + resolve(zipFile); + }); + }); + + const dDone = new Deferred(); + + zipFile.once("end", () => { + zipFile.close(); + dDone.resolve(); + }); + + // TODO: See benchmark if using a class here improves the performance over anonymous functions + class FileWriter { + constructor(private entry: yauzl.Entry) {} + + public async writeToFile(params: { + filePath: string; + modifiedData?: Buffer; + }): Promise { + const { filePath, modifiedData } = params; + + await fs.mkdir(pathDirname(filePath), { recursive: true }); + + if (modifiedData !== undefined) { + await fs.writeFile(filePath, modifiedData); + return; + } + + const readStream = await new Promise(resolve => + zipFile.openReadStream(this.entry, async (error, readStream) => { + if (error !== null) { + dDone.reject(error); + return; + } + + resolve(readStream); + }) + ); + + const dDoneWithFile = new Deferred(); + + stream.pipeline(readStream, fsSync.createWriteStream(filePath), error => { + if (error !== null) { + dDone.reject(error); + return; + } + + dDoneWithFile.resolve(); + }); + + await dDoneWithFile.pr; + } + + public readFile(): Promise { + return new Promise(resolve => + zipFile.openReadStream(this.entry, async (error, readStream) => { + if (error !== null) { + dDone.reject(error); + return; + } + + const chunks: Buffer[] = []; + + readStream.on("data", chunk => { + chunks.push(chunk); + }); + + readStream.on("end", () => { + resolve(Buffer.concat(chunks)); + }); + + readStream.on("error", error => { + dDone.reject(error); + }); + }) + ); + } + } + + zipFile.on("entry", async (entry: yauzl.Entry) => { + handle_file: { + // NOTE: Skip directories + if (entry.fileName.endsWith(pathSep)) { + break handle_file; + } + + const fileWriter = new FileWriter(entry); + + await onArchiveFile({ + relativeFilePathInArchive: entry.fileName.split("/").join(pathSep), + readFile: fileWriter.readFile.bind(fileWriter), + writeFile: fileWriter.writeToFile.bind(fileWriter) + }); + } + + zipFile.readEntry(); + }); + + zipFile.readEntry(); + + await dDone.pr; +} diff --git a/src/bin/tools/unzip.ts b/src/bin/tools/unzip.ts deleted file mode 100644 index eb4827be..00000000 --- a/src/bin/tools/unzip.ts +++ /dev/null @@ -1,149 +0,0 @@ -import fsp from "node:fs/promises"; -import fs from "fs"; -import path from "node:path"; -import yauzl from "yauzl"; -import yazl from "yazl"; -import stream from "node:stream"; -import { promisify } from "node:util"; - -const pipeline = promisify(stream.pipeline); - -async function pathExists(path: string) { - try { - await fsp.stat(path); - return true; - } catch (error) { - if ((error as { code: string }).code === "ENOENT") { - return false; - } - throw error; - } -} - -// Handlings of non posix path is not implemented correctly -// it work by coincidence. Don't have the time to fix but it should be fixed. -export async function unzip( - file: string, - targetFolder: string, - specificDirsToExtract?: string[] -) { - specificDirsToExtract = specificDirsToExtract?.map(dirPath => { - if (!dirPath.endsWith("/") || !dirPath.endsWith("\\")) { - dirPath += "/"; - } - - return dirPath; - }); - - if (!targetFolder.endsWith("/") || !targetFolder.endsWith("\\")) { - targetFolder += "/"; - } - if (!fs.existsSync(targetFolder)) { - fs.mkdirSync(targetFolder, { recursive: true }); - } - - return new Promise((resolve, reject) => { - yauzl.open(file, { lazyEntries: true }, async (err, zipfile) => { - if (err) { - reject(err); - return; - } - - zipfile.readEntry(); - - zipfile.on("entry", async entry => { - if (specificDirsToExtract !== undefined) { - const dirPath = specificDirsToExtract.find(dirPath => - entry.fileName.startsWith(dirPath) - ); - - // Skip files outside of the unzipSubPath - if (dirPath === undefined) { - zipfile.readEntry(); - return; - } - - // Remove the unzipSubPath from the file name - entry.fileName = entry.fileName.substring(dirPath.length); - } - - const target = path.join(targetFolder, entry.fileName); - - // Directory file names end with '/'. - // Note that entries for directories themselves are optional. - // An entry's fileName implicitly requires its parent directories to exist. - if (/[\/\\]$/.test(target)) { - await fsp.mkdir(target, { recursive: true }); - - zipfile.readEntry(); - return; - } - - // Skip existing files - if (await pathExists(target)) { - zipfile.readEntry(); - return; - } - - zipfile.openReadStream(entry, async (err, readStream) => { - if (err) { - reject(err); - return; - } - - await fsp.mkdir(path.dirname(target), { - recursive: true - }); - - await pipeline(readStream, fs.createWriteStream(target)); - - zipfile.readEntry(); - }); - }); - - zipfile.once("end", function () { - zipfile.close(); - resolve(); - }); - }); - }); -} - -// NOTE: This code was directly copied from ChatGPT and appears to function as expected. -// However, confidence in its complete accuracy and robustness is limited. -export async function zip(sourceFolder: string, targetZip: string) { - return new Promise(async (resolve, reject) => { - const zipfile = new yazl.ZipFile(); - const files: string[] = []; - - // Recursive function to explore directories and their subdirectories - async function exploreDir(dir: string) { - const dirContent = await fsp.readdir(dir); - for (const file of dirContent) { - const filePath = path.join(dir, file); - const stat = await fsp.stat(filePath); - if (stat.isDirectory()) { - await exploreDir(filePath); - } else if (stat.isFile()) { - files.push(filePath); - } - } - } - - // Collecting all files to be zipped - await exploreDir(sourceFolder); - - // Adding files to zip - for (const file of files) { - const relativePath = path.relative(sourceFolder, file); - zipfile.addFile(file, relativePath); - } - - zipfile.outputStream - .pipe(fs.createWriteStream(targetZip)) - .on("close", () => resolve()) - .on("error", err => reject(err)); // Listen to error events - - zipfile.end(); - }); -} diff --git a/yarn.lock b/yarn.lock index 742d1ceb..8dead454 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3562,13 +3562,6 @@ dependencies: "@types/node" "*" -"@types/yazl@^2.4.5": - version "2.4.5" - resolved "https://registry.yarnpkg.com/@types/yazl/-/yazl-2.4.5.tgz#0e21674799c7690afa23aeaff59806be5fe7494d" - integrity sha512-qpmPfx32HS7vlGJf7EsoM9qJnLZhXJBf1KH0hzfdc+D794rljQWh4H0I/UrZy+6Nhqn0l2jdBZXBGZtR1vnHqw== - dependencies: - "@types/node" "*" - "@typescript-eslint/scope-manager@5.59.0": version "5.59.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.59.0.tgz#86501d7a17885710b6716a23be2e93fc54a4fe8c" @@ -13307,13 +13300,6 @@ yauzl@^2.10.0: buffer-crc32 "~0.2.3" fd-slicer "~1.1.0" -yazl@^2.5.1: - version "2.5.1" - resolved "https://registry.yarnpkg.com/yazl/-/yazl-2.5.1.tgz#a3d65d3dd659a5b0937850e8609f22fffa2b5c35" - integrity sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw== - dependencies: - buffer-crc32 "~0.2.3" - yn@3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"