From 983af57842b6035c6810955ff466a279668cd84b Mon Sep 17 00:00:00 2001 From: garronej Date: Thu, 24 Aug 2023 08:58:00 +0200 Subject: [PATCH] Actually remove non used resources --- scripts/generate-i18n-messages.ts | 4 +- src/bin/copy-keycloak-resources-to-public.ts | 3 +- src/bin/download-builtin-keycloak-theme.ts | 4 +- .../downloadKeycloakStaticResources.ts | 36 ++++++++++- .../generateTheme/generateTheme.ts | 11 +++- .../generateTheme/readStaticResourcesUsage.ts | 6 +- src/bin/tools/downloadAndUnzip.ts | 62 ++++++++++++++++--- src/bin/tools/transformCodebase.ts | 13 ++-- test/bin/readStaticResourcesUsage.spec.ts | 19 +++++- 9 files changed, 129 insertions(+), 29 deletions(-) diff --git a/scripts/generate-i18n-messages.ts b/scripts/generate-i18n-messages.ts index 3a8adcee..756cef4f 100644 --- a/scripts/generate-i18n-messages.ts +++ b/scripts/generate-i18n-messages.ts @@ -24,9 +24,9 @@ async function main() { fs.rmSync(tmpDirPath, { "recursive": true, "force": true }); await downloadBuiltinKeycloakTheme({ + "projectDirPath": getProjectRoot(), keycloakVersion, - "destDirPath": tmpDirPath, - isSilent + "destDirPath": tmpDirPath }); type Dictionary = { [idiomId: string]: string }; diff --git a/src/bin/copy-keycloak-resources-to-public.ts b/src/bin/copy-keycloak-resources-to-public.ts index 09cad100..3fec0214 100644 --- a/src/bin/copy-keycloak-resources-to-public.ts +++ b/src/bin/copy-keycloak-resources-to-public.ts @@ -27,7 +27,8 @@ import * as fs from "fs"; projectDirPath, "keycloakVersion": buildOptions.keycloakVersionDefaultAssets, "themeType": themeType, - "themeDirPath": keycloakDirInPublicDir + "themeDirPath": keycloakDirInPublicDir, + "usedResources": undefined }); } diff --git a/src/bin/download-builtin-keycloak-theme.ts b/src/bin/download-builtin-keycloak-theme.ts index e37480ba..db098063 100644 --- a/src/bin/download-builtin-keycloak-theme.ts +++ b/src/bin/download-builtin-keycloak-theme.ts @@ -12,8 +12,6 @@ export async function downloadBuiltinKeycloakTheme(params: { projectDirPath: str const start = Date.now(); - console.log("Downloading Keycloak theme...", { keycloakVersion }); - await downloadAndUnzip({ "doUseCache": true, projectDirPath, @@ -21,7 +19,7 @@ export async function downloadBuiltinKeycloakTheme(params: { projectDirPath: str "url": `https://github.com/keycloak/keycloak/archive/refs/tags/${keycloakVersion}.zip`, "specificDirsToExtract": ["", "-community"].map(ext => `keycloak-${keycloakVersion}/themes/src/main/resources${ext}/theme`), "preCacheTransform": { - "actionCacheId": "Build Keycloak resources", + "actionCacheId": "npm install and build", "action": async ({ destDirPath }) => { install_common_node_modules: { const commonResourcesDirPath = pathJoin(destDirPath, "keycloak", "common", "resources"); diff --git a/src/bin/keycloakify/generateTheme/downloadKeycloakStaticResources.ts b/src/bin/keycloakify/generateTheme/downloadKeycloakStaticResources.ts index 334a45cd..04167cbb 100644 --- a/src/bin/keycloakify/generateTheme/downloadKeycloakStaticResources.ts +++ b/src/bin/keycloakify/generateTheme/downloadKeycloakStaticResources.ts @@ -17,9 +17,19 @@ export async function downloadKeycloakStaticResources( themeType: ThemeType; themeDirPath: string; keycloakVersion: string; + usedResources: { + resourcesCommonFilePaths: string[]; + resourcesFilePaths: string[]; + } | undefined } ) { - const { projectDirPath, themeType, themeDirPath, keycloakVersion } = params; + const { projectDirPath, themeType, themeDirPath, keycloakVersion, usedResources } = params; + + console.log({ + themeDirPath, + keycloakVersion, + usedResources + }); const tmpDirPath = pathJoin( themeDirPath, @@ -35,12 +45,32 @@ export async function downloadKeycloakStaticResources( transformCodebase({ "srcDirPath": pathJoin(tmpDirPath, "keycloak", themeType, "resources"), - "destDirPath": pathJoin(themeDirPath, pathRelative(basenameOfKeycloakDirInPublicDir, resourcesDirPathRelativeToPublicDir)) + "destDirPath": pathJoin(themeDirPath, pathRelative(basenameOfKeycloakDirInPublicDir, resourcesDirPathRelativeToPublicDir)), + "transformSourceCode": + usedResources === undefined + ? undefined + : ({ fileRelativePath, sourceCode }) => { + if (!usedResources.resourcesFilePaths.includes(fileRelativePath)) { + return undefined; + } + + return { "modifiedSourceCode": sourceCode }; + } }); transformCodebase({ "srcDirPath": pathJoin(tmpDirPath, "keycloak", "common", "resources"), - "destDirPath": pathJoin(themeDirPath, pathRelative(basenameOfKeycloakDirInPublicDir, resourcesCommonDirPathRelativeToPublicDir)) + "destDirPath": pathJoin(themeDirPath, pathRelative(basenameOfKeycloakDirInPublicDir, resourcesCommonDirPathRelativeToPublicDir)), + "transformSourceCode": + usedResources === undefined + ? undefined + : ({ fileRelativePath, sourceCode }) => { + if (!usedResources.resourcesCommonFilePaths.includes(fileRelativePath)) { + return undefined; + } + + return { "modifiedSourceCode": sourceCode }; + } }); fs.rmSync(tmpDirPath, { "recursive": true, "force": true }); diff --git a/src/bin/keycloakify/generateTheme/generateTheme.ts b/src/bin/keycloakify/generateTheme/generateTheme.ts index 74ecb315..c47bc22b 100644 --- a/src/bin/keycloakify/generateTheme/generateTheme.ts +++ b/src/bin/keycloakify/generateTheme/generateTheme.ts @@ -12,6 +12,7 @@ import { downloadKeycloakStaticResources } from "./downloadKeycloakStaticResourc import { readFieldNameUsage } from "./readFieldNameUsage"; import { readExtraPagesNames } from "./readExtraPageNames"; import { generateMessageProperties } from "./generateMessageProperties"; +import { readStaticResourcesUsage } from "./readStaticResourcesUsage"; export type BuildOptionsLike = { themeName: string; @@ -172,7 +173,8 @@ export async function generateTheme(params: { projectDirPath, "keycloakVersion": buildOptions.keycloakVersionDefaultAssets, "themeDirPath": keycloakDirInPublicDir, - themeType + themeType, + "usedResources": undefined }); if (themeType !== themeTypes[0]) { @@ -197,7 +199,12 @@ export async function generateTheme(params: { projectDirPath, "keycloakVersion": buildOptions.keycloakVersionDefaultAssets, themeDirPath, - themeType + themeType, + "usedResources": readStaticResourcesUsage({ + keycloakifySrcDirPath, + themeSrcDirPath, + themeType + }) }); fs.writeFileSync( diff --git a/src/bin/keycloakify/generateTheme/readStaticResourcesUsage.ts b/src/bin/keycloakify/generateTheme/readStaticResourcesUsage.ts index 646e1765..11805f3c 100644 --- a/src/bin/keycloakify/generateTheme/readStaticResourcesUsage.ts +++ b/src/bin/keycloakify/generateTheme/readStaticResourcesUsage.ts @@ -19,13 +19,11 @@ export function readStaticResourcesUsage(params: { keycloakifySrcDirPath: string for (const filePath of filePaths) { const rawSourceFile = fs.readFileSync(filePath).toString("utf8"); - if (!rawSourceFile.includes("resourcesCommonPath")) { + if (!rawSourceFile.includes("resourcesCommonPath") && !rawSourceFile.includes("resourcesPath")) { continue; } - if (!rawSourceFile.includes("resourcesPath")) { - continue; - } + console.log("=========>", filePath); const wrap = readPaths({ rawSourceFile }); diff --git a/src/bin/tools/downloadAndUnzip.ts b/src/bin/tools/downloadAndUnzip.ts index 0b1eb0ed..5d7dcc0c 100644 --- a/src/bin/tools/downloadAndUnzip.ts +++ b/src/bin/tools/downloadAndUnzip.ts @@ -10,8 +10,46 @@ import { unzip, zip } from "./unzip"; const exec = promisify(execCallback); -function sha256(s: string) { - return createHash("sha256").update(s).digest("hex"); +function generateFileNameFromURL(params: { + url: string; + preCacheTransform: + | { + actionCacheId: string; + actionFootprint: string; + } + | undefined; +}): string { + const { preCacheTransform } = params; + + // Parse the URL + const url = new URL(params.url); + + // Extract pathname and remove leading slashes + let fileName = url.pathname.replace(/^\//, "").replace(/\//g, "_"); + + // Optionally, add query parameters replacing special characters + if (url.search) { + fileName += url.search.replace(/[&=?]/g, "-"); + } + + // Replace any characters that are not valid in filenames + fileName = fileName.replace(/[^a-zA-Z0-9-_]/g, ""); + + // Trim or pad the fileName to a specific length + fileName = fileName.substring(0, 50); + + add_pre_cache_transform: { + if (preCacheTransform === undefined) { + break add_pre_cache_transform; + } + + // Sanitize actionCacheId the same way as other components + const sanitizedActionCacheId = preCacheTransform.actionCacheId.replace(/[^a-zA-Z0-9-_]/g, "_"); + + fileName += `_${sanitizedActionCacheId}_${createHash("sha256").update(preCacheTransform.actionFootprint).digest("hex").substring(0, 5)}`; + } + + return fileName; } async function exists(path: string) { @@ -133,14 +171,22 @@ export async function downloadAndUnzip( ) { const { url, destDirPath, specificDirsToExtract, preCacheTransform, ...rest } = params; - const hash = sha256( - JSON.stringify({ url }) + (preCacheTransform === undefined ? "" : `${preCacheTransform.actionCacheId}${preCacheTransform.action.toString()}`) - ).substring(0, 15); + const zipFileBasename = generateFileNameFromURL({ + url, + "preCacheTransform": + preCacheTransform === undefined + ? undefined + : { + "actionCacheId": preCacheTransform.actionCacheId, + "actionFootprint": preCacheTransform.action.toString() + } + }); + const cacheRoot = !rest.doUseCache ? `tmp_${Math.random().toString().slice(2, 12)}` : pathJoin(process.env.XDG_CACHE_HOME ?? pathJoin(rest.projectDirPath, "node_modules", ".cache"), "keycloakify"); - const zipFilePath = pathJoin(cacheRoot, `_${hash}.zip`); - const extractDirPath = pathJoin(cacheRoot, `tmp_unzip_${hash}`); + const zipFilePath = pathJoin(cacheRoot, `${zipFileBasename}.zip`); + const extractDirPath = pathJoin(cacheRoot, `tmp_unzip_${zipFileBasename}`); if (!(await exists(zipFilePath))) { const opts = await getFetchOptions(); @@ -181,5 +227,7 @@ export async function downloadAndUnzip( if (!rest.doUseCache) { await rm(cacheRoot, { "recursive": true }); + } else { + await rm(extractDirPath, { "recursive": true }); } } diff --git a/src/bin/tools/transformCodebase.ts b/src/bin/tools/transformCodebase.ts index 9e04074a..2064fe7d 100644 --- a/src/bin/tools/transformCodebase.ts +++ b/src/bin/tools/transformCodebase.ts @@ -3,7 +3,7 @@ import * as path from "path"; import { crawl } from "./crawl"; import { id } from "tsafe/id"; -type TransformSourceCode = (params: { sourceCode: Buffer; filePath: string }) => +type TransformSourceCode = (params: { sourceCode: Buffer; filePath: string; fileRelativePath: string }) => | { modifiedSourceCode: Buffer; newFileName?: string; @@ -20,26 +20,27 @@ export function transformCodebase(params: { srcDirPath: string; destDirPath: str })) } = params; - for (const file_relative_path of crawl({ "dirPath": srcDirPath, "returnedPathsType": "relative to dirPath" })) { - const filePath = path.join(srcDirPath, file_relative_path); + for (const fileRelativePath of crawl({ "dirPath": srcDirPath, "returnedPathsType": "relative to dirPath" })) { + const filePath = path.join(srcDirPath, fileRelativePath); const transformSourceCodeResult = transformSourceCode({ "sourceCode": fs.readFileSync(filePath), - filePath + filePath, + fileRelativePath }); if (transformSourceCodeResult === undefined) { continue; } - fs.mkdirSync(path.dirname(path.join(destDirPath, file_relative_path)), { + fs.mkdirSync(path.dirname(path.join(destDirPath, fileRelativePath)), { "recursive": true }); const { newFileName, modifiedSourceCode } = transformSourceCodeResult; fs.writeFileSync( - path.join(path.dirname(path.join(destDirPath, file_relative_path)), newFileName ?? path.basename(file_relative_path)), + path.join(path.dirname(path.join(destDirPath, fileRelativePath)), newFileName ?? path.basename(fileRelativePath)), modifiedSourceCode ); } diff --git a/test/bin/readStaticResourcesUsage.spec.ts b/test/bin/readStaticResourcesUsage.spec.ts index 1008b95f..74b74db2 100644 --- a/test/bin/readStaticResourcesUsage.spec.ts +++ b/test/bin/readStaticResourcesUsage.spec.ts @@ -7,7 +7,8 @@ describe("Ensure it's able to extract used Keycloak resources", () => { "resourcesCommonFilePaths": [ "node_modules/patternfly/dist/css/patternfly.min.css", "node_modules/patternfly/dist/css/patternfly-additions.min.css", - "lib/zocial/zocial.css" + "lib/zocial/zocial.css", + "node_modules/jquery/dist/jquery.min.js" ], "resourcesFilePaths": ["css/login.css"] }; @@ -28,6 +29,12 @@ describe("Ensure it's able to extract used Keycloak resources", () => { "htmlClassName": getClassName("kcHtmlClass"), "bodyClassName": undefined }); + + const { prLoaded, remove } = headInsert({ + "type": "javascript", + "src": \`\${kcContext.url.resourcesCommonPath}/node_modules/jquery/dist/jquery.min.js\` + }); + ` }); @@ -52,6 +59,11 @@ describe("Ensure it's able to extract used Keycloak resources", () => { "bodyClassName": undefined }); + const { prLoaded, remove } = headInsert({ + "type": "javascript", + "src": kcContext.url.resourcesCommonPath + "/node_modules/jquery/dist/jquery.min.js\" + }); + ` }); @@ -82,6 +94,11 @@ describe("Ensure it's able to extract used Keycloak resources", () => { "bodyClassName": undefined }); + const { prLoaded, remove } = headInsert({ + "type": "javascript", + "src": path.join(kcContext.url.resourcesCommonPath, "/node_modules/jquery/dist/jquery.min.js") + }); + ` });