diff --git a/lib/analyze-action-post.js b/lib/analyze-action-post.js index b4ee3c8d5b..4b90e5a46d 100644 --- a/lib/analyze-action-post.js +++ b/lib/analyze-action-post.js @@ -120735,6 +120735,23 @@ ${output}` ); } }, + async resolveDatabase(databasePath) { + const codeqlArgs = [ + "resolve", + "database", + databasePath, + "--format=json", + ...getExtraOptionsFromEnv(["resolve", "database"]) + ]; + const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true }); + try { + return JSON.parse(output); + } catch (e) { + throw new Error( + `Unexpected output from codeql resolve database --format=json: ${e}` + ); + } + }, async mergeResults(sarifFiles, outputFile, { mergeRunsFromEqualCategory = false }) { diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 1ba7f39e93..c4d488ab8a 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -88512,7 +88512,7 @@ function computeChangedFiles(baseFileOids, overlayFileOids) { var CACHE_VERSION = 1; var CACHE_PREFIX = "codeql-overlay-base-database"; var MAX_CACHE_OPERATION_MS = 6e5; -function checkOverlayBaseDatabase(config, logger, warningPrefix) { +async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) { const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); if (!fs3.existsSync(baseDatabaseOidsFilePath)) { logger.warning( @@ -88520,6 +88520,23 @@ function checkOverlayBaseDatabase(config, logger, warningPrefix) { ); return false; } + for (const language of config.languages) { + const dbPath = getCodeQLDatabasePath(config, language); + try { + const resolveDatabaseOutput = await codeql.resolveDatabase(dbPath); + if (resolveDatabaseOutput === void 0 || !("overlayBaseSpecifier" in resolveDatabaseOutput)) { + logger.info(`${warningPrefix}: no overlayBaseSpecifier defined`); + return false; + } else { + logger.info( + `Overlay base specifier for ${language} overlay-base database found: ${resolveDatabaseOutput.overlayBaseSpecifier}` + ); + } + } catch (e) { + logger.warning(`${warningPrefix}: failed to resolve database: ${e}`); + return false; + } + } return true; } async function cleanupAndUploadOverlayBaseDatabaseToCache(codeql, config, logger) { @@ -88542,7 +88559,8 @@ async function cleanupAndUploadOverlayBaseDatabaseToCache(codeql, config, logger ); return false; } - const databaseIsValid = checkOverlayBaseDatabase( + const databaseIsValid = await checkOverlayBaseDatabase( + codeql, config, logger, "Abort uploading overlay-base database to cache" @@ -90887,6 +90905,23 @@ ${output}` ); } }, + async resolveDatabase(databasePath) { + const codeqlArgs = [ + "resolve", + "database", + databasePath, + "--format=json", + ...getExtraOptionsFromEnv(["resolve", "database"]) + ]; + const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true }); + try { + return JSON.parse(output); + } catch (e) { + throw new Error( + `Unexpected output from codeql resolve database --format=json: ${e}` + ); + } + }, async mergeResults(sarifFiles, outputFile, { mergeRunsFromEqualCategory = false }) { diff --git a/lib/autobuild-action.js b/lib/autobuild-action.js index 10b30f64d5..20325380ce 100644 --- a/lib/autobuild-action.js +++ b/lib/autobuild-action.js @@ -84973,6 +84973,23 @@ ${output}` ); } }, + async resolveDatabase(databasePath) { + const codeqlArgs = [ + "resolve", + "database", + databasePath, + "--format=json", + ...getExtraOptionsFromEnv(["resolve", "database"]) + ]; + const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true }); + try { + return JSON.parse(output); + } catch (e) { + throw new Error( + `Unexpected output from codeql resolve database --format=json: ${e}` + ); + } + }, async mergeResults(sarifFiles, outputFile, { mergeRunsFromEqualCategory = false }) { diff --git a/lib/init-action-post.js b/lib/init-action-post.js index b6a4215af5..99780c4156 100644 --- a/lib/init-action-post.js +++ b/lib/init-action-post.js @@ -125350,6 +125350,23 @@ ${output}` ); } }, + async resolveDatabase(databasePath) { + const codeqlArgs = [ + "resolve", + "database", + databasePath, + "--format=json", + ...getExtraOptionsFromEnv(["resolve", "database"]) + ]; + const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true }); + try { + return JSON.parse(output); + } catch (e) { + throw new Error( + `Unexpected output from codeql resolve database --format=json: ${e}` + ); + } + }, async mergeResults(sarifFiles, outputFile, { mergeRunsFromEqualCategory = false }) { diff --git a/lib/init-action.js b/lib/init-action.js index 8f5b7f8871..0af2f67e84 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -85914,7 +85914,7 @@ function computeChangedFiles(baseFileOids, overlayFileOids) { var CACHE_VERSION = 1; var CACHE_PREFIX = "codeql-overlay-base-database"; var MAX_CACHE_OPERATION_MS = 6e5; -function checkOverlayBaseDatabase(config, logger, warningPrefix) { +async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) { const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); if (!fs3.existsSync(baseDatabaseOidsFilePath)) { logger.warning( @@ -85922,6 +85922,23 @@ function checkOverlayBaseDatabase(config, logger, warningPrefix) { ); return false; } + for (const language of config.languages) { + const dbPath = getCodeQLDatabasePath(config, language); + try { + const resolveDatabaseOutput = await codeql.resolveDatabase(dbPath); + if (resolveDatabaseOutput === void 0 || !("overlayBaseSpecifier" in resolveDatabaseOutput)) { + logger.info(`${warningPrefix}: no overlayBaseSpecifier defined`); + return false; + } else { + logger.info( + `Overlay base specifier for ${language} overlay-base database found: ${resolveDatabaseOutput.overlayBaseSpecifier}` + ); + } + } catch (e) { + logger.warning(`${warningPrefix}: failed to resolve database: ${e}`); + return false; + } + } return true; } async function downloadOverlayBaseDatabaseFromCache(codeql, config, logger) { @@ -86009,7 +86026,8 @@ async function downloadOverlayBaseDatabaseFromCache(codeql, config, logger) { ); return void 0; } - const databaseIsValid = checkOverlayBaseDatabase( + const databaseIsValid = await checkOverlayBaseDatabase( + codeql, config, logger, "Downloaded overlay-base database is invalid" @@ -89147,6 +89165,23 @@ ${output}` ); } }, + async resolveDatabase(databasePath) { + const codeqlArgs = [ + "resolve", + "database", + databasePath, + "--format=json", + ...getExtraOptionsFromEnv(["resolve", "database"]) + ]; + const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true }); + try { + return JSON.parse(output); + } catch (e) { + throw new Error( + `Unexpected output from codeql resolve database --format=json: ${e}` + ); + } + }, async mergeResults(sarifFiles, outputFile, { mergeRunsFromEqualCategory = false }) { diff --git a/lib/resolve-environment-action.js b/lib/resolve-environment-action.js index 6e159d31a4..ffaf4d1621 100644 --- a/lib/resolve-environment-action.js +++ b/lib/resolve-environment-action.js @@ -84672,6 +84672,23 @@ ${output}` ); } }, + async resolveDatabase(databasePath) { + const codeqlArgs = [ + "resolve", + "database", + databasePath, + "--format=json", + ...getExtraOptionsFromEnv(["resolve", "database"]) + ]; + const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true }); + try { + return JSON.parse(output); + } catch (e) { + throw new Error( + `Unexpected output from codeql resolve database --format=json: ${e}` + ); + } + }, async mergeResults(sarifFiles, outputFile, { mergeRunsFromEqualCategory = false }) { diff --git a/lib/setup-codeql-action.js b/lib/setup-codeql-action.js index 74b84a122f..4f9d6af4fc 100644 --- a/lib/setup-codeql-action.js +++ b/lib/setup-codeql-action.js @@ -85975,6 +85975,23 @@ ${output}` ); } }, + async resolveDatabase(databasePath) { + const codeqlArgs = [ + "resolve", + "database", + databasePath, + "--format=json", + ...getExtraOptionsFromEnv(["resolve", "database"]) + ]; + const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true }); + try { + return JSON.parse(output); + } catch (e) { + throw new Error( + `Unexpected output from codeql resolve database --format=json: ${e}` + ); + } + }, async mergeResults(sarifFiles, outputFile, { mergeRunsFromEqualCategory = false }) { diff --git a/lib/upload-lib.js b/lib/upload-lib.js index 4f3f1fa46b..aacb1837df 100644 --- a/lib/upload-lib.js +++ b/lib/upload-lib.js @@ -88697,6 +88697,23 @@ ${output}` ); } }, + async resolveDatabase(databasePath) { + const codeqlArgs = [ + "resolve", + "database", + databasePath, + "--format=json", + ...getExtraOptionsFromEnv(["resolve", "database"]) + ]; + const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true }); + try { + return JSON.parse(output); + } catch (e) { + throw new Error( + `Unexpected output from codeql resolve database --format=json: ${e}` + ); + } + }, async mergeResults(sarifFiles, outputFile, { mergeRunsFromEqualCategory = false }) { diff --git a/lib/upload-sarif-action.js b/lib/upload-sarif-action.js index ba2c9118cf..5908e956ef 100644 --- a/lib/upload-sarif-action.js +++ b/lib/upload-sarif-action.js @@ -89215,6 +89215,23 @@ ${output}` ); } }, + async resolveDatabase(databasePath) { + const codeqlArgs = [ + "resolve", + "database", + databasePath, + "--format=json", + ...getExtraOptionsFromEnv(["resolve", "database"]) + ]; + const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true }); + try { + return JSON.parse(output); + } catch (e) { + throw new Error( + `Unexpected output from codeql resolve database --format=json: ${e}` + ); + } + }, async mergeResults(sarifFiles, outputFile, { mergeRunsFromEqualCategory = false }) { diff --git a/src/codeql.ts b/src/codeql.ts index 2b86d843f3..f597931931 100644 --- a/src/codeql.ts +++ b/src/codeql.ts @@ -206,6 +206,7 @@ export interface CodeQL { * Run 'codeql resolve queries --format=startingpacks'. */ resolveQueriesStartingPacks(queries: string[]): Promise; + resolveDatabase(databasePath: string): Promise; /** * Run 'codeql github merge-results'. */ @@ -230,6 +231,10 @@ export interface VersionInfo { overlayVersion?: number; } +export interface ResolveDatabaseOutput { + [key: string]: string | [string]; +} + export interface ResolveLanguagesOutput { [language: string]: [string]; } @@ -493,6 +498,7 @@ export function createStubCodeQL(partialCodeql: Partial): CodeQL { partialCodeql, "resolveQueriesStartingPacks", ), + resolveDatabase: resolveFunction(partialCodeql, "resolveDatabase"), mergeResults: resolveFunction(partialCodeql, "mergeResults"), }; } @@ -1003,6 +1009,26 @@ async function getCodeQLForCmd( ); } }, + async resolveDatabase( + databasePath: string, + ): Promise { + const codeqlArgs = [ + "resolve", + "database", + databasePath, + "--format=json", + ...getExtraOptionsFromEnv(["resolve", "database"]), + ]; + const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true }); + + try { + return JSON.parse(output) as ResolveDatabaseOutput; + } catch (e) { + throw new Error( + `Unexpected output from codeql resolve database --format=json: ${e}`, + ); + } + }, async mergeResults( sarifFiles: string[], outputFile: string, diff --git a/src/overlay-database-utils.test.ts b/src/overlay-database-utils.test.ts index cee0d45f13..db47d4d879 100644 --- a/src/overlay-database-utils.test.ts +++ b/src/overlay-database-utils.test.ts @@ -7,7 +7,9 @@ import * as sinon from "sinon"; import * as actionsUtil from "./actions-util"; import * as apiClient from "./api-client"; +import { ResolveDatabaseOutput } from "./codeql"; import * as gitUtils from "./git-utils"; +import { KnownLanguage } from "./languages"; import { getRunnerLogger } from "./logging"; import { downloadOverlayBaseDatabaseFromCache, @@ -95,6 +97,7 @@ interface DownloadOverlayBaseDatabaseTestCase { hasBaseDatabaseOidsFile: boolean; tryGetFolderBytesSucceeds: boolean; codeQLVersion: string; + resolveDatabaseOutput: ResolveDatabaseOutput | Error; } const defaultDownloadTestCase: DownloadOverlayBaseDatabaseTestCase = { @@ -105,6 +108,7 @@ const defaultDownloadTestCase: DownloadOverlayBaseDatabaseTestCase = { hasBaseDatabaseOidsFile: true, tryGetFolderBytesSucceeds: true, codeQLVersion: "2.20.5", + resolveDatabaseOutput: { overlayBaseSpecifier: "20250626:XXX" }, }; const testDownloadOverlayBaseDatabaseFromCache = test.macro({ @@ -119,9 +123,11 @@ const testDownloadOverlayBaseDatabaseFromCache = test.macro({ await fs.promises.mkdir(dbLocation, { recursive: true }); const logger = getRunnerLogger(true); - const config = createTestConfig({ dbLocation }); - const testCase = { ...defaultDownloadTestCase, ...partialTestCase }; + const config = createTestConfig({ + dbLocation, + languages: [KnownLanguage.java], + }); config.overlayDatabaseMode = testCase.overlayDatabaseMode; config.useOverlayDatabaseCaching = testCase.useOverlayDatabaseCaching; @@ -163,9 +169,23 @@ const testDownloadOverlayBaseDatabaseFromCache = test.macro({ .resolves(testCase.tryGetFolderBytesSucceeds ? 1024 * 1024 : undefined); stubs.push(tryGetFolderBytesStub); + const codeql = mockCodeQLVersion(testCase.codeQLVersion); + + if (testCase.resolveDatabaseOutput instanceof Error) { + const resolveDatabaseStub = sinon + .stub(codeql, "resolveDatabase") + .rejects(testCase.resolveDatabaseOutput); + stubs.push(resolveDatabaseStub); + } else { + const resolveDatabaseStub = sinon + .stub(codeql, "resolveDatabase") + .resolves(testCase.resolveDatabaseOutput); + stubs.push(resolveDatabaseStub); + } + try { const result = await downloadOverlayBaseDatabaseFromCache( - mockCodeQLVersion(testCase.codeQLVersion), + codeql, config, logger, ); @@ -255,6 +275,24 @@ test( false, ); +test( + testDownloadOverlayBaseDatabaseFromCache, + "returns undefined when downloaded database doesn't have an overlayBaseSpecifier", + { + resolveDatabaseOutput: {}, + }, + false, +); + +test( + testDownloadOverlayBaseDatabaseFromCache, + "returns undefined when resolving database metadata fails", + { + resolveDatabaseOutput: new Error("Failed to resolve database metadata"), + }, + false, +); + test( testDownloadOverlayBaseDatabaseFromCache, "returns undefined when filesystem error occurs", diff --git a/src/overlay-database-utils.ts b/src/overlay-database-utils.ts index a340bfe2b0..7db248bd1d 100644 --- a/src/overlay-database-utils.ts +++ b/src/overlay-database-utils.ts @@ -17,6 +17,7 @@ import { getCommitOid, getFileOidsUnderPath } from "./git-utils"; import { Logger, withGroupAsync } from "./logging"; import { CleanupLevel, + getCodeQLDatabasePath, getErrorMessage, isInTestMode, tryGetFolderBytes, @@ -176,11 +177,12 @@ const MAX_CACHE_OPERATION_MS = 600_000; * @param warningPrefix Prefix for the check failure warning message * @returns True if the verification succeeded, false otherwise */ -function checkOverlayBaseDatabase( +async function checkOverlayBaseDatabase( + codeql: CodeQL, config: Config, logger: Logger, warningPrefix: string, -): boolean { +): Promise { // An overlay-base database should contain the base database OIDs file. const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); if (!fs.existsSync(baseDatabaseOidsFilePath)) { @@ -189,6 +191,29 @@ function checkOverlayBaseDatabase( ); return false; } + + for (const language of config.languages) { + const dbPath = getCodeQLDatabasePath(config, language); + try { + const resolveDatabaseOutput = await codeql.resolveDatabase(dbPath); + if ( + resolveDatabaseOutput === undefined || + !("overlayBaseSpecifier" in resolveDatabaseOutput) + ) { + logger.info(`${warningPrefix}: no overlayBaseSpecifier defined`); + return false; + } else { + logger.info( + `Overlay base specifier for ${language} overlay-base database found: ` + + `${resolveDatabaseOutput.overlayBaseSpecifier}`, + ); + } + } catch (e) { + logger.warning(`${warningPrefix}: failed to resolve database: ${e}`); + return false; + } + } + return true; } @@ -232,7 +257,8 @@ export async function cleanupAndUploadOverlayBaseDatabaseToCache( return false; } - const databaseIsValid = checkOverlayBaseDatabase( + const databaseIsValid = await checkOverlayBaseDatabase( + codeql, config, logger, "Abort uploading overlay-base database to cache", @@ -415,7 +441,8 @@ export async function downloadOverlayBaseDatabaseFromCache( return undefined; } - const databaseIsValid = checkOverlayBaseDatabase( + const databaseIsValid = await checkOverlayBaseDatabase( + codeql, config, logger, "Downloaded overlay-base database is invalid",