From 4b640c152a3bfcf09a903989eaa148b124c69360 Mon Sep 17 00:00:00 2001 From: Roman Bobrovskiy Date: Mon, 17 Feb 2025 17:12:39 +0100 Subject: [PATCH 1/6] feat: add support for next15 --- src/cacheHandler/strategy/s3.spec.ts | 480 +++++++++++++-------------- src/cacheHandler/strategy/s3.ts | 124 ++++--- src/common/project.ts | 27 +- src/lambdas/originRequest.ts | 5 +- 4 files changed, 346 insertions(+), 290 deletions(-) diff --git a/src/cacheHandler/strategy/s3.spec.ts b/src/cacheHandler/strategy/s3.spec.ts index d84e1cd..a5b2baa 100644 --- a/src/cacheHandler/strategy/s3.spec.ts +++ b/src/cacheHandler/strategy/s3.spec.ts @@ -1,240 +1,240 @@ -import { CacheEntry, CacheContext } from '@dbbs/next-cache-handler-core' -import { S3Cache, TAG_PREFIX } from './s3' - -const mockHtmlPage = '

My Page

' - -export const mockCacheEntry = { - value: { - pageData: {}, - html: mockHtmlPage, - kind: 'PAGE', - postponed: undefined, - headers: undefined, - status: 200 - }, - lastModified: 100000 -} satisfies CacheEntry - -const mockCacheContext: CacheContext = { - isAppRouter: false, - serverCacheDirPath: '' -} - -const mockBucketName = 'test-bucket' -const cacheKey = 'test' -const pageKey = 'index' -const s3Cache = new S3Cache(mockBucketName) - -const store = new Map() -const mockGetObject = jest.fn().mockImplementation(async ({ Key }) => { - const res = store.get(Key) - return res - ? { Body: { transformToString: () => res.Body }, Metadata: res.Metadata } - : { Body: undefined, Metadata: undefined } -}) -const mockPutObject = jest - .fn() - .mockImplementation(async ({ Key, Body, Metadata }) => store.set(Key, { Body, Metadata })) -const mockDeleteObject = jest.fn().mockImplementation(async ({ Key }) => store.delete(Key)) -const mockDeleteObjects = jest - .fn() - .mockImplementation(async ({ Delete: { Objects } }: { Delete: { Objects: { Key: string }[] } }) => - Objects.forEach(({ Key }) => store.delete(Key)) - ) -const mockGetObjectList = jest - .fn() - .mockImplementation(async () => ({ Contents: [...store.keys()].map((key) => ({ Key: key })) })) -const mockGetObjectTagging = jest - .fn() - .mockImplementation(() => ({ TagSet: [{ Key: 'revalidateTag0', Value: cacheKey }] })) - -jest.mock('@aws-sdk/client-s3', () => { - return { - S3: jest.fn().mockReturnValue({ - getObject: jest.fn((...params) => mockGetObject(...params)), - putObject: jest.fn((...params) => mockPutObject(...params)), - deleteObject: jest.fn((...params) => mockDeleteObject(...params)), - deleteObjects: jest.fn((...params) => mockDeleteObjects(...params)), - listObjectsV2: jest.fn((...params) => mockGetObjectList(...params)), - getObjectTagging: jest.fn((...params) => mockGetObjectTagging(...params)), - config: {} - }) - } -}) - -const mockDynamoQuery = jest.fn() -const mockDynamoPutItem = jest.fn() -jest.mock('@aws-sdk/client-dynamodb', () => { - return { - DynamoDB: jest.fn().mockReturnValue({ - query: jest.fn((...params) => mockDynamoQuery(...params)), - putItem: jest.fn((...params) => mockDynamoPutItem(...params)) - }) - } -}) - -describe('S3Cache', () => { - afterEach(() => { - jest.clearAllMocks() - store.clear() - }) - afterAll(() => { - jest.restoreAllMocks() - }) - - it('get should return null', async () => { - const result = await s3Cache.get() - expect(result).toBeNull() - }) - - it('should set cache for page router', async () => { - await s3Cache.set(cacheKey, cacheKey, mockCacheEntry, mockCacheContext) - expect(s3Cache.client.putObject).toHaveBeenCalledTimes(2) - expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { - Bucket: mockBucketName, - Key: `${cacheKey}/${cacheKey}.html`, - Body: mockHtmlPage, - ContentType: 'text/html', - Metadata: { - 'Cache-Fragment-Key': cacheKey - } - }) - expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(2, { - Bucket: mockBucketName, - Key: `${cacheKey}/${cacheKey}.json`, - Body: JSON.stringify(mockCacheEntry), - ContentType: 'application/json', - Metadata: { - 'Cache-Fragment-Key': cacheKey - } - }) - expect(mockDynamoPutItem).toHaveBeenCalledWith({ - TableName: process.env.DYNAMODB_CACHE_TABLE, - Item: { - pageKey: { S: cacheKey }, - cacheKey: { S: cacheKey }, - s3Key: { S: `${cacheKey}/${cacheKey}` }, - tags: { S: '' }, - createdAt: { S: expect.any(String) } - } - }) - }) - - it('should set cache for app router', async () => { - await s3Cache.set(cacheKey, cacheKey, mockCacheEntry, { ...mockCacheContext, isAppRouter: true }) - expect(s3Cache.client.putObject).toHaveBeenCalledTimes(3) - expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { - Bucket: mockBucketName, - Key: `${cacheKey}/${cacheKey}.html`, - Body: mockHtmlPage, - ContentType: 'text/html', - Metadata: { - 'Cache-Fragment-Key': cacheKey - } - }) - expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(2, { - Bucket: mockBucketName, - Key: `${cacheKey}/${cacheKey}.json`, - Body: JSON.stringify(mockCacheEntry), - ContentType: 'application/json', - Metadata: { - 'Cache-Fragment-Key': cacheKey - } - }) - expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(3, { - Bucket: mockBucketName, - Key: `${cacheKey}/${cacheKey}.rsc`, - Body: mockCacheEntry.value.pageData, - ContentType: 'text/x-component', - Metadata: { - 'Cache-Fragment-Key': cacheKey - } - }) - expect(mockDynamoPutItem).toHaveBeenCalledWith({ - TableName: process.env.DYNAMODB_CACHE_TABLE, - Item: { - pageKey: { S: cacheKey }, - cacheKey: { S: cacheKey }, - s3Key: { S: `${cacheKey}/${cacheKey}` }, - tags: { S: '' }, - createdAt: { S: expect.any(String) } - } - }) - }) - - it('should delete cache value', async () => { - await s3Cache.delete(cacheKey, cacheKey) - expect(s3Cache.client.deleteObjects).toHaveBeenCalledTimes(1) - expect(s3Cache.client.deleteObjects).toHaveBeenNthCalledWith(1, { - Bucket: mockBucketName, - Delete: { - Objects: [ - { Key: `${cacheKey}/${cacheKey}.json` }, - { Key: `${cacheKey}/${cacheKey}.html` }, - { Key: `${cacheKey}/${cacheKey}.rsc` } - ] - } - }) - }) - - it('should revalidate cache by tag and delete objects', async () => { - const s3Path = `${pageKey}/${cacheKey}` - const mockQueryResult = { - Items: [ - { - pageKey: { S: pageKey }, - cacheKey: { S: cacheKey } - } - ] - } - - mockDynamoQuery.mockResolvedValueOnce(mockQueryResult) - mockGetObjectTagging.mockResolvedValue({ TagSet: [{ Key: TAG_PREFIX, Value: 'test-tag' }] }) - mockGetObjectList.mockResolvedValueOnce({ - Contents: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] - }) - - await s3Cache.revalidateTag('test-tag') - - expect(mockDynamoQuery).toHaveBeenCalledWith({ - TableName: process.env.DYNAMODB_CACHE_TABLE, - KeyConditionExpression: '#field = :value', - ExpressionAttributeNames: { - '#field': 'tags' - }, - ExpressionAttributeValues: { - ':value': { S: 'test-tag' } - } - }) - - expect(s3Cache.client.deleteObjects).toHaveBeenCalledWith({ - Bucket: mockBucketName, - Delete: { - Objects: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] - } - }) - }) - - it('should revalidate cache by path', async () => { - const s3Path = `${pageKey}/${cacheKey}` - mockGetObjectList.mockResolvedValueOnce({ - Contents: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] - }) - - await s3Cache.deleteAllByKeyMatch(cacheKey, '') - - expect(s3Cache.client.listObjectsV2).toHaveBeenCalledWith({ - Bucket: mockBucketName, - ContinuationToken: undefined, - Prefix: `${cacheKey}/`, - Delimiter: '/' - }) - - expect(s3Cache.client.deleteObjects).toHaveBeenCalledWith({ - Bucket: mockBucketName, - Delete: { - Objects: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] - } - }) - }) -}) +// import { CacheEntry, CacheContext } from '@dbbs/next-cache-handler-core' +// import { S3Cache, TAG_PREFIX } from './s3' + +// const mockHtmlPage = '

My Page

' + +// export const mockCacheEntry = { +// value: { +// pageData: {}, +// html: mockHtmlPage, +// kind: 'PAGE', +// postponed: undefined, +// headers: undefined, +// status: 200 +// }, +// lastModified: 100000 +// } satisfies CacheEntry + +// const mockCacheContext: CacheContext = { +// isAppRouter: false, +// serverCacheDirPath: '' +// } + +// const mockBucketName = 'test-bucket' +// const cacheKey = 'test' +// const pageKey = 'index' +// const s3Cache = new S3Cache(mockBucketName) + +// const store = new Map() +// const mockGetObject = jest.fn().mockImplementation(async ({ Key }) => { +// const res = store.get(Key) +// return res +// ? { Body: { transformToString: () => res.Body }, Metadata: res.Metadata } +// : { Body: undefined, Metadata: undefined } +// }) +// const mockPutObject = jest +// .fn() +// .mockImplementation(async ({ Key, Body, Metadata }) => store.set(Key, { Body, Metadata })) +// const mockDeleteObject = jest.fn().mockImplementation(async ({ Key }) => store.delete(Key)) +// const mockDeleteObjects = jest +// .fn() +// .mockImplementation(async ({ Delete: { Objects } }: { Delete: { Objects: { Key: string }[] } }) => +// Objects.forEach(({ Key }) => store.delete(Key)) +// ) +// const mockGetObjectList = jest +// .fn() +// .mockImplementation(async () => ({ Contents: [...store.keys()].map((key) => ({ Key: key })) })) +// const mockGetObjectTagging = jest +// .fn() +// .mockImplementation(() => ({ TagSet: [{ Key: 'revalidateTag0', Value: cacheKey }] })) + +// jest.mock('@aws-sdk/client-s3', () => { +// return { +// S3: jest.fn().mockReturnValue({ +// getObject: jest.fn((...params) => mockGetObject(...params)), +// putObject: jest.fn((...params) => mockPutObject(...params)), +// deleteObject: jest.fn((...params) => mockDeleteObject(...params)), +// deleteObjects: jest.fn((...params) => mockDeleteObjects(...params)), +// listObjectsV2: jest.fn((...params) => mockGetObjectList(...params)), +// getObjectTagging: jest.fn((...params) => mockGetObjectTagging(...params)), +// config: {} +// }) +// } +// }) + +// const mockDynamoQuery = jest.fn() +// const mockDynamoPutItem = jest.fn() +// jest.mock('@aws-sdk/client-dynamodb', () => { +// return { +// DynamoDB: jest.fn().mockReturnValue({ +// query: jest.fn((...params) => mockDynamoQuery(...params)), +// putItem: jest.fn((...params) => mockDynamoPutItem(...params)) +// }) +// } +// }) + +// describe('S3Cache', () => { +// afterEach(() => { +// jest.clearAllMocks() +// store.clear() +// }) +// afterAll(() => { +// jest.restoreAllMocks() +// }) + +// it('get should return null', async () => { +// const result = await s3Cache.get() +// expect(result).toBeNull() +// }) + +// it('should set cache for page router', async () => { +// await s3Cache.set(cacheKey, cacheKey, mockCacheEntry, mockCacheContext) +// expect(s3Cache.client.putObject).toHaveBeenCalledTimes(2) +// expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { +// Bucket: mockBucketName, +// Key: `${cacheKey}/${cacheKey}.html`, +// Body: mockHtmlPage, +// ContentType: 'text/html', +// Metadata: { +// 'Cache-Fragment-Key': cacheKey +// } +// }) +// expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(2, { +// Bucket: mockBucketName, +// Key: `${cacheKey}/${cacheKey}.json`, +// Body: JSON.stringify(mockCacheEntry), +// ContentType: 'application/json', +// Metadata: { +// 'Cache-Fragment-Key': cacheKey +// } +// }) +// expect(mockDynamoPutItem).toHaveBeenCalledWith({ +// TableName: process.env.DYNAMODB_CACHE_TABLE, +// Item: { +// pageKey: { S: cacheKey }, +// cacheKey: { S: cacheKey }, +// s3Key: { S: `${cacheKey}/${cacheKey}` }, +// tags: { S: '' }, +// createdAt: { S: expect.any(String) } +// } +// }) +// }) + +// it('should set cache for app router', async () => { +// await s3Cache.set(cacheKey, cacheKey, mockCacheEntry, { ...mockCacheContext, isAppRouter: true }) +// expect(s3Cache.client.putObject).toHaveBeenCalledTimes(3) +// expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { +// Bucket: mockBucketName, +// Key: `${cacheKey}/${cacheKey}.html`, +// Body: mockHtmlPage, +// ContentType: 'text/html', +// Metadata: { +// 'Cache-Fragment-Key': cacheKey +// } +// }) +// expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(2, { +// Bucket: mockBucketName, +// Key: `${cacheKey}/${cacheKey}.json`, +// Body: JSON.stringify(mockCacheEntry), +// ContentType: 'application/json', +// Metadata: { +// 'Cache-Fragment-Key': cacheKey +// } +// }) +// expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(3, { +// Bucket: mockBucketName, +// Key: `${cacheKey}/${cacheKey}.rsc`, +// Body: mockCacheEntry.value.pageData, +// ContentType: 'text/x-component', +// Metadata: { +// 'Cache-Fragment-Key': cacheKey +// } +// }) +// expect(mockDynamoPutItem).toHaveBeenCalledWith({ +// TableName: process.env.DYNAMODB_CACHE_TABLE, +// Item: { +// pageKey: { S: cacheKey }, +// cacheKey: { S: cacheKey }, +// s3Key: { S: `${cacheKey}/${cacheKey}` }, +// tags: { S: '' }, +// createdAt: { S: expect.any(String) } +// } +// }) +// }) + +// it('should delete cache value', async () => { +// await s3Cache.delete(cacheKey, cacheKey) +// expect(s3Cache.client.deleteObjects).toHaveBeenCalledTimes(1) +// expect(s3Cache.client.deleteObjects).toHaveBeenNthCalledWith(1, { +// Bucket: mockBucketName, +// Delete: { +// Objects: [ +// { Key: `${cacheKey}/${cacheKey}.json` }, +// { Key: `${cacheKey}/${cacheKey}.html` }, +// { Key: `${cacheKey}/${cacheKey}.rsc` } +// ] +// } +// }) +// }) + +// it('should revalidate cache by tag and delete objects', async () => { +// const s3Path = `${pageKey}/${cacheKey}` +// const mockQueryResult = { +// Items: [ +// { +// pageKey: { S: pageKey }, +// cacheKey: { S: cacheKey } +// } +// ] +// } + +// mockDynamoQuery.mockResolvedValueOnce(mockQueryResult) +// mockGetObjectTagging.mockResolvedValue({ TagSet: [{ Key: TAG_PREFIX, Value: 'test-tag' }] }) +// mockGetObjectList.mockResolvedValueOnce({ +// Contents: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] +// }) + +// await s3Cache.revalidateTag('test-tag') + +// expect(mockDynamoQuery).toHaveBeenCalledWith({ +// TableName: process.env.DYNAMODB_CACHE_TABLE, +// KeyConditionExpression: '#field = :value', +// ExpressionAttributeNames: { +// '#field': 'tags' +// }, +// ExpressionAttributeValues: { +// ':value': { S: 'test-tag' } +// } +// }) + +// expect(s3Cache.client.deleteObjects).toHaveBeenCalledWith({ +// Bucket: mockBucketName, +// Delete: { +// Objects: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] +// } +// }) +// }) + +// it('should revalidate cache by path', async () => { +// const s3Path = `${pageKey}/${cacheKey}` +// mockGetObjectList.mockResolvedValueOnce({ +// Contents: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] +// }) + +// await s3Cache.deleteAllByKeyMatch(cacheKey, '') + +// expect(s3Cache.client.listObjectsV2).toHaveBeenCalledWith({ +// Bucket: mockBucketName, +// ContinuationToken: undefined, +// Prefix: `${cacheKey}/`, +// Delimiter: '/' +// }) + +// expect(s3Cache.client.deleteObjects).toHaveBeenCalledWith({ +// Bucket: mockBucketName, +// Delete: { +// Objects: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] +// } +// }) +// }) +// }) diff --git a/src/cacheHandler/strategy/s3.ts b/src/cacheHandler/strategy/s3.ts index 3442380..1d5444a 100644 --- a/src/cacheHandler/strategy/s3.ts +++ b/src/cacheHandler/strategy/s3.ts @@ -2,7 +2,7 @@ import { NEXT_CACHE_TAGS_HEADER } from 'next/dist/lib/constants' import { type ListObjectsV2CommandOutput, type PutObjectCommandInput, S3 } from '@aws-sdk/client-s3' import { DynamoDB } from '@aws-sdk/client-dynamodb' import { chunkArray } from '../../common/array' -import type { CacheEntry, CacheStrategy, CacheContext } from '@dbbs/next-cache-handler-core' +import { type CacheEntry, type CacheStrategy, type CacheContext, CachedRouteKind } from '@dbbs/next-cache-handler-core' export const TAG_PREFIX = 'revalidateTag' enum CacheExtension { @@ -12,7 +12,6 @@ enum CacheExtension { } const PAGE_CACHE_EXTENSIONS = Object.values(CacheExtension) const CHUNK_LIMIT = 1000 - export class S3Cache implements CacheStrategy { public readonly client: S3 public readonly bucketName: string @@ -41,7 +40,7 @@ export class S3Cache implements CacheStrategy { ) } - async get(): Promise { + async get(): Promise { // We always need to return null to make nextjs revalidate the page and create new file in s3 // caching retreiving logic is handled by CloudFront and origin response lambda // we can't use nextjs cache retrival since it is required to re-render page during validation @@ -51,80 +50,114 @@ export class S3Cache implements CacheStrategy { } async set(pageKey: string, cacheKey: string, data: CacheEntry, ctx: CacheContext): Promise { - const promises = [] + if (!data.value?.kind || data.value?.kind === CachedRouteKind.REDIRECT) return Promise.resolve() + + //@ts-expect-error - TODO: fix this + const headersTags = this.buildTagKeys(data.value?.headers?.[NEXT_CACHE_TAGS_HEADER]?.toString()) + const baseInput: PutObjectCommandInput = { Bucket: this.bucketName, Key: `${pageKey}/${cacheKey}`, Metadata: { 'Cache-Fragment-Key': cacheKey }, - ...(data.revalidate ? { CacheControl: `smax-age=${data.revalidate}, stale-while-revalidate` } : undefined) + // TODO: check if we want to cache page until next deployment, then next won't pass revalidate value + // check how to identify such case + ...(data.revalidate ? { CacheControl: `s-maxage=${data.revalidate}, stale-while-revalidate=120` } : undefined) } + const input: PutObjectCommandInput = { ...baseInput } - if (data.value?.kind === 'PAGE' || data.value?.kind === 'ROUTE') { - const headersTags = this.buildTagKeys(data.value.headers?.[NEXT_CACHE_TAGS_HEADER]?.toString()) - const input: PutObjectCommandInput = { ...baseInput } - - promises.push( - this.#dynamoDBClient.putItem({ - TableName: process.env.DYNAMODB_CACHE_TABLE!, - Item: { - pageKey: { S: pageKey }, - cacheKey: { S: cacheKey }, - s3Key: { S: baseInput.Key! }, - tags: { S: [headersTags, this.buildTagKeys(data.tags)].filter(Boolean).join('&') }, - createdAt: { S: new Date().toISOString() } - } - }) - ) + const promises = [ + this.#dynamoDBClient.putItem({ + TableName: process.env.DYNAMODB_CACHE_TABLE!, + Item: { + pageKey: { S: pageKey }, + cacheKey: { S: cacheKey }, + s3Key: { S: baseInput.Key! }, + tags: { S: [headersTags, this.buildTagKeys(data.tags)].filter(Boolean).join('&') }, + createdAt: { S: new Date().toISOString() } + } + }) + ] + + console.log('HERE_IS_DATA', data) + console.log('HERE_IS_DATA_VALUE', data.value) - if (data.value?.kind === 'PAGE') { + switch (data.value.kind) { + case CachedRouteKind.APP_PAGE: { + promises.push( + ...[ + this.client.putObject({ + ...input, + Key: `${input.Key}.${CacheExtension.HTML}`, + Body: data.value.html, + ContentType: 'text/html' + }), + this.client.putObject({ + ...input, + Key: `${input.Key}.${CacheExtension.RSC}`, + Body: data.value.rscData?.toString() as string, // for server react components we need to safe additional reference data for nextjs. + ContentType: 'text/x-component' + }) + ] + ) + break + } + case CachedRouteKind.FETCH: { promises.push( this.client.putObject({ ...input, - Key: `${input.Key}.${CacheExtension.HTML}`, - Body: data.value.html, - ContentType: 'text/html' + Key: `${input.Key}.${CacheExtension.JSON}`, + Body: data.value.data.body.toString(), + ContentType: 'application/json' }) ) + break + } + case CachedRouteKind.APP_ROUTE: + case CachedRouteKind.ROUTE: { promises.push( this.client.putObject({ ...input, Key: `${input.Key}.${CacheExtension.JSON}`, - Body: JSON.stringify(data), + Body: data.value.body.toString(), ContentType: 'application/json' }) ) + break + } + case CachedRouteKind.PAGE: + case CachedRouteKind.PAGES: { + promises.push( + this.client.putObject({ + ...input, + Key: `${input.Key}.${CacheExtension.HTML}`, + Body: data.value.html, + ContentType: 'text/html' + }) + ) + if (ctx.isAppRouter) { promises.push( this.client.putObject({ ...input, Key: `${input.Key}.${CacheExtension.RSC}`, + // @ts-expect-error - TODO: fix this Body: data.value.pageData as string, // for server react components we need to safe additional reference data for nextjs. ContentType: 'text/x-component' }) ) + } else { + promises.push( + this.client.putObject({ + ...input, + Key: `${input.Key}.${CacheExtension.JSON}`, + Body: JSON.stringify(data), + ContentType: 'application/json' + }) + ) } - } else { - promises.push( - this.client.putObject({ - ...input, - Key: `${input.Key}.${CacheExtension.JSON}`, - Body: JSON.stringify(data), - ContentType: 'application/json' - }) - ) } - } else { - promises.push( - this.client.putObject({ - ...baseInput, - Key: `${baseInput.Key}.${CacheExtension.JSON}`, - Body: JSON.stringify(data), - ContentType: 'application/json' - // ...(data.tags?.length ? { Tagging: `${this.buildTagKeys(data.tags)}` } : {}) - }) - ) } await Promise.all(promises) @@ -177,7 +210,6 @@ export class S3Cache implements CacheStrategy { } async delete(pageKey: string, cacheKey: string): Promise { - console.log('HERE_IS_CALL_DELETE') await this.client.deleteObjects({ Bucket: this.bucketName, Delete: { Objects: PAGE_CACHE_EXTENSIONS.map((ext) => ({ Key: `${pageKey}/${cacheKey}.${ext}` })) } diff --git a/src/common/project.ts b/src/common/project.ts index e000a90..349b36c 100644 --- a/src/common/project.ts +++ b/src/common/project.ts @@ -1,5 +1,7 @@ import fs from 'node:fs' import path from 'path' +import vm from 'node:vm' +import esbuild from 'esbuild' export interface ProjectPackager { type: 'npm' | 'yarn' | 'pnpm' @@ -27,7 +29,9 @@ export const findPackager = (appPath: string): ProjectPackager | undefined => { } export const findNextConfig = (appPath: string): string | undefined => { - return ['next.config.js', 'next.config.mjs'].find((config) => fs.existsSync(path.join(appPath, config))) + return ['next.config.js', 'next.config.mjs', 'next.config.ts'].find((config) => + fs.existsSync(path.join(appPath, config)) + ) } const checkIsAppDir = (appPath: string): boolean => { @@ -39,7 +43,7 @@ export const getProjectSettings = (projectPath: string): ProjectSettings | undef const nextConfig = findNextConfig(projectPath) if (!nextConfig) { - throw new Error('Could not find next.config.(js|mjs)') + throw new Error('Could not find next.config.(js|mjs|ts)') } while (currentPath !== '/') { @@ -61,5 +65,24 @@ export const getProjectSettings = (projectPath: string): ProjectSettings | undef } export const loadFile = async (filePath: string) => { + if (filePath.endsWith('.ts')) { + const fileContent = fs.readFileSync(filePath, 'utf-8') + const res = await esbuild.transform(fileContent, { + target: 'es2022', + format: 'cjs', + platform: 'node', + loader: 'ts' + }) + const script = new vm.Script(res.code) + const context = vm.createContext({ module: {}, exports: {}, require }) + script.runInContext(context) + console.log({ + context, + module: context.module, + default: context.module.exports.default + }) + return context.module.exports.default + } + return import(filePath).then((r) => r.default) } diff --git a/src/lambdas/originRequest.ts b/src/lambdas/originRequest.ts index be4fbf5..f66dd26 100644 --- a/src/lambdas/originRequest.ts +++ b/src/lambdas/originRequest.ts @@ -84,8 +84,9 @@ const shouldRevalidateFile = (s3FileMeta: { LastModified: Date | string; CacheCo const { LastModified, CacheControl } = s3FileMeta - const match = CacheControl.match(/max-age=(\d+)/) - const maxAge = match ? parseInt(match[1]) : 0 + const sMaxAgeMatch = CacheControl.match(/s-maxage=(\d+)/) + const maxAgeMatch = CacheControl.match(/max-age=(\d+)/) + const maxAge = sMaxAgeMatch ? parseInt(sMaxAgeMatch[1]) : maxAgeMatch ? parseInt(maxAgeMatch[1]) : 0 const isFileExpired = Date.now() - new Date(LastModified).getTime() > maxAge * 1000 From d92471c0ad7870424e7ccd3a2e36bfb6ee8d4996 Mon Sep 17 00:00:00 2001 From: Roman Bobrovskiy Date: Tue, 18 Feb 2025 20:18:21 +0100 Subject: [PATCH 2/6] feat: updated handling the data for next app --- jest.config.js | 3 +- package-lock.json | 11 +- package.json | 2 +- src/__tests__/__mocks__/cacheEntries.ts | 57 +++ .../cacheHandler/strategy/s3.spec.ts | 413 ++++++++++++++++++ src/{ => __tests__}/common/array.spec.ts | 2 +- src/cacheHandler/strategy/s3.spec.ts | 240 ---------- src/cacheHandler/strategy/s3.ts | 21 +- src/cdk/constructs/ViewerRequestLambdaEdge.ts | 6 +- 9 files changed, 493 insertions(+), 262 deletions(-) create mode 100644 src/__tests__/__mocks__/cacheEntries.ts create mode 100644 src/__tests__/cacheHandler/strategy/s3.spec.ts rename src/{ => __tests__}/common/array.spec.ts (95%) delete mode 100644 src/cacheHandler/strategy/s3.spec.ts diff --git a/jest.config.js b/jest.config.js index 563e0f2..138f386 100644 --- a/jest.config.js +++ b/jest.config.js @@ -2,5 +2,6 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', - modulePathIgnorePatterns: ['/dist/'] + modulePathIgnorePatterns: ['/dist/'], + testPathIgnorePatterns: ['__tests__/__mocks__/'] } diff --git a/package-lock.json b/package-lock.json index f0ed62c..857d79e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "@aws-sdk/client-sts": "3.590.0", "@aws-sdk/credential-providers": "3.590.0", "@aws-sdk/util-endpoints": "3.587.0", - "@dbbs/next-cache-handler-core": "1.3.0", + "@dbbs/next-cache-handler-core": "1.4.0", "aws-cdk-lib": "2.144.0", "aws-sdk": "2.1635.0", "body-parser": "^1.20.3", @@ -3007,13 +3007,16 @@ } }, "node_modules/@dbbs/next-cache-handler-core": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@dbbs/next-cache-handler-core/-/next-cache-handler-core-1.3.0.tgz", - "integrity": "sha512-KX7Y73yvGEydzRmEpzNCUnUhlg4Wt26tY+G/KjFQAAldbJmHFhB5NWqw9A2jUXBstzmZDNeCupX+QSZ6Oekytw==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@dbbs/next-cache-handler-core/-/next-cache-handler-core-1.4.0.tgz", + "integrity": "sha512-eGiqGdPcv23b4J9upyDEDnL+KxDcT4eluwoWvx/mV3V5v6MTTgLATtKL6bj9IX24pGgsbxJiVSov3Rks2zUMSA==", "dependencies": { "cookie": "0.6.0", "path-to-regexp": "6.2.2", "ua-parser-js": "1.0.37" + }, + "peerDependencies": { + "next": "^14.1.0 || ^15.0.0" } }, "node_modules/@esbuild/aix-ppc64": { diff --git a/package.json b/package.json index 35194c5..098f76f 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,7 @@ "@aws-sdk/client-sts": "3.590.0", "@aws-sdk/credential-providers": "3.590.0", "@aws-sdk/util-endpoints": "3.587.0", - "@dbbs/next-cache-handler-core": "1.3.0", + "@dbbs/next-cache-handler-core": "1.4.0", "aws-cdk-lib": "2.144.0", "aws-sdk": "2.1635.0", "body-parser": "^1.20.3", diff --git a/src/__tests__/__mocks__/cacheEntries.ts b/src/__tests__/__mocks__/cacheEntries.ts new file mode 100644 index 0000000..7b15d3f --- /dev/null +++ b/src/__tests__/__mocks__/cacheEntries.ts @@ -0,0 +1,57 @@ +import { + CachedRouteKind, + CachedRedirectValue, + IncrementalCachedPageValue, + IncrementalCachedAppPageValue, + CachedImageValue, + CachedFetchValue, + CachedRouteValue +} from '@dbbs/next-cache-handler-core' + +export const mockRedirectCacheEntry: CachedRedirectValue = { + kind: CachedRouteKind.REDIRECT, + props: {} +} + +export const mockPageCacheEntry: IncrementalCachedPageValue = { + kind: CachedRouteKind.PAGE, + html: '

My Page

', + pageData: {}, + headers: {}, + status: 200 +} + +export const mockAppPageCacheEntry: IncrementalCachedAppPageValue = { + kind: CachedRouteKind.APP_PAGE, + html: '

My Page

', + rscData: Buffer.from('123'), + headers: {}, + postponed: undefined, + status: 200, + segmentData: undefined +} + +export const mockFetchCacheEntry: CachedFetchValue = { + kind: CachedRouteKind.FETCH, + data: { + url: 'https://example.com', + headers: {}, + body: '123' + }, + tags: [], + revalidate: 1000 +} + +export const mockRouteCacheEntry: CachedRouteValue = { + kind: CachedRouteKind.ROUTE, + body: Buffer.from('123'), + status: 200, + headers: {} +} + +export const mockAppRouteCacheEntry: CachedRouteValue = { + kind: CachedRouteKind.APP_ROUTE, + body: Buffer.from('123'), + status: 200, + headers: {} +} diff --git a/src/__tests__/cacheHandler/strategy/s3.spec.ts b/src/__tests__/cacheHandler/strategy/s3.spec.ts new file mode 100644 index 0000000..6884fda --- /dev/null +++ b/src/__tests__/cacheHandler/strategy/s3.spec.ts @@ -0,0 +1,413 @@ +import { CacheEntry, CacheContext, CachedRouteKind } from '@dbbs/next-cache-handler-core' +import { S3Cache, TAG_PREFIX, CACHE_ONE_YEAR } from '../../../cacheHandler/strategy/s3' +import { + mockRedirectCacheEntry, + mockPageCacheEntry, + mockAppPageCacheEntry, + mockFetchCacheEntry, + mockRouteCacheEntry, + mockAppRouteCacheEntry +} from '../../__mocks__/cacheEntries' + +const mockHtmlPage = '

My Page

' + +export const mockCacheEntry = { + value: { + pageData: {}, + html: mockHtmlPage, + kind: CachedRouteKind.PAGE, + headers: undefined, + status: 200 + }, + lastModified: 100000 +} satisfies CacheEntry + +const mockRevalidate = 100 +const mockCacheControlRevalidateHeader = `s-maxage=${mockRevalidate}, stale-while-revalidate=${CACHE_ONE_YEAR - mockRevalidate}` + +const mockCacheContext: CacheContext = { + isAppRouter: false, + serverCacheDirPath: '' +} + +const mockBucketName = 'test-bucket' +const cacheKey = 'test' +const pageKey = 'index' +const s3Cache = new S3Cache(mockBucketName) + +const store = new Map() +const mockGetObject = jest.fn().mockImplementation(async ({ Key }) => { + const res = store.get(Key) + return res + ? { Body: { transformToString: () => res.Body }, Metadata: res.Metadata } + : { Body: undefined, Metadata: undefined } +}) +const mockPutObject = jest + .fn() + .mockImplementation(async ({ Key, Body, Metadata }) => store.set(Key, { Body, Metadata })) +const mockDeleteObject = jest.fn().mockImplementation(async ({ Key }) => store.delete(Key)) +const mockDeleteObjects = jest + .fn() + .mockImplementation(async ({ Delete: { Objects } }: { Delete: { Objects: { Key: string }[] } }) => + Objects.forEach(({ Key }) => store.delete(Key)) + ) +const mockGetObjectList = jest + .fn() + .mockImplementation(async () => ({ Contents: [...store.keys()].map((key) => ({ Key: key })) })) +const mockGetObjectTagging = jest + .fn() + .mockImplementation(() => ({ TagSet: [{ Key: 'revalidateTag0', Value: cacheKey }] })) + +jest.mock('@aws-sdk/client-s3', () => { + return { + S3: jest.fn().mockReturnValue({ + getObject: jest.fn((...params) => mockGetObject(...params)), + putObject: jest.fn((...params) => mockPutObject(...params)), + deleteObject: jest.fn((...params) => mockDeleteObject(...params)), + deleteObjects: jest.fn((...params) => mockDeleteObjects(...params)), + listObjectsV2: jest.fn((...params) => mockGetObjectList(...params)), + getObjectTagging: jest.fn((...params) => mockGetObjectTagging(...params)), + config: {} + }) + } +}) + +const mockDynamoQuery = jest.fn() +const mockDynamoPutItem = jest.fn() +jest.mock('@aws-sdk/client-dynamodb', () => { + return { + DynamoDB: jest.fn().mockReturnValue({ + query: jest.fn((...params) => mockDynamoQuery(...params)), + putItem: jest.fn((...params) => mockDynamoPutItem(...params)) + }) + } +}) + +describe('S3Cache', () => { + afterEach(() => { + jest.clearAllMocks() + store.clear() + }) + afterAll(() => { + jest.restoreAllMocks() + }) + + it('get should return null', async () => { + const result = await s3Cache.get() + expect(result).toBeNull() + }) + + it('should not write cache for redirect entry', async () => { + await s3Cache.set(cacheKey, cacheKey, { value: mockRedirectCacheEntry, lastModified: 0 }, mockCacheContext) + + expect(s3Cache.client.putObject).not.toHaveBeenCalled() + expect(mockDynamoPutItem).not.toHaveBeenCalled() + }) + + it('should not write cache if revalidate is 0', async () => { + await s3Cache.set( + cacheKey, + cacheKey, + { value: mockPageCacheEntry, lastModified: 0, revalidate: 0 }, + mockCacheContext + ) + + expect(s3Cache.client.putObject).not.toHaveBeenCalled() + expect(mockDynamoPutItem).not.toHaveBeenCalled() + }) + + it('should not write cache if value is null', async () => { + await s3Cache.set(cacheKey, cacheKey, { value: null, lastModified: 0 }, mockCacheContext) + + expect(s3Cache.client.putObject).not.toHaveBeenCalled() + expect(mockDynamoPutItem).not.toHaveBeenCalled() + }) + + it(`should write value for ${CachedRouteKind.APP_PAGE}`, async () => { + await s3Cache.set( + cacheKey, + cacheKey, + { value: mockAppPageCacheEntry, lastModified: 0, revalidate: mockRevalidate }, + mockCacheContext + ) + + expect(s3Cache.client.putObject).toHaveBeenCalledTimes(2) + expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { + Bucket: mockBucketName, + Key: `${cacheKey}/${cacheKey}.html`, + Body: mockHtmlPage, + ContentType: 'text/html', + CacheControl: mockCacheControlRevalidateHeader, + Metadata: { + 'Cache-Fragment-Key': cacheKey + } + }) + expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(2, { + Bucket: mockBucketName, + Key: `${cacheKey}/${cacheKey}.rsc`, + Body: mockAppPageCacheEntry.rscData?.toString(), + ContentType: 'text/x-component', + CacheControl: mockCacheControlRevalidateHeader, + Metadata: { + 'Cache-Fragment-Key': cacheKey + } + }) + expect(mockDynamoPutItem).toHaveBeenCalledWith({ + TableName: process.env.DYNAMODB_CACHE_TABLE, + Item: { + pageKey: { S: cacheKey }, + cacheKey: { S: cacheKey }, + s3Key: { S: `${cacheKey}/${cacheKey}` }, + tags: { S: '' }, + createdAt: { S: expect.any(String) } + } + }) + }) + + it(`should write value for ${CachedRouteKind.FETCH}`, async () => { + await s3Cache.set( + cacheKey, + cacheKey, + { value: mockFetchCacheEntry, lastModified: 0, revalidate: mockRevalidate }, + mockCacheContext + ) + + expect(s3Cache.client.putObject).toHaveBeenCalledTimes(1) + expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { + Bucket: mockBucketName, + Key: `${cacheKey}/${cacheKey}.json`, + Body: mockFetchCacheEntry.data.body, + ContentType: 'application/json', + CacheControl: mockCacheControlRevalidateHeader, + Metadata: { + 'Cache-Fragment-Key': cacheKey + } + }) + expect(mockDynamoPutItem).toHaveBeenCalledWith({ + TableName: process.env.DYNAMODB_CACHE_TABLE, + Item: { + pageKey: { S: cacheKey }, + cacheKey: { S: cacheKey }, + s3Key: { S: `${cacheKey}/${cacheKey}` }, + tags: { S: '' }, + createdAt: { S: expect.any(String) } + } + }) + }) + + it(`should write value for ${CachedRouteKind.APP_ROUTE}`, async () => { + await s3Cache.set( + cacheKey, + cacheKey, + { value: mockAppRouteCacheEntry, lastModified: 0, revalidate: mockRevalidate }, + mockCacheContext + ) + + expect(s3Cache.client.putObject).toHaveBeenCalledTimes(1) + expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { + Bucket: mockBucketName, + Key: `${cacheKey}/${cacheKey}.json`, + Body: mockAppRouteCacheEntry.body.toString(), + ContentType: 'application/json', + CacheControl: mockCacheControlRevalidateHeader, + Metadata: { + 'Cache-Fragment-Key': cacheKey + } + }) + expect(mockDynamoPutItem).toHaveBeenCalledWith({ + TableName: process.env.DYNAMODB_CACHE_TABLE, + Item: { + pageKey: { S: cacheKey }, + cacheKey: { S: cacheKey }, + s3Key: { S: `${cacheKey}/${cacheKey}` }, + tags: { S: '' }, + createdAt: { S: expect.any(String) } + } + }) + }) + + it(`should write value for ${CachedRouteKind.ROUTE}`, async () => { + await s3Cache.set( + cacheKey, + cacheKey, + { value: mockRouteCacheEntry, lastModified: 0, revalidate: mockRevalidate }, + mockCacheContext + ) + + expect(s3Cache.client.putObject).toHaveBeenCalledTimes(1) + expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { + Bucket: mockBucketName, + Key: `${cacheKey}/${cacheKey}.json`, + Body: mockRouteCacheEntry.body.toString(), + ContentType: 'application/json', + CacheControl: mockCacheControlRevalidateHeader, + Metadata: { + 'Cache-Fragment-Key': cacheKey + } + }) + expect(mockDynamoPutItem).toHaveBeenCalledWith({ + TableName: process.env.DYNAMODB_CACHE_TABLE, + Item: { + pageKey: { S: cacheKey }, + cacheKey: { S: cacheKey }, + s3Key: { S: `${cacheKey}/${cacheKey}` }, + tags: { S: '' }, + createdAt: { S: expect.any(String) } + } + }) + }) + + it(`should write value for ${CachedRouteKind.PAGE} with page router`, async () => { + const pageData = { value: mockPageCacheEntry, lastModified: 0, revalidate: mockRevalidate } + await s3Cache.set(cacheKey, cacheKey, pageData, mockCacheContext) + + expect(s3Cache.client.putObject).toHaveBeenCalledTimes(2) + expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { + Bucket: mockBucketName, + Key: `${cacheKey}/${cacheKey}.html`, + Body: mockHtmlPage, + ContentType: 'text/html', + CacheControl: mockCacheControlRevalidateHeader, + Metadata: { + 'Cache-Fragment-Key': cacheKey + } + }) + expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(2, { + Bucket: mockBucketName, + Key: `${cacheKey}/${cacheKey}.json`, + Body: JSON.stringify(pageData), + ContentType: 'application/json', + CacheControl: mockCacheControlRevalidateHeader, + Metadata: { + 'Cache-Fragment-Key': cacheKey + } + }) + expect(mockDynamoPutItem).toHaveBeenCalledWith({ + TableName: process.env.DYNAMODB_CACHE_TABLE, + Item: { + pageKey: { S: cacheKey }, + cacheKey: { S: cacheKey }, + s3Key: { S: `${cacheKey}/${cacheKey}` }, + tags: { S: '' }, + createdAt: { S: expect.any(String) } + } + }) + }) + + it(`should write value for ${CachedRouteKind.PAGE} with app router`, async () => { + await s3Cache.set( + cacheKey, + cacheKey, + { value: mockPageCacheEntry, lastModified: 0, revalidate: mockRevalidate }, + { ...mockCacheContext, isAppRouter: true } + ) + + expect(s3Cache.client.putObject).toHaveBeenCalledTimes(2) + expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { + Bucket: mockBucketName, + Key: `${cacheKey}/${cacheKey}.html`, + Body: mockHtmlPage, + ContentType: 'text/html', + CacheControl: mockCacheControlRevalidateHeader, + Metadata: { + 'Cache-Fragment-Key': cacheKey + } + }) + expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(2, { + Bucket: mockBucketName, + Key: `${cacheKey}/${cacheKey}.rsc`, + Body: mockPageCacheEntry.pageData, + ContentType: 'text/x-component', + CacheControl: mockCacheControlRevalidateHeader, + Metadata: { + 'Cache-Fragment-Key': cacheKey + } + }) + expect(mockDynamoPutItem).toHaveBeenCalledWith({ + TableName: process.env.DYNAMODB_CACHE_TABLE, + Item: { + pageKey: { S: cacheKey }, + cacheKey: { S: cacheKey }, + s3Key: { S: `${cacheKey}/${cacheKey}` }, + tags: { S: '' }, + createdAt: { S: expect.any(String) } + } + }) + }) + + it('should delete cache value', async () => { + await s3Cache.delete(cacheKey, cacheKey) + expect(s3Cache.client.deleteObjects).toHaveBeenCalledTimes(1) + expect(s3Cache.client.deleteObjects).toHaveBeenNthCalledWith(1, { + Bucket: mockBucketName, + Delete: { + Objects: [ + { Key: `${cacheKey}/${cacheKey}.json` }, + { Key: `${cacheKey}/${cacheKey}.html` }, + { Key: `${cacheKey}/${cacheKey}.rsc` } + ] + } + }) + }) + + it('should revalidate cache by tag and delete objects', async () => { + const s3Path = `${pageKey}/${cacheKey}` + const mockQueryResult = { + Items: [ + { + pageKey: { S: pageKey }, + cacheKey: { S: cacheKey } + } + ] + } + + mockDynamoQuery.mockResolvedValueOnce(mockQueryResult) + mockGetObjectTagging.mockResolvedValue({ TagSet: [{ Key: TAG_PREFIX, Value: 'test-tag' }] }) + mockGetObjectList.mockResolvedValueOnce({ + Contents: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] + }) + + await s3Cache.revalidateTag('test-tag') + + expect(mockDynamoQuery).toHaveBeenCalledWith({ + TableName: process.env.DYNAMODB_CACHE_TABLE, + KeyConditionExpression: '#field = :value', + ExpressionAttributeNames: { + '#field': 'tags' + }, + ExpressionAttributeValues: { + ':value': { S: 'test-tag' } + } + }) + + expect(s3Cache.client.deleteObjects).toHaveBeenCalledWith({ + Bucket: mockBucketName, + Delete: { + Objects: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] + } + }) + }) + + it('should revalidate cache by path', async () => { + const s3Path = `${pageKey}/${cacheKey}` + mockGetObjectList.mockResolvedValueOnce({ + Contents: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] + }) + + await s3Cache.deleteAllByKeyMatch(cacheKey, '') + + expect(s3Cache.client.listObjectsV2).toHaveBeenCalledWith({ + Bucket: mockBucketName, + ContinuationToken: undefined, + Prefix: `${cacheKey}/`, + Delimiter: '/' + }) + + expect(s3Cache.client.deleteObjects).toHaveBeenCalledWith({ + Bucket: mockBucketName, + Delete: { + Objects: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] + } + }) + }) +}) diff --git a/src/common/array.spec.ts b/src/__tests__/common/array.spec.ts similarity index 95% rename from src/common/array.spec.ts rename to src/__tests__/common/array.spec.ts index d6b0e01..76aca3b 100644 --- a/src/common/array.spec.ts +++ b/src/__tests__/common/array.spec.ts @@ -1,4 +1,4 @@ -import { chunkArray } from './array' +import { chunkArray } from '../../common/array' describe('chunkArray', () => { interface TestCase { diff --git a/src/cacheHandler/strategy/s3.spec.ts b/src/cacheHandler/strategy/s3.spec.ts deleted file mode 100644 index a5b2baa..0000000 --- a/src/cacheHandler/strategy/s3.spec.ts +++ /dev/null @@ -1,240 +0,0 @@ -// import { CacheEntry, CacheContext } from '@dbbs/next-cache-handler-core' -// import { S3Cache, TAG_PREFIX } from './s3' - -// const mockHtmlPage = '

My Page

' - -// export const mockCacheEntry = { -// value: { -// pageData: {}, -// html: mockHtmlPage, -// kind: 'PAGE', -// postponed: undefined, -// headers: undefined, -// status: 200 -// }, -// lastModified: 100000 -// } satisfies CacheEntry - -// const mockCacheContext: CacheContext = { -// isAppRouter: false, -// serverCacheDirPath: '' -// } - -// const mockBucketName = 'test-bucket' -// const cacheKey = 'test' -// const pageKey = 'index' -// const s3Cache = new S3Cache(mockBucketName) - -// const store = new Map() -// const mockGetObject = jest.fn().mockImplementation(async ({ Key }) => { -// const res = store.get(Key) -// return res -// ? { Body: { transformToString: () => res.Body }, Metadata: res.Metadata } -// : { Body: undefined, Metadata: undefined } -// }) -// const mockPutObject = jest -// .fn() -// .mockImplementation(async ({ Key, Body, Metadata }) => store.set(Key, { Body, Metadata })) -// const mockDeleteObject = jest.fn().mockImplementation(async ({ Key }) => store.delete(Key)) -// const mockDeleteObjects = jest -// .fn() -// .mockImplementation(async ({ Delete: { Objects } }: { Delete: { Objects: { Key: string }[] } }) => -// Objects.forEach(({ Key }) => store.delete(Key)) -// ) -// const mockGetObjectList = jest -// .fn() -// .mockImplementation(async () => ({ Contents: [...store.keys()].map((key) => ({ Key: key })) })) -// const mockGetObjectTagging = jest -// .fn() -// .mockImplementation(() => ({ TagSet: [{ Key: 'revalidateTag0', Value: cacheKey }] })) - -// jest.mock('@aws-sdk/client-s3', () => { -// return { -// S3: jest.fn().mockReturnValue({ -// getObject: jest.fn((...params) => mockGetObject(...params)), -// putObject: jest.fn((...params) => mockPutObject(...params)), -// deleteObject: jest.fn((...params) => mockDeleteObject(...params)), -// deleteObjects: jest.fn((...params) => mockDeleteObjects(...params)), -// listObjectsV2: jest.fn((...params) => mockGetObjectList(...params)), -// getObjectTagging: jest.fn((...params) => mockGetObjectTagging(...params)), -// config: {} -// }) -// } -// }) - -// const mockDynamoQuery = jest.fn() -// const mockDynamoPutItem = jest.fn() -// jest.mock('@aws-sdk/client-dynamodb', () => { -// return { -// DynamoDB: jest.fn().mockReturnValue({ -// query: jest.fn((...params) => mockDynamoQuery(...params)), -// putItem: jest.fn((...params) => mockDynamoPutItem(...params)) -// }) -// } -// }) - -// describe('S3Cache', () => { -// afterEach(() => { -// jest.clearAllMocks() -// store.clear() -// }) -// afterAll(() => { -// jest.restoreAllMocks() -// }) - -// it('get should return null', async () => { -// const result = await s3Cache.get() -// expect(result).toBeNull() -// }) - -// it('should set cache for page router', async () => { -// await s3Cache.set(cacheKey, cacheKey, mockCacheEntry, mockCacheContext) -// expect(s3Cache.client.putObject).toHaveBeenCalledTimes(2) -// expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { -// Bucket: mockBucketName, -// Key: `${cacheKey}/${cacheKey}.html`, -// Body: mockHtmlPage, -// ContentType: 'text/html', -// Metadata: { -// 'Cache-Fragment-Key': cacheKey -// } -// }) -// expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(2, { -// Bucket: mockBucketName, -// Key: `${cacheKey}/${cacheKey}.json`, -// Body: JSON.stringify(mockCacheEntry), -// ContentType: 'application/json', -// Metadata: { -// 'Cache-Fragment-Key': cacheKey -// } -// }) -// expect(mockDynamoPutItem).toHaveBeenCalledWith({ -// TableName: process.env.DYNAMODB_CACHE_TABLE, -// Item: { -// pageKey: { S: cacheKey }, -// cacheKey: { S: cacheKey }, -// s3Key: { S: `${cacheKey}/${cacheKey}` }, -// tags: { S: '' }, -// createdAt: { S: expect.any(String) } -// } -// }) -// }) - -// it('should set cache for app router', async () => { -// await s3Cache.set(cacheKey, cacheKey, mockCacheEntry, { ...mockCacheContext, isAppRouter: true }) -// expect(s3Cache.client.putObject).toHaveBeenCalledTimes(3) -// expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(1, { -// Bucket: mockBucketName, -// Key: `${cacheKey}/${cacheKey}.html`, -// Body: mockHtmlPage, -// ContentType: 'text/html', -// Metadata: { -// 'Cache-Fragment-Key': cacheKey -// } -// }) -// expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(2, { -// Bucket: mockBucketName, -// Key: `${cacheKey}/${cacheKey}.json`, -// Body: JSON.stringify(mockCacheEntry), -// ContentType: 'application/json', -// Metadata: { -// 'Cache-Fragment-Key': cacheKey -// } -// }) -// expect(s3Cache.client.putObject).toHaveBeenNthCalledWith(3, { -// Bucket: mockBucketName, -// Key: `${cacheKey}/${cacheKey}.rsc`, -// Body: mockCacheEntry.value.pageData, -// ContentType: 'text/x-component', -// Metadata: { -// 'Cache-Fragment-Key': cacheKey -// } -// }) -// expect(mockDynamoPutItem).toHaveBeenCalledWith({ -// TableName: process.env.DYNAMODB_CACHE_TABLE, -// Item: { -// pageKey: { S: cacheKey }, -// cacheKey: { S: cacheKey }, -// s3Key: { S: `${cacheKey}/${cacheKey}` }, -// tags: { S: '' }, -// createdAt: { S: expect.any(String) } -// } -// }) -// }) - -// it('should delete cache value', async () => { -// await s3Cache.delete(cacheKey, cacheKey) -// expect(s3Cache.client.deleteObjects).toHaveBeenCalledTimes(1) -// expect(s3Cache.client.deleteObjects).toHaveBeenNthCalledWith(1, { -// Bucket: mockBucketName, -// Delete: { -// Objects: [ -// { Key: `${cacheKey}/${cacheKey}.json` }, -// { Key: `${cacheKey}/${cacheKey}.html` }, -// { Key: `${cacheKey}/${cacheKey}.rsc` } -// ] -// } -// }) -// }) - -// it('should revalidate cache by tag and delete objects', async () => { -// const s3Path = `${pageKey}/${cacheKey}` -// const mockQueryResult = { -// Items: [ -// { -// pageKey: { S: pageKey }, -// cacheKey: { S: cacheKey } -// } -// ] -// } - -// mockDynamoQuery.mockResolvedValueOnce(mockQueryResult) -// mockGetObjectTagging.mockResolvedValue({ TagSet: [{ Key: TAG_PREFIX, Value: 'test-tag' }] }) -// mockGetObjectList.mockResolvedValueOnce({ -// Contents: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] -// }) - -// await s3Cache.revalidateTag('test-tag') - -// expect(mockDynamoQuery).toHaveBeenCalledWith({ -// TableName: process.env.DYNAMODB_CACHE_TABLE, -// KeyConditionExpression: '#field = :value', -// ExpressionAttributeNames: { -// '#field': 'tags' -// }, -// ExpressionAttributeValues: { -// ':value': { S: 'test-tag' } -// } -// }) - -// expect(s3Cache.client.deleteObjects).toHaveBeenCalledWith({ -// Bucket: mockBucketName, -// Delete: { -// Objects: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] -// } -// }) -// }) - -// it('should revalidate cache by path', async () => { -// const s3Path = `${pageKey}/${cacheKey}` -// mockGetObjectList.mockResolvedValueOnce({ -// Contents: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] -// }) - -// await s3Cache.deleteAllByKeyMatch(cacheKey, '') - -// expect(s3Cache.client.listObjectsV2).toHaveBeenCalledWith({ -// Bucket: mockBucketName, -// ContinuationToken: undefined, -// Prefix: `${cacheKey}/`, -// Delimiter: '/' -// }) - -// expect(s3Cache.client.deleteObjects).toHaveBeenCalledWith({ -// Bucket: mockBucketName, -// Delete: { -// Objects: [{ Key: s3Path + '.json' }, { Key: s3Path + '.html' }, { Key: s3Path + '.rsc' }] -// } -// }) -// }) -// }) diff --git a/src/cacheHandler/strategy/s3.ts b/src/cacheHandler/strategy/s3.ts index 1d5444a..7829437 100644 --- a/src/cacheHandler/strategy/s3.ts +++ b/src/cacheHandler/strategy/s3.ts @@ -12,6 +12,8 @@ enum CacheExtension { } const PAGE_CACHE_EXTENSIONS = Object.values(CacheExtension) const CHUNK_LIMIT = 1000 +export const CACHE_ONE_YEAR = 31536000 + export class S3Cache implements CacheStrategy { public readonly client: S3 public readonly bucketName: string @@ -50,10 +52,13 @@ export class S3Cache implements CacheStrategy { } async set(pageKey: string, cacheKey: string, data: CacheEntry, ctx: CacheContext): Promise { - if (!data.value?.kind || data.value?.kind === CachedRouteKind.REDIRECT) return Promise.resolve() + if (!data.value?.kind || data.value.kind === CachedRouteKind.REDIRECT || data.revalidate === 0) + return Promise.resolve() - //@ts-expect-error - TODO: fix this - const headersTags = this.buildTagKeys(data.value?.headers?.[NEXT_CACHE_TAGS_HEADER]?.toString()) + let headersTags = '' + if ('headers' in data.value) { + headersTags = this.buildTagKeys(data.value?.headers?.[NEXT_CACHE_TAGS_HEADER]?.toString()) + } const baseInput: PutObjectCommandInput = { Bucket: this.bucketName, @@ -61,9 +66,7 @@ export class S3Cache implements CacheStrategy { Metadata: { 'Cache-Fragment-Key': cacheKey }, - // TODO: check if we want to cache page until next deployment, then next won't pass revalidate value - // check how to identify such case - ...(data.revalidate ? { CacheControl: `s-maxage=${data.revalidate}, stale-while-revalidate=120` } : undefined) + CacheControl: `s-maxage=${data.revalidate || CACHE_ONE_YEAR}, stale-while-revalidate=${CACHE_ONE_YEAR - (data.revalidate || 0)}` } const input: PutObjectCommandInput = { ...baseInput } @@ -80,9 +83,6 @@ export class S3Cache implements CacheStrategy { }) ] - console.log('HERE_IS_DATA', data) - console.log('HERE_IS_DATA_VALUE', data.value) - switch (data.value.kind) { case CachedRouteKind.APP_PAGE: { promises.push( @@ -142,8 +142,7 @@ export class S3Cache implements CacheStrategy { this.client.putObject({ ...input, Key: `${input.Key}.${CacheExtension.RSC}`, - // @ts-expect-error - TODO: fix this - Body: data.value.pageData as string, // for server react components we need to safe additional reference data for nextjs. + Body: data.value.pageData as unknown as string, // for server react components we need to safe additional reference data for nextjs. ContentType: 'text/x-component' }) ) diff --git a/src/cdk/constructs/ViewerRequestLambdaEdge.ts b/src/cdk/constructs/ViewerRequestLambdaEdge.ts index dc8e4ee..6e4a066 100644 --- a/src/cdk/constructs/ViewerRequestLambdaEdge.ts +++ b/src/cdk/constructs/ViewerRequestLambdaEdge.ts @@ -13,7 +13,6 @@ interface ViewerRequestLambdaEdgeProps extends cdk.StackProps { nodejs?: string redirects?: NextRedirects internationalizationConfig?: DeployConfig['internationalization'] - trailingSlash?: boolean } const NodeJSEnvironmentMapping: Record = { @@ -25,7 +24,7 @@ export class ViewerRequestLambdaEdge extends Construct { public readonly lambdaEdge: cloudfront.experimental.EdgeFunction constructor(scope: Construct, id: string, props: ViewerRequestLambdaEdgeProps) { - const { nodejs, buildOutputPath, redirects, internationalizationConfig, trailingSlash = false } = props + const { nodejs, buildOutputPath, redirects, internationalizationConfig } = props super(scope, id) const nodeJSEnvironment = NodeJSEnvironmentMapping[nodejs ?? ''] ?? NodeJSEnvironmentMapping['20'] @@ -34,8 +33,7 @@ export class ViewerRequestLambdaEdge extends Construct { buildLambda(name, buildOutputPath, { define: { 'process.env.REDIRECTS': JSON.stringify(redirects ?? []), - 'process.env.LOCALES_CONFIG': JSON.stringify(internationalizationConfig ?? null), - 'process.env.IS_TRAILING_SLASH': JSON.stringify(trailingSlash) + 'process.env.LOCALES_CONFIG': JSON.stringify(internationalizationConfig ?? null) } }) From 3abe457306036ace1abae60661841c3bca6a9436 Mon Sep 17 00:00:00 2001 From: Roman Bobrovskiy Date: Tue, 18 Feb 2025 20:20:12 +0100 Subject: [PATCH 3/6] feat: updated redirect and locale redirects --- src/cdk/constructs/ViewerRequestLambdaEdge.ts | 8 ++++---- src/cdk/stacks/NextCloudfrontStack.ts | 11 +++++------ src/commands/deploy.ts | 3 ++- src/lambdas/viewerRequest.ts | 12 ++---------- src/types/index.ts | 6 ++---- 5 files changed, 15 insertions(+), 25 deletions(-) diff --git a/src/cdk/constructs/ViewerRequestLambdaEdge.ts b/src/cdk/constructs/ViewerRequestLambdaEdge.ts index 6e4a066..591d9b8 100644 --- a/src/cdk/constructs/ViewerRequestLambdaEdge.ts +++ b/src/cdk/constructs/ViewerRequestLambdaEdge.ts @@ -6,13 +6,13 @@ import * as logs from 'aws-cdk-lib/aws-logs' import * as iam from 'aws-cdk-lib/aws-iam' import path from 'node:path' import { buildLambda } from '../../common/esbuild' -import { NextRedirects, DeployConfig } from '../../types' +import { NextRedirects, NextI18nConfig } from '../../types' interface ViewerRequestLambdaEdgeProps extends cdk.StackProps { buildOutputPath: string nodejs?: string redirects?: NextRedirects - internationalizationConfig?: DeployConfig['internationalization'] + nextI18nConfig?: NextI18nConfig } const NodeJSEnvironmentMapping: Record = { @@ -24,7 +24,7 @@ export class ViewerRequestLambdaEdge extends Construct { public readonly lambdaEdge: cloudfront.experimental.EdgeFunction constructor(scope: Construct, id: string, props: ViewerRequestLambdaEdgeProps) { - const { nodejs, buildOutputPath, redirects, internationalizationConfig } = props + const { nodejs, buildOutputPath, redirects, nextI18nConfig } = props super(scope, id) const nodeJSEnvironment = NodeJSEnvironmentMapping[nodejs ?? ''] ?? NodeJSEnvironmentMapping['20'] @@ -33,7 +33,7 @@ export class ViewerRequestLambdaEdge extends Construct { buildLambda(name, buildOutputPath, { define: { 'process.env.REDIRECTS': JSON.stringify(redirects ?? []), - 'process.env.LOCALES_CONFIG': JSON.stringify(internationalizationConfig ?? null) + 'process.env.LOCALES_CONFIG': JSON.stringify(nextI18nConfig ?? null) } }) diff --git a/src/cdk/stacks/NextCloudfrontStack.ts b/src/cdk/stacks/NextCloudfrontStack.ts index 54e233f..2a0d485 100644 --- a/src/cdk/stacks/NextCloudfrontStack.ts +++ b/src/cdk/stacks/NextCloudfrontStack.ts @@ -5,7 +5,7 @@ import { OriginRequestLambdaEdge } from '../constructs/OriginRequestLambdaEdge' import { CloudFrontDistribution } from '../constructs/CloudFrontDistribution' import { ViewerResponseLambdaEdge } from '../constructs/ViewerResponseLambdaEdge' import { ViewerRequestLambdaEdge } from '../constructs/ViewerRequestLambdaEdge' -import { DeployConfig, NextRedirects } from '../../types' +import { DeployConfig, NextRedirects, NextI18nConfig } from '../../types' export interface NextCloudfrontStackProps extends StackProps { nodejs?: string @@ -16,7 +16,7 @@ export interface NextCloudfrontStackProps extends StackProps { deployConfig: DeployConfig imageTTL?: number redirects?: NextRedirects - trailingSlash?: boolean + nextI18nConfig?: NextI18nConfig nextCachedRoutesMatchers: string[] } @@ -37,8 +37,8 @@ export class NextCloudfrontStack extends Stack { deployConfig, imageTTL, redirects, - trailingSlash = false, - nextCachedRoutesMatchers + nextCachedRoutesMatchers, + nextI18nConfig } = props this.originRequestLambdaEdge = new OriginRequestLambdaEdge(this, `${id}-OriginRequestLambdaEdge`, { @@ -55,8 +55,7 @@ export class NextCloudfrontStack extends Stack { buildOutputPath, nodejs, redirects, - internationalizationConfig: deployConfig.internationalization, - trailingSlash + nextI18nConfig }) this.viewerResponseLambdaEdge = new ViewerResponseLambdaEdge(this, `${id}-ViewerResponseLambdaEdge`, { diff --git a/src/commands/deploy.ts b/src/commands/deploy.ts index b4243ba..394fb24 100644 --- a/src/commands/deploy.ts +++ b/src/commands/deploy.ts @@ -86,6 +86,7 @@ export const deploy = async (config: DeployConfig) => { const nextConfig = (await loadFile(projectSettings.nextConfigPath)) as NextConfig const nextRedirects = nextConfig.redirects ? await nextConfig.redirects() : undefined + const nextI18nConfig = nextConfig.i18n const outputPath = createOutputFolder() @@ -154,7 +155,7 @@ export const deploy = async (config: DeployConfig) => { region, deployConfig, imageTTL: nextConfig.imageTTL, - trailingSlash: nextConfig.trailingSlash, + nextI18nConfig, redirects: nextRedirects, nextCachedRoutesMatchers, env: { diff --git a/src/lambdas/viewerRequest.ts b/src/lambdas/viewerRequest.ts index fd1e421..44db9a4 100644 --- a/src/lambdas/viewerRequest.ts +++ b/src/lambdas/viewerRequest.ts @@ -1,5 +1,5 @@ import type { CloudFrontRequestCallback, Context, CloudFrontResponseEvent } from 'aws-lambda' -import type { NextRedirects, DeployConfig } from '../types' +import type { NextRedirects, NextI18nConfig } from '../types' import path from 'node:path' /** @@ -18,15 +18,7 @@ export const handler = async ( ) => { const request = event.Records[0].cf.request const redirectsConfig = process.env.REDIRECTS as unknown as NextRedirects - const localesConfig = process.env.LOCALES_CONFIG as unknown as DeployConfig['internationalization'] | null - const isTrailingSlash = process.env.IS_TRAILING_SLASH as unknown as boolean - const pathHasTrailingSlash = request.uri.endsWith('/') - - if (pathHasTrailingSlash && !isTrailingSlash) { - request.uri = request.uri.slice(0, -1) - } else if (!pathHasTrailingSlash && isTrailingSlash) { - request.uri += '/' - } + const localesConfig = process.env.LOCALES_CONFIG as unknown as NextI18nConfig | null let shouldRedirectWithLocale = false let pagePath = request.uri diff --git a/src/types/index.ts b/src/types/index.ts index 8fe246a..4f40cf1 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -8,11 +8,9 @@ export interface CacheConfig { } export interface DeployConfig { - internationalization?: { - locales: string[] - defaultLocale: string - } cache: CacheConfig } export type NextRedirects = Awaited['redirects']>> + +export type NextI18nConfig = NextConfig['i18n'] From ec0b9b91a6d809cf98d149cf6d81434f121b4282 Mon Sep 17 00:00:00 2001 From: Roman Bobrovskiy Date: Tue, 18 Feb 2025 20:25:36 +0100 Subject: [PATCH 4/6] feat: fixed lint --- src/__tests__/__mocks__/cacheEntries.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/__tests__/__mocks__/cacheEntries.ts b/src/__tests__/__mocks__/cacheEntries.ts index 7b15d3f..3dd3bf8 100644 --- a/src/__tests__/__mocks__/cacheEntries.ts +++ b/src/__tests__/__mocks__/cacheEntries.ts @@ -3,7 +3,6 @@ import { CachedRedirectValue, IncrementalCachedPageValue, IncrementalCachedAppPageValue, - CachedImageValue, CachedFetchValue, CachedRouteValue } from '@dbbs/next-cache-handler-core' From 49db1efbd92c86bec9e9e11d9d1a2d8c95fb0c07 Mon Sep 17 00:00:00 2001 From: Roman Bobrovskiy Date: Tue, 18 Feb 2025 20:34:43 +0100 Subject: [PATCH 5/6] feat: removed log --- src/common/project.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/common/project.ts b/src/common/project.ts index 349b36c..c25ef11 100644 --- a/src/common/project.ts +++ b/src/common/project.ts @@ -76,11 +76,7 @@ export const loadFile = async (filePath: string) => { const script = new vm.Script(res.code) const context = vm.createContext({ module: {}, exports: {}, require }) script.runInContext(context) - console.log({ - context, - module: context.module, - default: context.module.exports.default - }) + return context.module.exports.default } From 27984c4c4797b2b2232a3bfb79785c4c07d69c6c Mon Sep 17 00:00:00 2001 From: Roman Bobrovskiy Date: Wed, 19 Feb 2025 11:22:15 +0100 Subject: [PATCH 6/6] feat: re-used nextjs function for generating header --- src/cacheHandler/strategy/s3.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/cacheHandler/strategy/s3.ts b/src/cacheHandler/strategy/s3.ts index 7829437..b742eec 100644 --- a/src/cacheHandler/strategy/s3.ts +++ b/src/cacheHandler/strategy/s3.ts @@ -1,4 +1,5 @@ import { NEXT_CACHE_TAGS_HEADER } from 'next/dist/lib/constants' +import { formatRevalidate } from 'next/dist/server/lib/revalidate' import { type ListObjectsV2CommandOutput, type PutObjectCommandInput, S3 } from '@aws-sdk/client-s3' import { DynamoDB } from '@aws-sdk/client-dynamodb' import { chunkArray } from '../../common/array' @@ -52,7 +53,12 @@ export class S3Cache implements CacheStrategy { } async set(pageKey: string, cacheKey: string, data: CacheEntry, ctx: CacheContext): Promise { - if (!data.value?.kind || data.value.kind === CachedRouteKind.REDIRECT || data.revalidate === 0) + if ( + !data.value?.kind || + data.value.kind === CachedRouteKind.REDIRECT || + data.revalidate === 0 || + data.revalidate === undefined + ) return Promise.resolve() let headersTags = '' @@ -66,7 +72,10 @@ export class S3Cache implements CacheStrategy { Metadata: { 'Cache-Fragment-Key': cacheKey }, - CacheControl: `s-maxage=${data.revalidate || CACHE_ONE_YEAR}, stale-while-revalidate=${CACHE_ONE_YEAR - (data.revalidate || 0)}` + CacheControl: formatRevalidate({ + revalidate: data.revalidate, + swrDelta: data.revalidate ? CACHE_ONE_YEAR - data.revalidate : CACHE_ONE_YEAR + }) } const input: PutObjectCommandInput = { ...baseInput } @@ -156,6 +165,10 @@ export class S3Cache implements CacheStrategy { }) ) } + break + } + default: { + console.warn(`Unknown cache kind: ${data.value.kind}`) } }