From 5536f55ec3486fbba5411c6c8c58b8d79dd2abd3 Mon Sep 17 00:00:00 2001 From: Oguz Vuruskaner Date: Fri, 19 Apr 2024 15:52:49 +0300 Subject: [PATCH 1/6] disabled pre commit hooks --- .gitignore | 1 + .husky/pre-commit | 1 - .husky/prepare-commit-msg | 6 ------ 3 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 .husky/pre-commit delete mode 100644 .husky/prepare-commit-msg diff --git a/.gitignore b/.gitignore index dc563b7..fab3dfb 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,4 @@ docs dist **misc.ts docs +.husky diff --git a/.husky/pre-commit b/.husky/pre-commit deleted file mode 100644 index 605953f..0000000 --- a/.husky/pre-commit +++ /dev/null @@ -1 +0,0 @@ -. "$(dirname -- "$0")/_/husky.sh" diff --git a/.husky/prepare-commit-msg b/.husky/prepare-commit-msg deleted file mode 100644 index 339d9fd..0000000 --- a/.husky/prepare-commit-msg +++ /dev/null @@ -1,6 +0,0 @@ -npm run lint -npm run prettier -npm run build -npm run test - -exec < /dev/tty && git cz --hook || true From 6a22b1236f72c5d745a59229d981cbd3d9c9a847 Mon Sep 17 00:00:00 2001 From: Oguz Vuruskaner Date: Fri, 19 Apr 2024 16:00:47 +0300 Subject: [PATCH 2/6] ci(workflow): Unit test pipeline is implemented. --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d00793e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: Publish NPM Package + +on: + push: + branches: + - '**' + pull_request: + branches: + - '**' + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + publish-npm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2 + with: + node-version: '18.x' + registry-url: 'https://registry.npmjs.org' + + - name: Install pnpm + run: npm install -g pnpm + - run: pnpm install + - run: pnpm run test + - run: pnpm run build + From f8f87a02b3f154eb3a4b8eb4fee8d09544bb8cf3 Mon Sep 17 00:00:00 2001 From: Oguz Vuruskaner Date: Fri, 19 Apr 2024 16:02:11 +0300 Subject: [PATCH 3/6] ci(workflow): Updated name of the pipeline --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d00793e..272dbb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Publish NPM Package +name: Run Unit Tests on: push: From 5fdc0245c794d4d5d4b9468da56ae4e2e919eae1 Mon Sep 17 00:00:00 2001 From: Oguz Vuruskaner Date: Fri, 19 Apr 2024 17:19:07 +0300 Subject: [PATCH 4/6] test(pipeline): Implement E2E tests (WIP) --- .github/workflows/e2e-test.yml | 32 ++++++++++++++++++ jest.e2e.config.js | 22 ++++++++++++ package.json | 1 + src/lib/types/common/models.ts | 33 ++++++++++++++++++ test/e2e/index.e2e-test.ts | 61 ++++++++++++++++++++++++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 .github/workflows/e2e-test.yml create mode 100644 jest.e2e.config.js create mode 100644 src/lib/types/common/models.ts create mode 100644 test/e2e/index.e2e-test.ts diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml new file mode 100644 index 0000000..6f22b96 --- /dev/null +++ b/.github/workflows/e2e-test.yml @@ -0,0 +1,32 @@ +name: Run E2E Tests + +on: + pull_request: + branches: + - 'main' + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +env: + CI: true + DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} + +jobs: + publish-npm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2 + with: + node-version: '18.x' + registry-url: 'https://registry.npmjs.org' + + - name: Install pnpm + run: npm install -g pnpm + - run: pnpm install + - run: pnpm run test:e2e + - run: pnpm run build + diff --git a/jest.e2e.config.js b/jest.e2e.config.js new file mode 100644 index 0000000..ab80b08 --- /dev/null +++ b/jest.e2e.config.js @@ -0,0 +1,22 @@ +/** @type {import('ts-jest').JstConfigWithTsJest} */ +module.exports = { + testEnvironment: 'node', + extensionsToTreatAsEsm: [".ts"], + testTimeout: 10000, + coveragePathIgnorePatterns: [ + "/node_modules/", + "/examples/", + "/test/" + ], + moduleNameMapper: { + "^@/(.*)$": "/src/$1" + }, + testMatch: [ + "/test/**/*.e2e-test.ts" + ], + rootDir: ".", + transform: { + "^.+\\.tsx?$": "ts-jest" + }, + +}; diff --git a/package.json b/package.json index 27647ce..29fa161 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "misc": "npx ts-node -r tsconfig-paths/register src/misc.ts", "prepare": "husky", "test": "jest --passWithNoTests", + "test:e2e": "jest --config=jest.e2e.config.js --passWithNoTests --runInBand", "lint": "eslint . --ext .ts --fix", "prettier": "prettier --write ./src ./test", "build-docs": "typedoc --out docs src", diff --git a/src/lib/types/common/models.ts b/src/lib/types/common/models.ts new file mode 100644 index 0000000..5e2e187 --- /dev/null +++ b/src/lib/types/common/models.ts @@ -0,0 +1,33 @@ +export const enum ModelTypes { + EMBEDDINGS = 'embeddings', + FILL_MASK = 'fill-mask', + TEXT_GENERATION = 'text-generation', + AUTOMATIC_SPEECH_RECOGNITION = 'automatic-speech-recognition', + TOKEN_CLASSIFICATION = 'token-classification', + TEXT2TEXT_GENERATION = 'text2text-generation', + OBJECT_DETECTION = 'object-detection', + QUESTION_ANSWERING = 'question-answering', + IMAGE_CLASSIFICATION = 'image-classification', + TEXT_TO_IMAGE = 'text-to-image', + ZERO_SHOT_IMAGE_CLASSIFICATION = 'zero-shot-image-classification', + CUSTOM = 'custom', + TEXT_CLASSIFICATION = 'text-classification', + DREAMBOOTH = 'dreambooth', +} + + +export interface ModelDefinition { + model_name: string; + type: ModelTypes; + reported_type: ModelTypes; + description: string; + cover_img_url: string; + tags: string[]; + pricing: { + cents_per_sec: number; + type: string; + }; + max_tokens: number | null; +} + +export type ModelDefinitionList = ModelDefinition[]; diff --git a/test/e2e/index.e2e-test.ts b/test/e2e/index.e2e-test.ts new file mode 100644 index 0000000..31e96d0 --- /dev/null +++ b/test/e2e/index.e2e-test.ts @@ -0,0 +1,61 @@ +import axios from 'axios'; +import {ModelDefinition, ModelDefinitionList, ModelTypes} from "@/lib/types/common/models"; +import { + AutomaticSpeechRecognition, + Embeddings, + FillMask, + ObjectDetection, + QuestionAnswering, + Sdxl, + TextClassification, + TextGeneration, + TextToImage, + TokenClassification +} from "@/index"; + +const GET_MODELS = "https://api.deepinfra.com/models/list"; +const SDXL_MODEL = "stability-ai/sdxl"; +const TEXT_TO_IMAGE_PROMPT = "The quick brown fox jumps over the lazy dog."; + +describe('E2E tests', () => { + + + let allModels: ModelDefinitionList; + + beforeAll(async () => { + const response = await axios.get(GET_MODELS); + allModels = response.data as ModelDefinitionList; + }); + + it('should have at least one model', () => { + expect(allModels.length).toBeGreaterThan(0); + }); + + describe("Text to image models", () => { + const textToImageModels = allModels.filter(model => model.reported_type === ModelTypes.TEXT_TO_IMAGE).map(m => m.model_name); + it('should infer correctly.', () => { + textToImageModels.forEach(async (modelName) => { + + if (modelName === SDXL_MODEL) { + const model = new Sdxl(); + expect(model).toBeDefined(); + await model.generate({input: {prompt: TEXT_TO_IMAGE_PROMPT}}).then(response => { + expect(response).toBeDefined(); + }); + } else { + const model = new TextToImage(modelName); + expect(model).toBeDefined(); + + await model.generate({prompt: TEXT_TO_IMAGE_PROMPT}).then(response => { + expect(response).toBeDefined(); + }); + } + + }); + }); + + + }); + + +}); From 81d07b51eb175d30b6ec03dc226ce4548565f72e Mon Sep 17 00:00:00 2001 From: Oguz Vuruskaner Date: Fri, 19 Apr 2024 17:35:40 +0300 Subject: [PATCH 5/6] draft --- .github/workflows/e2e-test.yml | 3 +- package.json | 3 +- pnpm-lock.yaml | 15 ++++ test/e2e/index.e2e-test.ts | 153 ++++++++++++++++++++++++++++----- 4 files changed, 152 insertions(+), 22 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 6f22b96..1cb8089 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -14,7 +14,8 @@ env: DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} jobs: - publish-npm: + e2e-test: + if: github.event.pull_request.draft == false runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/package.json b/package.json index 29fa161..e85b469 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,8 @@ "dependencies": { "@swc/core": "^1.4.6", "@swc/wasm": "^1.4.6", - "axios": "^1.6.7" + "axios": "^1.6.7", + "p-limit": "^5.0.0" }, "devDependencies": { "@types/jest": "^29.5.12", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eae292b..8a7ec9d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ dependencies: axios: specifier: ^1.6.7 version: 1.6.7 + p-limit: + specifier: ^5.0.0 + version: 5.0.0 devDependencies: '@types/jest': @@ -3411,6 +3414,13 @@ packages: yocto-queue: 0.1.0 dev: true + /p-limit@5.0.0: + resolution: {integrity: sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==} + engines: {node: '>=18'} + dependencies: + yocto-queue: 1.0.0 + dev: false + /p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -4238,3 +4248,8 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} dev: true + + /yocto-queue@1.0.0: + resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} + engines: {node: '>=12.20'} + dev: false diff --git a/test/e2e/index.e2e-test.ts b/test/e2e/index.e2e-test.ts index 31e96d0..c2a913d 100644 --- a/test/e2e/index.e2e-test.ts +++ b/test/e2e/index.e2e-test.ts @@ -16,9 +16,16 @@ import { const GET_MODELS = "https://api.deepinfra.com/models/list"; const SDXL_MODEL = "stability-ai/sdxl"; const TEXT_TO_IMAGE_PROMPT = "The quick brown fox jumps over the lazy dog."; +const TEXT_INPUT = "This is a test."; + +/* +TODO: Add mock audio file for ASR. +TODO: Add mock image file for object detection. +TODO: Implement p-limit + */ -describe('E2E tests', () => { +describe('E2E tests', () => { let allModels: ModelDefinitionList; @@ -30,32 +37,138 @@ describe('E2E tests', () => { it('should have at least one model', () => { expect(allModels.length).toBeGreaterThan(0); }); - - describe("Text to image models", () => { + it('Text to image models should infer correctly.', () => { const textToImageModels = allModels.filter(model => model.reported_type === ModelTypes.TEXT_TO_IMAGE).map(m => m.model_name); - it('should infer correctly.', () => { - textToImageModels.forEach(async (modelName) => { - - if (modelName === SDXL_MODEL) { - const model = new Sdxl(); - expect(model).toBeDefined(); - await model.generate({input: {prompt: TEXT_TO_IMAGE_PROMPT}}).then(response => { - expect(response).toBeDefined(); - }); - } else { - const model = new TextToImage(modelName); - expect(model).toBeDefined(); - - await model.generate({prompt: TEXT_TO_IMAGE_PROMPT}).then(response => { - expect(response).toBeDefined(); - }); - } + textToImageModels.forEach(async (modelName) => { + + if (modelName === SDXL_MODEL) { + const model = new Sdxl(); + expect(model).toBeDefined(); + await model.generate({input: {prompt: TEXT_TO_IMAGE_PROMPT}}).then(response => { + expect(response).toBeDefined(); + }); + } else { + const model = new TextToImage(modelName); + expect(model).toBeDefined(); + + await model.generate({prompt: TEXT_TO_IMAGE_PROMPT}).then(response => { + expect(response).toBeDefined(); + }); + } + + }); + }); + + it('Text classification models should infer correctly.', () => { + const textClassificationModels = allModels.filter(model => model.reported_type === ModelTypes.TEXT_CLASSIFICATION).map(m => m.model_name); + textClassificationModels.forEach(async (modelName) => { + const model = new TextClassification(modelName); + expect(model).toBeDefined(); + + await model.generate({input: TEXT_INPUT}).then(response => { + expect(response).toBeDefined(); + }); + }); + }); + + it('Text generation models should infer correctly.', () => { + const textGenerationModels = allModels.filter(model => model.reported_type === ModelTypes.TEXT_GENERATION).map(m => m.model_name); + textGenerationModels.forEach(async (modelName) => { + const model = new TextGeneration(modelName); + expect(model).toBeDefined(); + await model.generate({input: TEXT_INPUT}).then(response => { + expect(response).toBeDefined(); }); }); + }); + + it('Fill mask models should infer correctly.', () => { + const fillMaskModels = allModels.filter(model => model.reported_type === ModelTypes.FILL_MASK).map(m => m.model_name); + fillMaskModels.forEach(async (modelName) => { + const model = new FillMask(modelName); + expect(model).toBeDefined(); + + await model.generate({input: "This is a [MASK]"}) + .then(response => { + expect(response).toBeDefined(); + }); + }); + }); + it('Embeddings models should infer correctly.', () => { + const embeddingsModels = allModels.filter(model => model.reported_type === ModelTypes.EMBEDDINGS).map(m => m.model_name); + embeddingsModels.forEach(async (modelName) => { + const model = new Embeddings(modelName); + expect(model).toBeDefined(); + await model.generate({inputs: [TEXT_INPUT]}).then(response => { + expect(response).toBeDefined(); + }); + }); }); + + it('Question answering models should infer correctly.', () => { + const questionAnsweringModels = allModels.filter(model => model.reported_type === ModelTypes.QUESTION_ANSWERING).map(m => m.model_name); + questionAnsweringModels.forEach(async (modelName) => { + const model = new QuestionAnswering(modelName); + expect(model).toBeDefined(); + + await model.generate({question: TEXT_INPUT, context: TEXT_INPUT}) + .then(response => { + expect(response).toBeDefined(); + }); + }); + }); + + it('Token classification models should infer correctly.', () => { + const tokenClassificationModels = allModels.filter(model => model.reported_type === ModelTypes.TOKEN_CLASSIFICATION).map(m => m.model_name); + tokenClassificationModels.forEach(async (modelName) => { + const model = new TokenClassification(modelName); + expect(model).toBeDefined(); + + await model.generate({input: TEXT_INPUT}).then(response => { + expect(response).toBeDefined(); + }); + }); + }); + + it('Text2Text generation models should infer correctly.', () => { + const text2TextGenerationModels = allModels.filter(model => model.reported_type === ModelTypes.TEXT2TEXT_GENERATION).map(m => m.model_name); + text2TextGenerationModels.forEach(async (modelName) => { + const model = new TextGeneration(modelName); + expect(model).toBeDefined(); + + await model.generate({input: TEXT_INPUT}).then(response => { + expect(response).toBeDefined(); + }); + }); + }); + + it('Object detection models should infer correctly.', () => { + const objectDetectionModels = allModels.filter(model => model.reported_type === ModelTypes.OBJECT_DETECTION).map(m => m.model_name); + objectDetectionModels.forEach(async (modelName) => { + const model = new ObjectDetection(modelName); + expect(model).toBeDefined(); + + await model.generate({input: TEXT_INPUT}).then(response => { + expect(response).toBeDefined(); + }); + }); + }); + + it('Automatic speech recognition models should infer correctly.', () => { + const automaticSpeechRecognitionModels = allModels.filter(model => model.reported_type === ModelTypes.AUTOMATIC_SPEECH_RECOGNITION).map(m => m.model_name); + automaticSpeechRecognitionModels.forEach(async (modelName) => { + const model = new AutomaticSpeechRecognition(modelName); + expect(model).toBeDefined(); + + await model.generate({input: TEXT_INPUT}).then(response => { + expect(response).toBeDefined(); + }); + }); + }); + }); From 469ad68c162ab5129c7dc3beadb7235f036754bd Mon Sep 17 00:00:00 2001 From: Oguz Vuruskaner Date: Fri, 19 Apr 2024 17:36:34 +0300 Subject: [PATCH 6/6] dsiabled --- .github/workflows/e2e-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 1cb8089..fc5b7f7 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -6,7 +6,7 @@ on: - 'main' concurrency: - group: ${{ github.ref }} + group: ${{ github.ref }}-e2e-tests cancel-in-progress: true env: @@ -15,7 +15,7 @@ env: jobs: e2e-test: - if: github.event.pull_request.draft == false + if: false runs-on: ubuntu-latest steps: - uses: actions/checkout@v2