From d91bd5882c3213e1e624563ae75d25181c3177ec Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Fri, 7 Nov 2025 14:13:22 +0100 Subject: [PATCH 1/2] fix(curations): Add tests and model fixes - VcsType was not properly accepting correct string values, but list instead. Fixed to proper validate entry - Added VcsInfoCurationData and replaced VcsInfo in PackageCurationData - Add Ort documentation examples as a test set Signed-off-by: Helio Chissini de Castro --- .github/workflows/build.yml | 3 - .github/workflows/testing.yml | 28 +++++++ src/ort/models/package_curation_data.py | 4 +- src/ort/models/vcsinfo_curation_data.py | 37 +++++++++ src/ort/models/vcstype.py | 55 +++++++------ tests/data/example_curations.yml | 89 ++++++++++++++++++++++ tests/data/example_simple_curation.yml | 33 ++++++++ tests/test_ort_repository_configuration.py | 24 ++---- tests/test_package_curation.py | 35 +++++++++ tests/utils/load_yaml_config.py | 27 +++++++ 10 files changed, 284 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/testing.yml create mode 100644 src/ort/models/vcsinfo_curation_data.py create mode 100644 tests/data/example_curations.yml create mode 100644 tests/data/example_simple_curation.yml create mode 100644 tests/test_package_curation.py create mode 100644 tests/utils/load_yaml_config.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d9b80b4..1819dc9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,6 +46,3 @@ jobs: shell: bash run: | uv build - - - name: Test with python ${{ matrix.python-version }} - run: uv run --frozen pytest diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 0000000..0fe7125 --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,28 @@ +name: pytest + +on: [pull_request, workflow_dispatch] + +jobs: + test-optima: + name: Run python-optima tests + runs-on: ubuntu-24.04 + strategy: + matrix: + python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + with: + enable-cache: true + + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 + with: + python-version: ${{ matrix.python-version }} + + - name: Install the project + run: uv sync --locked --all-extras --dev + + - name: Test with python ${{ matrix.python-version }} + run: uv run --frozen pytest diff --git a/src/ort/models/package_curation_data.py b/src/ort/models/package_curation_data.py index c700939..c9cdc33 100644 --- a/src/ort/models/package_curation_data.py +++ b/src/ort/models/package_curation_data.py @@ -7,7 +7,7 @@ from .hash import Hash from .source_code_origin import SourceCodeOrigin -from .vcsinfo import VcsInfo +from .vcsinfo_curation_data import VcsInfoCurationData class CurationArtifact(BaseModel): @@ -28,7 +28,7 @@ class PackageCurationData(BaseModel): homepage_url: str | None = None binary_artifact: CurationArtifact | None = None source_artifact: CurationArtifact | None = None - vcs: VcsInfo | None = None + vcs: VcsInfoCurationData | None = None is_metadata_only: bool | None = None is_modified: bool | None = None declared_license_mapping: dict[str, Any] = Field(default_factory=dict) diff --git a/src/ort/models/vcsinfo_curation_data.py b/src/ort/models/vcsinfo_curation_data.py new file mode 100644 index 0000000..8d138cb --- /dev/null +++ b/src/ort/models/vcsinfo_curation_data.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: 2025 Helio Chissini de Castro +# SPDX-License-Identifier: MIT + +from pydantic import AnyUrl, BaseModel, Field + +from .vcstype import VcsType + + +class VcsInfoCurationData(BaseModel): + """ + Bundles general Version Control System information. + + Attributes: + type(VcsType): The type of the VCS, for example Git, GitRepo, Mercurial, etc. + url(AnyUrl): The URL to the VCS repository. + revision(str): The VCS-specific revision (tag, branch, SHA1) that the version of the package maps to. + path(str): The path inside the VCS to take into account. + If the VCS supports checking out only a subdirectory, only this path is checked out. + """ + + type: VcsType | None = Field( + default=None, + description="The type of the VCS, for example Git, GitRepo, Mercurial, etc.", + ) + url: AnyUrl | None = Field( + default=None, + description="The URL to the VCS repository.", + ) + revision: str | None = Field( + default=None, + description="The VCS-specific revision (tag, branch, SHA1) that the version of the package maps to.", + ) + path: str | None = Field( + default=None, + description="The path inside the VCS to take into account." + "If the VCS supports checking out only a subdirectory, only this path is checked out.", + ) diff --git a/src/ort/models/vcstype.py b/src/ort/models/vcstype.py index 9c0160c..51e9b1c 100644 --- a/src/ort/models/vcstype.py +++ b/src/ort/models/vcstype.py @@ -3,6 +3,14 @@ from pydantic import BaseModel, Field, model_validator +# Define known VCS types as constants +GIT = ["Git", "GitHub", "GitLab"] +GIT_REPO = ["GitRepo", "git-repo", "repo"] +MERCURIAL = ["Mercurial", "hg"] +SUBVERSION = ["Subversion", "svn"] + +KNOWN_TYPES = GIT + GIT_REPO + MERCURIAL + SUBVERSION + class VcsType(BaseModel): """ @@ -12,35 +20,26 @@ class VcsType(BaseModel): alias for the string representation. Attributes: - aliases(list[str]): Primary name and aliases + name(str): Primary name and aliases """ - aliases: list[str] = Field(default_factory=list, description="Primary name and aliases") - - @model_validator(mode="after") - def ensure_non_empty(self): - """Ensure the aliases list is never empty.""" - if not self.aliases: - self.aliases = [""] - return self - - def __str__(self): - return self.aliases[0] if self.aliases else "" + name: str = Field(default_factory=str) + @model_validator(mode="before") @classmethod - def for_name(cls, name: str) -> "VcsType": - """Lookup known type by name, or create a new instance.""" - for t in KNOWN_TYPES: - if any(alias.lower() == name.lower() for alias in t.aliases): - return t - return cls(aliases=[name]) - - -# Define known VCS types as constants -GIT = VcsType(aliases=["Git", "GitHub", "GitLab"]) -GIT_REPO = VcsType(aliases=["GitRepo", "git-repo", "repo"]) -MERCURIAL = VcsType(aliases=["Mercurial", "hg"]) -SUBVERSION = VcsType(aliases=["Subversion", "svn"]) -UNKNOWN = VcsType(aliases=[""]) - -KNOWN_TYPES = [GIT, GIT_REPO, MERCURIAL, SUBVERSION] + def _forName(cls, value): + # Allow direct string input (e.g., "Git" or "gitlab") + if isinstance(value, str): + if any(item.lower() == value.lower() for item in KNOWN_TYPES): + return {"name": value} + else: + # Not a known type → default to empty string + return {"name": ""} + # Allow dict input or existing model + elif isinstance(value, dict): + name = value.get("name", "") + if any(item.lower() == name.lower() for item in KNOWN_TYPES): + return value + else: + return {"name": ""} + return {"name": ""} diff --git a/tests/data/example_curations.yml b/tests/data/example_curations.yml new file mode 100644 index 0000000..e5c4290 --- /dev/null +++ b/tests/data/example_curations.yml @@ -0,0 +1,89 @@ +--- +# Example for a complete curation object: +#- id: "Maven:org.hamcrest:hamcrest-core:1.3" +# curations: +# comment: "An explanation why the curation is needed or the reasoning for a license conclusion." +# concluded_license: "Apache-2.0 OR BSD-3-Clause" # Valid SPDX license expression to override the license findings. +# declared_license_mapping: +# "Copyright (C) 2013, Martin Journois": "NONE" +# "BSD": "BSD-3-Clause" +# description: "Curated description." +# homepage_url: "http://example.com" +# binary_artifact: +# url: "http://example.com/binary.zip" +# hash: +# value: "ddce269a1e3d054cae349621c198dd52" +# algorithm: "MD5" +# source_artifact: +# url: "http://example.com/sources.zip" +# hash: +# value: "ddce269a1e3d054cae349621c198dd52" +# algorithm: "MD5" +# vcs: +# type: "Git" +# url: "http://example.com/repo.git" +# revision: "1234abc" +# path: "subdirectory" +# is_metadata_only: true # Whether the package is metadata only. +# is_modified: true # Whether the package is modified compared to the original source. + +- id: 'Maven:asm:asm' # No version means the curation will be applied to all versions of the package. + curations: + comment: 'Repository moved to https://gitlab.ow2.org.' + vcs: + type: 'Giot' + url: 'https://gitlab.ow2.org/asm/asm.git' + +- id: 'NPM::ast-traverse:0.1.0' + curations: + comment: 'Revision found by comparing the NPM package with the sources from https://github.com/olov/ast-traverse.' + vcs: + revision: 'f864d24ba07cde4b79f16999b1c99bfb240a441e' + +- id: 'NPM::ast-traverse:0.1.1' + curations: + comment: 'Revision found by comparing the NPM package with the sources from https://github.com/olov/ast-traverse.' + vcs: + revision: '73f2b3c319af82fd8e490d40dd89a15951069b0d' + +- id: 'NPM::ramda:[0.21.0,0.25.0]' # Ivy-style version matchers are supported. + curations: + comment: >- + The package is licensed under MIT per `LICENSE` and `dist/ramda.js`. The project logo is CC-BY-NC-SA-3.0 but it is + not part of the distributed .tar.gz package, see the `README.md` which says: + "Ramda logo artwork © 2014 J. C. Phillipps. Licensed Creative Commons CC BY-NC-SA 3.0." + concluded_license: 'MIT' + +- id: 'Maven:org.jetbrains.kotlin:kotlin-bom' + curations: + comment: 'The package is a Maven BOM file and thus is metadata only.' + is_metadata_only: true + +- id: 'PyPI::pyramid-workflow:1.0.0' + curations: + comment: 'The package has an unmappable declared license entry.' + declared_license_mapping: + 'BSD-derived (http://www.repoze.org/LICENSE.txt)': 'LicenseRef-scancode-repoze' + +- id: 'PyPI::branca' + curations: + comment: 'A copyright statement was used to declare the license.' + declared_license_mapping: + 'Copyright (C) 2013, Martin Journois': 'NONE' + +- id: 'Maven:androidx.collection:collection:' + curations: + comment: 'Scan the source artifact, because the VCS revision and path are hard to figure out.' + source_code_origins: [ARTIFACT] + +- id: 'Maven:androidx.collection:collection:' + curations: + comment: 'Specify the platform for use within policy rules.' + labels: + platform: 'android' + +- id: 'NPM:@types:mime-types:2.1.0' + curations: + comment: 'Retrieve the vulnerabilities from Black Duck by the provided origin-id instead of by the purl.' + labels: + black-duck:origin-id: 'npmjs:@types/mime-types/2.1.0' diff --git a/tests/data/example_simple_curation.yml b/tests/data/example_simple_curation.yml new file mode 100644 index 0000000..48896d8 --- /dev/null +++ b/tests/data/example_simple_curation.yml @@ -0,0 +1,33 @@ +- id: "Maven:com.example.app:example:0.0.1" + curations: + comment: "An explanation why the curation is needed or the reasoning for a license conclusion" + purl: "pkg:Maven/com.example.app/example@0.0.1?arch=arm64-v8a#src/main" + authors: + - "Name of one author" + - "Name of another author" + cpe: "cpe:2.3:a:example-org:example-package:0.0.1:*:*:*:*:*:*:*" + concluded_license: "Valid SPDX license expression to override the license findings." + declared_license_mapping: + "license a": "Apache-2.0" + description: "Curated description." + homepage_url: "http://example.com" + binary_artifact: + url: "http://example.com/binary.zip" + hash: + value: "ddce269a1e3d054cae349621c198dd52" + algorithm: "MD5" + source_artifact: + url: "http://example.com/sources.zip" + hash: + value: "ddce269a1e3d054cae349621c198dd52" + algorithm: "MD5" + vcs: + type: "Git" + url: "http://example.com/repo.git" + revision: "1234abc" + path: "subdirectory" + is_metadata_only: true + is_modified: true + source_code_origins: [ARTIFACT, VCS] + labels: + my-key: "my-value" diff --git a/tests/test_ort_repository_configuration.py b/tests/test_ort_repository_configuration.py index 92f8434..2fcf81c 100644 --- a/tests/test_ort_repository_configuration.py +++ b/tests/test_ort_repository_configuration.py @@ -2,10 +2,8 @@ # SPDX-License-Identifier: MIT from pathlib import Path -from typing import Any import pytest -import yaml from ort.models.repository_configuration import ( OrtRepositoryConfiguration, @@ -13,31 +11,21 @@ OrtRepositoryConfigurationIncludesPath, PathIncludeReason, ) +from tests.utils.load_yaml_config import load_yaml_config # type: ignore REPO_CONFIG_DIR = Path(__file__).parent / "data" / "repo_config" -def load_yaml_config(filename) -> Any: - """ - Load a YAML configuration file from the REPO_CONFIG_DIR directory. - - Args: - filename (str): The name of the YAML file to load. - - Returns: - object: The parsed YAML data as a Python object (usually dict). - """ - with (REPO_CONFIG_DIR / filename).open() as f: - return yaml.safe_load(f) - - def test_only_include_valid(): """ Test that a valid repository configuration with a single path include is loaded correctly. Verifies that the pattern, reason, and comment fields are present and have expected values, and that the model objects are instantiated without error and contain the correct data. """ - config_data = load_yaml_config("only_include.yml") + config_data = load_yaml_config( + filename="only_include.yml", + data_dir=REPO_CONFIG_DIR, + ) includes = config_data.get("includes", {}) if "paths" not in includes: pytest.fail("Missing 'paths' in includes") @@ -80,7 +68,7 @@ def test_only_include_reason_fail(): raises a ValueError when instantiating OrtRepositoryConfigurationIncludesPath. The test expects failure when 'reason' is not a valid PathIncludeReason enum. """ - config_data = load_yaml_config("only_include_reason_fail.yml") + config_data = load_yaml_config("only_include_reason_fail.yml", REPO_CONFIG_DIR) includes = config_data.get("includes", {}) if "paths" not in includes: pytest.fail("Missing 'paths' in includes") diff --git a/tests/test_package_curation.py b/tests/test_package_curation.py new file mode 100644 index 0000000..db94f5e --- /dev/null +++ b/tests/test_package_curation.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: 2025 Helio Chissini de Castro +# SPDX-License-Identifier: MIT + + +import pytest +from pydantic import ValidationError + +from ort.models.config.curations import Curations +from tests.utils.load_yaml_config import load_yaml_config # type: ignore + + +def test_ort_docs_simple_curation_example(): + """ + Validate the curation example existing in Ort documentation for package curations. + Reference: https://oss-review-toolkit.org/ort/docs/configuration/package-curations + """ + config_data = load_yaml_config("example_simple_curation.yml") + + try: + Curations(packages=config_data) + except ValidationError as e: + pytest.fail(f"Failed to instantiate OrtRepositoryConfiguration: {e}") + + +def test_ort_docs_curation_example(): + """ + Validate the curation example existing in Ort documentation for package curations. + Reference: https://oss-review-toolkit.org/ort/docs/configuration/package-curations + """ + config_data = load_yaml_config("example_curations.yml") + + try: + Curations(packages=config_data) + except ValidationError as e: + pytest.fail(f"Failed to instantiate OrtRepositoryConfiguration: {e}") diff --git a/tests/utils/load_yaml_config.py b/tests/utils/load_yaml_config.py new file mode 100644 index 0000000..415dc90 --- /dev/null +++ b/tests/utils/load_yaml_config.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2025 Helio Chissini de Castro +# SPDX-License-Identifier: MIT + +from pathlib import Path +from typing import Any + +import pytest +import yaml + +DATA_CONFIG_DIR = Path(__file__).parent.parent / "data" + + +def load_yaml_config(filename: str, data_dir: Path = DATA_CONFIG_DIR) -> Any: + """ + Load a YAML configuration file from the REPO_CONFIG_DIR directory. + + Args: + filename (str): The name of the YAML file to load. + + Returns: + object: The parsed YAML data as a Python object (usually dict). + """ + try: + with (data_dir / filename).open() as f: + return yaml.safe_load(f) + except OSError: + pytest.fail("Fail to load test assets.") From fec512ae12111c6a55d224c7a3a9d6583046d002 Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Fri, 7 Nov 2025 14:35:22 +0100 Subject: [PATCH 2/2] chore(project): Bump version Signed-off-by: Helio Chissini de Castro --- pyproject.toml | 9 ++++----- uv.lock | 38 +++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 913ff8d..043e186 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,12 +2,9 @@ requires = ["uv_build>=0.8.12,<0.10.0"] build-backend = "uv_build" -[tool.hatch.build.targets.wheel] -packages = ["src/ort"] - [project] name = "python-ort" -version = "0.3.0" +version = "0.3.1" description = "A Python Ort model serialization library" readme = "README.md" license = "MIT" @@ -39,6 +36,7 @@ dev = [ "pycodestyle>=2.14.0", "pyrefly>=0.40.0", "pytest>=8.4.2", + "rich>=14.2.0", "ruff>=0.14.4", "types-pyyaml>=6.0.12.20250915", ] @@ -47,7 +45,8 @@ dev = [ addopts = ["--import-mode=importlib"] log_cli = true log_cli_level = "INFO" -pythonpath = "src" +pythonpath = ["src"] +testpaths = ["tests"] [tool.pylint.messages_control] disable = [ diff --git a/uv.lock b/uv.lock index 0a6c4dd..3d5219d 100644 --- a/uv.lock +++ b/uv.lock @@ -272,6 +272,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -357,6 +369,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "more-itertools" version = "10.8.0" @@ -623,7 +644,7 @@ wheels = [ [[package]] name = "python-ort" -version = "0.3.0" +version = "0.3.1" source = { editable = "." } dependencies = [ { name = "pydantic" }, @@ -636,6 +657,7 @@ dev = [ { name = "pycodestyle" }, { name = "pyrefly" }, { name = "pytest" }, + { name = "rich" }, { name = "ruff" }, { name = "types-pyyaml" }, ] @@ -650,6 +672,7 @@ dev = [ { name = "pycodestyle", specifier = ">=2.14.0" }, { name = "pyrefly", specifier = ">=0.40.0" }, { name = "pytest", specifier = ">=8.4.2" }, + { name = "rich", specifier = ">=14.2.0" }, { name = "ruff", specifier = ">=0.14.4" }, { name = "types-pyyaml", specifier = ">=6.0.12.20250915" }, ] @@ -727,6 +750,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "rich" +version = "14.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/d2/8920e102050a0de7bfabeb4c4614a49248cf8d5d7a8d01885fbb24dc767a/rich-14.2.0.tar.gz", hash = "sha256:73ff50c7c0c1c77c8243079283f4edb376f0f6442433aecb8ce7e6d0b92d1fe4", size = 219990, upload-time = "2025-10-09T14:16:53.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl", hash = "sha256:76bc51fe2e57d2b1be1f96c524b890b816e334ab4c1e45888799bfaab0021edd", size = 243393, upload-time = "2025-10-09T14:16:51.245Z" }, +] + [[package]] name = "ruff" version = "0.14.4"