diff --git a/src/fromager/packagesettings.py b/src/fromager/packagesettings.py index 7ae96c22..bee1eba7 100644 --- a/src/fromager/packagesettings.py +++ b/src/fromager/packagesettings.py @@ -1,5 +1,6 @@ from __future__ import annotations +import enum import logging import os import pathlib @@ -18,10 +19,10 @@ from pydantic import Field from pydantic_core import CoreSchema, core_schema -from . import overrides +from . import overrides, resolver if typing.TYPE_CHECKING: - from . import build_environment, context + from . import build_environment, context, requirements_file logger = logging.getLogger(__name__) @@ -354,6 +355,12 @@ class VariantInfo(pydantic.BaseModel): pre_built: bool = False """Use pre-built wheel from index server?""" + source: typing.Annotated[ + SourceResolver | None, + pydantic.Field(default=None, discriminator="provider"), + ] + """Source resolver and downloader""" + class GitOptions(pydantic.BaseModel): """Git repository cloning options @@ -385,6 +392,320 @@ class GitOptions(pydantic.BaseModel): """ +VERSION_QUOTED = "%7Bversion%7D" + + +class BuildSDist(enum.StrEnum): + pep517 = "pep517" + tarball = "tarball" + + +class AbstractResolver(pydantic.BaseModel): + model_config = MODEL_CONFIG + + provider: str + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.BaseProvider: + raise NotImplementedError + + +class PyPISDistResolver(AbstractResolver): + """Resolve version with PyPI, download sdist from PyPI + + The ``pypi-sdist` provider uses :pep:`503` *Simple Repository API* or + :pep:`691` *JSON-based Simple API* to resolve packages on PyPI or a + PyPI-compatible index. + + The provider downloads source distributions (tarballs) from the index. + It ignores releases that have only wheels and no sdist. + """ + + provider: typing.Literal["pypi-sdist"] + + index_url: pydantic.HttpUrl = pydantic.Field( + default=pydantic.HttpUrl("https://pypi.org/simple/"), + description="Python Package Index URL", + ) + + # It is not safe to use PEP 517 to re-generate a source distribution. + # Some PEP 517 backends require VCS to generate correct sdist. + build_sdist: typing.ClassVar[BuildSDist | None] = BuildSDist.tarball + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.PyPIProvider: + return resolver.PyPIProvider( + include_sdists=True, + include_wheels=False, + sdist_server_url=str(self.index_url), + constraints=ctx.constraints, + req_type=req_type, + ignore_platform=False, + ) + + +class PyPIPrebuiltResolver(AbstractResolver): + """Resolve version with PyPI, download pre-built wheel from PyPI + + The ``pypi-prebuilt` provider uses :pep:`503` *Simple Repository API* or + :pep:`691` *JSON-based Simple API* to resolve packages on PyPI or a + PyPI-compatible index. + + The provider downloads pre-built wheels from the index. It ignores + versions that have no compatible wheels (sdist-only or incompatible + OS, CPU arch, or glibc version). + """ + + provider: typing.Literal["pypi-prebuilt"] + + index_url: pydantic.HttpUrl = pydantic.Field( + default=pydantic.HttpUrl("https://pypi.org/simple/"), + description="Python Package Index URL", + ) + + build_sdist: typing.ClassVar[BuildSDist | None] = None + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.PyPIProvider: + return resolver.PyPIProvider( + include_sdists=False, + include_wheels=True, + sdist_server_url=str(self.index_url), + constraints=ctx.constraints, + req_type=req_type, + ignore_platform=False, + ) + + +class PyPIDownloadResolver(AbstractResolver): + """Resolve version with PyPI, download sdist from arbitrary URL + + The ``pypi-download` provider uses :pep:`503` *Simple Repository API* or + :pep:`691` *JSON-based Simple API* to resolve packages on PyPI or a + PyPI-compatible index. + + The provider takes all releases into account (sdist-only, wheel-only, + even incompatible wheels). + + It downloads tarball from an alternative download location. The download + URL must contain a ``{version}`` template, e.g. + ``https://download.example/mypackage-{version}.tar.gz``. + """ + + provider: typing.Literal["pypi-download"] + + index_url: pydantic.HttpUrl = pydantic.Field( + default=pydantic.HttpUrl("https://pypi.org/simple/"), + description="Python Package Index URL", + ) + + download_url: pydantic.HttpUrl + """Remote download URL + + URL must contain '{version}' template string. + """ + + build_sdist: typing.ClassVar[BuildSDist | None] = BuildSDist.tarball + + @pydantic.field_validator("download_url", mode="after") + @classmethod + def validate_download_url(cls, value: pydantic.HttpUrl) -> pydantic.HttpUrl: + if not value.path: + raise ValueError(f"url {value} has an empty path") + if VERSION_QUOTED not in value.path: + raise ValueError(f"missing '{{version}}' in url {value}") + return value + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.PyPIProvider: + return resolver.PyPIProvider( + include_sdists=True, + include_wheels=True, + sdist_server_url=str(self.index_url), + constraints=ctx.constraints, + req_type=req_type, + ignore_platform=True, + override_download_url=str(self.download_url).replace( + VERSION_QUOTED, "{version}" + ), + ) + + +class PyPIGitResolver(AbstractResolver): + """Resolve version with PyPI, build sdist from git clone + + The ``pypi-git` provider uses :pep:`503` *Simple Repository API* or + :pep:`691` *JSON-based Simple API* to resolve packages on PyPI or a + PyPI-compatible index. + + The provider takes all releases into account (sdist-only, wheel-only, + even incompatible wheels). + + It clones and retrieves a git repo + recursive submodules at a specific + tag. The tag must contain ``{version}`` template. + + """ + + provider: typing.Literal["pypi-git"] + + index_url: pydantic.HttpUrl = pydantic.Field( + default=pydantic.HttpUrl("https://pypi.org/simple/"), + description="Python Package Index URL", + ) + + clone_url: pydantic.AnyUrl + """git clone URL + + https://git.test/repo.git + """ + + tag: str + + build_sdist: BuildSDist = BuildSDist.pep517 + """Source distribution build method""" + + @pydantic.field_validator("clone_url", mode="after") + @classmethod + def validate_clone_url(cls, value: pydantic.AnyUrl) -> pydantic.AnyUrl: + if value.scheme not in {"https", "ssh"}: + raise ValueError(f"invalid scheme in url {value}") + if not value.path: + raise ValueError(f"url {value} has an empty path") + return value + + @pydantic.field_validator("tag", mode="after") + @classmethod + def validate_tag(cls, value: str) -> str: + if "{version}" not in value: + raise ValueError(f"missing '{{version}}' in tag {value}") + return value + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.PyPIProvider: + download_url = f"git+{self.clone_url}@{self.tag}" + return resolver.PyPIProvider( + include_sdists=True, + include_wheels=True, + sdist_server_url=str(self.index_url), + constraints=ctx.constraints, + req_type=req_type, + ignore_platform=True, + override_download_url=download_url, + ) + + +class GithubSourceResolver(AbstractResolver): + """Resolve version from Github tags, build sdist from tarball or git clone + + The `github` provider uses GitHub's REST API to resolve versions from tags. + + It can either directly download a tarball bundle or git clone a repository. + """ + + provider: typing.Literal["github"] + + url: pydantic.HttpUrl + """Full GitHub project URL""" + + tag_pattern: re.Pattern = re.compile(r"v?(\d+\..*)") + """Regular expression matching the tag""" + + retrieve_method: resolver.RetrieveMethod = resolver.RetrieveMethod.git_https + """Retrieve method (tar bundle, git clone)""" + + build_sdist: BuildSDist = BuildSDist.pep517 + """Source distribution build method""" + + @pydantic.field_validator("url", mode="after") + @classmethod + def validate_url(cls, value: pydantic.HttpUrl) -> pydantic.HttpUrl: + if value.host != "github.com": + raise ValueError(f"Expected 'github.com' in {value}") + if not value.path or value.path.count("/") != 2: + raise ValueError(f"Invalid path in {value}, expected two elements") + return value + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.GitHubTagProvider: + path = self.url.path + assert path is not None # for type checker + path = path.lstrip("/") + if path.endswith(".git"): + path = path[:-4] + organization, repo = path.split("/") + return resolver.GitHubTagProvider( + organization=organization, + repo=repo, + constraints=ctx.constraints, + matcher=self.tag_pattern, + req_type=req_type, + retrieve_method=self.retrieve_method, + ) + + +class GitlabSourceResolver(AbstractResolver): + """Resolve version from Gitlab tags, build sdist from download or clone + + The `gitlab` provider uses Gitlab's REST API to resolve versions from tags. + + It can either directly download a tarball bundle or git clone a repository. + """ + + provider: typing.Literal["gitlab"] + + url: pydantic.HttpUrl + """Full GitLab project URL""" + + tag_pattern: re.Pattern = re.compile(r"v?(\d+\..*)") + """Regular expression matching the tag""" + + retrieve_method: resolver.RetrieveMethod = resolver.RetrieveMethod.git_https + """Retrieve method (tar bundle, git clone)""" + + build_sdist: BuildSDist = BuildSDist.pep517 + """Source distribution build method""" + + @pydantic.field_validator("url", mode="after") + @classmethod + def validate_url(cls, value: pydantic.HttpUrl) -> pydantic.HttpUrl: + if value.scheme != "https" or not value.host or not value.path: + raise ValueError(f"invalid url {value}") + return value + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.GitLabTagProvider: + path = self.url.path + assert path is not None # for type checker + path = path.lstrip("/") + if path.endswith(".git"): + path = path[:-4] + return resolver.GitLabTagProvider( + project_path=path, + server_url=f"https://{self.url.host}", + constraints=ctx.constraints, + matcher=self.tag_pattern, + req_type=req_type, + retrieve_method=self.retrieve_method, + ) + + +SourceResolver = ( + PyPISDistResolver + | PyPIPrebuiltResolver + | PyPIDownloadResolver + | PyPIGitResolver + | GithubSourceResolver + | GitlabSourceResolver +) + + _DictStrAny = dict[str, typing.Any] @@ -453,6 +774,12 @@ class PackageSettings(pydantic.BaseModel): env: EnvVars = Field(default_factory=dict) """Common env var for all variants""" + source: typing.Annotated[ + SourceResolver | None, + pydantic.Field(default=None, discriminator="provider"), + ] + """Source resolver and downloader""" + download_source: DownloadSource = Field(default_factory=DownloadSource) """Alternative source download settings""" @@ -986,6 +1313,14 @@ def variants(self) -> Mapping[Variant, VariantInfo]: """Get the variant configuration for the current package""" return self._ps.variants + @property + def source_resolver(self) -> SourceResolver | None: + """Get source resolver settings (variant or global)""" + vi = self._ps.variants.get(self.variant) + if vi is not None and vi.source is not None: + return vi.source + return self._ps.source + def serialize(self, **kwargs: typing.Any) -> dict[str, typing.Any]: return self._ps.serialize(**kwargs) diff --git a/src/fromager/resolver.py b/src/fromager/resolver.py index 1ab015a8..9c0dd728 100644 --- a/src/fromager/resolver.py +++ b/src/fromager/resolver.py @@ -6,6 +6,7 @@ from __future__ import annotations import datetime +import enum import functools import logging import os @@ -14,7 +15,7 @@ from collections.abc import Iterable from operator import attrgetter from platform import python_version -from urllib.parse import quote, unquote, urljoin, urlparse +from urllib.parse import quote, unquote, urljoin, urlparse, urlsplit, urlunsplit import pypi_simple import resolvelib @@ -180,11 +181,42 @@ def resolve_from_provider( raise ValueError(f"Unable to resolve {req}") +class RetrieveMethod(enum.StrEnum): + tarball = "tarball" + git_https = "git+https" + git_ssh = "git+ssh" + + @classmethod + def from_url(cls, download_url: str) -> tuple[RetrieveMethod, str, str | None]: + """Parse a download URL into method, url, reference""" + scheme, netloc, path, query, fragment = urlsplit( + download_url, allow_fragments=False + ) + match scheme: + case "https": + return RetrieveMethod.tarball, download_url, None + case "git+https": + method = RetrieveMethod.git_https + case "git+ssh": + method = RetrieveMethod.git_ssh + case _: + raise ValueError(f"unsupported download URL {download_url!r}") + # remove git+ + scheme = scheme[4:] + # split off @ revision + if "@" not in path: + raise ValueError(f"git download url {download_url!r} is missing '@ref'") + path, ref = path.rsplit("@", 1) + return method, urlunsplit((scheme, netloc, path, query, fragment)), ref + + def get_project_from_pypi( project: str, extras: typing.Iterable[str], sdist_server_url: str, ignore_platform: bool = False, + *, + override_download_url: str | None = None, ) -> Candidates: """Return candidates created from the project name and extras.""" found_candidates: set[str] = set() @@ -345,6 +377,11 @@ def get_project_from_pypi( ignored_candidates.add(dp.filename) continue + if override_download_url is None: + url = dp.url + else: + url = override_download_url.format(version=version) + upload_time = dp.upload_time if upload_time is not None: upload_time = upload_time.astimezone(datetime.UTC) @@ -352,7 +389,7 @@ def get_project_from_pypi( c = Candidate( name=name, version=version, - url=dp.url, + url=url, extras=tuple(sorted(extras)), is_sdist=is_sdist, build_tag=build_tag, @@ -603,6 +640,7 @@ def __init__( ignore_platform: bool = False, *, use_resolver_cache: bool = True, + override_download_url: str | None = None, ): super().__init__( constraints=constraints, @@ -613,6 +651,7 @@ def __init__( self.include_wheels = include_wheels self.sdist_server_url = sdist_server_url self.ignore_platform = ignore_platform + self.override_download_url = override_download_url @property def cache_key(self) -> str: @@ -625,9 +664,10 @@ def cache_key(self) -> str: def find_candidates(self, identifier: str) -> Candidates: return get_project_from_pypi( identifier, - set(), - self.sdist_server_url, - self.ignore_platform, + extras=set(), + sdist_server_url=self.sdist_server_url, + ignore_platform=self.ignore_platform, + override_download_url=self.override_download_url, ) def validate_candidate( @@ -803,6 +843,7 @@ def __init__( *, req_type: RequirementType | None = None, use_resolver_cache: bool = True, + retrieve_method: RetrieveMethod = RetrieveMethod.tarball, ): super().__init__( constraints=constraints, @@ -813,6 +854,7 @@ def __init__( ) self.organization = organization self.repo = repo + self.retrieve_method = retrieve_method @property def cache_key(self) -> str: @@ -847,7 +889,14 @@ def _find_tags( logger.debug(f"{identifier}: match function ignores {tagname}") continue assert isinstance(version, Version) - url = entry["tarball_url"] + + match self.retrieve_method: + case RetrieveMethod.tarball: + url = entry["tarball_url"] + case RetrieveMethod.git_https: + url = f"git+https://{self.host}/{self.organization}/{self.repo}.git@{tagname}" + case RetrieveMethod.git_ssh: + url = f"git+ssh://git@{self.host}/{self.organization}/{self.repo}.git@{tagname}" # Github tag API endpoint does not include commit date information. # It would be too expensive to query every commit API endpoint. @@ -880,6 +929,7 @@ def __init__( *, req_type: RequirementType | None = None, use_resolver_cache: bool = True, + retrieve_method: RetrieveMethod = RetrieveMethod.tarball, ) -> None: super().__init__( constraints=constraints, @@ -889,6 +939,9 @@ def __init__( matcher=matcher, ) self.server_url = server_url.rstrip("/") + self.server_hostname = urlparse(server_url).hostname + if not self.server_hostname: + raise ValueError(f"invalid {server_url=}") self.project_path = project_path.lstrip("/") # URL-encode the project path as required by GitLab API. # The safe="" parameter tells quote() to encode ALL characters, @@ -899,6 +952,7 @@ def __init__( self.api_url = ( f"{self.server_url}/api/v4/projects/{encoded_path}/repository/tags" ) + self.retrieve_method = retrieve_method @property def cache_key(self) -> str: @@ -927,8 +981,14 @@ def _find_tags( continue assert isinstance(version, Version) - archive_path: str = f"{self.project_path}/-/archive/{tagname}/{self.project_path.split('/')[-1]}-{tagname}.tar.gz" - url = urljoin(self.server_url, archive_path) + match self.retrieve_method: + case RetrieveMethod.tarball: + archive_path: str = f"{self.project_path}/-/archive/{tagname}/{self.project_path.split('/')[-1]}-{tagname}.tar.gz" + url = urljoin(self.server_url, archive_path) + case RetrieveMethod.git_https: + url = f"git+https://{self.server_hostname}/{self.project_path}.git@{tagname}" + case RetrieveMethod.git_ssh: + url = f"git+ssh://git@{self.server_hostname}/{self.project_path}.git@{tagname}" # get tag creation time, fall back to commit creation time created_at_str: str | None = entry.get("created_at") diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 07bee01a..f8cf9072 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -8,7 +8,7 @@ from packaging.utils import NormalizedName from packaging.version import Version -from fromager import build_environment, context +from fromager import build_environment, context, requirements_file, resolver from fromager.packagesettings import ( Annotations, BuildDirectory, @@ -27,6 +27,12 @@ TEST_OTHER_PKG = "test-other-pkg" TEST_RELATED_PKG = "test-pkg-library" TEST_PREBUILT_PKG = "test-prebuilt-pkg" +TEST_GITHUB = "test-github" +TEST_GITLAB = "test-gitlab" +TEST_PYPIDOWNLOAD = "test-pypidownload" +TEST_PYPIPREBUILT = "test-pypiprebuilt" +TEST_PYPIGIT = "test-pypigit" +TEST_PYPISDIST = "test-pypisdist" FULL_EXPECTED: dict[str, typing.Any] = { "annotations": { @@ -81,6 +87,7 @@ "ignore_platform": True, "use_pypi_org_metadata": True, }, + "source": None, "variants": { "cpu": { "annotations": { @@ -89,6 +96,7 @@ "env": {"EGG": "spam ${EGG}", "EGG_AGAIN": "$EGG"}, "wheel_server_url": "https://wheel.test/simple", "pre_built": False, + "source": None, }, "rocm": { "annotations": { @@ -97,12 +105,14 @@ "env": {"SPAM": ""}, "wheel_server_url": None, "pre_built": True, + "source": None, }, "cuda": { "annotations": None, "env": {}, "wheel_server_url": None, "pre_built": False, + "source": None, }, }, } @@ -141,6 +151,7 @@ "ignore_platform": False, "use_pypi_org_metadata": None, }, + "source": None, "variants": {}, } @@ -180,12 +191,14 @@ "ignore_platform": False, "use_pypi_org_metadata": None, }, + "source": None, "variants": { "cpu": { "annotations": None, "env": {}, "pre_built": True, "wheel_server_url": None, + "source": None, }, }, } @@ -474,6 +487,12 @@ def test_settings_overrides(testdata_context: context.WorkContext) -> None: TEST_OTHER_PKG, TEST_RELATED_PKG, TEST_PREBUILT_PKG, + TEST_GITHUB, + TEST_GITLAB, + TEST_PYPIDOWNLOAD, + TEST_PYPIGIT, + TEST_PYPIPREBUILT, + TEST_PYPISDIST, } @@ -519,11 +538,17 @@ def test_global_changelog(testdata_context: context.WorkContext) -> None: def test_settings_list(testdata_context: context.WorkContext) -> None: assert testdata_context.settings.list_overrides() == { + TEST_PKG, TEST_EMPTY_PKG, TEST_OTHER_PKG, - TEST_PKG, TEST_RELATED_PKG, TEST_PREBUILT_PKG, + TEST_GITHUB, + TEST_GITLAB, + TEST_PYPIDOWNLOAD, + TEST_PYPIGIT, + TEST_PYPIPREBUILT, + TEST_PYPISDIST, } assert testdata_context.settings.list_pre_built() == {TEST_PREBUILT_PKG} assert testdata_context.settings.variant_changelog() == [] @@ -805,3 +830,83 @@ def test_use_pypi_org_metadata(testdata_context: context.WorkContext) -> None: "somepackage_without_customization" ) assert pbi.use_pypi_org_metadata + + +@pytest.mark.parametrize( + "name,expected", + [ + ( + TEST_GITHUB, + { + "provider": "github", + "tag_pattern": "(\\d.*)", + "url": "https://github.com/python-wheel-build/fromager", + }, + ), + ( + TEST_GITLAB, + { + "provider": "gitlab", + "tag_pattern": "(\\d.*)", + "url": "https://gitlab.test/python-wheel-build/fromager", + }, + ), + ( + TEST_PYPIDOWNLOAD, + { + "provider": "pypi-download", + "index_url": "https://pypi.test/simple", + "download_url": "https://download.test/test_pypidownload-%7Bversion%7D.tar.gz", + }, + ), + ( + TEST_PYPIGIT, + { + "provider": "pypi-git", + "index_url": "https://pypi.test/simple", + "clone_url": "https://github.com/python-wheel-build/fromager.git", + "tag": "{version}", + }, + ), + ( + TEST_PYPIPREBUILT, + { + "provider": "pypi-prebuilt", + "index_url": "https://pypi.test/simple", + }, + ), + ( + TEST_PYPISDIST, + { + "provider": "pypi-sdist", + "index_url": "https://pypi.test/simple", + }, + ), + ], +) +def test_source_resolvers( + name: str, expected: dict, testdata_context: context.WorkContext +) -> None: + pbi = testdata_context.settings.package_build_info(name) + assert pbi.source_resolver + assert pbi.source_resolver.provider == expected["provider"] + assert pbi.serialize(mode="json")["source"] == expected + + resolver_provider = pbi.source_resolver.resolver_provider( + ctx=testdata_context, + req_type=requirements_file.RequirementType.TOP_LEVEL, + ) + assert isinstance(resolver_provider, resolver.BaseProvider) + + +def test_source_resolver_variant(testdata_context: context.WorkContext) -> None: + pbi = testdata_context.settings.package_build_info(TEST_PYPISDIST) + assert pbi.source_resolver + assert pbi.source_resolver.provider == "pypi-sdist" + assert str(pbi.source_resolver.index_url) == "https://pypi.test/simple" + + testdata_context.settings.variant = Variant("rocm") + pbi = testdata_context.settings.package_build_info(TEST_PYPISDIST) + assert pbi.source_resolver + assert pbi.source_resolver.provider == "pypi-sdist" + assert str(pbi.source_resolver.index_url) == "https://rocm.test/simple" diff --git a/tests/test_resolver.py b/tests/test_resolver.py index cc7c5593..58a3ecdc 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -370,6 +370,26 @@ def test_provider_constraint_match() -> None: assert str(candidate.version) == "1.2.2" +def test_provider_override_download_url() -> None: + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/hydra-core/", + text=_hydra_core_simple_response, + ) + + provider = resolver.PyPIProvider( + override_download_url="https://server.test/hydr_core-{version}.tar.gz" + ) + reporter: resolvelib.BaseReporter = resolvelib.BaseReporter() + rslvr = resolvelib.Resolver(provider, reporter) + + result = rslvr.resolve([Requirement("hydra-core")]) + assert "hydra-core" in result.mapping + + candidate = result.mapping["hydra-core"] + assert candidate.url == "https://server.test/hydr_core-1.3.2.tar.gz" + + _ignore_platform_simple_response = """ @@ -715,6 +735,51 @@ def test_resolve_github() -> None: ) +@pytest.mark.parametrize( + ["retrieve_method", "expected_url"], + [ + ( + resolver.RetrieveMethod.tarball, + "https://api.github.com/repos/python-wheel-build/fromager/tarball/refs/tags/0.9.0", + ), + ( + resolver.RetrieveMethod.git_https, + "git+https://github.com:443/python-wheel-build/fromager.git@0.9.0", + ), + ( + resolver.RetrieveMethod.git_ssh, + "git+ssh://git@github.com:443/python-wheel-build/fromager.git@0.9.0", + ), + ], +) +def test_resolve_github_retrieve_method( + retrieve_method: resolver.RetrieveMethod, expected_url: str +) -> None: + with requests_mock.Mocker() as r: + r.get( + "https://api.github.com:443/repos/python-wheel-build/fromager", + text=_github_fromager_repo_response, + ) + r.get( + "https://api.github.com:443/repos/python-wheel-build/fromager/tags", + text=_github_fromager_tag_response, + ) + + provider = resolver.GitHubTagProvider( + organization="python-wheel-build", + repo="fromager", + retrieve_method=retrieve_method, + ) + reporter: resolvelib.BaseReporter = resolvelib.BaseReporter() + rslvr = resolvelib.Resolver(provider, reporter) + + result = rslvr.resolve([Requirement("fromager")]) + assert "fromager" in result.mapping + + candidate = result.mapping["fromager"] + assert candidate.url == expected_url + + def test_github_constraint_mismatch() -> None: constraint = constraints.Constraints() constraint.add_constraint("fromager>=1.0") @@ -922,6 +987,49 @@ def test_resolve_gitlab() -> None: ) +@pytest.mark.parametrize( + ["retrieve_method", "expected_url"], + [ + ( + resolver.RetrieveMethod.tarball, + "https://gitlab.com/mirrors/github/decile-team/submodlib/-/archive/v0.0.3/submodlib-v0.0.3.tar.gz", + ), + ( + resolver.RetrieveMethod.git_https, + "git+https://gitlab.com/mirrors/github/decile-team/submodlib.git@v0.0.3", + ), + ( + resolver.RetrieveMethod.git_ssh, + "git+ssh://git@gitlab.com/mirrors/github/decile-team/submodlib.git@v0.0.3", + ), + ], +) +def test_resolve_gitlab_retrieve_method( + retrieve_method: resolver.RetrieveMethod, expected_url: str +) -> None: + with requests_mock.Mocker() as r: + r.get( + "https://gitlab.com/api/v4/projects/mirrors%2Fgithub%2Fdecile-team%2Fsubmodlib/repository/tags", + text=_gitlab_submodlib_repo_response, + ) + + provider = resolver.GitLabTagProvider( + project_path="mirrors/github/decile-team/submodlib", + server_url="https://gitlab.com", + retrieve_method=retrieve_method, + ) + reporter: resolvelib.BaseReporter = resolvelib.BaseReporter() + rslvr = resolvelib.Resolver(provider, reporter) + + result = rslvr.resolve([Requirement("submodlib")]) + assert "submodlib" in result.mapping + + candidate = result.mapping["submodlib"] + assert str(candidate.version) == "0.0.3" + + assert candidate.url == expected_url + + def test_gitlab_constraint_mismatch() -> None: constraint = constraints.Constraints() constraint.add_constraint("submodlib>=1.0") @@ -1109,3 +1217,53 @@ def custom_resolver_provider( assert "pypi.org" not in error_message.lower(), ( f"Error message incorrectly mentions PyPI when using GitHub resolver: {error_message}" ) + + +@pytest.mark.parametrize( + ["download_url", "expected_method", "expected_url", "expected_ref"], + [ + ( + "https://api.github.com/repos/python-wheel-build/fromager/tarball/refs/tags/0.9.0", + resolver.RetrieveMethod.tarball, + "https://api.github.com/repos/python-wheel-build/fromager/tarball/refs/tags/0.9.0", + None, + ), + ( + "git+https://github.com:443/python-wheel-build/fromager.git@0.9.0", + resolver.RetrieveMethod.git_https, + "https://github.com:443/python-wheel-build/fromager.git", + "0.9.0", + ), + ( + "git+ssh://git@github.com:443/python-wheel-build/fromager.git@0.9.0", + resolver.RetrieveMethod.git_ssh, + "ssh://git@github.com:443/python-wheel-build/fromager.git", + "0.9.0", + ), + ], +) +def test_retrieve_method_from_url( + download_url: str, + expected_method: resolver.RetrieveMethod, + expected_url: str, + expected_ref: str | None, +) -> None: + assert resolver.RetrieveMethod.from_url(download_url) == ( + expected_method, + expected_url, + expected_ref, + ) + + +@pytest.mark.parametrize( + ["download_url"], + [ + ["http://insecure.test"], + ["hg+ssh://mercurial.test"], + ], +) +def test_retrieve_method_from_url_error( + download_url: str, +) -> None: + with pytest.raises(ValueError): + resolver.RetrieveMethod.from_url(download_url) diff --git a/tests/testdata/context/overrides/settings/test_github.yaml b/tests/testdata/context/overrides/settings/test_github.yaml new file mode 100644 index 00000000..2c442285 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_github.yaml @@ -0,0 +1,6 @@ +source: + provider: github + url: https://github.com/python-wheel-build/fromager + tag_pattern: "(\\d.*)" + retrieve_method: git+https + build_sdist: pep517 diff --git a/tests/testdata/context/overrides/settings/test_gitlab.yaml b/tests/testdata/context/overrides/settings/test_gitlab.yaml new file mode 100644 index 00000000..01be4330 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_gitlab.yaml @@ -0,0 +1,6 @@ +source: + provider: gitlab + url: https://gitlab.test/python-wheel-build/fromager + tag_pattern: "(\\d.*)" + retrieve_method: git+https + build_sdist: pep517 diff --git a/tests/testdata/context/overrides/settings/test_pypidownload.yaml b/tests/testdata/context/overrides/settings/test_pypidownload.yaml new file mode 100644 index 00000000..1045e714 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_pypidownload.yaml @@ -0,0 +1,4 @@ +source: + provider: pypi-download + index_url: https://pypi.test/simple + download_url: https://download.test/test_pypidownload-{version}.tar.gz diff --git a/tests/testdata/context/overrides/settings/test_pypigit.yaml b/tests/testdata/context/overrides/settings/test_pypigit.yaml new file mode 100644 index 00000000..b40e6794 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_pypigit.yaml @@ -0,0 +1,6 @@ +source: + provider: pypi-git + index_url: https://pypi.test/simple + clone_url: https://github.com/python-wheel-build/fromager.git + tag: '{version}' + build_sdist: pep517 diff --git a/tests/testdata/context/overrides/settings/test_pypiprebuilt.yaml b/tests/testdata/context/overrides/settings/test_pypiprebuilt.yaml new file mode 100644 index 00000000..8fe01538 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_pypiprebuilt.yaml @@ -0,0 +1,3 @@ +source: + provider: pypi-prebuilt + index_url: https://pypi.test/simple diff --git a/tests/testdata/context/overrides/settings/test_pypisdist.yaml b/tests/testdata/context/overrides/settings/test_pypisdist.yaml new file mode 100644 index 00000000..9b5ab385 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_pypisdist.yaml @@ -0,0 +1,8 @@ +source: + provider: pypi-sdist + index_url: https://pypi.test/simple +variants: + rocm: + source: + provider: pypi-sdist + index_url: https://rocm.test/simple