Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 82 additions & 13 deletions src/fromager/bootstrap_requirement_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from packaging.requirements import Requirement
from packaging.version import Version

from . import resolver, sources, wheels
from . import finders, resolver, sources, wheels
from .dependency_graph import DependencyGraph
from .requirements_file import RequirementType

Expand Down Expand Up @@ -40,15 +40,24 @@ def __init__(
self,
ctx: context.WorkContext,
prev_graph: DependencyGraph | None = None,
multiple_versions: bool = False,
cache_wheel_server_url: str = "",
) -> None:
"""Initialize requirement resolver.

Args:
ctx: Work context with constraints and settings
prev_graph: Optional previous dependency graph for caching
multiple_versions: If ``True``, age filtering returns an empty
list instead of falling back to all candidates, letting the
caller decide how to handle the case.
cache_wheel_server_url: URL of the remote wheel cache server.
Used as a fallback when age filtering produces no candidates.
"""
self.ctx = ctx
self.prev_graph = prev_graph
self.multiple_versions = multiple_versions
self.cache_wheel_server_url = cache_wheel_server_url
# Session-level resolution cache to avoid re-resolving same requirements
# Key: (requirement_string, pre_built) to distinguish source vs prebuilt
# Value: tuple of (url, version) tuples sorted by version (highest first)
Expand All @@ -72,6 +81,8 @@ def resolve(
1. Session cache (if previously resolved)
2. Previous dependency graph
3. PyPI resolution (source or prebuilt based on package build info)
4. Remote wheel cache server (multi-version mode only, when age
filtering produced no candidates)

Args:
req: Package requirement
Expand Down Expand Up @@ -113,8 +124,12 @@ def resolve(
# Resolve using strategies
results = self._resolve(req, req_type, parent_req, pre_built)
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is _resolve() its own method?


# Cache the result
self.cache_resolution(req, pre_built, results)
# Only cache non-empty results.
if results:
self.cache_resolution(req, pre_built, results)

if not results:
return []
return results if return_all_versions else [results[0]]
Comment thread
coderabbitai[bot] marked this conversation as resolved.

def _resolve(
Expand All @@ -129,6 +144,8 @@ def _resolve(
Tries resolution strategies in order:
1. Previous dependency graph
2. PyPI resolution (source or prebuilt)
3. Remote wheel cache server (multi-version mode only, when age
filtering produced no source candidates)

Args:
req: Package requirement
Expand Down Expand Up @@ -167,18 +184,70 @@ def _resolve(
wheel_server_urls=wheel_server_urls,
req_type=req_type,
)
else:
# Resolve source (sdist)
provider = sources.get_source_provider(
ctx=self.ctx,
req=req,
sdist_server_url=resolver.PYPI_SERVER_URL,
req_type=req_type,

# Resolve source (sdist)
provider = sources.get_source_provider(
ctx=self.ctx,
req=req,
sdist_server_url=resolver.PYPI_SERVER_URL,
req_type=req_type,
)
max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx)
results = resolver.find_all_matching_from_provider(
provider,
req,
max_age_cutoff=max_age_cutoff,
fallback_on_empty_age_filter=not self.multiple_versions,
)

if not results and self.multiple_versions and self.cache_wheel_server_url:
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This section deserves a comment explaining why we look at the cache server.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You might also consider if it makes sense to reverse the logic and break up each reason for returning to make the logic clearer.

if results:
    return results

if not self.multiple_versions:
    # without multiple-versions enabled we don't use the age filter so a failure to resolve a version means it is definitely not there
    logger.warning("resolver returned no results")
    return []

if not self.cache_wheel_server_url:
    # if we do not have a cache wheel server URL we cannot look for a previous build
    logger.warning("resolver returned no results and we have no wheel cache server URL to fall back on")
    return []

logger.info("no results found with normal resolution, falling back to the cache server %s", self.cache_wheel_server_url)
return self._resolve_from_cache_server(req)

results = self._resolve_from_cache_server(req)

if not results:
logger.warning(
"%s: could not find any versions (all filtered by "
"max-release-age and no cached wheel on server)",
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it would help someone debugging this error if the log message included the max-release-age value.

req.name,
)

return results

def _resolve_from_cache_server(self, req: Requirement) -> list[tuple[str, Version]]:
"""Fall back to the remote wheel cache server for a cached version.

When age filtering removes all candidates in multi-version mode,
queries the remote cache server for the newest available wheel.
Returns at most one version so that transitive dependencies are
re-processed without rebuilding every old version.
"""
logger.info(
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This log message assumes an execution order of the methods that might change in the future. We should log the detail about the max-release-age filtering before calling this function. See my suggested changes in the earlier comment.

"%s: all versions filtered by max-release-age, "
"checking cache server %s for existing wheel",
req.name,
self.cache_wheel_server_url,
)
try:
provider = finders.PyPICacheProvider(
cache_server_url=self.cache_wheel_server_url,
constraints=self.ctx.constraints,
)
max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx)
return resolver.find_all_matching_from_provider(
provider, req, max_age_cutoff=max_age_cutoff
results = resolver.find_all_matching_from_provider(provider, req)
if results:
url, version = results[0]
logger.info(
"%s: found version %s on cache server",
req.name,
version,
)
return [(url, version)]
except Exception as err:
logger.warning(
"%s: error checking cache server %s: %s",
req.name,
self.cache_wheel_server_url,
err,
)
return []

def get_cached_resolution(
self,
Expand Down
2 changes: 2 additions & 0 deletions src/fromager/bootstrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,8 @@ def __init__(
self._resolver = bootstrap_requirement_resolver.BootstrapRequirementResolver(
ctx=ctx,
prev_graph=prev_graph,
multiple_versions=multiple_versions,
cache_wheel_server_url=self.cache_wheel_server_url,
)
# Push items onto the stack as we start to resolve their
# dependencies so at the end we have a list of items that need to
Expand Down
16 changes: 15 additions & 1 deletion src/fromager/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ def find_all_matching_from_provider(
provider: BaseProvider,
req: Requirement,
max_age_cutoff: datetime.datetime | None = None,
fallback_on_empty_age_filter: bool = True,
) -> list[tuple[str, Version]]:
"""Find all matching candidates from provider without full dependency resolution.

Expand All @@ -255,6 +256,10 @@ def find_all_matching_from_provider(
max_age_cutoff: If set, reject candidates published before this time.
If all candidates are older than the cutoff, all are kept and
a warning is emitted to avoid empty resolution.
fallback_on_empty_age_filter: If ``True`` (default), keep all
candidates when age filtering would produce an empty result.
If ``False``, return an empty list instead, allowing the
caller to implement its own fallback strategy.

Returns list of (url, version) tuples sorted by version (highest first).

Expand Down Expand Up @@ -315,7 +320,7 @@ def find_all_matching_from_provider(
)
if filtered:
candidates_list = filtered
else:
elif fallback_on_empty_age_filter:
logger.warning(
"%s: all %d candidate(s) of %s are older than %d days, "
"keeping all to avoid empty resolution",
Expand All @@ -324,6 +329,15 @@ def find_all_matching_from_provider(
req,
max_age_days,
)
else:
logger.info(
"%s: all %d candidate(s) of %s are older than %d days",
req.name,
len(candidates_list),
req,
max_age_days,
)
candidates_list = []

# Convert candidates to list of (url, version) tuples
# Candidates are sorted by version (highest first) by BaseProvider.find_matches()
Expand Down
167 changes: 167 additions & 0 deletions tests/test_bootstrap_requirement_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -694,3 +694,170 @@ def test_resolve_prebuilt_after_source_uses_separate_cache(
url2, version2 = results2[0]
assert url2 == "https://files.pythonhosted.org/testpkg-1.5-py3-none-any.whl"
assert version2 == Version("1.5")


class TestResolveFromCacheServer:
"""Tests for the cache server fallback in _resolve_from_cache_server."""

def test_returns_newest_version_from_cache(self, tmp_context: WorkContext) -> None:
"""Falls back to cache server and returns only the newest version."""
brr = BootstrapRequirementResolver(
tmp_context,
multiple_versions=True,
cache_wheel_server_url="http://cache.test/simple",
)
req = Requirement("testpkg")

with patch(
"fromager.bootstrap_requirement_resolver.resolver"
".find_all_matching_from_provider",
return_value=[
("http://cache.test/testpkg-3.0.whl", Version("3.0")),
("http://cache.test/testpkg-2.0.whl", Version("2.0")),
],
):
result = brr._resolve_from_cache_server(req)

assert len(result) == 1
assert result[0][1] == Version("3.0")

def test_returns_empty_when_cache_has_no_match(
self, tmp_context: WorkContext
) -> None:
"""Returns empty list when cache server has nothing."""
brr = BootstrapRequirementResolver(
tmp_context,
multiple_versions=True,
cache_wheel_server_url="http://cache.test/simple",
)
req = Requirement("testpkg")

with patch(
"fromager.bootstrap_requirement_resolver.resolver"
".find_all_matching_from_provider",
return_value=[],
):
result = brr._resolve_from_cache_server(req)

assert result == []

def test_returns_empty_on_exception(self, tmp_context: WorkContext) -> None:
"""Returns empty list when cache server query fails."""
brr = BootstrapRequirementResolver(
tmp_context,
multiple_versions=True,
cache_wheel_server_url="http://cache.test/simple",
)
req = Requirement("testpkg")

with patch(
"fromager.bootstrap_requirement_resolver.resolver"
".find_all_matching_from_provider",
side_effect=RuntimeError("connection refused"),
):
result = brr._resolve_from_cache_server(req)

assert result == []

def test_resolve_uses_cache_fallback_when_age_filter_empties(
self, tmp_context: WorkContext
) -> None:
"""_resolve falls back to cache server when age filter produces empty result."""
brr = BootstrapRequirementResolver(
tmp_context,
multiple_versions=True,
cache_wheel_server_url="http://cache.test/simple",
)
req = Requirement("testpkg")

with (
patch.object(brr, "_resolve_from_graph", return_value=None),
patch(
"fromager.bootstrap_requirement_resolver.sources.get_source_provider",
),
patch(
"fromager.bootstrap_requirement_resolver.resolver"
".find_all_matching_from_provider",
return_value=[],
) as mock_pypi,
patch.object(
brr,
"_resolve_from_cache_server",
return_value=[("http://cache.test/testpkg-1.0.whl", Version("1.0"))],
) as mock_cache,
):
result = brr._resolve(
req,
RequirementType.INSTALL,
parent_req=None,
pre_built=False,
)

mock_pypi.assert_called_once()
mock_cache.assert_called_once_with(req)
assert len(result) == 1
assert result[0][1] == Version("1.0")

def test_resolve_skips_cache_fallback_in_single_version_mode(
self, tmp_context: WorkContext
) -> None:
"""_resolve does not fall back to cache server in single-version mode."""
brr = BootstrapRequirementResolver(
tmp_context,
multiple_versions=False,
cache_wheel_server_url="http://cache.test/simple",
)
req = Requirement("testpkg")

with (
patch.object(brr, "_resolve_from_graph", return_value=None),
patch(
"fromager.bootstrap_requirement_resolver.sources.get_source_provider",
),
patch(
"fromager.bootstrap_requirement_resolver.resolver"
".find_all_matching_from_provider",
return_value=[("url", Version("1.0"))],
),
patch.object(brr, "_resolve_from_cache_server") as mock_cache,
):
brr._resolve(
req,
RequirementType.INSTALL,
parent_req=None,
pre_built=False,
)

mock_cache.assert_not_called()

def test_resolve_skips_cache_fallback_when_no_server_url(
self, tmp_context: WorkContext
) -> None:
"""_resolve does not fall back when no cache_wheel_server_url is set."""
brr = BootstrapRequirementResolver(
tmp_context,
multiple_versions=True,
cache_wheel_server_url="",
)
req = Requirement("testpkg")

with (
patch.object(brr, "_resolve_from_graph", return_value=None),
patch(
"fromager.bootstrap_requirement_resolver.sources.get_source_provider",
),
patch(
"fromager.bootstrap_requirement_resolver.resolver"
".find_all_matching_from_provider",
return_value=[],
),
patch.object(brr, "_resolve_from_cache_server") as mock_cache,
):
brr._resolve(
req,
RequirementType.INSTALL,
parent_req=None,
pre_built=False,
)

mock_cache.assert_not_called()
14 changes: 14 additions & 0 deletions tests/test_bootstrapper_iterative.py
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,20 @@ def test_no_filtering_in_single_version_mode(
assert len(result) == 1
mock_cache.assert_not_called()

def test_empty_resolution_raises_runtime_error(
self, tmp_context: WorkContext
) -> None:
"""Empty resolution raises RuntimeError regardless of mode."""
for multi in (False, True):
bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=multi)
item = _make_resolve_item()

with (
patch.object(bt, "resolve_versions", return_value=[]),
pytest.raises(RuntimeError, match="Could not resolve"),
):
bt._phase_resolve(item)


class TestPhaseStart:
def test_new_item_advances_to_prepare_source(
Expand Down
Loading
Loading