Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 68 additions & 6 deletions crates/socket-patch-cli/src/commands/scan/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,53 @@ const BATCH_BODY_BYTE_CAP: usize = 256 * 1024;
/// authenticated API and [`DEFAULT_PROXY_BATCH_SIZE`] on the public proxy.
/// Floored at 1: `--batch-size 0` is otherwise unvalidated and would make
/// the chunking below panic, so it degrades to one-package batches.
/// `purls` plus, for each lockfile-only PyPI purl among them, its other
/// PEP 440 spellings of the same release (#604), deduplicated in order.
fn with_pypi_equivalents(purls: &[String], lockfile_only: &HashSet<PurlKey>) -> Vec<String> {
let mut out = purls.to_vec();
let mut seen: HashSet<PurlKey> = purls.iter().map(|p| PurlKey::new(p)).collect();
for purl in purls {
if !lockfile_only_contains(lockfile_only, purl) {
continue;
}
for spelling in socket_patch_core::utils::purl_key::pypi_equivalent_purls(purl) {
if seen.insert(PurlKey::new(&spelling)) {
out.push(spelling);
}
}
}
out
}

/// Mark each API purl that names a lockfile-only PyPI pin under another
/// PEP 440 spelling (`@1.16.0` for a lock's `@1.16`, #604) as lockfile-only
/// too, so the `notInstalled` flag, the `[NOT INSTALLED]` marker and the
/// vendored baseline pre-check treat it as the package it is. A spelling an
/// installed copy already carries is left alone.
fn adopt_pypi_equivalents(
packages: &[BatchPackagePatches],
scanned: &[String],
lockfile_only: &mut HashSet<PurlKey>,
) {
let scanned_keys: HashSet<PurlKey> = scanned.iter().map(|p| PurlKey::new(p)).collect();
let lock_only_pypi: Vec<&String> = scanned
.iter()
.filter(|p| p.starts_with("pkg:pypi/") && lockfile_only_contains(lockfile_only, p))
.collect();
for pkg in packages {
let key = PurlKey::new(&pkg.purl);
if scanned_keys.contains(&key) {
continue;
}
if lock_only_pypi
.iter()
.any(|lock| socket_patch_core::utils::purl_key::pypi_same_release(lock, &pkg.purl))
{
lockfile_only.insert(key);
}
}
}

fn effective_batch_size(requested: Option<usize>, use_public_proxy: bool) -> usize {
requested
.unwrap_or(if use_public_proxy {
Expand Down Expand Up @@ -1858,7 +1905,10 @@ async fn run_scan(
// that have NO installed copy (fresh clone, partial install). They join
// discovery and are flagged "not yet installed". Scoped to the crawled
// ecosystems.
let lockfile_only = lockfile_supplement(&ctx, &all_crawled, crawl_scope).await;
let mut lockfile_only = lockfile_supplement(&ctx, &all_crawled, crawl_scope).await;
// Counted once: #604 adds the API's spellings of lockfile-only PyPI pins
// to `lockfile_only.purls` after the batch query.
let lockfile_only_count = lockfile_only.purls.len();
// Unsupported layouts and malformed binary Bun locks, kept on empty
// scans too: an unreadable graph is not evidence of no dependencies.
let mut layout_refusals = unsupported_layout_warnings(&lockfile_only.unsupported);
Expand Down Expand Up @@ -2305,8 +2355,8 @@ async fn run_scan(
plural(package_count, "package", "packages")
));
if human {
if !lockfile_only.purls.is_empty() {
eprintln!("{}", render::lockfile_only_note(lockfile_only.purls.len()));
if lockfile_only_count > 0 {
eprintln!("{}", render::lockfile_only_note(lockfile_only_count));
}
print_layout_refusals(&layout_refusals, args.common.silent);
policy.print_warnings(args.common.silent);
Expand All @@ -2315,7 +2365,12 @@ async fn run_scan(
// Query API in batches
let mut all_packages_with_patches: Vec<BatchPackagePatches> = Vec::new();
let mut can_access_paid_patches = false;
let chunks: Vec<&[String]> = batch_chunks(&all_purls, batch_size, BATCH_BODY_BYTE_CAP);
// #604: a lockfile-only PyPI pin is spelled as the user wrote it
// (`six==1.16` → `@1.16`) while the API keys the release as the
// registry published it (`@1.16.0`); pip treats both as one release
// (PEP 440). Ask for its equivalent spellings too.
let query_purls = with_pypi_equivalents(&all_purls, &lockfile_only.purls);
let chunks: Vec<&[String]> = batch_chunks(&query_purls, batch_size, BATCH_BODY_BYTE_CAP);
let total_batches = chunks.len();
let mut batch_error_count = 0usize;
let mut last_batch_error: Option<String> = None;
Expand Down Expand Up @@ -2426,6 +2481,13 @@ async fn run_scan(
// drives the table, the `--json` `packages` array and the apply order,
// which operators diff across runs.
all_packages_with_patches.sort_by(|a, b| a.purl.cmp(&b.purl));
// #604: a patch the API returned under an equivalent spelling of a
// lockfile-only PyPI pin is that lockfile-only package.
adopt_pypi_equivalents(
&all_packages_with_patches,
&all_purls,
&mut lockfile_only.purls,
);

// If every batch errored, surface a full scan failure rather than
// silently reporting zero patches.
Expand All @@ -2440,7 +2502,7 @@ async fn run_scan(
"status": "error",
"error": { "code": API_BATCH_FAILED, "message": err },
"scannedPackages": package_count,
"lockfileOnlyPackages": lockfile_only.purls.len(),
"lockfileOnlyPackages": lockfile_only_count,
"packagesWithPatches": 0,
"totalPatches": 0,
"freePatches": 0,
Expand Down Expand Up @@ -2520,7 +2582,7 @@ async fn run_scan(
let mut result = serde_json::json!({
"status": "success",
"scannedPackages": package_count,
"lockfileOnlyPackages": lockfile_only.purls.len(),
"lockfileOnlyPackages": lockfile_only_count,
"packagesWithPatches": all_packages_with_patches.len(),
"totalPatches": total_patches,
"freePatches": free_patches,
Expand Down
53 changes: 53 additions & 0 deletions crates/socket-patch-cli/tests/mode_migration_pypi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2071,6 +2071,59 @@ async fn pipenv_hosted_to_vendored_names_the_unpatched_requirements() {
);
}

// ── #604: PEP 440-equivalent lock-only pins ──────────────────────────────

/// #604: on a fresh checkout (no venv), `six==1.16` (or `==1.16.0.0`,
/// `==01.16.0`) is the release the patch API keys as `six@1.16.0`. Hosted
/// mode must find the patch and rewrite the pin, as it does when a venv
/// holds six 1.16.0, and report the package as not installed.
#[tokio::test]
async fn lock_only_pep440_equivalent_pin_is_patched() {
use wiremock::matchers::body_string_contains;
for pin in ["1.16", "1.16.0.0", "01.16.0"] {
let server = MockServer::start().await;
// The API matches purls exactly: only `@1.16.0` has the patch.
Mock::given(method("POST"))
.and(path(format!("/v0/orgs/{ORG}/patches/batch")))
.and(body_string_contains(PURL))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"packages": [{ "purl": PURL, "patches": [{
"uuid": UUID, "purl": PURL, "tier": "free", "cveIds": [], "ghsaIds": [],
"severity": "high", "title": "pep440 fixture"
}]}],
"canAccessPaidPatches": false,
})))
.with_priority(1)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path(format!("/v0/orgs/{ORG}/patches/batch")))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"packages": [], "canAccessPaidPatches": false,
})))
.with_priority(2)
.mount(&server)
.await;
let hosted_url = mount_hosted_api(&server, true).await;
let (_tmp, root) = project();
std::fs::write(
root.join("requirements.txt"),
format!("idna==3.7\nsix=={pin}\n"),
)
.unwrap();
let (code, env) = hosted_scan(&root, &server);
assert_eq!(code, 0, "{pin}: {env:#}");
assert_eq!(env["redirect"]["redirected"], 1, "{pin}: {env:#}");
assert_eq!(env["packages"][0]["purl"], PURL, "{pin}: {env:#}");
assert_eq!(env["packages"][0]["notInstalled"], true, "{pin}: {env:#}");
let requirements = std::fs::read_to_string(root.join("requirements.txt")).unwrap();
assert!(
requirements.contains(&format!("six @ {hosted_url}")),
"{pin}: the pin is rewritten:\n{requirements}"
);
}
}

// ── #1138: a uv workspace member ─────────────────────────────────────────

/// A uv workspace (root `pyproject.toml` with `[tool.uv.workspace]` and its
Expand Down
37 changes: 37 additions & 0 deletions crates/socket-patch-cli/tests/scan_requirements_lock_only.rs
Original file line number Diff line number Diff line change
Expand Up @@ -376,3 +376,40 @@ async fn lock_only_scan_discovers_pin_with_dangling_eof_continuation() {
)
.await;
}

/// #604: pip resolves `six==1.16`, `six==1.16.0.0` and `six==01.16.0` to
/// the registry release `1.16.0` (PEP 440), but lock-only discovery queried
/// only the spelled version, so the patch keyed `@1.16.0` was never found.
/// Each spelling must also ask for the release's other spellings.
#[tokio::test]
async fn lock_only_scan_queries_pep440_equivalent_spellings() {
for (pin, spelled) in [
("1.16", "1.16"),
("1.16.0.0", "1.16.0.0"),
("01.16.0", "01.16.0"),
] {
for mode in [&[][..], &["--mode", "vendored"][..]] {
let mock = MockServer::start().await;
mount_empty_batch(&mock).await;
let tmp = tempfile::tempdir().unwrap();
std::fs::write(
tmp.path().join("requirements.txt"),
format!("sp-fixture-six=={pin}\n"),
)
.unwrap();
let (code, v) = run_scan(tmp.path(), &mock.uri(), mode, &[]);
assert_eq!(code, 0, "{pin} {mode:?}: {v}");
assert_eq!(v["lockfileOnlyPackages"].as_u64(), Some(1), "{v}");
let purls = batch_purls(&mock).await;
for want in [
format!("pkg:pypi/sp-fixture-six@{spelled}"),
"pkg:pypi/sp-fixture-six@1.16.0".to_string(),
] {
assert!(
purls.contains(&want),
"{pin} {mode:?}: {want} must reach the patch API; sent {purls:?}"
);
}
}
}
}
57 changes: 57 additions & 0 deletions crates/socket-patch-core/src/utils/pep440.rs
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,37 @@ pub(crate) fn is_exact_pin(specifier: &str) -> bool {
.is_some_and(|pinned| parse(pinned).is_some())
}

/// The other spellings a registry may have published the pure release
/// `version` (digits and dots only) under, as PEP 440 equality reads them:
/// leading zeros dropped and the release padded with `.0` from its
/// trailing-zero-trimmed form up to at least three segments (`1.16` →
/// `1.16.0`; `1.16.0` → `1.16`; `01.16.0.0` → `1.16`, `1.16.0`). `version`
/// itself is not included; anything that is not a pure release has none.
pub(crate) fn equivalent_release_spellings(version: &str) -> Vec<String> {
let valid = !version.is_empty()
&& version.split('.').all(|segment| {
!segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_digit())
});
if !valid {
return Vec::new();
}
let segments: Vec<String> = version.split('.').map(number).collect();
let mut trimmed = segments.clone();
while trimmed.len() > 1 && trimmed.last().is_some_and(|segment| segment == "0") {
trimmed.pop();
}
let mut out = Vec::new();
for len in trimmed.len()..=segments.len().max(3) {
let mut release = trimmed.clone();
release.resize(len, "0".to_string());
let spelling = release.join(".");
if spelling != version && !out.contains(&spelling) {
out.push(spelling);
}
}
out
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down Expand Up @@ -182,6 +213,32 @@ mod tests {
}
}

#[test]
fn equivalent_release_spellings_pad_and_trim() {
assert_eq!(equivalent_release_spellings("1.16"), ["1.16.0"]);
assert_eq!(equivalent_release_spellings("1.16.0"), ["1.16"]);
assert_eq!(equivalent_release_spellings("1.16.0.0"), ["1.16", "1.16.0"]);
assert_eq!(equivalent_release_spellings("01.16.0"), ["1.16", "1.16.0"]);
assert_eq!(equivalent_release_spellings("2"), ["2.0", "2.0.0"]);
assert!(equivalent_release_spellings("2.8.2").is_empty());
for other in [
"1.0rc1",
"2.9.0.post0",
"1.0+local",
"",
"1..2",
"v1.0",
"1.*",
] {
assert!(equivalent_release_spellings(other).is_empty(), "{other}");
}
for version in ["1.16", "1.16.0.0", "01.16.0"] {
for spelling in equivalent_release_spellings(version) {
assert!(versions_equal(version, &spelling), "{version} {spelling}");
}
}
}

#[test]
fn exact_pin_spellings() {
assert!(is_exact_pin_of("==1.16", "1.16.0"));
Expand Down
37 changes: 37 additions & 0 deletions crates/socket-patch-core/src/utils/purl_key.rs
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,43 @@ fn composer_identity(canonical: &str) -> Option<String> {
))
}

/// `pkg:pypi/<name>@<v>` purls for every other spelling of `purl`'s pure
/// release (`@1.16` → `@1.16.0`, see
/// [`crate::utils::pep440::equivalent_release_spellings`]): the lockfile
/// spells a pin as the user wrote it (`six==1.16`), while the patch API
/// keys the release as the registry published it (#604). Empty for any
/// other purl.
pub fn pypi_equivalent_purls(purl: &str) -> Vec<String> {
let canonical = canonical_base_purl(purl);
let Some((name, version)) = canonical
.strip_prefix("pkg:pypi/")
.and_then(|rest| rest.rsplit_once('@'))
else {
return Vec::new();
};
crate::utils::pep440::equivalent_release_spellings(version)
.into_iter()
.map(|spelling| format!("pkg:pypi/{name}@{spelling}"))
.collect()
}

/// Whether two PyPI purls name the same release under PEP 440 equality
/// (`@1.16` and `@1.16.0`), the way pip resolves an `==` pin. `false` for
/// anything that is not two PyPI purls of one (PEP 503) name.
pub fn pypi_same_release(a: &str, b: &str) -> bool {
let (a, b) = (canonical_base_purl(a), canonical_base_purl(b));
let split = |purl: &str| -> Option<(String, String)> {
let (name, version) = purl.strip_prefix("pkg:pypi/")?.rsplit_once('@')?;
Some((name.to_string(), version.to_string()))
};
match (split(&a), split(&b)) {
(Some((name_a, version_a)), Some((name_b, version_b))) => {
name_a == name_b && crate::utils::pep440::versions_equal(&version_a, &version_b)
}
_ => false,
}
}

#[cfg(test)]
#[path = "purl_key_nuget_vendor_tests.rs"]
mod nuget_vendor_tests;
Expand Down
5 changes: 5 additions & 0 deletions docs/testing/uv-compatibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,11 @@ frozen, locked, and ordinary installation outcomes separately where supported.
those cases requires marker-specific source mappings. Standalone PEP 751
rewriting selects the exact package version; duplicate entries for the same
name and version are refused when source selection is ambiguous.
- Lock-only discovery (a fresh checkout, no venv) queries the patch API with
every PEP 440 spelling of an exact pure-release pin as well as the one
written (`six==1.16` asks for `@1.16` and `@1.16.0`; `==1.16.0` also asks for
`@1.16`), so it finds the patch the registry keys under its own spelling, as a
venv-backed run does, and reports the package as not installed (#604).
- Hosted requirements select exact `==`/`===` pins or socket-patch's own
hosted archive URLs. A user-authored direct reference to the patched release
(`name @ <url>` on any other origin, `files.pythonhosted.org` and `file://`
Expand Down
Loading