Skip to content

Commit 6843538

Browse files
mikolalysenkoclaude
andcommitted
Find lock-only patches for PEP 440 pin spellings
On a fresh checkout (no venv), requirements.txt discovery sent a pin as written: `six==1.16` became `pkg:pypi/six@1.16`, `==1.16.0.0` became `@1.16.0.0`, `==01.16.0` became `@01.16.0`. pip installs the registry release 1.16.0 for all three, but the patch API keys it `@1.16.0`, so the scan said "No patches available" and the unpatched release was installed. The same file with a venv holding six was patched. Scan now also asks the API for the other PEP 440 spellings of a lockfile-only PyPI pure-release pin (leading zeros dropped, release padded or trimmed to at least three segments). A patch returned under another spelling is counted as that lockfile-only package, so `notInstalled` and the vendored baseline pre-check see it. The rewriters already match `==` pins under PEP 440 (#478). Fixes #604 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent bdc255a commit 6843538

6 files changed

Lines changed: 257 additions & 6 deletions

File tree

‎crates/socket-patch-cli/src/commands/scan/mod.rs‎

Lines changed: 68 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,53 @@ const BATCH_BODY_BYTE_CAP: usize = 256 * 1024;
102102
/// authenticated API and [`DEFAULT_PROXY_BATCH_SIZE`] on the public proxy.
103103
/// Floored at 1: `--batch-size 0` is otherwise unvalidated and would make
104104
/// the chunking below panic, so it degrades to one-package batches.
105+
/// `purls` plus, for each lockfile-only PyPI purl among them, its other
106+
/// PEP 440 spellings of the same release (#604), deduplicated in order.
107+
fn with_pypi_equivalents(purls: &[String], lockfile_only: &HashSet<PurlKey>) -> Vec<String> {
108+
let mut out = purls.to_vec();
109+
let mut seen: HashSet<PurlKey> = purls.iter().map(|p| PurlKey::new(p)).collect();
110+
for purl in purls {
111+
if !lockfile_only_contains(lockfile_only, purl) {
112+
continue;
113+
}
114+
for spelling in socket_patch_core::utils::purl_key::pypi_equivalent_purls(purl) {
115+
if seen.insert(PurlKey::new(&spelling)) {
116+
out.push(spelling);
117+
}
118+
}
119+
}
120+
out
121+
}
122+
123+
/// Mark each API purl that names a lockfile-only PyPI pin under another
124+
/// PEP 440 spelling (`@1.16.0` for a lock's `@1.16`, #604) as lockfile-only
125+
/// too, so the `notInstalled` flag, the `[NOT INSTALLED]` marker and the
126+
/// vendored baseline pre-check treat it as the package it is. A spelling an
127+
/// installed copy already carries is left alone.
128+
fn adopt_pypi_equivalents(
129+
packages: &[BatchPackagePatches],
130+
scanned: &[String],
131+
lockfile_only: &mut HashSet<PurlKey>,
132+
) {
133+
let scanned_keys: HashSet<PurlKey> = scanned.iter().map(|p| PurlKey::new(p)).collect();
134+
let lock_only_pypi: Vec<&String> = scanned
135+
.iter()
136+
.filter(|p| p.starts_with("pkg:pypi/") && lockfile_only_contains(lockfile_only, p))
137+
.collect();
138+
for pkg in packages {
139+
let key = PurlKey::new(&pkg.purl);
140+
if scanned_keys.contains(&key) {
141+
continue;
142+
}
143+
if lock_only_pypi
144+
.iter()
145+
.any(|lock| socket_patch_core::utils::purl_key::pypi_same_release(lock, &pkg.purl))
146+
{
147+
lockfile_only.insert(key);
148+
}
149+
}
150+
}
151+
105152
fn effective_batch_size(requested: Option<usize>, use_public_proxy: bool) -> usize {
106153
requested
107154
.unwrap_or(if use_public_proxy {
@@ -1829,7 +1876,10 @@ async fn run_scan(
18291876
// that have NO installed copy (fresh clone, partial install). They join
18301877
// discovery and are flagged "not yet installed". Scoped to the crawled
18311878
// ecosystems.
1832-
let lockfile_only = lockfile_supplement(&ctx, &all_crawled, crawl_scope).await;
1879+
let mut lockfile_only = lockfile_supplement(&ctx, &all_crawled, crawl_scope).await;
1880+
// Counted once: #604 adds the API's spellings of lockfile-only PyPI pins
1881+
// to `lockfile_only.purls` after the batch query.
1882+
let lockfile_only_count = lockfile_only.purls.len();
18331883
// Unsupported layouts and malformed binary Bun locks, kept on empty
18341884
// scans too: an unreadable graph is not evidence of no dependencies.
18351885
let mut layout_refusals = unsupported_layout_warnings(&lockfile_only.unsupported);
@@ -2266,8 +2316,8 @@ async fn run_scan(
22662316
plural(package_count, "package", "packages")
22672317
));
22682318
if human {
2269-
if !lockfile_only.purls.is_empty() {
2270-
eprintln!("{}", render::lockfile_only_note(lockfile_only.purls.len()));
2319+
if lockfile_only_count > 0 {
2320+
eprintln!("{}", render::lockfile_only_note(lockfile_only_count));
22712321
}
22722322
print_layout_refusals(&layout_refusals, args.common.silent);
22732323
policy.print_warnings(args.common.silent);
@@ -2276,7 +2326,12 @@ async fn run_scan(
22762326
// Query API in batches
22772327
let mut all_packages_with_patches: Vec<BatchPackagePatches> = Vec::new();
22782328
let mut can_access_paid_patches = false;
2279-
let chunks: Vec<&[String]> = batch_chunks(&all_purls, batch_size, BATCH_BODY_BYTE_CAP);
2329+
// #604: a lockfile-only PyPI pin is spelled as the user wrote it
2330+
// (`six==1.16` → `@1.16`) while the API keys the release as the
2331+
// registry published it (`@1.16.0`); pip treats both as one release
2332+
// (PEP 440). Ask for its equivalent spellings too.
2333+
let query_purls = with_pypi_equivalents(&all_purls, &lockfile_only.purls);
2334+
let chunks: Vec<&[String]> = batch_chunks(&query_purls, batch_size, BATCH_BODY_BYTE_CAP);
22802335
let total_batches = chunks.len();
22812336
let mut batch_error_count = 0usize;
22822337
let mut last_batch_error: Option<String> = None;
@@ -2387,6 +2442,13 @@ async fn run_scan(
23872442
// drives the table, the `--json` `packages` array and the apply order,
23882443
// which operators diff across runs.
23892444
all_packages_with_patches.sort_by(|a, b| a.purl.cmp(&b.purl));
2445+
// #604: a patch the API returned under an equivalent spelling of a
2446+
// lockfile-only PyPI pin is that lockfile-only package.
2447+
adopt_pypi_equivalents(
2448+
&all_packages_with_patches,
2449+
&all_purls,
2450+
&mut lockfile_only.purls,
2451+
);
23902452

23912453
// If every batch errored, surface a full scan failure rather than
23922454
// silently reporting zero patches.
@@ -2401,7 +2463,7 @@ async fn run_scan(
24012463
"status": "error",
24022464
"error": { "code": API_BATCH_FAILED, "message": err },
24032465
"scannedPackages": package_count,
2404-
"lockfileOnlyPackages": lockfile_only.purls.len(),
2466+
"lockfileOnlyPackages": lockfile_only_count,
24052467
"packagesWithPatches": 0,
24062468
"totalPatches": 0,
24072469
"freePatches": 0,
@@ -2481,7 +2543,7 @@ async fn run_scan(
24812543
let mut result = serde_json::json!({
24822544
"status": "success",
24832545
"scannedPackages": package_count,
2484-
"lockfileOnlyPackages": lockfile_only.purls.len(),
2546+
"lockfileOnlyPackages": lockfile_only_count,
24852547
"packagesWithPatches": all_packages_with_patches.len(),
24862548
"totalPatches": total_patches,
24872549
"freePatches": free_patches,

‎crates/socket-patch-cli/tests/mode_migration_pypi.rs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2002,3 +2002,56 @@ async fn pipenv_hosted_to_vendored_names_the_unpatched_requirements() {
20022002
"the takeover names requirements.txt as an unpatched install source: {env:#}"
20032003
);
20042004
}
2005+
2006+
// ── #604: PEP 440-equivalent lock-only pins ──────────────────────────────
2007+
2008+
/// #604: on a fresh checkout (no venv), `six==1.16` (or `==1.16.0.0`,
2009+
/// `==01.16.0`) is the release the patch API keys as `six@1.16.0`. Hosted
2010+
/// mode must find the patch and rewrite the pin, as it does when a venv
2011+
/// holds six 1.16.0, and report the package as not installed.
2012+
#[tokio::test]
2013+
async fn lock_only_pep440_equivalent_pin_is_patched() {
2014+
use wiremock::matchers::body_string_contains;
2015+
for pin in ["1.16", "1.16.0.0", "01.16.0"] {
2016+
let server = MockServer::start().await;
2017+
// The API matches purls exactly: only `@1.16.0` has the patch.
2018+
Mock::given(method("POST"))
2019+
.and(path(format!("/v0/orgs/{ORG}/patches/batch")))
2020+
.and(body_string_contains(PURL))
2021+
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
2022+
"packages": [{ "purl": PURL, "patches": [{
2023+
"uuid": UUID, "purl": PURL, "tier": "free", "cveIds": [], "ghsaIds": [],
2024+
"severity": "high", "title": "pep440 fixture"
2025+
}]}],
2026+
"canAccessPaidPatches": false,
2027+
})))
2028+
.with_priority(1)
2029+
.mount(&server)
2030+
.await;
2031+
Mock::given(method("POST"))
2032+
.and(path(format!("/v0/orgs/{ORG}/patches/batch")))
2033+
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
2034+
"packages": [], "canAccessPaidPatches": false,
2035+
})))
2036+
.with_priority(2)
2037+
.mount(&server)
2038+
.await;
2039+
let hosted_url = mount_hosted_api(&server, true).await;
2040+
let (_tmp, root) = project();
2041+
std::fs::write(
2042+
root.join("requirements.txt"),
2043+
format!("idna==3.7\nsix=={pin}\n"),
2044+
)
2045+
.unwrap();
2046+
let (code, env) = hosted_scan(&root, &server);
2047+
assert_eq!(code, 0, "{pin}: {env:#}");
2048+
assert_eq!(env["redirect"]["redirected"], 1, "{pin}: {env:#}");
2049+
assert_eq!(env["packages"][0]["purl"], PURL, "{pin}: {env:#}");
2050+
assert_eq!(env["packages"][0]["notInstalled"], true, "{pin}: {env:#}");
2051+
let requirements = std::fs::read_to_string(root.join("requirements.txt")).unwrap();
2052+
assert!(
2053+
requirements.contains(&format!("six @ {hosted_url}")),
2054+
"{pin}: the pin is rewritten:\n{requirements}"
2055+
);
2056+
}
2057+
}

‎crates/socket-patch-cli/tests/scan_requirements_lock_only.rs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -376,3 +376,40 @@ async fn lock_only_scan_discovers_pin_with_dangling_eof_continuation() {
376376
)
377377
.await;
378378
}
379+
380+
/// #604: pip resolves `six==1.16`, `six==1.16.0.0` and `six==01.16.0` to
381+
/// the registry release `1.16.0` (PEP 440), but lock-only discovery queried
382+
/// only the spelled version, so the patch keyed `@1.16.0` was never found.
383+
/// Each spelling must also ask for the release's other spellings.
384+
#[tokio::test]
385+
async fn lock_only_scan_queries_pep440_equivalent_spellings() {
386+
for (pin, spelled) in [
387+
("1.16", "1.16"),
388+
("1.16.0.0", "1.16.0.0"),
389+
("01.16.0", "01.16.0"),
390+
] {
391+
for mode in [&[][..], &["--mode", "vendored"][..]] {
392+
let mock = MockServer::start().await;
393+
mount_empty_batch(&mock).await;
394+
let tmp = tempfile::tempdir().unwrap();
395+
std::fs::write(
396+
tmp.path().join("requirements.txt"),
397+
format!("sp-fixture-six=={pin}\n"),
398+
)
399+
.unwrap();
400+
let (code, v) = run_scan(tmp.path(), &mock.uri(), mode, &[]);
401+
assert_eq!(code, 0, "{pin} {mode:?}: {v}");
402+
assert_eq!(v["lockfileOnlyPackages"].as_u64(), Some(1), "{v}");
403+
let purls = batch_purls(&mock).await;
404+
for want in [
405+
format!("pkg:pypi/sp-fixture-six@{spelled}"),
406+
"pkg:pypi/sp-fixture-six@1.16.0".to_string(),
407+
] {
408+
assert!(
409+
purls.contains(&want),
410+
"{pin} {mode:?}: {want} must reach the patch API; sent {purls:?}"
411+
);
412+
}
413+
}
414+
}
415+
}

‎crates/socket-patch-core/src/utils/pep440.rs‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,37 @@ pub(crate) fn is_exact_pin(specifier: &str) -> bool {
130130
.is_some_and(|pinned| parse(pinned).is_some())
131131
}
132132

133+
/// The other spellings a registry may have published the pure release
134+
/// `version` (digits and dots only) under, as PEP 440 equality reads them:
135+
/// leading zeros dropped and the release padded with `.0` from its
136+
/// trailing-zero-trimmed form up to at least three segments (`1.16` →
137+
/// `1.16.0`; `1.16.0` → `1.16`; `01.16.0.0` → `1.16`, `1.16.0`). `version`
138+
/// itself is not included; anything that is not a pure release has none.
139+
pub(crate) fn equivalent_release_spellings(version: &str) -> Vec<String> {
140+
let valid = !version.is_empty()
141+
&& version.split('.').all(|segment| {
142+
!segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_digit())
143+
});
144+
if !valid {
145+
return Vec::new();
146+
}
147+
let segments: Vec<String> = version.split('.').map(number).collect();
148+
let mut trimmed = segments.clone();
149+
while trimmed.len() > 1 && trimmed.last().is_some_and(|segment| segment == "0") {
150+
trimmed.pop();
151+
}
152+
let mut out = Vec::new();
153+
for len in trimmed.len()..=segments.len().max(3) {
154+
let mut release = trimmed.clone();
155+
release.resize(len, "0".to_string());
156+
let spelling = release.join(".");
157+
if spelling != version && !out.contains(&spelling) {
158+
out.push(spelling);
159+
}
160+
}
161+
out
162+
}
163+
133164
#[cfg(test)]
134165
mod tests {
135166
use super::*;
@@ -182,6 +213,32 @@ mod tests {
182213
}
183214
}
184215

216+
#[test]
217+
fn equivalent_release_spellings_pad_and_trim() {
218+
assert_eq!(equivalent_release_spellings("1.16"), ["1.16.0"]);
219+
assert_eq!(equivalent_release_spellings("1.16.0"), ["1.16"]);
220+
assert_eq!(equivalent_release_spellings("1.16.0.0"), ["1.16", "1.16.0"]);
221+
assert_eq!(equivalent_release_spellings("01.16.0"), ["1.16", "1.16.0"]);
222+
assert_eq!(equivalent_release_spellings("2"), ["2.0", "2.0.0"]);
223+
assert!(equivalent_release_spellings("2.8.2").is_empty());
224+
for other in [
225+
"1.0rc1",
226+
"2.9.0.post0",
227+
"1.0+local",
228+
"",
229+
"1..2",
230+
"v1.0",
231+
"1.*",
232+
] {
233+
assert!(equivalent_release_spellings(other).is_empty(), "{other}");
234+
}
235+
for version in ["1.16", "1.16.0.0", "01.16.0"] {
236+
for spelling in equivalent_release_spellings(version) {
237+
assert!(versions_equal(version, &spelling), "{version} {spelling}");
238+
}
239+
}
240+
}
241+
185242
#[test]
186243
fn exact_pin_spellings() {
187244
assert!(is_exact_pin_of("==1.16", "1.16.0"));

‎crates/socket-patch-core/src/utils/purl_key.rs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,43 @@ fn composer_identity(canonical: &str) -> Option<String> {
173173
))
174174
}
175175

176+
/// `pkg:pypi/<name>@<v>` purls for every other spelling of `purl`'s pure
177+
/// release (`@1.16` → `@1.16.0`, see
178+
/// [`crate::utils::pep440::equivalent_release_spellings`]): the lockfile
179+
/// spells a pin as the user wrote it (`six==1.16`), while the patch API
180+
/// keys the release as the registry published it (#604). Empty for any
181+
/// other purl.
182+
pub fn pypi_equivalent_purls(purl: &str) -> Vec<String> {
183+
let canonical = canonical_base_purl(purl);
184+
let Some((name, version)) = canonical
185+
.strip_prefix("pkg:pypi/")
186+
.and_then(|rest| rest.rsplit_once('@'))
187+
else {
188+
return Vec::new();
189+
};
190+
crate::utils::pep440::equivalent_release_spellings(version)
191+
.into_iter()
192+
.map(|spelling| format!("pkg:pypi/{name}@{spelling}"))
193+
.collect()
194+
}
195+
196+
/// Whether two PyPI purls name the same release under PEP 440 equality
197+
/// (`@1.16` and `@1.16.0`), the way pip resolves an `==` pin. `false` for
198+
/// anything that is not two PyPI purls of one (PEP 503) name.
199+
pub fn pypi_same_release(a: &str, b: &str) -> bool {
200+
let (a, b) = (canonical_base_purl(a), canonical_base_purl(b));
201+
let split = |purl: &str| -> Option<(String, String)> {
202+
let (name, version) = purl.strip_prefix("pkg:pypi/")?.rsplit_once('@')?;
203+
Some((name.to_string(), version.to_string()))
204+
};
205+
match (split(&a), split(&b)) {
206+
(Some((name_a, version_a)), Some((name_b, version_b))) => {
207+
name_a == name_b && crate::utils::pep440::versions_equal(&version_a, &version_b)
208+
}
209+
_ => false,
210+
}
211+
}
212+
176213
#[cfg(test)]
177214
#[path = "purl_key_nuget_vendor_tests.rs"]
178215
mod nuget_vendor_tests;

‎docs/testing/uv-compatibility.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,11 @@ frozen, locked, and ordinary installation outcomes separately where supported.
154154
those cases requires marker-specific source mappings. Standalone PEP 751
155155
rewriting selects the exact package version; duplicate entries for the same
156156
name and version are refused when source selection is ambiguous.
157+
- Lock-only discovery (a fresh checkout, no venv) queries the patch API with
158+
every PEP 440 spelling of an exact pure-release pin as well as the one
159+
written (`six==1.16` asks for `@1.16` and `@1.16.0`; `==1.16.0` also asks for
160+
`@1.16`), so it finds the patch the registry keys under its own spelling, as a
161+
venv-backed run does, and reports the package as not installed (#604).
157162
- Hosted requirements select exact `==`/`===` pins or identifiable archive URLs.
158163
Other versions remain unchanged. A bare requirement is rewritten only when
159164
one row and one override version identify the selection. Ranges, wildcard

0 commit comments

Comments
 (0)