From d477522263d8e905261a92d2070930ade68ee2fb Mon Sep 17 00:00:00 2001 From: vados Date: Mon, 11 Mar 2024 16:33:30 +0900 Subject: [PATCH] feat(cargo-pgrx): add support for downloading specific pg versions This commit adds support for downloading a specific version of Postgres when running `cargo pgrx init` via tarball. Some example invocations would look like: - `cargo pgx init --pg12=12.6` Where as previously you could only provide a path to an existing `pg_config` from a Postgres installation (which you needed to place yourself), with this PR we can now specify either the version or a tarball. This PR does not solve the problem of allowing *any* URL to be used as a download URL, but instead only uses https://ftp.postgresql.org derived URLs. --- cargo-pgrx/src/command/init.rs | 286 +++++++++++++++++++++++------- cargo-pgrx/src/command/mod.rs | 9 + cargo-pgrx/src/command/version.rs | 7 +- pgrx-pg-config/src/lib.rs | 58 ++++++ pgrx/src/toast.rs | 1 + 5 files changed, 292 insertions(+), 69 deletions(-) diff --git a/cargo-pgrx/src/command/init.rs b/cargo-pgrx/src/command/init.rs index de61e222fd..46307eb7d0 100644 --- a/cargo-pgrx/src/command/init.rs +++ b/cargo-pgrx/src/command/init.rs @@ -7,17 +7,6 @@ //LICENSE All rights reserved. //LICENSE //LICENSE Use of this source code is governed by the MIT license that can be found in the LICENSE file. -use crate::command::stop::stop_postgres; -use crate::command::version::pgrx_default; -use crate::CommandExecute; -use bzip2::bufread::BzDecoder; -use eyre::{eyre, WrapErr}; -use owo_colors::OwoColorize; -use pgrx_pg_config::{ - get_c_locale_flags, prefix_path, ConfigToml, PgConfig, PgConfigSelector, Pgrx, PgrxHomeError, -}; -use tar::Archive; - use std::collections::HashMap; use std::fs::File; use std::io::{Read, Write}; @@ -26,6 +15,22 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::OnceLock; +use bzip2::bufread::BzDecoder; +use eyre::{bail, eyre, WrapErr}; +use owo_colors::OwoColorize; +use pgrx_pg_config::{ + get_c_locale_flags, prefix_path, ConfigToml, PgConfig, PgConfigSelector, PgMinorVersion, + PgVersion, Pgrx, PgrxHomeError, +}; +use tar::Archive; +use url::Url; + +use crate::command::stop::stop_postgres; +use crate::command::version::pgrx_default; +use crate::CommandExecute; + +use super::generate_ftp_download_url; + static PROCESS_ENV_DENYLIST: &[&str] = &[ "DEBUG", "MAKEFLAGS", @@ -41,6 +46,109 @@ static PROCESS_ENV_DENYLIST: &[&str] = &[ "LIBRARY_PATH", // see https://github.com/pgcentralfoundation/pgrx/issues/16 ]; +/// Specifies how to retrieve/init a Postgres version +pub(crate) enum PostgresRetrievalMethod { + /// The path to a postgres pg_config directory + PgConfigPath(PathBuf), + /// When the postgres version should be downloaded from wherever cargo-pgrx wants + ManagedDownload, + /// A more fully specified version (possibly a point release, rc or beta) + /// + /// (ex. `download.1` or `download.beta2`) + ManagedDownloadWithMinor(String), +} + +impl PostgresRetrievalMethod { + fn from_method_with_minor(m: &str, minor_version: &Option) -> eyre::Result { + match (m, minor_version) { + // If the path is 'download' then we can prompt a managed download, + // of the latest version by default + ("download", None) => Ok(Self::ManagedDownload), + // Versions that are specified with dots are likely point releases, betas or RCs + ("download", Some(v)) => Ok(Self::ManagedDownloadWithMinor(v.clone())), + // If the method is the path to a valid file, then we can use it + (p, _) if std::path::Path::new(p).exists() => Ok(Self::PgConfigPath(PathBuf::from(p))), + // All other methods should be skipped + (m, _) => bail!( + "unrecognized input [{m}], please specify a valid PG_CONFIG path, or 'download'" + ), + } + } +} + +/// Default [`Pgrx`] instance that is used by implementation methods +static DEFAULT_PGRX: OnceLock> = OnceLock::new(); +fn get_default_pgrx() -> &'static eyre::Result { + DEFAULT_PGRX.get_or_init(|| pgrx_default()) +} + +impl PostgresRetrievalMethod { + /// Given a Postgres major version, resolve this version input (which can take many forms) + /// into a [`PgConfig`] which can be used as part of a [`Pgrx`] + /// + /// # Arguments + /// + /// * `pg_major_version` - Postgres major version, with the 'pg' prefix (ex. `"pg16"`) + /// * `pgrx_home` - A Path to the PGRX home directory + /// * `pg_init` - Init command configuration + fn resolve( + &self, + pg_major_version: &str, + pgrx_home: impl AsRef, + pg_init: &Init, + ) -> eyre::Result { + // Build a PgConfig based on the versions that were provided + let pg_config = match self { + // If a config path was specified, then we can look it up + Self::PgConfigPath(pg_config_path) => { + PgConfig::new_with_defaults(pg_config_path.clone()) + } + // If 'download' was specified, we can download teh version of + Self::ManagedDownload => { + let Ok(default_pgrx) = get_default_pgrx().as_ref() else { + bail!("failed to generate default pgrx config"); + }; + default_pgrx.get(pg_major_version).wrap_err_with(|| { + format!("{pg_major_version} is not a known Postgres major version") + })? + } + // If a specialized version was used (ex. a point release, beta or rc), then + // use that more specified version rather than the simply the major version + Self::ManagedDownloadWithMinor(minor) => { + let version = PgVersion::try_from(( + format!("{pg_major_version}.{minor}").as_ref(), + Some(Url::parse(&generate_ftp_download_url( + pg_major_version.trim_start_matches("pg"), + minor, + ))?), + ))?; + + // Beta and RC versions cannot be downloaded this way (sources aren't avaiable via FTP, anyway) + if let PgMinorVersion::Rc(_) | PgMinorVersion::Beta(_) = version.minor { + bail!("managed download not supported for Rc and Beta versions"); + } + + download_postgres(&PgConfig::from(version), pgrx_home.as_ref(), pg_init)? + } + }; + + // Check the label after building, only necessary if we're not doing managed download + // to ensure we didn't somehow give a pg_config with the wrong version + if let Self::ManagedDownloadWithMinor(_) | Self::PgConfigPath(_) = self { + let label = pg_config.label().ok(); + if label.is_some() && label.as_deref() != Some(pg_major_version) { + bail!( + "wrong `pg_config` given to `--{pg_major_version}` `{:?}` is for PostgreSQL {}", + pg_config.path(), + pg_config.major_version()?, + ); + } + } + + Ok(pg_config) + } +} + /// Initialize pgrx development environment for the first time #[derive(clap::Args, Debug)] #[clap(author)] @@ -48,26 +156,75 @@ pub(crate) struct Init { /// If installed locally, the path to PG12's `pgconfig` tool, or `download` to have pgrx download/compile/install it #[clap(env = "PG12_PG_CONFIG", long)] pg12: Option, + + /// Specify a minor version for PG12 + #[clap( + env = "PG12_MINOR_VERSION", + long, + help = "Postgres 12 minor version (ex. '1', 'beta2', 'rc3')" + )] + pg12_minor_version: Option, + /// If installed locally, the path to PG13's `pgconfig` tool, or `download` to have pgrx download/compile/install it #[clap(env = "PG13_PG_CONFIG", long)] pg13: Option, + + /// Specify a minor version for PG13 + #[clap( + env = "PG13_MINOR_VERSION", + long, + help = "Postgres 13 minor version (ex. '1', 'beta2', 'rc3')" + )] + pg13_minor_version: Option, + /// If installed locally, the path to PG14's `pgconfig` tool, or `download` to have pgrx download/compile/install it #[clap(env = "PG14_PG_CONFIG", long)] pg14: Option, + + /// Specify a minor version for PG14 + #[clap( + env = "PG14_MINOR_VERSION", + long, + help = "Postgres 14 minor version (ex. '1', 'beta2', 'rc3')" + )] + pg14_minor_version: Option, + /// If installed locally, the path to PG15's `pgconfig` tool, or `download` to have pgrx download/compile/install it #[clap(env = "PG15_PG_CONFIG", long)] pg15: Option, + + /// Specify a minor version for PG15 + #[clap( + env = "PG15_MINOR_VERSION", + long, + help = "Postgres 15 minor version (ex. '1', 'beta2', 'rc3')" + )] + pg15_minor_version: Option, + /// If installed locally, the path to PG16's `pgconfig` tool, or `download` to have pgrx download/compile/install it #[clap(env = "PG16_PG_CONFIG", long)] pg16: Option, + + /// Specify a minor version for PG16 + #[clap( + env = "PG16_MINOR_VERSION", + long, + help = "Postgres 16 minor version (ex. '1', 'beta2', 'rc3')" + )] + pg16_minor_version: Option, + #[clap(from_global, action = ArgAction::Count)] verbose: u8, + #[clap(long, help = "Base port number")] base_port: Option, + #[clap(long, help = "Base testing port number")] base_testing_port: Option, + #[clap(long, help = "Additional flags to pass to the configure script")] configure_flag: Vec, + /// Compile PostgreSQL with the necessary flags to detect a good amount of /// memory errors when run under Valgrind. /// @@ -75,8 +232,10 @@ pub(crate) struct Init { /// installed, but the resulting build is usable without valgrind. #[clap(long)] valgrind: bool, + #[clap(long, short, help = "Allow N make jobs at once")] jobs: Option, + #[clap(skip)] jobserver: OnceLock, } @@ -97,79 +256,76 @@ impl CommandExecute for Init { ) .unwrap(); + // Parse versions provided into a lookup of requested versions (if present) to the + // retrieval method that should be used for retreiving them let mut versions = HashMap::new(); - if let Some(ref version) = self.pg12 { - versions.insert("pg12", version.clone()); + // Since arguments to options like --pg12 are *methods* of obtaining the source + // (i.e. 'download' or a path on disk to a pg_config), we use them to generate a method + // which we will later `resolve()` + if let Some(ref method) = self.pg12 { + versions.insert( + "pg12", + PostgresRetrievalMethod::from_method_with_minor(method, &self.pg12_minor_version)?, + ); } - if let Some(ref version) = self.pg13 { - versions.insert("pg13", version.clone()); + if let Some(ref method) = self.pg13 { + versions.insert( + "pg13", + PostgresRetrievalMethod::from_method_with_minor(method, &self.pg13_minor_version)?, + ); } - if let Some(ref version) = self.pg14 { - versions.insert("pg14", version.clone()); + if let Some(ref method) = self.pg14 { + versions.insert( + "pg14", + PostgresRetrievalMethod::from_method_with_minor(method, &self.pg14_minor_version)?, + ); } - if let Some(ref version) = self.pg15 { - versions.insert("pg15", version.clone()); + if let Some(ref method) = self.pg15 { + versions.insert( + "pg15", + PostgresRetrievalMethod::from_method_with_minor(method, &self.pg15_minor_version)?, + ); } - if let Some(ref version) = self.pg16 { - versions.insert("pg16", version.clone()); + if let Some(ref method) = self.pg16 { + versions.insert( + "pg16", + PostgresRetrievalMethod::from_method_with_minor(method, &self.pg16_minor_version)?, + ); } - if versions.is_empty() { - // no arguments specified, so we'll just install our defaults - init_pgrx(&pgrx_default()?, &self) - } else { - // user specified arguments, so we'll only install those versions of Postgres - let mut default_pgrx = None; - let mut pgrx = Pgrx::default(); - - for (pgver, pg_config_path) in versions { - let config = if pg_config_path == "download" { - if default_pgrx.is_none() { - default_pgrx = Some(pgrx_default()?); - } - default_pgrx - .as_ref() - .unwrap() // We just set this - .get(pgver) - .wrap_err_with(|| format!("{pgver} is not a known Postgres version"))? - .clone() - } else { - let config = PgConfig::new_with_defaults(pg_config_path.as_str().into()); - let label = config.label().ok(); - // We allow None in case it's configured via the environment or something. - if label.is_some() && label.as_deref() != Some(pgver) { - return Err(eyre!( - "wrong `pg_config` given to `--{pgver}` `{pg_config_path:?}` is for PostgreSQL {}", - config.major_version()?, - )); - } - config - }; - pgrx.push(config); - } - - init_pgrx(&pgrx, &self) + // Resolve all versions required and then push them into the config + let pgrx_home = get_pgrx_home()?; + let mut pgrx = Pgrx::default(); + for (pg_version, version_input) in versions.into_iter() { + pgrx.push(version_input.resolve(pg_version, &pgrx_home, &self)?) } + + init_pgrx(&pgrx, &self) } } -#[tracing::instrument(skip_all)] -pub(crate) fn init_pgrx(pgrx: &Pgrx, init: &Init) -> eyre::Result<()> { - let pgrx_home = match Pgrx::home() { - Ok(path) => path, +/// Determine the home that pgrx should use +pub(crate) fn get_pgrx_home() -> eyre::Result { + match Pgrx::home() { + Ok(path) => Ok(path), Err(e) => match e { - PgrxHomeError::NoHomeDirectory => return Err(e.into()), - PgrxHomeError::IoError(e) => return Err(e.into()), + PgrxHomeError::NoHomeDirectory => Err(e.into()), + PgrxHomeError::IoError(e) => Err(e.into()), PgrxHomeError::MissingPgrxHome(path) => { // $PGRX_HOME doesn't exist, but that's okay as `cargo pgrx init` is the right time // to try and create it println!("{} PGRX_HOME at `{}`", " Creating".bold().green(), path.display()); std::fs::create_dir_all(&path)?; - path + Ok(path) } }, - }; + } +} + +#[tracing::instrument(skip_all)] +pub(crate) fn init_pgrx(pgrx: &Pgrx, init: &Init) -> eyre::Result<()> { + let pgrx_home = get_pgrx_home()?; let mut output_configs = std::thread::scope(|s| -> eyre::Result> { let span = tracing::Span::current(); @@ -186,7 +342,7 @@ pub(crate) fn init_pgrx(pgrx: &Pgrx, init: &Init) -> eyre::Result<()> { let mut pg_config = pg_config.clone(); stop_postgres(&pg_config).ok(); // no need to fail on errors trying to stop postgres while initializing if !pg_config.is_real() { - pg_config = match download_postgres(&pg_config, &pgrx_home, init) { + pg_config = match download_postgres(&pg_config, pgrx_home.as_path(), init) { Ok(pg_config) => pg_config, Err(e) => return Err(eyre!(e)), } diff --git a/cargo-pgrx/src/command/mod.rs b/cargo-pgrx/src/command/mod.rs index d309515893..7d16236f01 100644 --- a/cargo-pgrx/src/command/mod.rs +++ b/cargo-pgrx/src/command/mod.rs @@ -38,4 +38,13 @@ fn build_agent_for_url(url: &str) -> eyre::Result { } } +/// Generate the FTP download for a Postgres tarball, given the major and minor versions +fn generate_ftp_download_url(major: impl ToString, minor: impl ToString) -> String { + let major = major.to_string(); + let minor = minor.to_string(); + format!( + "https://ftp.postgresql.org/pub/source/v{major}.{minor}/postgresql-{major}.{minor}.tar.bz2" + ) +} + // TODO: Abstract over the repeated `fn perform`? diff --git a/cargo-pgrx/src/command/version.rs b/cargo-pgrx/src/command/version.rs index 4be017fc1a..12a613cfb2 100644 --- a/cargo-pgrx/src/command/version.rs +++ b/cargo-pgrx/src/command/version.rs @@ -27,7 +27,7 @@ mod rss { use std::collections::BTreeMap; use url::Url; - use crate::command::build_agent_for_url; + use crate::command::{build_agent_for_url, generate_ftp_download_url}; pub(super) struct PostgreSQLVersionRss; @@ -65,9 +65,8 @@ mod rss { if matches!(known_pgver.minor, PgMinorVersion::Latest) { // fill in the latest minor version number and its url known_pgver.minor = PgMinorVersion::Release(minor); - known_pgver.url = Some(Url::parse( - &format!("https://ftp.postgresql.org/pub/source/v{major}.{minor}/postgresql-{major}.{minor}.tar.bz2") - )?); + known_pgver.url = + Some(Url::parse(&generate_ftp_download_url(major, minor))?); } } } diff --git a/pgrx-pg-config/src/lib.rs b/pgrx-pg-config/src/lib.rs index 931589c574..486a38a0fc 100644 --- a/pgrx-pg-config/src/lib.rs +++ b/pgrx-pg-config/src/lib.rs @@ -86,6 +86,31 @@ impl PgMinorVersion { } } +impl TryFrom<&str> for PgMinorVersion { + type Error = eyre::Error; + fn try_from(other: &str) -> eyre::Result { + Ok(match other.trim_start_matches("pg") { + "latest" => Self::Latest, + s if s.chars().all(|c| c.is_ascii_digit()) => { + Self::Release(u16::from_str_radix(s, 10)?) + } + s if s.contains("beta") => { + let (_major, minor) = s + .split_once("beta") + .ok_or_else(|| eyre!("unexpectedly missing 'beta' from version"))?; + Self::Beta(u16::from_str_radix(minor, 10)?) + } + s if s.contains("rc") => { + let (_major, minor) = s + .split_once("rc") + .ok_or_else(|| eyre!("unexpectedly missing 'rc' from version"))?; + Self::Rc(u16::from_str_radix(minor, 10)?) + } + s => eyre::bail!("unexpected input string for minor version [{s}]"), + }) + } +} + #[derive(Clone, Debug)] pub struct PgVersion { pub major: u16, @@ -109,6 +134,39 @@ impl Display for PgVersion { } } +impl TryFrom<(&str, Option)> for PgVersion { + type Error = eyre::Error; + + fn try_from((s, url): (&str, Option)) -> eyre::Result { + let v = match s.trim_start_matches("pg") { + s if s.contains(".beta") => { + let (major, minor) = + s.split_once(".").ok_or_else(|| eyre!("unexpectedly missing '.beta'"))?; + let major = u16::from_str_radix(major, 10)?; + Self::new(major, PgMinorVersion::try_from(minor)?, url) + } + s if s.contains(".rc") => { + let (major, minor) = + s.split_once(".").ok_or_else(|| eyre!("unexpectedly missing '.rc'"))?; + let major = u16::from_str_radix(major, 10)?; + Self::new(major, PgMinorVersion::try_from(minor)?, url) + } + s if s.contains('.') => { + let (major, minor) = + s.split_once('.').ok_or_else(|| eyre!("unexpectedly missing '.'"))?; + let major = u16::from_str_radix(major, 10)?; + Self::new(major, PgMinorVersion::try_from(minor)?, url) + } + s if s.chars().all(|c| c.is_ascii_digit()) => { + let major = u16::from_str_radix(s, 10)?; + Self::new(major, PgMinorVersion::Latest, url) + } + s => eyre::bail!("unexpected input string for pg version [{s}]"), + }; + Ok(v) + } +} + #[derive(Clone, Debug)] pub struct PgConfig { version: Option, diff --git a/pgrx/src/toast.rs b/pgrx/src/toast.rs index bac574c7de..249caaeb7e 100644 --- a/pgrx/src/toast.rs +++ b/pgrx/src/toast.rs @@ -18,6 +18,7 @@ where } pub(crate) trait Toasty: Sized { + #[allow(dead_code)] fn detoast(self) -> Toast; /// Why does it always land butter-side down? unsafe fn drop_toast(&mut self);