feat(sys): improve linking behavior, separate features for vendoring DTLS

- using mbedtls or tinydtls will no longer force usage of the vendored
  library version of mbedtls-sys-auto/tinydtls-sys by default
- remove dependency on gnutls-sys crate as it is unmaintained, we now
  handle linking to gnutls ourselves
- mbedtls-sys-auto is only used if mbedtls should be vendored, as the
  crate doesn't allow for disabling vendoring independently and doesn't
  support mbedtls >= 3.0.0
  (see fortanix/rust-mbedtls#320)
- in order to use the vendored versions of DTLS crate, you now have to
  enable the `dtls_backend_[BACKEND]_vendored` feature of libcoap-sys
  (or the `dtls_[BACKEND]_vendored` feature of libcoap-rs)

Author: pulsastrix

Cargo.toml

@@ -9,3 +9,4 @@ members = [
     "libcoap",
     "libcoap-sys",
 ]
+resolver = "2"

libcoap-sys/Cargo.toml

@@ -17,6 +17,7 @@ authors = ["Hugo Hakim Damer <[email protected]>"]
 categories = ["external-ffi-bindings", "network-programming", "embedded"]
 keywords = ["coap", "libcoap"]
 exclude = ["src/libcoap/ext/"]
+resolver = "2"
 
 [features]
 # The default features match those of libcoaps configure script, except for dtls, which is disabled here because it
@@ -34,10 +35,13 @@ default = ["server", "client", "tcp", "async", "epoll", "vendored", "static"]
 # different backends), we select one based on the auto-detection order specified in
 # https://github.com/obgm/libcoap/blob/develop/configure.ac#L494 (gnutls > openssl > mbedtls > tinydtls).
 dtls = []
-dtls_backend_openssl = ["dtls", "openssl-sys"]
-dtls_backend_gnutls = ["dtls", "gnutls-sys"]
-dtls_backend_mbedtls = ["dtls", "mbedtls-sys-auto"]
-dtls_backend_tinydtls = ["dtls", "tinydtls-sys"]
+dtls_backend_openssl = ["dtls", "dep:openssl-sys"]
+dtls_backend_openssl_vendored = ["dtls_backend_openssl", "openssl-sys/vendored"]
+dtls_backend_gnutls = ["dtls"]
+dtls_backend_mbedtls = ["dtls"] # can't use mbedtls-sys-auto to generate linker flags here, as the crate doesn't support mbedtls >= 3.0.0
+dtls_backend_mbedtls_vendored = ["dep:mbedtls-sys-auto", "dtls_backend_mbedtls"]
+dtls_backend_tinydtls = ["dtls", "dep:tinydtls-sys"]
+dtls_backend_tinydtls_vendored = ["dtls_backend_tinydtls", "tinydtls-sys/vendored"]
 # Enabling this feature will force libcoap-sys to be built with and statically linked to a vendored version of libcoap,
 # which will be built by the build-script before building libcoap-sys.
 # This way, it is no longer required to have libcoap installed to use this crate.
@@ -61,11 +65,10 @@ client = []
 epoll = []
 
 [dependencies]
-gnutls-sys = {version = "^0.1.2", optional = true}
 openssl-sys = {version = "^0.9.74", optional = true}
 mbedtls-sys-auto = {version = "^2.26", optional = true}
 libc = "^0.2.126"
-tinydtls-sys = {version = "^0.1.2", optional = true}
+tinydtls-sys = {version = "^0.1.2", default-features = false, optional = true}
 
 [build-dependencies]
 bindgen = "^0.65.1"

libcoap-sys/build.rs

@@ -16,6 +16,7 @@ use std::{
 };
 
 use bindgen::EnumVariation;
+use pkg_config::probe_library;
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum DtlsBackend {
@@ -124,6 +125,7 @@ fn main() {
                 .enable("dtls", None)
                 .with(dtls_backend.to_string().as_str(), None);
 
+            // If DTLS is vendored we need to tell libcoap about the vendored version
             match dtls_backend {
                 DtlsBackend::TinyDtls => {
                     // We do not ship tinydtls with our source distribution. Instead, we use tinydtls-sys.
@@ -190,9 +192,7 @@ fn main() {
                     }
                 },
                 DtlsBackend::GnuTls => {
-                    // gnutls-sys doesn't provide include directory metadata, but doesn't support
-                    // vendoring the library either. Instead, it just uses the GnuTLS found via
-                    // pkg-config, which is already found by libcoap by default.
+                    // Vendoring not supported                
                 },
             }
         } else {
@@ -251,6 +251,8 @@ fn main() {
         );
         env::set_current_dir(current_dir_backup).expect("unable to switch back to source dir");
     }
+
+    // Tell cargo to link libcoap.
     println!(
         "cargo:rustc-link-lib={}{}",
         cfg!(feature = "static").then(|| "static=").unwrap_or(""),
@@ -263,6 +265,44 @@ fn main() {
         )
         .as_str()
     );
+    
+    // For the DTLS libraries, we need to tell cargo which external libraries to link.
+    // Note that these linker instructions have to be added *after* the linker instruction
+    // for libcoap itself, as some linkers require dependencies to be in reverse order. 
+    if let Some(dtls_backend) = dtls_backend {
+        match dtls_backend {
+            DtlsBackend::TinyDtls => {
+                // Handled by tinydtls-sys
+            },
+            DtlsBackend::OpenSsl => {
+                // Handled by openssl-sys
+            },
+            DtlsBackend::MbedTls => {
+                // If mbedtls is vendored, mbedtls-sys-auto already takes care of linking.
+                if env::var_os("DEP_MBEDTLS_INCLUDE").is_none() {
+                    // We aren't using mbedtls-sys-auto if we aren't vendoring (as it doesn't support 
+                    // mbedtls >= 3.0.0), so we need to tell cargo to link to mbedtls ourselves.
+                    
+                    // Try to find mbedtls using pkg-config
+                    if probe_library("mbedtls").is_err() {
+                        // couldn't find using pkg-config, just try linking with default library search path
+                        println!("cargo:rustc-link-lib=mbedtls");
+                        println!("cargo:rustc-link-lib=mbedx509");
+                        println!("cargo:rustc-link-lib=mbedcrypto");
+                    }
+                }
+            },
+            DtlsBackend::GnuTls => {
+                // gnutls-sys is unmaintained, so we need to link to gnutls ourselves.
+
+                // try pkg-config
+                if probe_library("gnutls").is_err() {
+                    // if that doesn't work, try using the standard library search path.
+                    println!("cargo:rustc-link-lib=gnutls")
+                }
+            },
+        }
+    }
 
     bindgen_builder = bindgen_builder
         .header("src/wrapper.h")

libcoap-sys/src/lib.rs

@@ -90,9 +90,9 @@ use libc::{epoll_event, fd_set, sockaddr, sockaddr_in, sockaddr_in6, socklen_t,
 
 use crate::coap_pdu_type_t::COAP_MESSAGE_RST;
 
-#[cfg(feature = "dtls_backend_gnutls")]
-use gnutls_sys as _;
-#[cfg(feature = "dtls_backend_mbedtls")]
+// use dtls backend libraries in cases where they set our linker flags, otherwise cargo will 
+// optimize them out.
+#[cfg(feature = "dtls_backend_mbedtls_vendored")]
 use mbedtls_sys as _;
 #[cfg(feature = "dtls_backend_openssl")]
 use openssl_sys as _;

libcoap/Cargo.toml

@@ -15,14 +15,18 @@ repository = "https://github.com/namib-project/libcoap-rs"
 authors = ["Hugo Hakim Damer <[email protected]>"]
 categories = ["api-bindings", "network-programming", "embedded"]
 keywords = ["coap", "libcoap"]
+resolver = "2"
 
 [features]
 default = ["dtls", "tcp", "dtls_openssl"]
 dtls = ["libcoap-sys/dtls"]
 dtls_tinydtls = ["libcoap-sys/dtls_backend_tinydtls"]
+dtls_tinydtls_vendored = ["dtls_tinydtls", "libcoap-sys/dtls_backend_tinydtls_vendored"]
 dtls_openssl = ["libcoap-sys/dtls_backend_openssl"]
+dtls_openssl_vendored = ["dtls_openssl", "libcoap-sys/dtls_backend_openssl_vendored"]
 dtls_gnutls = ["libcoap-sys/dtls_backend_gnutls"]
 dtls_mbedtls = ["libcoap-sys/dtls_backend_mbedtls"]
+dtls_mbedtls_vendored = ["dtls_mbedtls", "libcoap-sys/dtls_backend_mbedtls_vendored"]
 tcp = []
 nightly = []
 vendored = ["libcoap-sys/vendored"]