Merge branch 'patch-add-dist-bind-addr' of github.com:fslongjin/sccac…

…he into fea/nvcc-sccache-dist+build_addr

docs/DistributedQuickstart.md

@@ -75,6 +75,12 @@ cache_dir = "/tmp/toolchains"
 # toolchain_cache_size = 10737418240
 # A public IP address and port that clients will use to connect to this builder.
 public_addr = "192.168.1.1:10501"
+
+# The address this builder will listen on.
+# If unspecified the default is `public_addr`.
+# If you are running builder in Docker, you need to set this to `0.0.0.0:10501`
+bind_addr = "192.168.1.1:10501"
+
 # The URL used to connect to the scheduler (should use https, given an ideal
 # setup of a HTTPS server in front of the scheduler)
 scheduler_url = "https://192.168.1.1"

src/bin/sccache-dist/main.rs

@@ -225,10 +225,12 @@ fn run(command: Command) -> Result<i32> {
             builder,
             cache_dir,
             public_addr,
+            bind_addr,
             scheduler_url,
             scheduler_auth,
             toolchain_cache_size,
         }) => {
+            let bind_addr = bind_addr.unwrap_or_else(|| public_addr);
             let builder: Box<dyn dist::BuilderIncoming> = match builder {
                 #[cfg(not(target_os = "freebsd"))]
                 server_config::BuilderType::Docker => {
@@ -289,6 +291,7 @@ fn run(command: Command) -> Result<i32> {
                 .context("Failed to create sccache server instance")?;
             let http_server = dist::http::Server::new(
                 public_addr,
+                bind_addr,
                 scheduler_url.to_url(),
                 scheduler_auth,
                 server,

src/bin/sccache-dist/token_check.rs

@@ -100,7 +100,7 @@ impl MozillaCheck {
     pub fn new(required_groups: Vec<String>) -> Self {
         Self {
             auth_cache: Mutex::new(HashMap::new()),
-            client: new_reqwest_blocking_client(),
+            client: new_reqwest_blocking_client(None),
             required_groups,
         }
     }
@@ -269,7 +269,7 @@ impl ProxyTokenCheck {
         let maybe_auth_cache: Option<Mutex<(HashMap<String, Instant>, Duration)>> =
             cache_secs.map(|secs| Mutex::new((HashMap::new(), Duration::from_secs(secs))));
         Self {
-            client: new_reqwest_blocking_client(),
+            client: new_reqwest_blocking_client(None),
             maybe_auth_cache,
             url,
         }

src/config.rs

@@ -1207,6 +1207,7 @@ pub mod server {
         pub builder: BuilderType,
         pub cache_dir: PathBuf,
         pub public_addr: SocketAddr,
+        pub bind_addr: Option<SocketAddr>,
         pub scheduler_url: HTTPUrl,
         pub scheduler_auth: SchedulerAuth,
         #[serde(default = "default_toolchain_cache_size")]

src/dist/client_auth.rs

@@ -246,7 +246,7 @@ mod code_grant_pkce {
             grant_type: GRANT_TYPE_PARAM_VALUE,
             redirect_uri,
         };
-        let client = new_reqwest_blocking_client();
+        let client = new_reqwest_blocking_client(None);
         let res = client.post(token_url).json(&token_request).send()?;
         if !res.status().is_success() {
             bail!(

src/dist/http.rs

@@ -679,7 +679,7 @@ mod server {
                 check_server_auth,
             } = self;
             let requester = SchedulerRequester {
-                client: Mutex::new(new_reqwest_blocking_client()),
+                client: Mutex::new(new_reqwest_blocking_client(None)),
             };
 
             macro_rules! check_server_auth_or_err {
@@ -871,6 +871,7 @@ mod server {
 
     pub struct Server<S> {
         public_addr: SocketAddr,
+        bind_addr: SocketAddr,
         scheduler_url: reqwest::Url,
         scheduler_auth: String,
         // HTTPS pieces all the builders will use for connection encryption
@@ -887,6 +888,7 @@ mod server {
     impl<S: dist::ServerIncoming + 'static> Server<S> {
         pub fn new(
             public_addr: SocketAddr,
+            bind_addr: SocketAddr,
             scheduler_url: reqwest::Url,
             scheduler_auth: String,
             handler: S,
@@ -900,6 +902,7 @@ mod server {
 
             Ok(Self {
                 public_addr,
+                bind_addr,
                 scheduler_url,
                 scheduler_auth,
                 cert_digest,
@@ -914,6 +917,7 @@ mod server {
         pub fn start(self) -> Result<Infallible> {
             let Self {
                 public_addr,
+                bind_addr,
                 scheduler_url,
                 scheduler_auth,
                 cert_digest,
@@ -933,14 +937,14 @@ mod server {
             let job_authorizer = JWTJobAuthorizer::new(jwt_key);
             let heartbeat_url = urls::scheduler_heartbeat_server(&scheduler_url);
             let requester = ServerRequester {
-                client: new_reqwest_blocking_client(),
+                client: new_reqwest_blocking_client(Some(public_addr)),
                 scheduler_url,
                 scheduler_auth: scheduler_auth.clone(),
             };
 
             // TODO: detect if this panics
             thread::spawn(move || {
-                let client = new_reqwest_blocking_client();
+                let client = new_reqwest_blocking_client(Some(public_addr));
                 loop {
                     trace!("Performing heartbeat");
                     match bincode_req(
@@ -963,10 +967,13 @@ mod server {
                 }
             });
 
-            info!("Server listening for clients on {}", public_addr);
+            info!(
+                "Server listening for clients on {}, public_addr is: {}",
+                bind_addr, public_addr
+            );
             let request_count = atomic::AtomicUsize::new(0);
 
-            let server = rouille::Server::new_ssl(public_addr, move |request| {
+            let server = rouille::Server::new_ssl(bind_addr, move |request| {
                 let req_id = request_count.fetch_add(1, atomic::Ordering::SeqCst);
                 trace!("Req {} ({}): {:?}", req_id, request.remote_addr(), request);
                 let response = (|| router!(request,

src/util.rs

@@ -23,6 +23,7 @@ use std::cell::Cell;
 use std::ffi::{OsStr, OsString};
 use std::hash::Hasher;
 use std::io::prelude::*;
+use std::net::SocketAddr;
 use std::path::{Path, PathBuf};
 use std::process::{self, Stdio};
 use std::str;
@@ -938,8 +939,17 @@ pub fn daemonize() -> Result<()> {
 ///
 /// More details could be found at https://github.com/mozilla/sccache/pull/1563
 #[cfg(any(feature = "dist-server", feature = "dist-client"))]
-pub fn new_reqwest_blocking_client() -> reqwest::blocking::Client {
+pub fn new_reqwest_blocking_client(real_addr: Option<SocketAddr>) -> reqwest::blocking::Client {
+    let mut headers = reqwest::header::HeaderMap::new();
+    if let Some(addr) = real_addr {
+        headers.insert(
+            "X-Real-IP",
+            reqwest::header::HeaderValue::from_str(&format!("{}", addr.ip())).unwrap(),
+        );
+    }
+
     reqwest::blocking::Client::builder()
+        .default_headers(headers)
         .pool_max_idle_per_host(0)
         .build()
         .expect("http client must build with success")

tests/harness/mod.rs

@@ -195,6 +195,7 @@ fn sccache_server_cfg(
         },
         cache_dir: Path::new(CONFIGS_CONTAINER_PATH).join(relpath),
         public_addr: SocketAddr::new(server_ip, SERVER_PORT),
+        bind_addr: None,
         scheduler_url,
         scheduler_auth: sccache::config::server::SchedulerAuth::Token {
             token: DIST_SERVER_TOKEN.to_owned(),
@@ -409,9 +410,14 @@ impl DistSystem {
             listener.local_addr().unwrap()
         };
         let token = create_server_token(ServerId::new(server_addr), DIST_SERVER_TOKEN);
-        let server =
-            dist::http::Server::new(server_addr, self.scheduler_url().to_url(), token, handler)
-                .unwrap();
+        let server = dist::http::Server::new(
+            server_addr,
+            server_addr,
+            self.scheduler_url().to_url(),
+            token,
+            handler,
+        )
+        .unwrap();
         let pid = match unsafe { nix::unistd::fork() }.unwrap() {
             ForkResult::Parent { child } => {
                 self.server_pids.push(child);