First wave of fixes for remote-run (#4033)

* Describe 403 code for remote-run apis
    * Since OPA can deny remote-run requests, it'd be nice to have the right code returned from the API server rather than 500 cause the code is not in the spec (this was actually raise by @Qmando in #4022 but I turned the suggestion down, mistakenly, sorry)
* Add more output to remote-run cli
    * ... because it's way too silent now, and pod don't come online instantaneusly
* Fix wrong kwarg in paasta_cleanup_remote_run_resources
    * This is just linters failing me (and too much mocking in unit tests)
* Drop unneeded pod options for remote-run jobs
    * We do not want this to become a way for people to deploy services in prod, so in general these pods should not have routable IPs (it'll be a different story for toolbox containers as those will run sshd).
    * In my recent testing the job pod was failing to becomes ready due to hacheck. I think that may be due to some zookeeper locking which interfered with the existing replicas of the service, but at any rate, we don't want that, or any other sidecar, as this is effectively meant to be "local-run, but in a pod"

Author: piax93

paasta_tools/api/api_docs/oapi.yaml

@@ -2,7 +2,7 @@
 openapi: 3.0.0
 info:
   title: Paasta API
-  version: 1.1.0
+  version: 1.1.1
 servers:
 - url: "{scheme}://{host}/{basePath}"
   variables:
@@ -1216,6 +1216,8 @@ paths:
               schema:
                 $ref: '#/components/schemas/RemoteRunOutcome'
           description: Successfully started remote-run sandbox
+        "403":
+          description: Unauthorized to remote-run this service
         "404":
           description: Service instance not found
         "500":
@@ -1252,6 +1254,8 @@ paths:
               schema:
                 $ref: '#/components/schemas/RemoteRunOutcome'
           description: Remote run pod stopped
+        "403":
+          description: Unauthorized to remote-run this service
         "404":
           description: Service instance not found
         "500":
@@ -1288,6 +1292,8 @@ paths:
               schema:
                 $ref: '#/components/schemas/RemoteRunOutcome'
           description: Pod status information
+        "403":
+          description: Unauthorized to remote-run this service
         "404":
           description: Service instance not found
         "500":
@@ -1324,6 +1330,8 @@ paths:
               schema:
                 $ref: '#/components/schemas/RemoteRunToken'
           description: Token generated successfully
+        "403":
+          description: Unauthorized to remote-run this service
         "404":
           description: Service instance not found
         "500":

paasta_tools/api/api_docs/swagger.json

@@ -2,7 +2,7 @@
     "swagger": "2.0",
     "info": {
         "title": "Paasta API",
-        "version": "1.1.0"
+        "version": "1.1.1"
     },
     "basePath": "/v1",
     "schemes": [
@@ -856,6 +856,9 @@
                             "$ref": "#/definitions/RemoteRunOutcome"
                         }
                     },
+                    "403": {
+                        "description": "Unauthorized to remote-run this service"
+                    },
                     "404": {
                         "description": "Deployment key not found"
                     },
@@ -904,6 +907,9 @@
                             "$ref": "#/definitions/RemoteRunOutcome"
                         }
                     },
+                    "403": {
+                        "description": "Unauthorized to remote-run this service"
+                    },
                     "404": {
                         "description": "Service instance not found"
                     },
@@ -952,6 +958,9 @@
                             "$ref": "#/definitions/RemoteRunOutcome"
                         }
                     },
+                    "403": {
+                        "description": "Unauthorized to remote-run this service"
+                    },
                     "404": {
                         "description": "Service instance not found"
                     },
@@ -998,6 +1007,9 @@
                             "$ref": "#/definitions/RemoteRunToken"
                         }
                     },
+                    "403": {
+                        "description": "Unauthorized to remote-run this service"
+                    },
                     "404": {
                         "description": "Service instance not found"
                     },

paasta_tools/cli/cmds/remote_run.py

@@ -61,6 +61,9 @@ def paasta_remote_run_start(
         print(f"Error from PaaSTA APIs while starting job: {start_response.message}")
         return 1
 
+    print(
+        f"Triggered remote-run job for {args.service}. Waiting for pod to come online..."
+    )
     start_time = time.time()
     while time.time() - start_time < args.timeout:
         poll_response = client.remote_run.remote_run_poll(
@@ -69,7 +72,9 @@ def paasta_remote_run_start(
             start_response.job_name,
         )
         if poll_response.status == 200:
+            print("")
             break
+        print(f"\rStatus: {poll_response.message}", end="")
         time.sleep(10)
     else:
         print("Timed out while waiting for job to start")
@@ -79,6 +84,8 @@ def paasta_remote_run_start(
         print("Successfully started remote-run job")
         return 0
 
+    print("Pod ready, establishing interactive session...")
+
     token_response = client.remote_run.remote_run_token(
         args.service, args.instance, user
     )

paasta_tools/kubernetes/bin/paasta_cleanup_remote_run_resources.py

@@ -74,7 +74,7 @@ def parse_args() -> argparse.Namespace:
 def main():
     args = parse_args()
     kube_client = KubeClient()
-    age_limit = datetime.now(tzinfo=timezone.utc) - timedelta(seconds=args.max_age)
+    age_limit = datetime.now(tz=timezone.utc) - timedelta(seconds=args.max_age)
     for namespace in get_all_managed_namespaces(kube_client):
         clean_namespace(kube_client, namespace, age_limit)
 

paasta_tools/kubernetes_tools.py

@@ -1436,6 +1436,7 @@ def get_kubernetes_containers(
         aws_ebs_volumes: Sequence[AwsEbsVolume],
         secret_volumes: Sequence[SecretVolume],
         service_namespace_config: ServiceNamespaceConfig,
+        include_sidecars: bool = True,
     ) -> Sequence[V1Container]:
         ports = [self.get_container_port()]
         # MONK-1130
@@ -1471,11 +1472,13 @@ def get_kubernetes_containers(
                 projected_sa_volumes=self.get_projected_sa_volumes(),
             ),
         )
-        containers = [service_container] + self.get_sidecar_containers(  # type: ignore
-            system_paasta_config=system_paasta_config,
-            service_namespace_config=service_namespace_config,
-            hacheck_sidecar_volumes=hacheck_sidecar_volumes,
-        )
+        containers = [service_container]
+        if include_sidecars:
+            containers += self.get_sidecar_containers(  # type: ignore
+                system_paasta_config=system_paasta_config,
+                service_namespace_config=service_namespace_config,
+                hacheck_sidecar_volumes=hacheck_sidecar_volumes,
+            )
         return containers
 
     def get_readiness_probe(
@@ -2052,11 +2055,15 @@ def format_kubernetes_job(
         self,
         job_label: str,
         deadline_seconds: int = 3600,
+        keep_routable_ip: bool = False,
+        include_sidecars: bool = False,
     ) -> V1Job:
         """Create the config for launching the deployment as a Job
 
         :param str job_label: value to set for the "job type" label
         :param int deadline_seconds: maximum allowed duration for the job
+        :param bool keep_routable_ip: maintain routable IP annotation in pod template
+        :param bool include_sidecars: do not discard sidecar containers when building pod spec
         :return: job object
         """
         try:
@@ -2073,6 +2080,8 @@ def format_kubernetes_job(
                         git_sha=git_sha,
                         system_paasta_config=system_paasta_config,
                         restart_on_failure=False,
+                        include_sidecars=include_sidecars,
+                        force_no_routable_ip=not keep_routable_ip,
                     ),
                 ),
             )
@@ -2205,6 +2214,8 @@ def get_pod_template_spec(
         git_sha: str,
         system_paasta_config: SystemPaastaConfig,
         restart_on_failure: bool = True,
+        include_sidecars: bool = True,
+        force_no_routable_ip: bool = False,
     ) -> V1PodTemplateSpec:
         service_namespace_config = load_service_namespace_config(
             service=self.service, namespace=self.get_nerve_namespace()
@@ -2214,8 +2225,10 @@ def get_pod_template_spec(
         )
 
         hacheck_sidecar_volumes = system_paasta_config.get_hacheck_sidecar_volumes()
-        has_routable_ip = self.has_routable_ip(
-            service_namespace_config, system_paasta_config
+        has_routable_ip = (
+            "false"
+            if force_no_routable_ip
+            else self.has_routable_ip(service_namespace_config, system_paasta_config)
         )
         annotations: KubePodAnnotations = {
             "smartstack_registrations": json.dumps(self.get_registrations()),
@@ -2239,6 +2252,7 @@ def get_pod_template_spec(
                 secret_volumes=self.get_secret_volumes(),
                 system_paasta_config=system_paasta_config,
                 service_namespace_config=service_namespace_config,
+                include_sidecars=include_sidecars,
             ),
             share_process_namespace=True,
             node_selector=self.get_node_selector(),

paasta_tools/paastaapi/__init__.py

@@ -7,7 +7,7 @@
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
 
-    The version of the OpenAPI document: 1.1.0
+    The version of the OpenAPI document: 1.1.1
     Generated by: https://openapi-generator.tech
 """
 

paasta_tools/paastaapi/api/autoscaler_api.py

@@ -5,7 +5,7 @@
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
 
-    The version of the OpenAPI document: 1.1.0
+    The version of the OpenAPI document: 1.1.1
     Generated by: https://openapi-generator.tech
 """
 

paasta_tools/paastaapi/api/default_api.py

@@ -5,7 +5,7 @@
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
 
-    The version of the OpenAPI document: 1.1.0
+    The version of the OpenAPI document: 1.1.1
     Generated by: https://openapi-generator.tech
 """
 

paasta_tools/paastaapi/api/remote_run_api.py

@@ -5,7 +5,7 @@
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
 
-    The version of the OpenAPI document: 1.1.0
+    The version of the OpenAPI document: 1.1.1
     Generated by: https://openapi-generator.tech
 """
 

paasta_tools/paastaapi/api/resources_api.py

@@ -5,7 +5,7 @@
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
 
-    The version of the OpenAPI document: 1.1.0
+    The version of the OpenAPI document: 1.1.1
     Generated by: https://openapi-generator.tech
 """