Show logs for all containers

Author: markwinter

backend/apps/common/routes/get.py

@@ -1,12 +1,12 @@
-from flask import request
-
 from kubeflow.kubeflow.crud_backend import api, logging
 
 from .. import utils, versions
 from . import bp
 
 log = logging.getLogger(__name__)
 
+KSERVE_CONTAINER = "kserve-container"
+
 
 @bp.route("/api/namespaces/<namespace>/inferenceservices")
 def get_inference_services(namespace):
@@ -21,39 +21,71 @@ def get_inference_services(namespace):
 def get_inference_service(namespace, name):
     inference_service = api.get_custom_rsrc(**versions.inference_service_gvk(),
                                             namespace=namespace, name=name)
-    if request.args.get("logs", "false") == "true":
-        # find the logs
-        return api.success_response(
-            "serviceLogs", get_inference_service_logs(inference_service)
-        )
 
     return api.success_response("inferenceService", inference_service)
 
 
-def get_inference_service_logs(svc):
-    namespace = svc["metadata"]["namespace"]
-    components = request.args.getlist("component")
+@bp.route("/api/namespaces/<namespace>/inferenceservices/<name>/components/<component>/pods/containers")
+def get_inference_service_containers(namespace: str, name: str, component: str):
+    """Get all containers and init-containers for the latest pod of the given component.
+
+    The kserve-container will always be the first in the list if it exists
+
+    Return:
+        {
+            "containers": ["kserve-container", "container2", ...]
+        }
+    """
+    inference_service = api.get_custom_rsrc(**versions.inference_service_gvk(),
+                                            namespace=namespace, name=name)
+
+    latest_pod = utils.get_component_latest_pod(inference_service, component)
+
+    if latest_pod is None:
+        return api.failed_response(f"couldn't find latest pod for component: {component}", 404)
 
-    log.info(components)
+    containers = []
+    for container in latest_pod.spec.init_containers:
+        containers.append(container.name)
 
-    # dictionary{component: [pod-names]}
-    component_pods_dict = utils.get_inference_service_pods(svc, components)
+    for container in latest_pod.spec.containers:
+        containers.append(container.name)
+
+    # Make kserve-container always the first container in the list if it exists
+    try:
+        idx = containers.index(KSERVE_CONTAINER)
+        containers.insert(0, containers.pop(idx))
+    except ValueError:
+        # kserve-container not found in list
+        pass
+
+    return api.success_response("containers", containers)
+
+
+@bp.route("/api/namespaces/<namespace>/inferenceservices/<name>/components/<component>/pods/containers"
+          "/<container>/logs")
+def get_container_logs(namespace: str, name: str, component: str, container: str):
+    """Get logs for a particular container inside the latest pod of the given component
+
+    Logs are split on newline and returned as an array of lines
+
+    Return:
+        {
+            "logs": ["log\n", "text\n", ...]
+        }
+    """
+    inference_service = api.get_custom_rsrc(**versions.inference_service_gvk(),
+                                            namespace=namespace, name=name)
+    namespace = inference_service["metadata"]["namespace"]
 
-    if len(component_pods_dict.keys()) == 0:
-        return {}
+    latest_pod = utils.get_component_latest_pod(inference_service, component)
+    if latest_pod is None:
+        return api.failed_response(f"couldn't find latest pod for component: {component}", 404)
 
-    resp = {}
-    logging.info("Component pods: %s", component_pods_dict)
-    for component, pods in component_pods_dict.items():
-        if component not in resp:
-            resp[component] = []
+    logs = api.get_pod_logs(namespace, latest_pod.metadata.name, container, auth=False)
+    logs = logs.split("\n")
 
-        for pod in pods:
-            logs = api.get_pod_logs(namespace, pod, "kserve-container",
-                                    auth=False)
-            resp[component].append({"podName": pod,
-                                    "logs": logs.split("\n")})
-    return resp
+    return api.success_response("logs", logs)
 
 
 @bp.route("/api/namespaces/<namespace>/knativeServices/<name>")

backend/apps/common/utils.py

@@ -1,10 +1,12 @@
 import os
+from typing import Dict, Union
 
 from kubeflow.kubeflow.crud_backend import api, helpers, logging
 
 log = logging.getLogger(__name__)
 
 KNATIVE_REVISION_LABEL = "serving.knative.dev/revision"
+LATEST_CREATED_REVISION = "latestCreatedRevision"
 FILE_ABS_PATH = os.path.abspath(os.path.dirname(__file__))
 
 INFERENCESERVICE_TEMPLATE_YAML = os.path.join(
@@ -21,71 +23,54 @@ def load_inference_service_template(**kwargs):
     return helpers.load_param_yaml(INFERENCESERVICE_TEMPLATE_YAML, **kwargs)
 
 
-# helper functions for accessing the logs of an InferenceService
-def get_inference_service_pods(svc, components=[]):
-    """
-    Return a dictionary with (endpoint, component) keys,
-    i.e. ("default", "predictor") and a list of pod names as values
+def get_component_latest_pod(svc: Dict, component: str) -> Union[api.client.V1Pod, None]:
+    """Get pod of the latest Knative revision for the given component.
+
+    Return:
+        Latest pod: k8s V1Pod
     """
     namespace = svc["metadata"]["namespace"]
 
-    # dictionary{revisionName: (endpoint, component)}
-    revisions_dict = get_components_revisions_dict(components, svc)
+    latest_revision = get_component_latest_revision(svc, component)
 
-    if len(revisions_dict.keys()) == 0:
-        return {}
+    if latest_revision is None:
+        return None
 
     pods = api.list_pods(namespace, auth=False).items
-    component_pods_dict = {}
+
     for pod in pods:
-        for revision in revisions_dict:
-            if KNATIVE_REVISION_LABEL not in pod.metadata.labels:
-                continue
+        if KNATIVE_REVISION_LABEL not in pod.metadata.labels:
+            continue
+
+        if pod.metadata.labels[KNATIVE_REVISION_LABEL] != latest_revision:
+            continue
 
-            if pod.metadata.labels[KNATIVE_REVISION_LABEL] != revision:
-                continue
+        return pod
 
-            component = revisions_dict[revision]
-            curr_pod_names = component_pods_dict.get(component, [])
-            curr_pod_names.append(pod.metadata.name)
-            component_pods_dict[component] = curr_pod_names
+    log.info(f"No pods are found for inference service: {svc['metadata']['name']}")
 
-    if len(component_pods_dict.keys()) == 0:
-        log.info("No pods are found for inference service: %s",
-                 svc["metadata"]["name"])
+    return None
 
-    return component_pods_dict
 
+def get_component_latest_revision(svc: Dict, component: str) -> Union[str, None]:
+    """Get the name of the latest created knative revision for the given component.
 
-# FIXME(elikatsis,kimwnasptd): Change the logic of this function according to
-# https://github.com/arrikto/dev/issues/867
-def get_components_revisions_dict(components, svc):
-    """
-    Return a dictionary{revisionId: component}
+    Return:
+        Latest Created Knative Revision: str
     """
     status = svc["status"]
-    revisions_dict = {}
 
-    for component in components:
-        if "components" not in status:
-            log.info("Component '%s' not in inference service '%s'",
-                     component, svc["metadata"]["name"])
-            continue
-
-        if component not in status["components"]:
-            log.info("Component '%s' not in inference service '%s'",
-                     component, svc["metadata"]["name"])
-            continue
+    if "components" not in status:
+        log.info(f"Components field not found in status object of {svc['metadata']['name']}")
+        return None
 
-        if "latestReadyRevision" in status["components"][component]:
-            revision = status["components"][component]["latestReadyRevision"]
+    if component not in status["components"]:
+        log.info(f"Component {component} not found in inference service {svc['metadata']['name']}")
+        return None
 
-            revisions_dict[revision] = component
+    if LATEST_CREATED_REVISION in status["components"][component]:
+        return status["components"][component][LATEST_CREATED_REVISION]
 
-    if len(revisions_dict.keys()) == 0:
-        log.info(
-            "No revisions found for the inference service's components: %s",
-            svc["metadata"]["name"],
-        )
+    log.info(f"No {LATEST_CREATED_REVISION} found for the {component} in {svc['metadata']['name']}")
 
-    return revisions_dict
+    return None

frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.component.html

@@ -1,6 +1,3 @@
-<lib-heading-row [heading]="[heading]" [subHeading]="[subHeading]">
-</lib-heading-row>
-
 <cdk-virtual-scroll-viewport itemSize="18" class="logs-viewer">
   <div *cdkVirtualFor="let entry of logs; let index = index" class="log-entry">
     <span class="number">{{ index }}</span>

frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.component.ts

@@ -2,15 +2,10 @@ import {
   Component,
   Input,
   ViewChild,
-  NgZone,
-  SimpleChanges,
-  OnChanges,
   HostBinding,
-  ElementRef,
   AfterViewInit,
 } from '@angular/core';
 import { CdkVirtualScrollViewport } from '@angular/cdk/scrolling';
-import { take } from 'rxjs/operators';
 
 @Component({
   selector: 'app-logs-viewer',
@@ -22,8 +17,6 @@ export class LogsViewerComponent implements AfterViewInit {
   @ViewChild(CdkVirtualScrollViewport, { static: true })
   viewPort: CdkVirtualScrollViewport;
 
-  @Input() heading = 'Logs';
-  @Input() subHeading = 'tit';
   @Input() height = '400px';
   @Input()
   set logs(newLogs: string[]) {

frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.module.ts

@@ -1,12 +1,18 @@
 import { NgModule } from '@angular/core';
 import { CommonModule } from '@angular/common';
 import { ScrollingModule } from '@angular/cdk/scrolling';
+import { MatTabsModule } from '@angular/material/tabs';
 import { LogsViewerComponent } from './logs-viewer.component';
 import { HeadingSubheadingRowModule } from 'kubeflow';
 
 @NgModule({
   declarations: [LogsViewerComponent],
-  imports: [CommonModule, ScrollingModule, HeadingSubheadingRowModule],
+  imports: [
+    CommonModule,
+    MatTabsModule,
+    ScrollingModule,
+    HeadingSubheadingRowModule,
+  ],
   exports: [LogsViewerComponent],
 })
 export class LogsViewerModule {}

frontend/src/app/pages/server-info/logs/logs.component.html

@@ -1,50 +1,41 @@
-<lib-loading-spinner *ngIf="!logsRequestCompleted"></lib-loading-spinner>
-
-<!--if no logs are present at all then show a warning message-->
-<ng-container *ngIf="logsRequestCompleted && !logsNotEmpty">
+<ng-container>
   <lib-panel class="lib-panel">
-    No logs were found for this InferenceService.
+    Logs are shown for the latest created revision
   </lib-panel>
 </ng-container>
 
-<!--logs loaded successfully from the backend-->
-<ng-container *ngIf="logsRequestCompleted && logsNotEmpty">
-  <ng-container *ngIf="loadErrorMsg">
+<ng-container>
+  <mat-tab-group
+    #componentTabGroup
+    (selectedTabChange)="componentTabChange(componentTabGroup.selectedIndex)"
+    class="page-placement"
+    dynamicHeight
+    animationDuration="0ms"
+  >
+    <mat-tab *ngFor="let component of components" label="{{ component | titlecase }}">
+      <ng-template matTabContent>
+        <mat-tab-group
+          #containerTabGroup
+          (selectedTabChange)="containerTabChange(containerTabGroup.selectedIndex)"
+          class="page-placement"
+          dynamicHeight
+          animationDuration="0ms"
+        >
+          <mat-tab *ngFor="let container of isvcComponents[component].containers" label="{{ container }}"></mat-tab>
+        </mat-tab-group>
+      </ng-template>
+    </mat-tab>
+  </mat-tab-group>
+
+  <lib-loading-spinner *ngIf="!logsRequestCompleted"></lib-loading-spinner>
+
+  <ng-container *ngIf="loadErrorMsg.length">
     <lib-panel>
       {{ loadErrorMsg }}
     </lib-panel>
   </ng-container>
 
-  <div
-    *ngFor="let podLogs of currLogs?.predictor; trackBy: logsTrackFn"
-    class="margin-bottom"
-  >
-    <app-logs-viewer
-      heading="Predictor:"
-      [subHeading]="podLogs.podName"
-      [logs]="podLogs.logs"
-    ></app-logs-viewer>
-  </div>
-
-  <div
-    *ngFor="let podLogs of currLogs?.transformer; trackBy: logsTrackFn"
-    class="margin-bottom"
-  >
-    <app-logs-viewer
-      heading="Transformer:"
-      [subHeading]="podLogs.podName"
-      [logs]="podLogs.logs"
-    ></app-logs-viewer>
-  </div>
-
-  <div
-    *ngFor="let podLogs of currLogs?.explainer; trackBy: logsTrackFn"
-    class="margin-bottom"
-  >
-    <app-logs-viewer
-      heading="Explainer:"
-      [subHeading]="podLogs.podName"
-      [logs]="podLogs.logs"
-    ></app-logs-viewer>
-  </div>
+  <app-logs-viewer *ngIf="currLogs.length"
+    [logs]="currLogs"
+  ></app-logs-viewer>
 </ng-container>