From 5c78e31b5dcb4ccb1263d01cf3805b8970791202 Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 20 Dec 2023 14:51:01 +0700 Subject: [PATCH 01/18] initial impl --- .buildkite/ftr_configs.yml | 1 + .../performance/journeys/infra_hosts_table.ts | 53 +++++++++++++++++++ .../metrics/hosts/hooks/use_hosts_view.ts | 24 ++++++--- .../services/telemetry/telemetry_client.ts | 16 ++++++ .../infra/public/services/telemetry/types.ts | 11 ++++ 5 files changed, 99 insertions(+), 6 deletions(-) create mode 100644 x-pack/performance/journeys/infra_hosts_table.ts diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index 62ca848819d1c..ef5bc1cb368f1 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -456,6 +456,7 @@ enabled: - x-pack/performance/journeys/tags_listing_page.ts - x-pack/performance/journeys/cloud_security_dashboard.ts - x-pack/performance/journeys/apm_service_inventory.ts + - x-pack/performance/journeys/infra_hosts_table.ts - x-pack/test/custom_branding/config.ts - x-pack/test/profiling_api_integration/cloud/config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts diff --git a/x-pack/performance/journeys/infra_hosts_table.ts b/x-pack/performance/journeys/infra_hosts_table.ts new file mode 100644 index 0000000000000..23e1322b62b78 --- /dev/null +++ b/x-pack/performance/journeys/infra_hosts_table.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Journey } from '@kbn/journeys'; +import { createLogger, InfraSynthtraceEsClient, LogLevel } from '@kbn/apm-synthtrace'; +import { infra, timerange } from '@kbn/apm-synthtrace-client'; + +export const journey = new Journey({ + beforeSteps: async ({ kbnUrl, log, auth, es }) => { + const synthClient = new InfraSynthtraceEsClient({ + logger: createLogger(LogLevel.info), + client: es, + refreshAfterIndex: true, + }); + + const start = Date.now() - 1000 * 60 * 10; + await synthClient.index( + generateHostsData({ + from: new Date(start).toISOString(), + to: new Date().toISOString(), + count: 500, + }) + ); + }, +}).step('Navigate to Service Inventory Page', async ({ page, kbnUrl }) => { + await page.goto(kbnUrl.get(`app/metrics/hosts`)); + await page.waitForSelector(`[data-test-subj="hostsView-table"]`); +}); + +export function generateHostsData({ + from, + to, + count = 1, +}: { + from: string; + to: string; + count: number; +}) { + const range = timerange(from, to); + + const hosts = Array(count) + .fill(0) + .map((_, idx) => infra.host(`my-host-${idx}`)); + + return range + .interval('1m') + .rate(1) + .generator((timestamp, index) => hosts.map((host) => host.metrics().timestamp(timestamp))); +} diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/hooks/use_hosts_view.ts b/x-pack/plugins/infra/public/pages/metrics/hosts/hooks/use_hosts_view.ts index e56faa0cc2c1b..751f5afb6544d 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/hooks/use_hosts_view.ts +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/hooks/use_hosts_view.ts @@ -41,7 +41,7 @@ const BASE_INFRA_METRICS_PATH = '/api/metrics/infra'; export const useHostsView = () => { const { sourceId } = useSourceContext(); const { - services: { http, data }, + services: { http, data, telemetry }, } = useKibanaContextForPlugin(); const { buildQuery, parsedDateRange, searchCriteria } = useUnifiedSearchContext(); const abortCtrlRef = useRef(new AbortController()); @@ -59,14 +59,26 @@ export const useHostsView = () => { ); const [state, refetch] = useAsyncFn( - () => { + async () => { abortCtrlRef.current.abort(); abortCtrlRef.current = new AbortController(); - return http.post(`${BASE_INFRA_METRICS_PATH}`, { - signal: abortCtrlRef.current.signal, - body: JSON.stringify(baseRequest), - }); + const start = performance.now(); + const metricsResponse = await http.post( + `${BASE_INFRA_METRICS_PATH}`, + { + signal: abortCtrlRef.current.signal, + body: JSON.stringify(baseRequest), + } + ); + const duration = performance.now() - start; + telemetry.reportPerformanceMetricEvent( + 'infra_hosts_table_load', + duration, + { key1: 'data_load', value1: duration }, + { limit: searchCriteria.limit } + ); + return metricsResponse; }, [baseRequest, http], { loading: true } diff --git a/x-pack/plugins/infra/public/services/telemetry/telemetry_client.ts b/x-pack/plugins/infra/public/services/telemetry/telemetry_client.ts index d4acc0a8bd96d..2c8cac426635d 100644 --- a/x-pack/plugins/infra/public/services/telemetry/telemetry_client.ts +++ b/x-pack/plugins/infra/public/services/telemetry/telemetry_client.ts @@ -6,6 +6,7 @@ */ import type { AnalyticsServiceSetup } from '@kbn/core-analytics-server'; +import { reportPerformanceMetricEvent } from '@kbn/ebt-tools'; import { AssetDetailsFlyoutViewedParams, AssetDetailsPageViewedParams, @@ -15,6 +16,7 @@ import { HostsViewQuerySubmittedParams, InfraTelemetryEventTypes, ITelemetryClient, + PerformanceMetricInnerEvents, } from './types'; /** @@ -70,4 +72,18 @@ export class TelemetryClient implements ITelemetryClient { public reportAssetDetailsPageViewed = (params: AssetDetailsPageViewedParams) => { this.analytics.reportEvent(InfraTelemetryEventTypes.ASSET_DETAILS_PAGE_VIEWED, params); }; + + public reportPerformanceMetricEvent = ( + eventName: string, + duration: number, + innerEvents: PerformanceMetricInnerEvents = {}, + meta: Record = {} + ) => { + reportPerformanceMetricEvent(this.analytics, { + eventName, + duration, + meta, + ...innerEvents, + }); + }; } diff --git a/x-pack/plugins/infra/public/services/telemetry/types.ts b/x-pack/plugins/infra/public/services/telemetry/types.ts index 769cc303def50..0556b20af0fb4 100644 --- a/x-pack/plugins/infra/public/services/telemetry/types.ts +++ b/x-pack/plugins/infra/public/services/telemetry/types.ts @@ -61,6 +61,11 @@ export type InfraTelemetryEventParams = | HostsViewQueryHostsCountRetrievedParams | AssetDetailsFlyoutViewedParams; +export interface PerformanceMetricInnerEvents { + key1?: string; + value1?: number; +} + export interface ITelemetryClient { reportHostEntryClicked(params: HostEntryClickedParams): void; reportHostFlyoutFilterRemoved(params: HostFlyoutFilterActionParams): void; @@ -69,6 +74,12 @@ export interface ITelemetryClient { reportHostsViewQuerySubmitted(params: HostsViewQuerySubmittedParams): void; reportAssetDetailsFlyoutViewed(params: AssetDetailsFlyoutViewedParams): void; reportAssetDetailsPageViewed(params: AssetDetailsPageViewedParams): void; + reportPerformanceMetricEvent( + eventName: string, + duration: number, + innerEvents: PerformanceMetricInnerEvents, + meta: Record + ): void; } export type InfraTelemetryEvent = From 65326dcaab7508a694b253b3b861a3e3c9233030 Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 21 Dec 2023 14:29:04 +0700 Subject: [PATCH 02/18] add host.name to synthtrace host documents --- packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts index 8da22d5b140d7..f74fb27b94596 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts @@ -14,6 +14,7 @@ import { pod } from './pod'; interface HostDocument extends Fields { 'host.hostname': string; + 'host.name': string; } class Host extends Entity { @@ -47,5 +48,6 @@ class HostMetrics extends Serializable {} export function host(name: string): Host { return new Host({ 'host.hostname': name, + 'host.name': name, }); } From d7c0a84fdd5a5460b2f5f197b5630a2dac18fee2 Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 21 Dec 2023 17:07:00 +0700 Subject: [PATCH 03/18] add more system metricsets to synthtrace --- .../src/lib/infra/container.ts | 1 + .../src/lib/infra/host.ts | 68 ++++++++++++++++++- .../src/lib/infra/pod.ts | 1 + .../lib/infra/infra_synthtrace_es_client.ts | 12 +++- .../performance/journeys/infra_hosts_table.ts | 12 +++- 5 files changed, 90 insertions(+), 4 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/infra/container.ts b/packages/kbn-apm-synthtrace-client/src/lib/infra/container.ts index 2df8aa2d71ea3..2f1cb1b7c27e9 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/infra/container.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/infra/container.ts @@ -15,6 +15,7 @@ interface ContainerDocument extends Fields { 'container.id': string; 'kubernetes.pod.uid': string; 'kubernetes.node.name': string; + 'metricset.name'?: string; } export class Container extends Entity { diff --git a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts index f74fb27b94596..1f40673fd84e0 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts @@ -15,9 +15,62 @@ import { pod } from './pod'; interface HostDocument extends Fields { 'host.hostname': string; 'host.name': string; + 'metricset.name'?: string; } class Host extends Entity { + cpu() { + return new HostMetrics({ + ...this.fields, + 'system.cpu.total.norm.pct': 0.094, + 'system.cpu.user.pct': 0.805, + 'system.cpu.system.pct': 0.704, + 'system.cpu.cores': 16, + 'metricset.period': 10000, + 'metricset.name': 'cpu', + }); + } + + memory() { + return new HostMetrics({ + ...this.fields, + 'system.memory.actual.used.pct': 0.351, + 'system.memory.total': 68719476736, + 'system.memory.actual.used.bytes': 24141996032, + 'metricset.period': 10000, + 'metricset.name': 'cpu', + }); + } + + network() { + return new HostMetrics({ + ...this.fields, + 'host.network.ingress.bytes': 2871285, + 'host.network.egress.bytes': 2904987, + 'metricset.period': 10000, + 'metricset.name': 'network', + }); + } + + load() { + return new HostMetrics({ + ...this.fields, + 'system.load.1': 3, + 'system.load.cores': 16, + 'metricset.period': 10000, + 'metricset.name': 'load', + }); + } + + filesystem() { + return new HostMetrics({ + ...this.fields, + 'system.filesystem.used.pct': 12.23, + 'metricset.period': 10000, + 'metricset.name': 'filesystem', + }); + } + metrics() { return new HostMetrics({ ...this.fields, @@ -40,7 +93,20 @@ class Host extends Entity { } export interface HostMetricsDocument extends HostDocument { - 'system.cpu.total.norm.pct': number; + 'metricset.period'?: number; + 'metricset.name'?: string; + 'system.cpu.total.norm.pct'?: number; + 'system.cpu.user.pct'?: number; + 'system.cpu.system.pct'?: number; + 'system.cpu.cores'?: number; + 'system.filesystem.used.pct'?: number; + 'system.memory.actual.used.pct'?: number; + 'system.memory.total'?: number; + 'system.memory.actual.used.bytes'?: number; + 'system.load.1'?: number; + 'system.load.cores'?: number; + 'host.network.ingress.bytes'?: number; + 'host.network.egress.bytes'?: number; } class HostMetrics extends Serializable {} diff --git a/packages/kbn-apm-synthtrace-client/src/lib/infra/pod.ts b/packages/kbn-apm-synthtrace-client/src/lib/infra/pod.ts index 4876fd7291f53..618db5a69b77a 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/infra/pod.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/infra/pod.ts @@ -15,6 +15,7 @@ import { container } from './container'; interface PodDocument extends Fields { 'kubernetes.pod.uid': string; 'kubernetes.node.name': string; + 'metricset.name'?: string; } export class Pod extends Entity { diff --git a/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_es_client.ts index 897d813ae6718..30e28d7b400c0 100644 --- a/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_es_client.ts @@ -46,8 +46,18 @@ function getRoutingTransform() { return new Transform({ objectMode: true, transform(document: ESDocumentWithOperation, encoding, callback) { - if ('host.hostname' in document) { + const metricset = document['metricset.name']; + + if (metricset === 'cpu') { document._index = 'metrics-system.cpu-default'; + } else if (metricset === 'memory') { + document._index = 'metrics-system.memory-default'; + } else if (metricset === 'network') { + document._index = 'metrics-system.network-default'; + } else if (metricset === 'load') { + document._index = 'metrics-system.load-default'; + } else if (metricset === 'filesystem') { + document._index = 'metrics-system.filesystem-default'; } else if ('container.id' in document) { document._index = 'metrics-kubernetes.container-default'; } else if ('kubernetes.pod.uid' in document) { diff --git a/x-pack/performance/journeys/infra_hosts_table.ts b/x-pack/performance/journeys/infra_hosts_table.ts index 23e1322b62b78..6e5903de938e3 100644 --- a/x-pack/performance/journeys/infra_hosts_table.ts +++ b/x-pack/performance/journeys/infra_hosts_table.ts @@ -28,7 +28,7 @@ export const journey = new Journey({ }, }).step('Navigate to Service Inventory Page', async ({ page, kbnUrl }) => { await page.goto(kbnUrl.get(`app/metrics/hosts`)); - await page.waitForSelector(`[data-test-subj="hostsView-table"]`); + await page.waitForSelector(`[data-test-subj="hostsView-tableRow"]`); }); export function generateHostsData({ @@ -49,5 +49,13 @@ export function generateHostsData({ return range .interval('1m') .rate(1) - .generator((timestamp, index) => hosts.map((host) => host.metrics().timestamp(timestamp))); + .generator((timestamp, index) => + hosts.flatMap((host) => [ + host.cpu().timestamp(timestamp), + host.memory().timestamp(timestamp), + host.network().timestamp(timestamp), + host.load().timestamp(timestamp), + host.filesystem().timestamp(timestamp), + ]) + ); } From d6be8065135d7ccec7374bac0a2bf0fd65b348c6 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 22 Dec 2023 12:18:49 +0700 Subject: [PATCH 04/18] load 50/500 hosts --- .../performance/journeys/infra_hosts_table.ts | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/x-pack/performance/journeys/infra_hosts_table.ts b/x-pack/performance/journeys/infra_hosts_table.ts index 6e5903de938e3..35993d52d05fb 100644 --- a/x-pack/performance/journeys/infra_hosts_table.ts +++ b/x-pack/performance/journeys/infra_hosts_table.ts @@ -22,14 +22,23 @@ export const journey = new Journey({ generateHostsData({ from: new Date(start).toISOString(), to: new Date().toISOString(), - count: 500, + count: 1000, }) ); }, -}).step('Navigate to Service Inventory Page', async ({ page, kbnUrl }) => { - await page.goto(kbnUrl.get(`app/metrics/hosts`)); - await page.waitForSelector(`[data-test-subj="hostsView-tableRow"]`); -}); +}) + .step('Navigate to Hosts view and load 100 hosts (default)', async ({ page, kbnUrl }) => { + await page.goto(kbnUrl.get(`app/metrics/hosts`)); + await page.waitForSelector(`[data-test-subj="hostsView-tableRow"]`); + }) + .step('Load 500 hosts', async ({ page, kbnUrl }) => { + await page.click('[data-test-subj="hostsViewLimitSelection500Button"]'); + await page.waitForSelector(`[data-test-subj="hostsView-tableRow"]`); + }) + .step('Load 50 hosts', async ({ page, kbnUrl }) => { + await page.click('[data-test-subj="hostsViewLimitSelection50Button"]'); + await page.waitForSelector(`[data-test-subj="hostsView-tableRow"]`); + }); export function generateHostsData({ from, @@ -47,8 +56,8 @@ export function generateHostsData({ .map((_, idx) => infra.host(`my-host-${idx}`)); return range - .interval('1m') - .rate(1) + .interval('30s') + .rate(5) .generator((timestamp, index) => hosts.flatMap((host) => [ host.cpu().timestamp(timestamp), From b40969ec1e697401f4518dc74046e9318a8b8eea Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 22 Dec 2023 18:19:36 +0700 Subject: [PATCH 05/18] use querystring to set host limit --- .../performance/journeys/infra_hosts_table.ts | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/x-pack/performance/journeys/infra_hosts_table.ts b/x-pack/performance/journeys/infra_hosts_table.ts index 35993d52d05fb..1e17a0d732618 100644 --- a/x-pack/performance/journeys/infra_hosts_table.ts +++ b/x-pack/performance/journeys/infra_hosts_table.ts @@ -27,17 +27,29 @@ export const journey = new Journey({ ); }, }) - .step('Navigate to Hosts view and load 100 hosts (default)', async ({ page, kbnUrl }) => { - await page.goto(kbnUrl.get(`app/metrics/hosts`)); - await page.waitForSelector(`[data-test-subj="hostsView-tableRow"]`); + .step('Navigate to Hosts view and load 50 hosts', async ({ page, kbnUrl }) => { + await page.goto( + kbnUrl.get( + `app/metrics/hosts?_a=(dateRange:(from:now-15m,to:now),filters:!(),limit:50,panelFilters:!(),query:(language:kuery,query:''))` + ) + ); + await page.waitForSelector('[data-test-subj="hostsView-tableRow"]'); }) - .step('Load 500 hosts', async ({ page, kbnUrl }) => { - await page.click('[data-test-subj="hostsViewLimitSelection500Button"]'); - await page.waitForSelector(`[data-test-subj="hostsView-tableRow"]`); + .step('Navigate to Hosts view and load 100 hosts', async ({ page, kbnUrl }) => { + await page.goto( + kbnUrl.get( + `app/metrics/hosts?_a=(dateRange:(from:now-15m,to:now),filters:!(),limit:100,panelFilters:!(),query:(language:kuery,query:''))` + ) + ); + await page.waitForSelector('[data-test-subj="hostsView-tableRow"]'); }) - .step('Load 50 hosts', async ({ page, kbnUrl }) => { - await page.click('[data-test-subj="hostsViewLimitSelection50Button"]'); - await page.waitForSelector(`[data-test-subj="hostsView-tableRow"]`); + .step('Navigate to Hosts view and load 500 hosts', async ({ page, kbnUrl }) => { + await page.goto( + kbnUrl.get( + `app/metrics/hosts?_a=(dateRange:(from:now-15m,to:now),filters:!(),limit:500,panelFilters:!(),query:(language:kuery,query:''))` + ) + ); + await page.waitForSelector('[data-test-subj="hostsView-tableRow"]'); }); export function generateHostsData({ From 866b03384308518c25531eebd346266dc20396fd Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 26 Dec 2023 17:48:35 +0700 Subject: [PATCH 06/18] infra kibana client to install system package --- packages/kbn-apm-synthtrace/index.ts | 1 + .../infra/infra_synthtrace_kibana_client.ts | 78 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts diff --git a/packages/kbn-apm-synthtrace/index.ts b/packages/kbn-apm-synthtrace/index.ts index 880b97cac2a8f..378d7f610539d 100644 --- a/packages/kbn-apm-synthtrace/index.ts +++ b/packages/kbn-apm-synthtrace/index.ts @@ -12,6 +12,7 @@ export { ApmSynthtraceEsClient } from './src/lib/apm/client/apm_synthtrace_es_cl export { ApmSynthtraceKibanaClient } from './src/lib/apm/client/apm_synthtrace_kibana_client'; export { InfraSynthtraceEsClient } from './src/lib/infra/infra_synthtrace_es_client'; +export { InfraSynthtraceKibanaClient } from './src/lib/infra/infra_synthtrace_kibana_client'; export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_client'; diff --git a/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts b/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts new file mode 100644 index 0000000000000..7cc4dee1c2c09 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { join } from 'path'; +import fetch from 'node-fetch'; +import pRetry from 'p-retry'; +import { Logger } from '../utils/create_logger'; + +export class InfraSynthtraceKibanaClient { + private readonly logger: Logger; + private target: string; + + constructor(options: { logger: Logger; target: string; username: string; password: string }) { + this.logger = options.logger; + const url = new URL(options.target); + url.username = options.username; + url.password = options.password; + this.target = url.toString(); + } + + async fetchLatestSystemPackageVersion() { + const fleetPackageApiUrl = join(this.target, '/api/fleet/epm/packages/system?prerelease=true'); + this.logger.debug(`Fetching latest System package version from ${fleetPackageApiUrl}`); + const response = await fetch(fleetPackageApiUrl, { + method: 'GET', + headers: kibanaHeaders(), + }); + + const responseJson = await response.json(); + + if (response.status !== 200) { + throw new Error( + `Failed to fetch latest System package version, received HTTP ${response.status} and message: ${responseJson.message}` + ); + } + + const { latestVersion } = responseJson.item; + + return latestVersion as string; + } + + async installSystemPackage(packageVersion: string) { + this.logger.debug(`Installing System package ${packageVersion}`); + + const url = join(this.target, `/api/fleet/epm/packages/system/${packageVersion}`); + const response = await pRetry(() => { + return fetch(url, { + method: 'POST', + headers: kibanaHeaders(), + body: '{"force":true}', + }); + }); + + const responseJson = await response.json(); + + if (!responseJson.items) { + throw new Error( + `Failed to install System package version ${packageVersion}, received HTTP ${response.status} and message: ${responseJson.message} for url ${url}` + ); + } + + this.logger.info(`Installed System package ${packageVersion}`); + } +} + +function kibanaHeaders() { + return { + Accept: 'application/json', + 'Content-Type': 'application/json', + 'kbn-xsrf': 'kibana', + 'elastic-api-version': '2023-10-31', + }; +} From f173687a8801748b96765275ad75d8e23309f1dd Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 26 Dec 2023 17:50:32 +0700 Subject: [PATCH 07/18] add diskio metricset + dimension field --- .../src/lib/infra/host.ts | 38 ++++++++++++++++--- .../lib/infra/infra_synthtrace_es_client.ts | 2 + 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts index 1f40673fd84e0..67187a43d491b 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts @@ -13,6 +13,7 @@ import { Serializable } from '../serializable'; import { pod } from './pod'; interface HostDocument extends Fields { + 'agent.id': string; 'host.hostname': string; 'host.name': string; 'metricset.name'?: string; @@ -34,9 +35,12 @@ class Host extends Entity { memory() { return new HostMetrics({ ...this.fields, - 'system.memory.actual.used.pct': 0.351, + 'system.memory.actual.free': 44704067584, + 'system.memory.actual.used.bytes': 24015409152, + 'system.memory.actual.used.pct': 0.35, 'system.memory.total': 68719476736, - 'system.memory.actual.used.bytes': 24141996032, + 'system.memory.used.bytes': 39964708864, + 'system.memory.used.pct': 0.582, 'metricset.period': 10000, 'metricset.name': 'cpu', }); @@ -55,8 +59,10 @@ class Host extends Entity { load() { return new HostMetrics({ ...this.fields, - 'system.load.1': 3, - 'system.load.cores': 16, + 'system.load': { + 1: 3, + cores: 16, + }, 'metricset.period': 10000, 'metricset.name': 'load', }); @@ -71,6 +77,18 @@ class Host extends Entity { }); } + diskio() { + return new HostMetrics({ + ...this.fields, + 'system.diskio.read.count': 3538413, + 'system.diskio.write.count': 4694333, + 'system.diskio.read.bytes': 33147297792, + 'system.diskio.write.bytes': 48595652608, + 'metricset.period': 10000, + 'metricset.name': 'diskio', + }); + } + metrics() { return new HostMetrics({ ...this.fields, @@ -93,18 +111,25 @@ class Host extends Entity { } export interface HostMetricsDocument extends HostDocument { + 'agent.id': string; 'metricset.period'?: number; 'metricset.name'?: string; 'system.cpu.total.norm.pct'?: number; 'system.cpu.user.pct'?: number; 'system.cpu.system.pct'?: number; 'system.cpu.cores'?: number; + 'system.diskio.read.count'?: number; + 'system.diskio.write.count'?: number; + 'system.diskio.read.bytes'?: number; + 'system.diskio.write.bytes'?: number; 'system.filesystem.used.pct'?: number; 'system.memory.actual.used.pct'?: number; 'system.memory.total'?: number; 'system.memory.actual.used.bytes'?: number; - 'system.load.1'?: number; - 'system.load.cores'?: number; + 'system.memory.actual.free'?: number; + 'system.memory.used.bytes'?: number; + 'system.memory.used.pct'?: number; + 'system.load'?: { 1: number; cores: number }; 'host.network.ingress.bytes'?: number; 'host.network.egress.bytes'?: number; } @@ -113,6 +138,7 @@ class HostMetrics extends Serializable {} export function host(name: string): Host { return new Host({ + 'agent.id': 'synthtrace', 'host.hostname': name, 'host.name': name, }); diff --git a/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_es_client.ts index 30e28d7b400c0..031c40c9ccf19 100644 --- a/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_es_client.ts @@ -58,6 +58,8 @@ function getRoutingTransform() { document._index = 'metrics-system.load-default'; } else if (metricset === 'filesystem') { document._index = 'metrics-system.filesystem-default'; + } else if (metricset === 'diskio') { + document._index = 'metrics-system.diskio-default'; } else if ('container.id' in document) { document._index = 'metrics-kubernetes.container-default'; } else if ('kubernetes.pod.uid' in document) { From ecd2ab9b61751ff3e4057bef91bddbffb6e97950 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 26 Dec 2023 17:52:00 +0700 Subject: [PATCH 08/18] only run host view step with 500 hosts --- .../performance/journeys/infra_hosts_table.ts | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/x-pack/performance/journeys/infra_hosts_table.ts b/x-pack/performance/journeys/infra_hosts_table.ts index 1e17a0d732618..1c7c4c43243ad 100644 --- a/x-pack/performance/journeys/infra_hosts_table.ts +++ b/x-pack/performance/journeys/infra_hosts_table.ts @@ -6,19 +6,35 @@ */ import { Journey } from '@kbn/journeys'; -import { createLogger, InfraSynthtraceEsClient, LogLevel } from '@kbn/apm-synthtrace'; +import { + createLogger, + InfraSynthtraceEsClient, + LogLevel, + InfraSynthtraceKibanaClient, +} from '@kbn/apm-synthtrace'; import { infra, timerange } from '@kbn/apm-synthtrace-client'; export const journey = new Journey({ - beforeSteps: async ({ kbnUrl, log, auth, es }) => { - const synthClient = new InfraSynthtraceEsClient({ - logger: createLogger(LogLevel.info), + beforeSteps: async ({ kbnUrl, auth, es }) => { + const logger = createLogger(LogLevel.debug); + const synthKibanaClient = new InfraSynthtraceKibanaClient({ + logger, + target: kbnUrl.get(), + username: auth.getUsername(), + password: auth.getPassword(), + }); + + const pkgVersion = await synthKibanaClient.fetchLatestSystemPackageVersion(); + await synthKibanaClient.installSystemPackage(pkgVersion); + + const synthEsClient = new InfraSynthtraceEsClient({ + logger, client: es, refreshAfterIndex: true, }); const start = Date.now() - 1000 * 60 * 10; - await synthClient.index( + await synthEsClient.index( generateHostsData({ from: new Date(start).toISOString(), to: new Date().toISOString(), @@ -26,31 +42,14 @@ export const journey = new Journey({ }) ); }, -}) - .step('Navigate to Hosts view and load 50 hosts', async ({ page, kbnUrl }) => { - await page.goto( - kbnUrl.get( - `app/metrics/hosts?_a=(dateRange:(from:now-15m,to:now),filters:!(),limit:50,panelFilters:!(),query:(language:kuery,query:''))` - ) - ); - await page.waitForSelector('[data-test-subj="hostsView-tableRow"]'); - }) - .step('Navigate to Hosts view and load 100 hosts', async ({ page, kbnUrl }) => { - await page.goto( - kbnUrl.get( - `app/metrics/hosts?_a=(dateRange:(from:now-15m,to:now),filters:!(),limit:100,panelFilters:!(),query:(language:kuery,query:''))` - ) - ); - await page.waitForSelector('[data-test-subj="hostsView-tableRow"]'); - }) - .step('Navigate to Hosts view and load 500 hosts', async ({ page, kbnUrl }) => { - await page.goto( - kbnUrl.get( - `app/metrics/hosts?_a=(dateRange:(from:now-15m,to:now),filters:!(),limit:500,panelFilters:!(),query:(language:kuery,query:''))` - ) - ); - await page.waitForSelector('[data-test-subj="hostsView-tableRow"]'); - }); +}).step('Navigate to Hosts view and load 500 hosts', async ({ page, kbnUrl }) => { + await page.goto( + kbnUrl.get( + `app/metrics/hosts?_a=(dateRange:(from:now-15m,to:now),filters:!(),limit:500,panelFilters:!(),query:(language:kuery,query:''))` + ) + ); + await page.waitForSelector('[data-test-subj="hostsView-tableRow"]'); +}); export function generateHostsData({ from, @@ -69,14 +68,15 @@ export function generateHostsData({ return range .interval('30s') - .rate(5) + .rate(1) .generator((timestamp, index) => hosts.flatMap((host) => [ host.cpu().timestamp(timestamp), - host.memory().timestamp(timestamp), - host.network().timestamp(timestamp), - host.load().timestamp(timestamp), - host.filesystem().timestamp(timestamp), + host.memory().timestamp(timestamp + 1), + host.network().timestamp(timestamp + 2), + host.load().timestamp(timestamp + 3), + host.filesystem().timestamp(timestamp + 4), + host.diskio().timestamp(timestamp + 5), ]) ); } From 9de1787e898fc10972600f747bc3eb71155b0cb9 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 26 Dec 2023 11:55:31 +0000 Subject: [PATCH 09/18] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/infra/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/tsconfig.json b/x-pack/plugins/infra/tsconfig.json index cbcbdbbbc365f..955a3a18cf2e1 100644 --- a/x-pack/plugins/infra/tsconfig.json +++ b/x-pack/plugins/infra/tsconfig.json @@ -79,7 +79,8 @@ "@kbn/profiling-utils", "@kbn/profiling-data-access-plugin", "@kbn/core-http-request-handler-context-server", - "@kbn/observability-get-padded-alert-time-range-util" + "@kbn/observability-get-padded-alert-time-range-util", + "@kbn/ebt-tools" ], "exclude": ["target/**/*"] } From 399c5b484d9cd5ce65832ead9f9d38225ac7e899 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 26 Dec 2023 19:39:44 +0700 Subject: [PATCH 10/18] add mock fn --- .../infra/public/services/telemetry/telemetry_client.mock.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/infra/public/services/telemetry/telemetry_client.mock.ts b/x-pack/plugins/infra/public/services/telemetry/telemetry_client.mock.ts index 1f354ecd1670f..08a2f9fb9eedb 100644 --- a/x-pack/plugins/infra/public/services/telemetry/telemetry_client.mock.ts +++ b/x-pack/plugins/infra/public/services/telemetry/telemetry_client.mock.ts @@ -15,4 +15,5 @@ export const createTelemetryClientMock = (): jest.Mocked => ({ reportHostsViewTotalHostCountRetrieved: jest.fn(), reportAssetDetailsFlyoutViewed: jest.fn(), reportAssetDetailsPageViewed: jest.fn(), + reportPerformanceMetricEvent: jest.fn(), }); From e3158688577d1d700d2a56dd160d4de19d7b7b01 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 26 Dec 2023 22:24:25 +0700 Subject: [PATCH 11/18] fix tests --- packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts | 7 ------- .../api_integration/apis/asset_manager/tests/helpers.ts | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts index 67187a43d491b..05904d47683dd 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts @@ -89,13 +89,6 @@ class Host extends Entity { }); } - metrics() { - return new HostMetrics({ - ...this.fields, - 'system.cpu.total.norm.pct': 46, - }); - } - asset() { return new HostAsset({ 'asset.kind': 'host', diff --git a/x-pack/test/api_integration/apis/asset_manager/tests/helpers.ts b/x-pack/test/api_integration/apis/asset_manager/tests/helpers.ts index 8983b139d9462..3d1086ca8b8e4 100644 --- a/x-pack/test/api_integration/apis/asset_manager/tests/helpers.ts +++ b/x-pack/test/api_integration/apis/asset_manager/tests/helpers.ts @@ -100,5 +100,5 @@ export function generateHostsData({ return range .interval('1m') .rate(1) - .generator((timestamp, index) => hosts.map((host) => host.metrics().timestamp(timestamp))); + .generator((timestamp, index) => hosts.map((host) => host.cpu().timestamp(timestamp))); } From 066ae0b48e8516892d5cb70a70cd08220bcb0609 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Thu, 28 Dec 2023 12:39:52 +0100 Subject: [PATCH 12/18] update hosts table wait for render, add charts rendering wait in journey --- x-pack/performance/journeys/infra_hosts_table.ts | 8 ++++++-- .../public/pages/metrics/hosts/components/hosts_table.tsx | 2 +- x-pack/test/functional/apps/infra/hosts_view.ts | 2 +- x-pack/test/functional/page_objects/infra_hosts_view.ts | 6 +++--- .../test_suites/observability/infra/hosts_page.ts | 2 +- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/x-pack/performance/journeys/infra_hosts_table.ts b/x-pack/performance/journeys/infra_hosts_table.ts index 1c7c4c43243ad..9e64c1f86285b 100644 --- a/x-pack/performance/journeys/infra_hosts_table.ts +++ b/x-pack/performance/journeys/infra_hosts_table.ts @@ -13,6 +13,7 @@ import { InfraSynthtraceKibanaClient, } from '@kbn/apm-synthtrace'; import { infra, timerange } from '@kbn/apm-synthtrace-client'; +import { subj } from '@kbn/test-subj-selector'; export const journey = new Journey({ beforeSteps: async ({ kbnUrl, auth, es }) => { @@ -42,13 +43,16 @@ export const journey = new Journey({ }) ); }, -}).step('Navigate to Hosts view and load 500 hosts', async ({ page, kbnUrl }) => { +}).step('Navigate to Hosts view and load 500 hosts', async ({ page, kbnUrl, kibanaPage }) => { await page.goto( kbnUrl.get( `app/metrics/hosts?_a=(dateRange:(from:now-15m,to:now),filters:!(),limit:500,panelFilters:!(),query:(language:kuery,query:''))` ) ); - await page.waitForSelector('[data-test-subj="hostsView-tableRow"]'); + // wait for table to be loaded + await page.waitForSelector(subj('hostsView-table-loaded')); + // wait for metric charts to be loaded + await kibanaPage.waitForCharts({ count: 5, timeout: 60000 }); }); export function generateHostsData({ diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx index 2082689cb6e1a..bbb8c44a85f7e 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx @@ -44,7 +44,7 @@ export const HostsTable = () => { /> { await retry.waitFor( 'wait for table and KPI charts to load', async () => - (await pageObjects.infraHostsView.isHostTableLoading()) && + (await pageObjects.infraHostsView.isHostTableLoaded()) && (await pageObjects.infraHostsView.isKPIChartsLoaded()) ); diff --git a/x-pack/test/functional/page_objects/infra_hosts_view.ts b/x-pack/test/functional/page_objects/infra_hosts_view.ts index 3fbb3d361d879..5cf14ce8ab265 100644 --- a/x-pack/test/functional/page_objects/infra_hosts_view.ts +++ b/x-pack/test/functional/page_objects/infra_hosts_view.ts @@ -49,11 +49,11 @@ export function InfraHostsViewProvider({ getService }: FtrProviderContext) { // Table async getHostsTable() { - return testSubjects.find('hostsView-table'); + return testSubjects.find('hostsView-table-loaded'); }, - async isHostTableLoading() { - return !(await testSubjects.exists('tbody[class*=euiBasicTableBodyLoading]')); + async isHostTableLoaded() { + return !(await testSubjects.exists('hostsView-table-loading')); }, async getHostsTableData() { diff --git a/x-pack/test_serverless/functional/test_suites/observability/infra/hosts_page.ts b/x-pack/test_serverless/functional/test_suites/observability/infra/hosts_page.ts index 7e23b43b253c7..44d8580822dbc 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/infra/hosts_page.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/infra/hosts_page.ts @@ -36,7 +36,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { await retry.waitFor( 'wait for table and KPI charts to load', async () => - (await pageObjects.infraHostsView.isHostTableLoading()) && + (await pageObjects.infraHostsView.isHostTableLoaded()) && (await pageObjects.infraHostsView.isKPIChartsLoaded()) ); From 7ed68173b6b764c078c8686eb94fadf3fc60d42c Mon Sep 17 00:00:00 2001 From: klacabane Date: Mon, 8 Jan 2024 14:53:55 +0100 Subject: [PATCH 13/18] use correct dataset --- packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts index 05904d47683dd..db2fb3dd3d735 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts @@ -42,7 +42,7 @@ class Host extends Entity { 'system.memory.used.bytes': 39964708864, 'system.memory.used.pct': 0.582, 'metricset.period': 10000, - 'metricset.name': 'cpu', + 'metricset.name': 'memory', }); } From 2344594cfd9c5f34097753245aec4869c67db8f0 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 9 Jan 2024 14:07:32 +0100 Subject: [PATCH 14/18] track total render time --- .../hosts/components/hosts_content.tsx | 32 ++++++++++++++++--- .../metrics/hosts/components/hosts_table.tsx | 16 ++++++++-- .../metrics/hosts/hooks/use_hosts_view.ts | 13 +++----- .../infra/public/services/telemetry/types.ts | 4 +++ 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_content.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_content.tsx index 5496dcf64cd50..3bf9235440151 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_content.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_content.tsx @@ -13,12 +13,38 @@ import { Tabs } from './tabs/tabs'; import { AlertsQueryProvider } from '../hooks/use_alerts_query'; import { HostsViewProvider } from '../hooks/use_hosts_view'; import { HostsTableProvider } from '../hooks/use_hosts_table'; +import { useKibanaContextForPlugin } from '../../../../hooks/use_kibana'; import { ErrorCallout } from './error_callout'; import { useUnifiedSearchContext } from '../hooks/use_unified_search'; import { HostCountProvider } from '../hooks/use_host_count'; export const HostsContent = () => { - const { error } = useUnifiedSearchContext(); + const { error, searchCriteria } = useUnifiedSearchContext(); + const { + services: { telemetry }, + } = useKibanaContextForPlugin(); + + const renderHostsTable = () => { + const startTime = performance.now(); + return ( + { + const totalDuration = performance.now() - startTime; + telemetry.reportPerformanceMetricEvent( + 'infra_hosts_table_load', + totalDuration, + { + key1: 'data_load', + value1: dataLoadDuration, + key2: 'render_time', + value2: totalDuration - (dataLoadDuration ?? 0), + }, + { limit: searchCriteria.limit } + ); + }} + /> + ); + }; return ( <> @@ -33,9 +59,7 @@ export const HostsContent = () => { - - - + {renderHostsTable()} diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx index bbb8c44a85f7e..89847df987f7a 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React from 'react'; +import React, { useEffect } from 'react'; import { EuiBasicTable } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { EuiEmptyPrompt } from '@elastic/eui'; @@ -17,8 +17,12 @@ import { FilterAction } from './table/filter_action'; const PAGE_SIZE_OPTIONS = [5, 10, 20]; -export const HostsTable = () => { - const { loading } = useHostsViewContext(); +interface HostsTableOptions { + onRender?: ({ dataLoadDuration }: { dataLoadDuration?: number }) => void; +} + +export const HostsTable = ({ onRender }: HostsTableOptions) => { + const { loading, duration } = useHostsViewContext(); const { columns, @@ -36,6 +40,12 @@ export const HostsTable = () => { refs, } = useHostsTableContext(); + useEffect(() => { + if (!loading) { + onRender?.({ dataLoadDuration: duration }); + } + }, [loading, onRender, duration]); + return ( <> { const { sourceId } = useSourceContext(); const { - services: { http, data, telemetry }, + services: { http, data }, } = useKibanaContextForPlugin(); const { buildQuery, parsedDateRange, searchCriteria } = useUnifiedSearchContext(); const abortCtrlRef = useRef(new AbortController()); @@ -72,13 +72,7 @@ export const useHostsView = () => { } ); const duration = performance.now() - start; - telemetry.reportPerformanceMetricEvent( - 'infra_hosts_table_load', - duration, - { key1: 'data_load', value1: duration }, - { limit: searchCriteria.limit } - ); - return metricsResponse; + return { response: metricsResponse, duration }; }, [baseRequest, http], { loading: true } @@ -94,8 +88,9 @@ export const useHostsView = () => { return { loading, error, - hostNodes: value?.nodes ?? [], + hostNodes: value?.response?.nodes ?? [], searchSessionId, + duration: value?.duration, }; }; diff --git a/x-pack/plugins/infra/public/services/telemetry/types.ts b/x-pack/plugins/infra/public/services/telemetry/types.ts index 0556b20af0fb4..04a6c919b07d4 100644 --- a/x-pack/plugins/infra/public/services/telemetry/types.ts +++ b/x-pack/plugins/infra/public/services/telemetry/types.ts @@ -64,6 +64,10 @@ export type InfraTelemetryEventParams = export interface PerformanceMetricInnerEvents { key1?: string; value1?: number; + key2?: string; + value2?: number; + key3?: string; + value3?: number; } export interface ITelemetryClient { From 954b1e90f480f7c6ce213bd6e7b2aab1a7ef75ff Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 9 Jan 2024 17:21:13 +0100 Subject: [PATCH 15/18] Revert "track total render time" This reverts commit 2344594cfd9c5f34097753245aec4869c67db8f0. --- .../hosts/components/hosts_content.tsx | 32 +++---------------- .../metrics/hosts/components/hosts_table.tsx | 16 ++-------- .../metrics/hosts/hooks/use_hosts_view.ts | 13 +++++--- .../infra/public/services/telemetry/types.ts | 4 --- 4 files changed, 16 insertions(+), 49 deletions(-) diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_content.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_content.tsx index 3bf9235440151..5496dcf64cd50 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_content.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_content.tsx @@ -13,38 +13,12 @@ import { Tabs } from './tabs/tabs'; import { AlertsQueryProvider } from '../hooks/use_alerts_query'; import { HostsViewProvider } from '../hooks/use_hosts_view'; import { HostsTableProvider } from '../hooks/use_hosts_table'; -import { useKibanaContextForPlugin } from '../../../../hooks/use_kibana'; import { ErrorCallout } from './error_callout'; import { useUnifiedSearchContext } from '../hooks/use_unified_search'; import { HostCountProvider } from '../hooks/use_host_count'; export const HostsContent = () => { - const { error, searchCriteria } = useUnifiedSearchContext(); - const { - services: { telemetry }, - } = useKibanaContextForPlugin(); - - const renderHostsTable = () => { - const startTime = performance.now(); - return ( - { - const totalDuration = performance.now() - startTime; - telemetry.reportPerformanceMetricEvent( - 'infra_hosts_table_load', - totalDuration, - { - key1: 'data_load', - value1: dataLoadDuration, - key2: 'render_time', - value2: totalDuration - (dataLoadDuration ?? 0), - }, - { limit: searchCriteria.limit } - ); - }} - /> - ); - }; + const { error } = useUnifiedSearchContext(); return ( <> @@ -59,7 +33,9 @@ export const HostsContent = () => { - {renderHostsTable()} + + + diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx index 89847df987f7a..bbb8c44a85f7e 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useEffect } from 'react'; +import React from 'react'; import { EuiBasicTable } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { EuiEmptyPrompt } from '@elastic/eui'; @@ -17,12 +17,8 @@ import { FilterAction } from './table/filter_action'; const PAGE_SIZE_OPTIONS = [5, 10, 20]; -interface HostsTableOptions { - onRender?: ({ dataLoadDuration }: { dataLoadDuration?: number }) => void; -} - -export const HostsTable = ({ onRender }: HostsTableOptions) => { - const { loading, duration } = useHostsViewContext(); +export const HostsTable = () => { + const { loading } = useHostsViewContext(); const { columns, @@ -40,12 +36,6 @@ export const HostsTable = ({ onRender }: HostsTableOptions) => { refs, } = useHostsTableContext(); - useEffect(() => { - if (!loading) { - onRender?.({ dataLoadDuration: duration }); - } - }, [loading, onRender, duration]); - return ( <> { const { sourceId } = useSourceContext(); const { - services: { http, data }, + services: { http, data, telemetry }, } = useKibanaContextForPlugin(); const { buildQuery, parsedDateRange, searchCriteria } = useUnifiedSearchContext(); const abortCtrlRef = useRef(new AbortController()); @@ -72,7 +72,13 @@ export const useHostsView = () => { } ); const duration = performance.now() - start; - return { response: metricsResponse, duration }; + telemetry.reportPerformanceMetricEvent( + 'infra_hosts_table_load', + duration, + { key1: 'data_load', value1: duration }, + { limit: searchCriteria.limit } + ); + return metricsResponse; }, [baseRequest, http], { loading: true } @@ -88,9 +94,8 @@ export const useHostsView = () => { return { loading, error, - hostNodes: value?.response?.nodes ?? [], + hostNodes: value?.nodes ?? [], searchSessionId, - duration: value?.duration, }; }; diff --git a/x-pack/plugins/infra/public/services/telemetry/types.ts b/x-pack/plugins/infra/public/services/telemetry/types.ts index 04a6c919b07d4..0556b20af0fb4 100644 --- a/x-pack/plugins/infra/public/services/telemetry/types.ts +++ b/x-pack/plugins/infra/public/services/telemetry/types.ts @@ -64,10 +64,6 @@ export type InfraTelemetryEventParams = export interface PerformanceMetricInnerEvents { key1?: string; value1?: number; - key2?: string; - value2?: number; - key3?: string; - value3?: number; } export interface ITelemetryClient { From c6d80a165f80b13e25479d892f6f1a86d5af237d Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 10 Jan 2024 12:03:35 +0100 Subject: [PATCH 16/18] record infra source loading --- .buildkite/ftr_configs.yml | 2 +- ...fra_hosts_table.ts => infra_hosts_view.ts} | 0 .../containers/metrics_source/source.tsx | 22 ++++++++++++++----- 3 files changed, 18 insertions(+), 6 deletions(-) rename x-pack/performance/journeys/{infra_hosts_table.ts => infra_hosts_view.ts} (100%) diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index ccff79b175906..7befa57c2f36c 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -458,7 +458,7 @@ enabled: - x-pack/performance/journeys/tags_listing_page.ts - x-pack/performance/journeys/cloud_security_dashboard.ts - x-pack/performance/journeys/apm_service_inventory.ts - - x-pack/performance/journeys/infra_hosts_table.ts + - x-pack/performance/journeys/infra_hosts_view.ts - x-pack/test/custom_branding/config.ts - x-pack/test/profiling_api_integration/cloud/config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts diff --git a/x-pack/performance/journeys/infra_hosts_table.ts b/x-pack/performance/journeys/infra_hosts_view.ts similarity index 100% rename from x-pack/performance/journeys/infra_hosts_table.ts rename to x-pack/performance/journeys/infra_hosts_view.ts diff --git a/x-pack/plugins/infra/public/containers/metrics_source/source.tsx b/x-pack/plugins/infra/public/containers/metrics_source/source.tsx index ffbc7148bc017..4cce1e9540f75 100644 --- a/x-pack/plugins/infra/public/containers/metrics_source/source.tsx +++ b/x-pack/plugins/infra/public/containers/metrics_source/source.tsx @@ -8,8 +8,8 @@ import createContainer from 'constate'; import React, { useEffect, useState } from 'react'; -import { useKibana } from '@kbn/kibana-react-plugin/public'; import { IHttpFetchError } from '@kbn/core-http-browser'; +import { useKibanaContextForPlugin } from '../../hooks/use_kibana'; import type { MetricsSourceConfigurationResponse, MetricsSourceConfiguration, @@ -34,11 +34,13 @@ export const pickIndexPattern = ( }; export const useSource = ({ sourceId }: { sourceId: string }) => { - const { services } = useKibana(); + const { + services: { http, telemetry }, + } = useKibanaContextForPlugin(); const notify = useSourceNotifier(); - const fetchService = services.http; + const fetchService = http; const API_URL = `/api/metrics/source/${sourceId}`; const [source, setSource] = useState(undefined); @@ -46,12 +48,22 @@ export const useSource = ({ sourceId }: { sourceId: string }) => { const [loadSourceRequest, loadSource] = useTrackedPromise( { cancelPreviousOn: 'resolution', - createPromise: () => { + createPromise: async () => { if (!fetchService) { throw new MissingHttpClientException(); } - return fetchService.fetch(API_URL, { method: 'GET' }); + const start = performance.now(); + const response = await fetchService.fetch(API_URL, { + method: 'GET', + }); + telemetry.reportPerformanceMetricEvent( + 'infra_source_load', + performance.now() - start, + {}, + {} + ); + return response; }, onResolve: (response) => { if (response) { From 2bee796c0390c6098f1798e36d844067390296cb Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 11 Jan 2024 12:00:47 +0100 Subject: [PATCH 17/18] extract kibana headers --- .../apm/client/apm_synthtrace_kibana_client.ts | 10 +--------- .../lib/infra/infra_synthtrace_kibana_client.ts | 10 +--------- .../src/lib/shared/client_headers.ts | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 packages/kbn-apm-synthtrace/src/lib/shared/client_headers.ts diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts b/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts index 3304fc7bd3c9c..76faaac0c942c 100644 --- a/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts @@ -9,6 +9,7 @@ import fetch from 'node-fetch'; import pRetry from 'p-retry'; import { Logger } from '../../utils/create_logger'; +import { kibanaHeaders } from '../../shared/client_headers'; export class ApmSynthtraceKibanaClient { private readonly logger: Logger; @@ -63,12 +64,3 @@ export class ApmSynthtraceKibanaClient { this.logger.info(`Installed APM package ${packageVersion}`); } } - -function kibanaHeaders() { - return { - Accept: 'application/json', - 'Content-Type': 'application/json', - 'kbn-xsrf': 'kibana', - 'elastic-api-version': '2023-10-31', - }; -} diff --git a/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts b/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts index 7cc4dee1c2c09..c1ac555276a66 100644 --- a/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts @@ -10,6 +10,7 @@ import { join } from 'path'; import fetch from 'node-fetch'; import pRetry from 'p-retry'; import { Logger } from '../utils/create_logger'; +import { kibanaHeaders } from '../shared/client_headers'; export class InfraSynthtraceKibanaClient { private readonly logger: Logger; @@ -67,12 +68,3 @@ export class InfraSynthtraceKibanaClient { this.logger.info(`Installed System package ${packageVersion}`); } } - -function kibanaHeaders() { - return { - Accept: 'application/json', - 'Content-Type': 'application/json', - 'kbn-xsrf': 'kibana', - 'elastic-api-version': '2023-10-31', - }; -} diff --git a/packages/kbn-apm-synthtrace/src/lib/shared/client_headers.ts b/packages/kbn-apm-synthtrace/src/lib/shared/client_headers.ts new file mode 100644 index 0000000000000..c6a5a80d6ad7f --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/lib/shared/client_headers.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export function kibanaHeaders() { + return { + Accept: 'application/json', + 'Content-Type': 'application/json', + 'kbn-xsrf': 'kibana', + 'elastic-api-version': '2023-10-31', + }; +} From a6ac8297e0ab37ff198b35b87f3780e7210c3045 Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 11 Jan 2024 17:55:25 +0100 Subject: [PATCH 18/18] use same timestamp for synthtrace documents --- x-pack/performance/journeys/infra_hosts_view.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/performance/journeys/infra_hosts_view.ts b/x-pack/performance/journeys/infra_hosts_view.ts index 9e64c1f86285b..b936cc7c1719c 100644 --- a/x-pack/performance/journeys/infra_hosts_view.ts +++ b/x-pack/performance/journeys/infra_hosts_view.ts @@ -76,11 +76,11 @@ export function generateHostsData({ .generator((timestamp, index) => hosts.flatMap((host) => [ host.cpu().timestamp(timestamp), - host.memory().timestamp(timestamp + 1), - host.network().timestamp(timestamp + 2), - host.load().timestamp(timestamp + 3), - host.filesystem().timestamp(timestamp + 4), - host.diskio().timestamp(timestamp + 5), + host.memory().timestamp(timestamp), + host.network().timestamp(timestamp), + host.load().timestamp(timestamp), + host.filesystem().timestamp(timestamp), + host.diskio().timestamp(timestamp), ]) ); }