diff --git a/python/tests/integration/test_spark_job.py b/python/tests/integration/test_spark_job.py index 6e865c97..064d5dbf 100644 --- a/python/tests/integration/test_spark_job.py +++ b/python/tests/integration/test_spark_job.py @@ -35,6 +35,7 @@ HISTORY_SERVER = "history-server" PUSHGATEWAY = "pushgateway" PROMETHEUS = "prometheus" +LOKI = "loki" @pytest.fixture(scope="module") @@ -266,6 +267,34 @@ async def test_spark_metrics_in_prometheus( assert spark_id == spark_app_selector +@pytest.mark.abort_on_fail +@pytest.mark.asyncio +async def test_spark_logforwaring_to_loki( + ops_test: OpsTest, registry, service_account, cos +): + if not cos: + pytest.skip("Not possible to test without cos") + + _, stdout, _ = await ops_test.juju("status") + + logger.info(f"Show status: {stdout}") + with ops_test.model_context(COS_ALIAS) as cos_model: + status = await cos_model.get_status() + loki_address = status["applications"][LOKI]["units"][f"{LOKI}/0"]["address"] + + log_gl = urllib.parse.quote('{app="spark", pebble_service="sparkd"}') + query = json.loads( + urllib.request.urlopen( + f"http://{loki_address}:3100/loki/api/v1/query_range?query={log_gl}" + ).read() + ) + + # NOTE(rgildein): This check depends on previous tests, because without the previous ones + # there will be no logs. + logger.info(f"query: {query}") + assert len(query["data"]["result"]) != 0, "no logs was found" + + @pytest.mark.abort_on_fail @pytest.mark.asyncio async def test_history_server_metrics_in_cos(ops_test: OpsTest, cos): diff --git a/python/tests/integration/test_spark_job_azure.py b/python/tests/integration/test_spark_job_azure.py index f8f97067..8a5ee1c1 100644 --- a/python/tests/integration/test_spark_job_azure.py +++ b/python/tests/integration/test_spark_job_azure.py @@ -37,6 +37,7 @@ HISTORY_SERVER = "history-server" PUSHGATEWAY = "pushgateway" PROMETHEUS = "prometheus" +LOKI = "loki" @pytest.fixture(scope="module") @@ -263,6 +264,34 @@ async def test_job_in_prometheus(ops_test: OpsTest, registry, service_account, c assert spark_id == spark_app_selector +@pytest.mark.abort_on_fail +@pytest.mark.asyncio +async def test_spark_logforwaring_to_loki( + ops_test: OpsTest, registry, service_account, cos +): + if not cos: + pytest.skip("Not possible to test without cos") + + _, stdout, _ = await ops_test.juju("status") + + logger.info(f"Show status: {stdout}") + with ops_test.model_context(COS_ALIAS) as cos_model: + status = await cos_model.get_status() + loki_address = status["applications"][LOKI]["units"][f"{LOKI}/0"]["address"] + + log_gl = urllib.parse.quote('{app="spark", pebble_service="sparkd"}') + query = json.loads( + urllib.request.urlopen( + f"http://{loki_address}:3100/loki/api/v1/query_range?query={log_gl}" + ).read() + ) + + # NOTE(rgildein): This check depends on previous tests, because without the previous ones + # there will be no logs. + logger.info(f"query: {query}") + assert len(query["data"]["result"]) != 0, "no logs was found" + + @pytest.mark.abort_on_fail @pytest.mark.asyncio async def test_history_server_metrics_in_cos(ops_test: OpsTest, cos): diff --git a/releases/3.4/terraform/base/applications.tf b/releases/3.4/terraform/base/applications.tf index 98469e4e..0202f96a 100644 --- a/releases/3.4/terraform/base/applications.tf +++ b/releases/3.4/terraform/base/applications.tf @@ -2,86 +2,75 @@ # See LICENSE file for licensing details. resource "juju_application" "history_server" { - name = "history-server" - - model = var.model + name = "history-server" + model = var.model charm { - name = "spark-history-server-k8s" - channel = "3.4/edge" - revision = 30 + name = "spark-history-server-k8s" + channel = "3.4/edge" + revision = 33 } resources = { - spark-history-server-image = 17 # 3.4.2 + spark-history-server-image = 17 # 3.4.2 } - units = 1 - + units = 1 constraints = "arch=amd64" - } resource "juju_application" "s3" { - name = "s3" - - model = var.model + name = "s3" + model = var.model charm { - name = "s3-integrator" - channel = "latest/edge" + name = "s3-integrator" + channel = "latest/edge" revision = 17 } config = { - path = "spark-events" - bucket = var.s3.bucket - endpoint = var.s3.endpoint + path = "spark-events" + bucket = var.s3.bucket + endpoint = var.s3.endpoint } - units = 1 - + units = 1 constraints = "arch=amd64" - } resource "juju_application" "kyuubi" { - - name = "kyuubi" - - model = var.model + name = "kyuubi" + model = var.model charm { - name = "kyuubi-k8s" - channel = "latest/edge" + name = "kyuubi-k8s" + channel = "latest/edge" revision = 27 } resources = { - kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:9268d19a6eef91914e874734b320fab64908faf0f7adb8856be809bc60ecd1d0" + kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:9268d19a6eef91914e874734b320fab64908faf0f7adb8856be809bc60ecd1d0" } config = { - namespace = var.model + namespace = var.model service-account = var.kyuubi_user } - units = 3 - trust = true - + units = 3 + trust = true constraints = "arch=amd64" } resource "juju_application" "zookeeper" { - - name = "zookeeper" - - model = var.model + name = "zookeeper" + model = var.model charm { - name = "zookeeper-k8s" - channel = "3/edge" + name = "zookeeper-k8s" + channel = "3/edge" revision = 59 } @@ -89,72 +78,63 @@ resource "juju_application" "zookeeper" { zookeeper-image = 31 } - units = 3 + units = 3 constraints = "arch=amd64" } resource "juju_application" "kyuubi_users" { - name = "kyuubi-users" - - model = var.model + name = "kyuubi-users" + model = var.model charm { - name = "postgresql-k8s" - channel = "14/stable" + name = "postgresql-k8s" + channel = "14/stable" revision = 281 } resources = { - postgresql-image = 159 + postgresql-image = 159 } - units = 1 - trust = true - + units = 1 + trust = true constraints = "arch=amd64" - } resource "juju_application" "metastore" { - name = "metastore" - - model = var.model + name = "metastore" + model = var.model charm { - name = "postgresql-k8s" - channel = "14/stable" + name = "postgresql-k8s" + channel = "14/stable" revision = 281 } resources = { - postgresql-image = 159 + postgresql-image = 159 } - units = 1 - trust = true - + units = 1 + trust = true constraints = "arch=amd64" - } resource "juju_application" "hub" { - name = "integration-hub" - - model = var.model + name = "integration-hub" + model = var.model charm { - name = "spark-integration-hub-k8s" - channel = "latest/edge" - revision = 20 + name = "spark-integration-hub-k8s" + channel = "latest/edge" + revision = 22 } resources = { - integration-hub-image = 3 + integration-hub-image = 3 } - units = 1 - trust = true - + units = 1 + trust = true constraints = "arch=amd64" - -} \ No newline at end of file +} diff --git a/releases/3.4/terraform/cos/integrations.tf b/releases/3.4/terraform/cos/integrations.tf index e77e081f..06dad337 100644 --- a/releases/3.4/terraform/cos/integrations.tf +++ b/releases/3.4/terraform/cos/integrations.tf @@ -15,66 +15,66 @@ data "juju_offer" "loki" { resource "juju_integration" "cos_configuration_agent" { - model = var.model + model = var.model application { - name = juju_application.cos_configuration.name + name = juju_application.cos_configuration.name endpoint = "grafana-dashboards" } application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "grafana-dashboards-consumer" } } resource "juju_integration" "pushgateway_scrape_config" { - model = var.model + model = var.model application { - name = juju_application.pushgateway.name + name = juju_application.pushgateway.name endpoint = "metrics-endpoint" } application { - name = juju_application.scrape_config.name + name = juju_application.scrape_config.name endpoint = "configurable-scrape-jobs" } } resource "juju_integration" "scrape_config_agent" { - model = var.model + model = var.model application { - name = juju_application.scrape_config.name + name = juju_application.scrape_config.name endpoint = "metrics-endpoint" } application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "metrics-endpoint" } } resource "juju_integration" "pushgateway_integration_hub" { - model = var.model + model = var.model application { - name = juju_application.pushgateway.name + name = juju_application.pushgateway.name endpoint = "push-endpoint" } application { - name = var.integration_hub + name = var.integration_hub endpoint = "cos" } } resource "juju_integration" "agent_grafana_dashboards" { - model = var.model + model = var.model application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "grafana-dashboards-provider" } @@ -84,10 +84,10 @@ resource "juju_integration" "agent_grafana_dashboards" { } resource "juju_integration" "agent_prometheus" { - model = var.model + model = var.model application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "send-remote-write" } @@ -97,56 +97,70 @@ resource "juju_integration" "agent_prometheus" { } resource "juju_integration" "history_server_agent_dashboard" { - model = var.model + model = var.model application { - name = var.history_server + name = var.history_server endpoint = "grafana-dashboard" } application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "grafana-dashboards-consumer" } } resource "juju_integration" "history_server_agent_logging" { - model = var.model + model = var.model application { - name = var.history_server + name = var.history_server endpoint = "logging" } application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "logging-provider" } } resource "juju_integration" "history_server_agent_metrics" { - model = var.model + model = var.model application { - name = var.history_server + name = var.history_server endpoint = "metrics-endpoint" } application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "metrics-endpoint" } } resource "juju_integration" "agent_loki" { - model = var.model + model = var.model application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "logging-consumer" } application { offer_url = data.juju_offer.loki.url } -} \ No newline at end of file +} + +resource "juju_integration" "integration_hub_logging" { + model = var.model + + application { + name = juju_application.agent.name + endpoint = "logging-provider" + } + + application { + name = var.integration_hub + endpoint = "logging" + } +} diff --git a/releases/3.4/yaml/bundle-azure-storage.yaml.j2 b/releases/3.4/yaml/bundle-azure-storage.yaml.j2 index 463d9b34..49b04027 100644 --- a/releases/3.4/yaml/bundle-azure-storage.yaml.j2 +++ b/releases/3.4/yaml/bundle-azure-storage.yaml.j2 @@ -51,7 +51,7 @@ applications: history-server: charm: spark-history-server-k8s channel: 3.4/edge - revision: 30 + revision: 33 resources: spark-history-server-image: ghcr.io/canonical/charmed-spark@sha256:75a01dfca493b5a457fc7d3258daba3e9891f0408f0097f1fd189100d5de4891 # 3.4.2 scale: 1 @@ -59,7 +59,7 @@ applications: integration-hub: charm: spark-integration-hub-k8s channel: latest/edge - revision: 20 + revision: 22 resources: integration-hub-image: 3 scale: 1 @@ -75,4 +75,4 @@ relations: - - kyuubi:auth-db - kyuubi-users:database - - kyuubi:spark-service-account - - integration-hub:spark-service-account \ No newline at end of file + - integration-hub:spark-service-account diff --git a/releases/3.4/yaml/bundle.yaml.j2 b/releases/3.4/yaml/bundle.yaml.j2 index ce2726f8..dce32feb 100644 --- a/releases/3.4/yaml/bundle.yaml.j2 +++ b/releases/3.4/yaml/bundle.yaml.j2 @@ -58,7 +58,7 @@ applications: history-server: charm: spark-history-server-k8s channel: 3.4/edge - revision: 30 + revision: 33 resources: spark-history-server-image: ghcr.io/canonical/charmed-spark@sha256:75a01dfca493b5a457fc7d3258daba3e9891f0408f0097f1fd189100d5de4891 # 3.4.2 scale: 1 @@ -66,7 +66,7 @@ applications: integration-hub: charm: spark-integration-hub-k8s channel: latest/edge - revision: 20 + revision: 22 resources: integration-hub-image: 3 scale: 1 diff --git a/releases/3.4/yaml/overlays/cos-integration.yaml.j2 b/releases/3.4/yaml/overlays/cos-integration.yaml.j2 index 1b3e5d45..c4eb2a05 100644 --- a/releases/3.4/yaml/overlays/cos-integration.yaml.j2 +++ b/releases/3.4/yaml/overlays/cos-integration.yaml.j2 @@ -86,4 +86,6 @@ relations: - - kyuubi:grafana-dashboard - agent:grafana-dashboards-consumer - - kyuubi:logging - - agent:logging-provider \ No newline at end of file + - agent:logging-provider +- - integration-hub:logging + - agent:logging-provider diff --git a/releases/3.5/terraform/base/applications.tf b/releases/3.5/terraform/base/applications.tf index be2e6abb..0a03c8ae 100644 --- a/releases/3.5/terraform/base/applications.tf +++ b/releases/3.5/terraform/base/applications.tf @@ -2,86 +2,75 @@ # See LICENSE file for licensing details. resource "juju_application" "history_server" { - name = "history-server" - - model = var.model + name = "history-server" + model = var.model charm { - name = "spark-history-server-k8s" - channel = "3.4/edge" - revision = 30 + name = "spark-history-server-k8s" + channel = "3.4/edge" + revision = 33 } resources = { - spark-history-server-image = 17 # 3.4.2 + spark-history-server-image = 17 # 3.4.2 } - units = 1 - + units = 1 constraints = "arch=amd64" - } resource "juju_application" "s3" { - name = "s3" - - model = var.model + name = "s3" + model = var.model charm { - name = "s3-integrator" - channel = "latest/edge" + name = "s3-integrator" + channel = "latest/edge" revision = 17 } config = { - path = "spark-events" - bucket = var.s3.bucket - endpoint = var.s3.endpoint + path = "spark-events" + bucket = var.s3.bucket + endpoint = var.s3.endpoint } - units = 1 - + units = 1 constraints = "arch=amd64" - } resource "juju_application" "kyuubi" { - - name = "kyuubi" - - model = var.model + name = "kyuubi" + model = var.model charm { - name = "kyuubi-k8s" - channel = "latest/edge" + name = "kyuubi-k8s" + channel = "latest/edge" revision = 27 } resources = { - kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:9268d19a6eef91914e874734b320fab64908faf0f7adb8856be809bc60ecd1d0" + kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:9268d19a6eef91914e874734b320fab64908faf0f7adb8856be809bc60ecd1d0" } config = { - namespace = var.model + namespace = var.model service-account = var.kyuubi_user } - units = 3 - trust = true - + units = 3 + trust = true constraints = "arch=amd64" } resource "juju_application" "zookeeper" { - - name = "zookeeper" - - model = var.model + name = "zookeeper" + model = var.model charm { - name = "zookeeper-k8s" - channel = "3/edge" + name = "zookeeper-k8s" + channel = "3/edge" revision = 59 } @@ -89,73 +78,64 @@ resource "juju_application" "zookeeper" { zookeeper-image = 31 } - units = 3 + units = 3 constraints = "arch=amd64" } resource "juju_application" "kyuubi_users" { - name = "kyuubi-users" - - model = var.model + name = "kyuubi-users" + model = var.model charm { - name = "postgresql-k8s" - channel = "14/stable" + name = "postgresql-k8s" + channel = "14/stable" revision = 281 } resources = { - postgresql-image = 159 + postgresql-image = 159 } - units = 1 - trust = true - + units = 1 + trust = true constraints = "arch=amd64" - } resource "juju_application" "metastore" { - name = "metastore" - - model = var.model + name = "metastore" + model = var.model charm { - name = "postgresql-k8s" - channel = "14/stable" + name = "postgresql-k8s" + channel = "14/stable" revision = 281 } resources = { - postgresql-image = 159 + postgresql-image = 159 } - units = 1 - trust = true - + units = 1 + trust = true constraints = "arch=amd64" - } resource "juju_application" "hub" { - name = "integration-hub" - - model = var.model + name = "integration-hub" + model = var.model charm { - name = "spark-integration-hub-k8s" - channel = "latest/edge" - revision = 20 + name = "spark-integration-hub-k8s" + channel = "latest/edge" + revision = 22 } resources = { - integration-hub-image = 3 + integration-hub-image = 3 } - units = 1 - trust = true - + units = 1 + trust = true constraints = "arch=amd64" - -} \ No newline at end of file +} diff --git a/releases/3.5/terraform/cos/integrations.tf b/releases/3.5/terraform/cos/integrations.tf index e77e081f..06dad337 100644 --- a/releases/3.5/terraform/cos/integrations.tf +++ b/releases/3.5/terraform/cos/integrations.tf @@ -15,66 +15,66 @@ data "juju_offer" "loki" { resource "juju_integration" "cos_configuration_agent" { - model = var.model + model = var.model application { - name = juju_application.cos_configuration.name + name = juju_application.cos_configuration.name endpoint = "grafana-dashboards" } application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "grafana-dashboards-consumer" } } resource "juju_integration" "pushgateway_scrape_config" { - model = var.model + model = var.model application { - name = juju_application.pushgateway.name + name = juju_application.pushgateway.name endpoint = "metrics-endpoint" } application { - name = juju_application.scrape_config.name + name = juju_application.scrape_config.name endpoint = "configurable-scrape-jobs" } } resource "juju_integration" "scrape_config_agent" { - model = var.model + model = var.model application { - name = juju_application.scrape_config.name + name = juju_application.scrape_config.name endpoint = "metrics-endpoint" } application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "metrics-endpoint" } } resource "juju_integration" "pushgateway_integration_hub" { - model = var.model + model = var.model application { - name = juju_application.pushgateway.name + name = juju_application.pushgateway.name endpoint = "push-endpoint" } application { - name = var.integration_hub + name = var.integration_hub endpoint = "cos" } } resource "juju_integration" "agent_grafana_dashboards" { - model = var.model + model = var.model application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "grafana-dashboards-provider" } @@ -84,10 +84,10 @@ resource "juju_integration" "agent_grafana_dashboards" { } resource "juju_integration" "agent_prometheus" { - model = var.model + model = var.model application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "send-remote-write" } @@ -97,56 +97,70 @@ resource "juju_integration" "agent_prometheus" { } resource "juju_integration" "history_server_agent_dashboard" { - model = var.model + model = var.model application { - name = var.history_server + name = var.history_server endpoint = "grafana-dashboard" } application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "grafana-dashboards-consumer" } } resource "juju_integration" "history_server_agent_logging" { - model = var.model + model = var.model application { - name = var.history_server + name = var.history_server endpoint = "logging" } application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "logging-provider" } } resource "juju_integration" "history_server_agent_metrics" { - model = var.model + model = var.model application { - name = var.history_server + name = var.history_server endpoint = "metrics-endpoint" } application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "metrics-endpoint" } } resource "juju_integration" "agent_loki" { - model = var.model + model = var.model application { - name = juju_application.agent.name + name = juju_application.agent.name endpoint = "logging-consumer" } application { offer_url = data.juju_offer.loki.url } -} \ No newline at end of file +} + +resource "juju_integration" "integration_hub_logging" { + model = var.model + + application { + name = juju_application.agent.name + endpoint = "logging-provider" + } + + application { + name = var.integration_hub + endpoint = "logging" + } +} diff --git a/releases/3.5/yaml/bundle-azure-storage.yaml.j2 b/releases/3.5/yaml/bundle-azure-storage.yaml.j2 index 4d30fa2c..ad418fbb 100644 --- a/releases/3.5/yaml/bundle-azure-storage.yaml.j2 +++ b/releases/3.5/yaml/bundle-azure-storage.yaml.j2 @@ -51,7 +51,7 @@ applications: history-server: charm: spark-history-server-k8s channel: 3.4/edge - revision: 30 + revision: 33 resources: spark-history-server-image: ghcr.io/canonical/charmed-spark@sha256:75a01dfca493b5a457fc7d3258daba3e9891f0408f0097f1fd189100d5de4891 # 3.4.2 scale: 1 @@ -59,7 +59,7 @@ applications: integration-hub: charm: spark-integration-hub-k8s channel: latest/edge - revision: 20 + revision: 22 resources: integration-hub-image: 3 scale: 1 @@ -75,4 +75,4 @@ relations: - - kyuubi:auth-db - kyuubi-users:database - - kyuubi:spark-service-account - - integration-hub:spark-service-account \ No newline at end of file + - integration-hub:spark-service-account diff --git a/releases/3.5/yaml/bundle.yaml.j2 b/releases/3.5/yaml/bundle.yaml.j2 index 59eaf5b1..b5dca1ce 100644 --- a/releases/3.5/yaml/bundle.yaml.j2 +++ b/releases/3.5/yaml/bundle.yaml.j2 @@ -58,7 +58,7 @@ applications: history-server: charm: spark-history-server-k8s channel: 3.4/edge - revision: 30 + revision: 33 resources: # We use 3.4.2 here because we don't have metrics added in version 3.5.1 of the rock image spark-history-server-image: ghcr.io/canonical/charmed-spark@sha256:75a01dfca493b5a457fc7d3258daba3e9891f0408f0097f1fd189100d5de4891 # 3.4.2 @@ -67,7 +67,7 @@ applications: integration-hub: charm: spark-integration-hub-k8s channel: latest/edge - revision: 20 + revision: 22 resources: integration-hub-image: 3 scale: 1 @@ -87,4 +87,4 @@ relations: - - kyuubi:spark-service-account - integration-hub:spark-service-account - - kyuubi:zookeeper - - zookeeper:zookeeper \ No newline at end of file + - zookeeper:zookeeper diff --git a/releases/3.5/yaml/overlays/cos-integration.yaml.j2 b/releases/3.5/yaml/overlays/cos-integration.yaml.j2 index ce10d8dd..adf27a97 100644 --- a/releases/3.5/yaml/overlays/cos-integration.yaml.j2 +++ b/releases/3.5/yaml/overlays/cos-integration.yaml.j2 @@ -86,4 +86,6 @@ relations: - - kyuubi:grafana-dashboard - agent:grafana-dashboards-consumer - - kyuubi:logging - - agent:logging-provider \ No newline at end of file + - agent:logging-provider +- - integration-hub:logging + - agent:logging-provider