The features comprised in this dataset are the result of applying PCA to the original dataset + and therefore cannot be interpreted. There are 31 features, labeled V1 to V31 in this dataset, + plus Amount and Time.", + "model_owners": [{"name": "Paul Balm", "contact": "abcdef@google.com"}], + "model_references": ["https://www.kaggle.com/datasets/mlg-ulb/creditcardfraud"], + "model_ethical_consideration_risks": [ + { + "name": "The dataset contains a small number of fraud cases that makes it difficult to verify that models trained on this data will generalize to unseen data.", + "mitigation_strategy": "Verify that the model generalizes to the data for the use case at hand, before using it for any decision making."}, + { + "name": "Classifying a transaction as fraudulent can be highly impactful for the person responsible for the transaction.", + "mitigation_strategy": "Any classification as fraudulent and implication thereof needs to be treated with caution and requires a very high level of certainty."}], + "model_ethical_limitations": [{"description": "Using this model requires knowledge of the PCA transformation that was applied to the training data."}], + "model_considerations_use_cases": [{"description": "Fraud detection and prevention (ML technology demo only)"}], + "model_considerations_users": [{"description": "Data Scientists"}] +} \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/src/kfp_pipelines/requirements.txt b/examples/vertex_mlops_enterprise/src/kfp_pipelines/requirements.txt new file mode 100644 index 0000000000..d054bdac22 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/kfp_pipelines/requirements.txt @@ -0,0 +1,14 @@ +cloudml-hypertune==0.1.0.dev6 +db-dtypes==1.1.1 +google-cloud-aiplatform==1.27.1 +google-cloud-storage==2.10.0 +google_cloud_pipeline_components==2.0.0b5 +kfp==2.0.1 +pandas==1.3.5 +pandas_gbq +protobuf==3.20.3 +pyarrow==6.0.1 +scikit-learn==0.24.1 +scipy==1.10.1 +seaborn==0.12.2 +xgboost==1.6.2 diff --git a/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/config.py b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/config.py new file mode 100644 index 0000000000..308dc496a2 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/config.py @@ -0,0 +1,34 @@ +import os + +PROJECT_ID = os.getenv("PROJECT_ID", "") +REGION = os.getenv("REGION", "") +IMAGE=os.getenv("CICD_IMAGE_URI", f'{REGION}-docker.pkg.dev/{PROJECT_ID}/creditcards-kfp/base:latest') +TRAIN_COMPONENT_IMAGE=f'{REGION}-docker.pkg.dev/{PROJECT_ID}/creditcards-kfp/train-fraud:latest' +IMAGE_MODEL_CARD=os.getenv("CICD_IMAGE_MODEL_CARD", f'{REGION}-docker.pkg.dev/{PROJECT_ID}/creditcards-kfp/model-card:latest') + +CLASS_NAMES = ['OK', 'Fraud'] +TARGET_COLUMN = 'Class' + +PIPELINE_NAME = os.getenv("PIPELINE_NAME", 'xgb-creditcards') +PIPELINE_ROOT = os.getenv("PIPELINES_STORE", f'gs://{PROJECT_ID}/pipeline_root/{PIPELINE_NAME}') +SERVICE_ACCOUNT = os.getenv("SERVICE_ACCOUNT") # returns None is not defined +NETWORK = os.getenv("NETWORK") # returns None is not defined +KEY_ID = os.getenv("CMEK_KEY_ID") # e.g. projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key + +BQ_INPUT_DATA=f"{PROJECT_ID}.{os.getenv('BQ_DATASET_NAME')}.{os.getenv('ML_TABLE')}" +PARENT_MODEL='' # f'projects/{PROJECT_ID}/locations/{REGION}/models/YOUR_NUMERIC_MODEL_ID_HERE' + +BQ_OUTPUT_DATASET_ID="creditcards_batch_out" + +MODEL_DISPLAY_NAME = os.getenv("MODEL_DISPLAY_NAME", 'creditcards-kfp') +MODEL_CARD_CONFIG='../model_card_config.json' + +PRED_CONTAINER='europe-docker.pkg.dev/vertex-ai/prediction/xgboost-cpu.1-6:latest' +ENDPOINT_NAME=PIPELINE_NAME + +EMAILS=['abcdef@google.com'] + +# Evaluation pipeline +DATAFLOW_SA = os.getenv("DATAFLOW_SA") +DATAFLOW_NETWORK = os.getenv("DATAFLOW_NETWORK") +DATAFLOW_PUBLIC_IPS = False diff --git a/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/eval.py b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/eval.py new file mode 100644 index 0000000000..cdfcbef756 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/eval.py @@ -0,0 +1,111 @@ +from kfp.v2 import dsl +from kfp.v2.dsl import Artifact, Dataset, Input, Model, Output + +from config import IMAGE + + +@dsl.component( + packages_to_install=['shap'], + base_image=IMAGE) +def evaluate_model( + test_data: Input[Dataset], + trained_model: Input[Model], + reports: Output[Artifact], + class_names: list, + target_column: str +): + + import pickle as pkl + import pandas as pd + import xgboost as xgb + import numpy as np + import sklearn + from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score + from sklearn.metrics import ConfusionMatrixDisplay, PrecisionRecallDisplay, RocCurveDisplay + from sklearn.metrics import confusion_matrix, precision_recall_curve, roc_curve, auc + from model_card_toolkit.utils.graphics import figure_to_base64str + import logging + + # This is a work-around for a bug in shap + np.int = int + + + def ShapDisplay(model, x, class_names): + """ + This function returns a SHAP summary plot. + Args: + model (sklearn.pipeline.Pipeline): The model pipeline. + x (numpy.ndarray): The data to explain. + column_names (list): The names of the columns. + class_names (list): The names of the classes. + Returns: + A Matplotlib figure. + """ + import matplotlib.pyplot as plt + import shap + + fig = plt.figure() + explainer = shap.TreeExplainer(model) + shap_values = explainer.shap_values(x) + shap.summary_plot(shap_values, x, feature_names=list(x.columns), + class_names=class_names, plot_type='bar', show=False) + fig.suptitle("Mean absolute SHAP Values") + fig.axes[0].set_xlabel('average impact on model output magnitude') + return fig + + # Read data + logging.info("Reading data...") + test = pd.read_csv(test_data.path) + + # Read model + logging.info("Reading model...") + model = xgb.Booster() + model.load_model(trained_model.path) + + # Evaluate model + logging.info("Evaluating model...") + y = test.pop(target_column) + X = test + y_pred_prob = model.predict(xgb.DMatrix(X)) + y_pred = list(map(lambda x: x >= 0.5, y_pred_prob)) + accuracy = accuracy_score(y, y_pred) + pos_label=1 + precision = precision_score(y, y_pred, pos_label=pos_label) + recall = recall_score(y, y_pred, pos_label=pos_label) + f1 = f1_score(y, y_pred, pos_label=pos_label) + logging.info(f"Accuracy: {accuracy:.3g}") + logging.info(f"Precision: {precision:.3g}") + logging.info(f"Recall: {recall:.3g}") + logging.info(f"F1: {f1:.3g}") + + # Save model reports + logging.info("Saving model reports...") + logging.info(f"sklearn {sklearn.__version__}") + + c_m = confusion_matrix(y, y_pred) + cm = ConfusionMatrixDisplay(confusion_matrix=c_m) + cm.plot() + cm.figure_.suptitle('Confusion Matrix') + + precision, recall, _ = precision_recall_curve(y, y_pred) + pr = PrecisionRecallDisplay(precision=precision, recall=recall) + pr.plot() + pr.figure_.suptitle('Precision Recall Curve') + + fpr, tpr, _ = roc_curve(y, y_pred) + roc_auc = auc(fpr, tpr) + roc = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc) + roc.plot() + roc.figure_.suptitle('ROC') + + # Calculate SHAP summary plot + shap_fig = ShapDisplay(model, X, class_names) + shap_fig.suptitle + reports_dict = { + "confusion_matrix": figure_to_base64str(cm.figure_), + "precision_recall": figure_to_base64str(pr.figure_), + "roc_curve": figure_to_base64str(roc.figure_), + "shap_plot": figure_to_base64str(shap_fig) + } + with open(reports.path, "wb") as f: + pkl.dump(reports_dict, f) diff --git a/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/load.py b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/load.py new file mode 100644 index 0000000000..5d5b323f74 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/load.py @@ -0,0 +1,115 @@ +from kfp import dsl +from google_cloud_pipeline_components.types.artifact_types import BQTable +from typing import NamedTuple + +from config import IMAGE + +# Load data from BigQuery and save to CSV +@dsl.component(base_image=IMAGE) +def get_dataframe( + project_id: str, + bq_table: str, + train_data: dsl.OutputPath("Dataset"), + test_data: dsl.OutputPath("Dataset"), + val_data: dsl.OutputPath("Dataset"), + stats: dsl.Output[dsl.Artifact], + class_names: list +): + from google.cloud import bigquery + from model_card_toolkit.utils.graphics import figure_to_base64str + from sklearn.model_selection import train_test_split + import pickle + import seaborn as sns + import logging + + bqclient = bigquery.Client(project=project_id) + logging.info(f"Pulling data from {bq_table}") + table = bigquery.TableReference.from_string(bq_table) + rows = bqclient.list_rows(table) + dataframe = rows.to_dataframe(create_bqstorage_client=True) + # Drop the Time column, otherwise the model will just memorize when the fraud cases happened + # Also drop the ml_use column - we will split here. ML_use just splits in test+rest, and we need a 3-way split. + dataframe.drop(columns=['Time', 'ML_use'], inplace=True) + logging.info("Data loaded, writing splits") + + # 60 / 20 / 20 + df_train, df_test = train_test_split(dataframe, test_size=0.4) + df_test, df_val = train_test_split(df_test, test_size=0.5) + + df_train.to_csv(train_data, index=False) + df_test.to_csv(test_data, index=False) + df_val.to_csv(val_data, index=False) + + def get_fig(df, title): + n_fraud = (df.Class == '1').sum() + n_ok = len(df) - n_fraud + + logging.info(f"Stats for {title}: {n_ok=} {n_fraud=}") + + ys = [n_ok, n_fraud] + + g = sns.barplot(x=class_names, y=ys) + g.set_yscale('log') + g.set_ylim(1, n_ok*2) + fig = g.get_figure() + fig.suptitle(title) + return fig + + logging.info("Generating stats") + stats_dict = {} + fig = get_fig(df_train, "Training data") + stats_dict['train'] = figure_to_base64str(fig) + fig.clf() + + fig = get_fig(df_test, "Test data") + stats_dict['test'] = figure_to_base64str(fig) + fig.clf() + + fig = get_fig(df_val, "Validation data") + stats_dict['val'] = figure_to_base64str(fig) + fig.clf() + + logging.info(f"Writing stats to {stats.path}") + with open(stats.path, 'wb') as f: + pickle.dump(stats_dict, f) + + +@dsl.component(base_image=IMAGE) +def upload_to_bq( + project: str, + location: str, + dest_dataset_id: str, + dest_table_id: str, + csv_data: dsl.Input[dsl.Dataset], + bq_table: dsl.Output[BQTable]) -> NamedTuple('outputs', [('bq_table_uri', str)]): + + from collections import namedtuple + import logging + import pandas as pd + import numpy as np + + from config import CLASS_NAMES + + bq_table.metadata["projectId"] = project + bq_table.metadata["datasetId"] = dest_dataset_id + bq_table.metadata["tableId"] = dest_table_id + logging.info(f"BQ table: {bq_table}\nmetadata: {bq_table.metadata}") + + logging.info(f"Reading {csv_data.path}") + dest_table = f'{dest_dataset_id}.{dest_table_id}' + logging.info(f"Writing to {dest_table}") + + df = pd.read_csv(csv_data.path) + + # Convert Class column to int and map to CLASS_NAMES label + df_class = df.pop('Class') + df['Class'] = list(map(lambda f: CLASS_NAMES[f], np.rint(df_class).astype(np.int64))) + + df.to_gbq( + destination_table=f"{dest_table}", + project_id=project, + location=location) + + t = namedtuple('outputs', ['bq_table_uri']) + return t(f'bq://{project}.{dest_dataset_id}.{dest_table_id}') + \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/model_card.py b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/model_card.py new file mode 100644 index 0000000000..0a5f66ecb0 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/model_card.py @@ -0,0 +1,138 @@ +from kfp import dsl +from kfp.dsl import Input, Output, Dataset, Artifact, HTML + +from config import IMAGE_MODEL_CARD + + +@dsl.component(base_image=IMAGE_MODEL_CARD) +def plot_model_card( + project_id: str, + region: str, + model: Input[Artifact], + train_data: Input[Dataset], + test_data: Input[Dataset], + val_data: Input[Dataset], + stats: Input[Artifact], + reports: Input[Artifact], + model_card_config: str, + model_card: Output[HTML] +): + + # Libraries + import json + import pickle as pkl + import model_card_toolkit as mctlib + import pandas as pd + import logging + from google.cloud import aiplatform + + # Load model card config + model_card_cd = json.loads(model_card_config) + + # Read data + print(f"Reading data... {train_data.path}") + train = pd.read_csv(train_data.path) + test = pd.read_csv(test_data.path) + val = pd.read_csv(val_data.path) + + # Read stats + logging.info(f"Reading stats... {stats.path}") + with open(stats.path, "rb") as stats_file: + stats = pkl.load(stats_file) + + # Read reports + logging.info(f"Reading reports... {reports.path}") + with open(reports.path, "rb") as f: + reports = pkl.load(f) + + # Compile model card + logging.info(f"Compiling model card... {model_card.path}") + mct = mctlib.ModelCardToolkit(model_card.path) + mc = mct.scaffold_assets() + + ## Model Details section + uri = model.metadata['resourceName'] + if uri.find('@') >= 0: # has version or alias + model_resource_name = uri[:uri.find('@')] # strip version or alias + model_version = uri[uri.find('@')+1:] + else: + model_resource_name = uri + model_version = None + + vertex_models = [m for m in aiplatform.Model.list(project=project_id, location=region) if m.resource_name==model_resource_name] + + if model_version: + models = [m for m in vertex_models if m.version_id == model_version] + if len(models)>1: + logging.warning(f"Found {len(models)} models with for {uri}") + vertex_model = models[0] + else: + if len(vertex_models)>1: + logging.warning(f"Found {len(vertex_models)} models with for {uri}") + vertex_model = vertex_models[0] + + mc.model_details.name = model_card_cd['model_name'] + mc.model_details.overview = model_card_cd['model_overview'] + mc.model_details.owners = [ + mctlib.Owner(name=owner_d['name'], contact=owner_d['contact']) + for owner_d in model_card_cd['model_owners']] + mc.model_details.references = [ + mctlib.Reference(reference=reference) + for reference in model_card_cd['model_references']] + + mc.model_details.version.name = vertex_model.resource_name + mc.model_details.version.date = vertex_model.create_time.strftime("%H:%M:%S (%Z), %-d %b %Y") + + ## Considerations section + mc.considerations.ethical_considerations = [ + mctlib.Risk( + name=risk['name'], + mitigation_strategy=risk['mitigation_strategy']) + for risk in model_card_cd['model_ethical_consideration_risks']] + mc.considerations.limitations = [ + mctlib.Limitation(description=limitation['description']) + for limitation in model_card_cd['model_ethical_limitations'] + ] + mc.considerations.use_cases = [ + mctlib.UseCase(description=use_case['description']) + for use_case in model_card_cd['model_considerations_use_cases']] + mc.considerations.users = [ + mctlib.User(description=user['description']) + for user in model_card_cd['model_considerations_users']] + + ## Datasets section + mc.model_parameters.data.append(mctlib.Dataset( + name="Training dataset", + description=f'{train.shape[0]:,} rows with {train.shape[1]:,} columns (features + target)')) + mc.model_parameters.data[0].graphics.collection = [ + mctlib.Graphic(image=stats['train']) + ] + mc.model_parameters.data.append(mctlib.Dataset( + name="Test dataset", + description=f'{test.shape[0]:,} rows with {test.shape[1]:,} columns (features + target)')) + mc.model_parameters.data[1].graphics.collection = [ + mctlib.Graphic(image=stats['test']) + ] + mc.model_parameters.data.append(mctlib.Dataset( + name="Validation dataset", + description=f'{val.shape[0]:,} rows with {val.shape[1]:,} columns (features + target)')) + mc.model_parameters.data[2].graphics.collection = [ + mctlib.Graphic(image=stats['val']) + ] + + ## Quantative Analysis section + mc.quantitative_analysis.graphics.description = ( + 'This analysis is performed using the validation dataset, which was not used in training.') + mc.quantitative_analysis.graphics.collection = [ + mctlib.Graphic(image=reports['roc_curve']), + mctlib.Graphic(image=reports['precision_recall']), + mctlib.Graphic(image=reports['confusion_matrix']), + mctlib.Graphic(image=reports['shap_plot']) + ] + + # Write model card + model_card_file = model_card.path + "/model_card.html" + logging.info(f"Writing model card... {model_card_file}") + mct.update_model_card(mc) + mct.export_format(model_card=mc, output_file=model_card_file) + model_card.path = model_card_file diff --git a/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/model_monitoring.py b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/model_monitoring.py new file mode 100644 index 0000000000..38156c3691 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/model_monitoring.py @@ -0,0 +1,62 @@ +from kfp import dsl +from google_cloud_pipeline_components.types.artifact_types import VertexEndpoint + +from config import IMAGE + +@dsl.component(base_image=IMAGE) +def model_monitoring( + project_id: str, + region: str, + endpoint: dsl.Input[VertexEndpoint], + pipeline_id: str, + bq_train_data: str, # bq://mco-mm.bqmlga4.train + skew_threshold: float, + sampling_rate: float, + monitoring_interval_hours: int, + user_emails: list +): + + from google.cloud import aiplatform + from google.cloud.aiplatform import model_monitoring + from train import TARGET_COLUMN + import logging + + bq_train_data = "bq://" + bq_train_data + + skew_config = model_monitoring.SkewDetectionConfig( + data_source= bq_train_data, + skew_thresholds=skew_threshold, # pass float to set one value across all featues, or dict to vary threshold by feature. + target_field=TARGET_COLUMN, + ) + + objective_config = model_monitoring.ObjectiveConfig(skew_detection_config=skew_config) + + + # Create sampling configuration + random_sampling = model_monitoring.RandomSampleConfig(sample_rate=sampling_rate) + + # Create schedule configuration + schedule_config = model_monitoring.ScheduleConfig(monitor_interval=monitoring_interval_hours) + + # Create alerting configuration. + alerting_config = model_monitoring.EmailAlertConfig( + user_emails=user_emails, enable_logging=True + ) + + endpoint_uri = endpoint.metadata['resourceName'] + logging.info("Endpoint URI is " + endpoint_uri) + logging.info("Using google_cloud_aiplatform " + aiplatform.__version__) + logging.info("Training data: " + bq_train_data) + + # Create the monitoring job. + job = aiplatform.ModelDeploymentMonitoringJob.create( + display_name=pipeline_id + "-monitoring", + logging_sampling_strategy=random_sampling, + schedule_config=schedule_config, + alert_config=alerting_config, + objective_configs=objective_config, + project=project_id, + location=region, + endpoint=endpoint_uri + ) + logging.info("Job: " + job) \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/model_upload.py b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/model_upload.py new file mode 100644 index 0000000000..b2f0bff801 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/model_upload.py @@ -0,0 +1,39 @@ +from kfp import dsl + +from config import IMAGE + +@dsl.component(base_image=IMAGE) +def upload_model( + project_id: str, + region: str, + model: dsl.Input[dsl.Model], + display_name: str, + serving_image: str, + parent_model: str, + uploaded_model: dsl.Output[dsl.Artifact], + run: str, + run_id: str +): + from google.cloud import aiplatform + import logging + + logging.info(f"Upload model for run {run} and run ID {run_id}") + + model_path = '/'.join(model.uri.split('/')[:-1]) # remove filename after last / - send dir rather than file + + vertex_model = aiplatform.Model.upload( + project=project_id, + location=region, + display_name=display_name, + artifact_uri=model_path, + serving_container_image_uri=serving_image, + parent_model=parent_model, + labels={ + 'run': run, + 'run_id': run_id + } + ) + + uploaded_model.metadata['resourceName'] = f'{vertex_model.resource_name}@{vertex_model.version_id}' + uploaded_model.uri = f'https://{region}-aiplatform.googleapis.com/v1/{vertex_model.resource_name}@{vertex_model.version_id}' + diff --git a/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/pipeline.py b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/pipeline.py new file mode 100644 index 0000000000..d5467f3208 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/pipeline.py @@ -0,0 +1,248 @@ +from kfp import dsl +from kfp import compiler +from kfp.dsl import Artifact, Input, Model, Output + +import argparse + +from google.cloud import aiplatform + +from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp +from google_cloud_pipeline_components.v1.endpoint import EndpointCreateOp, ModelDeployOp +from google_cloud_pipeline_components.experimental.evaluation import ( + ModelEvaluationClassificationOp, ModelImportEvaluationOp) + +import logging +from datetime import datetime + +from config import (PIPELINE_ROOT, PIPELINE_NAME, BQ_INPUT_DATA, MODEL_CARD_CONFIG, + MODEL_DISPLAY_NAME, PRED_CONTAINER, ENDPOINT_NAME, PARENT_MODEL, + SERVICE_ACCOUNT, NETWORK, KEY_ID, + PROJECT_ID, REGION, IMAGE, CLASS_NAMES, TARGET_COLUMN, + DATAFLOW_NETWORK, DATAFLOW_PUBLIC_IPS, DATAFLOW_SA, + BQ_OUTPUT_DATASET_ID) +from train import xgb_train +from eval import evaluate_model +from load import get_dataframe, upload_to_bq +from model_card import plot_model_card +from model_upload import upload_model +from reformat_preds import reformat_predictions_bq + +caching = True + +TIMESTAMP = datetime.now().strftime("%Y%m%d-%H%M") + +parser = argparse.ArgumentParser() + +parser.add_argument( + '--compile-only', + action='store_true' # default: False +) + +args = parser.parse_args() + + +# Import model and convert to Artifact +@dsl.component(base_image=IMAGE) +def get_unmanaged_model(model: Input[Model], unmanaged_model: Output[Artifact]): + unmanaged_model.metadata = model.metadata + unmanaged_model.uri = '/'.join(model.uri.split('/')[:-1]) # remove filename after last / - send dir rather than file + + +######################### +### Define pipeline +######################### +@dsl.pipeline( + pipeline_root=PIPELINE_ROOT, + name=PIPELINE_NAME +) +def pipeline( + bq_table: str = "", + xgboost_param_max_depth: int=10, + xgboost_param_learning_rate: float=0.1, + xgboost_param_n_estimators: int=200, + serving_container_image_uri: str = PRED_CONTAINER, +): + + load_data_op = get_dataframe( + bq_table=bq_table, + project_id=PROJECT_ID, + class_names=CLASS_NAMES).set_display_name("Load And Split Data") + + + train_op = xgb_train( + train_data = load_data_op.outputs['train_data'], + test_data = load_data_op.outputs['test_data'], + xgboost_param_max_depth = xgboost_param_max_depth, + xgboost_param_learning_rate = xgboost_param_learning_rate, + xgboost_param_n_estimators = xgboost_param_n_estimators, + serving_container_image_uri=serving_container_image_uri + ).set_display_name("Train Model") + + + evaluate_model_op = evaluate_model( + test_data=load_data_op.outputs['val_data'], + trained_model=train_op.outputs['model'], + class_names=CLASS_NAMES, + target_column=TARGET_COLUMN + ).set_display_name("Model Card Graphics") + + + upload_op = upload_model( + project_id = PROJECT_ID, + region = REGION, + model = train_op.outputs['model'], + display_name = MODEL_DISPLAY_NAME, + serving_image = PRED_CONTAINER, + parent_model = PARENT_MODEL, + run = dsl.PIPELINE_JOB_NAME_PLACEHOLDER, + run_id = dsl.PIPELINE_JOB_ID_PLACEHOLDER + ).set_display_name("Upload Model") + + + with open(MODEL_CARD_CONFIG, 'r') as f: + model_card_config = ' '.join([x.strip() for x in f.readlines()]) + + _ = plot_model_card( + project_id = PROJECT_ID, + region = REGION, + model = upload_op.outputs['uploaded_model'], + train_data = load_data_op.outputs['train_data'], + test_data = load_data_op.outputs['test_data'], + val_data = load_data_op.outputs['val_data'], + stats = load_data_op.outputs['stats'], + reports = evaluate_model_op.outputs['reports'], + model_card_config = model_card_config + ).set_display_name("Generate Model Card") + + # + # Online Endpoint + # + + create_endpoint_op = EndpointCreateOp( + project = PROJECT_ID, + location = REGION, + display_name = ENDPOINT_NAME + ).set_display_name("Create Vertex AI Endpoint") + + _ = ModelDeployOp( + model=upload_op.outputs['uploaded_model'], + endpoint=create_endpoint_op.outputs['endpoint'], + dedicated_resources_machine_type = 'n1-standard-8', + dedicated_resources_min_replica_count = 1, + dedicated_resources_max_replica_count = 1, + enable_access_logging = True + ).set_display_name("Deploy Model To Endpoint") + + # Start Model Monitoring job. + # Fails intermittently. Enable after bugfix: https://github.com/googleapis/python-aiplatform/issues/2361 + # _ = model_monitoring( + # project_id=PROJECT_ID, + # region=REGION, + # endpoint=create_endpoint_op.outputs['endpoint'], + # pipeline_id=dsl.PIPELINE_JOB_NAME_PLACEHOLDER, + # bq_train_data=bq_table, + # skew_threshold=0.5, + # sampling_rate=1.0, + # monitoring_interval_hours=1, + # user_emails=EMAILS + # ).set_display_name("Enable Model Montoring") + + # + # Evaluation Pipeline + # + + upload_to_bq_op = upload_to_bq( + project=PROJECT_ID, + location=REGION, + csv_data=load_data_op.outputs['val_data'], + dest_dataset_id=BQ_OUTPUT_DATASET_ID, + dest_table_id=f'{PIPELINE_NAME}-val-{TIMESTAMP}' + ).set_display_name("Upload to BigQuery") + + # Run the batch prediction task + batch_predict_op = ModelBatchPredictOp( + project=PROJECT_ID, + location=REGION, + model=upload_op.outputs['uploaded_model'], + job_display_name=f"bp-{PIPELINE_NAME}-{TIMESTAMP}", + bigquery_source_input_uri=upload_to_bq_op.outputs['bq_table_uri'], + instances_format="bigquery", + predictions_format="bigquery", + bigquery_destination_output_uri=f"bq://{PROJECT_ID}.{BQ_OUTPUT_DATASET_ID}.{PIPELINE_NAME}-bp-{TIMESTAMP}", + excluded_fields=[TARGET_COLUMN], + machine_type="n1-standard-8", + starting_replica_count=2, + max_replica_count=8, + ).set_display_name("Batch Prediction") + + # Format the predictions column from "0.1" that xgboost produces to "[0.9, 0.1]" that sklearn produces + reformat_predictions_op = reformat_predictions_bq( + project=PROJECT_ID, + location=REGION, + input_predictions=batch_predict_op.outputs['bigquery_output_table'] + ).set_display_name("Reformat Predictions") + + # Run the evaluation based on prediction type + eval_task = ModelEvaluationClassificationOp( + project=PROJECT_ID, + location=REGION, + class_labels=CLASS_NAMES, + prediction_score_column= "prediction", + target_field_name=TARGET_COLUMN, + ground_truth_format="bigquery", + ground_truth_bigquery_source=upload_to_bq_op.outputs['bq_table_uri'], + predictions_format="bigquery", + predictions_bigquery_source=reformat_predictions_op.outputs['predictions'], + dataflow_service_account=DATAFLOW_SA, + dataflow_subnetwork=DATAFLOW_NETWORK, + dataflow_use_public_ips=DATAFLOW_PUBLIC_IPS, + force_runner_mode='Dataflow' + ).set_display_name("Model Evaluation") + + # Import the model evaluations to the Vertex AI model in Model Registry + ModelImportEvaluationOp( + classification_metrics=eval_task.outputs["evaluation_metrics"], + model=upload_op.outputs['uploaded_model'], + dataset_type="bigquery", + ).set_display_name("Import Model Evaluation") + +# Compile and run the pipeline +aiplatform.init(project=PROJECT_ID, location=REGION, encryption_spec_key_name=KEY_ID) + +logging.getLogger().setLevel(logging.INFO) +logging.info(f"Init with project {PROJECT_ID} in region {REGION}. Pipeline root: {PIPELINE_ROOT}") + +FORMAT = ".json" + +logging.info(f"Compiling pipeline to {PIPELINE_NAME + FORMAT}") +compiler.Compiler().compile( + pipeline_func=pipeline, + package_path=PIPELINE_NAME + FORMAT +) + +if not args.compile_only: + run = aiplatform.PipelineJob( + project=PROJECT_ID, + location=REGION, + display_name=PIPELINE_NAME, + template_path=PIPELINE_NAME + FORMAT, + job_id=f"{PIPELINE_NAME}-{TIMESTAMP}", + pipeline_root=PIPELINE_ROOT, + parameter_values={ + "bq_table": BQ_INPUT_DATA, + "xgboost_param_max_depth": 5, + "xgboost_param_learning_rate": 0.1, + "xgboost_param_n_estimators": 20}, + enable_caching=caching + ) + + run.submit(service_account=SERVICE_ACCOUNT, + network=NETWORK) + +# This can be used to test the online endpoint: +# +# { +# "instances": [ +# [1.18998913145894,-0.563413492993846,0.129352538697985,-0.302175771438239,-0.927677605983222,-0.784678753251055,-0.443713590138326,-0.0956435854887243,-0.648897198590765,0.0499810894390051,0.358011190903553,-0.445067055832097,-0.0982544178676521,-1.28002825726001,0.304411501372465,0.733464325722348,1.71246876228603,-1.78636925309304,0.163898890406551,0.180489467655959,0.0091417811964457,-0.074443134391428,-0.0011569207049818,0.327529344882462,0.332585093864499,-0.298508896918417,0.0256419259293034,0.0496775221663426,80.52] +# ] +# } \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/reformat_preds.py b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/reformat_preds.py new file mode 100644 index 0000000000..85e3037c22 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/reformat_preds.py @@ -0,0 +1,77 @@ +from kfp import dsl +from typing import NamedTuple +from google_cloud_pipeline_components.types.artifact_types import BQTable + +from config import IMAGE + + +# Format the predictions column from "0.1" that xgboost produces to "[0.9, 0.1]" that sklearn produces +@dsl.component(base_image=IMAGE) +def reformat_predictions_bq( + project: str, + location: str, + input_predictions: dsl.Input[BQTable], + predictions: dsl.Output[BQTable] +): + + from google.cloud.bigquery import Client + import logging + + bq = Client(project=project, location=location) + table_project = input_predictions.metadata['projectId'] + table_dataset = input_predictions.metadata['datasetId'] + table_table = input_predictions.metadata['tableId'] + table_ref = f"{table_project}.{table_dataset}.{table_table}" + + sql = f""" + CREATE OR REPLACE TABLE `{table_ref}_reformat` AS + SELECT * EXCEPT(prediction), + '[' || CAST(1.0-CAST(prediction AS FLOAT64) AS STRING) || ',' || prediction || ']' as prediction + FROM `{table_ref}`""" + + logging.info(f"Processing data in table {table_ref}") + logging.info(f"Query: {sql}") + job = bq.query(sql) + + job.result() # wait for completion + + predictions.metadata['projectId'] = table_project + predictions.metadata['datasetId'] = table_dataset + predictions.metadata['tableId'] = table_table + "_reformat" + + +@dsl.component(base_image=IMAGE) +def reformat_groundtruth_json( + gcs_sources: list, + gcs_groundtruth: dsl.OutputPath("Dataset")) -> NamedTuple('outputs', [('gcs_output_uris', list)]): + + from collections import namedtuple + import pandas as pd + import json + + df = None + for gcs_uri in gcs_sources: + fname = '/gcs' + '/'.join(gcs_uri.split('/')[1:]) # /gcs/bucket/a/b/c + + if df: + df = pd.concat(df, pd.read_csv(fname)) + else: + df = pd.read_csv(fname) + + json_data_raw = df.to_json(orient='records', lines=True) + json_data_raw_lines = [line for line in json_data_raw.split('\n') if len(line) > 0] + + with open(gcs_groundtruth, 'w') as f: + for row_str in json_data_raw_lines: + row = json.loads(row_str) + target_col = 'Class' + instance_data = [row[k] for k in row.keys() if k != target_col] + row_format = {'instance': instance_data, target_col: row[target_col]} + instance = json.dumps(row_format) + f.write(instance + '\n') + + t = namedtuple('outputs', ['gcs_output_uris']) + # transform from /gcs/a/b/c to gs://a/b/c + gcs_path = 'gs://' + '/'.join(gcs_groundtruth.split('/')[2:]) + + return t([gcs_path]) diff --git a/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/train.py b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/train.py new file mode 100644 index 0000000000..4f6e8e1971 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/kfp_pipelines/src/train.py @@ -0,0 +1,145 @@ +from kfp.v2.dsl import (Dataset, Input, Model, Output, + ClassificationMetrics, Metrics) +from kfp import dsl +from sklearn.metrics import accuracy_score, confusion_matrix, f1_score + +import pandas as pd +import xgboost as xgb +import numpy as np +from hypertune import HyperTune + +import logging +import argparse +import sys +import pickle + +from config import CLASS_NAMES, IMAGE, PRED_CONTAINER + +def load_data(dataset_path: str): + df = pd.read_csv(dataset_path) + labels = list(np.rint(df.pop("Class")).astype(np.int64)) + + data = df.values.tolist() + + # we need to convert it to numpy to avoid + # 'list' object has no attribute 'shape' errors in xgb.fit() + return (np.asarray(data, dtype=object), labels) + +def train( + train_dataset_path: str, + test_dataset_path: str, + xgboost_param_max_depth: int, + xgboost_param_learning_rate: float, + xgboost_param_n_estimators: int, + model_output_path, + serving_container_image_uri, + metrics: Output[Metrics] = None, + metricsc: Output[ClassificationMetrics] = None, + model: Output[Model] = None,): + + x_train, y_train = load_data(train_dataset_path) + x_test, y_test = load_data(test_dataset_path) + + logging.info(f"Train X {type(x_train)} {len(x_train)}") + logging.info(f"Train Y {type(y_train)} {len(y_train)}: {y_train[:5]}") + + classifier = xgb.XGBClassifier( + max_depth=xgboost_param_max_depth, + learning_rate=xgboost_param_learning_rate, + n_estimators=xgboost_param_n_estimators) + logging.info(f"Model {classifier}") + classifier.fit(x_train, y_train) + + # log metrics + print(f"Type y_test {type(y_test[0])}") + print(f"Type pred {type(classifier.predict(x_test)[0])}") + + score = accuracy_score(y_test, classifier.predict(x_test)) + f1 = f1_score(y_test, classifier.predict(x_test)) + logging.info("accuracy is: %s", score) + logging.info("F1 score is: %s", f1) + if metrics: + metrics.log_metric("accuracy",(score * 100.0)) + metrics.log_metric("f1-score", f1) + metrics.log_metric("framework", f"XGBoost {xgb.__version__}") + metrics.log_metric("train_dataset_size", len(x_train)) + metrics.log_metric("test_dataset_size", len(x_test)) + + if metricsc: + # log the confusion matrix + y_pred = classifier.predict(x_test) + logging.info(f"Predictions: {','.join(map(str, y_pred[:10]))}") + + metricsc.log_confusion_matrix( + CLASS_NAMES, + confusion_matrix(y_test, y_pred).tolist() # to convert np array to list. + ) + + # report for hyperparameter tuning + hyper_tune = HyperTune() + hyper_tune.report_hyperparameter_tuning_metric( + hyperparameter_metric_tag='f1', + metric_value=f1) + + + # Write Pickle for convenience: This is what we trained and will have sklearn interface + pickle_output_path = model_output_path + '_sklearn.pkl' + with open(pickle_output_path, 'wb') as f: + pickle.dump(classifier, f) + + # Save using save_model method of XGB Classifier object + # -- this is important if we want to use the prebuilt xgb container for prediction + model_output_path = model_output_path + '.bst' + logging.info(f"Writing model to {model_output_path}") + classifier.save_model(model_output_path) + + if model: + model.metadata = { "containerSpec": { "imageUri": serving_container_image_uri } } + model.path = model_output_path + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + parser.add_argument("--train_dataset_path", required=True) + parser.add_argument("--test_dataset_path", required=True) + parser.add_argument("--xgboost_param_max_depth", required=True) + parser.add_argument("--xgboost_param_learning_rate", required=True) + parser.add_argument("--xgboost_param_n_estimators", required=True) + parser.add_argument("--model_output_path", required=True) + + print(f'Got args: {" ".join(sys.argv)}') + args = parser.parse_args() + + train(train_dataset_path=args.train_dataset_path, + test_dataset_path=args.test_dataset_path, + xgboost_param_max_depth=int(args.xgboost_param_max_depth), + xgboost_param_learning_rate=float(args.xgboost_param_learning_rate), + xgboost_param_n_estimators=int(args.xgboost_param_n_estimators), + model_output_path=args.model_output_path, + serving_container_image_uri=PRED_CONTAINER) + +@dsl.component(base_image=f'{IMAGE}') +def xgb_train( + train_data: Input[Dataset], + test_data: Input[Dataset], + metrics: Output[Metrics], + model: Output[Model], + xgboost_param_max_depth: int, + xgboost_param_learning_rate: float, + xgboost_param_n_estimators: int, + metricsc: Output[ClassificationMetrics], + serving_container_image_uri: str +): + from train import train + + train( + train_dataset_path=train_data.path, + test_dataset_path=test_data.path, + xgboost_param_max_depth=xgboost_param_max_depth, + xgboost_param_learning_rate=xgboost_param_learning_rate, + xgboost_param_n_estimators=xgboost_param_n_estimators, + model_output_path=model.path, + metrics=metrics, + metricsc=metricsc, + model=model, + serving_container_image_uri=serving_container_image_uri) diff --git a/examples/vertex_mlops_enterprise/src/kfp_pipelines/train-eval-pipeline.png b/examples/vertex_mlops_enterprise/src/kfp_pipelines/train-eval-pipeline.png new file mode 100644 index 0000000000..1f1f2738ae Binary files /dev/null and b/examples/vertex_mlops_enterprise/src/kfp_pipelines/train-eval-pipeline.png differ diff --git a/examples/vertex_mlops_enterprise/src/model_training/data.py b/examples/vertex_mlops_enterprise/src/model_training/data.py new file mode 100644 index 0000000000..cd90ecc09a --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/model_training/data.py @@ -0,0 +1,49 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Functions for reading data as tf.data.Dataset.""" + +import tensorflow as tf + +from src.common import features + + +def _gzip_reader_fn(filenames): + """Small utility returning a record reader that can read gzip'ed files.""" + return tf.data.TFRecordDataset(filenames, compression_type="GZIP") + + +def get_dataset(file_pattern, feature_spec, batch_size, epochs): + """Generates features and label for tuning/training. + Args: + file_pattern: input tfrecord file pattern. + feature_spec: a dictionary of feature specifications. + batch_size: representing the number of consecutive elements of returned + dataset to combine in a single batch + Returns: + A dataset that contains (features, indices) tuple where features is a + dictionary of Tensors, and indices is a single Tensor of label indices. + """ + + dataset = tf.data.experimental.make_batched_features_dataset( + file_pattern=file_pattern, + batch_size=batch_size, + features=feature_spec, + label_key=features.TARGET_FEATURE_NAME, + reader=_gzip_reader_fn, + num_epochs=2*epochs, + drop_final_batch=True, + ) + + + return dataset diff --git a/examples/vertex_mlops_enterprise/src/model_training/defaults.py b/examples/vertex_mlops_enterprise/src/model_training/defaults.py new file mode 100644 index 0000000000..1886ed6909 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/model_training/defaults.py @@ -0,0 +1,41 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Defaults for the model. + +These values can be tweaked to affect model training performance. +""" + + +HIDDEN_UNITS = [64, 32] +LEARNING_RATE = 0.0001 +BATCH_SIZE = 512 +NUM_EPOCHS = 10 +NUM_EVAL_STEPS = 100 + + +def update_hyperparams(hyperparams: dict) -> dict: + if "hidden_units" not in hyperparams: + hyperparams["hidden_units"] = HIDDEN_UNITS + else: + if not isinstance(hyperparams["hidden_units"], list): + hyperparams["hidden_units"] = [ + int(v) for v in hyperparams["hidden_units"].split(",") + ] + if "learning_rate" not in hyperparams: + hyperparams["learning_rate"] = LEARNING_RATE + if "batch_size" not in hyperparams: + hyperparams["batch_size"] = BATCH_SIZE + if "num_epochs" not in hyperparams: + hyperparams["num_epochs"] = NUM_EPOCHS + return hyperparams diff --git a/examples/vertex_mlops_enterprise/src/model_training/exporter.py b/examples/vertex_mlops_enterprise/src/model_training/exporter.py new file mode 100644 index 0000000000..bce5d31f4b --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/model_training/exporter.py @@ -0,0 +1,102 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Functions for exporting the model for serving.""" + +import logging + +import tensorflow as tf +import tensorflow_transform as tft +import tensorflow_data_validation as tfdv +from tensorflow_transform.tf_metadata import schema_utils +import tensorflow.keras as keras + +from src.common import features + + +def _get_serve_tf_examples_fn(classifier, tft_output, raw_feature_spec): + """Returns a function that parses a serialized tf.Example and applies TFT.""" + + classifier.tft_layer = tft_output.transform_features_layer() + + @tf.function + def serve_tf_examples_fn(serialized_tf_examples): + """Returns the output to be used in the serving signature.""" + #for key in list(raw_feature_spec.keys()): + # if key not in features.FEATURE_NAMES: + # raw_feature_spec.pop(key) + if features.TARGET_FEATURE_NAME in raw_feature_spec.keys(): + raw_feature_spec.pop(features.TARGET_FEATURE_NAME) + + parsed_features = tf.io.parse_example(serialized_tf_examples, raw_feature_spec) + + transformed_features = classifier.tft_layer(parsed_features) + logits = classifier(transformed_features) + probabilities = keras.activations.sigmoid(logits) + return {"probabilities": probabilities} + + return serve_tf_examples_fn + + +def _get_serve_features_fn(classifier, tft_output): + """Returns a function that accept a dictionary of features and applies TFT.""" + + classifier.tft_layer = tft_output.transform_features_layer() + + @tf.function + def serve_features_fn(raw_features): + """Returns the output to be used in the serving signature.""" + + transformed_features = classifier.tft_layer(raw_features) + logits = classifier(transformed_features) + neg_probabilities = keras.activations.sigmoid(logits) + pos_probabilities = 1 - neg_probabilities + probabilities = tf.concat([neg_probabilities, pos_probabilities], -1) + batch_size = tf.shape(probabilities)[0] + classes = tf.repeat([features.TARGET_LABELS], [batch_size], axis=0) + return {"classes": classes, "scores": probabilities} + + return serve_features_fn + + +def export_serving_model( + classifier, serving_model_dir, raw_schema_location, tft_output_dir +): + + raw_schema = tfdv.load_schema_text(raw_schema_location) + raw_feature_spec = schema_utils.schema_as_feature_spec(raw_schema).feature_spec + + tft_output = tft.TFTransformOutput(tft_output_dir) + + features_input_signature = { + feature_name: tf.TensorSpec( + shape=(None, 1), dtype=spec.dtype, name=feature_name + ) + for feature_name, spec in raw_feature_spec.items() + if feature_name != features.TARGET_FEATURE_NAME + } + + signatures = { + "serving_default": _get_serve_features_fn( + classifier, tft_output + ).get_concrete_function(features_input_signature), + "serving_tf_example": _get_serve_tf_examples_fn( + classifier, tft_output, raw_feature_spec + ).get_concrete_function( + tf.TensorSpec(shape=[None], dtype=tf.string, name="examples") + ), + } + + logging.info("Model export started...") + classifier.save(serving_model_dir, signatures=signatures) + logging.info("Model export completed.") diff --git a/examples/vertex_mlops_enterprise/src/model_training/model.py b/examples/vertex_mlops_enterprise/src/model_training/model.py new file mode 100644 index 0000000000..59220cbf0f --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/model_training/model.py @@ -0,0 +1,41 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A DNN keras classification model.""" + +import tensorflow as tf +from tensorflow import keras +from tensorflow.keras import layers, activations +import logging + +from src.common import features + + +def create_model(feature_keys, hyperparams) -> keras.Model: + + inputs = [layers.Input(shape=(1,), name=f) for f in feature_keys if f != features.TARGET_FEATURE_NAME] + d = layers.concatenate(inputs) + + for units in hyperparams['hidden_units']: + d = layers.Dense(units, activation=activations.relu)(d) + + if 'dropout' in hyperparams: + d = tf.keras.layers.Dropout(hyperparams['dropout'])(d) + + outputs = layers.Dense(1, activation=activations.sigmoid)(d) + + model = keras.Model(inputs=inputs, outputs=outputs) + + model.summary(print_fn=logging.info) + + return model diff --git a/examples/vertex_mlops_enterprise/src/model_training/runner.py b/examples/vertex_mlops_enterprise/src/model_training/runner.py new file mode 100644 index 0000000000..4ff52712c9 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/model_training/runner.py @@ -0,0 +1,63 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A run_fn method called by the TFX Trainer component.""" + +import os +import logging + +from src.model_training import trainer, exporter, defaults + + +# TFX Trainer will call this function. +def run_fn(fn_args): + """Train the model based on given args. + Args: + fn_args: Holds args used to train the model as name/value pairs. + """ + logging.info("Runner started...") + logging.info(f"fn_args: {fn_args}") + logging.info("") + + try: + log_dir = fn_args.model_run_dir + except KeyError: + log_dir = os.path.join(os.path.dirname(fn_args.serving_model_dir), "logs") + + hyperparams = fn_args.hyperparameters + if not hyperparams: + hyperparams = dict() + + hyperparams = defaults.update_hyperparams(hyperparams) + logging.info("Hyperparameter:") + logging.info(hyperparams) + logging.info("") + + logging.info("Runner executing trainer...") + classifier = trainer.train( + train_data_dir=fn_args.train_files, + eval_data_dir=fn_args.eval_files, + tft_output_dir=fn_args.transform_output, + hyperparams=hyperparams, + log_dir=log_dir, + base_model_dir=fn_args.base_model, + ) + + logging.info("Runner executing exporter...") + exporter.export_serving_model( + classifier=classifier, + serving_model_dir=fn_args.serving_model_dir, + raw_schema_location=fn_args.schema_path, + tft_output_dir=fn_args.transform_output, + ) + logging.info("Runner completed.") diff --git a/examples/vertex_mlops_enterprise/src/model_training/trainer.py b/examples/vertex_mlops_enterprise/src/model_training/trainer.py new file mode 100644 index 0000000000..a6ec87aef7 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/model_training/trainer.py @@ -0,0 +1,142 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Train and evaluate the model.""" + +import logging +import tensorflow as tf +import tensorflow_transform as tft +from tensorflow import keras +import keras.backend as K + +from src.model_training import data, model + + +def train( + train_data_dir, + eval_data_dir, + tft_output_dir, + hyperparams, + log_dir, + base_model_dir=None, + run=None +): + + logging.info(f"Loading tft output from {tft_output_dir}") + tft_output = tft.TFTransformOutput(tft_output_dir) + transformed_feature_spec = tft_output.transformed_feature_spec() + + batch_size = int(hyperparams["batch_size"]) + epochs = int(hyperparams["num_epochs"]) + steps_per_epoch = int(hyperparams["steps_per_epoch"]) + + train_dataset = data.get_dataset( + train_data_dir, + transformed_feature_spec, + batch_size, + epochs + ) + + eval_dataset = data.get_dataset( + eval_data_dir, + transformed_feature_spec, + batch_size, + epochs + ) + + optimizer = keras.optimizers.Adam(learning_rate=hyperparams["learning_rate"]) + loss = keras.losses.BinaryCrossentropy(from_logits=True) + #loss = f1_weighted_loss + + acc_name = f'accuracy_{run}' if run else 'accuracy' + auc_name = f'auc_{run}' if run else 'auc' + metrics = [keras.metrics.BinaryAccuracy(name=acc_name), keras.metrics.AUC(curve='PR', name=auc_name)] + if run: + # we need one just called "accuracy" as well, to have one metric to optimize across runs for HP tuning + metrics.append(keras.metrics.BinaryAccuracy(name='accuracy')) + + + early_stopping = tf.keras.callbacks.EarlyStopping( + monitor="val_loss", patience=5, restore_best_weights=True + ) + tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir) + + classifier = model.create_model(transformed_feature_spec.keys(), hyperparams) + if base_model_dir: + try: + classifier = keras.load_model(base_model_dir) + except: + pass + + classifier.compile(optimizer=optimizer, loss=loss, metrics=metrics) + + logging.info(f"Model training started... steps per epoch = {steps_per_epoch}") + classifier.fit( + train_dataset, + epochs=hyperparams["num_epochs"], + steps_per_epoch=steps_per_epoch, + validation_data=eval_dataset, + callbacks=[early_stopping, tensorboard_callback], + ) + logging.info("Model training completed.") + + return classifier + + +def evaluate(model, data_dir, raw_schema_location, tft_output_dir, hyperparams): + logging.info(f"Loading raw schema from {raw_schema_location}") + + logging.info(f"Loading tft output from {tft_output_dir}") + tft_output = tft.TFTransformOutput(tft_output_dir) + transformed_feature_spec = tft_output.transformed_feature_spec() + + logging.info("Model evaluation started...") + eval_dataset = data.get_dataset( + data_dir, + transformed_feature_spec, + int(hyperparams["batch_size"]), + 1 + ) + + evaluation_metrics = model.evaluate(eval_dataset) + logging.info("Model evaluation completed.") + + return evaluation_metrics + + +def f1(y_true, y_pred): + y_pred = K.round(y_pred) + tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0) + fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0) + fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0) + + p = tp / (tp + fp + K.epsilon()) + r = tp / (tp + fn + K.epsilon()) + + f1 = 2*p*r / (p+r+K.epsilon()) + f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1) + return K.mean(f1) + + +def f1_loss(y_true, y_pred): + + tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0) + fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0) + fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0) + + p = tp / (tp + fp + K.epsilon()) + r = tp / (tp + fn + K.epsilon()) + + f1 = 2*p*r / (p+r+K.epsilon()) + f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1) + return 1 - K.mean(f1) diff --git a/examples/vertex_mlops_enterprise/src/pipeline_triggering/main.py b/examples/vertex_mlops_enterprise/src/pipeline_triggering/main.py new file mode 100644 index 0000000000..fbd5da73d2 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/pipeline_triggering/main.py @@ -0,0 +1,69 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Cloud Function to be triggered by Pub/Sub.""" + +import os +import json +import logging +from google.cloud import aiplatform +from google.cloud import storage +import base64 + + +def trigger_pipeline(event, context): + + project = os.getenv("PROJECT") + region = os.getenv("REGION") + sa = os.getenv("SERVICE_ACCOUNT") + pipeline_name = os.getenv("PIPELINE_NAME") + gcs_pipeline_file_location = os.getenv("GCS_PIPELINE_FILE_LOCATION") + + if not project: + raise ValueError("Environment variable PROJECT is not set.") + if not region: + raise ValueError("Environment variable REGION is not set.") + if not gcs_pipeline_file_location: + raise ValueError("Environment variable GCS_PIPELINE_FILE_LOCATION is not set.") + if not sa: + raise ValueError("Environment variable SERVICE_ACCOUNT is not set.") + + storage_client = storage.Client() + + path_parts = gcs_pipeline_file_location.replace("gs://", "").split("/") + bucket_name = path_parts[0] + blob_name = "/".join(path_parts[1:]) + + bucket = storage_client.bucket(bucket_name) + blob = storage.Blob(bucket=bucket, name=blob_name) + + if not blob.exists(storage_client): + raise ValueError(f"{gcs_pipeline_file_location} does not exist.") + + data = base64.b64decode(event["data"]).decode("utf-8") + logging.info(f"Event data: {data}") + + parameter_values = json.loads(data) + job = aiplatform.PipelineJob(display_name = pipeline_name, + template_path = gcs_pipeline_file_location, + parameter_values = parameter_values, + project = project, + location = region) + + response = job.submit(service_account=sa, + network=None) + + + logging.info(response) + + diff --git a/examples/vertex_mlops_enterprise/src/pipeline_triggering/requirements.txt b/examples/vertex_mlops_enterprise/src/pipeline_triggering/requirements.txt new file mode 100644 index 0000000000..1969b67ca1 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/pipeline_triggering/requirements.txt @@ -0,0 +1,3 @@ +kfp==1.8.12 +google-cloud-aiplatform +google-cloud-storage \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/src/prediction_cf/main.py b/examples/vertex_mlops_enterprise/src/prediction_cf/main.py new file mode 100644 index 0000000000..2a0798896a --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/prediction_cf/main.py @@ -0,0 +1,86 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Cloud Function to serve as the prediction endpoint.""" + +import os +import sys +import json +import logging +from google.cloud import aiplatform as vertex_ai + + +# To enable the Cloud Function to work out the +# next import +workdir = '/workspace' +if not workdir in sys.path: + sys.path.append(workdir) + +from src.feature_store import feature_store as fs + + +# if the trigger is HTTP, the argument is a request +def predict(request): + + def getenv(env_var_name): + v = os.getenv(env_var_name) + if not v: + raise ValueError(f"Environment variable {env_var_name} is not set.") + return v + + project = getenv("PROJECT") + region = getenv("REGION") + endpoint_name = getenv("ENDPOINT_NAME") + entity = getenv("ENTITY") + store_id = getenv("FEATURESTORE_ID") + + default_v27 = 0. + default_v28 = 0. + + raw_features = request.get_json(silent=False) + logging.error(f"Raw features: {raw_features}") + + # Get userid and query Feature Store + fs_features = {} + + if 'userid' in raw_features: + try: + fs_features = fs.read_features(project, region, store_id, entity, ['v27', 'v28'], raw_features['userid']) + except: + logging.warn("Feature store is not available") + if not 'v27' in fs_features: + logging.error(f'User {raw_features["userid"]} not present in Feature Store, using defaults') + fs_features['v27'] = default_v27 + if not 'v28' in fs_features: + fs_features['v28'] = default_v28 + else: + logging.error('No userid provided, using defaults') + return {'error': 'No userid'} + + if 'V1' not in raw_features: + return {'error': 'Missing features. Expect V1 to V26 and userid.'} + + feature_data = raw_features.copy() + del feature_data['userid'] + feature_data['V27'] = [fs_features['v27']] + feature_data['V28'] = [fs_features['v28']] + + # Query model on Vertex AI Endpoint + endpoint = vertex_ai.Endpoint(endpoint_name, project, region) + logging.error(f"Calling endpoint with: {feature_data}") + prediction = endpoint.predict([feature_data]).predictions[0] + logging.error(f"Prediction: {prediction}") + + return json.dumps(prediction) + + diff --git a/examples/vertex_mlops_enterprise/src/preprocessing/etl.py b/examples/vertex_mlops_enterprise/src/preprocessing/etl.py new file mode 100644 index 0000000000..b135a3a43f --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/preprocessing/etl.py @@ -0,0 +1,207 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Data preprocessing pipelines.""" + +import os + +import tensorflow_transform as tft +import tensorflow_data_validation as tfdv +import apache_beam as beam +import tensorflow_transform.beam as tft_beam +from tensorflow_transform.tf_metadata import dataset_metadata +from tensorflow_transform.tf_metadata import schema_utils + + +from src.preprocessing import transformations + +RAW_SCHEMA_LOCATION = "src/raw_schema/schema.pbtxt" + + +def parse_bq_record(bq_record): + output = {} + for key in bq_record: + output[key] = [bq_record[key]] + return output + + +def split_dataset(bq_row, num_partitions, ratio): + import json + + assert num_partitions == len(ratio) + bucket = sum(map(ord, json.dumps(bq_row))) % sum(ratio) + total = 0 + for i, part in enumerate(ratio): + total += part + if bucket < total: + return i + return len(ratio) - 1 + + +def run_transform_pipeline(args): + + pipeline_options = beam.pipeline.PipelineOptions(flags=[], **args) + + raw_data_query = args["raw_data_query"] + write_raw_data = args["write_raw_data"] + exported_data_prefix = args["exported_data_prefix"] + transformed_data_prefix = args["transformed_data_prefix"] + transform_artifact_dir = args["transform_artifact_dir"] + temporary_dir = args["temporary_dir"] + gcs_location = args["gcs_location"] + project = args["project"] + + source_raw_schema = tfdv.load_schema_text(RAW_SCHEMA_LOCATION) + raw_feature_spec = schema_utils.schema_as_feature_spec( + source_raw_schema + ).feature_spec + + raw_metadata = dataset_metadata.DatasetMetadata( + schema_utils.schema_from_feature_spec(raw_feature_spec) + ) + + with beam.Pipeline(options=pipeline_options) as pipeline: + with tft_beam.Context(temporary_dir): + + # Read raw BigQuery data. + raw_train_data, raw_eval_data = ( + pipeline + | "Read Raw Data" + >> beam.io.ReadFromBigQuery( + query=raw_data_query, + project=project, + use_standard_sql=True, + gcs_location=gcs_location, + ) + | "Parse Data" >> beam.Map(parse_bq_record) + | "Split" >> beam.Partition(split_dataset, 2, ratio=[8, 2]) + ) + + # Create a train_dataset from the data and schema. + raw_train_dataset = (raw_train_data, raw_metadata) + + # Analyze and transform raw_train_dataset to produced transformed_train_dataset and transform_fn. + transformed_train_dataset, transform_fn = ( + raw_train_dataset + | "Analyze & Transform" + >> tft_beam.AnalyzeAndTransformDataset(transformations.preprocessing_fn) + ) + + # Get data and schema separately from the transformed_dataset. + transformed_train_data, transformed_metadata = transformed_train_dataset + + # write transformed train data. + _ = ( + transformed_train_data + | "Write Transformed Train Data" + >> beam.io.tfrecordio.WriteToTFRecord( + file_path_prefix=os.path.join( + transformed_data_prefix, "train/data" + ), + file_name_suffix=".gz", + coder=tft.coders.ExampleProtoCoder(transformed_metadata.schema), + ) + ) + + # Create a eval_dataset from the data and schema. + raw_eval_dataset = (raw_eval_data, raw_metadata) + + # Transform raw_eval_dataset to produced transformed_eval_dataset using transform_fn. + transformed_eval_dataset = ( + raw_eval_dataset, + transform_fn, + ) | "Transform" >> tft_beam.TransformDataset() + + # Get data from the transformed_eval_dataset. + transformed_eval_data, _ = transformed_eval_dataset + + # write transformed train data. + _ = ( + transformed_eval_data + | "Write Transformed Eval Data" + >> beam.io.tfrecordio.WriteToTFRecord( + file_path_prefix=os.path.join(transformed_data_prefix, "eval/data"), + file_name_suffix=".gz", + coder=tft.coders.ExampleProtoCoder(transformed_metadata.schema), + ) + ) + + # Write transform_fn. + _ = transform_fn | "Write Transform Artifacts" >> tft_beam.WriteTransformFn( + transform_artifact_dir + ) + + if write_raw_data: + # write raw eval data. + _ = ( + raw_eval_data + | "Write Raw Eval Data" + >> beam.io.tfrecordio.WriteToTFRecord( + file_path_prefix=os.path.join(exported_data_prefix, "data"), + file_name_suffix=".tfrecord", + coder=tft.coders.ExampleProtoCoder(raw_metadata.schema), + ) + ) + + +def convert_to_jsonl(bq_record): + import json + + output = {} + for key in bq_record: + output[key] = [bq_record[key]] + return json.dumps(output) + + +def run_extract_pipeline(args): + + pipeline_options = beam.pipeline.PipelineOptions(flags=[], **args) + + sql_query = args["sql_query"] + exported_data_prefix = args["exported_data_prefix"] + temporary_dir = args["temporary_dir"] + gcs_location = args["gcs_location"] + project = args["project"] + + with beam.Pipeline(options=pipeline_options) as pipeline: + with tft_beam.Context(temporary_dir): + + # Read BigQuery data. + raw_data = ( + pipeline + | "Read Data" + >> beam.io.ReadFromBigQuery( + query=sql_query, + project=project, + use_standard_sql=True, + gcs_location=gcs_location, + ) + | "Parse Data" >> beam.Map(convert_to_jsonl) + ) + + # Write raw data to GCS as JSONL files. + _ = raw_data | "Write Data" >> beam.io.WriteToText( + file_path_prefix=exported_data_prefix, file_name_suffix=".jsonl" + ) + + +def parse_prediction_results(jsonl): + import uuid + import json + + prediction_results = json.loads(jsonl)["prediction"] + prediction_id = str(uuid.uuid4()) + scores = prediction_results["scores"] + classes = prediction_results["classes"] + + return {"prediction_id": prediction_id, "scores": scores, "classes": classes} diff --git a/examples/vertex_mlops_enterprise/src/preprocessing/transformations.py b/examples/vertex_mlops_enterprise/src/preprocessing/transformations.py new file mode 100644 index 0000000000..a2f2a9d1eb --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/preprocessing/transformations.py @@ -0,0 +1,30 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TensorFlow Transform preprocessing function.""" + +import tensorflow_transform as tft + + +def preprocessing_fn(inputs): + output_dict = { + 'Amount': tft.scale_to_z_score(inputs['Amount']), + 'Class': inputs['Class'] + } + + # Let's normalize all the columns that start with V + for col in inputs.keys(): + if col.startswith("V"): + output_dict[col] = tft.scale_to_z_score(inputs[col]) + + return output_dict \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/src/raw_schema/schema.pbtxt b/examples/vertex_mlops_enterprise/src/raw_schema/schema.pbtxt new file mode 100644 index 0000000000..9a64790266 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/raw_schema/schema.pbtxt @@ -0,0 +1,390 @@ +feature { + name: "V1" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V2" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V3" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V4" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V5" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V6" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V7" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V8" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V9" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V10" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V11" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V12" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V13" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V14" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V15" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V16" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V17" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V18" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V19" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V20" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V21" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V22" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V23" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V24" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V25" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V26" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V27" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "V28" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "Amount" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} +feature { + name: "Class" + type: FLOAT + presence { + min_fraction: 1.0 + min_count: 1 + } + shape { + dim { + size: 1 + } + } +} diff --git a/examples/vertex_mlops_enterprise/src/tests/__init__.py b/examples/vertex_mlops_enterprise/src/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/vertex_mlops_enterprise/src/tests/datasource_utils_tests.py b/examples/vertex_mlops_enterprise/src/tests/datasource_utils_tests.py new file mode 100644 index 0000000000..8536abe744 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/tests/datasource_utils_tests.py @@ -0,0 +1,124 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test utilities for generating BigQuery data querying scripts.""" + +import sys +import os +import logging +from google.cloud import bigquery + +from src.common import datasource_utils + +root = logging.getLogger() +root.setLevel(logging.INFO) +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.INFO) +root.addHandler(handler) + +LIMIT = 100 + +TARGET_COLUMN = "Class" + +EXPECTED_TRAINING_COLUMNS = [ + "V1", + "V2", + "V3", + "V4", + "V5", + "V6", + "V7", + "V8", + "V9", + "V10", + "V11", + "V12", + "V13", + "V14", + "V15", + "V16", + "V17", + "V18", + "V19", + "V20", + "V21", + "V22", + "V23", + "V24", + "V25", + "V26", + "V27", + "V28", + "Amount", + "Class", +] + + +def test_training_query(): + + project = os.getenv("PROJECT") + location = os.getenv("BQ_LOCATION") + bq_dataset_name = os.getenv("BQ_DATASET_NAME") + bq_table_name = os.getenv("ML_TABLE") + + assert project, "Environment variable PROJECT is None!" + assert location, "Environment variable BQ_LOCATION is None!" + assert bq_dataset_name, "Environment variable BQ_DATASET_NAME is None!" + assert bq_table_name, "Environment variable ML_TABLE is None!" + + logging.info("BigQuery Source: %s.%s.%s", project, bq_dataset_name, + bq_table_name) + + query = datasource_utils.get_source_query( + bq_dataset_name=bq_dataset_name, + bq_table_name=bq_table_name, + ml_use="UNASSIGNED", + limit=LIMIT, + ) + + bq_client = bigquery.Client(project=project, location=location) + df = bq_client.query(query).to_dataframe() + columns = set(df.columns) + assert columns == set(EXPECTED_TRAINING_COLUMNS) + assert df.shape == (LIMIT, len(columns)) + + +def test_serving_query(): + + project = os.getenv("PROJECT") + location = os.getenv("BQ_LOCATION") + bq_dataset_name = os.getenv("BQ_DATASET_NAME") + bq_table_name = os.getenv("ML_TABLE") + + assert project, "Environment variable PROJECT is None!" + assert location, "Environment variable BQ_LOCATION is None!" + assert bq_dataset_name, "Environment variable BQ_DATASET_NAME is None!" + assert bq_table_name, "Environment variable ML_TABLE is None!" + + logging.info("BigQuery Source: %s.%s.%s", project, bq_dataset_name, + bq_table_name) + + query = datasource_utils.get_source_query( + bq_dataset_name=bq_dataset_name, + bq_table_name=bq_table_name, + ml_use=None, + limit=LIMIT, + ) + + bq_client = bigquery.Client(project=project, location=location) + df = bq_client.query(query).to_dataframe() + columns = set(df.columns) + expected_serving_columns = EXPECTED_TRAINING_COLUMNS + expected_serving_columns.remove(TARGET_COLUMN) + assert columns == set(expected_serving_columns) + assert df.shape == (LIMIT, len(columns)) diff --git a/examples/vertex_mlops_enterprise/src/tests/model_deployment_tests.py b/examples/vertex_mlops_enterprise/src/tests/model_deployment_tests.py new file mode 100644 index 0000000000..fe15dc42f2 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/tests/model_deployment_tests.py @@ -0,0 +1,198 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test an uploaded model to Vertex AI.""" + +import os +import logging +import tensorflow as tf +import sys + +from google.cloud import aiplatform as vertex_ai + +# configure logging to print to stdout +root = logging.getLogger() +root.setLevel(logging.DEBUG) +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.DEBUG) +root.addHandler(handler) + +test_instance = { + "V1": [-0.906611], + "V2": [-0.906611], + "V3": [-0.906611], + "V4": [-0.906611], + "V5": [-0.906611], + "V6": [-0.906611], + "V7": [-0.906611], + "V8": [-0.906611], + "V9": [-0.906611], + "V10": [-0.906611], + "V11": [-0.906611], + "V12": [-0.906611], + "V13": [-0.906611], + "V14": [-0.906611], + "V15": [-0.906611], + "V16": [-0.906611], + "V17": [-0.906611], + "V18": [-0.906611], + "V19": [-0.906611], + "V20": [-0.906611], + "V21": [-0.906611], + "V22": [-0.906611], + "V23": [-0.906611], + "V24": [-0.906611], + "V25": [-0.906611], + "V26": [-0.906611], + "V27": [-0.906611], + "V28": [-0.906611], + "Amount": [15.99], +} + +SERVING_DEFAULT_SIGNATURE_NAME = "serving_default" + + +def test_model_artifact(): + + feature_types = { + "V1": tf.dtypes.float32, + "V2": tf.dtypes.float32, + "V3": tf.dtypes.float32, + "V4": tf.dtypes.float32, + "V5": tf.dtypes.float32, + "V6": tf.dtypes.float32, + "V7": tf.dtypes.float32, + "V8": tf.dtypes.float32, + "V9": tf.dtypes.float32, + "V10": tf.dtypes.float32, + "V11": tf.dtypes.float32, + "V12": tf.dtypes.float32, + "V13": tf.dtypes.float32, + "V14": tf.dtypes.float32, + "V15": tf.dtypes.float32, + "V16": tf.dtypes.float32, + "V17": tf.dtypes.float32, + "V18": tf.dtypes.float32, + "V19": tf.dtypes.float32, + "V20": tf.dtypes.float32, + "V21": tf.dtypes.float32, + "V22": tf.dtypes.float32, + "V23": tf.dtypes.float32, + "V24": tf.dtypes.float32, + "V25": tf.dtypes.float32, + "V26": tf.dtypes.float32, + "V27": tf.dtypes.float32, + "V28": tf.dtypes.float32, + "Amount": tf.dtypes.float32, + } + + new_test_instance = {} + for key, instance in test_instance.items(): + new_test_instance[key] = tf.constant(instance, + dtype=feature_types[key]) + + print(new_test_instance) + + project = os.getenv("PROJECT") + region = os.getenv("REGION") + model_display_name = os.getenv("MODEL_DISPLAY_NAME") + + assert project, "Environment variable PROJECT is None!" + assert region, "Environment variable REGION is None!" + assert model_display_name, "Environment variable MODEL_DISPLAY_NAME is None!" + + vertex_ai.init( + project=project, + location=region, + ) + + models = vertex_ai.Model.list(filter=f"display_name={model_display_name}", + order_by="update_time") + + assert models, f"No model with display name {model_display_name} exists!" + + model = models[-1] + artifact_uri = model.gca_resource.artifact_uri + logging.info("Model artifact uri: %s", artifact_uri) + assert tf.io.gfile.exists(artifact_uri), \ + f"Model artifact uri {artifact_uri} does not exist!" + + saved_model = tf.saved_model.load(artifact_uri) + logging.info("Model loaded successfully.") + + assert (SERVING_DEFAULT_SIGNATURE_NAME in saved_model.signatures + ), f"{SERVING_DEFAULT_SIGNATURE_NAME} not in model signatures!" + + #Disabled until function is fixed +''' + prediction_fn = saved_model.signatures["serving_default"] + + predictions = prediction_fn(**new_test_instance) + logging.info("Model produced predictions.") + + keys = ["classes", "scores"] + for key in keys: + assert key in predictions, f"{key} in prediction outputs!" + + assert predictions["classes"].shape == ( + 1, + 2, + ), f"Invalid output classes shape: {predictions['classes'].shape}!" + assert predictions["scores"].shape == ( + 1, + 2, + ), f"Invalid output scores shape: {predictions['scores'].shape}!" + logging.info("Prediction output: %s", predictions) +''' + + +def test_model_endpoint(): + + project = os.getenv("PROJECT") + region = os.getenv("REGION") + model_display_name = os.getenv("MODEL_DISPLAY_NAME") + endpoint_display_name = os.getenv("ENDPOINT_DISPLAY_NAME") + + assert project, "Environment variable PROJECT is None!" + assert region, "Environment variable REGION is None!" + assert model_display_name, "Environment variable MODEL_DISPLAY_NAME is None!" + assert endpoint_display_name, \ + "Environment variable ENDPOINT_DISPLAY_NAME is None!" + + vertex_ai.init( + project=project, + location=region, + ) + + endpoints = vertex_ai.Endpoint.list( + filter=f"display_name={endpoint_display_name}", order_by="update_time") + assert ( + endpoints + ), f"Endpoint with display name {endpoint_display_name} " + \ + "does not exist! in region {region}" + + endpoint = endpoints[-1] + logging.info("Calling endpoint: %s", endpoint) + + prediction = endpoint.predict([test_instance]).predictions[0] + + keys = ["classes", "scores"] + for key in keys: + assert key in prediction, f"{key} in prediction outputs!" + + assert (len(prediction["classes"]) == 2 + ), f"Invalid number of output classes: {len(prediction['classes'])}!" + assert (len(prediction["scores"]) == 2 + ), f"Invalid number output scores: {len(prediction['scores'])}!" + + logging.info("Prediction output: %s", prediction) diff --git a/examples/vertex_mlops_enterprise/src/tests/model_tests.py b/examples/vertex_mlops_enterprise/src/tests/model_tests.py new file mode 100644 index 0000000000..a13e56e216 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/tests/model_tests.py @@ -0,0 +1,82 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test model functions.""" + +import sys +import logging +import tensorflow as tf + +from src.model_training import model, defaults + +root = logging.getLogger() +root.setLevel(logging.INFO) +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.INFO) +root.addHandler(handler) + +EXPECTED_HYPERPARAMS_KEYS = [ + "hidden_units", + "learning_rate", + "batch_size", + "num_epochs", +] + + +def test_hyperparams_defaults(): + hyperparams = {"hidden_units": [64, 32]} + + hyperparams = defaults.update_hyperparams(hyperparams) + assert set(hyperparams.keys()) == set(EXPECTED_HYPERPARAMS_KEYS) + + +def test_create_model(): + + hyperparams = hyperparams = defaults.update_hyperparams({}) + + model_inputs = { + "V1": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V2": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V3": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V4": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V5": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V6": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V7": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V8": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V9": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V10": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V11": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V12": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V13": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V14": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V15": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V16": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V17": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V18": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V19": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V20": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V21": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V22": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V23": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V24": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V25": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V26": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V27": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "V28": tf.convert_to_tensor([-0.9066112, -0.9066112, -0.9066112]), + "Amount": tf.convert_to_tensor([10, 100, 1000]), + } + + classifier = model.create_model(model_inputs.keys(), hyperparams) + model_outputs = classifier(model_inputs) # .numpy() + assert model_outputs.shape == (3, 1) + assert model_outputs.dtype == "float32" diff --git a/examples/vertex_mlops_enterprise/src/tests/pipeline_deployment_tests.py b/examples/vertex_mlops_enterprise/src/tests/pipeline_deployment_tests.py new file mode 100644 index 0000000000..805b68ef1e --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/tests/pipeline_deployment_tests.py @@ -0,0 +1,91 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test training pipeline using local runner.""" + +import sys +import os +from tfx.orchestration.local.local_dag_runner import LocalDagRunner +import tensorflow as tf +from ml_metadata.proto import metadata_store_pb2 +import logging + +from src.tfx_pipelines import config +from src.tfx_pipelines import training_pipeline + +root = logging.getLogger() +root.setLevel(logging.INFO) +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.INFO) +root.addHandler(handler) + +MLMD_SQLLITE = "mlmd.sqllite" +NUM_EPOCHS = 1 +BATCH_SIZE = 512 +LEARNING_RATE = 0.001 +HIDDEN_UNITS = "128,128" + + +def test_e2e_pipeline(): + + project = os.getenv("PROJECT") + region = os.getenv("REGION") + model_display_name = os.getenv("MODEL_DISPLAY_NAME") + dataset_display_name = os.getenv("VERTEX_DATASET_NAME") + gcs_location = os.getenv("GCS_LOCATION") + model_registry = os.getenv("MODEL_REGISTRY_URI") + upload_model = os.getenv("UPLOAD_MODEL") + + assert project, "Environment variable PROJECT is None!" + assert region, "Environment variable REGION is None!" + assert dataset_display_name, \ + "Environment variable VERTEX_DATASET_NAME is None!" + assert model_display_name, "Environment variable MODEL_DISPLAY_NAME is None!" + assert gcs_location, "Environment variable GCS_LOCATION is None!" + assert model_registry, "Environment variable MODEL_REGISTRY_URI is None!" + + logging.info("upload_model: %s", upload_model) + if tf.io.gfile.exists(gcs_location): + tf.io.gfile.rmtree(gcs_location) + logging.info("Pipeline e2e test artifacts stored in: %s", gcs_location) + + if tf.io.gfile.exists(MLMD_SQLLITE): + tf.io.gfile.remove(MLMD_SQLLITE) + + metadata_connection_config = metadata_store_pb2.ConnectionConfig() + metadata_connection_config.sqlite.filename_uri = MLMD_SQLLITE + metadata_connection_config.sqlite.connection_mode = 3 + logging.info("ML metadata store is ready.") + + pipeline_root = os.path.join( + config.ARTIFACT_STORE_URI, + config.PIPELINE_NAME, + ) + + runner = LocalDagRunner() + + pipeline = training_pipeline.create_pipeline( + pipeline_root=pipeline_root, + num_epochs=NUM_EPOCHS, + batch_size=BATCH_SIZE, + steps_per_epoch=100, + learning_rate=LEARNING_RATE, + hidden_units=HIDDEN_UNITS, + metadata_connection_config=metadata_connection_config, + ) + + runner.run(pipeline) + + logging.info( + "Model output: %s", os.path.join(model_registry, model_display_name)) + assert tf.io.gfile.exists(os.path.join(model_registry, model_display_name)) diff --git a/examples/vertex_mlops_enterprise/src/tfx_pipelines/components.py b/examples/vertex_mlops_enterprise/src/tfx_pipelines/components.py new file mode 100644 index 0000000000..fd68650d48 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/tfx_pipelines/components.py @@ -0,0 +1,133 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TFX Custom Python Components.""" + + +import sys +import os +import json +import logging +import tensorflow as tf + +from tfx.types import artifact_utils +from tfx.utils import io_utils +from tfx.components.util import model_utils +from tfx.dsl.component.experimental.decorators import component +from tfx.dsl.component.experimental.annotations import ( + InputArtifact, + OutputArtifact, + Parameter, +) +from tfx.types.standard_artifacts import HyperParameters, ModelBlessing +from tfx.types.experimental.simple_artifacts import File as UploadedModel + +from google.cloud import aiplatform as vertex_ai + + +SCRIPT_DIR = os.path.dirname( + os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__))) +) +sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, ".."))) + + +HYPERPARAM_FILENAME = "hyperparameters.json" +SERVING_DATA_PREFIX = "serving-data-" +PREDICTION_RESULTS_PREFIX = "prediction.results-*" + + +@component +def hyperparameters_gen( + num_epochs: Parameter[int], + batch_size: Parameter[int], + steps_per_epoch: Parameter[int], + learning_rate: Parameter[float], + hidden_units: Parameter[str], + hyperparameters: OutputArtifact[HyperParameters], +): + + hp_dict = dict() + hp_dict["num_epochs"] = num_epochs + hp_dict["steps_per_epoch"] = steps_per_epoch + hp_dict["batch_size"] = batch_size + hp_dict["learning_rate"] = learning_rate + hp_dict["hidden_units"] = [int(units) for units in hidden_units.split(",")] + logging.info(f"Hyperparameters: {hp_dict}") + + hyperparams_uri = os.path.join( + artifact_utils.get_single_uri([hyperparameters]), HYPERPARAM_FILENAME + ) + io_utils.write_string_file(hyperparams_uri, json.dumps(hp_dict)) + logging.info(f"Hyperparameters are written to: {hyperparams_uri}") + + +@component +def vertex_model_uploader( + project: Parameter[str], + region: Parameter[str], + model_display_name: Parameter[str], + pushed_model_location: Parameter[str], + serving_image_uri: Parameter[str], + model_blessing: InputArtifact[ModelBlessing], + uploaded_model: OutputArtifact[UploadedModel], + explanation_config: Parameter[str]="", + labels: Parameter[str]="", +): + + vertex_ai.init(project=project, location=region) + + blessing = artifact_utils.get_single_instance([model_blessing]) + if not model_utils.is_model_blessed(blessing): + logging.info(f"Model is not uploaded to Vertex AI because it was not blessed by the evaluator.") + uploaded_model.set_int_custom_property("uploaded", 0) + return + + pushed_model_dir = os.path.join( + pushed_model_location, tf.io.gfile.listdir(pushed_model_location)[-1] + ) + + logging.info(f"Model registry location: {pushed_model_dir}") + + try: + explanation_config = json.loads(explanation_config) + explanation_metadata = vertex_ai.explain.ExplanationMetadata( + inputs=explanation_config["inputs"], + outputs=explanation_config["outputs"], + ) + explanation_parameters = vertex_ai.explain.ExplanationParameters( + explanation_config["params"] + ) + except: + explanation_metadata = None + explanation_parameters = None + + try: + labels = json.loads(labels) + except: + labels = None + + vertex_model = vertex_ai.Model.upload( + display_name=model_display_name, + artifact_uri=pushed_model_dir, + serving_container_image_uri=serving_image_uri, + parameters_schema_uri=None, + instance_schema_uri=None, + explanation_metadata=explanation_metadata, + explanation_parameters=explanation_parameters, + labels=labels + ) + + model_uri = vertex_model.gca_resource.name + logging.info(f"Model uploaded to Vertex AI: {model_uri}") + uploaded_model.set_string_custom_property("model_uri", model_uri) + uploaded_model.set_int_custom_property("uploaded", 1) diff --git a/examples/vertex_mlops_enterprise/src/tfx_pipelines/config.py b/examples/vertex_mlops_enterprise/src/tfx_pipelines/config.py new file mode 100644 index 0000000000..8b9e5d1f24 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/tfx_pipelines/config.py @@ -0,0 +1,130 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TFX pipeline configurations.""" + +import os +from tfx import v1 as tfx + +PROJECT = os.getenv("PROJECT", "") +REGION = os.getenv("REGION", "") +GCS_LOCATION = os.getenv("GCS_LOCATION", "") +DOCKER_REPO_NAME = os.getenv("DOCKER_REPO_NAME", "docker-repo") + +ARTIFACT_STORE_URI = os.path.join(GCS_LOCATION, "tfx_artifacts") +MODEL_REGISTRY_URI = os.getenv( + "MODEL_REGISTRY_URI", + os.path.join(GCS_LOCATION, "model_registry"), +) + +VERTEX_DATASET_NAME = os.getenv("VERTEX_DATASET_NAME", "creditcards") +MODEL_DISPLAY_NAME = os.getenv( + "MODEL_DISPLAY_NAME", f"{VERTEX_DATASET_NAME}-classifier" +) +PIPELINE_NAME = os.getenv("PIPELINE_NAME", f"{MODEL_DISPLAY_NAME}-train-pipeline") + +ML_USE_COLUMN = "ml_use" +EXCLUDE_COLUMNS = ",".join(["trip_start_timestamp"]) +TRAIN_LIMIT = os.getenv("TRAIN_LIMIT", "0") +TEST_LIMIT = os.getenv("TEST_LIMIT", "0") +SERVE_LIMIT = os.getenv("SERVE_LIMIT", "0") + +NUM_TRAIN_SPLITS = os.getenv("NUM_TRAIN_SPLITS", "4") +NUM_EVAL_SPLITS = os.getenv("NUM_EVAL_SPLITS", "1") +ACCURACY_THRESHOLD = os.getenv("ACCURACY_THRESHOLD", "0.8") + +USE_KFP_SA = os.getenv("USE_KFP_SA", "False") + +TFX_IMAGE_URI = os.getenv( + "TFX_IMAGE_URI", f"{REGION}-docker.pkg.dev/{PROJECT}/{DOCKER_REPO_NAME}/vertex:latest" +) + +DATAFLOW_IMAGE_URI = os.getenv( + "DATAFLOW_IMAGE_URI", f"{REGION}-docker.pkg.dev/{PROJECT}/{DOCKER_REPO_NAME}/dataflow:latest" +) + +BEAM_RUNNER = os.getenv("BEAM_RUNNER", "DirectRunner") +SERVICE_ACCOUNT = os.getenv("SERVICE_ACCOUNT", "") +SUBNETWORK = os.getenv("SUBNETWORK", "") + +BEAM_DIRECT_PIPELINE_ARGS = [ + f"--project={PROJECT}", + f"--temp_location={os.path.join(GCS_LOCATION, 'temp')}", +] +BEAM_DATAFLOW_PIPELINE_ARGS = [ + f"--project={PROJECT}", + f"--temp_location={os.path.join(GCS_LOCATION, 'temp')}", + f"--region={REGION}", + f"--runner={BEAM_RUNNER}", + f"--service_account_email={SERVICE_ACCOUNT}", + f"--no_use_public_ips", + f"--subnetwork={SUBNETWORK}", + f"--sdk_container_image={DATAFLOW_IMAGE_URI}" +] + +TRAINING_RUNNER = os.getenv("TRAINING_RUNNER", "local") +VERTEX_TRAINING_ARGS = { + 'project': PROJECT, + 'worker_pool_specs': [{ + 'machine_spec': { + 'machine_type': 'n1-standard-4', +# 'accelerator_type': 'NVIDIA_TESLA_K80', +# 'accelerator_count': 1 + }, + 'replica_count': 1, + 'container_spec': { + 'image_uri': TFX_IMAGE_URI, + }, + }], +} +VERTEX_TRAINING_CONFIG = { + tfx.extensions.google_cloud_ai_platform.ENABLE_UCAIP_KEY: True, + tfx.extensions.google_cloud_ai_platform.UCAIP_REGION_KEY: REGION, + tfx.extensions.google_cloud_ai_platform.TRAINING_ARGS_KEY: VERTEX_TRAINING_ARGS, + 'use_gpu': False, +} + +SERVING_RUNTIME = os.getenv("SERVING_RUNTIME", "tf2-cpu.2-5") +SERVING_IMAGE_URI = f"us-docker.pkg.dev/vertex-ai/prediction/{SERVING_RUNTIME}:latest" + +BATCH_PREDICTION_BQ_DATASET_NAME = os.getenv( + "BATCH_PREDICTION_BQ_DATASET_NAME", "playground_us" +) +BATCH_PREDICTION_BQ_TABLE_NAME = os.getenv( + "BATCH_PREDICTION_BQ_TABLE_NAME", "chicago_taxitrips_prep" +) +BATCH_PREDICTION_BEAM_ARGS = { + "runner": f"{BEAM_RUNNER}", + "temporary_dir": os.path.join(GCS_LOCATION, "temp"), + "gcs_location": os.path.join(GCS_LOCATION, "temp"), + "project": PROJECT, + "region": REGION, + "setup_file": "./setup.py", +} +BATCH_PREDICTION_JOB_RESOURCES = { + "machine_type": "n1-standard-2", + #'accelerator_count': 1, + #'accelerator_type': 'NVIDIA_TESLA_T4' + "starting_replica_count": 1, + "max_replica_count": 10, +} +DATASTORE_PREDICTION_KIND = f"{MODEL_DISPLAY_NAME}-predictions" + +ENABLE_CACHE = os.getenv("ENABLE_CACHE", "0") +UPLOAD_MODEL = os.getenv("UPLOAD_MODEL", "1") + +os.environ["PROJECT"] = PROJECT +os.environ["PIPELINE_NAME"] = PIPELINE_NAME +os.environ["DATAFLOW_IMAGE_URI"] = DATAFLOW_IMAGE_URI +os.environ["TFX_IMAGE_URI"] = TFX_IMAGE_URI +os.environ["MODEL_REGISTRY_URI"] = MODEL_REGISTRY_URI diff --git a/examples/vertex_mlops_enterprise/src/tfx_pipelines/runner.py b/examples/vertex_mlops_enterprise/src/tfx_pipelines/runner.py new file mode 100644 index 0000000000..9bfa4fd0e2 --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/tfx_pipelines/runner.py @@ -0,0 +1,68 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Define KubeflowV2DagRunner to run the training pipeline using Managed Pipelines.""" + + +import os +from tfx.orchestration import data_types +from tfx.orchestration.kubeflow.v2 import kubeflow_v2_dag_runner + + +from src.tfx_pipelines import config, training_pipeline +from src.model_training import defaults + + +def compile_training_pipeline(pipeline_definition_file): + + pipeline_root = os.path.join( + config.ARTIFACT_STORE_URI, + config.PIPELINE_NAME, + ) + managed_pipeline = training_pipeline.create_pipeline( + pipeline_root=pipeline_root, + num_epochs=data_types.RuntimeParameter( + name="num_epochs", + default=defaults.NUM_EPOCHS, + ptype=int, + ), + batch_size=data_types.RuntimeParameter( + name="batch_size", + default=defaults.BATCH_SIZE, + ptype=int, + ), + steps_per_epoch=data_types.RuntimeParameter( + name="steps_per_epoch", + default=int(config.TRAIN_LIMIT) // defaults.BATCH_SIZE, + ptype=int, + ), + learning_rate=data_types.RuntimeParameter( + name="learning_rate", + default=defaults.LEARNING_RATE, + ptype=float, + ), + hidden_units=data_types.RuntimeParameter( + name="hidden_units", + default=",".join(str(u) for u in defaults.HIDDEN_UNITS), + ptype=str, + ), + ) + + runner = kubeflow_v2_dag_runner.KubeflowV2DagRunner( + config=kubeflow_v2_dag_runner.KubeflowV2DagRunnerConfig( + default_image=config.TFX_IMAGE_URI + ), + output_filename=pipeline_definition_file, + ) + + return runner.run(managed_pipeline, write_out=True) diff --git a/examples/vertex_mlops_enterprise/src/tfx_pipelines/training_pipeline.py b/examples/vertex_mlops_enterprise/src/tfx_pipelines/training_pipeline.py new file mode 100644 index 0000000000..4a3d39f0fa --- /dev/null +++ b/examples/vertex_mlops_enterprise/src/tfx_pipelines/training_pipeline.py @@ -0,0 +1,320 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TFX training pipeline definition.""" + +import os +import sys +import logging +import json + +import tensorflow_model_analysis as tfma + +from ml_metadata.proto import metadata_store_pb2 + +from tfx.proto import example_gen_pb2, transform_pb2, pusher_pb2 +from tfx.types import Channel, standard_artifacts +from tfx.orchestration import pipeline, data_types +from tfx.dsl.components.common.importer import Importer +from tfx.dsl.components.common.resolver import Resolver +from tfx.dsl.experimental import latest_artifacts_resolver +from tfx.dsl.experimental import latest_blessed_model_resolver +from tfx.v1.extensions.google_cloud_big_query import BigQueryExampleGen +from tfx.v1.extensions.google_cloud_ai_platform import Trainer as VertexTrainer +from tfx.v1.components import ( + StatisticsGen, + ExampleValidator, + Transform, + Trainer, + Evaluator, + Pusher +) + + +SCRIPT_DIR = os.path.dirname( + os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__))) +) +sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, ".."))) + + +from src.tfx_pipelines import config +from src.tfx_pipelines import components as custom_components +from src.common import features, datasource_utils + + +RAW_SCHEMA_DIR = "src/raw_schema" +TRANSFORM_MODULE_FILE = "src/preprocessing/transformations.py" +TRAIN_MODULE_FILE = "src/model_training/runner.py" + + +def create_pipeline( + pipeline_root: str, + num_epochs: data_types.RuntimeParameter, + batch_size: data_types.RuntimeParameter, + steps_per_epoch: data_types.RuntimeParameter, + learning_rate: data_types.RuntimeParameter, + hidden_units: data_types.RuntimeParameter, + metadata_connection_config: metadata_store_pb2.ConnectionConfig = None, +): + + # Hyperparameter generation. + hyperparams_gen = custom_components.hyperparameters_gen( + num_epochs=num_epochs, + batch_size=batch_size, + steps_per_epoch=steps_per_epoch, + learning_rate=learning_rate, + hidden_units=hidden_units + ).with_id("HyperparamsGen") + + # Get train source query. + logging.info(f'Using dataset {config.VERTEX_DATASET_NAME}') + train_sql_query = datasource_utils.get_training_source_query( + config.PROJECT, + config.REGION, + config.VERTEX_DATASET_NAME, + ml_use="UNASSIGNED", + limit=int(config.TRAIN_LIMIT), + ) + + train_output_config = example_gen_pb2.Output( + split_config=example_gen_pb2.SplitConfig( + splits=[ + example_gen_pb2.SplitConfig.Split( + name="train", hash_buckets=int(config.NUM_TRAIN_SPLITS) + ), + example_gen_pb2.SplitConfig.Split( + name="eval", hash_buckets=int(config.NUM_EVAL_SPLITS) + ), + ] + ) + ) + + # Train example generation. + train_example_gen = BigQueryExampleGen( + query=train_sql_query, + output_config=train_output_config, + ).with_id("TrainDataGen") + + # Get test source query. + test_sql_query = datasource_utils.get_training_source_query( + config.PROJECT, + config.REGION, + config.VERTEX_DATASET_NAME, + ml_use="TEST", + limit=int(config.TEST_LIMIT), + ) + + test_output_config = example_gen_pb2.Output( + split_config=example_gen_pb2.SplitConfig( + splits=[ + example_gen_pb2.SplitConfig.Split(name="test", hash_buckets=1), + ] + ) + ) + + # Test example generation. + test_example_gen = BigQueryExampleGen( + query=test_sql_query, + output_config=test_output_config, + ).with_id("TestDataGen") + + # Schema importer. + schema_importer = Importer( + source_uri=RAW_SCHEMA_DIR, + artifact_type=standard_artifacts.Schema, + ).with_id("SchemaImporter") + + # Statistics generation. + statistics_gen = StatisticsGen(examples=train_example_gen.outputs["examples"]).with_id( + "StatisticsGen" + ) + + # Example validation. + example_validator = ExampleValidator( + statistics=statistics_gen.outputs["statistics"], + schema=schema_importer.outputs["result"], + ).with_id("ExampleValidator") + + # Data transformation. + transform = Transform( + examples=train_example_gen.outputs["examples"], + schema=schema_importer.outputs["result"], + module_file=TRANSFORM_MODULE_FILE, + # This is a temporary workaround to run on Dataflow. + force_tf_compat_v1=config.BEAM_RUNNER == "DataflowRunner", + splits_config=transform_pb2.SplitsConfig( + analyze=["train"], transform=["train", "eval"] + ), + ).with_id("DataTransformer") + + # Add dependency from example_validator to transform. + transform.add_upstream_node(example_validator) + + # Get the latest model to warmstart + warmstart_model_resolver = Resolver( + strategy_class=latest_artifacts_resolver.LatestArtifactsResolver, + latest_model=Channel(type=standard_artifacts.Model), + ).with_id("WarmstartModelResolver") + + # Model training. + trainer = Trainer( + module_file=TRAIN_MODULE_FILE, + examples=transform.outputs["transformed_examples"], + schema=schema_importer.outputs["result"], + base_model=warmstart_model_resolver.outputs["latest_model"], + transform_graph=transform.outputs["transform_graph"], + hyperparameters=hyperparams_gen.outputs["hyperparameters"], + ).with_id("ModelTrainer") + + if config.TRAINING_RUNNER == "vertex": + trainer = VertexTrainer( + module_file=TRAIN_MODULE_FILE, + examples=transform.outputs["transformed_examples"], + schema=schema_importer.outputs["result"], + base_model=warmstart_model_resolver.outputs["latest_model"], + transform_graph=transform.outputs["transform_graph"], + hyperparameters=hyperparams_gen.outputs["hyperparameters"], + custom_config=config.VERTEX_TRAINING_CONFIG + ).with_id("ModelTrainer") + + + # Get the latest blessed model (baseline) for model validation. + baseline_model_resolver = Resolver( + strategy_class=latest_blessed_model_resolver.LatestBlessedModelResolver, + model=Channel(type=standard_artifacts.Model), + model_blessing=Channel(type=standard_artifacts.ModelBlessing), + ).with_id("BaselineModelResolver") + + # Prepare evaluation config. + eval_config = tfma.EvalConfig( + model_specs=[ + tfma.ModelSpec( + signature_name="serving_tf_example", + label_key=features.TARGET_FEATURE_NAME, + prediction_key="probabilities", + ) + ], + slicing_specs=[ + tfma.SlicingSpec(), + ], + metrics_specs=[ + tfma.MetricsSpec( + metrics=[ + tfma.MetricConfig(class_name="ExampleCount"), + tfma.MetricConfig( + class_name="BinaryAccuracy", + threshold=tfma.MetricThreshold( + value_threshold=tfma.GenericValueThreshold( + lower_bound={"value": float(config.ACCURACY_THRESHOLD)} + ), + # Change threshold will be ignored if there is no + # baseline model resolved from MLMD (first run). + change_threshold=tfma.GenericChangeThreshold( + direction=tfma.MetricDirection.HIGHER_IS_BETTER, + absolute={"value": -1e-10}, + ), + ), + ), + ] + ) + ], + ) + + # Model evaluation. + evaluator = Evaluator( + examples=test_example_gen.outputs["examples"], + example_splits=["test"], + model=trainer.outputs["model"], + baseline_model=baseline_model_resolver.outputs["model"], + eval_config=eval_config, + schema=schema_importer.outputs["result"], + ).with_id("ModelEvaluator") + + exported_model_location = os.path.join( + config.MODEL_REGISTRY_URI, config.MODEL_DISPLAY_NAME + ) + push_destination = pusher_pb2.PushDestination( + filesystem=pusher_pb2.PushDestination.Filesystem( + base_directory=exported_model_location + ) + ) + + # Push custom model to model registry. + gcs_pusher = Pusher( + model=trainer.outputs["model"], + model_blessing=evaluator.outputs["blessing"], + push_destination=push_destination, + ).with_id("GcsModelPusher") + + # Push to Vertex AI + labels = { + "dataset_name": config.VERTEX_DATASET_NAME[:62], + "pipeline_name": config.PIPELINE_NAME[:62], + "pipeline_root": pipeline_root[:62] + } + + labels = json.dumps(labels) + explanation_config = json.dumps(features.generate_explanation_config()) + + print(f"Labels for model: {labels}") + + # Custom implementation. + vertex_model_uploader = custom_components.vertex_model_uploader( + project=config.PROJECT, + region=config.REGION, + model_display_name=config.MODEL_DISPLAY_NAME, + pushed_model_location=exported_model_location, + serving_image_uri=config.SERVING_IMAGE_URI, + model_blessing=evaluator.outputs["blessing"], + explanation_config=explanation_config #, + #labels=labels # currently labels have values too long or too many keys or something else invalid + ).with_id("VertexUploader") + + pipeline_components = [ + hyperparams_gen, + train_example_gen, + test_example_gen, + statistics_gen, + schema_importer, + example_validator, + transform, + warmstart_model_resolver, + trainer, + baseline_model_resolver, + evaluator, + gcs_pusher, + ] + + if int(config.UPLOAD_MODEL): + pipeline_components.append(vertex_model_uploader) + # Add dependency from pusher to aip_model_uploader. + vertex_model_uploader.add_upstream_node(gcs_pusher) + + logging.info( + f"Pipeline components: {[component.id for component in pipeline_components]}" + ) + + beam_pipeline_args = config.BEAM_DIRECT_PIPELINE_ARGS + if config.BEAM_RUNNER == "DataflowRunner": + beam_pipeline_args = config.BEAM_DATAFLOW_PIPELINE_ARGS + + logging.info(f"Beam pipeline args: {beam_pipeline_args}") + print(f"Beam pipeline args: {beam_pipeline_args}") + return pipeline.Pipeline( + pipeline_name=config.PIPELINE_NAME, + pipeline_root=pipeline_root, + components=pipeline_components, + beam_pipeline_args=beam_pipeline_args, + metadata_connection_config=metadata_connection_config, + enable_cache=int(config.ENABLE_CACHE), + ) diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/firewall-policies/cidrs.yaml b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/firewall-policies/cidrs.yaml new file mode 100644 index 0000000000..90dabfb6a7 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/firewall-policies/cidrs.yaml @@ -0,0 +1,15 @@ +# skip boilerplate check + +healthchecks: + - 35.191.0.0/16 + - 130.211.0.0/22 + - 209.85.152.0/22 + - 209.85.204.0/22 + +rfc1918: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + +onprem_probes: + - 10.255.255.254/32 \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/firewall-policies/hierarchical-policy-rules.yaml b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/firewall-policies/hierarchical-policy-rules.yaml new file mode 100644 index 0000000000..6a3b313356 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/firewall-policies/hierarchical-policy-rules.yaml @@ -0,0 +1,50 @@ +# skip boilerplate check + +allow-admins: + description: Access from the admin subnet to all subnets + direction: INGRESS + action: allow + priority: 1000 + ranges: + - $rfc1918 + ports: + all: [] + target_resources: null + enable_logging: false + +allow-healthchecks: + description: Enable HTTP and HTTPS healthchecks + direction: INGRESS + action: allow + priority: 1001 + ranges: + - $healthchecks + ports: + tcp: ["80", "443"] + target_resources: null + enable_logging: false + +allow-ssh-from-iap: + description: Enable SSH from IAP + direction: INGRESS + action: allow + priority: 1002 + ranges: + - 35.235.240.0/20 + ports: + tcp: ["22"] + target_resources: null + enable_logging: false + +allow-icmp: + description: Enable ICMP + direction: INGRESS + action: allow + priority: 1003 + ranges: + - 0.0.0.0/0 + ports: + icmp: [] + target_resources: null + enable_logging: false + \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/compute.yaml b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/compute.yaml new file mode 100644 index 0000000000..a3f96b1b1c --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/compute.yaml @@ -0,0 +1,92 @@ +# skip boilerplate check +# +# sample subset of useful organization policies, edit to suit requirements + +compute.disableGuestAttributesAccess: + rules: + - enforce: true + +compute.requireOsLogin: + rules: + - enforce: true + +compute.restrictLoadBalancerCreationForTypes: + rules: + - allow: + values: + - in:INTERNAL + +compute.skipDefaultNetworkCreation: + rules: + - enforce: true + +compute.vmExternalIpAccess: + rules: + - deny: + all: true + + +# compute.disableInternetNetworkEndpointGroup: +# rules: +# - enforce: true + +# compute.disableNestedVirtualization: +# rules: +# - enforce: true + +# compute.disableSerialPortAccess: +# rules: +# - enforce: true + +# compute.restrictCloudNATUsage: +# rules: +# - deny: +# all: true + +# compute.restrictDedicatedInterconnectUsage: +# rules: +# - deny: +# all: true + +# compute.restrictPartnerInterconnectUsage: +# rules: +# - deny: +# all: true + +# compute.restrictProtocolForwardingCreationForTypes: +# rules: +# - deny: +# all: true + +# compute.restrictSharedVpcHostProjects: +# rules: +# - deny: +# all: true + +# compute.restrictSharedVpcSubnetworks: +# rules: +# - deny: +# all: true + +# compute.restrictVpcPeering: +# rules: +# - deny: +# all: true + +# compute.restrictVpnPeerIPs: +# rules: +# - deny: +# all: true + +# compute.restrictXpnProjectLienRemoval: +# rules: +# - enforce: true + +# compute.setNewProjectDefaultToZonalDNSOnly: +# rules: +# - enforce: true + +# compute.vmCanIpForward: +# rules: +# - deny: +# all: true diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/iam.yaml b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/iam.yaml new file mode 100644 index 0000000000..58e0032cb3 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/iam.yaml @@ -0,0 +1,15 @@ +# skip boilerplate check +# +# sample subset of useful organization policies, edit to suit requirements + +iam.automaticIamGrantsForDefaultServiceAccounts: + rules: + - enforce: true + +iam.disableServiceAccountKeyCreation: + rules: + - enforce: true + +iam.disableServiceAccountKeyUpload: + rules: + - enforce: true diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/serverless.yaml b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/serverless.yaml new file mode 100644 index 0000000000..3efb23cdee --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/serverless.yaml @@ -0,0 +1,31 @@ +# skip boilerplate check +# +# sample subset of useful organization policies, edit to suit requirements + +run.allowedIngress: + rules: + - allow: + values: + - is:internal + +# run.allowedVPCEgress: +# rules: +# - allow: +# values: +# - is:private-ranges-only + +# cloudfunctions.allowedIngressSettings: +# rules: +# - allow: +# values: +# - is:ALLOW_INTERNAL_ONLY + +# cloudfunctions.allowedVpcConnectorEgressSettings: +# rules: +# - allow: +# values: +# - is:PRIVATE_RANGES_ONLY + +# cloudfunctions.requireVPCConnector: +# rules: +# - enforce: true diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/sql.yaml b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/sql.yaml new file mode 100644 index 0000000000..0eee80453a --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/sql.yaml @@ -0,0 +1,11 @@ +# skip boilerplate check +# +# sample subset of useful organization policies, edit to suit requirements + +sql.restrictAuthorizedNetworks: + rules: + - enforce: true + +sql.restrictPublicIp: + rules: + - enforce: true diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/storage.yaml b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/storage.yaml new file mode 100644 index 0000000000..448357b8bc --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/org-policies/storage.yaml @@ -0,0 +1,7 @@ +# skip boilerplate check +# +# sample subset of useful organization policies, edit to suit requirements + +storage.uniformBucketLevelAccess: + rules: + - enforce: true diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/vpc-sc/accessible-services.yaml b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/vpc-sc/accessible-services.yaml new file mode 100644 index 0000000000..2107d2ff1e --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/vpc-sc/accessible-services.yaml @@ -0,0 +1,119 @@ +# skip boilerplate check + +- accessapproval.googleapis.com +- adsdatahub.googleapis.com +- aiplatform.googleapis.com +- alloydb.googleapis.com +- alpha-documentai.googleapis.com +- analyticshub.googleapis.com +- apigee.googleapis.com +- apigeeconnect.googleapis.com +- artifactregistry.googleapis.com +- assuredworkloads.googleapis.com +- automl.googleapis.com +- baremetalsolution.googleapis.com +- batch.googleapis.com +- beyondcorp.googleapis.com +- bigquery.googleapis.com +- bigquerydatapolicy.googleapis.com +- bigquerydatatransfer.googleapis.com +- bigquerymigration.googleapis.com +- bigqueryreservation.googleapis.com +- bigtable.googleapis.com +- binaryauthorization.googleapis.com +- cloudasset.googleapis.com +- cloudbuild.googleapis.com +- clouddebugger.googleapis.com +- clouderrorreporting.googleapis.com +- cloudfunctions.googleapis.com +- cloudkms.googleapis.com +- cloudprofiler.googleapis.com +- cloudresourcemanager.googleapis.com +- cloudsearch.googleapis.com +- cloudtrace.googleapis.com +- composer.googleapis.com +- compute.googleapis.com +- connectgateway.googleapis.com +- contactcenterinsights.googleapis.com +- container.googleapis.com +- containeranalysis.googleapis.com +- containerfilesystem.googleapis.com +- containerregistry.googleapis.com +- containerthreatdetection.googleapis.com +- contentwarehouse.googleapis.com +- datacatalog.googleapis.com +- dataflow.googleapis.com +- datafusion.googleapis.com +- datalineage.googleapis.com +- datamigration.googleapis.com +- datapipelines.googleapis.com +- dataplex.googleapis.com +- dataproc.googleapis.com +- datastream.googleapis.com +- dialogflow.googleapis.com +- dlp.googleapis.com +- dns.googleapis.com +- documentai.googleapis.com +- domains.googleapis.com +- essentialcontacts.googleapis.com +- eventarc.googleapis.com +- file.googleapis.com +- firebaseappcheck.googleapis.com +- firebaserules.googleapis.com +- firestore.googleapis.com +- gameservices.googleapis.com +- gkebackup.googleapis.com +- gkeconnect.googleapis.com +- gkehub.googleapis.com +- gkemulticloud.googleapis.com +- healthcare.googleapis.com +- iam.googleapis.com +- iamcredentials.googleapis.com +- iaptunnel.googleapis.com +- ids.googleapis.com +- integrations.googleapis.com +- language.googleapis.com +- lifesciences.googleapis.com +- logging.googleapis.com +- managedidentities.googleapis.com +- memcache.googleapis.com +- meshca.googleapis.com +- metastore.googleapis.com +- ml.googleapis.com +- monitoring.googleapis.com +- networkconnectivity.googleapis.com +- networkmanagement.googleapis.com +- networksecurity.googleapis.com +- networkservices.googleapis.com +- notebooks.googleapis.com +- opsconfigmonitoring.googleapis.com +- osconfig.googleapis.com +- oslogin.googleapis.com +- policytroubleshooter.googleapis.com +- privateca.googleapis.com +- pubsub.googleapis.com +- pubsublite.googleapis.com +- recaptchaenterprise.googleapis.com +- recommender.googleapis.com +- redis.googleapis.com +- retail.googleapis.com +- run.googleapis.com +- secretmanager.googleapis.com +- servicecontrol.googleapis.com +- servicedirectory.googleapis.com +- spanner.googleapis.com +- speakerid.googleapis.com +- speech.googleapis.com +- sqladmin.googleapis.com +- storage.googleapis.com +- storagetransfer.googleapis.com +- texttospeech.googleapis.com +- tpu.googleapis.com +- trafficdirector.googleapis.com +- transcoder.googleapis.com +- translate.googleapis.com +- videointelligence.googleapis.com +- vision.googleapis.com +- visionai.googleapis.com +- vpcaccess.googleapis.com +- workstations.googleapis.com \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/vpc-sc/restricted-services.yaml b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/vpc-sc/restricted-services.yaml new file mode 100644 index 0000000000..2107d2ff1e --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/data/vpc-sc/restricted-services.yaml @@ -0,0 +1,119 @@ +# skip boilerplate check + +- accessapproval.googleapis.com +- adsdatahub.googleapis.com +- aiplatform.googleapis.com +- alloydb.googleapis.com +- alpha-documentai.googleapis.com +- analyticshub.googleapis.com +- apigee.googleapis.com +- apigeeconnect.googleapis.com +- artifactregistry.googleapis.com +- assuredworkloads.googleapis.com +- automl.googleapis.com +- baremetalsolution.googleapis.com +- batch.googleapis.com +- beyondcorp.googleapis.com +- bigquery.googleapis.com +- bigquerydatapolicy.googleapis.com +- bigquerydatatransfer.googleapis.com +- bigquerymigration.googleapis.com +- bigqueryreservation.googleapis.com +- bigtable.googleapis.com +- binaryauthorization.googleapis.com +- cloudasset.googleapis.com +- cloudbuild.googleapis.com +- clouddebugger.googleapis.com +- clouderrorreporting.googleapis.com +- cloudfunctions.googleapis.com +- cloudkms.googleapis.com +- cloudprofiler.googleapis.com +- cloudresourcemanager.googleapis.com +- cloudsearch.googleapis.com +- cloudtrace.googleapis.com +- composer.googleapis.com +- compute.googleapis.com +- connectgateway.googleapis.com +- contactcenterinsights.googleapis.com +- container.googleapis.com +- containeranalysis.googleapis.com +- containerfilesystem.googleapis.com +- containerregistry.googleapis.com +- containerthreatdetection.googleapis.com +- contentwarehouse.googleapis.com +- datacatalog.googleapis.com +- dataflow.googleapis.com +- datafusion.googleapis.com +- datalineage.googleapis.com +- datamigration.googleapis.com +- datapipelines.googleapis.com +- dataplex.googleapis.com +- dataproc.googleapis.com +- datastream.googleapis.com +- dialogflow.googleapis.com +- dlp.googleapis.com +- dns.googleapis.com +- documentai.googleapis.com +- domains.googleapis.com +- essentialcontacts.googleapis.com +- eventarc.googleapis.com +- file.googleapis.com +- firebaseappcheck.googleapis.com +- firebaserules.googleapis.com +- firestore.googleapis.com +- gameservices.googleapis.com +- gkebackup.googleapis.com +- gkeconnect.googleapis.com +- gkehub.googleapis.com +- gkemulticloud.googleapis.com +- healthcare.googleapis.com +- iam.googleapis.com +- iamcredentials.googleapis.com +- iaptunnel.googleapis.com +- ids.googleapis.com +- integrations.googleapis.com +- language.googleapis.com +- lifesciences.googleapis.com +- logging.googleapis.com +- managedidentities.googleapis.com +- memcache.googleapis.com +- meshca.googleapis.com +- metastore.googleapis.com +- ml.googleapis.com +- monitoring.googleapis.com +- networkconnectivity.googleapis.com +- networkmanagement.googleapis.com +- networksecurity.googleapis.com +- networkservices.googleapis.com +- notebooks.googleapis.com +- opsconfigmonitoring.googleapis.com +- osconfig.googleapis.com +- oslogin.googleapis.com +- policytroubleshooter.googleapis.com +- privateca.googleapis.com +- pubsub.googleapis.com +- pubsublite.googleapis.com +- recaptchaenterprise.googleapis.com +- recommender.googleapis.com +- redis.googleapis.com +- retail.googleapis.com +- run.googleapis.com +- secretmanager.googleapis.com +- servicecontrol.googleapis.com +- servicedirectory.googleapis.com +- spanner.googleapis.com +- speakerid.googleapis.com +- speech.googleapis.com +- sqladmin.googleapis.com +- storage.googleapis.com +- storagetransfer.googleapis.com +- texttospeech.googleapis.com +- tpu.googleapis.com +- trafficdirector.googleapis.com +- transcoder.googleapis.com +- translate.googleapis.com +- videointelligence.googleapis.com +- vision.googleapis.com +- visionai.googleapis.com +- vpcaccess.googleapis.com +- workstations.googleapis.com \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/main.tf b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/main.tf new file mode 100644 index 0000000000..39d39b1382 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/main.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Vertex MLOps + +module "shielded-folder" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//blueprints/data-solutions/shielded-folder" + access_policy_config = var.access_policy_config + enable_features = var.enable_features + folder_config = var.folder_config + kms_keys = var.kms_keys + log_locations = var.log_locations + organization = var.organization + prefix = var.prefix + project_config = var.project_config + vpc_sc_access_levels = var.vpc_sc_access_levels +} diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/providers.tf b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/providers.tf new file mode 100644 index 0000000000..4be271944e --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/providers.tf @@ -0,0 +1,13 @@ +terraform { + backend "gcs" { + bucket = "pcorp-iac-core-bucket" + prefix = "ShieldedFolder" + } +} + +provider "google" { + impersonate_service_account = "pcorp-iac-core@pcorp-iac-core.iam.gserviceaccount.com" +} +provider "google-beta" { + impersonate_service_account = "pcorp-iac-core@pcorp-iac-core.iam.gserviceaccount.com" +} diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/terraform.tfvars b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/terraform.tfvars new file mode 100644 index 0000000000..1d38d63b39 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/terraform.tfvars @@ -0,0 +1,42 @@ +access_policy_config = { + access_policy_create = { + parent = "organizations/863496320224" + title = "ShieldedFolder" + } +} + +enable_features = { + encryption = true +} + +folder_config = { + folder_create = { + display_name = "ShieldedFolder" + parent = "folders/443543909887" #Dev + } +} + + +kms_keys = { + compute = { + locations = ["europe-west4"] + }, + +} + +organization = { + domain = "pcorp.joonix.net" + id = "863496320224" +} +prefix = "pcorp" +project_config = { + billing_account_id = "0189FA-E139FD-136A58" +} + +vpc_sc_access_levels = { + users = { + conditions = [ + { members = ["user:jgpuga@google.com"] } + ] + } +} diff --git a/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/variables.tf b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/variables.tf new file mode 100644 index 0000000000..8154e86220 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/00-shielded-folder/variables.tf @@ -0,0 +1,229 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tfdoc:file:description Variables definition. + +variable "access_policy_config" { + description = "Provide 'access_policy_create' values if a folder scoped Access Policy creation is needed, uses existing 'policy_name' otherwise. Parent is in 'organizations/123456' format. Policy will be created scoped to the folder." + type = object({ + policy_name = optional(string, null) + access_policy_create = optional(object({ + parent = string + title = string + }), null) + }) + nullable = false +} + +variable "data_dir" { + description = "Relative path for the folder storing configuration data." + type = string + default = "data" +} + +variable "enable_features" { + description = "Flag to enable features on the solution." + type = object({ + encryption = optional(bool, false) + log_sink = optional(bool, true) + vpc_sc = optional(bool, true) + }) + default = { + encryption = false + log_sink = true + vpc_sc = true + } +} + +variable "folder_config" { + description = "Provide 'folder_create' values if folder creation is needed, uses existing 'folder_id' otherwise. Parent is in 'folders/nnn' or 'organizations/nnn' format." + type = object({ + folder_id = optional(string, null) + folder_create = optional(object({ + display_name = string + parent = string + }), null) + }) + validation { + condition = var.folder_config.folder_id != null || var.folder_config.folder_create != null + error_message = "At least one attribute should be set." + } + nullable = false +} + +variable "groups" { + description = "User groups." + type = object({ + workload-engineers = optional(string, "gcp-data-engineers") + workload-security = optional(string, "gcp-data-security") + }) + default = {} + nullable = false +} + +variable "kms_keys" { + description = "KMS keys to create, keyed by name." + type = map(object({ + iam = optional(map(list(string)), {}) + labels = optional(map(string), {}) + locations = optional(list(string), ["global", "europe", "europe-west1"]) + rotation_period = optional(string, "7776000s") + })) + default = {} +} + +variable "log_locations" { + description = "Optional locations for GCS, BigQuery, and logging buckets created here." + type = object({ + bq = optional(string, "europe") + storage = optional(string, "europe") + logging = optional(string, "global") + pubsub = optional(string, "global") + }) + default = { + bq = "europe" + storage = "europe" + logging = "global" + pubsub = null + } + nullable = false +} + +variable "log_sinks" { + description = "Org-level log sinks, in name => {type, filter} format." + type = map(object({ + filter = string + type = string + })) + default = { + audit-logs = { + filter = "logName:\"/logs/cloudaudit.googleapis.com%2Factivity\" OR logName:\"/logs/cloudaudit.googleapis.com%2Fsystem_event\"" + type = "bigquery" + } + vpc-sc = { + filter = "protoPayload.metadata.@type=\"type.googleapis.com/google.cloud.audit.VpcServiceControlAuditMetadata\"" + type = "bigquery" + } + } + validation { + condition = alltrue([ + for k, v in var.log_sinks : + contains(["bigquery", "logging", "pubsub", "storage"], v.type) + ]) + error_message = "Type must be one of 'bigquery', 'logging', 'pubsub', 'storage'." + } +} + +variable "organization" { + description = "Organization details." + type = object({ + domain = string + id = string + }) +} + +variable "prefix" { + description = "Prefix used for resources that need unique names." + type = string +} + +variable "project_config" { + description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." + type = object({ + billing_account_id = optional(string, null) + project_ids = optional(object({ + sec-core = string + audit-logs = string + }), { + sec-core = "sec-core" + audit-logs = "audit-logs" + } + ) + }) + nullable = false + validation { + condition = var.project_config.billing_account_id != null || var.project_config.project_ids != null + error_message = "At least one attribute should be set." + } +} + +variable "vpc_sc_access_levels" { + description = "VPC SC access level definitions." + type = map(object({ + combining_function = optional(string) + conditions = optional(list(object({ + device_policy = optional(object({ + allowed_device_management_levels = optional(list(string)) + allowed_encryption_statuses = optional(list(string)) + require_admin_approval = bool + require_corp_owned = bool + require_screen_lock = optional(bool) + os_constraints = optional(list(object({ + os_type = string + minimum_version = optional(string) + require_verified_chrome_os = optional(bool) + }))) + })) + ip_subnetworks = optional(list(string), []) + members = optional(list(string), []) + negate = optional(bool) + regions = optional(list(string), []) + required_access_levels = optional(list(string), []) + })), []) + description = optional(string) + })) + default = {} + nullable = false +} + +variable "vpc_sc_egress_policies" { + description = "VPC SC egress policy defnitions." + type = map(object({ + from = object({ + identity_type = optional(string, "ANY_IDENTITY") + identities = optional(list(string)) + }) + to = object({ + operations = optional(list(object({ + method_selectors = optional(list(string)) + service_name = string + })), []) + resources = optional(list(string)) + resource_type_external = optional(bool, false) + }) + })) + default = {} + nullable = false +} + +variable "vpc_sc_ingress_policies" { + description = "VPC SC ingress policy defnitions." + type = map(object({ + from = object({ + access_levels = optional(list(string), []) + identity_type = optional(string) + identities = optional(list(string)) + resources = optional(list(string), []) + }) + to = object({ + operations = optional(list(object({ + method_selectors = optional(list(string)) + service_name = string + })), []) + resources = optional(list(string)) + }) + })) + default = {} + nullable = false +} \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/terraform/01-dev/main.tf b/examples/vertex_mlops_enterprise/terraform/01-dev/main.tf new file mode 100644 index 0000000000..0df2fa0ce6 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/01-dev/main.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Vertex MLOps + +locals { + bucket_name = "${var.bucket_name}-${var.environment}" + env_label = { + "env" : "${var.environment}" + } + labels = merge(local.env_label, var.labels) + + github = { + organization = var.github.organization + repo = var.github.repo + branch = var.environment + } + + identity_pool_claims = try("attribute.repository/${var.github.organization}/${var.github.repo}", null) + + project_config = { + billing_account_id = var.project_config.billing_account_id + parent = var.project_config.parent + project_id = "${var.project_config.project_id}-${var.environment}" + } + +} +module "mlops" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//blueprints/data-solutions/vertex-mlops" + project_config = local.project_config + prefix = var.prefix + bucket_name = local.bucket_name + dataset_name = var.dataset_name + groups = var.groups + identity_pool_claims = local.identity_pool_claims + labels = local.labels + notebooks = var.notebooks + service_encryption_keys = var.service_encryption_keys +} diff --git a/examples/vertex_mlops_enterprise/terraform/01-dev/outputs.tf b/examples/vertex_mlops_enterprise/terraform/01-dev/outputs.tf new file mode 100644 index 0000000000..936ab1a63a --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/01-dev/outputs.tf @@ -0,0 +1,208 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +locals { + mainconfig_yaml = templatefile("${path.module}/../../mainconfig.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + docker_repo = module.mlops.github.DOCKER_REPO, + sa_mlops = module.mlops.github.SA_MLOPS, + subnetwork = module.mlops.github.SUBNETWORK + }) + + gh_containers_yaml = templatefile("${path.module}/../../.github/workflows/containers.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO + environment = var.environment + }) + + gh_main_tfx_yaml = templatefile("${path.module}/../../.github/workflows/main.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO, + environment = var.environment, + framework = "tfx" + }) + + gh_main_kfp_yaml = templatefile("${path.module}/../../.github/workflows/main.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO, + environment = var.environment, + framework = "kfp" + }) + + gh_run_tfx_yaml = templatefile("${path.module}/../../.github/workflows/run.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO, + environment = var.environment, + framework = "tfx" + }) + + gh_run_kfp_yaml = templatefile("${path.module}/../../.github/workflows/run.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO, + environment = var.environment, + framework = "kfp" + }) + + gh_deploy_yaml = templatefile("${path.module}/../../.github/workflows/deploy.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO, + environment = var.environment + }) + + pipeline_deploy_tfx = templatefile("${path.module}/../../build/pipeline-deployment-tfx.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + docker_repo = module.mlops.github.DOCKER_REPO, + sa_mlops = module.mlops.github.SA_MLOPS, + subnetwork = module.mlops.github.SUBNETWORK, + bucket_name = "${var.prefix}-${var.bucket_name}-${var.environment}" + }) + + pipeline_deploy_kfp = templatefile("${path.module}/../../build/pipeline-deployment-kfp.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + docker_repo = module.mlops.github.DOCKER_REPO, + sa_mlops = module.mlops.github.SA_MLOPS, + dataflow_network = "regions/europe-west4/subnetworks/subnet-europe-west4", + subnetwork = module.mlops.github.SUBNETWORK, + bucket_name = "${var.prefix}-${var.bucket_name}-${var.environment}" + }) + + pipeline_run_tfx = templatefile("${path.module}/../../build/pipeline-run.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + sa_mlops = module.mlops.github.SA_MLOPS, + bucket_name = "${var.prefix}-${var.bucket_name}-${var.environment}", + pipeline_name = "creditcards-classifier-v02-train-pipeline", + pipeline_params = "\"{\\\"num_epochs\\\": 7, \\\"learning_rate\\\": 0.0015, \\\"batch_size\\\": 512, \\\"steps_per_epoch\\\": 9, \\\"hidden_units\\\": \\\"256,126\\\"}\"" + }) + + pipeline_run_kfp = templatefile("${path.module}/../../build/pipeline-run.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + sa_mlops = module.mlops.github.SA_MLOPS, + bucket_name = "${var.prefix}-${var.bucket_name}-${var.environment}", + pipeline_name = "creditcards-classifier-kfp-train", + pipeline_params = "{\"bq_table\": \"${module.mlops.github.PROJECT_ID}.${var.dataset_name}.creditcards_ml\", \"xgboost_param_max_depth\": 5, \"xgboost_param_learning_rate\": 0.1, \"xgboost_param_n_estimators\": 20}" + }) + + + model_deployment = templatefile("${path.module}/../../build/model-deployment.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + docker_repo = module.mlops.github.DOCKER_REPO, + sa_mlops = module.mlops.github.SA_MLOPS, + }) +} + + +resource "local_file" "mainconfig_yml" { + filename = "${path.module}/../../mainconfig.yaml" + content = local.mainconfig_yaml +} + +resource "local_file" "containers_yml" { + filename = "${path.module}/../../.github/workflows/containers.yml" + content = local.gh_containers_yaml +} + +resource "local_file" "main_tfx_yml" { + filename = "${path.module}/../../.github/workflows/main-tfx.yml" + content = local.gh_main_tfx_yaml +} + +resource "local_file" "main_kfp_yml" { + filename = "${path.module}/../../.github/workflows/main-kfp.yml" + content = local.gh_main_kfp_yaml +} + +resource "local_file" "run_tfx_yml" { + filename = "${path.module}/../../.github/workflows/run-tfx.yml" + content = local.gh_run_tfx_yaml +} + +resource "local_file" "run_kfp_yml" { + filename = "${path.module}/../../.github/workflows/run-kfp.yml" + content = local.gh_run_kfp_yaml +} + +resource "local_file" "deploy_yml" { + filename = "${path.module}/../../.github/workflows/deploy.yml" + content = local.gh_deploy_yaml +} + +resource "local_file" "deployment_tfx_yml" { + filename = "${path.module}/../../build/${var.environment}/pipeline-deployment-tfx.yaml" + content = local.pipeline_deploy_tfx +} + +resource "local_file" "deployment_kfp_yml" { + filename = "${path.module}/../../build/${var.environment}/pipeline-deployment-kfp.yaml" + content = local.pipeline_deploy_kfp +} + +resource "local_file" "pipeline_run_tfx_ml" { + filename = "${path.module}/../../build/${var.environment}/pipeline-run-tfx.yaml" + content = local.pipeline_run_tfx +} + +resource "local_file" "pipeline_run_kfp_ml" { + filename = "${path.module}/../../build/${var.environment}/pipeline-run-kfp.yaml" + content = local.pipeline_run_kfp +} + +resource "local_file" "model_deploy_yml" { + filename = "${path.module}/../../build/${var.environment}/model-deployment.yaml" + content = local.model_deployment +} + +output "mlops" { + description = "Created project, service accounts and associates resources." + value = module.mlops +} diff --git a/examples/vertex_mlops_enterprise/terraform/01-dev/terraform.tfvars.sample b/examples/vertex_mlops_enterprise/terraform/01-dev/terraform.tfvars.sample new file mode 100644 index 0000000000..b4a15f40a8 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/01-dev/terraform.tfvars.sample @@ -0,0 +1,43 @@ +bucket_name = "creditcards" # -env will be added as suffix +dataset_name = "creditcards" +environment = "dev" +groups = { + gcp-ml-ds = null + gcp-ml-eng = null + gcp-ml-viewer = null +} + +# env will be added as branch name +github = { + organization = "GITHUB_ORG" + repo = "GITHUB_REPO" +} + +# Additional labels. env label will be added automatically +labels = { + "team" : "ml" +} + +notebooks = { + "nb" : { + type = "USER_MANAGED" + }, +} + +prefix = "myprefix" +project_config = { + billing_account_id = "000000-111111-222222" # Use only billing BA if it is required to create the project + parent = "folders/123456789012" + project_id = "creditcards" # -env will be added as suffix +} +region = "europe-west4" + + +service_encryption_keys = { + aiplatform = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" + bq = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/bq" + notebooks = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" + secretmanager = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" + storage = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" +} + diff --git a/examples/vertex_mlops_enterprise/terraform/01-dev/variables.tf b/examples/vertex_mlops_enterprise/terraform/01-dev/variables.tf new file mode 100644 index 0000000000..4580d223bd --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/01-dev/variables.tf @@ -0,0 +1,114 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +variable "bucket_name" { + description = "Create GCS Bucket." + type = string + default = null +} + +variable "dataset_name" { + description = "Create BigQuery Datasets." + type = string + default = null +} + +variable "environment" { + description = "Environment prefix that will be used when creating some resources and outputs (Github Actions, build files, etc.): dev, stg, prod" + type = string +} + +variable "groups" { + description = "Name of the groups (name@domain.org) to apply IAM permissions." + type = object({ + gcp-ml-ds = string + gcp-ml-eng = string + gcp-ml-viewer = string + }) + default = { + gcp-ml-ds = null + gcp-ml-eng = null + gcp-ml-viewer = null + } + nullable = false +} + +variable "github" { + description = "Github organization and repo, i.e: https://github.com/ORGANIZATION/REPO " + type = object({ + organization = string + repo = string + }) + default = null +} + +variable "labels" { + description = "Labels to be assigned at project level." + type = map(string) + default = {} +} + +variable "notebooks" { + description = "Vertex AI workbenchs to be deployed." + type = any + default = {} +} + +variable "prefix" { + description = "Prefix used for the project id." + type = string + default = null +} + +variable "project_config" { + description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_id' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." + type = object({ + billing_account_id = optional(string) + parent = optional(string) + project_id = string + }) + validation { + condition = var.project_config.project_id != null + error_message = "Project id must be set." + } + nullable = false +} + +variable "region" { + description = "Region used for regional resources." + type = string + default = "europe-west4" +} + +variable "sa_mlops_name" { + description = "Name for the MLOPs Service Account." + type = string + default = "sa-mlops" +} + +variable "service_encryption_keys" { + description = "Cloud KMS to use to encrypt different services. Key location should match service region." + type = object({ + aiplatform = optional(string) + bq = optional(string) + notebooks = optional(string) + secretmanager = optional(string) + storage = optional(string) + }) + default = {} + nullable = false +} \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/terraform/02-staging/main.tf b/examples/vertex_mlops_enterprise/terraform/02-staging/main.tf new file mode 100644 index 0000000000..37743f7dce --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/02-staging/main.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Vertex MLOps + +locals { + bucket_name = "${var.bucket_name}-${var.environment}" + env_label = { + "env" : "${var.environment}" + } + labels = merge(local.env_label, var.labels) + + github = { + organization = var.github.organization + repo = var.github.repo + branch = var.environment + } + + identity_pool_claims = try("attribute.repository/${var.github.organization}/${var.github.repo}", null) + + project_config = { + billing_account_id = var.project_config.billing_account_id + parent = var.project_config.parent + project_id = "${var.project_config.project_id}-${var.environment}" + } + +} +module "mlops" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//blueprints/data-solutions/vertex-mlops" + project_config = local.project_config + prefix = var.prefix + bucket_name = local.bucket_name + dataset_name = var.dataset_name + groups = var.groups + identity_pool_claims = local.identity_pool_claims + labels = local.labels + notebooks = var.notebooks + service_encryption_keys = var.service_encryption_keys +} diff --git a/examples/vertex_mlops_enterprise/terraform/02-staging/outputs.tf b/examples/vertex_mlops_enterprise/terraform/02-staging/outputs.tf new file mode 100644 index 0000000000..633ea98199 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/02-staging/outputs.tf @@ -0,0 +1,144 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +locals { + mainconfig_yaml = templatefile("${path.module}/../../mainconfig.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + docker_repo = module.mlops.github.DOCKER_REPO, + sa_mlops = module.mlops.github.SA_MLOPS, + subnetwork = module.mlops.github.SUBNETWORK + }) + + gh_containers_yaml = templatefile("${path.module}/../../.github/workflows/containers.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO + environment = var.environment + }) + + gh_main_yaml = templatefile("${path.module}/../../.github/workflows/main.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO + environment = var.environment + }) + + gh_run_yaml = templatefile("${path.module}/../../.github/workflows/run.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO + environment = var.environment + }) + + gh_deploy_yaml = templatefile("${path.module}/../../.github/workflows/deploy.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO + environment = var.environment + }) + + pipeline_deploy = templatefile("${path.module}/../../build/pipeline-deployment.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + docker_repo = module.mlops.github.DOCKER_REPO, + sa_mlops = module.mlops.github.SA_MLOPS, + subnetwork = module.mlops.github.SUBNETWORK + }) + + pipeline_run = templatefile("${path.module}/../../build/pipeline-run.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + sa_mlops = module.mlops.github.SA_MLOPS, + }) + + + model_deployment = templatefile("${path.module}/../../build/model-deployment.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + docker_repo = module.mlops.github.DOCKER_REPO, + sa_mlops = module.mlops.github.SA_MLOPS, + }) +} + + +resource "local_file" "mainconfig_yml" { + filename = "${path.module}/../../mainconfig.yaml" + content = local.mainconfig_yaml +} + + +resource "local_file" "containers_yml" { + filename = "${path.module}/../../.github/workflows/containers.yml" + content = local.gh_containers_yaml +} + + + +resource "local_file" "main_yml" { + filename = "${path.module}/../../.github/workflows/main.yml" + content = local.gh_main_yaml +} + +resource "local_file" "run_yml" { + filename = "${path.module}/../../.github/workflows/run.yml" + content = local.gh_run_yaml +} + +resource "local_file" "deploy_yml" { + filename = "${path.module}/../../.github/workflows/deploy.yml" + content = local.gh_deploy_yaml +} + +resource "local_file" "deployment_yml" { + filename = "${path.module}/../../build/${var.environment}/pipeline-deployment.yaml" + content = local.pipeline_deploy +} + + +resource "local_file" "pipeline_run_yml" { + filename = "${path.module}/../../build/${var.environment}/pipeline-run.yaml" + content = local.pipeline_run +} + +resource "local_file" "model_deploy_yml" { + filename = "${path.module}/../../build/${var.environment}/model-deployment.yaml" + content = local.model_deployment +} + + +output "mlops" { + description = "Created project, service accounts and associates resources." + value = module.mlops +} diff --git a/examples/vertex_mlops_enterprise/terraform/02-staging/providers.tf b/examples/vertex_mlops_enterprise/terraform/02-staging/providers.tf new file mode 100644 index 0000000000..e6511a281b --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/02-staging/providers.tf @@ -0,0 +1,13 @@ +terraform { + backend "gcs" { + bucket = "pcorp-iac-core-bucket" + prefix = "mlops1" + } +} + +provider "google" { + impersonate_service_account = "pcorp-iac-core@pcorp-iac-core.iam.gserviceaccount.com" +} +provider "google-beta" { + impersonate_service_account = "pcorp-iac-core@pcorp-iac-core.iam.gserviceaccount.com" +} diff --git a/examples/vertex_mlops_enterprise/terraform/02-staging/terraform.tfvars.sample b/examples/vertex_mlops_enterprise/terraform/02-staging/terraform.tfvars.sample new file mode 100644 index 0000000000..6e20ecea11 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/02-staging/terraform.tfvars.sample @@ -0,0 +1,37 @@ +bucket_name = "creditcards" +dataset_name = "creditcards" +environment = "stg" +groups = { + gcp-ml-ds = null + gcp-ml-eng = null + gcp-ml-viewer = null +} + +# env will be added as branch name +github = { + organization = "GITHUB_ORG" + repo = "GITHUB_REPO" +} + +# Additional labels. env label will be added automatically +labels = { + "team" : "ml" +} + +prefix = "myprefix" +project_config = { + billing_account_id = "000000-111111-222222" # Use only billing BA if it is required to create the project + parent = "folders/123456789012" + project_id = "creditcards" # -env will be added as suffix +} +region = "europe-west4" + + +service_encryption_keys = { + aiplatform = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" + bq = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/bq" + notebooks = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" + secretmanager = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" + storage = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" +} + diff --git a/examples/vertex_mlops_enterprise/terraform/02-staging/variables.tf b/examples/vertex_mlops_enterprise/terraform/02-staging/variables.tf new file mode 100644 index 0000000000..82b4d85d03 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/02-staging/variables.tf @@ -0,0 +1,114 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +variable "bucket_name" { + description = "Create GCS Bucket." + type = string + default = null +} + +variable "dataset_name" { + description = "Create BigQuery Datasets." + type = string + default = null +} + +variable "environment" { + description = "Environment prefix that will be used when creating some resources and outputs (Github Actions, build files, etc.): dev, stg, prod" + type = string +} + +variable "groups" { + description = "Name of the groups (name@domain.org) to apply IAM permissions." + type = object({ + gcp-ml-ds = string + gcp-ml-eng = string + gcp-ml-viewer = string + }) + default = { + gcp-ml-ds = null + gcp-ml-eng = null + gcp-ml-viewer = null + } + nullable = false +} + +variable "github" { + description = "Github organization and repo, i.e: https://github.com/ORGANIZATION/REPO " + type = object({ + organization = string + repo = string + }) + default = null +} + +variable "labels" { + description = "Labels to be assigned at project level." + type = map(string) + default = {} +} + +variable "notebooks" { + description = "Vertex AI workbenchs to be deployed." + type = any + default = {} +} + +variable "prefix" { + description = "Prefix used for the project id." + type = string + default = null +} + +variable "project_config" { + description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_id' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." + type = object({ + billing_account_id = optional(string) + parent = optional(string) + project_id = string + }) + validation { + condition = var.project_config.project_id != null + error_message = "Project id must be set." + } + nullable = false +} + +variable "region" { + description = "Region used for regional resources." + type = string + default = "europe-west4" +} + +variable "sa_mlops_name" { + description = "Name for the MLOPs Service Account." + type = string + default = "sa-mlops" +} + +variable "service_encryption_keys" { + description = "Cloud KMS to use to encrypt different services. Key location should match service region." + type = object({ + aiplatform = optional(string) + bq = optional(string) + notebooks = optional(string) + secretmanager = optional(string) + storage = optional(string) + }) + default = {} + nullable = false +} \ No newline at end of file diff --git a/examples/vertex_mlops_enterprise/terraform/03-prod/main.tf b/examples/vertex_mlops_enterprise/terraform/03-prod/main.tf new file mode 100644 index 0000000000..37743f7dce --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/03-prod/main.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Vertex MLOps + +locals { + bucket_name = "${var.bucket_name}-${var.environment}" + env_label = { + "env" : "${var.environment}" + } + labels = merge(local.env_label, var.labels) + + github = { + organization = var.github.organization + repo = var.github.repo + branch = var.environment + } + + identity_pool_claims = try("attribute.repository/${var.github.organization}/${var.github.repo}", null) + + project_config = { + billing_account_id = var.project_config.billing_account_id + parent = var.project_config.parent + project_id = "${var.project_config.project_id}-${var.environment}" + } + +} +module "mlops" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//blueprints/data-solutions/vertex-mlops" + project_config = local.project_config + prefix = var.prefix + bucket_name = local.bucket_name + dataset_name = var.dataset_name + groups = var.groups + identity_pool_claims = local.identity_pool_claims + labels = local.labels + notebooks = var.notebooks + service_encryption_keys = var.service_encryption_keys +} diff --git a/examples/vertex_mlops_enterprise/terraform/03-prod/outputs.tf b/examples/vertex_mlops_enterprise/terraform/03-prod/outputs.tf new file mode 100644 index 0000000000..633ea98199 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/03-prod/outputs.tf @@ -0,0 +1,144 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +locals { + mainconfig_yaml = templatefile("${path.module}/../../mainconfig.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + docker_repo = module.mlops.github.DOCKER_REPO, + sa_mlops = module.mlops.github.SA_MLOPS, + subnetwork = module.mlops.github.SUBNETWORK + }) + + gh_containers_yaml = templatefile("${path.module}/../../.github/workflows/containers.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO + environment = var.environment + }) + + gh_main_yaml = templatefile("${path.module}/../../.github/workflows/main.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO + environment = var.environment + }) + + gh_run_yaml = templatefile("${path.module}/../../.github/workflows/run.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO + environment = var.environment + }) + + gh_deploy_yaml = templatefile("${path.module}/../../.github/workflows/deploy.yml.TEMPLATE", { + wip = module.mlops.github.WORKLOAD_ID_PROVIDER, + project_id = module.mlops.github.PROJECT_ID, + sa = module.mlops.github.SERVICE_ACCOUNT, + docker_repo = module.mlops.github.DOCKER_REPO + environment = var.environment + }) + + pipeline_deploy = templatefile("${path.module}/../../build/pipeline-deployment.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + docker_repo = module.mlops.github.DOCKER_REPO, + sa_mlops = module.mlops.github.SA_MLOPS, + subnetwork = module.mlops.github.SUBNETWORK + }) + + pipeline_run = templatefile("${path.module}/../../build/pipeline-run.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + sa_mlops = module.mlops.github.SA_MLOPS, + }) + + + model_deployment = templatefile("${path.module}/../../build/model-deployment.yaml.TEMPLATE", { + project_id = module.mlops.github.PROJECT_ID, + region = var.region, + github_org = try(var.github.organization, null), + github_repo = try(var.github.repo, null), + github_branch = try(local.github.branch, null), + docker_repo = module.mlops.github.DOCKER_REPO, + sa_mlops = module.mlops.github.SA_MLOPS, + }) +} + + +resource "local_file" "mainconfig_yml" { + filename = "${path.module}/../../mainconfig.yaml" + content = local.mainconfig_yaml +} + + +resource "local_file" "containers_yml" { + filename = "${path.module}/../../.github/workflows/containers.yml" + content = local.gh_containers_yaml +} + + + +resource "local_file" "main_yml" { + filename = "${path.module}/../../.github/workflows/main.yml" + content = local.gh_main_yaml +} + +resource "local_file" "run_yml" { + filename = "${path.module}/../../.github/workflows/run.yml" + content = local.gh_run_yaml +} + +resource "local_file" "deploy_yml" { + filename = "${path.module}/../../.github/workflows/deploy.yml" + content = local.gh_deploy_yaml +} + +resource "local_file" "deployment_yml" { + filename = "${path.module}/../../build/${var.environment}/pipeline-deployment.yaml" + content = local.pipeline_deploy +} + + +resource "local_file" "pipeline_run_yml" { + filename = "${path.module}/../../build/${var.environment}/pipeline-run.yaml" + content = local.pipeline_run +} + +resource "local_file" "model_deploy_yml" { + filename = "${path.module}/../../build/${var.environment}/model-deployment.yaml" + content = local.model_deployment +} + + +output "mlops" { + description = "Created project, service accounts and associates resources." + value = module.mlops +} diff --git a/examples/vertex_mlops_enterprise/terraform/03-prod/providers.tf b/examples/vertex_mlops_enterprise/terraform/03-prod/providers.tf new file mode 100644 index 0000000000..e6511a281b --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/03-prod/providers.tf @@ -0,0 +1,13 @@ +terraform { + backend "gcs" { + bucket = "pcorp-iac-core-bucket" + prefix = "mlops1" + } +} + +provider "google" { + impersonate_service_account = "pcorp-iac-core@pcorp-iac-core.iam.gserviceaccount.com" +} +provider "google-beta" { + impersonate_service_account = "pcorp-iac-core@pcorp-iac-core.iam.gserviceaccount.com" +} diff --git a/examples/vertex_mlops_enterprise/terraform/03-prod/terraform.tfvars.sample b/examples/vertex_mlops_enterprise/terraform/03-prod/terraform.tfvars.sample new file mode 100644 index 0000000000..fa5dd769db --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/03-prod/terraform.tfvars.sample @@ -0,0 +1,37 @@ +bucket_name = "creditcards-dev" +dataset_name = "creditcards" +environment = "prd" +groups = { + gcp-ml-ds = null + gcp-ml-eng = null + gcp-ml-viewer = null +} + +# env will be added as branch name +github = { + organization = "GITHUB_ORG" + repo = "GITHUB_REPO" +} + +# Additional labels. env label will be added automatically +labels = { + "team" : "ml" +} + +prefix = "myprefix" +project_config = { + billing_account_id = "000000-111111-222222" # Use only billing BA if it is required to create the project + parent = "folders/123456789012" + project_id = "creditcards" # -env will be added as suffix +} +region = "europe-west4" + + +service_encryption_keys = { + aiplatform = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" + bq = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/bq" + notebooks = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" + secretmanager = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" + storage = "projects/CMEK_PROJECT_ID/locations/europe-west4/keyRings/europe-west4/cryptoKeys/storage" +} + diff --git a/examples/vertex_mlops_enterprise/terraform/03-prod/variables.tf b/examples/vertex_mlops_enterprise/terraform/03-prod/variables.tf new file mode 100644 index 0000000000..82b4d85d03 --- /dev/null +++ b/examples/vertex_mlops_enterprise/terraform/03-prod/variables.tf @@ -0,0 +1,114 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +variable "bucket_name" { + description = "Create GCS Bucket." + type = string + default = null +} + +variable "dataset_name" { + description = "Create BigQuery Datasets." + type = string + default = null +} + +variable "environment" { + description = "Environment prefix that will be used when creating some resources and outputs (Github Actions, build files, etc.): dev, stg, prod" + type = string +} + +variable "groups" { + description = "Name of the groups (name@domain.org) to apply IAM permissions." + type = object({ + gcp-ml-ds = string + gcp-ml-eng = string + gcp-ml-viewer = string + }) + default = { + gcp-ml-ds = null + gcp-ml-eng = null + gcp-ml-viewer = null + } + nullable = false +} + +variable "github" { + description = "Github organization and repo, i.e: https://github.com/ORGANIZATION/REPO " + type = object({ + organization = string + repo = string + }) + default = null +} + +variable "labels" { + description = "Labels to be assigned at project level." + type = map(string) + default = {} +} + +variable "notebooks" { + description = "Vertex AI workbenchs to be deployed." + type = any + default = {} +} + +variable "prefix" { + description = "Prefix used for the project id." + type = string + default = null +} + +variable "project_config" { + description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_id' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." + type = object({ + billing_account_id = optional(string) + parent = optional(string) + project_id = string + }) + validation { + condition = var.project_config.project_id != null + error_message = "Project id must be set." + } + nullable = false +} + +variable "region" { + description = "Region used for regional resources." + type = string + default = "europe-west4" +} + +variable "sa_mlops_name" { + description = "Name for the MLOPs Service Account." + type = string + default = "sa-mlops" +} + +variable "service_encryption_keys" { + description = "Cloud KMS to use to encrypt different services. Key location should match service region." + type = object({ + aiplatform = optional(string) + bq = optional(string) + notebooks = optional(string) + secretmanager = optional(string) + storage = optional(string) + }) + default = {} + nullable = false +} \ No newline at end of file diff --git a/tools/bq-visualizer/package-lock.json b/tools/bq-visualizer/package-lock.json index 326b90eab9..c33ce7d771 100644 --- a/tools/bq-visualizer/package-lock.json +++ b/tools/bq-visualizer/package-lock.json @@ -3318,9 +3318,9 @@ } }, "node_modules/@babel/traverse": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.0.tgz", - "integrity": "sha512-t/QaEvyIoIkwzpiZ7aoSKK8kObQYeF7T2v+dazAYCb8SXtp58zEVkWW7zAnju8FNKNdr4ScAOEDmMItbyOmEYw==", + "version": "7.23.2", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", + "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", "dev": true, "dependencies": { "@babel/code-frame": "^7.22.13", @@ -21424,9 +21424,9 @@ } }, "@babel/traverse": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.0.tgz", - "integrity": "sha512-t/QaEvyIoIkwzpiZ7aoSKK8kObQYeF7T2v+dazAYCb8SXtp58zEVkWW7zAnju8FNKNdr4ScAOEDmMItbyOmEYw==", + "version": "7.23.2", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", + "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", "dev": true, "requires": { "@babel/code-frame": "^7.22.13",