-
Notifications
You must be signed in to change notification settings - Fork 64
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #36 from ciaran28/main
Major Updates + Functionality
- Loading branch information
Showing
36 changed files
with
1,298 additions
and
339 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
31 changes: 31 additions & 0 deletions
31
.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsMIAuthentication.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
|
|
||
| AAD_TOKEN=$( az account get-access-token \ | ||
| --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d \ | ||
| --query "accessToken" \ | ||
| --output tsv ) | ||
|
|
||
| echo $AAD_TOKEN | ||
|
|
||
| CREATE_REPO_RESPONSE=$(curl -X POST -H "Authorization: Bearer $AAD_TOKEN" \ | ||
| -H "X-Databricks-Azure-Workspace-Resource-Id: $WORKSPACE_ID" \ | ||
| -H 'Content-Type: application/scim+json' \ | ||
| -d $JSON_STRING \ | ||
| '{ | ||
| "displayName": "My Service Principal", | ||
| "applicationId": "12a34b56-789c-0d12-e3fa-b456789c0123", | ||
| "entitlements": [ | ||
| { | ||
| "value": "allow-cluster-create" | ||
| } | ||
| ], | ||
| "schemas": [ | ||
| "urn:ietf:params:scim:schemas:core:2.0:ServicePrincipal" | ||
| ], | ||
| "active": true | ||
| }' https://$DATABRICKS_INSTANCE/api/2.0/preview/scim/v2/ServicePrincipals ) | ||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
2 changes: 2 additions & 0 deletions
2
.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
95 changes: 95 additions & 0 deletions
95
.azureDevOps/Pipelines/Managed_Identity_Auth/1-Master/onDeploy.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
|
|
||
| name: Managed Identity MLOps Databricks Deployment | ||
|
|
||
| trigger: none | ||
| pr: none | ||
|
|
||
|
|
||
|
|
||
| # TO DO : Retrofit This Code So You Can Choose The Environments You Want To Deploy. This Will be | ||
| # Helpfull if a Release Pipeline Fails And We Need To Rewind The Commit And Redeploy | ||
|
|
||
| # Create A Condition That This Will Not Run When A Pull Request Is Launched. Presumably a CI only Condition | ||
| # After The First Environment Deployment, This YAML Pipeline is Triggering Alongside onRealease. I have Set | ||
| # pr == none For Now To Prevent This Behaviour | ||
|
|
||
|
|
||
| pool: | ||
| name: 'vmss-linux-pool' | ||
|
|
||
|
|
||
| parameters: | ||
|
|
||
| - name: ENVIRONMENT | ||
| displayName: Choose Environment | ||
| default: GenesisDeployment | ||
| type: string | ||
| values: | ||
| - Sandbox | ||
| - Development | ||
| - UAT | ||
| - Production | ||
| - GenesisDeployment | ||
|
|
||
|
|
||
| - name: MI_VMSS_NAME | ||
| displayName: Enter name of Virtual Machine Scale Set With Managed Identity On Sub Enabled | ||
| default: vmsslinux | ||
| type: string | ||
|
|
||
|
|
||
| - name: azureSubscription | ||
| displayName: Enter Service Connection Name | ||
| default: mi-vmss-spn | ||
| type: string | ||
|
|
||
|
|
||
| - name: PULL_BRANCH | ||
| displayName: Branch For DBX Repo Folders | ||
| default: ciarand/dbx_pipeline | ||
| type: string | ||
|
|
||
|
|
||
| stages: | ||
|
|
||
| - stage: SandboxDeploy | ||
| condition: or(eq('${{ parameters.ENVIRONMENT }}', 'GenesisDeployment'), eq('${{ parameters.ENVIRONMENT }}', 'Sandbox')) | ||
| displayName: SandboxDeploy | ||
| jobs: | ||
| - template: ..\2-Jobs\jobDatabricks.yaml | ||
| parameters: | ||
| Environment: Sandbox | ||
| azureSubscription: ${{ parameters.azureSubscription }} | ||
| enableRepoPull: false | ||
| branchName: main | ||
| MI_VMSS_NAME: ${{ parameters.MI_VMSS_NAME }} | ||
|
|
||
| - stage: DevelopmentDeploy | ||
| condition: or(eq('${{ parameters.ENVIRONMENT }}', 'GenesisDeployment'), eq('${{ parameters.ENVIRONMENT }}', 'Development')) | ||
| displayName: DevelopmentDeploy | ||
| dependsOn: [] | ||
| jobs: | ||
| - template: ..\2-Jobs\jobDatabricks.yaml | ||
| parameters: | ||
| Environment: Development | ||
| azureSubscription: ${{ parameters.azureSubscription }} | ||
| enableRepoPull: false | ||
| branchName: main | ||
| updateFolder: None | ||
| MI_VMSS_NAME: ${{ parameters.MI_VMSS_NAME }} | ||
|
|
||
| - stage: UATDeploy | ||
| condition: or(eq('${{ parameters.ENVIRONMENT }}', 'GenesisDeployment'), eq('${{ parameters.ENVIRONMENT }}', 'UAT')) | ||
| displayName: UATDeploy | ||
| dependsOn: [] | ||
| jobs: | ||
| - template: ..\2-Jobs\jobDatabricks.yaml | ||
| parameters: | ||
| Environment: UAT | ||
| azureSubscription: ${{ parameters.azureSubscription }} | ||
| enableRepoPull: false | ||
| branchName: main | ||
| releaseBranch: 'release/1' | ||
| MI_VMSS_NAME: ${{ parameters.MI_VMSS_NAME }} | ||
|
|
||
|
|
116 changes: 116 additions & 0 deletions
116
.azureDevOps/Pipelines/Managed_Identity_Auth/1-Master/onRelease.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| # UPDATES PENDING - MAY NOT WORK | ||
|
|
||
|
|
||
|
|
||
| name: Databricks Deployment | ||
|
|
||
| trigger: | ||
| branches: | ||
| include: | ||
| - main | ||
| - dev | ||
| #- features/* | ||
| - releases/* | ||
| paths: | ||
| exclude: | ||
| - README.md | ||
|
|
||
| #pr: none | ||
|
|
||
| pool: | ||
| vmImage: 'ubuntu-latest' | ||
|
|
||
|
|
||
| #Secrets | ||
| variables: | ||
| - group: ADO-Secrets | ||
| - name: isPR | ||
| value: $[startsWith(variables['Build.SourceBranch'], 'refs/pull/')] | ||
|
|
||
| - name: isMain | ||
| value: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')] | ||
|
|
||
| - name: isPRFeatureOnMain | ||
| value: $[eq(variables['System.PullRequest.SourceBranch'], 'features/*')] | ||
|
|
||
|
|
||
| #resources: | ||
| # repositories: | ||
| # - repository: self | ||
| # ref: 'refs/heads/$(branchName)' | ||
|
|
||
|
|
||
| # Undo Pull Requests == None. Create conditions which run tests on the given branch. Pull Request will check out the | ||
| # Source Branch... i.e if you Pull Request from feature to Main , then it will use the feature branch code. This is ideal. | ||
| # Create a stage which runs tests on the feature branch code. If it comes back all green, then reviewer can appove the request. | ||
| # You could even have a deployment to a sandbox environment. Possibly overkill for Dev merges, but it might be good before promoting | ||
| # at higher levels! | ||
|
|
||
|
|
||
|
|
||
| # PROTECT THE MAIN BRANCH SO YOU CANT PUSH DIRECTLY TO IT | ||
| stages: | ||
| - stage: DevelopmentDeploy | ||
| condition: and(eq(variables['Build.SourceBranchName'], 'dev'), eq(variables['Build.Reason'], 'IndividualCI')) | ||
| displayName: DevelopmentDeploy | ||
| jobs: | ||
| - template: ..\2-Jobs\jobDatabricks.yaml | ||
| parameters: | ||
| Environment: Development | ||
| azureSubscription: DBX_ADO_DSTOOLKIT | ||
| enableRepoPull: true # should really be true when bash script is working | ||
| branchName: $(Build.SourceBranchName) | ||
| updateFolder: DevelopmentFolder | ||
|
|
||
| # This will deploy code on the source branch for the PR. If PR from Feature to Dev, then this will deploy Feature. | ||
| - stage: pullRequestChecks | ||
| condition: and(startsWith(variables['system.pullRequest.sourceBranch'], 'features/'), eq(variables['system.pullRequest.targetBranch'], 'dev')) | ||
| displayName: pullRequestChecks | ||
| jobs: | ||
| - job: | ||
| steps: | ||
| - checkout: self | ||
| - bash: echo "Dummy Test - This is changed in Feature Branch - If you see this then SUCCESSUL" | ||
|
|
||
|
|
||
| # TO DO - Test on PR From Dev To Release | ||
|
|
||
| # Change To Main Branch --> Deploy To Test Environment | ||
| - stage: UATDeploy | ||
| displayName: UATDeploy | ||
| condition: and(eq(variables['Build.SourceBranchName'], 'main'), eq(variables['Build.Reason'], 'IndividualCI')) | ||
| jobs: | ||
| - template: ..\2-Jobs\jobDatabricks.yaml | ||
| parameters: | ||
| Environment: UAT | ||
| azureSubscription: DBX_ADO_DSTOOLKIT | ||
| enableRepoPull: true # should really be true when bash script is working | ||
| branchName: $(Build.SourceBranchName) | ||
| updateFolder: DevelopmentFolder | ||
|
|
||
| - stage: pullRequestChecks_dev_to_main | ||
| condition: and(eq(variables['system.pullRequest.sourceBranch'], 'dev'), eq(variables['system.pullRequest.targetBranch'], 'main')) | ||
| displayName: pullRequestChecks | ||
| jobs: | ||
| - job: | ||
| steps: | ||
| - checkout: self | ||
| - bash: echo "Dummy Test " | ||
|
|
||
|
|
||
| # Tag Release Branch --> Deploy To Production Environment | ||
| # - stage: ProductionDeploy | ||
| # displayName: ProductionDeploy | ||
| # condition: and(startsWith(variables['Build.SourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'IndividualCI')) | ||
| # jobs: | ||
| # - template: ..\2-Jobs\Job-Databricks.yaml | ||
| # parameters: | ||
| # Environment: Production | ||
| # azureSubscription: DBX_ADO_DSTOOLKIT | ||
| # enableRepoPull: true | ||
| # branchName: $(Build.SourceBranchName) | ||
| # updateFolder: DevelopmentFolder | ||
|
|
||
|
|
||
| # Implement a condition to ignore a azure resource deployment if Infra folder is unchanged. | ||
| # https://pumpingco.de/blog/run-an-azure-pipelines-job-only-if-source-code-has-changed/ |
Oops, something went wrong.