Skip to content

Commit 98306e6

Browse files
fredms2fredms2
committed
Update source data storage
1 parent 13fb4f2 commit 98306e6

15 files changed

+293
-28
lines changed

‎.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_cli_samples.yml

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# This code is autogenerated.
2+
# Code is generated by running custom script: python3 readme.py
3+
# Any manual changes to this file may cause incorrect behavior.
4+
# Any manual changes will be overwritten if the code is regenerated.
5+
6+
name: sdk-featurestore_sample-automation-test-test_featurestore_cli_samples
7+
# This file is created by sdk/python/readme.py.
8+
# Please do not edit directly.
9+
on:
10+
workflow_dispatch:
11+
schedule:
12+
- cron: "56 10/12 * * *"
13+
pull_request:
14+
branches:
15+
- main
16+
paths:
17+
- sdk/python/featurestore_sample/automation-test/**
18+
- .github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_cli_samples.yml
19+
- sdk/python/dev-requirements.txt
20+
- infra/bootstrapping/**
21+
- sdk/python/setup.sh
22+
- sdk/python/featurestore_sample/**
23+
permissions:
24+
id-token: write
25+
concurrency:
26+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
27+
cancel-in-progress: true
28+
jobs:
29+
build:
30+
runs-on: ubuntu-latest
31+
steps:
32+
- name: check out repo
33+
uses: actions/checkout@v2
34+
- name: setup python
35+
uses: actions/setup-python@v2
36+
with:
37+
python-version: "3.10"
38+
- name: pip install notebook reqs
39+
run: pip install -r sdk/python/dev-requirements.txt
40+
- name: azure login
41+
uses: azure/login@v1
42+
with:
43+
client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }}
44+
tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }}
45+
subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }}
46+
- name: bootstrap resources
47+
run: |
48+
echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
49+
bash bootstrap.sh
50+
working-directory: infra/bootstrapping
51+
continue-on-error: false
52+
- name: setup SDK
53+
run: |
54+
source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
55+
source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
56+
bash setup.sh
57+
working-directory: sdk/python
58+
continue-on-error: true
59+
- name: validate readme
60+
run: |
61+
python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample"
62+
working-directory: infra/bootstrapping
63+
continue-on-error: false
64+
- name: setup-cli
65+
run: |
66+
source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
67+
source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
68+
bash setup.sh
69+
working-directory: cli
70+
continue-on-error: true
71+
- name: setup feature-store resources
72+
run: |
73+
bash -x automation-test/setup-resources-cli.sh automation-test/test_featurestore_cli_samples.ipynb
74+
working-directory: sdk/python/featurestore_sample
75+
continue-on-error: true
76+
- name: run featurestore_sample/automation-test/test_featurestore_cli_samples.ipynb
77+
run: |
78+
source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
79+
source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
80+
bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
81+
bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "test_featurestore_cli_samples.ipynb";
82+
[ -f "../../.azureml/config" ] && cat "../../.azureml/config";
83+
papermill -k python test_featurestore_cli_samples.ipynb test_featurestore_cli_samples.output.ipynb
84+
working-directory: sdk/python/featurestore_sample/automation-test
85+
- name: upload notebook's working folder as an artifact
86+
if: ${{ always() }}
87+
uses: actions/upload-artifact@v2
88+
with:
89+
name: test_featurestore_cli_samples
90+
path: sdk/python/featurestore_sample/automation-test

‎.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples.yml

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# This code is autogenerated.
2+
# Code is generated by running custom script: python3 readme.py
3+
# Any manual changes to this file may cause incorrect behavior.
4+
# Any manual changes will be overwritten if the code is regenerated.
5+
6+
name: sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples
7+
# This file is created by sdk/python/readme.py.
8+
# Please do not edit directly.
9+
on:
10+
workflow_dispatch:
11+
schedule:
12+
- cron: "22 2/12 * * *"
13+
pull_request:
14+
branches:
15+
- main
16+
paths:
17+
- sdk/python/featurestore_sample/automation-test/**
18+
- .github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples.yml
19+
- sdk/python/dev-requirements.txt
20+
- infra/bootstrapping/**
21+
- sdk/python/setup.sh
22+
- sdk/python/featurestore_sample/**
23+
permissions:
24+
id-token: write
25+
concurrency:
26+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
27+
cancel-in-progress: true
28+
jobs:
29+
build:
30+
runs-on: ubuntu-latest
31+
steps:
32+
- name: check out repo
33+
uses: actions/checkout@v2
34+
- name: setup python
35+
uses: actions/setup-python@v2
36+
with:
37+
python-version: "3.10"
38+
- name: pip install notebook reqs
39+
run: pip install -r sdk/python/dev-requirements.txt
40+
- name: azure login
41+
uses: azure/login@v1
42+
with:
43+
client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }}
44+
tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }}
45+
subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }}
46+
- name: bootstrap resources
47+
run: |
48+
echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
49+
bash bootstrap.sh
50+
working-directory: infra/bootstrapping
51+
continue-on-error: false
52+
- name: setup SDK
53+
run: |
54+
source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
55+
source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
56+
bash setup.sh
57+
working-directory: sdk/python
58+
continue-on-error: true
59+
- name: validate readme
60+
run: |
61+
python check-readme.py "${{ github.workspace }}/sdk/python/featurestore_sample"
62+
working-directory: infra/bootstrapping
63+
continue-on-error: false
64+
- name: setup-cli
65+
run: |
66+
source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
67+
source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
68+
bash setup.sh
69+
working-directory: cli
70+
continue-on-error: true
71+
- name: setup feature-store resources
72+
run: |
73+
bash -x automation-test/setup-resources.sh automation-test/test_featurestore_sdk_samples.ipynb
74+
working-directory: sdk/python/featurestore_sample
75+
continue-on-error: true
76+
- name: run featurestore_sample/automation-test/test_featurestore_sdk_samples.ipynb
77+
run: |
78+
source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
79+
source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
80+
bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
81+
bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "test_featurestore_sdk_samples.ipynb";
82+
[ -f "../../.azureml/config" ] && cat "../../.azureml/config";
83+
papermill -k python test_featurestore_sdk_samples.ipynb test_featurestore_sdk_samples.output.ipynb
84+
working-directory: sdk/python/featurestore_sample/automation-test
85+
- name: upload notebook's working folder as an artifact
86+
if: ${{ always() }}
87+
uses: actions/upload-artifact@v2
88+
with:
89+
name: test_featurestore_sdk_samples
90+
path: sdk/python/featurestore_sample/automation-test

‎.github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_vnet_samples.yml

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# This code is autogenerated.
2+
# Code is generated by running custom script: python3 readme.py
3+
# Any manual changes to this file may cause incorrect behavior.
4+
# Any manual changes will be overwritten if the code is regenerated.
5+
6+
name: sdk-featurestore_sample-automation-test-test_featurestore_vnet_samples
7+
# This file is created by sdk/python/readme.py.
8+
# Please do not edit directly.
9+
on:
10+
workflow_dispatch:
11+
schedule:
12+
- cron: "3 8/12 * * *"
13+
pull_request:
14+
branches:
15+
- main
16+
paths:
17+
- sdk/python/featurestore_sample/automation-test/**
18+
- .github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_vnet_samples.yml
19+
- sdk/python/dev-requirements.txt
20+
- infra/bootstrapping/**
21+
- sdk/python/setup.sh
22+
- sdk/python/featurestore_sample/**
23+
permissions:
24+
id-token: write
25+
concurrency:
26+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
27+
cancel-in-progress: true
28+
jobs:
29+
build:
30+
runs-on: ubuntu-latest
31+
steps:
32+
- name: check out repo
33+
uses: actions/checkout@v2
34+
- name: setup python
35+
uses: actions/setup-python@v2
36+
with:
37+
python-version: "3.10"
38+
- name: pip install notebook reqs
39+
run: pip install -r sdk/python/dev-requirements.txt
40+
- name: azure login
41+
uses: azure/login@v1
42+
with:
43+
client-id: ${{ secrets.OIDC_AZURE_CLIENT_ID }}
44+
tenant-id: ${{ secrets.OIDC_AZURE_TENANT_ID }}
45+
subscription-id: ${{ secrets.OIDC_AZURE_SUBSCRIPTION_ID }}
46+
- name: bootstrap resources
47+
run: |
48+
echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
49+
bash bootstrap.sh
50+
working-directory: infra/bootstrapping
51+
continue-on-error: false
52+
- name: setup SDK
53+
run: |
54+
source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
55+
source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
56+
bash setup.sh
57+
working-directory: sdk/python
58+
continue-on-error: true
59+
- name: setup-cli
60+
run: |
61+
source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
62+
source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
63+
bash setup.sh
64+
working-directory: cli
65+
continue-on-error: true
66+
- name: setup feature-store resources
67+
run: |
68+
bash -x automation-test/setup-resources-vnet.sh automation-test/test_featurestore_vnet_samples.ipynb
69+
working-directory: sdk/python/featurestore_sample
70+
continue-on-error: true
71+
- name: run featurestore_sample/automation-test/test_featurestore_vnet_samples.ipynb
72+
run: |
73+
source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
74+
source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
75+
bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
76+
bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "test_featurestore_vnet_samples.ipynb";
77+
[ -f "../../.azureml/config" ] && cat "../../.azureml/config";
78+
papermill -k python test_featurestore_vnet_samples.ipynb test_featurestore_vnet_samples.output.ipynb
79+
working-directory: sdk/python/featurestore_sample/automation-test
80+
- name: upload notebook's working folder as an artifact
81+
if: ${{ always() }}
82+
uses: actions/upload-artifact@v2
83+
with:
84+
name: test_featurestore_vnet_samples
85+
path: sdk/python/featurestore_sample/automation-test

‎sdk/python/featurestore_sample/featurestore/featuresets/accounts/spec/FeatureSetSpec.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ index_columns:
1111
- name: accountID
1212
type: string
1313
source:
14-
path: wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet
14+
path: wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet
1515
timestamp_column:
1616
name: timestamp
1717
type: parquet

‎sdk/python/featurestore_sample/featurestore/featuresets/transactions/spec/FeatureSetSpec.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ index_columns:
1919
- name: accountID
2020
type: string
2121
source:
22-
path: wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet
22+
path: wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet
2323
source_delay:
2424
days: 0
2525
hours: 0

‎sdk/python/featurestore_sample/notebooks/sdk_and_cli/1.Develop-feature-set-and-register.ipynb

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -565,7 +565,7 @@
565565
"outputs": [],
566566
"source": [
567567
"# Remove the \".\" in the roor directory path as we need to generate absolute path to read from Spark.\n",
568-
"transactions_source_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet\"\n",
568+
"transactions_source_data_path = \"wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet\"\n",
569569
"transactions_src_df = spark.read.parquet(transactions_source_data_path)\n",
570570
"\n",
571571
"display(transactions_src_df.head(5))\n",
@@ -640,7 +640,7 @@
640640
"transactions_featureset_spec = create_feature_set_spec(\n",
641641
" source=FeatureSource(\n",
642642
" type=SourceType.parquet,\n",
643-
" path=\"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet\",\n",
643+
" path=\"wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet\",\n",
644644
" timestamp_column=TimestampColumn(name=\"timestamp\"),\n",
645645
" source_delay=DateTimeOffset(days=0, hours=0, minutes=20),\n",
646646
" ),\n",
@@ -930,7 +930,7 @@
930930
},
931931
"outputs": [],
932932
"source": [
933-
"observation_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/observation_data/train/*.parquet\"\n",
933+
"observation_data_path = \"wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/observation_data/train/*.parquet\"\n",
934934
"observation_data_df = spark.read.parquet(observation_data_path)\n",
935935
"obs_data_timestamp_column = \"timestamp\"\n",
936936
"\n",

‎sdk/python/featurestore_sample/notebooks/sdk_and_cli/2.Experiment-train-models-using-features.ipynb

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,7 @@
471471
},
472472
"outputs": [],
473473
"source": [
474-
"accounts_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet\"\n",
474+
"accounts_data_path = \"wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet\"\n",
475475
"accounts_df = spark.read.parquet(accounts_data_path)\n",
476476
"\n",
477477
"display(accounts_df.head(5))"
@@ -526,7 +526,7 @@
526526
"accounts_featureset_spec = create_feature_set_spec(\n",
527527
" source=FeatureSource(\n",
528528
" type=SourceType.parquet,\n",
529-
" path=\"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet\",\n",
529+
" path=\"wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet\",\n",
530530
" timestamp_column=TimestampColumn(name=\"timestamp\"),\n",
531531
" ),\n",
532532
" index_columns=[Column(name=\"accountID\", type=ColumnType.string)],\n",
@@ -703,7 +703,7 @@
703703
"from azureml.featurestore import get_offline_features\n",
704704
"\n",
705705
"# Load the observation data. To understand observatio ndata, refer to part 1 of this tutorial\n",
706-
"observation_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/observation_data/train/*.parquet\"\n",
706+
"observation_data_path = \"wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/observation_data/train/*.parquet\"\n",
707707
"observation_data_df = spark.read.parquet(observation_data_path)\n",
708708
"obs_data_timestamp_column = \"timestamp\""
709709
]

‎sdk/python/featurestore_sample/notebooks/sdk_and_cli/network_isolation/Network-isolation-feature-store.ipynb

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -603,7 +603,7 @@
603603
"outputs": [],
604604
"source": [
605605
"# Copy sample source data for transactions feature set used in this tutorial series from the public storage account to the newly created storage account\n",
606-
"transactions_source_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet\"\n",
606+
"transactions_source_data_path = \"wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/datasources/transactions-source/*.parquet\"\n",
607607
"transactions_src_df = spark.read.parquet(transactions_source_data_path)\n",
608608
"\n",
609609
"transactions_src_df.write.parquet(\n",
@@ -643,7 +643,7 @@
643643
"outputs": [],
644644
"source": [
645645
"# Copy sample source data for account feature set used in this tutorial series from the public storage account to the newly created storage account\n",
646-
"accounts_data_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet\"\n",
646+
"accounts_data_path = \"wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/datasources/accounts-precalculated/*.parquet\"\n",
647647
"accounts_data_df = spark.read.parquet(accounts_data_path)\n",
648648
"\n",
649649
"accounts_data_df.write.parquet(\n",
@@ -683,7 +683,7 @@
683683
"outputs": [],
684684
"source": [
685685
"# Copy sample observation data used for training from the public storage account to the newly created storage account\n",
686-
"observation_data_train_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/observation_data/train/*.parquet\"\n",
686+
"observation_data_train_path = \"wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/observation_data/train/*.parquet\"\n",
687687
"observation_data_train_df = spark.read.parquet(observation_data_train_path)\n",
688688
"\n",
689689
"observation_data_train_df.write.parquet(\n",
@@ -723,7 +723,7 @@
723723
"outputs": [],
724724
"source": [
725725
"# Copy sample observation data used for batch inference from a public storage account to the newly created storage account\n",
726-
"observation_data_inference_path = \"wasbs://data@azuremlexampledata.blob.core.windows.net/feature-store-prp/observation_data/batch_inference/*.parquet\"\n",
726+
"observation_data_inference_path = \"wasbs://data@azuremlfsexampledata.blob.core.windows.net/feature-store-prp/observation_data/batch_inference/*.parquet\"\n",
727727
"observation_data_inference_df = spark.read.parquet(observation_data_inference_path)\n",
728728
"\n",
729729
"observation_data_inference_df.write.parquet(\n",

0 commit comments

Comments
 (0)