Skip to content

Commit

Permalink
fixup! SDKTECHNO-271: add dockerInfos for sub-apps backup & restore f…
Browse files Browse the repository at this point in the history 
…or apps storages
  • Loading branch information
@EmilienLeroux
EmilienLeroux committed Oct 8, 2024
1 parent a9ac766 commit c9e7d0b
Show file tree
Hide file tree
Showing 8 changed files with 129 additions and 125 deletions.
28 changes: 14 additions & 14 deletions technologies/app/saagie-app-storages-backup-restore/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,19 @@ Useless:
- `BACKUP_STORAGE_FOLDER`: Dossier de stockage, Path directory on image

Facultatives :
- `BACKUP_LIST_APP_ID`: Liste des ID d'applications à sauvegarder
- `RESTORE_LIST_APP_ID`: Liste des ID d'applications à restaurer
- `BACKUP_TMP_APP_PREFIX`: Préfixe nom de l'app temporaire
- `RESTORE_TMP_APP_PREFIX`: Préfixe nom de l'app temporaire
- `SAAGIE_APP_BACKUP_LIST_APP_ID`: Liste des ID d'applications à sauvegarder
- `SAAGIE_APP_RESTORE_LIST_APP_ID`: Liste des ID d'applications à restaurer
- `SAAGIE_APP_BACKUP_TMP_APP_PREFIX`: Préfixe nom de l'app temporaire
- `SAAGIE_APP_RESTORE_TMP_APP_PREFIX`: Préfixe nom de l'app temporaire

Mandatories :
- `BACKUP_S3_ENDPOINT`: Endpoint of the S3 provider of storage object
- `BACKUP_REGION_NAME`: Name of the S3 region of the storage object
- `BACKUP_S3_ACCESS_KEY_ID`: Access key of the storage object
- `BACKUP_S3_SECRET_ACCESS_KEY`: Secret access key of the storage object
- `BACKUP_S3_BUCKET_NAME`: Name of the bucket of the storage object where to back up the apps
- `BACKUP_URL`: URL of the Saagie environment
- `BACKUP_PF_ID`: ID of the current Saagie platform
- `BACKUP_USER`: Login of a Saagie user (required to use the Saagie API for access to your apps)
- `BACKUP_PWD`: Password of the Saagie user
- `BACKUP_APP_PROJECT_ID`: Project ID of the backup-restore application
- `SAAGIE_APP_BACKUP_S3_ENDPOINT`: Endpoint of the S3 provider of storage object
- `SAAGIE_APP_BACKUP_S3_REGION_NAME`: Name of the S3 region of the storage object
- `SAAGIE_APP_BACKUP_S3_ACCESS_KEY_ID`: Access key of the storage object
- `SAAGIE_APP_BACKUP_S3_SECRET_ACCESS_KEY`: Secret access key of the storage object
- `SAAGIE_APP_BACKUP_S3_BUCKET_NAME`: Name of the bucket of the storage object where to back up the apps
- `SAAGIE_APP_BACKUP_SAAGIE_URL`: URL of the Saagie environment
- `SAAGIE_APP_BACKUP_PF_ID`: ID of the current Saagie platform
- `SAAGIE_APP_BACKUP_SAAGIE_USER`: Login of a Saagie user (required to use the Saagie API for access to your apps)
- `SAAGIE_APP_BACKUP_SAAGIE_PWD`: Password of the Saagie user
- `SAAGIE_APP_BACKUP_CURRENT_APP_PROJECT_ID`: Project ID of the backup-restore application
4 changes: 2 additions & 2 deletions ...gie-app-storages-backup-restore/saagie-app-storages-backup-restore-2024.04/app/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ USER root
ENV APP_BACKUP_VERSION='2024.04-0.1-1.192.0_SDKTECHNO-271'
ENV APP_RESTORE_VERSION='2024.04-0.1-1.192.0_SDKTECHNO-271'

ENV BACKUP_TMP_APP_PREFIX='saagie-sub-app-backup'
ENV RESTORE_TMP_APP_PREFIX='saagie-sub-app-restore'
ENV SAAGIE_APP_BACKUP_TMP_APP_PREFIX='saagie-sub-app-backup'
ENV SAAGIE_APP_RESTORE_TMP_APP_PREFIX='saagie-sub-app-restore'

# In order to use dash-mantine-components>=0.14, you have to set up this variable to `18.2.0`
ENV REACT_VERSION=18.2.0
Expand Down
78 changes: 39 additions & 39 deletions .../saagie-app-storages-backup-restore/saagie-app-storages-backup-restore-2024.04/app/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,16 +49,16 @@



url = os.environ['BACKUP_URL']
pf = os.environ['BACKUP_PF_ID']
platformLogin = os.environ['BACKUP_USER']
platformPwd = os.environ['BACKUP_PWD']
url = os.environ['SAAGIE_APP_BACKUP_SAAGIE_URL']
pf = os.environ['SAAGIE_APP_BACKUP_PF_ID']
platformLogin = os.environ['SAAGIE_APP_BACKUP_SAAGIE_USER']
platformPwd = os.environ['SAAGIE_APP_BACKUP_SAAGIE_PWD']
realm = get_realm_from_url(url)
endpoint_url = os.environ["BACKUP_S3_ENDPOINT"]
region_name = os.environ["BACKUP_REGION_NAME"]
s3_bucket_name = os.environ["BACKUP_S3_BUCKET_NAME"] # Bucket name on S3, e.g. saagie-backup
app_prefix = os.environ["BACKUP_TMP_APP_PREFIX"] # Prefix of this app name
backup_project_id = os.environ["BACKUP_APP_PROJECT_ID"] # id du projet où est hébergé l'addOn
endpoint_url = os.environ["SAAGIE_APP_BACKUP_S3_ENDPOINT"]
region_name = os.environ["SAAGIE_APP_BACKUP_S3_REGION_NAME"]
s3_bucket_name = os.environ["SAAGIE_APP_BACKUP_S3_BUCKET_NAME"] # Bucket name on S3, e.g. saagie-backup
app_prefix = os.environ["SAAGIE_APP_BACKUP_TMP_APP_PREFIX"] # Prefix of this app name
backup_project_id = os.environ["SAAGIE_APP_BACKUP_CURRENT_APP_PROJECT_ID"] # id du projet où est hébergé l'addOn

saagie = SaagieApi(url_saagie=url, id_platform=pf, user=platformLogin, password=platformPwd, realm=realm)
# récupération des infos du projet où se trouve l'app backup
Expand All @@ -83,10 +83,10 @@

# Connect to S3
s3_client = boto3.client("s3",
endpoint_url=os.environ["BACKUP_S3_ENDPOINT"],
region_name=os.environ["BACKUP_REGION_NAME"],
aws_access_key_id=os.environ["BACKUP_S3_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["BACKUP_S3_SECRET_ACCESS_KEY"])
endpoint_url=os.environ["SAAGIE_APP_BACKUP_S3_ENDPOINT"],
region_name=os.environ["SAAGIE_APP_BACKUP_S3_REGION_NAME"],
aws_access_key_id=os.environ["SAAGIE_APP_BACKUP_S3_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["SAAGIE_APP_BACKUP_S3_SECRET_ACCESS_KEY"])

# Check if the bucket exist, if not create it
bucket_exist = [True for bucket in s3_client.list_buckets()["Buckets"] if bucket["Name"] == s3_bucket_name]
Expand Down Expand Up @@ -114,7 +114,7 @@
# to run locally
# app = Dash(__name__)


app.title = "Saagie Backup/Restore apps storages"
app.layout = dmc.MantineProvider(
children=[
html.Div(id="sui-root", children=[
Expand Down Expand Up @@ -179,8 +179,8 @@
children=[
html.Div(id='action-return'),
# dcc.Link(id='app-link', target="_blank", href="", title="Voir l'application restaurée"),
html.A(id='app-link', target="_blank", title="Voir l'application",
children="Voir l'application"),
html.A(id='app-link', target="_blank", title="See app",
children="Go to the app"),
html.Button(id='btn-close-notification', className="sui-o-notification__clear")
]
)
Expand Down Expand Up @@ -308,7 +308,7 @@ def render_content(tab):
data=get_select_data(saagie, list_path_to_dict(list_paths_backup)),
searchable=True,
clearable=True,
label="Liste des backups disponibles",
label="Available backups",
placeholder="Select an app to backup",
id="app-select",
w=400,
Expand All @@ -324,7 +324,7 @@ def render_content(tab):
data=[],
searchable=True,
# clearable=True,
label="Dates des backups",
label="Backup dates",
placeholder="Select a date",
id="date-select",
w=400,
Expand Down Expand Up @@ -426,10 +426,10 @@ def prepare_restore(value_app, value_date, n_clicks):
if value_app and len(value_app) > 0:
#########################################################
# maj de l'envvar en local
print(f"local RESTORE_LIST_APP_ID avant maj : {os.environ.get('RESTORE_LIST_APP_ID')}")
os.environ['RESTORE_LIST_APP_ID'] = value_app
test = os.environ['RESTORE_LIST_APP_ID']
print(f"local RESTORE_LIST_APP_ID après maj : {test}")
print(f"local SAAGIE_APP_RESTORE_LIST_APP_ID avant maj : {os.environ.get('SAAGIE_APP_RESTORE_LIST_APP_ID')}")
os.environ['SAAGIE_APP_RESTORE_LIST_APP_ID'] = value_app
test = os.environ['SAAGIE_APP_RESTORE_LIST_APP_ID']
print(f"local SAAGIE_APP_RESTORE_LIST_APP_ID après maj : {test}")

print(f"local RESTORE_DATE avant maj : {os.environ.get('RESTORE_DATE')}")
os.environ['RESTORE_DATE'] = value_date
Expand All @@ -438,16 +438,16 @@ def prepare_restore(value_app, value_date, n_clicks):

#########################################################
# maj de l'envvar projet
logging.info(f"----- Updating environment variables RESTORE_LIST_APP_ID...")
logging.info(f"----- Updating environment variables SAAGIE_APP_RESTORE_LIST_APP_ID...")
logging.info(f"pf_id : {pf}")
saagie.env_vars.create_or_update(
scope="PROJECT",
name="RESTORE_LIST_APP_ID",
name="SAAGIE_APP_RESTORE_LIST_APP_ID",
value=value_app,
description="List of apps to backup",
project_id=backup_project_id
)
logging.info(f"----- Fin Updating environment variables RESTORE_LIST_APP_ID")
logging.info(f"----- End updating environment variables SAAGIE_APP_RESTORE_LIST_APP_ID")

logging.info(f"----- Updating environment variables RESTORE_DATE...")
logging.info(f"pf_id : {pf}")
Expand All @@ -458,14 +458,14 @@ def prepare_restore(value_app, value_date, n_clicks):
description="Date of selected backup to restore",
project_id=backup_project_id
)
logging.info(f"----- Fin Updating environment variables RESTORE_DATE")
logging.info(f"----- End of updating environment variables RESTORE_DATE")

url_app = url + "/projects/platform/" + pf + "/project/" + backup_project_id + "/app/" + value_app
print(f"url_app :{url_app}")

launch_restore()
# time.sleep(2)
message = f"app {value_app} restaurée à la date du {value_date}"
message = f"App {value_app} restored to date of {value_date}"
return message, message, {'display': 'block'}, url_app
return "", "", {'display': 'none'}, ""

Expand Down Expand Up @@ -535,7 +535,7 @@ def prepare_backup(selectedRows, rowData, n_clicks):
# do something with values
# return something
# - afficher la liste des apps qui seront backupées dans la div 'selected-app'
# - mettre à jour la variable d'env BACKUP_LIST_APP_ID => comment trouver l'id du projet de l app courrante ?
# - mettre à jour la variable d'env SAAGIE_APP_BACKUP_LIST_APP_ID => comment trouver l'id du projet de l app courrante ?
# - lancer le script script_backup/__main__.py
# print(selectedRows)
# print(rowData)
Expand All @@ -546,24 +546,24 @@ def prepare_backup(selectedRows, rowData, n_clicks):
if len(backup_list_app_id) > 0:
#########################################################
# maj de l'envvar en local
print(f"local BACKUP_LIST_APP_ID avant maj : {os.environ.get('BACKUP_LIST_APP_ID', '')}")
os.environ['BACKUP_LIST_APP_ID'] = backup_list_app_id
test = os.environ['BACKUP_LIST_APP_ID']
print(f"local BACKUP_LIST_APP_ID après maj : {test}")
print(f"local SAAGIE_APP_BACKUP_LIST_APP_ID avant maj : {os.environ.get('SAAGIE_APP_BACKUP_LIST_APP_ID', '')}")
os.environ['SAAGIE_APP_BACKUP_LIST_APP_ID'] = backup_list_app_id
test = os.environ['SAAGIE_APP_BACKUP_LIST_APP_ID']
print(f"local SAAGIE_APP_BACKUP_LIST_APP_ID après maj : {test}")

#########################################################
# maj de l'envvar projet
logging.info(f"----- Updating environment variables BACKUP_LIST_APP_ID...")
logging.info(f"----- Updating environment variables SAAGIE_APP_BACKUP_LIST_APP_ID...")
logging.info(f"pf_id : {pf}")
saagie.env_vars.create_or_update(
scope="PROJECT",
name="BACKUP_LIST_APP_ID",
name="SAAGIE_APP_BACKUP_LIST_APP_ID",
value=backup_list_app_id,
description="List of apps to backup",
# project_id=project_id
project_id=backup_project_id
)
logging.info(f"----- Fin Updating environment variables BACKUP_LIST_APP_ID")
logging.info(f"----- End updating environment variables SAAGIE_APP_BACKUP_LIST_APP_ID")

launch_backup()
# time.sleep(2)
Expand All @@ -576,9 +576,9 @@ def prepare_backup(selectedRows, rowData, n_clicks):


def launch_backup():
logging.info(f"----- Lancement du script_backup ...")
logging.info(f"----- Launch of backup script ...")
script_backup.script_backup()
logging.info(f"----- Fin du script_backup ...")
logging.info(f"----- End du script_backup ...")
# return ""


Expand Down Expand Up @@ -764,9 +764,9 @@ def get_settings():


def launch_restore():
logging.info(f"----- Lancement du script_restore ...")
logging.info(f"----- Launch of restore script ...")
script_restore.script_restore()
logging.info(f"----- Fin du script_restore ...")
logging.info(f"----- End of restore script ...")


if __name__ == "__main__":
Expand Down
20 changes: 10 additions & 10 deletions ...es-backup-restore/saagie-app-storages-backup-restore-2024.04/app/scripts/script_backup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,27 +8,27 @@

def script_backup():
##############################################################################################
# liste des apps à sauvegarder BACKUP_LIST_APP_ID
# liste des apps à sauvegarder SAAGIE_APP_BACKUP_LIST_APP_ID

if os.environ.get('BACKUP_LIST_APP_ID'):
list_app = os.environ['BACKUP_LIST_APP_ID'].split(',')
if os.environ.get('SAAGIE_APP_BACKUP_LIST_APP_ID'):
list_app = os.environ['SAAGIE_APP_BACKUP_LIST_APP_ID'].split(',')
else:
logging.warning("==> BACKUP_LIST_APP_ID à paramétrer")
logging.warning("==> SAAGIE_APP_BACKUP_LIST_APP_ID à paramétrer")
return False

# TODO: define a timeout
init_timeout = 600
timeout = 600
project_id = os.environ["BACKUP_APP_PROJECT_ID"]
project_id = os.environ["SAAGIE_APP_BACKUP_CURRENT_APP_PROJECT_ID"]
finished_status = ["STOPPED", "FAILED", "UNKNOWN"]

# Get information to connect to Saagie
logging.info(f"Connect to Saagie ")
url = os.getenv('BACKUP_URL', 'https://saagie-tech.saagie.io')
platform_login = os.getenv('BACKUP_USER', 'tech_user')
platform_pwd = os.getenv('BACKUP_PWD', 'tech_user')
url = os.getenv('SAAGIE_APP_BACKUP_SAAGIE_URL', 'https://saagie-tech.saagie.io')
platform_login = os.getenv('SAAGIE_APP_BACKUP_SAAGIE_USER', 'tech_user')
platform_pwd = os.getenv('SAAGIE_APP_BACKUP_SAAGIE_PWD', 'tech_user')
realm = get_realm_from_url(url)
pf = os.getenv('BACKUP_PF_ID', '1')
pf = os.getenv('SAAGIE_APP_BACKUP_PF_ID', '1')

APP_BACKUP_VERSION = os.environ.get("APP_BACKUP_VERSION", '2024.04-0.1-1.192.0_SDKTECHNO-271')

Expand Down Expand Up @@ -166,7 +166,7 @@ def script_backup():
value=s3_file_prefix,
project_id=project_id
)
app_name = f"{os.environ['BACKUP_TMP_APP_PREFIX']} {datetime.timestamp(d)}"
app_name = f"{os.environ['SAAGIE_APP_BACKUP_TMP_APP_PREFIX']} {datetime.timestamp(d)}"

app_backup_baseName = 'saagie/saagie-app-storages-sub-app-backup'
app_backup_name = f'{app_backup_baseName}:{APP_BACKUP_VERSION}'
Expand Down
34 changes: 17 additions & 17 deletions ...s-backup-restore/saagie-app-storages-backup-restore-2024.04/app/scripts/script_restore.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@

def script_restore():
##############################################################################################
# liste des apps à restored RESTORE_LIST_APP_ID
if os.environ.get('RESTORE_LIST_APP_ID'):
list_app = os.environ['RESTORE_LIST_APP_ID'].split(',')
# liste des apps à restored SAAGIE_APP_RESTORE_LIST_APP_ID
if os.environ.get('SAAGIE_APP_RESTORE_LIST_APP_ID'):
list_app = os.environ['SAAGIE_APP_RESTORE_LIST_APP_ID'].split(',')
else:
logging.warning("==> RESTORE_LIST_APP_ID à paramétrer")
logging.warning("==> SAAGIE_APP_RESTORE_LIST_APP_ID à paramétrer")
return False

# date du backup
Expand All @@ -34,11 +34,11 @@ def script_restore():

# Get information to connect to Saagie
logging.info(f"Connect to Saagie ")
url = os.getenv('BACKUP_URL', 'https://saagie-tech.saagie.io')
platform_login = os.getenv('BACKUP_USER', 'tech_user')
platform_pwd = os.getenv('BACKUP_PWD', 'tech_user')
url = os.getenv('SAAGIE_APP_BACKUP_SAAGIE_URL', 'https://saagie-tech.saagie.io')
platform_login = os.getenv('SAAGIE_APP_BACKUP_SAAGIE_USER', 'tech_user')
platform_pwd = os.getenv('SAAGIE_APP_BACKUP_SAAGIE_PWD', 'tech_user')
realm = get_realm_from_url(url)
pf = os.getenv('BACKUP_PF_ID', '1')
pf = os.getenv('SAAGIE_APP_BACKUP_PF_ID', '1')

APP_RESTORE_VERSION = os.environ.get("APP_RESTORE_VERSION", '2024.04-0.1-1.192.0_SDKTECHNO-271')

Expand All @@ -55,11 +55,11 @@ def script_restore():
realm=realm)

s3_client = boto3.client("s3",
endpoint_url=os.environ["BACKUP_S3_ENDPOINT"],
region_name=os.environ["BACKUP_REGION_NAME"],
aws_access_key_id=os.environ["BACKUP_S3_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["BACKUP_S3_SECRET_ACCESS_KEY"])
s3_bucket_name = os.environ["BACKUP_S3_BUCKET_NAME"]
endpoint_url=os.environ["SAAGIE_APP_BACKUP_S3_ENDPOINT"],
region_name=os.environ["SAAGIE_APP_BACKUP_S3_REGION_NAME"],
aws_access_key_id=os.environ["SAAGIE_APP_BACKUP_S3_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["SAAGIE_APP_BACKUP_S3_SECRET_ACCESS_KEY"])
s3_bucket_name = os.environ["SAAGIE_APP_BACKUP_S3_BUCKET_NAME"]


for app_id in list_app:
Expand All @@ -69,7 +69,7 @@ def script_restore():
project_id = app_info['project']['id']
client_saagie.env_vars.create_or_update(
scope="PROJECT",
name="BACKUP_APP_PROJECT_ID",
name="SAAGIE_APP_BACKUP_CURRENT_APP_PROJECT_ID",
value=project_id,
project_id=project_id
)
Expand Down Expand Up @@ -148,15 +148,15 @@ def script_restore():

client_saagie.env_vars.create_or_update(
scope="PROJECT",
name="RESTORE_TMP_APP_PREFIX",
value=os.environ["RESTORE_TMP_APP_PREFIX"],
name="SAAGIE_APP_RESTORE_TMP_APP_PREFIX",
value=os.environ["SAAGIE_APP_RESTORE_TMP_APP_PREFIX"],
project_id=project_id
)


# Create a temporary app to restore the backup
#d = datetime.now()
app_name = f"{os.environ['RESTORE_TMP_APP_PREFIX']} {datetime.timestamp(d)}"
app_name = f"{os.environ['SAAGIE_APP_RESTORE_TMP_APP_PREFIX']} {datetime.timestamp(d)}"

app_restore_baseName = 'saagie/saagie-app-storages-sub-app-restore'
app_restore_name = f'{app_restore_baseName}:{APP_RESTORE_VERSION}'
Expand Down
Loading

0 comments on commit c9e7d0b

Please sign in to comment.