How to using client to upload data/artifacts to s3 bucket?

[namnd00]

How to use the client to upload data/artifacts to the s3 bucket?

[polyaxon-team]

To upload artifacts to a newly created run, you can use a snippet like this one:

...
# upload_to should be a value like `path/code`
upload_to = upload_to or DEFAULT_UPLOADS_PATH
meta_info = {META_UPLOAD_ARTIFACTS: upload_to}
client = RunClient(owner=owner, project=project_name)
run_instance = client.create(
    name=name,
    description=description,
    tags=tags,
    content=op_spec,
    is_managed=True,
    meta_info=meta_info,
   pending=V1RunPending.UPLOAD,  # Very important to tell the server to wait for the upload to finish
)
...
# If path from then check if file otherwise it's local directory
is_file = os.path.isfile(path_from) if path_from else False 
client = RunClient(owner=owner, project=project_name, run_uuid=run_instance.uuid)
if is_file:
    response = client.upload_artifact(filepath=path_from, path=path_to, overwrite=True)
else:
    response = client.upload_artifacts_dir(dirpath=path_from, path=path_to, overwrite=True, relative_to=path_from)

[namnd00]

Thank you for your answer. I pass some arguments into your snippet but I check on the s3 bucket not find the artifact that I want to upload to the s3 bucket. Additionally, I created AWS secret keys in my namespace that deployed polyaxon platform. My following snippet is like this one:

import os
from polyaxon.client.run import RunClient
from polyaxon.schemas import V1RunPending
from polyaxon.constants.globals import DEFAULT_UPLOADS_PATH
from polyaxon.constants.metadata import META_UPLOAD_ARTIFACTS
from qai.polyaxonfile import OperationSpecification


upload_to = "s3://<artifacts-path>"
path_from = "README.md"
path_to = "s3://<artifacts-path>/artifacts-store"

owner = "default"
project_name = "upload_data_to_s3_project"

name = "upload_data_to_s3"
description = "Upload data to S3 bucket."
tags = ["s3-upload-data"]

op_spec = "{'version':1.1,'kind':'operation','component':{'version':1.1,'kind':'component','name':'upload_data','tags':['upload_data'],'run':{'kind':'job','container':{'image':'nginx','command':'sleep 5'}}}}"

# upload_to should be a value like `path/code`
upload_to = upload_to or DEFAULT_UPLOADS_PATH
meta_info = {META_UPLOAD_ARTIFACTS: upload_to}
client = RunClient(owner=owner, project=project_name)
run_instance = client.create(
    name=name,
    description=description,
    tags=tags,
    content=op_spec,
    is_managed=True,
    meta_info=meta_info,
   pending=V1RunPending.UPLOAD,  # Very important to tell the server to wait for the upload to finish
) 
...
# If path from then check if file otherwise it's local directory
is_file = os.path.isfile(path_from) if path_from else False 
client = RunClient(owner=owner, project=project_name, run_uuid=run_instance.uuid)
if is_file:
    response = client.upload_artifact(filepath=path_from, path=path_to, overwrite=True)
else:
    response = client.upload_artifacts_dir(dirpath=path_from, path=path_to, overwrite=True, relative_to=path_from)