From d8f7ea0d1696a7c7e89aa7c62aec911147649ff0 Mon Sep 17 00:00:00 2001 From: caewok Date: Fri, 5 Nov 2021 14:04:53 -0700 Subject: [PATCH 1/4] Update `as.data.frame` to handle Backblaze S3 Backblaze S3 will return bucket lists without Owner:DisplayName. This causes `as.data.frame.s3_bucket` to fail when parsing the contents for `get_bucket_df`. This PR corrects that issue by ensuring that NULL values are changed to NA in the relevant cases. --- R/utils.R | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/R/utils.R b/R/utils.R index be85368..008e629 100644 --- a/R/utils.R +++ b/R/utils.R @@ -92,22 +92,28 @@ get_objectkey.s3_object <- function(x, ...) { #' @export as.data.frame.s3_bucket <- function(x, row.names = NULL, optional = FALSE, ...) { if (length(x)) { - out <- lapply(x, function(z) { - c(Key = z[["Key"]], - LastModified = z[["LastModified"]], - ETag = z[["ETag"]], - Size = z[["Size"]], - Owner_ID = - ifelse(is.null(z[["Owner"]]), NA, z[["Owner"]][["ID"]]), - Owner_DisplayName = - ifelse(is.null(z[["Owner"]]), NA, z[["Owner"]][["DisplayName"]]), - StorageClass = z[["StorageClass"]], - Bucket = z[["Bucket"]]) + x <- lapply(x, function(z) { + lst = c(Key = z[["Key"]], + LastModified = z[["LastModified"]], + ETag = z[["ETag"]], + Size = z[["Size"]], + Owner_ID = z[["Owner"]][["ID"]]), + Owner_DisplayName = z[["Owner"]][["DisplayName"]]), + StorageClass = z[["StorageClass"]], + Bucket = z[["Bucket"]]) + + # any null values will be dropped, so add back in + # only seems to be a problem for the "Owner" properties, so assuming for + # now that the others are correct. + if(is.null(lst[["Owner_DisplayName"]])) lst[["Owner_DisplayName"]] = NA + if(is.null(lst[["Owner_ID"]])) lst[["Owner_ID"]] = NA + return(lst) + }) op <- options(stringsAsFactors = FALSE) on.exit(options(op)) - out <- do.call("rbind.data.frame", unname(out)) - names(out) <- c("Key", "LastModified", "ETag", "Size", "Owner_ID", "Owner_DisplayName", "StorageClass", "Bucket") + out <- do.call("rbind.data.frame", unname(x)) + names(out) <- names(x$Contents) structure(out, row.names = if(!is.null(row.names)) row.names else seq_len(nrow(out)), Marker = attributes(x)[["Marker"]], IsTruncated = attributes(x)[["IsTruncated"]], From 55b1bb9ab52f230f5bb35e9d1fb1229d9f72d6c5 Mon Sep 17 00:00:00 2001 From: caewok Date: Fri, 5 Nov 2021 14:08:02 -0700 Subject: [PATCH 2/4] Fix parsing error. --- R/utils.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/utils.R b/R/utils.R index 008e629..fe0edf4 100644 --- a/R/utils.R +++ b/R/utils.R @@ -97,8 +97,8 @@ as.data.frame.s3_bucket <- function(x, row.names = NULL, optional = FALSE, ...) LastModified = z[["LastModified"]], ETag = z[["ETag"]], Size = z[["Size"]], - Owner_ID = z[["Owner"]][["ID"]]), - Owner_DisplayName = z[["Owner"]][["DisplayName"]]), + Owner_ID = z[["Owner"]][["ID"]], + Owner_DisplayName = z[["Owner"]][["DisplayName"]], StorageClass = z[["StorageClass"]], Bucket = z[["Bucket"]]) From 9d6a36b5f4571d67663a25f1c18edbeb0697a297 Mon Sep 17 00:00:00 2001 From: caewok Date: Fri, 5 Nov 2021 14:26:24 -0700 Subject: [PATCH 3/4] Update DESCRIPTION for PR --- DESCRIPTION | 1 + 1 file changed, 1 insertion(+) diff --git a/DESCRIPTION b/DESCRIPTION index 9c11671..cae8d2d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -15,6 +15,7 @@ Authors@R: c(person("Thomas J.", "Leeper", role = "aut", person("Andrii", "Degtiarov", role = "ctb"), person("Dhruv", "Aggarwal", role = "ctb"), person("Alyssa", "Columbus", role = "ctb"), + person("Michael", "Enion", role = "ctb"), person("Simon", "Urbanek", role = c("cre", "ctb"), email = "simon.urbanek@R-project.org") ) From bf2742ed05ef66fa0f28598265bde8887d854c67 Mon Sep 17 00:00:00 2001 From: caewok Date: Fri, 5 Nov 2021 14:29:24 -0700 Subject: [PATCH 4/4] Update NEWS.md --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 40d2d8f..0070410 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# aws.s3 0.3.22x + +* `get_bucket_df` no longer throws an error when listing bucket contents from a Backblaze S3 bucket. Fixes #407. + # aws.s3 0.3.22 ## API changes