Merge pull request #4 from tonyfujs/svy_list

Update `get_survey_list()` to always return the complete list of available surveys

Author: tonyfujs

DESCRIPTION

@@ -9,9 +9,11 @@ Description: More about what it does (maybe more than one line)
 License: MIT + file LICENSE
 Encoding: UTF-8
 LazyData: true
-Imports: httr,
-    jsonlite
-RoxygenNote: 6.1.1
+Imports: 
+    httr,
+    jsonlite,
+    purrr
+RoxygenNote: 7.1.1
 Suggests: knitr,
     rmarkdown
 VignetteBuilder: knitr

NAMESPACE

@@ -2,6 +2,7 @@
 
 export(connect_mdlib)
 export(get_access_types)
+export(get_data_dictionary)
 export(get_datafiles_list)
 export(get_doc_metadata)
 export(get_public_survey_list)

R/get_data_dictionary.R

@@ -0,0 +1,38 @@
+#' get_data_dictionary
+#'
+#' @param id numeric: Survey Unique ID
+#' @param token character: Microdata API authentication token
+#' @param limit character: maximum number of records per page
+#'
+#' @return list
+#' @export
+#'
+
+get_data_dictionary <- function(id, token, limit = NULL){
+  if (is.null(limit)) {
+    path <- paste0('index.php/api/v2/metadata/list_variables/', id)
+    out <- connect_mdlib(path = path, token = token)
+    out <- out$content$items
+
+    return(out)
+
+  } else {
+    out <- vector(mode = 'list', length = 1000)
+    counter = 1
+
+    records_number <- limit
+    while (records_number == limit) {
+      temp_offset <- (counter - 1) * limit
+      path <- paste0('index.php/api/v2/metadata/list_variables/', id, '/', limit, '/', temp_offset)
+      temp_out <- connect_mdlib(path = path, token = token)
+      temp_out <- temp_out$content$items
+      out[[counter]] <- temp_out
+      counter <- counter + 1
+      records_number <- length(temp_out)
+    }
+
+    out <- purrr::flatten(out)
+
+    return(out)
+  }
+}

R/get_survey_list.R

@@ -7,8 +7,33 @@
 #'
 
 get_survey_list <- function(token){
-  out <- connect_mdlib(path = 'index.php/api/catalog/search?ps=10000', token = token)
+
+  n_records <- get_number_of_surveys(token)
+  path <- paste0('index.php/api/catalog/search?ps=', n_records)
+  out <- connect_mdlib(path = path, token = token)
   out <- out$content$result$rows
 
   return(out)
 }
+
+get_number_of_surveys <- function(token) {
+  # Should be better integrated with the other functions (URL should not be hard coded)
+  url <- "http://microdatalib.worldbank.org/index.php/api/catalog/search?ps=1"
+
+  resp <- httr::GET(url,
+                    httr::add_headers(.headers = c('X-API-KEY' = token,
+                                                   'charset' = "utf-8")),
+                    httr::accept_json())
+
+  # Return useful message on error
+  httr::stop_for_status(resp, task = 'complete request to Microdata library API\n')
+  # CHECK: datatype is .JSON
+  if (httr::http_type(resp) != "application/json") {
+    stop("API did not return json", call. = FALSE)
+  }
+  # Parse response
+  parsed <- jsonlite::fromJSON(httr::content(resp, "text"), simplifyVector = FALSE)
+  n_records <- parsed$result$total
+
+  return(n_records)
+}

man/connect_mdlib.Rd

@@ -32,10 +32,14 @@ You can get your API KEY by:
 1. Loging using your World Bank SecureID token
 1. Clicking the __create new key__ button.
 
+API documentation can be accessed [here](http://microdatalibqa.worldbank.org/apidocumentation/#api-Metadata-datafile_variables)
+
 ### List all available surveys
 ```{r}
-library(mdlibconnect)
-get_survey_list(token = Sys.getenv('mdlib_token'))
+devtools::load_all()
+response <- get_survey_list(token = Sys.getenv('mdlib_token'))
+str(response, max.level = 2, list.len = 3)
+
 ```