open groups lazily

src/ZGroup.jl

@@ -1,35 +1,22 @@
 struct ZGroup{S<:AbstractStore}
     storage::S
     path::String
-    arrays::Dict{String, ZArray}
-    groups::Dict{String, ZGroup}
     attrs::Dict
     writeable::Bool
+    fill_as_missing::Bool
 end
 
 # path can also be a SubString{String}
-ZGroup(storage, path::AbstractString, arrays, groups, attrs, writeable) =
-    ZGroup(storage, String(path), arrays, groups, attrs, writeable)
+ZGroup(storage, path::AbstractString, attrs, writeable, fill_as_missing) =
+    ZGroup(storage, String(path), attrs, writeable, fill_as_missing)
 
 zname(g::ZGroup) = zname(g.path)
 
 #Open an existing ZGroup
 function ZGroup(s::T,mode="r",path="";fill_as_missing=false) where T <: AbstractStore
-  arrays = Dict{String, ZArray}()
-  groups = Dict{String, ZGroup}()
-
-  for d in subdirs(s,path)
-    dshort = split(d,'/')[end]
-    m = zopen_noerr(s,mode,path=_concatpath(path,dshort),fill_as_missing=fill_as_missing)
-    if isa(m, ZArray)
-      arrays[dshort] = m
-    elseif isa(m, ZGroup)
-      groups[dshort] = m
-    end
-  end
-  attrs = getattrs(s,path)
-  startswith(path,"/") && error("Paths should never start with a leading '/'")
-  ZGroup(s, path, arrays, groups, attrs,mode=="w")
+    attrs = getattrs(s,path)
+    startswith(path,"/") && error("Paths should never start with a leading '/'")
+    ZGroup(s, path, attrs, mode == "w", fill_as_missing)
 end
 
 """
@@ -59,21 +46,28 @@ end
 
 function Base.show(io::IO, g::ZGroup)
     print(io, "ZarrGroup at ", g.storage, " and path ", g.path)
-    !isempty(g.arrays) && print(io, "\nVariables: ", map(i -> string(zname(i), " "), values(g.arrays))...)
-    !isempty(g.groups) && print(io, "\nGroups: ", map(i -> string(zname(i), " "), values(g.groups))...)
-    nothing
+    for (i, d) in enumerate(subdirs(g.storage, g.path))
+        if i > 10  # don't print too many
+            print(io, "\n  ...")
+            break
+        end
+        path = _concatpath(g.path, d)
+        if is_zarray(g.storage, path)
+            print(io, "\n  ", d, " (Array)")
+        elseif is_zgroup(g.storage, path)
+            print(io, "\n  ", d, " (Group)")
+        end
+    end
 end
-Base.haskey(g::ZGroup,k)= haskey(g.groups,k) || haskey(g.arrays,k)
 
+function Base.haskey(g::ZGroup, k)
+    path = _concatpath(g.path, k)
+    is_zarray(g.storage, path) || is_zgroup(g.storage, path)
+end
 
 function Base.getindex(g::ZGroup, k)
-    if haskey(g.groups, k)
-        return g.groups[k]
-    elseif haskey(g.arrays, k)
-       return g.arrays[k]
-    else
-       throw(KeyError("Zarr Dataset does not contain $k"))
-    end
+    m = zopen_noerr(g.storage, g.writeable ? "w" : "r", path = _concatpath(g.path, k), fill_as_missing = g.fill_as_missing)
+    m !== nothing ? m : throw(KeyError(k))
 end
 
 """
@@ -135,24 +129,22 @@ function zgroup(s::AbstractStore, path::String=""; attrs=Dict())
     JSON.print(b,d)
     s[path,".zgroup"]=take!(b)
     writeattrs(s,path,attrs)
-    ZGroup(s, path, Dict{String,ZArray}(), Dict{String,ZGroup}(), attrs,true)
+    ZGroup(s, path, attrs, true, false)
 end
 
 zgroup(s::String;kwargs...)=zgroup(storefromstring(s, true)...;kwargs...)
 
 "Create a subgroup of the group g"
 function zgroup(g::ZGroup, name; attrs=Dict()) 
   g.writeable || throw(IOError("Zarr group is not writeable. Please re-open in write mode to create an array"))
-  g.groups[name] = zgroup(g.storage,_concatpath(g.path,name),attrs=attrs)
+  zgroup(g.storage, _concatpath(g.path, name), attrs = attrs)
 end
 
 "Create a new subarray of the group g"
 function zcreate(::Type{T},g::ZGroup, name::AbstractString, addargs...; kwargs...) where T
   g.writeable || throw(IOError("Zarr group is not writeable. Please re-open in write mode to create an array"))
   name = string(name)
-  z = zcreate(T, g.storage, addargs...; path = _concatpath(g.path,name), kwargs...)
-  g.arrays[name] = z
-  return z
+  zcreate(T, g.storage, addargs...; path = _concatpath(g.path, name), kwargs...)
 end
 
 HTTP.serve(s::Union{ZArray,ZGroup}, args...; kwargs...) = HTTP.serve(s.storage, s.path, args...; kwargs...)

test/storage.jl

@@ -176,7 +176,7 @@ end
   @test storagesize(S3,p) == 0
   @test Zarr.is_zgroup(S3,p) == true
   S3group = zopen(S3,path=p)
-  S3Array = S3group.groups["bar"].arrays["baz"]
+  S3Array = S3group["bar"]["baz"]
   @test eltype(S3Array) == Zarr.ASCIIChar
   @test storagesize(S3Array) == 69
   @test String(S3Array[:]) == "Hello from the cloud!"