Add Var._dates_to_seconds

This commit helps with preprocessing observational data. This commit add
Var._dates_to_seconds to the the constructor for reading NetCDF files so
that automatically converts dates to seconds in the time dimension. For
testing, a sample of precipitation observational data is added.

Author: ph-kev

NEWS.md

@@ -20,6 +20,28 @@ arrays are equispaced and span the entire range, then a periodic boundary condit
 for the longitude dimension and a flat boundary condition is added for the latitude
 dimension.
 
+### Preprocess dates and times
+There is now support for preprocessing dates and times. The constructor for reading NetCDF
+files now automatically converts dates to seconds in the time dimension. This is done
+because `ClimaAnalysis` does not support interpolating on dates which mean functions that
+rely on the interpolats, such as `resampled_as`, will not work otherwise.
+
+Also, the constructor supports two additional parameters `new_start_date` and `shift_by`.
+After converting from dates to seconds, the seconds are shifted to match `new_start_date`.
+If preprocessing of dates is needed before shifting to `new_start_date`, then the parameter
+`shift_by` can be used as it accepts a function that takes in `Dates.DateTime` elements and
+return Dates.DateTime elements. This function is applied to each element of the time array.
+```julia
+# Shift the dates to first day of month, convert to seconds, and adjust seconds to
+# match "1/1/2010"
+shift_var = OutputVar(
+        "test.nc",
+        "pr",
+        new_start_date = "1/1/2010", # or Dates.DateTime(2010, 1, 1)
+        shift_by = Dates.firstdayofmonth,
+    )
+```
+
 ## Bug fixes
 
 - Interpolation is not possible with dates. When dates are detected in any dimension, an

docs/src/var.md

@@ -80,6 +80,31 @@ and latitude dimensions, a periodic boundary condition and a flat boundary condi
 added, respectively, when the dimension array is equispaced and spans the entire range. For
 all other cases, extrapolating beyond the domain of the dimension will throw an error.
 
+## Preprocess dates and seconds
+When loading a NetCDF file, dates in the time dimension are automatically converted to
+seconds and a start date is added to the attributes of the `OutputVar`. This is done because
+`ClimaAnalysis` does not support interpolating on dates which mean functions that rely on
+the interpolats, such as `resampled_as`, will not work otherwise.
+
+Two additional parameters are provided to help preprocess dates which are `new_start_date`
+and `shift_by`. If `new_start_date` is provided, then dates in the time dimension will
+automatically be converted with reference to the `new_start_date` rather than the first date
+found in the NetCDF file. The parameter `new_start_date` can be any string parseable by the
+[Dates](https://docs.julialang.org/en/v1/stdlib/Dates/) module or a `Dates.DateTime` object.
+If additional preprocessing is needed, then one can provide a function that takes in and
+returns a `Date.DateTime` object. This function is applied to each date before converting
+each dates to seconds with reference with the start date.
+```@julia dates_to_seconds
+# Shift the dates to first day of month, convert to seconds, and adjust seconds to
+# match the date 1/1/2010
+obs_var = ClimaAnalysis.OutputVar(
+                "pr.nc",
+                "precip",
+                new_start_date = "2010-01-01T00:00:00", # or Dates.DateTime(2010, 1, 1)
+                shift_by = Dates.firstdayofmonth,
+            )
+```
+
 ## Integration
 
 `OutputVar`s can be integrated with respect to longitude, latitude, or both using

src/Var.jl

@@ -195,14 +195,41 @@ function OutputVar(dims, data)
 end
 
 """
-    OutputVar(path, short_name = nothing)
+    OutputVar(path,
+              short_name = nothing;
+              new_start_date = nothing,
+              shift_by = identity)
 
 Read the NetCDF file in `path` as an `OutputVar`.
 
 If `short_name` is `nothing`, automatically find the name.
-"""
-function OutputVar(path::String, short_name = nothing)
-    return read_var(path; short_name)
+
+Dates in the time dimension are automatically converted to seconds with respect to the first
+date in the time dimension array or the `new_start_date`. The parameter `new_start_date` can
+be any string parseable by the [Dates](https://docs.julialang.org/en/v1/stdlib/Dates/)
+module or a `Dates.DateTime` object. The parameter `shift_by` is a function that takes in
+Dates.DateTime elements and return Dates.DateTime elements. The start date is added to the
+attributes of the `OutputVar`. The parameter `shift_by` is a function that takes in
+`Dates.DateTime` elements and returns `Dates.DateTime` elements. This function is applied to
+each element of the time array. Shifting the dates and converting to seconds is done in that
+order.
+"""
+function OutputVar(
+    path::String,
+    short_name = nothing;
+    new_start_date = nothing,
+    shift_by = identity,
+)
+    var = read_var(path; short_name)
+    # Check if it is possible to convert dates to seconds in the time dimension
+    if (has_time(var) && eltype(times(var)) <: Dates.DateTime)
+        var = _dates_to_seconds(
+            read_var(path; short_name),
+            new_start_date = new_start_date,
+            shift_by = shift_by,
+        )
+    end
+    return var
 end
 
 """
@@ -1299,6 +1326,69 @@ function global_rmse(sim::OutputVar, obs::OutputVar)
     squared_error_var = squared_error(sim, obs)
     return squared_error_var.attributes["global_rmse"]
 end
+
+"""
+    _dates_to_seconds(var::OutputVar;
+                      new_start_date = nothing,
+                      shift_by = identity)
+
+Convert dates in time dimension to seconds with respect to the first date in the time
+dimension array or the `new_start_date`.
+
+Dates in the time dimension are automatically converted to seconds with respect to the first
+date in the time dimension array or the `new_start_date`. The parameter `new_start_date` can
+be any string parseable by the [Dates](https://docs.julialang.org/en/v1/stdlib/Dates/)
+module or a `Dates.DateTime` object. The parameter `shift_by` is a function that takes in
+Dates.DateTime elements and return Dates.DateTime elements. The start date is added to the
+attributes of the `OutputVar`. The parameter `shift_by` is a function that takes in
+`Dates.DateTime` elements and returns `Dates.DateTime` elements. This function is applied to
+each element of the time array. Shifting the dates and converting to seconds is done in that
+order.
+
+Note that this function only works for the time dimension and will not work for the date
+dimension.
+"""
+function _dates_to_seconds(
+    var::OutputVar;
+    new_start_date = nothing,
+    shift_by = identity,
+)
+    has_time(var) || error(
+        "Converting from dates to seconds is only supported for the time dimension",
+    )
+    eltype(times(var)) <: Dates.DateTime ||
+        error("Type of time dimension is not dates")
+
+    # Preprocess time_arr by shifting dates
+    time_arr = copy(times(var))
+    if !isnothing(shift_by)
+        time_arr .= shift_by.(time_arr)
+    end
+
+    # Convert from dates to seconds using the first date in the time dimension array as the
+    # start date or the new_start_date
+    start_date = isnothing(new_start_date) ? time_arr[begin] : new_start_date
+
+    # Handle the case if start_date is a DateTime or string; if it is the latter, then try
+    # to parse it as a DateTime
+    start_date isa AbstractString && (start_date = Dates.DateTime(start_date))
+    time_arr = map(date -> date_to_time(start_date, date), time_arr)
+
+    # Remake OutputVar
+    ret_attribs = deepcopy(var.attributes)
+    ret_attribs["start_date"] = string(start_date) # add start_date as an attribute
+    ret_dim_attribs = deepcopy(var.dim_attributes)
+    ret_dim_attribs[time_name(var)]["units"] = "s" # add unit
+    var_dims = deepcopy(var.dims)
+    ret_dims_generator = (
+        conventional_dim_name(dim_name) == "time" ? dim_name => time_arr :
+        dim_name => dim_data for (dim_name, dim_data) in var_dims
+    )
+    ret_dims = OrderedDict(ret_dims_generator...)
+    ret_data = copy(var.data)
+    return OutputVar(ret_attribs, ret_dims, ret_dim_attribs, ret_data)
+end
+
 """
     overload_binary_op(op)
 

test/sample_nc/test_pr.nc

@@ -1349,3 +1349,89 @@ end
     var_units = ClimaAnalysis.set_units(var, "idk")
     @test ClimaAnalysis.units(var_units) == "idk"
 end
+
+@testset "Dates to seconds for vars" begin
+    # Test for no start date
+    time_arr = [
+        Dates.DateTime(2020, 3, 1, 1, 1),
+        Dates.DateTime(2020, 3, 1, 1, 2),
+        Dates.DateTime(2020, 3, 1, 1, 3),
+    ]
+    data = ones(length(time_arr))
+    dims = OrderedDict("time" => time_arr)
+    dim_attribs = OrderedDict("time" => Dict("blah" => "blah"))
+    attribs =
+        Dict("long_name" => "idk", "short_name" => "short", "units" => "kg")
+    var = ClimaAnalysis.OutputVar(attribs, dims, dim_attribs, data)
+    var_s = ClimaAnalysis.Var._dates_to_seconds(var)
+    @test ClimaAnalysis.times(var_s) == [0.0, 60.0, 120.0]
+    @test var_s.attributes["start_date"] == "2020-03-01T01:01:00"
+
+    # Test for a new start date
+    var_s = ClimaAnalysis.Var._dates_to_seconds(
+        var;
+        new_start_date = "2020-03-01T01:03:00",
+    )
+    @test ClimaAnalysis.times(var_s) == [-120.0, -60.0, 0.0]
+    @test var_s.attributes["start_date"] == "2020-03-01T01:03:00"
+
+    # Test for a new start date as a DateTime object
+    var_s = ClimaAnalysis.Var._dates_to_seconds(
+        var;
+        new_start_date = Dates.DateTime("2020-03-01T01:03:00"),
+    )
+    @test ClimaAnalysis.times(var_s) == [-120.0, -60.0, 0.0]
+    @test var_s.attributes["start_date"] == "2020-03-01T01:03:00"
+
+    # Test for shifting dates
+    var_s = ClimaAnalysis.Var._dates_to_seconds(
+        var,
+        shift_by = t -> t - Dates.Day(15),
+    )
+    @test ClimaAnalysis.times(var_s) == [0.0, 60.0, 120.0]
+    @test var_s.attributes["start_date"] == "2020-02-15T01:01:00"
+
+    # Test for shifting dates and new date together
+    var_s = ClimaAnalysis.Var._dates_to_seconds(
+        var;
+        new_start_date = "2020-03-01T01:00:00",
+        shift_by = t -> t + Dates.Minute(4),
+    )
+    @test ClimaAnalysis.times(var_s) == [300.0, 360.0, 420.0]
+    @test var_s.attributes["start_date"] == "2020-03-01T01:00:00"
+
+    # Test constructor for OutputVar that uses _dates_to_seconds
+    ncpath = joinpath(@__DIR__, "sample_nc/test_pr.nc")
+    file_var = ClimaAnalysis.OutputVar(
+        ncpath;
+        new_start_date = nothing,
+        shift_by = identity,
+    )
+    @test ClimaAnalysis.times(file_var) == [0.0, 1398902400.0]
+    @test file_var.attributes["start_date"] == "1979-01-01T00:00:00"
+
+    # Test for error handling
+    # Use date dimension instead of time dimension
+    date_arr = [
+        Dates.DateTime(2020, 3, 1, 1, 1),
+        Dates.DateTime(2020, 3, 1, 1, 2),
+        Dates.DateTime(2020, 3, 1, 1, 3),
+    ]
+    data = ones(length(date_arr))
+    dims = OrderedDict("date" => date_arr)
+    dim_attribs = OrderedDict("date" => Dict("blah" => "blah"))
+    attribs =
+        Dict("long_name" => "idk", "short_name" => "short", "units" => "kg")
+    var = ClimaAnalysis.OutputVar(attribs, dims, dim_attribs, data)
+    @test_throws ErrorException ClimaAnalysis.Var._dates_to_seconds(var)
+
+    # Cannot convert if the element type of time array is float
+    time_arr = [0.0, 60.0, 120.0]
+    data = ones(length(time_arr))
+    dims = OrderedDict("time" => time_arr)
+    dim_attribs = OrderedDict("time" => Dict("blah" => "blah"))
+    attribs =
+        Dict("long_name" => "idk", "short_name" => "short", "units" => "kg")
+    var = ClimaAnalysis.OutputVar(attribs, dims, dim_attribs, data)
+    @test_throws ErrorException ClimaAnalysis.Var._dates_to_seconds(var)
+end