Better nanosecond timestamp support (#1200)

Author: Affie

src/Common.jl

@@ -179,3 +179,15 @@ function solveGraph! end
 Standard parametric graph solution (Experimental).
 """
 function solveGraphParametric! end
+
+# delta timestamps
+calcDeltatime(from::Nanosecond, to::Nanosecond) = Dates.value(to - from) / 10^9
+function calcDeltatime(from::TimeDateZone, to::TimeDateZone)
+    return Dates.value(convert(Nanosecond, to - from)) / 10^9
+end
+calcDeltatime(from_node, to_node) = calcDeltatime(from_node.timestamp, to_node.timestamp)
+
+function tdz_now(zone = tz"UTC") #TODO or default to slower localzone()? 
+    t = time()
+    return TimeDateZone(TimeDate(1970) + Nanosecond(t * 10^9), zone)
+end

src/DataBlobs/entities/BlobEntry.jl

@@ -44,7 +44,7 @@ StructUtils.@kwarg struct Blobentry
     """ Storage for a couple of bytes directly in the graph. Use with caution and keep it small and simple."""
     metadata::JSONText = JSONText("{}")
     """ When the Blob itself was first created. Serialized as an ISO 8601 string."""
-    timestamp::TimeDateZone = TimeDateZone(now(localzone()))
+    timestamp::TimeDateZone = tdz_now()
     """ Type version of this Blobentry."""
     version::VersionNumber = DFG.version(Blobentry)
 end

src/entities/DFGFactor.jl

@@ -68,7 +68,7 @@ StructUtils.@kwarg struct FactorDFG{T <: AbstractObservation, N} <: AbstractGrap
     variableorder::NTuple{N, Symbol} & (choosetype = x->NTuple{length(x), Symbol},) # NOTE v0.29 renamed from _variableOrderSymbols
     """Variable timestamp.
     Accessors: [`getTimestamp`](@ref)"""
-    timestamp::TimeDateZone = TimeDateZone(now(localzone())) # NOTE v0.29 changed from ZonedDateTime
+    timestamp::TimeDateZone = tdz_now() # NOTE v0.29 changed from ZonedDateTime
     # TODO
     # """(Optional) Steady (monotonic) time in nanoseconds `Nanosecond` (`Int64``)"""
     # nstime::Nanosecond #NOTE v0.29 REMOVED as not used, add when needed, or now as steadytime.
@@ -103,7 +103,7 @@ function FactorDFG(
     variableorder::Union{<:Tuple, Vector{Symbol}},
     observation::AbstractObservation;
     label::Symbol = assembleFactorName(variableorder),
-    timestamp::Union{TimeDateZone, ZonedDateTime} = TimeDateZone(now(localzone())),
+    timestamp::Union{TimeDateZone, ZonedDateTime} = tdz_now(),
     tags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}([:FACTOR]),
     bloblets::Bloblets = Bloblets(),
     multihypo::Vector{Float64} = Float64[],
@@ -160,9 +160,7 @@ function FactorDFG(
     state::Recipestate = Recipestate(),
     cache = nothing;
     tags::Set{Symbol} = Set{Symbol}([:FACTOR]),
-    timestamp::Union{DateTime, ZonedDateTime, TimeDateZone} = TimeDateZone(
-        now(localzone()),
-    ),
+    timestamp::Union{DateTime, ZonedDateTime, TimeDateZone} = tdz_now(),
     solvable::Int = 1,
     bloblets::Bloblets = Bloblets(),
     blobentries::Blobentries = Blobentries(),
@@ -237,7 +235,7 @@ end
 function FactorSummary(
     label::Symbol,
     variableorder::Union{Vector{Symbol}, Tuple};
-    timestamp::TimeDateZone = TimeDateZone(now(localzone())),
+    timestamp::TimeDateZone = tdz_now(),
     tags::Set{Symbol} = Set{Symbol}(),
 )
     return FactorSummary(label, tags, Tuple(variableorder), timestamp)

src/entities/DFGVariable.jl

@@ -128,11 +128,6 @@ end
 ##------------------------------------------------------------------------------
 # The Variable information packed in a way that accomdates multi-lang using json.
 
-# Notes:
-# - timestamp is a `ZonedDateTime` in UTC.
-# - nstime can be used as mission time, with the convention that the timestamp millis coincide with the mission start nstime
-#   - e.g. timestamp is `2020-01-01 06:30:01.250 UTC` and first nstime is `250_000_000`.
-
 """
 $(TYPEDEF)
 Complete variable structure for a DistributedFactorGraph variable.
@@ -147,9 +142,9 @@ $(TYPEDFIELDS)
     label::Symbol
     """Variable timestamp.
     Accessors: [`getTimestamp`](@ref)"""
-    timestamp::TimeDateZone = TimeDateZone(now(localzone())) #NOTE changed to TimeDateZone in v0.29
-    """Nanoseconds since a user-understood epoch (i.e unix epoch, robot boot time, etc.)"""
-    steadytime::Union{Nothing, Nanosecond} = nothing #NOTE changed to TimeDateZone in v0.29
+    timestamp::TimeDateZone = tdz_now() #NOTE changed to TimeDateZone in v0.29
+    # """Nanoseconds since a user-understood epoch (i.e unix epoch, robot boot time, etc.)"""
+    # steadytime::Union{Nothing, Nanosecond} = nothing #NOTE changed to TimeDateZone in v0.29
     #nstime::String = "0" #NOTE different uses, as 0-999_999 nanosecond part of timestamp now in timestamp, as steady timestamp now in steadytime
     """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK].
     Accessors: [`getTags`](@ref), [`mergeTags!`](@ref), and [`deleteTags!`](@ref)"""
@@ -179,7 +174,7 @@ function StructUtils.fielddefaults(
     ::Type{VariableDFG{T, P, N}},
 ) where {T, P, N}
     return (
-        timestamp = TimeDateZone(now(localzone())),
+        timestamp = tdz_now(),
         tags = Set{Symbol}(),
         # states = OrderedDict{Symbol, State{T, P, N}}(),
         bloblets = Bloblets(),
@@ -218,9 +213,9 @@ function VariableDFG(
     label::Symbol,
     statetype::Union{T, Type{T}};
     tags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}(),
-    timestamp::Union{TimeDateZone, ZonedDateTime} = TimeDateZone(now(localzone())),
+    timestamp::Union{TimeDateZone, ZonedDateTime} = tdz_now(),
     solvable::Union{Int, Base.RefValue{Int}} = Ref{Int}(1),
-    steadytime::Union{Nothing, Nanosecond} = nothing,
+    # steadytime::Union{Nothing, Nanosecond} = nothing,
     nanosecondtime = nothing,
     smalldata = nothing,
     kwargs...,
@@ -230,13 +225,10 @@ function VariableDFG(
         timestamp = TimeDateZone(timestamp)
     end
     if !isnothing(nanosecondtime)
-        @assert isnothing(steadytime),
-        "nanosecondtime is replaced by steadytime. Cannot specify both steadytime and nanosecondtime"
         Base.depwarn(
-            "nanosecondtime kwarg is deprecated, use steadytime instead",
+            "nanosecondtime kwarg is deprecated, use `timestamp` or `bloblets` instead",
             :VariableDFG,
         )
-        steadytime = Nanosecond(nanosecondtime)
     end
     if !isnothing(smalldata)
         Base.depwarn("smalldata kwarg is deprecated, use bloblets instead", :VariableDFG)
@@ -249,7 +241,7 @@ function VariableDFG(
 
     N = getDimension(T)
     P = getPointType(T)
-    return VariableDFG{T, P, N}(; label, steadytime, solvable, tags, timestamp, kwargs...)
+    return VariableDFG{T, P, N}(; label, solvable, tags, timestamp, kwargs...)
 end
 
 function VariableDFG(label::Symbol, state::State; kwargs...)

src/services/CustomPrinting.jl

@@ -36,7 +36,6 @@ function printVariable(
             nothing
         end
         println(ioc, "  timestamp:  ", vert.timestamp)
-        isnothing(vert.steadytime) || println(ioc, "  steadytime: ", vert.steadytime)
         print(ioc, "  label:      ")
         printstyled(ioc, vert.label; bold = true)
         println(ioc)

src/services/find.jl

@@ -6,15 +6,15 @@
 """
     $SIGNATURES
 
-Find and return the closest timestamp from two sets of Tuples.  Also return the minimum delta-time (`::Millisecond`) and how many elements match from the two sets are separated by the minimum delta-time.
+Find and return the closest timestamp from two sets of Tuples.  Also return the minimum delta-time (`::Nanosecond`) and how many elements match from the two sets are separated by the minimum delta-time.
 """
 function findClosestTimestamp(
     setA::Vector{Tuple{TimeDateZone, T}},
     setB::Vector{Tuple{TimeDateZone, S}},
 ) where {S, T}
     #
     # build matrix of delta times, ranges on rows x vars on columns
-    DT = Array{Millisecond, 2}(undef, length(setA), length(setB))
+    DT = Array{Nanosecond, 2}(undef, length(setA), length(setB))
     for i = 1:length(setA), j = 1:length(setB)
         DT[i, j] = setB[j][1] - setA[i][1]
     end
@@ -38,7 +38,7 @@ end
 Find and return nearest variable labels per delta time.  Function will filter on `regexFilter`, `tags`, and `solvable`.
 
 Notes
-- Returns `Vector{Tuple{Vector{Symbol}, Millisecond}}`
+- Returns `Vector{Tuple{Vector{Symbol}, Nanosecond}}`
 
 DevNotes:
 - TODO `number` should allow returning more than one for k-nearest matches.
@@ -68,7 +68,7 @@ function findVariableNearTimestamp(
     mask = BitArray{1}(undef, length(syms))
     fill!(mask, true)
 
-    RET = Vector{Tuple{Vector{Symbol}, Millisecond}}()
+    RET = Vector{Tuple{Vector{Symbol}, Nanosecond}}()
     SYMS = Symbol[]
     CORRS = 1
     NUMBER = number

test/GraphsDFGSummaryTypes.jl

@@ -57,7 +57,6 @@ if false
         tags = [:FACTOR],
         variableorder = [:a],
         timestamp = DFG.Dates.now(DFG.tz"Z"),
-        nstime = 0,
         fnctype = "PriorPose2",
         solvable = 1,
         data = "",
@@ -68,7 +67,6 @@ if false
         tags = [:FACTOR],
         variableorder = [:a, :b],
         timestamp = DFG.Dates.now(DFG.tz"Z"),
-        nstime = 0,
         fnctype = "Pose2Pose2",
         solvable = 1,
         data = "",
@@ -79,7 +77,6 @@ if false
         tags = [:FACTOR],
         variableorder = [:b, :c],
         timestamp = DFG.Dates.now(DFG.tz"Z"),
-        nstime = 0,
         fnctype = "Pose2Pose2",
         solvable = 1,
         data = "",

test/interfaceTests.jl

@@ -64,8 +64,7 @@ end
     @test printVariable(var1) === nothing
     @test printFactor(fac1) === nothing
 
-    @test printVariable(iobuf, var1; skipfields = [:timestamp, :solver, :ppe, :nstime]) ===
-          nothing
+    @test printVariable(iobuf, var1; skipfields = [:timestamp]) === nothing
 
     varstr = String(take!(iobuf))
     @test occursin(r"VariableDFG", varstr)
@@ -79,7 +78,7 @@ end
     @test occursin(r"bandwidths", varstr)
     #  == "VariableCompute{TestVariableType1}\nlabel: a\ntags: Set([:VARIABLE, :POSE])\nsize marginal samples: (1, 1)\nkde bandwidths: [0.0]\nNo PPEs\n"
 
-    @test printFactor(iobuf, fac1; skipfields = [:timestamp, :solver, :nstime]) === nothing
+    @test printFactor(iobuf, fac1; skipfields = [:timestamp]) === nothing
     @test occursin(r"FactorDFG.*\nlabel:\n:abf1", String(take!(iobuf)))
 
     @test printFactor(iobuf, fac1; short = true) === nothing

test/testBlocks.jl

@@ -319,6 +319,8 @@ function DFGVariableSCA()
     @test getDimension(testvar) == 1
     @test getManifold(testvar) == TranslationGroup(1)
 
+    @test DFG.calcDeltatime(v1, v2) isa Real
+
     # #TODO sort out
     # getState
     # getSolvedCount
@@ -389,6 +391,7 @@ function DFGFactorSCA()
     # create f0 here for a later timestamp
     f0 = FactorCompute(:af1, [:a], obs_prior; tags = Set([:PRIOR]))
 
+    @test DFG.calcDeltatime(f1, f2) isa Real
     #fill in undefined fields
     # f2.solverData.certainhypo = Int[]
     # f2.solverData.multihypo = Float64[]