first try of time controller

.gitignore

@@ -26,4 +26,5 @@ Manifest.toml
 *.out
 *.ok
 debug_psrclasses
+*.gz
 *.sqlite

profiling/Project.toml

@@ -0,0 +1,3 @@
+[deps]
+PProf = "e4faabce-9ead-11e9-39d9-4379958e3056"
+Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"

profiling/create_profile.jl

@@ -0,0 +1,13 @@
+# You should run the script from the profiling directory
+
+using Profile
+using PProf
+import Pkg
+root_path = dirname(@__DIR__)
+Pkg.activate(root_path)
+using PSRClassesInterface
+
+
+include("../script_time_controller.jl")
+@profile include("../script_time_controller.jl")
+pprof()

profiling/open_profile.jl

@@ -0,0 +1,8 @@
+# You should run the script from the profiling directory
+
+using Profile
+using PProf
+
+file_name = "profile.pb.gz"
+
+PProf.refresh(file=file_name, webport = 57998)

script.jl

@@ -70,17 +70,6 @@ function test_create_time_series()
     )
     @show results
 
-    @show labels = PSRDatabaseSQLite.read_scalar_parameters(db, "Resource", "label")
-
-    results = PSRDatabaseSQLite.read_time_series_dfs(
-        db,
-        "Resource",
-        "some_vector5";
-        date_time = DateTime(2010)
-    )
-    @show results
-
-
     PSRDatabaseSQLite.close!(db)
     rm(db_path)
     @test true

script_time_controller.jl

@@ -0,0 +1,76 @@
+using PSRClassesInterface.PSRDatabaseSQLite
+using SQLite
+using DataFrames
+using Dates
+using Test
+
+function test_create_time_series()
+    path_schema = joinpath(@__DIR__, "time_controller.sql")
+    db_path = joinpath(@__DIR__, "test_create_time_series.sqlite")
+    GC.gc()
+    GC.gc()
+    if isfile(db_path)
+        rm(db_path)
+    end
+
+    db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true)
+    PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0)
+    PSRDatabaseSQLite.SQLite.transaction(db.sqlite_db) do
+        for i in 1:500
+            df_timeseries_group1 = DataFrame(
+                date_time = vcat([DateTime(0)],[DateTime(i) for i in 1900:1979]),
+                some_vector1 = vcat([missing],[j for j in 1:80] .* i),
+                some_vector2 = vcat([1.0],[missing for j in 1:10], [j for j in 1:10] .* i, [missing for j in 1:60]),
+                some_vector3 = vcat([1.0], [missing for j in 1:80]),
+                some_vector4 = vcat([missing], [missing for j in 1:80]),
+            )
+            PSRDatabaseSQLite.create_element!(
+                db,
+                "Resource";
+                label = "Resource $i",
+                group1 = df_timeseries_group1,
+            )
+        end
+    end
+
+    PSRDatabaseSQLite.close!(db) 
+end
+
+function test_read_time_series()
+    db_path = joinpath(@__DIR__, "test_create_time_series.sqlite")
+
+    db = PSRDatabaseSQLite.load_db(db_path; read_only = true)
+
+    for date_time in [DateTime(i) for i in 1900:1901]
+        @show date_time
+        for i in 1:50
+            PSRDatabaseSQLite.read_mapped_timeseries(
+                db,
+                "Resource",
+                "some_vector1",
+                date_time = date_time
+            )
+
+            PSRDatabaseSQLite.read_mapped_timeseries(
+                db,
+                "Resource",
+                "some_vector2",
+                date_time = date_time
+            )
+
+            PSRDatabaseSQLite.read_mapped_timeseries(
+                db,
+                "Resource",
+                "some_vector3",
+                date_time = date_time
+            )
+        end
+    end
+
+
+    PSRDatabaseSQLite.close!(db)
+    rm(db_path)
+end
+
+test_create_time_series()
+test_read_time_series()

src/PSRDatabaseSQLite/PSRDatabaseSQLite.jl

@@ -20,6 +20,7 @@ include("exceptions.jl")
 include("utils.jl")
 include("attribute.jl")
 include("collection.jl")
+include("time_controller.jl")
 include("database_sqlite.jl")
 include("create.jl")
 include("read.jl")

src/PSRDatabaseSQLite/database_sqlite.jl

@@ -1,8 +1,17 @@
-mutable struct DatabaseSQLite
+Base.@kwdef mutable struct DatabaseSQLite
     sqlite_db::SQLite.DB
     collections_map::OrderedDict{String, Collection}
+    read_only::Bool = false
+    # TimeController is a cache that allows PSRDatabaseSQLite to
+    # store information about the last timeseries query. This is useful for avoiding to
+    # re-query the database when the same query is made multiple times.
+    # The TimeController is a private behaviour and whenever it is used
+    # it changes the database mode to read-only.
+    _time_controller::TimeController = TimeController()
 end
 
+_is_read_only(db::DatabaseSQLite) = db.read_only
+
 function _set_default_pragmas!(db::SQLite.DB)
     _set_foreign_keys_on!(db)
     _set_busy_timeout!(db, 5000)
@@ -42,7 +51,7 @@ function DatabaseSQLite_from_schema(
         rethrow(e)
     end
 
-    db = DatabaseSQLite(
+    db = DatabaseSQLite(;
         sqlite_db,
         collections_map,
     )
@@ -76,7 +85,7 @@ function DatabaseSQLite_from_migrations(
         rethrow(e)
     end
 
-    db = DatabaseSQLite(
+    db = DatabaseSQLite(;
         sqlite_db,
         collections_map,
     )
@@ -89,7 +98,7 @@ function DatabaseSQLite(
     read_only::Bool = false,
 )
     sqlite_db =
-        read_only ? SQLite.DB("file:" * database_path * "?mode=ro&immutable=1") :
+        # read_only ? SQLite.DB("file:" * database_path * "?mode=ro&immutable=1") :
         SQLite.DB(database_path)
 
     _set_default_pragmas!(sqlite_db)
@@ -102,9 +111,10 @@ function DatabaseSQLite(
         rethrow(e)
     end
 
-    db = DatabaseSQLite(
+    db = DatabaseSQLite(;
         sqlite_db,
         collections_map,
+        read_only
     )
     return db
 end

src/PSRDatabaseSQLite/read.jl

@@ -195,6 +195,27 @@ function read_time_series_df(
     )
 end
 
+function end_date_query(db::DatabaseSQLite, attribute::Attribute)
+     # First checks if the date or dimension value is within the range of the data.
+    # Then it queries the closest date before the provided date.
+    # If there is no date query the data with date 0 (which will probably return no data.)
+    end_date_query = "SELECT MAX(DATE(date_time)) FROM $(attribute.table_where_is_located)"
+    end_date = DBInterface.execute(db.sqlite_db, end_date_query) |> DataFrame
+    if isempty(end_date)
+        return DateTime(0)
+    end
+    return DateTime(end_date[!, 1][1])
+end
+
+function closest_date_query(db::DatabaseSQLite, attribute::Attribute, dim_value::DateTime)
+    closest_date_query_earlier = "SELECT DISTINCT date_time FROM $(attribute.table_where_is_located) WHERE $(attribute.id) IS NOT NULL AND DATE(date_time) <= DATE('$(dim_value)') ORDER BY DATE(date_time) DESC LIMIT 1"
+    closest_date = DBInterface.execute(db.sqlite_db, closest_date_query_earlier) |> DataFrame
+    if isempty(closest_date)
+        return DateTime(0)
+    end
+    return DateTime(closest_date[!, 1][1])
+end
+
 function _read_time_series_df(
     db::DatabaseSQLite,
     collection_id::String,
@@ -215,20 +236,12 @@ function _read_time_series_df(
                 if read_exact_date
                     query *= "DATE($dim_name) = DATE('$(dim_value)')"
                 else
-                    # First checks if the date or dimension value is within the range of the data.
-                    # Then it queries the closest date before the provided date.
-                    # If there is no date query the data with date 0 (which will probably return no data.)
-                    end_date_query = "SELECT MAX(DATE($dim_name)) FROM $(attribute.table_where_is_located)"
-                    end_date = DBInterface.execute(db.sqlite_db, end_date_query) |> DataFrame
-                    # Query the nearest date before the provided date
-                    closest_date_query_earlier = "SELECT DISTINCT $dim_name FROM $(attribute.table_where_is_located) WHERE $(attribute.id) IS NOT NULL AND DATE($dim_name) <= DATE('$(dim_value)') ORDER BY DATE($dim_name) DESC LIMIT 1"
-                    closest_date = DBInterface.execute(db.sqlite_db, closest_date_query_earlier) |> DataFrame
-                    date_to_equal_in_query = if dim_value > DateTime(end_date[!, 1][1])
-                        DateTime(0)
-                    elseif isempty(closest_date)
+                    end_date = end_date_query(db, attribute)
+                    closest_date = closest_date_query(db, attribute, dim_value)
+                    date_to_equal_in_query = if dim_value > end_date
                         DateTime(0)
                     else
-                        closest_date[!, 1][1]
+                        closest_date
                     end
                     # query the closest date and make it equal to the provided date.
                     query *= "DATE($dim_name) = DATE('$(date_to_equal_in_query)')"