From 872e52530cbd50d46b07e036818d712b284c7c53 Mon Sep 17 00:00:00 2001 From: Danny Winrow Date: Wed, 6 Apr 2022 16:25:00 +0100 Subject: [PATCH 1/8] Added datasource Bank of England database --- docs/src/downloads.md | 11 ++++ src/MarketData.jl | 3 +- src/downloads.jl | 113 ++++++++++++++++++++++++++++++++++++++++++ test/downloads.jl | 8 +++ 4 files changed, 134 insertions(+), 1 deletion(-) diff --git a/docs/src/downloads.md b/docs/src/downloads.md index 563eaad..fd9316c 100644 --- a/docs/src/downloads.md +++ b/docs/src/downloads.md @@ -35,4 +35,15 @@ This is the ONS' headline monthly price inflation statistic. ```@docs ons +``` + +## BOE + +For the `boe()` method, the default +seriescode is the `IUDSOIA`, which +represents the daily Sterling Overnight Index Average +which is one of the 10 most popular BoE timeseries. + +```@docs +boe ``` \ No newline at end of file diff --git a/src/MarketData.jl b/src/MarketData.jl index 8934f44..9f20a24 100644 --- a/src/MarketData.jl +++ b/src/MarketData.jl @@ -17,7 +17,8 @@ export AAPL, BA, CAT, DELL, EBAY, F, GE, TX, # downloads.jl export AbstractQueryOpt, YahooOpt, - yahoo, fred, ons + BoeOpt, + yahoo, fred, ons, boe ###### include ################## diff --git a/src/downloads.jl b/src/downloads.jl index 5fd4ec8..3eacc0e 100644 --- a/src/downloads.jl +++ b/src/downloads.jl @@ -193,3 +193,116 @@ function ons(timeseries::AbstractString = "L522", dataset::AbstractString = "MM2 end TimeArray(ta, meta = json["description"]) end + +""" + struct BoeOpt <: AbstractQueryOpt + Datefrom # the start time + Dateto # the end time + UsingCodes # indicate using series codes + CSVF # "TT" (Tabular with titles), "TN" (Tabular no titles), "CT" (Columnar with- titles) or "CN" (Columnar no titles) + VPD # provisional data is required + end + +The Bank of England Database API query object. + +# Examples + +```jl-repl +julia> t = Dates.today() +2022-04-06 + +julia> BoeOpt(Datefrom = t - Year(2), Dateto = t) +BoeOpt with 5 entries: + :Datefrom => "06/Apr/2020" + :Dateto => "06/Apr/2022" + :UsingCodes => "Y" + :CSVF => "TN" + :VPD => "Y" +``` +""" +struct BoeOpt <: AbstractQueryOpt + Datefrom::Date + Dateto::Date + UsingCodes::Bool + CSVF::String + VPD::Bool + + BoeOpt(; Datefrom::Date = Date(1963, 1, 1), + Dateto::Date = Dates.today(), + UsingCodes::Bool = true, + CSVF::String = "TN", + VPD::Bool = true + ) = + new(Datefrom, Dateto, UsingCodes, CSVF, VPD) +end + +function Base.iterate(opt::BoeOpt, state = 1) + (state > length(BoeOpt)) && return nothing + k = fieldname(BoeOpt, state) + v = getfield(opt, state) + v′ = v isa Date ? Dates.format(v,dateformat"dd/u/yyyy") : v isa Bool ? v ? "Y" : "N" : v + (k => v′, state + 1) +end + +""" + boe(seriescode::String="IUDSOIA")::TimeArray + +The boe() method is a wrapper to download financial and economic time series data from the Bank of England (BoE) database. + +The boe() method takes a string argument that corresponds to a series code from the BoE database. +It returns the data in the TimeSeries.TimeArray data structure. When no argument is provided, the +default data set is the daily Sterling Overnight Index Average (SONIA) rate. + +# Examples + +```julia +GBPUSD = boe("XUDLGBD") +SONIA = boe() +``` + +```jl-repl +julia> start = Date(2018, 1, 1) +2018-01-01 + +julia> boe(:IUDSOIA, BoeOpt(Datefrom = start)) +1078×1 TimeArray{Float64, 1, Date, Vector{Float64}} 2018-01-02 to 2022-04-04 +│ │ IUDSOIA │ +├────────────┼─────────┤ +│ 2018-01-02 │ 0.4622 │ +│ 2018-01-03 │ 0.4642 │ +... + +julia> boe("XUDLGBD,XUDLERS", BoeOpt(Datefrom = start)) +1079×2 TimeArray{Float64, 2, Date, Matrix{Float64}} 2018-01-02 to 2022-04-05 +│ │ XUDLGBD │ XUDLERS │ +├────────────┼─────────┼─────────┤ +│ 2018-01-02 │ 0.7364 │ 1.1274 │ +│ 2018-01-03 │ 0.74 │ 1.124 │ +... +``` + +# References + +https://www.bankofengland.co.uk/boeapps/database/ +https://www.bankofengland.co.uk/boeapps/database/Help.asp#CSV +https://www.bankofengland.co.uk/statistics/details + +# See Also + +- yahoo() which is a wrapper to download financial time series for stocks from Yahoo Finance. +- fred() which accesses the St. Louis Federal Reserve financial and economic data sets. +- ons() which is a wrapper to download financial and economic time series data from the Office for National Statistics (ONS). +""" +function boe(seriescodes::AbstractString = "XUDLGBD", opt::BoeOpt = BoeOpt()) + url = "http://www.bankofengland.co.uk/boeapps/iadb/fromshowcolumns.asp" + parameters = Dict( + "csv.x" => "yes", + "SeriesCodes" => seriescodes + ) + res = HTTP.get(url, query = merge(parameters,Dict(opt))) + @assert res.status == 200 + csv = CSV.File(res.body, dateformat=dateformat"dd u yyyy") + sch = TimeSeries.Tables.schema(csv) + TimeArray(csv, timestamp = first(sch.names)) |> cleanup_colname! +end +boe(s::Symbol, opt::BoeOpt = BoeOpt()) = boe(string(s), opt) diff --git a/test/downloads.jl b/test/downloads.jl index 8ec367e..66db0cd 100644 --- a/test/downloads.jl +++ b/test/downloads.jl @@ -20,4 +20,12 @@ using Test ta = ons() @test ta |> timestamp |> length > 100 end + + @testset "BOE" begin + t = Dates.today() - Year(2) + opt = BoeOpt(Datefrom = t) + ta = boe("XUDLGBD",opt) + @test ta |> timestamp |> length > 100 + @test timestamp(ta)[1] >= t + end end From a019cac643efe3c030b0d1c309865318de32e5e5 Mon Sep 17 00:00:00 2001 From: Danny Winrow Date: Wed, 6 Apr 2022 16:43:05 +0100 Subject: [PATCH 2/8] updated docs --- docs/src/downloads.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/downloads.md b/docs/src/downloads.md index fd9316c..f8c616f 100644 --- a/docs/src/downloads.md +++ b/docs/src/downloads.md @@ -1,8 +1,9 @@ # Downloading from remote data source Three eponymous methods are provided for downloading free data, from -Yahoo Finance (yahoo) the Federal Reserve St. Louis (fred) and the -United Kingdom's Office for National Statistics (ons) +Yahoo Finance (yahoo) the Federal Reserve St. Louis (fred), the +United Kingdom's Office for National Statistics (ons) and the +Bank of England's database (boe). These methods take a string argument that represents the name of the desired data set to be downloaded. @@ -36,7 +37,6 @@ This is the ONS' headline monthly price inflation statistic. ```@docs ons ``` - ## BOE For the `boe()` method, the default From 8abab1edacfd1f516c80fae1d793abe896547384 Mon Sep 17 00:00:00 2001 From: Danny Winrow Date: Wed, 6 Apr 2022 16:48:46 +0100 Subject: [PATCH 3/8] updated default series to SONIA --- src/downloads.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/downloads.jl b/src/downloads.jl index 3eacc0e..5abe819 100644 --- a/src/downloads.jl +++ b/src/downloads.jl @@ -293,7 +293,7 @@ https://www.bankofengland.co.uk/statistics/details - fred() which accesses the St. Louis Federal Reserve financial and economic data sets. - ons() which is a wrapper to download financial and economic time series data from the Office for National Statistics (ONS). """ -function boe(seriescodes::AbstractString = "XUDLGBD", opt::BoeOpt = BoeOpt()) +function boe(seriescodes::AbstractString = "IUDSOIA", opt::BoeOpt = BoeOpt()) url = "http://www.bankofengland.co.uk/boeapps/iadb/fromshowcolumns.asp" parameters = Dict( "csv.x" => "yes", From 4363d1f67227cbb8d00a953e2eb4ba1ae79d956d Mon Sep 17 00:00:00 2001 From: Danny Winrow Date: Wed, 6 Apr 2022 18:02:37 +0100 Subject: [PATCH 4/8] added some error handling for invalid seriescodes --- src/downloads.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/downloads.jl b/src/downloads.jl index 5abe819..a518a61 100644 --- a/src/downloads.jl +++ b/src/downloads.jl @@ -301,6 +301,13 @@ function boe(seriescodes::AbstractString = "IUDSOIA", opt::BoeOpt = BoeOpt()) ) res = HTTP.get(url, query = merge(parameters,Dict(opt))) @assert res.status == 200 + if Dict(res.headers)["Content-Type"] != "application/csv" + if Dict(res.headers)["Content-Type"] == "text/html" + mat = match(r"

\s+([\w\s]+)",String(res.body)) + !isnothing(mat) && throw("Error message from BoE: $(mat.captures[1])") + end + throw("CSV not returned from BoE, it's likely that the SeriesCodes, ($seriescodes) are invalid") + end csv = CSV.File(res.body, dateformat=dateformat"dd u yyyy") sch = TimeSeries.Tables.schema(csv) TimeArray(csv, timestamp = first(sch.names)) |> cleanup_colname! From d75a87c67e8bb2f6fc9e1f28a7030f3ee68d1bae Mon Sep 17 00:00:00 2001 From: Danny Winrow Date: Tue, 19 Apr 2022 14:39:22 +0100 Subject: [PATCH 5/8] improved codecov --- test/downloads.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/downloads.jl b/test/downloads.jl index 66db0cd..354a251 100644 --- a/test/downloads.jl +++ b/test/downloads.jl @@ -27,5 +27,6 @@ using Test ta = boe("XUDLGBD",opt) @test ta |> timestamp |> length > 100 @test timestamp(ta)[1] >= t + @test_throws String boe("asdfasdf") end end From 533c4cfe183637e76e87b963b9ef7a2be2d4884d Mon Sep 17 00:00:00 2001 From: Danny Winrow Date: Tue, 19 Apr 2022 14:52:12 +0100 Subject: [PATCH 6/8] lower case fieldnames for BoeOpt --- src/downloads.jl | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/downloads.jl b/src/downloads.jl index a518a61..d2eb8be 100644 --- a/src/downloads.jl +++ b/src/downloads.jl @@ -196,11 +196,11 @@ end """ struct BoeOpt <: AbstractQueryOpt - Datefrom # the start time - Dateto # the end time - UsingCodes # indicate using series codes - CSVF # "TT" (Tabular with titles), "TN" (Tabular no titles), "CT" (Columnar with- titles) or "CN" (Columnar no titles) - VPD # provisional data is required + datefrom # the start time + dateto # the end time + usingcodes # indicate using series codes + csvf # "TT" (Tabular with titles), "TN" (Tabular no titles), "CT" (Columnar with- titles) or "CN" (Columnar no titles) + vpd # provisional data is required end The Bank of England Database API query object. @@ -211,29 +211,29 @@ The Bank of England Database API query object. julia> t = Dates.today() 2022-04-06 -julia> BoeOpt(Datefrom = t - Year(2), Dateto = t) +julia> BoeOpt(datefrom = t - Year(2), dateto = t) BoeOpt with 5 entries: - :Datefrom => "06/Apr/2020" - :Dateto => "06/Apr/2022" - :UsingCodes => "Y" - :CSVF => "TN" - :VPD => "Y" + :datefrom => "06/Apr/2020" + :dateto => "06/Apr/2022" + :usingvodes => "Y" + :csvf => "TN" + :vpd => "Y" ``` """ struct BoeOpt <: AbstractQueryOpt - Datefrom::Date - Dateto::Date - UsingCodes::Bool - CSVF::String - VPD::Bool - - BoeOpt(; Datefrom::Date = Date(1963, 1, 1), - Dateto::Date = Dates.today(), - UsingCodes::Bool = true, - CSVF::String = "TN", - VPD::Bool = true + datefrom::Date + dateto::Date + usingcodes::Bool + csvf::String + vpd::Bool + + BoeOpt(; datefrom::Date = Date(1963, 1, 1), + dateto::Date = Dates.today(), + usingcodes::Bool = true, + csvf::String = "TN", + vpd::Bool = true ) = - new(Datefrom, Dateto, UsingCodes, CSVF, VPD) + new(datefrom, dateto, usingcodes, csvf, vpd) end function Base.iterate(opt::BoeOpt, state = 1) @@ -264,7 +264,7 @@ SONIA = boe() julia> start = Date(2018, 1, 1) 2018-01-01 -julia> boe(:IUDSOIA, BoeOpt(Datefrom = start)) +julia> boe(:IUDSOIA, BoeOpt(datefrom = start)) 1078×1 TimeArray{Float64, 1, Date, Vector{Float64}} 2018-01-02 to 2022-04-04 │ │ IUDSOIA │ ├────────────┼─────────┤ @@ -272,7 +272,7 @@ julia> boe(:IUDSOIA, BoeOpt(Datefrom = start)) │ 2018-01-03 │ 0.4642 │ ... -julia> boe("XUDLGBD,XUDLERS", BoeOpt(Datefrom = start)) +julia> boe("XUDLGBD,XUDLERS", BoeOpt(datefrom = start)) 1079×2 TimeArray{Float64, 2, Date, Matrix{Float64}} 2018-01-02 to 2022-04-05 │ │ XUDLGBD │ XUDLERS │ ├────────────┼─────────┼─────────┤ From 321c1ddd8ae46332172f20192743af9e1b75c6ff Mon Sep 17 00:00:00 2001 From: Danny Winrow Date: Tue, 19 Apr 2022 14:57:08 +0100 Subject: [PATCH 7/8] update tests consistent with last commit --- test/downloads.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/downloads.jl b/test/downloads.jl index 354a251..b57c2e7 100644 --- a/test/downloads.jl +++ b/test/downloads.jl @@ -23,7 +23,7 @@ using Test @testset "BOE" begin t = Dates.today() - Year(2) - opt = BoeOpt(Datefrom = t) + opt = BoeOpt(datefrom = t) ta = boe("XUDLGBD",opt) @test ta |> timestamp |> length > 100 @test timestamp(ta)[1] >= t From f49adfab99fbda27462201c616e721548715a157 Mon Sep 17 00:00:00 2001 From: Danny Winrow Date: Tue, 19 Apr 2022 15:56:07 +0100 Subject: [PATCH 8/8] included custom error types to improve compat with Julia 1 --- src/MarketData.jl | 1 + src/downloads.jl | 15 +++++++++++++-- test/downloads.jl | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/MarketData.jl b/src/MarketData.jl index 9f20a24..f47feb6 100644 --- a/src/MarketData.jl +++ b/src/MarketData.jl @@ -12,6 +12,7 @@ using JSON3 @reexport using TimeSeries export AAPL, BA, CAT, DELL, EBAY, F, GE, TX, + APIError, InvalidAPIReturn, cl, op, ohlc, ohlcv, datetime1, datetime2, mdata, o, h, l, c, v # downloads.jl diff --git a/src/downloads.jl b/src/downloads.jl index d2eb8be..255d9fd 100644 --- a/src/downloads.jl +++ b/src/downloads.jl @@ -8,6 +8,17 @@ function Base.iterate(aqo::T, state = 1) where {T<:AbstractQueryOpt} (fieldname(T, state) => getfield(aqo, state), state + 1) end +struct APIError <: Exception + errmsg::String + res::HTTP.Messages.Response +end +Base.showerror(io::IO, e::APIError) = print(io, "Error message from API: ", e.errmsg, "\nfrom query: $(HTTP.unescapeuri(HTTP.uri(e.res.request)))") +struct InvalidAPIReturn <: Exception + expected::String + res::HTTP.Messages.Response +end +Base.showerror(io::IO, e::InvalidAPIReturn) = print(io, "Expected type: $(e.expected), API returned: $(Dict(e.res.headers)["Content-Type"])\nfrom query: $(HTTP.unescapeuri(HTTP.uri(e.res.request)))") + """ struct YahooOpt <: AbstractQueryOpt period1 # the start time @@ -304,9 +315,9 @@ function boe(seriescodes::AbstractString = "IUDSOIA", opt::BoeOpt = BoeOpt()) if Dict(res.headers)["Content-Type"] != "application/csv" if Dict(res.headers)["Content-Type"] == "text/html" mat = match(r"

\s+([\w\s]+)",String(res.body)) - !isnothing(mat) && throw("Error message from BoE: $(mat.captures[1])") + !isnothing(mat) && throw(APIError(string(mat.captures[1]),res)) end - throw("CSV not returned from BoE, it's likely that the SeriesCodes, ($seriescodes) are invalid") + throw(InvalidAPIReturn("application/csv",res)) end csv = CSV.File(res.body, dateformat=dateformat"dd u yyyy") sch = TimeSeries.Tables.schema(csv) diff --git a/test/downloads.jl b/test/downloads.jl index b57c2e7..402f345 100644 --- a/test/downloads.jl +++ b/test/downloads.jl @@ -27,6 +27,6 @@ using Test ta = boe("XUDLGBD",opt) @test ta |> timestamp |> length > 100 @test timestamp(ta)[1] >= t - @test_throws String boe("asdfasdf") + @test_throws APIError boe("asdfasdf") end end