Clean up tests. Add Aqua. Run doctests. (#18)

* Note when errors are expected

* run doctests in CI

* Add aqua

* Fix aqua violations

* Fix dangling doc refs

* patch version bump

* document the new shadow framework

Project.toml

@@ -1,6 +1,6 @@
 name = "AWSCRT"
 uuid = "df31ea59-17a4-4ebd-9d69-4f45266dc2c7"
-version = "0.1.3"
+version = "0.1.4"
 
 [deps]
 AWSCRT_jll = "01db5350-6ea1-5d9a-9a47-8a31a394cb9c"
@@ -11,9 +11,19 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LibAWSCRT = "df7458b6-5204-493f-a0e7-404b4eb72fac"
 
 [compat]
+AWSCRT_jll = "0.1"
 CEnum = "0.4"
 CountDownLatches = "2"
 ForeignCallbacks = "0.1"
 JSON = "0.21"
 LibAWSCRT = "0.1"
 julia = "1.9"
+
+[extras]
+Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Aqua", "Documenter", "Random", "Test"]

README.md

@@ -1,7 +1,7 @@
 # AWSCRT
 
 | :exclamation: This is unfinished, early software. Expect bugs and breakages! |
-|------------------------------------------------------------------------------|
+| ---------------------------------------------------------------------------- |
 
 [![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://Octogonapus.github.io/AWSCRT.jl/stable)
 [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://Octogonapus.github.io/AWSCRT.jl/dev)
@@ -12,6 +12,23 @@
 A high-level wrapper for the code in [LibAWSCRT.jl](https://github.com/Octogonapus/LibAWSCRT.jl).
 Currently, only an MQTT client is implemented.
 
+- [AWSCRT](#awscrt)
+  - [Installation](#installation)
+  - [MQTT Client](#mqtt-client)
+    - [Create a client](#create-a-client)
+    - [Connect a client](#connect-a-client)
+    - [Subscribe](#subscribe)
+    - [Publish](#publish)
+    - [Clean Up](#clean-up)
+    - [See Also](#see-also)
+  - [AWS IoT Device Shadow Service](#aws-iot-device-shadow-service)
+  - [Create a ShadowFramework](#create-a-shadowframework)
+    - [Using the ShadowFramework](#using-the-shadowframework)
+    - [Update Callbacks: Ordering and Other Behaviors](#update-callbacks-ordering-and-other-behaviors)
+    - [Persisting the Shadow Document Locally](#persisting-the-shadow-document-locally)
+    - [See Also](#see-also-1)
+
+
 ## Installation
 
 ```julia
@@ -104,3 +121,87 @@ fetch(task) # wait for the connection to be closed
 - [AWS Protocol Port Mapping and Authentication Documentation](https://docs.aws.amazon.com/iot/latest/developerguide/protocols.html)
 - [AWS MQTT Topic Documentation](https://docs.aws.amazon.com/iot/latest/developerguide/topics.html)
 - [AWS IoT Client Certificate Documentation](https://docs.aws.amazon.com/iot/latest/developerguide/x509-client-certs.html)
+
+## AWS IoT Device Shadow Service
+
+The AWS IoT Device Shadow service adds persistent JSON documents you can use to e.g. manage device settings.
+This package provides both a high-level framework via the `ShadowFramework` and direct access via the `ShadowClient` object.
+
+## Create a ShadowFramework
+
+The `ShadowFramework` object must first be created.
+
+```julia
+mqtt_connection = ... # create this yourself
+thing_name = "thing1"
+shadow_name = "settings"
+doc = Dict()
+sf = ShadowFramework(mqtt_connection, thing_name, shadow_name, doc)
+```
+
+### Using the ShadowFramework
+
+```julia
+subscribe(sf) # Subscribe to all the shadow service topics and perform any initial state updates
+
+lock(sf) do
+    doc["k1"] = "v1" # update our copy of the shadow document
+end
+
+publish_current_state(sf) # tell the shadow service about it
+```
+
+These updates go both ways. Shadow document updates from the service are received asynchronously and the local
+shadow document is updated automatically. The remote state is always reconciled with the local state.
+
+### Update Callbacks: Ordering and Other Behaviors
+
+If you need to take action before, during, or after a local shadow document update, there are callbacks available.
+
+```julia
+cbs = Dict(
+    # This callback will run whenever the key `foo` is updated. We can do whatever we want, including update the
+    # shadow document itself, but be careful about potential update order conflicts and deadlocks if you jump to
+    # another thread and then update the shadow document.
+    "foo" => it -> do_something(it)
+)
+sf = ShadowFramework(...; shadow_document_property_callbacks = cbs)
+```
+
+### Persisting the Shadow Document Locally
+
+The post-update callback is great for persisting the shadow document to the local disk:
+
+```julia
+doc = Dict()
+sf = ShadowFramework(
+    ...,
+    doc;
+    shadow_document_post_update_callback = doc -> write(shadow_path, serialize_shadow(doc))
+)
+```
+
+The function `serialize_shadow` is a modified `JSON` serializer which handles version numbers better:
+
+```julia
+import JSON.Serializations: CommonSerialization, StandardSerialization
+import JSON.Writer: StructuralContext, show_json
+
+# Custom serialization for shadow documents so that version numbers serialize cleanly
+struct ShadowSerialization <: CommonSerialization end
+
+function show_json(io::StructuralContext, ::ShadowSerialization, f::VersionNumber)
+    show_json(io, StandardSerialization(), string(f))
+end
+
+serialize_shadow(shadow) = sprint(show_json, ShadowSerialization(), shadow)
+
+deserialize_shadow(text) = JSON.parse(text)
+```
+
+The next time your application starts, it can initialize the shadow document with the value from `deserialize_shadow`.
+Pass that value in to the `ShadowFramework` when creating it and run `subscribe` as usual.
+
+### See Also
+
+- [AWS IoT Device Shadow Service Documentation](https://docs.aws.amazon.com/iot/latest/developerguide/iot-device-shadows.html)

docs/Project.toml

@@ -1,3 +1,3 @@
 [deps]
-Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 AWSCRT = "df31ea59-17a4-4ebd-9d69-4f45266dc2c7"
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"

docs/make.jl

@@ -1,24 +1,18 @@
 using AWSCRT
 using Documenter
 
-DocMeta.setdocmeta!(AWSCRT, :DocTestSetup, :(using AWSCRT); recursive=true)
+DocMeta.setdocmeta!(AWSCRT, :DocTestSetup, :(using AWSCRT); recursive = true)
 
 makedocs(;
-    modules=[AWSCRT],
-    authors="Octogonapus <[email protected]> and contributors",
-    repo="https://github.com/Octogonapus/AWSCRT.jl/blob/{commit}{path}#{line}",
-    sitename="AWSCRT.jl",
-    format=Documenter.HTML(;
-        prettyurls=get(ENV, "CI", "false") == "true",
-        canonical="https://Octogonapus.github.io/AWSCRT.jl",
-        assets=String[],
+    modules = [AWSCRT],
+    repo = "https://github.com/Octogonapus/AWSCRT.jl/blob/{commit}{path}#{line}",
+    sitename = "AWSCRT.jl",
+    format = Documenter.HTML(;
+        prettyurls = get(ENV, "CI", "false") == "true",
+        canonical = "https://Octogonapus.github.io/AWSCRT.jl",
+        assets = String[],
     ),
-    pages=[
-        "Home" => "index.md",
-    ],
+    pages = ["Home" => "index.md"],
 )
 
-deploydocs(;
-    repo="github.com/Octogonapus/AWSCRT.jl",
-    devbranch="main",
-)
+deploydocs(; repo = "github.com/Octogonapus/AWSCRT.jl", devbranch = "main")

src/AWSCRT.jl

@@ -1,9 +1,9 @@
 """
 Environment variables:
 
-  - `AWS_CRT_MEMORY_TRACING`: Set to `0`, `1`, or `2` to enable memory tracing. Default is off. See [`aws_mem_trace_level`](@ref).
+  - `AWS_CRT_MEMORY_TRACING`: Set to `0`, `1`, or `2` to enable memory tracing. Default is off. See `aws_mem_trace_level`.
   - `AWS_CRT_MEMORY_TRACING_FRAMES_PER_STACK`: Set the number of frames per stack for memory tracing. Default is the AWS library's default.
-  - `AWS_CRT_LOG_LEVEL`: Set to `0` through `6` to enable logging. Default is off. See [`aws_log_level`](@ref).
+  - `AWS_CRT_LOG_LEVEL`: Set to `0` through `6` to enable logging. Default is off. See [`aws_log_level`](https://octogonapus.github.io/LibAWSCRT.jl/dev/#LibAWSCRT.aws_log_level).
   - `AWS_CRT_LOG_PATH`: Set to the log file path. Must be set if `AWS_CRT_LOG_LEVEL` is set.
 
 Note: all the symbols in this package that begin with underscores are private and are not part of this package's published interface. Please don't use them.

src/AWSIO.jl

@@ -19,8 +19,9 @@ A collection of event-loops.
 An event-loop is a thread for doing async work, such as I/O.
 
 Arguments:
-- `num_threads (Union{Int,Nothing}) (default=nothing)`: Maximum number of event-loops to create. If unspecified, one is created for each processor on the machine.
-- `cpu_group (Union{Int,Nothing}) (default=nothing)`: Optional processor group to which all threads will be pinned. Useful for systems with non-uniform memory access (NUMA) nodes. If specified, the number of threads will be capped at the number of processors in the group.
+
+  - `num_threads (Union{Int,Nothing}) (default=nothing)`: Maximum number of event-loops to create. If unspecified, one is created for each processor on the machine.
+  - `cpu_group (Union{Int,Nothing}) (default=nothing)`: Optional processor group to which all threads will be pinned. Useful for systems with non-uniform memory access (NUMA) nodes. If specified, the number of threads will be capped at the number of processors in the group.
 """
 function EventLoopGroup(num_threads::Union{Int,Nothing} = nothing, cpu_group::Union{Int,Nothing} = nothing)
     if num_threads === nothing
@@ -78,8 +79,9 @@ end
 Default DNS host resolver.
 
 Arguments:
-- `el_group (EventLoopGroup)`: EventLoopGroup to use.
-- `max_hosts (Int) (default=16)`: Max host names to cache.
+
+  - `el_group (EventLoopGroup)`: EventLoopGroup to use.
+  - `max_hosts (Int) (default=16)`: Max host names to cache.
 """
 function HostResolver(el_group::EventLoopGroup, max_hosts::Int = 16)
     if max_hosts <= 0
@@ -127,8 +129,9 @@ end
 Handles creation and setup of client socket connections.
 
 Arguments:
-- `el_group (EventLoopGroup)`: EventLoopGroup to use.
-- `host_resolver (HostResolver)`: DNS host resolver to use.
+
+  - `el_group (EventLoopGroup)`: EventLoopGroup to use.
+  - `host_resolver (HostResolver)`: DNS host resolver to use.
 """
 function ClientBootstrap(el_group::EventLoopGroup, host_resolver::HostResolver)
     options = Ref(aws_client_bootstrap_options(el_group.ptr, host_resolver.ptr, C_NULL, C_NULL, C_NULL))
@@ -160,7 +163,7 @@ const extra_tls_kwargs_docs = """
 - `ca_dirpath (Union{String,Nothing}) (default=nothing)`: Path to directory containing trusted certificates, which will overrides the default trust store. Only supported on Unix.
 - `ca_filepath (Union{String,Nothing}) (default=nothing)`: Path to file containing PEM armored chain of trusted CA certificates.
 - `ca_data (Union{String,Nothing}) (default=nothing)`: PEM armored chain of trusted CA certificates.
-- `alpn_list (Union{Vector{String},Nothing}) (default=nothing)`: If set, names to use in Application Layer Protocol Negotiation (ALPN). ALPN is not supported on all systems, see [`aws_tls_is_alpn_available`](@ref). This can be customized per connection; see [`TLSConnectionOptions`](@ref).
+- `alpn_list (Union{Vector{String},Nothing}) (default=nothing)`: If set, names to use in Application Layer Protocol Negotiation (ALPN). ALPN is not supported on all systems, see [`aws_tls_is_alpn_available`](https://octogonapus.github.io/LibAWSCRT.jl/dev/#LibAWSCRT.aws_tls_is_alpn_available-Tuple{}). This can be customized per connection; see [`TLSConnectionOptions`](@ref).
 """
 
 """
@@ -296,7 +299,8 @@ A context is expensive, but can be used for the lifetime of the application by a
 use the same TLS configuration.
 
 Arguments:
-- `options (TLSContextOptions)`: Configuration options.
+
+  - `options (TLSContextOptions)`: Configuration options.
 """
 function ClientTLSContext(options::TLSContextOptions)
     tls_ctx_opt = Ref(aws_tls_ctx_options(ntuple(_ -> UInt8(0), 200)))
@@ -388,9 +392,10 @@ Connection-specific TLS options.
 Note that while a TLS context is an expensive object, this object is cheap.
 
 Arguments:
-- `client_tls_context (ClientTLSContext)`: TLS context. A context can be shared by many connections.
-- `alpn_list (Union{Vector{String},Nothing}) (default=nothing)`: Connection-specific Application Layer Protocol Negotiation (ALPN) list. This overrides any ALPN list on the TLS context in the client this connection was made with. ALPN is not supported on all systems, see [`aws_tls_is_alpn_available`](@ref).
-- `server_name (Union{String,Nothing}) (default=nothing)`: Name for TLS Server Name Indication (SNI). Also used for x.509 validation.
+
+  - `client_tls_context (ClientTLSContext)`: TLS context. A context can be shared by many connections.
+  - `alpn_list (Union{Vector{String},Nothing}) (default=nothing)`: Connection-specific Application Layer Protocol Negotiation (ALPN) list. This overrides any ALPN list on the TLS context in the client this connection was made with. ALPN is not supported on all systems, see [`aws_tls_is_alpn_available`](https://octogonapus.github.io/LibAWSCRT.jl/dev/#LibAWSCRT.aws_tls_is_alpn_available-Tuple%7B%7D).
+  - `server_name (Union{String,Nothing}) (default=nothing)`: Name for TLS Server Name Indication (SNI). Also used for x.509 validation.
 """
 function TLSConnectionOptions(
     client_tls_context::ClientTLSContext,

src/AWSMQTT.jl

@@ -269,7 +269,7 @@ Arguments:
   - `username (Union{String,Nothing}) (default=nothing)`: Username to connect with.
   - `password (Union{String,Nothing}) (default=nothing)`: Password to connect with.
   - `socket_options (Ref(aws_socket_options}) (default=Ref(aws_socket_options(AWS_SOCKET_STREAM, AWS_SOCKET_IPV6, 5000, 0, 0, 0, false)))`: Optional socket options.
-  - `alpn_list (Union{Vector{String},Nothing}) (default=nothing)`: Connection-specific Application Layer Protocol Negotiation (ALPN) list. This overrides any ALPN list on the TLS context in the client this connection was made with. ALPN is not supported on all systems, see [`aws_tls_is_alpn_available`](@ref).
+  - `alpn_list (Union{Vector{String},Nothing}) (default=nothing)`: Connection-specific Application Layer Protocol Negotiation (ALPN) list. This overrides any ALPN list on the TLS context in the client this connection was made with. ALPN is not supported on all systems, see [`aws_tls_is_alpn_available`](https://octogonapus.github.io/LibAWSCRT.jl/dev/#LibAWSCRT.aws_tls_is_alpn_available-Tuple%7B%7D).
   - `use_websockets (Bool) (default=false)`: # TODO
   - `websocket_handshake_transform (nothing) (default=nothing)`: # TODO
   - `proxy_options (nothing) (default=nothing)`: # TODO

test/runtests.jl

@@ -5,11 +5,17 @@ ENV["AWS_CRT_LOG_LEVEL"] = "6"
 ENV["AWS_CRT_LOG_PATH"] = joinpath(@__DIR__, "log.txt")
 ENV["JULIA_DEBUG"] = "AWSCRT"
 
-using Test, AWSCRT, AWSCRT.LibAWSCRT, JSON, CountDownLatches, Random
+using Test, AWSCRT, LibAWSCRT, JSON, CountDownLatches, Random, Documenter, Aqua
 
 include("util.jl")
 
 @testset "AWSCRT" begin
+    doctest(AWSCRT)
+    @testset "Aqua" begin
+        # TODO: see how this issue resolves and update https://github.com/JuliaTesting/Aqua.jl/issues/77#issuecomment-1166304846
+        Aqua.test_all(AWSCRT, ambiguities = false)
+        Aqua.test_ambiguities(AWSCRT)
+    end
     @testset "mqtt_test.jl" begin
         @info "Starting mqtt_test.jl"
         include("mqtt_test.jl")

test/shadow_framework_integ_test.jl

@@ -443,6 +443,7 @@ end
     end
 
     @testset "desired state that can't be reconciled with the local shadow doesn't cause excessive publishing" begin
+        @warn "Missing property errors are expected in this test"
         connection = new_mqtt_connection()
         shadow_name = random_shadow_name()
         doc = ShadowDocMissingProperty(1, 0)