From 96742a87990758faf46426feccaa6228f6c98670 Mon Sep 17 00:00:00 2001 From: Jim Salem Date: Wed, 5 Mar 2025 17:48:19 -0500 Subject: [PATCH] Updated to v1.8.0 (copied from Notehub) --- go.mod | 4 +- go.sum | 4 +- hub/discover.go | 10 ++- hub/event.go | 11 ++- hub/http.go | 6 +- hub/session.go | 5 +- lib/bulk.go | 7 ++ lib/clang/notehub.pb.c | 12 ++- lib/clang/notehub.pb.h | 16 ++-- lib/concurrency_test.go | 20 +++-- lib/debug.go | 36 ++++++++- lib/discover.go | 29 +++++-- lib/event.go | 135 ++++++++++++++++++++++++++++++- lib/event_test.go | 63 +++++++++++++++ lib/hubreq.go | 174 ++++++++++++++++++++++++++-------------- lib/note.go | 20 ++--- lib/notebox.go | 110 ++++++++++++++++--------- lib/notefile.go | 87 ++++++++++++++------ lib/notehub-defs.go | 140 +++++++++++++++++++------------- lib/notehub.pb.go | 143 +++++++++++++++++++-------------- lib/notehub.proto | 2 + lib/notelib.go | 7 ++ lib/req.go | 21 +++-- lib/wire.go | 139 ++++++++++++++++++++++++-------- 24 files changed, 862 insertions(+), 339 deletions(-) mode change 100644 => 100755 lib/bulk.go create mode 100644 lib/event_test.go diff --git a/go.mod b/go.mod index 2d6359b..dc973d1 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/blues/note -go 1.19 +go 1.22 require ( - github.com/blues/note-go v1.7.2 + github.com/blues/note-go v1.8.0 github.com/golang/snappy v0.0.4 github.com/google/open-location-code/go v0.0.0-20220120191843-cafb35c0d74d github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 65ae733..03dcd66 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/blues/note-go v1.7.2 h1:hasFsMNTnvyp5MPpnhqL46LC1tqzxQ5sGg+NOcJUV8E= -github.com/blues/note-go v1.7.2/go.mod h1:GfslvbmFus7z05P1YykcbMedTKTuDNTf8ryBb1Qjq/4= +github.com/blues/note-go v1.8.0 h1:7h9xVXREnFK0bT7xcYyXq19s1yPcFhZjrkerqFV0TLg= +github.com/blues/note-go v1.8.0/go.mod h1:GfslvbmFus7z05P1YykcbMedTKTuDNTf8ryBb1Qjq/4= github.com/creack/goselect v0.1.2 h1:2DNy14+JPjRBgPzAd1thbQp4BSIihxcBf0IXhQXDRa0= github.com/creack/goselect v0.1.2/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglDzQP3hKY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/hub/discover.go b/hub/discover.go index 61fc725..8c26750 100644 --- a/hub/discover.go +++ b/hub/discover.go @@ -13,7 +13,7 @@ import ( ) // NotehubDiscover is responsible for discovery of information about the services and apps -func NotehubDiscover(deviceUID string, deviceSN string, productUID string, hostname string, packetHandlerVersion string) (info notelib.DiscoverInfo, err error) { +func NotehubDiscover(deviceUID string, deviceSN string, productUID string, appUID string, needHandlerInfo bool, hostname string, packetHandlerVersion string) (info notelib.DiscoverInfo, err error) { // Return basic info about the server info.HubEndpointID = note.DefaultHubEndpointID info.HubTimeNs = time.Now().UnixNano() @@ -25,11 +25,13 @@ func NotehubDiscover(deviceUID string, deviceSN string, productUID string, hostn err = err2 return } - info.HubSessionHandler = device.Handler - info.HubSessionTicket = device.Ticket - info.HubSessionTicketExpiresTimeNs = device.TicketExpiresTimeSec * int64(1000000000) info.HubDeviceStorageObject = notelib.FileStorageObject(deviceUID) info.HubDeviceAppUID = device.AppUID + if needHandlerInfo { + info.HubSessionHandler = device.Handler + info.HubSessionTicket = device.Ticket + info.HubSessionTicketExpiresTimeNs = device.TicketExpiresTimeSec * int64(1000000000) + } } // Return the tcps issuer rootca cert, used for device-side certificate rotation diff --git a/hub/event.go b/hub/event.go index 0e992b2..725cb76 100644 --- a/hub/event.go +++ b/hub/event.go @@ -14,7 +14,6 @@ import ( "github.com/blues/note-go/note" notelib "github.com/blues/note/lib" golc "github.com/google/open-location-code/go" - "github.com/google/uuid" ) // Event log directory @@ -27,12 +26,12 @@ func eventLogInit(dir string) { } // Event handling procedure -func notehubEvent(ctx context.Context, session *notelib.HubSession, local bool, file *notelib.Notefile, event *note.Event) (err error) { +func notehubEvent(ctx context.Context, session *notelib.HubSession, local bool, event *note.Event) (err error) { // Retrieve the session context // If this is a queue and this is a template note, recursively expand it to multiple notifications if event.Bulk { - eventBulk(session, local, file, *event) + eventBulk(session, local, *event) return } @@ -69,7 +68,7 @@ func notehubEvent(ctx context.Context, session *notelib.HubSession, local bool, } // For bulk data, process the template and payload, generating recursive notifications -func eventBulk(session *notelib.HubSession, local bool, file *notelib.Notefile, event note.Event) (err error) { +func eventBulk(session *notelib.HubSession, local bool, event note.Event) (err error) { // Get the template from the note bodyJSON, err := note.JSONMarshal(event.Body) if err != nil { @@ -107,8 +106,8 @@ func eventBulk(session *notelib.HubSession, local bool, file *notelib.Notefile, nn.Bulk = false nn.Body = &body nn.Payload = payload - nn.EventUID = uuid.New().String() - notehubEvent(context.Background(), session, local, file, &nn) + nn.EventUID = notelib.GenerateEventUid(&nn) + notehubEvent(context.Background(), session, local, &nn) } diff --git a/hub/http.go b/hub/http.go index bec1ef7..2e3b528 100644 --- a/hub/http.go +++ b/hub/http.go @@ -37,7 +37,7 @@ func httpReqHandler(httpRsp http.ResponseWriter, httpReq *http.Request) { reqJSON, err = io.ReadAll(httpReq.Body) if err != nil { err = fmt.Errorf("please supply a JSON request in the HTTP body") - io.WriteString(httpRsp, string(notelib.ErrorResponse(err))) + httpRsp.Write(notelib.ErrorResponse(err)) return } @@ -64,7 +64,7 @@ func httpReqHandler(httpRsp http.ResponseWriter, httpReq *http.Request) { // Get the hub endpoint ID and storage object var hubEndpointID, deviceStorageObject string if err == nil { - _, hubEndpointID, _, deviceStorageObject, err = notelib.HubDiscover(deviceUID, "", device.ProductUID) + hubEndpointID, _, deviceStorageObject, err = notelib.HubDiscover(deviceUID, "", device.ProductUID, appUID) } // Process the request @@ -73,7 +73,7 @@ func httpReqHandler(httpRsp http.ResponseWriter, httpReq *http.Request) { var box *notelib.Notebox box, err = notelib.OpenEndpointNotebox(ctx, hubEndpointID, deviceStorageObject, false) if err == nil { - box.SetEventInfo(deviceUID, device.DeviceSN, device.ProductUID, appUID, notehubEvent, nil) + box.SetEventInfo(deviceUID, device.DeviceSN, device.ProductUID, appUID, "api:http", notehubEvent, nil) rspJSON = box.Request(ctx, hubEndpointID, reqJSON) box.Close(ctx) } diff --git a/hub/session.go b/hub/session.go index 7d2d626..8431406 100644 --- a/hub/session.go +++ b/hub/session.go @@ -11,7 +11,6 @@ import ( "net" notelib "github.com/blues/note/lib" - "github.com/google/uuid" ) // Process requests for the duration of a session being open @@ -28,7 +27,7 @@ func sessionHandler(connSession net.Conn, secure bool) { } // Always start with a blank, inactive Session Context - sessionContext := notelib.NewHubSession(uuid.New().String(), sessionSource, secure) + sessionContext := notelib.NewHubSession(sessionSource, secure) // Set up golang context with session context stored within ctx := context.Background() @@ -39,7 +38,7 @@ func sessionHandler(connSession net.Conn, secure bool) { firstTransaction := sessionContext.Transactions == 0 // Extract a request from the wire, and exit if error - _, request, err = notelib.WireReadRequest(connSession, true) + _, request, err = notelib.WireReadRequest(ctx, connSession, true) if err != nil { if !notelib.ErrorContains(err, "{closed}") { fmt.Printf("session: error reading request: %s\n", err) diff --git a/lib/bulk.go b/lib/bulk.go old mode 100644 new mode 100755 index 10723e8..49b2d92 --- a/lib/bulk.go +++ b/lib/bulk.go @@ -608,6 +608,13 @@ func (tmplContext *BulkTemplateContext) BulkDecodeNextEntry() (body map[string]i } } + // Perform special processing of the body to remove a field added in the + // Notecard repo commit c29ba90d94687a442f3ed4e170372fb98f2200c9 wherein + // lora templates needed to be unique. This manifested itself as + // a field whose name is "_" being added to the template, which, + // if "noOmitEmpty", would actually show up to the user. + delete(body, "_") + // Exit if we'd encountered underrun if tmplContext.BinUnderrun { logDebug(context.Background(), "bin: bin underrun") diff --git a/lib/clang/notehub.pb.c b/lib/clang/notehub.pb.c index 6772a6d..ad4c97c 100644 --- a/lib/clang/notehub.pb.c +++ b/lib/clang/notehub.pb.c @@ -1,5 +1,5 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.3.5 at Mon Sep 11 17:31:55 2023. */ +/* Generated by nanopb-0.3.5 at Fri Sep 13 06:53:51 2024. */ #include "notehub.pb.h" @@ -10,7 +10,7 @@ -const pb_field_t notelib_NotehubPB_fields[60] = { +const pb_field_t notelib_NotehubPB_fields[62] = { PB_FIELD( 1, INT64 , OPTIONAL, STATIC , FIRST, notelib_NotehubPB, Version, Version, 0), PB_FIELD( 2, STRING , OPTIONAL, STATIC , OTHER, notelib_NotehubPB, MessageType, Version, 0), PB_FIELD( 3, STRING , OPTIONAL, STATIC , OTHER, notelib_NotehubPB, Error, MessageType, 0), @@ -70,8 +70,16 @@ const pb_field_t notelib_NotehubPB_fields[60] = { PB_FIELD( 57, STRING , OPTIONAL, STATIC , OTHER, notelib_NotehubPB, Where, SuppressResponse, 0), PB_FIELD( 58, INT64 , OPTIONAL, STATIC , OTHER, notelib_NotehubPB, WhereWhen, Where, 0), PB_FIELD( 59, STRING , OPTIONAL, STATIC , OTHER, notelib_NotehubPB, HubPacketHandler, WhereWhen, 0), + PB_FIELD( 60, UINT32 , OPTIONAL, STATIC , OTHER, notelib_NotehubPB, PowerSource, HubPacketHandler, 0), + PB_FIELD( 61, DOUBLE , OPTIONAL, STATIC , OTHER, notelib_NotehubPB, PowerMahUsed, PowerSource, 0), PB_LAST_FIELD }; +/* On some platforms (such as AVR), double is really float. + * These are not directly supported by nanopb, but see example_avr_double. + * To get rid of this error, remove any double fields from your .proto. + */ +PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES) + /* @@protoc_insertion_point(eof) */ diff --git a/lib/clang/notehub.pb.h b/lib/clang/notehub.pb.h index 446af4c..a94e10b 100644 --- a/lib/clang/notehub.pb.h +++ b/lib/clang/notehub.pb.h @@ -1,5 +1,5 @@ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.3.5 at Mon Sep 11 17:31:55 2023. */ +/* Generated by nanopb-0.3.5 at Fri Sep 13 06:53:51 2024. */ #ifndef PB_NOTEHUB_PB_H_INCLUDED #define PB_NOTEHUB_PB_H_INCLUDED @@ -134,14 +134,18 @@ typedef struct _notelib_NotehubPB { int64_t WhereWhen; bool has_HubPacketHandler; char HubPacketHandler[254]; + bool has_PowerSource; + uint32_t PowerSource; + bool has_PowerMahUsed; + double PowerMahUsed; /* @@protoc_insertion_point(struct:notelib_NotehubPB) */ } notelib_NotehubPB; /* Default values for struct fields */ /* Initializer values for message structs */ -#define notelib_NotehubPB_init_default {false, 0, false, "", false, "", false, "", false, "", false, 0, false, "", false, "", false, "", false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, "", false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, "", false, 0, false, ""} -#define notelib_NotehubPB_init_zero {false, 0, false, "", false, "", false, "", false, "", false, 0, false, "", false, "", false, "", false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, "", false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, "", false, 0, false, ""} +#define notelib_NotehubPB_init_default {false, 0, false, "", false, "", false, "", false, "", false, 0, false, "", false, "", false, "", false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, "", false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, "", false, 0, false, "", false, 0, false, 0} +#define notelib_NotehubPB_init_zero {false, 0, false, "", false, "", false, "", false, "", false, 0, false, "", false, "", false, "", false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, "", false, 0, false, 0, false, "", false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, 0, false, "", false, "", false, 0, false, 0, false, 0, false, "", false, 0, false, "", false, 0, false, 0} /* Field tags (for use in manual encoding/decoding) */ #define notelib_NotehubPB_Version_tag 1 @@ -203,12 +207,14 @@ typedef struct _notelib_NotehubPB { #define notelib_NotehubPB_Where_tag 57 #define notelib_NotehubPB_WhereWhen_tag 58 #define notelib_NotehubPB_HubPacketHandler_tag 59 +#define notelib_NotehubPB_PowerSource_tag 60 +#define notelib_NotehubPB_PowerMahUsed_tag 61 /* Struct field encoding specification for nanopb */ -extern const pb_field_t notelib_NotehubPB_fields[60]; +extern const pb_field_t notelib_NotehubPB_fields[62]; /* Maximum encoded size of messages (where known) */ -#define notelib_NotehubPB_size 4237 +#define notelib_NotehubPB_size 4254 /* Message IDs (where set with "msgid" option) */ #ifdef PB_MSGID diff --git a/lib/concurrency_test.go b/lib/concurrency_test.go index d49c629..882d0a7 100644 --- a/lib/concurrency_test.go +++ b/lib/concurrency_test.go @@ -87,7 +87,7 @@ func TestConcurrency(t *testing.T) { } -func parallelTest(t *testing.T, testName string, summarizeWork bool, workInstances int, workFn func(testid string) bool) { +func parallelTest(t *testing.T, testName string, summarizeWork bool, workInstances int, workFn func(ctx context.Context, testid string) bool) { ctx := context.Background() // Start the tests and notify the caller that we've started them @@ -130,7 +130,7 @@ func parallelTest(t *testing.T, testName string, summarizeWork bool, workInstanc } // Perform work in parallel -func parallelWork(workFn func(testid string) bool, summarizeWork bool, pActive *int, iterations *uint64, errors *uint64) { +func parallelWork(workFn func(ctx context.Context, testid string) bool, summarizeWork bool, pActive *int, iterations *uint64, errors *uint64) { ctx := context.Background() // Notify the callers that we've started, and wait for 'go' @@ -147,7 +147,7 @@ func parallelWork(workFn func(testid string) bool, summarizeWork bool, pActive * localIterations := 0 testid := fmt.Sprintf("concurrency_test_%04d", id) for !stopTests { - if !workFn(testid) { + if !workFn(ctx, testid) { atomic.AddUint64(errors, 1) } else { atomic.AddUint64(iterations, 1) @@ -168,9 +168,7 @@ func parallelWork(workFn func(testid string) bool, summarizeWork bool, pActive * } // Perform a single notebox test -func testNotebox(testid string) bool { - ctx := context.Background() - +func testNotebox(ctx context.Context, testid string) bool { box, err := OpenEndpointNotebox(ctx, "", FileStorageObject(testid), true) if err != nil { logDebug(ctx, "boxOpen: %s", err) @@ -207,7 +205,7 @@ func testNotebox(testid string) bool { } for i := 0; i < 100; i++ { - err := nf.DeleteNote("", fmt.Sprintf("note%d", i)) + err := nf.DeleteNote(ctx, "", fmt.Sprintf("note%d", i)) if err != nil { return false } @@ -245,7 +243,7 @@ func testNotebox(testid string) bool { } // Perform a single notefile test -func testNotefile(testid string) bool { +func testNotefile(ctx context.Context, testid string) bool { nf := CreateNotefile(false) for i := 0; i < 100; i++ { @@ -253,13 +251,13 @@ func testNotefile(testid string) bool { if err != nil { return false } - _, err = nf.AddNote("", fmt.Sprintf("note%d", i), n) + _, err = nf.AddNote(ctx, "", fmt.Sprintf("note%d", i), n) if err != nil { return false } } for i := 0; i < 100; i++ { - err := nf.DeleteNote("", fmt.Sprintf("note%d", i)) + err := nf.DeleteNote(ctx, "", fmt.Sprintf("note%d", i)) if err != nil { return false } @@ -267,7 +265,7 @@ func testNotefile(testid string) bool { if err != nil { return false } - _, err = nf.AddNote("", "", n) + _, err = nf.AddNote(ctx, "", "", n) if err != nil { return false } diff --git a/lib/debug.go b/lib/debug.go index 3ac9ef9..458dd5d 100644 --- a/lib/debug.go +++ b/lib/debug.go @@ -12,12 +12,13 @@ import ( "runtime" "strconv" "strings" + "time" ) // Debugging details var ( synchronous = true - debugEvent = true + DebugEvent = true debugBox = false debugSync = false debugSyncMax = false @@ -31,7 +32,7 @@ var ( ) var vars = [...]*bool{ - &debugEvent, &debugBox, &debugSync, &debugSyncMax, &debugCompress, &debugHubRequest, &debugRequest, &debugFile, + &DebugEvent, &debugBox, &debugSync, &debugSyncMax, &debugCompress, &debugHubRequest, &debugRequest, &debugFile, } var debugEnvInitialized = false @@ -207,3 +208,34 @@ func loggerHandler() { runtime.Gosched() } } + +// Default timewarn period used for suppressing LogInfo entirely +const LogPeriodSuppressSecs = 2 + +// PeriodicInfo is a struct that enables information to be displayed only periodically, +// for 'sampled' tracing purposes, rather than on a continuous basis +type LogPeriod struct { + LastLog time.Time + PeriodSecs int +} + +// LogInfoPeriodReset ensures that the very next period log does a trace +func LogInfoPeriodReset(period *LogPeriod) { + period.LastLog = time.Time{} +} + +// LogInfoPeriodically writes an unstructured log message with the INFO level but only periodically +func LogInfoPeriodically(ctx context.Context, period *LogPeriod, msg string, v ...interface{}) { + if !period.LastLog.IsZero() && time.Since(period.LastLog) < time.Duration(period.PeriodSecs)*time.Second { + return + } + logInfo(ctx, msg, v...) + period.LastLog = time.Now() +} + +// LogInfoBumpTrace bumps a trace interval +func LogInfoBumpTrace(prev *time.Time) (duration time.Duration) { + duration = time.Since(*prev) + *prev = time.Now() + return +} diff --git a/lib/discover.go b/lib/discover.go index 2b924be..7a1d8fc 100644 --- a/lib/discover.go +++ b/lib/discover.go @@ -23,7 +23,7 @@ type DiscoverInfo struct { } // DiscoverFunc is the func to retrieve discovery info for this server -type DiscoverFunc func(edgeUID string, deviceSN string, productUID string, hostname string, packetHandlerVersion string) (info DiscoverInfo, err error) +type DiscoverFunc func(edgeUID string, deviceSN string, productUID string, appUID string, needHandlerInfo bool, hostname string, packetHandlerVersion string) (info DiscoverInfo, err error) var fnDiscover DiscoverFunc @@ -37,23 +37,40 @@ func HubSetDiscover(fn DiscoverFunc) { } // HubDiscover ensures that we've read the local server's discover info, and return the Hub's Endpoint ID -func HubDiscover(deviceUID string, deviceSN string, productUID string) (hubSessionTicket string, hubEndpointID string, appUID string, deviceStorageObject string, err error) { +func HubDiscover(deviceUID string, deviceSN string, productUID string, appUID string) (hubEndpointID string, retAppUID string, deviceStorageObject string, err error) { if fnDiscover == nil { err = fmt.Errorf("no discovery function is available") return } // Call the discover func with the null edge UID just to get basic server info - discinfo, err := fnDiscover(deviceUID, deviceSN, productUID, "*", "") + discinfo, err := fnDiscover(deviceUID, deviceSN, productUID, appUID, false, "*", "") if err != nil { err = fmt.Errorf("error from discovery handler for %s: %s", deviceUID, err) return } - return discinfo.HubSessionTicket, discinfo.HubEndpointID, discinfo.HubDeviceAppUID, discinfo.HubDeviceStorageObject, nil + return discinfo.HubEndpointID, discinfo.HubDeviceAppUID, discinfo.HubDeviceStorageObject, nil } -// HubDiscover calls the discover function, and return discovery info +// HubDiscoverSessionTicket gets the session ticket for a session +func HubDiscoverSessionTicket(deviceUID string, deviceSN string, productUID string, appUID string) (hubSessionTicket string, err error) { + if fnDiscover == nil { + err = fmt.Errorf("no discovery function is available") + return + } + + // Call the discover func with the null edge UID just to get basic server info + discinfo, err := fnDiscover(deviceUID, deviceSN, productUID, appUID, true, "*", "") + if err != nil { + err = fmt.Errorf("error from discovery handler for %s: %s", deviceUID, err) + return + } + + return discinfo.HubSessionTicket, nil +} + +// HubProcessDiscoveryRequest calls the discover function, and return discovery info func hubProcessDiscoveryRequest(deviceUID string, deviceSN string, productUID string, hostname string, packetHandlerVersion string) (info DiscoverInfo, err error) { if fnDiscover == nil { err = fmt.Errorf("no discovery function is available") @@ -61,7 +78,7 @@ func hubProcessDiscoveryRequest(deviceUID string, deviceSN string, productUID st } // Call the discover func - info, err = fnDiscover(deviceUID, deviceSN, productUID, hostname, packetHandlerVersion) + info, err = fnDiscover(deviceUID, deviceSN, productUID, "", true, hostname, packetHandlerVersion) if err != nil { err = fmt.Errorf("error from discovery handler for %s: %s", deviceUID, err) return diff --git a/lib/event.go b/lib/event.go index 09265a6..74c4058 100644 --- a/lib/event.go +++ b/lib/event.go @@ -5,10 +5,143 @@ package notelib import ( + "bytes" "context" + "crypto/md5" + "encoding/binary" + "encoding/json" + "fmt" + "hash" + "sort" "github.com/blues/note-go/note" + "github.com/google/uuid" ) // EventFunc is the func to get called whenever there is a note add/update/delete -type EventFunc func(ctx context.Context, sess *HubSession, local bool, file *Notefile, data *note.Event) (err error) +type EventFunc func(ctx context.Context, sess *HubSession, local bool, data *note.Event) (err error) + +// Generate a UUID for an event in a way that will allow detection of duplicates of "user data" +func GenerateEventUid(event *note.Event) string { + + // If the captured date is unknown, we can't make any assumptions about duplication + if event == nil || event.When == 0 { + return uuid.New().String() + } + + // Create a new MD5 hash instance + var h hash.Hash = md5.New() + + // Hash deviceUID so we don't conflict with another device in the app + h.Write([]byte(event.DeviceUID)) + + // Convert the 'when' to a byte slice and hash it + whenBytes := make([]byte, 8) + binary.LittleEndian.PutUint64(whenBytes, uint64(event.When)) + h.Write(whenBytes) + + // Hash notefileID and noteID + h.Write([]byte(event.NotefileID)) + h.Write([]byte(event.NoteID)) + + // If body is available, hash it + if event.Body != nil { + h.Write(marshalSortedJSON(*event.Body)) + } + + // Hash the payload + h.Write(event.Payload) + + // If details is available, hash it + if event.Details != nil { + h.Write(marshalSortedJSON(*event.Details)) + } + + // Copy the MD5 checksum intoa 16-byte array for UUID conversion + var array16 [16]byte + copy(array16[:], h.Sum(nil)) + + // Convert to a compliant UUID + return makeCustomUuid(array16) + +} + +// Generate a cusstom (type 8) UUID from a 16-byte array and return it as a formatted string +// https://www.ietf.org/archive/id/draft-peabody-dispatch-new-uuid-format-04.html#name-uuid-version-8 +func makeCustomUuid(input [16]byte) string { + uuid := input // Copy the entire input into the uuid array + + // Set the version (type 8 UUID, which is 0b1000) + uuid[6] = (uuid[6] & 0x0F) | 0x80 + + // Set the variant (the two most significant bits: 0b10 for RFC 4122) + uuid[8] = (uuid[8] & 0x3F) | 0x80 + + // Format the UUID as 8-4-4-4-12 directly with %x + return fmt.Sprintf("%x-%x-%x-%x-%x", + uuid[0:4], // First 4 bytes (8 hex characters) + uuid[4:6], // Next 2 bytes (4 hex characters) + uuid[6:8], // Next 2 bytes (4 hex characters) + uuid[8:10], // Next 2 bytes (4 hex characters) + uuid[10:16]) // Final 6 bytes (12 hex characters) +} + +// This method recursively sorts JSON fields and marshals them. Thie purpose of +// this method is to account for the fact that golang maps always come back in an +// intentionally-arbitrary order, and for hashing we need them in a deterministic order. +func marshalSortedJSON(jsonObject interface{}) []byte { + + switch v := jsonObject.(type) { + + case map[string]interface{}: + + // Sort map keys + keys := make([]string, 0, len(v)) + for key := range v { + keys = append(keys, key) + } + sort.Strings(keys) + + // Create a buffer to hold the marshaled JSON + var buffer bytes.Buffer + buffer.WriteString("{") + + // Recursively marshal the value + for i, key := range keys { + valueBytes := marshalSortedJSON(v[key]) + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, valueBytes)) + } + + buffer.WriteString("}") + return buffer.Bytes() + + case []interface{}: + + // Handle JSON arrays + var buffer bytes.Buffer + buffer.WriteString("[") + + // Recursively marshal the elements + for i, elem := range v { + elemBytes := marshalSortedJSON(elem) + if i > 0 { + buffer.WriteString(",") + } + buffer.Write(elemBytes) + } + + buffer.WriteString("]") + return buffer.Bytes() + + default: + j, err := json.Marshal(v) + if err != nil { + return []byte{} + } + return j + } + +} diff --git a/lib/event_test.go b/lib/event_test.go new file mode 100644 index 0000000..9da1201 --- /dev/null +++ b/lib/event_test.go @@ -0,0 +1,63 @@ +package notelib + +import ( + "testing" + + "github.com/blues/note-go/note" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func isValidUUID(u string) bool { + _, err := uuid.Parse(u) + return err == nil +} + +func TestGenerateEventUid(t *testing.T) { + // nil events or events without a When field + // always generate a UUID + e := note.Event{} + a := GenerateEventUid(nil) + b := GenerateEventUid(&e) + c := GenerateEventUid(&e) + require.True(t, isValidUUID(a)) + require.True(t, isValidUUID(b)) + require.True(t, isValidUUID(c)) + require.NotEqual(t, a, b) + require.NotEqual(t, b, c) + + // an event with When field set should always + // generate a deterministic UID which will look + // something like this: + // + // 33cdeccc-cebe-8032-9f1f-dbee7f5874cb + e = note.Event{When: 1} + b = GenerateEventUid(&e) + c = GenerateEventUid(&e) + require.Equal(t, b, c) + require.Equal(t, b, "33cdeccc-cebe-8032-9f1f-dbee7f5874cb") + + // The `When`, `Body`, `Payload` and `Details` fields + // are all used to construct the UID + e = note.Event{ + When: 1, + Body: &map[string]interface{}{ + "key": "value", + "obj": map[string]int{ + "a": 1, + "b": 2, + }, + "list": []string{"a", "b", "c"}, + }, + Payload: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, + Details: &map[string]interface{}{ + "key": 1, + "key2": "value2", + }, + } + + b = GenerateEventUid(&e) + c = GenerateEventUid(&e) + require.Equal(t, b, c) + require.Equal(t, b, "edd13f0c-b9b8-836c-8c1c-631230e21e56") +} diff --git a/lib/hubreq.go b/lib/hubreq.go index 1885cbc..06f31e7 100644 --- a/lib/hubreq.go +++ b/lib/hubreq.go @@ -11,6 +11,7 @@ import ( "fmt" "hash/crc32" "net/http" + "regexp" "strconv" "strings" "time" @@ -23,6 +24,9 @@ import ( // Flag indicating TLS support on this server var serverSupportsTLS bool +// Maximum field length in our protobufs if null were included in the cstring (see notehub.options) +const maxProtobufStringLength = 253 + // NoteboxUpdateEnvFunc is the func to update environment variables for a device type NoteboxUpdateEnvFunc func(ctx context.Context, box *Notebox, deviceUID string, appUID string, updateFeeds bool) (err error) @@ -98,6 +102,7 @@ func HubCheckpointRequest() (message []byte, err error) { // - if err is returned, do not return a reply to the remote requestor // - if err is nil, always send result back to request func HubRequest(ctx context.Context, content []byte, event EventFunc, session *HubSession) (reqtype string, result []byte, suppress bool, err error) { + // Preset in case of error return var rsp notehubMessage rsp.Version = currentProtocolVersion @@ -116,10 +121,24 @@ func HubRequest(ctx context.Context, content []byte, event EventFunc, session *H return } + // Determine if the message type indicates that we want to suppress response + if strings.HasPrefix(req.MessageType, msgSuppressResponsePrefix) { + req.SuppressResponse = true + req.MessageType = strings.TrimPrefix(req.MessageType, msgSuppressResponsePrefix) + } + suppress = req.SuppressResponse reqtype = msgTypeName(req.MessageType) + + // Warn if transactions take too long reqStart := time.Now() + defer func() { + if time.Since(reqStart) > transactionErrorDuration && req.MessageType != msgGetNotification { + session.LogWarn(ctx, "%s %s [timewarn] excessively long request: %s", req.NotefileID, reqtype, time.Since(reqStart)) + } + }() + // Extract key info from the request filteredMsg := filterForLog(req) filteredJSON, _ := note.JSONMarshal(filteredMsg) session.LogInfo(ctx, "Request #%d (%db wire) %s %s", session.Transactions, wirelen, reqtype, filteredJSON) @@ -129,7 +148,7 @@ func HubRequest(ctx context.Context, content []byte, event EventFunc, session *H // Ensure that the device is valid. (If it had to be provisioned it would've been before // we ever arrived here.) - sessionTicket, _, _, _, err2 := HubDiscover(req.DeviceUID, req.DeviceSN, req.ProductUID) + sessionTicket, err2 := HubDiscoverSessionTicket(req.DeviceUID, req.DeviceSN, req.ProductUID, session.AppUID) if err2 != nil { err = fmt.Errorf("discover error "+note.ErrAuth+": %s", err2) } else { @@ -198,7 +217,7 @@ func HubRequest(ctx context.Context, content []byte, event EventFunc, session *H // After the first request has been successfully executed, mark it as active, and piggyback the session info if err != nil { - rsp.Error = fmt.Sprintf("%s", err) + rsp.Error = TruncateErrorString(err.Error(), maxProtobufStringLength) err = nil } else if !session.Active { @@ -245,6 +264,7 @@ func HubRequest(ctx context.Context, content []byte, event EventFunc, session *H // Convert back to on-wire format result, wirelen, err = msgToWire(rsp) if err != nil { + logError(ctx, "msgToWire: %s", err) return } @@ -271,7 +291,7 @@ func HubErrorResponse(ctx context.Context, session *HubSession, errorMessage str // Create a response message rsp.Version = currentProtocolVersion - rsp.Error = errorMessage + rsp.Error = TruncateErrorString(errorMessage, maxProtobufStringLength) // Convert to on-wire format response, wirelen, err := msgToWire(rsp) @@ -365,7 +385,7 @@ func processRequest(ctx context.Context, session *HubSession, req notehubMessage // If an error, return it this way if err != nil { - rsp.Error = fmt.Sprintf("%s", err) + rsp.Error = TruncateErrorString(err.Error(), maxProtobufStringLength) } return rsp @@ -373,9 +393,18 @@ func processRequest(ctx context.Context, session *HubSession, req notehubMessage // Open the endpoint's notebox func openHubNoteboxForDevice(ctx context.Context, session *HubSession, deviceUID string, deviceSN string, productUID string, endpointID string, event EventFunc) (box *Notebox, appUID string, err error) { + + // Warn if this take too long; it should be a few milliseconds + start := time.Now() + defer func() { + if time.Since(start) > time.Second { + session.LogWarn(ctx, "[timewarn] excessively long time to open the notebox: %s", time.Since(start)) + } + }() + // Ensure that hub endpoint is available var hubEndpointID, deviceStorageObject string - _, hubEndpointID, appUID, deviceStorageObject, err = HubDiscover(deviceUID, deviceSN, productUID) + hubEndpointID, appUID, deviceStorageObject, err = HubDiscover(deviceUID, deviceSN, productUID, appUID) if err != nil { return } @@ -387,7 +416,7 @@ func openHubNoteboxForDevice(ctx context.Context, session *HubSession, deviceUID } // Set the default notification context for files opened within this box instance - box.SetEventInfo(deviceUID, deviceSN, productUID, appUID, event, session) + box.SetEventInfo(deviceUID, deviceSN, productUID, appUID, session.Session.Transport, event, session) // If we haven't yet enum'ed the notefiles, do so sawNotefile(session, box, "") @@ -446,7 +475,8 @@ func hubWebRequest(ctx context.Context, session *HubSession, req notehubMessage, } reqtype := req.NotefileID if reqtype == "" { - return fmt.Errorf("web request requires a method") + // Added 2024-12-13 to optimize on-wire traffic for HAAS + reqtype = "POST" } reqcontent := req.MotionOrientation reqmaxbytes := int(req.MotionSecs) @@ -495,30 +525,51 @@ func hubWebRequest(ctx context.Context, session *HubSession, req notehubMessage, } } - // If a segmented payload upload has been requested (as indicated by HighPowerSecsTotal in wire protocol) + // A segmented payload upload request is indicated by totalPayloadLen being specified if totalPayloadLen == 0 { - // If not requesting a segmented upload, clear any pending retained payload + + // We're not requesting a segmented upload, so clear any pending retained payload session.PendingWebPayload = []byte{} - } else if len(snote.Payload) > 0 { - // It would be absurd for a developer to be sending this much data, but in the - // spirit of defensive coding let's make sure they don't do crazy things. + } else { + + // For robustness in segmented payload processing, we follow these strict rules: + // 1. Allow a 0 length writes as a way just to truncate lengths and proceed with a pending upload + // 2. If someone tries to upload more than the totalPayloadLen, FIX the indicated totalPayloadLen + // just to be forgiving to the developer + // 3. If there would be a gap in the uploaded data becase offset is > pending length, it's an error + // 4. If the offset is <= pending length, just truncate it because we may simply be retrying an upload + // 5. If the pending length is > an updated totalPayloadLen, just truncate pending length and proceed + // because this is a valid way to 'terminate' a segmented payload upload being appended-to + + // (2) above + if payloadOffset+len(snote.Payload) > totalPayloadLen { + totalPayloadLen = payloadOffset + len(snote.Payload) + } + + // Enforce the maximum amount we allow for segmented payloads. Note that there's nothing + // special about this number except to keep memory usage in check. If a customer needs more, + // within reason, we could certainly consider it and update this number as it really is + // just temporary storage that should be freed pretty quickly in practice. maxPayloadLen := 10000000 if totalPayloadLen > maxPayloadLen { session.PendingWebPayload = []byte{} return fmt.Errorf("segmented payloads are limited to %d bytes (%d requested) ", maxPayloadLen, totalPayloadLen) } - // If this is the first segment, clear out the pending payload - if payloadOffset == 0 { - session.PendingWebPayload = []byte{} + // (3) above (making sure we only do this iff we actually uploaded data to be appended) + if len(snote.Payload) > 0 && payloadOffset > len(session.PendingWebPayload) { + return fmt.Errorf("segmented payloads must be uploaded without gaps (%d already uploaded but offset is %d) "+note.ErrWebPayload, len(session.PendingWebPayload), payloadOffset) } - // Validate that the payload segments are arriving in-order as indicated by offset - if payloadOffset != len(session.PendingWebPayload) { - actual := len(session.PendingWebPayload) - session.PendingWebPayload = []byte{} - return fmt.Errorf("segmented payloads must be uploaded in exact order (%d already uploaded but offset is %d) "+note.ErrWebPayload, actual, payloadOffset) + // (4) above (making sure we only do this iff we actually uploaded data to be appended) + if len(snote.Payload) > 0 && payloadOffset < len(session.PendingWebPayload) { + session.PendingWebPayload = session.PendingWebPayload[:payloadOffset] + } + + // (5) above + if len(session.PendingWebPayload) > totalPayloadLen { + session.PendingWebPayload = session.PendingWebPayload[:totalPayloadLen] } // Append this chunk to the pending web payload @@ -529,7 +580,7 @@ func hubWebRequest(ctx context.Context, session *HubSession, req notehubMessage, rsp.MaxChanges = http.StatusContinue notefile = CreateNotefile(false) snote, _ = note.CreateNote(nil, nil) - _, err = notefile.AddNote(req.DeviceEndpointID, specialNoteID, snote) + _, err = notefile.AddNote(ctx, req.DeviceEndpointID, specialNoteID, snote) if err != nil { err = fmt.Errorf("addNote error when pending payload: %w", err) return @@ -542,13 +593,6 @@ func hubWebRequest(ctx context.Context, session *HubSession, req notehubMessage, return } - // If we've received too much, indicate so - if len(session.PendingWebPayload) > totalPayloadLen { - actual := len(session.PendingWebPayload) - session.PendingWebPayload = []byte{} - return fmt.Errorf("too much total segmented data received (%d actual, %d expected) "+note.ErrWebPayload, actual, totalPayloadLen) - } - // We've completed assembling the payload, so proceed with the web request snote.Payload = session.PendingWebPayload session.PendingWebPayload = []byte{} @@ -570,7 +614,7 @@ func hubWebRequest(ctx context.Context, session *HubSession, req notehubMessage, // continue processing the transaction. statuscode, rspHeader, rspBodyJSON, rspPayload, err := fnHubWebRequest(ctx, session, req.DeviceUID, req.ProductUID, alias, reqtype, reqcontent, reqoffset, reqmaxbytes, target, snote.GetBody(), snote.Payload) if err != nil { - rsp.Error = fmt.Sprintf("%s", err) + rsp.Error = TruncateErrorString(err.Error(), maxProtobufStringLength) } // If there's a payload, return its MD5 to the caller so the device can check for corruption @@ -611,7 +655,7 @@ func hubWebRequest(ctx context.Context, session *HubSession, req notehubMessage, } // Add the note to the notefile and add it to the response message - _, err = notefile.AddNote(req.DeviceEndpointID, specialNoteID, snote) + _, err = notefile.AddNote(ctx, req.DeviceEndpointID, specialNoteID, snote) if err != nil { return err } @@ -674,7 +718,7 @@ func hubGetNotification(ctx context.Context, session *HubSession, msg notehubMes snote, err = note.CreateNote(nil, payload) if err == nil { notefile := CreateNotefile(false) - _, err = notefile.AddNote(msg.DeviceEndpointID, "signal", snote) + _, err = notefile.AddNote(ctx, msg.DeviceEndpointID, "signal", snote) if err == nil { err = rsp.SetNotefile(notefile) } @@ -718,7 +762,7 @@ func hubDiscovery(ctx context.Context, session *HubSession, msg notehubMessage, xnote, err := note.CreateNote(nil, discinfo.HubCert) if err == nil { noteID := "result" - _, err = newNotefile.AddNote(discinfo.HubEndpointID, noteID, xnote) + _, err = newNotefile.AddNote(ctx, discinfo.HubEndpointID, noteID, xnote) if err == nil { if rsp.SetNotefile(newNotefile) == nil { rsp.NoteID = noteID @@ -863,6 +907,7 @@ func hubNotefileChanges(ctx context.Context, session *HubSession, req notehubMes // Notefile Merge func hubNotefileMerge(ctx context.Context, session *HubSession, req notehubMessage, rsp *notehubMessage, event EventFunc) (err error) { + // Open the box box, appUID, err2 := openHubNoteboxForDevice(ctx, session, req.DeviceUID, req.DeviceSN, req.ProductUID, req.DeviceEndpointID, event) if err2 != nil { @@ -880,7 +925,7 @@ func hubNotefileMerge(ctx context.Context, session *HubSession, req notehubMessa sawNotefile(session, box, req.NotefileID) // Set the notification context - file.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, event, session) + file.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, session.Session.Transport, event, session) // Unmarshal the notefile notefile, err5 := req.GetNotefile() @@ -892,7 +937,7 @@ func hubNotefileMerge(ctx context.Context, session *HubSession, req notehubMessa } // Merge the tracked changes - err = file.MergeNotefile(notefile) + err = file.MergeNotefile(ctx, notefile) if err != nil { openfile.Close(ctx) box.Close(ctx) @@ -939,10 +984,10 @@ func hubNotefilesMerge(ctx context.Context, session *HubSession, req notehubMess sawNotefile(session, box, NotefileID) // Set the notification context - file.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, event, session) + file.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, session.Session.Transport, event, session) // Merge the tracked changes - err = file.MergeNotefile(chgfile) + err = file.MergeNotefile(ctx, chgfile) if err != nil { openfile.Close(ctx) box.Close(ctx) @@ -998,6 +1043,7 @@ func hubNotefileUpdateChangeTracker(ctx context.Context, session *HubSession, re // Notebox Summary func hubNoteboxSummary(ctx context.Context, session *HubSession, req notehubMessage, rsp *notehubMessage, event EventFunc) (err error) { + // Open the box box, appUID, err2 := openHubNoteboxForDevice(ctx, session, req.DeviceUID, req.DeviceSN, req.ProductUID, req.DeviceEndpointID, event) if err2 != nil { @@ -1006,8 +1052,7 @@ func hubNoteboxSummary(ctx context.Context, session *HubSession, req notehubMess } // Validate the session ID, and delete all local trackers to force resync if we've gotten out of sync - // Then, flush the notebox to ensure that if this session for this device drops, that a follow-on session - // starting in another handler will pick up the proper sessionID so that it doesn't need to do a full sync. + // Note that this session ID will be flushed by our 5 minute checkpoint task and by normal session close. sessionIDPrev := box.Notefile().swapTrackerSessionID(req.DeviceEndpointID, req.SessionIDNext) if sessionIDPrev != req.SessionIDPrev { if debugSync { @@ -1016,13 +1061,9 @@ func hubNoteboxSummary(ctx context.Context, session *HubSession, req notehubMess box.ClearAllTrackers(ctx, req.DeviceEndpointID) rsp.SessionIDMismatch = true } - err2 = box.Checkpoint(ctx) - if err2 != nil { - logError(ctx, "Error checkpointing notebox: %v", err2) - } // Set the notification context - box.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, event, session) + box.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, session.Session.Transport, event, session) // Update the environment vars for the notebox, which may result in a changed _env.dbs hubUpdateEnvVars(ctx, box, req.DeviceUID, appUID, true) @@ -1043,18 +1084,9 @@ func hubNoteboxSummary(ctx context.Context, session *HubSession, req notehubMess } } - // Return the results so long as they fit within the protocol buffer (see notehub.options). + // Return the results truncated to fit within the protocol buffer (see notehub.options). // This is safe because it will simply take multiple passes to sync all of these files. - for { - rsp.NotefileIDs = strings.Join(fileChanges, ReservedIDDelimiter) - if len(rsp.NotefileIDs) < (250 - 10) { - break - } - if len(fileChanges) == 0 { - break - } - fileChanges = fileChanges[:len(fileChanges)-1] - } + rsp.NotefileIDs = fmt.Sprintf("%.*s", maxProtobufStringLength, strings.Join(fileChanges, ReservedIDDelimiter)) // Done box.Close(ctx) @@ -1091,7 +1123,7 @@ func hubNotefileAddNote(ctx context.Context, session *HubSession, req notehubMes sawNotefile(session, box, req.NotefileID) // Set the notification context - file.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, event, session) + file.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, session.Session.Transport, event, session) // Perform the operation var body []byte @@ -1112,7 +1144,7 @@ func hubNotefileAddNote(ctx context.Context, session *HubSession, req notehubMes newNote.Histories = &histories // Add the note to the notefile with history set up here - _, err = file.AddNoteWithHistory(req.DeviceEndpointID, req.NoteID, newNote) + _, err = file.AddNoteWithHistory(ctx, req.DeviceEndpointID, req.NoteID, newNote) if err == nil { // If this is an outbound queue, purge ALL tombstones from the @@ -1157,10 +1189,10 @@ func hubNotefileDeleteNote(ctx context.Context, session *HubSession, req notehub sawNotefile(session, box, req.NotefileID) // Set the notification context - file.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, event, session) + file.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, session.Session.Transport, event, session) // Perform the operation - err = file.DeleteNote(req.DeviceEndpointID, req.NoteID) + err = file.DeleteNote(ctx, req.DeviceEndpointID, req.NoteID) if err != nil { openfile.Close(ctx) box.Close(ctx) @@ -1193,7 +1225,7 @@ func hubNotefileUpdateNote(ctx context.Context, session *HubSession, req notehub sawNotefile(session, box, req.NotefileID) // Set the notification context - file.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, event, session) + file.SetEventInfo(req.DeviceUID, req.DeviceSN, req.ProductUID, appUID, session.Session.Transport, event, session) // Perform the operation xnote, err5 := file.GetNote(req.NoteID) @@ -1218,7 +1250,7 @@ func hubNotefileUpdateNote(ctx context.Context, session *HubSession, req notehub } } if err == nil { - err = file.UpdateNote(req.DeviceEndpointID, req.NoteID, xnote) + err = file.UpdateNote(ctx, req.DeviceEndpointID, req.NoteID, xnote) } // Done @@ -1264,6 +1296,12 @@ func hubReadFile(ctx context.Context, session *HubSession, req notehubMessage, r return } + // locally we have bogus firmware that's created for testing the UI. + // if any device tries to update to this firmware, always return an error + if metadata.Firmware != nil && metadata.Firmware.Description == notehub.TestFirmwareString { + return fmt.Errorf("this firmware is invalid and created for testing purposes only") + } + // If we'd like the binary returned uncompressed as the response payload, do so if returnUncompressedBinary { rsp.SetPayload(payload) @@ -1286,7 +1324,7 @@ func hubReadFile(ctx context.Context, session *HubSession, req notehubMessage, r box.Close(ctx) return } - _, err = newNotefile.AddNote(req.DeviceEndpointID, "result", xnote) + _, err = newNotefile.AddNote(ctx, req.DeviceEndpointID, "result", xnote) if err != nil { box.Close(ctx) return @@ -1382,3 +1420,17 @@ func msgTypeName(msgType string) string { } return "(" + msgType + ")" } + +// Truncate an error string, moving errors to the front if it's too long +func TruncateErrorString(errstr string, maxchars int) string { + if len(errstr) <= maxchars { + return errstr + } + re := regexp.MustCompile(`\{[^}]*\}`) + matches := re.FindAllString(errstr, -1) + if len(matches) > 0 { + errstr = strings.TrimSpace(re.ReplaceAllString(errstr, "")) + errstr = fmt.Sprintf("%s %s", strings.Join(matches, ""), errstr) + } + return fmt.Sprintf("%.*s", maxchars, errstr) +} diff --git a/lib/note.go b/lib/note.go index b7f139b..ca0a906 100644 --- a/lib/note.go +++ b/lib/note.go @@ -301,7 +301,7 @@ func newHistory(endpointID string, when int64, where string, wherewhen int64, se newHistory.EndpointID = endpointID if when == 0 { - newHistory.When = createNoteTimestamp(endpointID) + newHistory.When = createNoteTimestamp() } else { newHistory.When = when } @@ -440,20 +440,10 @@ func copyNote(xnote *note.Note) note.Note { return newNote } -// Return the current timestamp to be used for "when", in milliseconds -var lastTimestamp int64 - -func createNoteTimestamp(endpointID string) int64 { - // Return the later of the current time (in seconds) and the last one that we handed out - thisTimestamp := time.Now().UTC().UnixNano() / 1000000000 - if thisTimestamp <= lastTimestamp { - lastTimestamp++ - thisTimestamp = lastTimestamp - } else { - lastTimestamp = thisTimestamp - } - - return thisTimestamp +// because we only have second-level resolution, some notes can be created +// with the same captured date which might screw up ordering in some cases +func createNoteTimestamp() int64 { + return time.Now().UTC().Unix() } // compareNoteTimestamps is a standard Compare function diff --git a/lib/notebox.go b/lib/notebox.go index 2b2ea11..96eb55a 100644 --- a/lib/notebox.go +++ b/lib/notebox.go @@ -27,6 +27,7 @@ import ( "context" "fmt" "net/http" + "sort" "strings" "sync" "sync/atomic" @@ -45,22 +46,17 @@ var boxLock sync.RWMutex var autoCheckpointStarted bool var autoCheckpointStartedLock sync.Mutex +// Checkpoint timing. Note that we only purge on a longer basis than writes because as of 2024-08-29 +// I changed the session close code to purge closed notefiles from the in-memory cache whenever the +// session closes. The only remaining function of the AutoPurge is to remove closed notefiles +// from memory for continuous sessions. Because this will cause active notefiles to be immediately +// re-opened, we don't want to do it too often. var ( AutoCheckpointSeconds = 5 * 60 - AutoPurgeSeconds = 5 * 60 + AutoPurgeSeconds = 120 * 60 CheckpointSilently = false ) -// Information about files and templates -type NoteboxNotefileInfo struct { - NotefileID string - Info note.NotefileInfo - BodyTemplate string - PayloadTemplate uint32 - TemplateFormat uint32 - TemplatePort uint16 -} - // See if a string is in a string array func isNoteInNoteArray(val string, array []string) bool { for i := range array { @@ -101,7 +97,7 @@ func initNotebox(ctx context.Context, endpointID string, boxStorage string, stor } note, err := note.CreateNote(noteboxBodyToJSON(body), nil) if err == nil { - _, err = boxfile.AddNote(endpointID, compositeNoteID(endpointID, endpointID), note) + _, err = boxfile.AddNote(ctx, endpointID, compositeNoteID(endpointID, endpointID), note) } if err != nil { return err @@ -329,7 +325,7 @@ func Checkpoint(ctx context.Context) (err error) { return checkpointAllNoteboxes(ctx, true) } -// CheckpointNoteboxIfNeeded checkpoints a notebox if it's open +// CheckpointNoteboxIfNeeded checkpoints a notebox if it's open, purging closed files from the cache func CheckpointNoteboxIfNeeded(ctx context.Context, localEndpointID string, boxLocalStorage string) error { // Initialize debugging if we've not done so before debugEnvInit() @@ -349,7 +345,7 @@ func CheckpointNoteboxIfNeeded(ctx context.Context, localEndpointID string, boxL return err } - err = notebox.Checkpoint(ctx) + err = notebox.CheckpointPurge(ctx) _ = notebox.Close(ctx) return err } @@ -419,6 +415,12 @@ func (box *Notebox) Checkpoint(ctx context.Context) (err error) { return } +// Checkpoint a notebox and purge closed/cached files from memory +func (box *Notebox) CheckpointPurge(ctx context.Context) (err error) { + _, err = box.checkpoint(ctx, true, true, true, true) + return +} + // uCheckpoint checkpoints an open notebox func (box *Notebox) checkpoint(ctx context.Context, write bool, purgeClosed bool, purgeClosedForce bool, purgeClosedDeleted bool) (emptyBox bool, err error) { var firstError error @@ -461,11 +463,11 @@ func (box *Notebox) checkpoint(ctx context.Context, write bool, purgeClosed bool // thrash flash I/O unnecessarily. if purgeClosedForce || (purgeClosed && (time.Since(openfile.closeTime) >= time.Duration(AutoPurgeSeconds)*time.Second)) { box.instance.openfiles.Delete(fileStorage) - if debugBox { + if !CheckpointSilently { if openfile.deleted { - logDebug(ctx, "%s purged (had been deleted)", fileStorage) + logDebug(ctx, "CHECKPOINT PURGE %s (had been deleted)", fileStorage) } else { - logDebug(ctx, "%s purged", fileStorage) + logDebug(ctx, "CHECKPOINT PURGE %s", fileStorage) } } openfile.lock.Unlock() @@ -580,8 +582,14 @@ func (box *Notebox) EndpointID() string { } // GetChangedNotefiles determines, for a given tracker, if there are changes in any notebox and, -// if so, for which ones. +// if so, for which ones. NOTE that after significant performance measurement in Aug 2024 it became +// clear that this method would be far, far more efficient if we would design a box.OpenNotefiles() method +// that returns a vector of notefiles after doing a single app DB query of all of them to get their contents, +// using a new storage.readNotefiles() method. If and when we're looking to improve performance this would +// be a great addition, specifically because the msgNoteboxSummary transaction is done at the beginning +// of each and every sync and thus is inherently a hotspot. func (box *Notebox) GetChangedNotefiles(ctx context.Context, endpointID string) (changedNotefiles []string) { + // Get the names of all possible openable Notefiles allNotefiles := box.Notefiles(true) @@ -761,7 +769,7 @@ func (box *Notebox) AddNotefile(ctx context.Context, notefileID string, notefile notefileInfo = ¬e.NotefileInfo{} } - return box.SetNotefileInfo(notefileID, *notefileInfo) + return box.SetNotefileInfo(ctx, notefileID, *notefileInfo) } @@ -812,7 +820,7 @@ func (box *Notebox) GetNotefileInfo(notefileID string) (notefileInfo note.Notefi } // SetNotefileInfo sets the info about a notefile that is allowed to be changed after notefile creation -func (box *Notebox) SetNotefileInfo(notefileID string, notefileInfo note.NotefileInfo) (err error) { +func (box *Notebox) SetNotefileInfo(ctx context.Context, notefileID string, notefileInfo note.NotefileInfo) (err error) { // Now we're going to add things to the boxfile boxfile := box.Notefile() @@ -839,7 +847,7 @@ func (box *Notebox) SetNotefileInfo(notefileID string, notefileInfo note.Notefil _ = xnote.SetBody(noteboxBodyToJSON(body)) // Update the note - err = boxfile.UpdateNote(box.instance.endpointID, notefileID, xnote) + err = boxfile.UpdateNote(ctx, box.instance.endpointID, notefileID, xnote) if err != nil { return fmt.Errorf(note.ErrNotefileNoExist+" cannot set info for notefile %s: %s", notefileID, err) } @@ -1043,7 +1051,7 @@ func (box *Notebox) OpenNotefile(ctx context.Context, notefileID string) (iOpenf // then comes in, the event will be referencing the now-defunct session and // it will never get sent to the worker. This forces the session to be // refreshed every time we open the notefile, thus ensuring session freshness. - openfile.notefile.SetEventInfo(box.defaultEventDeviceUID, box.defaultEventDeviceSN, box.defaultEventProductUID, box.defaultEventAppUID, box.defaultEventFn, box.defaultEventSession) + openfile.notefile.SetEventInfo(box.defaultEventDeviceUID, box.defaultEventDeviceSN, box.defaultEventProductUID, box.defaultEventAppUID, box.defaultEventTransport, box.defaultEventFn, box.defaultEventSession) // Done return openfile, openfile.notefile, nil @@ -1078,7 +1086,7 @@ func (box *Notebox) OpenNotefile(ctx context.Context, notefileID string) (iOpenf // deviceUID or productUID on the client side because they are not always known // at the time we do the open. As such, client-side notifiers will need // to function without these values. - notefile.SetEventInfo(box.defaultEventDeviceUID, box.defaultEventDeviceSN, box.defaultEventProductUID, box.defaultEventAppUID, box.defaultEventFn, box.defaultEventSession) + notefile.SetEventInfo(box.defaultEventDeviceUID, box.defaultEventDeviceSN, box.defaultEventProductUID, box.defaultEventAppUID, box.defaultEventTransport, box.defaultEventFn, box.defaultEventSession) // Copy the notefile info for all but box notefiles if notefileID != box.instance.endpointID { @@ -1113,11 +1121,12 @@ func (box *Notebox) SetClientInfo(httpReq *http.Request, httpRsp http.ResponseWr } // SetEventInfo establishes default information used for change notification on notefiles opened in the box -func (box *Notebox) SetEventInfo(deviceUID string, deviceSN string, productUID string, appUID string, eventFn EventFunc, session *HubSession) { +func (box *Notebox) SetEventInfo(deviceUID string, deviceSN string, productUID string, appUID string, transport string, eventFn EventFunc, session *HubSession) { box.defaultEventDeviceUID = deviceUID box.defaultEventDeviceSN = deviceSN box.defaultEventProductUID = productUID box.defaultEventAppUID = appUID + box.defaultEventTransport = transport box.defaultEventFn = eventFn box.defaultEventSession = session } @@ -1210,7 +1219,7 @@ func (box *Notebox) DeleteNotefile(ctx context.Context, notefileID string) (err boxfile.lock.RUnlock() if !found { box.Openfile().lock.Unlock() - _ = boxfile.DeleteNote(box.instance.endpointID, notefileID) + _ = boxfile.DeleteNote(ctx, box.instance.endpointID, notefileID) return nil } @@ -1246,7 +1255,7 @@ func (box *Notebox) DeleteNotefile(ctx context.Context, notefileID string) (err // Delete the note used for managing notefiles across all instances, // ignoring errors because it may have been deleted by another instance - _ = boxfile.DeleteNote(box.instance.endpointID, notefileID) + _ = boxfile.DeleteNote(ctx, box.instance.endpointID, notefileID) // Delete the storage object _ = storage.delete(ctx, box.instance.storage, notefileBody.Notefile.Storage) @@ -1257,7 +1266,7 @@ func (box *Notebox) DeleteNotefile(ctx context.Context, notefileID string) (err box.Openfile().lock.Unlock() // Delete the note for managing the instance - _ = boxfile.DeleteNote(box.instance.endpointID, notefileNoteID) + _ = boxfile.DeleteNote(ctx, box.instance.endpointID, notefileNoteID) // Done return nil @@ -1309,7 +1318,7 @@ func (box *Notebox) AddNote(ctx context.Context, endpointID string, notefileID s return err } - _, err = file.AddNote(endpointID, noteID, note) + _, err = file.AddNote(ctx, endpointID, noteID, note) openfile.Close(ctx) @@ -1323,7 +1332,7 @@ func (box *Notebox) AddNoteWithHistory(ctx context.Context, endpointID string, n return err } - _, err = file.AddNoteWithHistory(endpointID, noteID, note) + _, err = file.AddNoteWithHistory(ctx, endpointID, noteID, note) openfile.Close(ctx) @@ -1337,7 +1346,7 @@ func (box *Notebox) UpdateNote(ctx context.Context, endpointID string, notefileI return err } - err = file.UpdateNote(endpointID, noteID, note) + err = file.UpdateNote(ctx, endpointID, noteID, note) openfile.Close(ctx) @@ -1351,7 +1360,7 @@ func (box *Notebox) DeleteNote(ctx context.Context, endpointID string, notefileI return err } - err = file.DeleteNote(endpointID, noteID) + err = file.DeleteNote(ctx, endpointID, noteID) openfile.Close(ctx) @@ -1403,6 +1412,9 @@ func (box *Notebox) MergeNotebox(ctx context.Context, fromBoxfile *Notefile) (er // delete notes that are our only pointers to local storage. We simply don't allow remote // endpoints to modify our local instance notes. for noteID := range fromBoxfile.Notes { + if noteID == "" { + logError(ctx, "MergeNotebox: empty notefileID coming from Notecard") + } isInstance, endpointID, _ := parseCompositeNoteID(noteID) if isInstance && endpointID == box.instance.endpointID { delete(fromBoxfile.Notes, noteID) @@ -1411,7 +1423,7 @@ func (box *Notebox) MergeNotebox(ctx context.Context, fromBoxfile *Notefile) (er // Next, merge what's remaining. This will add and remove remote all remote notefile changes, // as well as updating the global notefile notes that contain NotefileInfo - err = boxfile.MergeNotefile(fromBoxfile) + err = boxfile.MergeNotefile(ctx, fromBoxfile) if err != nil { return err } @@ -1440,6 +1452,12 @@ func (box *Notebox) MergeNotebox(ctx context.Context, fromBoxfile *Notefile) (er existsGloballyNoteID := []string{} deletedGloballyNoteID := []string{} for noteID, note := range boxfile.Notes { + if noteID == "" { + logError(ctx, "MergeNotebox: empty notefileID in merged contents of Notecard and Notehub") + delete(boxfile.Notes, noteID) + didSomething = true + continue + } isInstance, endpointID, notefileID := parseCompositeNoteID(noteID) // Skip notebox descriptors if endpointID == notefileID { @@ -1556,8 +1574,8 @@ func (box *Notebox) VerifyAccess(resource string, actions string) (err error) { return nil } -// NoteboxNotefileInfo gets info about all notefiles in a notebox -func (box *Notebox) NoteboxNotefileInfo() (allInfo []NoteboxNotefileInfo) { +// NoteboxNotefileDesc gets info about all notefiles in a notebox +func (box *Notebox) NoteboxNotefileDesc() (allInfo []note.NotefileDesc) { // Enum all notes in the boxfile, looking at the global files only boxfile := box.Notefile() @@ -1566,7 +1584,7 @@ func (box *Notebox) NoteboxNotefileInfo() (allInfo []NoteboxNotefileInfo) { isInstance, _, notefileID := parseCompositeNoteID(noteID) if !isInstance && !xnote.Deleted { notefileBody := noteboxBodyFromJSON(xnote.GetBody()) - info := NoteboxNotefileInfo{} + info := note.NotefileDesc{} info.NotefileID = notefileID if notefileBody.Notefile.Info != nil { info.Info = *notefileBody.Notefile.Info @@ -1580,12 +1598,30 @@ func (box *Notebox) NoteboxNotefileInfo() (allInfo []NoteboxNotefileInfo) { } boxfile.lock.RUnlock() + // Sort the allInfo slice by NotefileID in ascending order. (See comment related + // to 'determinism' in the NoteboxNotefileDescByPort() method below.) + sort.Slice(allInfo, func(i, j int) bool { + return allInfo[i].NotefileID < allInfo[j].NotefileID + }) + + // Done return allInfo } -// NoteboxNotefileInfoByPort gets info about all notefiles in a notebox -func NoteboxNotefileInfoByPort(allInfo []NoteboxNotefileInfo, port uint16) (info NoteboxNotefileInfo, present bool) { +// NoteboxNotefileDescByPort gets info about all notefiles in a notebox. Note that for +// determinism in the case where someone accidentally creates two notefiles with the same +// port, this method relies upon the fact that NoteboxNotefileDesc() returns the notefiles +// in alphabetized sort order. (The notecard DOES try to keep devs from assigning the same +// port to multiple notefiles, however this can happen by: +// a) add notefiles "a.qi port 1" +// b) sync +// c) factory reset notecard and quickly do a hub.set mode:off to keep it from syncing +// d) add notefiles "b.qi port 1" +// e) set hub.set mode:periodic and sync +// At the end of this, because a.qi comes back from the notehub, and b.qi goes up to +// the notehub, there are two notefiles with the same port number. +func NoteboxNotefileDescByPort(allInfo []note.NotefileDesc, port uint16) (info note.NotefileDesc, present bool) { for _, inf := range allInfo { if inf.TemplatePort == port { return inf, true diff --git a/lib/notefile.go b/lib/notefile.go index 6c24c08..14d2ac6 100644 --- a/lib/notefile.go +++ b/lib/notefile.go @@ -19,7 +19,6 @@ import ( "github.com/blues/note-go/note" olc "github.com/google/open-location-code/go" - "github.com/google/uuid" ) // Debug @@ -39,6 +38,7 @@ type Notefile struct { eventDeviceSN string // The deviceSN being dealt with at the time of the event setup eventProductUID string // The productUID being dealt with at the time of the event setup eventAppUID string // The appUID being dealt with at the time of the event setup + eventTransport string // The transport being dealt with at the time of the event setup notefileID string // The NotefileID for the open notefile notefileInfo note.NotefileInfo // The NotefileInfo for the open notefile Queue bool `json:"Q,omitempty"` @@ -167,7 +167,7 @@ func (nf *Notefile) Info() (info note.NotefileInfo) { } // SetEventInfo supplies information used for change notification -func (nf *Notefile) SetEventInfo(deviceUID string, deviceSN string, productUID string, appUID string, eventFn EventFunc, eventSession *HubSession) { +func (nf *Notefile) SetEventInfo(deviceUID string, deviceSN string, productUID string, appUID string, transport string, eventFn EventFunc, eventSession *HubSession) { // Lock for writing nf.lock.Lock() @@ -178,13 +178,14 @@ func (nf *Notefile) SetEventInfo(deviceUID string, deviceSN string, productUID s nf.eventDeviceSN = deviceSN nf.eventProductUID = productUID nf.eventAppUID = appUID + nf.eventTransport = transport // Unlock & exit nf.lock.Unlock() } // GetEventInfo Gets information used for change notification -func (nf *Notefile) GetEventInfo() (deviceUID string, deviceSN string, productUID string, appUID string, eventFn EventFunc, eventSession *HubSession) { +func (nf *Notefile) GetEventInfo() (deviceUID string, deviceSN string, productUID string, appUID string, transport string, eventFn EventFunc, eventSession *HubSession) { // Lock for reading nf.lock.RLock() @@ -196,6 +197,7 @@ func (nf *Notefile) GetEventInfo() (deviceUID string, deviceSN string, productUI deviceSN = nf.eventDeviceSN productUID = nf.eventProductUID appUID = nf.eventAppUID + transport = nf.eventTransport // Unlock & exit nf.lock.RUnlock() @@ -262,7 +264,7 @@ func (nf *Notefile) GetNote(noteID string) (xnote note.Note, err error) { } // event dispatches to the event proc based on the last change made to NoteID -func (nf *Notefile) event(local bool, NoteID string) { +func (nf *Notefile) event(ctx context.Context, local bool, NoteID string) { if nf.eventFn == nil { return } @@ -274,13 +276,15 @@ func (nf *Notefile) event(local bool, NoteID string) { if ErrorContains(err, note.ErrNoteNoExist) { return } - logError(context.Background(), "event: GetNote(%s) error: %s", NoteID, err) + logError(ctx, "event: GetNote(%s) error: %s", NoteID, err) return } - // Set up the notification structure + // Set up the notification structure. Note that prior to Oct 2024 we used to + // add the EndpointID, however the fact that Notehub-generated events failed + // to add EndpointID (combined with the fact that we don't document what + // EndpointID even means) means that it's best to not include it in the first place. event := note.Event{} - event.EventUID = uuid.New().String() if xnote.Deleted && !nf.Queue { event.Req = note.EventDelete } else if xnote.Updates != 0 { @@ -289,7 +293,6 @@ func (nf *Notefile) event(local bool, NoteID string) { event.Req = note.EventAdd } event.NoteID = NoteID - event.EndpointID = xnote.EndpointID() event.Deleted = xnote.Deleted event.Sent = xnote.Sent event.Bulk = xnote.Bulk @@ -301,6 +304,7 @@ func (nf *Notefile) event(local bool, NoteID string) { if nf.eventAppUID != "" { event.AppUID = nf.eventAppUID } + event.Transport = nf.eventTransport event.Payload = xnote.Payload if xnote.Body != nil { event.Body = &xnote.Body @@ -339,8 +343,19 @@ func (nf *Notefile) event(local bool, NoteID string) { event.Deleted = false } + // Now that the event has been initialized, generate a UID. Note that if we are + // generating it locally, generate a totally unique event UID because we know for a + // fact that the even isn't going to be a duplicate of one generated anywhere else, + // and otherwise generate a UID that is content-dependent so that if the notecard + // uploads duplicates they won't be saved. + if local { + event.EventUID = GenerateEventUid(nil) + } else { + event.EventUID = GenerateEventUid(&event) + } + // Debug - if debugEvent { + if DebugEvent { logString := fmt.Sprintf("event: %s %s %s %s %s", event.Req, event.NotefileID, nf.eventAppUID, event.DeviceUID, event.ProductUID) if len(event.Payload) > 0 { if event.Bulk { @@ -350,7 +365,7 @@ func (nf *Notefile) event(local bool, NoteID string) { } } // No need to log customer data body - logDebug(context.Background(), "%s", logString) + logDebug(ctx, "%s", logString) } // Disable the event function so as to avoid recursion @@ -359,9 +374,32 @@ func (nf *Notefile) event(local bool, NoteID string) { // Perform the notification (checking savedFn in case of race condition, because everything is unlocked if savedFn != nil { - err = savedFn(context.Background(), nf.eventSession, local, nf, &event) + start := time.Now() + err = savedFn(ctx, nf.eventSession, local, &event) + duration := time.Since(start) + // Since most note operations take a few milliseconds, logging the event should + // really only take a few milliseconds else the logging will dominate the operation itself, + // so issue a warning so that we can fix the logging. + if duration > 5*time.Second { + typeMsg := "event" + if local { + typeMsg = "local " + typeMsg + } + if event.Bulk { + typeMsg = "bulk " + typeMsg + } + if nf.eventSession == nil { + typeMsg = typeMsg + " (no session)" + } + message := fmt.Sprintf("%s %s %s [timewarn] warning: %s enqueue took %s", nf.eventAppUID, nf.eventDeviceUID, nf.notefileID, typeMsg, duration) + if duration > transactionErrorDuration { + logWarn(ctx, message) + } else { + logInfo(ctx, message) + } + } if err != nil { - logError(context.Background(), "event notification error: %s %s %s: %s", nf.eventAppUID, event.DeviceUID, event.NotefileID, err) + logError(ctx, "%s %s %s event notification error: %s", nf.eventAppUID, event.DeviceUID, event.NotefileID, err) } } @@ -430,9 +468,9 @@ func (nf *Notefile) MarkAsModified(reason string) { // undelete it and thus it will be present everywhere except the source. This is an intentional // asymmetry in synchronization that is primarily useful when sending "to hub" or "from hub". // By convention, sent notes should be deleted after they have been processed by the recipient(s). -func (nf *Notefile) AddNote(endpointID string, noteID string, xnote note.Note) (newNoteID string, err error) { +func (nf *Notefile) AddNote(ctx context.Context, endpointID string, noteID string, xnote note.Note) (newNoteID string, err error) { xnote.Histories = nil - return nf.AddNoteWithHistory(endpointID, noteID, xnote) + return nf.AddNoteWithHistory(ctx, endpointID, noteID, xnote) } // Set the 'when' date on a note immediately AFTER it has been added/updated @@ -460,7 +498,7 @@ func (nf *Notefile) SetNoteWhen(noteID string, when int64) (success bool) { } // AddNoteWithHistory is same as AddNote, but it retains history -func (nf *Notefile) AddNoteWithHistory(endpointID string, noteID string, xnote note.Note) (newNoteID string, err error) { +func (nf *Notefile) AddNoteWithHistory(ctx context.Context, endpointID string, noteID string, xnote note.Note) (newNoteID string, err error) { // Create a note ID if not specified if nf.Queue || noteID == "" { noteID = nf.NewNoteID(endpointID) @@ -469,7 +507,7 @@ func (nf *Notefile) AddNoteWithHistory(endpointID string, noteID string, xnote n // If the note exists as a deleted note, turn it into an update onote, present := nf.Notes[noteID] if !nf.Queue && present && onote.Deleted { - err = nf.UpdateNote(endpointID, noteID, xnote) + err = nf.UpdateNote(ctx, endpointID, noteID, xnote) return noteID, err } @@ -489,7 +527,7 @@ func (nf *Notefile) AddNoteWithHistory(endpointID string, noteID string, xnote n } // Event listeners - nf.event(endpointID != note.DefaultDeviceEndpointID, noteID) + nf.event(ctx, endpointID != note.DefaultDeviceEndpointID, noteID) // Done return noteID, err @@ -532,7 +570,7 @@ func (nf *Notefile) uUpdateNote(endpointID string, noteID string, xnote *note.No // UpdateNote updates an existing note within a Notefile, as well // as updating all its history and conflict metadata as appropriate. -func (nf *Notefile) UpdateNote(endpointID string, noteID string, xnote note.Note) error { +func (nf *Notefile) UpdateNote(ctx context.Context, endpointID string, noteID string, xnote note.Note) error { // Exit if trying to update within a queue if nf.Queue { return fmt.Errorf(note.ErrNotefileQueueDisallowed + " operation not allowed on queue notefiles") @@ -549,7 +587,7 @@ func (nf *Notefile) UpdateNote(endpointID string, noteID string, xnote note.Note // Event listeners if err == nil { - nf.event(endpointID != note.DefaultDeviceEndpointID, noteID) + nf.event(ctx, endpointID != note.DefaultDeviceEndpointID, noteID) } return err @@ -576,7 +614,7 @@ func (nf *Notefile) uDeleteNote(endpointID string, noteID string, xnote *note.No // DeleteNote sets the deleted flag on an existing note from in a Notefile, // marking it so that it will be purged at a later time when safe to do so // from a synchronization perspective. -func (nf *Notefile) DeleteNote(endpointID string, noteID string) error { +func (nf *Notefile) DeleteNote(ctx context.Context, endpointID string, noteID string) error { // Lock for writing nf.lock.Lock() @@ -595,7 +633,7 @@ func (nf *Notefile) DeleteNote(endpointID string, noteID string) error { // Event listeners if err == nil { - nf.event(endpointID != note.DefaultDeviceEndpointID, noteID) + nf.event(ctx, endpointID != note.DefaultDeviceEndpointID, noteID) } // Done @@ -665,7 +703,7 @@ func (nf *Notefile) uGetMergeNoteChangeList(fromNotefile *Notefile) (notelist [] } // MergeNotefile combines/merges the entire contents of one Notefile into another -func (nf *Notefile) MergeNotefile(fromNotefile *Notefile) error { +func (nf *Notefile) MergeNotefile(ctx context.Context, fromNotefile *Notefile) error { // Lock for writing nf.lock.Lock() @@ -718,10 +756,10 @@ func (nf *Notefile) MergeNotefile(fromNotefile *Notefile) error { // an inbound queue, delete the contents after modification because this is the core essential // behavior of queues for i := range modifiedNoteIDs { - nf.event(false, modifiedNoteIDs[i]) + nf.event(ctx, false, modifiedNoteIDs[i]) if inboundQueuesProcessedByNotifier { if nf.Queue { - _ = nf.DeleteNote(note.DefaultHubEndpointID, modifiedNoteIDs[i]) + _ = nf.DeleteNote(ctx, note.DefaultHubEndpointID, modifiedNoteIDs[i]) } } } @@ -919,6 +957,7 @@ func NotefileIDIsReservedWithExceptions(notefileID string) bool { if notefileID == note.TrackNotefile || notefileID == note.LogNotefile || notefileID == note.HealthNotefile || + notefileID == note.HealthHostNotefile || notefileID == note.NotecardRequestNotefile || notefileID == note.NotecardResponseNotefile { return false diff --git a/lib/notehub-defs.go b/lib/notehub-defs.go index d233681..06938b7 100644 --- a/lib/notehub-defs.go +++ b/lib/notehub-defs.go @@ -7,10 +7,13 @@ package notelib import ( "context" + "crypto/sha256" + "encoding/binary" "fmt" "time" "github.com/blues/note-go/note" + "github.com/google/uuid" ) // HTTPUserAgent is the HTTP user agent for all our uses of HTTP @@ -46,6 +49,9 @@ const ( msgCheckpoint = "Z" ) +// Msg type prefix; if present, we suppress response +const msgSuppressResponsePrefix = "-" + // nf is the native format of these structures // Note that the field names in this structure must be unique within // the notehubMessage structure because they are "flattened" to @@ -100,52 +106,54 @@ type notehubMessage struct { // no matter what MessageType the transaction happens to be. // THESE FIELDS SHOULD NEVER BE OVERLOADED BY RPC TRANSACTIONS FOR // THEIR OWN PER-TRANSACTION DATA TRANSFER REQUIREMENTS!!! - Version uint32 `json:"a,omitempty"` - DeviceUID string `json:"d,omitempty"` - DeviceEndpointID string `json:"e,omitempty"` - HubTimeNs int64 `json:"f,omitempty"` - HubEndpointID string `json:"g,omitempty"` - HubSessionHandler string `json:"h,omitempty"` - HubSessionFactoryResetID string `json:"i,omitempty"` - HubSessionTicket string `json:"j,omitempty"` - HubSessionTicketExpiresTimeSec int64 `json:"k,omitempty"` - SessionIDPrev int64 `json:"r,omitempty"` - SessionIDNext int64 `json:"s,omitempty"` - SessionIDMismatch bool `json:"t,omitempty"` - ProductUID string `json:"u,omitempty"` - UsageProvisioned int64 `json:"v,omitempty"` - UsageRcvdBytes uint32 `json:"w,omitempty"` - UsageSentBytes uint32 `json:"x,omitempty"` - UsageTCPSessions uint32 `json:"y,omitempty"` - UsageTLSSessions uint32 `json:"z,omitempty"` - UsageRcvdNotes uint32 `json:"A,omitempty"` - UsageSentNotes uint32 `json:"B,omitempty"` - HighPowerSecsTotal uint32 `json:"C,omitempty"` - HighPowerSecsData uint32 `json:"D,omitempty"` - HighPowerSecsGPS uint32 `json:"E,omitempty"` - HighPowerCyclesTotal uint32 `json:"F,omitempty"` - HighPowerCyclesData uint32 `json:"G,omitempty"` - HighPowerCyclesGPS uint32 `json:"H,omitempty"` - DeviceSN string `json:"I,omitempty"` - CellID string `json:"J,omitempty"` - NotificationSession bool `json:"K,omitempty"` - Voltage100 int32 `json:"L,omitempty"` - Temp100 int32 `json:"M,omitempty"` - Voltage1000 int32 `json:"N,omitempty"` - Temp1000 int32 `json:"O,omitempty"` - ContinuousSession bool `json:"P,omitempty"` - MotionSecs int64 `json:"Q,omitempty"` - MotionOrientation string `json:"R,omitempty"` - SessionTrigger string `json:"S,omitempty"` - DeviceSKU string `json:"T,omitempty"` - DeviceFirmware int64 `json:"U,omitempty"` - DevicePIN string `json:"V,omitempty"` - DeviceOrderingCode string `json:"W,omitempty"` - UsageRcvdBytesSecondary uint32 `json:"X,omitempty"` - UsageSentBytesSecondary uint32 `json:"Y,omitempty"` - Where string `json:"wh,omitempty"` - WhereWhen int64 `json:"ww,omitempty"` - HubPacketHandler string `json:"ph,omitempty"` + Version uint32 `json:"a,omitempty"` + DeviceUID string `json:"d,omitempty"` + DeviceEndpointID string `json:"e,omitempty"` + HubTimeNs int64 `json:"f,omitempty"` + HubEndpointID string `json:"g,omitempty"` + HubSessionHandler string `json:"h,omitempty"` + HubSessionFactoryResetID string `json:"i,omitempty"` + HubSessionTicket string `json:"j,omitempty"` + HubSessionTicketExpiresTimeSec int64 `json:"k,omitempty"` + SessionIDPrev int64 `json:"r,omitempty"` + SessionIDNext int64 `json:"s,omitempty"` + SessionIDMismatch bool `json:"t,omitempty"` + ProductUID string `json:"u,omitempty"` + UsageProvisioned int64 `json:"v,omitempty"` + UsageRcvdBytes uint32 `json:"w,omitempty"` + UsageSentBytes uint32 `json:"x,omitempty"` + UsageTCPSessions uint32 `json:"y,omitempty"` + UsageTLSSessions uint32 `json:"z,omitempty"` + UsageRcvdNotes uint32 `json:"A,omitempty"` + UsageSentNotes uint32 `json:"B,omitempty"` + HighPowerSecsTotal uint32 `json:"C,omitempty"` + HighPowerSecsData uint32 `json:"D,omitempty"` + HighPowerSecsGPS uint32 `json:"E,omitempty"` + HighPowerCyclesTotal uint32 `json:"F,omitempty"` + HighPowerCyclesData uint32 `json:"G,omitempty"` + HighPowerCyclesGPS uint32 `json:"H,omitempty"` + DeviceSN string `json:"I,omitempty"` + CellID string `json:"J,omitempty"` + NotificationSession bool `json:"K,omitempty"` + Voltage100 int32 `json:"L,omitempty"` + Temp100 int32 `json:"M,omitempty"` + Voltage1000 int32 `json:"N,omitempty"` + Temp1000 int32 `json:"O,omitempty"` + ContinuousSession bool `json:"P,omitempty"` + MotionSecs int64 `json:"Q,omitempty"` + MotionOrientation string `json:"R,omitempty"` + SessionTrigger string `json:"S,omitempty"` + DeviceSKU string `json:"T,omitempty"` + DeviceFirmware int64 `json:"U,omitempty"` + DevicePIN string `json:"V,omitempty"` + DeviceOrderingCode string `json:"W,omitempty"` + UsageRcvdBytesSecondary uint32 `json:"X,omitempty"` + UsageSentBytesSecondary uint32 `json:"Y,omitempty"` + Where string `json:"wh,omitempty"` + WhereWhen int64 `json:"ww,omitempty"` + HubPacketHandler string `json:"ph,omitempty"` + PowerSource uint32 `json:"ps,omitempty"` + PowerMahUsed float64 `json:"pm,omitempty"` // These fields are used within the handling of MessageType to mean things // that are specific to the transaction type specified in MessageType. Note @@ -169,6 +177,10 @@ type notehubMessage struct { *cf `json:",omitempty"` } +const NotecardPowerCharging = uint32(0x00000001) +const NotecardPowerUsb = uint32(0x00000002) +const NotecardPowerPrimary = uint32(0x00000004) + type HubSession struct { // These is all the fields sent from the notecard and decoded in wire.go Discovery bool @@ -201,23 +213,39 @@ type HubSession struct { Hub interface{} } -func NewHubSession(sessionUID, handlerAddr string, secure bool) HubSession { - now := time.Now() - deviceSession := note.DeviceSession{} - deviceSession.SessionUID = sessionUID - deviceSession.Handler = handlerAddr - deviceSession.Period().Since = now.Unix() - deviceSession.TLSSession = secure +// NewUUID returns a new UUID +func NewUUID() (randstr string) { + return uuid.New().String() +} +// NewUNameFromUID returns a name from a UUID +func NameFromUUID(uid string) (name string) { + hash := sha256.Sum256([]byte(uid)) + uint32Hash := binary.BigEndian.Uint32(hash[:4]) + return note.WordsFromNumber(uint32Hash) +} + +func InitDeviceSession(sess *note.DeviceSession, handlerAddr string, secure bool) time.Time { + now := time.Now() + sess.SessionUID = NewUUID() + sess.SessionBegan = now.UTC().Unix() + sess.Period().Since = sess.SessionBegan + sess.TLSSession = secure + sess.Handler = handlerAddr if secure { - deviceSession.Period().TLSSessions++ + sess.Period().TLSSessions++ } else { - deviceSession.Period().TCPSessions++ + sess.Period().TCPSessions++ } + return now +} +func NewHubSession(handlerAddr string, secure bool) HubSession { + deviceSession := note.DeviceSession{} + sessionBegan := InitDeviceSession(&deviceSession, handlerAddr, secure) return HubSession{ Session: deviceSession, - SessionStart: now, + SessionStart: sessionBegan, } } diff --git a/lib/notehub.pb.go b/lib/notehub.pb.go index 5c58037..31f92f0 100644 --- a/lib/notehub.pb.go +++ b/lib/notehub.pb.go @@ -31,65 +31,67 @@ type NotehubPB struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Version *int64 `protobuf:"varint,1,opt,name=Version" json:"Version,omitempty"` - MessageType *string `protobuf:"bytes,2,opt,name=MessageType" json:"MessageType,omitempty"` - Error *string `protobuf:"bytes,3,opt,name=Error" json:"Error,omitempty"` - DeviceUID *string `protobuf:"bytes,4,opt,name=DeviceUID" json:"DeviceUID,omitempty"` - DeviceEndpointID *string `protobuf:"bytes,5,opt,name=DeviceEndpointID" json:"DeviceEndpointID,omitempty"` - HubTimeNs *int64 `protobuf:"varint,6,opt,name=HubTimeNs" json:"HubTimeNs,omitempty"` - HubEndpointID *string `protobuf:"bytes,7,opt,name=HubEndpointID" json:"HubEndpointID,omitempty"` - HubSessionTicket *string `protobuf:"bytes,8,opt,name=HubSessionTicket" json:"HubSessionTicket,omitempty"` - HubSessionHandler *string `protobuf:"bytes,9,opt,name=HubSessionHandler" json:"HubSessionHandler,omitempty"` - HubSessionTicketExpiresTimeSec *int64 `protobuf:"varint,10,opt,name=HubSessionTicketExpiresTimeSec" json:"HubSessionTicketExpiresTimeSec,omitempty"` - NotefileID *string `protobuf:"bytes,11,opt,name=NotefileID" json:"NotefileID,omitempty"` - NotefileIDs *string `protobuf:"bytes,12,opt,name=NotefileIDs" json:"NotefileIDs,omitempty"` - Since *int64 `protobuf:"varint,13,opt,name=Since" json:"Since,omitempty"` - Until *int64 `protobuf:"varint,14,opt,name=Until" json:"Until,omitempty"` - MaxChanges *int64 `protobuf:"varint,15,opt,name=MaxChanges" json:"MaxChanges,omitempty"` - DeviceSN *string `protobuf:"bytes,16,opt,name=DeviceSN" json:"DeviceSN,omitempty"` - NoteID *string `protobuf:"bytes,17,opt,name=NoteID" json:"NoteID,omitempty"` - SessionIDPrev *int64 `protobuf:"varint,18,opt,name=SessionIDPrev" json:"SessionIDPrev,omitempty"` - SessionIDNext *int64 `protobuf:"varint,19,opt,name=SessionIDNext" json:"SessionIDNext,omitempty"` - SessionIDMismatch *bool `protobuf:"varint,20,opt,name=SessionIDMismatch" json:"SessionIDMismatch,omitempty"` - Bytes1 *int64 `protobuf:"varint,21,opt,name=Bytes1" json:"Bytes1,omitempty"` - Bytes2 *int64 `protobuf:"varint,22,opt,name=Bytes2" json:"Bytes2,omitempty"` - Bytes3 *int64 `protobuf:"varint,23,opt,name=Bytes3" json:"Bytes3,omitempty"` - Bytes4 *int64 `protobuf:"varint,24,opt,name=Bytes4" json:"Bytes4,omitempty"` - ProductUID *string `protobuf:"bytes,25,opt,name=ProductUID" json:"ProductUID,omitempty"` - UsageProvisioned *int64 `protobuf:"varint,26,opt,name=UsageProvisioned" json:"UsageProvisioned,omitempty"` - UsageRcvdBytes *uint32 `protobuf:"varint,27,opt,name=UsageRcvdBytes" json:"UsageRcvdBytes,omitempty"` - UsageSentBytes *uint32 `protobuf:"varint,28,opt,name=UsageSentBytes" json:"UsageSentBytes,omitempty"` - UsageTCPSessions *uint32 `protobuf:"varint,29,opt,name=UsageTCPSessions" json:"UsageTCPSessions,omitempty"` - UsageTLSSessions *uint32 `protobuf:"varint,30,opt,name=UsageTLSSessions" json:"UsageTLSSessions,omitempty"` - UsageRcvdNotes *uint32 `protobuf:"varint,31,opt,name=UsageRcvdNotes" json:"UsageRcvdNotes,omitempty"` - UsageSentNotes *uint32 `protobuf:"varint,32,opt,name=UsageSentNotes" json:"UsageSentNotes,omitempty"` - CellID *string `protobuf:"bytes,33,opt,name=CellID" json:"CellID,omitempty"` - NotificationSession *bool `protobuf:"varint,34,opt,name=NotificationSession" json:"NotificationSession,omitempty"` - Voltage100 *int32 `protobuf:"varint,35,opt,name=Voltage100" json:"Voltage100,omitempty"` - Temp100 *int32 `protobuf:"varint,36,opt,name=Temp100" json:"Temp100,omitempty"` - ContinuousSession *bool `protobuf:"varint,37,opt,name=ContinuousSession" json:"ContinuousSession,omitempty"` - MotionSecs *int64 `protobuf:"varint,38,opt,name=MotionSecs" json:"MotionSecs,omitempty"` - MotionOrientation *string `protobuf:"bytes,39,opt,name=MotionOrientation" json:"MotionOrientation,omitempty"` - SessionTrigger *string `protobuf:"bytes,40,opt,name=SessionTrigger" json:"SessionTrigger,omitempty"` - Voltage1000 *int32 `protobuf:"varint,41,opt,name=Voltage1000" json:"Voltage1000,omitempty"` - Temp1000 *int32 `protobuf:"varint,42,opt,name=Temp1000" json:"Temp1000,omitempty"` - HubSessionFactoryResetID *string `protobuf:"bytes,43,opt,name=HubSessionFactoryResetID" json:"HubSessionFactoryResetID,omitempty"` - HighPowerSecsTotal *uint32 `protobuf:"varint,44,opt,name=HighPowerSecsTotal" json:"HighPowerSecsTotal,omitempty"` - HighPowerSecsData *uint32 `protobuf:"varint,45,opt,name=HighPowerSecsData" json:"HighPowerSecsData,omitempty"` - HighPowerSecsGPS *uint32 `protobuf:"varint,46,opt,name=HighPowerSecsGPS" json:"HighPowerSecsGPS,omitempty"` - HighPowerCyclesTotal *uint32 `protobuf:"varint,47,opt,name=HighPowerCyclesTotal" json:"HighPowerCyclesTotal,omitempty"` - HighPowerCyclesData *uint32 `protobuf:"varint,48,opt,name=HighPowerCyclesData" json:"HighPowerCyclesData,omitempty"` - HighPowerCyclesGPS *uint32 `protobuf:"varint,49,opt,name=HighPowerCyclesGPS" json:"HighPowerCyclesGPS,omitempty"` - DeviceSKU *string `protobuf:"bytes,50,opt,name=DeviceSKU" json:"DeviceSKU,omitempty"` - DeviceFirmware *int64 `protobuf:"varint,51,opt,name=DeviceFirmware" json:"DeviceFirmware,omitempty"` - DevicePIN *string `protobuf:"bytes,52,opt,name=DevicePIN" json:"DevicePIN,omitempty"` - DeviceOrderingCode *string `protobuf:"bytes,53,opt,name=DeviceOrderingCode" json:"DeviceOrderingCode,omitempty"` - UsageRcvdBytesSecondary *uint32 `protobuf:"varint,54,opt,name=UsageRcvdBytesSecondary" json:"UsageRcvdBytesSecondary,omitempty"` - UsageSentBytesSecondary *uint32 `protobuf:"varint,55,opt,name=UsageSentBytesSecondary" json:"UsageSentBytesSecondary,omitempty"` - SuppressResponse *bool `protobuf:"varint,56,opt,name=SuppressResponse" json:"SuppressResponse,omitempty"` - Where *string `protobuf:"bytes,57,opt,name=Where" json:"Where,omitempty"` - WhereWhen *int64 `protobuf:"varint,58,opt,name=WhereWhen" json:"WhereWhen,omitempty"` - HubPacketHandler *string `protobuf:"bytes,59,opt,name=HubPacketHandler" json:"HubPacketHandler,omitempty"` + Version *int64 `protobuf:"varint,1,opt,name=Version" json:"Version,omitempty"` + MessageType *string `protobuf:"bytes,2,opt,name=MessageType" json:"MessageType,omitempty"` + Error *string `protobuf:"bytes,3,opt,name=Error" json:"Error,omitempty"` + DeviceUID *string `protobuf:"bytes,4,opt,name=DeviceUID" json:"DeviceUID,omitempty"` + DeviceEndpointID *string `protobuf:"bytes,5,opt,name=DeviceEndpointID" json:"DeviceEndpointID,omitempty"` + HubTimeNs *int64 `protobuf:"varint,6,opt,name=HubTimeNs" json:"HubTimeNs,omitempty"` + HubEndpointID *string `protobuf:"bytes,7,opt,name=HubEndpointID" json:"HubEndpointID,omitempty"` + HubSessionTicket *string `protobuf:"bytes,8,opt,name=HubSessionTicket" json:"HubSessionTicket,omitempty"` + HubSessionHandler *string `protobuf:"bytes,9,opt,name=HubSessionHandler" json:"HubSessionHandler,omitempty"` + HubSessionTicketExpiresTimeSec *int64 `protobuf:"varint,10,opt,name=HubSessionTicketExpiresTimeSec" json:"HubSessionTicketExpiresTimeSec,omitempty"` + NotefileID *string `protobuf:"bytes,11,opt,name=NotefileID" json:"NotefileID,omitempty"` + NotefileIDs *string `protobuf:"bytes,12,opt,name=NotefileIDs" json:"NotefileIDs,omitempty"` + Since *int64 `protobuf:"varint,13,opt,name=Since" json:"Since,omitempty"` + Until *int64 `protobuf:"varint,14,opt,name=Until" json:"Until,omitempty"` + MaxChanges *int64 `protobuf:"varint,15,opt,name=MaxChanges" json:"MaxChanges,omitempty"` + DeviceSN *string `protobuf:"bytes,16,opt,name=DeviceSN" json:"DeviceSN,omitempty"` + NoteID *string `protobuf:"bytes,17,opt,name=NoteID" json:"NoteID,omitempty"` + SessionIDPrev *int64 `protobuf:"varint,18,opt,name=SessionIDPrev" json:"SessionIDPrev,omitempty"` + SessionIDNext *int64 `protobuf:"varint,19,opt,name=SessionIDNext" json:"SessionIDNext,omitempty"` + SessionIDMismatch *bool `protobuf:"varint,20,opt,name=SessionIDMismatch" json:"SessionIDMismatch,omitempty"` + Bytes1 *int64 `protobuf:"varint,21,opt,name=Bytes1" json:"Bytes1,omitempty"` + Bytes2 *int64 `protobuf:"varint,22,opt,name=Bytes2" json:"Bytes2,omitempty"` + Bytes3 *int64 `protobuf:"varint,23,opt,name=Bytes3" json:"Bytes3,omitempty"` + Bytes4 *int64 `protobuf:"varint,24,opt,name=Bytes4" json:"Bytes4,omitempty"` + ProductUID *string `protobuf:"bytes,25,opt,name=ProductUID" json:"ProductUID,omitempty"` + UsageProvisioned *int64 `protobuf:"varint,26,opt,name=UsageProvisioned" json:"UsageProvisioned,omitempty"` + UsageRcvdBytes *uint32 `protobuf:"varint,27,opt,name=UsageRcvdBytes" json:"UsageRcvdBytes,omitempty"` + UsageSentBytes *uint32 `protobuf:"varint,28,opt,name=UsageSentBytes" json:"UsageSentBytes,omitempty"` + UsageTCPSessions *uint32 `protobuf:"varint,29,opt,name=UsageTCPSessions" json:"UsageTCPSessions,omitempty"` + UsageTLSSessions *uint32 `protobuf:"varint,30,opt,name=UsageTLSSessions" json:"UsageTLSSessions,omitempty"` + UsageRcvdNotes *uint32 `protobuf:"varint,31,opt,name=UsageRcvdNotes" json:"UsageRcvdNotes,omitempty"` + UsageSentNotes *uint32 `protobuf:"varint,32,opt,name=UsageSentNotes" json:"UsageSentNotes,omitempty"` + CellID *string `protobuf:"bytes,33,opt,name=CellID" json:"CellID,omitempty"` + NotificationSession *bool `protobuf:"varint,34,opt,name=NotificationSession" json:"NotificationSession,omitempty"` + Voltage100 *int32 `protobuf:"varint,35,opt,name=Voltage100" json:"Voltage100,omitempty"` + Temp100 *int32 `protobuf:"varint,36,opt,name=Temp100" json:"Temp100,omitempty"` + ContinuousSession *bool `protobuf:"varint,37,opt,name=ContinuousSession" json:"ContinuousSession,omitempty"` + MotionSecs *int64 `protobuf:"varint,38,opt,name=MotionSecs" json:"MotionSecs,omitempty"` + MotionOrientation *string `protobuf:"bytes,39,opt,name=MotionOrientation" json:"MotionOrientation,omitempty"` + SessionTrigger *string `protobuf:"bytes,40,opt,name=SessionTrigger" json:"SessionTrigger,omitempty"` + Voltage1000 *int32 `protobuf:"varint,41,opt,name=Voltage1000" json:"Voltage1000,omitempty"` + Temp1000 *int32 `protobuf:"varint,42,opt,name=Temp1000" json:"Temp1000,omitempty"` + HubSessionFactoryResetID *string `protobuf:"bytes,43,opt,name=HubSessionFactoryResetID" json:"HubSessionFactoryResetID,omitempty"` + HighPowerSecsTotal *uint32 `protobuf:"varint,44,opt,name=HighPowerSecsTotal" json:"HighPowerSecsTotal,omitempty"` + HighPowerSecsData *uint32 `protobuf:"varint,45,opt,name=HighPowerSecsData" json:"HighPowerSecsData,omitempty"` + HighPowerSecsGPS *uint32 `protobuf:"varint,46,opt,name=HighPowerSecsGPS" json:"HighPowerSecsGPS,omitempty"` + HighPowerCyclesTotal *uint32 `protobuf:"varint,47,opt,name=HighPowerCyclesTotal" json:"HighPowerCyclesTotal,omitempty"` + HighPowerCyclesData *uint32 `protobuf:"varint,48,opt,name=HighPowerCyclesData" json:"HighPowerCyclesData,omitempty"` + HighPowerCyclesGPS *uint32 `protobuf:"varint,49,opt,name=HighPowerCyclesGPS" json:"HighPowerCyclesGPS,omitempty"` + DeviceSKU *string `protobuf:"bytes,50,opt,name=DeviceSKU" json:"DeviceSKU,omitempty"` + DeviceFirmware *int64 `protobuf:"varint,51,opt,name=DeviceFirmware" json:"DeviceFirmware,omitempty"` + DevicePIN *string `protobuf:"bytes,52,opt,name=DevicePIN" json:"DevicePIN,omitempty"` + DeviceOrderingCode *string `protobuf:"bytes,53,opt,name=DeviceOrderingCode" json:"DeviceOrderingCode,omitempty"` + UsageRcvdBytesSecondary *uint32 `protobuf:"varint,54,opt,name=UsageRcvdBytesSecondary" json:"UsageRcvdBytesSecondary,omitempty"` + UsageSentBytesSecondary *uint32 `protobuf:"varint,55,opt,name=UsageSentBytesSecondary" json:"UsageSentBytesSecondary,omitempty"` + SuppressResponse *bool `protobuf:"varint,56,opt,name=SuppressResponse" json:"SuppressResponse,omitempty"` + Where *string `protobuf:"bytes,57,opt,name=Where" json:"Where,omitempty"` + WhereWhen *int64 `protobuf:"varint,58,opt,name=WhereWhen" json:"WhereWhen,omitempty"` + HubPacketHandler *string `protobuf:"bytes,59,opt,name=HubPacketHandler" json:"HubPacketHandler,omitempty"` + PowerSource *uint32 `protobuf:"varint,60,opt,name=PowerSource" json:"PowerSource,omitempty"` + PowerMahUsed *float64 `protobuf:"fixed64,61,opt,name=PowerMahUsed" json:"PowerMahUsed,omitempty"` } func (x *NotehubPB) Reset() { @@ -537,11 +539,25 @@ func (x *NotehubPB) GetHubPacketHandler() string { return "" } +func (x *NotehubPB) GetPowerSource() uint32 { + if x != nil && x.PowerSource != nil { + return *x.PowerSource + } + return 0 +} + +func (x *NotehubPB) GetPowerMahUsed() float64 { + if x != nil && x.PowerMahUsed != nil { + return *x.PowerMahUsed + } + return 0 +} + var File_notehub_proto protoreflect.FileDescriptor var file_notehub_proto_rawDesc = []byte{ 0x0a, 0x0d, 0x6e, 0x6f, 0x74, 0x65, 0x68, 0x75, 0x62, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, - 0x07, 0x6e, 0x6f, 0x74, 0x65, 0x6c, 0x69, 0x62, 0x22, 0xc3, 0x11, 0x0a, 0x09, 0x4e, 0x6f, 0x74, + 0x07, 0x6e, 0x6f, 0x74, 0x65, 0x6c, 0x69, 0x62, 0x22, 0x89, 0x12, 0x0a, 0x09, 0x4e, 0x6f, 0x74, 0x65, 0x68, 0x75, 0x62, 0x50, 0x42, 0x12, 0x18, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, @@ -681,7 +697,12 @@ var file_notehub_proto_rawDesc = []byte{ 0x6e, 0x18, 0x3a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x57, 0x68, 0x65, 0x72, 0x65, 0x57, 0x68, 0x65, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x48, 0x75, 0x62, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x18, 0x3b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x48, 0x75, - 0x62, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, + 0x62, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x12, 0x20, + 0x0a, 0x0b, 0x50, 0x6f, 0x77, 0x65, 0x72, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x3c, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x50, 0x6f, 0x77, 0x65, 0x72, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x12, 0x22, 0x0a, 0x0c, 0x50, 0x6f, 0x77, 0x65, 0x72, 0x4d, 0x61, 0x68, 0x55, 0x73, 0x65, 0x64, + 0x18, 0x3d, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0c, 0x50, 0x6f, 0x77, 0x65, 0x72, 0x4d, 0x61, 0x68, + 0x55, 0x73, 0x65, 0x64, } var ( diff --git a/lib/notehub.proto b/lib/notehub.proto index bcccedb..aa95ad4 100755 --- a/lib/notehub.proto +++ b/lib/notehub.proto @@ -67,6 +67,8 @@ message NotehubPB { optional string Where = 57; optional int64 WhereWhen = 58; optional string HubPacketHandler = 59; + optional uint32 PowerSource = 60; + optional double PowerMahUsed = 61; } diff --git a/lib/notelib.go b/lib/notelib.go index 24e7cc8..719941a 100644 --- a/lib/notelib.go +++ b/lib/notelib.go @@ -87,7 +87,14 @@ type Notebox struct { defaultEventDeviceSN string defaultEventProductUID string defaultEventAppUID string + defaultEventTransport string // For HTTP access control clientHTTPReq *http.Request clientHTTPRsp http.ResponseWriter } + +// If it takes more than 20 seconds, we are at risk of hitting the 30-second +// transaction timeout, which could cause massive batches of notes to be +// re-sent and re-enqueued. Make sure this is flagged as an error so that +// the code or operational infrastructure can be fixed. +const transactionErrorDuration = (20 * time.Second) diff --git a/lib/req.go b/lib/req.go index 80710b4..a62f20d 100644 --- a/lib/req.go +++ b/lib/req.go @@ -259,9 +259,9 @@ func (box *Notebox) Request(ctx context.Context, endpointID string, reqJSON []by rsp.Err = "no notefile specified" break } - isQueue, toHub, _, _, _, _ := NotefileAttributesFromID(req.NotefileID) + isQueue, _, _, _, _, _ := NotefileAttributesFromID(req.NotefileID) if req.NoteID == "" { - if !isQueue || !toHub { + if !isQueue { rsp.Err = "no note ID specified" break } @@ -279,8 +279,10 @@ func (box *Notebox) Request(ctx context.Context, endpointID string, reqJSON []by break } } - // Handle queues by getting the LRU note - if isQueue && toHub && req.NoteID == "" { + // Handle queues by getting the LRU note. Note that we allow either inbound + // or outbound queues to be accessed this way via API, explicitly to allow + // the API to be used to manage queues in either direction. + if isQueue && req.NoteID == "" { openfile, file, err2 := box.OpenNotefile(ctx, req.NotefileID) if err2 != nil { rsp.Err = fmt.Sprintf("%s", err2) @@ -354,6 +356,14 @@ func (box *Notebox) Request(ctx context.Context, endpointID string, reqJSON []by req.TrackerID = "^" } + // Special way, for use by support, to get a list of all notefiles + // and their info such as templates. + if req.Full { + allNotefiles := box.NoteboxNotefileDesc() + rsp.FileDesc = &allNotefiles + break + } + // If no tracker, generate the entire list of notefiles tracker := req.TrackerID if tracker == "" { @@ -577,13 +587,14 @@ func (box *Notebox) Request(ctx context.Context, endpointID string, reqJSON []by payload := xnote.GetPayload() info.Payload = &payload } + info.When = xnote.When() // Add it to the list to be returned infolist[noteIDs[i]] = info // Delete it as a side-effect, if desired if req.Delete { - _ = file.DeleteNote(endpointID, noteIDs[i]) + _ = file.DeleteNote(ctx, endpointID, noteIDs[i]) } } diff --git a/lib/wire.go b/lib/wire.go index 2ad7562..a6d7acc 100644 --- a/lib/wire.go +++ b/lib/wire.go @@ -735,6 +735,12 @@ func msgToWire(msg notehubMessage) (wire []byte, wirelen int, err error) { if msg.Temp1000 != 0 { pb.Temp1000 = &msg.Temp1000 } + if msg.PowerSource != 0 { + pb.PowerSource = &msg.PowerSource + } + if msg.PowerMahUsed != 0 { + pb.PowerMahUsed = &msg.PowerMahUsed + } if msg.CellID != "" { pb.CellID = &msg.CellID } @@ -957,40 +963,40 @@ func u32min(x, y uint32) uint32 { } // WireBarsFromSession extracts device's perception of the number of bars of signal from a session -func WireBarsFromSession(session *HubSession) (rat string, bars uint32) { +func WireBarsFromSession(session *note.DeviceSession) (rat string, bars uint32) { // Return the rat for the session - rat = session.Session.Rat + rat = session.Rat // Start by assuming great coverage bars = 4 // Handle GSM OR handle LTE at the state when RSRQ can't be computed - if session.Session.Rsrq == 0 { - if session.Session.Rssi < -70 { + if session.Rsrq == 0 { + if session.Rssi < -70 { bars = 3 } - if session.Session.Rssi < -85 { + if session.Rssi < -85 { bars = 2 } - if session.Session.Rssi < -100 { + if session.Rssi < -100 { bars = 1 } return } // RSRP is an integer indicating the reference signal received power in dBm - if session.Session.Rsrp < -80 { + if session.Rsrp < -80 { bars = u32min(bars, 3) } - if session.Session.Rsrp < -90 { + if session.Rsrp < -90 { bars = u32min(bars, 2) } - if session.Session.Rsrp < -100 { + if session.Rsrp < -100 { bars = u32min(bars, 1) } // SINR is an integer indicating the signal to interference plus noise ratio. // The logarithmic values (0-250) are in 1/5th of a dB, ranging from -20 to +30db - sinr := -20 + (session.Session.Sinr * 5) + sinr := -20 + (session.Sinr * 5) if sinr < 20 { bars = u32min(bars, 3) } @@ -1003,13 +1009,13 @@ func WireBarsFromSession(session *HubSession) (rat string, bars uint32) { // RSRQ is an integer indicating the reference signal received quality (RSRQ) in dB, // which is computed by the formula RSRQ = N*(RSRP/RSSI), where N is the number of // Resource Blocks of the E-UTRA carrier RSSI - if session.Session.Rsrq < -10 { + if session.Rsrq < -10 { bars = u32min(bars, 3) } - if session.Session.Rsrq < -15 { + if session.Rsrq < -15 { bars = u32min(bars, 2) } - if session.Session.Rsrq < -20 { + if session.Rsrq < -20 { bars = u32min(bars, 1) } @@ -1067,9 +1073,13 @@ func WireExtractSessionContext(wire []byte, session *HubSession) (suppressRespon if req.Temp1000 != 0 { session.Session.Temp = float64(req.Temp1000) / 1000 } + session.Session.PowerCharging = (req.PowerSource & NotecardPowerCharging) != 0 + session.Session.PowerUsb = (req.PowerSource & NotecardPowerUsb) != 0 + session.Session.PowerPrimary = (req.PowerSource & NotecardPowerPrimary) != 0 + session.Session.PowerMahUsed = req.PowerMahUsed session.Session.Moved = req.MotionSecs session.Session.Orientation = req.MotionOrientation - session.Session.Trigger = req.SessionTrigger + session.Session.WhySessionOpened = req.SessionTrigger session.Notification = req.NotificationSession session.Session.ContinuousSession = req.ContinuousSession if req.MessageType == msgDiscover { @@ -1155,25 +1165,77 @@ func WireExtractSessionContext(wire []byte, session *HubSession) (suppressRespon session.Session.Iccid = str1 session.Session.Apn = str2 } + + // Assign the base transport type + session.Session.Transport = rat + switch session.Session.Transport { + case "soft": + session.Session.Transport = "simulator" + case "gsm": + session.Session.Transport = "cell:gsm" + case "nbiot": + session.Session.Transport = "cell:nbiot" + case "cdma": + session.Session.Transport = "cell:cdma" + case "umts": + session.Session.Transport = "cell:umts" + case "emtc": + session.Session.Transport = "cell:emtc" + case "lte": + session.Session.Transport = "cell:lte" + } + + // Fix up bearer for old notecards that accidentally zero'ed out + // the bearer field rather than setting it to UNKNOWN (-1) + if bearer == notecard.NetworkBearerGsm && rat != "gsm" { + bearer = notecard.NetworkBearerUnknown + } + + // Other cleanups based on bearer switch bearer { case notecard.NetworkBearerGsm: session.Session.Bearer = "GSM" + if rat != "gsm" { + session.Session.Transport += ":gsm" + } case notecard.NetworkBearerTdScdma: session.Session.Bearer = "TD-SCDMA" + session.Session.Transport += ":td-scdma" case notecard.NetworkBearerWcdma: session.Session.Bearer = "WCDMA" + session.Session.Transport += ":wcdma" case notecard.NetworkBearerCdma2000: session.Session.Bearer = "CDMA2000" + session.Session.Transport += ":cdma2000" case notecard.NetworkBearerWiMax: session.Session.Bearer = "WIMAX" + session.Session.Transport += ":wimax" case notecard.NetworkBearerLteTdd: session.Session.Bearer = "LTE TDD" + if rat == "lte" { + session.Session.Transport += ":tdd" + } else { + session.Session.Transport += ":lte-tdd" + } case notecard.NetworkBearerLteFdd: session.Session.Bearer = "LTE FDD" + if rat == "lte" { + session.Session.Transport += ":fdd" + } else { + session.Session.Transport += ":lte-fdd" + } case notecard.NetworkBearerNBIot: session.Session.Bearer = "NB-IoT" + if rat != "nbiot" { + session.Session.Transport += ":nbiot" + } case notecard.NetworkBearerWLan: session.Session.Bearer = "WiFi" + if rat == "wifi-2.4" { + session.Session.Transport = "wifi" + } else if session.Session.Transport != "wifi" { + session.Session.Transport += ":wifi" + } case notecard.NetworkBearerBluetooth: session.Session.Bearer = "Bluetooth" case notecard.NetworkBearerIeee802p15p4: @@ -1277,6 +1339,8 @@ func msgFromWire(wire []byte) (msg notehubMessage, wirelen int, err error) { msg.Temp100 = pb.GetTemp100() msg.Voltage1000 = pb.GetVoltage1000() msg.Temp1000 = pb.GetTemp1000() + msg.PowerSource = pb.GetPowerSource() + msg.PowerMahUsed = pb.GetPowerMahUsed() msg.UsageProvisioned = pb.GetUsageProvisioned() msg.UsageRcvdBytes = pb.GetUsageRcvdBytes() msg.UsageSentBytes = pb.GetUsageSentBytes() @@ -1363,7 +1427,7 @@ func msgFromWire(wire []byte) (msg notehubMessage, wirelen int, err error) { } // WireReadRequest reads a message from the specified reader -func WireReadRequest(conn net.Conn, waitIndefinitely bool) (bytesRead uint32, request []byte, err error) { +func WireReadRequest(ctx context.Context, conn net.Conn, waitIndefinitely bool) (bytesRead uint32, request []byte, err error) { var n int var version []byte @@ -1376,13 +1440,16 @@ func WireReadRequest(conn net.Conn, waitIndefinitely bool) (bytesRead uint32, re var err2 error versionLen := 1 version = make([]byte, versionLen) - _ = conn.SetReadDeadline(time.Now().Add(timeoutDuration)) + if err := conn.SetReadDeadline(time.Now().Add(timeoutDuration)); err != nil { + logError(ctx, "SetReadDeadline: %v", err) + } + n, err2 = rdconn.Read(version) - if debugWireRead { - if err2 == nil { - logDebug(context.Background(), "\n\nrdVersion(%d) %d", len(version), n) - } + + if debugWireRead && err2 == nil { + logDebug(ctx, "\n\nrdVersion(%d) %d", len(version), n) } + if err2, ok := err2.(net.Error); ok && err2.Timeout() { if !waitIndefinitely { err = fmt.Errorf("wire read: " + note.ErrTimeout + " timeout on read") @@ -1406,7 +1473,7 @@ func WireReadRequest(conn net.Conn, waitIndefinitely bool) (bytesRead uint32, re break } - // Process the version byte to determine the length of th header that follows + // Process the version byte to determine the length of the header that follows isValidVersion, headerLength := wireProcessVersionByte(version[0]) if !isValidVersion { err = fmt.Errorf("wire read: unrecognized protocol") @@ -1415,16 +1482,18 @@ func WireReadRequest(conn net.Conn, waitIndefinitely bool) (bytesRead uint32, re // Read the header header := make([]byte, headerLength) - _ = conn.SetReadDeadline(time.Now().Add(timeoutDuration)) + if err := conn.SetReadDeadline(time.Now().Add(timeoutDuration)); err != nil { + logError(ctx, "SetReadDeadline: %v", err) + } if debugWireRead { - logDebug(context.Background(), "rdHeader(%d)", len(header)) + logDebug(ctx, "rdHeader(%d)", len(header)) } n, err = io.ReadFull(rdconn, header) if debugWireRead { if err == nil { - logDebug(context.Background(), "rdHeader(%d) %d", len(header), n) + logDebug(ctx, "rdHeader(%d) %d", len(header), n) } else { - logWarn(context.Background(), "rdHeader(%d) %d %s", len(header), n, err) + logWarn(ctx, "rdHeader(%d) %d %s", len(header), n, err) } } if err != nil { @@ -1447,16 +1516,18 @@ func WireReadRequest(conn net.Conn, waitIndefinitely bool) (bytesRead uint32, re var protobuf []byte if protobufLength != 0 { protobuf = make([]byte, protobufLength) - _ = conn.SetReadDeadline(time.Now().Add(timeoutDuration)) + if err := conn.SetReadDeadline(time.Now().Add(timeoutDuration)); err != nil { + logError(ctx, "SetReadDeadline: %v", err) + } if debugWireRead { - logDebug(context.Background(), "rdProtobuf(%d)", len(protobuf)) + logDebug(ctx, "rdProtobuf(%d)", len(protobuf)) } n, err = io.ReadFull(rdconn, protobuf) if debugWireRead { if err == nil { - logDebug(context.Background(), "rdProtobuf(%d) %d", len(protobuf), n) + logDebug(ctx, "rdProtobuf(%d) %d", len(protobuf), n) } else { - logWarn(context.Background(), "rdProtobuf(%d) %d %s", len(protobuf), n, err) + logWarn(ctx, "rdProtobuf(%d) %d %s", len(protobuf), n, err) } } if err != nil { @@ -1481,16 +1552,18 @@ func WireReadRequest(conn net.Conn, waitIndefinitely bool) (bytesRead uint32, re var binary []byte if binaryLength != 0 { binary = make([]byte, binaryLength) - _ = conn.SetReadDeadline(time.Now().Add(timeoutDuration)) + if err := conn.SetReadDeadline(time.Now().Add(timeoutDuration)); err != nil { + logError(ctx, "SetReadDeadline: %v", err) + } if debugWireRead { - logDebug(context.Background(), "rdBinary(%d)", len(binary)) + logDebug(ctx, "rdBinary(%d)", len(binary)) } n, err = io.ReadFull(rdconn, binary) if debugWireRead { if err == nil { - logDebug(context.Background(), "rdBinary(%d) %d", len(binary), n) + logDebug(ctx, "rdBinary(%d) %d", len(binary), n) } else { - logWarn(context.Background(), "rdBinary(%d) %d %s", len(binary), n, err) + logWarn(ctx, "rdBinary(%d) %d %s", len(binary), n, err) } } if err != nil {