util/thash: add a length getter fn

In order to have access to the length of datatypes with variable lengths
to correctly update memuse to calculate memcaps.

Bug 3910

src/app-layer-htp-range.c

@@ -172,8 +172,8 @@ void HttpRangeContainersInit(void)
 
     ContainerUrlRangeList.ht = THashInit("app-layer.protocols.http.byterange",
             sizeof(HttpRangeContainerFile), ContainerUrlRangeSet, ContainerUrlRangeFree,
-            ContainerUrlRangeHash, ContainerUrlRangeCompare, ContainerValueRangeTimeout, false,
-            memcap, CONTAINER_URLRANGE_HASH_SIZE);
+            ContainerUrlRangeHash, ContainerUrlRangeCompare, ContainerValueRangeTimeout, NULL,
+            false, memcap, CONTAINER_URLRANGE_HASH_SIZE);
     ContainerUrlRangeList.timeout = timeout;
 
     SCLogDebug("containers started");

src/datasets-string.c

@@ -98,6 +98,12 @@ uint32_t StringHash(void *s)
     return hash;
 }
 
+uint32_t StringGetLength(void *s)
+{
+    StringType *str = s;
+    return str->len;
+}
+
 // base data stays in hash
 void StringFree(void *s)
 {

src/datasets-string.h

@@ -35,6 +35,7 @@ typedef struct StringType {
 int StringSet(void *dst, void *src);
 bool StringCompare(void *a, void *b);
 uint32_t StringHash(void *s);
+uint32_t StringGetLength(void *s);
 void StringFree(void *s);
 int StringAsBase64(const void *s, char *out, size_t out_size);
 

src/datasets.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2017-2020 Open Information Security Foundation
+/* Copyright (C) 2017-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -701,7 +701,8 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
     switch (type) {
         case DATASET_TYPE_MD5:
             set->hash = THashInit(cnf_name, sizeof(Md5Type), Md5StrSet, Md5StrFree, Md5StrHash,
-                    Md5StrCompare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    Md5StrCompare, NULL, NULL, load != NULL ? 1 : 0,
+                    memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
@@ -710,7 +711,8 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_STRING:
             set->hash = THashInit(cnf_name, sizeof(StringType), StringSet, StringFree, StringHash,
-                    StringCompare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    StringCompare, NULL, StringGetLength, load != NULL ? 1 : 0,
+                    memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
@@ -719,7 +721,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_SHA256:
             set->hash = THashInit(cnf_name, sizeof(Sha256Type), Sha256StrSet, Sha256StrFree,
-                    Sha256StrHash, Sha256StrCompare, NULL, load != NULL ? 1 : 0,
+                    Sha256StrHash, Sha256StrCompare, NULL, NULL, load != NULL ? 1 : 0,
                     memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
@@ -728,18 +730,20 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
                 goto out_err;
             break;
         case DATASET_TYPE_IPV4:
-            set->hash = THashInit(cnf_name, sizeof(IPv4Type), IPv4Set, IPv4Free, IPv4Hash,
-                    IPv4Compare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
-                    hashsize > 0 ? hashsize : default_hashsize);
+            set->hash =
+                    THashInit(cnf_name, sizeof(IPv4Type), IPv4Set, IPv4Free, IPv4Hash, IPv4Compare,
+                            NULL, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                            hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
             if (DatasetLoadIPv4(set) < 0)
                 goto out_err;
             break;
         case DATASET_TYPE_IPV6:
-            set->hash = THashInit(cnf_name, sizeof(IPv6Type), IPv6Set, IPv6Free, IPv6Hash,
-                    IPv6Compare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
-                    hashsize > 0 ? hashsize : default_hashsize);
+            set->hash =
+                    THashInit(cnf_name, sizeof(IPv6Type), IPv6Set, IPv6Free, IPv6Hash, IPv6Compare,
+                            NULL, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                            hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
             if (DatasetLoadIPv6(set) < 0)

src/detect-engine-threshold.c

@@ -214,8 +214,8 @@ static int ThresholdsInit(struct Thresholds *t)
     }
 
     t->thash = THashInit("thresholds", sizeof(ThresholdEntry), ThresholdEntrySet,
-            ThresholdEntryFree, ThresholdEntryHash, ThresholdEntryCompare, ThresholdEntryExpire, 0,
-            memcap, hashsize);
+            ThresholdEntryFree, ThresholdEntryHash, ThresholdEntryCompare, ThresholdEntryExpire,
+            NULL, 0, memcap, hashsize);
     if (t->thash == NULL) {
         SCLogError("failed to initialize thresholds hash table");
         return -1;

src/util-thash.c

@@ -295,7 +295,7 @@ static int THashInitConfig(THashTableContext *ctx, const char *cnf_prefix)
 THashTableContext *THashInit(const char *cnf_prefix, size_t data_size,
         int (*DataSet)(void *, void *), void (*DataFree)(void *), uint32_t (*DataHash)(void *),
         bool (*DataCompare)(void *, void *), bool (*DataExpired)(void *, SCTime_t),
-        bool reset_memcap, uint64_t memcap, uint32_t hashsize)
+        uint32_t (*DataSize)(void *), bool reset_memcap, uint64_t memcap, uint32_t hashsize)
 {
     THashTableContext *ctx = SCCalloc(1, sizeof(*ctx));
     BUG_ON(!ctx);
@@ -306,6 +306,7 @@ THashTableContext *THashInit(const char *cnf_prefix, size_t data_size,
     ctx->config.DataHash = DataHash;
     ctx->config.DataCompare = DataCompare;
     ctx->config.DataExpired = DataExpired;
+    ctx->config.DataSize = DataSize;
 
     /* set defaults */
     ctx->config.hash_rand = (uint32_t)RandomGet();

src/util-thash.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2016 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -133,6 +133,7 @@ typedef struct THashDataConfig_ {
     uint32_t (*DataHash)(void *);
     bool (*DataCompare)(void *, void *);
     bool (*DataExpired)(void *, SCTime_t ts);
+    uint32_t (*DataSize)(void *);
 } THashConfig;
 
 #define THASH_DATA_SIZE(ctx) (sizeof(THashData) + (ctx)->config.data_size)
@@ -171,8 +172,8 @@ typedef struct THashTableContext_ {
 THashTableContext *THashInit(const char *cnf_prefix, size_t data_size,
         int (*DataSet)(void *dst, void *src), void (*DataFree)(void *),
         uint32_t (*DataHash)(void *), bool (*DataCompare)(void *, void *),
-        bool (*DataExpired)(void *, SCTime_t), bool reset_memcap, uint64_t memcap,
-        uint32_t hashsize);
+        bool (*DataExpired)(void *, SCTime_t), uint32_t (*DataSize)(void *), bool reset_memcap,
+        uint64_t memcap, uint32_t hashsize);
 
 void THashShutdown(THashTableContext *ctx);