Implement string compression when it is benefitial.

README.md

@@ -7,7 +7,7 @@ All of the serializers and deserializers were manually written, no automatic too
 ### Currently serializable primitives:
     - boolean
     - number (Automatically serializable as a float64 if not given a specific size)
-    - string
+    - string (May be compressesd using lzw when benefitial)
     - vector (Vector3's on Roblox)
     - table
     - buffer
@@ -21,7 +21,7 @@ Parsing `userdata` value types is not currently on the roadmap, and it may be tr
 ## How to install?
 - Download the `.rbxm` file on the latest release of this project.
 - Using wally:
-    - Add `tabletobuffer = "secondnewtonlaw/[email protected].1"`
+    - Add `tabletobuffer = "secondnewtonlaw/[email protected].2"`
 
 ## [How to use this?](https://secondnewtonlaw.github.io/TableToBuffer/)
 

lib/External/lualzw.lua

@@ -0,0 +1,164 @@
+--[[
+MIT License
+
+Copyright (c) 2016 Rochet2
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+]]
+
+local char = string.char
+local type = type
+local sub = string.sub
+local tconcat = table.concat
+
+local basedictcompress = {}
+local basedictdecompress = {}
+for i = 0, 255 do
+	local ic, iic = char(i), char(i, 0)
+	basedictcompress[ic] = iic
+	basedictdecompress[iic] = ic
+end
+
+local function dictAddA(str, dict, a, b)
+	if a >= 256 then
+		a, b = 0, b + 1
+		if b >= 256 then
+			dict = {}
+			b = 1
+		end
+	end
+	dict[str] = char(a, b)
+	a = a + 1
+	return dict, a, b
+end
+
+local function compress(input)
+	if type(input) ~= "string" then
+		return nil, "string expected, got " .. type(input)
+	end
+	local len = #input
+	if len <= 1 then
+		return "u" .. input
+	end
+
+	local dict = {}
+	local a, b = 0, 1
+
+	local result = { "c" }
+	local resultlen = 1
+	local n = 2
+	local word = ""
+	for i = 1, len do
+		local c = sub(input, i, i)
+		local wc = word .. c
+		if not (basedictcompress[wc] or dict[wc]) then
+			local write = basedictcompress[word] or dict[word]
+			if not write then
+				return nil, "algorithm error, could not fetch word"
+			end
+			result[n] = write
+			resultlen = resultlen + #write
+			n = n + 1
+			if len <= resultlen then
+				return "u" .. input
+			end
+			dict, a, b = dictAddA(wc, dict, a, b)
+			word = c
+		else
+			word = wc
+		end
+	end
+	result[n] = basedictcompress[word] or dict[word]
+	resultlen = resultlen + #result[n]
+	n = n + 1
+	if len <= resultlen then
+		return "u" .. input
+	end
+	return tconcat(result)
+end
+
+local function dictAddB(str, dict, a, b)
+	if a >= 256 then
+		a, b = 0, b + 1
+		if b >= 256 then
+			dict = {}
+			b = 1
+		end
+	end
+	dict[char(a, b)] = str
+	a = a + 1
+	return dict, a, b
+end
+
+local function decompress(input)
+	if type(input) ~= "string" then
+		return nil, "string expected, got " .. type(input)
+	end
+
+	if #input < 1 then
+		return nil, "invalid input - not a compressed string"
+	end
+
+	local control = sub(input, 1, 1)
+	if control == "u" then
+		return sub(input, 2)
+	elseif control ~= "c" then
+		return nil, "invalid input - not a compressed string"
+	end
+	input = sub(input, 2)
+	local len = #input
+
+	if len < 2 then
+		return nil, "invalid input - not a compressed string"
+	end
+
+	local dict = {}
+	local a, b = 0, 1
+
+	local result = {}
+	local n = 1
+	local last = sub(input, 1, 2)
+	result[n] = basedictdecompress[last] or dict[last]
+	n = n + 1
+	for i = 3, len, 2 do
+		local code = sub(input, i, i + 1)
+		local lastStr = basedictdecompress[last] or dict[last]
+		if not lastStr then
+			return nil, "could not find last from dict. Invalid input?"
+		end
+		local toAdd = basedictdecompress[code] or dict[code]
+		if toAdd then
+			result[n] = toAdd
+			n = n + 1
+			dict, a, b = dictAddB(lastStr .. sub(toAdd, 1, 1), dict, a, b)
+		else
+			local tmp = lastStr .. sub(lastStr, 1, 1)
+			result[n] = tmp
+			n = n + 1
+			dict, a, b = dictAddB(tmp, dict, a, b)
+		end
+		last = code
+	end
+	return tconcat(result)
+end
+
+return {
+	compress = compress,
+	decompress = decompress,
+}

lib/Requests/ReadString.luau

@@ -1,4 +1,6 @@
 --!strict
+local lualzw = require(script.Parent.Parent.External.lualzw)
+
 local request = {}
 
 function request.create(offset, maxStringSize)
@@ -16,22 +18,39 @@ function request.create(offset, maxStringSize)
 						.. maxStringSize
 				)
 			end
-			buffer.writeu32(buf, offset, maxStringSize)
-			buffer.writestring(buf, offset + 0x4, str, maxStringSize > #str and #str or maxStringSize)
+			local compressedStr = lualzw.compress(str)
+			local writeString = str
+			if #compressedStr > #str then
+				-- We only want to save a compressed string if it benefits us, thus, we want to leave a little flag for it to mark it, we can implement more compression algorithms in the future, and denominatee whcih to use based on this little u8.
+				writeString = str
+				buffer.writeu8(buf, offset, 0)
+			else
+				writeString = compressedStr
+				buffer.writeu8(buf, offset, 1)
+			end
+			buffer.writeu32(buf, offset + 0x1, maxStringSize)
+			buffer.writestring(
+				buf,
+				offset + 0x1 + 0x4,
+				writeString,
+				maxStringSize > #writeString and #writeString or maxStringSize
+			)
 		end,
 		--[=[
             Reads the string from the buffer.
         ]=]
 		read = function(buf: buffer): string
-			local sizeHeader = buffer.readu32(buf, offset)
-			local nStr = buffer.readstring(buf, offset + 0x4, sizeHeader)
-			return nStr
+			local isCompressed = buffer.readu8(buf, offset)
+			local sizeHeader = buffer.readu32(buf, offset + 0x1)
+			local nStr = buffer.readstring(buf, offset + 0x1 + 0x4, sizeHeader)
+
+			return isCompressed and lualzw.decompress(nStr) or nStr
 		end,
 		--[=[
             Returns the amount to increase the offset by after this operation.
         ]=]
 		advanceBy = function(_: buffer?): number
-			return maxStringSize + 0x4
+			return maxStringSize + 0x4 + 0x1
 		end,
 	}
 end

lib/SerializerBuilder.luau

@@ -200,7 +200,7 @@ function serializerBuilder.Create<T>()
 
 		-- uint32 = string_size
 		-- ...    = string
-		bufferSize += 0x4 + maxStringSize
+		bufferSize += 0x1 + 0x4 + maxStringSize
 		return __self
 	end
 

test/test_buffer.client.luau

@@ -1,3 +1,5 @@
+--!nocheck
+
 local ReplicatedStorage = game:GetService("ReplicatedStorage")
 
 local tableToBuffer = require(ReplicatedStorage.Packages.TableToBuffer)
@@ -62,11 +64,11 @@ do
 		NestedSerializer = table.clone(table_1),
 	}
 
-	print(buffer.tostring(serializer_1.Serialize(table_1)))
-	print(serializer_1.Deserialize(serializer_1.Serialize(table_1)))
+	--print(buffer.tostring(serializer_1.Serialize(table_1)))
+	--print(serializer_1.Deserialize(serializer_1.Serialize(table_1)))
 
-	print(buffer.tostring(serializer_2.Serialize(table_2)))
-	print(serializer_2.Deserialize(serializer_2.Serialize(table_2)))
+	--print(buffer.tostring(serializer_2.Serialize(table_2)))
+	--print(serializer_2.Deserialize(serializer_2.Serialize(table_2)))
 end
 
 do
@@ -78,13 +80,13 @@ do
 				Key = string.rep("C", 1024),
 			},
 		},
-		ThisIsAnotherIndex = "Hello!",
+		ThisIsAnotherIndex = "Hello, World!",
 		ThisIsABoolean = false,
 	})
 	print(tostring(tableAsBuffer))
 
 	tableAsBuffer.ThisIsAnIndex = {
-		AnotherIndex = "New Data!",
+		AnotherIndex = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
 		AnotherExtraIndex = { Key = string.rep("A", 1024) },
 	}
 
@@ -93,7 +95,7 @@ do
 	tableAsBuffer.ThisIsABoolean = true
 	print(tostring(tableAsBuffer))
 	tableAsBuffer.ThisIsAnIndex = {
-		AnotherIndex = "Adios",
+		AnotherIndex = "THIS IS A NORMAL STRING",
 		AnotherExtraIndex = {
 			Key = string.rep("B", 1024),
 		},

wally.toml

@@ -1,7 +1,7 @@
 [package]
 name = "secondnewtonlaw/tabletobuffer"
 description = "A utility to convert tables into buffers"
-version = "0.1.1"
+version = "0.1.2"
 registry = "https://github.com/UpliftGames/wally-index"
 realm = "shared"
 license = "MIT"