Skip to content

Commit 62aad2e

Browse files
JohnMcPMSJohnMcPMS
authored
Tool to investigate SQLite compression (#6074)
In an effort to investigate some SQLite compression improvements, this tool was created and some changes were made to that are being left as a null result. The investigated changes both increased the compressed file size: - Increased page size to maximum - Move the package hashes to their own table
1 parent 4087437 commit 62aad2e

File tree

14 files changed

+965
-5
lines changed

14 files changed

+965
-5
lines changed

‎.github/actions/spelling/expect.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,7 @@ ewgs
169169
execustom
170170
EXEHASH
171171
experimentalfeatures
172+
XPRESS
172173
fdw
173174
fdwgp
174175
FECAFEB

‎src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,18 @@
88

99
namespace AppInstaller::Repository::Microsoft
1010
{
11+
namespace
12+
{
13+
size_t GetPageSizeFromOptions(SQLiteIndex::CreateOptions options)
14+
{
15+
return WI_IsFlagSet(options, SQLiteIndex::CreateOptions::LargePageSize) ? 65536 : 0;
16+
}
17+
}
18+
1119
SQLiteIndex SQLiteIndex::CreateNew(const std::string& filePath, SQLite::Version version, CreateOptions options)
1220
{
1321
AICLI_LOG(Repo, Info, << "Creating new SQLite Index with version [" << version << "] at '" << filePath << "'");
14-
SQLiteIndex result{ filePath, version };
22+
SQLiteIndex result{ filePath, version, options };
1523

1624
SQLite::Savepoint savepoint = SQLite::Savepoint::Create(result.m_dbconn, "sqliteindex_createnew");
1725

@@ -37,7 +45,7 @@ namespace AppInstaller::Repository::Microsoft
3745
return { filePath, source };
3846
}
3947

40-
SQLiteIndex::SQLiteIndex(const std::string& target, const SQLite::Version& version) : SQLiteStorageBase(target, version)
48+
SQLiteIndex::SQLiteIndex(const std::string& target, const SQLite::Version& version, CreateOptions options) : SQLiteStorageBase(target, version, GetPageSizeFromOptions(options))
4149
{
4250
m_dbconn.EnableICU();
4351
m_interface = Schema::CreateISQLiteIndex(version);

‎src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ namespace AppInstaller::Repository::Microsoft
175175

176176
private:
177177
// Constructor used to create a new index.
178-
SQLiteIndex(const std::string& target, const SQLite::Version& version);
178+
SQLiteIndex(const std::string& target, const SQLite::Version& version, CreateOptions options);
179179

180180
// Constructor used to open an existing index.
181181
SQLiteIndex(const std::string& target, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile);

‎src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ namespace AppInstaller::Repository::Microsoft::Schema
5252
SupportPathless = 0x1,
5353
// Disable support for dependencies
5454
DisableDependenciesSupport = 0x2,
55+
// Use maximum page size in SQLite.
56+
// This was part of an exploration of ways to reduce file size but ultimately led to a larger
57+
// compressed file with a worse ratio (limited testing but was significant enough to warrant abandonment).
58+
// Leaving this here as a valid null result to prevent future maintainers from needing to investigate.
59+
LargePageSize = 0x4,
5560
};
5661

5762
// Contains both the object representation of the version key and the rows.

‎src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ namespace AppInstaller::SQLite
3939
static void RenameSQLiteDatabase(const std::filesystem::path& source, const std::filesystem::path& destination, bool overwrite = false);
4040

4141
protected:
42-
SQLiteStorageBase(const std::string& target, const Version& version);
42+
SQLiteStorageBase(const std::string& target, const Version& version, size_t pageSize = 0);
4343

4444
SQLiteStorageBase(const std::string& filePath, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile);
4545

‎src/AppInstallerSharedLib/Public/winget/SQLiteWrapper.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,11 @@ namespace AppInstaller::SQLite
264264
// Must be performed outside of a transaction.
265265
bool SetJournalMode(std::string_view mode);
266266

267+
// Sets the page size for a new, empty database.
268+
// Must be called before the first write to take effect.
269+
// Must be a power of two between 512 and 65536 (inclusive), but we let SQLite enforce that.
270+
void SetPageSize(size_t pageSize);
271+
267272
operator sqlite3* () const { return m_dbconn->Get(); }
268273

269274
protected:

‎src/AppInstallerSharedLib/SQLiteStorageBase.cpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,10 +156,14 @@ namespace AppInstaller::SQLite
156156
m_version = Version::GetSchemaVersion(m_dbconn);
157157
}
158158

159-
SQLiteStorageBase::SQLiteStorageBase(const std::string& target, const Version& version) :
159+
SQLiteStorageBase::SQLiteStorageBase(const std::string& target, const Version& version, size_t pageSize) :
160160
m_dbconn(SQLite::Connection::Create(target, SQLite::Connection::OpenDisposition::Create))
161161
{
162162
m_version = version;
163+
if (pageSize > 0)
164+
{
165+
m_dbconn.SetPageSize(pageSize);
166+
}
163167
MetadataTable::Create(m_dbconn);
164168

165169
// Write a new identifier for this database

‎src/AppInstallerSharedLib/SQLiteWrapper.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,15 @@ namespace AppInstaller::SQLite
251251
return ToLower(setJournalMode.GetColumn<std::string>(0)) == ToLower(mode);
252252
}
253253

254+
void Connection::SetPageSize(size_t pageSize)
255+
{
256+
std::ostringstream stream;
257+
stream << "PRAGMA page_size=" << pageSize;
258+
259+
Statement setPageSize = Statement::Create(*this, stream.str());
260+
setPageSize.Step();
261+
}
262+
254263
std::shared_ptr<details::SharedConnection> Connection::GetSharedConnection() const
255264
{
256265
return m_dbconn;

‎tools/IndexComparisonTool/IndexComparisonTool.vcxproj‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project DefaultTargets="Build" ToolsVersion="16.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<ItemGroup Label="ProjectConfigurations">
4+
<ProjectConfiguration Include="Debug|ARM64">
5+
<Configuration>Debug</Configuration>
6+
<Platform>ARM64</Platform>
7+
</ProjectConfiguration>
8+
<ProjectConfiguration Include="Debug|x64">
9+
<Configuration>Debug</Configuration>
10+
<Platform>x64</Platform>
11+
</ProjectConfiguration>
12+
<ProjectConfiguration Include="Release|ARM64">
13+
<Configuration>Release</Configuration>
14+
<Platform>ARM64</Platform>
15+
</ProjectConfiguration>
16+
<ProjectConfiguration Include="Release|x64">
17+
<Configuration>Release</Configuration>
18+
<Platform>x64</Platform>
19+
</ProjectConfiguration>
20+
</ItemGroup>
21+
22+
<PropertyGroup Label="Globals">
23+
<VCProjectVersion>16.0</VCProjectVersion>
24+
<ProjectGuid>{2F8A1C3E-7B4D-4E9F-A6C2-1D5E8B3F0A7C}</ProjectGuid>
25+
<Keyword>Win32Proj</Keyword>
26+
<RootNamespace>IndexComparisonTool</RootNamespace>
27+
<WindowsTargetPlatformVersion>10.0.26100.0</WindowsTargetPlatformVersion>
28+
<WindowsTargetPlatformMinVersion>10.0.17763.0</WindowsTargetPlatformMinVersion>
29+
<WindowsSDKDesktopARM64Support>true</WindowsSDKDesktopARM64Support>
30+
</PropertyGroup>
31+
32+
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
33+
34+
<PropertyGroup Label="Configuration">
35+
<ConfigurationType>Application</ConfigurationType>
36+
<CharacterSet>Unicode</CharacterSet>
37+
<PlatformToolset>v143</PlatformToolset>
38+
</PropertyGroup>
39+
<PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
40+
<UseDebugLibraries>true</UseDebugLibraries>
41+
<LinkIncremental>true</LinkIncremental>
42+
</PropertyGroup>
43+
<PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
44+
<UseDebugLibraries>false</UseDebugLibraries>
45+
<WholeProgramOptimization>true</WholeProgramOptimization>
46+
<LinkIncremental>false</LinkIncremental>
47+
</PropertyGroup>
48+
49+
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
50+
51+
<ImportGroup Label="PropertySheets">
52+
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props"
53+
Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')"
54+
Label="LocalAppDataPlatform" />
55+
</ImportGroup>
56+
57+
<PropertyGroup Label="UserMacros" />
58+
59+
<PropertyGroup>
60+
<OutDir>$(ProjectDir)bin\$(Platform)\$(Configuration)\</OutDir>
61+
<IntDir>$(ProjectDir)obj\$(Platform)\$(Configuration)\</IntDir>
62+
</PropertyGroup>
63+
64+
<ItemDefinitionGroup>
65+
<ClCompile>
66+
<PrecompiledHeader>Use</PrecompiledHeader>
67+
<PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
68+
<PrecompiledHeaderOutputFile>$(IntDir)pch.pch</PrecompiledHeaderOutputFile>
69+
<WarningLevel>Level4</WarningLevel>
70+
<TreatWarningAsError>true</TreatWarningAsError>
71+
<SDLCheck>true</SDLCheck>
72+
<AdditionalOptions>/permissive- /std:c++17 %(AdditionalOptions)</AdditionalOptions>
73+
<!-- WinGetUtil.h is local to this project; no extra include dir needed -->
74+
<PreprocessorDefinitions>UNICODE;_UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
75+
</ClCompile>
76+
<Link>
77+
<SubSystem>Console</SubSystem>
78+
<!-- Cabinet.lib and winsqlite3.lib are pulled in via #pragma comment(lib) in main.cpp -->
79+
<!-- WinGetUtil.dll is loaded at runtime via LoadLibrary; no import lib needed -->
80+
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
81+
</Link>
82+
</ItemDefinitionGroup>
83+
84+
<ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
85+
<ClCompile>
86+
<Optimization>Disabled</Optimization>
87+
<PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
88+
</ClCompile>
89+
</ItemDefinitionGroup>
90+
91+
<ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
92+
<ClCompile>
93+
<Optimization>MaxSpeed</Optimization>
94+
<FunctionLevelLinking>true</FunctionLevelLinking>
95+
<IntrinsicFunctions>true</IntrinsicFunctions>
96+
<PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
97+
</ClCompile>
98+
<Link>
99+
<EnableCOMDATFolding>true</EnableCOMDATFolding>
100+
<OptimizeReferences>true</OptimizeReferences>
101+
</Link>
102+
</ItemDefinitionGroup>
103+
104+
<ItemGroup>
105+
<ClInclude Include="pch.h" />
106+
<ClInclude Include="WinGetUtil.h" />
107+
</ItemGroup>
108+
<ItemGroup>
109+
<ClCompile Include="main.cpp" />
110+
<ClCompile Include="pch.cpp">
111+
<PrecompiledHeader>Create</PrecompiledHeader>
112+
</ClCompile>
113+
</ItemGroup>
114+
115+
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
116+
</Project>

‎tools/IndexComparisonTool/IndexComparisonTool.vcxproj.filters‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<ItemGroup>
4+
<Filter Include="Source Files">
5+
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
6+
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
7+
</Filter>
8+
<Filter Include="Header Files">
9+
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
10+
<Extensions>h;hh;hpp;hxx;hm;inl;inc;ipp;xsd</Extensions>
11+
</Filter>
12+
</ItemGroup>
13+
<ItemGroup>
14+
<ClCompile Include="main.cpp">
15+
<Filter>Source Files</Filter>
16+
</ClCompile>
17+
<ClCompile Include="pch.cpp">
18+
<Filter>Source Files</Filter>
19+
</ClCompile>
20+
</ItemGroup>
21+
<ItemGroup>
22+
<ClInclude Include="pch.h">
23+
<Filter>Header Files</Filter>
24+
</ClInclude>
25+
<ClInclude Include="WinGetUtil.h">
26+
<Filter>Header Files</Filter>
27+
</ClInclude>
28+
</ItemGroup>
29+
</Project>

0 commit comments

Comments
 (0)