Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

v3 Proof of Concept #199

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
121 commits
Select commit Hold shift + click to select a range
a640063
Add .gitattributes and .gitignore.
jeremy-visionaid Oct 10, 2024
54e3029
Add project files.
jeremy-visionaid Oct 10, 2024
1e7ff7f
Read/write headers
jeremy-visionaid Oct 10, 2024
43e3292
Read/write directory entries
jeremy-visionaid Oct 10, 2024
95a0c77
Enumerate directory entries
jeremy-visionaid Oct 10, 2024
264fe6c
Open and read streams
jeremy-visionaid Oct 10, 2024
e9baa90
Add benchmark
jeremy-visionaid Oct 10, 2024
aa8fd9e
Cache sectors
jeremy-visionaid Oct 10, 2024
c58349a
Add IOContext
jeremy-visionaid Oct 13, 2024
2e69028
Add enumerators
jeremy-visionaid Oct 13, 2024
cacc76a
Move EntryInfo enumerator etc. to Storage
jeremy-visionaid Oct 14, 2024
7821a5d
Improve Sector validation
jeremy-visionaid Oct 14, 2024
b2fc989
Allow storage directory entry enumeration
jeremy-visionaid Oct 14, 2024
4d511e0
Improve enumerator validation
jeremy-visionaid Oct 14, 2024
6c13bec
Read mini FAT sectors, chains and streams
jeremy-visionaid Oct 15, 2024
9a8738b
Improve stream/storage validation
jeremy-visionaid Oct 16, 2024
fd8f692
Add stream unit tests
jeremy-visionaid Oct 16, 2024
bb4fe95
Refactoring and bug fixes
jeremy-visionaid Oct 16, 2024
d7e0696
Merge branch '3.0-poc'
jeremy-visionaid Oct 16, 2024
64f3b07
Refactor and add comments
jeremy-visionaid Oct 16, 2024
4c55bd4
Improve header validation
jeremy-visionaid Oct 16, 2024
8ebe41a
Additional comments
jeremy-visionaid Oct 16, 2024
4d86001
Seal all the things
jeremy-visionaid Oct 16, 2024
1a47cef
Add comments and refactor
jeremy-visionaid Oct 16, 2024
3441205
Improve DirectoryEntryEnumerator validation
jeremy-visionaid Oct 16, 2024
124d527
Improve MiniFatSectorEnumerator validation
jeremy-visionaid Oct 16, 2024
5f09487
Improve exception detail
jeremy-visionaid Oct 16, 2024
b3d2995
Improve sector position terminology
jeremy-visionaid Oct 16, 2024
80ecf5e
Merge branch '3.0-poc' into 3.0-poc-draft
jeremy-visionaid Oct 17, 2024
96ac4a5
Improve stream read validation
jeremy-visionaid Oct 17, 2024
305761f
Improve ReadGuid validation
jeremy-visionaid Oct 19, 2024
30de2e4
Improved DirectoryEntry validation
jeremy-visionaid Oct 19, 2024
812d582
Refactor chain enumeration
jeremy-visionaid Oct 20, 2024
ad09603
Fix solution paths
jeremy-visionaid Oct 20, 2024
e3dd49d
Merge branch '3.0-poc' into 3.0-poc-draft
jeremy-visionaid Oct 20, 2024
a2ba07c
Improve FatSectorEnumerator validation
jeremy-visionaid Oct 20, 2024
7391580
Rename chain enumerators
jeremy-visionaid Oct 20, 2024
1ef3a73
Merge branch '3.0-poc' into 3.0-poc-draft
jeremy-visionaid Oct 20, 2024
3b68891
Add MiniFatEnumerator
jeremy-visionaid Oct 20, 2024
e1b78ac
Allow enumerating unallocated DirectoryEntries
jeremy-visionaid Oct 20, 2024
5e19c7f
Allow writing CFB streams
jeremy-visionaid Oct 19, 2024
7af7151
Merge branch '3.0-poc' into 3.0-poc-draft
jeremy-visionaid Nov 4, 2024
c14584e
Apply minor refactor
jeremy-visionaid Nov 4, 2024
8d411d1
Improve EntryInfo
jeremy-visionaid Nov 4, 2024
68f3392
Add modify stream test
jeremy-visionaid Nov 4, 2024
e84e8b8
Check stream is seekable
jeremy-visionaid Nov 4, 2024
8aa1f26
Add transaction stream
jeremy-visionaid Nov 4, 2024
f712200
Add transaction benchmark
jeremy-visionaid Nov 4, 2024
e489569
Fix write overloads
jeremy-visionaid Nov 5, 2024
982b4ec
Use stackalloc for writing bytes
jeremy-visionaid Nov 5, 2024
d46ecc6
Add netstandard2.1 stream overrides
jeremy-visionaid Nov 5, 2024
97794f7
Benchmark stuff
jeremy-visionaid Nov 5, 2024
45ace4f
Minor refactor
jeremy-visionaid Nov 5, 2024
315fd66
Optimize directory entry writes
jeremy-visionaid Nov 5, 2024
d496dda
Minor performance improvements
jeremy-visionaid Nov 5, 2024
e40aae6
Perf exe improvements
jeremy-visionaid Nov 5, 2024
f1f8a67
Cache current FAT sector
jeremy-visionaid Nov 5, 2024
dc11f6c
Optimize SetLength
jeremy-visionaid Nov 5, 2024
f4f548b
Benchmark improvement
jeremy-visionaid Nov 5, 2024
d0d86b0
Optimize adding new FAT sector
jeremy-visionaid Nov 5, 2024
8fb7b94
Simplify extending base stream
jeremy-visionaid Nov 5, 2024
41aee2a
Improve chain extension
jeremy-visionaid Nov 5, 2024
a35e371
Optimize transacted stream
jeremy-visionaid Nov 5, 2024
35ff390
Flush mini stream
jeremy-visionaid Nov 5, 2024
ad77175
Fix transaction stream length
jeremy-visionaid Nov 5, 2024
cb549bd
Cache the current mini FAT sector
jeremy-visionaid Nov 5, 2024
ea2d4bf
Benchmark improvements
jeremy-visionaid Nov 6, 2024
340bd2b
Improve dotnet 8 FatStream write
jeremy-visionaid Nov 6, 2024
f2f6ac2
Add transacted read-only test
jeremy-visionaid Nov 6, 2024
375f1a0
Only write directories when dirty
jeremy-visionaid Nov 6, 2024
07c6b14
Buffer transacted streams
jeremy-visionaid Nov 6, 2024
a265ef1
Fix building for netstandard2.0
jeremy-visionaid Nov 6, 2024
858eb99
Delete directory entries
jeremy-visionaid Nov 6, 2024
0ba7e81
Allow tracing directory entries
jeremy-visionaid Nov 6, 2024
ba9e15c
Add directories class
jeremy-visionaid Nov 6, 2024
cb689e7
Improve tracing
jeremy-visionaid Nov 6, 2024
a066e5c
Fix FAT sector caching when adding sector
jeremy-visionaid Nov 7, 2024
3103c0d
Fix FAT sector index calculation
jeremy-visionaid Nov 7, 2024
76b8dfc
Validate FAT in tests
jeremy-visionaid Nov 7, 2024
b407f17
Set Mini FAT sector count
jeremy-visionaid Nov 8, 2024
02c236c
Set directory sector count for V4
jeremy-visionaid Nov 8, 2024
cb9cb9d
Use byte[] for directory entry name
jeremy-visionaid Nov 6, 2024
92cc9d7
Add initial DirectoryTree implementation
jeremy-visionaid Nov 7, 2024
59e907f
Add structured storage reference tests
jeremy-visionaid Nov 7, 2024
1e39763
Fix leave open flag
jeremy-visionaid Nov 8, 2024
089876e
Fix warnings
jeremy-visionaid Nov 8, 2024
760bbc4
Improve benchmarks
jeremy-visionaid Nov 8, 2024
a1e5bdc
Fix structured storage empty writes
jeremy-visionaid Nov 8, 2024
47d7659
Add reference write read test
jeremy-visionaid Nov 8, 2024
7f3e51b
Fix reading reference streams
jeremy-visionaid Nov 8, 2024
ea58529
Add difat sector enumerator
jeremy-visionaid Nov 10, 2024
2ac6588
Improve benchmarks
jeremy-visionaid Nov 10, 2024
cc2d502
Delete transaction scratch file on dispose
jeremy-visionaid Nov 11, 2024
f1391ff
Add flush with optional consolidation
jeremy-visionaid Nov 11, 2024
5b12f1b
Add v4 to benchmarks
jeremy-visionaid Nov 11, 2024
45432d3
Add validation for FAT/DIFAT sector count
jeremy-visionaid Nov 11, 2024
d91bf30
Trace free/used sectors
jeremy-visionaid Nov 11, 2024
4d36a9d
Improve root storage flag handling
jeremy-visionaid Nov 11, 2024
0c596c0
Rename Directories to DirectoryEntries
jeremy-visionaid Nov 11, 2024
5e873e8
Port OpenMcdf Extensions (OLE)
jeremy-visionaid Nov 11, 2024
30bc811
Add StructuredStorageExplorer
jeremy-visionaid Nov 11, 2024
753071b
Refactor OLE extension
jeremy-visionaid Nov 11, 2024
7d68c67
Allow opening old word docs
jeremy-visionaid Nov 11, 2024
2a6b0a3
Fix warnings
jeremy-visionaid Nov 12, 2024
97ca43a
Refactor explorer
jeremy-visionaid Nov 12, 2024
a6a7ddf
Disable warnings for netstandard2.0
jeremy-visionaid Nov 12, 2024
0b52b2d
Enable spell checking
jeremy-visionaid Nov 12, 2024
43c449c
Rename to OpenMcdf
jeremy-visionaid Nov 12, 2024
36e34c7
Allow switching streams
jeremy-visionaid Nov 12, 2024
7159cd4
Improve directory/header validation
jeremy-visionaid Nov 12, 2024
5d599c0
Refactor RootContext
jeremy-visionaid Nov 12, 2024
2eae5e4
Improve comments
jeremy-visionaid Nov 12, 2024
4395d2a
Optimize FatChainEnumerator
jeremy-visionaid Nov 12, 2024
821cde4
Optimize MiniFatChainEnumerator
jeremy-visionaid Nov 12, 2024
584da52
Merge branch '3.0-poc' into 3.0-poc-draft
jeremy-visionaid Nov 12, 2024
94ad220
Seal StreamDataProvider
jeremy-visionaid Nov 12, 2024
c9b8d23
Fix OLE stack overflow
jeremy-visionaid Nov 12, 2024
7ddcf4e
Fix OLE BinaryReader/Writer usage
jeremy-visionaid Nov 12, 2024
502926c
Fix encoding provider registration
jeremy-visionaid Nov 12, 2024
f454dac
Add OLE tests
jeremy-visionaid Nov 12, 2024
f4692b9
Merge branch '3.0-poc' into 3.0-poc-draft
jeremy-visionaid Nov 13, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
272 changes: 177 additions & 95 deletions .editorconfig

Large diffs are not rendered by default.

63 changes: 63 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
###############################################################################
# Set default behavior to automatically normalize line endings.
###############################################################################
* text=auto

###############################################################################
# Set default behavior for command prompt diff.
#
# This is need for earlier builds of msysgit that does not have it on by
# default for csharp files.
# Note: This is only used by command line
###############################################################################
#*.cs diff=csharp

###############################################################################
# Set the merge driver for project and solution files
#
# Merging from the command prompt will add diff markers to the files if there
# are conflicts (Merging from VS is not affected by the settings below, in VS
# the diff markers are never inserted). Diff markers may cause the following
# file extensions to fail to load in VS. An alternative would be to treat
# these files as binary and thus will always conflict and require user
# intervention with every merge. To do so, just uncomment the entries below
###############################################################################
#*.sln merge=binary
#*.csproj merge=binary
#*.vbproj merge=binary
#*.vcxproj merge=binary
#*.vcproj merge=binary
#*.dbproj merge=binary
#*.fsproj merge=binary
#*.lsproj merge=binary
#*.wixproj merge=binary
#*.modelproj merge=binary
#*.sqlproj merge=binary
#*.wwaproj merge=binary

###############################################################################
# behavior for image files
#
# image files are treated as binary by default.
###############################################################################
#*.jpg binary
#*.png binary
#*.gif binary

###############################################################################
# diff behavior for common document formats
#
# Convert binary document formats to text before diffing them. This feature
# is only available from the command line. Turn it on by uncommenting the
# entries below.
###############################################################################
#*.doc diff=astextplain
#*.DOC diff=astextplain
#*.docx diff=astextplain
#*.DOCX diff=astextplain
#*.dot diff=astextplain
#*.DOT diff=astextplain
#*.pdf diff=astextplain
#*.PDF diff=astextplain
#*.rtf diff=astextplain
#*.RTF diff=astextplain
186 changes: 186 additions & 0 deletions .globalconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
# Remove the line below if you want to inherit .editorconfig settings from higher directories
root = true

dotnet_analyzer_diagnostic.category-Design.severity = warning
dotnet_analyzer_diagnostic.category-Interoperability.severity = suggestion
dotnet_analyzer_diagnostic.category-Maintainability.severity = warning
dotnet_analyzer_diagnostic.category-Naming.severity = warning
dotnet_analyzer_diagnostic.category-Performance.severity = warning
dotnet_analyzer_diagnostic.category-Reliability.severity = warning
dotnet_analyzer_diagnostic.category-Style.severity = warning

dotnet_analyzer_diagnostic.category-StyleCop.CSharp.DocumentationRules.severity = none

# IDE0010: Populate switch
dotnet_diagnostic.IDE0010.severity = none

# IDE0011: Add braces to 'if' statement
dotnet_diagnostic.IDE0011.severity = suggestion

# IDE0022: Use expression body for method
dotnet_diagnostic.IDE0022.severity = suggestion

# IDE0025: Use expression body for property
dotnet_diagnostic.IDE0025.severity = suggestion

# IDE0028: Collection initialization can be simplified (https://learn.microsoft.com/dotnet/fundamentals/code-analysis/style-rules/ide0028)
dotnet_diagnostic.IDE0028.severity = none

# IDE0028: Collection initialization can be simplified (https://learn.microsoft.com/dotnet/fundamentals/code-analysis/style-rules/ide0028)
dotnet_diagnostic.IDE0030.severity = suggestion

# IDE0040: Accessibility modifiers required (disabled on build)
dotnet_diagnostic.IDE0040.severity = none

# IDE0045: Use conditional expression for assignment
dotnet_diagnostic.IDE0045.severity = suggestion

# IDE0046: Use conditional expression for return
dotnet_diagnostic.IDE0046.severity = none

# IDE0047: Parentheses can be removed
dotnet_diagnostic.IDE0047.severity = suggestion

# IDE0052: Remove unread private member
dotnet_diagnostic.IDE0052.severity = warning

# IDE0058: Expression value is never used
dotnet_diagnostic.IDE0058.severity = suggestion

# IDE0060: Remove unused parameter
dotnet_diagnostic.IDE0060.severity = suggestion

# IDE0072: Populate switch
dotnet_diagnostic.IDE0072.severity = none

# IDE0130: Namespace does not match folder structure
dotnet_diagnostic.IDE0130.severity = suggestion

# IDE0251: Make member readonly
dotnet_diagnostic.IDE0251.severity = warning

# IDE0270: Null check can be simplified
dotnet_diagnostic.IDE0270.severity = suggestion

# IDE0290: Use primary constructor
dotnet_diagnostic.IDE0290.severity = suggestion

# IDE0300: Collection initialization can be simplified
dotnet_diagnostic.IDE0300.severity = suggestion

# IDE0300: Collection initialization can be simplified (false positive on empty arrays)
dotnet_diagnostic.IDE0301.severity = suggestion

# IDE0303: Collection initialization can be simplified
dotnet_diagnostic.IDE0302.severity = suggestion

# IDE0303: Collection initialization can be simplified
dotnet_diagnostic.IDE0303.severity = suggestion

# IDE0305: Collection initialization can be simplified
dotnet_diagnostic.IDE0305.severity = suggestion

#CA1008: Enums should have zero value
dotnet_diagnostic.CA1008.severity = suggestion

# CA1028: Enum storage should be Int32
dotnet_diagnostic.CA1028.severity = suggestion

# CA1031: Do not catch general exception types
dotnet_diagnostic.CA1031.severity = suggestion

# CA1051: Do not declare visible instance fields
dotnet_diagnostic.CA1051.severity = none

# CA1062: Validate arguments of public methods
dotnet_diagnostic.CA1062.severity = none

# CA1304: Specify CultureInfo
dotnet_diagnostic.CA1304.severity = suggestion

# CA1305: Specify IFormatProvider
dotnet_diagnostic.CA1305.severity = suggestion

# CA1308: Normalize strings to uppercase
dotnet_diagnostic.CA1308.severity = suggestion

# CA1309: Use ordinal string comparison
dotnet_diagnostic.CA1309.severity = suggestion

# CA1416: Validate platform compatibility
dotnet_diagnostic.CA1416.severity = none

# CA1848: Use the LoggerMessage delegates
dotnet_diagnostic.CA1848.severity = none

# CA1711: Identifiers should not have incorrect suffix
dotnet_code_quality.CA1711.allowed_suffixes = Flag|Flags

# CA1721: Property names should not match get methods
dotnet_diagnostic.CA1721.severity = suggestion

# CA1724: Type names should not match namespaces
dotnet_diagnostic.CA1724.severity = suggestion

# CA1814: Prefer jagged arrays over multidimensional
dotnet_diagnostic.CA1814.severity = suggestion

# CA1826: Use property instead of Linq Enumerable method
dotnet_code_quality.CA1826.exclude_ordefault_methods = true

# CA1863: Use 'CompositeFormat'
dotnet_diagnostic.CA1863.severity = none

# CA2007: Do not directly await a Task
dotnet_diagnostic.CA2007.severity = suggestion

# CA2008: Do not create tasks without passing a TaskScheduler
dotnet_diagnostic.CA2008.severity = suggestion

# CA2109: Review visible event handlers
dotnet_diagnostic.CA2109.severity = none

# CA2229: Implement serialization constructors
dotnet_diagnostic.CA2229.severity = none

# CA2300: Do not use insecure deserializer BinaryFormatter
dotnet_diagnostic.CA2300.severity = none

# CA2302: Ensure BinaryFormatter.Binder is set before calling BinaryFormatter.Deserialize
dotnet_diagnostic.CA2302.severity = suggestion

# CA5392: Use DefaultDllImportSearchPaths attribute for P/Invokes
dotnet_diagnostic.CA5392.severity = suggestion

# CA5393: Do not use unsafe DllImportSearchPath value
dotnet_diagnostic.CA5393.severity = none

# CA5394: Do not use insecure randomness
dotnet_diagnostic.CA5394.severity = suggestion

# CS1591: Missing XML comment for publicly visible type or member
dotnet_diagnostic.CS1591.severity = none

# MSTEST0015: Test method should not be ignored
dotnet_diagnostic.MSTEST0015.severity = none

# SYSLIB0011: Type or member is obsolete
dotnet_diagnostic.SYSLIB0011.severity = none

# SA0001: XML comment analysis disabled
dotnet_diagnostic.SA0001.severity = none

# SA1215: An instance readonly element is positioned beneath an instance non-readonly element of the same type
dotnet_diagnostic.SA1215.severity = warning

# SA1515: Single-line comment should be preceded by blank line
dotnet_diagnostic.SA1515.severity = suggestion

# SA1628: Documentation text should begin with a capital letter
dotnet_diagnostic.SA1628.severity = warning

# SA1201: Elements must appear in the correct order
dotnet_diagnostic.SA1201.severity = suggestion

# SYSLIB1045: Use GeneratedRegexAttribute to generate the regular expression implementation at compile time
dotnet_diagnostic.SYSLIB1045.severity = suggestion
55 changes: 55 additions & 0 deletions OpenMcdf.Benchmarks/FileStreamRead.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using OpenMcdf.Benchmarks;

namespace OpenMcdf.Benchmark;

[MediumRunJob]
[MemoryDiagnoser]
[HideColumns(Column.AllocRatio)]
[MarkdownExporter]
public class FileStreamRead : IDisposable
{
private string readFileName = "";
private byte[] buffer = Array.Empty<byte>();

[Params(Version.V3, Version.V4)]
public Version Version { get; set; }

[Params(512, 1024 * 1024)]
public int BufferSize { get; set; }

[Params(1024 * 1024)]
public int StreamLength { get; set; }

public void Dispose()
{
File.Delete(readFileName);
}

[GlobalSetup]
public void GlobalSetup()
{
readFileName = Path.GetTempFileName();

buffer = new byte[BufferSize];

using var storage = RootStorage.Create(readFileName, Version, StorageModeFlags.LeaveOpen);
using CfbStream stream = storage.CreateStream("Test");

int iterationCount = StreamLength / BufferSize;
for (int iteration = 0; iteration < iterationCount; ++iteration)
stream.Write(buffer);
}

[GlobalCleanup]
public void GlobalCleanup() => Dispose();

[Benchmark]
public void Read() => OpenMcdfBenchmarks.ReadStream(readFileName!, buffer);

#if WINDOWS
[Benchmark(Baseline = true)]
public void ReadStructuredStorage() => StructuredStorageBenchmarks.ReadStream(readFileName!, buffer);
#endif
}
49 changes: 49 additions & 0 deletions OpenMcdf.Benchmarks/FileStreamTransactedWrite.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using OpenMcdf.Benchmarks;

namespace OpenMcdf.Benchmark;

[MediumRunJob]
[MemoryDiagnoser]
[HideColumns(Column.AllocRatio)]
[MarkdownExporter]
public class FileStreamTransactedWrite : IDisposable
{
private string writeFileName = "";
private byte[] buffer = Array.Empty<byte>();

[Params(Version.V3, Version.V4)]
public Version Version { get; set; }

[Params(512, 1024 * 1024)]
public int BufferSize { get; set; }

[Params(1024 * 1024)]
public int StreamLength { get; set; }

public void Dispose()
{
File.Delete(writeFileName);
}

[GlobalSetup]
public void GlobalSetup()
{
writeFileName = Path.GetTempFileName();

buffer = new byte[BufferSize];
}

[GlobalCleanup]
public void GlobalCleanup() => Dispose();

[Benchmark]
public void WriteTransacted() => OpenMcdfBenchmarks.WriteStream(writeFileName!, Version, StorageModeFlags.None | StorageModeFlags.Transacted, buffer, StreamLength);

#if WINDOWS

[Benchmark(Baseline = true)]
public void WriteStructuredStorageTransacted() => StructuredStorageBenchmarks.WriteStream(writeFileName, Version, StorageModeFlags.Transacted, buffer, StreamLength);
#endif
}
Loading
Loading