Skip to content

Commit

Permalink
Merge pull request #18 from things-library/adding-services
Browse files Browse the repository at this point in the history
Updateds cloud file and file storage methods and interface
lanmark7 authored Aug 20, 2024
2 parents baa8f5d + 65afccb commit 15a947d
Showing 20 changed files with 152 additions and 125 deletions.
3 changes: 3 additions & 0 deletions Base/src/ThingsLibrary.Base/ThingsLibrary.Base.csproj
Original file line number Diff line number Diff line change
@@ -20,4 +20,7 @@
<RepositoryUrl></RepositoryUrl>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="ThingsLibrary.Schema.Library" Version="0.2.1" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
@@ -30,7 +30,7 @@
<PackageReference Include="Serilog.Extensions.Logging" Version="8.0.0" />
<PackageReference Include="Serilog.Settings.Configuration" Version="8.0.2" />
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
<PackageReference Include="ThingsLibrary.Schema.Library" Version="0.2.0" />
<PackageReference Include="ThingsLibrary.Schema.Library" Version="0.2.1" />
</ItemGroup>

<ItemGroup>
Original file line number Diff line number Diff line change
@@ -10,10 +10,13 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
<PackageReference Include="MSTest.TestAdapter" Version="3.1.1" />
<PackageReference Include="MSTest.TestFramework" Version="3.1.1" />
<PackageReference Include="coverlet.collector" Version="6.0.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.0" />
<PackageReference Include="MSTest.TestAdapter" Version="3.5.2" />
<PackageReference Include="MSTest.TestFramework" Version="3.5.2" />
</ItemGroup>

<ItemGroup>
80 changes: 34 additions & 46 deletions Storage/src/ThingsLibrary.Storage.Azure/Extensions/CloudFile.cs
Original file line number Diff line number Diff line change
@@ -7,71 +7,59 @@ public static class CloudFileExtensions
{
public static CloudFile ToCloudFile(this BlobItem blob)
{
if (blob.Properties != null)
var item = new CloudFile(blob.Name)
{
return new CloudFile
Attributes =
{
// blob items
FilePath = blob.Name,
Version = blob.VersionId,
Metadata = blob.Metadata,

// properties
FileSize = (long)blob.Properties.ContentLength,
CreatedOn = blob.Properties.CreatedOn,
UpdatedOn = blob.Properties.LastModified,
{ "version", blob.VersionId }
},
Metadata = blob.Metadata
};

ContentType = blob.Properties.ContentType,
ContentMD5 = (blob.Properties.ContentHash != null ? Convert.ToBase64String(blob.Properties.ContentHash) : null),

Tag = blob.Properties.ETag.ToString()
};
}
else
if (blob.Properties != null)
{
return new CloudFile
item.Add(new Dictionary<string, object>()
{
// blob items
FilePath = blob.Name,
Version = blob.VersionId,
Metadata = blob.Metadata
};
// blob items
{ "size", blob.Properties.ContentLength },
{ "content_type", blob.Properties.ContentType },
{ "content_md5", (blob.Properties.ContentHash != null ? Convert.ToBase64String(blob.Properties.ContentHash) : null) },
{ "content_etag", blob.Properties.ETag.ToString() },

{ "created", blob.Properties.CreatedOn },
{ "updated", blob.Properties.LastModified }
});
}

return item;
}

public static CloudFile ToCloudFile(this BlobClient fromEntity)
{
var item = new CloudFile(fromEntity.Name);

var response = fromEntity.GetProperties();

if (response != null)
{
var properties = response.Value;

return new CloudFile

item.Metadata = properties.Metadata;

item.Add(new Dictionary<string, object>
{
FilePath = fromEntity.Name,
{ "version", properties.VersionId },

FileSize = properties.ContentLength,
{ "size", properties.ContentLength },
{ "content_type", properties.ContentType },
{ "content_md5", (properties.ContentHash != null ? Convert.ToBase64String(properties.ContentHash) : null) },
{ "content_etag", properties.ETag.ToString() },

CreatedOn = properties.CreatedOn,
UpdatedOn = properties.LastModified,

ContentType = properties.ContentType,
ContentMD5 = (properties.ContentHash != null ? Convert.ToBase64String(properties.ContentHash) : null),

Metadata = properties.Metadata,

Tag = properties.ETag.ToString()
};
}
else
{
return new CloudFile
{
FilePath = fromEntity.Name
};
{ "created", properties.CreatedOn },
{ "updated", properties.LastModified }
});
}

return item;
}
}
}
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@ public static IServiceCollection AddFileStores(this IServiceCollection services,
var fileStores = new FileStores(connectionString);

// Register lib services here...
services.AddSingleton<IFileStores>(fileStores);
services.AddSingleton<ICloudFileStores>(fileStores);

return services;
}
4 changes: 2 additions & 2 deletions Storage/src/ThingsLibrary.Storage.Azure/FileStore.cs
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@

namespace ThingsLibrary.Storage.Azure
{
public class FileStore : IFileStore
public class FileStore : ICloudFileStore
{
#region --- Transfer Events ---

@@ -51,7 +51,7 @@ private void OnTransferComplete(Exception error, bool cancelled)
public string BucketName { get; private set; }

/// <inheritdoc/>
public FileStoreType StorageType { get; init; } = FileStoreType.Azure_Blob;
public CloudFileStoreType StorageType { get; init; } = CloudFileStoreType.Azure_Blob;

/// <inheritdoc/>
public bool IsVersioning { get; set; } = false;
6 changes: 3 additions & 3 deletions Storage/src/ThingsLibrary.Storage.Azure/FileStores.cs
Original file line number Diff line number Diff line change
@@ -3,10 +3,10 @@

namespace ThingsLibrary.Storage.Azure
{
public class FileStores : IFileStores
public class FileStores : ICloudFileStores
{
/// <inheritdoc />
public FileStoreType StoreType => FileStoreType.Azure_Blob;
public CloudFileStoreType StoreType => CloudFileStoreType.Azure_Blob;

private string StorageConnectionString { get; set; }
private BlobServiceClient BlobServiceClient { get; set; }
@@ -22,7 +22,7 @@ public FileStores(string storageConnectionString)
}

/// <inheritdoc />
public IFileStore GetStore(string bucketName)
public ICloudFileStore GetStore(string bucketName)
{
return new FileStore(this.BlobServiceClient, bucketName);
}
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Azure.Storage.Blobs" Version="12.21.1" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.21.2" />
</ItemGroup>

<ItemGroup>
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@

namespace ThingsLibrary.Storage.Local
{
public class FileStore : IFileStore
public class CloudFileStore : ICloudFileStore
{
// TRANSFER EVENTS
#region --- Observable Events ---
@@ -53,7 +53,7 @@ private void OnTransferComplete(Exception error, bool cancelled)
public bool IsVersioning { get; set; } = false;

/// <inheritdoc/>
public FileStoreType StorageType { get; init; } = FileStoreType.Local;
public CloudFileStoreType StorageType { get; init; } = CloudFileStoreType.Local;

/// <inheritdoc/>
public CancellationToken CancellationToken { get; set; } = default;
@@ -68,7 +68,7 @@ private void OnTransferComplete(Exception error, bool cancelled)
private Task ScanTask { get; set; }

/// <inheritdoc/>
public FileStore(string storageConnectionString, string bucketName)
public CloudFileStore(string storageConnectionString, string bucketName)
{
//Connection String:
// "RootStorePath=./TestDirectory"
@@ -85,7 +85,7 @@ public FileStore(string storageConnectionString, string bucketName)
};

// validate the bucket naming against cloud specs
FileStore.ValidateBucketName(bucketName);
CloudFileStore.ValidateBucketName(bucketName);

// set the core properties
this.BucketName = bucketName;
@@ -301,7 +301,7 @@ private void UpdateDatabase()
this.UpdateDatabase(this.BucketDirectoryPath);

// add the population complete record (use a invalid bucket name)
this.DataCollection.Upsert(new BsonValue("<Populated>"), new CloudFile());
this.DataCollection.Upsert(new BsonValue("<Populated>"), new CloudFile("populated")); //TODO?

Log.Information("================================================================================");
Log.Information("File Count: {FileCount}", this.DataCollection.Count());
@@ -421,11 +421,7 @@ private CloudFile GenerateCloudFile(string storageFilePath)
{ "content_md5", IO.File.ComputeMD5Base64(storageFilePath) }
};

var cloudFile = new CloudFile()
{
Key = this.GetCloudFilePath(storageFilePath),
Name = Path.GetFileName(storageFilePath)
};
var cloudFile = new CloudFile(this.GetCloudFilePath(storageFilePath));

cloudFile.Add(attributes);

37 changes: 26 additions & 11 deletions Storage/src/ThingsLibrary.Storage/CloudFile.cs
Original file line number Diff line number Diff line change
@@ -2,20 +2,19 @@

namespace ThingsLibrary.Storage
{
public class CloudFile : ItemDto//, ICloudFile
public class CloudFile : ItemDto, ICloudFile
{

/// <inheritdoc />
public string FilePath { get; init; }
public string FilePath => this["resource_path"];

/// <inheritdoc />
public string FileName => System.IO.Path.GetFileName(this.FilePath);

public string FileName => this.Name;
/// <inheritdoc />
public DateTimeOffset? CreatedOn => this.Get<DateTimeOffset?>("created", null);
public long FileSize => this.Get<long>("file_size", 0);

/// <inheritdoc />
public DateTimeOffset? UpdatedOn => this.Get<DateTimeOffset?>("updated", null);
public double FileSizeMB => this.FileSize / (double)1048576; //(1024x1024) == bytes to megs

/// <inheritdoc />
public string ContentType => this["content_type"];
@@ -24,11 +23,12 @@ public class CloudFile : ItemDto//, ICloudFile
public string ContentMD5 => this["content_md5"]; //MD5 Hash

/// <inheritdoc />
public long FileSize => this.Get<long>("file_size", 0);
public DateTimeOffset? CreatedOn => this.Get<DateTimeOffset?>("created", null);

/// <inheritdoc />
public double FileSizeMB => this.FileSize / (double)1048576; //(1024x1024) == bytes to megs

public DateTimeOffset? UpdatedOn => this.Get<DateTimeOffset?>("updated", null);


#region --- Local / Generic Properties ---

/// <inheritdoc />
@@ -47,5 +47,20 @@ public bool IsAlreadyTransferred()

#endregion

public CloudFile(string key, string name)
{
this.Key = key;
this.Name = name;
}

public CloudFile(string resourcePath)
{
this.Key = Path.GetFileName(resourcePath);
this.Name = Path.GetFileName(resourcePath);

// keep track of the entire resource path
this.Attributes["resource_path"] = resourcePath;
}

}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace ThingsLibrary.Storage
{
public abstract class FileStore
public abstract class CloudFileStore
{
/// <inheritdoc />
public string BucketPrefix { get; set; }
@@ -9,7 +9,7 @@ public abstract class FileStore
public string BucketName { get; set; }

/// <inheritdoc />
public FileStoreType StorageType { get; set; }
public CloudFileStoreType StorageType { get; set; }

/// <inheritdoc />
public abstract DateTime? CreatedOn { get; }
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@
/// <summary>
/// Storage System Type
/// </summary>
public enum FileStoreType : byte
public enum CloudFileStoreType : byte
{
Azure_Blob = 1,
AWS_S3 = 2,
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace Core.Cloud.Storage.File
namespace ThingsLibrary.Storage.Contracts
{
public class TransferProgressReport
public class TransferProgressReportDto
{
public string Message { get; set; } = "";
public DateTime? StartedOn { get; set; } = null;
@@ -28,7 +28,7 @@ public class TransferProgressReport
public Exception LastException { get; set; } = null;
public DateTime? LastExceptionOn { get; set; } = null;

public TransferProgressReport()
public TransferProgressReportDto()
{

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
namespace ThingsLibrary.Storage.Contracts
{
public class TransferProgressStatusDto
{
public string Message { get; private set; } = "";
public double PercentComplete { get; private set; } = 0;

public double BytesPerSecond { get; private set; } = 0;
public TimeSpan? EstimatedTimeLeft { get; set; } = null;

public double Mbps => BytesPerSecond / 1048576; //(1024x1024) == bytes to megs

public TransferProgressStatusDto(string message, double percentComplete, double bytesPerSecond, TimeSpan? timeLeft)
{
Message = message;
PercentComplete = percentComplete;

BytesPerSecond = bytesPerSecond;
EstimatedTimeLeft = timeLeft;
}
}
}
21 changes: 12 additions & 9 deletions Storage/src/ThingsLibrary.Storage/Interfaces/ICloudFile.cs
Original file line number Diff line number Diff line change
@@ -33,14 +33,9 @@ public interface ICloudFile
public double FileSizeMB { get; } // => this.FileSize / (double)1048576; //(1024x1024) == bytes to megs

/// <summary>
/// Locak File Path
/// </summary>
public string LocalFilePath { get; set; }

/// <summary>
/// Tag
/// Actual metadata stamped on the cloud files
/// </summary>
public object Tag { get; set; }
public IDictionary<string, string> Metadata { get; }

/// <summary>
/// Updated Date
@@ -52,11 +47,19 @@ public interface ICloudFile
/// </summary>
public DateTimeOffset? CreatedOn { get; }

#region --- Extended Local Properties ---

/// <summary>
/// Actual metadata stamped on the cloud files
/// Locak File Path
/// </summary>
public string LocalFilePath { get; set; }

/// <summary>
/// Tag
/// </summary>
public IDictionary<string, string> Metadata { get; init; }
public object Tag { get; set; }

#endregion

/// <summary>
/// Is the local file path the same one in cloud storage based on MD5
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace ThingsLibrary.Storage.Interfaces
{
public interface IFileStore
public interface ICloudFileStore
{
#region --- Observable Events ---

@@ -25,7 +25,7 @@ public interface IFileStore
/// <summary>
/// File Store Type
/// </summary>
public FileStoreType StorageType { get; }
public CloudFileStoreType StorageType { get; }

/// <summary>
/// If edits to files should be versioned / snapshots
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
namespace ThingsLibrary.Storage.Interfaces
{
public interface IFileStores
public interface ICloudFileStores
{
/// <summary>
/// Type of entity store (Azure_Table, GCP_DataStore)
/// </summary>
public FileStoreType StoreType { get; }
public CloudFileStoreType StoreType { get; }

/// <summary>
/// Get/Create Cloud File Store
/// </summary>
/// <returns><see cref="FileStore"/></returns>
public IFileStore GetStore(string bucketName);
/// <returns><see cref="CloudFileStore"/></returns>
public ICloudFileStore GetStore(string bucketName);
}
}
Original file line number Diff line number Diff line change
@@ -20,9 +20,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageReference Include="Serilog" Version="4.0.1" />

<PackageReference Include="ThingsLibrary.Schema.Library" Version="0.1.8" />
<PackageReference Include="Serilog" Version="4.0.1" />
</ItemGroup>

<ItemGroup>
22 changes: 0 additions & 22 deletions Storage/src/ThingsLibrary.Storage/TransferProgressStatus.cs

This file was deleted.

21 changes: 21 additions & 0 deletions ThingsLibrary.Frameworks.Deploy.sln
Original file line number Diff line number Diff line change
@@ -35,6 +35,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ThingsLibrary.Services.Test
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Storage", "Storage", "{F2EB0266-EB80-473F-99E4-6F282BECAA7A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ThingsLibrary.Storage", "Storage\src\ThingsLibrary.Storage\ThingsLibrary.Storage.csproj", "{B9EBE159-A07C-40DF-9DD1-DB91906982CA}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ThingsLibrary.Storage.Azure", "Storage\src\ThingsLibrary.Storage.Azure\ThingsLibrary.Storage.Azure.csproj", "{06028A42-648D-44A6-B394-5B430C73BA60}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ThingsLibrary.Storage.Local", "Storage\src\ThingsLibrary.Storage.Local\ThingsLibrary.Storage.Local.csproj", "{9E948F7D-F92C-4B57-AD45-92305A8821C8}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -65,6 +71,18 @@ Global
{A8F03533-0BA7-40C4-9770-C0EF4A7E1692}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A8F03533-0BA7-40C4-9770-C0EF4A7E1692}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A8F03533-0BA7-40C4-9770-C0EF4A7E1692}.Release|Any CPU.Build.0 = Release|Any CPU
{B9EBE159-A07C-40DF-9DD1-DB91906982CA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B9EBE159-A07C-40DF-9DD1-DB91906982CA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B9EBE159-A07C-40DF-9DD1-DB91906982CA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B9EBE159-A07C-40DF-9DD1-DB91906982CA}.Release|Any CPU.Build.0 = Release|Any CPU
{06028A42-648D-44A6-B394-5B430C73BA60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{06028A42-648D-44A6-B394-5B430C73BA60}.Debug|Any CPU.Build.0 = Debug|Any CPU
{06028A42-648D-44A6-B394-5B430C73BA60}.Release|Any CPU.ActiveCfg = Release|Any CPU
{06028A42-648D-44A6-B394-5B430C73BA60}.Release|Any CPU.Build.0 = Release|Any CPU
{9E948F7D-F92C-4B57-AD45-92305A8821C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9E948F7D-F92C-4B57-AD45-92305A8821C8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9E948F7D-F92C-4B57-AD45-92305A8821C8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9E948F7D-F92C-4B57-AD45-92305A8821C8}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -79,6 +97,9 @@ Global
{14BDE79B-8256-4F68-89E8-EA78153684F0} = {827FE983-CB41-4775-8502-46D8A8DB7FCB}
{07DB7DD5-6AAC-4982-8411-8D1930A29375} = {B6219468-26C8-4B0E-AB9F-D06FC5BED113}
{A8F03533-0BA7-40C4-9770-C0EF4A7E1692} = {F807700F-D771-4F0C-9A61-8817DA1D770B}
{B9EBE159-A07C-40DF-9DD1-DB91906982CA} = {F2EB0266-EB80-473F-99E4-6F282BECAA7A}
{06028A42-648D-44A6-B394-5B430C73BA60} = {F2EB0266-EB80-473F-99E4-6F282BECAA7A}
{9E948F7D-F92C-4B57-AD45-92305A8821C8} = {F2EB0266-EB80-473F-99E4-6F282BECAA7A}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {9CF34144-3CDD-4955-B934-A5278D7A1139}

0 comments on commit 15a947d

Please sign in to comment.