Skip to content

Mss/fun january2025 #53

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 9 commits into
base: main
Choose a base branch
from
Draft

Mss/fun january2025 #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
731ded6
added TableColumn logic for saving schema means that when OData table…
MatthiasSort Jan 16, 2025
047dcc1
added groupd to TableColumn together with tablecolumn.columns so they…
MatthiasSort Jan 20, 2025
49b1026
Work for today
MatthiasSort Jan 23, 2025
618393a
added GetExpandAsParameters so it can extend the URL with expand valu…
MatthiasSort Jan 29, 2025
6563a41
added replace for when working with default api on BC, still needs to…
MatthiasSort Jan 29, 2025
2200dfa
changed how entitysets-tables are fetched so it is after all entityty…
MatthiasSort Jan 30, 2025
f724284
work so far
MatthiasSort Feb 3, 2025
72159de
work for today
MatthiasSort Feb 3, 2025
ad68c4e
work so far
MatthiasSort Feb 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Version>10.8.4</Version>
<Version>10.9.0</Version>
<AssemblyVersion>1.0.0.0</AssemblyVersion>
<Title>OData Provider</Title>
<Description>The Odata Provider lets you fetch and map data from or to any OData endpoint.</Description>
Expand All @@ -24,11 +24,15 @@
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dynamicweb.DataIntegration" Version="10.8.0" />
<PackageReference Include="Dynamicweb.Ecommerce" Version="10.8.0" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
</ItemGroup>
<ItemGroup>
<None Include="odata-logo.png" Pack="true" Visible="false" PackagePath="\" />
</ItemGroup>
<ItemGroup>
<Reference Include="Dynamicweb.DataIntegration">
<HintPath>..\..\..\Dynamicweb10\src\Features\DataIntegration\Dynamicweb.DataIntegration\bin\Debug\net8.0\Dynamicweb.DataIntegration.dll</HintPath>
</Reference>
</ItemGroup>
</Project>
154 changes: 104 additions & 50 deletions src/ODataProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,27 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml;
using System.Xml.Linq;
using Dynamicweb.Core;
using Dynamicweb.Core;
using Dynamicweb.DataIntegration.EndpointManagement;
using Dynamicweb.DataIntegration.Integration;
using Dynamicweb.DataIntegration.Integration.ERPIntegration;
using Dynamicweb.DataIntegration.Integration.Interfaces;
using Dynamicweb.DataIntegration.ProviderHelpers;
using Dynamicweb.DataIntegration.Providers.ODataProvider.Interfaces;
using Dynamicweb.DataIntegration.Providers.ODataProvider.Model;
using Dynamicweb.Extensibility.AddIns;
using Dynamicweb.Extensibility.Editors;
using Dynamicweb.Logging;
using Dynamicweb.Security.Licensing;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml;
using System.Xml.Linq;

namespace Dynamicweb.DataIntegration.Providers.ODataProvider;

Expand All @@ -31,7 +33,7 @@ namespace Dynamicweb.DataIntegration.Providers.ODataProvider;
[ResponseMapping(true)]
public class ODataProvider : BaseProvider, ISource, IDestination, IParameterOptions, IODataBaseProvider, IParameterVisibility
{
internal readonly EndpointService _endpointService = new();
internal readonly EndpointService _endpointService = new();
internal readonly EndpointCollectionService _endpointCollectionService = new EndpointCollectionService();
internal Schema _schema;
internal Endpoint _endpoint;
Expand Down Expand Up @@ -219,20 +221,20 @@ IEnumerable<ParameterOption> IParameterOptions.GetParameterOptions(string parame
{
var result = new List<ParameterOption>();

foreach(var collection in _endpointCollectionService.GetEndpointCollections().OrderBy(ec => ec.Sorting))
{
var parameterOptions = _endpointCollectionService.GetEndpoints(collection.Id).Select(endpoint =>
new ParameterOption(endpoint.Name,new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, collection.Name, endpoint.Id.ToString()))
{
foreach (var collection in _endpointCollectionService.GetEndpointCollections().OrderBy(ec => ec.Sorting))
{
var parameterOptions = _endpointCollectionService.GetEndpoints(collection.Id).Select(endpoint =>
new ParameterOption(endpoint.Name, new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, collection.Name, endpoint.Id.ToString()))
{
Group = collection.Name
});
result.AddRange(parameterOptions);
}
result.AddRange(parameterOptions);
}
result.AddRange(_endpointService.GetEndpoints().Where(e => e.Collection == null).Select(endpoint =>
new ParameterOption(endpoint.Name, new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, "Dynamicweb 9 Endpoints", endpoint.Id.ToString()))
{
Group = "Dynamicweb 9 Endpoints"
}));
new ParameterOption(endpoint.Name, new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, "Dynamicweb 9 Endpoints", endpoint.Id.ToString()))
{
Group = "Dynamicweb 9 Endpoints"
}));

return result;
}
Expand Down Expand Up @@ -302,34 +304,36 @@ public override void OverwriteDestinationSchemaToOriginal()
/// <inheritdoc />
public override Schema GetOriginalSourceSchema()
{
var name = GetEntityName();
var entityTypeTables = new Schema();
var entitySetsTables = new Schema();

if (_endpoint == null)
{
return new Schema();
}

var name = GetEntityName();
var header = new Dictionary<string, string>
{
{ "accept", "text/html,application/xhtml+xml,application/xml" },
{ "Content-Type", "text/html" }
};
if (_endpoint != null)
var endpointAuthentication = _endpoint.Authentication;
if (endpointAuthentication != null)
{
var endpointAuthentication = _endpoint.Authentication;
if (endpointAuthentication != null)
{
SetCredentials();
}
Task metadataResponse;
if (endpointAuthentication.IsTokenBased())
{
string token = OAuthHelper.GetToken(_endpoint, endpointAuthentication);
metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, token);
}
else
{
metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, endpointAuthentication, header);
}
metadataResponse.Wait();
SetCredentials();
}
Task metadataResponse;
if (endpointAuthentication.IsTokenBased())
{
string token = OAuthHelper.GetToken(_endpoint, endpointAuthentication);
metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, token);
}
else
{
metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, endpointAuthentication, header);
}
metadataResponse.Wait();

var emptySchema = new Schema();
if (entitySetsTables == emptySchema)
Expand All @@ -356,9 +360,13 @@ void HandleStream(Stream responseStream, HttpStatusCode responseStatusCode, Dict
else if (xmlReader.NodeType == XmlNodeType.Element &&
xmlReader.Name.Equals("EntitySet", StringComparison.OrdinalIgnoreCase))
{
GetColumnsFromEntityTypeTableToEntitySetTable(entitySetsTables.AddTable(xmlReader.GetAttribute("Name")), entityTypeTables, xmlReader.GetAttribute("EntityType"));
var entityTypeName = xmlReader.GetAttribute("EntityType");
var SqlSchema = entityTypeName.Substring(entityTypeName.LastIndexOf(".") + 1);
var setTable = entitySetsTables.AddTable(xmlReader.GetAttribute("Name"), SqlSchema);
}
}

GetColumnsFromEntityTypeTableToEntitySetTable(entityTypeTables, entitySetsTables);
if (!EndpointIsLoadAllEntities(_endpoint.Url))
{
var singleEntitySetSelected = entitySetsTables.GetTables().FirstOrDefault(obj => obj.Name.Equals(name, StringComparison.OrdinalIgnoreCase));
Expand All @@ -377,20 +385,39 @@ void HandleStream(Stream responseStream, HttpStatusCode responseStatusCode, Dict
}
}

private void GetColumnsFromEntityTypeTableToEntitySetTable(Table table, Schema entityTypeSchema, string entityTypeName)
private void GetColumnsFromEntityTypeTableToEntitySetTable(Schema entityTypeSchema, Schema entitySetsTables)
{
var entityTypeNameClean = entityTypeName.Substring(entityTypeName.LastIndexOf(".") + 1);
Table result = entityTypeSchema.GetTables().Where(obj => obj.Name.Equals(entityTypeNameClean, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
if (result != null)
var entityTypeSchemaTables = entityTypeSchema.GetTables();
var entitySetSchemaTables = entitySetsTables.GetTables();
foreach (var table in entitySetSchemaTables)
{
foreach (var item in result.Columns)
Table result = entityTypeSchemaTables.Where(obj => obj.Name.Equals(table.SqlSchema, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
if (result != null)
{
if (table.Columns.Where(obj => obj.Name == item.Name).Count() == 0)
foreach (var item in result.Columns)
{
table.AddColumn(new Column(item.Name, item.Type, table, item.IsPrimaryKey, item.IsNew, item.ReadOnly));
if (table.Columns.Where(obj => obj.Name == item.Name).Count() == 0)
{
if (item is TableColumn tableColumn)
{
var tableGroupName = tableColumn.Group;
var columns = entityTypeSchemaTables.FirstOrDefault(obj => obj.Name.Equals(tableColumn.Group, StringComparison.OrdinalIgnoreCase))?.Columns ?? [];
var entitySetTableName = entitySetSchemaTables.FirstOrDefault(obj => obj.SqlSchema.Equals(tableColumn.Group));
if (entitySetTableName != null)
{
tableGroupName = entitySetTableName.Name;
}
table.AddColumn(new TableColumn(tableColumn.Name, tableGroupName, table, tableColumn.Type, columns));
}
else
{
table.AddColumn(new Column(item.Name, item.Type, table, item.IsPrimaryKey, item.IsNew, item.ReadOnly));
}
}
}
}
}
entitySetSchemaTables.ForEach(obj => obj.SqlSchema = string.Empty);
}

private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table, Schema result)
Expand All @@ -399,6 +426,7 @@ private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table,
string entityName = xmlReader.GetAttribute("Name");
List<string> primaryKeys = new List<string>();
Column column = null;
TableColumn tableColumn = null;
while (xmlReader.Read() && !(xmlReader.NodeType == XmlNodeType.EndElement && xmlReader.Name.Equals("EntityType", StringComparison.OrdinalIgnoreCase)))
{
if (xmlReader.NodeType == XmlNodeType.Element && xmlReader.Name.Equals("PropertyRef", StringComparison.OrdinalIgnoreCase))
Expand Down Expand Up @@ -429,6 +457,16 @@ private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table,
if (!string.IsNullOrEmpty(permission) && permission.ToLower().EndsWith("permissiontype/read"))
column.ReadOnly = true;
}
else if (xmlReader.NodeType == XmlNodeType.Element && xmlReader.Name.Equals("NavigationProperty", StringComparison.OrdinalIgnoreCase))
{
//var containsTarget = xmlReader.GetAttribute("ContainsTarget");
var navigationPropertyName = xmlReader.GetAttribute("Name");
var navigationPropertyTypeString = xmlReader.GetAttribute("Type");
var navigationPropertyType = GetColumnTableType(navigationPropertyTypeString);
var groupName = GetTableName(navigationPropertyTypeString);
tableColumn = new TableColumn(navigationPropertyName, groupName, table, navigationPropertyType, []);
table.AddTableColumn(tableColumn);
}
else if (xmlReader.Name.Equals("EntityType", StringComparison.OrdinalIgnoreCase) && xmlReader.GetAttribute("Name") != entityName)
{
break;
Expand Down Expand Up @@ -480,6 +518,22 @@ private static Type GetColumnType(string columnTypeString)
return typeof(object);
}

private static Type GetColumnTableType(string columnTableTypeString)
{
if (columnTableTypeString.StartsWith("Collection", StringComparison.OrdinalIgnoreCase))
return typeof(Collection<object>);

return typeof(object);
}

private static string GetTableName(string columnTableTypeString)
{
var result = columnTableTypeString.Replace("Collection(", "", StringComparison.OrdinalIgnoreCase);
result = result.Replace("Microsoft.NAV.", "", StringComparison.OrdinalIgnoreCase);
result = result.Replace("NAV.", "", StringComparison.OrdinalIgnoreCase);
return result.Replace(")", "", StringComparison.OrdinalIgnoreCase);
}

/// <inheritdoc />
public override ISourceReader GetReader(Mapping mapping)
{
Expand Down
28 changes: 24 additions & 4 deletions src/ODataSourceReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,21 +166,30 @@ internal void CallEndpoing(IDictionary<string, string> headers, bool readFromLas
var selectAsParameters = GetSelectAsParameters(readFromLastRequestResponse);
var modeAsParemters = GetModeAsParameters();
var filterAsParameters = GetFilterAsParameters(_mapping);
var expandAsParameters = GetExpandAsParameters(_mapping);

if (!string.IsNullOrEmpty(modeAsParemters))
{
filterAsParameters.Add(modeAsParemters);
}

if (selectAsParameters.Any())
if (selectAsParameters.Count != 0)
{
parameters.Add("$select", string.Join(",", selectAsParameters));
}

if (filterAsParameters.Any())
if (filterAsParameters.Count != 0)
{
parameters.Add("$filter", string.Join(" and ", filterAsParameters));
}

if (expandAsParameters.Count != 0)
{
//It is possible to use $select for the middle table by just separating select and expand with semicolon:
//Customers?$select=CustomerID&$expand=Orders($select=OrderID;$expand=Order_Details($select=UnitPrice))
parameters.Add("$expand", string.Join(",", expandAsParameters));
}

if (_endpoint.Parameters != null)
{
foreach (var parameter in _endpoint.Parameters)
Expand Down Expand Up @@ -336,10 +345,10 @@ private string GetModeAsParameters()
private List<string> GetSelectAsParameters(bool readFromLastRequestResponse)
{
List<string> result = new();
var activeColumnMappings = _mapping.GetColumnMappings().Where(obj => obj.Active).ToList();
var activeColumnMappings = _mapping.GetColumnMappings().Where(obj => obj.Active && obj.SourceColumn != null).ToList();
if (activeColumnMappings.Any())
{
var selectColumnNames = activeColumnMappings.Where(obj => obj.SourceColumn != null)?.Select(obj => obj.SourceColumn.Name).ToList();
var selectColumnNames = activeColumnMappings.Where(obj => string.IsNullOrEmpty(obj.SourceColumn.Group))?.Select(obj => obj.SourceColumn.Name).ToList();

if (readFromLastRequestResponse)
{
Expand Down Expand Up @@ -507,6 +516,17 @@ private void LogWarningForConditional(MappingConditional item)
_logger?.Warn($"Can only add {item.ConditionalOperator} on Edm.String and the {item.SourceColumn.Name} is a type of {item.SourceColumn.Type.Name} for the table mapping {_mapping.SourceTable.Name} to {_mapping.DestinationTable.Name}, so this have been removed from the $filter.");
}

private List<string> GetExpandAsParameters(Mapping mapping)
{
List<string> result = new();
var sourceColumnsWithGroups = mapping.SourceTable.Columns.Where(obj => !string.IsNullOrEmpty(obj.Group));
if (sourceColumnsWithGroups.Any())
{
result.AddRange(sourceColumnsWithGroups.DistinctBy(obj => obj.Group).Select(obj => obj.Name));
}
return result;
}

/// <summary>
/// Handles a specified response stream, by creating an IEnumerable with yield return, so we can later enumerate the result one object at a time without loading them all into memory
/// </summary>
Expand Down
Loading