Skip to content

Commit a999b62

Browse files
rohkhannseantleonardaaronburtle
authored
Adding pagination limits. (#2153)
## Why make this change? This change allows users to use the runtime config to provide limits on the number of records that can be retrieved through paginated calls. It also gives them the ability to define a default size which will be the size returned when no pagination input is given (no first in case of graphql and no limit in case of rest calls). ## What is this change? It adds the nullable paginationoptions property to runtimeoptions of runtimeConfig. Default page size is set to 100. Max page size is set to 100,000 ( can be altered by the user in their runtimeconfig). A call with -1 pagination input will result in max page size records being returned. Any call with a pagination number higher than max page size will be rejected with bad request. In config: default-page-size is an integer default-page-size default is 100 default-page-size value -1 means "same as max-page-size" default-page-size value 0 is an error default-page-size value less than -1 is an error default-page-size value more than than max-page-size is an error max-page-size is an integer max-page-size default is 100,000 max-page-size value -1 means "same as int.MaxValue" max-page-size value 0 is an error max-page-size value less than -1 is an error max-page-size value more than int.MaxValue is an error In a query (REST or GQL): $first=-1 means "whatever the max-page-size value is" $first=less than -1 is an error $first=0 is an error $first=(any value more than max-page-size) is an error Sample configuration file: ```json { "runtime": { "pagination": { "default-page-size": -1, "max-page-size": 1233 } } } ``` ## How was this tested? 1. For default page value of 100, invalid page value of 0 or <-1 the existing test cases should cover. 2. Added tests for both GQL and REST for above conditions. --------- Co-authored-by: Sean Leonard <[email protected]> Co-authored-by: aaronburtle <[email protected]>
1 parent e660ba3 commit a999b62

21 files changed

+476
-45
lines changed
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System.Diagnostics.CodeAnalysis;
5+
using System.Net;
6+
using System.Text.Json.Serialization;
7+
using Azure.DataApiBuilder.Service.Exceptions;
8+
9+
namespace Azure.DataApiBuilder.Config.ObjectModel;
10+
11+
/// <summary>
12+
/// Pagination options for the dab setup.
13+
/// Properties are nullable to support DAB CLI merge config
14+
/// expected behavior.
15+
/// </summary>
16+
public record PaginationOptions
17+
{
18+
/// <summary>
19+
/// Default page size.
20+
/// </summary>
21+
public const uint DEFAULT_PAGE_SIZE = 100;
22+
23+
/// <summary>
24+
/// Max page size.
25+
/// </summary>
26+
public const uint MAX_PAGE_SIZE = 100000;
27+
28+
/// <summary>
29+
/// The default page size for pagination.
30+
/// </summary>
31+
[JsonPropertyName("default-page-size")]
32+
public int? DefaultPageSize { get; init; } = null;
33+
34+
/// <summary>
35+
/// The max page size for pagination.
36+
/// </summary>
37+
[JsonPropertyName("max-page-size")]
38+
public int? MaxPageSize { get; init; } = null;
39+
40+
[JsonConstructor]
41+
public PaginationOptions(int? DefaultPageSize = null, int? MaxPageSize = null)
42+
{
43+
if (MaxPageSize is not null)
44+
{
45+
ValidatePageSize((int)MaxPageSize);
46+
this.MaxPageSize = MaxPageSize == -1 ? Int32.MaxValue : (int)MaxPageSize;
47+
UserProvidedMaxPageSize = true;
48+
}
49+
else
50+
{
51+
this.MaxPageSize = (int)MAX_PAGE_SIZE;
52+
}
53+
54+
if (DefaultPageSize is not null)
55+
{
56+
ValidatePageSize((int)DefaultPageSize);
57+
this.DefaultPageSize = DefaultPageSize == -1 ? (int)this.MaxPageSize : (int)DefaultPageSize;
58+
UserProvidedDefaultPageSize = true;
59+
}
60+
else
61+
{
62+
this.DefaultPageSize = (int)DEFAULT_PAGE_SIZE;
63+
}
64+
65+
if (this.DefaultPageSize > this.MaxPageSize)
66+
{
67+
throw new DataApiBuilderException(
68+
message: "Pagination options invalid. The default page size cannot be greater than max page size",
69+
statusCode: HttpStatusCode.ServiceUnavailable,
70+
subStatusCode: DataApiBuilderException.SubStatusCodes.ConfigValidationError);
71+
}
72+
}
73+
74+
/// <summary>
75+
/// Flag which informs CLI and JSON serializer whether to write default page size.
76+
/// property and value to the runtime config file.
77+
/// When user doesn't provide the default-page-size property/value, which signals DAB to use the default,
78+
/// the DAB CLI should not write the default value to a serialized config.
79+
/// This is because the user's intent is to use DAB's default value which could change
80+
/// and DAB CLI writing the property and value would lose the user's intent.
81+
/// This is because if the user were to use the CLI created config, a default-page-size
82+
/// property/value specified would be interpreted by DAB as "user explicitly default-page-size."
83+
/// </summary>
84+
[JsonIgnore(Condition = JsonIgnoreCondition.Always)]
85+
[MemberNotNullWhen(true, nameof(DefaultPageSize))]
86+
public bool UserProvidedDefaultPageSize { get; init; } = false;
87+
88+
/// <summary>
89+
/// Flag which informs CLI and JSON serializer whether to write max-page-size
90+
/// property and value to the runtime config file.
91+
/// When user doesn't provide the max-page-size property/value, which signals DAB to use the default,
92+
/// the DAB CLI should not write the default value to a serialized config.
93+
/// This is because the user's intent is to use DAB's default value which could change
94+
/// and DAB CLI writing the property and value would lose the user's intent.
95+
/// This is because if the user were to use the CLI created config, a max-page-size
96+
/// property/value specified would be interpreted by DAB as "user explicitly max-page-size."
97+
/// </summary>
98+
[JsonIgnore(Condition = JsonIgnoreCondition.Always)]
99+
[MemberNotNullWhen(true, nameof(MaxPageSize))]
100+
public bool UserProvidedMaxPageSize { get; init; } = false;
101+
102+
private static void ValidatePageSize(int pageSize)
103+
{
104+
if (pageSize < -1 || pageSize == 0 || pageSize > Int32.MaxValue)
105+
{
106+
throw new DataApiBuilderException(
107+
message: "Pagination options invalid. Page size arguments cannot be 0, exceed max int value or be less than -1",
108+
statusCode: HttpStatusCode.ServiceUnavailable,
109+
subStatusCode: DataApiBuilderException.SubStatusCodes.ConfigValidationError);
110+
}
111+
}
112+
}

src/Config/ObjectModel/RuntimeConfig.cs

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -471,4 +471,45 @@ Runtime.GraphQL.MultipleMutationOptions is not null &&
471471
Runtime.GraphQL.MultipleMutationOptions.MultipleCreateOptions is not null &&
472472
Runtime.GraphQL.MultipleMutationOptions.MultipleCreateOptions.Enabled);
473473
}
474+
475+
public uint DefaultPageSize()
476+
{
477+
return (uint?)Runtime?.Pagination?.DefaultPageSize ?? PaginationOptions.DEFAULT_PAGE_SIZE;
478+
}
479+
480+
public uint MaxPageSize()
481+
{
482+
return (uint?)Runtime?.Pagination?.MaxPageSize ?? PaginationOptions.MAX_PAGE_SIZE;
483+
}
484+
485+
/// <summary>
486+
/// Get the pagination limit from the runtime configuration.
487+
/// </summary>
488+
/// <param name="first">The pagination input from the user. Example: $first=10</param>
489+
/// <returns></returns>
490+
/// <exception cref="DataApiBuilderException"></exception>
491+
public uint GetPaginationLimit(int? first)
492+
{
493+
uint defaultPageSize = this.DefaultPageSize();
494+
uint maxPageSize = this.MaxPageSize();
495+
496+
if (first is not null)
497+
{
498+
if (first < -1 || first == 0 || first > maxPageSize)
499+
{
500+
throw new DataApiBuilderException(
501+
message: $"Invalid number of items requested, {nameof(first)} argument must be either -1 or a positive number within the max page size limit of {maxPageSize}. Actual value: {first}",
502+
statusCode: HttpStatusCode.BadRequest,
503+
subStatusCode: DataApiBuilderException.SubStatusCodes.BadRequest);
504+
}
505+
else
506+
{
507+
return (first == -1 ? maxPageSize : (uint)first);
508+
}
509+
}
510+
else
511+
{
512+
return defaultPageSize;
513+
}
514+
}
474515
}

src/Config/ObjectModel/RuntimeOptions.cs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ public record RuntimeOptions
1414
public string? BaseRoute { get; init; }
1515
public TelemetryOptions? Telemetry { get; init; }
1616
public EntityCacheOptions? Cache { get; init; }
17+
public PaginationOptions? Pagination { get; init; }
1718

1819
[JsonConstructor]
1920
public RuntimeOptions(
@@ -22,14 +23,16 @@ public RuntimeOptions(
2223
HostOptions? Host,
2324
string? BaseRoute = null,
2425
TelemetryOptions? Telemetry = null,
25-
EntityCacheOptions? Cache = null)
26+
EntityCacheOptions? Cache = null,
27+
PaginationOptions? Pagination = null)
2628
{
2729
this.Rest = Rest;
2830
this.GraphQL = GraphQL;
2931
this.Host = Host;
3032
this.BaseRoute = BaseRoute;
3133
this.Telemetry = Telemetry;
3234
this.Cache = Cache;
35+
this.Pagination = Pagination;
3336
}
3437

3538
/// <summary>

src/Core/Models/GraphQLFilterParsers.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,7 @@ private void HandleNestedFilterForCosmos(
315315
CosmosExistsQueryStructure existsQuery = new(
316316
ctx,
317317
new Dictionary<string, object?>(),
318+
_configProvider,
318319
metadataProvider,
319320
queryStructure.AuthorizationResolver,
320321
this,

src/Core/Models/RestRequestContexts/RestRequestContext.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ protected RestRequestContext(string entityName, DatabaseObject dbo)
8888
/// Based on request this property may or may not be populated.
8989
/// </summary>
9090

91-
public uint? First { get; set; }
91+
public int? First { get; set; }
9292
/// <summary>
9393
/// Is the result supposed to be multiple values or not.
9494
/// </summary>

src/Core/Resolvers/CosmosExistsQueryStructure.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Licensed under the MIT License.
33

44
using Azure.DataApiBuilder.Auth;
5+
using Azure.DataApiBuilder.Core.Configurations;
56
using Azure.DataApiBuilder.Core.Models;
67
using Azure.DataApiBuilder.Core.Services;
78
using HotChocolate.Resolvers;
@@ -15,13 +16,15 @@ public class CosmosExistsQueryStructure : CosmosQueryStructure
1516
/// </summary>
1617
public CosmosExistsQueryStructure(IMiddlewareContext context,
1718
IDictionary<string, object?> parameters,
19+
RuntimeConfigProvider runtimeConfigProvider,
1820
ISqlMetadataProvider metadataProvider,
1921
IAuthorizationResolver authorizationResolver,
2022
GQLFilterParser gQLFilterParser,
2123
IncrementingInteger? counter = null,
2224
List<Predicate>? predicates = null)
2325
: base(context,
2426
parameters,
27+
runtimeConfigProvider,
2528
metadataProvider,
2629
authorizationResolver,
2730
gQLFilterParser,

src/Core/Resolvers/CosmosQueryEngine.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public async Task<Tuple<JsonDocument, IMetadata>> ExecuteAsync(
6969

7070
ISqlMetadataProvider metadataStoreProvider = _metadataProviderFactory.GetMetadataProvider(dataSourceName);
7171

72-
CosmosQueryStructure structure = new(context, parameters, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
72+
CosmosQueryStructure structure = new(context, parameters, _runtimeConfigProvider, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
7373
RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig();
7474

7575
string queryString = _queryBuilder.Build(structure);
@@ -201,7 +201,7 @@ public async Task<Tuple<IEnumerable<JsonDocument>, IMetadata>> ExecuteListAsync(
201201
// TODO: add support for TOP and Order-by push-down
202202

203203
ISqlMetadataProvider metadataStoreProvider = _metadataProviderFactory.GetMetadataProvider(dataSourceName);
204-
CosmosQueryStructure structure = new(context, parameters, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
204+
CosmosQueryStructure structure = new(context, parameters, _runtimeConfigProvider, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
205205
CosmosClient client = _clientProvider.Clients[dataSourceName];
206206
Container container = client.GetDatabase(structure.Database).GetContainer(structure.Container);
207207
QueryDefinition querySpec = new(_queryBuilder.Build(structure));

src/Core/Resolvers/CosmosQueryStructure.cs

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using Azure.DataApiBuilder.Auth;
66
using Azure.DataApiBuilder.Config.DatabasePrimitives;
77
using Azure.DataApiBuilder.Config.ObjectModel;
8+
using Azure.DataApiBuilder.Core.Configurations;
89
using Azure.DataApiBuilder.Core.Models;
910
using Azure.DataApiBuilder.Core.Services;
1011
using Azure.DataApiBuilder.Core.Services.MetadataProviders;
@@ -36,10 +37,12 @@ public class CosmosQueryStructure : BaseQueryStructure
3637
public string Container { get; internal set; }
3738
public string Database { get; internal set; }
3839
public string? Continuation { get; internal set; }
39-
public int? MaxItemCount { get; internal set; }
40+
public uint? MaxItemCount { get; internal set; }
4041
public string? PartitionKeyValue { get; internal set; }
4142
public List<OrderByColumn> OrderByColumns { get; internal set; }
4243

44+
public RuntimeConfigProvider RuntimeConfigProvider { get; internal set; }
45+
4346
public string GetTableAlias()
4447
{
4548
return $"table{TableCounter.Next()}";
@@ -48,6 +51,7 @@ public string GetTableAlias()
4851
public CosmosQueryStructure(
4952
IMiddlewareContext context,
5053
IDictionary<string, object?> parameters,
54+
RuntimeConfigProvider provider,
5155
ISqlMetadataProvider metadataProvider,
5256
IAuthorizationResolver authorizationResolver,
5357
GQLFilterParser gQLFilterParser,
@@ -58,6 +62,7 @@ public CosmosQueryStructure(
5862
_context = context;
5963
SourceAlias = _containerAlias;
6064
DatabaseObject.Name = _containerAlias;
65+
RuntimeConfigProvider = provider;
6166
Init(parameters);
6267
}
6368

@@ -116,7 +121,6 @@ private void Init(IDictionary<string, object?> queryParams)
116121

117122
IsPaginated = QueryBuilder.IsPaginationType(underlyingType);
118123
OrderByColumns = new();
119-
120124
if (IsPaginated)
121125
{
122126
FieldNode? fieldNode = ExtractItemsQueryField(selection.SyntaxNode);
@@ -155,13 +159,21 @@ private void Init(IDictionary<string, object?> queryParams)
155159
(CosmosSqlMetadataProvider)MetadataProvider);
156160
}
157161

162+
RuntimeConfigProvider.TryGetConfig(out RuntimeConfig? runtimeConfig);
158163
// first and after will not be part of query parameters. They will be going into headers instead.
159164
// TODO: Revisit 'first' while adding support for TOP queries
160165
if (queryParams.ContainsKey(QueryBuilder.PAGE_START_ARGUMENT_NAME))
161166
{
162-
MaxItemCount = (int?)queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
167+
object? firstArgument = queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
168+
MaxItemCount = runtimeConfig?.GetPaginationLimit((int?)firstArgument);
169+
163170
queryParams.Remove(QueryBuilder.PAGE_START_ARGUMENT_NAME);
164171
}
172+
else
173+
{
174+
// set max item count to default value.
175+
MaxItemCount = runtimeConfig?.DefaultPageSize();
176+
}
165177

166178
if (queryParams.ContainsKey(QueryBuilder.PAGINATION_TOKEN_ARGUMENT_NAME))
167179
{

src/Core/Resolvers/Sql Query Structures/SqlQueryStructure.cs

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
using HotChocolate.Language;
1818
using HotChocolate.Resolvers;
1919
using Microsoft.AspNetCore.Http;
20-
2120
namespace Azure.DataApiBuilder.Core.Resolvers
2221
{
2322
/// <summary>
@@ -59,15 +58,10 @@ public class SqlQueryStructure : BaseSqlQueryStructure
5958
/// </summary>
6059
public Dictionary<string, string> ColumnLabelToParam { get; }
6160

62-
/// <summary>
63-
/// Default limit when no first param is specified for list queries
64-
/// </summary>
65-
private const uint DEFAULT_LIST_LIMIT = 100;
66-
6761
/// <summary>
6862
/// The maximum number of results this query should return.
6963
/// </summary>
70-
private uint? _limit = DEFAULT_LIST_LIMIT;
64+
private uint? _limit = PaginationOptions.DEFAULT_PAGE_SIZE;
7165

7266
/// <summary>
7367
/// If this query is built because of a GraphQL query (as opposed to
@@ -198,7 +192,9 @@ public SqlQueryStructure(
198192
}
199193

200194
AddColumnsForEndCursor();
201-
_limit = context.First is not null ? context.First + 1 : DEFAULT_LIST_LIMIT + 1;
195+
runtimeConfigProvider.TryGetConfig(out RuntimeConfig? runtimeConfig);
196+
_limit = runtimeConfig?.GetPaginationLimit((int?)context.First) + 1;
197+
202198
ParametrizeColumns();
203199
}
204200

@@ -334,24 +330,19 @@ private SqlQueryStructure(
334330
IsListQuery = outputType.IsListType();
335331
}
336332

337-
if (IsListQuery && queryParams.ContainsKey(QueryBuilder.PAGE_START_ARGUMENT_NAME))
333+
if (IsListQuery)
338334
{
339-
// parse first parameter for all list queries
340-
object? firstObject = queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
341-
342-
if (firstObject != null)
335+
runtimeConfigProvider.TryGetConfig(out RuntimeConfig? runtimeConfig);
336+
if (queryParams.ContainsKey(QueryBuilder.PAGE_START_ARGUMENT_NAME))
343337
{
344-
int first = (int)firstObject;
345-
346-
if (first <= 0)
347-
{
348-
throw new DataApiBuilderException(
349-
message: $"Invalid number of items requested, {QueryBuilder.PAGE_START_ARGUMENT_NAME} argument must be an integer greater than 0 for {schemaField.Name}. Actual value: {first.ToString()}",
350-
statusCode: HttpStatusCode.BadRequest,
351-
subStatusCode: DataApiBuilderException.SubStatusCodes.BadRequest);
352-
}
353-
354-
_limit = (uint)first;
338+
// parse first parameter for all list queries
339+
object? firstObject = queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
340+
_limit = runtimeConfig?.GetPaginationLimit((int?)firstObject);
341+
}
342+
else
343+
{
344+
// if first is not passed, we should use the default page size.
345+
_limit = runtimeConfig?.DefaultPageSize();
355346
}
356347
}
357348

0 commit comments

Comments
 (0)