Skip to content

Commit 83f549d

Browse files
ankohankoh
committed
Migrate to new buf api
1 parent 783ad7f commit 83f549d

35 files changed

+213
-259
lines changed

‎buf.gen.yaml

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,6 @@ plugins:
44
- name: protoc-gen-es
55
path: ./node_modules/.bin/protoc-gen-es
66
out: ./packages/dashql-protobuf/gen
7-
opt: target=ts
8-
9-
# Generate Web & Node.js gRPC library
10-
- name: protoc-gen-connect-es
11-
path: ./node_modules/.bin/protoc-gen-connect-es
12-
out: ./packages/dashql-protobuf/gen
13-
opt: target=ts
7+
opt:
8+
- target=ts
9+
- import_extension=js

‎package.json

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,7 @@
1111
},
1212
"devDependencies": {
1313
"@bufbuild/buf": "^1.53.0",
14-
"@bufbuild/protoc-gen-es": "^1.10.1",
15-
"@connectrpc/protoc-gen-connect-es": "^1.6.1",
14+
"@bufbuild/protoc-gen-es": "^2.2.5",
1615
"@eslint/js": "^9.25.1",
1716
"@tauri-apps/cli": "^2.5.0",
1817
"@types/node": "^22.15.1",

‎packages/dashql-app/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
"@ankoh/dashql-core": "link:../dashql-core-bindings",
1919
"@ankoh/dashql-protobuf": "link:../dashql-protobuf",
2020
"@bokuweb/zstd-wasm": "^0.0.27",
21-
"@bufbuild/protobuf": "^1.10.1",
21+
"@bufbuild/protobuf": "^2.2.5",
2222
"@codemirror/autocomplete": "^6.18.1",
2323
"@codemirror/commands": "^6.6.2",
2424
"@codemirror/language": "^6.10.3",

‎packages/dashql-app/src/compute/compute_bindings.test.ts

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import '@jest/globals';
33
import * as arrow from 'apache-arrow';
44
import * as compute from '@ankoh/dashql-compute';
55
import * as pb from '@ankoh/dashql-protobuf';
6+
import * as buf from "@bufbuild/protobuf";
67
import * as path from 'path';
78
import * as fs from 'fs';
89

@@ -63,16 +64,16 @@ describe('DashQLCompute Arrow IO', () => {
6364

6465
const testOrderByColumn = async (inTable: arrow.Table, columnName: string, asc: boolean, nullsFirst: boolean, mapper: (o: any) => any, expected: any[]) => {
6566
const dataFrame = createDataFrameFromTable(inTable);
66-
const dataFrameTransform = new pb.dashql.compute.DataFrameTransform({
67-
orderBy: new pb.dashql.compute.OrderByTransform({
67+
const dataFrameTransform = buf.create(pb.dashql.compute.DataFrameTransformSchema, {
68+
orderBy: buf.create(pb.dashql.compute.OrderByTransformSchema, {
6869
constraints: [{
6970
fieldName: columnName,
7071
ascending: asc,
7172
nullsFirst
7273
}]
7374
})
7475
});
75-
const orderByConfigBytes = dataFrameTransform.toBinary();
76+
const orderByConfigBytes = buf.toBinary(pb.dashql.compute.DataFrameTransformSchema, dataFrameTransform);
7677
const orderedFrame = await dataFrame.transform(orderByConfigBytes);
7778
dataFrame.free();
7879

@@ -100,32 +101,33 @@ describe('DashQLCompute OrderBy', () => {
100101

101102
const testBinning = async (inTable: arrow.Table, columnName: string, expectedStats: any[], expectedBins: any[]) => {
102103
const inFrame = createDataFrameFromTable(inTable);
103-
const statsTransform = new pb.dashql.compute.DataFrameTransform({
104-
groupBy: new pb.dashql.compute.GroupByTransform({
104+
const statsTransform = buf.create(pb.dashql.compute.DataFrameTransformSchema, {
105+
groupBy: buf.create(pb.dashql.compute.GroupByTransformSchema, {
105106
keys: [],
106107
aggregates: [
107-
new pb.dashql.compute.GroupByAggregate({
108+
buf.create(pb.dashql.compute.GroupByAggregateSchema, {
108109
fieldName: columnName,
109110
outputAlias: "min",
110111
aggregationFunction: pb.dashql.compute.AggregationFunction.Min,
111112
}),
112-
new pb.dashql.compute.GroupByAggregate({
113+
buf.create(pb.dashql.compute.GroupByAggregateSchema, {
113114
fieldName: columnName,
114115
outputAlias: "max",
115116
aggregationFunction: pb.dashql.compute.AggregationFunction.Max,
116117
})
117118
]
118119
})
119120
});
120-
const statsFrame = await inFrame.transform(statsTransform.toBinary());
121+
const statsTransformBuf = buf.toBinary(pb.dashql.compute.DataFrameTransformSchema, statsTransform);
122+
const statsFrame = await inFrame.transform(statsTransformBuf);
121123

122-
const binTransform = new pb.dashql.compute.DataFrameTransform({
123-
groupBy: new pb.dashql.compute.GroupByTransform({
124+
const binTransform = buf.create(pb.dashql.compute.DataFrameTransformSchema, {
125+
groupBy: buf.create(pb.dashql.compute.GroupByTransformSchema, {
124126
keys: [
125-
new pb.dashql.compute.GroupByKey({
127+
buf.create(pb.dashql.compute.GroupByKeySchema, {
126128
fieldName: columnName,
127129
outputAlias: "bin",
128-
binning: new pb.dashql.compute.GroupByKeyBinning({
130+
binning: buf.create(pb.dashql.compute.GroupByKeyBinningSchema, {
129131
statsMinimumFieldName: "min",
130132
statsMaximumFieldName: "max",
131133
binCount: 8,
@@ -136,22 +138,23 @@ const testBinning = async (inTable: arrow.Table, columnName: string, expectedSta
136138
})
137139
],
138140
aggregates: [
139-
new pb.dashql.compute.GroupByAggregate({
141+
buf.create(pb.dashql.compute.GroupByAggregateSchema, {
140142
fieldName: columnName,
141143
outputAlias: "count",
142144
aggregationFunction: pb.dashql.compute.AggregationFunction.CountStar,
143145
})
144146
]
145147
}),
146-
orderBy: new pb.dashql.compute.OrderByTransform({
148+
orderBy: buf.create(pb.dashql.compute.OrderByTransformSchema, {
147149
constraints: [{
148150
fieldName: "bin",
149151
ascending: true,
150152
nullsFirst: false,
151153
}]
152154
})
153155
});
154-
const binnedFrame = await inFrame.transformWithStats(binTransform.toBinary(), statsFrame);
156+
const binTransformBuf = buf.toBinary(pb.dashql.compute.DataFrameTransformSchema, binTransform);
157+
const binnedFrame = await inFrame.transform(binTransformBuf);
155158

156159
const statsTable = readDataFrame(statsFrame);
157160
const binnedTable = readDataFrame(binnedFrame);

‎packages/dashql-app/src/compute/compute_worker_bindings.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import * as arrow from 'apache-arrow';
22
import * as pb from '@ankoh/dashql-protobuf';
3+
import * as buf from "@bufbuild/protobuf";
34

45
import { Logger } from "../platform/logger.js";
56
import { ComputeWorkerRequestType, ComputeWorkerResponseType, ComputeWorkerResponseVariant, ComputeWorkerTask, ComputeWorkerTaskReturnType, ComputeWorkerTaskVariant } from "./compute_worker_request.js";
@@ -338,7 +339,7 @@ export class AsyncDataFrame {
338339

339340
/// Transform a data frame
340341
async transform(transform: pb.dashql.compute.DataFrameTransform, stats: AsyncDataFrame | null = null): Promise<AsyncDataFrame> {
341-
const bytes = transform.toBinary();
342+
const bytes = buf.toBinary(pb.dashql.compute.DataFrameTransformSchema, transform);
342343
const task = new ComputeWorkerTask<
343344
ComputeWorkerRequestType.DATAFRAME_TRANSFORM,
344345
{ frameId: number, buffer: Uint8Array, statsFrameId: number | null },

‎packages/dashql-app/src/compute/table_transforms.ts

Lines changed: 33 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import * as arrow from 'apache-arrow';
22
import * as pb from '@ankoh/dashql-protobuf';
3+
import * as buf from "@bufbuild/protobuf";
34

45
import { VariantKind } from '../utils/variant.js';
56
import { AsyncDataFrame } from './compute_worker_bindings.js';
@@ -330,7 +331,7 @@ export function createTableSummaryTransform(task: TableSummaryTask): [pb.dashql.
330331

331332
// Add count(*) aggregate
332333
const countColumn = `_count`;
333-
aggregates.push(new pb.dashql.compute.GroupByAggregate({
334+
aggregates.push(buf.create(pb.dashql.compute.GroupByAggregateSchema, {
334335
outputAlias: `_count`,
335336
aggregationFunction: pb.dashql.compute.AggregationFunction.CountStar,
336337
}));
@@ -348,17 +349,17 @@ export function createTableSummaryTransform(task: TableSummaryTask): [pb.dashql.
348349
const countAggregateColumn = `_${i}_count`;
349350
const minAggregateColumn = `_${i}_min`;
350351
const maxAggregateColumn = `_${i}_max`;
351-
aggregates.push(new pb.dashql.compute.GroupByAggregate({
352+
aggregates.push(buf.create(pb.dashql.compute.GroupByAggregateSchema, {
352353
fieldName: entry.value.inputFieldName,
353354
outputAlias: countAggregateColumn,
354355
aggregationFunction: pb.dashql.compute.AggregationFunction.Count,
355356
}));
356-
aggregates.push(new pb.dashql.compute.GroupByAggregate({
357+
aggregates.push(buf.create(pb.dashql.compute.GroupByAggregateSchema, {
357358
fieldName: entry.value.inputFieldName,
358359
outputAlias: minAggregateColumn,
359360
aggregationFunction: pb.dashql.compute.AggregationFunction.Min,
360361
}));
361-
aggregates.push(new pb.dashql.compute.GroupByAggregate({
362+
aggregates.push(buf.create(pb.dashql.compute.GroupByAggregateSchema, {
362363
fieldName: entry.value.inputFieldName,
363364
outputAlias: maxAggregateColumn,
364365
aggregationFunction: pb.dashql.compute.AggregationFunction.Max,
@@ -381,12 +382,12 @@ export function createTableSummaryTransform(task: TableSummaryTask): [pb.dashql.
381382
case STRING_COLUMN: {
382383
const countAggregateColumn = `_${i}_count`;
383384
const countDistinctAggregateColumn = `_${i}_countd`;
384-
aggregates.push(new pb.dashql.compute.GroupByAggregate({
385+
aggregates.push(buf.create(pb.dashql.compute.GroupByAggregateSchema, {
385386
fieldName: entry.value.inputFieldName,
386387
outputAlias: countAggregateColumn,
387388
aggregationFunction: pb.dashql.compute.AggregationFunction.Count,
388389
}));
389-
aggregates.push(new pb.dashql.compute.GroupByAggregate({
390+
aggregates.push(buf.create(pb.dashql.compute.GroupByAggregateSchema, {
390391
fieldName: entry.value.inputFieldName,
391392
outputAlias: countDistinctAggregateColumn,
392393
aggregationFunction: pb.dashql.compute.AggregationFunction.Count,
@@ -410,12 +411,12 @@ export function createTableSummaryTransform(task: TableSummaryTask): [pb.dashql.
410411
case LIST_COLUMN: {
411412
const countAggregateColumn = `_${i}_count`;
412413
const countDistinctAggregateColumn = `_${i}_countd`;
413-
aggregates.push(new pb.dashql.compute.GroupByAggregate({
414+
aggregates.push(buf.create(pb.dashql.compute.GroupByAggregateSchema, {
414415
fieldName: entry.value.inputFieldName,
415416
outputAlias: countAggregateColumn,
416417
aggregationFunction: pb.dashql.compute.AggregationFunction.Count,
417418
}));
418-
aggregates.push(new pb.dashql.compute.GroupByAggregate({
419+
aggregates.push(buf.create(pb.dashql.compute.GroupByAggregateSchema, {
419420
fieldName: entry.value.inputFieldName,
420421
outputAlias: countDistinctAggregateColumn,
421422
aggregationFunction: pb.dashql.compute.AggregationFunction.Count,
@@ -438,8 +439,8 @@ export function createTableSummaryTransform(task: TableSummaryTask): [pb.dashql.
438439
}
439440
}
440441
}
441-
const transform = new pb.dashql.compute.DataFrameTransform({
442-
groupBy: new pb.dashql.compute.GroupByTransform({
442+
const transform = buf.create(pb.dashql.compute.DataFrameTransformSchema, {
443+
groupBy: buf.create(pb.dashql.compute.GroupByTransformSchema, {
443444
keys: [],
444445
aggregates
445446
})
@@ -459,13 +460,13 @@ export function createColumnSummaryTransform(task: ColumnSummaryTask): pb.dashql
459460
case ORDINAL_COLUMN: {
460461
const minField = task.columnEntry.value.statsFields.minAggregateFieldName!;
461462
const maxField = task.columnEntry.value.statsFields.maxAggregateFieldName!;
462-
out = new pb.dashql.compute.DataFrameTransform({
463-
groupBy: new pb.dashql.compute.GroupByTransform({
463+
out = buf.create(pb.dashql.compute.DataFrameTransformSchema, {
464+
groupBy: buf.create(pb.dashql.compute.GroupByTransformSchema, {
464465
keys: [
465-
new pb.dashql.compute.GroupByKey({
466+
buf.create(pb.dashql.compute.GroupByKeySchema, {
466467
fieldName,
467468
outputAlias: "bin",
468-
binning: new pb.dashql.compute.GroupByKeyBinning({
469+
binning: buf.create(pb.dashql.compute.GroupByKeyBinningSchema, {
469470
statsMinimumFieldName: minField,
470471
statsMaximumFieldName: maxField,
471472
binCount: BIN_COUNT,
@@ -476,16 +477,16 @@ export function createColumnSummaryTransform(task: ColumnSummaryTask): pb.dashql
476477
})
477478
],
478479
aggregates: [
479-
new pb.dashql.compute.GroupByAggregate({
480+
buf.create(pb.dashql.compute.GroupByAggregateSchema, {
480481
fieldName,
481482
outputAlias: "count",
482483
aggregationFunction: pb.dashql.compute.AggregationFunction.CountStar,
483484
})
484485
]
485486
}),
486-
orderBy: new pb.dashql.compute.OrderByTransform({
487+
orderBy: buf.create(pb.dashql.compute.OrderByTransformSchema, {
487488
constraints: [
488-
new pb.dashql.compute.OrderByConstraint({
489+
buf.create(pb.dashql.compute.OrderByConstraintSchema, {
489490
fieldName: "bin",
490491
ascending: true,
491492
nullsFirst: false,
@@ -497,25 +498,25 @@ export function createColumnSummaryTransform(task: ColumnSummaryTask): pb.dashql
497498
}
498499
case LIST_COLUMN:
499500
case STRING_COLUMN: {
500-
out = new pb.dashql.compute.DataFrameTransform({
501-
groupBy: new pb.dashql.compute.GroupByTransform({
501+
out = buf.create(pb.dashql.compute.DataFrameTransformSchema, {
502+
groupBy: buf.create(pb.dashql.compute.GroupByTransformSchema, {
502503
keys: [
503-
new pb.dashql.compute.GroupByKey({
504+
buf.create(pb.dashql.compute.GroupByKeySchema, {
504505
fieldName,
505506
outputAlias: "key",
506507
})
507508
],
508509
aggregates: [
509-
new pb.dashql.compute.GroupByAggregate({
510+
buf.create(pb.dashql.compute.GroupByAggregateSchema, {
510511
fieldName,
511512
outputAlias: "count",
512513
aggregationFunction: pb.dashql.compute.AggregationFunction.CountStar,
513514
})
514515
]
515516
}),
516-
orderBy: new pb.dashql.compute.OrderByTransform({
517+
orderBy: buf.create(pb.dashql.compute.OrderByTransformSchema, {
517518
constraints: [
518-
new pb.dashql.compute.OrderByConstraint({
519+
buf.create(pb.dashql.compute.OrderByConstraintSchema, {
519520
fieldName: "count",
520521
ascending: false,
521522
nullsFirst: false,
@@ -531,8 +532,8 @@ export function createColumnSummaryTransform(task: ColumnSummaryTask): pb.dashql
531532
}
532533

533534
export function createOrderByTransform(constraints: pb.dashql.compute.OrderByConstraint[], limit?: number): pb.dashql.compute.DataFrameTransform {
534-
const out = new pb.dashql.compute.DataFrameTransform({
535-
orderBy: new pb.dashql.compute.OrderByTransform({
535+
const out = buf.create(pb.dashql.compute.DataFrameTransformSchema, {
536+
orderBy: buf.create(pb.dashql.compute.OrderByTransformSchema, {
536537
constraints,
537538
limit
538539
})
@@ -566,7 +567,7 @@ export function createPrecomputationTransform(schema: arrow.Schema, columns: Gri
566567

567568
// Prepend the row number column at position 0
568569
const rowNumberFieldName = createUniqueColumnName(`_rownum`, fieldNames);
569-
const rowNumberTransform = new pb.dashql.compute.RowNumberTransform({
570+
const rowNumberTransform = buf.create(pb.dashql.compute.RowNumberTransformSchema, {
570571
outputAlias: rowNumberFieldName
571572
});
572573
const rowNumberGridColumn: GridColumnGroup = {
@@ -590,7 +591,7 @@ export function createPrecomputationTransform(schema: arrow.Schema, columns: Gri
590591
case ORDINAL_COLUMN: {
591592
const binFieldId = nextOutputColumn++;
592593
const binFieldName = createUniqueColumnName(`_${i}_bin`, fieldNames);
593-
binningTransforms.push(new pb.dashql.compute.BinningTransform({
594+
binningTransforms.push(buf.create(pb.dashql.compute.BinningTransformSchema, {
594595
fieldName: column.value.inputFieldName,
595596
statsMaximumFieldName: column.value.statsFields!.maxAggregateFieldName!,
596597
statsMinimumFieldName: column.value.statsFields!.minAggregateFieldName!,
@@ -608,7 +609,7 @@ export function createPrecomputationTransform(schema: arrow.Schema, columns: Gri
608609
}
609610
case STRING_COLUMN: {
610611
const valueFieldName = createUniqueColumnName(`_${i}_id`, fieldNames);
611-
identifierTransforms.push(new pb.dashql.compute.ValueIdentifierTransform({
612+
identifierTransforms.push(buf.create(pb.dashql.compute.ValueIdentifierTransformSchema, {
612613
fieldName: column.value.inputFieldName,
613614
outputAlias: valueFieldName
614615
}));
@@ -623,7 +624,7 @@ export function createPrecomputationTransform(schema: arrow.Schema, columns: Gri
623624
}
624625
case LIST_COLUMN: {
625626
const valueFieldName = createUniqueColumnName(`_${i}_id`, fieldNames);
626-
identifierTransforms.push(new pb.dashql.compute.ValueIdentifierTransform({
627+
identifierTransforms.push(buf.create(pb.dashql.compute.ValueIdentifierTransformSchema, {
627628
fieldName: column.value.inputFieldName,
628629
outputAlias: valueFieldName
629630
}));
@@ -639,17 +640,17 @@ export function createPrecomputationTransform(schema: arrow.Schema, columns: Gri
639640
}
640641
}
641642

642-
const ordering = new pb.dashql.compute.OrderByTransform({
643+
const ordering = buf.create(pb.dashql.compute.OrderByTransformSchema, {
643644
constraints: [
644-
new pb.dashql.compute.OrderByConstraint({
645+
buf.create(pb.dashql.compute.OrderByConstraintSchema, {
645646
fieldName: rowNumberFieldName,
646647
ascending: true,
647648
nullsFirst: false
648649
})
649650
]
650651
});
651652

652-
const transform = new pb.dashql.compute.DataFrameTransform({
653+
const transform = buf.create(pb.dashql.compute.DataFrameTransformSchema, {
653654
rowNumber: rowNumberTransform,
654655
valueIdentifiers: identifierTransforms,
655656
binning: binningTransforms,

0 commit comments

Comments
 (0)