Skip to content

add support for array types and more functions #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

add support for array types and more functions #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
946699f
add support for array types and more functions
abhizer Jan 2, 2025
06644d5
add array functions
abhizer Jan 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/sqlancer/feldera/FelderaProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ protected void checkViewsAreValid(FelderaGlobalState globalState) {

@Override
public void generateDatabase(FelderaGlobalState globalState) throws Exception {
createTables(globalState, Randomly.fromOptions(4, 5, 6));
createViews(globalState, Randomly.fromOptions(4, 5, 6));
createTables(globalState, Randomly.fromOptions(8, 10, 12));
createViews(globalState, Randomly.fromOptions(8, 10, 12));
prepareTables(globalState);
}

Expand Down
218 changes: 163 additions & 55 deletions src/sqlancer/feldera/FelderaSchema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
package sqlancer.feldera;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.*;
import java.util.stream.Collectors;

import sqlancer.Randomly;
Expand Down Expand Up @@ -33,39 +30,6 @@ public FelderaSchema addTable(FelderaTable table) {
return new FelderaSchema(tables, this.pipelineName);
}

public static FelderaDataType getColumnType(String typeString) {
switch (typeString.toUpperCase()) {
case "BOOLEAN":
return FelderaDataType.BOOLEAN;
case "TINYINT":
return FelderaDataType.TINYINT;
case "SMALLINT":
return FelderaDataType.SMALLINT;
case "INT":
return FelderaDataType.INT;
case "BIGINT":
return FelderaDataType.BIGINT;
case "VARCHAR":
return FelderaDataType.VARCHAR;
case "CHAR":
return FelderaDataType.CHAR;
case "NULL":
return FelderaDataType.NULL;
case "TIME":
return FelderaDataType.TIME;
case "DATE":
return FelderaDataType.DATE;
case "TIMESTAMP":
return FelderaDataType.TIMESTAMP;
case "REAL":
return FelderaDataType.REAL;
case "DOUBLE":
return FelderaDataType.DOUBLE;
default:
throw new AssertionError(typeString);
}
}

public static FelderaSchema fromConnection(FelderaConnection con) throws Exception {
return new FelderaSchema(new ArrayList<>(), con.getPipelineName());
}
Expand All @@ -83,16 +47,14 @@ public String getPipelineName() {
}

public enum FelderaDataType {
BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, VARCHAR, CHAR, NULL, TIME, DATE, TIMESTAMP,
// DECIMAL,
BOOLEAN, INT, VARCHAR, CHAR, NULL, TIME, DATE, TIMESTAMP, DECIMAL, FLOAT, ANY,
// VARIANT,
// VARBINARY,
// INTERVAL,
// GEOMETRY,
// ROW,
// ARRAY,
// MAP,
// VARIANT,
REAL, DOUBLE;
ARRAY;

public static FelderaDataType getRandomNumericType() {
return Randomly
Expand All @@ -101,26 +63,57 @@ public static FelderaDataType getRandomNumericType() {

public boolean isNumeric() {
switch (this) {
case REAL:
case DOUBLE:
case TINYINT:
case SMALLINT:
case FLOAT:
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why this change?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I changed this type to only represent the primitive types, so FLOAT represents DOUBLE, BOOL, and INT represents all the different sizes of the integer types.
This is similar to the implementation done by other databases in sqlancer.

case INT:
case BIGINT:
case DECIMAL:
return true;
default:
return false;
}
}

public static FelderaDataType getRandomNonNullType() {
return Randomly.fromList(
Arrays.stream(values()).filter(t -> t != FelderaDataType.NULL).collect(Collectors.toList()));
return Randomly.fromList(Arrays.stream(values())
.filter(t -> t != FelderaDataType.NULL && t != FelderaDataType.ANY).collect(Collectors.toList()));
}

public static FelderaDataType[] nonNullValues() {
return Arrays.stream(values()).filter(t -> t != FelderaDataType.NULL && t != FelderaDataType.ANY)
.toArray(FelderaDataType[]::new);
}

public static FelderaDataType getRandomType() {
return Randomly.fromOptions(values());
}
}

public static class FelderaCompositeDataType {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is not general enough, MAP has two generic arguments.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe call this GenericType?
Too bad we aren't reusing the structures we already have in the compiler.

private final FelderaDataType dataType;
private final int size;
private final int scale;
private final FelderaCompositeDataType elementType;

public FelderaCompositeDataType(FelderaDataType dataType, int size, int scale) {
this.dataType = dataType;
this.size = size;
this.scale = scale;
this.elementType = null;
}

public static FelderaCompositeDataType arrayOf(FelderaDataType elementType) {
return new FelderaCompositeDataType(FelderaDataType.ARRAY, getRandomFromPrimitiveType(elementType));
}

public FelderaCompositeDataType(FelderaDataType dataType, FelderaCompositeDataType elementType) {
if (dataType != FelderaDataType.ARRAY) {
throw new IllegalArgumentException("dataType must be ARRAY");
}

this.dataType = dataType;
this.scale = -1;
this.size = -1;
this.elementType = elementType;
}

public FelderaExpression getRandomConstant(FelderaGlobalState globalState) {
if (Randomly.getBooleanWithSmallProbability()) {
Expand All @@ -129,29 +122,144 @@ public FelderaExpression getRandomConstant(FelderaGlobalState globalState) {

return FelderaConstant.getRandomConstant(globalState, this);
}

public boolean isNumeric() {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks wrong.
An array is not numeric.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This method is not specific to arrays.
It returns true for int types, FP types and decimal.

return this.getPrimitiveType().isNumeric();
}

public FelderaCompositeDataType getElementType() {
return this.elementType;
}

public static FelderaCompositeDataType getBooleanType() {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand what this class is supposed to be.
Maybe it's a type factory?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this code should be in a separate factory class, which may also implement a singleton pattern.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is designed represents all datatypes in feldera (doesn't quite support map yet).
Similar to other implementations:

sqlancer/src/sqlancer/cockroachdb/CockroachDBSchema.java

Line 36 in 23b8b43

public static class CockroachDBCompositeDataType {

return FelderaCompositeDataType.getRandomFromPrimitiveType(FelderaDataType.BOOLEAN);
}

public static FelderaCompositeDataType getRandomVarcharType() {
return FelderaCompositeDataType.getRandomFromPrimitiveType(FelderaDataType.VARCHAR);
}

public static FelderaCompositeDataType getRandomNumericType() {
FelderaDataType type = FelderaDataType.getRandomNumericType();
return FelderaCompositeDataType.getRandomFromPrimitiveType(type);
}

public FelderaDataType getPrimitiveType() {
return dataType;
}

public int getSize() {
return size;
}

public int getScale() {
return scale;
}

public boolean isArray() {
return dataType == FelderaDataType.ARRAY;
}

public static FelderaCompositeDataType getRandomFromPrimitiveType(FelderaDataType type) {
int size = -1;
int scale = -1;
switch (type) {
case FLOAT:
size = Randomly.fromOptions(32, 64);
return new FelderaCompositeDataType(type, size, scale);
case INT:
size = Randomly.fromOptions(8, 16, 32, 64);
return new FelderaCompositeDataType(type, size, scale);
case ARRAY:
return new FelderaCompositeDataType(type, FelderaCompositeDataType.getRandomWithoutNull());
case DECIMAL:
scale = (int) Randomly.getNotCachedInteger(0, 10);
size = (int) Randomly.getNotCachedInteger(scale, 25);
return new FelderaCompositeDataType(type, size, scale);
case CHAR:
size = (int) Randomly.getNotCachedInteger(1, 10);
return new FelderaCompositeDataType(type, size, scale);
case VARCHAR:
if (Randomly.getBoolean()) {
size = (int) Randomly.getNotCachedInteger(1, 30);
}
return new FelderaCompositeDataType(type, size, scale);
default:
return new FelderaCompositeDataType(type, size, scale);
}
}

public static FelderaCompositeDataType getRandomWithoutNull() {
FelderaDataType type = FelderaDataType.getRandomNonNullType();
return FelderaCompositeDataType.getRandomFromPrimitiveType(type);
}

@Override
public String toString() {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this suggests that Type should be an interface and these all subclasses.
What are the constraints imposed from SqlLancer?
Maybe you really should reuse the compiler type hierarchy.

switch (dataType) {
case INT:
switch (size) {
case 8:
return "TINYINT";
case 16:
return "SMALLINT";
case 32:
return "INT";
case 64:
return "BIGINT";
default:
throw new AssertionError(dataType.toString() + scale);
}
case FLOAT:
switch (size) {
case 32:
return "REAL";
case 64:
return "DOUBLE";
default:
throw new AssertionError(dataType.toString() + scale);
}
case ARRAY:
if (elementType == null) {
throw new AssertionError(this);
}
return elementType + " ARRAY";
case CHAR:
return "CHAR(" + size + ")";
case VARCHAR:
if (size == -1) {
return "VARCHAR";
}
return "VARCHAR(" + size + ")";
case DECIMAL:
return "DECIMAL(" + size + ", " + scale + ")";
default:
return dataType.toString();
}
}
}

public static class FelderaFieldColumn extends FelderaColumn {
public FelderaFieldColumn(String name, FelderaDataType columnType) {
public FelderaFieldColumn(String name, FelderaCompositeDataType columnType) {
super(name, columnType);
}

public FelderaFieldColumn(String name, FelderaDataType columnType, boolean isNullable) {
public FelderaFieldColumn(String name, FelderaCompositeDataType columnType, boolean isNullable) {
super(name, columnType, isNullable);
// Note to self: later, assert that the Field column isn't something like INTERVAL
}
}

public static class FelderaColumn extends AbstractTableColumn<FelderaTable, FelderaDataType> {
public static class FelderaColumn extends AbstractTableColumn<FelderaTable, FelderaCompositeDataType> {

private final boolean isNullable;

public FelderaColumn(String name, FelderaDataType columnType) {
public FelderaColumn(String name, FelderaCompositeDataType columnType) {
super(name, null, columnType);
this.isNullable = false;
}

public FelderaColumn(String name, FelderaDataType columnType, boolean isNullable) {
public FelderaColumn(String name, FelderaCompositeDataType columnType, boolean isNullable) {
super(name, null, columnType);
this.isNullable = isNullable;
}
Expand All @@ -161,7 +269,7 @@ public FelderaColumnReference asColumnReference() {
}

public static FelderaColumn createDummy(String name) {
return new FelderaColumn(name, FelderaDataType.getRandomType());
return new FelderaColumn(name, FelderaCompositeDataType.getRandomWithoutNull());
}

public boolean isNullable() {
Expand Down
22 changes: 12 additions & 10 deletions src/sqlancer/feldera/ast/FelderaAggregate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,30 +13,32 @@ public class FelderaAggregate implements FelderaExpression {
private boolean blackbox;

public enum FelderaAggregateFunction {
AVG(FelderaSchema.FelderaDataType.INT, FelderaSchema.FelderaDataType.DOUBLE),
COUNT(FelderaSchema.FelderaDataType.values()), EVERY(FelderaSchema.FelderaDataType.BOOLEAN),
MAX(FelderaSchema.FelderaDataType.values()), MIN(FelderaSchema.FelderaDataType.values()),
AVG(FelderaSchema.FelderaDataType.INT, FelderaSchema.FelderaDataType.FLOAT),
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we don't even support FLOAT in SQL, this is very confusing.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With FLOAT, we represent both DOUBLE and REAL here.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Many functions work on double but not on real.

COUNT(FelderaSchema.FelderaDataType.nonNullValues()), EVERY(FelderaSchema.FelderaDataType.BOOLEAN),
MAX(FelderaSchema.FelderaDataType.nonNullValues()), MIN(FelderaSchema.FelderaDataType.nonNullValues()),
SOME(FelderaSchema.FelderaDataType.BOOLEAN),
SUM(FelderaSchema.FelderaDataType.INT, FelderaSchema.FelderaDataType.DOUBLE),
STDDEV(FelderaSchema.FelderaDataType.INT, FelderaSchema.FelderaDataType.DOUBLE),
STDDEV_POP(FelderaSchema.FelderaDataType.INT, FelderaSchema.FelderaDataType.DOUBLE),;
SUM(FelderaSchema.FelderaDataType.INT, FelderaSchema.FelderaDataType.FLOAT),
STDDEV(FelderaSchema.FelderaDataType.INT, FelderaSchema.FelderaDataType.FLOAT),
STDDEV_POP(FelderaSchema.FelderaDataType.INT, FelderaSchema.FelderaDataType.FLOAT),
COUNTIF(FelderaSchema.FelderaDataType.BOOLEAN);

private final FelderaSchema.FelderaDataType[] supportedReturnTypes;

FelderaAggregateFunction(FelderaSchema.FelderaDataType... supportedReturnTypes) {
this.supportedReturnTypes = supportedReturnTypes.clone();
}

public List<FelderaSchema.FelderaDataType> getTypes(FelderaSchema.FelderaDataType returnType) {
public List<FelderaSchema.FelderaCompositeDataType> getTypes(
FelderaSchema.FelderaCompositeDataType returnType) {
return Collections.singletonList(returnType);
}

public boolean supportsReturnType(FelderaSchema.FelderaDataType returnType) {
return Arrays.stream(supportedReturnTypes).anyMatch(t -> t == returnType)
public boolean supportsReturnType(FelderaSchema.FelderaCompositeDataType returnType) {
return Arrays.stream(supportedReturnTypes).anyMatch(t -> t == returnType.getPrimitiveType())
|| supportedReturnTypes.length == 0;
}

public static List<FelderaAggregateFunction> getAggregates(FelderaSchema.FelderaDataType type) {
public static List<FelderaAggregateFunction> getAggregates(FelderaSchema.FelderaCompositeDataType type) {
return Arrays.stream(values()).filter(p -> p.supportsReturnType(type)).collect(Collectors.toList());
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/sqlancer/feldera/ast/FelderaCast.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@

public class FelderaCast implements FelderaExpression {
private final FelderaExpression expr;
private final FelderaSchema.FelderaDataType type;
private final FelderaSchema.FelderaCompositeDataType type;

public FelderaCast(FelderaExpression expr, FelderaSchema.FelderaDataType type) {
public FelderaCast(FelderaExpression expr, FelderaSchema.FelderaCompositeDataType type) {
this.expr = expr;
this.type = type;
}
Expand Down
Loading