Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history 
Signed-off-by: Morgan Douglas <[email protected]>
  • Loading branch information
@morgando
morgando committed Oct 2, 2024
1 parent 857f3a8 commit a560fce
Showing 1 changed file with 193 additions and 111 deletions.
304 changes: 193 additions & 111 deletions sqlite/ext/comdb2/files_util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,17 @@ static const struct compareInfo globCaseInfo = {'%', '_', '[', 0};
extern int patternCompare(const u8 *, const u8 *, const struct compareInfo *,
u32);

typedef struct constraint_value {
void *val;
constraint_type_t type;
} constraint_value_t;

typedef struct comdb2_files_constraints {
const char *file_name_constraint;
constraint_type_t file_name_constraint_flag;
constraint_value_t *file_name_constraints;
int n_file_name_constraints;

file_type_t file_type_constraint;
constraint_type_t file_type_constraint_flag;
constraint_value_t *file_type_constraints;
int n_file_type_constraints;

int chunk_size_constraint;
} comdb2_files_constraints_t;
Expand Down Expand Up @@ -141,33 +146,48 @@ static void set_chunk_size(db_file_t *f, size_t chunk_size)
dbfile_set_chunk_size(f->info, chunk_size);
}

static int file_name_matches_constraint(const comdb2_files_constraints_t * const constraints,
static int file_name_matches_constraint(const constraint_value_t * const constraints,
int n_constraints,
const char * const file_name) {
const int flag = constraints->file_name_constraint_flag;
if (!flag) { return 1; }

const char * const file_name_constraint = constraints->file_name_constraint;
if (flag & FILES_FILE_NAME_EQ_FLAG) {
return strcmp(file_name_constraint, file_name) == 0;
} else if (flag & FILES_FILE_NAME_NEQ_FLAG) {
return strcmp(file_name_constraint, file_name) != 0;
} else {
assert(flag & FILES_FILE_NAME_LIKE_FLAG);
return patternCompare((u8 *)file_name_constraint, (u8 *)file_name, &globCaseInfo, '[') == 0;
int is_match = 1;

for (int i=0; i<n_constraints && is_match; ++i) {
const constraint_value_t * const constraint = (constraints + i);
const char * c_val = (const char *) constraint->val;
const constraint_type_t c_type = constraint->type;

if (c_type == FILES_FILE_NAME_EQ_FLAG) {
is_match = strcmp((const char *) c_val, file_name) == 0;
} else if (c_type == FILES_FILE_NAME_NEQ_FLAG) {
is_match = strcmp((const char *) c_val, file_name) != 0;
} else {
assert(c_type == FILES_FILE_NAME_LIKE_FLAG);
is_match = patternCompare((u8 *)c_val, (u8 *)file_name, &globCaseInfo, '[') == 0;
}
}

return is_match;
}

static int file_type_matches_constraint(const comdb2_files_constraints_t * const constraints,
static int file_type_matches_constraint(const constraint_value_t * const constraints,
int n_constraints,
const file_type_t file_type) {
const int flag = constraints->file_type_constraint_flag;
if (!flag) { return 1; }
int is_match = 1;

if (flag & FILES_FILE_TYPE_EQ_FLAG) {
return (file_type == constraints->file_type_constraint);
} else {
assert(flag & FILES_FILE_TYPE_NEQ_FLAG);
return (file_type != constraints->file_type_constraint);
for (int i=0; i<n_constraints && is_match; ++i) {
const constraint_value_t * const constraint = (constraints + i);
const file_type_t c_val = (const file_type_t) constraint->val;
const constraint_type_t c_type = constraint->type;

if (c_type == FILES_FILE_TYPE_EQ_FLAG) {
is_match = (file_type == c_val);
} else {
assert(c_type == FILES_FILE_TYPE_NEQ_FLAG);
is_match = (file_type != c_val);
}
}

return is_match;
}

static int read_dir(const char *dirname, db_file_t **files, int *count, const comdb2_files_constraints_t *constraints)
Expand Down Expand Up @@ -226,7 +246,9 @@ static int read_dir(const char *dirname, db_file_t **files, int *count, const co
continue;
}

if (!file_name_matches_constraint(constraints, de->d_name)) {
if (!file_name_matches_constraint(constraints->file_name_constraints,
constraints->n_file_name_constraints,
de->d_name)) {
logmsg(LOGMSG_DEBUG, "%s:%d: ignoring %s\n", __func__, __LINE__,
de->d_name);
continue;
Expand All @@ -251,7 +273,9 @@ static int read_dir(const char *dirname, db_file_t **files, int *count, const co
type = FILES_TYPE_UNKNOWN;
}

if (!file_type_matches_constraint(constraints, type)) {
if (!file_type_matches_constraint(constraints->file_type_constraints,
constraints->n_file_type_constraints,
type)) {
logmsg(LOGMSG_DEBUG, "%s:%d: ignoring %s\n", __func__, __LINE__,
de->d_name);
continue;
Expand Down Expand Up @@ -437,45 +461,85 @@ static int file_cmp(const void *file1, const void *file2)
return strcmp(f1->name, f2->name);
}

static int is_file_name_constraint(const constraint_type_t constraint_name) {
return (constraint_name == FILES_FILE_NAME_LIKE_FLAG
|| constraint_name == FILES_FILE_NAME_EQ_FLAG
|| constraint_name == FILES_FILE_NAME_NEQ_FLAG);
}

static int is_file_type_constraint(const constraint_type_t constraint_name) {
return (constraint_name == FILES_FILE_TYPE_EQ_FLAG
|| constraint_name == FILES_FILE_TYPE_NEQ_FLAG);
}

int files_util_filter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,
const char *idxStr, int argc, sqlite3_value **argv)
{
int rc = SQLITE_OK;

systbl_files_cursor *pCur = (systbl_files_cursor *)pVtabCursor;
comdb2_files_constraints_t constraints = {0};
int i = 0;

if ((idxNum & FILES_FILE_NAME_LIKE_FLAG
|| idxNum & FILES_FILE_NAME_EQ_FLAG
|| idxNum & FILES_FILE_NAME_NEQ_FLAG)) {
logmsg(LOGMSG_DEBUG, "%s: Applying file name constraint\n", __func__);
constraints.file_name_constraint = (char *)sqlite3_value_text(argv[i++]);
constraints.file_name_constraint_flag = idxNum;
const constraint_type_t * const constraint_names = (const constraint_type_t * const) idxStr;
comdb2_files_constraints_t constraint_info = {0};

for (int i=0; i<argc; ++i) {
const constraint_type_t constraint_name = constraint_names[i];

if (is_file_name_constraint(constraint_name)) {
constraint_info.n_file_name_constraints++;
} else if (is_file_type_constraint(constraint_name)) {
constraint_info.n_file_type_constraints++;
}
}

if (idxNum & FILES_FILE_TYPE_EQ_FLAG
|| idxNum & FILES_FILE_TYPE_NEQ_FLAG) {
logmsg(LOGMSG_DEBUG, "%s: Applying file type constraint\n", __func__);
constraints.file_type_constraint = str_to_file_type((char *)sqlite3_value_text(argv[i++]));
constraints.file_type_constraint_flag = idxNum;
if (constraint_info.n_file_name_constraints) {
constraint_info.file_name_constraints = calloc(1,
constraint_info.n_file_name_constraints * sizeof(constraint_value_t));
}

if ((idxNum & FILES_CHUNK_SIZE_EQ_FLAG)) {
logmsg(LOGMSG_DEBUG, "%s: Applying chunk size constraint\n", __func__);
constraints.chunk_size_constraint = sqlite3_value_int64(argv[i++]);
if (constraint_info.n_file_type_constraints) {
constraint_info.file_type_constraints = calloc(1,
constraint_info.n_file_type_constraints * sizeof(constraint_value_t));
}

if (get_files((void **)&pCur->files, &pCur->nfiles, &constraints)) {
int file_name_idx = 0;
int file_type_idx = 0;

for (int i=0; i<argc; ++i) {
const constraint_type_t constraint_name = constraint_names[i];

if (is_file_name_constraint(constraint_name)) {
logmsg(LOGMSG_DEBUG, "%s: Applying file name constraint\n", __func__);
constraint_info.file_name_constraints[file_name_idx++] = (const constraint_value_t) {
(char *)sqlite3_value_text(argv[i]),
constraint_name,
};
} else if (is_file_type_constraint(constraint_name)) {
logmsg(LOGMSG_DEBUG, "%s: Applying file type constraint\n", __func__);
constraint_info.file_type_constraints[file_type_idx++] = (const constraint_value_t) {
(void *)str_to_file_type((char *)sqlite3_value_text(argv[i])),
constraint_name,
};
} else if (constraint_name == FILES_CHUNK_SIZE_EQ_FLAG) {
logmsg(LOGMSG_DEBUG, "%s: Applying chunk size constraint\n", __func__);
constraint_info.chunk_size_constraint = sqlite3_value_int64(argv[i]);
}
}

if (get_files((void **)&pCur->files, &pCur->nfiles, &constraint_info)) {
logmsg(LOGMSG_ERROR, "%s:%d: Failed to get files following filter\n",
__FILE__, __LINE__);
return SQLITE_ERROR;
rc = SQLITE_ERROR;
goto err;
}

pCur->rowid = 0;

qsort(pCur->files, pCur->nfiles, sizeof(db_file_t), file_cmp);

return SQLITE_OK;
err:
if (constraint_info.file_name_constraints) { free(constraint_info.file_name_constraints); }
if (constraint_info.file_type_constraints) { free(constraint_info.file_type_constraints); }
return rc;
}

static void add_constraint(sqlite3_index_info * const pIdxInfo,
Expand All @@ -489,67 +553,87 @@ static void add_constraint(sqlite3_index_info * const pIdxInfo,
}

// Very cursory cost estimate.
static double estimate_cost(const int constraint) {
static double estimate_cost(const constraint_type_t * const constraints, const int n_constraints) {
double cost_est = DBL_MAX;
if (constraint & FILES_FILE_NAME_EQ_FLAG) {
cost_est /= pow(10, 6);
}
if (constraint & FILES_FILE_NAME_LIKE_FLAG) {
cost_est /= pow(10, 5);
}
if (constraint & FILES_FILE_TYPE_EQ_FLAG) {
cost_est /= pow(10, 4);
for (int i=0; i<n_constraints; ++i) {
const constraint_type_t constraint = constraints[i];

if (constraint == FILES_FILE_NAME_EQ_FLAG) {
cost_est /= pow(10, 6);
}
if (constraint == FILES_FILE_NAME_LIKE_FLAG) {
cost_est /= pow(10, 5);
}
if (constraint == FILES_FILE_TYPE_EQ_FLAG) {
cost_est /= pow(10, 4);
}
if (constraint == FILES_FILE_TYPE_NEQ_FLAG) {
cost_est /= pow(10, 3);
}
if (constraint == FILES_FILE_NAME_NEQ_FLAG) {
cost_est /= pow(10, 2);
}
}
if (constraint & FILES_FILE_TYPE_NEQ_FLAG) {
cost_est /= pow(10, 3);
return cost_est;
}

static int parse_file_name_constraint(const struct sqlite3_index_constraint * const pConstraint, constraint_type_t * const constraint) {
if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_LIKE) {
*constraint = FILES_FILE_NAME_LIKE_FLAG;
} else if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_EQ) {
*constraint = FILES_FILE_NAME_EQ_FLAG;
} else if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_NE) {
*constraint = FILES_FILE_NAME_NEQ_FLAG;
} else {
logmsg(LOGMSG_ERROR, "%s:%d: Column '%s' can only be constrained with 'like', '=', or '!='\n",
__FILE__, __LINE__, print_column_name(FILES_COLUMN_FILENAME));
return SQLITE_ERROR;
}
if (FILES_FILE_NAME_NEQ_FLAG) {
cost_est /= pow(10, 2);

return SQLITE_OK;
}

static int parse_file_type_constraint(const struct sqlite3_index_constraint * const pConstraint, constraint_type_t * const constraint) {
if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_EQ) {
*constraint = FILES_FILE_TYPE_EQ_FLAG;
} else if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_NE) {
*constraint = FILES_FILE_TYPE_NEQ_FLAG;
} else {
logmsg(LOGMSG_ERROR, "%s:%d: Column '%s' can only be constrained with '=', or '!='\n",
__FILE__, __LINE__, print_column_name(FILES_COLUMN_TYPE));
return SQLITE_ERROR;
}
return cost_est;

return SQLITE_OK;
}

static int parse_constraint(const struct sqlite3_index_constraint * const pConstraint,
int *idxNum, int i, int *filenameIdx, int *filetypeIdx, int *chunkSizeIdx) {
switch (pConstraint->iColumn) {
case FILES_COLUMN_FILENAME:
if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_LIKE) {
*idxNum |= FILES_FILE_NAME_LIKE_FLAG;
} else if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_EQ) {
*idxNum |= FILES_FILE_NAME_EQ_FLAG;
} else if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_NE) {
*idxNum |= FILES_FILE_NAME_NEQ_FLAG;
} else {
logmsg(LOGMSG_ERROR, "%s:%d: Column '%s' can only be constrained with 'like', '=', or '!='\n",
__FILE__, __LINE__, print_column_name(FILES_COLUMN_FILENAME));
return SQLITE_ERROR;
}
static int parse_file_chunk_size_constraint(const struct sqlite3_index_constraint * const pConstraint, constraint_type_t * const constraint) {
if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_EQ) {
*constraint = FILES_CHUNK_SIZE_EQ_FLAG;
} else {
logmsg(LOGMSG_ERROR, "%s:%d: Column '%s' can only be constrained with '=', or '!='\n",
__FILE__, __LINE__, print_column_name(FILES_COLUMN_CHUNK_SIZE));
return SQLITE_ERROR;
}

*filenameIdx = i;
break;
case FILES_COLUMN_TYPE:
if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_EQ) {
*idxNum |= FILES_FILE_TYPE_EQ_FLAG;
} else if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_NE) {
*idxNum |= FILES_FILE_TYPE_NEQ_FLAG;
} else {
logmsg(LOGMSG_ERROR, "%s:%d: Column '%s' can only be constrained with '=' or '!='\n",
__FILE__, __LINE__, print_column_name(FILES_COLUMN_TYPE));
return SQLITE_ERROR;
}
return SQLITE_OK;
}

*filetypeIdx = i;
break;
case FILES_COLUMN_CHUNK_SIZE:
if (pConstraint->op != SQLITE_INDEX_CONSTRAINT_EQ) {
logmsg(LOGMSG_ERROR, "%s:%d: Column '%s' can only be constrained with '='\n",
__FILE__, __LINE__, print_column_name(FILES_COLUMN_CHUNK_SIZE));
return SQLITE_ERROR;
}
static int parse_constraint(const struct sqlite3_index_constraint * const pConstraint,
const int i, constraint_type_t * const constraints) {

*idxNum |= FILES_CHUNK_SIZE_EQ_FLAG;
*chunkSizeIdx = i;
break;
if (pConstraint->iColumn == FILES_COLUMN_FILENAME
&& parse_file_name_constraint(pConstraint, constraints + i))
{
return SQLITE_ERROR;
} else if (pConstraint->iColumn == FILES_COLUMN_TYPE
&& parse_file_type_constraint(pConstraint, constraints + i))
{
return SQLITE_ERROR;
} else if (pConstraint->iColumn == FILES_COLUMN_CHUNK_SIZE
&& parse_file_chunk_size_constraint(pConstraint, constraints + i))
{
return SQLITE_ERROR;
}

return SQLITE_OK;
Expand All @@ -558,29 +642,27 @@ static int parse_constraint(const struct sqlite3_index_constraint * const pConst
int files_util_best_index(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo)
{
int i; /* Loop over constraints */
int idxNum = 0; /* The query plan bitmask */
int nArg = 0; /* Number of arguments that filesFilter() expects */
int filenameIdx = -1;
int filetypeIdx = -1;
int chunkSizeIdx = -1;

const int omit = 1;

constraint_type_t * const constraints = sqlite3_malloc(sizeof(constraint_type_t)*pIdxInfo->nConstraint);
if (!constraints) { return ENOMEM; }
memset(constraints, 0, sizeof(constraint_type_t)*pIdxInfo->nConstraint);

const struct sqlite3_index_constraint *pConstraint;
pConstraint = pIdxInfo->aConstraint;
for (i = 0; i < pIdxInfo->nConstraint; i++, pConstraint++) {
if (pConstraint->usable == 0) continue;
if (parse_constraint(pConstraint, &idxNum, i, &filenameIdx, &filetypeIdx, &chunkSizeIdx)
if (parse_constraint(pConstraint, i, constraints)
!= SQLITE_OK) {
return SQLITE_ERROR;
}
if (constraints[i] != 0) { add_constraint(pIdxInfo, i, &nArg, omit); }
}

const int omit = 1;
add_constraint(pIdxInfo, filenameIdx, &nArg, omit);
add_constraint(pIdxInfo, filetypeIdx, &nArg, omit);
add_constraint(pIdxInfo, chunkSizeIdx, &nArg, omit);

pIdxInfo->estimatedCost = estimate_cost(idxNum);
pIdxInfo->orderByConsumed = 0;
pIdxInfo->idxNum = idxNum;
pIdxInfo->estimatedCost = estimate_cost(constraints, pIdxInfo->nConstraint);
pIdxInfo->idxStr = (char *) constraints;
pIdxInfo->needToFreeIdxStr = 1;
return SQLITE_OK;
}

0 comments on commit a560fce

Please sign in to comment.