Skip to content

Commit

Permalink
debug cursor point truncation, debug stopProp using excludeCursor wro…
Browse files Browse the repository at this point in the history 
…ngfully
  • Loading branch information
@scorpevans
scorpevans committed Apr 15, 2017
1 parent 0ee968a commit e82b194
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 47 deletions.
6 changes: 6 additions & 0 deletions nodejs/TODO
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
debug: indexListQuery sometimes returns NULL instead of []

implement Join expressions i.e. computed fields on joins

make simple checks on interface functions to ensure that parameters are sanitized

capture UIDs separately from member-suffixes for zset; this help abstract better CRUD commands
e.g. some get-commands have to be a range if additional info is appended to the member

Expand Down
41 changes: 24 additions & 17 deletions nodejs/join.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ join.mergeStreams = function mergeStreams(join_config, then){
streamOrder = joinConfigDict.order,
streamConfigs = join_config.streamConfigs,
joint = join_config.joint,
limit = join_config.limit;
limit = (join_config.limit == '' ? null : join_config.limit);
var streamProps = [];
for(var i=0; i < streamConfigs.length; i++){
streamProps[i] = {};
Expand Down Expand Up @@ -241,7 +241,7 @@ join.mergeStreams = function mergeStreams(join_config, then){
var boundIndexMask = {};
var innerJoinCount = 0;
var refreshIdx = null; // the index of streamIndexes where joins are left off to fetch data
var ord = null; // the merges ord of the rangeconfigs
var ord = null; // the merged ord of the rangeconfigs
var streamJointMangles = []; // mangle info while merging ords
var streamJointMaps = []; // the merge of the inner and outer jointMaps
var getMaskedMangledStreamField = function(jmask, str){
Expand All @@ -258,7 +258,9 @@ join.mergeStreams = function mergeStreams(join_config, then){
}
var loopCount = 0;
async.whilst(
function(){return (((isRangeCount && joinData < limit) || joinData.length < limit) && streamIndexes.length > 0);},
function(){
return (((isRangeCount && joinData < limit) || joinData.length < limit) && streamIndexes.length > 0);
},
function(callback){
loopCount++;
if(loopCount > join.max_loops){
Expand Down Expand Up @@ -385,7 +387,7 @@ join.mergeStreams = function mergeStreams(join_config, then){
var newCursorIndex = null;
if(joinType == 'fulljoin'){
newCursorIndex = streamProps[str].results[idx-1];
cursorReln = '='; // next resultset has to be '>' newCursorIndex
cursorReln = '='; // next resultset has to be '>' newCursorIndex TODO assumes 1 to 1 joins
}else if(joinType == 'innerjoin'){
if(str == boundIndexStr){
newCursorIndex = boundIndex;
Expand All @@ -403,15 +405,19 @@ join.mergeStreams = function mergeStreams(join_config, then){
}
}
}
var oldCursor = streamProps[str].argsCopy[cursorIdx];
if(oldCursor == null){
oldCursor = new query.rangeConfig(newCursorIndex);
streamProps[str].argsCopy[cursorIdx] = oldCursor;
}else if(utils.isObjectEmpty(oldCursor.index)){
oldCursor.index = newCursorIndex;
var oldCursorClone = streamProps[str].argsCopy[cursorIdx].clone(); // use a shallowCopy for mutation
streamProps[str].argsCopy[cursorIdx] = oldCursorClone; // attached clone
// use entire index-cursor point
oldCursorClone.startProp = null;
oldCursorClone.startValue = null;
if(oldCursorClone == null){
oldCursorClone = new query.rangeConfig(newCursorIndex);
streamProps[str].argsCopy[cursorIdx] = oldCursorClone;
}else if(utils.isObjectEmpty(oldCursorClone.index)){
oldCursorClone.index = newCursorIndex;
}else{ // NB: keep partitioned-field values (e.g. [0,1]) intact
// mutate newCursorIndex instead; oldCursor points to other objects
for(var fld in oldCursor.index){
// FYI: mutate newCursorIndex instead; oldCursorClone points to other objects
for(var fld in oldCursorClone.index){
// copy partitioned-field values from old cursor
var fldMangle = streamJointMangles[str][fld] || fld;
var fldConfig = comparison.getOrdMaskConfig(ord, fldMangle);
Expand All @@ -422,15 +428,15 @@ join.mergeStreams = function mergeStreams(join_config, then){
var fldField = comparison.getOrdMaskField(ord, fldMangle);
var fldIdx = datatype.getConfigFieldIdx(fldConfig, fldField);
if(datatype.isConfigFieldPartitioned(fldConfig, fldIdx)){
newCursorIndex[fld] = oldCursor.index[fld];
newCursorIndex[fld] = oldCursorClone.index[fld];
}
}
oldCursor.index = newCursorIndex;
oldCursorClone.index = newCursorIndex;
}
if(cursorReln == '<'){
oldCursor.excludeCursor = false;
oldCursorClone.excludeCursor = false;
}else{
oldCursor.excludeCursor = true;
oldCursorClone.excludeCursor = true;
}
// initiate refresh
refreshIdx = i; // continue from this iteration
Expand Down Expand Up @@ -493,11 +499,12 @@ join.mergeStreams = function mergeStreams(join_config, then){
var streamFld = mmsf.streamFld;
// when join expressions exist, joints have to be computed further
// boundIndexMask would be set to that result instead
// TODO expressions should yield new separate fields instead
if(expression == null){
boundIndexMask[jmask] = index[streamFld];
// update ord with info about synthetic field
comparison.addOrdMaskFromClone(ord, jmask, mangleFld);
}/*else{ expressions are not implemented
}/*else{ TODO expressions are not implemented
}*/
}
}
Expand Down
49 changes: 24 additions & 25 deletions nodejs/query_redis.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,14 +425,20 @@ getKeyRangeBoundValue = function getQueryKeyRangeBoundValue(range_order, key_con
// NB: without an index-prop, there's no config reference for processing (e.g. for keysuffix)
var value = range_config[boundValueFlag];
if(valuetype == 'score'){
value = (range_config.excludeCursor ? '(' : '') + value;
value = (range_config.excludeCursor && boundValueFlag == 'startProp' ? '(' : '') + value;
}else if(valuetype == 'member'){
// prepending bound operator is required by redis API for .bylex commands
value = (range_config.excludeCursor ? '(' : '[') + value;
value = (range_config.excludeCursor && boundValueFlag == 'startProp' ? '(' : '[') + value;
}
return value;
}else if(range_config.index == null){
return (range_order == asc ? ascExtremePosition : descExtremePosition);
var value = (range_order == asc ? ascExtremePosition : descExtremePosition);
if(valuetype == 'score'){
value = ''+value;
}else if(valuetype == 'member'){
value = '['+value;
}
return value;
}
var boundProp = range_config[boundPropFlag];
var boundValue = range_config[boundValueFlag];
Expand Down Expand Up @@ -535,12 +541,13 @@ getKeyRangeBoundValue = function getQueryKeyRangeBoundValue(range_order, key_con
// skipping the score would be wrong for .byscorelex commands
// since it is the uid/member value which should be skipped in this case
// excludecursor instead of range_config.excludeCursor is introduced to regulate this
// NB: in the case of stopProp, excludecursor doesn't apply <=> excludeCursor affect startProp only
if(truncated){
var high = 0;
if((boundPropFlag == 'startProp' && !excludecursor) || (boundPropFlag == 'stopProp' && excludecursor)){
high = (range_order == asc ? 0 : 1);
}else if((boundPropFlag == 'stopProp' && !excludecursor) || (boundPropFlag == 'startProp' && excludecursor)){
if(excludecursor || boundPropFlag == 'stopProp'){
high = (range_order == asc ? 1 : 0);
}else{
high = (range_order == asc ? 0 : 1);
}
var low = 0;
if((boundPropFlag == 'startProp' && range_order == asc) || (boundPropFlag == 'stopProp' && range_order != asc)){
Expand All @@ -550,12 +557,8 @@ getKeyRangeBoundValue = function getQueryKeyRangeBoundValue(range_order, key_con
}
value = value + datatype.getFactorizedConfigFieldValue(key_config, boundPropIdx, high) + low;
}else{
if(excludecursor){
if(boundPropFlag == 'startProp'){
value = value + (range_order == asc ? 1 : -1);
}else if(boundPropFlag == 'stopProp'){
value = value + (range_order == asc ? -1 : 1);
}
if(excludecursor || boundPropFlag == 'stopProp'){
value = value + (range_order == asc ? 1 : -1);
}
}
}else if(valuetype == 'member'){
Expand All @@ -566,19 +569,15 @@ getKeyRangeBoundValue = function getQueryKeyRangeBoundValue(range_order, key_con
value = '['+value;
if(truncated){ // use the left and right char of the separator_detail to delimit range
var delimiter = '';
if((boundPropFlag == 'startProp' && !excludecursor) || (boundPropFlag == 'stopProp' && excludecursor)){
delimiter = (range_order == asc ? getPreviousCodeWord(separator_detail) : getNextCodeWord(separator_detail));
}else if((boundPropFlag == 'stopProp' && !excludecursor) || (boundPropFlag == 'startProp' && excludecursor)){
if(excludecursor || boundPropFlag == 'stopProp'){
delimiter = (range_order == asc ? getNextCodeWord(separator_detail) : getPreviousCodeWord(separator_detail));
}else{
delimiter = (range_order == asc ? getPreviousCodeWord(separator_detail) : getNextCodeWord(separator_detail));
}
value = value + delimiter;
}else{ // just (in/de)crement last char of value in case of excludecursor
if(excludecursor){
if(boundPropFlag == 'startProp'){
value = (range_order == asc ? getNextCodeWord(value) : getPreviousCodeWord(value));
}else if(boundPropFlag == 'stopProp'){
value = (range_order == asc ? getPreviousCodeWord(value) : getNextCodeWord(value));
}
if(excludecursor || boundPropFlag == 'stopProp'){
value = (range_order == asc ? getNextCodeWord(value) : getPreviousCodeWord(value));
}
}
}
Expand Down Expand Up @@ -829,7 +828,7 @@ query_redis.getCIQA = function getCIQA(meta_cmd, keys, rangeConfig, attribute, f
startScore = parseFloat(startScore.slice(1), 10);
}
startScore = scoreMap[startScore] || startScore;
var stopScore = getKeyRangeStopScore(cmdOrder, keyConfig, rangeConfig, xidPrefixes, keySuffixes, null);
var stopScore = getKeyRangeStopScore(cmdOrder, keyConfig, rangeConfig, xidPrefixes, keySuffixes, rangeConfig.excludeCursor);
if(stopScore && stopScore[0] == '('){
stopScore = parseFloat(stopScore.slice(1), 10);
}
Expand All @@ -841,7 +840,7 @@ query_redis.getCIQA = function getCIQA(meta_cmd, keys, rangeConfig, attribute, f
startMember = startMember.slice(1);
}
startMember = memberMap[startMember] || startMember;
var stopMember = getKeyRangeStopMember(cmdOrder, keyConfig, rangeConfig, uidPrefixes, keySuffixes, null);
var stopMember = getKeyRangeStopMember(cmdOrder, keyConfig, rangeConfig, uidPrefixes, keySuffixes, rangeConfig.excludeCursor);
if(stopMember && stopMember[0] == '['){
stopMember = stopMember.slice(1);
}
Expand All @@ -856,7 +855,7 @@ query_redis.getCIQA = function getCIQA(meta_cmd, keys, rangeConfig, attribute, f
var negInf = (cmdOrder == asc ? '-' : '+');
var posInf = (cmdOrder == asc ? '+' : '-');
var startMember = getKeyRangeStartMember(cmdOrder, keyConfig, rangeConfig, uidPrefixes, keySuffixes, rangeConfig.excludeCursor);
var stopMember = getKeyRangeStopMember(cmdOrder, keyConfig, rangeConfig, uidPrefixes, keySuffixes, null);
var stopMember = getKeyRangeStopMember(cmdOrder, keyConfig, rangeConfig, uidPrefixes, keySuffixes, rangeConfig.excludeCursor);
if(utils.startsWith(cmdType, 'countbylex') && startMember == negInf && stopMember == posInf){
cmd = command.getMode(datatype.getConfigCommand(keyConfig).count).bykey;
}else{
Expand All @@ -869,7 +868,7 @@ query_redis.getCIQA = function getCIQA(meta_cmd, keys, rangeConfig, attribute, f
var negInf = (cmdOrder == asc ? '-inf' : '+inf');
var posInf = (cmdOrder == asc ? '+inf' : '-inf');
var startScore = getKeyRangeStartScore(cmdOrder, keyConfig, rangeConfig, xidPrefixes, keySuffixes, rangeConfig.excludeCursor);
var stopScore = getKeyRangeStopScore(cmdOrder, keyConfig, rangeConfig, xidPrefixes, keySuffixes, null);
var stopScore = getKeyRangeStopScore(cmdOrder, keyConfig, rangeConfig, xidPrefixes, keySuffixes, rangeConfig.excludeCursor);
if(utils.startsWith(cmdType, 'countbyscore') && startScore == negInf && stopScore == posInf){
cmd = command.getMode(datatype.getConfigCommand(keyConfig).count).bykey;
}else{
Expand Down
10 changes: 5 additions & 5 deletions nodejs/redislayer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,12 @@ Key objects also have a command-set depending on the Struct under which they are
*/
getCollisionBreaker: datatype.getCollisionBreaker,
/**
* @return {string} see getDetailSeparator
* @return {string} the string used to join data of fields e.g. '|' as in evans:akai:bekoe
*/
getDetailSeparator: datatype.getDetailSeparator,

/**
* @return {number} the maximum
* @return {number} the maximum power of 10 which can be stored in the redis Score
*/
getRedisMaxScoreFactor: datatype.getRedisMaxScoreFactor,

Expand Down Expand Up @@ -267,9 +267,9 @@ A query Attributes is an object with a subset of the following redis attributes
* @param {object} arg - a dict
* @param {object} arg.cmd - select a command from $key.getCommand()
* @param {object} arg.key - a key
* @param {object} arg.indexList - a list of objects
* @param {object} arg.indexList.index - an index
* @param {object} arg.indexList.attribute - an attribute
* @param {object} arg.indexlist - a list of objects
* @param {object} arg.indexlist.index - an index
* @param {object} arg.indexlist.attribute - an attribute
* @param {resultsetCallback} callback handler
*/
indexListQuery: function indexListQuery(arg, then){
Expand Down

0 comments on commit e82b194

Please sign in to comment.