feat:[TS-4897] virtual table

Jenkinsfile2

@@ -165,7 +165,7 @@ def pre_test(){
     script {
         sh '''
             cd ${WK}
-            git checkout ''' + env.CHANGE_TARGET + '''
+            git checkout feat/3.0/TS-4897-2
             cd ${WKC}
             git checkout ''' + env.CHANGE_TARGET + '''
         '''
@@ -274,7 +274,7 @@ def pre_test_win(){
     script {
         bat '''
             cd %WIN_INTERNAL_ROOT%
-            git checkout ''' + env.CHANGE_TARGET + '''
+            git checkout feat/3.0/TS-4897-2
         '''
         bat '''
             cd %WIN_COMMUNITY_ROOT%

include/common/taosdef.h

@@ -39,7 +39,9 @@ typedef enum {
   TSDB_SYSTEM_TABLE = 5,
   TSDB_TSMA_TABLE = 6,  // time-range-wise sma
   TSDB_VIEW_TABLE = 7,
-  TSDB_TABLE_MAX = 8
+  TSDB_VIRTUAL_TABLE = 8,
+  TSDB_VIRTUAL_CHILD_TABLE = 9,
+  TSDB_TABLE_MAX = 10
 } ETableType;
 
 typedef enum {

include/common/tcol.h

@@ -80,6 +80,7 @@ uint8_t     columnEncodeVal(const char* encode);
 uint16_t    columnCompressVal(const char* compress);
 
 bool useCompress(uint8_t tableType);
+bool hasRefCol(uint8_t tableType);
 bool checkColumnEncode(char encode[TSDB_CL_COMPRESS_OPTION_LEN]);
 bool checkColumnEncodeOrSetDefault(uint8_t type, char encode[TSDB_CL_COMPRESS_OPTION_LEN]);
 bool checkColumnCompress(char compress[TSDB_CL_COMPRESS_OPTION_LEN]);

include/common/tmsg.h

@@ -182,6 +182,9 @@ typedef enum _mgmt_table {
 #define TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS          13
 #define TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION 14
 #define TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL            15
+#define TSDB_ALTER_TABLE_ALTER_COLUMN_REF                16
+#define TSDB_ALTER_TABLE_REMOVE_COLUMN_REF               17
+#define TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COLUMN_REF      18
 
 #define TSDB_FILL_NONE        0
 #define TSDB_FILL_NULL        1
@@ -222,9 +225,11 @@ typedef enum _mgmt_table {
 #define TSDB_COL_IS_UD_COL(f)     ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC)
 #define TSDB_COL_REQ_NULL(f)      (((f)&TSDB_COL_NULL) != 0)
 
-#define TD_SUPER_TABLE  TSDB_SUPER_TABLE
-#define TD_CHILD_TABLE  TSDB_CHILD_TABLE
-#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE
+#define TD_SUPER_TABLE         TSDB_SUPER_TABLE
+#define TD_CHILD_TABLE         TSDB_CHILD_TABLE
+#define TD_NORMAL_TABLE        TSDB_NORMAL_TABLE
+#define TD_VIRTUAL_TABLE       TSDB_VIRTUAL_TABLE
+#define TD_VIRTUAL_CHILD_TABLE TSDB_VIRTUAL_CHILD_TABLE
 
 typedef enum ENodeType {
   // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN,
@@ -270,6 +275,7 @@ typedef enum ENodeType {
   QUERY_NODE_ANOMALY_WINDOW,
   QUERY_NODE_RANGE_AROUND,
   QUERY_NODE_STREAM_NOTIFY_OPTIONS,
+  QUERY_NODE_VIRTUAL_TABLE,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR = 100,
@@ -327,6 +333,13 @@ typedef enum ENodeType {
   QUERY_NODE_REVOKE_STMT,
   QUERY_NODE_ALTER_CLUSTER_STMT,
   QUERY_NODE_S3MIGRATE_DATABASE_STMT,
+  QUERY_NODE_CREATE_TSMA_STMT,
+  QUERY_NODE_DROP_TSMA_STMT,
+  QUERY_NODE_CREATE_VIRTUAL_TABLE_STMT,
+  QUERY_NODE_CREATE_VIRTUAL_SUBTABLE_STMT,
+  QUERY_NODE_DROP_VIRTUAL_TABLE_STMT,
+  QUERY_NODE_ALTER_VIRTUAL_TABLE_STMT,
+
   // placeholder for [154, 180]
   QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181,
   QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
@@ -359,6 +372,8 @@ typedef enum ENodeType {
   QUERY_NODE_CREATE_ANODE_STMT,
   QUERY_NODE_DROP_ANODE_STMT,
   QUERY_NODE_UPDATE_ANODE_STMT,
+  QUERY_NODE_SHOW_CREATE_TSMA_STMT,
+  QUERY_NODE_SHOW_CREATE_VTABLE_STMT,
 
   // show statement nodes
   // see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET'
@@ -403,11 +418,9 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_ANODES_STMT,
   QUERY_NODE_SHOW_ANODES_FULL_STMT,
   QUERY_NODE_SHOW_USAGE_STMT,
-  QUERY_NODE_CREATE_TSMA_STMT,
-  QUERY_NODE_SHOW_CREATE_TSMA_STMT,
-  QUERY_NODE_DROP_TSMA_STMT,
   QUERY_NODE_SHOW_FILESETS_STMT,
   QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT,
+  QUERY_NODE_SHOW_VTABLES_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@@ -428,6 +441,7 @@ typedef enum ENodeType {
   QUERY_NODE_LOGIC_PLAN_GROUP_CACHE,
   QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL,
   QUERY_NODE_LOGIC_PLAN_FORECAST_FUNC,
+  QUERY_NODE_LOGIC_PLAN_VIRTUAL_TABLE_SCAN,
 
   // physical plan node
   QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100,
@@ -484,6 +498,7 @@ typedef enum ENodeType {
   QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC,
   QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERP_FUNC,
   QUERY_NODE_RESET_STREAM_STMT,
+  QUERY_NODE_PHYSICAL_PLAN_VIRTUAL_TABLE_SCAN,
 } ENodeType;
 
 typedef struct {
@@ -585,6 +600,19 @@ STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
 // for debug
 int32_t tPrintFixedSchemaSubmitReq(SSubmitReq* pReq, STSchema* pSchema);
 
+typedef struct {
+  bool     hasRef;
+  col_id_t id;
+  char     refTableName[TSDB_TABLE_NAME_LEN];
+  char     refColName[TSDB_COL_NAME_LEN];
+} SColRef;
+
+typedef struct {
+  int32_t   nCols;
+  int32_t   version;
+  SColRef*  pColRef;
+} SColRefWrapper;
+
 struct SSchema {
   int8_t   type;
   int8_t   flags;
@@ -625,6 +653,9 @@ typedef struct {
   int8_t      sysInfo;
   SSchema*    pSchemas;
   SSchemaExt* pSchemaExt;
+  int8_t      virtualStb;
+  int32_t     numOfColRefs;
+  SColRef*    pColRefs;
 } STableMetaRsp;
 
 typedef struct {
@@ -709,6 +740,23 @@ typedef struct {
   SColCmpr* pColCmpr;
 } SColCmprWrapper;
 
+
+static FORCE_INLINE int32_t tInitDefaultSColRefWrapperByCols(SColRefWrapper* pRef, int32_t nCols) {
+  if (pRef->pColRef) {
+    return TSDB_CODE_INVALID_PARA;
+  }
+  pRef->pColRef = (SColRef*)taosMemoryCalloc(nCols, sizeof(SColRef));
+  if (pRef->pColRef == NULL) {
+    return terrno;
+  }
+  pRef->nCols = nCols;
+  for (int32_t i = 0; i < nCols; i++) {
+    pRef->pColRef[i].hasRef = false;
+    pRef->pColRef[i].id = (col_id_t)(i + 1);
+  }
+  return 0;
+}
+
 static FORCE_INLINE SColCmprWrapper* tCloneSColCmprWrapper(const SColCmprWrapper* pSrcWrapper) {
   if (pSrcWrapper->pColCmpr == NULL || pSrcWrapper->nCols == 0) {
     terrno = TSDB_CODE_INVALID_PARA;
@@ -851,6 +899,27 @@ static FORCE_INLINE int32_t tDecodeSSchemaExt(SDecoder* pDecoder, SSchemaExt* pS
   return 0;
 }
 
+static FORCE_INLINE int32_t tEncodeSColRef(SEncoder* pEncoder, const SColRef* pColRef) {
+  TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pColRef->hasRef));
+  TAOS_CHECK_RETURN(tEncodeI16(pEncoder, pColRef->id));
+  if (pColRef->hasRef) {
+    TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pColRef->refTableName));
+    TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pColRef->refColName));
+  }
+  return 0;
+}
+
+static FORCE_INLINE int32_t tDecodeSColRef(SDecoder* pDecoder, SColRef* pColRef) {
+  TAOS_CHECK_RETURN(tDecodeI8(pDecoder, (int8_t*)&pColRef->hasRef));
+  TAOS_CHECK_RETURN(tDecodeI16(pDecoder, &pColRef->id));
+  if (pColRef->hasRef) {
+    TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pColRef->refTableName));
+    TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pColRef->refColName));
+  }
+
+  return 0;
+}
+
 static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) {
   int32_t tlen = 0;
   tlen += taosEncodeVariantI32(buf, pSW->nCols);
@@ -947,6 +1016,7 @@ typedef struct {
   int64_t  deleteMark2;
   int32_t  sqlLen;
   char*    sql;
+  int8_t   virtualStb;
 } SMCreateStbReq;
 
 int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq);
@@ -1301,6 +1371,8 @@ typedef struct {
   int32_t     tagsLen;
   char*       pTags;
   SSchemaExt* pSchemaExt;
+  int8_t      virtualStb;
+  SColRef*    pColRefs;
 } STableCfg;
 
 typedef STableCfg STableCfgRsp;
@@ -3194,6 +3266,7 @@ typedef struct SVCreateStbReq {
   int8_t          source;
   int8_t          colCmpred;
   SColCmprWrapper colCmpr;
+  int8_t          virtualStb;
 } SVCreateStbReq;
 
 int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq);
@@ -3234,6 +3307,7 @@ typedef struct SVCreateTbReq {
   int32_t         sqlLen;
   char*           sql;
   SColCmprWrapper colCmpr;
+  SColRefWrapper  colRef; // col reference for virtual table
 } SVCreateTbReq;
 
 int  tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq);
@@ -3248,15 +3322,16 @@ static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) {
   taosMemoryFreeClear(req->sql);
   taosMemoryFreeClear(req->name);
   taosMemoryFreeClear(req->comment);
-  if (req->type == TSDB_CHILD_TABLE) {
+  if (req->type == TSDB_CHILD_TABLE || req->type == TSDB_VIRTUAL_CHILD_TABLE) {
     taosMemoryFreeClear(req->ctb.pTag);
     taosMemoryFreeClear(req->ctb.stbName);
     taosArrayDestroy(req->ctb.tagName);
     req->ctb.tagName = NULL;
-  } else if (req->type == TSDB_NORMAL_TABLE) {
+  } else if (req->type == TSDB_NORMAL_TABLE || req->type == TSDB_VIRTUAL_TABLE) {
     taosMemoryFreeClear(req->ntb.schemaRow.pSchema);
   }
   taosMemoryFreeClear(req->colCmpr.pColCmpr);
+  taosMemoryFreeClear(req->colRef.pColRef);
 }
 
 typedef struct {
@@ -3304,6 +3379,7 @@ typedef struct {
   uint64_t suid;  // for tmq in wal format
   int64_t  uid;
   int8_t   igNotExists;
+  int8_t   isVirtual;
 } SVDropTbReq;
 
 typedef struct {
@@ -3343,7 +3419,7 @@ typedef struct SMultiTagUpateVal {
   int8_t   isNull;
   SArray*  pTagArray;
 } SMultiTagUpateVal;
-typedef struct {
+typedef struct SVAlterTbReq{
   char*   tbName;
   int8_t  action;
   char*   colName;
@@ -3373,6 +3449,10 @@ typedef struct {
   int8_t   source;     // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
   uint32_t compress;   // TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS
   SArray*  pMultiTag;  // TSDB_ALTER_TABLE_ADD_MULTI_TAGS
+  // TSDB_ALTER_TABLE_ALTER_COLUMN_REF
+  char*    refTbName;
+  char*    refColName;
+  // TSDB_ALTER_TABLE_REMOVE_COLUMN_REF
 } SVAlterTbReq;
 
 int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);

include/libs/executor/storageapi.h

@@ -83,6 +83,7 @@ typedef struct SMetaEntry {
   uint8_t* pBuf;
 
   SColCmprWrapper colCmpr;  // col compress alg
+  SColRefWrapper  colRef;   // col reference for virtual table
 } SMetaEntry;
 
 typedef struct SMetaReader {

include/libs/function/functionMgt.h

@@ -273,6 +273,7 @@ bool fmIsForbidSysTableFunc(int32_t funcId);
 bool fmIsIntervalInterpoFunc(int32_t funcId);
 bool fmIsInterpFunc(int32_t funcId);
 bool fmIsLastRowFunc(int32_t funcId);
+bool fmIsLastFunc(int32_t funcId);
 bool fmIsForecastFunc(int32_t funcId);
 bool fmIsNotNullOutputFunc(int32_t funcId);
 bool fmIsSelectValueFunc(int32_t funcId);

include/libs/nodes/cmdnodes.h

@@ -25,10 +25,12 @@ extern "C" {
 
 #define DESCRIBE_RESULT_COLS               4
 #define DESCRIBE_RESULT_COLS_COMPRESS      7
+#define DESCRIBE_RESULT_COLS_REF           5
 #define DESCRIBE_RESULT_FIELD_LEN          (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE)
 #define DESCRIBE_RESULT_TYPE_LEN           (20 + VARSTR_HEADER_SIZE)
 #define DESCRIBE_RESULT_NOTE_LEN           (16 + VARSTR_HEADER_SIZE)
 #define DESCRIBE_RESULT_COPRESS_OPTION_LEN (TSDB_CL_COMPRESS_OPTION_LEN + VARSTR_HEADER_SIZE)
+#define DESCRIBE_RESULT_COL_REF_LEN         (TSDB_COL_FNAME_LEN + VARSTR_HEADER_SIZE)
 
 #define SHOW_CREATE_DB_RESULT_COLS       2
 #define SHOW_CREATE_DB_RESULT_FIELD1_LEN (TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE)
@@ -180,6 +182,7 @@ typedef struct SCompactVgroupsStmt {
 
 typedef struct STableOptions {
   ENodeType  type;
+  bool       virtualStb;
   bool       commentNull;
   char       comment[TSDB_TB_COMMENT_LEN];
   SNodeList* pMaxDelay;
@@ -204,7 +207,11 @@ typedef struct SColumnOptions {
   char      compress[TSDB_CL_COMPRESS_OPTION_LEN];
   char      compressLevel[TSDB_CL_COMPRESS_OPTION_LEN];
   bool      bPrimaryKey;
+  bool      hasRef;
+  char      refTable[TSDB_TABLE_NAME_LEN];
+  char      refColumn[TSDB_COL_NAME_LEN];
 } SColumnOptions;
+
 typedef struct SColumnDefNode {
   ENodeType type;
   char      colName[TSDB_COL_NAME_LEN];
@@ -223,6 +230,27 @@ typedef struct SCreateTableStmt {
   STableOptions* pOptions;
 } SCreateTableStmt;
 
+typedef struct SCreateVTableStmt {
+  ENodeType      type;
+  char           dbName[TSDB_DB_NAME_LEN];
+  char           tableName[TSDB_TABLE_NAME_LEN];
+  bool           ignoreExists;
+  SNodeList*     pCols;
+} SCreateVTableStmt;
+
+typedef struct SCreateVSubTableStmt {
+  ENodeType      type;
+  char           dbName[TSDB_DB_NAME_LEN];
+  char           tableName[TSDB_TABLE_NAME_LEN];
+  char           useDbName[TSDB_DB_NAME_LEN];
+  char           useTableName[TSDB_TABLE_NAME_LEN];
+  bool           ignoreExists;
+  SNodeList*     pSpecificTags;
+  SNodeList*     pValsOfTags;
+  SNodeList*     pSpecificColRefs;
+  SNodeList*     pColRefs;
+} SCreateVSubTableStmt;
+
 typedef struct SCreateSubTableClause {
   ENodeType      type;
   char           dbName[TSDB_DB_NAME_LEN];
@@ -272,6 +300,14 @@ typedef struct SDropSuperTableStmt {
   bool      withOpt;
 } SDropSuperTableStmt;
 
+typedef struct SDropVirtualTableStmt {
+  ENodeType type;
+  char      dbName[TSDB_DB_NAME_LEN];
+  char      tableName[TSDB_TABLE_NAME_LEN];
+  bool      ignoreNotExists;
+  bool      withOpt;
+} SDropVirtualTableStmt;
+
 typedef struct SAlterTableStmt {
   ENodeType       type;
   char            dbName[TSDB_DB_NAME_LEN];
@@ -284,6 +320,8 @@ typedef struct SAlterTableStmt {
   SValueNode*     pVal;
   SColumnOptions* pColOptions;
   SNodeList*      pNodeListTagValue;
+  char            refTableName[TSDB_TABLE_NAME_LEN];
+  char            refColName[TSDB_COL_NAME_LEN];
 } SAlterTableStmt;
 
 typedef struct SAlterTableMultiStmt {