taosdata
/
TDengine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #32362 from taosdata/feat/TS-6100-3.0-ming 

Feat/ts 6100 3.0 ming
This commit is contained in:
WANG MINGMING 2025-07-29 18:04:23 +08:00 committed by GitHub
parent a163744ab0 89d27dbfc1
commit 694da6eb6e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 23 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/libs/new-stream/streamReader.h
View File

@ -36,7 +36,6 @@ typedef struct SStreamTriggerReaderInfo {
SSDataBlock* triggerResBlock; SSDataBlock* triggerResBlock;
SSDataBlock* calcResBlock; SSDataBlock* calcResBlock;
SSDataBlock* tsBlock; SSDataBlock* tsBlock;
SSDataBlock* calcResBlockTmp;
SExprInfo* pExprInfo; SExprInfo* pExprInfo;
int32_t numOfExpr; int32_t numOfExpr;
SArray* uidList; // for virtual table stream, uid list SArray* uidList; // for virtual table stream, uid list

8
source/common/src/ttime.c
View File

@ -641,7 +641,13 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision, t
} }
if (!IS_CALENDAR_TIME_DURATION(unit)) { if (!IS_CALENDAR_TIME_DURATION(unit)) {
return t + duration; double tmp = t;
if (tmp + duration >= (double)INT64_MAX || tmp + duration <= (double)INT64_MIN) {
uError("time overflow, t:%" PRId64 ", duration:%" PRId64 ", unit:%c, precision:%d", t, duration, unit, precision);
return t;
} else {
return t + duration;
}
} }
// The following code handles the y/n time duration // The following code handles the y/n time duration

16
source/dnode/vnode/src/vnd/vnodeStream.c
View File

@ -1487,6 +1487,7 @@ static int32_t vnodeProcessStreamTsdbCalcDataReq(SVnode* pVnode, SRpcMsg* pMsg,
int32_t lino = 0; int32_t lino = 0;
void* buf = NULL; void* buf = NULL;
size_t size = 0; size_t size = 0;
SSDataBlock* pBlockRes = NULL;
STREAM_CHECK_NULL_GOTO(sStreamReaderInfo, terrno); STREAM_CHECK_NULL_GOTO(sStreamReaderInfo, terrno);
void* pTask = sStreamReaderInfo->pTask; void* pTask = sStreamReaderInfo->pTask;
@ -1505,9 +1506,8 @@ static int32_t vnodeProcessStreamTsdbCalcDataReq(SVnode* pVnode, SRpcMsg* pMsg,
STREAM_CHECK_RET_GOTO(createStreamTask(pVnode, &options, &pTaskInner, sStreamReaderInfo->triggerResBlock, NULL, &api)); STREAM_CHECK_RET_GOTO(createStreamTask(pVnode, &options, &pTaskInner, sStreamReaderInfo->triggerResBlock, NULL, &api));
STREAM_CHECK_RET_GOTO(taosHashPut(sStreamReaderInfo->streamTaskMap, &key, LONG_BYTES, &pTaskInner, sizeof(pTaskInner))); STREAM_CHECK_RET_GOTO(taosHashPut(sStreamReaderInfo->streamTaskMap, &key, LONG_BYTES, &pTaskInner, sizeof(pTaskInner)));
STREAM_CHECK_RET_GOTO(createOneDataBlock(sStreamReaderInfo->triggerResBlock, false, &pTaskInner->pResBlockDst));
STREAM_CHECK_RET_GOTO(createOneDataBlock(sStreamReaderInfo->calcResBlock, false, &pTaskInner->pResBlockDst)); STREAM_CHECK_RET_GOTO(createOneDataBlock(sStreamReaderInfo->calcResBlock, false, &pBlockRes));
} else { } else {
void** tmp = taosHashGet(sStreamReaderInfo->streamTaskMap, &key, LONG_BYTES); void** tmp = taosHashGet(sStreamReaderInfo->streamTaskMap, &key, LONG_BYTES);
STREAM_CHECK_NULL_GOTO(tmp, TSDB_CODE_STREAM_NO_CONTEXT); STREAM_CHECK_NULL_GOTO(tmp, TSDB_CODE_STREAM_NO_CONTEXT);
@ -1527,14 +1527,14 @@ static int32_t vnodeProcessStreamTsdbCalcDataReq(SVnode* pVnode, SRpcMsg* pMsg,
SSDataBlock* pBlock = NULL; SSDataBlock* pBlock = NULL;
STREAM_CHECK_RET_GOTO(getTableData(pTaskInner, &pBlock)); STREAM_CHECK_RET_GOTO(getTableData(pTaskInner, &pBlock));
STREAM_CHECK_RET_GOTO(qStreamFilter(pBlock, pTaskInner->pFilterInfo)); STREAM_CHECK_RET_GOTO(qStreamFilter(pBlock, pTaskInner->pFilterInfo));
blockDataTransform(sStreamReaderInfo->calcResBlockTmp, pBlock); STREAM_CHECK_RET_GOTO(blockDataMerge(pTaskInner->pResBlockDst, pBlock));
STREAM_CHECK_RET_GOTO(blockDataMerge(pTaskInner->pResBlockDst, sStreamReaderInfo->calcResBlockTmp));
if (pTaskInner->pResBlockDst->info.rows >= STREAM_RETURN_ROWS_NUM) { if (pTaskInner->pResBlockDst->info.rows >= STREAM_RETURN_ROWS_NUM) {
break; break;
} }
} }
STREAM_CHECK_RET_GOTO(buildRsp(pTaskInner->pResBlockDst, &buf, &size)); blockDataTransform(pBlockRes, pTaskInner->pResBlockDst);
ST_TASK_DLOG("vgId:%d %s get result rows:%" PRId64, TD_VID(pVnode), __func__, pTaskInner->pResBlockDst->info.rows); STREAM_CHECK_RET_GOTO(buildRsp(pBlockRes, &buf, &size));
ST_TASK_DLOG("vgId:%d %s get result rows:%" PRId64, TD_VID(pVnode), __func__, pBlockRes->info.rows);
if (!hasNext) { if (!hasNext) {
taosHashRemove(sStreamReaderInfo->streamTaskMap, &key, LONG_BYTES); taosHashRemove(sStreamReaderInfo->streamTaskMap, &key, LONG_BYTES);
} }
@ -1544,7 +1544,7 @@ end:
SRpcMsg rsp = { SRpcMsg rsp = {
.msgType = TDMT_STREAM_TRIGGER_PULL_RSP, .info = pMsg->info, .pCont = buf, .contLen = size, .code = code}; .msgType = TDMT_STREAM_TRIGGER_PULL_RSP, .info = pMsg->info, .pCont = buf, .contLen = size, .code = code};
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
blockDataDestroy(pBlockRes);
return code; return code;
} }

23
source/libs/executor/src/dynqueryctrloperator.c
View File

@ -1782,32 +1782,13 @@ static int32_t resetDynQueryCtrlOperState(SOperatorInfo* pOper) {
case DYN_QTYPE_STB_HASH:{ case DYN_QTYPE_STB_HASH:{
pDyn->stbJoin.execInfo = (SDynQueryCtrlExecInfo){0}; pDyn->stbJoin.execInfo = (SDynQueryCtrlExecInfo){0};
SStbJoinDynCtrlInfo* pStbJoin = &pDyn->stbJoin; SStbJoinDynCtrlInfo* pStbJoin = &pDyn->stbJoin;
if (pStbJoin->basic.batchFetch) { destroyStbJoinDynCtrlInfo(&pDyn->stbJoin);
if (pStbJoin->ctx.prev.leftHash) {
tSimpleHashSetFreeFp(pStbJoin->ctx.prev.leftHash, freeVgTableList);
tSimpleHashClear(pStbJoin->ctx.prev.leftHash);
}
if (pStbJoin->ctx.prev.rightHash) {
tSimpleHashSetFreeFp(pStbJoin->ctx.prev.rightHash, freeVgTableList);
tSimpleHashClear(pStbJoin->ctx.prev.rightHash);
}
} else {
if (pStbJoin->ctx.prev.leftCache) {
tSimpleHashClear(pStbJoin->ctx.prev.leftCache);
}
if (pStbJoin->ctx.prev.rightCache) {
tSimpleHashClear(pStbJoin->ctx.prev.rightCache);
}
if (pStbJoin->ctx.prev.onceTable) {
tSimpleHashClear(pStbJoin->ctx.prev.onceTable);
}
}
int32_t code = initSeqStbJoinTableHash(&pDyn->stbJoin.ctx.prev, pDyn->stbJoin.basic.batchFetch); int32_t code = initSeqStbJoinTableHash(&pDyn->stbJoin.ctx.prev, pDyn->stbJoin.basic.batchFetch);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
qError("initSeqStbJoinTableHash failed since %s", tstrerror(code)); qError("initSeqStbJoinTableHash failed since %s", tstrerror(code));
return code; return code;
} }
destroyStbJoinTableList(pStbJoin->ctx.prev.pListHead);
pStbJoin->ctx.prev.pListHead = NULL; pStbJoin->ctx.prev.pListHead = NULL;
pStbJoin->ctx.prev.joinBuild = false; pStbJoin->ctx.prev.joinBuild = false;
pStbJoin->ctx.prev.pListTail = NULL; pStbJoin->ctx.prev.pListTail = NULL;

10
source/libs/executor/src/groupcacheoperator.c
View File

@ -641,8 +641,8 @@ static int32_t retrieveBlkFromBufCache(SGroupCacheOperatorInfo* pGCache, SGroupC
static FORCE_INLINE void initGcVgroupCtx(SOperatorInfo* pOperator, SGcVgroupCtx* pVgCtx, int32_t downstreamId, int32_t vgId, SArray* pTbList) { static FORCE_INLINE void initGcVgroupCtx(SOperatorInfo* pOperator, SGcVgroupCtx* pVgCtx, int32_t downstreamId, int32_t vgId, SArray* pTbList) {
pVgCtx->pTbList = pTbList; pVgCtx->pTbList = pTbList;
pVgCtx->id = vgId; pVgCtx->id = vgId;
(void)snprintf(pVgCtx->fileCtx.baseFilename, sizeof(pVgCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d_%d", (void)snprintf(pVgCtx->fileCtx.baseFilename, sizeof(pVgCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%p_%d_%d",
tsTempDir, taosGetPId(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, downstreamId, vgId); tsTempDir, taosGetPId(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, pOperator, downstreamId, vgId);
pVgCtx->fileCtx.baseFilename[sizeof(pVgCtx->fileCtx.baseFilename) - 1] = 0; pVgCtx->fileCtx.baseFilename[sizeof(pVgCtx->fileCtx.baseFilename) - 1] = 0;
pVgCtx->fileCtx.baseNameLen = strlen(pVgCtx->fileCtx.baseFilename); pVgCtx->fileCtx.baseNameLen = strlen(pVgCtx->fileCtx.baseFilename);
@ -1454,8 +1454,8 @@ static int32_t initGroupCacheDownstreamCtx(SOperatorInfo* pOperator) {
return terrno; return terrno;
} }
(void)snprintf(pCtx->fileCtx.baseFilename, sizeof(pCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d", (void)snprintf(pCtx->fileCtx.baseFilename, sizeof(pCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%p_%d",
tsTempDir, taosGetPId(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, pCtx->id); tsTempDir, taosGetPId(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, pOperator, pCtx->id);
pCtx->fileCtx.baseFilename[sizeof(pCtx->fileCtx.baseFilename) - 1] = 0; pCtx->fileCtx.baseFilename[sizeof(pCtx->fileCtx.baseFilename) - 1] = 0;
pCtx->fileCtx.baseNameLen = strlen(pCtx->fileCtx.baseFilename); pCtx->fileCtx.baseNameLen = strlen(pCtx->fileCtx.baseFilename);
} }
@ -1542,7 +1542,7 @@ static int32_t resetGroupCacheDownstreamCtx(SOperatorInfo* pOper) {
tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx)); tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx));
} }
taosArrayClear(pCtx->pFreeBlock); taosArrayClearEx(pCtx->pFreeBlock, freeGcBlockInList);
taosHashClear(pCtx->pSessions); taosHashClear(pCtx->pSessions);
taosHashClear(pCtx->pWaitSessions); taosHashClear(pCtx->pWaitSessions);
freeSGcFileCacheCtx(&pCtx->fileCtx); freeSGcFileCacheCtx(&pCtx->fileCtx);

2
source/libs/new-stream/src/streamReader.c
View File

@ -214,7 +214,6 @@ static void releaseStreamReaderInfo(void* p) {
blockDataDestroy(pInfo->triggerResBlock); blockDataDestroy(pInfo->triggerResBlock);
blockDataDestroy(pInfo->calcResBlock); blockDataDestroy(pInfo->calcResBlock);
blockDataDestroy(pInfo->tsBlock); blockDataDestroy(pInfo->tsBlock);
blockDataDestroy(pInfo->calcResBlockTmp);
destroyExprInfo(pInfo->pExprInfo, pInfo->numOfExpr); destroyExprInfo(pInfo->pExprInfo, pInfo->numOfExpr);
taosMemoryFreeClear(pInfo->pExprInfo); taosMemoryFreeClear(pInfo->pExprInfo);
taosArrayDestroy(pInfo->uidList); taosArrayDestroy(pInfo->uidList);
@ -349,7 +348,6 @@ static SStreamTriggerReaderInfo* createStreamReaderInfo(void* pTask, const SStre
SNodeList* pScanCols = ((STableScanPhysiNode*)(sStreamReaderInfo->calcAst->pNode))->scan.pScanCols; SNodeList* pScanCols = ((STableScanPhysiNode*)(sStreamReaderInfo->calcAst->pNode))->scan.pScanCols;
setColIdForCalcResBlock(pseudoCols, sStreamReaderInfo->calcResBlock->pDataBlock); setColIdForCalcResBlock(pseudoCols, sStreamReaderInfo->calcResBlock->pDataBlock);
setColIdForCalcResBlock(pScanCols, sStreamReaderInfo->calcResBlock->pDataBlock); setColIdForCalcResBlock(pScanCols, sStreamReaderInfo->calcResBlock->pDataBlock);
STREAM_CHECK_RET_GOTO(createOneDataBlock(sStreamReaderInfo->calcResBlock, false, &sStreamReaderInfo->calcResBlockTmp));
} }
STREAM_CHECK_RET_GOTO(createDataBlockForTs(&sStreamReaderInfo->tsBlock)); STREAM_CHECK_RET_GOTO(createDataBlockForTs(&sStreamReaderInfo->tsBlock));

2
test/cases/13-StreamProcessing/20-UseCase/test_three_gorges_second_case18.py
View File

@ -60,7 +60,7 @@ class Test_ThreeGorges:
self.checkStreamRunning() self.checkStreamRunning()
self.sxny_data2() self.sxny_data2()
self.dataIn() self.dataIn()
tdSql.checkRowsLoop(500,f"select val,tablename,point, ps_code, cnstationno, index_code from {self.dbname}.{self.outTbname} order by tablename;",100,1) tdSql.checkRowsLoop(500,f"select val,tablename,point, ps_code, cnstationno, index_code from {self.dbname}.{self.outTbname} order by tablename;",200,1)
self.checkResultWithResultFile() self.checkResultWithResultFile()