From d1f23ccab698038a97f5a1fdce02a24381ebfd53 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 15 Jul 2026 10:27:40 +0800 Subject: [PATCH 01/14] fix: cancel ISCP index consumers on drop index --- pkg/iscp/data_retriever.go | 60 +++++- pkg/iscp/executor.go | 26 ++- pkg/iscp/index_consumer.go | 68 +++++++ pkg/iscp/index_consumer_test.go | 65 ++++++- pkg/iscp/iteration.go | 78 +++++++- pkg/iscp/iteration_consumer_context_test.go | 1 + pkg/iscp/runtime_cancel.go | 205 ++++++++++++++++++++ pkg/iscp/runtime_cancel_test.go | 171 ++++++++++++++++ pkg/iscp/table_entry.go | 3 + pkg/iscp/types.go | 20 ++ pkg/iscp/worker.go | 11 +- pkg/objectio/injects.go | 21 +- pkg/sql/compile/ddl.go | 14 +- pkg/sql/compile/ddl_util_test.go | 30 +++ pkg/sql/compile/iscp_util.go | 24 +++ pkg/sql/compile/iscp_util_extra_test.go | 54 ++++++ pkg/util/fault/fault.go | 58 +++++- pkg/util/fault/fault_test.go | 94 +++++++++ 18 files changed, 983 insertions(+), 20 deletions(-) create mode 100644 pkg/iscp/runtime_cancel.go create mode 100644 pkg/iscp/runtime_cancel_test.go diff --git a/pkg/iscp/data_retriever.go b/pkg/iscp/data_retriever.go index 3c91e9faa0f11..666e07dd296e2 100644 --- a/pkg/iscp/data_retriever.go +++ b/pkg/iscp/data_retriever.go @@ -81,6 +81,20 @@ func (d *ISCPData) Done() { } } +func (d *ISCPData) Close() { + if d == nil { + return + } + if d.insertBatch != nil { + d.insertBatch.Close() + d.insertBatch = nil + } + if d.deleteBatch != nil { + d.deleteBatch.Close() + d.deleteBatch = nil + } +} + const ( ISCPDataType_Snapshot int8 = iota ISCPDataType_Tail @@ -145,6 +159,9 @@ func (r *DataRetrieverImpl) UpdateWatermark(ctx context.Context, cnUUID string, txn client.TxnOperator) error { + if r.IsCanceled() { + return r.terminalError() + } ctxWithSysAccount := context.WithValue(ctx, defines.TenantIDKey{}, catalog.System_Account) ctxWithSysAccount, cancel := context.WithTimeout(ctxWithSysAccount, time.Minute*5) defer cancel() @@ -187,17 +204,24 @@ func (r *DataRetrieverImpl) GetTableID() uint64 { return r.tableID } -func (r *DataRetrieverImpl) SetNextBatch(data *ISCPData) { +func (r *DataRetrieverImpl) SetNextBatch(data *ISCPData) bool { if r.hasError() { - data.Done() - return + return false } select { case r.insertDataCh <- data: - return + if r.IsCanceled() { + select { + case queued := <-r.insertDataCh: + if queued == data { + return false + } + default: + } + } + return true case <-r.ctx.Done(): - data.Done() - return + return false } } @@ -218,6 +242,10 @@ func (r *DataRetrieverImpl) terminalError() error { // after error occurs, the data retriever won't consume any more data func (r *DataRetrieverImpl) SetError(err error) { + r.Cancel(err) +} + +func (r *DataRetrieverImpl) Cancel(err error) { if r.hasError() { return } @@ -228,6 +256,26 @@ func (r *DataRetrieverImpl) SetError(err error) { } r.cancel() r.err = err + for { + select { + case data := <-r.insertDataCh: + data.Done() + default: + return + } + } +} + +func (r *DataRetrieverImpl) IsCanceled() bool { + if r.hasError() { + return true + } + select { + case <-r.ctx.Done(): + return true + default: + return false + } } func (r *DataRetrieverImpl) Close() { diff --git a/pkg/iscp/executor.go b/pkg/iscp/executor.go index eae7ee46908b5..2bb4a477bf8af 100644 --- a/pkg/iscp/executor.go +++ b/pkg/iscp/executor.go @@ -182,6 +182,10 @@ func NewISCPTaskExecutor( tableMu: sync.RWMutex{}, option: option, mp: mp, + fencedJobs: make(map[JobRuntimeKey]struct{}), + runningConsumers: make( + map[JobRuntimeKey]map[uint64]*RunningJobConsumer, + ), } return exec, nil } @@ -266,10 +270,21 @@ func (exec *ISCPTaskExecutor) initStateLocked() error { "ISCP-Task Start", ) ctx, cancel := context.WithCancel(context.Background()) - worker := NewWorker(exec.cnUUID, exec.txnEngine, exec.cnTxnClient, exec.mp) + worker := NewWorker(exec, exec.cnUUID, exec.txnEngine, exec.cnTxnClient, exec.mp) exec.worker = worker exec.ctx = ctx exec.cancel = cancel + initialized := false + defer func() { + if initialized { + return + } + worker.Stop() + cancel() + exec.running = false + exec.ctx, exec.cancel = nil, nil + exec.worker = nil + }() err := retry( ctx, func() error { @@ -295,6 +310,8 @@ func (exec *ISCPTaskExecutor) initStateLocked() error { return err } exec.wg.Add(1) + RegisterExecutorRuntime(exec.cnUUID, exec) + initialized = true return nil } @@ -311,6 +328,7 @@ func (exec *ISCPTaskExecutor) Stop() { exec.worker.Stop() exec.cancel() exec.wg.Wait() + UnregisterExecutorRuntime(exec.cnUUID, exec) exec.ctx, exec.cancel = nil, nil exec.worker = nil } @@ -474,6 +492,12 @@ func (exec *ISCPTaskExecutor) run(ctx context.Context) { onErrorFn(err) continue } + if objectio.WaitInjected(objectio.FJ_ISCPCancelAfterSubmit) { + logutil.Infof("ISCP-Task cancel fault wait %s", objectio.FJ_ISCPCancelAfterSubmit) + } + if msg, injected := objectio.ISCPExecutorInjected(); injected && msg == "iscp:after-submit" { + logutil.Infof("ISCP-Task injected hook %s", msg) + } } else { if !ok2 { logutil.Error( diff --git a/pkg/iscp/index_consumer.go b/pkg/iscp/index_consumer.go index 62c9c61c1210a..2b74f8dbc4f30 100644 --- a/pkg/iscp/index_consumer.go +++ b/pkg/iscp/index_consumer.go @@ -155,6 +155,7 @@ func RunIndex(c *IndexConsumer, ctx context.Context, errch chan error, r DataRet func runIndex(c *IndexConsumer, ctx context.Context, errch chan error, r DataRetriever) { datatype := r.GetDataType() + indexName := c.indexName() if datatype == ISCPDataType_Snapshot { // SNAPSHOT @@ -180,6 +181,12 @@ func runIndex(c *IndexConsumer, ctx context.Context, errch chan error, r DataRet func(sqlproc *sqlexec.SqlProcess, cbdata any) (err error) { sqlctx := sqlproc.SqlCtx + if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeExec, indexName) { + logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongBeforeExec, indexName) + if err := ctx.Err(); err != nil { + return err + } + } if objectio.ISCPIndexExecErrorInjected() { return injectedISCPIndexErr("exec") } @@ -230,6 +237,12 @@ func runIndex(c *IndexConsumer, ctx context.Context, errch chan error, r DataRet // update SQL var res executor.Result + if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeExec, indexName) { + logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongBeforeExec, indexName) + if err := ctx.Err(); err != nil { + return err + } + } if objectio.ISCPIndexExecErrorInjected() { return injectedISCPIndexErr("exec") } @@ -264,6 +277,7 @@ func RunHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch c func runHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch chan error, r DataRetriever) { datatype := r.GetDataType() + indexName := c.indexName() // Suppose we shoult not use transaction here for Snapshot type and commit every time a new batch comes. // However, HNSW only run in local without save to database until Sync.Save(). @@ -328,6 +342,18 @@ func runHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch c if objectio.ISCPIndexHnswSaveErrInjected() { return injectedISCPIndexErr("hnsw save") } + if objectio.WaitInjected(objectio.FJ_ISCPCancelHnswBeforeSave) { + logutil.Infof("ISCP-Task cancel fault wait %s index=%s", objectio.FJ_ISCPCancelHnswBeforeSave, indexName) + } + if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongHnswBeforeSave, indexName) { + logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongHnswBeforeSave, indexName) + if err := ctx.Err(); err != nil { + return err + } + } + if msg, injected := objectio.ISCPExecutorInjected(); injected && msg == "iscp:hnsw-before-save:"+indexName { + logutil.Infof("ISCP-Task injected hook %s", msg) + } err = sync.Save(sqlproc) if err != nil { return @@ -338,6 +364,18 @@ func runHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch c if objectio.ISCPIndexWatermarkErrInjected() { return injectedISCPIndexErr("watermark") } + if objectio.WaitInjected(objectio.FJ_ISCPCancelBeforeUpdateWatermark) { + logutil.Infof("ISCP-Task cancel fault wait %s index=%s", objectio.FJ_ISCPCancelBeforeUpdateWatermark, indexName) + } + if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeWatermark, indexName) { + logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongBeforeWatermark, indexName) + if err := ctx.Err(); err != nil { + return err + } + } + if msg, injected := objectio.ISCPExecutorInjected(); injected && msg == "iscp:before-update-watermark:"+indexName { + logutil.Infof("ISCP-Task injected hook %s", msg) + } err = r.UpdateWatermark(sqlproc.GetContext(), sqlctx.GetService(), sqlctx.Txn()) if err != nil { return @@ -365,6 +403,12 @@ func runHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch c // HNSW models are already in local so hnsw Update should not require executing SQL or should be read-only. No transaction required. err = runTxnWithSqlContext(ctx, c.cnEngine, c.cnTxnClient, c.cnUUID, r.GetAccountID(), 30*time.Minute, nil, nil, func(sqlproc *sqlexec.SqlProcess, cbdata any) (err error) { + if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongHnswBeforeUpdate, indexName) { + logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongHnswBeforeUpdate, indexName) + if err := ctx.Err(); err != nil { + return err + } + } if objectio.ISCPIndexHnswUpdateErrInjected() { return injectedISCPIndexErr("hnsw update") } @@ -404,6 +448,20 @@ func reportIndexConsumerErr(errch chan error, err error) { } } +func waitISCPIndexCancelCtx(ctx context.Context, key, indexName string) bool { + if indexName != "" && objectio.WaitInjectedCtx(ctx, key+":"+indexName) { + return true + } + return objectio.WaitInjectedCtx(ctx, key) +} + +func (c *IndexConsumer) indexName() string { + if c == nil || c.info == nil { + return "" + } + return c.info.IndexName +} + func injectedISCPIndexErr(point string) error { return moerr.NewInternalErrorNoCtx("injected ISCP index " + point + " failure") } @@ -659,6 +717,16 @@ func (c *IndexConsumer) sendSql(ctx context.Context, errch chan error, writer In if writer.Empty() { return nil } + if err := ctx.Err(); err != nil { + return err + } + indexName := c.indexName() + if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeSend, indexName) { + logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongBeforeSend, indexName) + if err := ctx.Err(); err != nil { + return err + } + } // generate sql from cdc sql, err := writer.ToSql() diff --git a/pkg/iscp/index_consumer_test.go b/pkg/iscp/index_consumer_test.go index f2f80fb37a7e7..882dae5f6bcef 100644 --- a/pkg/iscp/index_consumer_test.go +++ b/pkg/iscp/index_consumer_test.go @@ -29,11 +29,13 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/testutil/testengine" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/util/executor" + "github.com/matrixorigin/matrixone/pkg/util/fault" "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/prashantv/gostub" @@ -419,6 +421,38 @@ func TestIndexConsumerSendSqlReturnsContextErrorWhenBlocked(t *testing.T) { require.ErrorIs(t, err, context.Canceled) } +func TestIndexConsumerSendSqlCtxFaultReturnsContextError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fault.Enable() + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(ctx, objectio.FJ_ISCPCancelLongBeforeSend, ":::", "wait", 0, "", false)) + require.NoError(t, fault.AddFaultPoint(ctx, "get-send-waiters", ":::", "getwaiters", 0, objectio.FJ_ISCPCancelLongBeforeSend, false)) + + consumer := &IndexConsumer{ + info: newTestIvfConsumerInfo(), + sqlBufSendCh: make(chan []byte), + } + done := make(chan error, 1) + go func() { + done <- consumer.sendSql(ctx, make(chan error, 1), &flushEveryRowWriter{rows: 1}) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := fault.TriggerFault("get-send-waiters") + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("sendSql stayed blocked in ctx-aware fault wait after context cancellation") + } +} + func TestIndexConsumerWorkerErrorCancelsBlockedNext(t *testing.T) { proc := testutil.NewProcess(t) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) @@ -626,7 +660,7 @@ func TestIndexConsumerTailFinalizationUsesParentContext(t *testing.T) { require.ErrorIs(t, err, context.Canceled) } -func TestDataRetrieverSetNextBatchReleasesRejectedData(t *testing.T) { +func TestDataRetrieverSetNextBatchRejectsCanceledRetriever(t *testing.T) { ctx := context.Background() status := &JobStatus{} retriever := NewDataRetriever(ctx, 0, 0, "job", 0, status, 0, ISCPDataType_Snapshot) @@ -639,10 +673,37 @@ func TestDataRetrieverSetNextBatchReleasesRejectedData(t *testing.T) { } data.Set(1) - retriever.SetNextBatch(data) + accepted := retriever.SetNextBatch(data) + require.False(t, accepted) + data.Done() require.Nil(t, data.insertBatch) } +func TestDataRetrieverSetNextBatchDoesNotQueueAfterContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + status := &JobStatus{} + retriever := NewDataRetriever(ctx, 0, 0, "job", 0, status, 0, ISCPDataType_Snapshot) + cancel() + + data := &ISCPData{ + insertBatch: &AtomicBatch{ + Rows: btree.NewBTreeGOptions(AtomicBatchRow.Less, btree.Options{Degree: 64}), + }, + } + data.Set(1) + + accepted := retriever.SetNextBatch(data) + + require.False(t, accepted) + data.Done() + require.Nil(t, data.insertBatch) + select { + case queued := <-retriever.insertDataCh: + require.Nil(t, queued) + default: + } +} + func TestIndexConsumerTerminalFlushErrorDoesNotCloseChannel(t *testing.T) { proc := testutil.NewProcess(t) ctx := context.Background() diff --git a/pkg/iscp/iteration.go b/pkg/iscp/iteration.go index d8ebfec6d9de6..5c1f7908b852c 100644 --- a/pkg/iscp/iteration.go +++ b/pkg/iscp/iteration.go @@ -42,8 +42,10 @@ import ( type DataRetrieverConsumer interface { DataRetriever - SetNextBatch(*ISCPData) + SetNextBatch(*ISCPData) bool SetError(error) + Cancel(error) + IsCanceled() bool Close() } @@ -60,6 +62,22 @@ func ExecuteIteration( iterCtx *IterationContext, mp *mpool.MPool, ) (err error) { + return ExecuteIterationWithRuntime(ctx, nil, cnUUID, cnEngine, cnTxnClient, iterCtx, mp) +} + +func ExecuteIterationWithRuntime( + ctx context.Context, + runtime *ISCPTaskExecutor, + cnUUID string, + cnEngine engine.Engine, + cnTxnClient client.TxnClient, + iterCtx *IterationContext, + mp *mpool.MPool, +) (err error) { + iterCtx = runtime.filterFencedIteration(iterCtx) + if iterCtx == nil { + return nil + } packer := types.NewPacker() defer packer.Close() @@ -276,6 +294,7 @@ func ExecuteIteration( runISCPTaskIterationConsumers( ctxWithoutTimeout, + runtime, iterCtx, changes, consumers, @@ -289,6 +308,9 @@ func ExecuteIteration( delCompositedPkColIdx, ) for i, status := range statuses { + if runtime != nil && runtime.IsJobFenced(NewJobRuntimeKey(iterCtx.accountID, iterCtx.tableID, iterCtx.jobNames[i])) { + continue + } if status.ErrorCode != 0 || typ == ISCPDataType_Snapshot { state := ISCPJobState_Completed if status.PermanentlyFailed() { @@ -335,6 +357,7 @@ func ExecuteIteration( func runISCPTaskIterationConsumers( ctx context.Context, + runtime *ISCPTaskExecutor, iterCtx *IterationContext, changes engine.ChangesHandle, consumers []Consumer, @@ -450,9 +473,26 @@ func runISCPTaskIterationConsumers( } noMoreData := data.noMoreData - data.Set(len(consumers)) - for i := range consumers { - dataRetrievers[i].SetNextBatch(data) + active := make([]DataRetrieverConsumer, 0, len(dataRetrievers)) + for i := range dataRetrievers { + if dataRetrievers[i] != nil && !dataRetrievers[i].IsCanceled() { + active = append(active, dataRetrievers[i]) + } + } + data.Set(len(active)) + if len(active) == 0 { + data.Close() + } + for _, retriever := range active { + if objectio.WaitInjected(objectio.FJ_ISCPCancelFanoutBeforeSend) { + logutil.Infof("ISCP-Task cancel fault wait %s", objectio.FJ_ISCPCancelFanoutBeforeSend) + } + if msg, injected := objectio.ISCPExecutorInjected(); injected && strings.HasPrefix(msg, "iscp:fanout-before-send:") { + logutil.Infof("ISCP-Task injected hook %s", msg) + } + if !retriever.SetNextBatch(data) { + data.Done() + } } if noMoreData { @@ -469,9 +509,37 @@ func runISCPTaskIterationConsumers( waitGroups[i].Add(1) go func(i int) { defer waitGroups[i].Done() - consumerCtx := context.WithValue(ctxWithCancel, defines.TenantIDKey{}, catalog.System_Account) + consumerCtx, consumerCancel := context.WithCancel(ctx) + defer consumerCancel() + consumerCtx = context.WithValue(consumerCtx, defines.TenantIDKey{}, catalog.System_Account) + var handle *RunningJobConsumer + key := NewJobRuntimeKey(iterCtx.accountID, iterCtx.tableID, iterCtx.jobNames[i]) + if runtime != nil { + var ok bool + handle, ok = runtime.RegisterRunningConsumer( + key, + iterCtx.jobIDs[i], + iterCtx.lsn[i], + consumerCancel, + dataRetrievers[i].Cancel, + ) + if !ok { + dataRetrievers[i].Cancel(errors.New("iscp job consumer canceled")) + return + } + if objectio.WaitInjected(objectio.FJ_ISCPCancelAfterRegisterConsumer) { + logutil.Infof("ISCP-Task cancel fault wait %s job=%s", objectio.FJ_ISCPCancelAfterRegisterConsumer, iterCtx.jobNames[i]) + } + if msg, injected := objectio.ISCPExecutorInjected(); injected && msg == "iscp:after-register-consumer:"+iterCtx.jobNames[i] { + logutil.Infof("ISCP-Task injected hook %s", msg) + } + defer runtime.UnregisterRunningConsumer(handle) + } err := consumerEntry.Consume(consumerCtx, dataRetrievers[i]) if err != nil { + if runtime != nil && runtime.IsJobFenced(key) { + return + } logutil.Error( "ISCP-Task sink consume failed", zap.Uint32("tenantID", iterCtx.accountID), diff --git a/pkg/iscp/iteration_consumer_context_test.go b/pkg/iscp/iteration_consumer_context_test.go index cdb0de8c2c5a3..cfbc605ce0f92 100644 --- a/pkg/iscp/iteration_consumer_context_test.go +++ b/pkg/iscp/iteration_consumer_context_test.go @@ -144,6 +144,7 @@ func runIterationConsumersWithStatusesForTest( defer packer.Close() runISCPTaskIterationConsumers( ctx, + nil, iterCtx, changes, consumers, diff --git a/pkg/iscp/runtime_cancel.go b/pkg/iscp/runtime_cancel.go new file mode 100644 index 0000000000000..768a56ce920bb --- /dev/null +++ b/pkg/iscp/runtime_cancel.go @@ -0,0 +1,205 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iscp + +import ( + "context" + "errors" + "sync" +) + +var iscpExecutors sync.Map // map[cnUUID]*ISCPTaskExecutor + +func RegisterExecutorRuntime(cnUUID string, exec *ISCPTaskExecutor) { + if cnUUID == "" || exec == nil { + return + } + iscpExecutors.Store(cnUUID, exec) +} + +func UnregisterExecutorRuntime(cnUUID string, exec *ISCPTaskExecutor) { + if cnUUID == "" || exec == nil { + return + } + if current, ok := iscpExecutors.Load(cnUUID); ok && current == exec { + iscpExecutors.Delete(cnUUID) + } +} + +func GetExecutorRuntime(cnUUID string) (*ISCPTaskExecutor, bool) { + v, ok := iscpExecutors.Load(cnUUID) + if !ok { + var found *ISCPTaskExecutor + count := 0 + iscpExecutors.Range(func(_, v any) bool { + exec, ok := v.(*ISCPTaskExecutor) + if !ok { + return true + } + found = exec + count++ + return count < 2 + }) + if count == 1 { + return found, true + } + return nil, false + } + exec, ok := v.(*ISCPTaskExecutor) + return exec, ok +} + +func NewJobRuntimeKey(accountID uint32, tableID uint64, jobName string) JobRuntimeKey { + return JobRuntimeKey{AccountID: accountID, TableID: tableID, JobName: jobName} +} + +func (exec *ISCPTaskExecutor) ensureRuntimeMapsLocked() { + if exec.fencedJobs == nil { + exec.fencedJobs = make(map[JobRuntimeKey]struct{}) + } + if exec.runningConsumers == nil { + exec.runningConsumers = make(map[JobRuntimeKey]map[uint64]*RunningJobConsumer) + } +} + +func (exec *ISCPTaskExecutor) IsJobFenced(key JobRuntimeKey) bool { + if exec == nil { + return false + } + exec.runtimeMu.Lock() + defer exec.runtimeMu.Unlock() + _, ok := exec.fencedJobs[key] + return ok +} + +func (exec *ISCPTaskExecutor) CancelAndDrainJobConsumer( + ctx context.Context, + accountID uint32, + tableID uint64, + jobName string, +) error { + if exec == nil { + return nil + } + key := NewJobRuntimeKey(accountID, tableID, jobName) + cancelErr := errors.New("iscp job consumer canceled") + + exec.runtimeMu.Lock() + exec.ensureRuntimeMapsLocked() + exec.fencedJobs[key] = struct{}{} + handles := make([]*RunningJobConsumer, 0, len(exec.runningConsumers[key])) + for _, h := range exec.runningConsumers[key] { + handles = append(handles, h) + if h.cancel != nil { + h.cancel() + } + if h.cancelRetriever != nil { + h.cancelRetriever(cancelErr) + } + } + exec.runtimeMu.Unlock() + + for _, h := range handles { + select { + case <-h.done: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil +} + +func (exec *ISCPTaskExecutor) RegisterRunningConsumer( + key JobRuntimeKey, + jobID uint64, + iterID uint64, + cancel context.CancelFunc, + cancelRetriever func(error), +) (*RunningJobConsumer, bool) { + if exec == nil { + return nil, true + } + exec.runtimeMu.Lock() + defer exec.runtimeMu.Unlock() + exec.ensureRuntimeMapsLocked() + if _, fenced := exec.fencedJobs[key]; fenced { + return nil, false + } + h := &RunningJobConsumer{ + key: key, + jobID: jobID, + iterID: iterID, + cancel: cancel, + cancelRetriever: cancelRetriever, + done: make(chan struct{}), + } + byJobID := exec.runningConsumers[key] + if byJobID == nil { + byJobID = make(map[uint64]*RunningJobConsumer) + exec.runningConsumers[key] = byJobID + } + byJobID[jobID] = h + return h, true +} + +func (exec *ISCPTaskExecutor) UnregisterRunningConsumer(h *RunningJobConsumer) { + if exec == nil || h == nil { + return + } + exec.runtimeMu.Lock() + if byJobID := exec.runningConsumers[h.key]; byJobID != nil { + if byJobID[h.jobID] == h { + delete(byJobID, h.jobID) + } + if len(byJobID) == 0 { + delete(exec.runningConsumers, h.key) + } + } + exec.runtimeMu.Unlock() + close(h.done) +} + +func (exec *ISCPTaskExecutor) filterFencedIteration(iter *IterationContext) *IterationContext { + if exec == nil || iter == nil || len(iter.jobNames) == 0 { + return iter + } + keepJobNames := make([]string, 0, len(iter.jobNames)) + keepJobIDs := make([]uint64, 0, len(iter.jobIDs)) + keepLSN := make([]uint64, 0, len(iter.lsn)) + for i, jobName := range iter.jobNames { + key := NewJobRuntimeKey(iter.accountID, iter.tableID, jobName) + if exec.IsJobFenced(key) { + continue + } + keepJobNames = append(keepJobNames, jobName) + keepJobIDs = append(keepJobIDs, iter.jobIDs[i]) + keepLSN = append(keepLSN, iter.lsn[i]) + } + if len(keepJobNames) == 0 { + return nil + } + if len(keepJobNames) == len(iter.jobNames) { + return iter + } + return &IterationContext{ + accountID: iter.accountID, + tableID: iter.tableID, + jobNames: keepJobNames, + jobIDs: keepJobIDs, + lsn: keepLSN, + fromTS: iter.fromTS, + toTS: iter.toTS, + } +} diff --git a/pkg/iscp/runtime_cancel_test.go b/pkg/iscp/runtime_cancel_test.go new file mode 100644 index 0000000000000..2f76c99965c01 --- /dev/null +++ b/pkg/iscp/runtime_cancel_test.go @@ -0,0 +1,171 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iscp + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +func newRuntimeTestExecutor() *ISCPTaskExecutor { + return &ISCPTaskExecutor{ + fencedJobs: make(map[JobRuntimeKey]struct{}), + runningConsumers: make(map[JobRuntimeKey]map[uint64]*RunningJobConsumer), + option: fillDefaultOption(nil), + ctx: context.Background(), + } +} + +func TestCancelAndDrainJobConsumerFencesAndWaitsForRunningConsumer(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01") + consumerCtx, consumerCancel := context.WithCancel(context.Background()) + defer consumerCancel() + retrieverCanceled := make(chan error, 1) + + h, ok := exec.RegisterRunningConsumer(key, 7, 11, consumerCancel, func(err error) { + retrieverCanceled <- err + }) + require.True(t, ok) + + exited := make(chan struct{}) + go func() { + defer close(exited) + <-consumerCtx.Done() + time.Sleep(20 * time.Millisecond) + exec.UnregisterRunningConsumer(h) + }() + + start := time.Now() + err := exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName) + require.NoError(t, err) + require.GreaterOrEqual(t, time.Since(start), 20*time.Millisecond) + require.True(t, exec.IsJobFenced(key)) + require.ErrorContains(t, <-retrieverCanceled, "iscp job consumer canceled") + <-exited +} + +func TestRegisterRunningConsumerRejectsFencedJob(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01") + + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName)) + _, ok := exec.RegisterRunningConsumer(key, 1, 1, func() {}, nil) + + require.False(t, ok) +} + +func TestCancelAndDrainJobConsumerOnlyCancelsMatchingJob(t *testing.T) { + exec := newRuntimeTestExecutor() + key1 := NewJobRuntimeKey(1, 2, "index_idx01") + key2 := NewJobRuntimeKey(1, 2, "index_idx02") + ctx1, cancel1 := context.WithCancel(context.Background()) + ctx2, cancel2 := context.WithCancel(context.Background()) + defer cancel2() + + h1, ok := exec.RegisterRunningConsumer(key1, 1, 1, cancel1, nil) + require.True(t, ok) + _, ok = exec.RegisterRunningConsumer(key2, 2, 1, cancel2, nil) + require.True(t, ok) + go func() { + <-ctx1.Done() + exec.UnregisterRunningConsumer(h1) + }() + + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key1.AccountID, key1.TableID, key1.JobName)) + + select { + case <-ctx2.Done(): + t.Fatal("non-matching job was canceled") + default: + } + exec.runtimeMu.Lock() + _, stillRunning := exec.runningConsumers[key2][uint64(2)] + exec.runtimeMu.Unlock() + require.True(t, stillRunning) +} + +func TestFilterFencedIterationRemovesOnlyFencedJobs(t *testing.T) { + exec := newRuntimeTestExecutor() + iter := NewIterationContext( + 1, + 2, + []string{"index_idx01", "index_idx02"}, + []uint64{10, 20}, + []uint64{100, 200}, + types.BuildTS(1, 0), + types.BuildTS(2, 0), + ) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01")) + + filtered := exec.filterFencedIteration(iter) + + require.NotNil(t, filtered) + require.Equal(t, []string{"index_idx02"}, filtered.jobNames) + require.Equal(t, []uint64{20}, filtered.jobIDs) + require.Equal(t, []uint64{200}, filtered.lsn) + require.True(t, filtered.fromTS.EQ(&iter.fromTS)) + require.True(t, filtered.toTS.EQ(&iter.toTS)) +} + +func TestFilterFencedIterationDropsAllJobs(t *testing.T) { + exec := newRuntimeTestExecutor() + iter := NewIterationContext( + 1, + 2, + []string{"index_idx01"}, + []uint64{10}, + []uint64{100}, + types.BuildTS(1, 0), + types.BuildTS(2, 0), + ) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01")) + + require.Nil(t, exec.filterFencedIteration(iter)) +} + +func TestTableEntryGetCandidateSkipsFencedJob(t *testing.T) { + exec := newRuntimeTestExecutor() + table := NewTableEntry(exec, 1, 10, 2, "db", "tbl") + spec := &JobSpec{TriggerSpec: TriggerSpec{JobType: TriggerType_Default}} + table.jobs[JobKey{JobName: "index_idx01", JobID: 1}] = NewJobEntry(table, "index_idx01", spec, 1, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + table.jobs[JobKey{JobName: "index_idx02", JobID: 2}] = NewJobEntry(table, "index_idx02", spec, 2, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01")) + + iters, _ := table.getCandidate() + + require.Len(t, iters, 1) + require.Equal(t, []string{"index_idx02"}, iters[0].jobNames) +} + +func TestCancelAndDrainJobConsumerHonorsCallerContext(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01") + consumerCancelCalled := make(chan struct{}) + _, ok := exec.RegisterRunningConsumer(key, 1, 1, func() { close(consumerCancelCalled) }, nil) + require.True(t, ok) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := exec.CancelAndDrainJobConsumer(ctx, key.AccountID, key.TableID, key.JobName) + + require.True(t, errors.Is(err, context.Canceled)) + <-consumerCancelCalled +} diff --git a/pkg/iscp/table_entry.go b/pkg/iscp/table_entry.go index dbc00980cdcad..548049ccdd8dd 100644 --- a/pkg/iscp/table_entry.go +++ b/pkg/iscp/table_entry.go @@ -139,6 +139,9 @@ func (t *TableEntry) getCandidate() (iter []*IterationContext, minFromTS types.T if sinker.dropAt != 0 { continue } + if t.exec != nil && t.exec.IsJobFenced(NewJobRuntimeKey(t.accountID, t.tableID, sinker.jobName)) { + continue + } candidates = append(candidates, sinker) } iterations := make([]*IterationContext, 0, len(candidates)) diff --git a/pkg/iscp/types.go b/pkg/iscp/types.go index a810a90598e13..2645fc071bb64 100644 --- a/pkg/iscp/types.go +++ b/pkg/iscp/types.go @@ -157,6 +157,26 @@ type ISCPTaskExecutor struct { running bool runningMu sync.Mutex + + runtimeMu sync.Mutex + fencedJobs map[JobRuntimeKey]struct{} + runningConsumers map[JobRuntimeKey]map[uint64]*RunningJobConsumer +} + +type JobRuntimeKey struct { + AccountID uint32 + TableID uint64 + JobName string +} + +type RunningJobConsumer struct { + key JobRuntimeKey + jobID uint64 + iterID uint64 + + cancel context.CancelFunc + cancelRetriever func(error) + done chan struct{} } // Intra-System Change Propagation Job Entry diff --git a/pkg/iscp/worker.go b/pkg/iscp/worker.go index 7ceba379cf766..b0f27a6ee9240 100644 --- a/pkg/iscp/worker.go +++ b/pkg/iscp/worker.go @@ -41,6 +41,7 @@ type Worker interface { } type worker struct { + exec *ISCPTaskExecutor cnUUID string cnEngine engine.Engine cnTxnClient client.TxnClient @@ -52,8 +53,9 @@ type worker struct { closed atomic.Bool } -func NewWorker(cnUUID string, cnEngine engine.Engine, cnTxnClient client.TxnClient, mp *mpool.MPool) Worker { +func NewWorker(exec *ISCPTaskExecutor, cnUUID string, cnEngine engine.Engine, cnTxnClient client.TxnClient, mp *mpool.MPool) Worker { worker := &worker{ + exec: exec, cnUUID: cnUUID, cnEngine: cnEngine, cnTxnClient: cnTxnClient, @@ -95,11 +97,16 @@ func (w *worker) Submit(iteration *IterationContext) error { } func (w *worker) onItem(iterCtx *IterationContext) { + iterCtx = w.exec.filterFencedIteration(iterCtx) + if iterCtx == nil { + return + } err := retry( w.ctx, func() error { - err := ExecuteIteration( + err := ExecuteIterationWithRuntime( w.ctx, + w.exec, w.cnUUID, w.cnEngine, w.cnTxnClient, diff --git a/pkg/objectio/injects.go b/pkg/objectio/injects.go index 9ba17f0867f93..0f0fa761d75ac 100644 --- a/pkg/objectio/injects.go +++ b/pkg/objectio/injects.go @@ -67,6 +67,17 @@ const ( FJ_ISCPIndexCuvsAppendErr = "fj/iscp/index/cuvs/append/error" FJ_ISCPIndexCuvsSaveErr = "fj/iscp/index/cuvs/save/error" + FJ_ISCPCancelAfterSubmit = "fj/iscp/cancel/after-submit" + FJ_ISCPCancelAfterRegisterConsumer = "fj/iscp/cancel/after-register-consumer" + FJ_ISCPCancelFanoutBeforeSend = "fj/iscp/cancel/fanout-before-send" + FJ_ISCPCancelHnswBeforeSave = "fj/iscp/cancel/hnsw-before-save" + FJ_ISCPCancelBeforeUpdateWatermark = "fj/iscp/cancel/before-update-watermark" + FJ_ISCPCancelLongBeforeSend = "fj/iscp/cancel/long/before-send" + FJ_ISCPCancelLongBeforeExec = "fj/iscp/cancel/long/before-exec" + FJ_ISCPCancelLongHnswBeforeUpdate = "fj/iscp/cancel/long/hnsw-before-update" + FJ_ISCPCancelLongHnswBeforeSave = "fj/iscp/cancel/long/hnsw-before-save" + FJ_ISCPCancelLongBeforeWatermark = "fj/iscp/cancel/long/before-watermark" + FJ_PublicationSnapshotFinished = "fj/publication/snapshot/finished" FJ_UpstreamSQLHelper = "fj/publication/upstream/sqlhelper" @@ -436,8 +447,14 @@ func GCDumpTableInjected() (string, bool) { return sarg, injected } -func WaitInjected(key string) { - fault.TriggerFault(key) +func WaitInjected(key string) bool { + _, _, injected := fault.TriggerFault(key) + return injected +} + +func WaitInjectedCtx(ctx context.Context, key string) bool { + _, _, injected := fault.TriggerFaultWithContext(ctx, key) + return injected } func NotifyInjected(key string) { diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index 0f52aaaa6f027..33cf9ad84a2a9 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -737,6 +737,9 @@ func (s *Scope) AlterTableInplace(c *Compile) error { hasUpdateConstraints = true var notDroppedIndex []*plan.IndexDef var newIndexes []uint64 + if err = DrainIndexCdcTaskConsumer(c, oTableDef, constraintName); err != nil { + return err + } for idx, indexdef := range oTableDef.Indexes { if indexdef.IndexName == constraintName { dropIndexMap[indexdef.IndexName] = true @@ -2512,6 +2515,10 @@ func (s *Scope) DropIndex(c *Compile) error { if err != nil { return err } + err = DrainIndexCdcTaskConsumer(c, oldTableDef, qry.IndexName) + if err != nil { + return err + } err = r.UpdateConstraint(c.proc.Ctx, newCt) if err != nil { return err @@ -2582,7 +2589,12 @@ func makeNewDropConstraint(oldCt *engine.ConstraintDef, dropName string) (*engin case *engine.IndexDef: pred := func(index *plan.IndexDef) bool { if index.IndexName == dropName && len(index.IndexTableName) > 0 { - dropIndexTableNames = append(dropIndexTableNames, index.IndexTableName) + if catalog.ToLower(index.IndexAlgo) == "hnsw" && + catalog.ToLower(index.IndexAlgoTableType) == catalog.Hnsw_TblType_Storage { + dropIndexTableNames = append([]string{index.IndexTableName}, dropIndexTableNames...) + } else { + dropIndexTableNames = append(dropIndexTableNames, index.IndexTableName) + } } return index.IndexName == dropName } diff --git a/pkg/sql/compile/ddl_util_test.go b/pkg/sql/compile/ddl_util_test.go index 02e6f1ee1c310..c68832e5def93 100644 --- a/pkg/sql/compile/ddl_util_test.go +++ b/pkg/sql/compile/ddl_util_test.go @@ -20,15 +20,45 @@ import ( "strings" "testing" + "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/sqlquote" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/sql/parsers" "github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect" + "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestMakeNewDropConstraintDropsHnswStorageBeforeMetadata(t *testing.T) { + oldCt := &engine.ConstraintDef{ + Cts: []engine.Constraint{ + &engine.IndexDef{ + Indexes: []*plan.IndexDef{ + { + IndexName: "idx01", + IndexTableName: "__meta", + IndexAlgo: "hnsw", + IndexAlgoTableType: catalog.Hnsw_TblType_Metadata, + }, + { + IndexName: "idx01", + IndexTableName: "__storage", + IndexAlgo: "hnsw", + IndexAlgoTableType: catalog.Hnsw_TblType_Storage, + }, + }, + }, + }, + } + + _, dropIndexTableNames, err := makeNewDropConstraint(oldCt, "idx01") + + require.NoError(t, err) + require.Equal(t, []string{"__storage", "__meta"}, dropIndexTableNames) +} + // TestAlterIndexVisibleEscaping is a regression for the ALTER TABLE ... ALTER // INDEX ... VISIBLE/INVISIBLE catalog write (ddl.go, updateMoIndexesVisibleFormat). // A backticked index identifier may contain single quotes and backslashes, and diff --git a/pkg/sql/compile/iscp_util.go b/pkg/sql/compile/iscp_util.go index 1544714ce8ed5..53a769332196c 100644 --- a/pkg/sql/compile/iscp_util.go +++ b/pkg/sql/compile/iscp_util.go @@ -20,6 +20,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/defines" indexplugin "github.com/matrixorigin/matrixone/pkg/indexplugin" "github.com/matrixorigin/matrixone/pkg/iscp" "github.com/matrixorigin/matrixone/pkg/logutil" @@ -31,6 +32,7 @@ import ( var ( iscpRegisterJobFunc = iscp.RegisterJob iscpUnregisterJobFunc = iscp.UnregisterJob + iscpGetExecutorFunc = iscp.GetExecutorRuntime isTableInCCPRFunc = isTableInCCPRImpl ) @@ -192,6 +194,28 @@ func DropIndexCdcTask(c *Compile, tableDef *plan.TableDef, dbname string, tablen return nil } +func DrainIndexCdcTaskConsumer(c *Compile, tableDef *plan.TableDef, indexname string) error { + valid, err := checkValidIndexCdc(tableDef, indexname) + if err != nil { + return err + } + if !valid { + return nil + } + exec, ok := iscpGetExecutorFunc(c.proc.GetService()) + if !ok || exec == nil { + logutil.Infof("skip draining index cdc task consumer, iscp executor not found: tableID=%d index=%s", tableDef.TblId, indexname) + return nil + } + accountID, err := defines.GetAccountId(c.proc.Ctx) + if err != nil { + return err + } + jobName := genCdcTaskJobID(indexname) + logutil.Infof("drain index cdc task consumer: accountID=%d tableID=%d jobName=%s", accountID, tableDef.TblId, jobName) + return exec.CancelAndDrainJobConsumer(c.proc.Ctx, accountID, tableDef.TblId, jobName) +} + // drop all cdc tasks according to tableDef func DropAllIndexCdcTasks(c *Compile, tabledef *plan.TableDef, dbname string, tablename string) error { idxmap := make(map[string]bool) diff --git a/pkg/sql/compile/iscp_util_extra_test.go b/pkg/sql/compile/iscp_util_extra_test.go index 00762d48761b0..f9fac779c1f35 100644 --- a/pkg/sql/compile/iscp_util_extra_test.go +++ b/pkg/sql/compile/iscp_util_extra_test.go @@ -318,6 +318,60 @@ func TestCoverage_DropIndexCdcTask_InvalidIndex(t *testing.T) { require.Nil(t, err) } +func TestDrainIndexCdcTaskConsumerFencesRuntimeJob(t *testing.T) { + exec := &iscp.ISCPTaskExecutor{} + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return exec, true + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + err := DrainIndexCdcTaskConsumer(c, tbldef, "idx1") + + require.NoError(t, err) + require.True(t, exec.IsJobFenced(iscp.NewJobRuntimeKey(0, 42, "index_idx1"))) +} + +func TestDrainIndexCdcTaskConsumerNoExecutorIsNoop(t *testing.T) { + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "idx1")) +} + func TestCoverage_DropAllIndexCdcTasks_DuplicateNames(t *testing.T) { dropCount := 0 iscpUnregisterJobFunc = func(ctx context.Context, cnUUID string, txn client.TxnOperator, job *iscp.JobID) (bool, error) { diff --git a/pkg/util/fault/fault.go b/pkg/util/fault/fault.go index 2081ddeee8ca4..cbb4bb4bc45b6 100644 --- a/pkg/util/fault/fault.go +++ b/pkg/util/fault/fault.go @@ -115,11 +115,19 @@ func (fm *faultMap) run() { } case REMOVE: if e.name == "all" { + for _, v := range fm.faultPoints { + if v.action == WAIT && v.cond != nil { + v.cond.Broadcast() + } + } fm.faultPoints = make(map[string]*faultEntry) fm.chOut <- e continue } if v, ok := fm.faultPoints[e.name]; ok { + if v.action == WAIT && v.cond != nil { + v.cond.Broadcast() + } delete(fm.faultPoints, e.name) fm.chOut <- v } else { @@ -192,6 +200,38 @@ func (e *faultEntry) do() (int64, string) { return 0, "" } +func (e *faultEntry) doWithContext(ctx context.Context) (int64, string) { + if e.action == SLEEP { + timer := time.NewTimer(time.Duration(e.iarg) * time.Second) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + } + return 0, "" + } + if e.action != WAIT { + return e.do() + } + e.mutex.Lock() + e.nWaiters += 1 + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + e.mutex.Lock() + e.cond.Broadcast() + e.mutex.Unlock() + case <-done: + } + }() + e.cond.Wait() + e.nWaiters -= 1 + close(done) + e.mutex.Unlock() + return 0, "" +} + func startFaultMap(domain Domain) bool { if enabled[domain].Load() != nil { return false @@ -276,7 +316,19 @@ func TriggerFault(name string) (iret int64, sret string, exist bool) { return TriggerFaultInDomain(DomainDefault, name) } +func TriggerFaultWithContext(ctx context.Context, name string) (iret int64, sret string, exist bool) { + return TriggerFaultInDomainWithContext(ctx, DomainDefault, name) +} + func TriggerFaultInDomain(domain Domain, name string) (iret int64, sret string, exist bool) { + return triggerFaultInDomain(context.Background(), domain, name, false) +} + +func TriggerFaultInDomainWithContext(ctx context.Context, domain Domain, name string) (iret int64, sret string, exist bool) { + return triggerFaultInDomain(ctx, domain, name, true) +} + +func triggerFaultInDomain(ctx context.Context, domain Domain, name string, useCtx bool) (iret int64, sret string, exist bool) { fm := enabled[domain].Load() if fm == nil { return @@ -296,7 +348,11 @@ func TriggerFaultInDomain(domain Domain, name string) (iret int64, sret string, return } exist = true - iret, sret = out.do() + if useCtx { + iret, sret = out.doWithContext(ctx) + } else { + iret, sret = out.do() + } return } diff --git a/pkg/util/fault/fault_test.go b/pkg/util/fault/fault_test.go index 9329c38f96fb2..5cee62feb52e3 100644 --- a/pkg/util/fault/fault_test.go +++ b/pkg/util/fault/fault_test.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "testing" + "time" "github.com/stretchr/testify/require" @@ -182,6 +183,99 @@ func TestWait(t *testing.T) { Disable() } +func TestRemoveWaitFaultPointNotifiesWaiters(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + require.NoError(t, AddFaultPoint(ctx, "gw", ":::", "getwaiters", 0, "w", false)) + + done := make(chan struct{}) + go func() { + TriggerFault("w") + close(done) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := TriggerFault("gw") + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + removed, err := RemoveFaultPoint(ctx, "w") + require.NoError(t, err) + require.True(t, removed) + require.Eventually(t, func() bool { + select { + case <-done: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) +} + +func TestTriggerWaitFaultWithContextReturnsOnCancel(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + require.NoError(t, AddFaultPoint(ctx, "gw", ":::", "getwaiters", 0, "w", false)) + + waitCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + TriggerFaultWithContext(waitCtx, "w") + close(done) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := TriggerFault("gw") + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + cancel() + require.Eventually(t, func() bool { + select { + case <-done: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) +} + +func TestTriggerSleepFaultWithContextReturnsOnCancel(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "s", ":::", "sleep", 30, "", false)) + + sleepCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + start := time.Now() + go func() { + TriggerFaultWithContext(sleepCtx, "s") + close(done) + }() + + cancel() + require.Eventually(t, func() bool { + select { + case <-done: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) + require.Less(t, time.Since(start), 5*time.Second) +} + func Test_panic(t *testing.T) { var ctx = context.TODO() From fcce6f3f8ae2104032037d1da5ef79bd427ed1e3 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 15 Jul 2026 11:49:30 +0800 Subject: [PATCH 02/14] test: cover ISCP cancel error paths --- pkg/iscp/index_consumer.go | 55 ++++----- pkg/iscp/index_consumer_test.go | 190 ++++++++++++++++++++++++++++++++ pkg/iscp/iteration.go | 2 +- pkg/iscp/runtime_cancel.go | 5 +- 4 files changed, 216 insertions(+), 36 deletions(-) diff --git a/pkg/iscp/index_consumer.go b/pkg/iscp/index_consumer.go index 2b74f8dbc4f30..2572eae5e194c 100644 --- a/pkg/iscp/index_consumer.go +++ b/pkg/iscp/index_consumer.go @@ -181,11 +181,8 @@ func runIndex(c *IndexConsumer, ctx context.Context, errch chan error, r DataRet func(sqlproc *sqlexec.SqlProcess, cbdata any) (err error) { sqlctx := sqlproc.SqlCtx - if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeExec, indexName) { - logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongBeforeExec, indexName) - if err := ctx.Err(); err != nil { - return err - } + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeExec, indexName); err != nil { + return err } if objectio.ISCPIndexExecErrorInjected() { return injectedISCPIndexErr("exec") @@ -237,11 +234,8 @@ func runIndex(c *IndexConsumer, ctx context.Context, errch chan error, r DataRet // update SQL var res executor.Result - if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeExec, indexName) { - logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongBeforeExec, indexName) - if err := ctx.Err(); err != nil { - return err - } + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeExec, indexName); err != nil { + return err } if objectio.ISCPIndexExecErrorInjected() { return injectedISCPIndexErr("exec") @@ -345,11 +339,8 @@ func runHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch c if objectio.WaitInjected(objectio.FJ_ISCPCancelHnswBeforeSave) { logutil.Infof("ISCP-Task cancel fault wait %s index=%s", objectio.FJ_ISCPCancelHnswBeforeSave, indexName) } - if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongHnswBeforeSave, indexName) { - logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongHnswBeforeSave, indexName) - if err := ctx.Err(); err != nil { - return err - } + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongHnswBeforeSave, indexName); err != nil { + return err } if msg, injected := objectio.ISCPExecutorInjected(); injected && msg == "iscp:hnsw-before-save:"+indexName { logutil.Infof("ISCP-Task injected hook %s", msg) @@ -367,11 +358,8 @@ func runHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch c if objectio.WaitInjected(objectio.FJ_ISCPCancelBeforeUpdateWatermark) { logutil.Infof("ISCP-Task cancel fault wait %s index=%s", objectio.FJ_ISCPCancelBeforeUpdateWatermark, indexName) } - if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeWatermark, indexName) { - logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongBeforeWatermark, indexName) - if err := ctx.Err(); err != nil { - return err - } + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeWatermark, indexName); err != nil { + return err } if msg, injected := objectio.ISCPExecutorInjected(); injected && msg == "iscp:before-update-watermark:"+indexName { logutil.Infof("ISCP-Task injected hook %s", msg) @@ -403,11 +391,8 @@ func runHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch c // HNSW models are already in local so hnsw Update should not require executing SQL or should be read-only. No transaction required. err = runTxnWithSqlContext(ctx, c.cnEngine, c.cnTxnClient, c.cnUUID, r.GetAccountID(), 30*time.Minute, nil, nil, func(sqlproc *sqlexec.SqlProcess, cbdata any) (err error) { - if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongHnswBeforeUpdate, indexName) { - logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongHnswBeforeUpdate, indexName) - if err := ctx.Err(); err != nil { - return err - } + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongHnswBeforeUpdate, indexName); err != nil { + return err } if objectio.ISCPIndexHnswUpdateErrInjected() { return injectedISCPIndexErr("hnsw update") @@ -448,11 +433,18 @@ func reportIndexConsumerErr(errch chan error, err error) { } } -func waitISCPIndexCancelCtx(ctx context.Context, key, indexName string) bool { +func waitISCPIndexCancelCtx(ctx context.Context, key, indexName string) error { + injected := false if indexName != "" && objectio.WaitInjectedCtx(ctx, key+":"+indexName) { - return true + injected = true + } else if objectio.WaitInjectedCtx(ctx, key) { + injected = true + } + if !injected { + return nil } - return objectio.WaitInjectedCtx(ctx, key) + logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", key, indexName) + return ctx.Err() } func (c *IndexConsumer) indexName() string { @@ -721,11 +713,8 @@ func (c *IndexConsumer) sendSql(ctx context.Context, errch chan error, writer In return err } indexName := c.indexName() - if waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeSend, indexName) { - logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", objectio.FJ_ISCPCancelLongBeforeSend, indexName) - if err := ctx.Err(); err != nil { - return err - } + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeSend, indexName); err != nil { + return err } // generate sql from cdc diff --git a/pkg/iscp/index_consumer_test.go b/pkg/iscp/index_consumer_test.go index 882dae5f6bcef..d3895f75fbe29 100644 --- a/pkg/iscp/index_consumer_test.go +++ b/pkg/iscp/index_consumer_test.go @@ -453,6 +453,102 @@ func TestIndexConsumerSendSqlCtxFaultReturnsContextError(t *testing.T) { } } +func TestIndexConsumerSendSqlCtxFaultMatchesIndexName(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + info := newTestIvfConsumerInfo() + key := objectio.FJ_ISCPCancelLongBeforeSend + ":" + info.IndexName + waitersKey := "get-send-index-waiters" + + fault.Enable() + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(ctx, key, ":::", "wait", 0, "", false)) + require.NoError(t, fault.AddFaultPoint(ctx, waitersKey, ":::", "getwaiters", 0, key, false)) + + consumer := &IndexConsumer{ + info: info, + sqlBufSendCh: make(chan []byte), + } + done := make(chan error, 1) + go func() { + done <- consumer.sendSql(ctx, make(chan error, 1), &flushEveryRowWriter{rows: 1}) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := fault.TriggerFault(waitersKey) + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("sendSql stayed blocked in index-specific ctx-aware fault wait after context cancellation") + } +} + +func TestRunIndexBeforeExecCtxFaultReturnsContextError(t *testing.T) { + for _, dtype := range []int8{ISCPDataType_Snapshot, ISCPDataType_Tail} { + t.Run(fmt.Sprintf("type-%d", dtype), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + info := newTestIvfConsumerInfo() + key := objectio.FJ_ISCPCancelLongBeforeExec + ":" + info.IndexName + waitersKey := fmt.Sprintf("get-before-exec-waiters-%d", dtype) + + fault.Enable() + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(ctx, key, ":::", "wait", 0, "", false)) + require.NoError(t, fault.AddFaultPoint(ctx, waitersKey, ":::", "getwaiters", 0, key, false)) + + stubRunTxn := gostub.Stub(&runTxnWithSqlContext, func( + ctx context.Context, + _ engine.Engine, + _ client.TxnClient, + cnUUID string, + accountID uint32, + _ time.Duration, + resolveVariableFunc func(varName string, isSystemVar, isGlobalVar bool) (interface{}, error), + cbdata any, + f func(sqlproc *sqlexec.SqlProcess, data any) error, + ) error { + sqlproc := sqlexec.NewSqlProcessWithContext(sqlexec.NewSqlContext(ctx, cnUUID, nil, accountID, resolveVariableFunc)) + return f(sqlproc, cbdata) + }) + defer stubRunTxn.Reset() + + consumer := &IndexConsumer{ + info: info, + sqlBufSendCh: make(chan []byte), + } + retriever := &MockRetriever{dtype: dtype} + errch := make(chan error, 2) + done := make(chan struct{}) + go func() { + runIndex(consumer, ctx, errch, retriever) + close(done) + }() + + consumer.sqlBufSendCh <- []byte("select 1") + require.Eventually(t, func() bool { + cnt, _, ok := fault.TriggerFault(waitersKey) + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("runIndex stayed blocked in before-exec ctx-aware fault wait after context cancellation") + } + require.ErrorIs(t, <-errch, context.Canceled) + }) + } +} + func TestIndexConsumerWorkerErrorCancelsBlockedNext(t *testing.T) { proc := testutil.NewProcess(t) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) @@ -679,6 +775,100 @@ func TestDataRetrieverSetNextBatchRejectsCanceledRetriever(t *testing.T) { require.Nil(t, data.insertBatch) } +func TestDataRetrieverCanceledWatermarkAndIsCanceledErrorPath(t *testing.T) { + ctx := context.Background() + retriever := NewDataRetriever(ctx, 0, 0, "job", 0, &JobStatus{}, 0, ISCPDataType_Tail) + cancelErr := errors.New("consumer canceled") + retriever.Cancel(cancelErr) + + require.True(t, retriever.IsCanceled()) + require.ErrorIs(t, retriever.UpdateWatermark(ctx, "", nil), cancelErr) +} + +func TestDataRetrieverStatusJSONAndAccessors(t *testing.T) { + status := &JobStatus{ + LSN: 7, + Stage: JobStage_Running, + ErrorCode: 1, + ErrorMsg: "failed", + } + statusJSON, err := MarshalJobStatus(status) + require.NoError(t, err) + statusByteJSON, err := types.ParseStringToByteJson(statusJSON) + require.NoError(t, err) + encodedStatusJSON, err := types.EncodeJson(statusByteJSON) + require.NoError(t, err) + + decoded, err := UnmarshalJobStatus(encodedStatusJSON) + require.NoError(t, err) + require.Equal(t, status.LSN, decoded.LSN) + require.Equal(t, status.Stage, decoded.Stage) + require.Equal(t, status.ErrorCode, decoded.ErrorCode) + require.Equal(t, status.ErrorMsg, decoded.ErrorMsg) + + retriever := NewDataRetriever(context.Background(), 42, 99, "job", 3, status, 11, ISCPDataType_Tail) + require.Equal(t, int8(ISCPDataType_Tail), retriever.GetDataType()) + require.Equal(t, uint32(42), retriever.GetAccountID()) + require.Equal(t, uint64(99), retriever.GetTableID()) +} + +func TestDataRetrieverUpdateWatermarkSnapshotNoop(t *testing.T) { + retriever := NewDataRetriever(context.Background(), 0, 0, "job", 0, &JobStatus{}, 0, ISCPDataType_Snapshot) + require.NoError(t, retriever.UpdateWatermark(context.Background(), "", nil)) +} + +func TestDataRetrieverUpdateWatermarkTailUsesExecWithResult(t *testing.T) { + ctx := context.Background() + status := &JobStatus{To: types.BuildTS(10, 1)} + retriever := NewDataRetriever(ctx, 7, 8, "job", 9, status, 11, ISCPDataType_Tail) + + var execSQL string + stubExec := gostub.Stub(&ExecWithResult, func(execCtx context.Context, sql string, cnUUID string, txn client.TxnOperator) (executor.Result, error) { + execSQL = sql + require.Equal(t, "cn", cnUUID) + require.Equal(t, catalog.System_Account, execCtx.Value(defines.TenantIDKey{})) + return executor.Result{}, nil + }) + defer stubExec.Reset() + + require.NoError(t, retriever.UpdateWatermark(ctx, "cn", nil)) + require.Equal(t, uint64(11), status.LSN) + require.Contains(t, execSQL, "UPDATE mo_catalog.mo_iscp_log") + require.Contains(t, execSQL, "job") +} + +func TestDataRetrieverUpdateWatermarkTailExecError(t *testing.T) { + expected := errors.New("watermark update failed") + retriever := NewDataRetriever(context.Background(), 7, 8, "job", 9, &JobStatus{}, 11, ISCPDataType_Tail) + stubExec := gostub.Stub(&ExecWithResult, func(context.Context, string, string, client.TxnOperator) (executor.Result, error) { + return executor.Result{}, expected + }) + defer stubExec.Reset() + + require.ErrorIs(t, retriever.UpdateWatermark(context.Background(), "cn", nil), expected) +} + +func TestISCPDataCloseReleasesBothBatches(t *testing.T) { + data := &ISCPData{ + insertBatch: &AtomicBatch{ + Rows: btree.NewBTreeGOptions(AtomicBatchRow.Less, btree.Options{Degree: 64}), + Batches: []*batch.Batch{}, + }, + deleteBatch: &AtomicBatch{ + Rows: btree.NewBTreeGOptions(AtomicBatchRow.Less, btree.Options{Degree: 64}), + Batches: []*batch.Batch{}, + }, + } + + data.Close() + + require.Nil(t, data.insertBatch) + require.Nil(t, data.deleteBatch) + + var nilData *ISCPData + require.NotPanics(t, nilData.Close) +} + func TestDataRetrieverSetNextBatchDoesNotQueueAfterContextCancel(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) status := &JobStatus{} diff --git a/pkg/iscp/iteration.go b/pkg/iscp/iteration.go index ad7751456cc81..c6bb85680ff0b 100644 --- a/pkg/iscp/iteration.go +++ b/pkg/iscp/iteration.go @@ -515,7 +515,7 @@ func runISCPTaskIterationConsumers( dataRetrievers[i].Cancel, ) if !ok { - dataRetrievers[i].Cancel(errors.New("iscp job consumer canceled")) + dataRetrievers[i].Cancel(moerr.NewInternalErrorNoCtx("iscp job consumer canceled")) return } if objectio.WaitInjected(objectio.FJ_ISCPCancelAfterRegisterConsumer) { diff --git a/pkg/iscp/runtime_cancel.go b/pkg/iscp/runtime_cancel.go index 768a56ce920bb..724c34d7c97a7 100644 --- a/pkg/iscp/runtime_cancel.go +++ b/pkg/iscp/runtime_cancel.go @@ -16,8 +16,9 @@ package iscp import ( "context" - "errors" "sync" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" ) var iscpExecutors sync.Map // map[cnUUID]*ISCPTaskExecutor @@ -94,7 +95,7 @@ func (exec *ISCPTaskExecutor) CancelAndDrainJobConsumer( return nil } key := NewJobRuntimeKey(accountID, tableID, jobName) - cancelErr := errors.New("iscp job consumer canceled") + cancelErr := moerr.NewInternalErrorNoCtx("iscp job consumer canceled") exec.runtimeMu.Lock() exec.ensureRuntimeMapsLocked() From fa72cd8a78a5a16d25346cb6eb51c353c84f89af Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 15 Jul 2026 15:02:16 +0800 Subject: [PATCH 03/14] fix: close ISCP cancel lifecycle gaps --- pkg/iscp/iteration.go | 4 +- pkg/iscp/runtime_cancel.go | 69 +++++++++++--------- pkg/iscp/runtime_cancel_test.go | 54 +++++++++++---- pkg/iscp/table_entry.go | 2 +- pkg/iscp/types.go | 1 + pkg/iscp/watermark_updater.go | 35 ++++++++++ pkg/sql/compile/ddl.go | 21 +++--- pkg/sql/compile/iscp_util.go | 59 ++++++++++++++--- pkg/sql/compile/iscp_util_extra_test.go | 87 +++++++++++++++++++++++-- pkg/util/fault/fault.go | 15 ++++- pkg/util/fault/fault_test.go | 27 ++++++++ 11 files changed, 303 insertions(+), 71 deletions(-) diff --git a/pkg/iscp/iteration.go b/pkg/iscp/iteration.go index c6bb85680ff0b..a35466f4812aa 100644 --- a/pkg/iscp/iteration.go +++ b/pkg/iscp/iteration.go @@ -311,7 +311,7 @@ func ExecuteIterationWithRuntime( delCompositedPkColIdx, ) for i, status := range statuses { - if runtime != nil && runtime.IsJobFenced(NewJobRuntimeKey(iterCtx.accountID, iterCtx.tableID, iterCtx.jobNames[i])) { + if runtime != nil && runtime.IsJobFenced(NewJobRuntimeKey(iterCtx.accountID, iterCtx.tableID, iterCtx.jobNames[i], iterCtx.jobIDs[i])) { continue } if status.ErrorCode != 0 || typ == ISCPDataType_Snapshot { @@ -504,7 +504,7 @@ func runISCPTaskIterationConsumers( defer consumerCancel() consumerCtx = context.WithValue(consumerCtx, defines.TenantIDKey{}, catalog.System_Account) var handle *RunningJobConsumer - key := NewJobRuntimeKey(iterCtx.accountID, iterCtx.tableID, iterCtx.jobNames[i]) + key := NewJobRuntimeKey(iterCtx.accountID, iterCtx.tableID, iterCtx.jobNames[i], iterCtx.jobIDs[i]) if runtime != nil { var ok bool handle, ok = runtime.RegisterRunningConsumer( diff --git a/pkg/iscp/runtime_cancel.go b/pkg/iscp/runtime_cancel.go index 724c34d7c97a7..9e81bfa65a0fe 100644 --- a/pkg/iscp/runtime_cancel.go +++ b/pkg/iscp/runtime_cancel.go @@ -42,28 +42,19 @@ func UnregisterExecutorRuntime(cnUUID string, exec *ISCPTaskExecutor) { func GetExecutorRuntime(cnUUID string) (*ISCPTaskExecutor, bool) { v, ok := iscpExecutors.Load(cnUUID) if !ok { - var found *ISCPTaskExecutor - count := 0 - iscpExecutors.Range(func(_, v any) bool { - exec, ok := v.(*ISCPTaskExecutor) - if !ok { - return true - } - found = exec - count++ - return count < 2 - }) - if count == 1 { - return found, true - } return nil, false } exec, ok := v.(*ISCPTaskExecutor) return exec, ok } -func NewJobRuntimeKey(accountID uint32, tableID uint64, jobName string) JobRuntimeKey { - return JobRuntimeKey{AccountID: accountID, TableID: tableID, JobName: jobName} +func NewJobRuntimeKey(accountID uint32, tableID uint64, jobName string, jobID uint64) JobRuntimeKey { + return JobRuntimeKey{AccountID: accountID, TableID: tableID, JobName: jobName, JobID: jobID} +} + +func (key JobRuntimeKey) consumerGroupKey() JobRuntimeKey { + key.JobID = 0 + return key } func (exec *ISCPTaskExecutor) ensureRuntimeMapsLocked() { @@ -90,24 +81,31 @@ func (exec *ISCPTaskExecutor) CancelAndDrainJobConsumer( accountID uint32, tableID uint64, jobName string, + jobID uint64, ) error { if exec == nil { return nil } - key := NewJobRuntimeKey(accountID, tableID, jobName) + key := NewJobRuntimeKey(accountID, tableID, jobName, jobID) + groupKey := key.consumerGroupKey() cancelErr := moerr.NewInternalErrorNoCtx("iscp job consumer canceled") exec.runtimeMu.Lock() exec.ensureRuntimeMapsLocked() exec.fencedJobs[key] = struct{}{} - handles := make([]*RunningJobConsumer, 0, len(exec.runningConsumers[key])) - for _, h := range exec.runningConsumers[key] { - handles = append(handles, h) - if h.cancel != nil { - h.cancel() - } - if h.cancelRetriever != nil { - h.cancelRetriever(cancelErr) + handles := make([]*RunningJobConsumer, 0, 1) + if byJobID := exec.runningConsumers[groupKey]; byJobID != nil { + for runningJobID, h := range byJobID { + if jobID != 0 && runningJobID != jobID { + continue + } + handles = append(handles, h) + if h.cancel != nil { + h.cancel() + } + if h.cancelRetriever != nil { + h.cancelRetriever(cancelErr) + } } } exec.runtimeMu.Unlock() @@ -122,6 +120,15 @@ func (exec *ISCPTaskExecutor) CancelAndDrainJobConsumer( return nil } +func (exec *ISCPTaskExecutor) RemoveJobFence(key JobRuntimeKey) { + if exec == nil { + return + } + exec.runtimeMu.Lock() + delete(exec.fencedJobs, key) + exec.runtimeMu.Unlock() +} + func (exec *ISCPTaskExecutor) RegisterRunningConsumer( key JobRuntimeKey, jobID uint64, @@ -138,6 +145,7 @@ func (exec *ISCPTaskExecutor) RegisterRunningConsumer( if _, fenced := exec.fencedJobs[key]; fenced { return nil, false } + groupKey := key.consumerGroupKey() h := &RunningJobConsumer{ key: key, jobID: jobID, @@ -146,10 +154,10 @@ func (exec *ISCPTaskExecutor) RegisterRunningConsumer( cancelRetriever: cancelRetriever, done: make(chan struct{}), } - byJobID := exec.runningConsumers[key] + byJobID := exec.runningConsumers[groupKey] if byJobID == nil { byJobID = make(map[uint64]*RunningJobConsumer) - exec.runningConsumers[key] = byJobID + exec.runningConsumers[groupKey] = byJobID } byJobID[jobID] = h return h, true @@ -160,12 +168,13 @@ func (exec *ISCPTaskExecutor) UnregisterRunningConsumer(h *RunningJobConsumer) { return } exec.runtimeMu.Lock() - if byJobID := exec.runningConsumers[h.key]; byJobID != nil { + groupKey := h.key.consumerGroupKey() + if byJobID := exec.runningConsumers[groupKey]; byJobID != nil { if byJobID[h.jobID] == h { delete(byJobID, h.jobID) } if len(byJobID) == 0 { - delete(exec.runningConsumers, h.key) + delete(exec.runningConsumers, groupKey) } } exec.runtimeMu.Unlock() @@ -180,7 +189,7 @@ func (exec *ISCPTaskExecutor) filterFencedIteration(iter *IterationContext) *Ite keepJobIDs := make([]uint64, 0, len(iter.jobIDs)) keepLSN := make([]uint64, 0, len(iter.lsn)) for i, jobName := range iter.jobNames { - key := NewJobRuntimeKey(iter.accountID, iter.tableID, jobName) + key := NewJobRuntimeKey(iter.accountID, iter.tableID, jobName, iter.jobIDs[i]) if exec.IsJobFenced(key) { continue } diff --git a/pkg/iscp/runtime_cancel_test.go b/pkg/iscp/runtime_cancel_test.go index 2f76c99965c01..2674aeb809e7c 100644 --- a/pkg/iscp/runtime_cancel_test.go +++ b/pkg/iscp/runtime_cancel_test.go @@ -33,9 +33,23 @@ func newRuntimeTestExecutor() *ISCPTaskExecutor { } } +func TestGetExecutorRuntimeRequiresExactCN(t *testing.T) { + exec := newRuntimeTestExecutor() + RegisterExecutorRuntime("runner-cn", exec) + defer UnregisterExecutorRuntime("runner-cn", exec) + + found, ok := GetExecutorRuntime("ddl-cn") + require.False(t, ok) + require.Nil(t, found) + + found, ok = GetExecutorRuntime("runner-cn") + require.True(t, ok) + require.Same(t, exec, found) +} + func TestCancelAndDrainJobConsumerFencesAndWaitsForRunningConsumer(t *testing.T) { exec := newRuntimeTestExecutor() - key := NewJobRuntimeKey(1, 2, "index_idx01") + key := NewJobRuntimeKey(1, 2, "index_idx01", 7) consumerCtx, consumerCancel := context.WithCancel(context.Background()) defer consumerCancel() retrieverCanceled := make(chan error, 1) @@ -54,7 +68,7 @@ func TestCancelAndDrainJobConsumerFencesAndWaitsForRunningConsumer(t *testing.T) }() start := time.Now() - err := exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName) + err := exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID) require.NoError(t, err) require.GreaterOrEqual(t, time.Since(start), 20*time.Millisecond) require.True(t, exec.IsJobFenced(key)) @@ -64,9 +78,9 @@ func TestCancelAndDrainJobConsumerFencesAndWaitsForRunningConsumer(t *testing.T) func TestRegisterRunningConsumerRejectsFencedJob(t *testing.T) { exec := newRuntimeTestExecutor() - key := NewJobRuntimeKey(1, 2, "index_idx01") + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) - require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName)) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) _, ok := exec.RegisterRunningConsumer(key, 1, 1, func() {}, nil) require.False(t, ok) @@ -74,8 +88,8 @@ func TestRegisterRunningConsumerRejectsFencedJob(t *testing.T) { func TestCancelAndDrainJobConsumerOnlyCancelsMatchingJob(t *testing.T) { exec := newRuntimeTestExecutor() - key1 := NewJobRuntimeKey(1, 2, "index_idx01") - key2 := NewJobRuntimeKey(1, 2, "index_idx02") + key1 := NewJobRuntimeKey(1, 2, "index_idx01", 1) + key2 := NewJobRuntimeKey(1, 2, "index_idx02", 2) ctx1, cancel1 := context.WithCancel(context.Background()) ctx2, cancel2 := context.WithCancel(context.Background()) defer cancel2() @@ -89,7 +103,7 @@ func TestCancelAndDrainJobConsumerOnlyCancelsMatchingJob(t *testing.T) { exec.UnregisterRunningConsumer(h1) }() - require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key1.AccountID, key1.TableID, key1.JobName)) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key1.AccountID, key1.TableID, key1.JobName, key1.JobID)) select { case <-ctx2.Done(): @@ -97,7 +111,7 @@ func TestCancelAndDrainJobConsumerOnlyCancelsMatchingJob(t *testing.T) { default: } exec.runtimeMu.Lock() - _, stillRunning := exec.runningConsumers[key2][uint64(2)] + _, stillRunning := exec.runningConsumers[key2.consumerGroupKey()][uint64(2)] exec.runtimeMu.Unlock() require.True(t, stillRunning) } @@ -113,7 +127,7 @@ func TestFilterFencedIterationRemovesOnlyFencedJobs(t *testing.T) { types.BuildTS(1, 0), types.BuildTS(2, 0), ) - require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01")) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01", 10)) filtered := exec.filterFencedIteration(iter) @@ -136,7 +150,7 @@ func TestFilterFencedIterationDropsAllJobs(t *testing.T) { types.BuildTS(1, 0), types.BuildTS(2, 0), ) - require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01")) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01", 10)) require.Nil(t, exec.filterFencedIteration(iter)) } @@ -147,7 +161,7 @@ func TestTableEntryGetCandidateSkipsFencedJob(t *testing.T) { spec := &JobSpec{TriggerSpec: TriggerSpec{JobType: TriggerType_Default}} table.jobs[JobKey{JobName: "index_idx01", JobID: 1}] = NewJobEntry(table, "index_idx01", spec, 1, types.BuildTS(1, 0), ISCPJobState_Completed, 0) table.jobs[JobKey{JobName: "index_idx02", JobID: 2}] = NewJobEntry(table, "index_idx02", spec, 2, types.BuildTS(1, 0), ISCPJobState_Completed, 0) - require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01")) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01", 1)) iters, _ := table.getCandidate() @@ -155,16 +169,30 @@ func TestTableEntryGetCandidateSkipsFencedJob(t *testing.T) { require.Equal(t, []string{"index_idx02"}, iters[0].jobNames) } +func TestTableEntryGetCandidateDoesNotSkipRecreatedSameNameJob(t *testing.T) { + exec := newRuntimeTestExecutor() + table := NewTableEntry(exec, 1, 10, 2, "db", "tbl") + spec := &JobSpec{TriggerSpec: TriggerSpec{JobType: TriggerType_Default}} + table.jobs[JobKey{JobName: "index_idx01", JobID: 1}] = NewJobEntry(table, "index_idx01", spec, 1, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + table.jobs[JobKey{JobName: "index_idx01", JobID: 2}] = NewJobEntry(table, "index_idx01", spec, 2, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01", 1)) + + iters, _ := table.getCandidate() + + require.Len(t, iters, 1) + require.Equal(t, []uint64{2}, iters[0].jobIDs) +} + func TestCancelAndDrainJobConsumerHonorsCallerContext(t *testing.T) { exec := newRuntimeTestExecutor() - key := NewJobRuntimeKey(1, 2, "index_idx01") + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) consumerCancelCalled := make(chan struct{}) _, ok := exec.RegisterRunningConsumer(key, 1, 1, func() { close(consumerCancelCalled) }, nil) require.True(t, ok) ctx, cancel := context.WithCancel(context.Background()) cancel() - err := exec.CancelAndDrainJobConsumer(ctx, key.AccountID, key.TableID, key.JobName) + err := exec.CancelAndDrainJobConsumer(ctx, key.AccountID, key.TableID, key.JobName, key.JobID) require.True(t, errors.Is(err, context.Canceled)) <-consumerCancelCalled diff --git a/pkg/iscp/table_entry.go b/pkg/iscp/table_entry.go index 548049ccdd8dd..668128717c2e4 100644 --- a/pkg/iscp/table_entry.go +++ b/pkg/iscp/table_entry.go @@ -139,7 +139,7 @@ func (t *TableEntry) getCandidate() (iter []*IterationContext, minFromTS types.T if sinker.dropAt != 0 { continue } - if t.exec != nil && t.exec.IsJobFenced(NewJobRuntimeKey(t.accountID, t.tableID, sinker.jobName)) { + if t.exec != nil && t.exec.IsJobFenced(NewJobRuntimeKey(t.accountID, t.tableID, sinker.jobName, sinker.jobID)) { continue } candidates = append(candidates, sinker) diff --git a/pkg/iscp/types.go b/pkg/iscp/types.go index 2645fc071bb64..6a0638c7e37e5 100644 --- a/pkg/iscp/types.go +++ b/pkg/iscp/types.go @@ -167,6 +167,7 @@ type JobRuntimeKey struct { AccountID uint32 TableID uint64 JobName string + JobID uint64 } type RunningJobConsumer struct { diff --git a/pkg/iscp/watermark_updater.go b/pkg/iscp/watermark_updater.go index a8d679a68d07d..5a585eae23621 100644 --- a/pkg/iscp/watermark_updater.go +++ b/pkg/iscp/watermark_updater.go @@ -327,6 +327,41 @@ func UnregisterJob( ) } +func LookupJobLog( + ctx context.Context, + cnUUID string, + txn client.TxnOperator, + jobID *JobID, +) (accountID uint32, tableID uint64, internalJobID uint64, exists bool, dropped bool, err error) { + ctxWithSysAccount := context.WithValue(ctx, defines.TenantIDKey{}, catalog.System_Account) + ctxWithSysAccount, cancel := context.WithTimeout(ctxWithSysAccount, time.Minute*5) + defer cancel() + accountID, err = defines.GetAccountId(ctx) + if err != nil { + return + } + tableID, _, err = getTableID( + ctxWithSysAccount, + cnUUID, + txn, + accountID, + jobID.DBName, + jobID.TableName, + ) + if err != nil { + return + } + exists, dropped, internalJobID, err = queryIndexLog( + ctxWithSysAccount, + cnUUID, + txn, + accountID, + tableID, + jobID.JobName, + ) + return +} + func RenameSrcTable( ctx context.Context, cnUUID string, diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index 406b39fc1c2e0..0e600ff6eb8c2 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -726,7 +726,10 @@ func (s *Scope) AlterTableInplace(c *Compile) error { hasUpdateConstraints = true var notDroppedIndex []*plan.IndexDef var newIndexes []uint64 - if err = DrainIndexCdcTaskConsumer(c, oTableDef, constraintName); err != nil { + if err = DropIndexCdcTask(c, oTableDef, dbName, tblName, constraintName); err != nil { + return err + } + if err = DrainIndexCdcTaskConsumer(c, oTableDef, dbName, tblName, constraintName); err != nil { return err } for idx, indexdef := range oTableDef.Indexes { @@ -2504,7 +2507,11 @@ func (s *Scope) DropIndex(c *Compile) error { if err != nil { return err } - err = DrainIndexCdcTaskConsumer(c, oldTableDef, qry.IndexName) + err = DropIndexCdcTask(c, oldTableDef, qry.Database, qry.Table, qry.IndexName) + if err != nil { + return err + } + err = DrainIndexCdcTaskConsumer(c, oldTableDef, qry.Database, qry.Table, qry.IndexName) if err != nil { return err } @@ -2534,13 +2541,7 @@ func (s *Scope) DropIndex(c *Compile) error { } } - //3. delete iscp job for vector, fulltext index - err = DropIndexCdcTask(c, oldTableDef, qry.Database, qry.Table, qry.IndexName) - if err != nil { - return err - } - - // 4. unregister index update + // 3. unregister index update err = idxcron.UnregisterUpdate(c.proc.Ctx, c.proc.GetService(), c.proc.GetTxnOperator(), @@ -2551,7 +2552,7 @@ func (s *Scope) DropIndex(c *Compile) error { return err } - //5. delete index object from mo_catalog.mo_indexes + // 4. delete index object from mo_catalog.mo_indexes deleteSql := fmt.Sprintf(deleteMoIndexesWithTableIdAndIndexNameFormat, r.GetTableID(c.proc.Ctx), qry.IndexName) err = c.runSqlWithOptions( deleteSql, executor.StatementOption{}.WithDisableLog(), diff --git a/pkg/sql/compile/iscp_util.go b/pkg/sql/compile/iscp_util.go index 53a769332196c..470991a098e86 100644 --- a/pkg/sql/compile/iscp_util.go +++ b/pkg/sql/compile/iscp_util.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" indexplugin "github.com/matrixorigin/matrixone/pkg/indexplugin" @@ -32,6 +33,7 @@ import ( var ( iscpRegisterJobFunc = iscp.RegisterJob iscpUnregisterJobFunc = iscp.UnregisterJob + iscpLookupJobLogFunc = iscp.LookupJobLog iscpGetExecutorFunc = iscp.GetExecutorRuntime isTableInCCPRFunc = isTableInCCPRImpl ) @@ -194,7 +196,7 @@ func DropIndexCdcTask(c *Compile, tableDef *plan.TableDef, dbname string, tablen return nil } -func DrainIndexCdcTaskConsumer(c *Compile, tableDef *plan.TableDef, indexname string) error { +func DrainIndexCdcTaskConsumer(c *Compile, tableDef *plan.TableDef, dbname string, tablename string, indexname string) error { valid, err := checkValidIndexCdc(tableDef, indexname) if err != nil { return err @@ -202,18 +204,56 @@ func DrainIndexCdcTaskConsumer(c *Compile, tableDef *plan.TableDef, indexname st if !valid { return nil } - exec, ok := iscpGetExecutorFunc(c.proc.GetService()) - if !ok || exec == nil { - logutil.Infof("skip draining index cdc task consumer, iscp executor not found: tableID=%d index=%s", tableDef.TblId, indexname) - return nil - } accountID, err := defines.GetAccountId(c.proc.Ctx) if err != nil { return err } jobName := genCdcTaskJobID(indexname) - logutil.Infof("drain index cdc task consumer: accountID=%d tableID=%d jobName=%s", accountID, tableDef.TblId, jobName) - return exec.CancelAndDrainJobConsumer(c.proc.Ctx, accountID, tableDef.TblId, jobName) + _, tableID, jobID, exists, _, err := iscpLookupJobLogFunc( + c.proc.Ctx, + c.proc.GetService(), + c.proc.GetTxnOperator(), + &iscp.JobID{DBName: dbname, TableName: tablename, JobName: jobName}, + ) + if err != nil { + return err + } + if !exists { + logutil.Infof("skip draining index cdc task consumer, iscp job not found: tableID=%d index=%s", tableDef.TblId, indexname) + return nil + } + if tableID == 0 { + tableID = tableDef.TblId + } + exec, ok := iscpGetExecutorFunc(c.proc.GetService()) + if !ok || exec == nil { + return moerr.NewInternalErrorf( + c.proc.Ctx, + "cannot confirm ISCP consumer quiescence on CN %s for tableID=%d jobName=%s jobID=%d", + c.proc.GetService(), + tableID, + jobName, + jobID, + ) + } + key := iscp.NewJobRuntimeKey(accountID, tableID, jobName, jobID) + logutil.Infof("drain index cdc task consumer: accountID=%d tableID=%d jobName=%s jobID=%d", accountID, tableID, jobName, jobID) + if err := exec.CancelAndDrainJobConsumer(c.proc.Ctx, accountID, tableID, jobName, jobID); err != nil { + exec.RemoveJobFence(key) + return err + } + if txnOp := c.proc.GetTxnOperator(); txnOp != nil { + cleanup := client.NewTxnEventCallback(func(_ context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { + if !event.CostEvent { + return nil + } + exec.RemoveJobFence(key) + return nil + }) + txnOp.AppendEventCallback(client.CommitEvent, cleanup) + txnOp.AppendEventCallback(client.RollbackEvent, cleanup) + } + return nil } // drop all cdc tasks according to tableDef @@ -238,6 +278,9 @@ func DropAllIndexCdcTasks(c *Compile, tabledef *plan.TableDef, dbname string, ta if e != nil { return e } + if e = DrainIndexCdcTaskConsumer(c, tabledef, dbname, tablename, idx.IndexName); e != nil { + return e + } } } return nil diff --git a/pkg/sql/compile/iscp_util_extra_test.go b/pkg/sql/compile/iscp_util_extra_test.go index f9fac779c1f35..a7b7ca3bf2f17 100644 --- a/pkg/sql/compile/iscp_util_extra_test.go +++ b/pkg/sql/compile/iscp_util_extra_test.go @@ -18,6 +18,8 @@ import ( "context" "testing" + "github.com/golang/mock/gomock" + mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" "github.com/matrixorigin/matrixone/pkg/iscp" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/testutil" @@ -323,8 +325,12 @@ func TestDrainIndexCdcTaskConsumerFencesRuntimeJob(t *testing.T) { iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { return exec, true } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } defer func() { iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpLookupJobLogFunc = iscp.LookupJobLog }() c := &Compile{} @@ -341,18 +347,22 @@ func TestDrainIndexCdcTaskConsumerFencesRuntimeJob(t *testing.T) { }, } - err := DrainIndexCdcTaskConsumer(c, tbldef, "idx1") + err := DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1") require.NoError(t, err) - require.True(t, exec.IsJobFenced(iscp.NewJobRuntimeKey(0, 42, "index_idx1"))) + require.True(t, exec.IsJobFenced(iscp.NewJobRuntimeKey(0, 42, "index_idx1", 7))) } -func TestDrainIndexCdcTaskConsumerNoExecutorIsNoop(t *testing.T) { +func TestDrainIndexCdcTaskConsumerNoExecutorFailsClosed(t *testing.T) { iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { return nil, false } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } defer func() { iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpLookupJobLogFunc = iscp.LookupJobLog }() c := &Compile{} @@ -369,7 +379,58 @@ func TestDrainIndexCdcTaskConsumerNoExecutorIsNoop(t *testing.T) { }, } - require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "idx1")) + require.ErrorContains(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1"), "cannot confirm ISCP consumer quiescence") +} + +func TestDrainIndexCdcTaskConsumerRegistersRollbackFenceCleanup(t *testing.T) { + exec := &iscp.ISCPTaskExecutor{} + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return exec, true + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpLookupJobLogFunc = iscp.LookupJobLog + }() + + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + var rollbackCleanup client.TxnEventCallback + txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).Times(1) + txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + rollbackCleanup = cb + }, + ).Times(1) + + c := &Compile{} + c.proc = testutil.NewProcess(t) + c.proc.Base.TxnOperator = txnOp + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + key := iscp.NewJobRuntimeKey(0, 42, "index_idx1", 7) + require.True(t, exec.IsJobFenced(key)) + require.NotNil(t, rollbackCleanup.Func) + + require.NoError(t, rollbackCleanup.Func(context.Background(), txnOp, client.TxnEvent{}, nil)) + require.True(t, exec.IsJobFenced(key)) + require.NoError(t, rollbackCleanup.Func(context.Background(), txnOp, client.TxnEvent{CostEvent: true}, nil)) + require.False(t, exec.IsJobFenced(key)) + _, ok := exec.RegisterRunningConsumer(key, 7, 1, func() {}, nil) + require.True(t, ok) } func TestCoverage_DropAllIndexCdcTasks_DuplicateNames(t *testing.T) { @@ -378,8 +439,17 @@ func TestCoverage_DropAllIndexCdcTasks_DuplicateNames(t *testing.T) { dropCount++ return true, nil } + exec := &iscp.ISCPTaskExecutor{} + iscpGetExecutorFunc = func(string) (*iscp.ISCPTaskExecutor, bool) { + return exec, true + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, uint64(dropCount), true, true, nil + } defer func() { iscpUnregisterJobFunc = iscp.UnregisterJob + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpLookupJobLogFunc = iscp.LookupJobLog }() c := &Compile{} @@ -413,8 +483,17 @@ func TestCoverage_DropAllIndexCdcTasks_MixedIndexes(t *testing.T) { dropCount++ return true, nil } + exec := &iscp.ISCPTaskExecutor{} + iscpGetExecutorFunc = func(string) (*iscp.ISCPTaskExecutor, bool) { + return exec, true + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, uint64(dropCount), true, true, nil + } defer func() { iscpUnregisterJobFunc = iscp.UnregisterJob + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpLookupJobLogFunc = iscp.LookupJobLog }() c := &Compile{} diff --git a/pkg/util/fault/fault.go b/pkg/util/fault/fault.go index cbb4bb4bc45b6..cde32839861c3 100644 --- a/pkg/util/fault/fault.go +++ b/pkg/util/fault/fault.go @@ -84,6 +84,7 @@ type faultEntry struct { constant bool nWaiters int + removed bool mutex sync.Mutex cond *sync.Cond scope Domain @@ -117,7 +118,10 @@ func (fm *faultMap) run() { if e.name == "all" { for _, v := range fm.faultPoints { if v.action == WAIT && v.cond != nil { + v.mutex.Lock() + v.removed = true v.cond.Broadcast() + v.mutex.Unlock() } } fm.faultPoints = make(map[string]*faultEntry) @@ -126,7 +130,10 @@ func (fm *faultMap) run() { } if v, ok := fm.faultPoints[e.name]; ok { if v.action == WAIT && v.cond != nil { + v.mutex.Lock() + v.removed = true v.cond.Broadcast() + v.mutex.Unlock() } delete(fm.faultPoints, e.name) fm.chOut <- v @@ -168,9 +175,11 @@ func (e *faultEntry) do() (int64, string) { } case WAIT: e.mutex.Lock() - e.nWaiters += 1 - e.cond.Wait() - e.nWaiters -= 1 + if !e.removed { + e.nWaiters += 1 + e.cond.Wait() + e.nWaiters -= 1 + } e.mutex.Unlock() case GETWAITERS: if ee := lookup(e.scope, e.sarg); ee != nil { diff --git a/pkg/util/fault/fault_test.go b/pkg/util/fault/fault_test.go index 5cee62feb52e3..79e3afc88af75 100644 --- a/pkg/util/fault/fault_test.go +++ b/pkg/util/fault/fault_test.go @@ -216,6 +216,33 @@ func TestRemoveWaitFaultPointNotifiesWaiters(t *testing.T) { }, time.Second, 10*time.Millisecond) } +func TestRemoveWaitFaultPointBeforeWaitRegistrationDoesNotHang(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + entry := lookup(DomainDefault, "w") + require.NotNil(t, entry) + + removed, err := RemoveFaultPoint(ctx, "w") + require.NoError(t, err) + require.True(t, removed) + + done := make(chan struct{}) + go func() { + entry.do() + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("wait fault point blocked after it was removed before waiter registration") + } +} + func TestTriggerWaitFaultWithContextReturnsOnCancel(t *testing.T) { ctx := context.Background() From 526420654710d8860488bb44f2545d4357551c3b Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Wed, 15 Jul 2026 15:31:37 +0800 Subject: [PATCH 04/14] fix: unblock removed context fault waits --- pkg/util/fault/fault.go | 4 ++++ pkg/util/fault/fault_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/pkg/util/fault/fault.go b/pkg/util/fault/fault.go index cde32839861c3..0fcf71b7c3b70 100644 --- a/pkg/util/fault/fault.go +++ b/pkg/util/fault/fault.go @@ -223,6 +223,10 @@ func (e *faultEntry) doWithContext(ctx context.Context) (int64, string) { return e.do() } e.mutex.Lock() + if e.removed { + e.mutex.Unlock() + return 0, "" + } e.nWaiters += 1 done := make(chan struct{}) go func() { diff --git a/pkg/util/fault/fault_test.go b/pkg/util/fault/fault_test.go index 79e3afc88af75..a9f4ee3fa135f 100644 --- a/pkg/util/fault/fault_test.go +++ b/pkg/util/fault/fault_test.go @@ -243,6 +243,39 @@ func TestRemoveWaitFaultPointBeforeWaitRegistrationDoesNotHang(t *testing.T) { } } +func TestRemoveWaitFaultPointBeforeContextWaitRegistrationDoesNotHang(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + fm := enabled[DomainDefault].Load() + require.NotNil(t, fm) + + fm.chIn <- &faultEntry{cmd: TRIGGER, name: "w"} + entry := <-fm.chOut + require.NotNil(t, entry) + + removed, err := RemoveFaultPoint(ctx, "w") + require.NoError(t, err) + require.True(t, removed) + + waitCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan struct{}) + go func() { + entry.doWithContext(waitCtx) + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("context-aware wait fault point blocked after it was removed before waiter registration") + } +} + func TestTriggerWaitFaultWithContextReturnsOnCancel(t *testing.T) { ctx := context.Background() From 8d6cb35feefb8ee91828f87a30d7f6d398c0a4d3 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 16 Jul 2026 10:39:16 +0800 Subject: [PATCH 05/14] fix iscp cancel finalization --- pkg/cdc/sql_builder.go | 35 +++++- pkg/iscp/executor.go | 41 ++++++- pkg/iscp/runtime_cancel.go | 146 ++++++++++++++++++++++++ pkg/iscp/runtime_cancel_test.go | 95 +++++++++++++++ pkg/iscp/types.go | 3 + pkg/iscp/watermark_updater.go | 1 + pkg/objectio/injects.go | 2 + pkg/sql/compile/iscp_util.go | 23 +++- pkg/sql/compile/iscp_util_extra_test.go | 36 ++++++ 9 files changed, 376 insertions(+), 6 deletions(-) diff --git a/pkg/cdc/sql_builder.go b/pkg/cdc/sql_builder.go index a2fd3cad10d83..1c9fa43c8ff08 100644 --- a/pkg/cdc/sql_builder.go +++ b/pkg/cdc/sql_builder.go @@ -282,7 +282,7 @@ const ( `AND table_id = %d ` + `AND job_name = '%s'` + `AND job_id = %d ` + - `AND job_state != 4 ` + + `AND job_state NOT IN (4, 5, 6) ` + `AND JSON_EXTRACT(job_status, '$.LSN') = '%d'` CDCUpdateMOISCPLogJobSpecSqlTemplate = `UPDATE mo_catalog.mo_iscp_log SET ` + `job_spec = '%s'` + @@ -292,12 +292,20 @@ const ( `AND job_name = '%s'` + `AND job_id = %d` CDCUpdateMOISCPLogDropAtSqlTemplate = `UPDATE mo_catalog.mo_iscp_log SET ` + + `job_state = %d,` + `drop_at = now()` + `WHERE` + ` account_id = %d ` + `AND table_id = %d ` + `AND job_name = '%s'` + `AND job_id = %d` + CDCUpdateMOISCPLogStateSqlTemplate = `UPDATE mo_catalog.mo_iscp_log SET ` + + `job_state = %d ` + + `WHERE` + + ` account_id = %d ` + + `AND table_id = %d ` + + `AND job_name = '%s'` + + `AND job_id = %d` CDCDeleteMOISCPLogSqlTemplate = `DELETE FROM mo_catalog.mo_iscp_log WHERE ` + `drop_at < '%s'` CDCSelectMOISCPLogSqlTemplate = `SELECT * from mo_catalog.mo_iscp_log` @@ -348,8 +356,9 @@ const ( CDCGetTableIDTemplate_Idx = 29 CDCUpdateTaskStateByTaskIdSQL_Idx = 30 CDCUpdateTaskStateByTaskIdAndStateSQL_Idx = 31 + CDCUpdateMOISCPLogStateSqlTemplate_Idx = 32 - CDCSqlTemplateCount = 32 + CDCSqlTemplateCount = 33 ) var CDCSQLTemplates = [CDCSqlTemplateCount]struct { @@ -478,6 +487,9 @@ var CDCSQLTemplates = [CDCSqlTemplateCount]struct { CDCUpdateMOISCPLogDropAtSqlTemplate_Idx: { SQL: CDCUpdateMOISCPLogDropAtSqlTemplate, }, + CDCUpdateMOISCPLogStateSqlTemplate_Idx: { + SQL: CDCUpdateMOISCPLogStateSqlTemplate, + }, CDCDeleteMOISCPLogSqlTemplate_Idx: { SQL: CDCDeleteMOISCPLogSqlTemplate, }, @@ -961,9 +973,28 @@ func (b cdcSQLBuilder) ISCPLogUpdateDropAtSQL( tableID uint64, jobName string, jobID uint64, + jobState int8, ) string { return fmt.Sprintf( CDCSQLTemplates[CDCUpdateMOISCPLogDropAtSqlTemplate_Idx].SQL, + jobState, + accountID, + tableID, + jobName, + jobID, + ) +} + +func (b cdcSQLBuilder) ISCPLogUpdateStateSQL( + accountID uint32, + tableID uint64, + jobName string, + jobID uint64, + jobState int8, +) string { + return fmt.Sprintf( + CDCSQLTemplates[CDCUpdateMOISCPLogStateSqlTemplate_Idx].SQL, + jobState, accountID, tableID, jobName, diff --git a/pkg/iscp/executor.go b/pkg/iscp/executor.go index 4b5fc2538cb9b..2b4cf1aa5115c 100644 --- a/pkg/iscp/executor.go +++ b/pkg/iscp/executor.go @@ -60,6 +60,7 @@ const ( DefaultSyncTaskInterval = time.Second * 10 DefaultFlushWatermarkInterval = time.Hour DefaultFlushWatermarkTTL = time.Hour + DefaultCancelJobQueueSize = 1024 DefaultRetryTimes = 5 DefaultRetryInterval = time.Second @@ -191,6 +192,8 @@ func NewISCPTaskExecutor( runningConsumers: make( map[JobRuntimeKey]map[uint64]*RunningJobConsumer, ), + cancelJobs: make(chan JobRuntimeKey, DefaultCancelJobQueueSize), + cancelingJobs: make(map[JobRuntimeKey]struct{}), } return exec, nil } @@ -380,7 +383,7 @@ func (exec *ISCPTaskExecutor) stopLocked() { func (exec *ISCPTaskExecutor) repairAbandonedJobs(ctx context.Context) (err error) { ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Minute*5) defer cancel() - sql := fmt.Sprintf( + activeJobsSQL := fmt.Sprintf( `UPDATE mo_catalog.mo_iscp_log SET job_state = %d WHERE job_state IN (%d, %d); @@ -389,6 +392,14 @@ func (exec *ISCPTaskExecutor) repairAbandonedJobs(ctx context.Context) (err erro ISCPJobState_Pending, ISCPJobState_Running, ) + droppedJobsSQL := fmt.Sprintf( + `UPDATE mo_catalog.mo_iscp_log + SET job_state = %d + WHERE drop_at IS NOT NULL AND job_state != %d; + `, + ISCPJobState_Canceled, + ISCPJobState_Canceled, + ) nowTs := exec.txnEngine.LatestLogtailAppliedTime() createByOpt := client.WithTxnCreateBy( 0, @@ -412,11 +423,16 @@ func (exec *ISCPTaskExecutor) repairAbandonedJobs(ctx context.Context) (err erro if err != nil { return } - result, err := ExecWithResult(ctxWithTimeout, sql, exec.cnUUID, txnOp) + result, err := ExecWithResult(ctxWithTimeout, activeJobsSQL, exec.cnUUID, txnOp) if err != nil { return } - defer result.Close() + result.Close() + result, err = ExecWithResult(ctxWithTimeout, droppedJobsSQL, exec.cnUUID, txnOp) + if err != nil { + return + } + result.Close() return nil } @@ -446,6 +462,18 @@ func (exec *ISCPTaskExecutor) run(ctx context.Context, worker Worker) { select { case <-ctx.Done(): return + case key := <-exec.cancelJobs: + if err := exec.finishCanceledJob(ctx, key); err != nil { + logutil.Error( + "ISCP-Task cancel job failed", + zap.Uint32("accountID", key.AccountID), + zap.Uint64("tableID", key.TableID), + zap.String("jobName", key.JobName), + zap.Uint64("jobID", key.JobID), + zap.Error(err), + ) + exec.requeueCancelJob(ctx, key) + } case <-syncTaskTrigger.C: // apply iscp log from := exec.iscpLogWm.Next() @@ -908,6 +936,9 @@ func (exec *ISCPTaskExecutor) addOrUpdateRecoveredJob( if state == ISCPJobState_Pending || state == ISCPJobState_Running { state = ISCPJobState_Completed } + if dropAt != 0 && state != ISCPJobState_Canceled { + state = ISCPJobState_Canceled + } return exec.addOrUpdateJob( accountID, tableID, @@ -972,6 +1003,7 @@ func (exec *ISCPTaskExecutor) addOrUpdateJob( notPrint bool, ) error { var newCreate bool + key := NewJobRuntimeKey(accountID, tableID, jobName, jobID) var watermark types.TS defer func() { @@ -998,6 +1030,9 @@ func (exec *ISCPTaskExecutor) addOrUpdateJob( if err != nil { return err } + if dropAt != 0 && state != ISCPJobState_Canceled { + exec.enqueueCancelJob(key) + } var table *TableEntry table, ok := exec.getTable(accountID, tableID) if !ok { diff --git a/pkg/iscp/runtime_cancel.go b/pkg/iscp/runtime_cancel.go index 9e81bfa65a0fe..343c777547b2c 100644 --- a/pkg/iscp/runtime_cancel.go +++ b/pkg/iscp/runtime_cancel.go @@ -16,9 +16,17 @@ package iscp import ( "context" + "errors" "sync" + "time" + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/cdc" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/util/fault" ) var iscpExecutors sync.Map // map[cnUUID]*ISCPTaskExecutor @@ -64,6 +72,9 @@ func (exec *ISCPTaskExecutor) ensureRuntimeMapsLocked() { if exec.runningConsumers == nil { exec.runningConsumers = make(map[JobRuntimeKey]map[uint64]*RunningJobConsumer) } + if exec.cancelingJobs == nil { + exec.cancelingJobs = make(map[JobRuntimeKey]struct{}) + } } func (exec *ISCPTaskExecutor) IsJobFenced(key JobRuntimeKey) bool { @@ -120,6 +131,141 @@ func (exec *ISCPTaskExecutor) CancelAndDrainJobConsumer( return nil } +func (exec *ISCPTaskExecutor) enqueueCancelJob(key JobRuntimeKey) { + if exec == nil { + return + } + exec.runtimeMu.Lock() + exec.ensureRuntimeMapsLocked() + if _, ok := exec.cancelingJobs[key]; ok { + exec.runtimeMu.Unlock() + return + } + exec.cancelingJobs[key] = struct{}{} + cancelJobs := exec.cancelJobs + exec.runtimeMu.Unlock() + + if cancelJobs == nil { + exec.clearCancelingJob(key) + return + } + select { + case cancelJobs <- key: + default: + go func() { + ctx := exec.ctx + if ctx == nil { + exec.clearCancelingJob(key) + return + } + select { + case cancelJobs <- key: + case <-ctx.Done(): + exec.clearCancelingJob(key) + } + }() + } +} + +func (exec *ISCPTaskExecutor) requeueCancelJob(ctx context.Context, key JobRuntimeKey) { + if exec == nil { + return + } + go func() { + timer := time.NewTimer(DefaultRetryInterval) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + exec.clearCancelingJob(key) + return + } + cancelJobs := exec.cancelJobs + if cancelJobs == nil { + exec.clearCancelingJob(key) + return + } + select { + case cancelJobs <- key: + case <-ctx.Done(): + exec.clearCancelingJob(key) + } + }() +} + +func (exec *ISCPTaskExecutor) clearCancelingJob(key JobRuntimeKey) { + exec.runtimeMu.Lock() + delete(exec.cancelingJobs, key) + exec.runtimeMu.Unlock() +} + +func (exec *ISCPTaskExecutor) finishCanceledJob(ctx context.Context, key JobRuntimeKey) error { + if exec == nil { + return nil + } + if err := exec.CancelAndDrainJobConsumer(ctx, key.AccountID, key.TableID, key.JobName, key.JobID); err != nil { + return err + } + if err := exec.markJobCanceled(ctx, key); err != nil { + return err + } + exec.clearCancelingJob(key) + return nil +} + +func (exec *ISCPTaskExecutor) markJobCanceled(ctx context.Context, key JobRuntimeKey) (err error) { + ctx = context.WithValue(ctx, defines.TenantIDKey{}, catalog.System_Account) + ctx, cancel := context.WithTimeoutCause(ctx, time.Minute*5, moerr.NewInternalErrorNoCtx("iscp cancel job timeout")) + defer cancel() + if _, _, injected := fault.TriggerFaultWithContext(ctx, objectio.FJ_ISCPCancelBeforeMarkCanceled); injected { + logutil.Infof( + "ISCP-Task cancel fault triggered %s: accountID=%d tableID=%d jobName=%s jobID=%d", + objectio.FJ_ISCPCancelBeforeMarkCanceled, + key.AccountID, + key.TableID, + key.JobName, + key.JobID, + ) + } + if _, msg, injected := fault.TriggerFault(objectio.FJ_ISCPCancelMarkCanceledError); injected { + if msg == "" { + msg = objectio.FJ_ISCPCancelMarkCanceledError + } + return moerr.NewInternalErrorNoCtxf("injected ISCP cancel mark error: %s", msg) + } + txnOp, err := getTxn(ctx, exec.txnEngine, exec.cnTxnClient, "iscp cancel job") + if err != nil { + return err + } + defer func() { + if err != nil { + err = errors.Join(err, txnOp.Rollback(ctx)) + return + } + err = txnOp.Commit(ctx) + }() + sql := cdc.CDCSQLBuilder.ISCPLogUpdateStateSQL( + key.AccountID, + key.TableID, + key.JobName, + key.JobID, + ISCPJobState_Canceled, + ) + result, err := ExecWithResult(ctx, sql, exec.cnUUID, txnOp) + if err != nil { + return err + } + result.Close() + logutil.Infof( + "ISCP-Task marked job canceled: accountID=%d tableID=%d jobName=%s jobID=%d", + key.AccountID, + key.TableID, + key.JobName, + key.JobID, + ) + return nil +} + func (exec *ISCPTaskExecutor) RemoveJobFence(key JobRuntimeKey) { if exec == nil { return diff --git a/pkg/iscp/runtime_cancel_test.go b/pkg/iscp/runtime_cancel_test.go index 2674aeb809e7c..de2d9dc906253 100644 --- a/pkg/iscp/runtime_cancel_test.go +++ b/pkg/iscp/runtime_cancel_test.go @@ -28,11 +28,23 @@ func newRuntimeTestExecutor() *ISCPTaskExecutor { return &ISCPTaskExecutor{ fencedJobs: make(map[JobRuntimeKey]struct{}), runningConsumers: make(map[JobRuntimeKey]map[uint64]*RunningJobConsumer), + cancelJobs: make(chan JobRuntimeKey, DefaultCancelJobQueueSize), + cancelingJobs: make(map[JobRuntimeKey]struct{}), option: fillDefaultOption(nil), ctx: context.Background(), + tables: newISCPTableTree(), } } +func encodeRuntimeCancelTestJSON(t *testing.T, value string) []byte { + t.Helper() + byteJSON, err := types.ParseStringToByteJson(value) + require.NoError(t, err) + encoded, err := types.EncodeJson(byteJSON) + require.NoError(t, err) + return encoded +} + func TestGetExecutorRuntimeRequiresExactCN(t *testing.T) { exec := newRuntimeTestExecutor() RegisterExecutorRuntime("runner-cn", exec) @@ -197,3 +209,86 @@ func TestCancelAndDrainJobConsumerHonorsCallerContext(t *testing.T) { require.True(t, errors.Is(err, context.Canceled)) <-consumerCancelCalled } + +func TestDroppedJobIsQueuedForCancel(t *testing.T) { + exec := newRuntimeTestExecutor() + table := NewTableEntry(exec, 1, 10, 2, "db", "tbl") + exec.setTable(table) + spec := &JobSpec{ + ConsumerInfo: ConsumerInfo{ + SrcTable: TableInfo{DBID: 10, TableID: 2, DBName: "db", TableName: "tbl"}, + }, + TriggerSpec: TriggerSpec{JobType: TriggerType_Default}, + } + specJSON, err := MarshalJobSpec(spec) + require.NoError(t, err) + statusJSON, err := MarshalJobStatus(&JobStatus{}) + require.NoError(t, err) + dropAt := types.Timestamp(time.Now().UnixNano()) + + err = exec.addOrUpdateJob( + 1, + 2, + "index_idx01", + 7, + ISCPJobState_Canceling, + types.BuildTS(1, 0).ToString(), + encodeRuntimeCancelTestJSON(t, specJSON), + encodeRuntimeCancelTestJSON(t, statusJSON), + dropAt, + true, + ) + require.NoError(t, err) + + key := NewJobRuntimeKey(1, 2, "index_idx01", 7) + require.Eventually(t, func() bool { + select { + case got := <-exec.cancelJobs: + return got == key + default: + return false + } + }, time.Second, 10*time.Millisecond) + exec.runtimeMu.Lock() + _, queued := exec.cancelingJobs[key] + exec.runtimeMu.Unlock() + require.True(t, queued) +} + +func TestRecoveredDroppedJobBecomesCanceledWithoutQueue(t *testing.T) { + exec := newRuntimeTestExecutor() + table := NewTableEntry(exec, 1, 10, 2, "db", "tbl") + exec.setTable(table) + spec := &JobSpec{ + ConsumerInfo: ConsumerInfo{ + SrcTable: TableInfo{DBID: 10, TableID: 2, DBName: "db", TableName: "tbl"}, + }, + TriggerSpec: TriggerSpec{JobType: TriggerType_Default}, + } + specJSON, err := MarshalJobSpec(spec) + require.NoError(t, err) + statusJSON, err := MarshalJobStatus(&JobStatus{}) + require.NoError(t, err) + dropAt := types.Timestamp(time.Now().UnixNano()) + + err = exec.addOrUpdateRecoveredJob( + 1, + 2, + "index_idx01", + 7, + ISCPJobState_Canceling, + types.BuildTS(1, 0).ToString(), + encodeRuntimeCancelTestJSON(t, specJSON), + encodeRuntimeCancelTestJSON(t, statusJSON), + dropAt, + true, + ) + require.NoError(t, err) + + select { + case got := <-exec.cancelJobs: + t.Fatalf("recovered dropped job should not be queued, got %+v", got) + default: + } + require.Equal(t, int8(ISCPJobState_Canceled), table.jobs[JobKey{JobName: "index_idx01", JobID: 7}].state) +} diff --git a/pkg/iscp/types.go b/pkg/iscp/types.go index 2fdfc19105496..2e1e3e4ee1179 100644 --- a/pkg/iscp/types.go +++ b/pkg/iscp/types.go @@ -49,6 +49,7 @@ const ( ISCPJobState_Completed ISCPJobState_Error ISCPJobState_Canceled + ISCPJobState_Canceling ) type DataRetriever interface { @@ -163,6 +164,8 @@ type ISCPTaskExecutor struct { runtimeMu sync.Mutex fencedJobs map[JobRuntimeKey]struct{} runningConsumers map[JobRuntimeKey]map[uint64]*RunningJobConsumer + cancelJobs chan JobRuntimeKey + cancelingJobs map[JobRuntimeKey]struct{} } type JobRuntimeKey struct { diff --git a/pkg/iscp/watermark_updater.go b/pkg/iscp/watermark_updater.go index 5a585eae23621..ac29c6e39345f 100644 --- a/pkg/iscp/watermark_updater.go +++ b/pkg/iscp/watermark_updater.go @@ -629,6 +629,7 @@ func unregisterJob( tableID, jobID.JobName, internalJobID, + ISCPJobState_Canceling, ) result, err := ExecWithResult(ctxWithSysAccount, sql, cnUUID, txn) if err != nil { diff --git a/pkg/objectio/injects.go b/pkg/objectio/injects.go index 0f0fa761d75ac..a2d00c07092be 100644 --- a/pkg/objectio/injects.go +++ b/pkg/objectio/injects.go @@ -77,6 +77,8 @@ const ( FJ_ISCPCancelLongHnswBeforeUpdate = "fj/iscp/cancel/long/hnsw-before-update" FJ_ISCPCancelLongHnswBeforeSave = "fj/iscp/cancel/long/hnsw-before-save" FJ_ISCPCancelLongBeforeWatermark = "fj/iscp/cancel/long/before-watermark" + FJ_ISCPCancelBeforeMarkCanceled = "fj/iscp/cancel/before-mark-canceled" + FJ_ISCPCancelMarkCanceledError = "fj/iscp/cancel/mark-canceled-error" FJ_PublicationSnapshotFinished = "fj/publication/snapshot/finished" diff --git a/pkg/sql/compile/iscp_util.go b/pkg/sql/compile/iscp_util.go index 470991a098e86..8d8f6a0a55be6 100644 --- a/pkg/sql/compile/iscp_util.go +++ b/pkg/sql/compile/iscp_util.go @@ -197,6 +197,17 @@ func DropIndexCdcTask(c *Compile, tableDef *plan.TableDef, dbname string, tablen } func DrainIndexCdcTaskConsumer(c *Compile, tableDef *plan.TableDef, dbname string, tablename string, indexname string) error { + return drainIndexCdcTaskConsumer(c, tableDef, dbname, tablename, indexname, true) +} + +func drainIndexCdcTaskConsumer( + c *Compile, + tableDef *plan.TableDef, + dbname string, + tablename string, + indexname string, + failIfNoLocalExecutor bool, +) error { valid, err := checkValidIndexCdc(tableDef, indexname) if err != nil { return err @@ -227,6 +238,16 @@ func DrainIndexCdcTaskConsumer(c *Compile, tableDef *plan.TableDef, dbname strin } exec, ok := iscpGetExecutorFunc(c.proc.GetService()) if !ok || exec == nil { + if !failIfNoLocalExecutor { + logutil.Infof( + "skip local drain for index cdc task consumer, iscp executor not found: cn=%s tableID=%d jobName=%s jobID=%d", + c.proc.GetService(), + tableID, + jobName, + jobID, + ) + return nil + } return moerr.NewInternalErrorf( c.proc.Ctx, "cannot confirm ISCP consumer quiescence on CN %s for tableID=%d jobName=%s jobID=%d", @@ -278,7 +299,7 @@ func DropAllIndexCdcTasks(c *Compile, tabledef *plan.TableDef, dbname string, ta if e != nil { return e } - if e = DrainIndexCdcTaskConsumer(c, tabledef, dbname, tablename, idx.IndexName); e != nil { + if e = drainIndexCdcTaskConsumer(c, tabledef, dbname, tablename, idx.IndexName, false); e != nil { return e } } diff --git a/pkg/sql/compile/iscp_util_extra_test.go b/pkg/sql/compile/iscp_util_extra_test.go index a7b7ca3bf2f17..e5e46bd27cee5 100644 --- a/pkg/sql/compile/iscp_util_extra_test.go +++ b/pkg/sql/compile/iscp_util_extra_test.go @@ -477,6 +477,42 @@ func TestCoverage_DropAllIndexCdcTasks_DuplicateNames(t *testing.T) { assert.Equal(t, 1, dropCount, "duplicate index names should be deduplicated") } +func TestDropAllIndexCdcTasksNoLocalExecutorContinuesAfterUnregister(t *testing.T) { + dropCount := 0 + iscpUnregisterJobFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (bool, error) { + dropCount++ + return true, nil + } + iscpGetExecutorFunc = func(string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 1, true, true, nil + } + defer func() { + iscpUnregisterJobFunc = iscp.UnregisterJob + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpLookupJobLogFunc = iscp.LookupJobLog + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DropAllIndexCdcTasks(c, tbldef, "db", "tbl")) + require.Equal(t, 1, dropCount) +} + func TestCoverage_DropAllIndexCdcTasks_MixedIndexes(t *testing.T) { dropCount := 0 iscpUnregisterJobFunc = func(ctx context.Context, cnUUID string, txn client.TxnOperator, job *iscp.JobID) (bool, error) { From 76ddc86855d9c5f1fc376c78e2e827f05077ad79 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 16 Jul 2026 11:26:58 +0800 Subject: [PATCH 06/14] fix drop index iscp cancel path --- pkg/sql/compile/iscp_util.go | 2 +- pkg/sql/compile/iscp_util_extra_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/sql/compile/iscp_util.go b/pkg/sql/compile/iscp_util.go index 8d8f6a0a55be6..cb993ec1684d7 100644 --- a/pkg/sql/compile/iscp_util.go +++ b/pkg/sql/compile/iscp_util.go @@ -197,7 +197,7 @@ func DropIndexCdcTask(c *Compile, tableDef *plan.TableDef, dbname string, tablen } func DrainIndexCdcTaskConsumer(c *Compile, tableDef *plan.TableDef, dbname string, tablename string, indexname string) error { - return drainIndexCdcTaskConsumer(c, tableDef, dbname, tablename, indexname, true) + return drainIndexCdcTaskConsumer(c, tableDef, dbname, tablename, indexname, false) } func drainIndexCdcTaskConsumer( diff --git a/pkg/sql/compile/iscp_util_extra_test.go b/pkg/sql/compile/iscp_util_extra_test.go index e5e46bd27cee5..1e67e2bd0e431 100644 --- a/pkg/sql/compile/iscp_util_extra_test.go +++ b/pkg/sql/compile/iscp_util_extra_test.go @@ -353,7 +353,7 @@ func TestDrainIndexCdcTaskConsumerFencesRuntimeJob(t *testing.T) { require.True(t, exec.IsJobFenced(iscp.NewJobRuntimeKey(0, 42, "index_idx1", 7))) } -func TestDrainIndexCdcTaskConsumerNoExecutorFailsClosed(t *testing.T) { +func TestDrainIndexCdcTaskConsumerNoLocalExecutorContinues(t *testing.T) { iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { return nil, false } @@ -379,7 +379,7 @@ func TestDrainIndexCdcTaskConsumerNoExecutorFailsClosed(t *testing.T) { }, } - require.ErrorContains(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1"), "cannot confirm ISCP consumer quiescence") + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) } func TestDrainIndexCdcTaskConsumerRegistersRollbackFenceCleanup(t *testing.T) { From 4ad3702debf6b12383b81e563ca40604a3ff0b2c Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 16 Jul 2026 14:08:50 +0800 Subject: [PATCH 07/14] fix iscp runner drain and fence cleanup --- pkg/iscp/executor.go | 1 + pkg/iscp/runtime_cancel.go | 14 +++++ pkg/iscp/runtime_cancel_test.go | 16 ++++++ pkg/iscp/util.go | 25 ++++++--- pkg/sql/compile/iscp_util.go | 27 +++++----- pkg/sql/compile/iscp_util_extra_test.go | 68 +++++++++++++++++++++++-- 6 files changed, 126 insertions(+), 25 deletions(-) diff --git a/pkg/iscp/executor.go b/pkg/iscp/executor.go index 2b4cf1aa5115c..01a6a217d6cc3 100644 --- a/pkg/iscp/executor.go +++ b/pkg/iscp/executor.go @@ -1065,6 +1065,7 @@ func (exec *ISCPTaskExecutor) GCInMemoryJob(threshold time.Duration) { tids := make([]uint64, 0, len(tablesToDelete)) for _, table := range tablesToDelete { exec.deleteTableEntry(table) + exec.RemoveTableJobFences(table.accountID, table.tableID) tids = append(tids, table.tableID) } logutil.Infof("ISCP-Task delete table %v", tids) diff --git a/pkg/iscp/runtime_cancel.go b/pkg/iscp/runtime_cancel.go index 343c777547b2c..4c12273c16aee 100644 --- a/pkg/iscp/runtime_cancel.go +++ b/pkg/iscp/runtime_cancel.go @@ -210,6 +210,7 @@ func (exec *ISCPTaskExecutor) finishCanceledJob(ctx context.Context, key JobRunt return err } exec.clearCancelingJob(key) + exec.RemoveJobFence(key) return nil } @@ -275,6 +276,19 @@ func (exec *ISCPTaskExecutor) RemoveJobFence(key JobRuntimeKey) { exec.runtimeMu.Unlock() } +func (exec *ISCPTaskExecutor) RemoveTableJobFences(accountID uint32, tableID uint64) { + if exec == nil { + return + } + exec.runtimeMu.Lock() + for key := range exec.fencedJobs { + if key.AccountID == accountID && key.TableID == tableID { + delete(exec.fencedJobs, key) + } + } + exec.runtimeMu.Unlock() +} + func (exec *ISCPTaskExecutor) RegisterRunningConsumer( key JobRuntimeKey, jobID uint64, diff --git a/pkg/iscp/runtime_cancel_test.go b/pkg/iscp/runtime_cancel_test.go index de2d9dc906253..b15ad2d900f38 100644 --- a/pkg/iscp/runtime_cancel_test.go +++ b/pkg/iscp/runtime_cancel_test.go @@ -195,6 +195,22 @@ func TestTableEntryGetCandidateDoesNotSkipRecreatedSameNameJob(t *testing.T) { require.Equal(t, []uint64{2}, iters[0].jobIDs) } +func TestRemoveTableJobFencesOnlyClearsMatchingTable(t *testing.T) { + exec := newRuntimeTestExecutor() + key1 := NewJobRuntimeKey(1, 2, "index_idx01", 1) + key2 := NewJobRuntimeKey(1, 2, "index_idx01", 2) + keyOtherTable := NewJobRuntimeKey(1, 3, "index_idx01", 1) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key1.AccountID, key1.TableID, key1.JobName, key1.JobID)) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key2.AccountID, key2.TableID, key2.JobName, key2.JobID)) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), keyOtherTable.AccountID, keyOtherTable.TableID, keyOtherTable.JobName, keyOtherTable.JobID)) + + exec.RemoveTableJobFences(1, 2) + + require.False(t, exec.IsJobFenced(key1)) + require.False(t, exec.IsJobFenced(key2)) + require.True(t, exec.IsJobFenced(keyOtherTable)) +} + func TestCancelAndDrainJobConsumerHonorsCallerContext(t *testing.T) { exec := newRuntimeTestExecutor() key := NewJobRuntimeKey(1, 2, "index_idx01", 1) diff --git a/pkg/iscp/util.go b/pkg/iscp/util.go index d97795657bb34..3532892e4c207 100644 --- a/pkg/iscp/util.go +++ b/pkg/iscp/util.go @@ -441,14 +441,8 @@ func checkLease( } defer txn.Commit(ctxWithTimeout) - sql := `select task_runner from mo_task.sys_daemon_task where task_type = "ISCP" and task_runner is not null` - result, err := ExecWithResult(ctxWithTimeout, sql, cnUUID, txn) - if err != nil { - return - } - defer result.Close() var runner string - runner, err = readSingleTaskRunner(result) + runner, err = GetTaskRunner(ctxWithTimeout, cnUUID, txn) if err != nil { return } @@ -467,6 +461,23 @@ func checkLease( return } +func GetTaskRunner( + ctx context.Context, + cnUUID string, + txn client.TxnOperator, +) (string, error) { + ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, time.Minute*5, moerr.NewInternalErrorNoCtx("iscp get task runner timeout")) + defer cancel() + + sql := `select task_runner from mo_task.sys_daemon_task where task_type = "ISCP" and task_runner is not null` + result, err := ExecWithResult(ctxWithTimeout, sql, cnUUID, txn) + if err != nil { + return "", err + } + defer result.Close() + return readSingleTaskRunner(result) +} + func readSingleTaskRunner(result executor.Result) (string, error) { runners := make([]string, 0, 1) result.ReadRows(func(rows int, cols []*vector.Vector) bool { diff --git a/pkg/sql/compile/iscp_util.go b/pkg/sql/compile/iscp_util.go index cb993ec1684d7..2d6d0e717581c 100644 --- a/pkg/sql/compile/iscp_util.go +++ b/pkg/sql/compile/iscp_util.go @@ -35,6 +35,7 @@ var ( iscpUnregisterJobFunc = iscp.UnregisterJob iscpLookupJobLogFunc = iscp.LookupJobLog iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner isTableInCCPRFunc = isTableInCCPRImpl ) @@ -197,7 +198,7 @@ func DropIndexCdcTask(c *Compile, tableDef *plan.TableDef, dbname string, tablen } func DrainIndexCdcTaskConsumer(c *Compile, tableDef *plan.TableDef, dbname string, tablename string, indexname string) error { - return drainIndexCdcTaskConsumer(c, tableDef, dbname, tablename, indexname, false) + return drainIndexCdcTaskConsumer(c, tableDef, dbname, tablename, indexname) } func drainIndexCdcTaskConsumer( @@ -206,7 +207,6 @@ func drainIndexCdcTaskConsumer( dbname string, tablename string, indexname string, - failIfNoLocalExecutor bool, ) error { valid, err := checkValidIndexCdc(tableDef, indexname) if err != nil { @@ -236,22 +236,19 @@ func drainIndexCdcTaskConsumer( if tableID == 0 { tableID = tableDef.TblId } - exec, ok := iscpGetExecutorFunc(c.proc.GetService()) + runnerCN, err := iscpGetTaskRunnerFunc(c.proc.Ctx, c.proc.GetService(), c.proc.GetTxnOperator()) + if err != nil { + return err + } + if runnerCN == "" { + runnerCN = c.proc.GetService() + } + exec, ok := iscpGetExecutorFunc(runnerCN) if !ok || exec == nil { - if !failIfNoLocalExecutor { - logutil.Infof( - "skip local drain for index cdc task consumer, iscp executor not found: cn=%s tableID=%d jobName=%s jobID=%d", - c.proc.GetService(), - tableID, - jobName, - jobID, - ) - return nil - } return moerr.NewInternalErrorf( c.proc.Ctx, "cannot confirm ISCP consumer quiescence on CN %s for tableID=%d jobName=%s jobID=%d", - c.proc.GetService(), + runnerCN, tableID, jobName, jobID, @@ -299,7 +296,7 @@ func DropAllIndexCdcTasks(c *Compile, tabledef *plan.TableDef, dbname string, ta if e != nil { return e } - if e = drainIndexCdcTaskConsumer(c, tabledef, dbname, tablename, idx.IndexName, false); e != nil { + if e = drainIndexCdcTaskConsumer(c, tabledef, dbname, tablename, idx.IndexName); e != nil { return e } } diff --git a/pkg/sql/compile/iscp_util_extra_test.go b/pkg/sql/compile/iscp_util_extra_test.go index 1e67e2bd0e431..96a8dd0a90148 100644 --- a/pkg/sql/compile/iscp_util_extra_test.go +++ b/pkg/sql/compile/iscp_util_extra_test.go @@ -325,11 +325,15 @@ func TestDrainIndexCdcTaskConsumerFencesRuntimeJob(t *testing.T) { iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { return exec, true } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { return 0, 42, 7, true, true, nil } defer func() { iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner iscpLookupJobLogFunc = iscp.LookupJobLog }() @@ -353,15 +357,23 @@ func TestDrainIndexCdcTaskConsumerFencesRuntimeJob(t *testing.T) { require.True(t, exec.IsJobFenced(iscp.NewJobRuntimeKey(0, 42, "index_idx1", 7))) } -func TestDrainIndexCdcTaskConsumerNoLocalExecutorContinues(t *testing.T) { +func TestDrainIndexCdcTaskConsumerRoutesToRunnerExecutor(t *testing.T) { + runnerExec := &iscp.ISCPTaskExecutor{} iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + if cnUUID == "runner-cn" { + return runnerExec, true + } return nil, false } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { return 0, 42, 7, true, true, nil } defer func() { iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner iscpLookupJobLogFunc = iscp.LookupJobLog }() @@ -380,6 +392,40 @@ func TestDrainIndexCdcTaskConsumerNoLocalExecutorContinues(t *testing.T) { } require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + require.True(t, runnerExec.IsJobFenced(iscp.NewJobRuntimeKey(0, 42, "index_idx1", 7))) +} + +func TestDrainIndexCdcTaskConsumerNoRunnerExecutorFailsClosed(t *testing.T) { + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.ErrorContains(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1"), "cannot confirm ISCP consumer quiescence on CN runner-cn") } func TestDrainIndexCdcTaskConsumerRegistersRollbackFenceCleanup(t *testing.T) { @@ -387,11 +433,15 @@ func TestDrainIndexCdcTaskConsumerRegistersRollbackFenceCleanup(t *testing.T) { iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { return exec, true } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { return 0, 42, 7, true, true, nil } defer func() { iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner iscpLookupJobLogFunc = iscp.LookupJobLog }() @@ -443,12 +493,16 @@ func TestCoverage_DropAllIndexCdcTasks_DuplicateNames(t *testing.T) { iscpGetExecutorFunc = func(string) (*iscp.ISCPTaskExecutor, bool) { return exec, true } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { return 0, 42, uint64(dropCount), true, true, nil } defer func() { iscpUnregisterJobFunc = iscp.UnregisterJob iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner iscpLookupJobLogFunc = iscp.LookupJobLog }() @@ -477,7 +531,7 @@ func TestCoverage_DropAllIndexCdcTasks_DuplicateNames(t *testing.T) { assert.Equal(t, 1, dropCount, "duplicate index names should be deduplicated") } -func TestDropAllIndexCdcTasksNoLocalExecutorContinuesAfterUnregister(t *testing.T) { +func TestDropAllIndexCdcTasksNoRunnerExecutorFailsAfterUnregister(t *testing.T) { dropCount := 0 iscpUnregisterJobFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (bool, error) { dropCount++ @@ -486,12 +540,16 @@ func TestDropAllIndexCdcTasksNoLocalExecutorContinuesAfterUnregister(t *testing. iscpGetExecutorFunc = func(string) (*iscp.ISCPTaskExecutor, bool) { return nil, false } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { return 0, 42, 1, true, true, nil } defer func() { iscpUnregisterJobFunc = iscp.UnregisterJob iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner iscpLookupJobLogFunc = iscp.LookupJobLog }() @@ -509,7 +567,7 @@ func TestDropAllIndexCdcTasksNoLocalExecutorContinuesAfterUnregister(t *testing. }, } - require.NoError(t, DropAllIndexCdcTasks(c, tbldef, "db", "tbl")) + require.ErrorContains(t, DropAllIndexCdcTasks(c, tbldef, "db", "tbl"), "cannot confirm ISCP consumer quiescence on CN runner-cn") require.Equal(t, 1, dropCount) } @@ -523,12 +581,16 @@ func TestCoverage_DropAllIndexCdcTasks_MixedIndexes(t *testing.T) { iscpGetExecutorFunc = func(string) (*iscp.ISCPTaskExecutor, bool) { return exec, true } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { return 0, 42, uint64(dropCount), true, true, nil } defer func() { iscpUnregisterJobFunc = iscp.UnregisterJob iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner iscpLookupJobLogFunc = iscp.LookupJobLog }() From dce285f1a72755200ca1bc4cb67ca30fc62289e5 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 16 Jul 2026 14:58:49 +0800 Subject: [PATCH 08/14] fix: route iscp drain to runner cn --- pkg/cdc/sql_builder.go | 35 +- pkg/cnservice/server_query.go | 32 + pkg/iscp/executor.go | 41 +- pkg/iscp/runtime_cancel.go | 147 --- pkg/iscp/runtime_cancel_test.go | 94 -- pkg/iscp/types.go | 3 - pkg/iscp/watermark_updater.go | 1 - pkg/objectio/injects.go | 2 - pkg/pb/query/query.pb.go | 1199 +++++++++++++++++------ pkg/queryservice/client/query_client.go | 1 + pkg/sql/compile/iscp_util.go | 98 +- pkg/sql/compile/iscp_util_extra_test.go | 175 ++++ proto/query.proto | 15 + 13 files changed, 1222 insertions(+), 621 deletions(-) diff --git a/pkg/cdc/sql_builder.go b/pkg/cdc/sql_builder.go index 1c9fa43c8ff08..a2fd3cad10d83 100644 --- a/pkg/cdc/sql_builder.go +++ b/pkg/cdc/sql_builder.go @@ -282,7 +282,7 @@ const ( `AND table_id = %d ` + `AND job_name = '%s'` + `AND job_id = %d ` + - `AND job_state NOT IN (4, 5, 6) ` + + `AND job_state != 4 ` + `AND JSON_EXTRACT(job_status, '$.LSN') = '%d'` CDCUpdateMOISCPLogJobSpecSqlTemplate = `UPDATE mo_catalog.mo_iscp_log SET ` + `job_spec = '%s'` + @@ -292,20 +292,12 @@ const ( `AND job_name = '%s'` + `AND job_id = %d` CDCUpdateMOISCPLogDropAtSqlTemplate = `UPDATE mo_catalog.mo_iscp_log SET ` + - `job_state = %d,` + `drop_at = now()` + `WHERE` + ` account_id = %d ` + `AND table_id = %d ` + `AND job_name = '%s'` + `AND job_id = %d` - CDCUpdateMOISCPLogStateSqlTemplate = `UPDATE mo_catalog.mo_iscp_log SET ` + - `job_state = %d ` + - `WHERE` + - ` account_id = %d ` + - `AND table_id = %d ` + - `AND job_name = '%s'` + - `AND job_id = %d` CDCDeleteMOISCPLogSqlTemplate = `DELETE FROM mo_catalog.mo_iscp_log WHERE ` + `drop_at < '%s'` CDCSelectMOISCPLogSqlTemplate = `SELECT * from mo_catalog.mo_iscp_log` @@ -356,9 +348,8 @@ const ( CDCGetTableIDTemplate_Idx = 29 CDCUpdateTaskStateByTaskIdSQL_Idx = 30 CDCUpdateTaskStateByTaskIdAndStateSQL_Idx = 31 - CDCUpdateMOISCPLogStateSqlTemplate_Idx = 32 - CDCSqlTemplateCount = 33 + CDCSqlTemplateCount = 32 ) var CDCSQLTemplates = [CDCSqlTemplateCount]struct { @@ -487,9 +478,6 @@ var CDCSQLTemplates = [CDCSqlTemplateCount]struct { CDCUpdateMOISCPLogDropAtSqlTemplate_Idx: { SQL: CDCUpdateMOISCPLogDropAtSqlTemplate, }, - CDCUpdateMOISCPLogStateSqlTemplate_Idx: { - SQL: CDCUpdateMOISCPLogStateSqlTemplate, - }, CDCDeleteMOISCPLogSqlTemplate_Idx: { SQL: CDCDeleteMOISCPLogSqlTemplate, }, @@ -973,28 +961,9 @@ func (b cdcSQLBuilder) ISCPLogUpdateDropAtSQL( tableID uint64, jobName string, jobID uint64, - jobState int8, ) string { return fmt.Sprintf( CDCSQLTemplates[CDCUpdateMOISCPLogDropAtSqlTemplate_Idx].SQL, - jobState, - accountID, - tableID, - jobName, - jobID, - ) -} - -func (b cdcSQLBuilder) ISCPLogUpdateStateSQL( - accountID uint32, - tableID uint64, - jobName string, - jobID uint64, - jobState int8, -) string { - return fmt.Sprintf( - CDCSQLTemplates[CDCUpdateMOISCPLogStateSqlTemplate_Idx].SQL, - jobState, accountID, tableID, jobName, diff --git a/pkg/cnservice/server_query.go b/pkg/cnservice/server_query.go index d4aeb3f568e7e..c5bf988e1f2cc 100644 --- a/pkg/cnservice/server_query.go +++ b/pkg/cnservice/server_query.go @@ -26,6 +26,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/iscp" "github.com/matrixorigin/matrixone/pkg/lockservice" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio" @@ -102,6 +103,7 @@ func (s *service) initQueryCommandHandler() { s.queryService.AddHandleFunc(query.CmdMethod_WorkspaceThreshold, s.handleWorkspaceThresholdRequest, false) s.queryService.AddHandleFunc(query.CmdMethod_MinTimestamp, s.handleGetMinTimestamp, false) s.queryService.AddHandleFunc(query.CmdMethod_CtlPrefetchOnSubscribed, s.handleCtlPrefetchOnSubscribed, false) + s.queryService.AddHandleFunc(query.CmdMethod_ISCPDrainConsumer, s.handleISCPDrainConsumer, false) } func (s *service) handleKillConn(ctx context.Context, req *query.Request, resp *query.Response, _ *morpc.Buffer) error { @@ -192,6 +194,36 @@ func (s *service) handleCtlPrefetchOnSubscribed(ctx context.Context, req *query. return nil } +func (s *service) handleISCPDrainConsumer(ctx context.Context, req *query.Request, resp *query.Response, _ *morpc.Buffer) error { + if req == nil || req.ISCPDrainConsumerRequest == nil { + return moerr.NewInternalError(ctx, "bad request") + } + r := req.ISCPDrainConsumerRequest + exec, ok := iscp.GetExecutorRuntime(s.cfg.UUID) + if !ok || exec == nil { + return moerr.NewInternalErrorf( + ctx, + "cannot confirm ISCP consumer quiescence on CN %s for tableID=%d jobName=%s jobID=%d", + s.cfg.UUID, + r.TableID, + r.JobName, + r.JobID, + ) + } + key := iscp.NewJobRuntimeKey(r.AccountID, r.TableID, r.JobName, r.JobID) + if r.RemoveFenceOnly { + exec.RemoveJobFence(key) + resp.ISCPDrainConsumerResponse = &query.ISCPDrainConsumerResponse{Success: true} + return nil + } + if err := exec.CancelAndDrainJobConsumer(ctx, r.AccountID, r.TableID, r.JobName, r.JobID); err != nil { + exec.RemoveJobFence(key) + return err + } + resp.ISCPDrainConsumerResponse = &query.ISCPDrainConsumerResponse{Success: true} + return nil +} + // handleGetLockInfo sends the lock info on current cn to another cn that needs. func (s *service) handleGetLockInfo(ctx context.Context, req *query.Request, resp *query.Response, _ *morpc.Buffer) error { resp.GetLockInfoResponse = new(query.GetLockInfoResponse) diff --git a/pkg/iscp/executor.go b/pkg/iscp/executor.go index 01a6a217d6cc3..ac448ccd6993c 100644 --- a/pkg/iscp/executor.go +++ b/pkg/iscp/executor.go @@ -60,7 +60,6 @@ const ( DefaultSyncTaskInterval = time.Second * 10 DefaultFlushWatermarkInterval = time.Hour DefaultFlushWatermarkTTL = time.Hour - DefaultCancelJobQueueSize = 1024 DefaultRetryTimes = 5 DefaultRetryInterval = time.Second @@ -192,8 +191,6 @@ func NewISCPTaskExecutor( runningConsumers: make( map[JobRuntimeKey]map[uint64]*RunningJobConsumer, ), - cancelJobs: make(chan JobRuntimeKey, DefaultCancelJobQueueSize), - cancelingJobs: make(map[JobRuntimeKey]struct{}), } return exec, nil } @@ -383,7 +380,7 @@ func (exec *ISCPTaskExecutor) stopLocked() { func (exec *ISCPTaskExecutor) repairAbandonedJobs(ctx context.Context) (err error) { ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Minute*5) defer cancel() - activeJobsSQL := fmt.Sprintf( + sql := fmt.Sprintf( `UPDATE mo_catalog.mo_iscp_log SET job_state = %d WHERE job_state IN (%d, %d); @@ -392,14 +389,6 @@ func (exec *ISCPTaskExecutor) repairAbandonedJobs(ctx context.Context) (err erro ISCPJobState_Pending, ISCPJobState_Running, ) - droppedJobsSQL := fmt.Sprintf( - `UPDATE mo_catalog.mo_iscp_log - SET job_state = %d - WHERE drop_at IS NOT NULL AND job_state != %d; - `, - ISCPJobState_Canceled, - ISCPJobState_Canceled, - ) nowTs := exec.txnEngine.LatestLogtailAppliedTime() createByOpt := client.WithTxnCreateBy( 0, @@ -423,16 +412,11 @@ func (exec *ISCPTaskExecutor) repairAbandonedJobs(ctx context.Context) (err erro if err != nil { return } - result, err := ExecWithResult(ctxWithTimeout, activeJobsSQL, exec.cnUUID, txnOp) - if err != nil { - return - } - result.Close() - result, err = ExecWithResult(ctxWithTimeout, droppedJobsSQL, exec.cnUUID, txnOp) + result, err := ExecWithResult(ctxWithTimeout, sql, exec.cnUUID, txnOp) if err != nil { return } - result.Close() + defer result.Close() return nil } @@ -462,18 +446,6 @@ func (exec *ISCPTaskExecutor) run(ctx context.Context, worker Worker) { select { case <-ctx.Done(): return - case key := <-exec.cancelJobs: - if err := exec.finishCanceledJob(ctx, key); err != nil { - logutil.Error( - "ISCP-Task cancel job failed", - zap.Uint32("accountID", key.AccountID), - zap.Uint64("tableID", key.TableID), - zap.String("jobName", key.JobName), - zap.Uint64("jobID", key.JobID), - zap.Error(err), - ) - exec.requeueCancelJob(ctx, key) - } case <-syncTaskTrigger.C: // apply iscp log from := exec.iscpLogWm.Next() @@ -936,9 +908,6 @@ func (exec *ISCPTaskExecutor) addOrUpdateRecoveredJob( if state == ISCPJobState_Pending || state == ISCPJobState_Running { state = ISCPJobState_Completed } - if dropAt != 0 && state != ISCPJobState_Canceled { - state = ISCPJobState_Canceled - } return exec.addOrUpdateJob( accountID, tableID, @@ -1003,7 +972,6 @@ func (exec *ISCPTaskExecutor) addOrUpdateJob( notPrint bool, ) error { var newCreate bool - key := NewJobRuntimeKey(accountID, tableID, jobName, jobID) var watermark types.TS defer func() { @@ -1030,9 +998,6 @@ func (exec *ISCPTaskExecutor) addOrUpdateJob( if err != nil { return err } - if dropAt != 0 && state != ISCPJobState_Canceled { - exec.enqueueCancelJob(key) - } var table *TableEntry table, ok := exec.getTable(accountID, tableID) if !ok { diff --git a/pkg/iscp/runtime_cancel.go b/pkg/iscp/runtime_cancel.go index 4c12273c16aee..3552a7500ac30 100644 --- a/pkg/iscp/runtime_cancel.go +++ b/pkg/iscp/runtime_cancel.go @@ -16,17 +16,9 @@ package iscp import ( "context" - "errors" "sync" - "time" - "github.com/matrixorigin/matrixone/pkg/catalog" - "github.com/matrixorigin/matrixone/pkg/cdc" "github.com/matrixorigin/matrixone/pkg/common/moerr" - "github.com/matrixorigin/matrixone/pkg/defines" - "github.com/matrixorigin/matrixone/pkg/logutil" - "github.com/matrixorigin/matrixone/pkg/objectio" - "github.com/matrixorigin/matrixone/pkg/util/fault" ) var iscpExecutors sync.Map // map[cnUUID]*ISCPTaskExecutor @@ -72,9 +64,6 @@ func (exec *ISCPTaskExecutor) ensureRuntimeMapsLocked() { if exec.runningConsumers == nil { exec.runningConsumers = make(map[JobRuntimeKey]map[uint64]*RunningJobConsumer) } - if exec.cancelingJobs == nil { - exec.cancelingJobs = make(map[JobRuntimeKey]struct{}) - } } func (exec *ISCPTaskExecutor) IsJobFenced(key JobRuntimeKey) bool { @@ -131,142 +120,6 @@ func (exec *ISCPTaskExecutor) CancelAndDrainJobConsumer( return nil } -func (exec *ISCPTaskExecutor) enqueueCancelJob(key JobRuntimeKey) { - if exec == nil { - return - } - exec.runtimeMu.Lock() - exec.ensureRuntimeMapsLocked() - if _, ok := exec.cancelingJobs[key]; ok { - exec.runtimeMu.Unlock() - return - } - exec.cancelingJobs[key] = struct{}{} - cancelJobs := exec.cancelJobs - exec.runtimeMu.Unlock() - - if cancelJobs == nil { - exec.clearCancelingJob(key) - return - } - select { - case cancelJobs <- key: - default: - go func() { - ctx := exec.ctx - if ctx == nil { - exec.clearCancelingJob(key) - return - } - select { - case cancelJobs <- key: - case <-ctx.Done(): - exec.clearCancelingJob(key) - } - }() - } -} - -func (exec *ISCPTaskExecutor) requeueCancelJob(ctx context.Context, key JobRuntimeKey) { - if exec == nil { - return - } - go func() { - timer := time.NewTimer(DefaultRetryInterval) - defer timer.Stop() - select { - case <-timer.C: - case <-ctx.Done(): - exec.clearCancelingJob(key) - return - } - cancelJobs := exec.cancelJobs - if cancelJobs == nil { - exec.clearCancelingJob(key) - return - } - select { - case cancelJobs <- key: - case <-ctx.Done(): - exec.clearCancelingJob(key) - } - }() -} - -func (exec *ISCPTaskExecutor) clearCancelingJob(key JobRuntimeKey) { - exec.runtimeMu.Lock() - delete(exec.cancelingJobs, key) - exec.runtimeMu.Unlock() -} - -func (exec *ISCPTaskExecutor) finishCanceledJob(ctx context.Context, key JobRuntimeKey) error { - if exec == nil { - return nil - } - if err := exec.CancelAndDrainJobConsumer(ctx, key.AccountID, key.TableID, key.JobName, key.JobID); err != nil { - return err - } - if err := exec.markJobCanceled(ctx, key); err != nil { - return err - } - exec.clearCancelingJob(key) - exec.RemoveJobFence(key) - return nil -} - -func (exec *ISCPTaskExecutor) markJobCanceled(ctx context.Context, key JobRuntimeKey) (err error) { - ctx = context.WithValue(ctx, defines.TenantIDKey{}, catalog.System_Account) - ctx, cancel := context.WithTimeoutCause(ctx, time.Minute*5, moerr.NewInternalErrorNoCtx("iscp cancel job timeout")) - defer cancel() - if _, _, injected := fault.TriggerFaultWithContext(ctx, objectio.FJ_ISCPCancelBeforeMarkCanceled); injected { - logutil.Infof( - "ISCP-Task cancel fault triggered %s: accountID=%d tableID=%d jobName=%s jobID=%d", - objectio.FJ_ISCPCancelBeforeMarkCanceled, - key.AccountID, - key.TableID, - key.JobName, - key.JobID, - ) - } - if _, msg, injected := fault.TriggerFault(objectio.FJ_ISCPCancelMarkCanceledError); injected { - if msg == "" { - msg = objectio.FJ_ISCPCancelMarkCanceledError - } - return moerr.NewInternalErrorNoCtxf("injected ISCP cancel mark error: %s", msg) - } - txnOp, err := getTxn(ctx, exec.txnEngine, exec.cnTxnClient, "iscp cancel job") - if err != nil { - return err - } - defer func() { - if err != nil { - err = errors.Join(err, txnOp.Rollback(ctx)) - return - } - err = txnOp.Commit(ctx) - }() - sql := cdc.CDCSQLBuilder.ISCPLogUpdateStateSQL( - key.AccountID, - key.TableID, - key.JobName, - key.JobID, - ISCPJobState_Canceled, - ) - result, err := ExecWithResult(ctx, sql, exec.cnUUID, txnOp) - if err != nil { - return err - } - result.Close() - logutil.Infof( - "ISCP-Task marked job canceled: accountID=%d tableID=%d jobName=%s jobID=%d", - key.AccountID, - key.TableID, - key.JobName, - key.JobID, - ) - return nil -} - func (exec *ISCPTaskExecutor) RemoveJobFence(key JobRuntimeKey) { if exec == nil { return diff --git a/pkg/iscp/runtime_cancel_test.go b/pkg/iscp/runtime_cancel_test.go index b15ad2d900f38..db4553c97d4e7 100644 --- a/pkg/iscp/runtime_cancel_test.go +++ b/pkg/iscp/runtime_cancel_test.go @@ -28,23 +28,12 @@ func newRuntimeTestExecutor() *ISCPTaskExecutor { return &ISCPTaskExecutor{ fencedJobs: make(map[JobRuntimeKey]struct{}), runningConsumers: make(map[JobRuntimeKey]map[uint64]*RunningJobConsumer), - cancelJobs: make(chan JobRuntimeKey, DefaultCancelJobQueueSize), - cancelingJobs: make(map[JobRuntimeKey]struct{}), option: fillDefaultOption(nil), ctx: context.Background(), tables: newISCPTableTree(), } } -func encodeRuntimeCancelTestJSON(t *testing.T, value string) []byte { - t.Helper() - byteJSON, err := types.ParseStringToByteJson(value) - require.NoError(t, err) - encoded, err := types.EncodeJson(byteJSON) - require.NoError(t, err) - return encoded -} - func TestGetExecutorRuntimeRequiresExactCN(t *testing.T) { exec := newRuntimeTestExecutor() RegisterExecutorRuntime("runner-cn", exec) @@ -225,86 +214,3 @@ func TestCancelAndDrainJobConsumerHonorsCallerContext(t *testing.T) { require.True(t, errors.Is(err, context.Canceled)) <-consumerCancelCalled } - -func TestDroppedJobIsQueuedForCancel(t *testing.T) { - exec := newRuntimeTestExecutor() - table := NewTableEntry(exec, 1, 10, 2, "db", "tbl") - exec.setTable(table) - spec := &JobSpec{ - ConsumerInfo: ConsumerInfo{ - SrcTable: TableInfo{DBID: 10, TableID: 2, DBName: "db", TableName: "tbl"}, - }, - TriggerSpec: TriggerSpec{JobType: TriggerType_Default}, - } - specJSON, err := MarshalJobSpec(spec) - require.NoError(t, err) - statusJSON, err := MarshalJobStatus(&JobStatus{}) - require.NoError(t, err) - dropAt := types.Timestamp(time.Now().UnixNano()) - - err = exec.addOrUpdateJob( - 1, - 2, - "index_idx01", - 7, - ISCPJobState_Canceling, - types.BuildTS(1, 0).ToString(), - encodeRuntimeCancelTestJSON(t, specJSON), - encodeRuntimeCancelTestJSON(t, statusJSON), - dropAt, - true, - ) - require.NoError(t, err) - - key := NewJobRuntimeKey(1, 2, "index_idx01", 7) - require.Eventually(t, func() bool { - select { - case got := <-exec.cancelJobs: - return got == key - default: - return false - } - }, time.Second, 10*time.Millisecond) - exec.runtimeMu.Lock() - _, queued := exec.cancelingJobs[key] - exec.runtimeMu.Unlock() - require.True(t, queued) -} - -func TestRecoveredDroppedJobBecomesCanceledWithoutQueue(t *testing.T) { - exec := newRuntimeTestExecutor() - table := NewTableEntry(exec, 1, 10, 2, "db", "tbl") - exec.setTable(table) - spec := &JobSpec{ - ConsumerInfo: ConsumerInfo{ - SrcTable: TableInfo{DBID: 10, TableID: 2, DBName: "db", TableName: "tbl"}, - }, - TriggerSpec: TriggerSpec{JobType: TriggerType_Default}, - } - specJSON, err := MarshalJobSpec(spec) - require.NoError(t, err) - statusJSON, err := MarshalJobStatus(&JobStatus{}) - require.NoError(t, err) - dropAt := types.Timestamp(time.Now().UnixNano()) - - err = exec.addOrUpdateRecoveredJob( - 1, - 2, - "index_idx01", - 7, - ISCPJobState_Canceling, - types.BuildTS(1, 0).ToString(), - encodeRuntimeCancelTestJSON(t, specJSON), - encodeRuntimeCancelTestJSON(t, statusJSON), - dropAt, - true, - ) - require.NoError(t, err) - - select { - case got := <-exec.cancelJobs: - t.Fatalf("recovered dropped job should not be queued, got %+v", got) - default: - } - require.Equal(t, int8(ISCPJobState_Canceled), table.jobs[JobKey{JobName: "index_idx01", JobID: 7}].state) -} diff --git a/pkg/iscp/types.go b/pkg/iscp/types.go index 2e1e3e4ee1179..2fdfc19105496 100644 --- a/pkg/iscp/types.go +++ b/pkg/iscp/types.go @@ -49,7 +49,6 @@ const ( ISCPJobState_Completed ISCPJobState_Error ISCPJobState_Canceled - ISCPJobState_Canceling ) type DataRetriever interface { @@ -164,8 +163,6 @@ type ISCPTaskExecutor struct { runtimeMu sync.Mutex fencedJobs map[JobRuntimeKey]struct{} runningConsumers map[JobRuntimeKey]map[uint64]*RunningJobConsumer - cancelJobs chan JobRuntimeKey - cancelingJobs map[JobRuntimeKey]struct{} } type JobRuntimeKey struct { diff --git a/pkg/iscp/watermark_updater.go b/pkg/iscp/watermark_updater.go index ac29c6e39345f..5a585eae23621 100644 --- a/pkg/iscp/watermark_updater.go +++ b/pkg/iscp/watermark_updater.go @@ -629,7 +629,6 @@ func unregisterJob( tableID, jobID.JobName, internalJobID, - ISCPJobState_Canceling, ) result, err := ExecWithResult(ctxWithSysAccount, sql, cnUUID, txn) if err != nil { diff --git a/pkg/objectio/injects.go b/pkg/objectio/injects.go index a2d00c07092be..0f0fa761d75ac 100644 --- a/pkg/objectio/injects.go +++ b/pkg/objectio/injects.go @@ -77,8 +77,6 @@ const ( FJ_ISCPCancelLongHnswBeforeUpdate = "fj/iscp/cancel/long/hnsw-before-update" FJ_ISCPCancelLongHnswBeforeSave = "fj/iscp/cancel/long/hnsw-before-save" FJ_ISCPCancelLongBeforeWatermark = "fj/iscp/cancel/long/before-watermark" - FJ_ISCPCancelBeforeMarkCanceled = "fj/iscp/cancel/before-mark-canceled" - FJ_ISCPCancelMarkCanceledError = "fj/iscp/cancel/mark-canceled-error" FJ_PublicationSnapshotFinished = "fj/publication/snapshot/finished" diff --git a/pkg/pb/query/query.pb.go b/pkg/pb/query/query.pb.go index d86faf806c77e..d5915dee4f1b5 100644 --- a/pkg/pb/query/query.pb.go +++ b/pkg/pb/query/query.pb.go @@ -103,6 +103,7 @@ const ( CmdMethod_WorkspaceThreshold CmdMethod = 34 CmdMethod_MinTimestamp CmdMethod = 35 CmdMethod_CtlPrefetchOnSubscribed CmdMethod = 36 + CmdMethod_ISCPDrainConsumer CmdMethod = 37 ) var CmdMethod_name = map[int32]string{ @@ -143,6 +144,7 @@ var CmdMethod_name = map[int32]string{ 34: "WorkspaceThreshold", 35: "MinTimestamp", 36: "CtlPrefetchOnSubscribed", + 37: "ISCPDrainConsumer", } var CmdMethod_value = map[string]int32{ @@ -183,6 +185,7 @@ var CmdMethod_value = map[string]int32{ "WorkspaceThreshold": 34, "MinTimestamp": 35, "CtlPrefetchOnSubscribed": 36, + "ISCPDrainConsumer": 37, } func (x CmdMethod) String() string { @@ -969,6 +972,7 @@ type Request struct { CtlMoTableStatsRequest CtlMoTableStatsRequest `protobuf:"bytes,36,opt,name=CtlMoTableStatsRequest,proto3" json:"CtlMoTableStatsRequest"` WorkspaceThresholdRequest *WorkspaceThresholdRequest `protobuf:"bytes,37,opt,name=WorkspaceThresholdRequest,proto3" json:"WorkspaceThresholdRequest,omitempty"` CtlPrefetchOnSubscribedRequest *CtlPrefetchOnSubscribedRequest `protobuf:"bytes,38,opt,name=CtlPrefetchOnSubscribedRequest,proto3" json:"CtlPrefetchOnSubscribedRequest,omitempty"` + ISCPDrainConsumerRequest *ISCPDrainConsumerRequest `protobuf:"bytes,39,opt,name=ISCPDrainConsumerRequest,proto3" json:"ISCPDrainConsumerRequest,omitempty"` } func (m *Request) Reset() { *m = Request{} } @@ -1270,6 +1274,13 @@ func (m *Request) GetCtlPrefetchOnSubscribedRequest() *CtlPrefetchOnSubscribedRe return nil } +func (m *Request) GetISCPDrainConsumerRequest() *ISCPDrainConsumerRequest { + if m != nil { + return m.ISCPDrainConsumerRequest + } + return nil +} + // ShowProcessListResponse is the response of command ShowProcessList. type ShowProcessListResponse struct { Sessions []*status.Session `protobuf:"bytes,1,rep,name=Sessions,proto3" json:"Sessions,omitempty"` @@ -1369,6 +1380,7 @@ type Response struct { WorkspaceThresholdResponse *WorkspaceThresholdResponse `protobuf:"bytes,37,opt,name=WorkspaceThresholdResponse,proto3" json:"WorkspaceThresholdResponse,omitempty"` MinTimestampResponse *MinTimestampResponse `protobuf:"bytes,38,opt,name=MinTimestampResponse,proto3" json:"MinTimestampResponse,omitempty"` CtlPrefetchOnSubscribedResponse *CtlPrefetchOnSubscribedResponse `protobuf:"bytes,39,opt,name=CtlPrefetchOnSubscribedResponse,proto3" json:"CtlPrefetchOnSubscribedResponse,omitempty"` + ISCPDrainConsumerResponse *ISCPDrainConsumerResponse `protobuf:"bytes,40,opt,name=ISCPDrainConsumerResponse,proto3" json:"ISCPDrainConsumerResponse,omitempty"` } func (m *Response) Reset() { *m = Response{} } @@ -1677,6 +1689,13 @@ func (m *Response) GetCtlPrefetchOnSubscribedResponse() *CtlPrefetchOnSubscribed return nil } +func (m *Response) GetISCPDrainConsumerResponse() *ISCPDrainConsumerResponse { + if m != nil { + return m.ISCPDrainConsumerResponse + } + return nil +} + // AlterAccountRequest is the "alter account restricted" query request. type AlterAccountRequest struct { // Tenant is the tenant which to alter. @@ -2068,6 +2087,126 @@ func (m *CtlPrefetchOnSubscribedResponse) GetResp() string { return "" } +type ISCPDrainConsumerRequest struct { + AccountID uint32 `protobuf:"varint,1,opt,name=AccountID,proto3" json:"AccountID,omitempty"` + TableID uint64 `protobuf:"varint,2,opt,name=TableID,proto3" json:"TableID,omitempty"` + JobName string `protobuf:"bytes,3,opt,name=JobName,proto3" json:"JobName,omitempty"` + JobID uint64 `protobuf:"varint,4,opt,name=JobID,proto3" json:"JobID,omitempty"` + RemoveFenceOnly bool `protobuf:"varint,5,opt,name=RemoveFenceOnly,proto3" json:"RemoveFenceOnly,omitempty"` +} + +func (m *ISCPDrainConsumerRequest) Reset() { *m = ISCPDrainConsumerRequest{} } +func (m *ISCPDrainConsumerRequest) String() string { return proto.CompactTextString(m) } +func (*ISCPDrainConsumerRequest) ProtoMessage() {} +func (*ISCPDrainConsumerRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_5c6ac9b241082464, []int{27} +} +func (m *ISCPDrainConsumerRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ISCPDrainConsumerRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ISCPDrainConsumerRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ISCPDrainConsumerRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ISCPDrainConsumerRequest.Merge(m, src) +} +func (m *ISCPDrainConsumerRequest) XXX_Size() int { + return m.ProtoSize() +} +func (m *ISCPDrainConsumerRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ISCPDrainConsumerRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ISCPDrainConsumerRequest proto.InternalMessageInfo + +func (m *ISCPDrainConsumerRequest) GetAccountID() uint32 { + if m != nil { + return m.AccountID + } + return 0 +} + +func (m *ISCPDrainConsumerRequest) GetTableID() uint64 { + if m != nil { + return m.TableID + } + return 0 +} + +func (m *ISCPDrainConsumerRequest) GetJobName() string { + if m != nil { + return m.JobName + } + return "" +} + +func (m *ISCPDrainConsumerRequest) GetJobID() uint64 { + if m != nil { + return m.JobID + } + return 0 +} + +func (m *ISCPDrainConsumerRequest) GetRemoveFenceOnly() bool { + if m != nil { + return m.RemoveFenceOnly + } + return false +} + +type ISCPDrainConsumerResponse struct { + Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` +} + +func (m *ISCPDrainConsumerResponse) Reset() { *m = ISCPDrainConsumerResponse{} } +func (m *ISCPDrainConsumerResponse) String() string { return proto.CompactTextString(m) } +func (*ISCPDrainConsumerResponse) ProtoMessage() {} +func (*ISCPDrainConsumerResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_5c6ac9b241082464, []int{28} +} +func (m *ISCPDrainConsumerResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ISCPDrainConsumerResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ISCPDrainConsumerResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ISCPDrainConsumerResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ISCPDrainConsumerResponse.Merge(m, src) +} +func (m *ISCPDrainConsumerResponse) XXX_Size() int { + return m.ProtoSize() +} +func (m *ISCPDrainConsumerResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ISCPDrainConsumerResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ISCPDrainConsumerResponse proto.InternalMessageInfo + +func (m *ISCPDrainConsumerResponse) GetSuccess() bool { + if m != nil { + return m.Success + } + return false +} + type TraceSpanRequest struct { Cmd string `protobuf:"bytes,1,opt,name=Cmd,proto3" json:"Cmd,omitempty"` Spans string `protobuf:"bytes,2,opt,name=Spans,proto3" json:"Spans,omitempty"` @@ -2078,7 +2217,7 @@ func (m *TraceSpanRequest) Reset() { *m = TraceSpanRequest{} } func (m *TraceSpanRequest) String() string { return proto.CompactTextString(m) } func (*TraceSpanRequest) ProtoMessage() {} func (*TraceSpanRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{27} + return fileDescriptor_5c6ac9b241082464, []int{29} } func (m *TraceSpanRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2136,7 +2275,7 @@ func (m *TraceSpanResponse) Reset() { *m = TraceSpanResponse{} } func (m *TraceSpanResponse) String() string { return proto.CompactTextString(m) } func (*TraceSpanResponse) ProtoMessage() {} func (*TraceSpanResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{28} + return fileDescriptor_5c6ac9b241082464, []int{30} } func (m *TraceSpanResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2179,7 +2318,7 @@ func (m *GetLockInfoRequest) Reset() { *m = GetLockInfoRequest{} } func (m *GetLockInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetLockInfoRequest) ProtoMessage() {} func (*GetLockInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{29} + return fileDescriptor_5c6ac9b241082464, []int{31} } func (m *GetLockInfoRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2221,7 +2360,7 @@ func (m *LockInfo) Reset() { *m = LockInfo{} } func (m *LockInfo) String() string { return proto.CompactTextString(m) } func (*LockInfo) ProtoMessage() {} func (*LockInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{30} + return fileDescriptor_5c6ac9b241082464, []int{32} } func (m *LockInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2301,7 +2440,7 @@ func (m *GetLockInfoResponse) Reset() { *m = GetLockInfoResponse{} } func (m *GetLockInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetLockInfoResponse) ProtoMessage() {} func (*GetLockInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{31} + return fileDescriptor_5c6ac9b241082464, []int{33} } func (m *GetLockInfoResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2351,7 +2490,7 @@ func (m *GetTxnInfoRequest) Reset() { *m = GetTxnInfoRequest{} } func (m *GetTxnInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetTxnInfoRequest) ProtoMessage() {} func (*GetTxnInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{32} + return fileDescriptor_5c6ac9b241082464, []int{34} } func (m *GetTxnInfoRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2397,7 +2536,7 @@ func (m *TxnLockInfo) Reset() { *m = TxnLockInfo{} } func (m *TxnLockInfo) String() string { return proto.CompactTextString(m) } func (*TxnLockInfo) ProtoMessage() {} func (*TxnLockInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{33} + return fileDescriptor_5c6ac9b241082464, []int{35} } func (m *TxnLockInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2461,7 +2600,7 @@ func (m *TxnInfo) Reset() { *m = TxnInfo{} } func (m *TxnInfo) String() string { return proto.CompactTextString(m) } func (*TxnInfo) ProtoMessage() {} func (*TxnInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{34} + return fileDescriptor_5c6ac9b241082464, []int{36} } func (m *TxnInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2527,7 +2666,7 @@ func (m *GetTxnInfoResponse) Reset() { *m = GetTxnInfoResponse{} } func (m *GetTxnInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetTxnInfoResponse) ProtoMessage() {} func (*GetTxnInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{35} + return fileDescriptor_5c6ac9b241082464, []int{37} } func (m *GetTxnInfoResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2577,7 +2716,7 @@ func (m *GetCacheInfoRequest) Reset() { *m = GetCacheInfoRequest{} } func (m *GetCacheInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetCacheInfoRequest) ProtoMessage() {} func (*GetCacheInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{36} + return fileDescriptor_5c6ac9b241082464, []int{38} } func (m *GetCacheInfoRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2625,7 +2764,7 @@ func (m *CacheInfo) Reset() { *m = CacheInfo{} } func (m *CacheInfo) String() string { return proto.CompactTextString(m) } func (*CacheInfo) ProtoMessage() {} func (*CacheInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{37} + return fileDescriptor_5c6ac9b241082464, []int{39} } func (m *CacheInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2704,7 +2843,7 @@ func (m *GetCacheInfoResponse) Reset() { *m = GetCacheInfoResponse{} } func (m *GetCacheInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetCacheInfoResponse) ProtoMessage() {} func (*GetCacheInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{38} + return fileDescriptor_5c6ac9b241082464, []int{40} } func (m *GetCacheInfoResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2750,7 +2889,7 @@ func (m *RemoveRemoteLockTableRequest) Reset() { *m = RemoveRemoteLockTa func (m *RemoveRemoteLockTableRequest) String() string { return proto.CompactTextString(m) } func (*RemoveRemoteLockTableRequest) ProtoMessage() {} func (*RemoveRemoteLockTableRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{39} + return fileDescriptor_5c6ac9b241082464, []int{41} } func (m *RemoveRemoteLockTableRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2808,7 +2947,7 @@ func (m *RemoveRemoteLockTableResponse) Reset() { *m = RemoveRemoteLockT func (m *RemoveRemoteLockTableResponse) String() string { return proto.CompactTextString(m) } func (*RemoveRemoteLockTableResponse) ProtoMessage() {} func (*RemoveRemoteLockTableResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{40} + return fileDescriptor_5c6ac9b241082464, []int{42} } func (m *RemoveRemoteLockTableResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2853,7 +2992,7 @@ func (m *GetLatestBindRequest) Reset() { *m = GetLatestBindRequest{} } func (m *GetLatestBindRequest) String() string { return proto.CompactTextString(m) } func (*GetLatestBindRequest) ProtoMessage() {} func (*GetLatestBindRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{41} + return fileDescriptor_5c6ac9b241082464, []int{43} } func (m *GetLatestBindRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2904,7 +3043,7 @@ func (m *GetLatestBindResponse) Reset() { *m = GetLatestBindResponse{} } func (m *GetLatestBindResponse) String() string { return proto.CompactTextString(m) } func (*GetLatestBindResponse) ProtoMessage() {} func (*GetLatestBindResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{42} + return fileDescriptor_5c6ac9b241082464, []int{44} } func (m *GetLatestBindResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2953,7 +3092,7 @@ func (m *UnsubscribeTableRequest) Reset() { *m = UnsubscribeTableRequest func (m *UnsubscribeTableRequest) String() string { return proto.CompactTextString(m) } func (*UnsubscribeTableRequest) ProtoMessage() {} func (*UnsubscribeTableRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{43} + return fileDescriptor_5c6ac9b241082464, []int{45} } func (m *UnsubscribeTableRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3005,7 +3144,7 @@ func (m *UnsubscribeTableResponse) Reset() { *m = UnsubscribeTableRespon func (m *UnsubscribeTableResponse) String() string { return proto.CompactTextString(m) } func (*UnsubscribeTableResponse) ProtoMessage() {} func (*UnsubscribeTableResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{44} + return fileDescriptor_5c6ac9b241082464, []int{46} } func (m *UnsubscribeTableResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3051,7 +3190,7 @@ func (m *CacheKey) Reset() { *m = CacheKey{} } func (m *CacheKey) String() string { return proto.CompactTextString(m) } func (*CacheKey) ProtoMessage() {} func (*CacheKey) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{45} + return fileDescriptor_5c6ac9b241082464, []int{47} } func (m *CacheKey) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3109,7 +3248,7 @@ func (m *CacheKeys) Reset() { *m = CacheKeys{} } func (m *CacheKeys) String() string { return proto.CompactTextString(m) } func (*CacheKeys) ProtoMessage() {} func (*CacheKeys) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{46} + return fileDescriptor_5c6ac9b241082464, []int{48} } func (m *CacheKeys) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3154,7 +3293,7 @@ func (m *RequestCacheKey) Reset() { *m = RequestCacheKey{} } func (m *RequestCacheKey) String() string { return proto.CompactTextString(m) } func (*RequestCacheKey) ProtoMessage() {} func (*RequestCacheKey) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{47} + return fileDescriptor_5c6ac9b241082464, []int{49} } func (m *RequestCacheKey) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3207,7 +3346,7 @@ func (m *GetCacheDataRequest) Reset() { *m = GetCacheDataRequest{} } func (m *GetCacheDataRequest) String() string { return proto.CompactTextString(m) } func (*GetCacheDataRequest) ProtoMessage() {} func (*GetCacheDataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{48} + return fileDescriptor_5c6ac9b241082464, []int{50} } func (m *GetCacheDataRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3254,7 +3393,7 @@ func (m *ResponseCacheData) Reset() { *m = ResponseCacheData{} } func (m *ResponseCacheData) String() string { return proto.CompactTextString(m) } func (*ResponseCacheData) ProtoMessage() {} func (*ResponseCacheData) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{49} + return fileDescriptor_5c6ac9b241082464, []int{51} } func (m *ResponseCacheData) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3313,7 +3452,7 @@ func (m *GetCacheDataResponse) Reset() { *m = GetCacheDataResponse{} } func (m *GetCacheDataResponse) String() string { return proto.CompactTextString(m) } func (*GetCacheDataResponse) ProtoMessage() {} func (*GetCacheDataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{50} + return fileDescriptor_5c6ac9b241082464, []int{52} } func (m *GetCacheDataResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3357,7 +3496,7 @@ func (m *GetStatsInfoRequest) Reset() { *m = GetStatsInfoRequest{} } func (m *GetStatsInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetStatsInfoRequest) ProtoMessage() {} func (*GetStatsInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{51} + return fileDescriptor_5c6ac9b241082464, []int{53} } func (m *GetStatsInfoRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3401,7 +3540,7 @@ func (m *GetStatsInfoResponse) Reset() { *m = GetStatsInfoResponse{} } func (m *GetStatsInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetStatsInfoResponse) ProtoMessage() {} func (*GetStatsInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{52} + return fileDescriptor_5c6ac9b241082464, []int{54} } func (m *GetStatsInfoResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3447,7 +3586,7 @@ func (m *PrepareStmt) Reset() { *m = PrepareStmt{} } func (m *PrepareStmt) String() string { return proto.CompactTextString(m) } func (*PrepareStmt) ProtoMessage() {} func (*PrepareStmt) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{53} + return fileDescriptor_5c6ac9b241082464, []int{55} } func (m *PrepareStmt) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3505,7 +3644,7 @@ func (m *MigrateConnFromRequest) Reset() { *m = MigrateConnFromRequest{} func (m *MigrateConnFromRequest) String() string { return proto.CompactTextString(m) } func (*MigrateConnFromRequest) ProtoMessage() {} func (*MigrateConnFromRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{54} + return fileDescriptor_5c6ac9b241082464, []int{56} } func (m *MigrateConnFromRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3550,7 +3689,7 @@ func (m *MigrateConnFromResponse) Reset() { *m = MigrateConnFromResponse func (m *MigrateConnFromResponse) String() string { return proto.CompactTextString(m) } func (*MigrateConnFromResponse) ProtoMessage() {} func (*MigrateConnFromResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{55} + return fileDescriptor_5c6ac9b241082464, []int{57} } func (m *MigrateConnFromResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3604,7 +3743,7 @@ func (m *MigrateConnToRequest) Reset() { *m = MigrateConnToRequest{} } func (m *MigrateConnToRequest) String() string { return proto.CompactTextString(m) } func (*MigrateConnToRequest) ProtoMessage() {} func (*MigrateConnToRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{56} + return fileDescriptor_5c6ac9b241082464, []int{58} } func (m *MigrateConnToRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3669,7 +3808,7 @@ func (m *MigrateConnToResponse) Reset() { *m = MigrateConnToResponse{} } func (m *MigrateConnToResponse) String() string { return proto.CompactTextString(m) } func (*MigrateConnToResponse) ProtoMessage() {} func (*MigrateConnToResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{57} + return fileDescriptor_5c6ac9b241082464, []int{59} } func (m *MigrateConnToResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3714,7 +3853,7 @@ func (m *ReloadAutoIncrementCacheRequest) Reset() { *m = ReloadAutoIncre func (m *ReloadAutoIncrementCacheRequest) String() string { return proto.CompactTextString(m) } func (*ReloadAutoIncrementCacheRequest) ProtoMessage() {} func (*ReloadAutoIncrementCacheRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{58} + return fileDescriptor_5c6ac9b241082464, []int{60} } func (m *ReloadAutoIncrementCacheRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3758,7 +3897,7 @@ func (m *ReloadAutoIncrementCacheResponse) Reset() { *m = ReloadAutoIncr func (m *ReloadAutoIncrementCacheResponse) String() string { return proto.CompactTextString(m) } func (*ReloadAutoIncrementCacheResponse) ProtoMessage() {} func (*ReloadAutoIncrementCacheResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{59} + return fileDescriptor_5c6ac9b241082464, []int{61} } func (m *ReloadAutoIncrementCacheResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3795,7 +3934,7 @@ func (m *GetReplicaCountRequest) Reset() { *m = GetReplicaCountRequest{} func (m *GetReplicaCountRequest) String() string { return proto.CompactTextString(m) } func (*GetReplicaCountRequest) ProtoMessage() {} func (*GetReplicaCountRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{60} + return fileDescriptor_5c6ac9b241082464, []int{62} } func (m *GetReplicaCountRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3839,7 +3978,7 @@ func (m *GetReplicaCountResponse) Reset() { *m = GetReplicaCountResponse func (m *GetReplicaCountResponse) String() string { return proto.CompactTextString(m) } func (*GetReplicaCountResponse) ProtoMessage() {} func (*GetReplicaCountResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{61} + return fileDescriptor_5c6ac9b241082464, []int{63} } func (m *GetReplicaCountResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3885,7 +4024,7 @@ func (m *ResetSessionRequest) Reset() { *m = ResetSessionRequest{} } func (m *ResetSessionRequest) String() string { return proto.CompactTextString(m) } func (*ResetSessionRequest) ProtoMessage() {} func (*ResetSessionRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{62} + return fileDescriptor_5c6ac9b241082464, []int{64} } func (m *ResetSessionRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3933,7 +4072,7 @@ func (m *ResetSessionResponse) Reset() { *m = ResetSessionResponse{} } func (m *ResetSessionResponse) String() string { return proto.CompactTextString(m) } func (*ResetSessionResponse) ProtoMessage() {} func (*ResetSessionResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{63} + return fileDescriptor_5c6ac9b241082464, []int{65} } func (m *ResetSessionResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3987,7 +4126,7 @@ func (m *GoMaxProcsRequest) Reset() { *m = GoMaxProcsRequest{} } func (m *GoMaxProcsRequest) String() string { return proto.CompactTextString(m) } func (*GoMaxProcsRequest) ProtoMessage() {} func (*GoMaxProcsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{64} + return fileDescriptor_5c6ac9b241082464, []int{66} } func (m *GoMaxProcsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4034,7 +4173,7 @@ func (m *GoMaxProcsResponse) Reset() { *m = GoMaxProcsResponse{} } func (m *GoMaxProcsResponse) String() string { return proto.CompactTextString(m) } func (*GoMaxProcsResponse) ProtoMessage() {} func (*GoMaxProcsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{65} + return fileDescriptor_5c6ac9b241082464, []int{67} } func (m *GoMaxProcsResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4078,7 +4217,7 @@ func (m *GoMemLimitRequest) Reset() { *m = GoMemLimitRequest{} } func (m *GoMemLimitRequest) String() string { return proto.CompactTextString(m) } func (*GoMemLimitRequest) ProtoMessage() {} func (*GoMemLimitRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{66} + return fileDescriptor_5c6ac9b241082464, []int{68} } func (m *GoMemLimitRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4125,7 +4264,7 @@ func (m *GoMemLimitResponse) Reset() { *m = GoMemLimitResponse{} } func (m *GoMemLimitResponse) String() string { return proto.CompactTextString(m) } func (*GoMemLimitResponse) ProtoMessage() {} func (*GoMemLimitResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{67} + return fileDescriptor_5c6ac9b241082464, []int{69} } func (m *GoMemLimitResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4172,7 +4311,7 @@ func (m *FileServiceCacheRequest) Reset() { *m = FileServiceCacheRequest func (m *FileServiceCacheRequest) String() string { return proto.CompactTextString(m) } func (*FileServiceCacheRequest) ProtoMessage() {} func (*FileServiceCacheRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{68} + return fileDescriptor_5c6ac9b241082464, []int{70} } func (m *FileServiceCacheRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4226,7 +4365,7 @@ func (m *FileServiceCacheResponse) Reset() { *m = FileServiceCacheRespon func (m *FileServiceCacheResponse) String() string { return proto.CompactTextString(m) } func (*FileServiceCacheResponse) ProtoMessage() {} func (*FileServiceCacheResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{69} + return fileDescriptor_5c6ac9b241082464, []int{71} } func (m *FileServiceCacheResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4284,7 +4423,7 @@ func (m *FileServiceCacheEvictRequest) Reset() { *m = FileServiceCacheEv func (m *FileServiceCacheEvictRequest) String() string { return proto.CompactTextString(m) } func (*FileServiceCacheEvictRequest) ProtoMessage() {} func (*FileServiceCacheEvictRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{70} + return fileDescriptor_5c6ac9b241082464, []int{72} } func (m *FileServiceCacheEvictRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4330,7 +4469,7 @@ func (m *FileServiceCacheEvictResponse) Reset() { *m = FileServiceCacheE func (m *FileServiceCacheEvictResponse) String() string { return proto.CompactTextString(m) } func (*FileServiceCacheEvictResponse) ProtoMessage() {} func (*FileServiceCacheEvictResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{71} + return fileDescriptor_5c6ac9b241082464, []int{73} } func (m *FileServiceCacheEvictResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4388,7 +4527,7 @@ func (m *MetadataCacheRequest) Reset() { *m = MetadataCacheRequest{} } func (m *MetadataCacheRequest) String() string { return proto.CompactTextString(m) } func (*MetadataCacheRequest) ProtoMessage() {} func (*MetadataCacheRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{72} + return fileDescriptor_5c6ac9b241082464, []int{74} } func (m *MetadataCacheRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4432,7 +4571,7 @@ func (m *MetadataCacheResponse) Reset() { *m = MetadataCacheResponse{} } func (m *MetadataCacheResponse) String() string { return proto.CompactTextString(m) } func (*MetadataCacheResponse) ProtoMessage() {} func (*MetadataCacheResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{73} + return fileDescriptor_5c6ac9b241082464, []int{75} } func (m *MetadataCacheResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4476,7 +4615,7 @@ func (m *GoGCPercentRequest) Reset() { *m = GoGCPercentRequest{} } func (m *GoGCPercentRequest) String() string { return proto.CompactTextString(m) } func (*GoGCPercentRequest) ProtoMessage() {} func (*GoGCPercentRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{74} + return fileDescriptor_5c6ac9b241082464, []int{76} } func (m *GoGCPercentRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4523,7 +4662,7 @@ func (m *GoGCPercentResponse) Reset() { *m = GoGCPercentResponse{} } func (m *GoGCPercentResponse) String() string { return proto.CompactTextString(m) } func (*GoGCPercentResponse) ProtoMessage() {} func (*GoGCPercentResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{75} + return fileDescriptor_5c6ac9b241082464, []int{77} } func (m *GoGCPercentResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4568,7 +4707,7 @@ func (m *FaultInjectRequest) Reset() { *m = FaultInjectRequest{} } func (m *FaultInjectRequest) String() string { return proto.CompactTextString(m) } func (*FaultInjectRequest) ProtoMessage() {} func (*FaultInjectRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{76} + return fileDescriptor_5c6ac9b241082464, []int{78} } func (m *FaultInjectRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4619,7 +4758,7 @@ func (m *FaultInjectResponse) Reset() { *m = FaultInjectResponse{} } func (m *FaultInjectResponse) String() string { return proto.CompactTextString(m) } func (*FaultInjectResponse) ProtoMessage() {} func (*FaultInjectResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{77} + return fileDescriptor_5c6ac9b241082464, []int{79} } func (m *FaultInjectResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4663,7 +4802,7 @@ func (m *CtlMoTableStatsRequest) Reset() { *m = CtlMoTableStatsRequest{} func (m *CtlMoTableStatsRequest) String() string { return proto.CompactTextString(m) } func (*CtlMoTableStatsRequest) ProtoMessage() {} func (*CtlMoTableStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{78} + return fileDescriptor_5c6ac9b241082464, []int{80} } func (m *CtlMoTableStatsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4707,7 +4846,7 @@ func (m *CtlMoTableStatsResponse) Reset() { *m = CtlMoTableStatsResponse func (m *CtlMoTableStatsResponse) String() string { return proto.CompactTextString(m) } func (*CtlMoTableStatsResponse) ProtoMessage() {} func (*CtlMoTableStatsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{79} + return fileDescriptor_5c6ac9b241082464, []int{81} } func (m *CtlMoTableStatsResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4752,7 +4891,7 @@ func (m *WorkspaceThresholdRequest) Reset() { *m = WorkspaceThresholdReq func (m *WorkspaceThresholdRequest) String() string { return proto.CompactTextString(m) } func (*WorkspaceThresholdRequest) ProtoMessage() {} func (*WorkspaceThresholdRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{80} + return fileDescriptor_5c6ac9b241082464, []int{82} } func (m *WorkspaceThresholdRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4804,7 +4943,7 @@ func (m *WorkspaceThresholdResponse) Reset() { *m = WorkspaceThresholdRe func (m *WorkspaceThresholdResponse) String() string { return proto.CompactTextString(m) } func (*WorkspaceThresholdResponse) ProtoMessage() {} func (*WorkspaceThresholdResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{81} + return fileDescriptor_5c6ac9b241082464, []int{83} } func (m *WorkspaceThresholdResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4856,7 +4995,7 @@ func (m *MinTimestampResponse) Reset() { *m = MinTimestampResponse{} } func (m *MinTimestampResponse) String() string { return proto.CompactTextString(m) } func (*MinTimestampResponse) ProtoMessage() {} func (*MinTimestampResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{82} + return fileDescriptor_5c6ac9b241082464, []int{84} } func (m *MinTimestampResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4922,6 +5061,8 @@ func init() { proto.RegisterType((*CtlReaderResponse)(nil), "query.CtlReaderResponse") proto.RegisterType((*CtlPrefetchOnSubscribedRequest)(nil), "query.CtlPrefetchOnSubscribedRequest") proto.RegisterType((*CtlPrefetchOnSubscribedResponse)(nil), "query.CtlPrefetchOnSubscribedResponse") + proto.RegisterType((*ISCPDrainConsumerRequest)(nil), "query.ISCPDrainConsumerRequest") + proto.RegisterType((*ISCPDrainConsumerResponse)(nil), "query.ISCPDrainConsumerResponse") proto.RegisterType((*TraceSpanRequest)(nil), "query.TraceSpanRequest") proto.RegisterType((*TraceSpanResponse)(nil), "query.TraceSpanResponse") proto.RegisterType((*GetLockInfoRequest)(nil), "query.GetLockInfoRequest") @@ -4983,223 +5124,231 @@ func init() { func init() { proto.RegisterFile("query.proto", fileDescriptor_5c6ac9b241082464) } var fileDescriptor_5c6ac9b241082464 = []byte{ - // 3448 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x5b, 0xdd, 0x72, 0xdb, 0xc6, - 0xf5, 0x37, 0xf5, 0x49, 0x1e, 0x7d, 0x41, 0x6b, 0x4a, 0x82, 0x64, 0x99, 0x92, 0x61, 0xc7, 0x76, - 0xec, 0x44, 0xf2, 0xdf, 0x89, 0xfd, 0x6f, 0x92, 0xb6, 0x13, 0x89, 0xb2, 0x14, 0xc5, 0x92, 0x25, - 0x2f, 0xe9, 0xd8, 0x71, 0x67, 0x3c, 0x03, 0x91, 0x2b, 0x09, 0x35, 0x09, 0x30, 0x00, 0x98, 0x48, - 0x99, 0xe9, 0x4c, 0x1f, 0x21, 0x97, 0xbd, 0x6c, 0x1f, 0xa3, 0x6f, 0x90, 0xcb, 0x5c, 0xe6, 0xaa, - 0xed, 0xc4, 0xef, 0xd1, 0xe9, 0xec, 0xe2, 0x2c, 0x80, 0x05, 0x16, 0x94, 0x93, 0x49, 0x6e, 0x34, - 0xd8, 0xb3, 0xe7, 0xfc, 0xf6, 0xec, 0x62, 0xf7, 0xe0, 0xb7, 0xe7, 0x50, 0x30, 0xf1, 0x55, 0x9f, - 0xf9, 0xe7, 0x6b, 0x3d, 0xdf, 0x0b, 0x3d, 0x32, 0x2a, 0x1a, 0x4b, 0x93, 0x41, 0x68, 0x87, 0xfd, - 0x20, 0x12, 0x2e, 0x41, 0xc7, 0x6b, 0xbd, 0xc6, 0xe7, 0x4a, 0x78, 0xe6, 0xe2, 0xe3, 0x4c, 0xe8, - 0x74, 0x59, 0x10, 0xda, 0xdd, 0x9e, 0x14, 0x70, 0xab, 0xc0, 0x71, 0x8f, 0x3d, 0x14, 0xbc, 0x7f, - 0xe2, 0x84, 0xa7, 0xfd, 0xa3, 0xb5, 0x96, 0xd7, 0x5d, 0x3f, 0xf1, 0x4e, 0xbc, 0x75, 0x21, 0x3e, - 0xea, 0x1f, 0x8b, 0x96, 0x68, 0x88, 0x27, 0x54, 0x5f, 0x39, 0xf1, 0xbc, 0x93, 0x0e, 0x4b, 0xb4, - 0x32, 0x03, 0x58, 0x37, 0x60, 0xf2, 0x29, 0xf7, 0x8f, 0xb2, 0xaf, 0xfa, 0x2c, 0x08, 0x49, 0x15, - 0x46, 0x45, 0xdb, 0x2c, 0xad, 0x96, 0x6e, 0x57, 0x68, 0xd4, 0xb0, 0x9e, 0xc0, 0x7c, 0xe3, 0xd4, - 0xfb, 0xe6, 0xd0, 0xf7, 0x5a, 0x2c, 0x08, 0xf6, 0x9c, 0x20, 0x94, 0xfa, 0xf3, 0x30, 0xd6, 0x64, - 0xae, 0xed, 0x86, 0x68, 0x80, 0x2d, 0xb2, 0x0c, 0x95, 0xc6, 0x79, 0x80, 0x5d, 0x43, 0xab, 0xa5, - 0xdb, 0x65, 0x9a, 0x08, 0xac, 0xe7, 0x30, 0xdb, 0x38, 0x77, 0x5b, 0x75, 0xaf, 0xdb, 0x75, 0x62, - 0xa8, 0x4d, 0x98, 0xde, 0xb3, 0x43, 0x16, 0x84, 0x91, 0xb8, 0xd9, 0x10, 0x90, 0x13, 0xf7, 0xab, - 0x6b, 0x89, 0xd3, 0x4d, 0xf9, 0xb4, 0x39, 0xf2, 0xfd, 0xbf, 0x56, 0x2e, 0xd1, 0x8c, 0x85, 0xf5, - 0x12, 0x48, 0x1a, 0x38, 0xe8, 0x79, 0x6e, 0xc0, 0xc8, 0x16, 0xcc, 0xd4, 0xfb, 0xbe, 0xcf, 0xdc, - 0x9f, 0x03, 0x9d, 0x35, 0xb1, 0x08, 0x18, 0x3b, 0x2c, 0x54, 0x7c, 0xb6, 0xbe, 0x84, 0xd9, 0x94, - 0xec, 0x57, 0x1d, 0x6e, 0x1d, 0xe6, 0xea, 0x9e, 0xcf, 0xb6, 0xfa, 0xdd, 0x5e, 0xdd, 0x73, 0x8f, - 0x9d, 0x93, 0xd4, 0x92, 0x6f, 0xb4, 0x42, 0xc7, 0x73, 0xe5, 0x92, 0x47, 0x2d, 0xcb, 0x84, 0xf9, - 0xac, 0x41, 0xe4, 0x90, 0x75, 0x05, 0x16, 0x77, 0x58, 0x78, 0xc8, 0x5f, 0x78, 0xcb, 0xeb, 0x7c, - 0xc1, 0xfc, 0xc0, 0xf1, 0x5c, 0x39, 0x85, 0x87, 0xb0, 0xa4, 0xeb, 0xc4, 0xb9, 0x98, 0x30, 0x8e, - 0x22, 0x31, 0xda, 0x30, 0x95, 0x4d, 0xeb, 0x01, 0x2c, 0x36, 0x8a, 0x40, 0x07, 0x98, 0x3d, 0x84, - 0xa5, 0xc6, 0x2f, 0x19, 0xee, 0x3d, 0x98, 0xa6, 0x7d, 0xb7, 0x69, 0x07, 0xaf, 0xe5, 0x18, 0x4b, - 0x50, 0xe6, 0xcd, 0xba, 0xd7, 0x66, 0x42, 0x79, 0x94, 0xc6, 0x6d, 0xeb, 0x5d, 0x98, 0x89, 0xb5, - 0x11, 0x7a, 0x1e, 0xc6, 0x28, 0x0b, 0xfa, 0x9d, 0x78, 0xa7, 0x46, 0x2d, 0xbe, 0x6c, 0x7c, 0xfe, - 0x4e, 0x8f, 0x75, 0x1c, 0x97, 0xed, 0xba, 0xc7, 0x9e, 0x5c, 0x99, 0x75, 0x58, 0xc8, 0xf5, 0x20, - 0x58, 0x15, 0x46, 0xeb, 0x5e, 0x1f, 0x77, 0xfd, 0x30, 0x8d, 0x1a, 0xd6, 0x7f, 0xe7, 0x61, 0x5c, - 0x7a, 0xb7, 0x0c, 0x15, 0x7c, 0xdc, 0xdd, 0x12, 0x5a, 0x23, 0x34, 0x11, 0x90, 0x35, 0xa8, 0xd4, - 0xbb, 0xed, 0x7d, 0x16, 0x9e, 0x7a, 0x6d, 0x71, 0x3c, 0xa6, 0xef, 0x1b, 0x6b, 0x51, 0xd4, 0x88, - 0xe5, 0x34, 0x51, 0x21, 0xff, 0xaf, 0x1e, 0x53, 0x73, 0x58, 0xec, 0xa7, 0xcb, 0x68, 0x92, 0xee, - 0xa2, 0xea, 0x79, 0x7e, 0x56, 0x74, 0x72, 0xcd, 0x11, 0x01, 0x71, 0x15, 0x21, 0xf4, 0x4a, 0xb4, - 0xe8, 0xd8, 0xef, 0xc1, 0xe5, 0x8d, 0x4e, 0xc8, 0xfc, 0x8d, 0x56, 0x8b, 0xcf, 0x5c, 0x62, 0x8e, - 0x0a, 0xcc, 0x25, 0xc4, 0xd4, 0x68, 0x50, 0x9d, 0x19, 0xf9, 0x14, 0x66, 0x1e, 0x3b, 0x9d, 0x4e, - 0xdd, 0x73, 0xe5, 0x06, 0x32, 0xc7, 0x04, 0xd2, 0x3c, 0x22, 0x65, 0x7a, 0x69, 0x56, 0x9d, 0xd4, - 0xc1, 0x68, 0xfa, 0x76, 0x8b, 0x35, 0x7a, 0x76, 0x0c, 0x31, 0x2e, 0x20, 0x16, 0x10, 0x22, 0xdb, - 0x4d, 0x73, 0x06, 0x64, 0x17, 0xc8, 0x0e, 0x0b, 0xf7, 0xbc, 0xd6, 0xeb, 0xd4, 0x2e, 0x30, 0xcb, - 0x02, 0x66, 0x11, 0x61, 0xf2, 0x0a, 0x54, 0x63, 0x44, 0xb6, 0x45, 0x5c, 0x68, 0x9e, 0xb9, 0x69, - 0xa4, 0x8a, 0x40, 0x32, 0x13, 0x24, 0xb5, 0x9f, 0xe6, 0x4d, 0xf8, 0x3a, 0xf3, 0xf8, 0x62, 0xb7, - 0x4e, 0xd3, 0x3b, 0xd3, 0x04, 0x65, 0x9d, 0x35, 0x1a, 0x54, 0x67, 0x46, 0x7e, 0x07, 0xd0, 0x38, - 0x6f, 0xb9, 0x51, 0x88, 0x31, 0x27, 0x14, 0x77, 0x72, 0xf1, 0x98, 0xa6, 0x74, 0xc9, 0x03, 0xa8, - 0xc4, 0x71, 0xce, 0x9c, 0x54, 0x16, 0x36, 0x1b, 0x13, 0x69, 0xa2, 0x49, 0x0e, 0xc5, 0x8a, 0x66, - 0x0e, 0xbb, 0x39, 0x25, 0xec, 0x57, 0x13, 0x7b, 0x7d, 0x10, 0xa1, 0x1a, 0x5b, 0x8e, 0x98, 0x0f, - 0x1f, 0xe6, 0xb4, 0x82, 0xd8, 0x28, 0x46, 0xcc, 0x77, 0x91, 0x2d, 0x98, 0x56, 0xc3, 0xa6, 0x39, - 0x23, 0xd0, 0x96, 0xe5, 0x79, 0xd4, 0x05, 0x61, 0x9a, 0xb1, 0x21, 0xeb, 0x30, 0x8e, 0x01, 0xc7, - 0x34, 0x84, 0xf9, 0x1c, 0x9a, 0xab, 0x41, 0x8b, 0x4a, 0x2d, 0xf2, 0x25, 0xcc, 0x51, 0xd6, 0xf5, - 0xbe, 0x66, 0xfc, 0x6f, 0xc8, 0xf8, 0x06, 0x6a, 0xda, 0x47, 0x1d, 0x66, 0xce, 0x0a, 0xf3, 0xeb, - 0xd2, 0x5c, 0xa7, 0x23, 0xc1, 0xf4, 0x08, 0x64, 0x03, 0xa6, 0xf8, 0x96, 0x14, 0x5f, 0xc6, 0x4d, - 0xc7, 0x6d, 0x9b, 0x44, 0x40, 0x5e, 0x49, 0x6d, 0xe1, 0xb8, 0x4f, 0x42, 0xa9, 0x16, 0xe4, 0x73, - 0x30, 0x9e, 0xb9, 0x41, 0xff, 0x28, 0x68, 0xf9, 0xce, 0x11, 0x8b, 0x1c, 0xbb, 0x2c, 0x50, 0x6a, - 0x88, 0x92, 0xed, 0x8e, 0x8f, 0x55, 0xb6, 0x23, 0xbd, 0x87, 0xb7, 0xec, 0xd0, 0x96, 0x7b, 0xb8, - 0xaa, 0xdd, 0xc3, 0x29, 0x0d, 0xaa, 0x33, 0x43, 0xb4, 0x06, 0xa7, 0x45, 0xe9, 0x13, 0x31, 0x97, - 0x45, 0xcb, 0x6a, 0x50, 0x9d, 0x19, 0x0f, 0x8f, 0xfa, 0xe0, 0x6f, 0xce, 0x2b, 0xe1, 0x51, 0xaf, - 0x44, 0x0b, 0x8c, 0x39, 0xec, 0xbe, 0x73, 0xe2, 0xdb, 0x21, 0xe3, 0x41, 0x6a, 0xdb, 0xf7, 0xba, - 0x12, 0x76, 0x41, 0x81, 0xd5, 0x2b, 0xd1, 0x02, 0x63, 0x72, 0x00, 0xd5, 0x54, 0x4f, 0x33, 0xf6, - 0xd5, 0x54, 0xde, 0xaf, 0x4e, 0x85, 0x6a, 0x0d, 0xc9, 0x11, 0x98, 0x94, 0x75, 0x3c, 0xbb, 0xbd, - 0xd1, 0x0f, 0xbd, 0x5d, 0xb7, 0xe5, 0xb3, 0x2e, 0xa7, 0x20, 0x7c, 0xcd, 0xcd, 0x45, 0x01, 0x7a, - 0x33, 0xde, 0x87, 0x7a, 0x35, 0x89, 0x5f, 0x88, 0xc3, 0x43, 0x73, 0x3d, 0xec, 0x50, 0x66, 0xb7, - 0x99, 0x2f, 0x1d, 0x5e, 0x52, 0x22, 0x48, 0xb6, 0x9b, 0xe6, 0x0c, 0xc8, 0x3e, 0xcc, 0xec, 0xb0, - 0x90, 0xb2, 0x5e, 0xc7, 0x69, 0xd9, 0xd1, 0x97, 0xf7, 0x4a, 0xf6, 0x05, 0xa5, 0x7b, 0xd1, 0x4e, - 0x72, 0xab, 0x4c, 0x2f, 0xdf, 0x44, 0x94, 0x05, 0x2c, 0x6c, 0xb0, 0x20, 0x15, 0x1e, 0xcc, 0x65, - 0x65, 0x13, 0x69, 0x34, 0xa8, 0xce, 0x8c, 0xec, 0xc1, 0xec, 0x8e, 0xb7, 0x6f, 0x9f, 0xf1, 0xef, - 0x64, 0x20, 0xb1, 0xae, 0xaa, 0xc1, 0x3e, 0xdb, 0x8f, 0x9e, 0xe5, 0x0d, 0x11, 0x8d, 0x75, 0xf7, - 0x9c, 0x24, 0xa6, 0x9a, 0xb5, 0x2c, 0x9a, 0xda, 0x9f, 0x42, 0x53, 0x3b, 0xc8, 0x2b, 0x58, 0xd8, - 0x76, 0x3a, 0xac, 0xc1, 0xfc, 0xaf, 0x9d, 0x16, 0x4b, 0xbf, 0x32, 0x73, 0x45, 0x39, 0xcf, 0x05, - 0x5a, 0x88, 0x5c, 0x04, 0x42, 0xba, 0xb0, 0x9c, 0xed, 0x7a, 0xf4, 0xb5, 0xd3, 0x8a, 0x1d, 0x5f, - 0x55, 0xa2, 0xd9, 0x20, 0x55, 0x1c, 0x69, 0x20, 0x1c, 0x79, 0x06, 0xd5, 0x7d, 0x16, 0xda, 0x6d, - 0x3b, 0xb4, 0x95, 0xb9, 0x5c, 0x53, 0x4f, 0x80, 0x46, 0x05, 0xe1, 0xb5, 0xe6, 0xe4, 0x00, 0xc8, - 0x8e, 0xb7, 0x53, 0x3f, 0x64, 0x7e, 0x8b, 0x25, 0x6c, 0xc6, 0x52, 0xbf, 0xfc, 0x39, 0x05, 0x84, - 0xd4, 0x98, 0x72, 0x2a, 0xb1, 0x6d, 0xf7, 0x3b, 0xe1, 0xae, 0xfb, 0x67, 0x96, 0x2c, 0xc6, 0x75, - 0x05, 0x30, 0xaf, 0x40, 0x35, 0x46, 0xe4, 0x4f, 0x30, 0x5f, 0x0f, 0x3b, 0xfb, 0x9e, 0x08, 0xa6, - 0x22, 0x80, 0x49, 0xb8, 0x1b, 0xca, 0x09, 0xd0, 0x2b, 0xa1, 0x8f, 0x05, 0x10, 0xe4, 0x15, 0x2c, - 0x3e, 0xf7, 0xfc, 0xd7, 0x41, 0xcf, 0x6e, 0xb1, 0xe6, 0xa9, 0xcf, 0x82, 0x53, 0xaf, 0x23, 0xbf, - 0x09, 0xe6, 0x3b, 0xca, 0x57, 0xb5, 0x50, 0x8f, 0x16, 0x43, 0x90, 0x2e, 0xd4, 0xea, 0x61, 0xe7, - 0xd0, 0x67, 0xc7, 0x2c, 0x6c, 0x9d, 0x1e, 0xb8, 0x0d, 0xf9, 0x69, 0x88, 0x07, 0xb9, 0x29, 0x06, - 0x79, 0x27, 0x99, 0xc4, 0x00, 0x65, 0x7a, 0x01, 0x98, 0xb5, 0x0d, 0x0b, 0x39, 0xc2, 0x8a, 0x8c, - 0xfd, 0x2e, 0x94, 0xf1, 0xd8, 0x06, 0x66, 0x69, 0x75, 0xf8, 0xf6, 0xc4, 0xfd, 0x99, 0x35, 0xbc, - 0x92, 0xcb, 0xe3, 0x1c, 0x2b, 0x58, 0x7f, 0x35, 0xa1, 0x1c, 0x5b, 0xfe, 0xba, 0x4c, 0xbe, 0x0a, - 0xa3, 0x8f, 0x7c, 0xdf, 0xf3, 0x05, 0x85, 0x9f, 0xa4, 0x51, 0x83, 0xbc, 0x28, 0x74, 0x1c, 0x79, - 0x7a, 0xad, 0x88, 0xa7, 0x47, 0x5a, 0xb4, 0x70, 0xde, 0x07, 0x50, 0x55, 0x29, 0x37, 0xc2, 0x8e, - 0x2a, 0x27, 0x46, 0xa7, 0x42, 0xb5, 0x86, 0x3c, 0x9e, 0x27, 0xec, 0x1b, 0xc1, 0xc6, 0x94, 0x78, - 0x9e, 0xed, 0xa6, 0x39, 0x03, 0xce, 0x8f, 0x53, 0xf4, 0x1b, 0x51, 0xc6, 0x95, 0x20, 0x97, 0xeb, - 0xa7, 0x79, 0x13, 0x64, 0x03, 0x09, 0xfb, 0x46, 0xa4, 0x72, 0x96, 0x0d, 0x64, 0x35, 0xa8, 0xce, - 0x0c, 0x2f, 0x00, 0x31, 0x05, 0x47, 0xb0, 0x4a, 0xf6, 0x02, 0x90, 0x51, 0xa0, 0x1a, 0x23, 0xbe, - 0xec, 0x2a, 0x03, 0x47, 0x30, 0xc8, 0x52, 0xb1, 0x9c, 0x0a, 0xd5, 0x1a, 0x92, 0x8f, 0x38, 0x77, - 0x97, 0x14, 0x1d, 0xb9, 0xfb, 0xa2, 0x86, 0xbb, 0x23, 0x48, 0x4a, 0x99, 0x3c, 0xcc, 0x93, 0x77, - 0x33, 0x4f, 0xde, 0xd1, 0x30, 0xc5, 0xde, 0x9f, 0x0e, 0x60, 0xef, 0xd7, 0x06, 0xb0, 0xf7, 0xd4, - 0xb2, 0x64, 0xc9, 0xf6, 0xd3, 0x01, 0xf4, 0xfd, 0xda, 0x00, 0xfa, 0x2e, 0x21, 0x35, 0xfc, 0xfd, - 0x51, 0x01, 0x7f, 0xbf, 0x5a, 0xc0, 0xdf, 0x11, 0x2a, 0x4b, 0xe0, 0xef, 0x65, 0x09, 0xfc, 0x7c, - 0x96, 0xc0, 0xa3, 0x61, 0xcc, 0xe0, 0x5f, 0x0e, 0x66, 0xf0, 0x37, 0x06, 0x33, 0x78, 0x44, 0x2b, - 0xa0, 0xf0, 0x9b, 0x7a, 0x0a, 0xbf, 0xac, 0xa7, 0xf0, 0x88, 0x95, 0xe1, 0xf0, 0x8f, 0x0b, 0x39, - 0xfc, 0x4a, 0x21, 0x87, 0x97, 0x07, 0x36, 0x47, 0xe2, 0x53, 0xfb, 0x39, 0x62, 0xe3, 0xb8, 0x9f, - 0xab, 0xda, 0xfd, 0x9c, 0x56, 0xa1, 0x5a, 0x43, 0x04, 0x4c, 0x11, 0x72, 0x04, 0x9c, 0xcb, 0x02, - 0xe6, 0x54, 0xa8, 0xd6, 0x90, 0x87, 0xd0, 0x82, 0x6c, 0x0d, 0x72, 0xf9, 0x5a, 0x11, 0x97, 0x97, - 0x21, 0xb4, 0x28, 0xd9, 0xf3, 0x02, 0x16, 0x72, 0x84, 0x1c, 0x91, 0x17, 0x14, 0xe4, 0x02, 0x2d, - 0x5a, 0x64, 0x4e, 0x28, 0xcc, 0x65, 0x78, 0x39, 0xe2, 0x9a, 0xca, 0xeb, 0xd6, 0xea, 0x50, 0xbd, - 0x29, 0x69, 0x5d, 0xc8, 0xe9, 0x6f, 0x5d, 0xc8, 0xe9, 0x71, 0x84, 0x62, 0x52, 0xbf, 0x0d, 0xb3, - 0x29, 0x8e, 0x8e, 0x4e, 0x2f, 0x29, 0xa1, 0x25, 0xd7, 0x4f, 0xf3, 0x26, 0xe4, 0x49, 0x11, 0xaf, - 0xaf, 0x15, 0xf1, 0xfa, 0xc8, 0xb0, 0x88, 0xd8, 0x1f, 0x40, 0x55, 0x65, 0xe8, 0xe8, 0xda, 0xb2, - 0xb2, 0xab, 0x74, 0x2a, 0x54, 0x6b, 0x18, 0x31, 0xc3, 0x84, 0xa2, 0x23, 0xdc, 0xd5, 0x0c, 0x33, - 0xcc, 0x2a, 0x24, 0xcc, 0x30, 0xdb, 0x83, 0x80, 0x31, 0x4b, 0x47, 0xc0, 0x5a, 0x16, 0x30, 0xa3, - 0x90, 0x02, 0xcc, 0xf4, 0x10, 0x1b, 0xcc, 0x3c, 0x39, 0x47, 0xd8, 0x15, 0xe5, 0xb8, 0x17, 0xa9, - 0x21, 0x78, 0x21, 0x0c, 0xe9, 0xc1, 0xd5, 0x02, 0x56, 0x8e, 0xe3, 0xac, 0x2a, 0x11, 0x6f, 0xa0, - 0x2e, 0x0e, 0x36, 0x18, 0x90, 0xbc, 0x80, 0xb9, 0x0c, 0x51, 0xc7, 0x91, 0xae, 0xa9, 0x07, 0x43, - 0xa7, 0x83, 0x23, 0xe8, 0x01, 0x08, 0x85, 0xcb, 0x0a, 0x5f, 0x47, 0x5c, 0x4b, 0x65, 0x0c, 0x79, - 0x0d, 0x44, 0xd5, 0x19, 0x73, 0x16, 0xa2, 0x10, 0x77, 0xc4, 0xbc, 0xae, 0x60, 0x6a, 0x34, 0xa8, - 0xce, 0x8c, 0x5f, 0xd9, 0x72, 0x6c, 0x1d, 0x11, 0x6f, 0x28, 0x67, 0xa3, 0x40, 0x4b, 0x5e, 0xd9, - 0x0a, 0xba, 0x89, 0x0d, 0x4b, 0x3a, 0xc2, 0x8e, 0x43, 0xbc, 0xa3, 0x7c, 0x8b, 0x8b, 0x15, 0xe9, - 0x00, 0x90, 0x28, 0x51, 0xe1, 0xc6, 0x25, 0x8e, 0x18, 0xfc, 0x66, 0x26, 0x51, 0x91, 0x57, 0xa1, - 0x5a, 0x43, 0xd2, 0x83, 0x95, 0x42, 0xea, 0x8f, 0xd8, 0xb7, 0x94, 0x7c, 0xc5, 0x05, 0xda, 0xf4, - 0x22, 0x38, 0x6b, 0x57, 0x9b, 0xe1, 0x16, 0x45, 0x07, 0x51, 0xc3, 0xda, 0x6d, 0x63, 0xee, 0x3f, - 0x6e, 0x93, 0x79, 0x18, 0x6b, 0x88, 0x1b, 0x85, 0xe0, 0xf6, 0x15, 0x8a, 0x2d, 0xeb, 0x63, 0x3d, - 0x05, 0x27, 0x16, 0x4c, 0xda, 0x5c, 0xde, 0xe8, 0xb7, 0x38, 0x6d, 0x17, 0x78, 0x65, 0xaa, 0xc8, - 0xac, 0xdd, 0x5c, 0x6a, 0x9c, 0xdf, 0x47, 0x10, 0x09, 0xef, 0x23, 0xc3, 0x34, 0x11, 0xa4, 0x2b, - 0x28, 0x43, 0xe2, 0xae, 0x92, 0xaa, 0xa0, 0xe4, 0x79, 0xb8, 0x09, 0xe3, 0xea, 0xe8, 0xb2, 0x69, - 0xed, 0xe5, 0xd3, 0x36, 0xc4, 0x80, 0xe1, 0x7a, 0xb7, 0x8d, 0xf5, 0x13, 0xfe, 0x28, 0x24, 0xc7, - 0x27, 0x62, 0x24, 0x2e, 0x39, 0x3e, 0x11, 0xf7, 0x9b, 0xb3, 0xd0, 0xb7, 0xe3, 0xfb, 0x0d, 0x6f, - 0x58, 0xb7, 0x34, 0xdf, 0x0b, 0x42, 0x60, 0x84, 0x3f, 0x23, 0x9e, 0x78, 0xb6, 0x7e, 0x7f, 0xd1, - 0x85, 0x91, 0xbf, 0x81, 0x43, 0x3b, 0x0c, 0x99, 0x8f, 0x17, 0xb9, 0x0a, 0x8d, 0xdb, 0xd6, 0x83, - 0x0b, 0xb7, 0x89, 0x76, 0xd0, 0x17, 0xf9, 0xea, 0x81, 0x66, 0xae, 0x55, 0x18, 0xe5, 0x0a, 0x01, - 0xce, 0x36, 0x6a, 0xf0, 0xb7, 0x11, 0xef, 0x7f, 0x31, 0xe7, 0x61, 0x9a, 0x08, 0xf8, 0xbc, 0xf3, - 0x97, 0x16, 0x9d, 0x0b, 0x55, 0x5d, 0xed, 0xc1, 0xfa, 0xb1, 0x04, 0x65, 0x29, 0xe3, 0xef, 0x4a, - 0x9c, 0x66, 0xdc, 0x79, 0x23, 0x54, 0x36, 0x39, 0xe0, 0x63, 0x76, 0xce, 0x1d, 0x1b, 0xbe, 0x3d, - 0x49, 0xc5, 0x33, 0xb9, 0x13, 0x59, 0xee, 0x7b, 0x6d, 0x26, 0xdc, 0x9a, 0xbe, 0x3f, 0xbd, 0x26, - 0x8a, 0xce, 0x52, 0x4a, 0xe3, 0x7e, 0xb2, 0x0a, 0x13, 0x4e, 0x40, 0x6d, 0xf7, 0x44, 0x30, 0x50, - 0x71, 0xe3, 0x2c, 0xd3, 0xb4, 0x88, 0xdc, 0x82, 0xf1, 0xcf, 0xbc, 0x4e, 0x9b, 0xf9, 0x81, 0x39, - 0x2a, 0x2e, 0xcf, 0x53, 0x11, 0xd8, 0x73, 0xdb, 0xe1, 0x57, 0x1f, 0x2a, 0x7b, 0xb9, 0x22, 0x97, - 0x71, 0xc5, 0x31, 0xad, 0x22, 0xf6, 0x5a, 0xaf, 0xb4, 0x37, 0x37, 0x3e, 0x95, 0xba, 0xbb, 0x2b, - 0xd7, 0x5d, 0x3c, 0x93, 0x0f, 0x60, 0x52, 0xea, 0xf1, 0xab, 0xad, 0x98, 0x26, 0xbf, 0xbe, 0x47, - 0x27, 0x3d, 0x86, 0x50, 0x94, 0xac, 0xcb, 0x9a, 0x0a, 0x8c, 0x75, 0x0a, 0x13, 0xcd, 0x33, 0xf7, - 0xed, 0x56, 0x94, 0x7a, 0xdf, 0xc4, 0x2b, 0xca, 0x9f, 0xc9, 0x5d, 0x18, 0x3f, 0xe8, 0x85, 0x22, - 0x81, 0x10, 0x95, 0xdf, 0x66, 0x93, 0x05, 0xc5, 0x0e, 0x2a, 0x35, 0xac, 0x7f, 0x96, 0x60, 0x1c, - 0x07, 0x27, 0x9f, 0x42, 0xb9, 0xee, 0x33, 0x3b, 0x64, 0x1b, 0x21, 0x16, 0x82, 0x97, 0xd6, 0xa2, - 0xba, 0xfc, 0x9a, 0xac, 0xcb, 0xa7, 0xca, 0xc1, 0x65, 0x1e, 0xbd, 0xbf, 0xfb, 0xf7, 0x4a, 0x89, - 0xc6, 0x56, 0x64, 0x15, 0x46, 0xf8, 0xd7, 0x4c, 0xec, 0xbc, 0x89, 0xfb, 0x93, 0x6b, 0xe1, 0x99, - 0xbb, 0xd6, 0x3c, 0x73, 0xb9, 0x8c, 0x8a, 0x1e, 0x3e, 0x95, 0x67, 0x01, 0xf3, 0x9b, 0x67, 0xae, - 0x70, 0xae, 0x4c, 0x65, 0x93, 0xdc, 0x83, 0x0a, 0x5f, 0x73, 0xee, 0x65, 0x60, 0x8e, 0x88, 0xa5, - 0x23, 0xf2, 0x8a, 0x9d, 0xac, 0x05, 0x4d, 0x94, 0xac, 0x97, 0xba, 0x6b, 0xb0, 0xf6, 0xcd, 0xdc, - 0x13, 0xeb, 0x99, 0x79, 0x31, 0xd3, 0x09, 0xba, 0x00, 0x48, 0xab, 0x58, 0x73, 0xda, 0x82, 0x96, - 0xf5, 0x8f, 0x12, 0x54, 0x62, 0x21, 0x3f, 0xe2, 0x4f, 0xbc, 0x36, 0x6b, 0x9e, 0xf7, 0x18, 0x0e, - 0x17, 0xb7, 0x79, 0x90, 0xe5, 0xcf, 0xbb, 0x6d, 0x3c, 0x86, 0xd8, 0xe2, 0xe7, 0x50, 0x00, 0x08, - 0xa3, 0x28, 0xfe, 0x26, 0x02, 0xee, 0xfc, 0xb3, 0x80, 0xb5, 0xc5, 0xd6, 0x1e, 0xa1, 0xe2, 0x99, - 0xcb, 0xb6, 0x7d, 0x16, 0x65, 0x42, 0x46, 0xa8, 0x78, 0xe6, 0x23, 0x7f, 0xe6, 0x84, 0xd4, 0x0e, - 0x1d, 0x4f, 0x24, 0x35, 0x86, 0x68, 0xdc, 0xb6, 0x9e, 0xe8, 0xaf, 0xf4, 0xe4, 0x21, 0x4c, 0xc5, - 0x42, 0xb1, 0x0c, 0x51, 0x7a, 0x29, 0xce, 0x02, 0xc5, 0x06, 0xaa, 0x9a, 0xd5, 0x81, 0xe5, 0x41, - 0xd5, 0x1d, 0xfe, 0x4a, 0x77, 0x7c, 0xaf, 0xdf, 0xc3, 0x28, 0x3f, 0x45, 0x65, 0x33, 0xd9, 0xb7, - 0x5b, 0x32, 0xc6, 0x63, 0x33, 0x1d, 0xfd, 0x87, 0xd5, 0xe8, 0xff, 0x00, 0xae, 0x0e, 0xbc, 0x89, - 0xaa, 0x25, 0xed, 0x51, 0x59, 0xd2, 0xfe, 0x5c, 0x4c, 0x3a, 0x57, 0x2f, 0xfa, 0x25, 0xce, 0x59, - 0x77, 0x61, 0x4e, 0x7b, 0x71, 0xe5, 0x6f, 0x42, 0x5c, 0x72, 0x71, 0x6b, 0xf1, 0x67, 0xab, 0x01, - 0x0b, 0x05, 0x25, 0x26, 0x52, 0x03, 0xe0, 0x57, 0xc9, 0x23, 0x3b, 0x60, 0x71, 0x46, 0x2e, 0x25, - 0x19, 0xe0, 0xc1, 0x87, 0x60, 0x16, 0xdd, 0x79, 0x07, 0x7c, 0x0a, 0xb7, 0xa1, 0x2c, 0xde, 0xdc, - 0x63, 0x76, 0xce, 0x5d, 0x3d, 0xb4, 0xc3, 0x53, 0xe9, 0x2a, 0x7f, 0xe6, 0x5b, 0xf2, 0xe0, 0xf8, - 0x38, 0x60, 0xd1, 0x0f, 0x5d, 0x86, 0x29, 0xb6, 0xc8, 0x34, 0x0c, 0x35, 0xbe, 0xc5, 0x6f, 0xc2, - 0x50, 0xe3, 0x5b, 0xeb, 0x21, 0x6e, 0x51, 0x11, 0x9f, 0xdf, 0x85, 0x91, 0xd7, 0x3c, 0x66, 0x97, - 0x94, 0x60, 0x26, 0xfb, 0x91, 0xc3, 0x09, 0x15, 0xab, 0x09, 0x33, 0x38, 0xf5, 0xd8, 0x8d, 0x2a, - 0x8c, 0xee, 0xba, 0x6d, 0x76, 0x26, 0x5f, 0x96, 0x68, 0x90, 0xbb, 0x89, 0xa3, 0x18, 0x2a, 0xb2, - 0xb8, 0x34, 0x56, 0xb0, 0x9e, 0x6b, 0xcb, 0x72, 0xe4, 0xd3, 0xdc, 0x60, 0xe8, 0x62, 0x9c, 0x0f, - 0x51, 0x7b, 0x69, 0x56, 0xdd, 0x3a, 0x80, 0x59, 0xb9, 0xa8, 0x31, 0x7a, 0x81, 0xc3, 0x06, 0x0c, - 0x7f, 0xe6, 0xc8, 0xdf, 0x07, 0xf1, 0x47, 0xbe, 0xbe, 0x5c, 0x1f, 0xd9, 0x83, 0x78, 0xb6, 0x5e, - 0xe9, 0x73, 0x0f, 0xfc, 0x12, 0x9a, 0x1b, 0x08, 0x9d, 0x35, 0x93, 0x9b, 0x9e, 0xda, 0x4f, 0xf3, - 0x26, 0x16, 0xd5, 0x96, 0x14, 0xc9, 0x27, 0x30, 0x19, 0xcb, 0xa2, 0x65, 0x88, 0x92, 0x9c, 0xc9, - 0x4f, 0xb2, 0xd2, 0xdd, 0x54, 0x51, 0xc6, 0x73, 0x93, 0xcf, 0x52, 0xdc, 0x87, 0x4a, 0x2c, 0x8c, - 0x7f, 0x15, 0xa4, 0x41, 0xa4, 0x89, 0x9a, 0xd5, 0x80, 0x89, 0x43, 0x9f, 0xf5, 0x6c, 0x9f, 0x35, - 0xc2, 0xae, 0x58, 0xa2, 0x27, 0x76, 0x57, 0x46, 0x46, 0xf1, 0xcc, 0x17, 0xb2, 0xf1, 0x74, 0x4f, - 0xf2, 0xb0, 0xc6, 0xd3, 0x3d, 0x7e, 0x48, 0x0e, 0x6d, 0xdf, 0xee, 0xf2, 0xf0, 0x17, 0xe0, 0x72, - 0xa6, 0x24, 0xd6, 0xbd, 0xa2, 0x12, 0x25, 0xdf, 0xce, 0x5c, 0x14, 0x9f, 0x6c, 0x6c, 0x59, 0x76, - 0x61, 0x1a, 0x84, 0xef, 0xf4, 0xad, 0x4d, 0x74, 0x68, 0x68, 0x6b, 0x93, 0x3c, 0x84, 0xc9, 0x94, - 0xc7, 0x01, 0x7e, 0x18, 0xe4, 0x67, 0x27, 0xd5, 0x45, 0x15, 0x3d, 0xeb, 0x6f, 0x25, 0x7d, 0x85, - 0xb3, 0xc8, 0x27, 0x1c, 0x78, 0x28, 0x1e, 0x78, 0x15, 0x26, 0x1a, 0x2c, 0xfc, 0xc2, 0xf6, 0xa3, - 0x71, 0x87, 0x05, 0x3f, 0x4c, 0x8b, 0x72, 0xae, 0x8d, 0xbc, 0xa5, 0x6b, 0xff, 0x57, 0x90, 0xaa, - 0x19, 0x10, 0x37, 0x3e, 0x81, 0x95, 0x0b, 0xca, 0xa6, 0xe9, 0x50, 0x55, 0x52, 0x43, 0x95, 0x05, - 0xab, 0x17, 0xe5, 0x67, 0xac, 0xdb, 0xa2, 0x7a, 0xad, 0xa9, 0x7b, 0xf2, 0x75, 0xa9, 0x3f, 0x91, - 0x2f, 0xa4, 0xfe, 0x04, 0x7f, 0xca, 0xa4, 0xcb, 0xa4, 0x14, 0xfc, 0x94, 0xe9, 0x7d, 0x6d, 0x85, - 0xb4, 0x70, 0x6f, 0x1c, 0xea, 0xf3, 0x2e, 0xc5, 0x8b, 0xc3, 0xf7, 0xe7, 0x46, 0x3f, 0x3c, 0x6d, - 0x84, 0xbe, 0xe3, 0x46, 0x17, 0x88, 0x49, 0x9a, 0x92, 0x58, 0xeb, 0x9a, 0xa2, 0x2a, 0xff, 0x3c, - 0x4b, 0x91, 0xfc, 0xc9, 0x97, 0x6c, 0x5b, 0xf7, 0x74, 0x99, 0x9a, 0x81, 0x16, 0x1f, 0x69, 0x2a, - 0xad, 0xe4, 0x06, 0x4c, 0x49, 0xd1, 0xe6, 0x79, 0xc8, 0x02, 0x5c, 0x16, 0x55, 0x68, 0x7d, 0xac, - 0xcb, 0xe2, 0xbc, 0xa5, 0xed, 0x69, 0x61, 0x49, 0x96, 0xac, 0xc3, 0x48, 0x4c, 0x7a, 0xa6, 0xe3, - 0x7b, 0x72, 0x56, 0x9b, 0xab, 0x50, 0xa1, 0x18, 0xb3, 0x9e, 0x86, 0xf3, 0x2d, 0xc3, 0xaf, 0x4f, - 0x22, 0xb0, 0xce, 0x8a, 0x53, 0x43, 0xaa, 0x65, 0x29, 0x63, 0xc9, 0x67, 0x22, 0x1a, 0x75, 0xbb, - 0x67, 0xb7, 0x9c, 0xf0, 0x1c, 0xb1, 0x55, 0x21, 0x7f, 0xbb, 0xfb, 0x2c, 0x08, 0xec, 0x13, 0xc9, - 0xb8, 0x64, 0xd3, 0x3a, 0x18, 0x5c, 0x16, 0xfe, 0xd9, 0x13, 0xb5, 0xfe, 0x72, 0x41, 0x0a, 0xea, - 0x37, 0x9e, 0xcf, 0x87, 0xfa, 0xba, 0xf3, 0xe0, 0x51, 0xad, 0x3f, 0x14, 0x64, 0xb1, 0xf2, 0xee, - 0x94, 0x34, 0xee, 0x58, 0x6b, 0xba, 0xaa, 0x34, 0x77, 0x12, 0x25, 0xb8, 0xa1, 0x65, 0xd3, 0x5a, - 0xd7, 0xa6, 0xb6, 0x06, 0x18, 0xec, 0xe9, 0xaa, 0xd4, 0xfc, 0x8c, 0x63, 0x39, 0x13, 0x7f, 0x28, - 0x89, 0x95, 0x4b, 0xf9, 0x45, 0x61, 0xe2, 0x76, 0x17, 0xc5, 0xdc, 0x94, 0xc4, 0x7a, 0x57, 0x9b, - 0x05, 0xd3, 0xde, 0x76, 0xef, 0x14, 0xd5, 0xb4, 0xf3, 0xd7, 0x6e, 0xeb, 0xfd, 0xc2, 0x74, 0x98, - 0x16, 0xba, 0x3b, 0xa0, 0xa2, 0x4d, 0x6e, 0xc3, 0x0c, 0xfe, 0xbe, 0x36, 0xbe, 0xb2, 0x47, 0x61, - 0x37, 0x2b, 0x26, 0x37, 0x61, 0xfa, 0xb9, 0xef, 0x84, 0x09, 0x04, 0x52, 0xc9, 0x8c, 0xd4, 0x72, - 0x07, 0x25, 0xd3, 0x7e, 0x83, 0xf1, 0xbe, 0xd0, 0x67, 0xd6, 0xc8, 0x1f, 0x61, 0x32, 0x2d, 0x7f, - 0x8b, 0x1f, 0x1c, 0x2b, 0xfa, 0x77, 0x7e, 0x1c, 0x4d, 0xd5, 0xb1, 0x49, 0x05, 0x7f, 0x05, 0x6e, - 0x5c, 0x22, 0x97, 0x61, 0x26, 0x53, 0x5a, 0x36, 0x4a, 0xc4, 0x80, 0xc9, 0x74, 0x46, 0xcb, 0x18, - 0x22, 0x93, 0x50, 0x96, 0xc9, 0x25, 0x63, 0x98, 0x4c, 0x41, 0x25, 0x4e, 0x7b, 0x18, 0x23, 0x64, - 0x06, 0x26, 0x52, 0x77, 0x7d, 0x63, 0x94, 0x4c, 0x03, 0x24, 0x37, 0x4c, 0x63, 0x8c, 0xe3, 0xa5, - 0xaf, 0x56, 0xc6, 0x38, 0xd7, 0x48, 0x2a, 0x98, 0x46, 0x99, 0x23, 0xc6, 0x85, 0x49, 0xa3, 0x42, - 0xe6, 0x75, 0xa5, 0x49, 0x03, 0xb8, 0x3c, 0x5f, 0x22, 0x34, 0x26, 0x08, 0xc9, 0x16, 0x09, 0x8d, - 0x49, 0x32, 0x11, 0x57, 0xfc, 0x8c, 0x29, 0xb2, 0x58, 0x50, 0xcc, 0x33, 0xa6, 0xc9, 0x6c, 0xa6, - 0x16, 0x67, 0xcc, 0x90, 0x6a, 0xbe, 0xb4, 0x66, 0x18, 0xe9, 0x59, 0x70, 0x5e, 0x69, 0xcc, 0xa2, - 0x24, 0x66, 0x72, 0x06, 0xe1, 0xcb, 0x99, 0x29, 0x33, 0x19, 0x97, 0xb9, 0x30, 0xc3, 0xac, 0x8c, - 0x2a, 0x1f, 0x56, 0x21, 0x1c, 0xc6, 0x1c, 0x59, 0x2e, 0x2e, 0xed, 0x18, 0xf3, 0x7c, 0x89, 0xe2, - 0x1c, 0x9b, 0xb1, 0x80, 0x23, 0xa5, 0x3f, 0xf9, 0x86, 0xc9, 0x1d, 0x4a, 0x7f, 0xa7, 0x8d, 0x45, - 0xf1, 0x2a, 0x0e, 0xf6, 0x37, 0x5e, 0x1c, 0xd2, 0x83, 0x7a, 0xc3, 0x58, 0xc2, 0xf6, 0xa3, 0xfd, - 0xbd, 0xdd, 0xfd, 0xdd, 0xa6, 0x71, 0x85, 0x4f, 0x35, 0x1b, 0x78, 0x8d, 0x65, 0xbe, 0x5c, 0xda, - 0x70, 0x6c, 0x5c, 0x15, 0x7e, 0xa7, 0x83, 0x9e, 0x51, 0x13, 0xef, 0xff, 0x20, 0x0e, 0x4c, 0xc6, - 0x0a, 0x17, 0xa4, 0x42, 0x85, 0xb1, 0xca, 0x9d, 0xcd, 0x1c, 0x72, 0xe3, 0x1a, 0x7f, 0x99, 0xf9, - 0xb3, 0x65, 0x58, 0x7c, 0x12, 0xe9, 0xbd, 0x6b, 0x5c, 0x27, 0x57, 0x44, 0x8c, 0xd0, 0xe5, 0xfd, - 0x8c, 0x1b, 0x77, 0xde, 0x83, 0xaa, 0xee, 0xc3, 0x42, 0x80, 0xc7, 0xb9, 0xae, 0x27, 0x76, 0x79, - 0x19, 0x46, 0xb6, 0x9c, 0xe0, 0xb5, 0x51, 0xda, 0xdc, 0xfb, 0xfe, 0xa7, 0x5a, 0xe9, 0x87, 0x9f, - 0x6a, 0xa5, 0xff, 0xfc, 0x54, 0xbb, 0xf4, 0xdd, 0x9b, 0xda, 0xa5, 0xbf, 0xbf, 0xa9, 0x95, 0x7e, - 0x78, 0x53, 0xbb, 0xf4, 0xe3, 0x9b, 0xda, 0xa5, 0x97, 0x6b, 0xa9, 0x7f, 0xbf, 0xe8, 0xda, 0xa1, - 0xef, 0x9c, 0x79, 0xbe, 0x73, 0xe2, 0xb8, 0xb2, 0xe1, 0xb2, 0xf5, 0xde, 0xeb, 0x93, 0xf5, 0xde, - 0xd1, 0xba, 0xf8, 0xa2, 0x1d, 0x8d, 0x89, 0x04, 0xcf, 0x07, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, - 0xb6, 0x07, 0xc8, 0x96, 0x12, 0x32, 0x00, 0x00, + // 3571 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x5b, 0xdb, 0x72, 0xdc, 0x46, + 0x73, 0xd6, 0xf2, 0xb8, 0xdb, 0x3c, 0x81, 0xa3, 0x25, 0x09, 0x52, 0xd4, 0x92, 0x82, 0x4e, 0xb4, + 0xf4, 0x9b, 0x54, 0xf4, 0xff, 0x52, 0x62, 0x3b, 0x49, 0x99, 0x5c, 0x8a, 0xf4, 0x4a, 0xa4, 0x48, + 0xcd, 0x52, 0x96, 0x2c, 0x57, 0xa9, 0x0a, 0xdc, 0x1d, 0x92, 0x88, 0x76, 0x81, 0x35, 0x80, 0xb5, + 0x49, 0x57, 0xe5, 0x1d, 0x7c, 0x99, 0xcb, 0xe4, 0x3e, 0x2f, 0x90, 0x37, 0xf0, 0xa5, 0x2f, 0x5d, + 0x95, 0xaa, 0xc4, 0x65, 0x3d, 0x40, 0x5e, 0x21, 0x35, 0x83, 0x1e, 0x00, 0x03, 0x0c, 0x96, 0xb2, + 0xcb, 0xbe, 0x61, 0xed, 0xf4, 0x74, 0x7f, 0x33, 0x98, 0x43, 0x4f, 0x7f, 0x3d, 0x43, 0x98, 0xf8, + 0xa6, 0xcf, 0xfc, 0x8b, 0xf5, 0x9e, 0xef, 0x85, 0x1e, 0x19, 0x15, 0x85, 0xa5, 0xc9, 0x20, 0xb4, + 0xc3, 0x7e, 0x10, 0x09, 0x97, 0xa0, 0xe3, 0xb5, 0xde, 0xe1, 0xef, 0x4a, 0x78, 0xee, 0xe2, 0xcf, + 0x99, 0xd0, 0xe9, 0xb2, 0x20, 0xb4, 0xbb, 0x3d, 0x29, 0xe0, 0x56, 0x81, 0xe3, 0x9e, 0x78, 0x28, + 0xf8, 0xf8, 0xd4, 0x09, 0xcf, 0xfa, 0xc7, 0xeb, 0x2d, 0xaf, 0xbb, 0x71, 0xea, 0x9d, 0x7a, 0x1b, + 0x42, 0x7c, 0xdc, 0x3f, 0x11, 0x25, 0x51, 0x10, 0xbf, 0x50, 0x7d, 0xe5, 0xd4, 0xf3, 0x4e, 0x3b, + 0x2c, 0xd1, 0xca, 0x34, 0x60, 0xdd, 0x82, 0xc9, 0x17, 0xbc, 0x7f, 0x94, 0x7d, 0xd3, 0x67, 0x41, + 0x48, 0xaa, 0x30, 0x2a, 0xca, 0x66, 0x69, 0xb5, 0xb4, 0x56, 0xa1, 0x51, 0xc1, 0x7a, 0x0e, 0xf3, + 0xcd, 0x33, 0xef, 0xbb, 0x43, 0xdf, 0x6b, 0xb1, 0x20, 0xd8, 0x73, 0x82, 0x50, 0xea, 0xcf, 0xc3, + 0xd8, 0x11, 0x73, 0x6d, 0x37, 0x44, 0x03, 0x2c, 0x91, 0x65, 0xa8, 0x34, 0x2f, 0x02, 0xac, 0x1a, + 0x5a, 0x2d, 0xad, 0x95, 0x69, 0x22, 0xb0, 0x5e, 0xc1, 0x6c, 0xf3, 0xc2, 0x6d, 0xd5, 0xbd, 0x6e, + 0xd7, 0x89, 0xa1, 0xb6, 0x60, 0x7a, 0xcf, 0x0e, 0x59, 0x10, 0x46, 0xe2, 0xa3, 0xa6, 0x80, 0x9c, + 0x78, 0x58, 0x5d, 0x4f, 0x3a, 0x7d, 0x24, 0x7f, 0x6d, 0x8d, 0xfc, 0xf8, 0x3f, 0x2b, 0x57, 0x68, + 0xc6, 0xc2, 0x7a, 0x03, 0x24, 0x0d, 0x1c, 0xf4, 0x3c, 0x37, 0x60, 0x64, 0x1b, 0x66, 0xea, 0x7d, + 0xdf, 0x67, 0xee, 0x6f, 0x81, 0xce, 0x9a, 0x58, 0x04, 0x8c, 0x5d, 0x16, 0x2a, 0x7d, 0xb6, 0xbe, + 0x82, 0xd9, 0x94, 0xec, 0x0f, 0x6d, 0x6e, 0x03, 0xe6, 0xea, 0x9e, 0xcf, 0xb6, 0xfb, 0xdd, 0x5e, + 0xdd, 0x73, 0x4f, 0x9c, 0xd3, 0xd4, 0x90, 0x6f, 0xb6, 0x42, 0xc7, 0x73, 0xe5, 0x90, 0x47, 0x25, + 0xcb, 0x84, 0xf9, 0xac, 0x41, 0xd4, 0x21, 0xeb, 0x1a, 0x2c, 0xee, 0xb2, 0xf0, 0x90, 0x4f, 0x78, + 0xcb, 0xeb, 0x7c, 0xc9, 0xfc, 0xc0, 0xf1, 0x5c, 0xf9, 0x09, 0x8f, 0x61, 0x49, 0x57, 0x89, 0xdf, + 0x62, 0xc2, 0x38, 0x8a, 0x44, 0x6b, 0xc3, 0x54, 0x16, 0xad, 0x47, 0xb0, 0xd8, 0x2c, 0x02, 0x1d, + 0x60, 0xf6, 0x18, 0x96, 0x9a, 0xbf, 0xa7, 0xb9, 0xbf, 0xc0, 0x34, 0xed, 0xbb, 0x47, 0x76, 0xf0, + 0x4e, 0xb6, 0xb1, 0x04, 0x65, 0x5e, 0xac, 0x7b, 0x6d, 0x26, 0x94, 0x47, 0x69, 0x5c, 0xb6, 0x3e, + 0x82, 0x99, 0x58, 0x1b, 0xa1, 0xe7, 0x61, 0x8c, 0xb2, 0xa0, 0xdf, 0x89, 0x57, 0x6a, 0x54, 0xe2, + 0xc3, 0xc6, 0xbf, 0xdf, 0xe9, 0xb1, 0x8e, 0xe3, 0xb2, 0x86, 0x7b, 0xe2, 0xc9, 0x91, 0xd9, 0x80, + 0x85, 0x5c, 0x0d, 0x82, 0x55, 0x61, 0xb4, 0xee, 0xf5, 0x71, 0xd5, 0x0f, 0xd3, 0xa8, 0x60, 0xfd, + 0xf7, 0x02, 0x8c, 0xcb, 0xde, 0x2d, 0x43, 0x05, 0x7f, 0x36, 0xb6, 0x85, 0xd6, 0x08, 0x4d, 0x04, + 0x64, 0x1d, 0x2a, 0xf5, 0x6e, 0x7b, 0x9f, 0x85, 0x67, 0x5e, 0x5b, 0x6c, 0x8f, 0xe9, 0x87, 0xc6, + 0x7a, 0xe4, 0x35, 0x62, 0x39, 0x4d, 0x54, 0xc8, 0xdf, 0xab, 0xdb, 0xd4, 0x1c, 0x16, 0xeb, 0xe9, + 0x2a, 0x9a, 0xa4, 0xab, 0xa8, 0xba, 0x9f, 0x5f, 0x16, 0xed, 0x5c, 0x73, 0x44, 0x40, 0x5c, 0x47, + 0x08, 0xbd, 0x12, 0x2d, 0xda, 0xf6, 0x7b, 0x70, 0x75, 0xb3, 0x13, 0x32, 0x7f, 0xb3, 0xd5, 0xe2, + 0x5f, 0x2e, 0x31, 0x47, 0x05, 0xe6, 0x12, 0x62, 0x6a, 0x34, 0xa8, 0xce, 0x8c, 0x7c, 0x0e, 0x33, + 0xcf, 0x9c, 0x4e, 0xa7, 0xee, 0xb9, 0x72, 0x01, 0x99, 0x63, 0x02, 0x69, 0x1e, 0x91, 0x32, 0xb5, + 0x34, 0xab, 0x4e, 0xea, 0x60, 0x1c, 0xf9, 0x76, 0x8b, 0x35, 0x7b, 0x76, 0x0c, 0x31, 0x2e, 0x20, + 0x16, 0x10, 0x22, 0x5b, 0x4d, 0x73, 0x06, 0xa4, 0x01, 0x64, 0x97, 0x85, 0x7b, 0x5e, 0xeb, 0x5d, + 0x6a, 0x15, 0x98, 0x65, 0x01, 0xb3, 0x88, 0x30, 0x79, 0x05, 0xaa, 0x31, 0x22, 0x3b, 0xc2, 0x2f, + 0x1c, 0x9d, 0xbb, 0x69, 0xa4, 0x8a, 0x40, 0x32, 0x13, 0x24, 0xb5, 0x9e, 0xe6, 0x4d, 0xf8, 0x38, + 0x73, 0xff, 0x62, 0xb7, 0xce, 0xd2, 0x2b, 0xd3, 0x04, 0x65, 0x9c, 0x35, 0x1a, 0x54, 0x67, 0x46, + 0xfe, 0x01, 0xa0, 0x79, 0xd1, 0x72, 0x23, 0x17, 0x63, 0x4e, 0x28, 0xdd, 0xc9, 0xf9, 0x63, 0x9a, + 0xd2, 0x25, 0x8f, 0xa0, 0x12, 0xfb, 0x39, 0x73, 0x52, 0x19, 0xd8, 0xac, 0x4f, 0xa4, 0x89, 0x26, + 0x39, 0x14, 0x23, 0x9a, 0xd9, 0xec, 0xe6, 0x94, 0xb0, 0x5f, 0x4d, 0xec, 0xf5, 0x4e, 0x84, 0x6a, + 0x6c, 0x39, 0x62, 0xde, 0x7d, 0x98, 0xd3, 0x0a, 0x62, 0xb3, 0x18, 0x31, 0x5f, 0x45, 0xb6, 0x61, + 0x5a, 0x75, 0x9b, 0xe6, 0x8c, 0x40, 0x5b, 0x96, 0xfb, 0x51, 0xe7, 0x84, 0x69, 0xc6, 0x86, 0x6c, + 0xc0, 0x38, 0x3a, 0x1c, 0xd3, 0x10, 0xe6, 0x73, 0x68, 0xae, 0x3a, 0x2d, 0x2a, 0xb5, 0xc8, 0x57, + 0x30, 0x47, 0x59, 0xd7, 0xfb, 0x96, 0xf1, 0xbf, 0x21, 0xe3, 0x0b, 0xe8, 0xc8, 0x3e, 0xee, 0x30, + 0x73, 0x56, 0x98, 0xdf, 0x94, 0xe6, 0x3a, 0x1d, 0x09, 0xa6, 0x47, 0x20, 0x9b, 0x30, 0xc5, 0x97, + 0xa4, 0x38, 0x19, 0xb7, 0x1c, 0xb7, 0x6d, 0x12, 0x01, 0x79, 0x2d, 0xb5, 0x84, 0xe3, 0x3a, 0x09, + 0xa5, 0x5a, 0x90, 0xa7, 0x60, 0xbc, 0x74, 0x83, 0xfe, 0x71, 0xd0, 0xf2, 0x9d, 0x63, 0x16, 0x75, + 0xec, 0xaa, 0x40, 0xa9, 0x21, 0x4a, 0xb6, 0x3a, 0xde, 0x56, 0xd9, 0x8a, 0xf4, 0x1a, 0xde, 0xb6, + 0x43, 0x5b, 0xae, 0xe1, 0xaa, 0x76, 0x0d, 0xa7, 0x34, 0xa8, 0xce, 0x0c, 0xd1, 0x9a, 0x3c, 0x2c, + 0x4a, 0xef, 0x88, 0xb9, 0x2c, 0x5a, 0x56, 0x83, 0xea, 0xcc, 0xb8, 0x7b, 0xd4, 0x3b, 0x7f, 0x73, + 0x5e, 0x71, 0x8f, 0x7a, 0x25, 0x5a, 0x60, 0xcc, 0x61, 0xf7, 0x9d, 0x53, 0xdf, 0x0e, 0x19, 0x77, + 0x52, 0x3b, 0xbe, 0xd7, 0x95, 0xb0, 0x0b, 0x0a, 0xac, 0x5e, 0x89, 0x16, 0x18, 0x93, 0x03, 0xa8, + 0xa6, 0x6a, 0x8e, 0xe2, 0xbe, 0x9a, 0xca, 0xfc, 0xea, 0x54, 0xa8, 0xd6, 0x90, 0x1c, 0x83, 0x49, + 0x59, 0xc7, 0xb3, 0xdb, 0x9b, 0xfd, 0xd0, 0x6b, 0xb8, 0x2d, 0x9f, 0x75, 0x79, 0x08, 0xc2, 0xc7, + 0xdc, 0x5c, 0x14, 0xa0, 0x77, 0xe2, 0x75, 0xa8, 0x57, 0x93, 0xf8, 0x85, 0x38, 0xdc, 0x35, 0xd7, + 0xc3, 0x0e, 0x65, 0x76, 0x9b, 0xf9, 0xb2, 0xc3, 0x4b, 0x8a, 0x07, 0xc9, 0x56, 0xd3, 0x9c, 0x01, + 0xd9, 0x87, 0x99, 0x5d, 0x16, 0x52, 0xd6, 0xeb, 0x38, 0x2d, 0x3b, 0x3a, 0x79, 0xaf, 0x65, 0x27, + 0x28, 0x5d, 0x8b, 0x76, 0x32, 0xb6, 0xca, 0xd4, 0xf2, 0x45, 0x44, 0x59, 0xc0, 0xc2, 0x26, 0x0b, + 0x52, 0xee, 0xc1, 0x5c, 0x56, 0x16, 0x91, 0x46, 0x83, 0xea, 0xcc, 0xc8, 0x1e, 0xcc, 0xee, 0x7a, + 0xfb, 0xf6, 0x39, 0x3f, 0x27, 0x03, 0x89, 0x75, 0x5d, 0x75, 0xf6, 0xd9, 0x7a, 0xec, 0x59, 0xde, + 0x10, 0xd1, 0x58, 0x77, 0xcf, 0x49, 0x7c, 0xaa, 0x59, 0xcb, 0xa2, 0xa9, 0xf5, 0x29, 0x34, 0xb5, + 0x82, 0xbc, 0x85, 0x85, 0x1d, 0xa7, 0xc3, 0x9a, 0xcc, 0xff, 0xd6, 0x69, 0xb1, 0xf4, 0x94, 0x99, + 0x2b, 0xca, 0x7e, 0x2e, 0xd0, 0x42, 0xe4, 0x22, 0x10, 0xd2, 0x85, 0xe5, 0x6c, 0xd5, 0x93, 0x6f, + 0x9d, 0x56, 0xdc, 0xf1, 0x55, 0xc5, 0x9b, 0x0d, 0x52, 0xc5, 0x96, 0x06, 0xc2, 0x91, 0x97, 0x50, + 0xdd, 0x67, 0xa1, 0xdd, 0xb6, 0x43, 0x5b, 0xf9, 0x96, 0x1b, 0xea, 0x0e, 0xd0, 0xa8, 0x20, 0xbc, + 0xd6, 0x9c, 0x1c, 0x00, 0xd9, 0xf5, 0x76, 0xeb, 0x87, 0xcc, 0x6f, 0xb1, 0x24, 0x9a, 0xb1, 0xd4, + 0x93, 0x3f, 0xa7, 0x80, 0x90, 0x1a, 0x53, 0x1e, 0x4a, 0xec, 0xd8, 0xfd, 0x4e, 0xd8, 0x70, 0xff, + 0x85, 0x25, 0x83, 0x71, 0x53, 0x01, 0xcc, 0x2b, 0x50, 0x8d, 0x11, 0xf9, 0x1a, 0xe6, 0xeb, 0x61, + 0x67, 0xdf, 0x13, 0xce, 0x54, 0x38, 0x30, 0x09, 0x77, 0x4b, 0xd9, 0x01, 0x7a, 0x25, 0xec, 0x63, + 0x01, 0x04, 0x79, 0x0b, 0x8b, 0xaf, 0x3c, 0xff, 0x5d, 0xd0, 0xb3, 0x5b, 0xec, 0xe8, 0xcc, 0x67, + 0xc1, 0x99, 0xd7, 0x91, 0x67, 0x82, 0x79, 0x5b, 0x39, 0x55, 0x0b, 0xf5, 0x68, 0x31, 0x04, 0xe9, + 0x42, 0xad, 0x1e, 0x76, 0x0e, 0x7d, 0x76, 0xc2, 0xc2, 0xd6, 0xd9, 0x81, 0xdb, 0x94, 0x47, 0x43, + 0xdc, 0xc8, 0x1d, 0xd1, 0xc8, 0xed, 0xe4, 0x23, 0x06, 0x28, 0xd3, 0x4b, 0xc0, 0xc8, 0xd7, 0x60, + 0x36, 0x9a, 0xf5, 0xc3, 0x6d, 0xdf, 0x76, 0xdc, 0xba, 0xe7, 0x06, 0xfd, 0x6e, 0xe2, 0x73, 0xee, + 0x8a, 0x86, 0x56, 0xb0, 0xa1, 0x22, 0x35, 0x5a, 0x08, 0x60, 0xed, 0xc0, 0x42, 0x2e, 0x1a, 0x46, + 0x3a, 0x70, 0x1f, 0xca, 0xe8, 0x13, 0x02, 0xb3, 0xb4, 0x3a, 0xbc, 0x36, 0xf1, 0x70, 0x66, 0x1d, + 0xf9, 0xbe, 0xf4, 0x15, 0xb1, 0x82, 0xf5, 0x8b, 0x09, 0xe5, 0xd8, 0xf2, 0x8f, 0xa5, 0x09, 0x55, + 0x18, 0x7d, 0xe2, 0xfb, 0x9e, 0x2f, 0xf8, 0xc1, 0x24, 0x8d, 0x0a, 0xe4, 0x75, 0x61, 0xc7, 0x91, + 0x04, 0xd4, 0x8a, 0x48, 0x40, 0xa4, 0x45, 0x0b, 0xbf, 0xfb, 0x00, 0xaa, 0x6a, 0x3c, 0x8f, 0xb0, + 0xa3, 0xca, 0x76, 0xd4, 0xa9, 0x50, 0xad, 0x21, 0x3f, 0x2c, 0x92, 0xd0, 0x1e, 0xc1, 0xc6, 0x94, + 0xc3, 0x22, 0x5b, 0x4d, 0x73, 0x06, 0x3c, 0xf8, 0x4e, 0xc5, 0xf6, 0x88, 0x32, 0xae, 0x78, 0xd0, + 0x5c, 0x3d, 0xcd, 0x9b, 0x60, 0xa8, 0x91, 0x84, 0xf6, 0x88, 0x54, 0xce, 0x86, 0x1a, 0x59, 0x0d, + 0xaa, 0x33, 0x43, 0x76, 0x11, 0xc7, 0xf7, 0x08, 0x56, 0xc9, 0xb2, 0x8b, 0x8c, 0x02, 0xd5, 0x18, + 0xf1, 0x61, 0x57, 0xc3, 0x7b, 0x04, 0x83, 0x6c, 0x9c, 0x97, 0x53, 0xa1, 0x5a, 0x43, 0xf2, 0x09, + 0x27, 0x06, 0x32, 0xfe, 0x47, 0x62, 0xb0, 0xa8, 0x21, 0x06, 0x08, 0x92, 0x52, 0x26, 0x8f, 0xf3, + 0xcc, 0xc0, 0xcc, 0x33, 0x03, 0x34, 0x4c, 0x51, 0x83, 0x17, 0x03, 0xa8, 0xc1, 0x8d, 0x01, 0xd4, + 0x20, 0x35, 0x2c, 0xd9, 0x48, 0xfe, 0xc5, 0x00, 0x6e, 0x70, 0x63, 0x00, 0x37, 0x90, 0x90, 0x1a, + 0x72, 0xf0, 0xa4, 0x80, 0x1c, 0x5c, 0x2f, 0x20, 0x07, 0x08, 0x95, 0x65, 0x07, 0x0f, 0xb2, 0xec, + 0x60, 0x3e, 0xcb, 0x0e, 0xd0, 0x30, 0xa6, 0x07, 0x6f, 0x06, 0xd3, 0x83, 0x5b, 0x83, 0xe9, 0x01, + 0xa2, 0x15, 0xf0, 0x83, 0x2d, 0x3d, 0x3f, 0x58, 0xd6, 0xf3, 0x03, 0xc4, 0xca, 0x10, 0x84, 0x67, + 0x85, 0x04, 0x61, 0xa5, 0x90, 0x20, 0xc8, 0x0d, 0x9b, 0x63, 0x08, 0xa9, 0xf5, 0x1c, 0x85, 0xfa, + 0xb8, 0x9e, 0xab, 0xda, 0xf5, 0x9c, 0x56, 0xa1, 0x5a, 0x43, 0x04, 0x4c, 0x45, 0xfb, 0x08, 0x38, + 0x97, 0x05, 0xcc, 0xa9, 0x50, 0xad, 0x21, 0x77, 0xa1, 0x05, 0xa9, 0x20, 0x24, 0x0a, 0xb5, 0x22, + 0xa2, 0x20, 0x5d, 0x68, 0x51, 0x26, 0xe9, 0x35, 0x2c, 0xe4, 0xa2, 0x7d, 0x44, 0x5e, 0x50, 0x90, + 0x0b, 0xb4, 0x68, 0x91, 0x39, 0xa1, 0x30, 0x97, 0x09, 0xfa, 0x11, 0xd7, 0x54, 0xa6, 0x5b, 0xab, + 0x43, 0xf5, 0xa6, 0xa4, 0x75, 0x29, 0x61, 0xb8, 0x7b, 0x29, 0x61, 0xc0, 0x16, 0x8a, 0x19, 0xc3, + 0x0e, 0xcc, 0xa6, 0x08, 0x00, 0x76, 0x7a, 0x49, 0x71, 0x2d, 0xb9, 0x7a, 0x9a, 0x37, 0x21, 0xcf, + 0x8b, 0x48, 0x43, 0xad, 0x88, 0x34, 0x44, 0x86, 0x45, 0xac, 0xe1, 0x00, 0xaa, 0x6a, 0xf8, 0x8f, + 0x5d, 0x5b, 0x56, 0x56, 0x95, 0x4e, 0x85, 0x6a, 0x0d, 0xa3, 0xb0, 0x33, 0x89, 0xff, 0x11, 0xee, + 0x7a, 0x26, 0xec, 0xcc, 0x2a, 0x24, 0x61, 0x67, 0xb6, 0x06, 0x01, 0x63, 0x0a, 0x80, 0x80, 0xb5, + 0x2c, 0x60, 0x46, 0x21, 0x05, 0x98, 0xa9, 0x21, 0x36, 0x98, 0xf9, 0xc8, 0x1f, 0x61, 0x57, 0x94, + 0xed, 0x5e, 0xa4, 0x86, 0xe0, 0x85, 0x30, 0xa4, 0x07, 0xd7, 0x0b, 0x42, 0x7e, 0x6c, 0x67, 0x55, + 0xf1, 0x78, 0x03, 0x75, 0xb1, 0xb1, 0xc1, 0x80, 0xe4, 0x35, 0xcc, 0x65, 0x58, 0x00, 0xb6, 0x74, + 0x43, 0xdd, 0x18, 0x3a, 0x1d, 0x6c, 0x41, 0x0f, 0x40, 0x28, 0x5c, 0x55, 0xc8, 0x00, 0xe2, 0x5a, + 0x6a, 0xc4, 0x90, 0xd7, 0x40, 0x54, 0x9d, 0x31, 0x8f, 0x42, 0x14, 0x56, 0x80, 0x98, 0x37, 0x15, + 0x4c, 0x8d, 0x06, 0xd5, 0x99, 0x71, 0x3e, 0x98, 0xa3, 0x02, 0x88, 0x78, 0x4b, 0xd9, 0x1b, 0x05, + 0x5a, 0x92, 0x0f, 0x16, 0x54, 0x13, 0x1b, 0x96, 0x74, 0x6c, 0x00, 0x9b, 0xb8, 0xad, 0x9c, 0xc5, + 0xc5, 0x8a, 0x74, 0x00, 0x48, 0x94, 0x05, 0x71, 0xe3, 0xfb, 0x93, 0x18, 0xfc, 0x4e, 0x26, 0x0b, + 0x92, 0x57, 0xa1, 0x5a, 0x43, 0xd2, 0x83, 0x95, 0x42, 0x5e, 0x81, 0xd8, 0x77, 0x95, 0x64, 0xc8, + 0x25, 0xda, 0xf4, 0x32, 0x38, 0x4e, 0xbb, 0x34, 0x34, 0x03, 0xdb, 0x5a, 0x53, 0x68, 0x57, 0xa1, + 0x1e, 0x2d, 0x86, 0xb0, 0x1a, 0xda, 0xf4, 0xbc, 0xb8, 0x31, 0x11, 0x17, 0x70, 0x8d, 0x36, 0x5e, + 0x5c, 0xc4, 0x65, 0x32, 0x0f, 0x63, 0x4d, 0xc1, 0x58, 0x04, 0x77, 0xa8, 0x50, 0x2c, 0x59, 0x9f, + 0xea, 0x43, 0x7c, 0x62, 0xc1, 0xa4, 0xcd, 0xe5, 0xcd, 0x7e, 0x8b, 0xd3, 0x02, 0x81, 0x57, 0xa6, + 0x8a, 0xcc, 0x6a, 0xe4, 0xf2, 0xfa, 0x9c, 0xef, 0x20, 0x12, 0xf2, 0x9d, 0x61, 0x9a, 0x08, 0xd2, + 0xd7, 0x3f, 0x43, 0x82, 0x0b, 0xa5, 0xae, 0x7f, 0xf2, 0x71, 0xbe, 0x09, 0xe3, 0x6a, 0xeb, 0xb2, + 0x68, 0xed, 0xe5, 0x73, 0x4e, 0xc4, 0x80, 0xe1, 0x7a, 0xb7, 0x8d, 0x97, 0x3f, 0xfc, 0xa7, 0x90, + 0x9c, 0x9c, 0x8a, 0x96, 0xb8, 0xe4, 0xe4, 0x54, 0xf0, 0xa7, 0xf3, 0xd0, 0xb7, 0x63, 0xfe, 0xc4, + 0x0b, 0xd6, 0x5d, 0xcd, 0x79, 0x44, 0x08, 0x8c, 0xf0, 0xdf, 0x88, 0x27, 0x7e, 0x5b, 0xff, 0x78, + 0x19, 0xdb, 0xe5, 0x33, 0x70, 0x68, 0x87, 0x21, 0xf3, 0x91, 0x28, 0x56, 0x68, 0x5c, 0xb6, 0x1e, + 0x5d, 0xba, 0x0c, 0xb5, 0x8d, 0xfe, 0x67, 0xa9, 0x98, 0xf4, 0xe6, 0x87, 0x7b, 0x2a, 0x33, 0xdc, + 0x62, 0x0b, 0x37, 0xb6, 0xe5, 0x70, 0x63, 0x91, 0xd7, 0x3c, 0xf5, 0x8e, 0x9f, 0xdb, 0x5d, 0x86, + 0xcb, 0x41, 0x16, 0xf9, 0x10, 0x3d, 0xf5, 0x8e, 0x1b, 0xdb, 0x82, 0x3a, 0x8e, 0xd0, 0xa8, 0x40, + 0xd6, 0x60, 0x26, 0x8a, 0x35, 0x77, 0x98, 0xdb, 0x62, 0x07, 0x6e, 0xe7, 0x42, 0x70, 0xc0, 0x32, + 0xcd, 0x8a, 0xad, 0x47, 0x03, 0x96, 0xfe, 0x80, 0x19, 0x7d, 0x9d, 0xbf, 0xe0, 0xd1, 0xcc, 0x68, + 0x15, 0x46, 0xb9, 0x42, 0x80, 0x73, 0x1a, 0x15, 0xf8, 0x20, 0xc4, 0x5e, 0x44, 0x7c, 0xce, 0x30, + 0x4d, 0x04, 0x7c, 0x76, 0xf3, 0xd4, 0x4f, 0x37, 0xd0, 0x55, 0xdd, 0xf5, 0x90, 0xf5, 0x73, 0x09, + 0xca, 0x52, 0x96, 0x0c, 0x68, 0x1b, 0xb9, 0xbc, 0x2c, 0x72, 0xc0, 0x67, 0xec, 0x82, 0x77, 0x6c, + 0x78, 0x6d, 0x92, 0x8a, 0xdf, 0xe4, 0x5e, 0x64, 0xb9, 0xef, 0xb5, 0xa3, 0x51, 0x9e, 0x7e, 0x38, + 0xbd, 0x2e, 0xde, 0x05, 0x48, 0x29, 0x8d, 0xeb, 0xc9, 0x2a, 0x4c, 0x38, 0x01, 0xb5, 0xdd, 0x53, + 0x11, 0xc7, 0x8b, 0xc1, 0x2f, 0xd3, 0xb4, 0x88, 0xdc, 0x85, 0xf1, 0x2f, 0xbc, 0x4e, 0x9b, 0xf9, + 0x81, 0x39, 0x2a, 0x52, 0x10, 0x53, 0x11, 0xd8, 0x2b, 0xdb, 0xe1, 0x04, 0x92, 0xca, 0x5a, 0xae, + 0xc8, 0x65, 0x5c, 0x71, 0x4c, 0xab, 0x88, 0xb5, 0xd6, 0x5b, 0x2d, 0xff, 0xe5, 0x9f, 0x52, 0x77, + 0x1b, 0x72, 0xdc, 0xc5, 0x6f, 0xf2, 0x57, 0x98, 0x94, 0x7a, 0x7b, 0x4e, 0x10, 0x8a, 0xcf, 0x9c, + 0x78, 0x38, 0x83, 0x3e, 0x2c, 0x86, 0x50, 0x94, 0xac, 0xab, 0x9a, 0x4b, 0x32, 0xeb, 0x0c, 0x26, + 0x8e, 0xce, 0xdd, 0x0f, 0x1b, 0x51, 0xea, 0x7d, 0x17, 0x8f, 0x28, 0xff, 0x4d, 0xee, 0xc3, 0xf8, + 0x41, 0x2f, 0x14, 0x69, 0x98, 0xe8, 0x86, 0x74, 0x36, 0x19, 0x50, 0xac, 0xa0, 0x52, 0xc3, 0xfa, + 0xaf, 0x12, 0x8c, 0x63, 0xe3, 0xe4, 0x73, 0x28, 0xd7, 0x7d, 0x66, 0x87, 0x6c, 0x33, 0xc4, 0xbb, + 0xfa, 0xa5, 0xf5, 0xe8, 0xe9, 0xc4, 0xba, 0x7c, 0x3a, 0x91, 0xba, 0xb1, 0x2f, 0xf3, 0x33, 0xf0, + 0x87, 0xff, 0x5d, 0x29, 0xd1, 0xd8, 0x8a, 0xac, 0xc2, 0x08, 0x8f, 0x09, 0xc4, 0xca, 0x9b, 0x78, + 0x38, 0xb9, 0x1e, 0x9e, 0xbb, 0xeb, 0x47, 0xe7, 0x2e, 0x97, 0x51, 0x51, 0xc3, 0x3f, 0xe5, 0x65, + 0xc0, 0xfc, 0xa3, 0x73, 0x57, 0x74, 0xae, 0x4c, 0x65, 0x91, 0x3c, 0x80, 0x0a, 0x1f, 0x73, 0xde, + 0xcb, 0xc0, 0x1c, 0x11, 0x43, 0x47, 0x64, 0xa2, 0x22, 0x19, 0x0b, 0x9a, 0x28, 0x59, 0x6f, 0x74, + 0xc9, 0x04, 0xed, 0xcc, 0x3c, 0x10, 0xe3, 0x99, 0x99, 0x98, 0xe9, 0x04, 0x5d, 0x00, 0xa4, 0x55, + 0xac, 0x39, 0xed, 0x9d, 0xa3, 0xf5, 0x1f, 0x25, 0xa8, 0xc4, 0x42, 0xee, 0xc8, 0x9e, 0x7b, 0x6d, + 0x76, 0x74, 0xd1, 0x63, 0xd8, 0x5c, 0x5c, 0xe6, 0x47, 0x09, 0xff, 0xdd, 0x68, 0xe3, 0x36, 0xc4, + 0x12, 0xdf, 0x87, 0x02, 0x40, 0x18, 0x45, 0x6e, 0x25, 0x11, 0xf0, 0xce, 0xbf, 0x0c, 0x58, 0x1b, + 0xfd, 0x8a, 0xf8, 0xcd, 0x65, 0x3b, 0x3e, 0x8b, 0xf2, 0x49, 0x23, 0x54, 0xfc, 0xe6, 0x2d, 0x7f, + 0xe1, 0x84, 0xd4, 0x0e, 0x1d, 0x4f, 0xa4, 0x86, 0x86, 0x68, 0x5c, 0xb6, 0x9e, 0xeb, 0x13, 0x23, + 0xe4, 0x31, 0x4c, 0xc5, 0x42, 0x31, 0x0c, 0x51, 0x92, 0x2e, 0xce, 0xa5, 0xc5, 0x06, 0xaa, 0x9a, + 0xd5, 0x81, 0xe5, 0x41, 0x17, 0x70, 0x7c, 0x4a, 0x77, 0x7d, 0xaf, 0xdf, 0x8b, 0x9d, 0xab, 0x2c, + 0x0e, 0x76, 0xad, 0xf2, 0x8c, 0x1b, 0x56, 0xcf, 0xb8, 0x47, 0x70, 0x7d, 0x20, 0x9f, 0x57, 0x5f, + 0x1d, 0x8c, 0xca, 0x57, 0x07, 0x4f, 0xc5, 0x47, 0xe7, 0xae, 0xf4, 0x7e, 0x4f, 0xe7, 0xac, 0xfb, + 0x30, 0xa7, 0xa5, 0xff, 0x7c, 0x26, 0x44, 0xaa, 0x00, 0x97, 0x16, 0xff, 0x6d, 0x35, 0x61, 0xa1, + 0xe0, 0x16, 0x90, 0xd4, 0x00, 0x38, 0x21, 0x3f, 0xb6, 0x03, 0x16, 0xe7, 0x35, 0x53, 0x92, 0x01, + 0x3d, 0xf8, 0x1b, 0x98, 0x45, 0x99, 0x83, 0x01, 0xc7, 0xc3, 0x0e, 0x94, 0xc5, 0xcc, 0x3d, 0x63, + 0x17, 0xbc, 0xab, 0x87, 0x76, 0x78, 0x26, 0xbb, 0xca, 0x7f, 0xf3, 0x25, 0x79, 0x70, 0x72, 0x12, + 0xb0, 0xe8, 0x2d, 0xd2, 0x30, 0xc5, 0x12, 0x99, 0x86, 0xa1, 0xe6, 0xf7, 0x78, 0x26, 0x0c, 0x35, + 0xbf, 0xb7, 0x1e, 0xe3, 0x12, 0x15, 0xfe, 0xf9, 0x23, 0x18, 0x79, 0xc7, 0x7d, 0x76, 0x49, 0x71, + 0x66, 0xb2, 0x1e, 0x23, 0x61, 0xa1, 0x62, 0x1d, 0xf1, 0xf3, 0x4f, 0x7c, 0x7a, 0xdc, 0x8d, 0x2a, + 0x8c, 0x36, 0xdc, 0x36, 0x3b, 0x97, 0x93, 0x25, 0x0a, 0xe4, 0x7e, 0xd2, 0x51, 0x74, 0x15, 0x59, + 0x5c, 0x1a, 0x2b, 0x58, 0xaf, 0xb4, 0x37, 0xa7, 0xe4, 0xf3, 0x5c, 0x63, 0xd8, 0xc5, 0x38, 0xab, + 0xa4, 0xd6, 0xd2, 0xac, 0xba, 0x75, 0x00, 0xb3, 0x72, 0x50, 0x63, 0xf4, 0x82, 0x0e, 0x1b, 0x30, + 0xfc, 0x85, 0x23, 0x9f, 0x70, 0xf1, 0x9f, 0x7c, 0x7c, 0xb9, 0x3e, 0xc6, 0x48, 0xe2, 0xb7, 0xf5, + 0x56, 0x9f, 0xc1, 0xe1, 0x54, 0x3e, 0xd7, 0x10, 0x76, 0xd6, 0x4c, 0xf8, 0xb2, 0x5a, 0x4f, 0xf3, + 0x26, 0x16, 0xd5, 0xde, 0xfa, 0x92, 0xcf, 0x60, 0x32, 0x96, 0x45, 0xc3, 0x10, 0xa5, 0x8a, 0x93, + 0x57, 0x73, 0xe9, 0x6a, 0xaa, 0x28, 0xe3, 0xbe, 0xc9, 0xe7, 0x7a, 0x1e, 0x42, 0x25, 0x16, 0xc6, + 0x0f, 0xb7, 0x34, 0x88, 0x34, 0x51, 0xb3, 0x9a, 0x30, 0x71, 0xe8, 0xb3, 0x9e, 0xed, 0xb3, 0x66, + 0xd8, 0x15, 0x43, 0x24, 0x62, 0x27, 0x5c, 0x82, 0x22, 0x70, 0x32, 0x60, 0xb8, 0xf9, 0x62, 0x4f, + 0x46, 0x9b, 0xcd, 0x17, 0x7b, 0x7c, 0x93, 0x1c, 0xda, 0xbe, 0xdd, 0xe5, 0xee, 0x2f, 0xc0, 0xe1, + 0x4c, 0x49, 0xac, 0x07, 0x45, 0xb7, 0xc8, 0x7c, 0x39, 0x73, 0x51, 0xbc, 0xb3, 0xb1, 0x64, 0xd9, + 0x85, 0xc9, 0x24, 0xbe, 0xd2, 0xb7, 0xb7, 0xb0, 0x43, 0x43, 0xdb, 0x5b, 0xe4, 0x31, 0x4c, 0xa6, + 0x7a, 0x1c, 0xe0, 0xc1, 0x20, 0x8f, 0x9d, 0x54, 0x15, 0x55, 0xf4, 0xac, 0x7f, 0x2b, 0xe9, 0x2f, + 0xa1, 0x8b, 0xfa, 0x84, 0x0d, 0x0f, 0xc5, 0x0d, 0xaf, 0xc2, 0x44, 0x93, 0x85, 0x5f, 0xda, 0x7e, + 0xd4, 0xee, 0xb0, 0x88, 0x82, 0xd3, 0xa2, 0x5c, 0xd7, 0x46, 0x3e, 0xb0, 0x6b, 0x7f, 0x57, 0x90, + 0xf0, 0x1a, 0xe0, 0x37, 0x3e, 0x83, 0x95, 0x4b, 0x6e, 0xb6, 0xd3, 0xae, 0xaa, 0xa4, 0xba, 0x2a, + 0x0b, 0x56, 0x2f, 0xcb, 0x72, 0x59, 0x6b, 0xe2, 0x81, 0x81, 0xe6, 0x6a, 0x9a, 0x8f, 0x4b, 0xfd, + 0xb9, 0x9c, 0x90, 0xfa, 0x73, 0x7c, 0x6d, 0xa6, 0xcb, 0x47, 0x15, 0xbc, 0x36, 0xfb, 0x58, 0x7b, + 0x89, 0x5d, 0xb8, 0x36, 0x0e, 0xf5, 0xd9, 0xab, 0xe2, 0xc1, 0xe1, 0xeb, 0x73, 0xb3, 0x1f, 0x9e, + 0x35, 0x43, 0xdf, 0x71, 0x23, 0x9a, 0x34, 0x49, 0x53, 0x12, 0x6b, 0x43, 0x73, 0xef, 0xcd, 0x8f, + 0x67, 0x29, 0x92, 0xaf, 0xf2, 0x64, 0xd9, 0x7a, 0xa0, 0xcb, 0x77, 0x0d, 0xb4, 0xf8, 0x44, 0x73, + 0x19, 0x4e, 0x6e, 0xc1, 0x94, 0x14, 0x6d, 0x5d, 0x84, 0x2c, 0xc0, 0x61, 0x51, 0x85, 0xd6, 0xa7, + 0xba, 0x5c, 0xd8, 0x07, 0xda, 0x9e, 0x15, 0xde, 0x9a, 0x93, 0x0d, 0x18, 0x89, 0x83, 0x9e, 0xe9, + 0x38, 0xdb, 0x90, 0xd5, 0xe6, 0x2a, 0x54, 0x28, 0xc6, 0x51, 0x4f, 0xd3, 0xf9, 0x9e, 0xe1, 0xe9, + 0x93, 0x08, 0xac, 0xf3, 0xe2, 0x04, 0x9b, 0x6a, 0x59, 0xca, 0x58, 0xf2, 0x2f, 0x11, 0x85, 0xba, + 0xdd, 0xb3, 0x5b, 0x4e, 0x78, 0x81, 0xd8, 0xaa, 0x90, 0xcf, 0xee, 0x3e, 0x0b, 0x02, 0xfb, 0x34, + 0x26, 0x72, 0x58, 0xb4, 0x0e, 0x06, 0xdf, 0xdc, 0xff, 0xe6, 0x0f, 0xb5, 0xfe, 0xf5, 0x92, 0x44, + 0xde, 0x9f, 0xfc, 0x3d, 0x7f, 0xd3, 0x3f, 0x0d, 0x18, 0xdc, 0xaa, 0xf5, 0x4f, 0x05, 0xb9, 0xc0, + 0x7c, 0x77, 0x4a, 0x9a, 0xee, 0x58, 0xeb, 0xba, 0x87, 0x03, 0xbc, 0x93, 0x28, 0xc1, 0x05, 0x2d, + 0x8b, 0xd6, 0x86, 0x36, 0x41, 0x38, 0xc0, 0x60, 0x4f, 0xf7, 0x90, 0x80, 0xef, 0x71, 0xbc, 0x14, + 0xc6, 0xb7, 0xac, 0x78, 0xff, 0x2b, 0x4f, 0x14, 0x26, 0xd8, 0x5d, 0xe4, 0x73, 0x53, 0x12, 0xeb, + 0x23, 0x6d, 0x2e, 0x51, 0xcb, 0x76, 0xef, 0x15, 0x3d, 0x3b, 0xc8, 0xd3, 0x6e, 0xeb, 0xe3, 0xc2, + 0xa4, 0xa2, 0x16, 0xba, 0x3b, 0xe0, 0xd1, 0x01, 0x59, 0x83, 0x19, 0x7c, 0x02, 0x1d, 0x53, 0xf6, + 0xc8, 0xed, 0x66, 0xc5, 0xe4, 0x0e, 0x4c, 0xbf, 0xf2, 0x9d, 0x30, 0x81, 0xc0, 0x50, 0x32, 0x23, + 0xb5, 0xdc, 0x41, 0x29, 0xc9, 0x3f, 0xa1, 0xbd, 0x2f, 0xf5, 0xf9, 0x49, 0xf2, 0xcf, 0x30, 0x99, + 0x96, 0x7f, 0xc0, 0x9b, 0x70, 0x45, 0xff, 0xde, 0xff, 0x8d, 0xa6, 0x5e, 0x03, 0x90, 0x0a, 0x3e, + 0xd4, 0x37, 0xae, 0x90, 0xab, 0x30, 0x93, 0xb9, 0xa0, 0x37, 0x4a, 0xc4, 0x80, 0xc9, 0x74, 0xde, + 0xce, 0x18, 0x22, 0x93, 0x50, 0x96, 0x29, 0x34, 0x63, 0x98, 0x4c, 0x41, 0x25, 0x4e, 0x7b, 0x18, + 0x23, 0x64, 0x06, 0x26, 0x52, 0x5c, 0xdf, 0x18, 0x25, 0xd3, 0x00, 0x09, 0xc3, 0x34, 0xc6, 0x38, + 0x5e, 0x9a, 0x5a, 0x19, 0xe3, 0x5c, 0x23, 0xb9, 0x07, 0x36, 0xca, 0x1c, 0x31, 0xbe, 0xde, 0x35, + 0x2a, 0x64, 0x5e, 0x77, 0xc1, 0x6b, 0x00, 0x97, 0xe7, 0x2f, 0x5a, 0x8d, 0x09, 0x42, 0xb2, 0x57, + 0xad, 0xc6, 0x24, 0x99, 0x88, 0xef, 0x4d, 0x8d, 0x29, 0xb2, 0x58, 0x70, 0x25, 0x6a, 0x4c, 0x93, + 0xd9, 0xcc, 0x8d, 0xa6, 0x31, 0x43, 0xaa, 0xf9, 0x0b, 0x4a, 0xc3, 0x48, 0x7f, 0x05, 0x8f, 0x2b, + 0x8d, 0x59, 0x94, 0xc4, 0x91, 0x9c, 0x41, 0xf8, 0x70, 0x66, 0x2e, 0xeb, 0x8c, 0xab, 0x5c, 0x98, + 0x89, 0xac, 0x8c, 0x2a, 0x6f, 0x56, 0x09, 0x38, 0x8c, 0x39, 0xb2, 0x5c, 0x7c, 0x41, 0x66, 0xcc, + 0xf3, 0x21, 0x8a, 0x33, 0x89, 0xc6, 0x02, 0xb6, 0x94, 0x3e, 0xf2, 0x0d, 0x93, 0x77, 0x28, 0x7d, + 0x4e, 0x1b, 0x8b, 0x62, 0x2a, 0x0e, 0xf6, 0x37, 0x5f, 0x1f, 0xd2, 0x83, 0x7a, 0xd3, 0x58, 0xc2, + 0xf2, 0x93, 0xfd, 0xbd, 0xc6, 0x7e, 0xe3, 0xc8, 0xb8, 0xc6, 0x3f, 0x35, 0xeb, 0x78, 0x8d, 0x65, + 0x3e, 0x5c, 0x5a, 0x77, 0x6c, 0x5c, 0x17, 0xfd, 0x4e, 0x3b, 0x3d, 0xa3, 0x26, 0xe6, 0xff, 0x20, + 0x76, 0x4c, 0xc6, 0x0a, 0x17, 0xa4, 0x5c, 0x85, 0xb1, 0xca, 0x3b, 0x9b, 0xd9, 0xe4, 0xc6, 0x0d, + 0x3e, 0x99, 0xf9, 0xbd, 0x65, 0x58, 0xfc, 0x23, 0xd2, 0x6b, 0xd7, 0xb8, 0x49, 0xae, 0x09, 0x1f, + 0xa1, 0xcb, 0x6e, 0x1a, 0xb7, 0xc8, 0x1c, 0xcc, 0xe6, 0x92, 0x82, 0xc6, 0xed, 0x7b, 0x7f, 0x81, + 0xaa, 0xee, 0xbc, 0x21, 0xc0, 0xdd, 0x5f, 0xd7, 0x13, 0x8b, 0xbf, 0x0c, 0x23, 0xdb, 0x4e, 0xf0, + 0xce, 0x28, 0x6d, 0xed, 0xfd, 0xf8, 0x6b, 0xad, 0xf4, 0xd3, 0xaf, 0xb5, 0xd2, 0x2f, 0xbf, 0xd6, + 0xae, 0xfc, 0xf0, 0xbe, 0x76, 0xe5, 0xdf, 0xdf, 0xd7, 0x4a, 0x3f, 0xbd, 0xaf, 0x5d, 0xf9, 0xf9, + 0x7d, 0xed, 0xca, 0x9b, 0xf5, 0xd4, 0x3f, 0xce, 0x74, 0xed, 0xd0, 0x77, 0xce, 0x3d, 0xdf, 0x39, + 0x75, 0x5c, 0x59, 0x70, 0xd9, 0x46, 0xef, 0xdd, 0xe9, 0x46, 0xef, 0x78, 0x43, 0x1c, 0x74, 0xc7, + 0x63, 0x22, 0xef, 0xf3, 0xd7, 0xff, 0x0f, 0x00, 0x00, 0xff, 0xff, 0xeb, 0x7b, 0x29, 0x25, 0xcc, + 0x33, 0x00, 0x00, } func (m *QueryRequest) Marshal() (dAtA []byte, err error) { @@ -5683,6 +5832,20 @@ func (m *Request) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ISCPDrainConsumerRequest != nil { + { + size, err := m.ISCPDrainConsumerRequest.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0xba + } if m.CtlPrefetchOnSubscribedRequest != nil { { size, err := m.CtlPrefetchOnSubscribedRequest.MarshalToSizedBuffer(dAtA[:i]) @@ -6215,6 +6378,20 @@ func (m *Response) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ISCPDrainConsumerResponse != nil { + { + size, err := m.ISCPDrainConsumerResponse.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0xc2 + } if m.CtlPrefetchOnSubscribedResponse != nil { { size, err := m.CtlPrefetchOnSubscribedResponse.MarshalToSizedBuffer(dAtA[:i]) @@ -6969,7 +7146,7 @@ func (m *CtlPrefetchOnSubscribedResponse) MarshalToSizedBuffer(dAtA []byte) (int return len(dAtA) - i, nil } -func (m *TraceSpanRequest) Marshal() (dAtA []byte, err error) { +func (m *ISCPDrainConsumerRequest) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -6979,34 +7156,122 @@ func (m *TraceSpanRequest) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *TraceSpanRequest) MarshalTo(dAtA []byte) (int, error) { +func (m *ISCPDrainConsumerRequest) MarshalTo(dAtA []byte) (int, error) { size := m.ProtoSize() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *TraceSpanRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *ISCPDrainConsumerRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if m.Threshold != 0 { - i = encodeVarintQuery(dAtA, i, uint64(m.Threshold)) + if m.RemoveFenceOnly { i-- - dAtA[i] = 0x18 + if m.RemoveFenceOnly { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x28 } - if len(m.Spans) > 0 { - i -= len(m.Spans) - copy(dAtA[i:], m.Spans) - i = encodeVarintQuery(dAtA, i, uint64(len(m.Spans))) + if m.JobID != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.JobID)) i-- - dAtA[i] = 0x12 + dAtA[i] = 0x20 } - if len(m.Cmd) > 0 { - i -= len(m.Cmd) - copy(dAtA[i:], m.Cmd) - i = encodeVarintQuery(dAtA, i, uint64(len(m.Cmd))) + if len(m.JobName) > 0 { + i -= len(m.JobName) + copy(dAtA[i:], m.JobName) + i = encodeVarintQuery(dAtA, i, uint64(len(m.JobName))) i-- - dAtA[i] = 0xa + dAtA[i] = 0x1a + } + if m.TableID != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.TableID)) + i-- + dAtA[i] = 0x10 + } + if m.AccountID != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.AccountID)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *ISCPDrainConsumerResponse) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ISCPDrainConsumerResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ISCPDrainConsumerResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Success { + i-- + if m.Success { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *TraceSpanRequest) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TraceSpanRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *TraceSpanRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Threshold != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.Threshold)) + i-- + dAtA[i] = 0x18 + } + if len(m.Spans) > 0 { + i -= len(m.Spans) + copy(dAtA[i:], m.Spans) + i = encodeVarintQuery(dAtA, i, uint64(len(m.Spans))) + i-- + dAtA[i] = 0x12 + } + if len(m.Cmd) > 0 { + i -= len(m.Cmd) + copy(dAtA[i:], m.Cmd) + i = encodeVarintQuery(dAtA, i, uint64(len(m.Cmd))) + i-- + dAtA[i] = 0xa } return len(dAtA) - i, nil } @@ -7316,12 +7581,12 @@ func (m *TxnInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x12 } - n78, err78 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.CreateAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.CreateAt):]) - if err78 != nil { - return 0, err78 + n80, err80 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.CreateAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.CreateAt):]) + if err80 != nil { + return 0, err80 } - i -= n78 - i = encodeVarintQuery(dAtA, i, uint64(n78)) + i -= n80 + i = encodeVarintQuery(dAtA, i, uint64(n80)) i-- dAtA[i] = 0xa return len(dAtA) - i, nil @@ -9302,6 +9567,10 @@ func (m *Request) ProtoSize() (n int) { l = m.CtlPrefetchOnSubscribedRequest.ProtoSize() n += 2 + l + sovQuery(uint64(l)) } + if m.ISCPDrainConsumerRequest != nil { + l = m.ISCPDrainConsumerRequest.ProtoSize() + n += 2 + l + sovQuery(uint64(l)) + } return n } @@ -9464,6 +9733,10 @@ func (m *Response) ProtoSize() (n int) { l = m.CtlPrefetchOnSubscribedResponse.ProtoSize() n += 2 + l + sovQuery(uint64(l)) } + if m.ISCPDrainConsumerResponse != nil { + l = m.ISCPDrainConsumerResponse.ProtoSize() + n += 2 + l + sovQuery(uint64(l)) + } return n } @@ -9584,6 +9857,43 @@ func (m *CtlPrefetchOnSubscribedResponse) ProtoSize() (n int) { return n } +func (m *ISCPDrainConsumerRequest) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.AccountID != 0 { + n += 1 + sovQuery(uint64(m.AccountID)) + } + if m.TableID != 0 { + n += 1 + sovQuery(uint64(m.TableID)) + } + l = len(m.JobName) + if l > 0 { + n += 1 + l + sovQuery(uint64(l)) + } + if m.JobID != 0 { + n += 1 + sovQuery(uint64(m.JobID)) + } + if m.RemoveFenceOnly { + n += 2 + } + return n +} + +func (m *ISCPDrainConsumerResponse) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Success { + n += 2 + } + return n +} + func (m *TraceSpanRequest) ProtoSize() (n int) { if m == nil { return 0 @@ -12930,6 +13240,42 @@ func (m *Request) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 39: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ISCPDrainConsumerRequest", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ISCPDrainConsumerRequest == nil { + m.ISCPDrainConsumerRequest = &ISCPDrainConsumerRequest{} + } + if err := m.ISCPDrainConsumerRequest.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) @@ -14408,6 +14754,42 @@ func (m *Response) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 40: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ISCPDrainConsumerResponse", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ISCPDrainConsumerResponse == nil { + m.ISCPDrainConsumerResponse = &ISCPDrainConsumerResponse{} + } + if err := m.ISCPDrainConsumerResponse.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) @@ -15152,6 +15534,235 @@ func (m *CtlPrefetchOnSubscribedResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *ISCPDrainConsumerRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ISCPDrainConsumerRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ISCPDrainConsumerRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountID", wireType) + } + m.AccountID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountID |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TableID", wireType) + } + m.TableID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TableID |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field JobName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.JobName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field JobID", wireType) + } + m.JobID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.JobID |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RemoveFenceOnly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RemoveFenceOnly = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipQuery(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthQuery + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ISCPDrainConsumerResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ISCPDrainConsumerResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ISCPDrainConsumerResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Success", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Success = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipQuery(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthQuery + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *TraceSpanRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/queryservice/client/query_client.go b/pkg/queryservice/client/query_client.go index 2d5ed07f81c1e..91da52bb9c916 100644 --- a/pkg/queryservice/client/query_client.go +++ b/pkg/queryservice/client/query_client.go @@ -60,6 +60,7 @@ var methodVersions = map[pb.CmdMethod]int64{ pb.CmdMethod_WorkspaceThreshold: defines.MORPCVersion4, pb.CmdMethod_MinTimestamp: defines.MORPCVersion4, pb.CmdMethod_CtlPrefetchOnSubscribed: defines.MORPCVersion4, + pb.CmdMethod_ISCPDrainConsumer: defines.MORPCVersion4, } type queryClient struct { diff --git a/pkg/sql/compile/iscp_util.go b/pkg/sql/compile/iscp_util.go index 2d6d0e717581c..92881eba03587 100644 --- a/pkg/sql/compile/iscp_util.go +++ b/pkg/sql/compile/iscp_util.go @@ -19,13 +19,17 @@ import ( "fmt" "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/clusterservice" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" indexplugin "github.com/matrixorigin/matrixone/pkg/indexplugin" "github.com/matrixorigin/matrixone/pkg/iscp" "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/pb/metadata" "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/pb/query" + qclient "github.com/matrixorigin/matrixone/pkg/queryservice/client" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/vectorindex/idxcron" ) @@ -36,6 +40,7 @@ var ( iscpLookupJobLogFunc = iscp.LookupJobLog iscpGetExecutorFunc = iscp.GetExecutorRuntime iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpGetCNQueryAddress = getCNQueryAddress isTableInCCPRFunc = isTableInCCPRImpl ) @@ -243,8 +248,28 @@ func drainIndexCdcTaskConsumer( if runnerCN == "" { runnerCN = c.proc.GetService() } - exec, ok := iscpGetExecutorFunc(runnerCN) - if !ok || exec == nil { + key := iscp.NewJobRuntimeKey(accountID, tableID, jobName, jobID) + logutil.Infof("drain index cdc task consumer: accountID=%d tableID=%d jobName=%s jobID=%d", accountID, tableID, jobName, jobID) + if exec, ok := iscpGetExecutorFunc(runnerCN); ok && exec != nil { + if err := exec.CancelAndDrainJobConsumer(c.proc.Ctx, accountID, tableID, jobName, jobID); err != nil { + exec.RemoveJobFence(key) + return err + } + if txnOp := c.proc.GetTxnOperator(); txnOp != nil { + cleanup := client.NewTxnEventCallback(func(_ context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { + if !event.CostEvent { + return nil + } + exec.RemoveJobFence(key) + return nil + }) + txnOp.AppendEventCallback(client.CommitEvent, cleanup) + txnOp.AppendEventCallback(client.RollbackEvent, cleanup) + } + return nil + } + qc := c.proc.GetQueryClient() + if qc == nil { return moerr.NewInternalErrorf( c.proc.Ctx, "cannot confirm ISCP consumer quiescence on CN %s for tableID=%d jobName=%s jobID=%d", @@ -254,19 +279,22 @@ func drainIndexCdcTaskConsumer( jobID, ) } - key := iscp.NewJobRuntimeKey(accountID, tableID, jobName, jobID) - logutil.Infof("drain index cdc task consumer: accountID=%d tableID=%d jobName=%s jobID=%d", accountID, tableID, jobName, jobID) - if err := exec.CancelAndDrainJobConsumer(c.proc.Ctx, accountID, tableID, jobName, jobID); err != nil { - exec.RemoveJobFence(key) + queryAddress, err := iscpGetCNQueryAddress(c.proc.Ctx, c.proc.GetService(), runnerCN) + if err != nil { + return err + } + if err := sendISCPDrainConsumerRequest(c.proc.Ctx, qc, queryAddress, accountID, tableID, jobName, jobID, false); err != nil { return err } if txnOp := c.proc.GetTxnOperator(); txnOp != nil { - cleanup := client.NewTxnEventCallback(func(_ context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { + cleanup := client.NewTxnEventCallback(func(ctx context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { if !event.CostEvent { return nil } - exec.RemoveJobFence(key) - return nil + if ctx == nil { + ctx = c.proc.Ctx + } + return sendISCPDrainConsumerRequest(ctx, qc, queryAddress, accountID, tableID, jobName, jobID, true) }) txnOp.AppendEventCallback(client.CommitEvent, cleanup) txnOp.AppendEventCallback(client.RollbackEvent, cleanup) @@ -274,6 +302,58 @@ func drainIndexCdcTaskConsumer( return nil } +func getCNQueryAddress(ctx context.Context, service string, cnUUID string) (string, error) { + cluster, err := clusterservice.GetMOClusterWithContext(ctx, service) + if err != nil { + return "", err + } + var queryAddress string + err = clusterservice.GetCNServiceWithoutWorkingStateWithContext( + ctx, + cluster, + clusterservice.NewServiceIDSelector(cnUUID), + func(cn metadata.CNService) bool { + queryAddress = cn.QueryAddress + return false + }, + ) + if err != nil { + return "", err + } + if queryAddress == "" { + return "", moerr.NewInternalErrorf(ctx, "cannot find query address for CN %s", cnUUID) + } + return queryAddress, nil +} + +func sendISCPDrainConsumerRequest( + ctx context.Context, + qc qclient.QueryClient, + queryAddress string, + accountID uint32, + tableID uint64, + jobName string, + jobID uint64, + removeFenceOnly bool, +) error { + req := qc.NewRequest(query.CmdMethod_ISCPDrainConsumer) + req.ISCPDrainConsumerRequest = &query.ISCPDrainConsumerRequest{ + AccountID: accountID, + TableID: tableID, + JobName: jobName, + JobID: jobID, + RemoveFenceOnly: removeFenceOnly, + } + resp, err := qc.SendMessage(ctx, queryAddress, req) + if err != nil { + return err + } + if resp != nil { + qc.Release(resp) + } + return nil +} + // drop all cdc tasks according to tableDef func DropAllIndexCdcTasks(c *Compile, tabledef *plan.TableDef, dbname string, tablename string) error { idxmap := make(map[string]bool) diff --git a/pkg/sql/compile/iscp_util_extra_test.go b/pkg/sql/compile/iscp_util_extra_test.go index 96a8dd0a90148..999db6eee6f78 100644 --- a/pkg/sql/compile/iscp_util_extra_test.go +++ b/pkg/sql/compile/iscp_util_extra_test.go @@ -19,15 +19,45 @@ import ( "testing" "github.com/golang/mock/gomock" + "github.com/matrixorigin/matrixone/pkg/common/moerr" mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" "github.com/matrixorigin/matrixone/pkg/iscp" "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/pb/query" "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type iscpDrainTestQueryClient struct { + serviceID string + requests []*query.Request + err error +} + +func (q *iscpDrainTestQueryClient) ServiceID() string { + return q.serviceID +} + +func (q *iscpDrainTestQueryClient) SendMessage(_ context.Context, _ string, req *query.Request) (*query.Response, error) { + q.requests = append(q.requests, req) + if q.err != nil { + return nil, q.err + } + return &query.Response{CmdMethod: req.CmdMethod}, nil +} + +func (q *iscpDrainTestQueryClient) NewRequest(method query.CmdMethod) *query.Request { + return &query.Request{CmdMethod: method} +} + +func (q *iscpDrainTestQueryClient) Release(*query.Response) {} + +func (q *iscpDrainTestQueryClient) Close() error { + return nil +} + func TestCoverage_genCdcTaskJobID(t *testing.T) { tests := []struct { name string @@ -428,6 +458,151 @@ func TestDrainIndexCdcTaskConsumerNoRunnerExecutorFailsClosed(t *testing.T) { require.ErrorContains(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1"), "cannot confirm ISCP consumer quiescence on CN runner-cn") } +func TestDrainIndexCdcTaskConsumerRoutesToRemoteRunnerQueryService(t *testing.T) { + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + iscpGetCNQueryAddress = func(context.Context, string, string) (string, error) { + return "runner-cn:18101", nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + iscpGetCNQueryAddress = getCNQueryAddress + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + qc := &iscpDrainTestQueryClient{serviceID: "ddl-cn"} + c.proc.Base.QueryClient = qc + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + require.Len(t, qc.requests, 1) + req := qc.requests[0] + require.Equal(t, query.CmdMethod_ISCPDrainConsumer, req.CmdMethod) + require.NotNil(t, req.ISCPDrainConsumerRequest) + require.Equal(t, uint32(0), req.ISCPDrainConsumerRequest.AccountID) + require.Equal(t, uint64(42), req.ISCPDrainConsumerRequest.TableID) + require.Equal(t, "index_idx1", req.ISCPDrainConsumerRequest.JobName) + require.Equal(t, uint64(7), req.ISCPDrainConsumerRequest.JobID) + require.False(t, req.ISCPDrainConsumerRequest.RemoveFenceOnly) +} + +func TestDrainIndexCdcTaskConsumerRemoteFenceCleanupOnTxnEvent(t *testing.T) { + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + iscpGetCNQueryAddress = func(context.Context, string, string) (string, error) { + return "runner-cn:18101", nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + iscpGetCNQueryAddress = getCNQueryAddress + }() + + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + var rollbackCleanup client.TxnEventCallback + txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).Times(1) + txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + rollbackCleanup = cb + }, + ).Times(1) + + c := &Compile{} + c.proc = testutil.NewProcess(t) + c.proc.Base.TxnOperator = txnOp + qc := &iscpDrainTestQueryClient{serviceID: "ddl-cn"} + c.proc.Base.QueryClient = qc + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + require.NotNil(t, rollbackCleanup.Func) + require.NoError(t, rollbackCleanup.Func(context.Background(), txnOp, client.TxnEvent{CostEvent: true}, nil)) + + require.Len(t, qc.requests, 2) + require.False(t, qc.requests[0].ISCPDrainConsumerRequest.RemoveFenceOnly) + require.True(t, qc.requests[1].ISCPDrainConsumerRequest.RemoveFenceOnly) +} + +func TestDrainIndexCdcTaskConsumerRemoteQueryFailureFailsClosed(t *testing.T) { + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + iscpGetCNQueryAddress = func(context.Context, string, string) (string, error) { + return "runner-cn:18101", nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + iscpGetCNQueryAddress = getCNQueryAddress + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + c.proc.Base.QueryClient = &iscpDrainTestQueryClient{ + serviceID: "ddl-cn", + err: moerr.NewInternalErrorNoCtx("remote drain failed"), + } + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.ErrorContains(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1"), "remote drain failed") +} + func TestDrainIndexCdcTaskConsumerRegistersRollbackFenceCleanup(t *testing.T) { exec := &iscp.ISCPTaskExecutor{} iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { diff --git a/proto/query.proto b/proto/query.proto index d32e5cf1d7097..3142a3406a1fd 100644 --- a/proto/query.proto +++ b/proto/query.proto @@ -102,6 +102,7 @@ enum CmdMethod { WorkspaceThreshold = 34; MinTimestamp = 35; CtlPrefetchOnSubscribed = 36; + ISCPDrainConsumer = 37; } // QueryRequest is the common query request. It contains the query @@ -242,6 +243,7 @@ message Request { CtlMoTableStatsRequest CtlMoTableStatsRequest = 36 [ (gogoproto.nullable) = false ]; WorkspaceThresholdRequest WorkspaceThresholdRequest = 37; CtlPrefetchOnSubscribedRequest CtlPrefetchOnSubscribedRequest = 38; + ISCPDrainConsumerRequest ISCPDrainConsumerRequest = 39; } // ShowProcessListResponse is the response of command ShowProcessList. @@ -307,6 +309,7 @@ message Response { WorkspaceThresholdResponse WorkspaceThresholdResponse = 37; MinTimestampResponse MinTimestampResponse = 38; CtlPrefetchOnSubscribedResponse CtlPrefetchOnSubscribedResponse = 39; + ISCPDrainConsumerResponse ISCPDrainConsumerResponse = 40; } // AlterAccountRequest is the "alter account restricted" query request. @@ -352,6 +355,18 @@ message CtlPrefetchOnSubscribedResponse { string Resp = 1; } +message ISCPDrainConsumerRequest { + uint32 AccountID = 1; + uint64 TableID = 2; + string JobName = 3; + uint64 JobID = 4; + bool RemoveFenceOnly = 5; +} + +message ISCPDrainConsumerResponse { + bool Success = 1; +} + message TraceSpanRequest { string Cmd = 1; string Spans = 2; From 1083581c6127bf4121e509a990f824c06b8650b9 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 16 Jul 2026 15:54:22 +0800 Subject: [PATCH 09/14] fix: retain iscp fences until drop logtail --- pkg/cnservice/server_query.go | 6 ++ pkg/iscp/executor.go | 5 +- pkg/iscp/runtime_cancel.go | 34 ++++++- pkg/iscp/runtime_cancel_test.go | 116 +++++++++++++++++++++++- pkg/iscp/types.go | 6 +- pkg/objectio/injects.go | 2 + pkg/sql/compile/iscp_util.go | 13 +-- pkg/sql/compile/iscp_util_extra_test.go | 57 +++++++++++- 8 files changed, 223 insertions(+), 16 deletions(-) diff --git a/pkg/cnservice/server_query.go b/pkg/cnservice/server_query.go index c5bf988e1f2cc..8e0c062190d4d 100644 --- a/pkg/cnservice/server_query.go +++ b/pkg/cnservice/server_query.go @@ -212,6 +212,12 @@ func (s *service) handleISCPDrainConsumer(ctx context.Context, req *query.Reques } key := iscp.NewJobRuntimeKey(r.AccountID, r.TableID, r.JobName, r.JobID) if r.RemoveFenceOnly { + if _, msg, injected := fault.TriggerFault(objectio.FJ_ISCPCancelRemoveFenceError); injected { + if msg == "" { + msg = objectio.FJ_ISCPCancelRemoveFenceError + } + return moerr.NewInternalErrorNoCtxf("injected ISCP remove fence error: %s", msg) + } exec.RemoveJobFence(key) resp.ISCPDrainConsumerResponse = &query.ISCPDrainConsumerResponse{Success: true} return nil diff --git a/pkg/iscp/executor.go b/pkg/iscp/executor.go index ac448ccd6993c..9fe5a757ccee7 100644 --- a/pkg/iscp/executor.go +++ b/pkg/iscp/executor.go @@ -187,7 +187,7 @@ func NewISCPTaskExecutor( tableMu: sync.RWMutex{}, option: option, mp: mp, - fencedJobs: make(map[JobRuntimeKey]struct{}), + fencedJobs: make(map[JobRuntimeKey]JobFence), runningConsumers: make( map[JobRuntimeKey]map[uint64]*RunningJobConsumer, ), @@ -972,6 +972,9 @@ func (exec *ISCPTaskExecutor) addOrUpdateJob( notPrint bool, ) error { var newCreate bool + if dropAt != 0 { + exec.RemoveJobFence(NewJobRuntimeKey(accountID, tableID, jobName, jobID)) + } var watermark types.TS defer func() { diff --git a/pkg/iscp/runtime_cancel.go b/pkg/iscp/runtime_cancel.go index 3552a7500ac30..449d880b82634 100644 --- a/pkg/iscp/runtime_cancel.go +++ b/pkg/iscp/runtime_cancel.go @@ -17,12 +17,17 @@ package iscp import ( "context" "sync" + "time" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/util/fault" ) var iscpExecutors sync.Map // map[cnUUID]*ISCPTaskExecutor +const DefaultRollbackFenceTTL = 5 * time.Minute + func RegisterExecutorRuntime(cnUUID string, exec *ISCPTaskExecutor) { if cnUUID == "" || exec == nil { return @@ -59,21 +64,40 @@ func (key JobRuntimeKey) consumerGroupKey() JobRuntimeKey { func (exec *ISCPTaskExecutor) ensureRuntimeMapsLocked() { if exec.fencedJobs == nil { - exec.fencedJobs = make(map[JobRuntimeKey]struct{}) + exec.fencedJobs = make(map[JobRuntimeKey]JobFence) } if exec.runningConsumers == nil { exec.runningConsumers = make(map[JobRuntimeKey]map[uint64]*RunningJobConsumer) } } +func rollbackFenceTTL() time.Duration { + ttl := DefaultRollbackFenceTTL + if seconds, _, injected := fault.TriggerFault(objectio.FJ_ISCPCancelRollbackFenceTTL); injected && seconds > 0 { + ttl = time.Duration(seconds) * time.Second + } + return ttl +} + func (exec *ISCPTaskExecutor) IsJobFenced(key JobRuntimeKey) bool { if exec == nil { return false } exec.runtimeMu.Lock() defer exec.runtimeMu.Unlock() - _, ok := exec.fencedJobs[key] - return ok + return exec.isJobFencedLocked(key, time.Now()) +} + +func (exec *ISCPTaskExecutor) isJobFencedLocked(key JobRuntimeKey, now time.Time) bool { + fence, ok := exec.fencedJobs[key] + if !ok { + return false + } + if !fence.ExpireAt.IsZero() && now.After(fence.ExpireAt) { + delete(exec.fencedJobs, key) + return false + } + return true } func (exec *ISCPTaskExecutor) CancelAndDrainJobConsumer( @@ -92,7 +116,7 @@ func (exec *ISCPTaskExecutor) CancelAndDrainJobConsumer( exec.runtimeMu.Lock() exec.ensureRuntimeMapsLocked() - exec.fencedJobs[key] = struct{}{} + exec.fencedJobs[key] = JobFence{ExpireAt: time.Now().Add(rollbackFenceTTL())} handles := make([]*RunningJobConsumer, 0, 1) if byJobID := exec.runningConsumers[groupKey]; byJobID != nil { for runningJobID, h := range byJobID { @@ -155,7 +179,7 @@ func (exec *ISCPTaskExecutor) RegisterRunningConsumer( exec.runtimeMu.Lock() defer exec.runtimeMu.Unlock() exec.ensureRuntimeMapsLocked() - if _, fenced := exec.fencedJobs[key]; fenced { + if exec.isJobFencedLocked(key, time.Now()) { return nil, false } groupKey := key.consumerGroupKey() diff --git a/pkg/iscp/runtime_cancel_test.go b/pkg/iscp/runtime_cancel_test.go index db4553c97d4e7..2c823f57cee5c 100644 --- a/pkg/iscp/runtime_cancel_test.go +++ b/pkg/iscp/runtime_cancel_test.go @@ -21,12 +21,14 @@ import ( "time" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/util/fault" "github.com/stretchr/testify/require" ) func newRuntimeTestExecutor() *ISCPTaskExecutor { return &ISCPTaskExecutor{ - fencedJobs: make(map[JobRuntimeKey]struct{}), + fencedJobs: make(map[JobRuntimeKey]JobFence), runningConsumers: make(map[JobRuntimeKey]map[uint64]*RunningJobConsumer), option: fillDefaultOption(nil), ctx: context.Background(), @@ -87,6 +89,40 @@ func TestRegisterRunningConsumerRejectsFencedJob(t *testing.T) { require.False(t, ok) } +func TestExpiredJobFenceIsClearedWhenChecked(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + exec.fencedJobs[key] = JobFence{ExpireAt: time.Now().Add(-time.Second)} + + require.False(t, exec.IsJobFenced(key)) + + exec.runtimeMu.Lock() + _, exists := exec.fencedJobs[key] + exec.runtimeMu.Unlock() + require.False(t, exists) + + _, ok := exec.RegisterRunningConsumer(key, 1, 1, func() {}, nil) + require.True(t, ok) +} + +func TestRollbackFenceTTLCanBeInjected(t *testing.T) { + require.True(t, fault.Enable()) + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL, ":::", "echo", 2, "", false)) + defer func() { + _, _ = fault.RemoveFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL) + }() + + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + + exec.runtimeMu.Lock() + fence := exec.fencedJobs[key] + exec.runtimeMu.Unlock() + require.WithinDuration(t, time.Now().Add(2*time.Second), fence.ExpireAt, 500*time.Millisecond) +} + func TestCancelAndDrainJobConsumerOnlyCancelsMatchingJob(t *testing.T) { exec := newRuntimeTestExecutor() key1 := NewJobRuntimeKey(1, 2, "index_idx01", 1) @@ -184,6 +220,24 @@ func TestTableEntryGetCandidateDoesNotSkipRecreatedSameNameJob(t *testing.T) { require.Equal(t, []uint64{2}, iters[0].jobIDs) } +func TestTableEntryGetCandidateClearsExpiredRollbackFence(t *testing.T) { + exec := newRuntimeTestExecutor() + table := NewTableEntry(exec, 1, 10, 2, "db", "tbl") + spec := &JobSpec{TriggerSpec: TriggerSpec{JobType: TriggerType_Default}} + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + table.jobs[JobKey{JobName: key.JobName, JobID: key.JobID}] = NewJobEntry(table, key.JobName, spec, key.JobID, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + exec.runtimeMu.Lock() + exec.fencedJobs[key] = JobFence{ExpireAt: time.Now().Add(-time.Second)} + exec.runtimeMu.Unlock() + + iters, _ := table.getCandidate() + + require.Len(t, iters, 1) + require.Equal(t, []string{key.JobName}, iters[0].jobNames) + require.False(t, exec.IsJobFenced(key)) +} + func TestRemoveTableJobFencesOnlyClearsMatchingTable(t *testing.T) { exec := newRuntimeTestExecutor() key1 := NewJobRuntimeKey(1, 2, "index_idx01", 1) @@ -200,6 +254,54 @@ func TestRemoveTableJobFencesOnlyClearsMatchingTable(t *testing.T) { require.True(t, exec.IsJobFenced(keyOtherTable)) } +func TestAddOrUpdateJobDropAtClearsGenerationFence(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + table := NewTableEntry(exec, key.AccountID, 1, key.TableID, "db", "tbl") + exec.setTable(table) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + require.True(t, exec.IsJobFenced(key)) + + err := exec.addOrUpdateJob( + key.AccountID, + key.TableID, + key.JobName, + key.JobID, + ISCPJobState_Completed, + types.BuildTS(1, 0).ToString(), + mustEncodeRuntimeJobSpec(t), + encodeJSONRows(t, []string{mustMarshalJobStatus(t, 1, JobStage_Running)})[0], + types.Timestamp(time.Now().UnixNano()), + false, + ) + + require.NoError(t, err) + require.False(t, exec.IsJobFenced(key)) +} + +func TestAddOrUpdateJobDropAtClearsGenerationFenceWhenTableIsAbsent(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + require.True(t, exec.IsJobFenced(key)) + + err := exec.addOrUpdateJob( + key.AccountID, + key.TableID, + key.JobName, + key.JobID, + ISCPJobState_Completed, + types.BuildTS(1, 0).ToString(), + mustEncodeRuntimeJobSpec(t), + encodeJSONRows(t, []string{mustMarshalJobStatus(t, 1, JobStage_Running)})[0], + types.Timestamp(time.Now().UnixNano()), + false, + ) + + require.NoError(t, err) + require.False(t, exec.IsJobFenced(key)) +} + func TestCancelAndDrainJobConsumerHonorsCallerContext(t *testing.T) { exec := newRuntimeTestExecutor() key := NewJobRuntimeKey(1, 2, "index_idx01", 1) @@ -214,3 +316,15 @@ func TestCancelAndDrainJobConsumerHonorsCallerContext(t *testing.T) { require.True(t, errors.Is(err, context.Canceled)) <-consumerCancelCalled } + +func mustEncodeRuntimeJobSpec(t *testing.T) []byte { + t.Helper() + spec, err := MarshalJobSpec(&JobSpec{ + ConsumerInfo: ConsumerInfo{ + SrcTable: TableInfo{DBID: 1, TableID: 2, DBName: "db", TableName: "tbl"}, + IndexName: "idx01", + }, + }) + require.NoError(t, err) + return encodeJSONRows(t, []string{spec})[0] +} diff --git a/pkg/iscp/types.go b/pkg/iscp/types.go index 2fdfc19105496..05547806ff6a1 100644 --- a/pkg/iscp/types.go +++ b/pkg/iscp/types.go @@ -161,7 +161,7 @@ type ISCPTaskExecutor struct { runningMu sync.Mutex runtimeMu sync.Mutex - fencedJobs map[JobRuntimeKey]struct{} + fencedJobs map[JobRuntimeKey]JobFence runningConsumers map[JobRuntimeKey]map[uint64]*RunningJobConsumer } @@ -172,6 +172,10 @@ type JobRuntimeKey struct { JobID uint64 } +type JobFence struct { + ExpireAt time.Time +} + type RunningJobConsumer struct { key JobRuntimeKey jobID uint64 diff --git a/pkg/objectio/injects.go b/pkg/objectio/injects.go index 0f0fa761d75ac..86200c55c36ab 100644 --- a/pkg/objectio/injects.go +++ b/pkg/objectio/injects.go @@ -77,6 +77,8 @@ const ( FJ_ISCPCancelLongHnswBeforeUpdate = "fj/iscp/cancel/long/hnsw-before-update" FJ_ISCPCancelLongHnswBeforeSave = "fj/iscp/cancel/long/hnsw-before-save" FJ_ISCPCancelLongBeforeWatermark = "fj/iscp/cancel/long/before-watermark" + FJ_ISCPCancelRemoveFenceError = "fj/iscp/cancel/remove-fence-error" + FJ_ISCPCancelRollbackFenceTTL = "fj/iscp/cancel/rollback-fence-ttl" FJ_PublicationSnapshotFinished = "fj/publication/snapshot/finished" diff --git a/pkg/sql/compile/iscp_util.go b/pkg/sql/compile/iscp_util.go index 92881eba03587..9f94a4d3b8157 100644 --- a/pkg/sql/compile/iscp_util.go +++ b/pkg/sql/compile/iscp_util.go @@ -263,7 +263,6 @@ func drainIndexCdcTaskConsumer( exec.RemoveJobFence(key) return nil }) - txnOp.AppendEventCallback(client.CommitEvent, cleanup) txnOp.AppendEventCallback(client.RollbackEvent, cleanup) } return nil @@ -291,12 +290,14 @@ func drainIndexCdcTaskConsumer( if !event.CostEvent { return nil } - if ctx == nil { - ctx = c.proc.Ctx - } - return sendISCPDrainConsumerRequest(ctx, qc, queryAddress, accountID, tableID, jobName, jobID, true) + cleanupCtx, cancel := context.WithTimeoutCause( + context.Background(), + iscp.DefaultRollbackFenceTTL, + moerr.NewInternalErrorNoCtx("iscp rollback fence cleanup timeout"), + ) + defer cancel() + return sendISCPDrainConsumerRequest(cleanupCtx, qc, queryAddress, accountID, tableID, jobName, jobID, true) }) - txnOp.AppendEventCallback(client.CommitEvent, cleanup) txnOp.AppendEventCallback(client.RollbackEvent, cleanup) } return nil diff --git a/pkg/sql/compile/iscp_util_extra_test.go b/pkg/sql/compile/iscp_util_extra_test.go index 999db6eee6f78..c02f33721479d 100644 --- a/pkg/sql/compile/iscp_util_extra_test.go +++ b/pkg/sql/compile/iscp_util_extra_test.go @@ -529,7 +529,6 @@ func TestDrainIndexCdcTaskConsumerRemoteFenceCleanupOnTxnEvent(t *testing.T) { ctrl := gomock.NewController(t) txnOp := mock_frontend.NewMockTxnOperator(ctrl) var rollbackCleanup client.TxnEventCallback - txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).Times(1) txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).DoAndReturn( func(_ client.EventType, cb client.TxnEventCallback) { rollbackCleanup = cb @@ -562,6 +561,61 @@ func TestDrainIndexCdcTaskConsumerRemoteFenceCleanupOnTxnEvent(t *testing.T) { require.True(t, qc.requests[1].ISCPDrainConsumerRequest.RemoveFenceOnly) } +func TestDrainIndexCdcTaskConsumerRemoteRollbackCleanupIgnoresCanceledCallerContext(t *testing.T) { + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + iscpGetCNQueryAddress = func(context.Context, string, string) (string, error) { + return "runner-cn:18101", nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + iscpGetCNQueryAddress = getCNQueryAddress + }() + + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + var rollbackCleanup client.TxnEventCallback + txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + rollbackCleanup = cb + }, + ).Times(1) + + c := &Compile{} + c.proc = testutil.NewProcess(t) + c.proc.Base.TxnOperator = txnOp + qc := &iscpDrainTestQueryClient{serviceID: "ddl-cn"} + c.proc.Base.QueryClient = qc + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + require.NoError(t, rollbackCleanup.Func(canceledCtx, txnOp, client.TxnEvent{CostEvent: true}, nil)) + + require.Len(t, qc.requests, 2) + require.True(t, qc.requests[1].ISCPDrainConsumerRequest.RemoveFenceOnly) +} + func TestDrainIndexCdcTaskConsumerRemoteQueryFailureFailsClosed(t *testing.T) { iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { return nil, false @@ -623,7 +677,6 @@ func TestDrainIndexCdcTaskConsumerRegistersRollbackFenceCleanup(t *testing.T) { ctrl := gomock.NewController(t) txnOp := mock_frontend.NewMockTxnOperator(ctrl) var rollbackCleanup client.TxnEventCallback - txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).Times(1) txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).DoAndReturn( func(_ client.EventType, cb client.TxnEventCallback) { rollbackCleanup = cb From 195e9f36d0ff1ab030493273b2e986d8e859b78d Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 16 Jul 2026 16:53:56 +0800 Subject: [PATCH 10/14] fix: query iscp runner from system account --- pkg/iscp/util.go | 5 ++++- pkg/iscp/util_readrows_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/pkg/iscp/util.go b/pkg/iscp/util.go index 3532892e4c207..f61d01d996d86 100644 --- a/pkg/iscp/util.go +++ b/pkg/iscp/util.go @@ -30,6 +30,7 @@ import ( "go.uber.org/zap" + "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/batch" @@ -38,6 +39,7 @@ import ( // "github.com/matrixorigin/matrixone/pkg/container/bytejson" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/util/executor" @@ -466,7 +468,8 @@ func GetTaskRunner( cnUUID string, txn client.TxnOperator, ) (string, error) { - ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, time.Minute*5, moerr.NewInternalErrorNoCtx("iscp get task runner timeout")) + ctxWithSysAccount := context.WithValue(ctx, defines.TenantIDKey{}, catalog.System_Account) + ctxWithTimeout, cancel := context.WithTimeoutCause(ctxWithSysAccount, time.Minute*5, moerr.NewInternalErrorNoCtx("iscp get task runner timeout")) defer cancel() sql := `select task_runner from mo_task.sys_daemon_task where task_type = "ISCP" and task_runner is not null` diff --git a/pkg/iscp/util_readrows_test.go b/pkg/iscp/util_readrows_test.go index 0ce829862e229..92a590403a134 100644 --- a/pkg/iscp/util_readrows_test.go +++ b/pkg/iscp/util_readrows_test.go @@ -15,10 +15,14 @@ package iscp import ( + "context" "testing" + "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/util/executor" "github.com/stretchr/testify/require" ) @@ -77,6 +81,34 @@ func TestReadSingleTaskRunnerRejectsEmptyRunner(t *testing.T) { require.Contains(t, err.Error(), "task runner is null") } +func TestGetTaskRunnerQueriesMOTaskWithSystemAccount(t *testing.T) { + oldExecWithResult := ExecWithResult + defer func() { + ExecWithResult = oldExecWithResult + }() + + result, mp := newTaskRunnerResult(t, [][]string{{"cn1"}}) + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + mpool.DeleteMPool(mp) + }() + + ExecWithResult = func(ctx context.Context, sql string, cnUUID string, txn client.TxnOperator) (executor.Result, error) { + accountID, err := defines.GetAccountId(ctx) + require.NoError(t, err) + require.Equal(t, uint32(catalog.System_Account), accountID) + require.Contains(t, sql, "mo_task.sys_daemon_task") + require.Equal(t, "cn0", cnUUID) + require.Nil(t, txn) + return result, nil + } + + tenantCtx := defines.AttachAccountId(context.Background(), 1001) + runner, err := GetTaskRunner(tenantCtx, "cn0", nil) + require.NoError(t, err) + require.Equal(t, "cn1", runner) +} + func newTaskRunnerResult(t *testing.T, batches [][]string) (executor.Result, *mpool.MPool) { t.Helper() From a08c234d400a64cf040ba7e2b20b682e92ca08bb Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 16 Jul 2026 17:03:59 +0800 Subject: [PATCH 11/14] fix: close iscp fence handoff window --- pkg/iscp/executor.go | 8 +-- pkg/iscp/runtime_cancel_test.go | 30 +++++++++++ pkg/util/fault/fault.go | 24 ++++++--- pkg/util/fault/fault_test.go | 90 +++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 9 deletions(-) diff --git a/pkg/iscp/executor.go b/pkg/iscp/executor.go index 9fe5a757ccee7..8847e71868d9f 100644 --- a/pkg/iscp/executor.go +++ b/pkg/iscp/executor.go @@ -972,9 +972,7 @@ func (exec *ISCPTaskExecutor) addOrUpdateJob( notPrint bool, ) error { var newCreate bool - if dropAt != 0 { - exec.RemoveJobFence(NewJobRuntimeKey(accountID, tableID, jobName, jobID)) - } + fenceKey := NewJobRuntimeKey(accountID, tableID, jobName, jobID) var watermark types.TS defer func() { @@ -1005,6 +1003,7 @@ func (exec *ISCPTaskExecutor) addOrUpdateJob( table, ok := exec.getTable(accountID, tableID) if !ok { if dropAt != 0 { + exec.RemoveJobFence(fenceKey) return nil } table = NewTableEntry( @@ -1018,6 +1017,9 @@ func (exec *ISCPTaskExecutor) addOrUpdateJob( exec.setTable(table) } newCreate = table.AddOrUpdateSinker(exec.ctx, jobName, jobSpec, jobStatus, jobID, watermark, state, dropAt) + if dropAt != 0 { + exec.RemoveJobFence(fenceKey) + } return nil } diff --git a/pkg/iscp/runtime_cancel_test.go b/pkg/iscp/runtime_cancel_test.go index 2c823f57cee5c..26b150c5934c6 100644 --- a/pkg/iscp/runtime_cancel_test.go +++ b/pkg/iscp/runtime_cancel_test.go @@ -277,6 +277,36 @@ func TestAddOrUpdateJobDropAtClearsGenerationFence(t *testing.T) { require.NoError(t, err) require.False(t, exec.IsJobFenced(key)) + iters, _ := table.getCandidate() + require.Empty(t, iters) +} + +func TestAddOrUpdateJobDropAtPreservesFenceOnParseFailure(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + table := NewTableEntry(exec, key.AccountID, 1, key.TableID, "db", "tbl") + spec := &JobSpec{TriggerSpec: TriggerSpec{JobType: TriggerType_Default}} + table.jobs[JobKey{JobName: key.JobName, JobID: key.JobID}] = NewJobEntry(table, key.JobName, spec, key.JobID, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + exec.setTable(table) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + + err := exec.addOrUpdateJob( + key.AccountID, + key.TableID, + key.JobName, + key.JobID, + ISCPJobState_Completed, + types.BuildTS(1, 0).ToString(), + []byte("bad-json"), + encodeJSONRows(t, []string{mustMarshalJobStatus(t, 1, JobStage_Running)})[0], + types.Timestamp(time.Now().UnixNano()), + false, + ) + + require.Error(t, err) + require.True(t, exec.IsJobFenced(key)) + iters, _ := table.getCandidate() + require.Empty(t, iters) } func TestAddOrUpdateJobDropAtClearsGenerationFenceWhenTableIsAbsent(t *testing.T) { diff --git a/pkg/util/fault/fault.go b/pkg/util/fault/fault.go index 0fcf71b7c3b70..130b56aeecd59 100644 --- a/pkg/util/fault/fault.go +++ b/pkg/util/fault/fault.go @@ -83,11 +83,12 @@ type faultEntry struct { sarg string // string arg constant bool - nWaiters int - removed bool - mutex sync.Mutex - cond *sync.Cond - scope Domain + nWaiters int + notifySeq int64 + removed bool + mutex sync.Mutex + cond *sync.Cond + scope Domain } type faultMap struct { @@ -190,11 +191,17 @@ func (e *faultEntry) do() (int64, string) { } case NOTIFY: if ee := lookup(e.scope, e.sarg); ee != nil { + ee.mutex.Lock() + ee.notifySeq++ ee.cond.Signal() + ee.mutex.Unlock() } case NOTIFYALL: if ee := lookup(e.scope, e.sarg); ee != nil { + ee.mutex.Lock() + ee.notifySeq++ ee.cond.Broadcast() + ee.mutex.Unlock() } case PANIC: switch e.iarg { @@ -228,17 +235,22 @@ func (e *faultEntry) doWithContext(ctx context.Context) (int64, string) { return 0, "" } e.nWaiters += 1 + notifySeq := e.notifySeq + canceled := false done := make(chan struct{}) go func() { select { case <-ctx.Done(): e.mutex.Lock() + canceled = true e.cond.Broadcast() e.mutex.Unlock() case <-done: } }() - e.cond.Wait() + for !e.removed && !canceled && e.notifySeq == notifySeq { + e.cond.Wait() + } e.nWaiters -= 1 close(done) e.mutex.Unlock() diff --git a/pkg/util/fault/fault_test.go b/pkg/util/fault/fault_test.go index a9f4ee3fa135f..fa84b5b6cbd9d 100644 --- a/pkg/util/fault/fault_test.go +++ b/pkg/util/fault/fault_test.go @@ -308,6 +308,96 @@ func TestTriggerWaitFaultWithContextReturnsOnCancel(t *testing.T) { }, time.Second, 10*time.Millisecond) } +func TestTriggerWaitFaultWithContextCancelDoesNotReleaseOtherWaiters(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + require.NoError(t, AddFaultPoint(ctx, "gw", ":::", "getwaiters", 0, "w", false)) + + waitCtx1, cancel1 := context.WithCancel(ctx) + waitCtx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + done1 := make(chan struct{}) + done2 := make(chan struct{}) + go func() { + TriggerFaultWithContext(waitCtx1, "w") + close(done1) + }() + go func() { + TriggerFaultWithContext(waitCtx2, "w") + close(done2) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := TriggerFault("gw") + return ok && cnt == 2 + }, time.Second, 10*time.Millisecond) + + cancel1() + require.Eventually(t, func() bool { + select { + case <-done1: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) + + select { + case <-done2: + t.Fatal("canceling one context-aware wait released another live waiter") + default: + } + cnt, _, ok := TriggerFault("gw") + require.True(t, ok) + require.Equal(t, int64(1), cnt) + + cancel2() + require.Eventually(t, func() bool { + select { + case <-done2: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) +} + +func TestTriggerWaitFaultWithContextReturnsOnNotifyAll(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + require.NoError(t, AddFaultPoint(ctx, "gw", ":::", "getwaiters", 0, "w", false)) + require.NoError(t, AddFaultPoint(ctx, "nall", ":::", "notifyall", 0, "w", false)) + + done := make(chan struct{}) + go func() { + TriggerFaultWithContext(ctx, "w") + close(done) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := TriggerFault("gw") + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + TriggerFault("nall") + require.Eventually(t, func() bool { + select { + case <-done: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) +} + func TestTriggerSleepFaultWithContextReturnsOnCancel(t *testing.T) { ctx := context.Background() From 6f143db29a1ec53920eb43e470062fffa2e485ba Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 16 Jul 2026 17:40:32 +0800 Subject: [PATCH 12/14] fix: renew iscp fences during open ddl txn --- pkg/cnservice/server_query.go | 5 ++ pkg/cnservice/server_query_test.go | 49 ++++++++++++++ pkg/iscp/runtime_cancel.go | 16 ++++- pkg/pb/query/query.pb.go | 41 ++++++++++++ pkg/sql/compile/iscp_util.go | 78 +++++++++++++++++++++- pkg/sql/compile/iscp_util_extra_test.go | 88 +++++++++++++++++++++++++ 6 files changed, 273 insertions(+), 4 deletions(-) diff --git a/pkg/cnservice/server_query.go b/pkg/cnservice/server_query.go index 8e0c062190d4d..0b118db53e826 100644 --- a/pkg/cnservice/server_query.go +++ b/pkg/cnservice/server_query.go @@ -222,6 +222,11 @@ func (s *service) handleISCPDrainConsumer(ctx context.Context, req *query.Reques resp.ISCPDrainConsumerResponse = &query.ISCPDrainConsumerResponse{Success: true} return nil } + if r.RenewFenceOnly { + exec.RenewJobFence(key, iscp.RollbackFenceTTL()) + resp.ISCPDrainConsumerResponse = &query.ISCPDrainConsumerResponse{Success: true} + return nil + } if err := exec.CancelAndDrainJobConsumer(ctx, r.AccountID, r.TableID, r.JobName, r.JobID); err != nil { exec.RemoveJobFence(key) return err diff --git a/pkg/cnservice/server_query_test.go b/pkg/cnservice/server_query_test.go index 2641ac3387771..e60f2ac52a84d 100644 --- a/pkg/cnservice/server_query_test.go +++ b/pkg/cnservice/server_query_test.go @@ -21,6 +21,7 @@ import ( goruntime "runtime" "runtime/debug" "testing" + "time" "unsafe" "github.com/golang/mock/gomock" @@ -40,7 +41,9 @@ import ( "github.com/matrixorigin/matrixone/pkg/frontend/test/mock_shard" "github.com/matrixorigin/matrixone/pkg/frontend/test/mock_task" "github.com/matrixorigin/matrixone/pkg/incrservice" + "github.com/matrixorigin/matrixone/pkg/iscp" "github.com/matrixorigin/matrixone/pkg/lockservice" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/lock" "github.com/matrixorigin/matrixone/pkg/pb/query" "github.com/matrixorigin/matrixone/pkg/pb/statsinfo" @@ -50,6 +53,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/taskservice" "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/txn/client" + "github.com/matrixorigin/matrixone/pkg/util/fault" "github.com/matrixorigin/matrixone/pkg/util/trace" "github.com/matrixorigin/matrixone/pkg/vm/engine" ) @@ -57,6 +61,51 @@ import ( var dummyBadRequestErr = moerr.NewInternalError(context.TODO(), "bad request") var dummyErr = moerr.NewInternalError(context.TODO(), "dummy error") +func Test_service_handleISCPDrainConsumerRenewFenceOnly(t *testing.T) { + require.True(t, fault.Enable()) + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL, ":::", "echo", 1, "", false)) + defer func() { + _, _ = fault.RemoveFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL) + }() + + exec := &iscp.ISCPTaskExecutor{} + iscp.RegisterExecutorRuntime("runner-cn", exec) + defer iscp.UnregisterExecutorRuntime("runner-cn", exec) + + s := &service{cfg: &Config{UUID: "runner-cn"}} + key := iscp.NewJobRuntimeKey(1, 42, "index_idx1", 7) + + resp := &query.Response{} + require.NoError(t, s.handleISCPDrainConsumer(context.Background(), &query.Request{ + ISCPDrainConsumerRequest: &query.ISCPDrainConsumerRequest{ + AccountID: key.AccountID, + TableID: key.TableID, + JobName: key.JobName, + JobID: key.JobID, + RenewFenceOnly: true, + }, + }, resp, nil)) + require.NotNil(t, resp.ISCPDrainConsumerResponse) + require.False(t, exec.IsJobFenced(key)) + + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + time.Sleep(700 * time.Millisecond) + + resp = &query.Response{} + require.NoError(t, s.handleISCPDrainConsumer(context.Background(), &query.Request{ + ISCPDrainConsumerRequest: &query.ISCPDrainConsumerRequest{ + AccountID: key.AccountID, + TableID: key.TableID, + JobName: key.JobName, + JobID: key.JobID, + RenewFenceOnly: true, + }, + }, resp, nil)) + time.Sleep(700 * time.Millisecond) + require.True(t, exec.IsJobFenced(key)) +} + func Test_service_handleGoMaxProcs(t *testing.T) { ctx := context.Background() type fields struct{} diff --git a/pkg/iscp/runtime_cancel.go b/pkg/iscp/runtime_cancel.go index 449d880b82634..68c05af942502 100644 --- a/pkg/iscp/runtime_cancel.go +++ b/pkg/iscp/runtime_cancel.go @@ -71,7 +71,7 @@ func (exec *ISCPTaskExecutor) ensureRuntimeMapsLocked() { } } -func rollbackFenceTTL() time.Duration { +func RollbackFenceTTL() time.Duration { ttl := DefaultRollbackFenceTTL if seconds, _, injected := fault.TriggerFault(objectio.FJ_ISCPCancelRollbackFenceTTL); injected && seconds > 0 { ttl = time.Duration(seconds) * time.Second @@ -116,7 +116,7 @@ func (exec *ISCPTaskExecutor) CancelAndDrainJobConsumer( exec.runtimeMu.Lock() exec.ensureRuntimeMapsLocked() - exec.fencedJobs[key] = JobFence{ExpireAt: time.Now().Add(rollbackFenceTTL())} + exec.fencedJobs[key] = JobFence{ExpireAt: time.Now().Add(RollbackFenceTTL())} handles := make([]*RunningJobConsumer, 0, 1) if byJobID := exec.runningConsumers[groupKey]; byJobID != nil { for runningJobID, h := range byJobID { @@ -153,6 +153,18 @@ func (exec *ISCPTaskExecutor) RemoveJobFence(key JobRuntimeKey) { exec.runtimeMu.Unlock() } +func (exec *ISCPTaskExecutor) RenewJobFence(key JobRuntimeKey, ttl time.Duration) { + if exec == nil || ttl <= 0 { + return + } + exec.runtimeMu.Lock() + exec.ensureRuntimeMapsLocked() + if _, ok := exec.fencedJobs[key]; ok { + exec.fencedJobs[key] = JobFence{ExpireAt: time.Now().Add(ttl)} + } + exec.runtimeMu.Unlock() +} + func (exec *ISCPTaskExecutor) RemoveTableJobFences(accountID uint32, tableID uint64) { if exec == nil { return diff --git a/pkg/pb/query/query.pb.go b/pkg/pb/query/query.pb.go index d5915dee4f1b5..47f7a10ed336c 100644 --- a/pkg/pb/query/query.pb.go +++ b/pkg/pb/query/query.pb.go @@ -2093,6 +2093,7 @@ type ISCPDrainConsumerRequest struct { JobName string `protobuf:"bytes,3,opt,name=JobName,proto3" json:"JobName,omitempty"` JobID uint64 `protobuf:"varint,4,opt,name=JobID,proto3" json:"JobID,omitempty"` RemoveFenceOnly bool `protobuf:"varint,5,opt,name=RemoveFenceOnly,proto3" json:"RemoveFenceOnly,omitempty"` + RenewFenceOnly bool `protobuf:"varint,6,opt,name=RenewFenceOnly,proto3" json:"RenewFenceOnly,omitempty"` } func (m *ISCPDrainConsumerRequest) Reset() { *m = ISCPDrainConsumerRequest{} } @@ -2163,6 +2164,13 @@ func (m *ISCPDrainConsumerRequest) GetRemoveFenceOnly() bool { return false } +func (m *ISCPDrainConsumerRequest) GetRenewFenceOnly() bool { + if m != nil { + return m.RenewFenceOnly + } + return false +} + type ISCPDrainConsumerResponse struct { Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` } @@ -7166,6 +7174,16 @@ func (m *ISCPDrainConsumerRequest) MarshalToSizedBuffer(dAtA []byte) (int, error _ = i var l int _ = l + if m.RenewFenceOnly { + i-- + if m.RenewFenceOnly { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x30 + } if m.RemoveFenceOnly { i-- if m.RemoveFenceOnly { @@ -9879,6 +9897,9 @@ func (m *ISCPDrainConsumerRequest) ProtoSize() (n int) { if m.RemoveFenceOnly { n += 2 } + if m.RenewFenceOnly { + n += 2 + } return n } @@ -15672,6 +15693,26 @@ func (m *ISCPDrainConsumerRequest) Unmarshal(dAtA []byte) error { } } m.RemoveFenceOnly = bool(v != 0) + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RenewFenceOnly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RenewFenceOnly = bool(v != 0) default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) diff --git a/pkg/sql/compile/iscp_util.go b/pkg/sql/compile/iscp_util.go index 9f94a4d3b8157..2a71227cceaf8 100644 --- a/pkg/sql/compile/iscp_util.go +++ b/pkg/sql/compile/iscp_util.go @@ -17,6 +17,8 @@ package compile import ( "context" "fmt" + "sync" + "time" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/clusterservice" @@ -256,7 +258,12 @@ func drainIndexCdcTaskConsumer( return err } if txnOp := c.proc.GetTxnOperator(); txnOp != nil { + lease := startISCPJobFenceLease(func() error { + exec.RenewJobFence(key, iscp.RollbackFenceTTL()) + return nil + }) cleanup := client.NewTxnEventCallback(func(_ context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { + lease.Stop() if !event.CostEvent { return nil } @@ -264,6 +271,12 @@ func drainIndexCdcTaskConsumer( return nil }) txnOp.AppendEventCallback(client.RollbackEvent, cleanup) + txnOp.AppendEventCallback(client.CommitEvent, client.NewTxnEventCallback(func(_ context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { + if !event.CostEvent { + lease.Stop() + } + return nil + })) } return nil } @@ -282,11 +295,21 @@ func drainIndexCdcTaskConsumer( if err != nil { return err } - if err := sendISCPDrainConsumerRequest(c.proc.Ctx, qc, queryAddress, accountID, tableID, jobName, jobID, false); err != nil { + if err := sendISCPDrainConsumerRequest(c.proc.Ctx, qc, queryAddress, accountID, tableID, jobName, jobID, false, false); err != nil { return err } if txnOp := c.proc.GetTxnOperator(); txnOp != nil { + lease := startISCPJobFenceLease(func() error { + renewCtx, cancel := context.WithTimeoutCause( + context.Background(), + iscp.RollbackFenceTTL(), + moerr.NewInternalErrorNoCtx("iscp fence lease renew timeout"), + ) + defer cancel() + return sendISCPDrainConsumerRequest(renewCtx, qc, queryAddress, accountID, tableID, jobName, jobID, false, true) + }) cleanup := client.NewTxnEventCallback(func(ctx context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { + lease.Stop() if !event.CostEvent { return nil } @@ -296,13 +319,62 @@ func drainIndexCdcTaskConsumer( moerr.NewInternalErrorNoCtx("iscp rollback fence cleanup timeout"), ) defer cancel() - return sendISCPDrainConsumerRequest(cleanupCtx, qc, queryAddress, accountID, tableID, jobName, jobID, true) + return sendISCPDrainConsumerRequest(cleanupCtx, qc, queryAddress, accountID, tableID, jobName, jobID, true, false) }) txnOp.AppendEventCallback(client.RollbackEvent, cleanup) + txnOp.AppendEventCallback(client.CommitEvent, client.NewTxnEventCallback(func(_ context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { + if !event.CostEvent { + lease.Stop() + } + return nil + })) } return nil } +type iscpJobFenceLease struct { + stop func() +} + +func startISCPJobFenceLease(renew func() error) iscpJobFenceLease { + ttl := iscp.RollbackFenceTTL() + if ttl <= 0 || renew == nil { + return iscpJobFenceLease{stop: func() {}} + } + interval := ttl / 2 + if interval < 10*time.Millisecond { + interval = 10 * time.Millisecond + } + stopCh := make(chan struct{}) + var once sync.Once + go func() { + timer := time.NewTimer(interval) + defer timer.Stop() + for { + select { + case <-timer.C: + if err := renew(); err != nil { + logutil.Warnf("failed to renew ISCP job fence lease: %v", err) + } + timer.Reset(interval) + case <-stopCh: + return + } + } + }() + return iscpJobFenceLease{stop: func() { + once.Do(func() { + close(stopCh) + }) + }} +} + +func (l iscpJobFenceLease) Stop() { + if l.stop != nil { + l.stop() + } +} + func getCNQueryAddress(ctx context.Context, service string, cnUUID string) (string, error) { cluster, err := clusterservice.GetMOClusterWithContext(ctx, service) if err != nil { @@ -336,6 +408,7 @@ func sendISCPDrainConsumerRequest( jobName string, jobID uint64, removeFenceOnly bool, + renewFenceOnly bool, ) error { req := qc.NewRequest(query.CmdMethod_ISCPDrainConsumer) req.ISCPDrainConsumerRequest = &query.ISCPDrainConsumerRequest{ @@ -344,6 +417,7 @@ func sendISCPDrainConsumerRequest( JobName: jobName, JobID: jobID, RemoveFenceOnly: removeFenceOnly, + RenewFenceOnly: renewFenceOnly, } resp, err := qc.SendMessage(ctx, queryAddress, req) if err != nil { diff --git a/pkg/sql/compile/iscp_util_extra_test.go b/pkg/sql/compile/iscp_util_extra_test.go index c02f33721479d..f8387d5616155 100644 --- a/pkg/sql/compile/iscp_util_extra_test.go +++ b/pkg/sql/compile/iscp_util_extra_test.go @@ -17,15 +17,18 @@ package compile import ( "context" "testing" + "time" "github.com/golang/mock/gomock" "github.com/matrixorigin/matrixone/pkg/common/moerr" mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" "github.com/matrixorigin/matrixone/pkg/iscp" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/pb/query" "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/txn/client" + "github.com/matrixorigin/matrixone/pkg/util/fault" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -529,11 +532,17 @@ func TestDrainIndexCdcTaskConsumerRemoteFenceCleanupOnTxnEvent(t *testing.T) { ctrl := gomock.NewController(t) txnOp := mock_frontend.NewMockTxnOperator(ctrl) var rollbackCleanup client.TxnEventCallback + var commitStart client.TxnEventCallback txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).DoAndReturn( func(_ client.EventType, cb client.TxnEventCallback) { rollbackCleanup = cb }, ).Times(1) + txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + commitStart = cb + }, + ).Times(1) c := &Compile{} c.proc = testutil.NewProcess(t) @@ -554,6 +563,7 @@ func TestDrainIndexCdcTaskConsumerRemoteFenceCleanupOnTxnEvent(t *testing.T) { require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) require.NotNil(t, rollbackCleanup.Func) + require.NotNil(t, commitStart.Func) require.NoError(t, rollbackCleanup.Func(context.Background(), txnOp, client.TxnEvent{CostEvent: true}, nil)) require.Len(t, qc.requests, 2) @@ -584,11 +594,17 @@ func TestDrainIndexCdcTaskConsumerRemoteRollbackCleanupIgnoresCanceledCallerCont ctrl := gomock.NewController(t) txnOp := mock_frontend.NewMockTxnOperator(ctrl) var rollbackCleanup client.TxnEventCallback + var commitStart client.TxnEventCallback txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).DoAndReturn( func(_ client.EventType, cb client.TxnEventCallback) { rollbackCleanup = cb }, ).Times(1) + txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + commitStart = cb + }, + ).Times(1) c := &Compile{} c.proc = testutil.NewProcess(t) @@ -608,6 +624,7 @@ func TestDrainIndexCdcTaskConsumerRemoteRollbackCleanupIgnoresCanceledCallerCont } require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + require.NotNil(t, commitStart.Func) canceledCtx, cancel := context.WithCancel(context.Background()) cancel() require.NoError(t, rollbackCleanup.Func(canceledCtx, txnOp, client.TxnEvent{CostEvent: true}, nil)) @@ -677,11 +694,17 @@ func TestDrainIndexCdcTaskConsumerRegistersRollbackFenceCleanup(t *testing.T) { ctrl := gomock.NewController(t) txnOp := mock_frontend.NewMockTxnOperator(ctrl) var rollbackCleanup client.TxnEventCallback + var commitStart client.TxnEventCallback txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).DoAndReturn( func(_ client.EventType, cb client.TxnEventCallback) { rollbackCleanup = cb }, ).Times(1) + txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + commitStart = cb + }, + ).Times(1) c := &Compile{} c.proc = testutil.NewProcess(t) @@ -702,6 +725,7 @@ func TestDrainIndexCdcTaskConsumerRegistersRollbackFenceCleanup(t *testing.T) { key := iscp.NewJobRuntimeKey(0, 42, "index_idx1", 7) require.True(t, exec.IsJobFenced(key)) require.NotNil(t, rollbackCleanup.Func) + require.NotNil(t, commitStart.Func) require.NoError(t, rollbackCleanup.Func(context.Background(), txnOp, client.TxnEvent{}, nil)) require.True(t, exec.IsJobFenced(key)) @@ -711,6 +735,70 @@ func TestDrainIndexCdcTaskConsumerRegistersRollbackFenceCleanup(t *testing.T) { require.True(t, ok) } +func TestDrainIndexCdcTaskConsumerRenewsFenceUntilCommitStarts(t *testing.T) { + require.True(t, fault.Enable()) + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL, ":::", "echo", 1, "", false)) + defer func() { + _, _ = fault.RemoveFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL) + }() + + exec := &iscp.ISCPTaskExecutor{} + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return exec, true + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + }() + + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + var commitStart client.TxnEventCallback + txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).Times(1) + txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + commitStart = cb + }, + ).Times(1) + + c := &Compile{} + c.proc = testutil.NewProcess(t) + c.proc.Base.TxnOperator = txnOp + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + key := iscp.NewJobRuntimeKey(0, 42, "index_idx1", 7) + defer exec.RemoveJobFence(key) + require.NotNil(t, commitStart.Func) + require.True(t, exec.IsJobFenced(key)) + + time.Sleep(1500 * time.Millisecond) + require.True(t, exec.IsJobFenced(key)) + + require.NoError(t, commitStart.Func(context.Background(), txnOp, client.TxnEvent{}, nil)) + require.Eventually(t, func() bool { + return !exec.IsJobFenced(key) + }, 1500*time.Millisecond, 50*time.Millisecond) +} + func TestCoverage_DropAllIndexCdcTasks_DuplicateNames(t *testing.T) { dropCount := 0 iscpUnregisterJobFunc = func(ctx context.Context, cnUUID string, txn client.TxnOperator, job *iscp.JobID) (bool, error) { From 70aba40f90436cca40d5b6685a7d399dd7b392ac Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 16 Jul 2026 17:50:36 +0800 Subject: [PATCH 13/14] fix: extend iscp fence lease timeout --- pkg/iscp/runtime_cancel.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/iscp/runtime_cancel.go b/pkg/iscp/runtime_cancel.go index 68c05af942502..f4cb59f9b4c27 100644 --- a/pkg/iscp/runtime_cancel.go +++ b/pkg/iscp/runtime_cancel.go @@ -26,7 +26,7 @@ import ( var iscpExecutors sync.Map // map[cnUUID]*ISCPTaskExecutor -const DefaultRollbackFenceTTL = 5 * time.Minute +const DefaultRollbackFenceTTL = 30 * time.Minute func RegisterExecutorRuntime(cnUUID string, exec *ISCPTaskExecutor) { if cnUUID == "" || exec == nil { From 0c3b673406606722c14aa908415efb022b38ff28 Mon Sep 17 00:00:00 2001 From: jiangxinmeng Date: Thu, 16 Jul 2026 18:03:12 +0800 Subject: [PATCH 14/14] fix: add iscp renew fence field to proto --- proto/query.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/query.proto b/proto/query.proto index 3142a3406a1fd..e5cda849fa80f 100644 --- a/proto/query.proto +++ b/proto/query.proto @@ -361,6 +361,7 @@ message ISCPDrainConsumerRequest { string JobName = 3; uint64 JobID = 4; bool RemoveFenceOnly = 5; + bool RenewFenceOnly = 6; } message ISCPDrainConsumerResponse {