diff --git a/pkg/cnservice/server_query.go b/pkg/cnservice/server_query.go index d4aeb3f568e7e..c74ab1c5fec6d 100644 --- a/pkg/cnservice/server_query.go +++ b/pkg/cnservice/server_query.go @@ -26,6 +26,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/iscp" "github.com/matrixorigin/matrixone/pkg/lockservice" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio" @@ -102,6 +103,7 @@ func (s *service) initQueryCommandHandler() { s.queryService.AddHandleFunc(query.CmdMethod_WorkspaceThreshold, s.handleWorkspaceThresholdRequest, false) s.queryService.AddHandleFunc(query.CmdMethod_MinTimestamp, s.handleGetMinTimestamp, false) s.queryService.AddHandleFunc(query.CmdMethod_CtlPrefetchOnSubscribed, s.handleCtlPrefetchOnSubscribed, false) + s.queryService.AddHandleFunc(query.CmdMethod_ISCPDrainConsumer, s.handleISCPDrainConsumer, false) } func (s *service) handleKillConn(ctx context.Context, req *query.Request, resp *query.Response, _ *morpc.Buffer) error { @@ -192,6 +194,56 @@ func (s *service) handleCtlPrefetchOnSubscribed(ctx context.Context, req *query. return nil } +func (s *service) handleISCPDrainConsumer(ctx context.Context, req *query.Request, resp *query.Response, _ *morpc.Buffer) error { + if req == nil || req.ISCPDrainConsumerRequest == nil { + return moerr.NewInternalError(ctx, "bad request") + } + r := req.ISCPDrainConsumerRequest + exec, ok := iscp.GetExecutorRuntime(s.cfg.UUID) + if !ok || exec == nil { + return moerr.NewInternalErrorf( + ctx, + "cannot confirm ISCP consumer quiescence on CN %s for tableID=%d jobName=%s jobID=%d", + s.cfg.UUID, + r.TableID, + r.JobName, + r.JobID, + ) + } + key := iscp.NewJobRuntimeKey(r.AccountID, r.TableID, r.JobName, r.JobID) + if r.RemoveFenceOnly { + if _, msg, injected := fault.TriggerFault(objectio.FJ_ISCPCancelRemoveFenceError); injected { + if msg == "" { + msg = objectio.FJ_ISCPCancelRemoveFenceError + } + return moerr.NewInternalErrorNoCtxf("injected ISCP remove fence error: %s", msg) + } + exec.RemoveJobFence(key) + resp.ISCPDrainConsumerResponse = &query.ISCPDrainConsumerResponse{Success: true} + return nil + } + if r.RenewFenceOnly { + if !exec.RenewJobFence(key, iscp.RollbackFenceTTL()) { + return moerr.NewInternalErrorf( + ctx, + "cannot renew ISCP consumer quiescence fence on CN %s for tableID=%d jobName=%s jobID=%d", + s.cfg.UUID, + r.TableID, + r.JobName, + r.JobID, + ) + } + resp.ISCPDrainConsumerResponse = &query.ISCPDrainConsumerResponse{Success: true} + return nil + } + if err := exec.CancelAndDrainJobConsumer(ctx, r.AccountID, r.TableID, r.JobName, r.JobID); err != nil { + exec.RemoveJobFence(key) + return err + } + resp.ISCPDrainConsumerResponse = &query.ISCPDrainConsumerResponse{Success: true} + return nil +} + // handleGetLockInfo sends the lock info on current cn to another cn that needs. func (s *service) handleGetLockInfo(ctx context.Context, req *query.Request, resp *query.Response, _ *morpc.Buffer) error { resp.GetLockInfoResponse = new(query.GetLockInfoResponse) diff --git a/pkg/cnservice/server_query_test.go b/pkg/cnservice/server_query_test.go index 2641ac3387771..26763893f5621 100644 --- a/pkg/cnservice/server_query_test.go +++ b/pkg/cnservice/server_query_test.go @@ -21,6 +21,7 @@ import ( goruntime "runtime" "runtime/debug" "testing" + "time" "unsafe" "github.com/golang/mock/gomock" @@ -40,7 +41,9 @@ import ( "github.com/matrixorigin/matrixone/pkg/frontend/test/mock_shard" "github.com/matrixorigin/matrixone/pkg/frontend/test/mock_task" "github.com/matrixorigin/matrixone/pkg/incrservice" + "github.com/matrixorigin/matrixone/pkg/iscp" "github.com/matrixorigin/matrixone/pkg/lockservice" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/lock" "github.com/matrixorigin/matrixone/pkg/pb/query" "github.com/matrixorigin/matrixone/pkg/pb/statsinfo" @@ -50,6 +53,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/taskservice" "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/txn/client" + "github.com/matrixorigin/matrixone/pkg/util/fault" "github.com/matrixorigin/matrixone/pkg/util/trace" "github.com/matrixorigin/matrixone/pkg/vm/engine" ) @@ -57,6 +61,64 @@ import ( var dummyBadRequestErr = moerr.NewInternalError(context.TODO(), "bad request") var dummyErr = moerr.NewInternalError(context.TODO(), "dummy error") +func Test_service_handleISCPDrainConsumerRenewFenceOnly(t *testing.T) { + require.True(t, fault.Enable()) + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL, ":::", "echo", 1, "", false)) + defer func() { + _, _ = fault.RemoveFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL) + }() + + exec := &iscp.ISCPTaskExecutor{} + iscp.RegisterExecutorRuntime("runner-cn", exec) + defer iscp.UnregisterExecutorRuntime("runner-cn", exec) + + s := &service{cfg: &Config{UUID: "runner-cn"}} + key := iscp.NewJobRuntimeKey(1, 42, "index_idx1", 7) + + resp := &query.Response{} + require.ErrorContains(t, s.handleISCPDrainConsumer(context.Background(), &query.Request{ + ISCPDrainConsumerRequest: &query.ISCPDrainConsumerRequest{ + AccountID: key.AccountID, + TableID: key.TableID, + JobName: key.JobName, + JobID: key.JobID, + RenewFenceOnly: true, + }, + }, resp, nil), "cannot renew ISCP consumer quiescence fence") + require.Nil(t, resp.ISCPDrainConsumerResponse) + require.False(t, exec.IsJobFenced(key)) + + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + time.Sleep(700 * time.Millisecond) + + resp = &query.Response{} + require.NoError(t, s.handleISCPDrainConsumer(context.Background(), &query.Request{ + ISCPDrainConsumerRequest: &query.ISCPDrainConsumerRequest{ + AccountID: key.AccountID, + TableID: key.TableID, + JobName: key.JobName, + JobID: key.JobID, + RenewFenceOnly: true, + }, + }, resp, nil)) + time.Sleep(700 * time.Millisecond) + require.True(t, exec.IsJobFenced(key)) + + time.Sleep(1100 * time.Millisecond) + require.False(t, exec.IsJobFenced(key)) + resp = &query.Response{} + require.ErrorContains(t, s.handleISCPDrainConsumer(context.Background(), &query.Request{ + ISCPDrainConsumerRequest: &query.ISCPDrainConsumerRequest{ + AccountID: key.AccountID, + TableID: key.TableID, + JobName: key.JobName, + JobID: key.JobID, + RenewFenceOnly: true, + }, + }, resp, nil), "cannot renew ISCP consumer quiescence fence") +} + func Test_service_handleGoMaxProcs(t *testing.T) { ctx := context.Background() type fields struct{} diff --git a/pkg/iscp/data_retriever.go b/pkg/iscp/data_retriever.go index 3c91e9faa0f11..666e07dd296e2 100644 --- a/pkg/iscp/data_retriever.go +++ b/pkg/iscp/data_retriever.go @@ -81,6 +81,20 @@ func (d *ISCPData) Done() { } } +func (d *ISCPData) Close() { + if d == nil { + return + } + if d.insertBatch != nil { + d.insertBatch.Close() + d.insertBatch = nil + } + if d.deleteBatch != nil { + d.deleteBatch.Close() + d.deleteBatch = nil + } +} + const ( ISCPDataType_Snapshot int8 = iota ISCPDataType_Tail @@ -145,6 +159,9 @@ func (r *DataRetrieverImpl) UpdateWatermark(ctx context.Context, cnUUID string, txn client.TxnOperator) error { + if r.IsCanceled() { + return r.terminalError() + } ctxWithSysAccount := context.WithValue(ctx, defines.TenantIDKey{}, catalog.System_Account) ctxWithSysAccount, cancel := context.WithTimeout(ctxWithSysAccount, time.Minute*5) defer cancel() @@ -187,17 +204,24 @@ func (r *DataRetrieverImpl) GetTableID() uint64 { return r.tableID } -func (r *DataRetrieverImpl) SetNextBatch(data *ISCPData) { +func (r *DataRetrieverImpl) SetNextBatch(data *ISCPData) bool { if r.hasError() { - data.Done() - return + return false } select { case r.insertDataCh <- data: - return + if r.IsCanceled() { + select { + case queued := <-r.insertDataCh: + if queued == data { + return false + } + default: + } + } + return true case <-r.ctx.Done(): - data.Done() - return + return false } } @@ -218,6 +242,10 @@ func (r *DataRetrieverImpl) terminalError() error { // after error occurs, the data retriever won't consume any more data func (r *DataRetrieverImpl) SetError(err error) { + r.Cancel(err) +} + +func (r *DataRetrieverImpl) Cancel(err error) { if r.hasError() { return } @@ -228,6 +256,26 @@ func (r *DataRetrieverImpl) SetError(err error) { } r.cancel() r.err = err + for { + select { + case data := <-r.insertDataCh: + data.Done() + default: + return + } + } +} + +func (r *DataRetrieverImpl) IsCanceled() bool { + if r.hasError() { + return true + } + select { + case <-r.ctx.Done(): + return true + default: + return false + } } func (r *DataRetrieverImpl) Close() { diff --git a/pkg/iscp/executor.go b/pkg/iscp/executor.go index 895c55f80b27a..8847e71868d9f 100644 --- a/pkg/iscp/executor.go +++ b/pkg/iscp/executor.go @@ -187,6 +187,10 @@ func NewISCPTaskExecutor( tableMu: sync.RWMutex{}, option: option, mp: mp, + fencedJobs: make(map[JobRuntimeKey]JobFence), + runningConsumers: make( + map[JobRuntimeKey]map[uint64]*RunningJobConsumer, + ), } return exec, nil } @@ -341,9 +345,10 @@ func (exec *ISCPTaskExecutor) initStateLocked(parent context.Context) (err error // Publish a fully initialized generation only after replay and state repair // have both succeeded. NewWorker seals all worker goroutines before return. - exec.worker = NewWorker(exec.cnUUID, exec.txnEngine, exec.cnTxnClient, exec.mp) + exec.worker = NewWorker(exec, exec.cnUUID, exec.txnEngine, exec.cnTxnClient, exec.mp) exec.wg.Add(1) exec.running = true + RegisterExecutorRuntime(exec.cnUUID, exec) logutil.Info( "ISCP-Task Start", ) @@ -367,6 +372,7 @@ func (exec *ISCPTaskExecutor) stopLocked() { exec.cancel() exec.worker.Stop() exec.wg.Wait() + UnregisterExecutorRuntime(exec.cnUUID, exec) exec.ctx, exec.cancel = nil, nil exec.worker = nil } @@ -530,6 +536,12 @@ func (exec *ISCPTaskExecutor) run(ctx context.Context, worker Worker) { go exec.Cancel() return } + if objectio.WaitInjected(objectio.FJ_ISCPCancelAfterSubmit) { + logutil.Infof("ISCP-Task cancel fault wait %s", objectio.FJ_ISCPCancelAfterSubmit) + } + if msg, injected := objectio.ISCPExecutorInjected(); injected && msg == "iscp:after-submit" { + logutil.Infof("ISCP-Task injected hook %s", msg) + } } else { table.UpdateWatermark(iter) } @@ -960,6 +972,7 @@ func (exec *ISCPTaskExecutor) addOrUpdateJob( notPrint bool, ) error { var newCreate bool + fenceKey := NewJobRuntimeKey(accountID, tableID, jobName, jobID) var watermark types.TS defer func() { @@ -990,6 +1003,7 @@ func (exec *ISCPTaskExecutor) addOrUpdateJob( table, ok := exec.getTable(accountID, tableID) if !ok { if dropAt != 0 { + exec.RemoveJobFence(fenceKey) return nil } table = NewTableEntry( @@ -1003,6 +1017,9 @@ func (exec *ISCPTaskExecutor) addOrUpdateJob( exec.setTable(table) } newCreate = table.AddOrUpdateSinker(exec.ctx, jobName, jobSpec, jobStatus, jobID, watermark, state, dropAt) + if dropAt != 0 { + exec.RemoveJobFence(fenceKey) + } return nil } @@ -1018,6 +1035,7 @@ func (exec *ISCPTaskExecutor) GCInMemoryJob(threshold time.Duration) { tids := make([]uint64, 0, len(tablesToDelete)) for _, table := range tablesToDelete { exec.deleteTableEntry(table) + exec.RemoveTableJobFences(table.accountID, table.tableID) tids = append(tids, table.tableID) } logutil.Infof("ISCP-Task delete table %v", tids) diff --git a/pkg/iscp/index_consumer.go b/pkg/iscp/index_consumer.go index 62c9c61c1210a..2572eae5e194c 100644 --- a/pkg/iscp/index_consumer.go +++ b/pkg/iscp/index_consumer.go @@ -155,6 +155,7 @@ func RunIndex(c *IndexConsumer, ctx context.Context, errch chan error, r DataRet func runIndex(c *IndexConsumer, ctx context.Context, errch chan error, r DataRetriever) { datatype := r.GetDataType() + indexName := c.indexName() if datatype == ISCPDataType_Snapshot { // SNAPSHOT @@ -180,6 +181,9 @@ func runIndex(c *IndexConsumer, ctx context.Context, errch chan error, r DataRet func(sqlproc *sqlexec.SqlProcess, cbdata any) (err error) { sqlctx := sqlproc.SqlCtx + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeExec, indexName); err != nil { + return err + } if objectio.ISCPIndexExecErrorInjected() { return injectedISCPIndexErr("exec") } @@ -230,6 +234,9 @@ func runIndex(c *IndexConsumer, ctx context.Context, errch chan error, r DataRet // update SQL var res executor.Result + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeExec, indexName); err != nil { + return err + } if objectio.ISCPIndexExecErrorInjected() { return injectedISCPIndexErr("exec") } @@ -264,6 +271,7 @@ func RunHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch c func runHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch chan error, r DataRetriever) { datatype := r.GetDataType() + indexName := c.indexName() // Suppose we shoult not use transaction here for Snapshot type and commit every time a new batch comes. // However, HNSW only run in local without save to database until Sync.Save(). @@ -328,6 +336,15 @@ func runHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch c if objectio.ISCPIndexHnswSaveErrInjected() { return injectedISCPIndexErr("hnsw save") } + if objectio.WaitInjected(objectio.FJ_ISCPCancelHnswBeforeSave) { + logutil.Infof("ISCP-Task cancel fault wait %s index=%s", objectio.FJ_ISCPCancelHnswBeforeSave, indexName) + } + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongHnswBeforeSave, indexName); err != nil { + return err + } + if msg, injected := objectio.ISCPExecutorInjected(); injected && msg == "iscp:hnsw-before-save:"+indexName { + logutil.Infof("ISCP-Task injected hook %s", msg) + } err = sync.Save(sqlproc) if err != nil { return @@ -338,6 +355,15 @@ func runHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch c if objectio.ISCPIndexWatermarkErrInjected() { return injectedISCPIndexErr("watermark") } + if objectio.WaitInjected(objectio.FJ_ISCPCancelBeforeUpdateWatermark) { + logutil.Infof("ISCP-Task cancel fault wait %s index=%s", objectio.FJ_ISCPCancelBeforeUpdateWatermark, indexName) + } + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeWatermark, indexName); err != nil { + return err + } + if msg, injected := objectio.ISCPExecutorInjected(); injected && msg == "iscp:before-update-watermark:"+indexName { + logutil.Infof("ISCP-Task injected hook %s", msg) + } err = r.UpdateWatermark(sqlproc.GetContext(), sqlctx.GetService(), sqlctx.Txn()) if err != nil { return @@ -365,6 +391,9 @@ func runHnsw[T types.RealNumbers](c *IndexConsumer, ctx context.Context, errch c // HNSW models are already in local so hnsw Update should not require executing SQL or should be read-only. No transaction required. err = runTxnWithSqlContext(ctx, c.cnEngine, c.cnTxnClient, c.cnUUID, r.GetAccountID(), 30*time.Minute, nil, nil, func(sqlproc *sqlexec.SqlProcess, cbdata any) (err error) { + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongHnswBeforeUpdate, indexName); err != nil { + return err + } if objectio.ISCPIndexHnswUpdateErrInjected() { return injectedISCPIndexErr("hnsw update") } @@ -404,6 +433,27 @@ func reportIndexConsumerErr(errch chan error, err error) { } } +func waitISCPIndexCancelCtx(ctx context.Context, key, indexName string) error { + injected := false + if indexName != "" && objectio.WaitInjectedCtx(ctx, key+":"+indexName) { + injected = true + } else if objectio.WaitInjectedCtx(ctx, key) { + injected = true + } + if !injected { + return nil + } + logutil.Infof("ISCP-Task cancel ctx fault wait %s index=%s", key, indexName) + return ctx.Err() +} + +func (c *IndexConsumer) indexName() string { + if c == nil || c.info == nil { + return "" + } + return c.info.IndexName +} + func injectedISCPIndexErr(point string) error { return moerr.NewInternalErrorNoCtx("injected ISCP index " + point + " failure") } @@ -659,6 +709,13 @@ func (c *IndexConsumer) sendSql(ctx context.Context, errch chan error, writer In if writer.Empty() { return nil } + if err := ctx.Err(); err != nil { + return err + } + indexName := c.indexName() + if err := waitISCPIndexCancelCtx(ctx, objectio.FJ_ISCPCancelLongBeforeSend, indexName); err != nil { + return err + } // generate sql from cdc sql, err := writer.ToSql() diff --git a/pkg/iscp/index_consumer_test.go b/pkg/iscp/index_consumer_test.go index f2f80fb37a7e7..d3895f75fbe29 100644 --- a/pkg/iscp/index_consumer_test.go +++ b/pkg/iscp/index_consumer_test.go @@ -29,11 +29,13 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/testutil/testengine" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/util/executor" + "github.com/matrixorigin/matrixone/pkg/util/fault" "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/prashantv/gostub" @@ -419,6 +421,134 @@ func TestIndexConsumerSendSqlReturnsContextErrorWhenBlocked(t *testing.T) { require.ErrorIs(t, err, context.Canceled) } +func TestIndexConsumerSendSqlCtxFaultReturnsContextError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fault.Enable() + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(ctx, objectio.FJ_ISCPCancelLongBeforeSend, ":::", "wait", 0, "", false)) + require.NoError(t, fault.AddFaultPoint(ctx, "get-send-waiters", ":::", "getwaiters", 0, objectio.FJ_ISCPCancelLongBeforeSend, false)) + + consumer := &IndexConsumer{ + info: newTestIvfConsumerInfo(), + sqlBufSendCh: make(chan []byte), + } + done := make(chan error, 1) + go func() { + done <- consumer.sendSql(ctx, make(chan error, 1), &flushEveryRowWriter{rows: 1}) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := fault.TriggerFault("get-send-waiters") + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("sendSql stayed blocked in ctx-aware fault wait after context cancellation") + } +} + +func TestIndexConsumerSendSqlCtxFaultMatchesIndexName(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + info := newTestIvfConsumerInfo() + key := objectio.FJ_ISCPCancelLongBeforeSend + ":" + info.IndexName + waitersKey := "get-send-index-waiters" + + fault.Enable() + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(ctx, key, ":::", "wait", 0, "", false)) + require.NoError(t, fault.AddFaultPoint(ctx, waitersKey, ":::", "getwaiters", 0, key, false)) + + consumer := &IndexConsumer{ + info: info, + sqlBufSendCh: make(chan []byte), + } + done := make(chan error, 1) + go func() { + done <- consumer.sendSql(ctx, make(chan error, 1), &flushEveryRowWriter{rows: 1}) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := fault.TriggerFault(waitersKey) + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("sendSql stayed blocked in index-specific ctx-aware fault wait after context cancellation") + } +} + +func TestRunIndexBeforeExecCtxFaultReturnsContextError(t *testing.T) { + for _, dtype := range []int8{ISCPDataType_Snapshot, ISCPDataType_Tail} { + t.Run(fmt.Sprintf("type-%d", dtype), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + info := newTestIvfConsumerInfo() + key := objectio.FJ_ISCPCancelLongBeforeExec + ":" + info.IndexName + waitersKey := fmt.Sprintf("get-before-exec-waiters-%d", dtype) + + fault.Enable() + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(ctx, key, ":::", "wait", 0, "", false)) + require.NoError(t, fault.AddFaultPoint(ctx, waitersKey, ":::", "getwaiters", 0, key, false)) + + stubRunTxn := gostub.Stub(&runTxnWithSqlContext, func( + ctx context.Context, + _ engine.Engine, + _ client.TxnClient, + cnUUID string, + accountID uint32, + _ time.Duration, + resolveVariableFunc func(varName string, isSystemVar, isGlobalVar bool) (interface{}, error), + cbdata any, + f func(sqlproc *sqlexec.SqlProcess, data any) error, + ) error { + sqlproc := sqlexec.NewSqlProcessWithContext(sqlexec.NewSqlContext(ctx, cnUUID, nil, accountID, resolveVariableFunc)) + return f(sqlproc, cbdata) + }) + defer stubRunTxn.Reset() + + consumer := &IndexConsumer{ + info: info, + sqlBufSendCh: make(chan []byte), + } + retriever := &MockRetriever{dtype: dtype} + errch := make(chan error, 2) + done := make(chan struct{}) + go func() { + runIndex(consumer, ctx, errch, retriever) + close(done) + }() + + consumer.sqlBufSendCh <- []byte("select 1") + require.Eventually(t, func() bool { + cnt, _, ok := fault.TriggerFault(waitersKey) + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("runIndex stayed blocked in before-exec ctx-aware fault wait after context cancellation") + } + require.ErrorIs(t, <-errch, context.Canceled) + }) + } +} + func TestIndexConsumerWorkerErrorCancelsBlockedNext(t *testing.T) { proc := testutil.NewProcess(t) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) @@ -626,7 +756,7 @@ func TestIndexConsumerTailFinalizationUsesParentContext(t *testing.T) { require.ErrorIs(t, err, context.Canceled) } -func TestDataRetrieverSetNextBatchReleasesRejectedData(t *testing.T) { +func TestDataRetrieverSetNextBatchRejectsCanceledRetriever(t *testing.T) { ctx := context.Background() status := &JobStatus{} retriever := NewDataRetriever(ctx, 0, 0, "job", 0, status, 0, ISCPDataType_Snapshot) @@ -639,10 +769,131 @@ func TestDataRetrieverSetNextBatchReleasesRejectedData(t *testing.T) { } data.Set(1) - retriever.SetNextBatch(data) + accepted := retriever.SetNextBatch(data) + require.False(t, accepted) + data.Done() require.Nil(t, data.insertBatch) } +func TestDataRetrieverCanceledWatermarkAndIsCanceledErrorPath(t *testing.T) { + ctx := context.Background() + retriever := NewDataRetriever(ctx, 0, 0, "job", 0, &JobStatus{}, 0, ISCPDataType_Tail) + cancelErr := errors.New("consumer canceled") + retriever.Cancel(cancelErr) + + require.True(t, retriever.IsCanceled()) + require.ErrorIs(t, retriever.UpdateWatermark(ctx, "", nil), cancelErr) +} + +func TestDataRetrieverStatusJSONAndAccessors(t *testing.T) { + status := &JobStatus{ + LSN: 7, + Stage: JobStage_Running, + ErrorCode: 1, + ErrorMsg: "failed", + } + statusJSON, err := MarshalJobStatus(status) + require.NoError(t, err) + statusByteJSON, err := types.ParseStringToByteJson(statusJSON) + require.NoError(t, err) + encodedStatusJSON, err := types.EncodeJson(statusByteJSON) + require.NoError(t, err) + + decoded, err := UnmarshalJobStatus(encodedStatusJSON) + require.NoError(t, err) + require.Equal(t, status.LSN, decoded.LSN) + require.Equal(t, status.Stage, decoded.Stage) + require.Equal(t, status.ErrorCode, decoded.ErrorCode) + require.Equal(t, status.ErrorMsg, decoded.ErrorMsg) + + retriever := NewDataRetriever(context.Background(), 42, 99, "job", 3, status, 11, ISCPDataType_Tail) + require.Equal(t, int8(ISCPDataType_Tail), retriever.GetDataType()) + require.Equal(t, uint32(42), retriever.GetAccountID()) + require.Equal(t, uint64(99), retriever.GetTableID()) +} + +func TestDataRetrieverUpdateWatermarkSnapshotNoop(t *testing.T) { + retriever := NewDataRetriever(context.Background(), 0, 0, "job", 0, &JobStatus{}, 0, ISCPDataType_Snapshot) + require.NoError(t, retriever.UpdateWatermark(context.Background(), "", nil)) +} + +func TestDataRetrieverUpdateWatermarkTailUsesExecWithResult(t *testing.T) { + ctx := context.Background() + status := &JobStatus{To: types.BuildTS(10, 1)} + retriever := NewDataRetriever(ctx, 7, 8, "job", 9, status, 11, ISCPDataType_Tail) + + var execSQL string + stubExec := gostub.Stub(&ExecWithResult, func(execCtx context.Context, sql string, cnUUID string, txn client.TxnOperator) (executor.Result, error) { + execSQL = sql + require.Equal(t, "cn", cnUUID) + require.Equal(t, catalog.System_Account, execCtx.Value(defines.TenantIDKey{})) + return executor.Result{}, nil + }) + defer stubExec.Reset() + + require.NoError(t, retriever.UpdateWatermark(ctx, "cn", nil)) + require.Equal(t, uint64(11), status.LSN) + require.Contains(t, execSQL, "UPDATE mo_catalog.mo_iscp_log") + require.Contains(t, execSQL, "job") +} + +func TestDataRetrieverUpdateWatermarkTailExecError(t *testing.T) { + expected := errors.New("watermark update failed") + retriever := NewDataRetriever(context.Background(), 7, 8, "job", 9, &JobStatus{}, 11, ISCPDataType_Tail) + stubExec := gostub.Stub(&ExecWithResult, func(context.Context, string, string, client.TxnOperator) (executor.Result, error) { + return executor.Result{}, expected + }) + defer stubExec.Reset() + + require.ErrorIs(t, retriever.UpdateWatermark(context.Background(), "cn", nil), expected) +} + +func TestISCPDataCloseReleasesBothBatches(t *testing.T) { + data := &ISCPData{ + insertBatch: &AtomicBatch{ + Rows: btree.NewBTreeGOptions(AtomicBatchRow.Less, btree.Options{Degree: 64}), + Batches: []*batch.Batch{}, + }, + deleteBatch: &AtomicBatch{ + Rows: btree.NewBTreeGOptions(AtomicBatchRow.Less, btree.Options{Degree: 64}), + Batches: []*batch.Batch{}, + }, + } + + data.Close() + + require.Nil(t, data.insertBatch) + require.Nil(t, data.deleteBatch) + + var nilData *ISCPData + require.NotPanics(t, nilData.Close) +} + +func TestDataRetrieverSetNextBatchDoesNotQueueAfterContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + status := &JobStatus{} + retriever := NewDataRetriever(ctx, 0, 0, "job", 0, status, 0, ISCPDataType_Snapshot) + cancel() + + data := &ISCPData{ + insertBatch: &AtomicBatch{ + Rows: btree.NewBTreeGOptions(AtomicBatchRow.Less, btree.Options{Degree: 64}), + }, + } + data.Set(1) + + accepted := retriever.SetNextBatch(data) + + require.False(t, accepted) + data.Done() + require.Nil(t, data.insertBatch) + select { + case queued := <-retriever.insertDataCh: + require.Nil(t, queued) + default: + } +} + func TestIndexConsumerTerminalFlushErrorDoesNotCloseChannel(t *testing.T) { proc := testutil.NewProcess(t) ctx := context.Background() diff --git a/pkg/iscp/iteration.go b/pkg/iscp/iteration.go index 8a06dfb4b9325..a35466f4812aa 100644 --- a/pkg/iscp/iteration.go +++ b/pkg/iscp/iteration.go @@ -42,8 +42,10 @@ import ( type DataRetrieverConsumer interface { DataRetriever - SetNextBatch(*ISCPData) + SetNextBatch(*ISCPData) bool SetError(error) + Cancel(error) + IsCanceled() bool Close() } @@ -60,6 +62,22 @@ func ExecuteIteration( iterCtx *IterationContext, mp *mpool.MPool, ) (err error) { + return ExecuteIterationWithRuntime(ctx, nil, cnUUID, cnEngine, cnTxnClient, iterCtx, mp) +} + +func ExecuteIterationWithRuntime( + ctx context.Context, + runtime *ISCPTaskExecutor, + cnUUID string, + cnEngine engine.Engine, + cnTxnClient client.TxnClient, + iterCtx *IterationContext, + mp *mpool.MPool, +) (err error) { + iterCtx = runtime.filterFencedIteration(iterCtx) + if iterCtx == nil { + return nil + } packer := types.NewPacker() defer packer.Close() @@ -279,6 +297,7 @@ func ExecuteIteration( runISCPTaskIterationConsumers( ctxWithoutTimeout, + runtime, iterCtx, changes, consumers, @@ -292,6 +311,9 @@ func ExecuteIteration( delCompositedPkColIdx, ) for i, status := range statuses { + if runtime != nil && runtime.IsJobFenced(NewJobRuntimeKey(iterCtx.accountID, iterCtx.tableID, iterCtx.jobNames[i], iterCtx.jobIDs[i])) { + continue + } if status.ErrorCode != 0 || typ == ISCPDataType_Snapshot { state := ISCPJobState_Completed if status.PermanentlyFailed() { @@ -326,6 +348,7 @@ func ExecuteIteration( func runISCPTaskIterationConsumers( ctx context.Context, + runtime *ISCPTaskExecutor, iterCtx *IterationContext, changes engine.ChangesHandle, consumers []Consumer, @@ -441,9 +464,26 @@ func runISCPTaskIterationConsumers( } noMoreData := data.noMoreData - data.Set(len(consumers)) - for i := range consumers { - dataRetrievers[i].SetNextBatch(data) + active := make([]DataRetrieverConsumer, 0, len(dataRetrievers)) + for i := range dataRetrievers { + if dataRetrievers[i] != nil && !dataRetrievers[i].IsCanceled() { + active = append(active, dataRetrievers[i]) + } + } + data.Set(len(active)) + if len(active) == 0 { + data.Close() + } + for _, retriever := range active { + if objectio.WaitInjected(objectio.FJ_ISCPCancelFanoutBeforeSend) { + logutil.Infof("ISCP-Task cancel fault wait %s", objectio.FJ_ISCPCancelFanoutBeforeSend) + } + if msg, injected := objectio.ISCPExecutorInjected(); injected && strings.HasPrefix(msg, "iscp:fanout-before-send:") { + logutil.Infof("ISCP-Task injected hook %s", msg) + } + if !retriever.SetNextBatch(data) { + data.Done() + } } if noMoreData { @@ -460,9 +500,37 @@ func runISCPTaskIterationConsumers( waitGroups[i].Add(1) go func(i int) { defer waitGroups[i].Done() - consumerCtx := context.WithValue(ctxWithCancel, defines.TenantIDKey{}, catalog.System_Account) + consumerCtx, consumerCancel := context.WithCancel(ctx) + defer consumerCancel() + consumerCtx = context.WithValue(consumerCtx, defines.TenantIDKey{}, catalog.System_Account) + var handle *RunningJobConsumer + key := NewJobRuntimeKey(iterCtx.accountID, iterCtx.tableID, iterCtx.jobNames[i], iterCtx.jobIDs[i]) + if runtime != nil { + var ok bool + handle, ok = runtime.RegisterRunningConsumer( + key, + iterCtx.jobIDs[i], + iterCtx.lsn[i], + consumerCancel, + dataRetrievers[i].Cancel, + ) + if !ok { + dataRetrievers[i].Cancel(moerr.NewInternalErrorNoCtx("iscp job consumer canceled")) + return + } + if objectio.WaitInjected(objectio.FJ_ISCPCancelAfterRegisterConsumer) { + logutil.Infof("ISCP-Task cancel fault wait %s job=%s", objectio.FJ_ISCPCancelAfterRegisterConsumer, iterCtx.jobNames[i]) + } + if msg, injected := objectio.ISCPExecutorInjected(); injected && msg == "iscp:after-register-consumer:"+iterCtx.jobNames[i] { + logutil.Infof("ISCP-Task injected hook %s", msg) + } + defer runtime.UnregisterRunningConsumer(handle) + } err := consumerEntry.Consume(consumerCtx, dataRetrievers[i]) if err != nil { + if runtime != nil && runtime.IsJobFenced(key) { + return + } logutil.Error( "ISCP-Task sink consume failed", zap.Uint32("tenantID", iterCtx.accountID), diff --git a/pkg/iscp/iteration_consumer_context_test.go b/pkg/iscp/iteration_consumer_context_test.go index cdb0de8c2c5a3..cfbc605ce0f92 100644 --- a/pkg/iscp/iteration_consumer_context_test.go +++ b/pkg/iscp/iteration_consumer_context_test.go @@ -144,6 +144,7 @@ func runIterationConsumersWithStatusesForTest( defer packer.Close() runISCPTaskIterationConsumers( ctx, + nil, iterCtx, changes, consumers, diff --git a/pkg/iscp/runtime_cancel.go b/pkg/iscp/runtime_cancel.go new file mode 100644 index 0000000000000..f72931060fa0f --- /dev/null +++ b/pkg/iscp/runtime_cancel.go @@ -0,0 +1,266 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iscp + +import ( + "context" + "sync" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/util/fault" +) + +var iscpExecutors sync.Map // map[cnUUID]*ISCPTaskExecutor + +const DefaultRollbackFenceTTL = 30 * time.Minute + +func RegisterExecutorRuntime(cnUUID string, exec *ISCPTaskExecutor) { + if cnUUID == "" || exec == nil { + return + } + iscpExecutors.Store(cnUUID, exec) +} + +func UnregisterExecutorRuntime(cnUUID string, exec *ISCPTaskExecutor) { + if cnUUID == "" || exec == nil { + return + } + if current, ok := iscpExecutors.Load(cnUUID); ok && current == exec { + iscpExecutors.Delete(cnUUID) + } +} + +func GetExecutorRuntime(cnUUID string) (*ISCPTaskExecutor, bool) { + v, ok := iscpExecutors.Load(cnUUID) + if !ok { + return nil, false + } + exec, ok := v.(*ISCPTaskExecutor) + return exec, ok +} + +func NewJobRuntimeKey(accountID uint32, tableID uint64, jobName string, jobID uint64) JobRuntimeKey { + return JobRuntimeKey{AccountID: accountID, TableID: tableID, JobName: jobName, JobID: jobID} +} + +func (key JobRuntimeKey) consumerGroupKey() JobRuntimeKey { + key.JobID = 0 + return key +} + +func (exec *ISCPTaskExecutor) ensureRuntimeMapsLocked() { + if exec.fencedJobs == nil { + exec.fencedJobs = make(map[JobRuntimeKey]JobFence) + } + if exec.runningConsumers == nil { + exec.runningConsumers = make(map[JobRuntimeKey]map[uint64]*RunningJobConsumer) + } +} + +func RollbackFenceTTL() time.Duration { + ttl := DefaultRollbackFenceTTL + if seconds, _, injected := fault.TriggerFault(objectio.FJ_ISCPCancelRollbackFenceTTL); injected && seconds > 0 { + ttl = time.Duration(seconds) * time.Second + } + return ttl +} + +func (exec *ISCPTaskExecutor) IsJobFenced(key JobRuntimeKey) bool { + if exec == nil { + return false + } + exec.runtimeMu.Lock() + defer exec.runtimeMu.Unlock() + return exec.isJobFencedLocked(key, time.Now()) +} + +func (exec *ISCPTaskExecutor) isJobFencedLocked(key JobRuntimeKey, now time.Time) bool { + fence, ok := exec.fencedJobs[key] + if !ok { + return false + } + if !fence.ExpireAt.IsZero() && now.After(fence.ExpireAt) { + delete(exec.fencedJobs, key) + return false + } + return true +} + +func (exec *ISCPTaskExecutor) CancelAndDrainJobConsumer( + ctx context.Context, + accountID uint32, + tableID uint64, + jobName string, + jobID uint64, +) error { + if exec == nil { + return nil + } + key := NewJobRuntimeKey(accountID, tableID, jobName, jobID) + groupKey := key.consumerGroupKey() + cancelErr := moerr.NewInternalErrorNoCtx("iscp job consumer canceled") + + exec.runtimeMu.Lock() + exec.ensureRuntimeMapsLocked() + exec.fencedJobs[key] = JobFence{ExpireAt: time.Now().Add(RollbackFenceTTL())} + handles := make([]*RunningJobConsumer, 0, 1) + if byJobID := exec.runningConsumers[groupKey]; byJobID != nil { + for runningJobID, h := range byJobID { + if jobID != 0 && runningJobID != jobID { + continue + } + handles = append(handles, h) + if h.cancel != nil { + h.cancel() + } + if h.cancelRetriever != nil { + h.cancelRetriever(cancelErr) + } + } + } + exec.runtimeMu.Unlock() + + for _, h := range handles { + select { + case <-h.done: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil +} + +func (exec *ISCPTaskExecutor) RemoveJobFence(key JobRuntimeKey) { + if exec == nil { + return + } + exec.runtimeMu.Lock() + delete(exec.fencedJobs, key) + exec.runtimeMu.Unlock() +} + +func (exec *ISCPTaskExecutor) RenewJobFence(key JobRuntimeKey, ttl time.Duration) bool { + if exec == nil || ttl <= 0 { + return false + } + exec.runtimeMu.Lock() + defer exec.runtimeMu.Unlock() + exec.ensureRuntimeMapsLocked() + if !exec.isJobFencedLocked(key, time.Now()) { + return false + } + exec.fencedJobs[key] = JobFence{ExpireAt: time.Now().Add(ttl)} + return true +} + +func (exec *ISCPTaskExecutor) RemoveTableJobFences(accountID uint32, tableID uint64) { + if exec == nil { + return + } + exec.runtimeMu.Lock() + for key := range exec.fencedJobs { + if key.AccountID == accountID && key.TableID == tableID { + delete(exec.fencedJobs, key) + } + } + exec.runtimeMu.Unlock() +} + +func (exec *ISCPTaskExecutor) RegisterRunningConsumer( + key JobRuntimeKey, + jobID uint64, + iterID uint64, + cancel context.CancelFunc, + cancelRetriever func(error), +) (*RunningJobConsumer, bool) { + if exec == nil { + return nil, true + } + exec.runtimeMu.Lock() + defer exec.runtimeMu.Unlock() + exec.ensureRuntimeMapsLocked() + if exec.isJobFencedLocked(key, time.Now()) { + return nil, false + } + groupKey := key.consumerGroupKey() + h := &RunningJobConsumer{ + key: key, + jobID: jobID, + iterID: iterID, + cancel: cancel, + cancelRetriever: cancelRetriever, + done: make(chan struct{}), + } + byJobID := exec.runningConsumers[groupKey] + if byJobID == nil { + byJobID = make(map[uint64]*RunningJobConsumer) + exec.runningConsumers[groupKey] = byJobID + } + byJobID[jobID] = h + return h, true +} + +func (exec *ISCPTaskExecutor) UnregisterRunningConsumer(h *RunningJobConsumer) { + if exec == nil || h == nil { + return + } + exec.runtimeMu.Lock() + groupKey := h.key.consumerGroupKey() + if byJobID := exec.runningConsumers[groupKey]; byJobID != nil { + if byJobID[h.jobID] == h { + delete(byJobID, h.jobID) + } + if len(byJobID) == 0 { + delete(exec.runningConsumers, groupKey) + } + } + exec.runtimeMu.Unlock() + close(h.done) +} + +func (exec *ISCPTaskExecutor) filterFencedIteration(iter *IterationContext) *IterationContext { + if exec == nil || iter == nil || len(iter.jobNames) == 0 { + return iter + } + keepJobNames := make([]string, 0, len(iter.jobNames)) + keepJobIDs := make([]uint64, 0, len(iter.jobIDs)) + keepLSN := make([]uint64, 0, len(iter.lsn)) + for i, jobName := range iter.jobNames { + key := NewJobRuntimeKey(iter.accountID, iter.tableID, jobName, iter.jobIDs[i]) + if exec.IsJobFenced(key) { + continue + } + keepJobNames = append(keepJobNames, jobName) + keepJobIDs = append(keepJobIDs, iter.jobIDs[i]) + keepLSN = append(keepLSN, iter.lsn[i]) + } + if len(keepJobNames) == 0 { + return nil + } + if len(keepJobNames) == len(iter.jobNames) { + return iter + } + return &IterationContext{ + accountID: iter.accountID, + tableID: iter.tableID, + jobNames: keepJobNames, + jobIDs: keepJobIDs, + lsn: keepLSN, + fromTS: iter.fromTS, + toTS: iter.toTS, + } +} diff --git a/pkg/iscp/runtime_cancel_test.go b/pkg/iscp/runtime_cancel_test.go new file mode 100644 index 0000000000000..26b150c5934c6 --- /dev/null +++ b/pkg/iscp/runtime_cancel_test.go @@ -0,0 +1,360 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iscp + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/util/fault" + "github.com/stretchr/testify/require" +) + +func newRuntimeTestExecutor() *ISCPTaskExecutor { + return &ISCPTaskExecutor{ + fencedJobs: make(map[JobRuntimeKey]JobFence), + runningConsumers: make(map[JobRuntimeKey]map[uint64]*RunningJobConsumer), + option: fillDefaultOption(nil), + ctx: context.Background(), + tables: newISCPTableTree(), + } +} + +func TestGetExecutorRuntimeRequiresExactCN(t *testing.T) { + exec := newRuntimeTestExecutor() + RegisterExecutorRuntime("runner-cn", exec) + defer UnregisterExecutorRuntime("runner-cn", exec) + + found, ok := GetExecutorRuntime("ddl-cn") + require.False(t, ok) + require.Nil(t, found) + + found, ok = GetExecutorRuntime("runner-cn") + require.True(t, ok) + require.Same(t, exec, found) +} + +func TestCancelAndDrainJobConsumerFencesAndWaitsForRunningConsumer(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 7) + consumerCtx, consumerCancel := context.WithCancel(context.Background()) + defer consumerCancel() + retrieverCanceled := make(chan error, 1) + + h, ok := exec.RegisterRunningConsumer(key, 7, 11, consumerCancel, func(err error) { + retrieverCanceled <- err + }) + require.True(t, ok) + + exited := make(chan struct{}) + go func() { + defer close(exited) + <-consumerCtx.Done() + time.Sleep(20 * time.Millisecond) + exec.UnregisterRunningConsumer(h) + }() + + start := time.Now() + err := exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID) + require.NoError(t, err) + require.GreaterOrEqual(t, time.Since(start), 20*time.Millisecond) + require.True(t, exec.IsJobFenced(key)) + require.ErrorContains(t, <-retrieverCanceled, "iscp job consumer canceled") + <-exited +} + +func TestRegisterRunningConsumerRejectsFencedJob(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + _, ok := exec.RegisterRunningConsumer(key, 1, 1, func() {}, nil) + + require.False(t, ok) +} + +func TestExpiredJobFenceIsClearedWhenChecked(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + exec.fencedJobs[key] = JobFence{ExpireAt: time.Now().Add(-time.Second)} + + require.False(t, exec.IsJobFenced(key)) + + exec.runtimeMu.Lock() + _, exists := exec.fencedJobs[key] + exec.runtimeMu.Unlock() + require.False(t, exists) + + _, ok := exec.RegisterRunningConsumer(key, 1, 1, func() {}, nil) + require.True(t, ok) +} + +func TestRollbackFenceTTLCanBeInjected(t *testing.T) { + require.True(t, fault.Enable()) + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL, ":::", "echo", 2, "", false)) + defer func() { + _, _ = fault.RemoveFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL) + }() + + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + + exec.runtimeMu.Lock() + fence := exec.fencedJobs[key] + exec.runtimeMu.Unlock() + require.WithinDuration(t, time.Now().Add(2*time.Second), fence.ExpireAt, 500*time.Millisecond) +} + +func TestCancelAndDrainJobConsumerOnlyCancelsMatchingJob(t *testing.T) { + exec := newRuntimeTestExecutor() + key1 := NewJobRuntimeKey(1, 2, "index_idx01", 1) + key2 := NewJobRuntimeKey(1, 2, "index_idx02", 2) + ctx1, cancel1 := context.WithCancel(context.Background()) + ctx2, cancel2 := context.WithCancel(context.Background()) + defer cancel2() + + h1, ok := exec.RegisterRunningConsumer(key1, 1, 1, cancel1, nil) + require.True(t, ok) + _, ok = exec.RegisterRunningConsumer(key2, 2, 1, cancel2, nil) + require.True(t, ok) + go func() { + <-ctx1.Done() + exec.UnregisterRunningConsumer(h1) + }() + + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key1.AccountID, key1.TableID, key1.JobName, key1.JobID)) + + select { + case <-ctx2.Done(): + t.Fatal("non-matching job was canceled") + default: + } + exec.runtimeMu.Lock() + _, stillRunning := exec.runningConsumers[key2.consumerGroupKey()][uint64(2)] + exec.runtimeMu.Unlock() + require.True(t, stillRunning) +} + +func TestFilterFencedIterationRemovesOnlyFencedJobs(t *testing.T) { + exec := newRuntimeTestExecutor() + iter := NewIterationContext( + 1, + 2, + []string{"index_idx01", "index_idx02"}, + []uint64{10, 20}, + []uint64{100, 200}, + types.BuildTS(1, 0), + types.BuildTS(2, 0), + ) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01", 10)) + + filtered := exec.filterFencedIteration(iter) + + require.NotNil(t, filtered) + require.Equal(t, []string{"index_idx02"}, filtered.jobNames) + require.Equal(t, []uint64{20}, filtered.jobIDs) + require.Equal(t, []uint64{200}, filtered.lsn) + require.True(t, filtered.fromTS.EQ(&iter.fromTS)) + require.True(t, filtered.toTS.EQ(&iter.toTS)) +} + +func TestFilterFencedIterationDropsAllJobs(t *testing.T) { + exec := newRuntimeTestExecutor() + iter := NewIterationContext( + 1, + 2, + []string{"index_idx01"}, + []uint64{10}, + []uint64{100}, + types.BuildTS(1, 0), + types.BuildTS(2, 0), + ) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01", 10)) + + require.Nil(t, exec.filterFencedIteration(iter)) +} + +func TestTableEntryGetCandidateSkipsFencedJob(t *testing.T) { + exec := newRuntimeTestExecutor() + table := NewTableEntry(exec, 1, 10, 2, "db", "tbl") + spec := &JobSpec{TriggerSpec: TriggerSpec{JobType: TriggerType_Default}} + table.jobs[JobKey{JobName: "index_idx01", JobID: 1}] = NewJobEntry(table, "index_idx01", spec, 1, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + table.jobs[JobKey{JobName: "index_idx02", JobID: 2}] = NewJobEntry(table, "index_idx02", spec, 2, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01", 1)) + + iters, _ := table.getCandidate() + + require.Len(t, iters, 1) + require.Equal(t, []string{"index_idx02"}, iters[0].jobNames) +} + +func TestTableEntryGetCandidateDoesNotSkipRecreatedSameNameJob(t *testing.T) { + exec := newRuntimeTestExecutor() + table := NewTableEntry(exec, 1, 10, 2, "db", "tbl") + spec := &JobSpec{TriggerSpec: TriggerSpec{JobType: TriggerType_Default}} + table.jobs[JobKey{JobName: "index_idx01", JobID: 1}] = NewJobEntry(table, "index_idx01", spec, 1, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + table.jobs[JobKey{JobName: "index_idx01", JobID: 2}] = NewJobEntry(table, "index_idx01", spec, 2, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), 1, 2, "index_idx01", 1)) + + iters, _ := table.getCandidate() + + require.Len(t, iters, 1) + require.Equal(t, []uint64{2}, iters[0].jobIDs) +} + +func TestTableEntryGetCandidateClearsExpiredRollbackFence(t *testing.T) { + exec := newRuntimeTestExecutor() + table := NewTableEntry(exec, 1, 10, 2, "db", "tbl") + spec := &JobSpec{TriggerSpec: TriggerSpec{JobType: TriggerType_Default}} + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + table.jobs[JobKey{JobName: key.JobName, JobID: key.JobID}] = NewJobEntry(table, key.JobName, spec, key.JobID, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + exec.runtimeMu.Lock() + exec.fencedJobs[key] = JobFence{ExpireAt: time.Now().Add(-time.Second)} + exec.runtimeMu.Unlock() + + iters, _ := table.getCandidate() + + require.Len(t, iters, 1) + require.Equal(t, []string{key.JobName}, iters[0].jobNames) + require.False(t, exec.IsJobFenced(key)) +} + +func TestRemoveTableJobFencesOnlyClearsMatchingTable(t *testing.T) { + exec := newRuntimeTestExecutor() + key1 := NewJobRuntimeKey(1, 2, "index_idx01", 1) + key2 := NewJobRuntimeKey(1, 2, "index_idx01", 2) + keyOtherTable := NewJobRuntimeKey(1, 3, "index_idx01", 1) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key1.AccountID, key1.TableID, key1.JobName, key1.JobID)) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key2.AccountID, key2.TableID, key2.JobName, key2.JobID)) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), keyOtherTable.AccountID, keyOtherTable.TableID, keyOtherTable.JobName, keyOtherTable.JobID)) + + exec.RemoveTableJobFences(1, 2) + + require.False(t, exec.IsJobFenced(key1)) + require.False(t, exec.IsJobFenced(key2)) + require.True(t, exec.IsJobFenced(keyOtherTable)) +} + +func TestAddOrUpdateJobDropAtClearsGenerationFence(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + table := NewTableEntry(exec, key.AccountID, 1, key.TableID, "db", "tbl") + exec.setTable(table) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + require.True(t, exec.IsJobFenced(key)) + + err := exec.addOrUpdateJob( + key.AccountID, + key.TableID, + key.JobName, + key.JobID, + ISCPJobState_Completed, + types.BuildTS(1, 0).ToString(), + mustEncodeRuntimeJobSpec(t), + encodeJSONRows(t, []string{mustMarshalJobStatus(t, 1, JobStage_Running)})[0], + types.Timestamp(time.Now().UnixNano()), + false, + ) + + require.NoError(t, err) + require.False(t, exec.IsJobFenced(key)) + iters, _ := table.getCandidate() + require.Empty(t, iters) +} + +func TestAddOrUpdateJobDropAtPreservesFenceOnParseFailure(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + table := NewTableEntry(exec, key.AccountID, 1, key.TableID, "db", "tbl") + spec := &JobSpec{TriggerSpec: TriggerSpec{JobType: TriggerType_Default}} + table.jobs[JobKey{JobName: key.JobName, JobID: key.JobID}] = NewJobEntry(table, key.JobName, spec, key.JobID, types.BuildTS(1, 0), ISCPJobState_Completed, 0) + exec.setTable(table) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + + err := exec.addOrUpdateJob( + key.AccountID, + key.TableID, + key.JobName, + key.JobID, + ISCPJobState_Completed, + types.BuildTS(1, 0).ToString(), + []byte("bad-json"), + encodeJSONRows(t, []string{mustMarshalJobStatus(t, 1, JobStage_Running)})[0], + types.Timestamp(time.Now().UnixNano()), + false, + ) + + require.Error(t, err) + require.True(t, exec.IsJobFenced(key)) + iters, _ := table.getCandidate() + require.Empty(t, iters) +} + +func TestAddOrUpdateJobDropAtClearsGenerationFenceWhenTableIsAbsent(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + require.NoError(t, exec.CancelAndDrainJobConsumer(context.Background(), key.AccountID, key.TableID, key.JobName, key.JobID)) + require.True(t, exec.IsJobFenced(key)) + + err := exec.addOrUpdateJob( + key.AccountID, + key.TableID, + key.JobName, + key.JobID, + ISCPJobState_Completed, + types.BuildTS(1, 0).ToString(), + mustEncodeRuntimeJobSpec(t), + encodeJSONRows(t, []string{mustMarshalJobStatus(t, 1, JobStage_Running)})[0], + types.Timestamp(time.Now().UnixNano()), + false, + ) + + require.NoError(t, err) + require.False(t, exec.IsJobFenced(key)) +} + +func TestCancelAndDrainJobConsumerHonorsCallerContext(t *testing.T) { + exec := newRuntimeTestExecutor() + key := NewJobRuntimeKey(1, 2, "index_idx01", 1) + consumerCancelCalled := make(chan struct{}) + _, ok := exec.RegisterRunningConsumer(key, 1, 1, func() { close(consumerCancelCalled) }, nil) + require.True(t, ok) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := exec.CancelAndDrainJobConsumer(ctx, key.AccountID, key.TableID, key.JobName, key.JobID) + + require.True(t, errors.Is(err, context.Canceled)) + <-consumerCancelCalled +} + +func mustEncodeRuntimeJobSpec(t *testing.T) []byte { + t.Helper() + spec, err := MarshalJobSpec(&JobSpec{ + ConsumerInfo: ConsumerInfo{ + SrcTable: TableInfo{DBID: 1, TableID: 2, DBName: "db", TableName: "tbl"}, + IndexName: "idx01", + }, + }) + require.NoError(t, err) + return encodeJSONRows(t, []string{spec})[0] +} diff --git a/pkg/iscp/table_entry.go b/pkg/iscp/table_entry.go index 05d08fb4effad..88c8ebe2ed85e 100644 --- a/pkg/iscp/table_entry.go +++ b/pkg/iscp/table_entry.go @@ -151,6 +151,9 @@ func (t *TableEntry) getCandidate() (iter []*IterationContext, minFromTS types.T if sinker.dropAt != 0 { continue } + if t.exec != nil && t.exec.IsJobFenced(NewJobRuntimeKey(t.accountID, t.tableID, sinker.jobName, sinker.jobID)) { + continue + } candidates = append(candidates, sinker) } iterations := make([]*IterationContext, 0, len(candidates)) diff --git a/pkg/iscp/types.go b/pkg/iscp/types.go index 34b0fa5cd3906..05547806ff6a1 100644 --- a/pkg/iscp/types.go +++ b/pkg/iscp/types.go @@ -159,6 +159,31 @@ type ISCPTaskExecutor struct { running bool runningMu sync.Mutex + + runtimeMu sync.Mutex + fencedJobs map[JobRuntimeKey]JobFence + runningConsumers map[JobRuntimeKey]map[uint64]*RunningJobConsumer +} + +type JobRuntimeKey struct { + AccountID uint32 + TableID uint64 + JobName string + JobID uint64 +} + +type JobFence struct { + ExpireAt time.Time +} + +type RunningJobConsumer struct { + key JobRuntimeKey + jobID uint64 + iterID uint64 + + cancel context.CancelFunc + cancelRetriever func(error) + done chan struct{} } // Intra-System Change Propagation Job Entry diff --git a/pkg/iscp/util.go b/pkg/iscp/util.go index d97795657bb34..f61d01d996d86 100644 --- a/pkg/iscp/util.go +++ b/pkg/iscp/util.go @@ -30,6 +30,7 @@ import ( "go.uber.org/zap" + "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/batch" @@ -38,6 +39,7 @@ import ( // "github.com/matrixorigin/matrixone/pkg/container/bytejson" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/util/executor" @@ -441,14 +443,8 @@ func checkLease( } defer txn.Commit(ctxWithTimeout) - sql := `select task_runner from mo_task.sys_daemon_task where task_type = "ISCP" and task_runner is not null` - result, err := ExecWithResult(ctxWithTimeout, sql, cnUUID, txn) - if err != nil { - return - } - defer result.Close() var runner string - runner, err = readSingleTaskRunner(result) + runner, err = GetTaskRunner(ctxWithTimeout, cnUUID, txn) if err != nil { return } @@ -467,6 +463,24 @@ func checkLease( return } +func GetTaskRunner( + ctx context.Context, + cnUUID string, + txn client.TxnOperator, +) (string, error) { + ctxWithSysAccount := context.WithValue(ctx, defines.TenantIDKey{}, catalog.System_Account) + ctxWithTimeout, cancel := context.WithTimeoutCause(ctxWithSysAccount, time.Minute*5, moerr.NewInternalErrorNoCtx("iscp get task runner timeout")) + defer cancel() + + sql := `select task_runner from mo_task.sys_daemon_task where task_type = "ISCP" and task_runner is not null` + result, err := ExecWithResult(ctxWithTimeout, sql, cnUUID, txn) + if err != nil { + return "", err + } + defer result.Close() + return readSingleTaskRunner(result) +} + func readSingleTaskRunner(result executor.Result) (string, error) { runners := make([]string, 0, 1) result.ReadRows(func(rows int, cols []*vector.Vector) bool { diff --git a/pkg/iscp/util_readrows_test.go b/pkg/iscp/util_readrows_test.go index 0ce829862e229..92a590403a134 100644 --- a/pkg/iscp/util_readrows_test.go +++ b/pkg/iscp/util_readrows_test.go @@ -15,10 +15,14 @@ package iscp import ( + "context" "testing" + "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/util/executor" "github.com/stretchr/testify/require" ) @@ -77,6 +81,34 @@ func TestReadSingleTaskRunnerRejectsEmptyRunner(t *testing.T) { require.Contains(t, err.Error(), "task runner is null") } +func TestGetTaskRunnerQueriesMOTaskWithSystemAccount(t *testing.T) { + oldExecWithResult := ExecWithResult + defer func() { + ExecWithResult = oldExecWithResult + }() + + result, mp := newTaskRunnerResult(t, [][]string{{"cn1"}}) + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + mpool.DeleteMPool(mp) + }() + + ExecWithResult = func(ctx context.Context, sql string, cnUUID string, txn client.TxnOperator) (executor.Result, error) { + accountID, err := defines.GetAccountId(ctx) + require.NoError(t, err) + require.Equal(t, uint32(catalog.System_Account), accountID) + require.Contains(t, sql, "mo_task.sys_daemon_task") + require.Equal(t, "cn0", cnUUID) + require.Nil(t, txn) + return result, nil + } + + tenantCtx := defines.AttachAccountId(context.Background(), 1001) + runner, err := GetTaskRunner(tenantCtx, "cn0", nil) + require.NoError(t, err) + require.Equal(t, "cn1", runner) +} + func newTaskRunnerResult(t *testing.T, batches [][]string) (executor.Result, *mpool.MPool) { t.Helper() diff --git a/pkg/iscp/watermark_updater.go b/pkg/iscp/watermark_updater.go index a8d679a68d07d..5a585eae23621 100644 --- a/pkg/iscp/watermark_updater.go +++ b/pkg/iscp/watermark_updater.go @@ -327,6 +327,41 @@ func UnregisterJob( ) } +func LookupJobLog( + ctx context.Context, + cnUUID string, + txn client.TxnOperator, + jobID *JobID, +) (accountID uint32, tableID uint64, internalJobID uint64, exists bool, dropped bool, err error) { + ctxWithSysAccount := context.WithValue(ctx, defines.TenantIDKey{}, catalog.System_Account) + ctxWithSysAccount, cancel := context.WithTimeout(ctxWithSysAccount, time.Minute*5) + defer cancel() + accountID, err = defines.GetAccountId(ctx) + if err != nil { + return + } + tableID, _, err = getTableID( + ctxWithSysAccount, + cnUUID, + txn, + accountID, + jobID.DBName, + jobID.TableName, + ) + if err != nil { + return + } + exists, dropped, internalJobID, err = queryIndexLog( + ctxWithSysAccount, + cnUUID, + txn, + accountID, + tableID, + jobID.JobName, + ) + return +} + func RenameSrcTable( ctx context.Context, cnUUID string, diff --git a/pkg/iscp/worker.go b/pkg/iscp/worker.go index 658089b600ad1..cc4069384b6b9 100644 --- a/pkg/iscp/worker.go +++ b/pkg/iscp/worker.go @@ -44,6 +44,7 @@ type Worker interface { } type worker struct { + exec *ISCPTaskExecutor cnUUID string cnEngine engine.Engine cnTxnClient client.TxnClient @@ -88,8 +89,9 @@ func (g *workerAdmissionGate) seal(cancel context.CancelFunc) { g.wg.Wait() } -func NewWorker(cnUUID string, cnEngine engine.Engine, cnTxnClient client.TxnClient, mp *mpool.MPool) Worker { +func NewWorker(exec *ISCPTaskExecutor, cnUUID string, cnEngine engine.Engine, cnTxnClient client.TxnClient, mp *mpool.MPool) Worker { worker := &worker{ + exec: exec, cnUUID: cnUUID, cnEngine: cnEngine, cnTxnClient: cnTxnClient, @@ -144,11 +146,16 @@ func (w *worker) Submit(iteration *IterationContext) error { } func (w *worker) onItem(iterCtx *IterationContext) { + iterCtx = w.exec.filterFencedIteration(iterCtx) + if iterCtx == nil { + return + } err := retry( w.ctx, func() error { - err := ExecuteIteration( + err := ExecuteIterationWithRuntime( w.ctx, + w.exec, w.cnUUID, w.cnEngine, w.cnTxnClient, diff --git a/pkg/iscp/worker_test.go b/pkg/iscp/worker_test.go index 8a4ff78da3939..02a7507b1e6ec 100644 --- a/pkg/iscp/worker_test.go +++ b/pkg/iscp/worker_test.go @@ -28,14 +28,14 @@ func TestWorkerStopImmediatelyAfterConstruction(t *testing.T) { defer runtime.GOMAXPROCS(previous) for range 32 { - worker := NewWorker("", nil, nil, nil) + worker := NewWorker(nil, "", nil, nil, nil) worker.Stop() runtime.Gosched() } } func TestWorkerStopIsIdempotent(t *testing.T) { - worker := NewWorker("", nil, nil, nil) + worker := NewWorker(nil, "", nil, nil, nil) var stops sync.WaitGroup for range 8 { @@ -49,7 +49,7 @@ func TestWorkerStopIsIdempotent(t *testing.T) { } func TestWorkerRejectsInvalidOrClosedSubmissions(t *testing.T) { - worker := NewWorker("", nil, nil, nil) + worker := NewWorker(nil, "", nil, nil, nil) require.Error(t, worker.Submit(nil)) worker.Stop() diff --git a/pkg/objectio/injects.go b/pkg/objectio/injects.go index 9ba17f0867f93..86200c55c36ab 100644 --- a/pkg/objectio/injects.go +++ b/pkg/objectio/injects.go @@ -67,6 +67,19 @@ const ( FJ_ISCPIndexCuvsAppendErr = "fj/iscp/index/cuvs/append/error" FJ_ISCPIndexCuvsSaveErr = "fj/iscp/index/cuvs/save/error" + FJ_ISCPCancelAfterSubmit = "fj/iscp/cancel/after-submit" + FJ_ISCPCancelAfterRegisterConsumer = "fj/iscp/cancel/after-register-consumer" + FJ_ISCPCancelFanoutBeforeSend = "fj/iscp/cancel/fanout-before-send" + FJ_ISCPCancelHnswBeforeSave = "fj/iscp/cancel/hnsw-before-save" + FJ_ISCPCancelBeforeUpdateWatermark = "fj/iscp/cancel/before-update-watermark" + FJ_ISCPCancelLongBeforeSend = "fj/iscp/cancel/long/before-send" + FJ_ISCPCancelLongBeforeExec = "fj/iscp/cancel/long/before-exec" + FJ_ISCPCancelLongHnswBeforeUpdate = "fj/iscp/cancel/long/hnsw-before-update" + FJ_ISCPCancelLongHnswBeforeSave = "fj/iscp/cancel/long/hnsw-before-save" + FJ_ISCPCancelLongBeforeWatermark = "fj/iscp/cancel/long/before-watermark" + FJ_ISCPCancelRemoveFenceError = "fj/iscp/cancel/remove-fence-error" + FJ_ISCPCancelRollbackFenceTTL = "fj/iscp/cancel/rollback-fence-ttl" + FJ_PublicationSnapshotFinished = "fj/publication/snapshot/finished" FJ_UpstreamSQLHelper = "fj/publication/upstream/sqlhelper" @@ -436,8 +449,14 @@ func GCDumpTableInjected() (string, bool) { return sarg, injected } -func WaitInjected(key string) { - fault.TriggerFault(key) +func WaitInjected(key string) bool { + _, _, injected := fault.TriggerFault(key) + return injected +} + +func WaitInjectedCtx(ctx context.Context, key string) bool { + _, _, injected := fault.TriggerFaultWithContext(ctx, key) + return injected } func NotifyInjected(key string) { diff --git a/pkg/pb/query/query.pb.go b/pkg/pb/query/query.pb.go index 160c9534d471e..ec501954c4126 100644 --- a/pkg/pb/query/query.pb.go +++ b/pkg/pb/query/query.pb.go @@ -103,6 +103,7 @@ const ( CmdMethod_WorkspaceThreshold CmdMethod = 34 CmdMethod_MinTimestamp CmdMethod = 35 CmdMethod_CtlPrefetchOnSubscribed CmdMethod = 36 + CmdMethod_ISCPDrainConsumer CmdMethod = 37 ) var CmdMethod_name = map[int32]string{ @@ -143,6 +144,7 @@ var CmdMethod_name = map[int32]string{ 34: "WorkspaceThreshold", 35: "MinTimestamp", 36: "CtlPrefetchOnSubscribed", + 37: "ISCPDrainConsumer", } var CmdMethod_value = map[string]int32{ @@ -183,6 +185,7 @@ var CmdMethod_value = map[string]int32{ "WorkspaceThreshold": 34, "MinTimestamp": 35, "CtlPrefetchOnSubscribed": 36, + "ISCPDrainConsumer": 37, } func (x CmdMethod) String() string { @@ -969,6 +972,7 @@ type Request struct { CtlMoTableStatsRequest CtlMoTableStatsRequest `protobuf:"bytes,36,opt,name=CtlMoTableStatsRequest,proto3" json:"CtlMoTableStatsRequest"` WorkspaceThresholdRequest *WorkspaceThresholdRequest `protobuf:"bytes,37,opt,name=WorkspaceThresholdRequest,proto3" json:"WorkspaceThresholdRequest,omitempty"` CtlPrefetchOnSubscribedRequest *CtlPrefetchOnSubscribedRequest `protobuf:"bytes,38,opt,name=CtlPrefetchOnSubscribedRequest,proto3" json:"CtlPrefetchOnSubscribedRequest,omitempty"` + ISCPDrainConsumerRequest *ISCPDrainConsumerRequest `protobuf:"bytes,39,opt,name=ISCPDrainConsumerRequest,proto3" json:"ISCPDrainConsumerRequest,omitempty"` } func (m *Request) Reset() { *m = Request{} } @@ -1270,6 +1274,13 @@ func (m *Request) GetCtlPrefetchOnSubscribedRequest() *CtlPrefetchOnSubscribedRe return nil } +func (m *Request) GetISCPDrainConsumerRequest() *ISCPDrainConsumerRequest { + if m != nil { + return m.ISCPDrainConsumerRequest + } + return nil +} + // ShowProcessListResponse is the response of command ShowProcessList. type ShowProcessListResponse struct { Sessions []*status.Session `protobuf:"bytes,1,rep,name=Sessions,proto3" json:"Sessions,omitempty"` @@ -1369,6 +1380,7 @@ type Response struct { WorkspaceThresholdResponse *WorkspaceThresholdResponse `protobuf:"bytes,37,opt,name=WorkspaceThresholdResponse,proto3" json:"WorkspaceThresholdResponse,omitempty"` MinTimestampResponse *MinTimestampResponse `protobuf:"bytes,38,opt,name=MinTimestampResponse,proto3" json:"MinTimestampResponse,omitempty"` CtlPrefetchOnSubscribedResponse *CtlPrefetchOnSubscribedResponse `protobuf:"bytes,39,opt,name=CtlPrefetchOnSubscribedResponse,proto3" json:"CtlPrefetchOnSubscribedResponse,omitempty"` + ISCPDrainConsumerResponse *ISCPDrainConsumerResponse `protobuf:"bytes,40,opt,name=ISCPDrainConsumerResponse,proto3" json:"ISCPDrainConsumerResponse,omitempty"` } func (m *Response) Reset() { *m = Response{} } @@ -1677,6 +1689,13 @@ func (m *Response) GetCtlPrefetchOnSubscribedResponse() *CtlPrefetchOnSubscribed return nil } +func (m *Response) GetISCPDrainConsumerResponse() *ISCPDrainConsumerResponse { + if m != nil { + return m.ISCPDrainConsumerResponse + } + return nil +} + // AlterAccountRequest is the "alter account restricted" query request. type AlterAccountRequest struct { // Tenant is the tenant which to alter. @@ -2068,6 +2087,134 @@ func (m *CtlPrefetchOnSubscribedResponse) GetResp() string { return "" } +type ISCPDrainConsumerRequest struct { + AccountID uint32 `protobuf:"varint,1,opt,name=AccountID,proto3" json:"AccountID,omitempty"` + TableID uint64 `protobuf:"varint,2,opt,name=TableID,proto3" json:"TableID,omitempty"` + JobName string `protobuf:"bytes,3,opt,name=JobName,proto3" json:"JobName,omitempty"` + JobID uint64 `protobuf:"varint,4,opt,name=JobID,proto3" json:"JobID,omitempty"` + RemoveFenceOnly bool `protobuf:"varint,5,opt,name=RemoveFenceOnly,proto3" json:"RemoveFenceOnly,omitempty"` + RenewFenceOnly bool `protobuf:"varint,6,opt,name=RenewFenceOnly,proto3" json:"RenewFenceOnly,omitempty"` +} + +func (m *ISCPDrainConsumerRequest) Reset() { *m = ISCPDrainConsumerRequest{} } +func (m *ISCPDrainConsumerRequest) String() string { return proto.CompactTextString(m) } +func (*ISCPDrainConsumerRequest) ProtoMessage() {} +func (*ISCPDrainConsumerRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_5c6ac9b241082464, []int{27} +} +func (m *ISCPDrainConsumerRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ISCPDrainConsumerRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ISCPDrainConsumerRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ISCPDrainConsumerRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ISCPDrainConsumerRequest.Merge(m, src) +} +func (m *ISCPDrainConsumerRequest) XXX_Size() int { + return m.ProtoSize() +} +func (m *ISCPDrainConsumerRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ISCPDrainConsumerRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ISCPDrainConsumerRequest proto.InternalMessageInfo + +func (m *ISCPDrainConsumerRequest) GetAccountID() uint32 { + if m != nil { + return m.AccountID + } + return 0 +} + +func (m *ISCPDrainConsumerRequest) GetTableID() uint64 { + if m != nil { + return m.TableID + } + return 0 +} + +func (m *ISCPDrainConsumerRequest) GetJobName() string { + if m != nil { + return m.JobName + } + return "" +} + +func (m *ISCPDrainConsumerRequest) GetJobID() uint64 { + if m != nil { + return m.JobID + } + return 0 +} + +func (m *ISCPDrainConsumerRequest) GetRemoveFenceOnly() bool { + if m != nil { + return m.RemoveFenceOnly + } + return false +} + +func (m *ISCPDrainConsumerRequest) GetRenewFenceOnly() bool { + if m != nil { + return m.RenewFenceOnly + } + return false +} + +type ISCPDrainConsumerResponse struct { + Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` +} + +func (m *ISCPDrainConsumerResponse) Reset() { *m = ISCPDrainConsumerResponse{} } +func (m *ISCPDrainConsumerResponse) String() string { return proto.CompactTextString(m) } +func (*ISCPDrainConsumerResponse) ProtoMessage() {} +func (*ISCPDrainConsumerResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_5c6ac9b241082464, []int{28} +} +func (m *ISCPDrainConsumerResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ISCPDrainConsumerResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ISCPDrainConsumerResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ISCPDrainConsumerResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ISCPDrainConsumerResponse.Merge(m, src) +} +func (m *ISCPDrainConsumerResponse) XXX_Size() int { + return m.ProtoSize() +} +func (m *ISCPDrainConsumerResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ISCPDrainConsumerResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ISCPDrainConsumerResponse proto.InternalMessageInfo + +func (m *ISCPDrainConsumerResponse) GetSuccess() bool { + if m != nil { + return m.Success + } + return false +} + type TraceSpanRequest struct { Cmd string `protobuf:"bytes,1,opt,name=Cmd,proto3" json:"Cmd,omitempty"` Spans string `protobuf:"bytes,2,opt,name=Spans,proto3" json:"Spans,omitempty"` @@ -2078,7 +2225,7 @@ func (m *TraceSpanRequest) Reset() { *m = TraceSpanRequest{} } func (m *TraceSpanRequest) String() string { return proto.CompactTextString(m) } func (*TraceSpanRequest) ProtoMessage() {} func (*TraceSpanRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{27} + return fileDescriptor_5c6ac9b241082464, []int{29} } func (m *TraceSpanRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2136,7 +2283,7 @@ func (m *TraceSpanResponse) Reset() { *m = TraceSpanResponse{} } func (m *TraceSpanResponse) String() string { return proto.CompactTextString(m) } func (*TraceSpanResponse) ProtoMessage() {} func (*TraceSpanResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{28} + return fileDescriptor_5c6ac9b241082464, []int{30} } func (m *TraceSpanResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2179,7 +2326,7 @@ func (m *GetLockInfoRequest) Reset() { *m = GetLockInfoRequest{} } func (m *GetLockInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetLockInfoRequest) ProtoMessage() {} func (*GetLockInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{29} + return fileDescriptor_5c6ac9b241082464, []int{31} } func (m *GetLockInfoRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2221,7 +2368,7 @@ func (m *LockInfo) Reset() { *m = LockInfo{} } func (m *LockInfo) String() string { return proto.CompactTextString(m) } func (*LockInfo) ProtoMessage() {} func (*LockInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{30} + return fileDescriptor_5c6ac9b241082464, []int{32} } func (m *LockInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2301,7 +2448,7 @@ func (m *GetLockInfoResponse) Reset() { *m = GetLockInfoResponse{} } func (m *GetLockInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetLockInfoResponse) ProtoMessage() {} func (*GetLockInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{31} + return fileDescriptor_5c6ac9b241082464, []int{33} } func (m *GetLockInfoResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2351,7 +2498,7 @@ func (m *GetTxnInfoRequest) Reset() { *m = GetTxnInfoRequest{} } func (m *GetTxnInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetTxnInfoRequest) ProtoMessage() {} func (*GetTxnInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{32} + return fileDescriptor_5c6ac9b241082464, []int{34} } func (m *GetTxnInfoRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2397,7 +2544,7 @@ func (m *TxnLockInfo) Reset() { *m = TxnLockInfo{} } func (m *TxnLockInfo) String() string { return proto.CompactTextString(m) } func (*TxnLockInfo) ProtoMessage() {} func (*TxnLockInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{33} + return fileDescriptor_5c6ac9b241082464, []int{35} } func (m *TxnLockInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2461,7 +2608,7 @@ func (m *TxnInfo) Reset() { *m = TxnInfo{} } func (m *TxnInfo) String() string { return proto.CompactTextString(m) } func (*TxnInfo) ProtoMessage() {} func (*TxnInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{34} + return fileDescriptor_5c6ac9b241082464, []int{36} } func (m *TxnInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2527,7 +2674,7 @@ func (m *GetTxnInfoResponse) Reset() { *m = GetTxnInfoResponse{} } func (m *GetTxnInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetTxnInfoResponse) ProtoMessage() {} func (*GetTxnInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{35} + return fileDescriptor_5c6ac9b241082464, []int{37} } func (m *GetTxnInfoResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2577,7 +2724,7 @@ func (m *GetCacheInfoRequest) Reset() { *m = GetCacheInfoRequest{} } func (m *GetCacheInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetCacheInfoRequest) ProtoMessage() {} func (*GetCacheInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{36} + return fileDescriptor_5c6ac9b241082464, []int{38} } func (m *GetCacheInfoRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2625,7 +2772,7 @@ func (m *CacheInfo) Reset() { *m = CacheInfo{} } func (m *CacheInfo) String() string { return proto.CompactTextString(m) } func (*CacheInfo) ProtoMessage() {} func (*CacheInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{37} + return fileDescriptor_5c6ac9b241082464, []int{39} } func (m *CacheInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2704,7 +2851,7 @@ func (m *GetCacheInfoResponse) Reset() { *m = GetCacheInfoResponse{} } func (m *GetCacheInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetCacheInfoResponse) ProtoMessage() {} func (*GetCacheInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{38} + return fileDescriptor_5c6ac9b241082464, []int{40} } func (m *GetCacheInfoResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2750,7 +2897,7 @@ func (m *RemoveRemoteLockTableRequest) Reset() { *m = RemoveRemoteLockTa func (m *RemoveRemoteLockTableRequest) String() string { return proto.CompactTextString(m) } func (*RemoveRemoteLockTableRequest) ProtoMessage() {} func (*RemoveRemoteLockTableRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{39} + return fileDescriptor_5c6ac9b241082464, []int{41} } func (m *RemoveRemoteLockTableRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2808,7 +2955,7 @@ func (m *RemoveRemoteLockTableResponse) Reset() { *m = RemoveRemoteLockT func (m *RemoveRemoteLockTableResponse) String() string { return proto.CompactTextString(m) } func (*RemoveRemoteLockTableResponse) ProtoMessage() {} func (*RemoveRemoteLockTableResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{40} + return fileDescriptor_5c6ac9b241082464, []int{42} } func (m *RemoveRemoteLockTableResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2853,7 +3000,7 @@ func (m *GetLatestBindRequest) Reset() { *m = GetLatestBindRequest{} } func (m *GetLatestBindRequest) String() string { return proto.CompactTextString(m) } func (*GetLatestBindRequest) ProtoMessage() {} func (*GetLatestBindRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{41} + return fileDescriptor_5c6ac9b241082464, []int{43} } func (m *GetLatestBindRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2904,7 +3051,7 @@ func (m *GetLatestBindResponse) Reset() { *m = GetLatestBindResponse{} } func (m *GetLatestBindResponse) String() string { return proto.CompactTextString(m) } func (*GetLatestBindResponse) ProtoMessage() {} func (*GetLatestBindResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{42} + return fileDescriptor_5c6ac9b241082464, []int{44} } func (m *GetLatestBindResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2953,7 +3100,7 @@ func (m *UnsubscribeTableRequest) Reset() { *m = UnsubscribeTableRequest func (m *UnsubscribeTableRequest) String() string { return proto.CompactTextString(m) } func (*UnsubscribeTableRequest) ProtoMessage() {} func (*UnsubscribeTableRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{43} + return fileDescriptor_5c6ac9b241082464, []int{45} } func (m *UnsubscribeTableRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3005,7 +3152,7 @@ func (m *UnsubscribeTableResponse) Reset() { *m = UnsubscribeTableRespon func (m *UnsubscribeTableResponse) String() string { return proto.CompactTextString(m) } func (*UnsubscribeTableResponse) ProtoMessage() {} func (*UnsubscribeTableResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{44} + return fileDescriptor_5c6ac9b241082464, []int{46} } func (m *UnsubscribeTableResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3051,7 +3198,7 @@ func (m *CacheKey) Reset() { *m = CacheKey{} } func (m *CacheKey) String() string { return proto.CompactTextString(m) } func (*CacheKey) ProtoMessage() {} func (*CacheKey) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{45} + return fileDescriptor_5c6ac9b241082464, []int{47} } func (m *CacheKey) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3109,7 +3256,7 @@ func (m *CacheKeys) Reset() { *m = CacheKeys{} } func (m *CacheKeys) String() string { return proto.CompactTextString(m) } func (*CacheKeys) ProtoMessage() {} func (*CacheKeys) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{46} + return fileDescriptor_5c6ac9b241082464, []int{48} } func (m *CacheKeys) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3154,7 +3301,7 @@ func (m *RequestCacheKey) Reset() { *m = RequestCacheKey{} } func (m *RequestCacheKey) String() string { return proto.CompactTextString(m) } func (*RequestCacheKey) ProtoMessage() {} func (*RequestCacheKey) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{47} + return fileDescriptor_5c6ac9b241082464, []int{49} } func (m *RequestCacheKey) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3207,7 +3354,7 @@ func (m *GetCacheDataRequest) Reset() { *m = GetCacheDataRequest{} } func (m *GetCacheDataRequest) String() string { return proto.CompactTextString(m) } func (*GetCacheDataRequest) ProtoMessage() {} func (*GetCacheDataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{48} + return fileDescriptor_5c6ac9b241082464, []int{50} } func (m *GetCacheDataRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3254,7 +3401,7 @@ func (m *ResponseCacheData) Reset() { *m = ResponseCacheData{} } func (m *ResponseCacheData) String() string { return proto.CompactTextString(m) } func (*ResponseCacheData) ProtoMessage() {} func (*ResponseCacheData) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{49} + return fileDescriptor_5c6ac9b241082464, []int{51} } func (m *ResponseCacheData) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3313,7 +3460,7 @@ func (m *GetCacheDataResponse) Reset() { *m = GetCacheDataResponse{} } func (m *GetCacheDataResponse) String() string { return proto.CompactTextString(m) } func (*GetCacheDataResponse) ProtoMessage() {} func (*GetCacheDataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{50} + return fileDescriptor_5c6ac9b241082464, []int{52} } func (m *GetCacheDataResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3357,7 +3504,7 @@ func (m *GetStatsInfoRequest) Reset() { *m = GetStatsInfoRequest{} } func (m *GetStatsInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetStatsInfoRequest) ProtoMessage() {} func (*GetStatsInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{51} + return fileDescriptor_5c6ac9b241082464, []int{53} } func (m *GetStatsInfoRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3401,7 +3548,7 @@ func (m *GetStatsInfoResponse) Reset() { *m = GetStatsInfoResponse{} } func (m *GetStatsInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetStatsInfoResponse) ProtoMessage() {} func (*GetStatsInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{52} + return fileDescriptor_5c6ac9b241082464, []int{54} } func (m *GetStatsInfoResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3447,7 +3594,7 @@ func (m *PrepareStmt) Reset() { *m = PrepareStmt{} } func (m *PrepareStmt) String() string { return proto.CompactTextString(m) } func (*PrepareStmt) ProtoMessage() {} func (*PrepareStmt) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{53} + return fileDescriptor_5c6ac9b241082464, []int{55} } func (m *PrepareStmt) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3505,7 +3652,7 @@ func (m *MigrateConnFromRequest) Reset() { *m = MigrateConnFromRequest{} func (m *MigrateConnFromRequest) String() string { return proto.CompactTextString(m) } func (*MigrateConnFromRequest) ProtoMessage() {} func (*MigrateConnFromRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{54} + return fileDescriptor_5c6ac9b241082464, []int{56} } func (m *MigrateConnFromRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3551,7 +3698,7 @@ func (m *MigrateConnFromResponse) Reset() { *m = MigrateConnFromResponse func (m *MigrateConnFromResponse) String() string { return proto.CompactTextString(m) } func (*MigrateConnFromResponse) ProtoMessage() {} func (*MigrateConnFromResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{55} + return fileDescriptor_5c6ac9b241082464, []int{57} } func (m *MigrateConnFromResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3613,7 +3760,7 @@ func (m *MigrateConnToRequest) Reset() { *m = MigrateConnToRequest{} } func (m *MigrateConnToRequest) String() string { return proto.CompactTextString(m) } func (*MigrateConnToRequest) ProtoMessage() {} func (*MigrateConnToRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{56} + return fileDescriptor_5c6ac9b241082464, []int{58} } func (m *MigrateConnToRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3685,7 +3832,7 @@ func (m *MigrateConnToResponse) Reset() { *m = MigrateConnToResponse{} } func (m *MigrateConnToResponse) String() string { return proto.CompactTextString(m) } func (*MigrateConnToResponse) ProtoMessage() {} func (*MigrateConnToResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{57} + return fileDescriptor_5c6ac9b241082464, []int{59} } func (m *MigrateConnToResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3730,7 +3877,7 @@ func (m *ReloadAutoIncrementCacheRequest) Reset() { *m = ReloadAutoIncre func (m *ReloadAutoIncrementCacheRequest) String() string { return proto.CompactTextString(m) } func (*ReloadAutoIncrementCacheRequest) ProtoMessage() {} func (*ReloadAutoIncrementCacheRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{58} + return fileDescriptor_5c6ac9b241082464, []int{60} } func (m *ReloadAutoIncrementCacheRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3774,7 +3921,7 @@ func (m *ReloadAutoIncrementCacheResponse) Reset() { *m = ReloadAutoIncr func (m *ReloadAutoIncrementCacheResponse) String() string { return proto.CompactTextString(m) } func (*ReloadAutoIncrementCacheResponse) ProtoMessage() {} func (*ReloadAutoIncrementCacheResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{59} + return fileDescriptor_5c6ac9b241082464, []int{61} } func (m *ReloadAutoIncrementCacheResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3811,7 +3958,7 @@ func (m *GetReplicaCountRequest) Reset() { *m = GetReplicaCountRequest{} func (m *GetReplicaCountRequest) String() string { return proto.CompactTextString(m) } func (*GetReplicaCountRequest) ProtoMessage() {} func (*GetReplicaCountRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{60} + return fileDescriptor_5c6ac9b241082464, []int{62} } func (m *GetReplicaCountRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3855,7 +4002,7 @@ func (m *GetReplicaCountResponse) Reset() { *m = GetReplicaCountResponse func (m *GetReplicaCountResponse) String() string { return proto.CompactTextString(m) } func (*GetReplicaCountResponse) ProtoMessage() {} func (*GetReplicaCountResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{61} + return fileDescriptor_5c6ac9b241082464, []int{63} } func (m *GetReplicaCountResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3901,7 +4048,7 @@ func (m *ResetSessionRequest) Reset() { *m = ResetSessionRequest{} } func (m *ResetSessionRequest) String() string { return proto.CompactTextString(m) } func (*ResetSessionRequest) ProtoMessage() {} func (*ResetSessionRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{62} + return fileDescriptor_5c6ac9b241082464, []int{64} } func (m *ResetSessionRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3949,7 +4096,7 @@ func (m *ResetSessionResponse) Reset() { *m = ResetSessionResponse{} } func (m *ResetSessionResponse) String() string { return proto.CompactTextString(m) } func (*ResetSessionResponse) ProtoMessage() {} func (*ResetSessionResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{63} + return fileDescriptor_5c6ac9b241082464, []int{65} } func (m *ResetSessionResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4003,7 +4150,7 @@ func (m *GoMaxProcsRequest) Reset() { *m = GoMaxProcsRequest{} } func (m *GoMaxProcsRequest) String() string { return proto.CompactTextString(m) } func (*GoMaxProcsRequest) ProtoMessage() {} func (*GoMaxProcsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{64} + return fileDescriptor_5c6ac9b241082464, []int{66} } func (m *GoMaxProcsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4050,7 +4197,7 @@ func (m *GoMaxProcsResponse) Reset() { *m = GoMaxProcsResponse{} } func (m *GoMaxProcsResponse) String() string { return proto.CompactTextString(m) } func (*GoMaxProcsResponse) ProtoMessage() {} func (*GoMaxProcsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{65} + return fileDescriptor_5c6ac9b241082464, []int{67} } func (m *GoMaxProcsResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4094,7 +4241,7 @@ func (m *GoMemLimitRequest) Reset() { *m = GoMemLimitRequest{} } func (m *GoMemLimitRequest) String() string { return proto.CompactTextString(m) } func (*GoMemLimitRequest) ProtoMessage() {} func (*GoMemLimitRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{66} + return fileDescriptor_5c6ac9b241082464, []int{68} } func (m *GoMemLimitRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4141,7 +4288,7 @@ func (m *GoMemLimitResponse) Reset() { *m = GoMemLimitResponse{} } func (m *GoMemLimitResponse) String() string { return proto.CompactTextString(m) } func (*GoMemLimitResponse) ProtoMessage() {} func (*GoMemLimitResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{67} + return fileDescriptor_5c6ac9b241082464, []int{69} } func (m *GoMemLimitResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4188,7 +4335,7 @@ func (m *FileServiceCacheRequest) Reset() { *m = FileServiceCacheRequest func (m *FileServiceCacheRequest) String() string { return proto.CompactTextString(m) } func (*FileServiceCacheRequest) ProtoMessage() {} func (*FileServiceCacheRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{68} + return fileDescriptor_5c6ac9b241082464, []int{70} } func (m *FileServiceCacheRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4242,7 +4389,7 @@ func (m *FileServiceCacheResponse) Reset() { *m = FileServiceCacheRespon func (m *FileServiceCacheResponse) String() string { return proto.CompactTextString(m) } func (*FileServiceCacheResponse) ProtoMessage() {} func (*FileServiceCacheResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{69} + return fileDescriptor_5c6ac9b241082464, []int{71} } func (m *FileServiceCacheResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4300,7 +4447,7 @@ func (m *FileServiceCacheEvictRequest) Reset() { *m = FileServiceCacheEv func (m *FileServiceCacheEvictRequest) String() string { return proto.CompactTextString(m) } func (*FileServiceCacheEvictRequest) ProtoMessage() {} func (*FileServiceCacheEvictRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{70} + return fileDescriptor_5c6ac9b241082464, []int{72} } func (m *FileServiceCacheEvictRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4346,7 +4493,7 @@ func (m *FileServiceCacheEvictResponse) Reset() { *m = FileServiceCacheE func (m *FileServiceCacheEvictResponse) String() string { return proto.CompactTextString(m) } func (*FileServiceCacheEvictResponse) ProtoMessage() {} func (*FileServiceCacheEvictResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{71} + return fileDescriptor_5c6ac9b241082464, []int{73} } func (m *FileServiceCacheEvictResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4404,7 +4551,7 @@ func (m *MetadataCacheRequest) Reset() { *m = MetadataCacheRequest{} } func (m *MetadataCacheRequest) String() string { return proto.CompactTextString(m) } func (*MetadataCacheRequest) ProtoMessage() {} func (*MetadataCacheRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{72} + return fileDescriptor_5c6ac9b241082464, []int{74} } func (m *MetadataCacheRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4448,7 +4595,7 @@ func (m *MetadataCacheResponse) Reset() { *m = MetadataCacheResponse{} } func (m *MetadataCacheResponse) String() string { return proto.CompactTextString(m) } func (*MetadataCacheResponse) ProtoMessage() {} func (*MetadataCacheResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{73} + return fileDescriptor_5c6ac9b241082464, []int{75} } func (m *MetadataCacheResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4492,7 +4639,7 @@ func (m *GoGCPercentRequest) Reset() { *m = GoGCPercentRequest{} } func (m *GoGCPercentRequest) String() string { return proto.CompactTextString(m) } func (*GoGCPercentRequest) ProtoMessage() {} func (*GoGCPercentRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{74} + return fileDescriptor_5c6ac9b241082464, []int{76} } func (m *GoGCPercentRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4539,7 +4686,7 @@ func (m *GoGCPercentResponse) Reset() { *m = GoGCPercentResponse{} } func (m *GoGCPercentResponse) String() string { return proto.CompactTextString(m) } func (*GoGCPercentResponse) ProtoMessage() {} func (*GoGCPercentResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{75} + return fileDescriptor_5c6ac9b241082464, []int{77} } func (m *GoGCPercentResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4584,7 +4731,7 @@ func (m *FaultInjectRequest) Reset() { *m = FaultInjectRequest{} } func (m *FaultInjectRequest) String() string { return proto.CompactTextString(m) } func (*FaultInjectRequest) ProtoMessage() {} func (*FaultInjectRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{76} + return fileDescriptor_5c6ac9b241082464, []int{78} } func (m *FaultInjectRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4635,7 +4782,7 @@ func (m *FaultInjectResponse) Reset() { *m = FaultInjectResponse{} } func (m *FaultInjectResponse) String() string { return proto.CompactTextString(m) } func (*FaultInjectResponse) ProtoMessage() {} func (*FaultInjectResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{77} + return fileDescriptor_5c6ac9b241082464, []int{79} } func (m *FaultInjectResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4679,7 +4826,7 @@ func (m *CtlMoTableStatsRequest) Reset() { *m = CtlMoTableStatsRequest{} func (m *CtlMoTableStatsRequest) String() string { return proto.CompactTextString(m) } func (*CtlMoTableStatsRequest) ProtoMessage() {} func (*CtlMoTableStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{78} + return fileDescriptor_5c6ac9b241082464, []int{80} } func (m *CtlMoTableStatsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4723,7 +4870,7 @@ func (m *CtlMoTableStatsResponse) Reset() { *m = CtlMoTableStatsResponse func (m *CtlMoTableStatsResponse) String() string { return proto.CompactTextString(m) } func (*CtlMoTableStatsResponse) ProtoMessage() {} func (*CtlMoTableStatsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{79} + return fileDescriptor_5c6ac9b241082464, []int{81} } func (m *CtlMoTableStatsResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4768,7 +4915,7 @@ func (m *WorkspaceThresholdRequest) Reset() { *m = WorkspaceThresholdReq func (m *WorkspaceThresholdRequest) String() string { return proto.CompactTextString(m) } func (*WorkspaceThresholdRequest) ProtoMessage() {} func (*WorkspaceThresholdRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{80} + return fileDescriptor_5c6ac9b241082464, []int{82} } func (m *WorkspaceThresholdRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4820,7 +4967,7 @@ func (m *WorkspaceThresholdResponse) Reset() { *m = WorkspaceThresholdRe func (m *WorkspaceThresholdResponse) String() string { return proto.CompactTextString(m) } func (*WorkspaceThresholdResponse) ProtoMessage() {} func (*WorkspaceThresholdResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{81} + return fileDescriptor_5c6ac9b241082464, []int{83} } func (m *WorkspaceThresholdResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4872,7 +5019,7 @@ func (m *MinTimestampResponse) Reset() { *m = MinTimestampResponse{} } func (m *MinTimestampResponse) String() string { return proto.CompactTextString(m) } func (*MinTimestampResponse) ProtoMessage() {} func (*MinTimestampResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5c6ac9b241082464, []int{82} + return fileDescriptor_5c6ac9b241082464, []int{84} } func (m *MinTimestampResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -4938,6 +5085,8 @@ func init() { proto.RegisterType((*CtlReaderResponse)(nil), "query.CtlReaderResponse") proto.RegisterType((*CtlPrefetchOnSubscribedRequest)(nil), "query.CtlPrefetchOnSubscribedRequest") proto.RegisterType((*CtlPrefetchOnSubscribedResponse)(nil), "query.CtlPrefetchOnSubscribedResponse") + proto.RegisterType((*ISCPDrainConsumerRequest)(nil), "query.ISCPDrainConsumerRequest") + proto.RegisterType((*ISCPDrainConsumerResponse)(nil), "query.ISCPDrainConsumerResponse") proto.RegisterType((*TraceSpanRequest)(nil), "query.TraceSpanRequest") proto.RegisterType((*TraceSpanResponse)(nil), "query.TraceSpanResponse") proto.RegisterType((*GetLockInfoRequest)(nil), "query.GetLockInfoRequest") @@ -4999,225 +5148,233 @@ func init() { func init() { proto.RegisterFile("query.proto", fileDescriptor_5c6ac9b241082464) } var fileDescriptor_5c6ac9b241082464 = []byte{ - // 3475 bytes of a gzipped FileDescriptorProto + // 3610 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x5b, 0xdd, 0x72, 0xdb, 0xc6, - 0xf5, 0x37, 0xf5, 0x49, 0x1e, 0x7d, 0x41, 0x6b, 0x4a, 0x82, 0x64, 0x99, 0x92, 0x61, 0xc7, 0x76, - 0xec, 0x44, 0xf2, 0xdf, 0x89, 0xfd, 0x6f, 0x92, 0xb6, 0x13, 0x89, 0xb2, 0x14, 0xc5, 0x92, 0x25, - 0x2f, 0xe9, 0xd8, 0x71, 0x67, 0x3c, 0x03, 0x91, 0x2b, 0x09, 0x35, 0x09, 0x30, 0x00, 0x98, 0x48, - 0x99, 0xe9, 0x4c, 0x5f, 0xa0, 0x33, 0x79, 0x84, 0xf6, 0x31, 0x7a, 0xdf, 0x8b, 0x5c, 0xe6, 0x32, - 0x57, 0x6d, 0x27, 0x7e, 0x8f, 0x4e, 0x67, 0x17, 0x67, 0x01, 0x2c, 0xb0, 0xa0, 0x92, 0x34, 0xb9, - 0xd1, 0x60, 0xcf, 0x9e, 0xf3, 0xdb, 0xb3, 0x8b, 0xdd, 0x83, 0xdf, 0x9e, 0x43, 0xc1, 0xc4, 0x17, - 0x7d, 0xe6, 0x9f, 0xaf, 0xf5, 0x7c, 0x2f, 0xf4, 0xc8, 0xa8, 0x68, 0x2c, 0x4d, 0x06, 0xa1, 0x1d, - 0xf6, 0x83, 0x48, 0xb8, 0x04, 0x1d, 0xaf, 0xf5, 0x1a, 0x9f, 0x2b, 0xe1, 0x99, 0x8b, 0x8f, 0x33, - 0xa1, 0xd3, 0x65, 0x41, 0x68, 0x77, 0x7b, 0x52, 0xc0, 0xad, 0x02, 0xc7, 0x3d, 0xf6, 0x50, 0xf0, - 0xee, 0x89, 0x13, 0x9e, 0xf6, 0x8f, 0xd6, 0x5a, 0x5e, 0x77, 0xfd, 0xc4, 0x3b, 0xf1, 0xd6, 0x85, - 0xf8, 0xa8, 0x7f, 0x2c, 0x5a, 0xa2, 0x21, 0x9e, 0x50, 0x7d, 0xe5, 0xc4, 0xf3, 0x4e, 0x3a, 0x2c, - 0xd1, 0xca, 0x0c, 0x60, 0xdd, 0x80, 0xc9, 0xa7, 0xdc, 0x3f, 0xca, 0xbe, 0xe8, 0xb3, 0x20, 0x24, - 0x55, 0x18, 0x15, 0x6d, 0xb3, 0xb4, 0x5a, 0xba, 0x5d, 0xa1, 0x51, 0xc3, 0x7a, 0x02, 0xf3, 0x8d, - 0x53, 0xef, 0xab, 0x43, 0xdf, 0x6b, 0xb1, 0x20, 0xd8, 0x73, 0x82, 0x50, 0xea, 0xcf, 0xc3, 0x58, - 0x93, 0xb9, 0xb6, 0x1b, 0xa2, 0x01, 0xb6, 0xc8, 0x32, 0x54, 0x1a, 0xe7, 0x01, 0x76, 0x0d, 0xad, - 0x96, 0x6e, 0x97, 0x69, 0x22, 0xb0, 0x9e, 0xc3, 0x6c, 0xe3, 0xdc, 0x6d, 0xd5, 0xbd, 0x6e, 0xd7, - 0x89, 0xa1, 0x36, 0x61, 0x7a, 0xcf, 0x0e, 0x59, 0x10, 0x46, 0xe2, 0x66, 0x43, 0x40, 0x4e, 0xdc, - 0xaf, 0xae, 0x25, 0x4e, 0x37, 0xe5, 0xd3, 0xe6, 0xc8, 0xb7, 0xff, 0x5c, 0xb9, 0x44, 0x33, 0x16, - 0xd6, 0x4b, 0x20, 0x69, 0xe0, 0xa0, 0xe7, 0xb9, 0x01, 0x23, 0x5b, 0x30, 0x53, 0xef, 0xfb, 0x3e, - 0x73, 0x7f, 0x0a, 0x74, 0xd6, 0xc4, 0x22, 0x60, 0xec, 0xb0, 0x50, 0xf1, 0xd9, 0xfa, 0x1c, 0x66, - 0x53, 0xb2, 0x5f, 0x74, 0xb8, 0x75, 0x98, 0xab, 0x7b, 0x3e, 0xdb, 0xea, 0x77, 0x7b, 0x75, 0xcf, - 0x3d, 0x76, 0x4e, 0x52, 0x4b, 0xbe, 0xd1, 0x0a, 0x1d, 0xcf, 0x95, 0x4b, 0x1e, 0xb5, 0x2c, 0x13, - 0xe6, 0xb3, 0x06, 0x91, 0x43, 0xd6, 0x15, 0x58, 0xdc, 0x61, 0xe1, 0x21, 0x7f, 0xe1, 0x2d, 0xaf, - 0xf3, 0x19, 0xf3, 0x03, 0xc7, 0x73, 0xe5, 0x14, 0x1e, 0xc2, 0x92, 0xae, 0x13, 0xe7, 0x62, 0xc2, - 0x38, 0x8a, 0xc4, 0x68, 0xc3, 0x54, 0x36, 0xad, 0x07, 0xb0, 0xd8, 0x28, 0x02, 0x1d, 0x60, 0xf6, - 0x10, 0x96, 0x1a, 0x3f, 0x67, 0xb8, 0x77, 0x60, 0x9a, 0xf6, 0xdd, 0xa6, 0x1d, 0xbc, 0x96, 0x63, - 0x2c, 0x41, 0x99, 0x37, 0xeb, 0x5e, 0x9b, 0x09, 0xe5, 0x51, 0x1a, 0xb7, 0xad, 0xb7, 0x61, 0x26, - 0xd6, 0x46, 0xe8, 0x79, 0x18, 0xa3, 0x2c, 0xe8, 0x77, 0xe2, 0x9d, 0x1a, 0xb5, 0xf8, 0xb2, 0xf1, - 0xf9, 0x3b, 0x3d, 0xd6, 0x71, 0x5c, 0xb6, 0xeb, 0x1e, 0x7b, 0x72, 0x65, 0xd6, 0x61, 0x21, 0xd7, - 0x83, 0x60, 0x55, 0x18, 0xad, 0x7b, 0x7d, 0xdc, 0xf5, 0xc3, 0x34, 0x6a, 0x58, 0xff, 0x99, 0x87, - 0x71, 0xe9, 0xdd, 0x32, 0x54, 0xf0, 0x71, 0x77, 0x4b, 0x68, 0x8d, 0xd0, 0x44, 0x40, 0xd6, 0xa0, - 0x52, 0xef, 0xb6, 0xf7, 0x59, 0x78, 0xea, 0xb5, 0xc5, 0xf1, 0x98, 0xbe, 0x6f, 0xac, 0x45, 0x51, - 0x23, 0x96, 0xd3, 0x44, 0x85, 0xfc, 0xbf, 0x7a, 0x4c, 0xcd, 0x61, 0xb1, 0x9f, 0x2e, 0xa3, 0x49, - 0xba, 0x8b, 0xaa, 0xe7, 0xf9, 0x59, 0xd1, 0xc9, 0x35, 0x47, 0x04, 0xc4, 0x55, 0x84, 0xd0, 0x2b, - 0xd1, 0xa2, 0x63, 0xbf, 0x07, 0x97, 0x37, 0x3a, 0x21, 0xf3, 0x37, 0x5a, 0x2d, 0x3e, 0x73, 0x89, - 0x39, 0x2a, 0x30, 0x97, 0x10, 0x53, 0xa3, 0x41, 0x75, 0x66, 0xe4, 0x63, 0x98, 0x79, 0xec, 0x74, - 0x3a, 0x75, 0xcf, 0x95, 0x1b, 0xc8, 0x1c, 0x13, 0x48, 0xf3, 0x88, 0x94, 0xe9, 0xa5, 0x59, 0x75, - 0x52, 0x07, 0xa3, 0xe9, 0xdb, 0x2d, 0xd6, 0xe8, 0xd9, 0x31, 0xc4, 0xb8, 0x80, 0x58, 0x40, 0x88, - 0x6c, 0x37, 0xcd, 0x19, 0x90, 0x5d, 0x20, 0x3b, 0x2c, 0xdc, 0xf3, 0x5a, 0xaf, 0x53, 0xbb, 0xc0, - 0x2c, 0x0b, 0x98, 0x45, 0x84, 0xc9, 0x2b, 0x50, 0x8d, 0x11, 0xd9, 0x16, 0x71, 0xa1, 0x79, 0xe6, - 0xa6, 0x91, 0x2a, 0x02, 0xc9, 0x4c, 0x90, 0xd4, 0x7e, 0x9a, 0x37, 0xe1, 0xeb, 0xcc, 0xe3, 0x8b, - 0xdd, 0x3a, 0x4d, 0xef, 0x4c, 0x13, 0x94, 0x75, 0xd6, 0x68, 0x50, 0x9d, 0x19, 0xf9, 0x0d, 0x40, - 0xe3, 0xbc, 0xe5, 0x46, 0x21, 0xc6, 0x9c, 0x50, 0xdc, 0xc9, 0xc5, 0x63, 0x9a, 0xd2, 0x25, 0x0f, - 0xa0, 0x12, 0xc7, 0x39, 0x73, 0x52, 0x59, 0xd8, 0x6c, 0x4c, 0xa4, 0x89, 0x26, 0x39, 0x14, 0x2b, - 0x9a, 0x39, 0xec, 0xe6, 0x94, 0xb0, 0x5f, 0x4d, 0xec, 0xf5, 0x41, 0x84, 0x6a, 0x6c, 0x39, 0x62, - 0x3e, 0x7c, 0x98, 0xd3, 0x0a, 0x62, 0xa3, 0x18, 0x31, 0xdf, 0x45, 0xb6, 0x60, 0x5a, 0x0d, 0x9b, - 0xe6, 0x8c, 0x40, 0x5b, 0x96, 0xe7, 0x51, 0x17, 0x84, 0x69, 0xc6, 0x86, 0xac, 0xc3, 0x38, 0x06, - 0x1c, 0xd3, 0x10, 0xe6, 0x73, 0x68, 0xae, 0x06, 0x2d, 0x2a, 0xb5, 0xc8, 0xe7, 0x30, 0x47, 0x59, - 0xd7, 0xfb, 0x92, 0xf1, 0xbf, 0x21, 0xe3, 0x1b, 0xa8, 0x69, 0x1f, 0x75, 0x98, 0x39, 0x2b, 0xcc, - 0xaf, 0x4b, 0x73, 0x9d, 0x8e, 0x04, 0xd3, 0x23, 0x90, 0x0d, 0x98, 0xe2, 0x5b, 0x52, 0x7c, 0x19, - 0x37, 0x1d, 0xb7, 0x6d, 0x12, 0x01, 0x79, 0x25, 0xb5, 0x85, 0xe3, 0x3e, 0x09, 0xa5, 0x5a, 0x90, - 0x4f, 0xc1, 0x78, 0xe6, 0x06, 0xfd, 0xa3, 0xa0, 0xe5, 0x3b, 0x47, 0x2c, 0x72, 0xec, 0xb2, 0x40, - 0xa9, 0x21, 0x4a, 0xb6, 0x3b, 0x3e, 0x56, 0xd9, 0x8e, 0xf4, 0x1e, 0xde, 0xb2, 0x43, 0x5b, 0xee, - 0xe1, 0xaa, 0x76, 0x0f, 0xa7, 0x34, 0xa8, 0xce, 0x0c, 0xd1, 0x1a, 0x9c, 0x16, 0xa5, 0x4f, 0xc4, - 0x5c, 0x16, 0x2d, 0xab, 0x41, 0x75, 0x66, 0x3c, 0x3c, 0xea, 0x83, 0xbf, 0x39, 0xaf, 0x84, 0x47, - 0xbd, 0x12, 0x2d, 0x30, 0xe6, 0xb0, 0xfb, 0xce, 0x89, 0x6f, 0x87, 0x8c, 0x07, 0xa9, 0x6d, 0xdf, - 0xeb, 0x4a, 0xd8, 0x05, 0x05, 0x56, 0xaf, 0x44, 0x0b, 0x8c, 0xc9, 0x01, 0x54, 0x53, 0x3d, 0xcd, - 0xd8, 0x57, 0x53, 0x79, 0xbf, 0x3a, 0x15, 0xaa, 0x35, 0x24, 0x47, 0x60, 0x52, 0xd6, 0xf1, 0xec, - 0xf6, 0x46, 0x3f, 0xf4, 0x76, 0xdd, 0x96, 0xcf, 0xba, 0x9c, 0x82, 0xf0, 0x35, 0x37, 0x17, 0x05, - 0xe8, 0xcd, 0x78, 0x1f, 0xea, 0xd5, 0x24, 0x7e, 0x21, 0x0e, 0x0f, 0xcd, 0xf5, 0xb0, 0x43, 0x99, - 0xdd, 0x66, 0xbe, 0x74, 0x78, 0x49, 0x89, 0x20, 0xd9, 0x6e, 0x9a, 0x33, 0x20, 0xfb, 0x30, 0xb3, - 0xc3, 0x42, 0xca, 0x7a, 0x1d, 0xa7, 0x65, 0x47, 0x5f, 0xde, 0x2b, 0xd9, 0x17, 0x94, 0xee, 0x45, - 0x3b, 0xc9, 0xad, 0x32, 0xbd, 0x7c, 0x13, 0x51, 0x16, 0xb0, 0xb0, 0xc1, 0x82, 0x54, 0x78, 0x30, - 0x97, 0x95, 0x4d, 0xa4, 0xd1, 0xa0, 0x3a, 0x33, 0xb2, 0x07, 0xb3, 0x3b, 0xde, 0xbe, 0x7d, 0xc6, - 0xbf, 0x93, 0x81, 0xc4, 0xba, 0xaa, 0x06, 0xfb, 0x6c, 0x3f, 0x7a, 0x96, 0x37, 0x44, 0x34, 0xd6, - 0xdd, 0x73, 0x92, 0x98, 0x6a, 0xd6, 0xb2, 0x68, 0x6a, 0x7f, 0x0a, 0x4d, 0xed, 0x20, 0xaf, 0x60, - 0x61, 0xdb, 0xe9, 0xb0, 0x06, 0xf3, 0xbf, 0x74, 0x5a, 0x2c, 0xfd, 0xca, 0xcc, 0x15, 0xe5, 0x3c, - 0x17, 0x68, 0x21, 0x72, 0x11, 0x08, 0xe9, 0xc2, 0x72, 0xb6, 0xeb, 0xd1, 0x97, 0x4e, 0x2b, 0x76, - 0x7c, 0x55, 0x89, 0x66, 0x83, 0x54, 0x71, 0xa4, 0x81, 0x70, 0xe4, 0x19, 0x54, 0xf7, 0x59, 0x68, - 0xb7, 0xed, 0xd0, 0x56, 0xe6, 0x72, 0x4d, 0x3d, 0x01, 0x1a, 0x15, 0x84, 0xd7, 0x9a, 0x93, 0x03, - 0x20, 0x3b, 0xde, 0x4e, 0xfd, 0x90, 0xf9, 0x2d, 0x96, 0xb0, 0x19, 0x4b, 0xfd, 0xf2, 0xe7, 0x14, - 0x10, 0x52, 0x63, 0xca, 0xa9, 0xc4, 0xb6, 0xdd, 0xef, 0x84, 0xbb, 0xee, 0x1f, 0x59, 0xb2, 0x18, - 0xd7, 0x15, 0xc0, 0xbc, 0x02, 0xd5, 0x18, 0x91, 0x3f, 0xc0, 0x7c, 0x3d, 0xec, 0xec, 0x7b, 0x22, - 0x98, 0x8a, 0x00, 0x26, 0xe1, 0x6e, 0x28, 0x27, 0x40, 0xaf, 0x84, 0x3e, 0x16, 0x40, 0x90, 0x57, - 0xb0, 0xf8, 0xdc, 0xf3, 0x5f, 0x07, 0x3d, 0xbb, 0xc5, 0x9a, 0xa7, 0x3e, 0x0b, 0x4e, 0xbd, 0x8e, - 0xfc, 0x26, 0x98, 0x6f, 0x29, 0x5f, 0xd5, 0x42, 0x3d, 0x5a, 0x0c, 0x41, 0xba, 0x50, 0xab, 0x87, - 0x9d, 0x43, 0x9f, 0x1d, 0xb3, 0xb0, 0x75, 0x7a, 0xe0, 0x36, 0xe4, 0xa7, 0x21, 0x1e, 0xe4, 0xa6, - 0x18, 0xe4, 0xad, 0x64, 0x12, 0x03, 0x94, 0xe9, 0x05, 0x60, 0xd6, 0x36, 0x2c, 0xe4, 0x08, 0x2b, - 0x32, 0xf6, 0xbb, 0x50, 0xc6, 0x63, 0x1b, 0x98, 0xa5, 0xd5, 0xe1, 0xdb, 0x13, 0xf7, 0x67, 0xd6, - 0xf0, 0x4a, 0x2e, 0x8f, 0x73, 0xac, 0x60, 0xfd, 0xd9, 0x84, 0x72, 0x6c, 0xf9, 0xcb, 0x32, 0xf9, - 0x2a, 0x8c, 0x3e, 0xf2, 0x7d, 0xcf, 0x17, 0x14, 0x7e, 0x92, 0x46, 0x0d, 0xf2, 0xa2, 0xd0, 0x71, - 0xe4, 0xe9, 0xb5, 0x22, 0x9e, 0x1e, 0x69, 0xd1, 0xc2, 0x79, 0x1f, 0x40, 0x55, 0xa5, 0xdc, 0x08, - 0x3b, 0xaa, 0x9c, 0x18, 0x9d, 0x0a, 0xd5, 0x1a, 0xf2, 0x78, 0x9e, 0xb0, 0x6f, 0x04, 0x1b, 0x53, - 0xe2, 0x79, 0xb6, 0x9b, 0xe6, 0x0c, 0x38, 0x3f, 0x4e, 0xd1, 0x6f, 0x44, 0x19, 0x57, 0x82, 0x5c, - 0xae, 0x9f, 0xe6, 0x4d, 0x90, 0x0d, 0x24, 0xec, 0x1b, 0x91, 0xca, 0x59, 0x36, 0x90, 0xd5, 0xa0, - 0x3a, 0x33, 0xbc, 0x00, 0xc4, 0x14, 0x1c, 0xc1, 0x2a, 0xd9, 0x0b, 0x40, 0x46, 0x81, 0x6a, 0x8c, - 0xf8, 0xb2, 0xab, 0x0c, 0x1c, 0xc1, 0x20, 0x4b, 0xc5, 0x72, 0x2a, 0x54, 0x6b, 0x48, 0x3e, 0xe0, - 0xdc, 0x5d, 0x52, 0x74, 0xe4, 0xee, 0x8b, 0x1a, 0xee, 0x8e, 0x20, 0x29, 0x65, 0xf2, 0x30, 0x4f, - 0xde, 0xcd, 0x3c, 0x79, 0x47, 0xc3, 0x14, 0x7b, 0x7f, 0x3a, 0x80, 0xbd, 0x5f, 0x1b, 0xc0, 0xde, - 0x53, 0xcb, 0x92, 0x25, 0xdb, 0x4f, 0x07, 0xd0, 0xf7, 0x6b, 0x03, 0xe8, 0xbb, 0x84, 0xd4, 0xf0, - 0xf7, 0x47, 0x05, 0xfc, 0xfd, 0x6a, 0x01, 0x7f, 0x47, 0xa8, 0x2c, 0x81, 0xbf, 0x97, 0x25, 0xf0, - 0xf3, 0x59, 0x02, 0x8f, 0x86, 0x31, 0x83, 0x7f, 0x39, 0x98, 0xc1, 0xdf, 0x18, 0xcc, 0xe0, 0x11, - 0xad, 0x80, 0xc2, 0x6f, 0xea, 0x29, 0xfc, 0xb2, 0x9e, 0xc2, 0x23, 0x56, 0x86, 0xc3, 0x3f, 0x2e, - 0xe4, 0xf0, 0x2b, 0x85, 0x1c, 0x5e, 0x1e, 0xd8, 0x1c, 0x89, 0x4f, 0xed, 0xe7, 0x88, 0x8d, 0xe3, - 0x7e, 0xae, 0x6a, 0xf7, 0x73, 0x5a, 0x85, 0x6a, 0x0d, 0x11, 0x30, 0x45, 0xc8, 0x11, 0x70, 0x2e, - 0x0b, 0x98, 0x53, 0xa1, 0x5a, 0x43, 0x1e, 0x42, 0x0b, 0xb2, 0x35, 0xc8, 0xe5, 0x6b, 0x45, 0x5c, - 0x5e, 0x86, 0xd0, 0xa2, 0x64, 0xcf, 0x0b, 0x58, 0xc8, 0x11, 0x72, 0x44, 0x5e, 0x50, 0x90, 0x0b, - 0xb4, 0x68, 0x91, 0x39, 0xa1, 0x30, 0x97, 0xe1, 0xe5, 0x88, 0x6b, 0x2a, 0xaf, 0x5b, 0xab, 0x43, - 0xf5, 0xa6, 0xa4, 0x75, 0x21, 0xa7, 0xbf, 0x75, 0x21, 0xa7, 0xc7, 0x11, 0x8a, 0x49, 0xfd, 0x36, - 0xcc, 0xa6, 0x38, 0x3a, 0x3a, 0xbd, 0xa4, 0x84, 0x96, 0x5c, 0x3f, 0xcd, 0x9b, 0x90, 0x27, 0x45, - 0xbc, 0xbe, 0x56, 0xc4, 0xeb, 0x23, 0xc3, 0x22, 0x62, 0x7f, 0x00, 0x55, 0x95, 0xa1, 0xa3, 0x6b, - 0xcb, 0xca, 0xae, 0xd2, 0xa9, 0x50, 0xad, 0x61, 0xc4, 0x0c, 0x13, 0x8a, 0x8e, 0x70, 0x57, 0x33, - 0xcc, 0x30, 0xab, 0x90, 0x30, 0xc3, 0x6c, 0x0f, 0x02, 0xc6, 0x2c, 0x1d, 0x01, 0x6b, 0x59, 0xc0, - 0x8c, 0x42, 0x0a, 0x30, 0xd3, 0x43, 0x6c, 0x30, 0xf3, 0xe4, 0x1c, 0x61, 0x57, 0x94, 0xe3, 0x5e, - 0xa4, 0x86, 0xe0, 0x85, 0x30, 0xa4, 0x07, 0x57, 0x0b, 0x58, 0x39, 0x8e, 0xb3, 0xaa, 0x44, 0xbc, - 0x81, 0xba, 0x38, 0xd8, 0x60, 0x40, 0xf2, 0x02, 0xe6, 0x32, 0x44, 0x1d, 0x47, 0xba, 0xa6, 0x1e, - 0x0c, 0x9d, 0x0e, 0x8e, 0xa0, 0x07, 0x20, 0x14, 0x2e, 0x2b, 0x7c, 0x1d, 0x71, 0x2d, 0x95, 0x31, - 0xe4, 0x35, 0x10, 0x55, 0x67, 0xcc, 0x59, 0x88, 0x42, 0xdc, 0x11, 0xf3, 0xba, 0x82, 0xa9, 0xd1, - 0xa0, 0x3a, 0x33, 0x7e, 0x65, 0xcb, 0xb1, 0x75, 0x44, 0xbc, 0xa1, 0x9c, 0x8d, 0x02, 0x2d, 0x79, - 0x65, 0x2b, 0xe8, 0x26, 0x36, 0x2c, 0xe9, 0x08, 0x3b, 0x0e, 0xf1, 0x96, 0xf2, 0x2d, 0x2e, 0x56, - 0xa4, 0x03, 0x40, 0xa2, 0x44, 0x85, 0x1b, 0x97, 0x38, 0x62, 0xf0, 0x9b, 0x99, 0x44, 0x45, 0x5e, - 0x85, 0x6a, 0x0d, 0x49, 0x0f, 0x56, 0x0a, 0xa9, 0x3f, 0x62, 0xdf, 0x52, 0xf2, 0x15, 0x17, 0x68, - 0xd3, 0x8b, 0xe0, 0xac, 0x5d, 0x6d, 0x86, 0x5b, 0x14, 0x1d, 0x44, 0x0d, 0x6b, 0xb7, 0x8d, 0xb9, - 0xff, 0xb8, 0x4d, 0xe6, 0x61, 0xac, 0x21, 0x6e, 0x14, 0x82, 0xdb, 0x57, 0x28, 0xb6, 0xac, 0x0f, - 0xf5, 0x14, 0x9c, 0x58, 0x30, 0x69, 0x73, 0x79, 0xa3, 0xdf, 0xe2, 0xb4, 0x5d, 0xe0, 0x95, 0xa9, - 0x22, 0xb3, 0x76, 0x73, 0xa9, 0x71, 0x7e, 0x1f, 0x41, 0x24, 0xbc, 0x8f, 0x0c, 0xd3, 0x44, 0x90, - 0xae, 0xa0, 0x0c, 0x89, 0xbb, 0x4a, 0xaa, 0x82, 0x92, 0xe7, 0xe1, 0x26, 0x8c, 0xab, 0xa3, 0xcb, - 0xa6, 0xb5, 0x97, 0x4f, 0xdb, 0x10, 0x03, 0x86, 0xeb, 0xdd, 0x36, 0xd6, 0x4f, 0xf8, 0xa3, 0x90, - 0x1c, 0x9f, 0x88, 0x91, 0xb8, 0xe4, 0xf8, 0x44, 0xdc, 0x6f, 0xce, 0x42, 0xdf, 0x8e, 0xef, 0x37, - 0xbc, 0x61, 0xdd, 0xd2, 0x7c, 0x2f, 0x08, 0x81, 0x11, 0xfe, 0x8c, 0x78, 0xe2, 0xd9, 0xfa, 0xed, - 0x45, 0x17, 0x46, 0xfe, 0x06, 0x0e, 0xed, 0x30, 0x64, 0x3e, 0x5e, 0xe4, 0x2a, 0x34, 0x6e, 0x5b, - 0x0f, 0x2e, 0xdc, 0x26, 0xda, 0x41, 0x5f, 0xe4, 0xab, 0x07, 0x9a, 0xb9, 0x56, 0x61, 0x94, 0x2b, - 0x04, 0x38, 0xdb, 0xa8, 0xc1, 0xdf, 0x46, 0xbc, 0xff, 0xc5, 0x9c, 0x87, 0x69, 0x22, 0xe0, 0xf3, - 0xce, 0x5f, 0x5a, 0x74, 0x2e, 0x54, 0x75, 0xb5, 0x07, 0xeb, 0xfb, 0x12, 0x94, 0xa5, 0x8c, 0xbf, - 0x2b, 0x71, 0x9a, 0x71, 0xe7, 0x8d, 0x50, 0xd9, 0xe4, 0x80, 0x8f, 0xd9, 0x39, 0x77, 0x6c, 0xf8, - 0xf6, 0x24, 0x15, 0xcf, 0xe4, 0x4e, 0x64, 0xb9, 0xef, 0xb5, 0x99, 0x70, 0x6b, 0xfa, 0xfe, 0xf4, - 0x9a, 0x28, 0x3a, 0x4b, 0x29, 0x8d, 0xfb, 0xc9, 0x2a, 0x4c, 0x38, 0x01, 0xb5, 0xdd, 0x13, 0xc1, - 0x40, 0xc5, 0x8d, 0xb3, 0x4c, 0xd3, 0x22, 0x72, 0x0b, 0xc6, 0x3f, 0xf1, 0x3a, 0x6d, 0xe6, 0x07, - 0xe6, 0xa8, 0xb8, 0x3c, 0x4f, 0x45, 0x60, 0xcf, 0x6d, 0x87, 0x5f, 0x7d, 0xa8, 0xec, 0xe5, 0x8a, - 0x5c, 0xc6, 0x15, 0xc7, 0xb4, 0x8a, 0xd8, 0x6b, 0xbd, 0xd2, 0xde, 0xdc, 0xf8, 0x54, 0xea, 0xee, - 0xae, 0x5c, 0x77, 0xf1, 0x4c, 0xde, 0x83, 0x49, 0xa9, 0xc7, 0xaf, 0xb6, 0x62, 0x9a, 0xfc, 0xfa, - 0x1e, 0x9d, 0xf4, 0x18, 0x42, 0x51, 0xb2, 0x2e, 0x6b, 0x2a, 0x30, 0xd6, 0x29, 0x4c, 0x34, 0xcf, - 0xdc, 0x1f, 0xb7, 0xa2, 0xd4, 0xfb, 0x2a, 0x5e, 0x51, 0xfe, 0x4c, 0xee, 0xc2, 0xf8, 0x41, 0x2f, - 0x14, 0x09, 0x84, 0xa8, 0xfc, 0x36, 0x9b, 0x2c, 0x28, 0x76, 0x50, 0xa9, 0x61, 0xfd, 0xbd, 0x04, - 0xe3, 0x38, 0x38, 0xf9, 0x18, 0xca, 0x75, 0x9f, 0xd9, 0x21, 0xdb, 0x08, 0xb1, 0x10, 0xbc, 0xb4, - 0x16, 0xd5, 0xe5, 0xd7, 0x64, 0x5d, 0x3e, 0x55, 0x0e, 0x2e, 0xf3, 0xe8, 0xfd, 0xcd, 0xbf, 0x56, - 0x4a, 0x34, 0xb6, 0x22, 0xab, 0x30, 0xc2, 0xbf, 0x66, 0x62, 0xe7, 0x4d, 0xdc, 0x9f, 0x5c, 0x0b, - 0xcf, 0xdc, 0xb5, 0xe6, 0x99, 0xcb, 0x65, 0x54, 0xf4, 0xf0, 0xa9, 0x3c, 0x0b, 0x98, 0xdf, 0x3c, - 0x73, 0x85, 0x73, 0x65, 0x2a, 0x9b, 0xe4, 0x1e, 0x54, 0xf8, 0x9a, 0x73, 0x2f, 0x03, 0x73, 0x44, - 0x2c, 0x1d, 0x91, 0x57, 0xec, 0x64, 0x2d, 0x68, 0xa2, 0x64, 0xbd, 0xd4, 0x5d, 0x83, 0xb5, 0x6f, - 0xe6, 0x9e, 0x58, 0xcf, 0xcc, 0x8b, 0x99, 0x4e, 0xd0, 0x05, 0x40, 0x5a, 0xc5, 0x9a, 0xd3, 0x16, - 0xb4, 0xac, 0xbf, 0x95, 0xa0, 0x12, 0x0b, 0xf9, 0x11, 0x7f, 0xe2, 0xb5, 0x59, 0xf3, 0xbc, 0xc7, - 0x70, 0xb8, 0xb8, 0xcd, 0x83, 0x2c, 0x7f, 0xde, 0x6d, 0xe3, 0x31, 0xc4, 0x16, 0x3f, 0x87, 0x02, - 0x40, 0x18, 0x45, 0xf1, 0x37, 0x11, 0x70, 0xe7, 0x9f, 0x05, 0xac, 0x2d, 0xb6, 0xf6, 0x08, 0x15, - 0xcf, 0x5c, 0xb6, 0xed, 0xb3, 0x28, 0x13, 0x32, 0x42, 0xc5, 0x33, 0x1f, 0xf9, 0x13, 0x27, 0xa4, - 0x76, 0xe8, 0x78, 0x22, 0xa9, 0x31, 0x44, 0xe3, 0xb6, 0xf5, 0x44, 0x7f, 0xa5, 0x27, 0x0f, 0x61, - 0x2a, 0x16, 0x8a, 0x65, 0x88, 0xd2, 0x4b, 0x71, 0x16, 0x28, 0x36, 0x50, 0xd5, 0xac, 0x0e, 0x2c, - 0x0f, 0xaa, 0xee, 0xf0, 0x57, 0xba, 0xe3, 0x7b, 0xfd, 0x1e, 0x46, 0xf9, 0x29, 0x2a, 0x9b, 0xc9, - 0xbe, 0xdd, 0x92, 0x31, 0x1e, 0x9b, 0xe9, 0xe8, 0x3f, 0xac, 0x46, 0xff, 0x07, 0x70, 0x75, 0xe0, - 0x4d, 0x54, 0x2d, 0x69, 0x8f, 0xca, 0x92, 0xf6, 0xa7, 0x62, 0xd2, 0xb9, 0x7a, 0xd1, 0xcf, 0x71, - 0xce, 0xba, 0x0b, 0x73, 0xda, 0x8b, 0x2b, 0x7f, 0x13, 0xe2, 0x92, 0x8b, 0x5b, 0x8b, 0x3f, 0x5b, - 0x0d, 0x58, 0x28, 0x28, 0x31, 0x91, 0x1a, 0x00, 0xbf, 0x4a, 0x1e, 0xd9, 0x01, 0x8b, 0x33, 0x72, - 0x29, 0xc9, 0x00, 0x0f, 0xde, 0x07, 0xb3, 0xe8, 0xce, 0x3b, 0xe0, 0x53, 0xb8, 0x0d, 0x65, 0xf1, - 0xe6, 0x1e, 0xb3, 0x73, 0xee, 0xea, 0xa1, 0x1d, 0x9e, 0x4a, 0x57, 0xf9, 0x33, 0xdf, 0x92, 0x07, - 0xc7, 0xc7, 0x01, 0x8b, 0x7e, 0xe8, 0x32, 0x4c, 0xb1, 0x45, 0xa6, 0x61, 0xa8, 0xf1, 0x35, 0x7e, - 0x13, 0x86, 0x1a, 0x5f, 0x5b, 0x0f, 0x71, 0x8b, 0x8a, 0xf8, 0xfc, 0x36, 0x8c, 0xbc, 0xe6, 0x31, - 0xbb, 0xa4, 0x04, 0x33, 0xd9, 0x8f, 0x1c, 0x4e, 0xa8, 0x58, 0x4d, 0x98, 0xc1, 0xa9, 0xc7, 0x6e, - 0x54, 0x61, 0x74, 0xd7, 0x6d, 0xb3, 0x33, 0xf9, 0xb2, 0x44, 0x83, 0xdc, 0x4d, 0x1c, 0xc5, 0x50, - 0x91, 0xc5, 0xa5, 0xb1, 0x82, 0xf5, 0x5c, 0x5b, 0x96, 0x23, 0x1f, 0xe7, 0x06, 0x43, 0x17, 0xe3, - 0x7c, 0x88, 0xda, 0x4b, 0xb3, 0xea, 0xd6, 0x01, 0xcc, 0xca, 0x45, 0x8d, 0xd1, 0x0b, 0x1c, 0x36, - 0x60, 0xf8, 0x13, 0x47, 0xfe, 0x3e, 0x88, 0x3f, 0xf2, 0xf5, 0xe5, 0xfa, 0xc8, 0x1e, 0xc4, 0xb3, - 0xf5, 0x4a, 0x9f, 0x7b, 0xe0, 0x97, 0xd0, 0xdc, 0x40, 0xe8, 0xac, 0x99, 0xdc, 0xf4, 0xd4, 0x7e, - 0x9a, 0x37, 0xb1, 0xa8, 0xb6, 0xa4, 0x48, 0x3e, 0x82, 0xc9, 0x58, 0x16, 0x2d, 0x43, 0x94, 0xe4, - 0x4c, 0x7e, 0x92, 0x95, 0xee, 0xa6, 0x8a, 0x32, 0x9e, 0x9b, 0x7c, 0x96, 0xe2, 0x3e, 0x54, 0x62, - 0x61, 0xfc, 0xab, 0x20, 0x0d, 0x22, 0x4d, 0xd4, 0xac, 0x06, 0x4c, 0x1c, 0xfa, 0xac, 0x67, 0xfb, - 0xac, 0x11, 0x76, 0xc5, 0x12, 0x3d, 0xb1, 0xbb, 0x32, 0x32, 0x8a, 0x67, 0xbe, 0x90, 0x8d, 0xa7, - 0x7b, 0x92, 0x87, 0x35, 0x9e, 0xee, 0xf1, 0x43, 0x72, 0x68, 0xfb, 0x76, 0x97, 0x87, 0xbf, 0x00, - 0x97, 0x33, 0x25, 0xb1, 0xee, 0x15, 0x95, 0x28, 0xf9, 0x76, 0xe6, 0xa2, 0xf8, 0x64, 0x63, 0xcb, - 0xfa, 0x4b, 0xa9, 0x30, 0x0f, 0xc2, 0xb7, 0xfa, 0xd6, 0x26, 0x7a, 0x34, 0xb4, 0xb5, 0x49, 0x1e, - 0xc2, 0x64, 0xca, 0xe5, 0x00, 0xbf, 0x0c, 0xf2, 0xbb, 0x93, 0xea, 0xa2, 0x8a, 0x1e, 0xb9, 0x03, - 0xc6, 0x9e, 0x1d, 0x84, 0x1b, 0xc7, 0xc7, 0xac, 0x15, 0xb2, 0xb6, 0xf8, 0x06, 0x47, 0x07, 0x28, - 0x27, 0xb7, 0xfe, 0x51, 0xd2, 0x97, 0x43, 0x8b, 0x26, 0x80, 0x4e, 0x0e, 0xc5, 0x4e, 0xae, 0xc2, - 0x44, 0x83, 0x85, 0x9f, 0xd9, 0x7e, 0xe4, 0xe3, 0xb0, 0x20, 0x93, 0x69, 0x51, 0x6e, 0x1a, 0x23, - 0xff, 0xc3, 0x34, 0x46, 0x0b, 0xa6, 0xf1, 0x7f, 0x05, 0x39, 0xa0, 0x01, 0x01, 0xe9, 0x23, 0x58, - 0xb9, 0xa0, 0x1e, 0x9b, 0x8e, 0x81, 0x25, 0x35, 0x06, 0x5a, 0xb0, 0x7a, 0x51, 0xe2, 0xc7, 0xba, - 0x2d, 0xca, 0xe2, 0x9a, 0x82, 0x2a, 0x5f, 0xc3, 0xfa, 0x13, 0xf9, 0xa2, 0xeb, 0x4f, 0xf0, 0x37, - 0x52, 0xba, 0x14, 0x4d, 0xc1, 0x6f, 0xa4, 0xde, 0xd5, 0x96, 0x5e, 0x0b, 0x37, 0xdd, 0xa1, 0x3e, - 0xa1, 0x53, 0xbc, 0x38, 0x7c, 0xe3, 0x6f, 0xf4, 0xc3, 0xd3, 0x46, 0xe8, 0x3b, 0x6e, 0x74, 0x33, - 0x99, 0xa4, 0x29, 0x89, 0xb5, 0xae, 0xa9, 0xd6, 0xf2, 0xef, 0xbe, 0x14, 0xc9, 0xdf, 0x92, 0xc9, - 0xb6, 0x75, 0x4f, 0x97, 0x02, 0x1a, 0x68, 0xf1, 0x81, 0xa6, 0x84, 0x4b, 0x6e, 0xc0, 0x94, 0x14, - 0x6d, 0x9e, 0x87, 0x2c, 0xc0, 0x65, 0x51, 0x85, 0xd6, 0x87, 0xba, 0xf4, 0xd0, 0x8f, 0xb4, 0x3d, - 0x2d, 0xac, 0xf5, 0x92, 0x75, 0x18, 0x89, 0xd9, 0xd4, 0x74, 0x7c, 0x01, 0xcf, 0x6a, 0x73, 0x15, - 0x2a, 0x14, 0x63, 0x3a, 0xd5, 0x70, 0xbe, 0x66, 0xf8, 0x59, 0x4b, 0x04, 0xd6, 0x59, 0x71, 0xce, - 0x49, 0xb5, 0x2c, 0x65, 0x2c, 0xf9, 0x4c, 0x44, 0xa3, 0x6e, 0xf7, 0xec, 0x96, 0x13, 0x9e, 0x23, - 0xb6, 0x2a, 0xe4, 0x6f, 0x77, 0x9f, 0x05, 0x81, 0x7d, 0x22, 0xa9, 0x9c, 0x6c, 0x5a, 0x07, 0x83, - 0xeb, 0xcd, 0x3f, 0x79, 0xa2, 0xd6, 0x9f, 0x2e, 0xc8, 0x6d, 0xfd, 0xca, 0xf3, 0x79, 0x5f, 0x5f, - 0xd0, 0x1e, 0x3c, 0xaa, 0xf5, 0xbb, 0x82, 0xf4, 0x58, 0xde, 0x9d, 0x92, 0xc6, 0x1d, 0x6b, 0x4d, - 0x57, 0xee, 0xe6, 0x4e, 0xa2, 0x04, 0x37, 0xb4, 0x6c, 0x5a, 0xeb, 0xda, 0x9c, 0xd9, 0x00, 0x83, - 0x3d, 0x5d, 0xf9, 0x9b, 0x9f, 0x71, 0xac, 0x93, 0xe2, 0x2f, 0x30, 0xb1, 0x24, 0x2a, 0x3f, 0x55, - 0x4c, 0x5c, 0x1b, 0xa3, 0xf8, 0x9c, 0x92, 0x58, 0x6f, 0x6b, 0xd3, 0x6b, 0xda, 0x6b, 0xf4, 0x9d, - 0xa2, 0x62, 0x79, 0xfe, 0x3e, 0x6f, 0xbd, 0x5b, 0x98, 0x67, 0xd3, 0x42, 0x77, 0x07, 0x94, 0xca, - 0xc9, 0x6d, 0x98, 0xc1, 0x1f, 0xee, 0xc6, 0xb9, 0x80, 0x28, 0xec, 0x66, 0xc5, 0xe4, 0x26, 0x4c, - 0x3f, 0xf7, 0x9d, 0x30, 0x81, 0x40, 0x8e, 0x9a, 0x91, 0x5a, 0xee, 0xa0, 0x2c, 0xdd, 0xaf, 0x30, - 0xde, 0x67, 0xfa, 0x94, 0x1d, 0xf9, 0x3d, 0x4c, 0xa6, 0xe5, 0x3f, 0xe2, 0x97, 0xcc, 0x8a, 0xfe, - 0x9d, 0xef, 0x47, 0x53, 0x05, 0x72, 0x52, 0xc1, 0x9f, 0x97, 0x1b, 0x97, 0xc8, 0x65, 0x98, 0xc9, - 0xd4, 0xac, 0x8d, 0x12, 0x31, 0x60, 0x32, 0x9d, 0x2a, 0x33, 0x86, 0xc8, 0x24, 0x94, 0x65, 0xd6, - 0xca, 0x18, 0x26, 0x53, 0x50, 0x89, 0xf3, 0x29, 0xc6, 0x08, 0x99, 0x81, 0x89, 0x54, 0x12, 0xc1, - 0x18, 0x25, 0xd3, 0x00, 0xc9, 0xd5, 0xd5, 0x18, 0xe3, 0x78, 0xe9, 0x3b, 0x9b, 0x31, 0xce, 0x35, - 0x92, 0xd2, 0xa8, 0x51, 0xe6, 0x88, 0x71, 0xc5, 0xd3, 0xa8, 0x90, 0x79, 0x5d, 0xcd, 0xd3, 0x00, - 0x2e, 0xcf, 0xd7, 0x1e, 0x8d, 0x09, 0x42, 0xb2, 0xd5, 0x47, 0x63, 0x92, 0x4c, 0xc4, 0xa5, 0x44, - 0x63, 0x8a, 0x2c, 0x16, 0x54, 0x09, 0x8d, 0x69, 0x32, 0x9b, 0x29, 0xf2, 0x19, 0x33, 0xa4, 0x9a, - 0xaf, 0xd9, 0x19, 0x46, 0x7a, 0x16, 0x9c, 0xb0, 0x1a, 0xb3, 0x28, 0x89, 0x29, 0xa2, 0x41, 0xf8, - 0x72, 0x66, 0xea, 0x57, 0xc6, 0x65, 0x2e, 0xcc, 0x30, 0x36, 0xa3, 0xca, 0x87, 0x55, 0x08, 0x87, - 0x31, 0x47, 0x96, 0x8b, 0x6b, 0x46, 0xc6, 0x3c, 0x5f, 0xa2, 0x38, 0x79, 0x67, 0x2c, 0xe0, 0x48, - 0xe9, 0x4f, 0xbe, 0x61, 0x72, 0x87, 0xd2, 0xdf, 0x69, 0x63, 0x51, 0xbc, 0x8a, 0x83, 0xfd, 0x8d, - 0x17, 0x87, 0xf4, 0xa0, 0xde, 0x30, 0x96, 0xb0, 0xfd, 0x68, 0x7f, 0x6f, 0x77, 0x7f, 0xb7, 0x69, - 0x5c, 0xe1, 0x53, 0xcd, 0x06, 0x5e, 0x63, 0x99, 0x2f, 0x97, 0x36, 0x1c, 0x1b, 0x57, 0x85, 0xdf, - 0xe9, 0xa0, 0x67, 0xd4, 0xc4, 0xfb, 0x3f, 0x88, 0x03, 0x93, 0xb1, 0xc2, 0x05, 0xa9, 0x50, 0x61, - 0xac, 0x72, 0x67, 0x33, 0x87, 0xdc, 0xb8, 0xc6, 0x5f, 0x66, 0xfe, 0x6c, 0x19, 0x16, 0x9f, 0x44, - 0x7a, 0xef, 0x1a, 0xd7, 0xc9, 0x15, 0x11, 0x23, 0x74, 0x09, 0x45, 0xe3, 0xc6, 0x9d, 0x77, 0xa0, - 0xaa, 0xfb, 0xb0, 0x10, 0xe0, 0x71, 0xae, 0xeb, 0x89, 0x5d, 0x5e, 0x86, 0x91, 0x2d, 0x27, 0x78, - 0x6d, 0x94, 0x36, 0xf7, 0xbe, 0xfd, 0xa1, 0x56, 0xfa, 0xee, 0x87, 0x5a, 0xe9, 0xdf, 0x3f, 0xd4, - 0x2e, 0x7d, 0xf3, 0xa6, 0x76, 0xe9, 0xaf, 0x6f, 0x6a, 0xa5, 0xef, 0xde, 0xd4, 0x2e, 0x7d, 0xff, - 0xa6, 0x76, 0xe9, 0xe5, 0x5a, 0xea, 0xff, 0x3a, 0xba, 0x76, 0xe8, 0x3b, 0x67, 0x9e, 0xef, 0x9c, - 0x38, 0xae, 0x6c, 0xb8, 0x6c, 0xbd, 0xf7, 0xfa, 0x64, 0xbd, 0x77, 0xb4, 0x2e, 0xbe, 0x68, 0x47, - 0x63, 0x22, 0x73, 0xf4, 0xde, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xd7, 0xce, 0xb4, 0xa1, 0x6b, - 0x32, 0x00, 0x00, + 0x77, 0x37, 0xf5, 0x49, 0x1e, 0x7d, 0x41, 0x6b, 0x4a, 0x82, 0x64, 0x99, 0x92, 0xe1, 0x2f, 0xc5, + 0xfe, 0x47, 0x72, 0xfd, 0x8f, 0xdd, 0x26, 0x69, 0x3b, 0x91, 0x28, 0x4b, 0xa1, 0x2d, 0x59, 0xf2, + 0x52, 0x8e, 0x1d, 0x67, 0xc6, 0x33, 0x10, 0xb9, 0x92, 0x50, 0x93, 0x00, 0x03, 0x80, 0x89, 0x94, + 0x99, 0x3e, 0x42, 0x67, 0xf2, 0x08, 0xed, 0x63, 0xf4, 0xbe, 0x17, 0xb9, 0xcc, 0x45, 0x2f, 0x32, + 0xd3, 0x99, 0x36, 0x13, 0x3f, 0x40, 0x5f, 0xa1, 0xb3, 0x8b, 0xb3, 0x00, 0x16, 0x58, 0x50, 0x49, + 0x9a, 0xdc, 0x70, 0xb0, 0x67, 0xcf, 0xf9, 0xed, 0xf7, 0xd9, 0xf3, 0xb1, 0x84, 0x89, 0xaf, 0xfb, + 0xcc, 0xbf, 0x58, 0xef, 0xf9, 0x5e, 0xe8, 0x91, 0x51, 0x51, 0x58, 0x9a, 0x0c, 0x42, 0x3b, 0xec, + 0x07, 0x11, 0x71, 0x09, 0x3a, 0x5e, 0xeb, 0x1d, 0x7e, 0x57, 0xc2, 0x73, 0x17, 0x3f, 0x67, 0x42, + 0xa7, 0xcb, 0x82, 0xd0, 0xee, 0xf6, 0x24, 0x81, 0x4b, 0x05, 0x8e, 0x7b, 0xe2, 0x21, 0xe1, 0xc3, + 0x53, 0x27, 0x3c, 0xeb, 0x1f, 0xaf, 0xb7, 0xbc, 0xee, 0xc6, 0xa9, 0x77, 0xea, 0x6d, 0x08, 0xf2, + 0x71, 0xff, 0x44, 0x94, 0x44, 0x41, 0x7c, 0x21, 0xfb, 0xca, 0xa9, 0xe7, 0x9d, 0x76, 0x58, 0xc2, + 0x95, 0x69, 0xc0, 0xba, 0x05, 0x93, 0x2f, 0x78, 0xff, 0x28, 0xfb, 0xba, 0xcf, 0x82, 0x90, 0x54, + 0x61, 0x54, 0x94, 0xcd, 0xd2, 0x6a, 0x69, 0xad, 0x42, 0xa3, 0x82, 0xf5, 0x1c, 0xe6, 0x9b, 0x67, + 0xde, 0xb7, 0x87, 0xbe, 0xd7, 0x62, 0x41, 0xb0, 0xe7, 0x04, 0xa1, 0xe4, 0x9f, 0x87, 0xb1, 0x23, + 0xe6, 0xda, 0x6e, 0x88, 0x02, 0x58, 0x22, 0xcb, 0x50, 0x69, 0x5e, 0x04, 0x58, 0x35, 0xb4, 0x5a, + 0x5a, 0x2b, 0xd3, 0x84, 0x60, 0xbd, 0x82, 0xd9, 0xe6, 0x85, 0xdb, 0xaa, 0x7b, 0xdd, 0xae, 0x13, + 0x43, 0x6d, 0xc1, 0xf4, 0x9e, 0x1d, 0xb2, 0x20, 0x8c, 0xc8, 0x47, 0x4d, 0x01, 0x39, 0xf1, 0xb0, + 0xba, 0x9e, 0x74, 0xfa, 0x48, 0x7e, 0x6d, 0x8d, 0xfc, 0xf0, 0xdf, 0x2b, 0x57, 0x68, 0x46, 0xc2, + 0x7a, 0x03, 0x24, 0x0d, 0x1c, 0xf4, 0x3c, 0x37, 0x60, 0x64, 0x1b, 0x66, 0xea, 0x7d, 0xdf, 0x67, + 0xee, 0x6f, 0x81, 0xce, 0x8a, 0x58, 0x04, 0x8c, 0x5d, 0x16, 0x2a, 0x7d, 0xb6, 0xbe, 0x84, 0xd9, + 0x14, 0xed, 0x0f, 0x6d, 0x6e, 0x03, 0xe6, 0xea, 0x9e, 0xcf, 0xb6, 0xfb, 0xdd, 0x5e, 0xdd, 0x73, + 0x4f, 0x9c, 0xd3, 0xd4, 0x94, 0x6f, 0xb6, 0x42, 0xc7, 0x73, 0xe5, 0x94, 0x47, 0x25, 0xcb, 0x84, + 0xf9, 0xac, 0x40, 0xd4, 0x21, 0xeb, 0x1a, 0x2c, 0xee, 0xb2, 0xf0, 0x90, 0x2f, 0x78, 0xcb, 0xeb, + 0x7c, 0xc1, 0xfc, 0xc0, 0xf1, 0x5c, 0x39, 0x84, 0xc7, 0xb0, 0xa4, 0xab, 0xc4, 0xb1, 0x98, 0x30, + 0x8e, 0x24, 0xd1, 0xda, 0x30, 0x95, 0x45, 0xeb, 0x11, 0x2c, 0x36, 0x8b, 0x40, 0x07, 0x88, 0x3d, + 0x86, 0xa5, 0xe6, 0xef, 0x69, 0xee, 0x2f, 0x30, 0x4d, 0xfb, 0xee, 0x91, 0x1d, 0xbc, 0x93, 0x6d, + 0x2c, 0x41, 0x99, 0x17, 0xeb, 0x5e, 0x9b, 0x09, 0xe6, 0x51, 0x1a, 0x97, 0xad, 0x0f, 0x60, 0x26, + 0xe6, 0x46, 0xe8, 0x79, 0x18, 0xa3, 0x2c, 0xe8, 0x77, 0xe2, 0x9d, 0x1a, 0x95, 0xf8, 0xb4, 0xf1, + 0xf1, 0x3b, 0x3d, 0xd6, 0x71, 0x5c, 0xd6, 0x70, 0x4f, 0x3c, 0x39, 0x33, 0x1b, 0xb0, 0x90, 0xab, + 0x41, 0xb0, 0x2a, 0x8c, 0xd6, 0xbd, 0x3e, 0xee, 0xfa, 0x61, 0x1a, 0x15, 0xac, 0xff, 0x5a, 0x80, + 0x71, 0xd9, 0xbb, 0x65, 0xa8, 0xe0, 0x67, 0x63, 0x5b, 0x70, 0x8d, 0xd0, 0x84, 0x40, 0xd6, 0xa1, + 0x52, 0xef, 0xb6, 0xf7, 0x59, 0x78, 0xe6, 0xb5, 0xc5, 0xf1, 0x98, 0x7e, 0x68, 0xac, 0x47, 0x5a, + 0x23, 0xa6, 0xd3, 0x84, 0x85, 0xfc, 0xad, 0x7a, 0x4c, 0xcd, 0x61, 0xb1, 0x9f, 0xae, 0xa2, 0x48, + 0xba, 0x8a, 0xaa, 0xe7, 0xf9, 0x65, 0xd1, 0xc9, 0x35, 0x47, 0x04, 0xc4, 0x75, 0x84, 0xd0, 0x33, + 0xd1, 0xa2, 0x63, 0xbf, 0x07, 0x57, 0x37, 0x3b, 0x21, 0xf3, 0x37, 0x5b, 0x2d, 0x3e, 0x72, 0x89, + 0x39, 0x2a, 0x30, 0x97, 0x10, 0x53, 0xc3, 0x41, 0x75, 0x62, 0xe4, 0x33, 0x98, 0x79, 0xe6, 0x74, + 0x3a, 0x75, 0xcf, 0x95, 0x1b, 0xc8, 0x1c, 0x13, 0x48, 0xf3, 0x88, 0x94, 0xa9, 0xa5, 0x59, 0x76, + 0x52, 0x07, 0xe3, 0xc8, 0xb7, 0x5b, 0xac, 0xd9, 0xb3, 0x63, 0x88, 0x71, 0x01, 0xb1, 0x80, 0x10, + 0xd9, 0x6a, 0x9a, 0x13, 0x20, 0x0d, 0x20, 0xbb, 0x2c, 0xdc, 0xf3, 0x5a, 0xef, 0x52, 0xbb, 0xc0, + 0x2c, 0x0b, 0x98, 0x45, 0x84, 0xc9, 0x33, 0x50, 0x8d, 0x10, 0xd9, 0x11, 0x7a, 0xe1, 0xe8, 0xdc, + 0x4d, 0x23, 0x55, 0x04, 0x92, 0x99, 0x20, 0xa9, 0xf5, 0x34, 0x2f, 0xc2, 0xe7, 0x99, 0xeb, 0x17, + 0xbb, 0x75, 0x96, 0xde, 0x99, 0x26, 0x28, 0xf3, 0xac, 0xe1, 0xa0, 0x3a, 0x31, 0xf2, 0x77, 0x00, + 0xcd, 0x8b, 0x96, 0x1b, 0xa9, 0x18, 0x73, 0x42, 0xe9, 0x4e, 0x4e, 0x1f, 0xd3, 0x14, 0x2f, 0x79, + 0x04, 0x95, 0x58, 0xcf, 0x99, 0x93, 0xca, 0xc4, 0x66, 0x75, 0x22, 0x4d, 0x38, 0xc9, 0xa1, 0x98, + 0xd1, 0xcc, 0x61, 0x37, 0xa7, 0x84, 0xfc, 0x6a, 0x22, 0xaf, 0x57, 0x22, 0x54, 0x23, 0xcb, 0x11, + 0xf3, 0xea, 0xc3, 0x9c, 0x56, 0x10, 0x9b, 0xc5, 0x88, 0xf9, 0x2a, 0xb2, 0x0d, 0xd3, 0xaa, 0xda, + 0x34, 0x67, 0x04, 0xda, 0xb2, 0x3c, 0x8f, 0x3a, 0x25, 0x4c, 0x33, 0x32, 0x64, 0x03, 0xc6, 0x51, + 0xe1, 0x98, 0x86, 0x10, 0x9f, 0x43, 0x71, 0x55, 0x69, 0x51, 0xc9, 0x45, 0xbe, 0x84, 0x39, 0xca, + 0xba, 0xde, 0x37, 0x8c, 0xff, 0x86, 0x8c, 0x6f, 0xa0, 0x23, 0xfb, 0xb8, 0xc3, 0xcc, 0x59, 0x21, + 0x7e, 0x53, 0x8a, 0xeb, 0x78, 0x24, 0x98, 0x1e, 0x81, 0x6c, 0xc2, 0x14, 0xdf, 0x92, 0xe2, 0x66, + 0xdc, 0x72, 0xdc, 0xb6, 0x49, 0x04, 0xe4, 0xb5, 0xd4, 0x16, 0x8e, 0xeb, 0x24, 0x94, 0x2a, 0x41, + 0x9e, 0x82, 0xf1, 0xd2, 0x0d, 0xfa, 0xc7, 0x41, 0xcb, 0x77, 0x8e, 0x59, 0xd4, 0xb1, 0xab, 0x02, + 0xa5, 0x86, 0x28, 0xd9, 0xea, 0xf8, 0x58, 0x65, 0x2b, 0xd2, 0x7b, 0x78, 0xdb, 0x0e, 0x6d, 0xb9, + 0x87, 0xab, 0xda, 0x3d, 0x9c, 0xe2, 0xa0, 0x3a, 0x31, 0x44, 0x6b, 0x72, 0xb3, 0x28, 0x7d, 0x22, + 0xe6, 0xb2, 0x68, 0x59, 0x0e, 0xaa, 0x13, 0xe3, 0xea, 0x51, 0xaf, 0xfc, 0xcd, 0x79, 0x45, 0x3d, + 0xea, 0x99, 0x68, 0x81, 0x30, 0x87, 0xdd, 0x77, 0x4e, 0x7d, 0x3b, 0x64, 0x5c, 0x49, 0xed, 0xf8, + 0x5e, 0x57, 0xc2, 0x2e, 0x28, 0xb0, 0x7a, 0x26, 0x5a, 0x20, 0x4c, 0x0e, 0xa0, 0x9a, 0xaa, 0x39, + 0x8a, 0xfb, 0x6a, 0x2a, 0xeb, 0xab, 0x63, 0xa1, 0x5a, 0x41, 0x72, 0x0c, 0x26, 0x65, 0x1d, 0xcf, + 0x6e, 0x6f, 0xf6, 0x43, 0xaf, 0xe1, 0xb6, 0x7c, 0xd6, 0xe5, 0x26, 0x08, 0x9f, 0x73, 0x73, 0x51, + 0x80, 0xde, 0x89, 0xf7, 0xa1, 0x9e, 0x4d, 0xe2, 0x17, 0xe2, 0x70, 0xd5, 0x5c, 0x0f, 0x3b, 0x94, + 0xd9, 0x6d, 0xe6, 0xcb, 0x0e, 0x2f, 0x29, 0x1a, 0x24, 0x5b, 0x4d, 0x73, 0x02, 0x64, 0x1f, 0x66, + 0x76, 0x59, 0x48, 0x59, 0xaf, 0xe3, 0xb4, 0xec, 0xe8, 0xe6, 0xbd, 0x96, 0x5d, 0xa0, 0x74, 0x2d, + 0xca, 0x49, 0xdb, 0x2a, 0x53, 0xcb, 0x37, 0x11, 0x65, 0x01, 0x0b, 0x9b, 0x2c, 0x48, 0xa9, 0x07, + 0x73, 0x59, 0xd9, 0x44, 0x1a, 0x0e, 0xaa, 0x13, 0x23, 0x7b, 0x30, 0xbb, 0xeb, 0xed, 0xdb, 0xe7, + 0xfc, 0x9e, 0x0c, 0x24, 0xd6, 0x75, 0x55, 0xd9, 0x67, 0xeb, 0xb1, 0x67, 0x79, 0x41, 0x44, 0x63, + 0xdd, 0x3d, 0x27, 0xd1, 0xa9, 0x66, 0x2d, 0x8b, 0xa6, 0xd6, 0xa7, 0xd0, 0xd4, 0x0a, 0xf2, 0x16, + 0x16, 0x76, 0x9c, 0x0e, 0x6b, 0x32, 0xff, 0x1b, 0xa7, 0xc5, 0xd2, 0x4b, 0x66, 0xae, 0x28, 0xe7, + 0xb9, 0x80, 0x0b, 0x91, 0x8b, 0x40, 0x48, 0x17, 0x96, 0xb3, 0x55, 0x4f, 0xbe, 0x71, 0x5a, 0x71, + 0xc7, 0x57, 0x15, 0x6d, 0x36, 0x88, 0x15, 0x5b, 0x1a, 0x08, 0x47, 0x5e, 0x42, 0x75, 0x9f, 0x85, + 0x76, 0xdb, 0x0e, 0x6d, 0x65, 0x2c, 0x37, 0xd4, 0x13, 0xa0, 0x61, 0x41, 0x78, 0xad, 0x38, 0x39, + 0x00, 0xb2, 0xeb, 0xed, 0xd6, 0x0f, 0x99, 0xdf, 0x62, 0x89, 0x35, 0x63, 0xa9, 0x37, 0x7f, 0x8e, + 0x01, 0x21, 0x35, 0xa2, 0xdc, 0x94, 0xd8, 0xb1, 0xfb, 0x9d, 0xb0, 0xe1, 0xfe, 0x13, 0x4b, 0x26, + 0xe3, 0xa6, 0x02, 0x98, 0x67, 0xa0, 0x1a, 0x21, 0xf2, 0x15, 0xcc, 0xd7, 0xc3, 0xce, 0xbe, 0x27, + 0x94, 0xa9, 0x50, 0x60, 0x12, 0xee, 0x96, 0x72, 0x02, 0xf4, 0x4c, 0xd8, 0xc7, 0x02, 0x08, 0xf2, + 0x16, 0x16, 0x5f, 0x79, 0xfe, 0xbb, 0xa0, 0x67, 0xb7, 0xd8, 0xd1, 0x99, 0xcf, 0x82, 0x33, 0xaf, + 0x23, 0xef, 0x04, 0xf3, 0xb6, 0x72, 0xab, 0x16, 0xf2, 0xd1, 0x62, 0x08, 0xd2, 0x85, 0x5a, 0x3d, + 0xec, 0x1c, 0xfa, 0xec, 0x84, 0x85, 0xad, 0xb3, 0x03, 0xb7, 0x29, 0xaf, 0x86, 0xb8, 0x91, 0x3b, + 0xa2, 0x91, 0xdb, 0xc9, 0x20, 0x06, 0x30, 0xd3, 0x4b, 0xc0, 0xc8, 0x57, 0x60, 0x36, 0x9a, 0xf5, + 0xc3, 0x6d, 0xdf, 0x76, 0xdc, 0xba, 0xe7, 0x06, 0xfd, 0x6e, 0xa2, 0x73, 0xee, 0x8a, 0x86, 0x56, + 0xb0, 0xa1, 0x22, 0x36, 0x5a, 0x08, 0x60, 0xed, 0xc0, 0x42, 0xce, 0x1a, 0x46, 0x77, 0xe0, 0x3e, + 0x94, 0x51, 0x27, 0x04, 0x66, 0x69, 0x75, 0x78, 0x6d, 0xe2, 0xe1, 0xcc, 0x3a, 0xfa, 0xfb, 0x52, + 0x57, 0xc4, 0x0c, 0xd6, 0xcf, 0x26, 0x94, 0x63, 0xc9, 0x3f, 0xd6, 0x4d, 0xa8, 0xc2, 0xe8, 0x13, + 0xdf, 0xf7, 0x7c, 0xe1, 0x1f, 0x4c, 0xd2, 0xa8, 0x40, 0x5e, 0x17, 0x76, 0x1c, 0x9d, 0x80, 0x5a, + 0x91, 0x13, 0x10, 0x71, 0xd1, 0xc2, 0x71, 0x1f, 0x40, 0x55, 0xb5, 0xe7, 0x11, 0x76, 0x54, 0x39, + 0x8e, 0x3a, 0x16, 0xaa, 0x15, 0xe4, 0x97, 0x45, 0x62, 0xda, 0x23, 0xd8, 0x98, 0x72, 0x59, 0x64, + 0xab, 0x69, 0x4e, 0x80, 0x1b, 0xdf, 0x29, 0xdb, 0x1e, 0x51, 0xc6, 0x15, 0x0d, 0x9a, 0xab, 0xa7, + 0x79, 0x11, 0x34, 0x35, 0x12, 0xd3, 0x1e, 0x91, 0xca, 0x59, 0x53, 0x23, 0xcb, 0x41, 0x75, 0x62, + 0xe8, 0x5d, 0xc4, 0xf6, 0x3d, 0x82, 0x55, 0xb2, 0xde, 0x45, 0x86, 0x81, 0x6a, 0x84, 0xf8, 0xb4, + 0xab, 0xe6, 0x3d, 0x82, 0x41, 0xd6, 0xce, 0xcb, 0xb1, 0x50, 0xad, 0x20, 0xf9, 0x98, 0x3b, 0x06, + 0xd2, 0xfe, 0x47, 0xc7, 0x60, 0x51, 0xe3, 0x18, 0x20, 0x48, 0x8a, 0x99, 0x3c, 0xce, 0x7b, 0x06, + 0x66, 0xde, 0x33, 0x40, 0xc1, 0x94, 0x6b, 0xf0, 0x62, 0x80, 0x6b, 0x70, 0x63, 0x80, 0x6b, 0x90, + 0x9a, 0x96, 0xac, 0x25, 0xff, 0x62, 0x80, 0x6f, 0x70, 0x63, 0x80, 0x6f, 0x20, 0x21, 0x35, 0xce, + 0xc1, 0x93, 0x02, 0xe7, 0xe0, 0x7a, 0x81, 0x73, 0x80, 0x50, 0x59, 0xef, 0xe0, 0x41, 0xd6, 0x3b, + 0x98, 0xcf, 0x7a, 0x07, 0x28, 0x18, 0xbb, 0x07, 0x6f, 0x06, 0xbb, 0x07, 0xb7, 0x06, 0xbb, 0x07, + 0x88, 0x56, 0xe0, 0x1f, 0x6c, 0xe9, 0xfd, 0x83, 0x65, 0xbd, 0x7f, 0x80, 0x58, 0x19, 0x07, 0xe1, + 0x59, 0xa1, 0x83, 0xb0, 0x52, 0xe8, 0x20, 0xc8, 0x03, 0x9b, 0xf3, 0x10, 0x52, 0xfb, 0x39, 0x32, + 0xf5, 0x71, 0x3f, 0x57, 0xb5, 0xfb, 0x39, 0xcd, 0x42, 0xb5, 0x82, 0x08, 0x98, 0xb2, 0xf6, 0x11, + 0x70, 0x2e, 0x0b, 0x98, 0x63, 0xa1, 0x5a, 0x41, 0xae, 0x42, 0x0b, 0x42, 0x41, 0xe8, 0x28, 0xd4, + 0x8a, 0x1c, 0x05, 0xa9, 0x42, 0x8b, 0x22, 0x49, 0xaf, 0x61, 0x21, 0x67, 0xed, 0x23, 0xf2, 0x82, + 0x82, 0x5c, 0xc0, 0x45, 0x8b, 0xc4, 0x09, 0x85, 0xb9, 0x8c, 0xd1, 0x8f, 0xb8, 0xa6, 0xb2, 0xdc, + 0x5a, 0x1e, 0xaa, 0x17, 0x25, 0xad, 0x4b, 0x1d, 0x86, 0xbb, 0x97, 0x3a, 0x0c, 0xd8, 0x42, 0xb1, + 0xc7, 0xb0, 0x03, 0xb3, 0x29, 0x07, 0x00, 0x3b, 0xbd, 0xa4, 0xa8, 0x96, 0x5c, 0x3d, 0xcd, 0x8b, + 0x90, 0xe7, 0x45, 0x4e, 0x43, 0xad, 0xc8, 0x69, 0x88, 0x04, 0x8b, 0xbc, 0x86, 0x03, 0xa8, 0xaa, + 0xe6, 0x3f, 0x76, 0x6d, 0x59, 0xd9, 0x55, 0x3a, 0x16, 0xaa, 0x15, 0x8c, 0xcc, 0xce, 0xc4, 0xfe, + 0x47, 0xb8, 0xeb, 0x19, 0xb3, 0x33, 0xcb, 0x90, 0x98, 0x9d, 0xd9, 0x1a, 0x04, 0x8c, 0x5d, 0x00, + 0x04, 0xac, 0x65, 0x01, 0x33, 0x0c, 0x29, 0xc0, 0x4c, 0x0d, 0xb1, 0xc1, 0xcc, 0x5b, 0xfe, 0x08, + 0xbb, 0xa2, 0x1c, 0xf7, 0x22, 0x36, 0x04, 0x2f, 0x84, 0x21, 0x3d, 0xb8, 0x5e, 0x60, 0xf2, 0x63, + 0x3b, 0xab, 0x8a, 0xc6, 0x1b, 0xc8, 0x8b, 0x8d, 0x0d, 0x06, 0x24, 0xaf, 0x61, 0x2e, 0xe3, 0x05, + 0x60, 0x4b, 0x37, 0xd4, 0x83, 0xa1, 0xe3, 0xc1, 0x16, 0xf4, 0x00, 0x84, 0xc2, 0x55, 0xc5, 0x19, + 0x40, 0x5c, 0x4b, 0xb5, 0x18, 0xf2, 0x1c, 0x88, 0xaa, 0x13, 0xe6, 0x56, 0x88, 0xe2, 0x15, 0x20, + 0xe6, 0x4d, 0x05, 0x53, 0xc3, 0x41, 0x75, 0x62, 0xdc, 0x1f, 0xcc, 0xb9, 0x02, 0x88, 0x78, 0x4b, + 0x39, 0x1b, 0x05, 0x5c, 0xd2, 0x1f, 0x2c, 0xa8, 0x26, 0x36, 0x2c, 0xe9, 0xbc, 0x01, 0x6c, 0xe2, + 0xb6, 0x72, 0x17, 0x17, 0x33, 0xd2, 0x01, 0x20, 0x51, 0x14, 0xc4, 0x8d, 0xf3, 0x27, 0x31, 0xf8, + 0x9d, 0x4c, 0x14, 0x24, 0xcf, 0x42, 0xb5, 0x82, 0xa4, 0x07, 0x2b, 0x85, 0x7e, 0x05, 0x62, 0xdf, + 0x55, 0x82, 0x21, 0x97, 0x70, 0xd3, 0xcb, 0xe0, 0xb8, 0xdb, 0xa5, 0x71, 0x33, 0xb0, 0xad, 0x35, + 0xc5, 0xed, 0x2a, 0xe4, 0xa3, 0xc5, 0x10, 0x56, 0x43, 0x1b, 0x9e, 0x17, 0x19, 0x13, 0x91, 0x80, + 0x6b, 0xb4, 0x31, 0x71, 0x11, 0x97, 0xc9, 0x3c, 0x8c, 0x35, 0x85, 0xc7, 0x22, 0x7c, 0x87, 0x0a, + 0xc5, 0x92, 0xf5, 0x89, 0xde, 0xc4, 0x27, 0x16, 0x4c, 0xda, 0x9c, 0xde, 0xec, 0xb7, 0xb8, 0x5b, + 0x20, 0xf0, 0xca, 0x54, 0xa1, 0x59, 0x8d, 0x5c, 0x5c, 0x9f, 0xfb, 0x3b, 0x88, 0x84, 0xfe, 0xce, + 0x30, 0x4d, 0x08, 0xe9, 0xf4, 0xcf, 0x90, 0xf0, 0x85, 0x52, 0xe9, 0x9f, 0xbc, 0x9d, 0x6f, 0xc2, + 0xb8, 0xda, 0xba, 0x2c, 0x5a, 0x7b, 0xf9, 0x98, 0x13, 0x31, 0x60, 0xb8, 0xde, 0x6d, 0x63, 0xf2, + 0x87, 0x7f, 0x0a, 0xca, 0xc9, 0xa9, 0x68, 0x89, 0x53, 0x4e, 0x4e, 0x85, 0xff, 0x74, 0x1e, 0xfa, + 0x76, 0xec, 0x3f, 0xf1, 0x82, 0x75, 0x57, 0x73, 0x1f, 0x11, 0x02, 0x23, 0xfc, 0x1b, 0xf1, 0xc4, + 0xb7, 0xf5, 0xf7, 0x97, 0x79, 0xbb, 0x7c, 0x05, 0x0e, 0xed, 0x30, 0x64, 0x3e, 0x3a, 0x8a, 0x15, + 0x1a, 0x97, 0xad, 0x47, 0x97, 0x6e, 0x43, 0x6d, 0xa3, 0xff, 0x59, 0x2a, 0x76, 0x7a, 0xf3, 0xd3, + 0x3d, 0x95, 0x99, 0x6e, 0x71, 0x84, 0x1b, 0xdb, 0x72, 0xba, 0xb1, 0xc8, 0x6b, 0x9e, 0x7a, 0xc7, + 0xcf, 0xed, 0x2e, 0xc3, 0xed, 0x20, 0x8b, 0x7c, 0x8a, 0x9e, 0x7a, 0xc7, 0x8d, 0x6d, 0xe1, 0x3a, + 0x8e, 0xd0, 0xa8, 0x40, 0xd6, 0x60, 0x26, 0xb2, 0x35, 0x77, 0x98, 0xdb, 0x62, 0x07, 0x6e, 0xe7, + 0x42, 0xf8, 0x80, 0x65, 0x9a, 0x25, 0x93, 0x3b, 0x30, 0x4d, 0x99, 0xcb, 0xbe, 0x4d, 0x18, 0xc7, + 0x04, 0x63, 0x86, 0x6a, 0x3d, 0x1a, 0x70, 0x44, 0x06, 0xac, 0xfc, 0xeb, 0x7c, 0x22, 0x48, 0xb3, + 0xf2, 0x55, 0x18, 0xe5, 0x0c, 0x01, 0xae, 0x7d, 0x54, 0xe0, 0x93, 0x15, 0x6b, 0x1b, 0x31, 0xec, + 0x61, 0x9a, 0x10, 0xf8, 0x2e, 0xc8, 0xbb, 0x88, 0xba, 0x05, 0xa9, 0xea, 0xd2, 0x48, 0xd6, 0x4f, + 0x25, 0x28, 0x4b, 0x5a, 0x32, 0xf1, 0x6d, 0xf4, 0xf9, 0x65, 0x91, 0x03, 0x3e, 0x63, 0x17, 0xbc, + 0x63, 0xc3, 0x6b, 0x93, 0x54, 0x7c, 0x93, 0x7b, 0x91, 0xe4, 0xbe, 0xd7, 0x8e, 0x56, 0x63, 0xfa, + 0xe1, 0xf4, 0xba, 0x78, 0x3f, 0x20, 0xa9, 0x34, 0xae, 0x27, 0xab, 0x30, 0xe1, 0x04, 0xd4, 0x76, + 0x4f, 0x85, 0xbd, 0x2f, 0x16, 0xa9, 0x4c, 0xd3, 0x24, 0x72, 0x17, 0xc6, 0x3f, 0xf7, 0x3a, 0x6d, + 0xe6, 0x07, 0xe6, 0xa8, 0x08, 0x55, 0x4c, 0x45, 0x60, 0xaf, 0x6c, 0x87, 0x3b, 0x9a, 0x54, 0xd6, + 0x72, 0x46, 0x4e, 0xe3, 0x8c, 0x63, 0x5a, 0x46, 0xac, 0xb5, 0xde, 0x6a, 0xfd, 0x64, 0x3e, 0x94, + 0xba, 0xdb, 0x90, 0xf3, 0x2e, 0xbe, 0xc9, 0x5f, 0x61, 0x52, 0xf2, 0xed, 0x39, 0x41, 0x28, 0x86, + 0x39, 0xf1, 0x70, 0x06, 0x75, 0x5d, 0x0c, 0xa1, 0x30, 0x59, 0x57, 0x35, 0xc9, 0x34, 0xeb, 0x0c, + 0x26, 0x8e, 0xce, 0xdd, 0x5f, 0x37, 0xa3, 0xd4, 0xfb, 0x36, 0x9e, 0x51, 0xfe, 0x4d, 0xee, 0xc3, + 0xf8, 0x41, 0x2f, 0x14, 0xe1, 0x9a, 0x28, 0x93, 0x3a, 0x9b, 0x4c, 0x28, 0x56, 0x50, 0xc9, 0x61, + 0xfd, 0x7b, 0x09, 0xc6, 0xb1, 0x71, 0xf2, 0x19, 0x94, 0xeb, 0x3e, 0xb3, 0x43, 0xb6, 0x19, 0x62, + 0x4e, 0x7f, 0x69, 0x3d, 0x7a, 0x62, 0xb1, 0x2e, 0x9f, 0x58, 0xa4, 0x32, 0xfb, 0x65, 0x7e, 0x57, + 0x7e, 0xff, 0x3f, 0x2b, 0x25, 0x1a, 0x4b, 0x91, 0x55, 0x18, 0xe1, 0xb6, 0x83, 0xd8, 0x79, 0x13, + 0x0f, 0x27, 0xd7, 0xc3, 0x73, 0x77, 0xfd, 0xe8, 0xdc, 0xe5, 0x34, 0x2a, 0x6a, 0xf8, 0x50, 0x5e, + 0x06, 0xcc, 0x3f, 0x3a, 0x77, 0x45, 0xe7, 0xca, 0x54, 0x16, 0xc9, 0x03, 0xa8, 0xf0, 0x39, 0xe7, + 0xbd, 0x0c, 0xcc, 0x11, 0x31, 0x75, 0x44, 0x06, 0x34, 0x92, 0xb9, 0xa0, 0x09, 0x93, 0xf5, 0x46, + 0x17, 0x74, 0xd0, 0xae, 0xcc, 0x03, 0x31, 0x9f, 0x99, 0x85, 0x99, 0x4e, 0xd0, 0x05, 0x40, 0x9a, + 0xc5, 0x9a, 0xd3, 0xe6, 0x26, 0xad, 0x7f, 0x2b, 0x41, 0x25, 0x26, 0x72, 0x85, 0xf7, 0xdc, 0x6b, + 0xb3, 0xa3, 0x8b, 0x1e, 0xc3, 0xe6, 0xe2, 0x32, 0xbf, 0x72, 0xf8, 0x77, 0xa3, 0x8d, 0xc7, 0x10, + 0x4b, 0xfc, 0x1c, 0x0a, 0x00, 0x21, 0x14, 0xa9, 0x9f, 0x84, 0xc0, 0x3b, 0xff, 0x32, 0x60, 0x6d, + 0xd4, 0x3f, 0xe2, 0x9b, 0xd3, 0x76, 0x7c, 0x16, 0xc5, 0x9d, 0x46, 0xa8, 0xf8, 0xe6, 0x2d, 0x7f, + 0xee, 0x84, 0xd4, 0x0e, 0x1d, 0x4f, 0xa8, 0x98, 0x21, 0x1a, 0x97, 0xad, 0xe7, 0xfa, 0x00, 0x0a, + 0x79, 0x0c, 0x53, 0x31, 0x51, 0x4c, 0x43, 0x14, 0xcc, 0x8b, 0x63, 0x6e, 0xb1, 0x80, 0xca, 0x66, + 0x75, 0x60, 0x79, 0x50, 0xa2, 0x8e, 0x2f, 0xe9, 0xae, 0xef, 0xf5, 0x7b, 0xb1, 0x12, 0x96, 0xc5, + 0xc1, 0x2a, 0x58, 0xde, 0x85, 0xc3, 0xea, 0x5d, 0xf8, 0x08, 0xae, 0x0f, 0xf4, 0xfb, 0xd5, 0xd7, + 0x09, 0xa3, 0xf2, 0x75, 0xc2, 0x53, 0x31, 0xe8, 0x5c, 0xea, 0xef, 0xf7, 0x74, 0xce, 0xba, 0x0f, + 0x73, 0xda, 0x30, 0x01, 0x5f, 0x09, 0x11, 0x52, 0xc0, 0xad, 0xc5, 0xbf, 0xad, 0x26, 0x2c, 0x14, + 0x64, 0x0b, 0x49, 0x0d, 0x80, 0x3b, 0xee, 0xc7, 0x76, 0xc0, 0xe2, 0xf8, 0x67, 0x8a, 0x32, 0xa0, + 0x07, 0x1f, 0x81, 0x59, 0x14, 0x61, 0x18, 0x70, 0x3d, 0xec, 0x40, 0x59, 0xac, 0xdc, 0x33, 0x76, + 0xc1, 0xbb, 0x7a, 0x68, 0x87, 0x67, 0xb2, 0xab, 0xfc, 0x9b, 0x6f, 0xc9, 0x83, 0x93, 0x93, 0x80, + 0x45, 0x6f, 0x96, 0x86, 0x29, 0x96, 0xc8, 0x34, 0x0c, 0x35, 0xbf, 0xc3, 0x3b, 0x61, 0xa8, 0xf9, + 0x9d, 0xf5, 0x18, 0xb7, 0xa8, 0xd0, 0xcf, 0x1f, 0xc0, 0xc8, 0x3b, 0xae, 0xb3, 0x4b, 0x8a, 0x32, + 0x93, 0xf5, 0x68, 0x31, 0x0b, 0x16, 0xeb, 0x88, 0xdf, 0x93, 0x62, 0xe8, 0x71, 0x37, 0xaa, 0x30, + 0xda, 0x70, 0xdb, 0xec, 0x5c, 0x2e, 0x96, 0x28, 0x90, 0xfb, 0x49, 0x47, 0x51, 0x55, 0x64, 0x71, + 0x69, 0xcc, 0x60, 0xbd, 0xd2, 0x66, 0x58, 0xc9, 0x67, 0xb9, 0xc6, 0xb0, 0x8b, 0x71, 0xf4, 0x49, + 0xad, 0xa5, 0x59, 0x76, 0xeb, 0x00, 0x66, 0xe5, 0xa4, 0xc6, 0xe8, 0x05, 0x1d, 0x36, 0x60, 0xf8, + 0x73, 0x47, 0x3e, 0xf5, 0xe2, 0x9f, 0x7c, 0x7e, 0x39, 0x3f, 0xda, 0x52, 0xe2, 0xdb, 0x7a, 0xab, + 0x8f, 0xf4, 0x70, 0x97, 0x3f, 0xd7, 0x10, 0x76, 0xd6, 0x4c, 0xfc, 0x6a, 0xb5, 0x9e, 0xe6, 0x45, + 0x2c, 0xaa, 0xcd, 0x0e, 0x93, 0x4f, 0x61, 0x32, 0xa6, 0x45, 0xd3, 0x10, 0x85, 0x94, 0x93, 0xd7, + 0x75, 0xe9, 0x6a, 0xaa, 0x30, 0xe3, 0xb9, 0xc9, 0xc7, 0x84, 0x1e, 0x42, 0x25, 0x26, 0xc6, 0x0f, + 0xbc, 0x34, 0x88, 0x34, 0x61, 0xb3, 0x9a, 0x30, 0x71, 0xe8, 0xb3, 0x9e, 0xed, 0xb3, 0x66, 0xd8, + 0x15, 0x53, 0x24, 0x6c, 0x2c, 0xdc, 0x82, 0xc2, 0xc0, 0x32, 0x60, 0xb8, 0xf9, 0x62, 0x4f, 0x5a, + 0xa5, 0xcd, 0x17, 0x7b, 0xfc, 0x90, 0x1c, 0xda, 0xbe, 0xdd, 0xe5, 0xea, 0x2f, 0xc0, 0xe9, 0x4c, + 0x51, 0xac, 0x07, 0x45, 0xd9, 0x66, 0xbe, 0x9d, 0x39, 0x29, 0x3e, 0xd9, 0x58, 0xb2, 0xfe, 0xa5, + 0x54, 0x18, 0x75, 0xe2, 0x5b, 0x7d, 0x7b, 0x0b, 0x7b, 0x34, 0xb4, 0xbd, 0x45, 0x1e, 0xc3, 0x64, + 0xaa, 0xcb, 0x01, 0xde, 0x0c, 0xf2, 0xde, 0x49, 0x55, 0x51, 0x85, 0x8f, 0xdc, 0x03, 0x63, 0xcf, + 0x0e, 0xc2, 0xcd, 0x93, 0x13, 0xd6, 0x0a, 0x59, 0x5b, 0xdc, 0xc1, 0xd1, 0x01, 0xca, 0xd1, 0xad, + 0xff, 0x28, 0xe9, 0x33, 0xdb, 0x45, 0x03, 0xc0, 0x4e, 0x0e, 0xc5, 0x9d, 0x5c, 0x85, 0x89, 0x26, + 0x0b, 0xbf, 0xb0, 0xfd, 0xa8, 0x8f, 0xc3, 0xc2, 0xb4, 0x4e, 0x93, 0x72, 0xc3, 0x18, 0xf9, 0x7f, + 0x0c, 0x63, 0xb4, 0x60, 0x18, 0x7f, 0x53, 0x10, 0x71, 0x1b, 0xa0, 0x90, 0x3e, 0x85, 0x95, 0x4b, + 0x52, 0xeb, 0x69, 0x1d, 0x58, 0x52, 0x75, 0xa0, 0x05, 0xab, 0x97, 0x85, 0xd9, 0xac, 0x35, 0xf1, + 0xc2, 0x41, 0x93, 0x1b, 0xe7, 0x73, 0x58, 0x7f, 0x2e, 0x17, 0xba, 0xfe, 0x1c, 0x9f, 0xbb, 0xe9, + 0x02, 0x62, 0x05, 0xcf, 0xdd, 0x3e, 0xd4, 0x66, 0xd1, 0x0b, 0x37, 0xdd, 0xa1, 0x3e, 0x7c, 0x56, + 0x3c, 0x39, 0x7c, 0xe3, 0x6f, 0xf6, 0xc3, 0xb3, 0x66, 0xe8, 0x3b, 0x6e, 0xe4, 0xa7, 0x4d, 0xd2, + 0x14, 0xc5, 0xda, 0xd0, 0x24, 0xde, 0xf9, 0xbd, 0x2f, 0x49, 0xf2, 0x59, 0xa0, 0x2c, 0x5b, 0x0f, + 0x74, 0x01, 0xb7, 0x81, 0x12, 0x1f, 0x6b, 0xb2, 0xf1, 0xe4, 0x16, 0x4c, 0x49, 0xd2, 0xd6, 0x45, + 0xc8, 0x02, 0x9c, 0x16, 0x95, 0x68, 0x7d, 0xa2, 0x0b, 0xc6, 0xfd, 0x4a, 0xd9, 0xb3, 0xc2, 0xb4, + 0x3d, 0xd9, 0x80, 0x91, 0xd8, 0x9a, 0x9a, 0x8e, 0xc3, 0x1d, 0x59, 0x6e, 0xce, 0x42, 0x05, 0x63, + 0x6c, 0x4e, 0x35, 0x9d, 0xef, 0x18, 0x5e, 0x6b, 0x09, 0xc1, 0x3a, 0x2f, 0x8e, 0xf0, 0xa9, 0x92, + 0xa5, 0x8c, 0x24, 0x1f, 0x89, 0x28, 0xd4, 0xed, 0x9e, 0xdd, 0x72, 0xc2, 0x0b, 0xc4, 0x56, 0x89, + 0x7c, 0x75, 0xf7, 0x59, 0x10, 0xd8, 0xa7, 0xb1, 0x27, 0x89, 0x45, 0xeb, 0x60, 0xf0, 0xd3, 0x81, + 0xdf, 0x3c, 0x50, 0xeb, 0x9f, 0x2f, 0x89, 0x24, 0xfe, 0xc9, 0xe3, 0xf9, 0x48, 0xff, 0x36, 0x61, + 0x70, 0xab, 0xd6, 0x3f, 0x14, 0x04, 0x23, 0xf3, 0xdd, 0x29, 0x69, 0xba, 0x63, 0xad, 0xeb, 0x5e, + 0x2e, 0xf0, 0x4e, 0x22, 0x05, 0x37, 0xb4, 0x2c, 0x5a, 0x1b, 0xda, 0x08, 0xe5, 0x00, 0x81, 0x3d, + 0xdd, 0x4b, 0x06, 0x7e, 0xc6, 0x31, 0x2b, 0x8d, 0x8f, 0x69, 0x31, 0x01, 0x2d, 0xaf, 0x2a, 0x26, + 0xdc, 0xc6, 0x48, 0x3f, 0xa7, 0x28, 0xd6, 0x07, 0xda, 0x60, 0xa6, 0xd6, 0x8d, 0xbe, 0x57, 0xf4, + 0xee, 0x21, 0xef, 0xcf, 0x5b, 0x1f, 0x16, 0x46, 0x35, 0xb5, 0xd0, 0xdd, 0x01, 0xaf, 0x1e, 0xc8, + 0x1a, 0xcc, 0xe0, 0x1b, 0xec, 0x38, 0x16, 0x10, 0xa9, 0xdd, 0x2c, 0x99, 0xdc, 0x81, 0xe9, 0x57, + 0xbe, 0x13, 0x26, 0x10, 0x68, 0xa3, 0x66, 0xa8, 0x96, 0x3b, 0x28, 0x26, 0xfa, 0x27, 0xb4, 0xf7, + 0x85, 0x3e, 0x40, 0x4a, 0xfe, 0x11, 0x26, 0xd3, 0xf4, 0x5f, 0xf1, 0x28, 0x5d, 0xe1, 0xbf, 0xf7, + 0xbf, 0xa3, 0xa9, 0xe7, 0x08, 0xa4, 0x82, 0xff, 0x14, 0x30, 0xae, 0x90, 0xab, 0x30, 0x93, 0x79, + 0x21, 0x60, 0x94, 0x88, 0x01, 0x93, 0xe9, 0xc0, 0xa1, 0x31, 0x44, 0x26, 0xa1, 0x2c, 0x63, 0x78, + 0xc6, 0x30, 0x99, 0x82, 0x4a, 0x1c, 0x4f, 0x31, 0x46, 0xc8, 0x0c, 0x4c, 0xa4, 0x82, 0x08, 0xc6, + 0x28, 0x99, 0x06, 0x48, 0x5c, 0x57, 0x63, 0x8c, 0xe3, 0xa5, 0x7d, 0x36, 0x63, 0x9c, 0x73, 0x24, + 0x89, 0x68, 0xa3, 0xcc, 0x11, 0xe3, 0xfc, 0xb2, 0x51, 0x21, 0xf3, 0xba, 0x0c, 0xb3, 0x01, 0x9c, + 0x9e, 0xcf, 0xf4, 0x1a, 0x13, 0x84, 0x64, 0x73, 0xbd, 0xc6, 0x24, 0x99, 0x88, 0x13, 0xb7, 0xc6, + 0x14, 0x59, 0x2c, 0xc8, 0xc9, 0x1a, 0xd3, 0x64, 0x36, 0x93, 0x52, 0x35, 0x66, 0x48, 0x35, 0x9f, + 0x21, 0x35, 0x8c, 0xf4, 0x28, 0xb8, 0xc1, 0x6a, 0xcc, 0x22, 0x25, 0x36, 0x11, 0x0d, 0xc2, 0xa7, + 0x33, 0x93, 0x2d, 0x34, 0xae, 0x72, 0x62, 0xc6, 0x62, 0x33, 0xaa, 0xbc, 0x59, 0xc5, 0xe0, 0x30, + 0xe6, 0xc8, 0x72, 0x71, 0x86, 0xce, 0x98, 0xe7, 0x53, 0x14, 0x87, 0x32, 0x8d, 0x05, 0x6c, 0x29, + 0x7d, 0xe5, 0x1b, 0x26, 0xef, 0x50, 0xfa, 0x9e, 0x36, 0x16, 0xc5, 0x52, 0x1c, 0xec, 0x6f, 0xbe, + 0x3e, 0xa4, 0x07, 0xf5, 0xa6, 0xb1, 0x84, 0xe5, 0x27, 0xfb, 0x7b, 0x8d, 0xfd, 0xc6, 0x91, 0x71, + 0x8d, 0x0f, 0x35, 0xab, 0x78, 0x8d, 0x65, 0x3e, 0x5d, 0x5a, 0x75, 0x6c, 0x5c, 0x17, 0xfd, 0x4e, + 0x2b, 0x3d, 0xa3, 0x26, 0xd6, 0xff, 0x20, 0x56, 0x4c, 0xc6, 0x0a, 0x27, 0xa4, 0x54, 0x85, 0xb1, + 0xca, 0x3b, 0x9b, 0x39, 0xe4, 0xc6, 0x0d, 0xbe, 0x98, 0xf9, 0xb3, 0x65, 0x58, 0x7c, 0x10, 0xe9, + 0xbd, 0x6b, 0xdc, 0x24, 0xd7, 0x84, 0x8e, 0xd0, 0x85, 0x57, 0x8d, 0x5b, 0x64, 0x0e, 0x66, 0x73, + 0xd1, 0x46, 0xe3, 0xf6, 0xbd, 0xbf, 0x40, 0x55, 0x77, 0xdf, 0x10, 0xe0, 0xea, 0xaf, 0xeb, 0x89, + 0xcd, 0x5f, 0x86, 0x91, 0x6d, 0x27, 0x78, 0x67, 0x94, 0xb6, 0xf6, 0x7e, 0xf8, 0xa5, 0x56, 0xfa, + 0xf1, 0x97, 0x5a, 0xe9, 0xe7, 0x5f, 0x6a, 0x57, 0xbe, 0x7f, 0x5f, 0xbb, 0xf2, 0xaf, 0xef, 0x6b, + 0xa5, 0x1f, 0xdf, 0xd7, 0xae, 0xfc, 0xf4, 0xbe, 0x76, 0xe5, 0xcd, 0x7a, 0xea, 0x9f, 0x3b, 0x5d, + 0x3b, 0xf4, 0x9d, 0x73, 0xcf, 0x77, 0x4e, 0x1d, 0x57, 0x16, 0x5c, 0xb6, 0xd1, 0x7b, 0x77, 0xba, + 0xd1, 0x3b, 0xde, 0x10, 0x17, 0xdd, 0xf1, 0x98, 0x08, 0x28, 0xfd, 0xf5, 0xff, 0x02, 0x00, 0x00, + 0xff, 0xff, 0x38, 0x49, 0xc6, 0x6b, 0x4d, 0x34, 0x00, 0x00, } func (m *QueryRequest) Marshal() (dAtA []byte, err error) { @@ -5701,6 +5858,20 @@ func (m *Request) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ISCPDrainConsumerRequest != nil { + { + size, err := m.ISCPDrainConsumerRequest.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0xba + } if m.CtlPrefetchOnSubscribedRequest != nil { { size, err := m.CtlPrefetchOnSubscribedRequest.MarshalToSizedBuffer(dAtA[:i]) @@ -6233,6 +6404,20 @@ func (m *Response) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ISCPDrainConsumerResponse != nil { + { + size, err := m.ISCPDrainConsumerResponse.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0xc2 + } if m.CtlPrefetchOnSubscribedResponse != nil { { size, err := m.CtlPrefetchOnSubscribedResponse.MarshalToSizedBuffer(dAtA[:i]) @@ -6987,7 +7172,7 @@ func (m *CtlPrefetchOnSubscribedResponse) MarshalToSizedBuffer(dAtA []byte) (int return len(dAtA) - i, nil } -func (m *TraceSpanRequest) Marshal() (dAtA []byte, err error) { +func (m *ISCPDrainConsumerRequest) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -6997,36 +7182,134 @@ func (m *TraceSpanRequest) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *TraceSpanRequest) MarshalTo(dAtA []byte) (int, error) { +func (m *ISCPDrainConsumerRequest) MarshalTo(dAtA []byte) (int, error) { size := m.ProtoSize() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *TraceSpanRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *ISCPDrainConsumerRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if m.Threshold != 0 { - i = encodeVarintQuery(dAtA, i, uint64(m.Threshold)) + if m.RenewFenceOnly { i-- - dAtA[i] = 0x18 + if m.RenewFenceOnly { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x30 } - if len(m.Spans) > 0 { - i -= len(m.Spans) - copy(dAtA[i:], m.Spans) - i = encodeVarintQuery(dAtA, i, uint64(len(m.Spans))) + if m.RemoveFenceOnly { i-- - dAtA[i] = 0x12 + if m.RemoveFenceOnly { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x28 } - if len(m.Cmd) > 0 { - i -= len(m.Cmd) - copy(dAtA[i:], m.Cmd) - i = encodeVarintQuery(dAtA, i, uint64(len(m.Cmd))) + if m.JobID != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.JobID)) i-- - dAtA[i] = 0xa + dAtA[i] = 0x20 } - return len(dAtA) - i, nil + if len(m.JobName) > 0 { + i -= len(m.JobName) + copy(dAtA[i:], m.JobName) + i = encodeVarintQuery(dAtA, i, uint64(len(m.JobName))) + i-- + dAtA[i] = 0x1a + } + if m.TableID != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.TableID)) + i-- + dAtA[i] = 0x10 + } + if m.AccountID != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.AccountID)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *ISCPDrainConsumerResponse) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ISCPDrainConsumerResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ISCPDrainConsumerResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Success { + i-- + if m.Success { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *TraceSpanRequest) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TraceSpanRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *TraceSpanRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Threshold != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.Threshold)) + i-- + dAtA[i] = 0x18 + } + if len(m.Spans) > 0 { + i -= len(m.Spans) + copy(dAtA[i:], m.Spans) + i = encodeVarintQuery(dAtA, i, uint64(len(m.Spans))) + i-- + dAtA[i] = 0x12 + } + if len(m.Cmd) > 0 { + i -= len(m.Cmd) + copy(dAtA[i:], m.Cmd) + i = encodeVarintQuery(dAtA, i, uint64(len(m.Cmd))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *TraceSpanResponse) Marshal() (dAtA []byte, err error) { @@ -7334,12 +7617,12 @@ func (m *TxnInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x12 } - n78, err78 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.CreateAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.CreateAt):]) - if err78 != nil { - return 0, err78 + n80, err80 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.CreateAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.CreateAt):]) + if err80 != nil { + return 0, err80 } - i -= n78 - i = encodeVarintQuery(dAtA, i, uint64(n78)) + i -= n80 + i = encodeVarintQuery(dAtA, i, uint64(n80)) i-- dAtA[i] = 0xa return len(dAtA) - i, nil @@ -9330,6 +9613,10 @@ func (m *Request) ProtoSize() (n int) { l = m.CtlPrefetchOnSubscribedRequest.ProtoSize() n += 2 + l + sovQuery(uint64(l)) } + if m.ISCPDrainConsumerRequest != nil { + l = m.ISCPDrainConsumerRequest.ProtoSize() + n += 2 + l + sovQuery(uint64(l)) + } return n } @@ -9492,6 +9779,10 @@ func (m *Response) ProtoSize() (n int) { l = m.CtlPrefetchOnSubscribedResponse.ProtoSize() n += 2 + l + sovQuery(uint64(l)) } + if m.ISCPDrainConsumerResponse != nil { + l = m.ISCPDrainConsumerResponse.ProtoSize() + n += 2 + l + sovQuery(uint64(l)) + } return n } @@ -9612,6 +9903,46 @@ func (m *CtlPrefetchOnSubscribedResponse) ProtoSize() (n int) { return n } +func (m *ISCPDrainConsumerRequest) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.AccountID != 0 { + n += 1 + sovQuery(uint64(m.AccountID)) + } + if m.TableID != 0 { + n += 1 + sovQuery(uint64(m.TableID)) + } + l = len(m.JobName) + if l > 0 { + n += 1 + l + sovQuery(uint64(l)) + } + if m.JobID != 0 { + n += 1 + sovQuery(uint64(m.JobID)) + } + if m.RemoveFenceOnly { + n += 2 + } + if m.RenewFenceOnly { + n += 2 + } + return n +} + +func (m *ISCPDrainConsumerResponse) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Success { + n += 2 + } + return n +} + func (m *TraceSpanRequest) ProtoSize() (n int) { if m == nil { return 0 @@ -12964,6 +13295,42 @@ func (m *Request) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 39: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ISCPDrainConsumerRequest", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ISCPDrainConsumerRequest == nil { + m.ISCPDrainConsumerRequest = &ISCPDrainConsumerRequest{} + } + if err := m.ISCPDrainConsumerRequest.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) @@ -14442,6 +14809,42 @@ func (m *Response) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 40: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ISCPDrainConsumerResponse", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ISCPDrainConsumerResponse == nil { + m.ISCPDrainConsumerResponse = &ISCPDrainConsumerResponse{} + } + if err := m.ISCPDrainConsumerResponse.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) @@ -15186,6 +15589,255 @@ func (m *CtlPrefetchOnSubscribedResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *ISCPDrainConsumerRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ISCPDrainConsumerRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ISCPDrainConsumerRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountID", wireType) + } + m.AccountID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountID |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TableID", wireType) + } + m.TableID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TableID |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field JobName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.JobName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field JobID", wireType) + } + m.JobID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.JobID |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RemoveFenceOnly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RemoveFenceOnly = bool(v != 0) + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RenewFenceOnly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RenewFenceOnly = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipQuery(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthQuery + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ISCPDrainConsumerResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ISCPDrainConsumerResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ISCPDrainConsumerResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Success", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Success = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipQuery(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthQuery + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *TraceSpanRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/queryservice/client/query_client.go b/pkg/queryservice/client/query_client.go index 2d5ed07f81c1e..91da52bb9c916 100644 --- a/pkg/queryservice/client/query_client.go +++ b/pkg/queryservice/client/query_client.go @@ -60,6 +60,7 @@ var methodVersions = map[pb.CmdMethod]int64{ pb.CmdMethod_WorkspaceThreshold: defines.MORPCVersion4, pb.CmdMethod_MinTimestamp: defines.MORPCVersion4, pb.CmdMethod_CtlPrefetchOnSubscribed: defines.MORPCVersion4, + pb.CmdMethod_ISCPDrainConsumer: defines.MORPCVersion4, } type queryClient struct { diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index b72a2051793cd..f44a10bd67df7 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -728,6 +728,12 @@ func (s *Scope) AlterTableInplace(c *Compile) error { hasUpdateConstraints = true var notDroppedIndex []*plan.IndexDef var newIndexes []uint64 + if err = DropIndexCdcTask(c, oTableDef, dbName, tblName, constraintName); err != nil { + return err + } + if err = DrainIndexCdcTaskConsumer(c, oTableDef, dbName, tblName, constraintName); err != nil { + return err + } for idx, indexdef := range oTableDef.Indexes { if indexdef.IndexName == constraintName { dropIndexMap[indexdef.IndexName] = true @@ -2503,6 +2509,14 @@ func (s *Scope) DropIndex(c *Compile) error { if err != nil { return err } + err = DropIndexCdcTask(c, oldTableDef, qry.Database, qry.Table, qry.IndexName) + if err != nil { + return err + } + err = DrainIndexCdcTaskConsumer(c, oldTableDef, qry.Database, qry.Table, qry.IndexName) + if err != nil { + return err + } err = r.UpdateConstraint(c.proc.Ctx, newCt) if err != nil { return err @@ -2529,13 +2543,7 @@ func (s *Scope) DropIndex(c *Compile) error { } } - //3. delete iscp job for vector, fulltext index - err = DropIndexCdcTask(c, oldTableDef, qry.Database, qry.Table, qry.IndexName) - if err != nil { - return err - } - - // 4. unregister index update + // 3. unregister index update err = idxcron.UnregisterUpdate(c.proc.Ctx, c.proc.GetService(), c.proc.GetTxnOperator(), @@ -2546,7 +2554,7 @@ func (s *Scope) DropIndex(c *Compile) error { return err } - //5. delete index object from mo_catalog.mo_indexes + // 4. delete index object from mo_catalog.mo_indexes deleteSql := fmt.Sprintf(deleteMoIndexesWithTableIdAndIndexNameFormat, r.GetTableID(c.proc.Ctx), qry.IndexName) err = c.runSqlWithOptions( deleteSql, executor.StatementOption{}.WithDisableLog(), @@ -2573,7 +2581,12 @@ func makeNewDropConstraint(oldCt *engine.ConstraintDef, dropName string) (*engin case *engine.IndexDef: pred := func(index *plan.IndexDef) bool { if index.IndexName == dropName && len(index.IndexTableName) > 0 { - dropIndexTableNames = append(dropIndexTableNames, index.IndexTableName) + if catalog.ToLower(index.IndexAlgo) == "hnsw" && + catalog.ToLower(index.IndexAlgoTableType) == catalog.Hnsw_TblType_Storage { + dropIndexTableNames = append([]string{index.IndexTableName}, dropIndexTableNames...) + } else { + dropIndexTableNames = append(dropIndexTableNames, index.IndexTableName) + } } return index.IndexName == dropName } diff --git a/pkg/sql/compile/ddl_util_test.go b/pkg/sql/compile/ddl_util_test.go index 02e6f1ee1c310..c68832e5def93 100644 --- a/pkg/sql/compile/ddl_util_test.go +++ b/pkg/sql/compile/ddl_util_test.go @@ -20,15 +20,45 @@ import ( "strings" "testing" + "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/sqlquote" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/sql/parsers" "github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect" + "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestMakeNewDropConstraintDropsHnswStorageBeforeMetadata(t *testing.T) { + oldCt := &engine.ConstraintDef{ + Cts: []engine.Constraint{ + &engine.IndexDef{ + Indexes: []*plan.IndexDef{ + { + IndexName: "idx01", + IndexTableName: "__meta", + IndexAlgo: "hnsw", + IndexAlgoTableType: catalog.Hnsw_TblType_Metadata, + }, + { + IndexName: "idx01", + IndexTableName: "__storage", + IndexAlgo: "hnsw", + IndexAlgoTableType: catalog.Hnsw_TblType_Storage, + }, + }, + }, + }, + } + + _, dropIndexTableNames, err := makeNewDropConstraint(oldCt, "idx01") + + require.NoError(t, err) + require.Equal(t, []string{"__storage", "__meta"}, dropIndexTableNames) +} + // TestAlterIndexVisibleEscaping is a regression for the ALTER TABLE ... ALTER // INDEX ... VISIBLE/INVISIBLE catalog write (ddl.go, updateMoIndexesVisibleFormat). // A backticked index identifier may contain single quotes and backslashes, and diff --git a/pkg/sql/compile/iscp_util.go b/pkg/sql/compile/iscp_util.go index 1544714ce8ed5..23b4f6f66ffbb 100644 --- a/pkg/sql/compile/iscp_util.go +++ b/pkg/sql/compile/iscp_util.go @@ -17,13 +17,21 @@ package compile import ( "context" "fmt" + "sync" + "time" "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/clusterservice" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/defines" indexplugin "github.com/matrixorigin/matrixone/pkg/indexplugin" "github.com/matrixorigin/matrixone/pkg/iscp" "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/pb/metadata" "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/pb/query" + qclient "github.com/matrixorigin/matrixone/pkg/queryservice/client" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/vectorindex/idxcron" ) @@ -31,6 +39,10 @@ import ( var ( iscpRegisterJobFunc = iscp.RegisterJob iscpUnregisterJobFunc = iscp.UnregisterJob + iscpLookupJobLogFunc = iscp.LookupJobLog + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpGetCNQueryAddress = getCNQueryAddress isTableInCCPRFunc = isTableInCCPRImpl ) @@ -192,6 +204,248 @@ func DropIndexCdcTask(c *Compile, tableDef *plan.TableDef, dbname string, tablen return nil } +func DrainIndexCdcTaskConsumer(c *Compile, tableDef *plan.TableDef, dbname string, tablename string, indexname string) error { + return drainIndexCdcTaskConsumer(c, tableDef, dbname, tablename, indexname) +} + +func drainIndexCdcTaskConsumer( + c *Compile, + tableDef *plan.TableDef, + dbname string, + tablename string, + indexname string, +) error { + valid, err := checkValidIndexCdc(tableDef, indexname) + if err != nil { + return err + } + if !valid { + return nil + } + accountID, err := defines.GetAccountId(c.proc.Ctx) + if err != nil { + return err + } + jobName := genCdcTaskJobID(indexname) + _, tableID, jobID, exists, _, err := iscpLookupJobLogFunc( + c.proc.Ctx, + c.proc.GetService(), + c.proc.GetTxnOperator(), + &iscp.JobID{DBName: dbname, TableName: tablename, JobName: jobName}, + ) + if err != nil { + return err + } + if !exists { + logutil.Infof("skip draining index cdc task consumer, iscp job not found: tableID=%d index=%s", tableDef.TblId, indexname) + return nil + } + if tableID == 0 { + tableID = tableDef.TblId + } + runnerCN, err := iscpGetTaskRunnerFunc(c.proc.Ctx, c.proc.GetService(), c.proc.GetTxnOperator()) + if err != nil { + return err + } + if runnerCN == "" { + runnerCN = c.proc.GetService() + } + key := iscp.NewJobRuntimeKey(accountID, tableID, jobName, jobID) + logutil.Infof("drain index cdc task consumer: accountID=%d tableID=%d jobName=%s jobID=%d", accountID, tableID, jobName, jobID) + if exec, ok := iscpGetExecutorFunc(runnerCN); ok && exec != nil { + if err := exec.CancelAndDrainJobConsumer(c.proc.Ctx, accountID, tableID, jobName, jobID); err != nil { + exec.RemoveJobFence(key) + return err + } + if txnOp := c.proc.GetTxnOperator(); txnOp != nil { + lease := startISCPJobFenceLease(func() error { + if !exec.RenewJobFence(key, iscp.RollbackFenceTTL()) { + return moerr.NewInternalErrorf( + c.proc.Ctx, + "cannot renew ISCP consumer quiescence fence for tableID=%d jobName=%s jobID=%d", + tableID, + jobName, + jobID, + ) + } + return nil + }) + cleanup := client.NewTxnEventCallback(func(_ context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { + lease.Stop() + if !event.CostEvent { + return nil + } + exec.RemoveJobFence(key) + return nil + }) + txnOp.AppendEventCallback(client.RollbackEvent, cleanup) + txnOp.AppendEventCallback(client.CommitEvent, client.NewTxnEventCallback(func(_ context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { + if !event.CostEvent { + lease.Stop() + } + return nil + })) + } + return nil + } + qc := c.proc.GetQueryClient() + if qc == nil { + return moerr.NewInternalErrorf( + c.proc.Ctx, + "cannot confirm ISCP consumer quiescence on CN %s for tableID=%d jobName=%s jobID=%d", + runnerCN, + tableID, + jobName, + jobID, + ) + } + queryAddress, err := iscpGetCNQueryAddress(c.proc.Ctx, c.proc.GetService(), runnerCN) + if err != nil { + return err + } + if err := sendISCPDrainConsumerRequest(c.proc.Ctx, qc, queryAddress, accountID, tableID, jobName, jobID, false, false); err != nil { + return err + } + if txnOp := c.proc.GetTxnOperator(); txnOp != nil { + lease := startISCPJobFenceLease(func() error { + ttl := iscp.RollbackFenceTTL() + renewCtx, cancel := context.WithTimeoutCause( + context.Background(), + iscpFenceRenewTimeout(ttl), + moerr.NewInternalErrorNoCtx("iscp fence lease renew timeout"), + ) + defer cancel() + return sendISCPDrainConsumerRequest(renewCtx, qc, queryAddress, accountID, tableID, jobName, jobID, false, true) + }) + cleanup := client.NewTxnEventCallback(func(ctx context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { + lease.Stop() + if !event.CostEvent { + return nil + } + cleanupCtx, cancel := context.WithTimeoutCause( + context.Background(), + iscp.DefaultRollbackFenceTTL, + moerr.NewInternalErrorNoCtx("iscp rollback fence cleanup timeout"), + ) + defer cancel() + return sendISCPDrainConsumerRequest(cleanupCtx, qc, queryAddress, accountID, tableID, jobName, jobID, true, false) + }) + txnOp.AppendEventCallback(client.RollbackEvent, cleanup) + txnOp.AppendEventCallback(client.CommitEvent, client.NewTxnEventCallback(func(_ context.Context, _ client.TxnOperator, event client.TxnEvent, _ any) error { + if !event.CostEvent { + lease.Stop() + } + return nil + })) + } + return nil +} + +type iscpJobFenceLease struct { + stop func() +} + +func startISCPJobFenceLease(renew func() error) iscpJobFenceLease { + ttl := iscp.RollbackFenceTTL() + if ttl <= 0 || renew == nil { + return iscpJobFenceLease{stop: func() {}} + } + interval := ttl / 2 + if interval < 10*time.Millisecond { + interval = 10 * time.Millisecond + } + stopCh := make(chan struct{}) + var once sync.Once + go func() { + timer := time.NewTimer(interval) + defer timer.Stop() + for { + select { + case <-timer.C: + if err := renew(); err != nil { + logutil.Warnf("failed to renew ISCP job fence lease: %v", err) + } + timer.Reset(interval) + case <-stopCh: + return + } + } + }() + return iscpJobFenceLease{stop: func() { + once.Do(func() { + close(stopCh) + }) + }} +} + +func iscpFenceRenewTimeout(ttl time.Duration) time.Duration { + timeout := ttl / 4 + if timeout < 10*time.Millisecond { + timeout = 10 * time.Millisecond + } + return timeout +} + +func (l iscpJobFenceLease) Stop() { + if l.stop != nil { + l.stop() + } +} + +func getCNQueryAddress(ctx context.Context, service string, cnUUID string) (string, error) { + cluster, err := clusterservice.GetMOClusterWithContext(ctx, service) + if err != nil { + return "", err + } + var queryAddress string + err = clusterservice.GetCNServiceWithoutWorkingStateWithContext( + ctx, + cluster, + clusterservice.NewServiceIDSelector(cnUUID), + func(cn metadata.CNService) bool { + queryAddress = cn.QueryAddress + return false + }, + ) + if err != nil { + return "", err + } + if queryAddress == "" { + return "", moerr.NewInternalErrorf(ctx, "cannot find query address for CN %s", cnUUID) + } + return queryAddress, nil +} + +func sendISCPDrainConsumerRequest( + ctx context.Context, + qc qclient.QueryClient, + queryAddress string, + accountID uint32, + tableID uint64, + jobName string, + jobID uint64, + removeFenceOnly bool, + renewFenceOnly bool, +) error { + req := qc.NewRequest(query.CmdMethod_ISCPDrainConsumer) + req.ISCPDrainConsumerRequest = &query.ISCPDrainConsumerRequest{ + AccountID: accountID, + TableID: tableID, + JobName: jobName, + JobID: jobID, + RemoveFenceOnly: removeFenceOnly, + RenewFenceOnly: renewFenceOnly, + } + resp, err := qc.SendMessage(ctx, queryAddress, req) + if err != nil { + return err + } + if resp != nil { + qc.Release(resp) + } + return nil +} + // drop all cdc tasks according to tableDef func DropAllIndexCdcTasks(c *Compile, tabledef *plan.TableDef, dbname string, tablename string) error { idxmap := make(map[string]bool) @@ -214,6 +468,9 @@ func DropAllIndexCdcTasks(c *Compile, tabledef *plan.TableDef, dbname string, ta if e != nil { return e } + if e = drainIndexCdcTaskConsumer(c, tabledef, dbname, tablename, idx.IndexName); e != nil { + return e + } } } return nil diff --git a/pkg/sql/compile/iscp_util_extra_test.go b/pkg/sql/compile/iscp_util_extra_test.go index 00762d48761b0..f8387d5616155 100644 --- a/pkg/sql/compile/iscp_util_extra_test.go +++ b/pkg/sql/compile/iscp_util_extra_test.go @@ -17,15 +17,50 @@ package compile import ( "context" "testing" + "time" + "github.com/golang/mock/gomock" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" "github.com/matrixorigin/matrixone/pkg/iscp" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/pb/query" "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/txn/client" + "github.com/matrixorigin/matrixone/pkg/util/fault" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type iscpDrainTestQueryClient struct { + serviceID string + requests []*query.Request + err error +} + +func (q *iscpDrainTestQueryClient) ServiceID() string { + return q.serviceID +} + +func (q *iscpDrainTestQueryClient) SendMessage(_ context.Context, _ string, req *query.Request) (*query.Response, error) { + q.requests = append(q.requests, req) + if q.err != nil { + return nil, q.err + } + return &query.Response{CmdMethod: req.CmdMethod}, nil +} + +func (q *iscpDrainTestQueryClient) NewRequest(method query.CmdMethod) *query.Request { + return &query.Request{CmdMethod: method} +} + +func (q *iscpDrainTestQueryClient) Release(*query.Response) {} + +func (q *iscpDrainTestQueryClient) Close() error { + return nil +} + func TestCoverage_genCdcTaskJobID(t *testing.T) { tests := []struct { name string @@ -318,14 +353,473 @@ func TestCoverage_DropIndexCdcTask_InvalidIndex(t *testing.T) { require.Nil(t, err) } +func TestDrainIndexCdcTaskConsumerFencesRuntimeJob(t *testing.T) { + exec := &iscp.ISCPTaskExecutor{} + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return exec, true + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + err := DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1") + + require.NoError(t, err) + require.True(t, exec.IsJobFenced(iscp.NewJobRuntimeKey(0, 42, "index_idx1", 7))) +} + +func TestDrainIndexCdcTaskConsumerRoutesToRunnerExecutor(t *testing.T) { + runnerExec := &iscp.ISCPTaskExecutor{} + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + if cnUUID == "runner-cn" { + return runnerExec, true + } + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + require.True(t, runnerExec.IsJobFenced(iscp.NewJobRuntimeKey(0, 42, "index_idx1", 7))) +} + +func TestDrainIndexCdcTaskConsumerNoRunnerExecutorFailsClosed(t *testing.T) { + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.ErrorContains(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1"), "cannot confirm ISCP consumer quiescence on CN runner-cn") +} + +func TestDrainIndexCdcTaskConsumerRoutesToRemoteRunnerQueryService(t *testing.T) { + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + iscpGetCNQueryAddress = func(context.Context, string, string) (string, error) { + return "runner-cn:18101", nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + iscpGetCNQueryAddress = getCNQueryAddress + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + qc := &iscpDrainTestQueryClient{serviceID: "ddl-cn"} + c.proc.Base.QueryClient = qc + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + require.Len(t, qc.requests, 1) + req := qc.requests[0] + require.Equal(t, query.CmdMethod_ISCPDrainConsumer, req.CmdMethod) + require.NotNil(t, req.ISCPDrainConsumerRequest) + require.Equal(t, uint32(0), req.ISCPDrainConsumerRequest.AccountID) + require.Equal(t, uint64(42), req.ISCPDrainConsumerRequest.TableID) + require.Equal(t, "index_idx1", req.ISCPDrainConsumerRequest.JobName) + require.Equal(t, uint64(7), req.ISCPDrainConsumerRequest.JobID) + require.False(t, req.ISCPDrainConsumerRequest.RemoveFenceOnly) +} + +func TestDrainIndexCdcTaskConsumerRemoteFenceCleanupOnTxnEvent(t *testing.T) { + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + iscpGetCNQueryAddress = func(context.Context, string, string) (string, error) { + return "runner-cn:18101", nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + iscpGetCNQueryAddress = getCNQueryAddress + }() + + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + var rollbackCleanup client.TxnEventCallback + var commitStart client.TxnEventCallback + txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + rollbackCleanup = cb + }, + ).Times(1) + txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + commitStart = cb + }, + ).Times(1) + + c := &Compile{} + c.proc = testutil.NewProcess(t) + c.proc.Base.TxnOperator = txnOp + qc := &iscpDrainTestQueryClient{serviceID: "ddl-cn"} + c.proc.Base.QueryClient = qc + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + require.NotNil(t, rollbackCleanup.Func) + require.NotNil(t, commitStart.Func) + require.NoError(t, rollbackCleanup.Func(context.Background(), txnOp, client.TxnEvent{CostEvent: true}, nil)) + + require.Len(t, qc.requests, 2) + require.False(t, qc.requests[0].ISCPDrainConsumerRequest.RemoveFenceOnly) + require.True(t, qc.requests[1].ISCPDrainConsumerRequest.RemoveFenceOnly) +} + +func TestDrainIndexCdcTaskConsumerRemoteRollbackCleanupIgnoresCanceledCallerContext(t *testing.T) { + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + iscpGetCNQueryAddress = func(context.Context, string, string) (string, error) { + return "runner-cn:18101", nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + iscpGetCNQueryAddress = getCNQueryAddress + }() + + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + var rollbackCleanup client.TxnEventCallback + var commitStart client.TxnEventCallback + txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + rollbackCleanup = cb + }, + ).Times(1) + txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + commitStart = cb + }, + ).Times(1) + + c := &Compile{} + c.proc = testutil.NewProcess(t) + c.proc.Base.TxnOperator = txnOp + qc := &iscpDrainTestQueryClient{serviceID: "ddl-cn"} + c.proc.Base.QueryClient = qc + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + require.NotNil(t, commitStart.Func) + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + require.NoError(t, rollbackCleanup.Func(canceledCtx, txnOp, client.TxnEvent{CostEvent: true}, nil)) + + require.Len(t, qc.requests, 2) + require.True(t, qc.requests[1].ISCPDrainConsumerRequest.RemoveFenceOnly) +} + +func TestDrainIndexCdcTaskConsumerRemoteQueryFailureFailsClosed(t *testing.T) { + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + iscpGetCNQueryAddress = func(context.Context, string, string) (string, error) { + return "runner-cn:18101", nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + iscpGetCNQueryAddress = getCNQueryAddress + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + c.proc.Base.QueryClient = &iscpDrainTestQueryClient{ + serviceID: "ddl-cn", + err: moerr.NewInternalErrorNoCtx("remote drain failed"), + } + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.ErrorContains(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1"), "remote drain failed") +} + +func TestDrainIndexCdcTaskConsumerRegistersRollbackFenceCleanup(t *testing.T) { + exec := &iscp.ISCPTaskExecutor{} + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return exec, true + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + }() + + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + var rollbackCleanup client.TxnEventCallback + var commitStart client.TxnEventCallback + txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + rollbackCleanup = cb + }, + ).Times(1) + txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + commitStart = cb + }, + ).Times(1) + + c := &Compile{} + c.proc = testutil.NewProcess(t) + c.proc.Base.TxnOperator = txnOp + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + key := iscp.NewJobRuntimeKey(0, 42, "index_idx1", 7) + require.True(t, exec.IsJobFenced(key)) + require.NotNil(t, rollbackCleanup.Func) + require.NotNil(t, commitStart.Func) + + require.NoError(t, rollbackCleanup.Func(context.Background(), txnOp, client.TxnEvent{}, nil)) + require.True(t, exec.IsJobFenced(key)) + require.NoError(t, rollbackCleanup.Func(context.Background(), txnOp, client.TxnEvent{CostEvent: true}, nil)) + require.False(t, exec.IsJobFenced(key)) + _, ok := exec.RegisterRunningConsumer(key, 7, 1, func() {}, nil) + require.True(t, ok) +} + +func TestDrainIndexCdcTaskConsumerRenewsFenceUntilCommitStarts(t *testing.T) { + require.True(t, fault.Enable()) + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL, ":::", "echo", 1, "", false)) + defer func() { + _, _ = fault.RemoveFaultPoint(context.Background(), objectio.FJ_ISCPCancelRollbackFenceTTL) + }() + + exec := &iscp.ISCPTaskExecutor{} + iscpGetExecutorFunc = func(cnUUID string) (*iscp.ISCPTaskExecutor, bool) { + return exec, true + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 7, true, true, nil + } + defer func() { + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + }() + + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + var commitStart client.TxnEventCallback + txnOp.EXPECT().AppendEventCallback(client.RollbackEvent, gomock.Any()).Times(1) + txnOp.EXPECT().AppendEventCallback(client.CommitEvent, gomock.Any()).DoAndReturn( + func(_ client.EventType, cb client.TxnEventCallback) { + commitStart = cb + }, + ).Times(1) + + c := &Compile{} + c.proc = testutil.NewProcess(t) + c.proc.Base.TxnOperator = txnOp + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.NoError(t, DrainIndexCdcTaskConsumer(c, tbldef, "db", "tbl", "idx1")) + key := iscp.NewJobRuntimeKey(0, 42, "index_idx1", 7) + defer exec.RemoveJobFence(key) + require.NotNil(t, commitStart.Func) + require.True(t, exec.IsJobFenced(key)) + + time.Sleep(1500 * time.Millisecond) + require.True(t, exec.IsJobFenced(key)) + + require.NoError(t, commitStart.Func(context.Background(), txnOp, client.TxnEvent{}, nil)) + require.Eventually(t, func() bool { + return !exec.IsJobFenced(key) + }, 1500*time.Millisecond, 50*time.Millisecond) +} + func TestCoverage_DropAllIndexCdcTasks_DuplicateNames(t *testing.T) { dropCount := 0 iscpUnregisterJobFunc = func(ctx context.Context, cnUUID string, txn client.TxnOperator, job *iscp.JobID) (bool, error) { dropCount++ return true, nil } + exec := &iscp.ISCPTaskExecutor{} + iscpGetExecutorFunc = func(string) (*iscp.ISCPTaskExecutor, bool) { + return exec, true + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, uint64(dropCount), true, true, nil + } defer func() { iscpUnregisterJobFunc = iscp.UnregisterJob + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog }() c := &Compile{} @@ -353,14 +847,67 @@ func TestCoverage_DropAllIndexCdcTasks_DuplicateNames(t *testing.T) { assert.Equal(t, 1, dropCount, "duplicate index names should be deduplicated") } +func TestDropAllIndexCdcTasksNoRunnerExecutorFailsAfterUnregister(t *testing.T) { + dropCount := 0 + iscpUnregisterJobFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (bool, error) { + dropCount++ + return true, nil + } + iscpGetExecutorFunc = func(string) (*iscp.ISCPTaskExecutor, bool) { + return nil, false + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, 1, true, true, nil + } + defer func() { + iscpUnregisterJobFunc = iscp.UnregisterJob + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog + }() + + c := &Compile{} + c.proc = testutil.NewProcess(t) + tbldef := &plan.TableDef{ + TblId: 42, + Indexes: []*plan.IndexDef{ + { + TableExist: true, + IndexName: "idx1", + IndexAlgo: "hnsw", + IndexAlgoParams: `{"async":"true"}`, + }, + }, + } + + require.ErrorContains(t, DropAllIndexCdcTasks(c, tbldef, "db", "tbl"), "cannot confirm ISCP consumer quiescence on CN runner-cn") + require.Equal(t, 1, dropCount) +} + func TestCoverage_DropAllIndexCdcTasks_MixedIndexes(t *testing.T) { dropCount := 0 iscpUnregisterJobFunc = func(ctx context.Context, cnUUID string, txn client.TxnOperator, job *iscp.JobID) (bool, error) { dropCount++ return true, nil } + exec := &iscp.ISCPTaskExecutor{} + iscpGetExecutorFunc = func(string) (*iscp.ISCPTaskExecutor, bool) { + return exec, true + } + iscpGetTaskRunnerFunc = func(context.Context, string, client.TxnOperator) (string, error) { + return "runner-cn", nil + } + iscpLookupJobLogFunc = func(context.Context, string, client.TxnOperator, *iscp.JobID) (uint32, uint64, uint64, bool, bool, error) { + return 0, 42, uint64(dropCount), true, true, nil + } defer func() { iscpUnregisterJobFunc = iscp.UnregisterJob + iscpGetExecutorFunc = iscp.GetExecutorRuntime + iscpGetTaskRunnerFunc = iscp.GetTaskRunner + iscpLookupJobLogFunc = iscp.LookupJobLog }() c := &Compile{} diff --git a/pkg/util/fault/fault.go b/pkg/util/fault/fault.go index 2081ddeee8ca4..130b56aeecd59 100644 --- a/pkg/util/fault/fault.go +++ b/pkg/util/fault/fault.go @@ -83,10 +83,12 @@ type faultEntry struct { sarg string // string arg constant bool - nWaiters int - mutex sync.Mutex - cond *sync.Cond - scope Domain + nWaiters int + notifySeq int64 + removed bool + mutex sync.Mutex + cond *sync.Cond + scope Domain } type faultMap struct { @@ -115,11 +117,25 @@ func (fm *faultMap) run() { } case REMOVE: if e.name == "all" { + for _, v := range fm.faultPoints { + if v.action == WAIT && v.cond != nil { + v.mutex.Lock() + v.removed = true + v.cond.Broadcast() + v.mutex.Unlock() + } + } fm.faultPoints = make(map[string]*faultEntry) fm.chOut <- e continue } if v, ok := fm.faultPoints[e.name]; ok { + if v.action == WAIT && v.cond != nil { + v.mutex.Lock() + v.removed = true + v.cond.Broadcast() + v.mutex.Unlock() + } delete(fm.faultPoints, e.name) fm.chOut <- v } else { @@ -160,9 +176,11 @@ func (e *faultEntry) do() (int64, string) { } case WAIT: e.mutex.Lock() - e.nWaiters += 1 - e.cond.Wait() - e.nWaiters -= 1 + if !e.removed { + e.nWaiters += 1 + e.cond.Wait() + e.nWaiters -= 1 + } e.mutex.Unlock() case GETWAITERS: if ee := lookup(e.scope, e.sarg); ee != nil { @@ -173,11 +191,17 @@ func (e *faultEntry) do() (int64, string) { } case NOTIFY: if ee := lookup(e.scope, e.sarg); ee != nil { + ee.mutex.Lock() + ee.notifySeq++ ee.cond.Signal() + ee.mutex.Unlock() } case NOTIFYALL: if ee := lookup(e.scope, e.sarg); ee != nil { + ee.mutex.Lock() + ee.notifySeq++ ee.cond.Broadcast() + ee.mutex.Unlock() } case PANIC: switch e.iarg { @@ -192,6 +216,47 @@ func (e *faultEntry) do() (int64, string) { return 0, "" } +func (e *faultEntry) doWithContext(ctx context.Context) (int64, string) { + if e.action == SLEEP { + timer := time.NewTimer(time.Duration(e.iarg) * time.Second) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + } + return 0, "" + } + if e.action != WAIT { + return e.do() + } + e.mutex.Lock() + if e.removed { + e.mutex.Unlock() + return 0, "" + } + e.nWaiters += 1 + notifySeq := e.notifySeq + canceled := false + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + e.mutex.Lock() + canceled = true + e.cond.Broadcast() + e.mutex.Unlock() + case <-done: + } + }() + for !e.removed && !canceled && e.notifySeq == notifySeq { + e.cond.Wait() + } + e.nWaiters -= 1 + close(done) + e.mutex.Unlock() + return 0, "" +} + func startFaultMap(domain Domain) bool { if enabled[domain].Load() != nil { return false @@ -276,7 +341,19 @@ func TriggerFault(name string) (iret int64, sret string, exist bool) { return TriggerFaultInDomain(DomainDefault, name) } +func TriggerFaultWithContext(ctx context.Context, name string) (iret int64, sret string, exist bool) { + return TriggerFaultInDomainWithContext(ctx, DomainDefault, name) +} + func TriggerFaultInDomain(domain Domain, name string) (iret int64, sret string, exist bool) { + return triggerFaultInDomain(context.Background(), domain, name, false) +} + +func TriggerFaultInDomainWithContext(ctx context.Context, domain Domain, name string) (iret int64, sret string, exist bool) { + return triggerFaultInDomain(ctx, domain, name, true) +} + +func triggerFaultInDomain(ctx context.Context, domain Domain, name string, useCtx bool) (iret int64, sret string, exist bool) { fm := enabled[domain].Load() if fm == nil { return @@ -296,7 +373,11 @@ func TriggerFaultInDomain(domain Domain, name string) (iret int64, sret string, return } exist = true - iret, sret = out.do() + if useCtx { + iret, sret = out.doWithContext(ctx) + } else { + iret, sret = out.do() + } return } diff --git a/pkg/util/fault/fault_test.go b/pkg/util/fault/fault_test.go index 9329c38f96fb2..fa84b5b6cbd9d 100644 --- a/pkg/util/fault/fault_test.go +++ b/pkg/util/fault/fault_test.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "testing" + "time" "github.com/stretchr/testify/require" @@ -182,6 +183,249 @@ func TestWait(t *testing.T) { Disable() } +func TestRemoveWaitFaultPointNotifiesWaiters(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + require.NoError(t, AddFaultPoint(ctx, "gw", ":::", "getwaiters", 0, "w", false)) + + done := make(chan struct{}) + go func() { + TriggerFault("w") + close(done) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := TriggerFault("gw") + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + removed, err := RemoveFaultPoint(ctx, "w") + require.NoError(t, err) + require.True(t, removed) + require.Eventually(t, func() bool { + select { + case <-done: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) +} + +func TestRemoveWaitFaultPointBeforeWaitRegistrationDoesNotHang(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + entry := lookup(DomainDefault, "w") + require.NotNil(t, entry) + + removed, err := RemoveFaultPoint(ctx, "w") + require.NoError(t, err) + require.True(t, removed) + + done := make(chan struct{}) + go func() { + entry.do() + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("wait fault point blocked after it was removed before waiter registration") + } +} + +func TestRemoveWaitFaultPointBeforeContextWaitRegistrationDoesNotHang(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + fm := enabled[DomainDefault].Load() + require.NotNil(t, fm) + + fm.chIn <- &faultEntry{cmd: TRIGGER, name: "w"} + entry := <-fm.chOut + require.NotNil(t, entry) + + removed, err := RemoveFaultPoint(ctx, "w") + require.NoError(t, err) + require.True(t, removed) + + waitCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan struct{}) + go func() { + entry.doWithContext(waitCtx) + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("context-aware wait fault point blocked after it was removed before waiter registration") + } +} + +func TestTriggerWaitFaultWithContextReturnsOnCancel(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + require.NoError(t, AddFaultPoint(ctx, "gw", ":::", "getwaiters", 0, "w", false)) + + waitCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + TriggerFaultWithContext(waitCtx, "w") + close(done) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := TriggerFault("gw") + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + cancel() + require.Eventually(t, func() bool { + select { + case <-done: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) +} + +func TestTriggerWaitFaultWithContextCancelDoesNotReleaseOtherWaiters(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + require.NoError(t, AddFaultPoint(ctx, "gw", ":::", "getwaiters", 0, "w", false)) + + waitCtx1, cancel1 := context.WithCancel(ctx) + waitCtx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + done1 := make(chan struct{}) + done2 := make(chan struct{}) + go func() { + TriggerFaultWithContext(waitCtx1, "w") + close(done1) + }() + go func() { + TriggerFaultWithContext(waitCtx2, "w") + close(done2) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := TriggerFault("gw") + return ok && cnt == 2 + }, time.Second, 10*time.Millisecond) + + cancel1() + require.Eventually(t, func() bool { + select { + case <-done1: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) + + select { + case <-done2: + t.Fatal("canceling one context-aware wait released another live waiter") + default: + } + cnt, _, ok := TriggerFault("gw") + require.True(t, ok) + require.Equal(t, int64(1), cnt) + + cancel2() + require.Eventually(t, func() bool { + select { + case <-done2: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) +} + +func TestTriggerWaitFaultWithContextReturnsOnNotifyAll(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "w", ":::", "wait", 0, "", false)) + require.NoError(t, AddFaultPoint(ctx, "gw", ":::", "getwaiters", 0, "w", false)) + require.NoError(t, AddFaultPoint(ctx, "nall", ":::", "notifyall", 0, "w", false)) + + done := make(chan struct{}) + go func() { + TriggerFaultWithContext(ctx, "w") + close(done) + }() + + require.Eventually(t, func() bool { + cnt, _, ok := TriggerFault("gw") + return ok && cnt == 1 + }, time.Second, 10*time.Millisecond) + + TriggerFault("nall") + require.Eventually(t, func() bool { + select { + case <-done: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) +} + +func TestTriggerSleepFaultWithContextReturnsOnCancel(t *testing.T) { + ctx := context.Background() + + Enable() + defer Disable() + + require.NoError(t, AddFaultPoint(ctx, "s", ":::", "sleep", 30, "", false)) + + sleepCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + start := time.Now() + go func() { + TriggerFaultWithContext(sleepCtx, "s") + close(done) + }() + + cancel() + require.Eventually(t, func() bool { + select { + case <-done: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) + require.Less(t, time.Since(start), 5*time.Second) +} + func Test_panic(t *testing.T) { var ctx = context.TODO() diff --git a/proto/query.proto b/proto/query.proto index 234b1427f0a85..3869c53b1f8c3 100644 --- a/proto/query.proto +++ b/proto/query.proto @@ -102,6 +102,7 @@ enum CmdMethod { WorkspaceThreshold = 34; MinTimestamp = 35; CtlPrefetchOnSubscribed = 36; + ISCPDrainConsumer = 37; } // QueryRequest is the common query request. It contains the query @@ -242,6 +243,7 @@ message Request { CtlMoTableStatsRequest CtlMoTableStatsRequest = 36 [ (gogoproto.nullable) = false ]; WorkspaceThresholdRequest WorkspaceThresholdRequest = 37; CtlPrefetchOnSubscribedRequest CtlPrefetchOnSubscribedRequest = 38; + ISCPDrainConsumerRequest ISCPDrainConsumerRequest = 39; } // ShowProcessListResponse is the response of command ShowProcessList. @@ -307,6 +309,7 @@ message Response { WorkspaceThresholdResponse WorkspaceThresholdResponse = 37; MinTimestampResponse MinTimestampResponse = 38; CtlPrefetchOnSubscribedResponse CtlPrefetchOnSubscribedResponse = 39; + ISCPDrainConsumerResponse ISCPDrainConsumerResponse = 40; } // AlterAccountRequest is the "alter account restricted" query request. @@ -352,6 +355,19 @@ message CtlPrefetchOnSubscribedResponse { string Resp = 1; } +message ISCPDrainConsumerRequest { + uint32 AccountID = 1; + uint64 TableID = 2; + string JobName = 3; + uint64 JobID = 4; + bool RemoveFenceOnly = 5; + bool RenewFenceOnly = 6; +} + +message ISCPDrainConsumerResponse { + bool Success = 1; +} + message TraceSpanRequest { string Cmd = 1; string Spans = 2;