From 0791a281864faa77622a2f4e67b93ad9642e376f Mon Sep 17 00:00:00 2001 From: Wenxuan Date: Mon, 20 Jan 2025 17:44:00 +0900 Subject: [PATCH] playground: Add --mode=tiflash-disagg (#2492) * playground: Add back --mode=tiflash-disagg (which is tidb-disagg previously) Signed-off-by: Wish * Fix Signed-off-by: Wish * Update Signed-off-by: Wish * Update Signed-off-by: Wish * Fix Signed-off-by: Wish * Fix CI Signed-off-by: Wish * Only set maxReplica=1 when starts 1 tikv instance Signed-off-by: Wish --------- Signed-off-by: Wish --- .github/workflows/integrate-cluster-cmd.yaml | 3 +- .../workflows/integrate-cluster-scale.yaml | 3 +- .github/workflows/integrate-dm.yaml | 3 +- .github/workflows/integrate-playground.yaml | 3 +- components/playground/instance/pd.go | 12 +++--- components/playground/instance/pd_config.go | 9 +++-- components/playground/instance/tidb.go | 6 +-- components/playground/instance/tidb_config.go | 5 ++- components/playground/instance/tiflash.go | 7 +++- .../playground/instance/tiflash_config.go | 30 +++++++++------ components/playground/instance/tikv.go | 6 +-- components/playground/instance/tikv_config.go | 2 +- components/playground/main.go | 38 +++++++++---------- components/playground/playground.go | 35 +++++++++-------- 14 files changed, 94 insertions(+), 68 deletions(-) diff --git a/.github/workflows/integrate-cluster-cmd.yaml b/.github/workflows/integrate-cluster-cmd.yaml index 9bac1e5702..a8d909aebe 100644 --- a/.github/workflows/integrate-cluster-cmd.yaml +++ b/.github/workflows/integrate-cluster-cmd.yaml @@ -100,8 +100,9 @@ jobs: - name: Upload component log if: ${{ failure() }} # if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: + overwrite: true name: component_logs path: ${{ env.working-directory }}/logs/ diff --git a/.github/workflows/integrate-cluster-scale.yaml b/.github/workflows/integrate-cluster-scale.yaml index 78d4b76580..878fe85164 100644 --- a/.github/workflows/integrate-cluster-scale.yaml +++ b/.github/workflows/integrate-cluster-scale.yaml @@ -100,8 +100,9 @@ jobs: - name: Upload component log if: ${{ failure() }} # if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: + overwrite: true name: cluster_logs path: ${{ env.working-directory }}/logs/ diff --git a/.github/workflows/integrate-dm.yaml b/.github/workflows/integrate-dm.yaml index cb18ae3387..c9b539a016 100644 --- a/.github/workflows/integrate-dm.yaml +++ b/.github/workflows/integrate-dm.yaml @@ -97,8 +97,9 @@ jobs: - name: Upload component log if: ${{ failure() }} # if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: + overwrite: true name: dm_logs path: ${{ env.working-directory }}/dm.logs.tar.gz diff --git a/.github/workflows/integrate-playground.yaml b/.github/workflows/integrate-playground.yaml index 48d075c22c..b39d7dd849 100644 --- a/.github/workflows/integrate-playground.yaml +++ b/.github/workflows/integrate-playground.yaml @@ -77,8 +77,9 @@ jobs: - name: Upload component log if: ${{ failure() }} # if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: + overwrite: true name: playground_logs path: ${{ env.working-directory }}/playground.logs.tar.gz diff --git a/components/playground/instance/pd.go b/components/playground/instance/pd.go index 4ce7392904..7de46702f6 100644 --- a/components/playground/instance/pd.go +++ b/components/playground/instance/pd.go @@ -46,11 +46,12 @@ type PDInstance struct { joinEndpoints []*PDInstance pds []*PDInstance Process - isCSEMode bool + mode string + kvIsSingleReplica bool } // NewPDInstance return a PDInstance -func NewPDInstance(role PDRole, binPath, dir, host, configPath string, portOffset int, id int, pds []*PDInstance, port int, isCSEMode bool) *PDInstance { +func NewPDInstance(role PDRole, binPath, dir, host, configPath string, portOffset int, id int, pds []*PDInstance, port int, mode string, kvIsSingleReplica bool) *PDInstance { if port <= 0 { port = 2379 } @@ -64,9 +65,10 @@ func NewPDInstance(role PDRole, binPath, dir, host, configPath string, portOffse StatusPort: utils.MustGetFreePort(host, port, portOffset), ConfigPath: configPath, }, - Role: role, - pds: pds, - isCSEMode: isCSEMode, + Role: role, + pds: pds, + mode: mode, + kvIsSingleReplica: kvIsSingleReplica, } } diff --git a/components/playground/instance/pd_config.go b/components/playground/instance/pd_config.go index 60091a3b0c..4d557adf2b 100644 --- a/components/playground/instance/pd_config.go +++ b/components/playground/instance/pd_config.go @@ -16,13 +16,16 @@ package instance func (inst *PDInstance) getConfig() map[string]any { config := make(map[string]any) config["schedule.patrol-region-interval"] = "100ms" + config["schedule.low-space-ratio"] = 1.0 - if inst.isCSEMode { + if inst.kvIsSingleReplica { + config["replication.max-replica"] = 1 + } + + if inst.mode == "tidb-cse" { config["keyspace.pre-alloc"] = []string{"mykeyspace"} config["replication.enable-placement-rules"] = true - config["replication.max-replica"] = 1 config["schedule.merge-schedule-limit"] = 0 - config["schedule.low-space-ratio"] = 1.0 config["schedule.replica-schedule-limit"] = 500 } diff --git a/components/playground/instance/tidb.go b/components/playground/instance/tidb.go index b5496ffca1..0d1c842cc1 100644 --- a/components/playground/instance/tidb.go +++ b/components/playground/instance/tidb.go @@ -30,11 +30,11 @@ type TiDBInstance struct { Process tiproxyCertDir string enableBinlog bool - isCSEMode bool + mode string } // NewTiDBInstance return a TiDBInstance -func NewTiDBInstance(binPath string, dir, host, configPath string, portOffset int, id, port int, pds []*PDInstance, tiproxyCertDir string, enableBinlog bool, isCSEMode bool) *TiDBInstance { +func NewTiDBInstance(binPath string, dir, host, configPath string, portOffset int, id, port int, pds []*PDInstance, tiproxyCertDir string, enableBinlog bool, mode string) *TiDBInstance { if port <= 0 { port = 4000 } @@ -51,7 +51,7 @@ func NewTiDBInstance(binPath string, dir, host, configPath string, portOffset in tiproxyCertDir: tiproxyCertDir, pds: pds, enableBinlog: enableBinlog, - isCSEMode: isCSEMode, + mode: mode, } } diff --git a/components/playground/instance/tidb_config.go b/components/playground/instance/tidb_config.go index a48e30df97..56af37045c 100644 --- a/components/playground/instance/tidb_config.go +++ b/components/playground/instance/tidb_config.go @@ -22,7 +22,7 @@ func (inst *TiDBInstance) getConfig() map[string]any { config := make(map[string]any) config["security.auto-tls"] = true - if inst.isCSEMode { + if inst.mode == "tidb-cse" { config["keyspace-name"] = "mykeyspace" config["enable-safe-point-v2"] = true config["force-enable-vector-type"] = true @@ -52,6 +52,9 @@ func (inst *TiDBInstance) getConfig() map[string]any { config["tiflash-replicas.group-id"] = "enable_s3_wn_region" config["tiflash-replicas.extra-s3-rule"] = false config["tiflash-replicas.min-count"] = 1 + } else if inst.mode == "tiflash-disagg" { + config["use-autoscaler"] = false + config["disaggregated-tiflash"] = true } tiproxyCrtPath := filepath.Join(inst.tiproxyCertDir, "tiproxy.crt") diff --git a/components/playground/instance/tiflash.go b/components/playground/instance/tiflash.go index 6a35d8a3d9..b5e421b095 100644 --- a/components/playground/instance/tiflash.go +++ b/components/playground/instance/tiflash.go @@ -42,6 +42,7 @@ const ( type TiFlashInstance struct { instance Role TiFlashRole + mode string cseOpts CSEOptions TCPPort int ServicePort int @@ -53,10 +54,13 @@ type TiFlashInstance struct { } // NewTiFlashInstance return a TiFlashInstance -func NewTiFlashInstance(role TiFlashRole, cseOptions CSEOptions, binPath, dir, host, configPath string, portOffset int, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance { +func NewTiFlashInstance(mode string, role TiFlashRole, cseOptions CSEOptions, binPath, dir, host, configPath string, portOffset int, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance { if role != TiFlashRoleNormal && role != TiFlashRoleDisaggWrite && role != TiFlashRoleDisaggCompute { panic(fmt.Sprintf("Unknown TiFlash role %s", role)) } + if (role == TiFlashRoleDisaggCompute || role == TiFlashRoleDisaggWrite) && mode != "tidb-cse" && mode != "tiflash-disagg" { + panic(fmt.Sprintf("Unsupported disagg role in mode %s", mode)) + } httpPort := 8123 if !tidbver.TiFlashNotNeedHTTPPortConfig(version) { @@ -72,6 +76,7 @@ func NewTiFlashInstance(role TiFlashRole, cseOptions CSEOptions, binPath, dir, h StatusPort: utils.MustGetFreePort(host, 8234, portOffset), ConfigPath: configPath, }, + mode: mode, Role: role, cseOpts: cseOptions, TCPPort: utils.MustGetFreePort(host, 9100, portOffset), // 9000 for default object store port diff --git a/components/playground/instance/tiflash_config.go b/components/playground/instance/tiflash_config.go index 869a1571fe..0cbc4ad582 100644 --- a/components/playground/instance/tiflash_config.go +++ b/components/playground/instance/tiflash_config.go @@ -23,14 +23,16 @@ func (inst *TiFlashInstance) getProxyConfig() map[string]any { config["storage.reserve-raft-space"] = 0 if inst.Role == TiFlashRoleDisaggWrite { - config["storage.api-version"] = 2 - config["storage.enable-ttl"] = true - config["dfs.prefix"] = "tikv" - config["dfs.s3-endpoint"] = inst.cseOpts.S3Endpoint - config["dfs.s3-key-id"] = inst.cseOpts.AccessKey - config["dfs.s3-secret-key"] = inst.cseOpts.SecretKey - config["dfs.s3-bucket"] = inst.cseOpts.Bucket - config["dfs.s3-region"] = "local" + if inst.mode == "tidb-cse" { + config["storage.api-version"] = 2 + config["storage.enable-ttl"] = true + config["dfs.prefix"] = "tikv" + config["dfs.s3-endpoint"] = inst.cseOpts.S3Endpoint + config["dfs.s3-key-id"] = inst.cseOpts.AccessKey + config["dfs.s3-secret-key"] = inst.cseOpts.SecretKey + config["dfs.s3-bucket"] = inst.cseOpts.Bucket + config["dfs.s3-region"] = "local" + } } return config @@ -43,8 +45,6 @@ func (inst *TiFlashInstance) getConfig() map[string]any { config["logger.level"] = "debug" if inst.Role == TiFlashRoleDisaggWrite { - config["enable_safe_point_v2"] = true - config["storage.api_version"] = 2 config["storage.s3.endpoint"] = inst.cseOpts.S3Endpoint config["storage.s3.bucket"] = inst.cseOpts.Bucket config["storage.s3.root"] = "/tiflash-cse/" @@ -52,17 +52,23 @@ func (inst *TiFlashInstance) getConfig() map[string]any { config["storage.s3.secret_access_key"] = inst.cseOpts.SecretKey config["storage.main.dir"] = []string{filepath.Join(inst.Dir, "main_data")} config["flash.disaggregated_mode"] = "tiflash_write" + if inst.mode == "tidb-cse" { + config["enable_safe_point_v2"] = true + config["storage.api_version"] = 2 + } } else if inst.Role == TiFlashRoleDisaggCompute { - config["enable_safe_point_v2"] = true config["storage.s3.endpoint"] = inst.cseOpts.S3Endpoint config["storage.s3.bucket"] = inst.cseOpts.Bucket config["storage.s3.root"] = "/tiflash-cse/" config["storage.s3.access_key_id"] = inst.cseOpts.AccessKey config["storage.s3.secret_access_key"] = inst.cseOpts.SecretKey config["storage.remote.cache.dir"] = filepath.Join(inst.Dir, "remote_cache") - config["storage.remote.cache.capacity"] = 20000000000 // 20GB + config["storage.remote.cache.capacity"] = 50000000000 // 50GB config["storage.main.dir"] = []string{filepath.Join(inst.Dir, "main_data")} config["flash.disaggregated_mode"] = "tiflash_compute" + if inst.mode == "tidb-cse" { + config["enable_safe_point_v2"] = true + } } return config diff --git a/components/playground/instance/tikv.go b/components/playground/instance/tikv.go index 8c97acd018..532914d589 100644 --- a/components/playground/instance/tikv.go +++ b/components/playground/instance/tikv.go @@ -30,13 +30,13 @@ type TiKVInstance struct { pds []*PDInstance tsos []*PDInstance Process - isCSEMode bool + mode string cseOpts CSEOptions isPDMSMode bool } // NewTiKVInstance return a TiKVInstance -func NewTiKVInstance(binPath string, dir, host, configPath string, portOffset int, id int, port int, pds []*PDInstance, tsos []*PDInstance, isCSEMode bool, cseOptions CSEOptions, isPDMSMode bool) *TiKVInstance { +func NewTiKVInstance(binPath string, dir, host, configPath string, portOffset int, id int, port int, pds []*PDInstance, tsos []*PDInstance, mode string, cseOptions CSEOptions, isPDMSMode bool) *TiKVInstance { if port <= 0 { port = 20160 } @@ -52,7 +52,7 @@ func NewTiKVInstance(binPath string, dir, host, configPath string, portOffset in }, pds: pds, tsos: tsos, - isCSEMode: isCSEMode, + mode: mode, cseOpts: cseOptions, isPDMSMode: isPDMSMode, } diff --git a/components/playground/instance/tikv_config.go b/components/playground/instance/tikv_config.go index 45fab76a0e..44abac6ac1 100644 --- a/components/playground/instance/tikv_config.go +++ b/components/playground/instance/tikv_config.go @@ -20,7 +20,7 @@ func (inst *TiKVInstance) getConfig() map[string]any { config["storage.reserve-space"] = 0 config["storage.reserve-raft-space"] = 0 - if inst.isCSEMode { + if inst.mode == "tidb-cse" { config["storage.api-version"] = 2 config["storage.enable-ttl"] = true config["dfs.prefix"] = "tikv" diff --git a/components/playground/main.go b/components/playground/main.go index 09ce160f2c..4fe5774f2a 100644 --- a/components/playground/main.go +++ b/components/playground/main.go @@ -64,16 +64,16 @@ type BootOptions struct { TiProxy instance.Config `yaml:"tiproxy"` TiDB instance.Config `yaml:"tidb"` TiKV instance.Config `yaml:"tikv"` - TiFlash instance.Config `yaml:"tiflash"` // ignored when mode == tidb-cse - TiFlashWrite instance.Config `yaml:"tiflash_write"` // Only available when mode == tidb-cse - TiFlashCompute instance.Config `yaml:"tiflash_compute"` // Only available when mode == tidb-cse + TiFlash instance.Config `yaml:"tiflash"` // ignored when mode == tidb-cse or tiflash-disagg + TiFlashWrite instance.Config `yaml:"tiflash_write"` // Only available when mode == tidb-cse or tiflash-disagg + TiFlashCompute instance.Config `yaml:"tiflash_compute"` // Only available when mode == tidb-cse or tiflash-disagg TiCDC instance.Config `yaml:"ticdc"` TiKVCDC instance.Config `yaml:"tikv_cdc"` Pump instance.Config `yaml:"pump"` Drainer instance.Config `yaml:"drainer"` Host string `yaml:"host"` Monitor bool `yaml:"monitor"` - CSEOpts instance.CSEOptions `yaml:"cse"` // Only available when mode == tidb-cse + CSEOpts instance.CSEOptions `yaml:"cse"` // Only available when mode == tidb-cse or tiflash-disagg GrafanaPort int `yaml:"grafana_port"` PortOffset int `yaml:"port_offset"` DMMaster instance.Config `yaml:"dm_master"` @@ -277,7 +277,7 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. }, } - rootCmd.Flags().StringVar(&options.Mode, "mode", "tidb", "TiUP playground mode: 'tidb', 'tidb-cse', 'tikv-slim'") + rootCmd.Flags().StringVar(&options.Mode, "mode", "tidb", "TiUP playground mode: 'tidb', 'tidb-cse', 'tiflash-disagg', 'tikv-slim'") rootCmd.Flags().StringVar(&options.PDMode, "pd.mode", "pd", "PD mode: 'pd', 'ms'") rootCmd.PersistentFlags().StringVarP(&tag, "tag", "T", "", "Specify a tag for playground, data dir of this tag will not be removed after exit") rootCmd.Flags().Bool("without-monitor", false, "Don't start prometheus and grafana component") @@ -294,9 +294,9 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().IntVar(&options.TSO.Num, "tso", 0, "TSO instance number") rootCmd.Flags().IntVar(&options.Scheduling.Num, "scheduling", 0, "Scheduling instance number") rootCmd.Flags().IntVar(&options.TiProxy.Num, "tiproxy", 0, "TiProxy instance number") - rootCmd.Flags().IntVar(&options.TiFlash.Num, "tiflash", 0, "TiFlash instance number, when --mode=tidb-cse this will set instance number for both Write Node and Compute Node") - rootCmd.Flags().IntVar(&options.TiFlashWrite.Num, "tiflash.write", 0, "TiFlash Write instance number, available when --mode=tidb-cse, take precedence over --tiflash") - rootCmd.Flags().IntVar(&options.TiFlashCompute.Num, "tiflash.compute", 0, "TiFlash Compute instance number, available when --mode=tidb-cse, take precedence over --tiflash") + rootCmd.Flags().IntVar(&options.TiFlash.Num, "tiflash", 0, "TiFlash instance number, when --mode=tidb-cse or --mode=tiflash-disagg this will set instance number for both Write Node and Compute Node") + rootCmd.Flags().IntVar(&options.TiFlashWrite.Num, "tiflash.write", 0, "TiFlash Write instance number, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash") + rootCmd.Flags().IntVar(&options.TiFlashCompute.Num, "tiflash.compute", 0, "TiFlash Compute instance number, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash") rootCmd.Flags().IntVar(&options.TiCDC.Num, "ticdc", 0, "TiCDC instance number") rootCmd.Flags().IntVar(&options.TiKVCDC.Num, "kvcdc", 0, "TiKV-CDC instance number") rootCmd.Flags().IntVar(&options.Pump.Num, "pump", 0, "Pump instance number") @@ -330,9 +330,9 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().StringVar(&options.TSO.ConfigPath, "tso.config", "", "TSO instance configuration file") rootCmd.Flags().StringVar(&options.Scheduling.ConfigPath, "scheduling.config", "", "Scheduling instance configuration file") rootCmd.Flags().StringVar(&options.TiProxy.ConfigPath, "tiproxy.config", "", "TiProxy instance configuration file") - rootCmd.Flags().StringVar(&options.TiFlash.ConfigPath, "tiflash.config", "", "TiFlash instance configuration file, when --mode=tidb-cse this will set config file for both Write Node and Compute Node") - rootCmd.Flags().StringVar(&options.TiFlashWrite.ConfigPath, "tiflash.write.config", "", "TiFlash Write instance configuration file, available when --mode=tidb-cse, take precedence over --tiflash.config") - rootCmd.Flags().StringVar(&options.TiFlashCompute.ConfigPath, "tiflash.compute.config", "", "TiFlash Compute instance configuration file, available when --mode=tidb-cse, take precedence over --tiflash.config") + rootCmd.Flags().StringVar(&options.TiFlash.ConfigPath, "tiflash.config", "", "TiFlash instance configuration file, when --mode=tidb-cse or --mode=tiflash-disagg this will set config file for both Write Node and Compute Node") + rootCmd.Flags().StringVar(&options.TiFlashWrite.ConfigPath, "tiflash.write.config", "", "TiFlash Write instance configuration file, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash.config") + rootCmd.Flags().StringVar(&options.TiFlashCompute.ConfigPath, "tiflash.compute.config", "", "TiFlash Compute instance configuration file, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash.config") rootCmd.Flags().StringVar(&options.Pump.ConfigPath, "pump.config", "", "Pump instance configuration file") rootCmd.Flags().StringVar(&options.Drainer.ConfigPath, "drainer.config", "", "Drainer instance configuration file") rootCmd.Flags().StringVar(&options.TiCDC.ConfigPath, "ticdc.config", "", "TiCDC instance configuration file") @@ -347,9 +347,9 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().StringVar(&options.Scheduling.BinPath, "scheduling.binpath", "", "Scheduling instance binary path") rootCmd.Flags().StringVar(&options.TiProxy.BinPath, "tiproxy.binpath", "", "TiProxy instance binary path") rootCmd.Flags().StringVar(&options.TiProxy.Version, "tiproxy.version", "", "TiProxy instance version") - rootCmd.Flags().StringVar(&options.TiFlash.BinPath, "tiflash.binpath", "", "TiFlash instance binary path, when --mode=tidb-cse this will set binary path for both Write Node and Compute Node") - rootCmd.Flags().StringVar(&options.TiFlashWrite.BinPath, "tiflash.write.binpath", "", "TiFlash Write instance binary path, available when --mode=tidb-cse, take precedence over --tiflash.binpath") - rootCmd.Flags().StringVar(&options.TiFlashCompute.BinPath, "tiflash.compute.binpath", "", "TiFlash Compute instance binary path, available when --mode=tidb-cse, take precedence over --tiflash.binpath") + rootCmd.Flags().StringVar(&options.TiFlash.BinPath, "tiflash.binpath", "", "TiFlash instance binary path, when --mode=tidb-cse or --mode=tiflash-disagg this will set binary path for both Write Node and Compute Node") + rootCmd.Flags().StringVar(&options.TiFlashWrite.BinPath, "tiflash.write.binpath", "", "TiFlash Write instance binary path, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash.binpath") + rootCmd.Flags().StringVar(&options.TiFlashCompute.BinPath, "tiflash.compute.binpath", "", "TiFlash Compute instance binary path, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash.binpath") rootCmd.Flags().StringVar(&options.TiCDC.BinPath, "ticdc.binpath", "", "TiCDC instance binary path") rootCmd.Flags().StringVar(&options.TiKVCDC.BinPath, "kvcdc.binpath", "", "TiKV-CDC instance binary path") rootCmd.Flags().StringVar(&options.Pump.BinPath, "pump.binpath", "", "Pump instance binary path") @@ -359,10 +359,10 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().StringVar(&options.TiKVCDC.Version, "kvcdc.version", "", "TiKV-CDC instance version") - rootCmd.Flags().StringVar(&options.CSEOpts.S3Endpoint, "cse.s3_endpoint", "http://127.0.0.1:9000", "Object store URL for --mode=tidb-cse") - rootCmd.Flags().StringVar(&options.CSEOpts.Bucket, "cse.bucket", "tiflash", "Object store bucket for --mode=tidb-cse") - rootCmd.Flags().StringVar(&options.CSEOpts.AccessKey, "cse.access_key", "minioadmin", "Object store access key for --mode=tidb-cse") - rootCmd.Flags().StringVar(&options.CSEOpts.SecretKey, "cse.secret_key", "minioadmin", "Object store secret key for --mode=tidb-cse") + rootCmd.Flags().StringVar(&options.CSEOpts.S3Endpoint, "cse.s3_endpoint", "http://127.0.0.1:9000", "Object store URL for --mode=tidb-cse or --mode=tiflash-disagg") + rootCmd.Flags().StringVar(&options.CSEOpts.Bucket, "cse.bucket", "tiflash", "Object store bucket for --mode=tidb-cse or --mode=tiflash-disagg") + rootCmd.Flags().StringVar(&options.CSEOpts.AccessKey, "cse.access_key", "minioadmin", "Object store access key for --mode=tidb-cse or --mode=tiflash-disagg") + rootCmd.Flags().StringVar(&options.CSEOpts.SecretKey, "cse.secret_key", "minioadmin", "Object store secret key for --mode=tidb-cse or --mode=tiflash-disagg") rootCmd.AddCommand(newDisplay()) rootCmd.AddCommand(newScaleOut()) @@ -396,7 +396,7 @@ func populateDefaultOpt(flagSet *pflag.FlagSet) error { defaultInt(&options.TiFlash.Num, "tiflash", 1) case "tikv-slim": defaultInt(&options.TiKV.Num, "kv", 1) - case "tidb-cse": + case "tidb-cse", "tiflash-disagg": defaultInt(&options.TiDB.Num, "db", 1) defaultInt(&options.TiKV.Num, "kv", 1) defaultInt(&options.TiFlash.Num, "tiflash", 1) diff --git a/components/playground/playground.go b/components/playground/playground.go index 62d3401f55..b86d03203b 100644 --- a/components/playground/playground.go +++ b/components/playground/playground.go @@ -792,7 +792,7 @@ func (p *Playground) addInstance(componentID string, pdRole instance.PDRole, tif switch componentID { case spec.ComponentPD: - inst := instance.NewPDInstance(pdRole, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, cfg.Port, p.bootOptions.Mode == "tidb-cse") + inst := instance.NewPDInstance(pdRole, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, cfg.Port, p.bootOptions.Mode, p.bootOptions.TiKV.Num == 1) ins = inst if pdRole == instance.PDRoleNormal || pdRole == instance.PDRoleAPI { if p.booted { @@ -810,23 +810,23 @@ func (p *Playground) addInstance(componentID string, pdRole instance.PDRole, tif p.schedulings = append(p.schedulings, inst) } case spec.ComponentTSO: - inst := instance.NewPDInstance(instance.PDRoleTSO, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, cfg.Port, p.bootOptions.Mode == "tidb-cse") + inst := instance.NewPDInstance(instance.PDRoleTSO, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, cfg.Port, p.bootOptions.Mode, p.bootOptions.TiKV.Num == 1) ins = inst p.tsos = append(p.tsos, inst) case spec.ComponentScheduling: - inst := instance.NewPDInstance(instance.PDRoleScheduling, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, cfg.Port, p.bootOptions.Mode == "tidb-cse") + inst := instance.NewPDInstance(instance.PDRoleScheduling, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, cfg.Port, p.bootOptions.Mode, p.bootOptions.TiKV.Num == 1) ins = inst p.schedulings = append(p.schedulings, inst) case spec.ComponentTiDB: - inst := instance.NewTiDBInstance(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, cfg.Port, p.pds, dataDir, p.enableBinlog(), p.bootOptions.Mode == "tidb-cse") + inst := instance.NewTiDBInstance(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, cfg.Port, p.pds, dataDir, p.enableBinlog(), p.bootOptions.Mode) ins = inst p.tidbs = append(p.tidbs, inst) case spec.ComponentTiKV: - inst := instance.NewTiKVInstance(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, cfg.Port, p.pds, p.tsos, p.bootOptions.Mode == "tidb-cse", p.bootOptions.CSEOpts, p.bootOptions.PDMode == "ms") + inst := instance.NewTiKVInstance(cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, cfg.Port, p.pds, p.tsos, p.bootOptions.Mode, p.bootOptions.CSEOpts, p.bootOptions.PDMode == "ms") ins = inst p.tikvs = append(p.tikvs, inst) case spec.ComponentTiFlash: - inst := instance.NewTiFlashInstance(tiflashRole, p.bootOptions.CSEOpts, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, p.tidbs, cfg.Version) + inst := instance.NewTiFlashInstance(p.bootOptions.Mode, tiflashRole, p.bootOptions.CSEOpts, cfg.BinPath, dir, host, cfg.ConfigPath, options.PortOffset, id, p.pds, p.tidbs, cfg.Version) ins = inst p.tiflashs = append(p.tiflashs, inst) case spec.ComponentTiProxy: @@ -1101,14 +1101,14 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme instances = append(instances, InstancePair{spec.ComponentTiFlash, instance.PDRoleNormal, instance.TiFlashRoleNormal, options.TiFlash}, ) - } else if options.Mode == "tidb-cse" { + } else if options.Mode == "tidb-cse" || options.Mode == "tiflash-disagg" { if !tidbver.TiFlashPlaygroundNewStartMode(options.Version) { // For simplicity, currently we only implemented disagg mode when TiFlash can run without config. - return fmt.Errorf("TiUP playground only supports CSE mode for TiDB cluster >= v7.1.0 (or nightly)") + return fmt.Errorf("TiUP playground only supports CSE/Disagg mode for TiDB cluster >= v7.1.0 (or nightly)") } if !strings.HasPrefix(options.CSEOpts.S3Endpoint, "https://") && !strings.HasPrefix(options.CSEOpts.S3Endpoint, "http://") { - return fmt.Errorf("CSE mode requires S3 endpoint to start with http:// or https://") + return fmt.Errorf("CSE/Disagg mode requires S3 endpoint to start with http:// or https://") } isSecure := strings.HasPrefix(options.CSEOpts.S3Endpoint, "https://") @@ -1117,7 +1117,7 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme // Currently we always assign region=local. Other regions are not supported. if strings.Contains(rawEndpoint, "amazonaws.com") { - return fmt.Errorf("Currently TiUP playground CSE mode only supports local S3 (like minio). S3 on AWS Regions are not supported. Contributions are welcome!") + return fmt.Errorf("Currently TiUP playground CSE/Disagg mode only supports local S3 (like minio). S3 on AWS Regions are not supported. Contributions are welcome!") } // Preflight check whether specified object storage is available. @@ -1126,7 +1126,7 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme Secure: isSecure, }) if err != nil { - return errors.Annotate(err, "CSE mode preflight check failed") + return errors.Annotate(err, "CSE/Disagg mode preflight check failed") } ctxCheck, cancel := context.WithTimeout(ctx, 5*time.Second) @@ -1134,14 +1134,14 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme bucketExists, err := s3Client.BucketExists(ctxCheck, options.CSEOpts.Bucket) if err != nil { - return errors.Annotate(err, "CSE mode preflight check failed") + return errors.Annotate(err, "CSE/Disagg mode preflight check failed") } if !bucketExists { // Try to create bucket. err := s3Client.MakeBucket(ctxCheck, options.CSEOpts.Bucket, minio.MakeBucketOptions{}) if err != nil { - return fmt.Errorf("CSE mode preflight check failed: Bucket %s doesn't exist and fail to create automatically (your bucket name may be invalid?)", options.CSEOpts.Bucket) + return fmt.Errorf("CSE/Disagg mode preflight check failed: Bucket %s doesn't exist and fail to create automatically (your bucket name may be invalid?)", options.CSEOpts.Bucket) } } @@ -1251,11 +1251,14 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme p.tiflashs = started p.waitAllTiFlashUp() - fmt.Println() - color.New(color.FgYellow, color.Bold).Println("TiDB Playground Cluster will delete all data of the cluster after exit. Please use --tag xx to pin the data dir.") - fmt.Println() color.New(color.FgGreen, color.Bold).Println("🎉 TiDB Playground Cluster is started, enjoy!") + + if deleteWhenExit { + fmt.Println() + colorstr.Printf("[yellow][bold]Warning[reset][bold]: cluster data will be destroyed after exit. To persist data after exit, specify [tiup_command]--tag [reset].\n") + } + fmt.Println() mysql := mysqlCommand() for _, dbAddr := range tidbSucc {