playground: Add --mode=tiflash-disagg

.github/workflows/integrate-cluster-cmd.yaml

@@ -100,8 +100,9 @@ jobs:
       - name: Upload component log
         if: ${{ failure() }}
         # if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          overwrite: true
           name: component_logs
           path: ${{ env.working-directory }}/logs/
 

.github/workflows/integrate-cluster-scale.yaml

@@ -100,8 +100,9 @@ jobs:
       - name: Upload component log
         if: ${{ failure() }}
         # if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          overwrite: true
           name: cluster_logs
           path: ${{ env.working-directory }}/logs/
 

.github/workflows/integrate-dm.yaml

@@ -97,8 +97,9 @@ jobs:
       - name: Upload component log
         if: ${{ failure() }}
         # if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          overwrite: true
           name: dm_logs
           path: ${{ env.working-directory }}/dm.logs.tar.gz
 

.github/workflows/integrate-playground.yaml

@@ -77,8 +77,9 @@ jobs:
       - name: Upload component log
         if: ${{ failure() }}
         # if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          overwrite: true
           name: playground_logs
           path: ${{ env.working-directory }}/playground.logs.tar.gz
 

components/playground/instance/pd.go

@@ -46,11 +46,12 @@ type PDInstance struct {
 	joinEndpoints []*PDInstance
 	pds           []*PDInstance
 	Process
-	isCSEMode bool
+	mode              string
+	kvIsSingleReplica bool
 }
 
 // NewPDInstance return a PDInstance
-func NewPDInstance(role PDRole, binPath, dir, host, configPath string, portOffset int, id int, pds []*PDInstance, port int, isCSEMode bool) *PDInstance {
+func NewPDInstance(role PDRole, binPath, dir, host, configPath string, portOffset int, id int, pds []*PDInstance, port int, mode string, kvIsSingleReplica bool) *PDInstance {
 	if port <= 0 {
 		port = 2379
 	}
@@ -64,9 +65,10 @@ func NewPDInstance(role PDRole, binPath, dir, host, configPath string, portOffse
 			StatusPort: utils.MustGetFreePort(host, port, portOffset),
 			ConfigPath: configPath,
 		},
-		Role:      role,
-		pds:       pds,
-		isCSEMode: isCSEMode,
+		Role:              role,
+		pds:               pds,
+		mode:              mode,
+		kvIsSingleReplica: kvIsSingleReplica,
 	}
 }
 

components/playground/instance/pd_config.go

@@ -16,13 +16,16 @@ package instance
 func (inst *PDInstance) getConfig() map[string]any {
 	config := make(map[string]any)
 	config["schedule.patrol-region-interval"] = "100ms"
+	config["schedule.low-space-ratio"] = 1.0
 
-	if inst.isCSEMode {
+	if inst.kvIsSingleReplica {
+		config["replication.max-replica"] = 1
+	}
+
+	if inst.mode == "tidb-cse" {
 		config["keyspace.pre-alloc"] = []string{"mykeyspace"}
 		config["replication.enable-placement-rules"] = true
-		config["replication.max-replica"] = 1
 		config["schedule.merge-schedule-limit"] = 0
-		config["schedule.low-space-ratio"] = 1.0
 		config["schedule.replica-schedule-limit"] = 500
 	}
 

components/playground/instance/tidb.go

@@ -30,11 +30,11 @@ type TiDBInstance struct {
 	Process
 	tiproxyCertDir string
 	enableBinlog   bool
-	isCSEMode      bool
+	mode           string
 }
 
 // NewTiDBInstance return a TiDBInstance
-func NewTiDBInstance(binPath string, dir, host, configPath string, portOffset int, id, port int, pds []*PDInstance, tiproxyCertDir string, enableBinlog bool, isCSEMode bool) *TiDBInstance {
+func NewTiDBInstance(binPath string, dir, host, configPath string, portOffset int, id, port int, pds []*PDInstance, tiproxyCertDir string, enableBinlog bool, mode string) *TiDBInstance {
 	if port <= 0 {
 		port = 4000
 	}
@@ -51,7 +51,7 @@ func NewTiDBInstance(binPath string, dir, host, configPath string, portOffset in
 		tiproxyCertDir: tiproxyCertDir,
 		pds:            pds,
 		enableBinlog:   enableBinlog,
-		isCSEMode:      isCSEMode,
+		mode:           mode,
 	}
 }
 

components/playground/instance/tidb_config.go

@@ -22,7 +22,7 @@ func (inst *TiDBInstance) getConfig() map[string]any {
 	config := make(map[string]any)
 	config["security.auto-tls"] = true
 
-	if inst.isCSEMode {
+	if inst.mode == "tidb-cse" {
 		config["keyspace-name"] = "mykeyspace"
 		config["enable-safe-point-v2"] = true
 		config["force-enable-vector-type"] = true
@@ -52,6 +52,9 @@ func (inst *TiDBInstance) getConfig() map[string]any {
 		config["tiflash-replicas.group-id"] = "enable_s3_wn_region"
 		config["tiflash-replicas.extra-s3-rule"] = false
 		config["tiflash-replicas.min-count"] = 1
+	} else if inst.mode == "tiflash-disagg" {
+		config["use-autoscaler"] = false
+		config["disaggregated-tiflash"] = true
 	}
 
 	tiproxyCrtPath := filepath.Join(inst.tiproxyCertDir, "tiproxy.crt")

components/playground/instance/tiflash.go

@@ -42,6 +42,7 @@ const (
 type TiFlashInstance struct {
 	instance
 	Role            TiFlashRole
+	mode            string
 	cseOpts         CSEOptions
 	TCPPort         int
 	ServicePort     int
@@ -53,10 +54,13 @@ type TiFlashInstance struct {
 }
 
 // NewTiFlashInstance return a TiFlashInstance
-func NewTiFlashInstance(role TiFlashRole, cseOptions CSEOptions, binPath, dir, host, configPath string, portOffset int, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance {
+func NewTiFlashInstance(mode string, role TiFlashRole, cseOptions CSEOptions, binPath, dir, host, configPath string, portOffset int, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance {
 	if role != TiFlashRoleNormal && role != TiFlashRoleDisaggWrite && role != TiFlashRoleDisaggCompute {
 		panic(fmt.Sprintf("Unknown TiFlash role %s", role))
 	}
+	if (role == TiFlashRoleDisaggCompute || role == TiFlashRoleDisaggWrite) && mode != "tidb-cse" && mode != "tiflash-disagg" {
+		panic(fmt.Sprintf("Unsupported disagg role in mode %s", mode))
+	}
 
 	httpPort := 8123
 	if !tidbver.TiFlashNotNeedHTTPPortConfig(version) {
@@ -72,6 +76,7 @@ func NewTiFlashInstance(role TiFlashRole, cseOptions CSEOptions, binPath, dir, h
 			StatusPort: utils.MustGetFreePort(host, 8234, portOffset),
 			ConfigPath: configPath,
 		},
+		mode:            mode,
 		Role:            role,
 		cseOpts:         cseOptions,
 		TCPPort:         utils.MustGetFreePort(host, 9100, portOffset), // 9000 for default object store port

components/playground/instance/tiflash_config.go

@@ -23,14 +23,16 @@ func (inst *TiFlashInstance) getProxyConfig() map[string]any {
 	config["storage.reserve-raft-space"] = 0
 
 	if inst.Role == TiFlashRoleDisaggWrite {
-		config["storage.api-version"] = 2
-		config["storage.enable-ttl"] = true
-		config["dfs.prefix"] = "tikv"
-		config["dfs.s3-endpoint"] = inst.cseOpts.S3Endpoint
-		config["dfs.s3-key-id"] = inst.cseOpts.AccessKey
-		config["dfs.s3-secret-key"] = inst.cseOpts.SecretKey
-		config["dfs.s3-bucket"] = inst.cseOpts.Bucket
-		config["dfs.s3-region"] = "local"
+		if inst.mode == "tidb-cse" {
+			config["storage.api-version"] = 2
+			config["storage.enable-ttl"] = true
+			config["dfs.prefix"] = "tikv"
+			config["dfs.s3-endpoint"] = inst.cseOpts.S3Endpoint
+			config["dfs.s3-key-id"] = inst.cseOpts.AccessKey
+			config["dfs.s3-secret-key"] = inst.cseOpts.SecretKey
+			config["dfs.s3-bucket"] = inst.cseOpts.Bucket
+			config["dfs.s3-region"] = "local"
+		}
 	}
 
 	return config
@@ -43,26 +45,30 @@ func (inst *TiFlashInstance) getConfig() map[string]any {
 	config["logger.level"] = "debug"
 
 	if inst.Role == TiFlashRoleDisaggWrite {
-		config["enable_safe_point_v2"] = true
-		config["storage.api_version"] = 2
 		config["storage.s3.endpoint"] = inst.cseOpts.S3Endpoint
 		config["storage.s3.bucket"] = inst.cseOpts.Bucket
 		config["storage.s3.root"] = "/tiflash-cse/"
 		config["storage.s3.access_key_id"] = inst.cseOpts.AccessKey
 		config["storage.s3.secret_access_key"] = inst.cseOpts.SecretKey
 		config["storage.main.dir"] = []string{filepath.Join(inst.Dir, "main_data")}
 		config["flash.disaggregated_mode"] = "tiflash_write"
+		if inst.mode == "tidb-cse" {
+			config["enable_safe_point_v2"] = true
+			config["storage.api_version"] = 2
+		}
 	} else if inst.Role == TiFlashRoleDisaggCompute {
-		config["enable_safe_point_v2"] = true
 		config["storage.s3.endpoint"] = inst.cseOpts.S3Endpoint
 		config["storage.s3.bucket"] = inst.cseOpts.Bucket
 		config["storage.s3.root"] = "/tiflash-cse/"
 		config["storage.s3.access_key_id"] = inst.cseOpts.AccessKey
 		config["storage.s3.secret_access_key"] = inst.cseOpts.SecretKey
 		config["storage.remote.cache.dir"] = filepath.Join(inst.Dir, "remote_cache")
-		config["storage.remote.cache.capacity"] = 20000000000 // 20GB
+		config["storage.remote.cache.capacity"] = 50000000000 // 50GB
 		config["storage.main.dir"] = []string{filepath.Join(inst.Dir, "main_data")}
 		config["flash.disaggregated_mode"] = "tiflash_compute"
+		if inst.mode == "tidb-cse" {
+			config["enable_safe_point_v2"] = true
+		}
 	}
 
 	return config

components/playground/instance/tikv.go

@@ -30,13 +30,13 @@ type TiKVInstance struct {
 	pds  []*PDInstance
 	tsos []*PDInstance
 	Process
-	isCSEMode  bool
+	mode       string
 	cseOpts    CSEOptions
 	isPDMSMode bool
 }
 
 // NewTiKVInstance return a TiKVInstance
-func NewTiKVInstance(binPath string, dir, host, configPath string, portOffset int, id int, port int, pds []*PDInstance, tsos []*PDInstance, isCSEMode bool, cseOptions CSEOptions, isPDMSMode bool) *TiKVInstance {
+func NewTiKVInstance(binPath string, dir, host, configPath string, portOffset int, id int, port int, pds []*PDInstance, tsos []*PDInstance, mode string, cseOptions CSEOptions, isPDMSMode bool) *TiKVInstance {
 	if port <= 0 {
 		port = 20160
 	}
@@ -52,7 +52,7 @@ func NewTiKVInstance(binPath string, dir, host, configPath string, portOffset in
 		},
 		pds:        pds,
 		tsos:       tsos,
-		isCSEMode:  isCSEMode,
+		mode:       mode,
 		cseOpts:    cseOptions,
 		isPDMSMode: isPDMSMode,
 	}

components/playground/instance/tikv_config.go

@@ -20,7 +20,7 @@ func (inst *TiKVInstance) getConfig() map[string]any {
 	config["storage.reserve-space"] = 0
 	config["storage.reserve-raft-space"] = 0
 
-	if inst.isCSEMode {
+	if inst.mode == "tidb-cse" {
 		config["storage.api-version"] = 2
 		config["storage.enable-ttl"] = true
 		config["dfs.prefix"] = "tikv"

components/playground/main.go

@@ -64,16 +64,16 @@ type BootOptions struct {
 	TiProxy        instance.Config     `yaml:"tiproxy"`
 	TiDB           instance.Config     `yaml:"tidb"`
 	TiKV           instance.Config     `yaml:"tikv"`
-	TiFlash        instance.Config     `yaml:"tiflash"`         // ignored when mode == tidb-cse
-	TiFlashWrite   instance.Config     `yaml:"tiflash_write"`   // Only available when mode == tidb-cse
-	TiFlashCompute instance.Config     `yaml:"tiflash_compute"` // Only available when mode == tidb-cse
+	TiFlash        instance.Config     `yaml:"tiflash"`         // ignored when mode == tidb-cse or tiflash-disagg
+	TiFlashWrite   instance.Config     `yaml:"tiflash_write"`   // Only available when mode == tidb-cse or tiflash-disagg
+	TiFlashCompute instance.Config     `yaml:"tiflash_compute"` // Only available when mode == tidb-cse or tiflash-disagg
 	TiCDC          instance.Config     `yaml:"ticdc"`
 	TiKVCDC        instance.Config     `yaml:"tikv_cdc"`
 	Pump           instance.Config     `yaml:"pump"`
 	Drainer        instance.Config     `yaml:"drainer"`
 	Host           string              `yaml:"host"`
 	Monitor        bool                `yaml:"monitor"`
-	CSEOpts        instance.CSEOptions `yaml:"cse"` // Only available when mode == tidb-cse
+	CSEOpts        instance.CSEOptions `yaml:"cse"` // Only available when mode == tidb-cse or tiflash-disagg
 	GrafanaPort    int                 `yaml:"grafana_port"`
 	PortOffset     int                 `yaml:"port_offset"`
 	DMMaster       instance.Config     `yaml:"dm_master"`
@@ -277,7 +277,7 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset].
 		},
 	}
 
-	rootCmd.Flags().StringVar(&options.Mode, "mode", "tidb", "TiUP playground mode: 'tidb', 'tidb-cse', 'tikv-slim'")
+	rootCmd.Flags().StringVar(&options.Mode, "mode", "tidb", "TiUP playground mode: 'tidb', 'tidb-cse', 'tiflash-disagg', 'tikv-slim'")
 	rootCmd.Flags().StringVar(&options.PDMode, "pd.mode", "pd", "PD mode: 'pd', 'ms'")
 	rootCmd.PersistentFlags().StringVarP(&tag, "tag", "T", "", "Specify a tag for playground, data dir of this tag will not be removed after exit")
 	rootCmd.Flags().Bool("without-monitor", false, "Don't start prometheus and grafana component")
@@ -294,9 +294,9 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset].
 	rootCmd.Flags().IntVar(&options.TSO.Num, "tso", 0, "TSO instance number")
 	rootCmd.Flags().IntVar(&options.Scheduling.Num, "scheduling", 0, "Scheduling instance number")
 	rootCmd.Flags().IntVar(&options.TiProxy.Num, "tiproxy", 0, "TiProxy instance number")
-	rootCmd.Flags().IntVar(&options.TiFlash.Num, "tiflash", 0, "TiFlash instance number, when --mode=tidb-cse this will set instance number for both Write Node and Compute Node")
-	rootCmd.Flags().IntVar(&options.TiFlashWrite.Num, "tiflash.write", 0, "TiFlash Write instance number, available when --mode=tidb-cse, take precedence over --tiflash")
-	rootCmd.Flags().IntVar(&options.TiFlashCompute.Num, "tiflash.compute", 0, "TiFlash Compute instance number, available when --mode=tidb-cse, take precedence over --tiflash")
+	rootCmd.Flags().IntVar(&options.TiFlash.Num, "tiflash", 0, "TiFlash instance number, when --mode=tidb-cse or --mode=tiflash-disagg this will set instance number for both Write Node and Compute Node")
+	rootCmd.Flags().IntVar(&options.TiFlashWrite.Num, "tiflash.write", 0, "TiFlash Write instance number, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash")
+	rootCmd.Flags().IntVar(&options.TiFlashCompute.Num, "tiflash.compute", 0, "TiFlash Compute instance number, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash")
 	rootCmd.Flags().IntVar(&options.TiCDC.Num, "ticdc", 0, "TiCDC instance number")
 	rootCmd.Flags().IntVar(&options.TiKVCDC.Num, "kvcdc", 0, "TiKV-CDC instance number")
 	rootCmd.Flags().IntVar(&options.Pump.Num, "pump", 0, "Pump instance number")
@@ -330,9 +330,9 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset].
 	rootCmd.Flags().StringVar(&options.TSO.ConfigPath, "tso.config", "", "TSO instance configuration file")
 	rootCmd.Flags().StringVar(&options.Scheduling.ConfigPath, "scheduling.config", "", "Scheduling instance configuration file")
 	rootCmd.Flags().StringVar(&options.TiProxy.ConfigPath, "tiproxy.config", "", "TiProxy instance configuration file")
-	rootCmd.Flags().StringVar(&options.TiFlash.ConfigPath, "tiflash.config", "", "TiFlash instance configuration file, when --mode=tidb-cse this will set config file for both Write Node and Compute Node")
-	rootCmd.Flags().StringVar(&options.TiFlashWrite.ConfigPath, "tiflash.write.config", "", "TiFlash Write instance configuration file, available when --mode=tidb-cse, take precedence over --tiflash.config")
-	rootCmd.Flags().StringVar(&options.TiFlashCompute.ConfigPath, "tiflash.compute.config", "", "TiFlash Compute instance configuration file, available when --mode=tidb-cse, take precedence over --tiflash.config")
+	rootCmd.Flags().StringVar(&options.TiFlash.ConfigPath, "tiflash.config", "", "TiFlash instance configuration file, when --mode=tidb-cse or --mode=tiflash-disagg this will set config file for both Write Node and Compute Node")
+	rootCmd.Flags().StringVar(&options.TiFlashWrite.ConfigPath, "tiflash.write.config", "", "TiFlash Write instance configuration file, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash.config")
+	rootCmd.Flags().StringVar(&options.TiFlashCompute.ConfigPath, "tiflash.compute.config", "", "TiFlash Compute instance configuration file, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash.config")
 	rootCmd.Flags().StringVar(&options.Pump.ConfigPath, "pump.config", "", "Pump instance configuration file")
 	rootCmd.Flags().StringVar(&options.Drainer.ConfigPath, "drainer.config", "", "Drainer instance configuration file")
 	rootCmd.Flags().StringVar(&options.TiCDC.ConfigPath, "ticdc.config", "", "TiCDC instance configuration file")
@@ -347,9 +347,9 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset].
 	rootCmd.Flags().StringVar(&options.Scheduling.BinPath, "scheduling.binpath", "", "Scheduling instance binary path")
 	rootCmd.Flags().StringVar(&options.TiProxy.BinPath, "tiproxy.binpath", "", "TiProxy instance binary path")
 	rootCmd.Flags().StringVar(&options.TiProxy.Version, "tiproxy.version", "", "TiProxy instance version")
-	rootCmd.Flags().StringVar(&options.TiFlash.BinPath, "tiflash.binpath", "", "TiFlash instance binary path, when --mode=tidb-cse this will set binary path for both Write Node and Compute Node")
-	rootCmd.Flags().StringVar(&options.TiFlashWrite.BinPath, "tiflash.write.binpath", "", "TiFlash Write instance binary path, available when --mode=tidb-cse, take precedence over --tiflash.binpath")
-	rootCmd.Flags().StringVar(&options.TiFlashCompute.BinPath, "tiflash.compute.binpath", "", "TiFlash Compute instance binary path, available when --mode=tidb-cse, take precedence over --tiflash.binpath")
+	rootCmd.Flags().StringVar(&options.TiFlash.BinPath, "tiflash.binpath", "", "TiFlash instance binary path, when --mode=tidb-cse or --mode=tiflash-disagg this will set binary path for both Write Node and Compute Node")
+	rootCmd.Flags().StringVar(&options.TiFlashWrite.BinPath, "tiflash.write.binpath", "", "TiFlash Write instance binary path, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash.binpath")
+	rootCmd.Flags().StringVar(&options.TiFlashCompute.BinPath, "tiflash.compute.binpath", "", "TiFlash Compute instance binary path, available when --mode=tidb-cse or --mode=tiflash-disagg, take precedence over --tiflash.binpath")
 	rootCmd.Flags().StringVar(&options.TiCDC.BinPath, "ticdc.binpath", "", "TiCDC instance binary path")
 	rootCmd.Flags().StringVar(&options.TiKVCDC.BinPath, "kvcdc.binpath", "", "TiKV-CDC instance binary path")
 	rootCmd.Flags().StringVar(&options.Pump.BinPath, "pump.binpath", "", "Pump instance binary path")
@@ -359,10 +359,10 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset].
 
 	rootCmd.Flags().StringVar(&options.TiKVCDC.Version, "kvcdc.version", "", "TiKV-CDC instance version")
 
-	rootCmd.Flags().StringVar(&options.CSEOpts.S3Endpoint, "cse.s3_endpoint", "http://127.0.0.1:9000", "Object store URL for --mode=tidb-cse")
-	rootCmd.Flags().StringVar(&options.CSEOpts.Bucket, "cse.bucket", "tiflash", "Object store bucket for --mode=tidb-cse")
-	rootCmd.Flags().StringVar(&options.CSEOpts.AccessKey, "cse.access_key", "minioadmin", "Object store access key for --mode=tidb-cse")
-	rootCmd.Flags().StringVar(&options.CSEOpts.SecretKey, "cse.secret_key", "minioadmin", "Object store secret key for --mode=tidb-cse")
+	rootCmd.Flags().StringVar(&options.CSEOpts.S3Endpoint, "cse.s3_endpoint", "http://127.0.0.1:9000", "Object store URL for --mode=tidb-cse or --mode=tiflash-disagg")
+	rootCmd.Flags().StringVar(&options.CSEOpts.Bucket, "cse.bucket", "tiflash", "Object store bucket for --mode=tidb-cse or --mode=tiflash-disagg")
+	rootCmd.Flags().StringVar(&options.CSEOpts.AccessKey, "cse.access_key", "minioadmin", "Object store access key for --mode=tidb-cse or --mode=tiflash-disagg")
+	rootCmd.Flags().StringVar(&options.CSEOpts.SecretKey, "cse.secret_key", "minioadmin", "Object store secret key for --mode=tidb-cse or --mode=tiflash-disagg")
 
 	rootCmd.AddCommand(newDisplay())
 	rootCmd.AddCommand(newScaleOut())
@@ -396,7 +396,7 @@ func populateDefaultOpt(flagSet *pflag.FlagSet) error {
 		defaultInt(&options.TiFlash.Num, "tiflash", 1)
 	case "tikv-slim":
 		defaultInt(&options.TiKV.Num, "kv", 1)
-	case "tidb-cse":
+	case "tidb-cse", "tiflash-disagg":
 		defaultInt(&options.TiDB.Num, "db", 1)
 		defaultInt(&options.TiKV.Num, "kv", 1)
 		defaultInt(&options.TiFlash.Num, "tiflash", 1)