diff --git a/CHANGELOG.md b/CHANGELOG.md index bf9f33d59..25eb50559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `tt coredump inspect`: fails for tarantool-ee coredump archive if the source directory is missing. +- `tt pack`: fails if `etcd` or `tcs` are present in the configuration + and not available. ## [2.6.0] - 2024-11-29 diff --git a/cli/pack/opts.go b/cli/pack/opts.go index bbfad2bf0..8212e7a69 100644 --- a/cli/pack/opts.go +++ b/cli/pack/opts.go @@ -49,7 +49,7 @@ func initAppsInfo(cliOpts *config.CliOpts, cmdCtx *cmdcontext.CmdCtx, packCtx *P } packCtx.AppList = appList packCtx.AppsInfo, err = running.CollectInstancesForApps(packCtx.AppList, cliOpts, - cmdCtx.Cli.ConfigDir, cmdCtx.Integrity, true) + cmdCtx.Cli.ConfigDir, cmdCtx.Integrity, false) if err != nil { return fmt.Errorf("failed to collect applications info: %s", err) } diff --git a/cli/running/running.go b/cli/running/running.go index 01fe2b087..9348ab78b 100644 --- a/cli/running/running.go +++ b/cli/running/running.go @@ -348,7 +348,7 @@ func loadInstanceConfig(configPath, instName string, // collectInstancesFromAppDir collects instances information from application directory. func collectInstancesFromAppDir(appDir string, selectedInstName string, - integrityCtx integrity.IntegrityCtx, instancesScriptsRequired bool) ( + integrityCtx integrity.IntegrityCtx, configRequired bool) ( []InstanceCtx, error, ) { @@ -376,7 +376,7 @@ func collectInstancesFromAppDir(appDir string, selectedInstName string, InstName: filepath.Base(appDir), AppDir: appDir, SingleApp: true}}, nil - } else if instancesScriptsRequired { + } else if configRequired { return nil, fmt.Errorf("require files are missing in application directory %q: "+ "there must be instances config or the default instance script (%q)", appDir, "init.lua") @@ -410,7 +410,7 @@ func collectInstancesFromAppDir(appDir string, selectedInstName string, log.Debugf("Instance %q", instance.InstName) if instance.Configuration, err = loadInstanceConfig(instance.ClusterConfigPath, - instance.InstName, integrityCtx); err != nil { + instance.InstName, integrityCtx); err != nil && configRequired { return instances, fmt.Errorf("error loading instance %q configuration from "+ "config %q: %w", instance.InstName, instance.ClusterConfigPath, err) } @@ -418,7 +418,7 @@ func collectInstancesFromAppDir(appDir string, selectedInstName string, instance.SingleApp = false if instance.InstanceScript, err = findInstanceScriptInAppDir(appDir, instance.InstName, appDirFiles.clusterCfgPath, appDirFiles.defaultLuaPath); err != nil && - instancesScriptsRequired { + configRequired { return instances, fmt.Errorf("cannot find instance script for %q in config %q: %w ", instance.InstName, appDirFiles.clusterCfgPath, err) } @@ -434,7 +434,7 @@ func collectInstancesFromAppDir(appDir string, selectedInstName string, // CollectInstances searches all instances available in application. func CollectInstances(appName string, applicationsDir string, - integrityCtx integrity.IntegrityCtx, instancesScriptsRequired bool) ([]InstanceCtx, error) { + integrityCtx integrity.IntegrityCtx, configRequired bool) ([]InstanceCtx, error) { // The user can select a specific instance from the application. // Example: `tt status application:server`. selectedInstName := "" @@ -463,7 +463,7 @@ func CollectInstances(appName string, applicationsDir string, } return collectInstancesFromAppDir(appDir, selectedInstName, integrityCtx, - instancesScriptsRequired) + configRequired) } // cleanup removes runtime artifacts. @@ -625,7 +625,7 @@ func GetClusterConfigPath(cliOpts *config.CliOpts, // CollectInstancesForApps collects instances information per application. func CollectInstancesForApps(appList []string, cliOpts *config.CliOpts, - ttConfigDir string, integrityCtx integrity.IntegrityCtx, instancesScriptsRequired bool) ( + ttConfigDir string, integrityCtx integrity.IntegrityCtx, configRequired bool) ( map[string][]InstanceCtx, error) { instEnabledPath := cliOpts.Env.InstancesEnabled if cliOpts.Env.InstancesEnabled == "." { @@ -635,7 +635,7 @@ func CollectInstancesForApps(appList []string, cliOpts *config.CliOpts, for _, appName := range appList { appName = strings.TrimSuffix(appName, ".lua") collectedInstances, err := CollectInstances(appName, instEnabledPath, integrityCtx, - instancesScriptsRequired) + configRequired) if err != nil { return apps, fmt.Errorf("can't collect instance information for %s: %w", appName, err) @@ -676,7 +676,7 @@ func createInstanceDataDirectories(instance InstanceCtx) error { // FillCtx fills the RunningCtx context. func FillCtx(cliOpts *config.CliOpts, cmdCtx *cmdcontext.CmdCtx, - runningCtx *RunningCtx, args []string, instancesScriptsRequired bool) error { + runningCtx *RunningCtx, args []string, configRequired bool) error { var err error if len(args) > 1 && cmdCtx.CommandName != "run" && cmdCtx.CommandName != "connect" && @@ -703,7 +703,7 @@ func FillCtx(cliOpts *config.CliOpts, cmdCtx *cmdcontext.CmdCtx, } instances, err := CollectInstancesForApps(appList, cliOpts, - cmdCtx.Cli.ConfigDir, cmdCtx.Integrity, instancesScriptsRequired) + cmdCtx.Cli.ConfigDir, cmdCtx.Integrity, configRequired) if err != nil { return err } diff --git a/test/integration/pack/test_bundles/vshard_app/instances.enabled/test_app b/test/integration/pack/test_bundles/vshard_app/instances.enabled/test_app new file mode 120000 index 000000000..1ab0d0931 --- /dev/null +++ b/test/integration/pack/test_bundles/vshard_app/instances.enabled/test_app @@ -0,0 +1 @@ +../test_app \ No newline at end of file diff --git a/test/integration/pack/test_bundles/vshard_app/test_app/config.yaml b/test/integration/pack/test_bundles/vshard_app/test_app/config.yaml new file mode 100644 index 000000000..cf7fe68c4 --- /dev/null +++ b/test/integration/pack/test_bundles/vshard_app/test_app/config.yaml @@ -0,0 +1,75 @@ +credentials: + users: + client: + password: 'secret' + roles: [super] + replicator: + password: 'secret' + roles: [replication] + storage: + password: 'secret' + roles: [sharding] + +iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + +sharding: + bucket_count: 3000 + +groups: + storages: + app: + module: storage + sharding: + roles: [storage] + replication: + failover: manual + replicasets: + storage-001: + leader: storage-001-a + instances: + storage-001-a: + iproto: + listen: + - uri: localhost:3301 + storage-001-b: + iproto: + listen: + - uri: localhost:3302 + storage-002: + leader: storage-002-a + instances: + storage-002-a: + iproto: + listen: + - uri: localhost:3303 + storage-002-b: + iproto: + listen: + - uri: localhost:3304 + routers: + app: + module: router + sharding: + roles: [router] + replicasets: + router-001: + instances: + router-001-a: + iproto: + listen: + - uri: localhost:3305 +config: + etcd: + endpoints: + - http://localhost:2379 + prefix: /test_app + username: client + password: secret + http: + request: + timeout: 3 diff --git a/test/integration/pack/test_bundles/vshard_app/test_app/instances.yaml b/test/integration/pack/test_bundles/vshard_app/test_app/instances.yaml new file mode 100644 index 000000000..910b35b9f --- /dev/null +++ b/test/integration/pack/test_bundles/vshard_app/test_app/instances.yaml @@ -0,0 +1,11 @@ +--- +storage-001-a: + +storage-001-b: + +storage-002-a: + +storage-002-b: + +router-001-a: + diff --git a/test/integration/pack/test_bundles/vshard_app/test_app/router.lua b/test/integration/pack/test_bundles/vshard_app/test_app/router.lua new file mode 100644 index 000000000..6d23f34e7 --- /dev/null +++ b/test/integration/pack/test_bundles/vshard_app/test_app/router.lua @@ -0,0 +1,34 @@ +local vshard = require('vshard') +local log = require('log') + +-- Bootstrap the vshard router. +while true do + local ok, err = vshard.router.bootstrap({ + if_not_bootstrapped = true, + }) + if ok then + break + end + log.info(('Router bootstrap error: %s'):format(err)) +end + +-- Put data into the cluster. +function put(id, name, age) + local bucket_id = vshard.router.bucket_id_mpcrc32({id}) + vshard.router.callrw(bucket_id, 'put', {id, bucket_id, name, age}) +end + +-- Get data from the cluster. +function get(id) + local bucket_id = vshard.router.bucket_id_mpcrc32({id}) + return vshard.router.callro(bucket_id, 'get', {id}) +end + +-- Put sample data. +function put_sample_data() + put(1, 'Elizabeth', 12) + put(2, 'Mary', 46) + put(3, 'David', 33) + put(4, 'William', 81) + put(5, 'Jack', 35) +end diff --git a/test/integration/pack/test_bundles/vshard_app/test_app/storage.lua b/test/integration/pack/test_bundles/vshard_app/test_app/storage.lua new file mode 100644 index 000000000..61aecd71c --- /dev/null +++ b/test/integration/pack/test_bundles/vshard_app/test_app/storage.lua @@ -0,0 +1,56 @@ +local vshard = require('vshard') + +-- Create 'customers' space. +box.once('customers', function() + box.schema.create_space('customers', { + format = {{ + name = 'id', + type = 'unsigned' + }, { + name = 'bucket_id', + type = 'unsigned' + }, { + name = 'name', + type = 'string' + }, { + name = 'age', + type = 'number' + }} + }) + box.space.customers:create_index('primary_index', { + parts = {{ + field = 1, + type = 'unsigned' + }} + }) + box.space.customers:create_index('bucket_id', { + parts = {{ + field = 2, + type = 'unsigned' + }}, + unique = false + }) + box.space.customers:create_index('age', { + parts = {{ + field = 4, + type = 'number' + }}, + unique = false + }) +end) + +-- Put data to the 'customers' space. +-- Function should be called by the router. +function put(id, bucket_id, name, age) + box.space.customers:insert({id, bucket_id, name, age}) +end + +-- Get data from the 'customers' space. +-- Function should be called by the router. +function get(id) + local tuple = box.space.customers:get(id) + if tuple == nil then + return nil + end + return {tuple.id, tuple.name, tuple.age} +end diff --git a/test/integration/pack/test_bundles/vshard_app/test_app/test_app-scm-1.rockspec b/test/integration/pack/test_bundles/vshard_app/test_app/test_app-scm-1.rockspec new file mode 100644 index 000000000..245497e8e --- /dev/null +++ b/test/integration/pack/test_bundles/vshard_app/test_app/test_app-scm-1.rockspec @@ -0,0 +1,11 @@ +package = 'test_app' +version = 'scm-1' +source = { + url = '/dev/null', +} +dependencies = { + 'vshard == 0.1.25' +} +build = { + type = 'none'; +} diff --git a/test/integration/pack/test_bundles/vshard_app/tt.yaml b/test/integration/pack/test_bundles/vshard_app/tt.yaml new file mode 100644 index 000000000..5a925d0dd --- /dev/null +++ b/test/integration/pack/test_bundles/vshard_app/tt.yaml @@ -0,0 +1,21 @@ +env: + bin_dir: bin + inc_dir: include + instances_enabled: instances.enabled + restart_on_failure: false + tarantoolctl_layout: false +modules: + directory: modules +app: + run_dir: var/run + log_dir: var/log + wal_dir: var/lib + memtx_dir: var/lib + vinyl_dir: var/lib +ee: + credential_path: "" +templates: +- path: templates +repo: + rocks: "" + distfiles: distfiles diff --git a/test/integration/pack/test_pack.py b/test/integration/pack/test_pack.py index e2ae3b2e5..b6c00fd93 100644 --- a/test/integration/pack/test_pack.py +++ b/test/integration/pack/test_pack.py @@ -55,6 +55,12 @@ def assert_single_app_env(config): assert config["env"]["bin_dir"] == "bin" +def assert_vshard_app_env(config): + assert config["env"]["instances_enabled"] == "instances.enabled" + assert config["env"]["bin_dir"] == "bin" + assert config["env"]["inc_dir"] == "include" + + def assert_artifacts_env(config): assert config["app"]["wal_dir"] == "var/lib" assert config["app"]["vinyl_dir"] == "var/lib" @@ -520,6 +526,36 @@ def prepare_tgz_test_cases(tt_cmd) -> list: ], "check_env": ["myapp", assert_single_app_env, assert_artifacts_env] }, + { + "name": "Vshard app packing", + "bundle_src": "vshard_app", + "cmd": tt_cmd, + "pack_type": "tgz", + "args": ["--name", "test_app"], + "res_file": "test_app-0.1.0.0." + get_arch() + ".tar.gz", + "check_exist": [ + os.path.join("bin", "tarantool"), + os.path.join("bin", "tt"), + os.path.join("instances.enabled", "test_app"), + "tt.yaml", + "test_app/config.yaml", + "test_app/instances.yaml", + "test_app/router.lua", + "test_app/storage.lua", + ], + "check_not_exist": [ + "test_app/test_app-scm-1.rockspec", + "test_app/tt.yaml", + "include", + "modules", + "templates", + os.path.join("test_app", "include"), + os.path.join("test_app", "modules"), + os.path.join("test_app", "templates"), + os.path.join("test_app", "distfiles"), + ], + "check_env": [".", assert_vshard_app_env, assert_artifacts_env] + }, ]