Merge branch 'master' into tsc/bishopbm1

Author: cognifloyd

.gitignore

@@ -50,6 +50,12 @@ nosetests.xml
 htmlcov
 benchmark_histograms/
 
+# Pants workspace files
+/.pants.d/
+/dist/
+/.pids
+/.pants.workdir.file_lock*
+
 # Mr Developer
 .idea
 .DS_Store

CHANGELOG.rst

@@ -19,6 +19,10 @@ Fixed
 
   Contributed by @S-T-A-R-L-O-R-D
 
+* Fixed a bug where calling 'get_by_name' on client for getting key details was not returning any results despite key being stored. #5677
+
+  Contributed by @bharath-orchestral
+
 
 * Fixed ``st2client/st2client/base.py`` file to use ``https_proxy``(not ``http_proxy``) to check HTTPS_PROXY environment variables.
 
@@ -29,6 +33,12 @@ Fixed
 
 * Fixed eventlet monkey patching so more of the unit tests work under pytest. #5689
 
+* Fix and reenable prance-based openapi spec validation, but make our custom ``x-api-model`` validation optional as the spec is out-of-date. #5709
+  Contributed by @cognifloyd
+
+* Fixed generation of `st2.conf.sample` to show correct syntax for `[sensorcontainer].partition_provider` (space separated `key:value` pairs). #5710
+  Contributed by @cognifloyd
+
 Added
 ~~~~~
 
@@ -41,6 +51,11 @@ Added
 * Added purging of old tokens. #5679
   Contributed by Amanda McGuinness (@amanda11 intive)
 
+* Begin introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
+  working on StackStorm, improve our security posture, and improve CI reliability thanks in part
+  to pants' use of PEX lockfiles. This is not a user-facing addition. #5713
+  Contributed by @cognifloyd
+
 Changed
 ~~~~~~~
 
@@ -91,7 +106,10 @@ Changed
 * Move from udatetime to ciso8601 for date functionality ahead of supporting python3.9 #5692
   Contributed by Amanda McGuinness (@amanda11 intive)
 
-* Refactor tests to use python imports to identify test fixtures. #5699
+* Refactor tests to use python imports to identify test fixtures. #5699 #5702 #5703 #5704 #5705 #5706
+  Contributed by @cognifloyd
+
+* Refactor ``st2-generate-schemas`` so that logic is in an importable module. #5708
   Contributed by @cognifloyd
 
 Removed

Makefile

@@ -317,10 +317,7 @@ configgen: requirements .configgen
 	@echo
 	@echo "================== config gen ===================="
 	@echo
-	echo "# Sample config which contains all the available options which the corresponding descriptions" > conf/st2.conf.sample;
-	echo "# Note: This file is automatically generated using tools/config_gen.py - DO NOT UPDATE MANUALLY" >> conf/st2.conf.sample
-	echo "" >> conf/st2.conf.sample
-	. $(VIRTUALENV_DIR)/bin/activate; python ./tools/config_gen.py >> conf/st2.conf.sample;
+	. $(VIRTUALENV_DIR)/bin/activate; python ./tools/config_gen.py > conf/st2.conf.sample;
 
 .PHONY: schemasgen
 schemasgen: requirements .schemasgen

conf/st2.conf.sample

@@ -278,7 +278,7 @@ version = 4
 # location of the logging.conf file
 logging = /etc/st2/logging.sensorcontainer.conf
 # Provider of sensor node partition config.
-partition_provider = {'name': 'default'}
+partition_provider = name:default
 # name of the sensor node.
 sensor_node_name = sensornode1
 # Run in a single sensor mode where parent process exits when a sensor crashes / dies. This is useful in environments where partitioning, sensor process life cycle and failover is handled by a 3rd party service such as kubernetes.

contrib/core/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 2022 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from st2tests import fixturesloader
+
+PACK_NAME, PACK_PATH = fixturesloader.get_fixture_name_and_path(__file__)

contrib/core/fixture.py

@@ -25,6 +25,7 @@
 
 from st2common.constants import action as action_constants
 
+from st2tests.fixtures.packs.core.fixture import PACK_NAME
 from st2tests.fixturesloader import FixturesLoader
 from st2tests.base import RunnerTestCase
 from st2tests.base import CleanDbTestCase
@@ -38,6 +39,10 @@
 MOCK_EXECUTION.id = "598dbf0c0640fd54bffc688b"
 HOSTNAME = socket.gethostname()
 
+# we need the core pack to also be in st2tests.fixtures so we can use FixturesLoader()
+# The PACK_NAME import tells pants to include that, so use the var here.
+FIXTURE_PACK = "packs/" + PACK_NAME
+
 
 class SendmailActionTestCase(RunnerTestCase, CleanDbTestCase, CleanFilesTestCase):
     """
@@ -237,11 +242,11 @@ def _run_action(self, action_parameters):
         parse the output email data.
         """
         models = self.fixtures_loader.load_models(
-            fixtures_pack="packs/core", fixtures_dict={"actions": ["sendmail.yaml"]}
+            fixtures_pack=FIXTURE_PACK, fixtures_dict={"actions": ["sendmail.yaml"]}
         )
         action_db = models["actions"]["sendmail.yaml"]
         entry_point = self.fixtures_loader.get_fixture_file_path_abs(
-            "packs/core", "actions", "send_mail/send_mail"
+            FIXTURE_PACK, "actions", "send_mail/send_mail"
         )
 
         runner = self._get_runner(action_db, entry_point=entry_point)

contrib/core/tests/test_action_sendmail.py

@@ -0,0 +1,16 @@
+# Copyright 2022 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+FIXTURES_DIR = os.path.dirname(os.path.abspath(__file__))

contrib/packs/tests/__init__.py

@@ -37,7 +37,8 @@
 
 from pack_mgmt.download import DownloadGitRepoAction
 
-BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+# this import allows pants to detect that the fixtures are used in this file
+from .fixtures import FIXTURES_DIR
 
 PACK_INDEX = {
     "test": {
@@ -639,7 +640,7 @@ def test_run_pack_dowload_local_git_repo_detached_head_state(self):
             side_effect=TypeError("detached head")
         )
 
-        pack_path = os.path.join(BASE_DIR, "fixtures/stackstorm-test")
+        pack_path = os.path.join(FIXTURES_DIR, "stackstorm-test")
 
         result = action.run(
             packs=["file://%s" % (pack_path)], abs_repo_base=self.repo_base
@@ -665,7 +666,7 @@ def test_run_pack_download_local_directory(self):
         )
 
         # 2. Local pack which is not a git repository
-        pack_path = os.path.join(BASE_DIR, "fixtures/stackstorm-test4")
+        pack_path = os.path.join(FIXTURES_DIR, "stackstorm-test4")
 
         result = action.run(
             packs=["file://%s" % (pack_path)], abs_repo_base=self.repo_base

contrib/packs/tests/fixtures/__init__.py

@@ -35,6 +35,7 @@
 from st2common.util import action_db as action_db_util
 from st2common.exceptions.action import ParameterRenderingFailedException
 from st2tests import ExecutionDbTestCase
+from st2tests.fixtures.generic.fixture import PACK_NAME as FIXTURES_PACK
 from st2tests.fixturesloader import FixturesLoader
 
 
@@ -45,8 +46,6 @@ def __init__(self, status=LIVEACTION_STATUS_SUCCEEDED, result=""):
         self.result = result
 
 
-FIXTURES_PACK = "generic"
-
 TEST_MODELS = {
     "actions": ["a1.yaml", "a2.yaml", "action_4_action_context_param.yaml"],
     "runners": ["testrunner1.yaml"],

contrib/packs/tests/test_action_download.py

@@ -35,7 +35,11 @@
 from st2common.transport.publishers import CUDPublisher
 
 from st2tests import ExecutionDbTestCase
-from st2tests import fixturesloader
+from st2tests.fixtures.packs.action_chain_tests.fixture import (
+    PACK_NAME as TEST_PACK,
+    PACK_PATH as TEST_PACK_PATH,
+)
+from st2tests.fixtures.packs.core.fixture import PACK_PATH as CORE_PACK_PATH
 from st2tests.mocks.liveaction import MockLiveActionPublisherNonBlocking
 from six.moves import range
 
@@ -45,10 +49,7 @@
     "actions": ["test_cancel.yaml", "test_cancel_with_subworkflow.yaml"],
 }
 
-TEST_PACK = "action_chain_tests"
-TEST_PACK_PATH = fixturesloader.get_fixtures_packs_base_path() + "/" + TEST_PACK
-
-PACKS = [TEST_PACK_PATH, fixturesloader.get_fixtures_packs_base_path() + "/core"]
+PACKS = [TEST_PACK_PATH, CORE_PACK_PATH]
 
 USERNAME = "stanley"
 

contrib/runners/action_chain_runner/tests/unit/test_actionchain.py

@@ -35,6 +35,12 @@
 from st2common.transport.liveaction import LiveActionPublisher
 from st2common.transport.publishers import CUDPublisher
 
+from st2tests.fixtures.generic.fixture import PACK_NAME as FIXTURES_PACK
+from st2tests.fixtures.packs.action_chain_tests.fixture import (
+    PACK_NAME as TEST_PACK,
+    PACK_PATH as TEST_PACK_PATH,
+)
+from st2tests.fixtures.packs.core.fixture import PACK_PATH as CORE_PACK_PATH
 from st2tests.mocks.liveaction import MockLiveActionPublisherNonBlocking
 
 
@@ -45,8 +51,6 @@ def __init__(self, status=action_constants.LIVEACTION_STATUS_SUCCEEDED, result="
         self.result = result
 
 
-FIXTURES_PACK = "generic"
-
 TEST_MODELS = {"actions": ["a1.yaml", "a2.yaml"], "runners": ["testrunner1.yaml"]}
 
 MODELS = fixturesloader.FixturesLoader().load_models(
@@ -60,10 +64,7 @@ def __init__(self, status=action_constants.LIVEACTION_STATUS_SUCCEEDED, result="
     FIXTURES_PACK, "actionchains", "chain_with_notifications.yaml"
 )
 
-TEST_PACK = "action_chain_tests"
-TEST_PACK_PATH = fixturesloader.get_fixtures_packs_base_path() + "/" + TEST_PACK
-
-PACKS = [TEST_PACK_PATH, fixturesloader.get_fixtures_packs_base_path() + "/core"]
+PACKS = [TEST_PACK_PATH, CORE_PACK_PATH]
 
 MOCK_NOTIFY = {
     "on-complete": {

contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py

@@ -37,7 +37,11 @@
 from st2common.util import date as date_utils
 
 from st2tests import ExecutionDbTestCase
-from st2tests import fixturesloader
+from st2tests.fixtures.packs.action_chain_tests.fixture import (
+    PACK_NAME as TEST_PACK,
+    PACK_PATH as TEST_PACK_PATH,
+)
+from st2tests.fixtures.packs.core.fixture import PACK_PATH as CORE_PACK_PATH
 from st2tests.mocks.liveaction import MockLiveActionPublisherNonBlocking
 from six.moves import range
 
@@ -67,10 +71,7 @@
     ],
 }
 
-TEST_PACK = "action_chain_tests"
-TEST_PACK_PATH = fixturesloader.get_fixtures_packs_base_path() + "/" + TEST_PACK
-
-PACKS = [TEST_PACK_PATH, fixturesloader.get_fixtures_packs_base_path() + "/core"]
+PACKS = [TEST_PACK_PATH, CORE_PACK_PATH]
 
 USERNAME = "stanley"