Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 53 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ Options:
(e.g., localhost,127.0.0.1).
--parallel-pagination Enable parallel pagination for faster case fetching
(experimental).
--dry-run Preview write operations without sending mutating
requests to TestRail.
--help Show this message and exit.

Commands:
Expand All @@ -100,6 +102,55 @@ Commands:
update Update TRCLI to the latest version from PyPI.
```

### Dry-run mode

TRCLI supports a client-side preview mode via `--dry-run`.

In dry-run mode:
- local parsing and validation still run
- read-only API calls may still run
- mutating requests to TestRail are not sent

Current dry-run support is aimed at write-oriented commands such as:
- `add_run`
- `import_gherkin`
- parser commands (`parse_junit`, `parse_robot`, `parse_cucumber`, `parse_openapi`)
- mutating label/reference commands

Example:

```shell
trcli --dry-run parse_junit -f results.xml --title "Nightly Run"
```

Important limitations:
- this is a TRCLI-side preview, not a server-side transaction preview
- TestRail IDs for newly created resources are not generated in dry-run mode
- parser dry-run summaries are based on local parsing and do not complete the full write workflow

### JSON output

TRCLI now supports machine-readable output for commands that naturally produce structured results.

Current `--json` support includes:
- `status`
- `add_run`
- parser commands: `parse_junit`, `parse_robot`, `parse_cucumber`, `parse_openapi`
- `import_gherkin`

Notes:
- `--json-output` is still accepted as a compatibility alias on commands that already exposed it.
- In JSON mode, stdout is intended for the final JSON document. Human-readable progress output is redirected away from stdout.
- Commands still use normal exit codes. In JSON mode, a failing command returns a non-zero exit code and emits an `"ok": false` payload when the failure can be represented structurally.

Example:

```shell
trcli status --json
trcli add_run --title "Nightly Run" --suite-id 1 --json
trcli parse_junit -f results.xml --title "Nightly Run" --json
```

Uploading automated test results
--------

Expand Down Expand Up @@ -151,7 +202,8 @@ Options:
test results to.
--test-run-ref Comma-separated list of reference IDs to append to the
test run (up to 250 characters total).
--json-output Output reference operation results in JSON format.
--json-output, --json
Output structured results in JSON format.
--update-existing-cases Update existing TestRail cases with values from
JUnit properties (default: no).
--update-strategy Strategy for combining incoming values with
Expand Down
30 changes: 30 additions & 0 deletions tests/test_api_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,36 @@ def test_send_post_status_code_success(self, api_resources, requests_mock):
check_calls_count(requests_mock)
check_response(201, FAKE_PROJECT_DATA, "", response)

@pytest.mark.api_client
@patch("requests.post")
def test_send_post_is_suppressed_in_dry_run_mode(self, mock_post, api_resources_maker, mocker):
environment = mocker.patch("trcli.cli.Environment")
api_client = api_resources_maker(environment=environment)
api_client.dry_run = True

response = api_client.send_post("add_project", {"name": "Preview Project"})

mock_post.assert_not_called()
check_response(
200,
{"uri": "add_project", "payload": {"name": "Preview Project"}, "has_files": False, "dry_run": True},
"",
response,
)
environment.log.assert_any_call("Dry run: skipping POST add_project")

@pytest.mark.api_client
def test_send_get_still_executes_in_dry_run_mode(self, api_resources_maker, requests_mock, mocker):
requests_mock.get(create_url("get_projects"), status_code=200, json=FAKE_PROJECT_DATA)
environment = mocker.patch("trcli.cli.Environment")
api_client = api_resources_maker(environment=environment)
api_client.dry_run = True

response = api_client.send_get("get_projects")

check_calls_count(requests_mock)
check_response(200, FAKE_PROJECT_DATA, "", response)

@pytest.mark.api_client
def test_send_get_status_code_not_success(self, api_resources, requests_mock):
"""The purpose of this test is to check behaviour of send_get one receiving not successful status code.
Expand Down
30 changes: 28 additions & 2 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from pathlib import Path

import pytest
import click

from click.testing import CliRunner

Expand Down Expand Up @@ -63,6 +64,29 @@ def test_run_with_help_parameter(self, cli_resources):
"Options:" in result.output
), "'Options:' is not present in output message when calling trcli with --help parameter."

@pytest.mark.cli
def test_help_uses_colored_palette(self):
ctx = click.Context(cli, info_name="trcli", color=True)
help_text = cli.get_help(ctx)

assert "\x1b[33mUsage: " in help_text
assert "\x1b[93mparse_junit\x1b[0m" in help_text
assert "\x1b[93m-h, --host " in help_text
assert "\x1b[93m--dry-run\x1b[0m" in help_text

@pytest.mark.cli
def test_dry_run_option_is_set_on_environment(self, mocker, cli_resources):
cli_agrs_helper, cli_runner = cli_resources
args = cli_agrs_helper.get_all_required_parameters_plus_optional(["--dry-run"])

mocker.patch("sys.argv", ["trcli", *args])
setattr_mock = mocker.patch("trcli.cli.setattr")

with cli_runner.isolated_filesystem():
_ = cli_runner.invoke(cli, args)

setattr_mock.assert_any_call(mocker.ANY, "dry_run", True)

@pytest.mark.cli
def test_run_without_command(self, mocker, cli_resources):
"""The purpose of this test is to check that calling trcli without command will result is
Expand Down Expand Up @@ -99,7 +123,8 @@ def test_check_error_message_for_required_parameters(
)

mocker.patch("sys.argv", ["trcli", *args])
result = cli_runner.invoke(cli, args)
with cli_runner.isolated_filesystem():
result = cli_runner.invoke(cli, args)
assert (
result.exit_code == expected_exit_code
), f"Exit code {expected_exit_code} expected. Got: {result.exit_code} instead."
Expand All @@ -121,7 +146,8 @@ def test_host_syntax_is_validated(self, host, cli_resources, mocker):
args = ["--host", host, *args]

mocker.patch("sys.argv", ["trcli", *args])
result = cli_runner.invoke(cli, args)
with cli_runner.isolated_filesystem():
result = cli_runner.invoke(cli, args)
assert (
result.exit_code == expected_exit_code
), f"Exit code {expected_exit_code} expected. Got: {result.exit_code} instead."
Expand Down
148 changes: 148 additions & 0 deletions tests/test_cmd_add_run.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
from unittest import mock
import json
import pytest
from click.testing import CliRunner

from trcli.cli import cli as root_cli
from trcli.cli import Environment
from trcli.commands import cmd_add_run

Expand Down Expand Up @@ -114,6 +116,152 @@ def test_refs_action_parameter_parsing(self):
assert "--run-refs-action" in result.output
assert "Action to perform on references" in result.output

@mock.patch("trcli.commands.cmd_add_run.write_run_to_file")
@mock.patch("trcli.commands.cmd_add_run.ProjectBasedClient")
@mock.patch("trcli.cli.check_for_updates", return_value=None)
def test_dry_run_does_not_write_run_file(
self, _mock_update_check, mock_project_client_class, mock_write_run_to_file
):
runner = CliRunner()
mock_project_client = mock_project_client_class.return_value
mock_project_client.resolve_project.return_value = None
mock_project_client.resolve_suite.return_value = None
mock_project_client.create_or_update_test_run.return_value = (0, "")

args = [
"--host",
"https://example.testrail.io",
"--project",
"Example Project",
"--username",
"user@example.com",
"--key",
"secret",
"--dry-run",
"add_run",
"--title",
"Preview Run",
"--suite-id",
"1",
"--file",
"preview.yml",
]

with runner.isolated_filesystem():
with mock.patch("sys.argv", ["trcli", *args]):
result = runner.invoke(root_cli, args)

assert result.exit_code == 0
assert "Dry run: no TestRail changes were made." in result.output
assert "run_id: <not created>" in result.output
mock_write_run_to_file.assert_not_called()

@mock.patch("trcli.commands.cmd_add_run.ProjectBasedClient")
@mock.patch("trcli.cli.check_for_updates", return_value=None)
def test_add_run_json_output_create(self, _mock_update_check, mock_project_client_class):
runner = CliRunner()
mock_project_client = mock_project_client_class.return_value
mock_project_client.resolve_project.return_value = None
mock_project_client.resolve_suite.return_value = None
mock_project_client.create_or_update_test_run.return_value = (321, "")

args = [
"--host",
"https://example.testrail.io",
"--project",
"Example Project",
"--username",
"user@example.com",
"--key",
"secret",
"add_run",
"--title",
"JSON Run",
"--suite-id",
"1",
"--json",
]

with mock.patch("sys.argv", ["trcli", *args]):
result = runner.invoke(root_cli, args)

assert result.exit_code == 0
payload = json.loads(result.output[result.output.find("{"):])
assert payload["ok"] is True
assert payload["command"] == "add_run"
assert payload["data"]["action"] == "create"
assert payload["data"]["run_id"] == 321
assert payload["data"]["title"] == "JSON Run"

@mock.patch("trcli.commands.cmd_add_run.ProjectBasedClient")
@mock.patch("trcli.cli.check_for_updates", return_value=None)
def test_add_run_json_output_dry_run(self, _mock_update_check, mock_project_client_class):
runner = CliRunner()
mock_project_client = mock_project_client_class.return_value
mock_project_client.resolve_project.return_value = None
mock_project_client.resolve_suite.return_value = None
mock_project_client.create_or_update_test_run.return_value = (0, "")

args = [
"--host",
"https://example.testrail.io",
"--project",
"Example Project",
"--username",
"user@example.com",
"--key",
"secret",
"--dry-run",
"add_run",
"--title",
"Preview Run",
"--suite-id",
"1",
"--json",
]

with mock.patch("sys.argv", ["trcli", *args]):
result = runner.invoke(root_cli, args)

assert result.exit_code == 0
payload = json.loads(result.output[result.output.find("{"):])
assert payload["ok"] is True
assert payload["dry_run"] is True
assert payload["data"]["action"] == "create"
assert payload["data"]["run_id"] is None

@mock.patch("trcli.cli.check_for_updates", return_value=None)
def test_add_run_json_output_validation_error(self, _mock_update_check):
runner = CliRunner()
long_refs = "A" * 251

args = [
"--host",
"https://example.testrail.io",
"--project",
"Example Project",
"--username",
"user@example.com",
"--key",
"secret",
"add_run",
"--title",
"JSON Run",
"--suite-id",
"1",
"--run-refs",
long_refs,
"--json",
]

with mock.patch("sys.argv", ["trcli", *args]):
result = runner.invoke(root_cli, args)

assert result.exit_code == 1
payload = json.loads(result.output[result.output.find("{"):])
assert payload["ok"] is False
assert "References field cannot exceed 250 characters." in payload["errors"][0]


class TestApiRequestHandlerReferences:
"""Test class for reference management functionality"""
Expand Down
Loading