add further tests and fixes

.cspell/custom-dictionary.txt

@@ -34,6 +34,7 @@ dapolymatrix
 dataconverter
 dataframe
 delaystage
+delenv
 dtype
 dxda
 dxde

docs/specsscan/helpers.rst

@@ -3,3 +3,7 @@ Helpers
 .. automodule:: specsscan.helpers
    :members:
    :undoc-members:
+
+.. automodule:: specsscan.metadata
+   :members:
+   :undoc-members:

src/specsanalyzer/config.py

@@ -25,6 +25,7 @@
     if platform.system() == "Windows"
     else Path("/etc/").joinpath("specsanalyzer")
 )
+ENV_DIR = Path(".env")
 
 # Configure logging
 logger = setup_logging("config")
@@ -276,7 +277,7 @@ def read_env_var(var_name: str) -> str | None:
         return value
 
     # 2. check .env in current directory
-    local_vars = _parse_env_file(Path(".env"))
+    local_vars = _parse_env_file(ENV_DIR)
     if var_name in local_vars:
         logger.debug(f"Found {var_name} in ./.env file")
         return local_vars[var_name]

src/specsscan/metadata.py

@@ -6,6 +6,7 @@
 
 import datetime
 import json
+from copy import deepcopy
 from urllib.error import HTTPError
 from urllib.error import URLError
 from urllib.request import urlopen
@@ -36,6 +37,8 @@ def __init__(self, metadata_config: dict, token: str = None) -> None:
             token (str, optional): The token to use for fetching metadata. If provided,
                 will be saved to .env file for future use.
         """
+        self._config = deepcopy(metadata_config)
+
         # Token handling
         if token:
             self.token = token
@@ -44,17 +47,24 @@ def __init__(self, metadata_config: dict, token: str = None) -> None:
             # Try to load token from config or .env file
             self.token = read_env_var("ELAB_TOKEN")
 
-        self._config = metadata_config
+        if not self.token:
+            logger.warning(
+                "No valid token provided for elabFTW. Fetching elabFTW metadata will be skipped.",
+            )
+            return
 
-        self.url = str(metadata_config.get("elab_url"))
+        self.url = metadata_config.get("elab_url")
         if not self.url:
-            raise ValueError("No URL provided for fetching metadata from elabFTW.")
+            logger.warning(
+                "No URL provided for elabFTW. Fetching elabFTW metadata will be skipped.",
+            )
+            return
 
         # Config
         self.configuration = elabapi_python.Configuration()
         self.configuration.api_key["api_key"] = self.token
         self.configuration.api_key_prefix["api_key"] = "Authorization"
-        self.configuration.host = self.url
+        self.configuration.host = str(self.url)
         self.configuration.debug = False
         self.configuration.verify_ssl = False
 
@@ -95,7 +105,7 @@ def fetch_epics_metadata(self, ts_from: float, ts_to: float, metadata: dict) ->
         except KeyError:
             epics_channels = []
 
-        channels_missing = set(epics_channels) - set(metadata["scan_info"].keys())
+        channels_missing = set(epics_channels) - set(metadata.get("scan_info", {}).keys())
         if channels_missing:
             logger.info("Collecting data from the EPICS archive...")
             for channel in channels_missing:
@@ -147,6 +157,14 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict:
                 "a token parameter or set the ELAB_TOKEN environment variable.",
             )
             return metadata
+
+        if not self.url:
+            logger.warning(
+                "No URL provided for fetching metadata from elabFTW. "
+                "Fetching elabFTW metadata will be skipped.",
+            )
+            return metadata
+
         logger.info("Collecting data from the elabFTW instance...")
         # Get the experiment
         try:

tests/test_config.py

@@ -8,7 +8,9 @@
 from specsanalyzer.config import complete_dictionary
 from specsanalyzer.config import load_config
 from specsanalyzer.config import parse_config
+from specsanalyzer.config import read_env_var
 from specsanalyzer.config import save_config
+from specsanalyzer.config import save_env_var
 
 test_dir = os.path.dirname(__file__)
 default_config_keys = [
@@ -88,3 +90,140 @@ def test_save_dict():
             save_config(config_dict, filename, overwrite=True)
             config = load_config(filename)
             assert "test_entry" not in config.keys()
+
+
+@pytest.fixture
+def mock_env_file(tmp_path, monkeypatch):
+    """Mock the .env file for testing"""
+    monkeypatch.setattr("specsanalyzer.config.USER_CONFIG_PATH", tmp_path)
+    yield tmp_path
+
+
+def test_env_var_read_write(mock_env_file):  # noqa: ARG001
+    """Test reading and writing environment variables."""
+    # Test writing a new variable
+    save_env_var("TEST_VAR", "test_value")
+    assert read_env_var("TEST_VAR") == "test_value"
+
+    # Test writing multiple variables
+    save_env_var("TEST_VAR2", "test_value2")
+    assert read_env_var("TEST_VAR") == "test_value"
+    assert read_env_var("TEST_VAR2") == "test_value2"
+
+    # Test overwriting an existing variable
+    save_env_var("TEST_VAR", "new_value")
+    assert read_env_var("TEST_VAR") == "new_value"
+    assert read_env_var("TEST_VAR2") == "test_value2"  # Other variables unchanged
+
+    # Test reading non-existent variable
+    assert read_env_var("NON_EXISTENT_VAR") is None
+
+
+def test_env_var_read_no_file(mock_env_file):  # noqa: ARG001
+    """Test reading environment variables when .env file doesn't exist."""
+    # Test reading from non-existent file
+    assert read_env_var("TEST_VAR") is None
+
+
+def test_env_var_special_characters(mock_env_file):  # noqa: ARG001
+    """Test reading and writing environment variables with special characters."""
+    test_cases = {
+        "TEST_URL": "http://example.com/path?query=value",
+        "TEST_PATH": "/path/to/something/with/spaces and special=chars",
+        "TEST_QUOTES": "value with 'single' and \"double\" quotes",
+    }
+
+    for var_name, value in test_cases.items():
+        save_env_var(var_name, value)
+        assert read_env_var(var_name) == value
+
+
+def test_env_var_precedence(mock_env_file, tmp_path, monkeypatch):  # noqa: ARG001
+    """Test that environment variables are read in correct order of precedence"""
+    # Create local .env directory if it doesn't exist
+    local_env_dir = tmp_path / "local"
+    local_env_dir.mkdir(exist_ok=True)
+    system_env_dir = tmp_path / "system"
+    system_env_dir.mkdir(exist_ok=True)
+    monkeypatch.setattr("specsanalyzer.config.ENV_DIR", local_env_dir / ".env")
+    monkeypatch.setattr("specsanalyzer.config.SYSTEM_CONFIG_PATH", system_env_dir)
+
+    # Set up test values in different locations
+    os.environ["TEST_VAR"] = "os_value"
+
+    # Save to system config first (4th precedence)
+    with open(system_env_dir / ".env", "w") as f:
+        f.write("TEST_VAR=system_value\n")
+
+    # Save to user config first (3rd precedence)
+    save_env_var("TEST_VAR", "user_value")
+
+    # Create local .env file (2nd precedence)
+    with open(local_env_dir / ".env", "w") as f:
+        f.write("TEST_VAR=local_value\n")
+
+    assert read_env_var("TEST_VAR") == "os_value"
+
+    # Remove from OS env to test other precedence levels
+    monkeypatch.delenv("TEST_VAR", raising=False)
+    assert read_env_var("TEST_VAR") == "local_value"
+
+    # Remove local .env and should get user config value
+    (local_env_dir / ".env").unlink()
+    assert read_env_var("TEST_VAR") == "user_value"
+
+    # Remove user config and should get system value
+    (mock_env_file / ".env").unlink()
+    assert read_env_var("TEST_VAR") == "system_value"
+
+    # Remove system config and should get None
+    (system_env_dir / ".env").unlink()
+    assert read_env_var("TEST_VAR") is None
+
+
+def test_env_var_save_and_load(mock_env_file, monkeypatch):  # noqa: ARG001
+    """Test saving and loading environment variables"""
+    # Clear any existing OS environment variables
+    monkeypatch.delenv("TEST_VAR", raising=False)
+    monkeypatch.delenv("OTHER_VAR", raising=False)
+
+    # Save a variable
+    save_env_var("TEST_VAR", "test_value")
+
+    # Should be able to read it back
+    assert read_env_var("TEST_VAR") == "test_value"
+
+    # Save another variable - should preserve existing ones
+    save_env_var("OTHER_VAR", "other_value")
+    assert read_env_var("TEST_VAR") == "test_value"
+    assert read_env_var("OTHER_VAR") == "other_value"
+
+
+def test_env_var_not_found(mock_env_file):  # noqa: ARG001
+    """Test behavior when environment variable is not found"""
+    assert read_env_var("NONEXISTENT_VAR") is None
+
+
+def test_env_file_format(mock_env_file, monkeypatch):  # noqa: ARG001
+    """Test that .env file parsing handles different formats correctly"""
+    # Clear any existing OS environment variables
+    monkeypatch.delenv("TEST_VAR", raising=False)
+    monkeypatch.delenv("SPACES_VAR", raising=False)
+    monkeypatch.delenv("EMPTY_VAR", raising=False)
+    monkeypatch.delenv("COMMENT", raising=False)
+
+    with open(mock_env_file / ".env", "w") as f:
+        f.write(
+            """
+                TEST_VAR=value1
+                SPACES_VAR  =  value2
+                EMPTY_VAR=
+                #COMMENT=value3
+                INVALID_LINE
+                """,
+        )
+
+    assert read_env_var("TEST_VAR") == "value1"
+    assert read_env_var("SPACES_VAR") == "value2"
+    assert read_env_var("EMPTY_VAR") == ""
+    assert read_env_var("COMMENT") is None

tests/test_specsscan_metadata.py

@@ -11,6 +11,7 @@
 
 from specsscan.metadata import get_archiver_data
 from specsscan.metadata import MetadataRetriever
+from tests.test_config import mock_env_file  # noqa: F401
 
 
 @pytest.fixture
@@ -33,7 +34,7 @@ def metadata_config():
 
 
 @pytest.fixture
-def metadata_retriever(metadata_config):
+def metadata_retriever(metadata_config, mock_env_file):  # noqa: ARG001
     return MetadataRetriever(metadata_config, "dummy_token")
 
 
@@ -42,6 +43,30 @@ def test_metadata_retriever_init(metadata_retriever):
     assert metadata_retriever.url == "http://example.com"
 
 
+def test_metadata_retriever_no_token(metadata_config, tmp_path, monkeypatch):
+    monkeypatch.setattr("specsanalyzer.config.ENV_DIR", tmp_path / ".dummy_env")
+    monkeypatch.setattr("specsanalyzer.config.SYSTEM_CONFIG_PATH", tmp_path / "system")
+    monkeypatch.setattr("specsanalyzer.config.USER_CONFIG_PATH", tmp_path / "user")
+    retriever = MetadataRetriever(metadata_config)
+    assert retriever.token is None
+
+    metadata = {}
+    runs = ["run1"]
+    updated_metadata = retriever.fetch_elab_metadata(runs, metadata)
+    assert updated_metadata == metadata
+
+
+def test_metadata_retriever_no_url(metadata_config, mock_env_file):  # noqa: ARG001
+    metadata_config.pop("elab_url")
+    retriever = MetadataRetriever(metadata_config, "dummy_token")
+    assert retriever.url is None
+
+    metadata = {}
+    runs = ["run1"]
+    updated_metadata = retriever.fetch_elab_metadata(runs, metadata)
+    assert updated_metadata == metadata
+
+
 @patch("specsscan.metadata.urlopen")
 def test_get_archiver_data(mock_urlopen):
     """Test get_archiver_data using a mock of urlopen."""
@@ -75,8 +100,21 @@ def test_fetch_epics_metadata(mock_get_archiver_data, metadata_retriever):
     assert updated_metadata["scan_info"]["channel1"] == 10
 
 
+@patch("sed.loader.mpes.metadata.get_archiver_data")
+def test_fetch_epics_metadata_missing_channels(mock_get_archiver_data, metadata_retriever):
+    """Test fetch_epics_metadata with missing EPICS channels."""
+    mock_get_archiver_data.return_value = (np.array([1.5]), np.array([10]))
+    metadata = {"file": {"channel1": 10}}
+    ts_from = datetime.datetime(2023, 1, 1).timestamp()
+    ts_to = datetime.datetime(2023, 1, 2).timestamp()
+
+    updated_metadata = metadata_retriever.fetch_epics_metadata(ts_from, ts_to, metadata)
+
+    assert "channel1" in updated_metadata["file"]
+
+
 @patch("specsscan.metadata.elabapi_python")
-def test_fetch_elab_metadata(mock_elabapi_python, metadata_config):
+def test_fetch_elab_metadata(mock_elabapi_python, metadata_config, mock_env_file):  # noqa: ARG001
     """Test fetch_elab_metadata using a mock of elabapi_python."""
     mock_experiment = MagicMock()
     mock_experiment.id = 1