minor changes to:

1) ht_on in __init__.py
2) logging of exploration\exploitation

Author: kvas7andy

cyberbattle/__init__.py

@@ -124,15 +124,15 @@ def register(id: str, cyberbattle_env_identifiers: model.Identifiers, **kwargs):
 # )
 
 
-max_ht_on = len(ht_on)
+max_ht_on = len(configuration.honeytokens_on)
 for i in range(1, max_ht_on + 1):
     for j in range(i + 1, max_ht_on + 1):
 
         ver = str(i) + str(j)
         if 'CyberBattleTinyMicro-v' + ver in registry.env_specs:
             del registry.env_specs['CyberBattleTinyMicro-v' + ver]
 
-        ht_on = dict(zip(list(ht_on.keys()),
+        ht_on = dict(zip(list(sorted(configuration.honeytokens_on.keys())),
                          [True if k in [i, j] else False for k in range(1, max_ht_on + 1)]))
         tinymicro_deception_constructor.reconfigure_environment(ht_on)
 
@@ -150,14 +150,13 @@ def register(id: str, cyberbattle_env_identifiers: model.Identifiers, **kwargs):
             max_episode_steps=50,
         )
 
-max_ht_on = len(ht_on)
 for i in range(1, max_ht_on + 1):
 
     ver = ''.join([str(k) for k in range(1, max_ht_on + 1) if k != i])
     if 'CyberBattleTinyMicro-v' + ver in registry.env_specs:
         del registry.env_specs['CyberBattleTinyMicro-v' + ver]
 
-    ht_on = dict(zip(list(ht_on.keys()),
+    ht_on = dict(zip(list(sorted(configuration.honeytokens_on.keys())),
                      [True if k != i else False for k in range(1, max_ht_on + 1)]))
     tinymicro_deception_constructor.reconfigure_environment(ht_on)
 

cyberbattle/agents/baseline/agent_dql.py

@@ -415,7 +415,6 @@ def explore(self, wrapped_env: w.AgentWrapper
                 ) -> Tuple[str, cyberbattle_env.Action, object]:
         """Random exploration that avoids repeating actions previously taken in the same state"""
         # sample local and remote actions only (excludes connect action)
-        logger.info("Enter exploration phase instead of exploitation.")
         gym_action = wrapped_env.env.sample_valid_action(kinds=[0, 1, 2])
         metadata = self.metadata_from_gymaction(wrapped_env, gym_action)
         return "explore", gym_action, metadata

cyberbattle/agents/baseline/agent_wrapper.py

@@ -386,7 +386,7 @@ def specialize_to_gymaction(self, source_node: np.int32, observation, abstract_a
             if discovered_nodes_count <= 1:
                 return None
 
-            # # NOTE: We can do better here than random pick: ultimately this
+            # # NOTE: BEFORE We can do better here than random pick: ultimately this
             # # should be learnt from target node properties
             # # NOTE: NOW we have each vulnerability unique based on node where it was writter
             # # choosing remote vulnerability among identifiers.remote_vulnerabilities,

cyberbattle/agents/baseline/learner.py

@@ -561,9 +561,11 @@ def epsilon_greedy_search(
             x = np.random.rand()
             if x <= epsilon:
                 action_style, gym_action, action_metadata = learner.explore(wrapped_env)
+                logger.info("Choose exploration phase")
             else:
                 action_style, gym_action, action_metadata = learner.exploit(wrapped_env, observation)
                 if not gym_action:
+                    logger.info("Enter exploration phase instead of exploitation")
                     stats['exploit_deflected_to_explore'] += 1
                     _, gym_action, action_metadata = learner.explore(wrapped_env)
 

cyberbattle/agents/baseline/notebooks/notebook_dql_debug_with_tinymicro.py

@@ -50,7 +50,7 @@
 log_results = os.getenv("LOG_RESULTS", 'False').lower() in ('true', '1', 't')
 gymid = os.getenv("GYMID", 'CyberBattleTinyMicro-v0')
 log_level = os.getenv('LOG_LEVEL', "info")
-seed = int(os.getenv('SEED', 0))
+seed = float(os.getenv('SEED', 0))
 iteration_count = None
 honeytokens_on = None
 training_episode_count = None