setup.py to run exmples accessing package globally

Author: ivanbelenky

examples/blackjack.py

@@ -1,5 +1,7 @@
 import random
 
+from rl.solvers import alpha_mc 
+
 VALUES = ['A','2','3','4','5','6','7','8','9','10','J','Q','K']
 SUITS = ['♠','♥','♦','♣']
 CARDS = [(value,suit) for value in VALUES for suit in SUITS]
@@ -59,7 +61,6 @@ def black_jack_transition(state, action):
             new_state = (player_sum, usable_ace, dealer_showing)
             return (new_state, 0.), False
 
-
     dealer_cards = [dealer_showing]
     dealer_sum = count(dealer_cards)
     if action == 'stand':
@@ -79,4 +80,10 @@ def black_jack_transition(state, action):
     elif dealer_sum < player_sum:
         return (state, 1.), True
     elif dealer_sum == player_sum:
-        return (state, 0.), True
+        return (state, 0.), True
+
+
+vqpi, samples = alpha_mc(states, actions, black_jack_transition, gamma=0.9,
+    use_N=True, n_episodes=1E4, first_visit=False)
+
+ 

examples/gridworld.py

@@ -5,7 +5,7 @@
 
 import numpy as np
 
-from src.mdp import MDP, TabularReward
+from rl.mdp import MDP, TabularReward
 
 GRID_SIZE = 5 # 5x5 gridworld
 

rl/__init__.py

@@ -0,0 +1,17 @@
+from .model_free import ModelFree, ModelFreePolicy, EpsilonSoftPolicy
+from .solvers import (
+    tdn, 
+    alpha_mc, 
+    off_policy_mc
+)
+
+
+__all__ = [
+    'utils',
+    'ModelFree',
+    'ModelFreePolicy',
+    'EpsilonSoftPolicy',
+    'tdn',
+    'alpha_mc',
+    'off_policy_mc',
+]

src/armed_bandits.py renamed to rl/armed_bandits.py

@@ -7,7 +7,7 @@
 import numpy as np
 import numpy.random as rnd
 
-from utils import Policy, RewardGenerator
+from rl.utils import Policy, RewardGenerator
 
 
 GAUSSIAN = [RewardGenerator('normal', rnd.random(), rnd.random()) for _ in range(10)]

src/mdp.py renamed to rl/mdp.py

@@ -14,7 +14,7 @@
 
 import numpy as np
 
-from utils import (
+from rl.utils import (
     State,
     Action,
     Policy,
@@ -143,7 +143,7 @@ def π(self, state: int):
 
     def __call__(self, state: int) -> np.ndarray:
         '''
-        collapses the policy to a single action, i.e. a sample from the
+        Collapses the policy to a single action, i.e. a sample from the
         random variable that represents the policy.
         '''
         return np.random.choice(self.pi_sa[state], p=self.pi_sa[state])
@@ -220,9 +220,9 @@ def vq_pi(
         method: str = 'iter_n'
         ) -> np.ndarray:
         '''
-            Individual state value functions and action-value functions
-            vpi and qpi cannot be calculated for bigger problems. That
-            constraint will give rise to parametrizations via DL.
+        Individual state value functions and action-value functions
+        vpi and qpi cannot be calculated for bigger problems. That
+        constraint will give rise to parametrizations via DL.
         '''
         policy = policy if policy else self.policy
         solver = self.VQ_PI_SOLVERS.get(method)
@@ -237,9 +237,9 @@ def optimize_policy(
         policy: MarkovPolicy = None
         ) -> MarkovPolicy:
         '''
-            Optimal policy is the policy that maximizes the expected
-            discounted return. It is the policy that maximizes the
-            value function for each possible state.
+        Optimal policy is the policy that maximizes the expected
+        discounted return. It is the policy that maximizes the
+        value function for each possible state.
         '''
         policy = policy if policy else self.policy
         solver = self.OPTIMAL_POLICY_SOLVERS.get(method)

src/model_free.py renamed to rl/model_free.py

@@ -14,13 +14,13 @@
 
 import numpy as np
 
-from utils import Policy, State, Action, MAX_ITER, MAX_STEPS
-from solvers import (
-    first_visit_monte_carlo,
-    every_visit_monte_carlo,
-    off_policy_first_visit,
-    off_policy_every_visit,
-    tdn
+from rl.utils import (
+    Policy, 
+    State, 
+    Action,
+    StateAction, 
+    MAX_ITER, 
+    MAX_STEPS
 )
 
 EpisodeStep = NewType(
@@ -83,6 +83,8 @@ def __init__(
         self.policy = policy
         self.states = State(states)
         self.actions = Action(actions)
+        self.stateaction = StateAction(
+            [(s,a) for s,a in zip(states, actions)])
         self.transition = transition
         self.gamma = gamma
         self.policy = policy if policy else ModelFreePolicy(

src/solvers.py renamed to rl/solvers.py

@@ -10,12 +10,12 @@
 import numpy as np
 from numpy.linalg import norm as lnorm
 
-from model_free import (
+from rl.model_free import (
     ModelFree,
     ModelFreePolicy,
     EpsilonSoftPolicy
 )
-from utils import (
+from rl.utils import (
     Policy,
     _typecheck_all,
     _get_sample_step,
@@ -31,8 +31,9 @@
 ) 
 
 
-def get_sample(v, q, π, n_episode, optimize):
-    _idx, _v, _q = n_episode, Vpi(v.copy()), Qpi(q.copy())
+def get_sample(MF, v, q, π, n_episode, optimize):
+    _idx = n_episode
+    _v, _q = Vpi(v.copy(), MF.states), Qpi(q.copy(), MF.stateaction)
     _pi = None
     if optimize:
         _pi = π.pi.copy() 
@@ -118,10 +119,9 @@ def value_iteration(MDP, policy: Policy = None, tol: float = TOL,
     policy.update_policy(qᵢ)
 
 
-
 def alpha_mc(states: Sequence[Any], actions: Sequence[Any], transition: Transition,
     gamma: float=0.9, alpha: float=0.05, use_N :bool=False, first_visit: bool=True,
-    exploring_starts: bool=False, n_episodes: int=MAX_ITER, max_steps: int=MAX_STEPS,
+    exploring_starts: bool=True, n_episodes: int=MAX_ITER, max_steps: int=MAX_STEPS,
     samples: int=1000, optimize: bool=False, policy: ModelFreePolicy=None, 
     eps: float=None) -> Tuple[VQPi, Samples]:
     '''α-MC state and action-value function estimation, policy optimization
@@ -176,9 +176,9 @@ def alpha_mc(states: Sequence[Any], actions: Sequence[Any], transition: Transiti
     '''
     if not policy and eps:
         _check_ranges(values=[eps], ranges=[(0,1)])
-        policy = EpsilonSoftPolicy(states, actions, eps=eps)
+        policy = EpsilonSoftPolicy(actions, states, eps=eps)
     elif not policy:
-        policy = ModelFreePolicy(states, actions)
+        policy = ModelFreePolicy(actions, states)
 
     _typecheck_all(tabular_idxs=[states, actions],transition=transition,
         constants=[gamma, alpha, n_episodes, max_steps, samples],
@@ -264,16 +264,16 @@ def _visit_monte_carlo(MF, first_visit, exploring_starts, use_N, alpha,
         n_episode += 1
 
         if sample_step and n_episode % sample_step == 0:
-            samples.append(get_sample(MF, v, q, π, n_episode))
-        
-    return v, q
+            samples.append(get_sample(MF, v, q, π, n_episode, optimize))
+
+    return v, q, samples
 
 
 def off_policy_mc(states: Sequence[Any], actions: Sequence[Any], transition: Transition,
     gamma: float=0.9, first_visit: bool=True, ordinary: bool=True,  
     n_episodes: int=MAX_ITER, max_steps: int=MAX_STEPS, samples: int=1000, 
     optimize: bool=False, policy: ModelFreePolicy=None, eps: float=None, 
-    b :ModelFreePolicy=None) -> Tuple[VQPi, Samples]: 
+    b: ModelFreePolicy=None) -> Tuple[VQPi, Samples]: 
     '''Off-policy Monte Carlo state and action value function estimation, policy 
     
     Off policy Monte Carlo method for estimating state and action-value functtions
@@ -326,11 +326,11 @@ def off_policy_mc(states: Sequence[Any], actions: Sequence[Any], transition: Tra
     if not policy and eps:
         _typecheck_all(constants=[eps])
         _check_ranges(values=[eps], ranges=[(0,1)])
-        policy = EpsilonSoftPolicy(states, actions, eps=eps)
+        policy = EpsilonSoftPolicy(actions, states, eps=eps)
     elif not policy:
-        policy = ModelFreePolicy(states, actions)
+        policy = ModelFreePolicy(actions, states)
     elif not b:
-        b = ModelFreePolicy(states, actions)
+        b = ModelFreePolicy(actions, states)
 
     _typecheck_all(tabular_idxs=[states, actions],transition=transition,
         constants=[gamma, n_episodes, max_steps, samples],
@@ -409,7 +409,7 @@ def _off_policy_monte_carlo(MF, off_policy, max_episodes, max_steps, first_visit
         if sample_step and n_episode % sample_step == 0:
             samples.append(get_sample(MF, v, q, π, n_episode))
 
-    return v, q
+    return v, q, samples
 
 
 
@@ -485,9 +485,9 @@ def tdn(states: Sequence[Any], actions: Sequence[Any], transition: Transition,
     if not policy and eps:
         _typecheck_all(constants=[eps])
         _check_ranges(values=[eps], ranges=[(0,1)])
-        policy = EpsilonSoftPolicy(states, actions, eps=eps)
+        policy = EpsilonSoftPolicy(actions, states, eps=eps)
     elif not policy:
-        policy = ModelFreePolicy(states, actions)
+        policy = ModelFreePolicy(actions, states)
 
     _typecheck_all(tabular_idxs=[states,actions], tansition=transition,
         constants=[gamma, n, alpha, n_episodes, samples, max_steps], 
@@ -541,7 +541,7 @@ def _tdn(MF, n, alpha, n_episodes, max_steps, optimize, sample_step):
         n_episode += 1
 
         if sample_step and n_episode % sample_step == 0:
-            samples.append(get_sample(MF, v, q, π, n_episode))
+            samples.append(get_sample(MF, v, q, π, n_episode, optimize))
 
     return v, q, samples
 

src/utils.py renamed to rl/utils.py

@@ -16,6 +16,7 @@
 TOL = 1E-6
 MEAN_ITERS = int(1E4)
 
+
 class Policy(ABC):
     def __init__(self):
         pass
@@ -61,21 +62,27 @@ class Action(_TabularIndexer):
     pass
 
 
+class StateAction(_TabularIndexer):
+    pass
+
+
 class _TabularValues:
     def __init__(self, values: np.ndarray, idx: _TabularIndexer):
         self.v = values
         self.idx = idx
         self.idx_val = {k:v for k,v in zip(idx.index.keys(), values)}
 
     def values(self):
-        return self.idx_val
+        return self.v
 
 
 class Vpi(_TabularValues):
-    pass    
+    def __str__(self):
+        return f'Vpi({self.v[:10]}...)'
 
 class Qpi(_TabularValues):
-    pass
+    def __str__(self):
+        return f'Vpi({self.v[:10]}...)'
 
 
 
@@ -114,13 +121,13 @@ def _typecheck_transition(transition):
     if not isinstance(transition, Callable):
         raise TypeError(
             f"transition must be a Callable, not {type(transition)}")
-    
-    if len(transition.__code__.co_varnames) != 2:
-        raise TypeError(
-            "transition must have 2 arguments, not ",
-            len(transition.__code__.co_varnames))
-    
 
+    #check that transition function has just two positional arguments
+    if transition.__code__.co_argcount != 2:
+        raise TypeError(
+            f"transition must have two positional arguments,"
+            f" not {transition.__code__.co_argcount}")   
+ 
 def _typecheck_constants(*args):
     for arg in args:
         if not isinstance(arg, (float, int)):

setup.py

@@ -0,0 +1,2 @@
+from setuptools import setup, find_packages  
+setup(name = 'rl', packages = find_packages())