Implemented Django style filtering

Closes #113

Signed-off-by: Jakub Filak <[email protected]>

Author: bartonip

docs/usage/querying.rst

@@ -66,6 +66,36 @@ Print unique identification (Id) of all employees with name John Smith:
         print(smith.EmployeeID)
 
 
+Get entities matching a filter in ORM style
+---------------------------------------------------
+
+Print unique identification (Id) of all employees with name John Smith:
+
+.. code-block:: python
+
+    from pyodata.v2.service import GetEntitySetFilter as esf
+
+    smith_employees_request = northwind.entity_sets.Employees.get_entities()
+    smith_employees_request = smith_employees_request.filter(FirstName="John", LastName="Smith")
+    for smith in smith_employees_request.execute():
+        print(smith.EmployeeID)
+
+
+Get entities matching a complex filter in ORM style
+---------------------------------------------------
+
+Print unique identification (Id) of all employees with name John Smith:
+
+.. code-block:: python
+
+    from pyodata.v2.service import GetEntitySetFilter as esf
+
+    smith_employees_request = northwind.entity_sets.Employees.get_entities()
+    smith_employees_request = smith_employees_request.filter(FirstName__contains="oh", LastName__startswith="Smi")
+    for smith in smith_employees_request.execute():
+        print(smith.EmployeeID)
+
+
 Get a count of entities
 -----------------------
 

pyodata/v2/model.py

@@ -1287,6 +1287,9 @@ def proprty(self, property_name):
     def proprties(self):
         return list(self._properties.values())
 
+    def has_proprty(self, proprty_name):
+        return proprty_name in self._properties
+
     @classmethod
     def from_etree(cls, type_node, config: Config):
         name = type_node.get('Name')

pyodata/v2/service.py

@@ -314,8 +314,9 @@ def execute(self):
         if body:
             self._logger.debug('  body: %s', body)
 
+        params = "&".join("%s=%s" % (k, v) for k, v in self.get_query_params().items())
         response = self._connection.request(
-            self.get_method(), url, headers=headers, params=self.get_query_params(), data=body)
+            self.get_method(), url, headers=headers, params=params, data=body)
 
         self._logger.debug('Received response')
         self._logger.debug('  url: %s', response.url)
@@ -623,7 +624,7 @@ def expand(self, expand):
     def filter(self, filter_val):
         """Sets the filter expression."""
         # returns QueryRequest
-        self._filter = quote(filter_val)
+        self._filter = filter_val
         return self
 
     # def nav(self, key_value, nav_property):
@@ -993,6 +994,212 @@ def __gt__(self, value):
         return GetEntitySetFilter.format_filter(self._proprty, 'gt', value)
 
 
+class FilterExpression:
+    """A class representing named expression of OData $filter"""
+
+    def __init__(self, **kwargs):
+        self._expressions = kwargs
+        self._other = None
+        self._operator = None
+
+    @property
+    def expressions(self):
+        """Get expressions where key is property name with the operator suffix
+           and value is the left hand side operand.
+        """
+
+        return self._expressions.items()
+
+    @property
+    def other(self):
+        """Get an instance of the other operand"""
+
+        return self._other
+
+    @property
+    def operator(self):
+        """The other operand"""
+
+        return self._operator
+
+    def __or__(self, other):
+        if self._other is not None:
+            raise RuntimeError('The FilterExpression already initialized')
+
+        self._other = other
+        self._operator = "or"
+        return self
+
+    def __and__(self, other):
+        if self._other is not None:
+            raise RuntimeError('The FilterExpression already initialized')
+
+        self._other = other
+        self._operator = "and"
+        return self
+
+
+class GetEntitySetFilterChainable:
+    """
+    Example expressions
+        FirstName='Tim'
+        FirstName__contains='Tim'
+        Age__gt=56
+        Age__gte=6
+        Age__lt=78
+        Age__lte=90
+        Age__range=(5,9)
+        FirstName__in=['Tim', 'Bob', 'Sam']
+        FirstName__startswith='Tim'
+        FirstName__endswith='mothy'
+        Addresses__Suburb='Chatswood'
+        Addresses__Suburb__contains='wood'
+    """
+
+    OPERATORS = [
+        'startswith',
+        'endswith',
+        'lt',
+        'lte',
+        'gt',
+        'gte',
+        'contains',
+        'range',
+        'in',
+        'length',
+        'eq'
+    ]
+
+    def __init__(self, entity_type, filter_expressions, exprs):
+        self._entity_type = entity_type
+        self._filter_expressions = filter_expressions
+        self._expressions = exprs
+
+    @property
+    def expressions(self):
+        """Get expressions as a list of tuples where the first item
+           is a property name with the operator suffix and the second item
+           is a left hand side value.
+        """
+
+        return self._expressions.items()
+
+    def proprty_obj(self, name):
+        """Returns a model property for a particular property"""
+
+        return self._entity_type.proprty(name)
+
+    def _decode_and_combine_filter_expression(self, filter_expression):
+        filter_expressions = [self._decode_expression(expr, val) for expr, val in filter_expression.expressions]
+        return self._combine_expressions(filter_expressions)
+
+    def _process_query_objects(self):
+        """Processes FilterExpression objects to OData lookups"""
+
+        filter_expressions = []
+
+        for expr in self._filter_expressions:
+            lhs_expressions = self._decode_and_combine_filter_expression(expr)
+
+            if expr.other is not None:
+                rhs_expressions = self._decode_and_combine_filter_expression(expr.other)
+                filter_expressions.append(f'({lhs_expressions}) {expr.operator} ({rhs_expressions})')
+            else:
+                filter_expressions.append(lhs_expressions)
+
+        return filter_expressions
+
+    def _process_expressions(self):
+        filter_expressions = [self._decode_expression(expr, val) for expr, val in self.expressions]
+
+        filter_expressions.extend(self._process_query_objects())
+
+        return filter_expressions
+
+    def _decode_expression(self, expr, val):
+        field = None
+        # field_heirarchy = []
+        operator = 'eq'
+        exprs = expr.split('__')
+
+        for part in exprs:
+            if self._entity_type.has_proprty(part):
+                field = part
+                # field_heirarchy.append(part)
+            elif part in self.__class__.OPERATORS:
+                operator = part
+            else:
+                raise ValueError(f'"{part}" is not a valid property or operator')
+        # field = '/'.join(field_heirarchy)
+
+        # target_field = self.proprty_obj(field_heirarchy[-1])
+        expression = self._build_expression(field, operator, val)
+
+        return expression
+
+    # pylint: disable=no-self-use
+    def _combine_expressions(self, expressions):
+        return ' and '.join(expressions)
+
+    # pylint: disable=too-many-return-statements, too-many-branches
+    def _build_expression(self, field_name, operator, value):
+        target_field = self.proprty_obj(field_name)
+
+        if operator not in ['length', 'in', 'range']:
+            value = target_field.to_literal(value)
+
+        if operator == 'lt':
+            return f'{field_name} lt {value}'
+
+        if operator == 'lte':
+            return f'{field_name} le {value}'
+
+        if operator == 'gte':
+            return f'{field_name} ge {value}'
+
+        if operator == 'gt':
+            return f'{field_name} gt {value}'
+
+        if operator == 'startswith':
+            return f'startswith({field_name}, {value}) eq true'
+
+        if operator == 'endswith':
+            return f'endswith({field_name}, {value}) eq true'
+
+        if operator == 'length':
+            value = int(value)
+            return f'length({field_name}) eq {value}'
+
+        if operator in ['contains']:
+            return f'substringof({value}, {field_name}) eq true'
+
+        if operator == 'range':
+            if not isinstance(value, (tuple, list)):
+                raise TypeError('Range must be tuple or list not {}'.format(type(value)))
+
+            if len(value) != 2:
+                raise ValueError('Only two items can be passed in a range.')
+
+            low_bound = target_field.to_literal(value[0])
+            high_bound = target_field.to_literal(value[1])
+
+            return f'{field_name} gte {low_bound} and {field_name} lte {high_bound}'
+
+        if operator == 'in':
+            literal_values = (f'{field_name} eq {target_field.to_literal(item)}' for item in value)
+            return ' or '.join(literal_values)
+
+        if operator == 'eq':
+            return f'{field_name} eq {value}'
+
+        raise ValueError(f'Invalid expression {operator}')
+
+    def __str__(self):
+        expressions = self._process_expressions()
+        result = self._combine_expressions(expressions)
+        return quote(result)
+
+
 class GetEntitySetRequest(QueryRequest):
     """GET on EntitySet"""
 
@@ -1005,6 +1212,19 @@ def __getattr__(self, name):
         proprty = self._entity_type.proprty(name)
         return GetEntitySetFilter(proprty)
 
+    def _set_filter(self, filter_val):
+        filter_text = self._filter + ' and ' if self._filter else ''
+        filter_text += filter_val
+        self._filter = filter_text
+
+    def filter(self, *args, **kwargs):
+        if args and len(args) == 1 and isinstance(args[0], str):
+            self._filter = args[0]
+        else:
+            self._set_filter(str(GetEntitySetFilterChainable(self._entity_type, args, kwargs)))
+
+        return self
+
 
 class EntitySetProxy:
     """EntitySet Proxy"""

tests/test_model_v2.py

@@ -6,7 +6,7 @@
 import pytest
 from pyodata.v2.model import Schema, Typ, StructTypeProperty, Types, EntityType, EdmStructTypeSerializer, \
     Association, AssociationSet, EndRole, AssociationSetEndRole, TypeInfo, MetadataBuilder, ParserError, PolicyWarning, \
-    PolicyIgnore, Config, PolicyFatal, NullType, NullAssociation, current_timezone
+    PolicyIgnore, Config, PolicyFatal, NullType, NullAssociation, current_timezone, StructType
 from pyodata.exceptions import PyODataException, PyODataModelError, PyODataParserError
 from tests.conftest import assert_logging_policy
 
@@ -1404,3 +1404,23 @@ def test_missing_property_referenced_in_annotation(mock_warning, xml_builder_fac
     )).build()
 
     assert mock_warning.called is False
+
+
+def test_struct_type_has_property_initial_instance():
+    struct_type = StructType('Name', 'Label', False)
+
+    assert struct_type.has_proprty('proprty') == False
+
+
+def test_struct_type_has_property_no():
+    struct_type = StructType('Name', 'Label', False)
+    struct_type._properties['foo'] = 'ugly test hack'
+
+    assert not struct_type.has_proprty('proprty')
+
+
+def test_struct_type_has_property_yes():
+    struct_type = StructType('Name', 'Label', False)
+    struct_type._properties['proprty'] = 'ugly test hack'
+
+    assert struct_type.has_proprty('proprty')

tests/test_service_v2.py

@@ -1640,14 +1640,14 @@ def test_navigation_count(service):
 
 
 @responses.activate
-def test_navigation_count_with_filter(service):
-    """Check getting $count via navigation property with $filter"""
+def test_count_with_filter(service):
+    """Check getting $count with $filter"""
 
     # pylint: disable=redefined-outer-name
 
     responses.add(
         responses.GET,
-        "{0}/Employees(23)/Addresses/$count?%24filter=City%2520eq%2520%2527London%2527".format(service.url),
+        "{0}/Employees(23)/Addresses/$count?%24filter=City%20eq%20%27London%27".format(service.url),
         json=3,
         status=200)
 
@@ -1659,6 +1659,369 @@ def test_navigation_count_with_filter(service):
     assert request.execute() == 3
 
 
+@responses.activate
+def test_count_with_chainable_filter(service):
+    """Check getting $count with $filter and using new filter syntax"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees(23)/Addresses/$count?%24filter=City%20eq%20%27London%27".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entity(23).nav('Addresses').get_entities()
+    request = employees.filter(City="London").count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_lt_operator(service):
+    """Check getting $count with $filter with new filter syntax using multiple filters"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?%24filter=ID%20lt%2023".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(ID__lt=23).count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_lte_operator(service):
+    """Check getting $count with $filter with new filter syntax using multiple filters"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?%24filter=ID%20le%2023".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(ID__lte=23).count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_gt_operator(service):
+    """Check getting $count with $filter with new filter syntax using multiple filters"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?%24filter=ID%20gt%2023".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(ID__gt=23).count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_gte_operator(service):
+    """Check getting $count with $filter with new filter syntax using multiple filters"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?%24filter=ID%20ge%2023".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(ID__gte=23).count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_eq_operator(service):
+    """Check getting $count with $filter with new filter syntax using multiple filters"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?%24filter=ID%20eq%2023".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(ID__eq=23).count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_in_operator(service):
+    """Check getting $count with $filter in"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?$filter=ID%20eq%201%20or%20ID%20eq%202%20or%20ID%20eq%203".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(ID__in=[1,2,3]).count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_startswith_operator(service):
+    """Check getting $count with $filter in"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?$filter=startswith%28NickName%2C%20%27Tim%27%29%20eq%20true".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(NickName__startswith="Tim").count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_endswith_operator(service):
+    """Check getting $count with $filter in"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?$filter=endswith%28NickName%2C%20%27othy%27%29%20eq%20true".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(NickName__endswith="othy").count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_length_operator(service):
+    """Check getting $count with $filter in"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?$filter=length%28NickName%29%20eq%206".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(NickName__length=6).count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_length_operator_as_string(service):
+    """Check getting $count with $filter in"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?$filter=length%28NickName%29%20eq%206".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(NickName__length="6").count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_contains_operator(service):
+    """Check getting $count with $filter in"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?$filter=substringof%28%27Tim%27%2C%20NickName%29%20eq%20true".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(NickName__contains="Tim").count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_range_operator(service):
+    """Check getting $count with $filter in"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?$filter=ID%20gte%2020%20and%20ID%20lte%2050".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(ID__range=(20, 50)).count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_multiple(service):
+    """Check getting $count with $filter with new filter syntax using multiple filters"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?%24filter=ID%20eq%2023%20and%20NickName%20eq%20%27Steve%27".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(ID=23, NickName="Steve").count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filter_or(service):
+    """Check getting $count with $filter with FilterExpression syntax or"""
+    from pyodata.v2.service import FilterExpression as Q
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?$filter=%28ID%20eq%2023%20and%20NickName%20eq%20%27Steve%27%29%20or%20%28ID%20eq%2025%20and%20NickName%20eq%20%27Tim%27%29".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(Q(ID=23, NickName="Steve") | Q(ID=25, NickName="Tim")).count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+@responses.activate
+def test_count_with_multiple_chainable_filters_startswith(service):
+    """Check getting $count with $filter calling startswith"""
+    from pyodata.v2.service import FilterExpression as Q
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?$filter=%28ID%20eq%2023%20and%20startswith%28NickName%2C%20%27Ste%27%29%20eq%20true%29%20or%20%28ID%20eq%2025%20and%20NickName%20eq%20%27Tim%27%29".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(Q(ID=23, NickName__startswith="Ste") | Q(ID=25, NickName="Tim")).count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
+@responses.activate
+def test_count_with_chainable_filters_invalid_property_lookup(service):
+    """Check getting $count with $filter calling startswith"""
+    # pylint: disable=redefined-outer-name
+
+    employees = service.entity_sets.Employees.get_entities()
+    with pytest.raises(ValueError) as ex:
+        request = employees.filter(Foo="Bar")
+
+    assert str(ex.value) == '"Foo" is not a valid property or operator'
+
+
+@responses.activate
+def test_count_with_chainable_filters_invalid_operator_lookup(service):
+    """Check getting $count with $filter calling startswith"""
+    # pylint: disable=redefined-outer-name
+
+    employees = service.entity_sets.Employees.get_entities()
+    with pytest.raises(ValueError) as ex:
+        request = employees.filter(NickName__foo="Bar")
+
+    assert str(ex.value) == '"foo" is not a valid property or operator'
+
+
+@responses.activate
+def test_count_with_chained_filters(service):
+    """Check getting $count with chained filters"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        "{0}/Employees/$count?$filter=ID%20gte%2020%20and%20ID%20lte%2050%20and%20NickName%20eq%20%27Tim%27".format(service.url),
+        json=3,
+        status=200)
+
+    employees = service.entity_sets.Employees.get_entities()
+    request = employees.filter(ID__range=(20, 50)).filter(NickName="Tim").count()
+
+    assert isinstance(request, pyodata.v2.service.GetEntitySetRequest)
+
+    assert request.execute() == 3
+
+
 @responses.activate
 def test_create_entity_with_datetime(service):
     """