Merge branch 'master' into snyk-fix-fcd6018f6c54219238f772fd46115704

Author: mikaelfrykholm

.github/workflows/python-package.yml

@@ -0,0 +1,48 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-24.04", "ubuntu-22.04", "ubuntu-20.04"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
+    runs-on:  ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Lint with Ruff
+      run: |
+        pip install ruff
+        ruff check --output-format=github .
+      continue-on-error: true
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        if [ -f test_requirements.txt ]; then pip install -r test_requirements.txt; fi
+        pip install coveralls
+        pip install mypy
+        python -m pip install --editable .
+
+
+    - name: Run tests
+      run: |
+        make quick_test
+        make test_coverage
+        #make typecheck
+    

NEWS.txt

@@ -173,4 +173,9 @@ to sign using HSMs. The only mandatory non-python dependency now is lxml.
 
 2.1.3
 -----
+* Release date: ons  10 sep 2024 17:17:10 CET
+
 * Add DiscoveryResponse info to SPs in discojson
+* Remove cherrypy imports
+* Fix logging
+* suport SP trust metadata in an entity attribute as JSON blob

requirements.txt

@@ -1,4 +1,4 @@
-urllib3==1.26.18
+urllib3==1.26.19
 pytz
 accept_types >=0.4.1
 apscheduler==3.6.3
@@ -24,3 +24,4 @@ wsgi-intercept
 xmldiff
 str2bool
 setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability
+zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability

src/pyff/api.py

@@ -487,24 +487,11 @@ def cors_headers(request: Request, response: Response) -> None:
     event.request.add_response_callback(cors_headers)
 
 
-def launch_memory_usage_server(port: int = 9002) -> None:
-    import cherrypy
-    import dowser
-
-    cherrypy.tree.mount(dowser.Root())
-    cherrypy.config.update({'environment': 'embedded', 'server.socket_port': port})
-
-    cherrypy.engine.start()
-
-
 def mkapp(*args: Any, **kwargs: Any) -> Any:
     md = kwargs.pop('md', None)
     if md is None:
         md = MDRepository()
 
-    if config.devel_memory_profile:
-        launch_memory_usage_server()
-
     with Configurator(debug_logger=log) as ctx:
         ctx.add_subscriber(add_cors_headers_response_callback, NewRequest)
 

src/pyff/builtins.py

@@ -31,6 +31,7 @@
 from pyff.samlmd import (
     annotate_entity,
     discojson_sp_t,
+    discojson_sp_attr_t,
     discojson_t,
     entitiesdescriptor,
     find_in_document,
@@ -731,7 +732,7 @@ def select(req: Plumbing.Request, *opts):
     Select a set of EntityDescriptor elements as the working document.
 
     :param req: The request
-    :param opts: Options - used for select alias
+    :param opts: Options - see Options below
     :return: returns the result of the operation as a working document
 
     Select picks and expands elements (with optional filtering) from the active repository you setup using calls
@@ -778,25 +779,60 @@ def select(req: Plumbing.Request, *opts):
     would terminate the plumbing at select if there are no SPs in the local repository. This is useful in
     combination with fork for handling multiple cases in your plumbings.
 
-    The 'as' keyword allows a select to be stored as an alias in the local repository. For instance
+    Options are put directly after "select". E.g:
 
     .. code-block:: yaml
 
-        - select as /foo-2.0: "!//md:EntityDescriptor[md:IDPSSODescriptor]"
+        - select as /foo-2.0 dedup True: "!//md:EntityDescriptor[md:IDPSSODescriptor]"
 
-    would allow you to use /foo-2.0.json to refer to the JSON-version of all IdPs in the current repository.
-    Note that you should not include an extension in your "as foo-bla-something" since that would make your
-    alias invisible for anything except the corresponding mime type.
+    **Options**
+    Defaults are marked with (*)
+    - as <name> : The 'as' keyword allows a select to be stored as an alias in the local repository. For instance
+
+        .. code-block:: yaml
+
+            - select as /foo-2.0: "!//md:EntityDescriptor[md:IDPSSODescriptor]"
+
+        would allow you to use /foo-2.0.json to refer to the JSON-version of all IdPs in the current repository.
+        Note that you should not include an extension in your "as foo-bla-something" since that would make your
+        alias invisible for anything except the corresponding mime type.
+
+    - dedup <True*|False> : Whether to deduplicate the results by entityID.
+
+        Note: When select is used after a load pipe with more than one source, if dedup is set to True
+        and there are entity properties that may differ from one source to another, these will be squashed
+        rather than merged.
     """
+    opt_names = ('as', 'dedup')
+    if len(opts) % 2 == 0:
+        _opts = dict(list(zip(opts[::2], opts[1::2])))
+    else:
+        _opts = {}
+        for i in range(0, len(opts), 2):
+            if opts[i] in opt_names:
+                _opts[opts[i]] = opts[i + 1]
+            else:
+                _opts['as'] = opts[i]
+                if i + 1 < len(opts):
+                    more_opts = opts[i + 1:]
+                    _opts.update(dict(list(zip(more_opts[::2], more_opts[1::2]))))
+                    break
+
+    _opts.setdefault('dedup', "True")
+    _opts.setdefault('name', req.plumbing.id)
+    _opts['dedup'] = bool(str2bool(_opts['dedup']))
+
     args = _select_args(req)
-    name = req.plumbing.id
+    name = _opts['name']
+    dedup = _opts['dedup']
+
     if len(opts) > 0:
         if opts[0] != 'as' and len(opts) == 1:
             name = opts[0]
         if opts[0] == 'as' and len(opts) == 2:
             name = opts[1]
 
-    entities = resolve_entities(args, lookup_fn=req.md.store.select)
+    entities = resolve_entities(args, lookup_fn=req.md.store.select, dedup=dedup)
 
     if req.state.get('match', None):  # TODO - allow this to be passed in via normal arguments
 
@@ -1044,6 +1080,36 @@ def _discojson_sp(req, *opts):
     return json.dumps(res)
 
 
+@pipe(name='discojson_sp_attr')
+def _discojson_sp_attr(req, *opts):
+    """
+
+    Return a json representation of the trust information
+
+    .. code-block:: yaml
+      discojson_sp_attr:
+
+    SP Entities can carry trust information as a base64 encoded json blob
+    as an entity attribute with name `https://refeds.org/entity-selection-profile`.
+    The schema of this json is the same as the one produced above from XML
+    with the pipe `discojson_sp`, and published at:
+
+    https://github.com/TheIdentitySelector/thiss-mdq/blob/master/trustinfo.schema.json
+
+    :param req: The request
+    :param opts: Options (unusued)
+    :return: returns a JSON doc
+
+    """
+
+    if req.t is None:
+        raise PipeException("Your pipeline is missing a select statement.")
+
+    res = discojson_sp_attr_t(req)
+
+    return json.dumps(res)
+
+
 @pipe
 def sign(req: Plumbing.Request, *_opts):
     """

src/pyff/constants.py

@@ -264,7 +264,7 @@ class Config(object):
     allow_shutdown = S("allow_shutdown", default=False, typeconv=as_bool, deprecated=True)
     ds_template = S("ds_template", default="ds.html", deprecated=True)
 
-    loglevel = S("loglevel", default=logging.WARN, info="set the loglevel")
+    loglevel = S("loglevel", default='WARN', info="set the loglevel")
 
     access_log = S("access_log", cmdline=['pyffd'], info="a log target (file) to use for access logs")
 
@@ -290,7 +290,7 @@ class Config(object):
 
     caching_enabled = S("caching_enabled", default=True, typeconv=as_bool, info="enable caching?")
 
-    no_cashing = N('no_cashing', invert=caching_enabled, short='C', info="disable all caches")
+    no_caching = N('no_caching', invert=caching_enabled, short='C', info="disable all caches")
 
     daemonize = S("daemonize", default=True, cmdline=['pyffd'], info="run in background")
 
@@ -523,7 +523,7 @@ def parse_options(program, docs):
         sys.exit(2)
 
     if config.loglevel is None:
-        config.loglevel = logging.INFO
+        config.loglevel = 'INFO'
 
     if config.aliases is None or len(config.aliases) == 0:
         config.aliases = dict(metadata=entities)
@@ -536,7 +536,7 @@ def parse_options(program, docs):
             if o in ('-h', '--help'):
                 print(docs)
                 sys.exit(0)
-            elif o in ('--version',):
+            elif o in ('-v', '--version'):
                 print("{} version {}".format(program, pyff_version))
                 sys.exit(0)
             elif o in ('-A', '--alias'):

src/pyff/fetch.py

@@ -96,7 +96,7 @@ def schedule(self, url):
         :param url: the url to fetch
         :return: nothing is returned.
         """
-        log.debug("scheduling fetch of {}".format(url))
+        log.info("scheduling fetch of {}".format(url))
         self.request.put(url)
 
     def stop(self):

src/pyff/logs.py

@@ -7,12 +7,6 @@
 
 import six
 
-try:
-    import cherrypy
-except ImportError as e:
-    logging.debug("cherrypy logging disabled")
-    cherrypy = None
-
 
 class PyFFLogger(object):
     def __init__(self, name=None):
@@ -29,9 +23,7 @@ def __init__(self, name=None):
         }
 
     def _l(self, severity, msg):
-        if cherrypy is not None and '' in cherrypy.tree.apps:
-            cherrypy.tree.apps[''].log(str(msg), severity=severity)
-        elif severity in self._loggers:
+        if severity in self._loggers:
             self._loggers[severity](str(msg))
         else:
             raise ValueError("unknown severity %s" % severity)

src/pyff/samlmd.py

@@ -1,4 +1,6 @@
+import json
 import traceback
+from base64 import b64decode
 from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from str2bool import str2bool
@@ -400,7 +402,7 @@ def filter_or_validate(
     return t
 
 
-def resolve_entities(entities, lookup_fn=None):
+def resolve_entities(entities, lookup_fn=None, dedup=True):
     """
 
     :param entities: a set of entities specifiers (lookup is used to find entities from this set)
@@ -414,13 +416,21 @@ def _resolve(m, l_fn):
         else:
             return l_fn(m)
 
-    resolved_entities = dict()  # a set won't do since __compare__ doesn't use @entityID
+    if dedup:
+        resolved_entities = dict()  # a set won't do since __compare__ doesn't use @entityID
+    else:
+        resolved_entities = []
     for member in entities:
         for entity in _resolve(member, lookup_fn):
             entity_id = entity.get('entityID', None)
             if entity is not None and entity_id is not None:
-                resolved_entities[entity_id] = entity
-    return resolved_entities.values()
+                if dedup:
+                    resolved_entities[entity_id] = entity
+                else:
+                    resolved_entities.append(entity)
+    if dedup:
+        return resolved_entities.values()
+    return resolved_entities
 
 
 def entitiesdescriptor(
@@ -1030,6 +1040,25 @@ def discojson_sp(e, global_trust_info=None, global_md_sources=None):
     return sp
 
 
+def discojson_sp_attr(e):
+
+    attribute = "https://refeds.org/entity-selection-profile"
+    b64_trustinfos = entity_attribute(e, attribute)
+    if b64_trustinfos is None:
+        return None
+
+    sp = {}
+    sp['entityID'] = e.get('entityID', None)
+    sp['profiles'] = {}
+
+    for b64_trustinfo in b64_trustinfos:
+        str_trustinfo = b64decode(b64_trustinfo.encode('ascii'))
+        trustinfo = json.loads(str_trustinfo.decode('utf8'))
+        sp['profiles'].update(trustinfo['profiles'])
+
+    return sp
+
+
 def discojson_sp_t(req):
     d = []
     t = req.t
@@ -1041,6 +1070,22 @@ def discojson_sp_t(req):
         if sp is not None:
             d.append(sp)
 
+        sp = discojson_sp_attr(e)
+        if sp is not None:
+            d.append(sp)
+
+    return d
+
+
+def discojson_sp_attr_t(req):
+    d = []
+    t = req.t
+
+    for e in iter_entities(t):
+        sp = discojson_sp_attr(e)
+        if sp is not None:
+            d.append(sp)
+
     return d
 
 

src/pyff/schema/schema.xsd

@@ -5,7 +5,7 @@
 <import namespace="urn:oasis:names:tc:SAML:2.0:assertion" schemaLocation="sstc-saml-schema-assertion-2.0.xsd"/>
 <import namespace="urn:oasis:names:tc:SAML:2.0:metadata" schemaLocation="sstc-saml-schema-metadata-2.0.xsd"/>
 <import namespace="urn:oasis:names:tc:SAML:1.0:assertion" schemaLocation="cs-sstc-schema-assertion-1.1.xsd"/>
-<import namespace="urn:oasis:names:tc:SAML:2.0:metadata:ui" schemaLocation="sstc-saml-metadata-ui-v1.0.xsd"/>
+<import namespace="urn:oasis:names:tc:SAML:metadata:ui" schemaLocation="sstc-saml-metadata-ui-v1.0.xsd"/>
 <import namespace="urn:mace:shibboleth:metadata:1.0" schemaLocation="shibboleth-metadata-1.0.xsd"/>
 <import namespace="urn:oasis:names:tc:SAML:metadata:attribute" schemaLocation="sstc-metadata-attr.xsd"/>
 <import namespace="urn:oasis:names:tc:SAML:metadata:algsupport" schemaLocation="sstc-saml-metadata-algsupport.xsd"/>