Merge pull request #670 from veselypeta/petr/563/structure_type

[UR] Automatically generate ur_structure_type_t

Author: pbalcer

include/ur.py

@@ -265,79 +265,6 @@ etors:
       value: "0x7ffffffe"
       desc: "Unknown or internal error"
 --- #--------------------------------------------------------------------------
-type: enum
-desc: "Defines structure types"
-name: $x_structure_type_t
-etors:
-    - name: CONTEXT_PROPERTIES
-      desc: $x_context_properties_t
-    - name: IMAGE_DESC
-      desc: $x_image_desc_t
-    - name: BUFFER_PROPERTIES
-      desc: $x_buffer_properties_t
-    - name: BUFFER_REGION
-      desc: $x_buffer_region_t
-    - name: BUFFER_CHANNEL_PROPERTIES
-      desc: $x_buffer_channel_properties_t
-    - name: BUFFER_ALLOC_LOCATION_PROPERTIES
-      desc: $x_buffer_alloc_location_properties_t
-    - name: PROGRAM_PROPERTIES
-      desc: $x_program_properties_t
-    - name: USM_DESC
-      desc: $x_usm_desc_t
-    - name: USM_HOST_DESC
-      desc: $x_usm_host_desc_t
-    - name: USM_DEVICE_DESC
-      desc: $x_usm_device_desc_t
-    - name: USM_POOL_DESC
-      desc: $x_usm_pool_desc_t
-    - name: USM_POOL_LIMITS_DESC
-      desc: $x_usm_pool_limits_desc_t
-    - name: DEVICE_BINARY
-      desc: $x_device_binary_t
-    - name: SAMPLER_DESC
-      desc: $x_sampler_desc_t
-    - name: QUEUE_PROPERTIES
-      desc: $x_queue_properties_t
-    - name: QUEUE_INDEX_PROPERTIES
-      desc: $x_queue_properties_t
-    - name: CONTEXT_NATIVE_PROPERTIES
-      desc: $x_context_native_properties_t
-    - name: KERNEL_NATIVE_PROPERTIES
-      desc: $x_kernel_native_properties_t
-    - name: QUEUE_NATIVE_PROPERTIES
-      desc: $x_queue_native_properties_t
-    - name: MEM_NATIVE_PROPERTIES
-      desc: $x_mem_native_properties_t
-    - name: EVENT_NATIVE_PROPERTIES
-      desc: $x_event_native_properties_t
-    - name: PLATFORM_NATIVE_PROPERTIES
-      desc: $x_platform_native_properties_t
-    - name: DEVICE_NATIVE_PROPERTIES
-      desc: $x_device_native_properties_t
-    - name: PROGRAM_NATIVE_PROPERTIES
-      desc: $x_program_native_properties_t
-    - name: SAMPLER_NATIVE_PROPERTIES
-      desc: $x_sampler_native_properties_t
-    - name: QUEUE_NATIVE_DESC
-      desc: $x_queue_native_desc_t
-    - name: DEVICE_PARTITION_PROPERTIES
-      desc: $x_device_partition_properties_t
-    - name: KERNEL_ARG_MEM_OBJ_PROPERTIES
-      desc: $x_kernel_arg_mem_obj_properties_t
-    - name: PHYSICAL_MEM_PROPERTIES
-      desc: $x_physical_mem_properties_t
-    - name: KERNEL_ARG_POINTER_PROPERTIES
-      desc: $x_kernel_arg_pointer_properties_t
-    - name: KERNEL_ARG_SAMPLER_PROPERTIES
-      desc: $x_kernel_arg_sampler_properties_t
-    - name: KERNEL_EXEC_INFO_PROPERTIES
-      desc: $x_kernel_exec_info_properties_t
-    - name: KERNEL_ARG_VALUE_PROPERTIES
-      desc: $x_kernel_arg_value_properties_t
-    - name: KERNEL_ARG_LOCAL_PROPERTIES
-      desc: $x_kernel_arg_local_properties_t
---- #--------------------------------------------------------------------------
 type: struct
 desc: "Base for all properties types"
 name: $x_base_properties_t

include/ur_api.h

@@ -1,7 +1,7 @@
 ---
 type: header
 desc: "Intel $OneApi Unified Runtime function registry"
-ordinal: "9"
+ordinal: "-1"
 ---
 name: $x_function_t
 type: enum
@@ -502,3 +502,110 @@ etors:
 - name: COMMAND_BUFFER_APPEND_MEMBUFFER_READ_RECT_EXP
   desc: Enumerator for $xCommandBufferAppendMembufferReadRectExp
   value: '171'
+---
+type: enum
+desc: Defines structure types
+name: $x_structure_type_t
+etors:
+- name: CONTEXT_PROPERTIES
+  desc: $x_context_properties_t
+  value: '0'
+- name: IMAGE_DESC
+  desc: $x_image_desc_t
+  value: '1'
+- name: BUFFER_PROPERTIES
+  desc: $x_buffer_properties_t
+  value: '2'
+- name: BUFFER_REGION
+  desc: $x_buffer_region_t
+  value: '3'
+- name: BUFFER_CHANNEL_PROPERTIES
+  desc: $x_buffer_channel_properties_t
+  value: '4'
+- name: BUFFER_ALLOC_LOCATION_PROPERTIES
+  desc: $x_buffer_alloc_location_properties_t
+  value: '5'
+- name: PROGRAM_PROPERTIES
+  desc: $x_program_properties_t
+  value: '6'
+- name: USM_DESC
+  desc: $x_usm_desc_t
+  value: '7'
+- name: USM_HOST_DESC
+  desc: $x_usm_host_desc_t
+  value: '8'
+- name: USM_DEVICE_DESC
+  desc: $x_usm_device_desc_t
+  value: '9'
+- name: USM_POOL_DESC
+  desc: $x_usm_pool_desc_t
+  value: '10'
+- name: USM_POOL_LIMITS_DESC
+  desc: $x_usm_pool_limits_desc_t
+  value: '11'
+- name: DEVICE_BINARY
+  desc: $x_device_binary_t
+  value: '12'
+- name: SAMPLER_DESC
+  desc: $x_sampler_desc_t
+  value: '13'
+- name: QUEUE_PROPERTIES
+  desc: $x_queue_properties_t
+  value: '14'
+- name: QUEUE_INDEX_PROPERTIES
+  desc: $x_queue_index_properties_t
+  value: '15'
+- name: CONTEXT_NATIVE_PROPERTIES
+  desc: $x_context_native_properties_t
+  value: '16'
+- name: KERNEL_NATIVE_PROPERTIES
+  desc: $x_kernel_native_properties_t
+  value: '17'
+- name: QUEUE_NATIVE_PROPERTIES
+  desc: $x_queue_native_properties_t
+  value: '18'
+- name: MEM_NATIVE_PROPERTIES
+  desc: $x_mem_native_properties_t
+  value: '19'
+- name: EVENT_NATIVE_PROPERTIES
+  desc: $x_event_native_properties_t
+  value: '20'
+- name: PLATFORM_NATIVE_PROPERTIES
+  desc: $x_platform_native_properties_t
+  value: '21'
+- name: DEVICE_NATIVE_PROPERTIES
+  desc: $x_device_native_properties_t
+  value: '22'
+- name: PROGRAM_NATIVE_PROPERTIES
+  desc: $x_program_native_properties_t
+  value: '23'
+- name: SAMPLER_NATIVE_PROPERTIES
+  desc: $x_sampler_native_properties_t
+  value: '24'
+- name: QUEUE_NATIVE_DESC
+  desc: $x_queue_native_desc_t
+  value: '25'
+- name: DEVICE_PARTITION_PROPERTIES
+  desc: $x_device_partition_properties_t
+  value: '26'
+- name: KERNEL_ARG_MEM_OBJ_PROPERTIES
+  desc: $x_kernel_arg_mem_obj_properties_t
+  value: '27'
+- name: PHYSICAL_MEM_PROPERTIES
+  desc: $x_physical_mem_properties_t
+  value: '28'
+- name: KERNEL_ARG_POINTER_PROPERTIES
+  desc: $x_kernel_arg_pointer_properties_t
+  value: '29'
+- name: KERNEL_ARG_SAMPLER_PROPERTIES
+  desc: $x_kernel_arg_sampler_properties_t
+  value: '30'
+- name: KERNEL_EXEC_INFO_PROPERTIES
+  desc: $x_kernel_exec_info_properties_t
+  value: '31'
+- name: KERNEL_ARG_VALUE_PROPERTIES
+  desc: $x_kernel_arg_value_properties_t
+  value: '32'
+- name: KERNEL_ARG_LOCAL_PROPERTIES
+  desc: $x_kernel_arg_local_properties_t
+  value: '33'

scripts/core/common.yml

@@ -9,6 +9,8 @@
 from fileinput import FileInput
 import util
 import yaml
+import re
+import copy
 
 ENUM_NAME = '$x_function_t'
 
@@ -18,29 +20,100 @@ class quoted(str):
 def quoted_presenter(dumper, data):
     return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
 
-def generate_registry(path, specs):
+def get_registry_header():
+    return {'type': 'header', 'desc': quoted('Intel $OneApi Unified Runtime function registry'), 'ordinal': quoted(-1)}
+
+def write_registry(data, path):
+    with open(path, 'w') as fout:
+        yaml.add_representer(quoted, quoted_presenter)
+        yaml.dump_all(data, fout,
+            default_flow_style=False,
+            sort_keys=False,
+            explicit_start=True)
+        
+def find_type_in_specs(specs, type):
+    return [obj for s in specs for obj in s['objects'] if obj['name'] == type][0]
+
+def get_max_enum(enum):
+    return int(max(enum['etors'], key=lambda x : int(x['value']))['value'])
+
+def copy_and_strip_prefix_from_enums(enum, prefix):
+    cpy = copy.deepcopy(enum)
+    for etor in cpy['etors']:
+        etor['name'] = etor['name'][len(prefix):]
+    return cpy
+        
+
+def generate_function_type(specs, meta, update_fn) -> dict:
+    existing_function_type = find_type_in_specs(specs, '$x_function_t')
+    existing_etors = {etor['name'] : etor['value'] for etor in existing_function_type['etors']}
+    max_etor = get_max_enum(existing_function_type)
+    functions = [obj['class'][len('$x'):] + obj['name'] for s in specs for obj in s['objects'] if obj['type'] == 'function']
+    registry = list()
+    for fname in functions:
+        etor_name = "$X_FUNCTION_" + util.to_snake_case(fname).upper()
+        id = existing_etors.get(etor_name)
+        if id is None:
+            max_etor += 1
+            id = max_etor
+        registry.append({
+            'name': etor_name, 
+            'desc': f'Enumerator for $x{fname}', 
+            'value': str(id)}
+            )
+    registry = sorted(registry, key=lambda x : int(x['value']))
+    existing_function_type['etors'] = registry
+    update_fn(existing_function_type, meta)
+
+    ## create a copy to write back to registry.yml
+    return copy_and_strip_prefix_from_enums(existing_function_type, '$X_FUNCTION_')
+
+
+def generate_structure_type(specs, meta, refresh_fn) -> dict:
+    structure_type = find_type_in_specs(specs, '$x_structure_type_t')
+    extended_structs = [obj for s in specs for obj in s['objects'] if re.match(r"struct|union", obj['type']) and 'base' in obj]
+    max_enum = get_max_enum(structure_type)
+
+    structure_type_etors = list()
+    for struct in extended_structs:
+        # skip experimental enumerations
+        if struct['name'].startswith('$x_exp_'):
+            continue
+
+        etor = [mem for mem in struct['members'] if mem['name'] == 'stype'][0]['init']
+
+        # try and match the etor
+        matched_etor = [e for e in structure_type['etors'] if e['name'] == etor]
+
+        out_etor = {
+            'name': etor,
+            'desc': struct['name']
+        }
+
+        # if no match exists we assign it a new value
+        if len(matched_etor) == 0:
+            max_enum += 1
+            out_etor['value'] = str(max_enum)
+        else:
+            out_etor['value'] = matched_etor[0]['value']
+
+        structure_type_etors.append(out_etor)
+    
+    structure_type_etors = sorted(structure_type_etors, key = lambda x : int(x['value']))
+    structure_type['etors'] = structure_type_etors
+    refresh_fn(structure_type, meta)
+
+    ## create a copy to write back to registry.yml
+    return copy_and_strip_prefix_from_enums(structure_type, '$X_STRUCTURE_TYPE_')
+
+def generate_registry(path, specs, meta, update_fn):
     try:
-        existing_registry = list(util.yamlRead(path))[1]['etors']
-        existing_etors = {etor["name"]: etor["value"] for etor in existing_registry}
-        max_etor = int(max(existing_registry, key = lambda x : int(x["value"]))["value"])
-        functions = [obj['class'][len('$x'):] + obj['name'] for s in specs for obj in s['objects'] if obj['type'] == 'function']
-        registry = list()
-        for fname in functions:
-            etor_name = util.to_snake_case(fname).upper()
-            id = existing_etors.get(etor_name)
-            if id is None:
-                max_etor += 1
-                id = max_etor
-            registry.append({'name': util.to_snake_case(fname).upper(), 'desc': 'Enumerator for $x'+fname, 'value': str(id)})
-        registry = sorted(registry, key=lambda x: int(x['value']))
-        wrapper = { 'name': ENUM_NAME, 'type': 'enum', 'desc': 'Defines unique stable identifiers for all functions' , 'etors': registry}
-        header = {'type': 'header', 'desc': quoted('Intel $OneApi Unified Runtime function registry'), 'ordinal': quoted(9)}
-        with open(path, 'w') as fout:
-            yaml.add_representer(quoted, quoted_presenter)
-            yaml.dump_all([header, wrapper], fout,
-                default_flow_style=False,
-                sort_keys=False,
-                explicit_start=True)
+        write_registry([
+            get_registry_header(), 
+            generate_function_type(specs, meta, update_fn), 
+            generate_structure_type(specs, meta, update_fn)
+            ], path)
+
     except BaseException as e:
         print("Failed to generate registry.yml... %s", e)
         raise e

scripts/core/registry.yml

@@ -763,14 +763,20 @@ def _append(lst, key, val):
         obj['returns'] = rets
     return obj
 
+
+def _inline_extended_structs(specs, meta):
+    for s in specs:
+        for i, obj in enumerate(s['objects']):
+            obj = _inline_base(obj, meta)
+            s['objects'][i] = obj
+
 """
     generates extra content
 """
 def _generate_extra(specs, meta):
     for s in specs:
         for i, obj in enumerate(s['objects']):
             obj = _generate_hash(obj)
-            obj = _inline_base(obj, meta)
             obj = _generate_returns(obj, meta)
             s['objects'][i] = obj
 
@@ -809,7 +815,8 @@ def _refresh_enum_meta(obj, meta):
     if obj.get('class'):
         meta['class'][obj['class']]['enum'].remove(obj['name'])
 
-    del meta['enum'][obj['name']]
+    if meta['enum'].get(obj['name']):
+        del meta['enum'][obj['name']]
     ## re-generate meta
     meta = _generate_meta(obj, None, meta)
 
@@ -856,13 +863,10 @@ def parse(section, version, tags, meta, ref):
     specs = []
 
     files = util.findFiles(path, "*.yml")
-    # make sure registry is last, because it's autogenerated based on the rest of the spec
-    files = sorted(files, key=lambda f: 1 if f.endswith('registry.yml') else 0)
+    registry = [f for f in files if f.endswith('registry.yml')][0]
 
     enum_extensions = []
     for f in files:
-        if f.endswith('registry.yml'):
-            generate_ids.generate_registry(f, specs)
 
         print("Parsing %s..."%f)
         docs = util.yamlRead(f)
@@ -914,6 +918,8 @@ def parse(section, version, tags, meta, ref):
             })
 
     specs = sorted(specs, key=lambda s: s['header']['ordinal'])
+    _inline_extended_structs(specs, meta)
+    generate_ids.generate_registry(registry, specs, meta, _refresh_enum_meta)
     _extend_enums(enum_extensions, specs, meta)
     _generate_extra(specs, meta)