Modernize kb layouts

data/README.osk-layouts

@@ -0,0 +1,33 @@
+Gnome-shell OSK layouts are extracted from CLDR layout definitions:
+https://www.unicode.org/cldr/charts/latest/keyboards/layouts/index.html
+
+Updating these involves several steps:
+
+1) Downloading and unzipping the tarball found at:
+   http://www.unicode.org/Public/cldr/latest/keyboards.zip
+
+   This file contains XML files describing the keyboard layouts.
+
+2) Cloning the cldr2json script at:
+   git://repo.or.cz/cldr2json.git
+
+   It will be used to convert the XML files into JSON that can be
+   directly consumed by gnome-shell.
+
+3) Running the script to produce the files:
+   ./cldr2json <input-directory> <output-directory>
+
+   We shall usually use the "android" folder, since that's most
+   complete, and similar to our UI and target sizes. And the target
+   directory must be data/osk-layouts in this repository.
+
+4) Modify gnome-shell-osk-layouts.gresource.xml to include the files
+
+5) Do git add on the updated/new files, and git commit.
+
+
+Or alternatively:
+
+1) Run update-osk-layouts.sh
+
+2) Do git add and git commit

data/cinnamon-osk-layouts.gresource.xml

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<gresources>
+  <gresource prefix="/org/cinnamon/osk-layouts">
+    <file>am.json</file>
+    <file>ara.json</file>
+    <file>be.json</file>
+    <file>bg.json</file>
+    <file>by.json</file>
+    <file>ca.json</file>
+    <file>cz.json</file>
+    <file>de.json</file>
+    <file>dk.json</file>
+    <file>ee.json</file>
+    <file>epo.json</file>
+    <file>es+cat.json</file>
+    <file>es.json</file>
+    <file>fi.json</file>
+    <file>fr.json</file>
+    <file>ge.json</file>
+    <file>gr.json</file>
+    <file>hr.json</file>
+    <file>hu.json</file>
+    <file>id.json</file>
+    <file>il.json</file>
+    <file>in+bolnagri.json</file>
+    <file>in+mal.json</file>
+    <file>ir.json</file>
+    <file>is.json</file>
+    <file>it.json</file>
+    <file>ke.json</file>
+    <file>kg.json</file>
+    <file>kh.json</file>
+    <file>la.json</file>
+    <file>latam.json</file>
+    <file>lt.json</file>
+    <file>lv.json</file>
+    <file>mk.json</file>
+    <file>mn.json</file>
+    <file>my.json</file>
+    <file>nl.json</file>
+    <file>no.json</file>
+    <file>ph.json</file>
+    <file>pl.json</file>
+    <file>pt.json</file>
+    <file>ro.json</file>
+    <file>rs.json</file>
+    <file>ru.json</file>
+    <file>se.json</file>
+    <file>si.json</file>
+    <file>sk.json</file>
+    <file>th.json</file>
+    <file>tr.json</file>
+    <file>ua.json</file>
+    <file>uk.json</file>
+    <file>us.json</file>
+    <file>vn.json</file>
+    <file>za.json</file>
+    <file>emoji.json</file>
+    <file>keyboard-caps-lock-filled-symbolic.svg</file>
+    <file>keyboard-enter-symbolic.svg</file>
+    <file>keyboard-shift-filled-symbolic.svg</file>
+  </gresource>
+</gresources>

data/cldr2json/README.md

@@ -0,0 +1,40 @@
+cldr2json
+=========
+
+This script converts Unicode CLDR android keyboard layouts to JSON usable by
+GNOME Shell.
+
+CLDR keyboard layouts can be found at
+<http://www.unicode.org/Public/cldr/latest/keyboards.zip>
+
+
+Usage
+=====
+
+    ./cldr2json <input file or directory> <output directory>
+
+example:
+
+    ./cldr2json cldr/keyboards/android/ json_layouts/
+
+
+Keyboard layout mapping
+=======================
+
+Unicode CLDR layout identifiers are language codes, while XKB layout
+identifiers are... something else. The mapping between the two currently uses
+heuristic based on the layout descriptions, in this order:
+
+- if the CLDR layout description matches an XKB layout description, chose its
+  XKB identifier
+- if one word of the CLDR layout description matches an XKB layout
+  description, chose its XKB identifier
+- if the CLDR layout description matches one word of an XKB layout description,
+  chose its XKB identifier
+
+That doesn't always work. For instance it fails for "en" language, that should
+match "us" XKB identifier. For such cases, there is a mapping in
+LOCALE_TO_XKB_OVERRIDES at the top of the script. If you discover a weird
+mapping of if you get a "failed to find XKB mapping for <locale>" warning then
+please consider adding an override there.
+

data/cldr2json/cldr2json.py

@@ -0,0 +1,212 @@
+#!/usr/bin/python3
+#
+# Copyright 2015  Daiki Ueno <dueno@src.gnome.org>
+#           2016  Parag Nemade <pnemade@redhat.com>
+#           2017  Alan <alan@boum.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, see
+# <http://www.gnu.org/licenses/>.
+
+import glob
+import json
+import locale
+import logging
+import os
+import re
+import sys
+import xml.etree.ElementTree
+
+import gi
+gi.require_version('GnomeDesktop', '3.0')   # NOQA: E402
+from gi.repository import GnomeDesktop
+
+ESCAPE_PATTERN = re.compile(r'\\u\{([0-9A-Fa-f]+?)\}')
+ISO_PATTERN = re.compile(r'[A-E]([0-9]+)')
+
+LOCALE_TO_XKB_OVERRIDES = {
+    'af':    'za',
+    'en':    'us',
+    'en-GB': 'uk',
+    'es-US': 'latam',
+    'fr-CA': 'ca',
+    'hi':    'in+bolnagri',
+    'ky':    'kg',
+    'nl-BE': 'be',
+    'zu':    None
+}
+
+
+def parse_single_key(value):
+    def unescape(m):
+        return chr(int(m.group(1), 16))
+    value = ESCAPE_PATTERN.sub(unescape, value)
+    return value
+
+
+def parse_rows(keymap):
+    unsorted_rows = {}
+    for _map in keymap.iter('map'):
+        value = _map.get('to')
+        key = [parse_single_key(value)]
+        iso = _map.get('iso')
+        if not ISO_PATTERN.match(iso):
+            sys.stderr.write('invalid ISO key name: %s\n' % iso)
+            continue
+        if not iso[0] in unsorted_rows:
+            unsorted_rows[iso[0]] = []
+        unsorted_rows[iso[0]].append((int(iso[1:]), key))
+        # add subkeys
+        longPress = _map.get('longPress')
+        if longPress:
+            for value in longPress.split(' '):
+                subkey = parse_single_key(value)
+                key.append(subkey)
+
+    rows = []
+    for k, v in sorted(list(unsorted_rows.items()),
+                       key=lambda x: x[0],
+                       reverse=True):
+        row = []
+        for key in sorted(v, key=lambda x: x):
+            row.append(key[1])
+        rows.append(row)
+
+    return rows
+
+
+def convert_xml(tree):
+    root = {}
+    for xml_keyboard in tree.iter("keyboard"):
+        locale_full = xml_keyboard.get("locale")
+        locale, sep, end = locale_full.partition("-t-")
+    root["locale"] = locale
+    for xml_name in tree.iter("name"):
+        name = xml_name.get("value")
+    root["name"] = name
+    root["levels"] = []
+    # parse levels
+    for index, keymap in enumerate(tree.iter('keyMap')):
+        # FIXME: heuristics here
+        modifiers = keymap.get('modifiers')
+        if not modifiers:
+            mode = 'default'
+            modifiers = ''
+        elif 'shift' in modifiers.split(' '):
+            mode = 'latched'
+            modifiers = 'shift'
+        else:
+            mode = 'locked'
+        level = {}
+        level["level"] = modifiers
+        level["mode"] = mode
+        level["rows"] = parse_rows(keymap)
+        root["levels"].append(level)
+    return root
+
+
+def locale_to_xkb(locale, name):
+    if locale in sorted(LOCALE_TO_XKB_OVERRIDES.keys()):
+        xkb = LOCALE_TO_XKB_OVERRIDES[locale]
+        logging.debug("override for %s → %s",
+                      locale, xkb)
+        if xkb:
+            return xkb
+        else:
+            raise KeyError("layout %s explicitely disabled in overrides"
+                           % locale)
+    xkb_names = sorted(name_to_xkb.keys())
+    if name in xkb_names:
+        return name_to_xkb[name]
+    else:
+        logging.debug("name %s failed" % name)
+    for sub_name in name.split(' '):
+        if sub_name in xkb_names:
+            xkb = name_to_xkb[sub_name]
+            logging.debug("dumb mapping failed but match with locale word: "
+                          "%s (%s) → %s (%s)",
+                          locale, name, xkb, sub_name)
+            return xkb
+        else:
+            logging.debug("sub_name failed")
+    for xkb_name in xkb_names:
+        for xkb_sub_name in xkb_name.split(' '):
+            if xkb_sub_name.strip('()') == name:
+                xkb = name_to_xkb[xkb_name]
+                logging.debug("dumb mapping failed but match with xkb word: "
+                              "%s (%s) → %s (%s)",
+                              locale, name, xkb, xkb_name)
+                return xkb
+    raise KeyError("failed to find XKB mapping for %s" % locale)
+
+
+def convert_file(source_file, destination_path):
+    logging.info("Parsing %s", source_file)
+
+    itree = xml.etree.ElementTree.ElementTree()
+    itree.parse(source_file)
+
+    root = convert_xml(itree)
+
+    try:
+        xkb_name = locale_to_xkb(root["locale"], root["name"])
+    except KeyError as e:
+        logging.warning(e)
+        return False
+    destination_file = os.path.join(destination_path, xkb_name + ".json")
+
+    try:
+        with open(destination_file, 'x', encoding="utf-8") as dest_fd:
+            json.dump(root, dest_fd, ensure_ascii=False, indent=2, sort_keys=True)
+    except FileExistsError as e:
+        logging.info("File %s exists, not updating", destination_file)
+        return False
+
+    logging.debug("written %s", destination_file)
+
+
+def load_xkb_mappings():
+    xkb = GnomeDesktop.XkbInfo()
+    layouts = xkb.get_all_layouts()
+    name_to_xkb = {}
+
+    for layout in layouts:
+        name = xkb.get_layout_info(layout).display_name
+        name_to_xkb[name] = layout
+
+    return name_to_xkb
+
+
+locale.setlocale(locale.LC_ALL, "C")
+name_to_xkb = load_xkb_mappings()
+
+
+if __name__ == "__main__":
+    if "DEBUG" in os.environ:
+        logging.basicConfig(level=logging.DEBUG)
+
+    if len(sys.argv) < 2:
+        print("supply a CLDR keyboard file")
+        sys.exit(1)
+
+    if len(sys.argv) < 3:
+        print("supply an output directory")
+        sys.exit(1)
+
+    source = sys.argv[1]
+    destination = sys.argv[2]
+    if os.path.isfile(source):
+        convert_file(source, destination)
+    elif os.path.isdir(source):
+        for path in glob.glob(source + "/*-t-k0-android.xml"):
+            convert_file(path, destination)