partitions portion of dictionary re-written

Author: azhou-datadog

mysql/assets/configuration/spec.yaml

@@ -385,6 +385,7 @@ files:
             hidden: true
             description: |
               Configure collection of schemas (databases).
+              Only tables and schemas for which the user has been granted SELECT privileges are collected.
             options:
               - name: enabled
                 description: |

mysql/datadog_checks/mysql/databases_data.py

@@ -9,6 +9,7 @@
 import json
 import time
 from contextlib import closing
+from collections import defaultdict
 
 import pymysql
 
@@ -209,6 +210,7 @@ def _collect_databases_data(self, tags):
                             - index_type (str): The index method used.
                             - columns (list): A list of column dictionaries
                                 - column (dict): A dictionary representing a column.
+                                    - name (str): The name of the column.
                                     - sub_part (int): The number of indexed characters if column is partially indexed.
                                     - packed (str): How the index is packed.
                                     - nullable (bool): Whether the column is nullable.
@@ -340,6 +342,8 @@ def _populate_with_index_data(self, table_name_to_table_index, table_list, table
         self._cursor_run(cursor, query=SQL_INDEXES.format(table_names), params=db_name)
         rows = cursor.fetchall()
         for row in rows:
+            print("ALLEN!!!")
+            print("row:", json.dumps(row, indent=4))
             table_name = str(row.pop("table_name"))
             table_list[table_name_to_table_index[table_name]].setdefault("indexes", [])
             if "nullables" in row:
@@ -351,7 +355,6 @@ def _populate_with_index_data(self, table_name_to_table_index, table_list, table
                     else:
                         nullables_converted += "false,"
                 row["nullables"] = nullables_converted[:-1]
-                # TODO ALLEN: make this booleans rather than strings
             table_list[table_name_to_table_index[table_name]]["indexes"].append(row)
 
     @tracked_method(agent_check_getter=agent_check_getter, track_result_length=True)
@@ -367,7 +370,46 @@ def _populate_with_foreign_keys_data(self, table_name_to_table_index, table_list
     def _populate_with_partitions_data(self, table_name_to_table_index, table_list, table_names, db_name, cursor):
         self._cursor_run(cursor, query=SQL_PARTITION.format(table_names), params=db_name)
         rows = cursor.fetchall()
+        if not rows:
+            return
+        table_partitions_dict = defaultdict(lambda: defaultdict(lambda: {
+            "name": None,
+            "subpartitions": [],
+            "partition_ordinal_position": None,
+            "partition_method": None,
+            "partition_expression": None,
+            "partition_description": None,
+            "table_rows": 0,
+            "data_length": 0,
+        }))
+
         for row in rows:
-            table_name = str(row.pop("table_name"))
+            table_name = str(row["table_name"])
             table_list[table_name_to_table_index[table_name]].setdefault("partitions", [])
-            table_list[table_name_to_table_index[table_name]]["partitions"].append(row)
+            partition_name = str(row["name"])
+            partition_data = table_partitions_dict[table_name][partition_name]
+
+            # Update partition-level info
+            partition_data["name"] = partition_name
+            partition_data["partition_ordinal_position"] = int(row["partition_ordinal_position"])
+            partition_data["partition_method"] = str(row["partition_method"])
+            partition_data["partition_expression"] = str(row["partition_expression"]).strip().lower()
+            partition_data["partition_description"] = str(row["partition_description"])
+            partition_data["table_rows"] += int(row["table_rows"])
+            partition_data["data_length"] += int(row["data_length"])
+
+            # Add subpartition info, if exists
+            if row["subpartition_name"]:
+                subpartition = {
+                    "name": row["subpartition_name"],
+                    "subpartition_ordinal_position": int(row["subpartition_ordinal_position"]),
+                    "subpartition_method": str(row["subpartition_method"]),
+                    "subpartition_expression": str(row["subpartition_expression"]).strip().lower(),
+                    "table_rows": int(row["table_rows"]),
+                    "data_length": int(row["data_length"]),
+                }
+                partition_data["subpartitions"].append(subpartition)
+        for table_name, partitions_dict in table_partitions_dict.items():
+            table_list[table_name_to_table_index[table_name]]["partitions"] = list(partitions_dict.values())
+        
+        print(json.dumps(partition_data, indent=4))

mysql/datadog_checks/mysql/queries.py

@@ -118,19 +118,19 @@
 SQL_INDEXES = """\
 SELECT
     table_name as `table_name`,
-    index_schema as `index_schema`,
     index_name as `name`,
     collation as `collation`,
+    cardinality as `cardinality`,
     index_type as `index_type`,
-    group_concat(seq_in_index order by seq_in_index asc) as seq_in_index,
-    group_concat(column_name order by seq_in_index asc) as columns,
-    group_concat(sub_part order by seq_in_index asc) as sub_parts,
-    group_concat(packed order by seq_in_index asc) as packed,
-    group_concat(nullable order by seq_in_index asc) as nullables,
-    group_concat(non_unique order by seq_in_index asc) as non_uniques
+    seq_in_index as `seq_in_index`,
+	column_name as `column_name`,
+    sub_part as `sub_part`,
+    packed as `packed`,
+    nullable as `nullable`,
+    non_unique as `non_unique`,
+    IFNULL(expression, NULL) as `expression`
 FROM INFORMATION_SCHEMA.STATISTICS
-WHERE table_schema = %s AND table_name IN ({})
-GROUP BY table_name, index_schema, index_name, collation, index_type;
+WHERE table_schema = %s AND table_name IN ({});
 """
 
 SQL_FOREIGN_KEYS = """\
@@ -163,12 +163,7 @@
     subpartition_expression as `subpartition_expression`,
     partition_description as `partition_description`,
     table_rows as `table_rows`,
-    data_length as `data_length`,
-    max_data_length as `max_data_length`,
-    index_length as `index_length`,
-    data_free as `data_free`,
-    partition_comment as `partition_comment`,
-    tablespace_name as `tablespace_name`
+    data_length as `data_length`
 FROM INFORMATION_SCHEMA.PARTITIONS
 WHERE
     table_schema = %s AND table_name in ({}) AND partition_name IS NOT NULL