WIP: Singlestore test 4.2.x

tests/_utils/local_test.sh

@@ -0,0 +1,48 @@
+export DJANGO_HOME=`pwd`
+
+export PYTHONPATH=$DJANGO_HOME:$DJANGO_HOME/tests:$DJANGO_HOME/tests/singlestore_settings:$PYTHONPATH
+
+# export NOT_ENFORCED_UNIQUE=1
+
+export TABLE_STORAGE_TYPE_ADMIN="ROWSTORE REFERENCE"
+export TABLE_STORAGE_TYPE_AUTH="ROWSTORE REFERENCE"
+export TABLE_STORAGE_TYPE_CONTENTTYPES="ROWSTORE REFERENCE"
+export TABLE_STORAGE_TYPE_SITES="ROWSTORE REFERENCE"
+
+
+export TABLE_STORAGE_TYPE_SERIALIZERS="ROWSTORE"
+export TABLE_STORAGE_TYPE_ADMIN_INLINES="ROWSTORE REFERENCE"
+export TABLE_STORAGE_TYPE_AUTH_TESTS="REFERENCE"
+export TABLE_STORAGE_TYPE_INTROSPECTION="ROWSTORE REFERENCE"
+export TABLE_STORAGE_TYPE_VALIDATION="ROWSTORE REFERENCE"
+export TABLE_STORAGE_TYPE_CONSTRAINTS="ROWSTORE REFERENCE"
+export TABLE_STORAGE_TYPE_BULK_CREATE="ROWSTORE REFERENCE"
+
+# 12 many-to-many fields
+export TABLE_STORAGE_TYPE_PREFETCH_RELATED="ROWSTORE REFERENCE"
+
+# abstract models - specifying through is tricky 
+export TABLE_STORAGE_TYPE_MANY_TO_MANY="ROWSTORE REFERENCE"
+
+
+
+prepare_settings() {
+    for dir in $(find tests -maxdepth 1 -type d | awk -F '/' '{print $2}'); do
+        # echo $dir
+        sed -e "s|TEST_MODULE|$dir|g" tests/singlestore_settings_TMPL > tests/singlestore_settings/singlestore_settings_$dir.py
+    done
+}
+
+# prepare_settings
+
+
+run_all_tests() {
+    for dir in $(find tests -maxdepth 1 -type d | awk -F '/' '{print $2}'); do
+        echo $dir
+        ./tests/runtests.py --settings=singlestore_settings_$dir --noinput -v 3 $dir >> django_test_result_$dir.txt 2>&1
+    done
+}
+# python run_tests_parallel.py
+# run_all_tests
+
+./tests/runtests.py --settings=singlestore_settings --noinput -v 3 queries --keepdb

tests/_utils/model_transformer.py

@@ -0,0 +1,81 @@
+import re
+
+
+def generate_intermediary_code(original_code):
+    # Extract class name and many-to-many field definition
+    class_match = re.search(r'class (\w+)\((?:models\.Model|[\w\.]+)\):', original_code)
+    m2m_match = re.search(r'(\w+)\s*=\s*models\.ManyToManyField\(\s*"?(self|\w+)"?\s*(.*)\)', original_code)
+
+    if not class_match or not m2m_match:
+        return "Invalid original code format"
+
+    class_name = class_match.group(1)
+    field_name = m2m_match.group(1)
+    related_model = m2m_match.group(2)
+    additional_args = m2m_match.group(3).strip()
+
+    # Handle self-referencing models
+    if related_model == "self":
+        related_model = class_name
+        intermediary_model_name = f"{class_name}Friend"
+        from_field_name = f"from_{class_name.lower()}"
+        to_field_name = f"to_{class_name.lower()}"
+    else:
+        intermediary_model_name = f"{class_name}{related_model}"
+        from_field_name = class_name.lower()
+        to_field_name = related_model.lower()
+
+    # Properly format the new code with a ManyToManyField that uses through
+    additional_args_str = f", {additional_args}" if additional_args else ""
+    through_str = f'through="{intermediary_model_name}"'
+    new_code = re.sub(
+        r'(\s*' + field_name + r'\s*=\s*models\.ManyToManyField\([^\)]*\))',
+        rf'\n    {field_name} = models.ManyToManyField("{related_model}"{additional_args_str}, {through_str})',
+        original_code
+    ).replace(",,", ",")  # This will clean up any double commas
+
+    intermediary_code = f"""
+
+class {intermediary_model_name}(models.Model):
+    {from_field_name} = models.ForeignKey({class_name}, on_delete=models.CASCADE)
+    {to_field_name} = models.ForeignKey({related_model}, on_delete=models.CASCADE)
+
+    class Meta:
+        unique_together = (('{from_field_name}', '{to_field_name}'),)
+        db_table = "{class_name.lower()}_{related_model.lower()}"
+"""
+
+    # Generate the SQL query for creating the intermediary table
+    sql_query = f"""
+CREATE TABLE `{class_name.lower()}_{related_model.lower()}` (
+  `{from_field_name}_id` BIGINT NOT NULL,
+  `{to_field_name}_id` BIGINT NOT NULL,
+  SHARD KEY (`{from_field_name}_id`),
+  UNIQUE KEY (`{from_field_name}_id`, `{to_field_name}_id`),
+  KEY (`{from_field_name}_id`),
+  KEY (`{to_field_name}_id`)
+);
+"""
+
+    return new_code + intermediary_code + "\nSQL Query:\n" + sql_query
+
+
+# Example usage
+original_code = """
+class Article(models.Model):
+    headline = models.CharField(max_length=100)
+    # Assign a string as name to make sure the intermediary model is
+    # correctly created. Refs #20207
+    authors = models.ManyToManyField("User", through="UserArticle")
+
+    objects = NoDeletedArticleManager()
+
+    class Meta:
+        ordering = ("headline",)
+
+    def __str__(self):
+        return self.headline
+"""
+
+new_code = generate_intermediary_code(original_code)
+print(new_code)

tests/_utils/run_tests_parallel.py

@@ -0,0 +1,80 @@
+import os
+import subprocess
+from multiprocessing import Pool
+
+
+TEST_MODULES = [d for d in os.listdir("tests") if os.path.isdir(os.path.join("tests", d))]
+TEST_MODULES = [t for t in TEST_MODULES if t != "requirements" and t != "__pycache__"]
+MAX_NUM_TEST_MODULES = 80
+DEFAULT_OUT_FILE = "django_test_results/results.txt"
+DEFAULT_SETTINGS = "singlestore_settings"
+
+
+def run_single_test(module_name, out_file=None, settings=None):
+    print(f"Running tests for {module_name}")
+    if out_file is None:
+        out_file = f"django_test_results/{module_name}.txt"
+    if settings is None:
+        settings = f"singlestore_settings_{module_name}"
+    cmd = f"./tests/runtests.py --settings={settings} --noinput -v 3 {module_name} >> {out_file} 2>&1"
+    result = subprocess.run(cmd, shell=True, text=True, capture_output=True)
+    print(result.returncode)
+    return result
+
+
+def run_all_tests():
+
+    with Pool(6) as workers:
+        results = workers.map(run_single_test, TEST_MODULES)
+    print(results)
+
+
+def run_chunked_tests():
+    current_tests = []
+    for t in TEST_MODULES:
+        current_tests.append(t)
+        if len(current_tests) >= MAX_NUM_TEST_MODULES:
+            module_name = " ".join(current_tests)
+            run_single_test(module_name, DEFAULT_OUT_FILE, DEFAULT_SETTINGS)
+            current_tests = []
+
+    if len(current_tests) > 0:
+        run_single_test(module_name, DEFAULT_OUT_FILE, DEFAULT_SETTINGS)
+
+
+def analyze_results():
+    ok_list = []
+    fail_list = []
+    total_tests = 0
+    for f_name in os.listdir("django_test_results"):
+        ok = False
+        with open(os.path.join("django_test_results", f_name), "r") as f:
+            all_lines = list(f.readlines())
+            for ln in all_lines:
+                if " tests in " in ln:
+                    tokens = ln.strip().split(" ")
+                    total_tests += int(tokens[1])
+            lines = [line.strip() for line in all_lines[-3:]]
+            lines = [line for line in lines if len(line)]
+
+            for ln in lines:
+                if "OK" in ln:
+                    ok_list.append((f_name, lines))
+                    ok = True
+                    break
+            if not ok:
+                result = None
+                for ln in lines:
+                    if "FAILED" in ln:
+                        result = ln
+                fail_list.append((f_name, result))
+
+    for elem in fail_list:
+        print(elem)
+    print(f"Total tests {total_tests}")
+
+
+if __name__ == "__main__":
+    analyze_results()
+    # run_all_tests()
+    # run_chunked_tests()