toolchain: Add pagination to check-security-tools.py DOCS-340 (codacy#1256)

* toolchain: Add .idea IDE files to .gitignore

* fix: Review ignored tool UUIDs

* fix: Implement simple pagination mechanism to fetch all code patterns

* fix: Make sure that Pylint is detected

Author: Paulo Ribeiro

.gitignore

@@ -7,6 +7,7 @@ site/
 # IDE files
 .vscode
 .metals
+.idea
 
 # Backup files
 *.swp

tools/check-security-tools.py

@@ -8,9 +8,8 @@
 DOCUMENTATION_PATH = "../docs/repositories/security-monitor.md"
 ENDPOINT_URL_TOOLS = "https://api.codacy.com/api/v3/tools"
 ENDPOINT_URL_CODE_PATTERNS = Template("https://api.codacy.com/api/v3/tools/${toolUuid}/patterns")
-IGNORED_TOOL_UUIDS = ["647dddc1-17c4-4840-acea-4c2c2bbecb45", # Codacy Scalameta Pro
-                      "31677b6d-4ae0-4f56-8041-606a8d7a8e61", # Pylint 2 (Python 3)
-                      "cf05f3aa-fd23-4586-8cce-5368917ec3e5"] # ESLint 7 (deprecated)
+IGNORED_TOOL_UUIDS = ["34225275-f79e-4b85-8126-c7512c987c0d",  # Pylint 1.9 (legacy)
+                      "cf05f3aa-fd23-4586-8cce-5368917ec3e5"]  # ESLint 7 (deprecated)
 
 
 def check_security_tools():
@@ -24,16 +23,25 @@ def check_security_tools():
             continue
         tool_name = tool["name"]
         tool_short_name = tool["shortName"]
+        # Hack to ensure that Pylint is detected
+        if tool_short_name == "pylintpython3":
+            tool_name = "Pylint"
         tool_languages = tool["languages"]
-        code_patterns = requests.get(ENDPOINT_URL_CODE_PATTERNS.substitute(toolUuid=tool["uuid"])).json()["data"]
+        r = requests.get(ENDPOINT_URL_CODE_PATTERNS.substitute(toolUuid=tool["uuid"]))
+        code_patterns = r.json()["data"]
+        cursor = r.json()["pagination"].get("cursor", False)
+        while cursor:
+            r = requests.get(ENDPOINT_URL_CODE_PATTERNS.substitute(toolUuid=tool["uuid"]) + f"?cursor={cursor}")
+            code_patterns += r.json()["data"]
+            cursor = r.json()["pagination"].get("cursor", False)
         for code_pattern in code_patterns:
             if code_pattern["category"] == "Security":
                 if tool_name.lower() in documentation or tool_short_name.lower() in documentation:
-                    print(emoji.emojize(f":check_mark_button: {tool_name} is included "
-                                        f"({', '.join(map(str, tool_languages))})"))
+                    print(emoji.emojize(f":check_mark_button: {tool_name} ({', '.join(map(str, tool_languages))}) "
+                                        f"is included, checked {len(code_patterns)} patterns"))
                 else:
-                    print(emoji.emojize(f":cross_mark: {tool_name} ISN'T included "
-                                        f"({', '.join(map(str, tool_languages))})"))
+                    print(emoji.emojize(f":cross_mark: {tool_name} ({', '.join(map(str, tool_languages))}) "
+                                        f"ISN'T included, checked {len(code_patterns)} patterns"))
                     count += 1
                 break
     if count:

tools/check-supported-tools.py

@@ -5,9 +5,9 @@
 
 DOCUMENTATION_PATH = "../docs/getting-started/supported-languages-and-tools.md"
 ENDPOINT_URL = "https://api.codacy.com/api/v3/tools"
-IGNORED_TOOL_UUIDS = ["647dddc1-17c4-4840-acea-4c2c2bbecb45", # Codacy Scalameta Pro
-                      "31677b6d-4ae0-4f56-8041-606a8d7a8e61", # Pylint 2 (Python 3)
-                      "cf05f3aa-fd23-4586-8cce-5368917ec3e5"] # ESLint 7 (deprecated)
+IGNORED_TOOL_UUIDS = ["647dddc1-17c4-4840-acea-4c2c2bbecb45",  # Codacy Scalameta Pro
+                      "34225275-f79e-4b85-8126-c7512c987c0d",  # Pylint 1.9 (legacy)
+                      "cf05f3aa-fd23-4586-8cce-5368917ec3e5"]  # ESLint 7 (deprecated)
 
 
 def check_supported_tools():
@@ -21,6 +21,9 @@ def check_supported_tools():
             continue
         tool_name = tool["name"]
         tool_short_name = tool["shortName"]
+        # Hack to ensure that Pylint is detected
+        if tool_short_name == "pylintpython3":
+            tool_name = "Pylint"
         tool_languages = tool["languages"]
         if tool_name.lower() in documentation or tool_short_name.lower() in documentation:
             print(emoji.emojize(f":check_mark_button: {tool_name} is included "