Reintroduce pylint and fix all the violations. Hopefully we'll all se…

…e consistent errors now (compiler-explorer#827)

.pre-commit-config.yaml

@@ -34,6 +34,11 @@ repos:
       - id: shellcheck
   - repo: local
     hooks:
+      - id: pylint
+        name: pylint
+        entry: .poetry/bin/poetry run pylint
+        language: system
+        types: [python]
       - id: test
         name: Run tests
         entry: make test

bin/lib/amazon_properties.py

@@ -26,7 +26,7 @@ def get_properties_compilers_and_libraries(language, logger):
 
     encoded_language = urllib.parse.quote(language)
     url = f"https://raw.githubusercontent.com/compiler-explorer/compiler-explorer/main/etc/config/{encoded_language}.amazon.properties"
-    request = requests.get(url)
+    request = requests.get(url, timeout=30)
     if not request.ok:
         raise RuntimeError(f"Fetch failure for {url}: {request}")
     lines = request.text.splitlines(keepends=False)

bin/lib/cli/builds.py

@@ -125,6 +125,7 @@ def builds_set_current(cfg: Config, branch: Optional[str], version: str, raw: bo
                 f"https://sentry.io/api/0/organizations/compiler-explorer/releases/{release.version}/deploys/",
                 data=dict(environment=cfg.env.value),
                 headers=dict(Authorization=f"Bearer {token}"),
+                timeout=30,
             )
             if not result.ok:
                 raise RuntimeError(f"Failed to send to sentry: {result} {result.content.decode('utf-8')}")

bin/lib/library_builder.py

@@ -21,6 +21,8 @@
 from lib.library_build_config import LibraryBuildConfig
 from lib.staging import StagingDir
 
+_TIMEOUT = 600
+
 build_supported_os = ["Linux"]
 build_supported_buildtype = ["Debug"]
 build_supported_arch = ["x86_64", "x86"]
@@ -604,7 +606,9 @@ def conanproxy_login(self):
         login_body = defaultdict(lambda: [])
         login_body["password"] = get_ssm_param("/compiler-explorer/conanpwd")
 
-        request = requests.post(url, data=json.dumps(login_body), headers={"Content-Type": "application/json"})
+        request = requests.post(
+            url, data=json.dumps(login_body), headers={"Content-Type": "application/json"}, timeout=_TIMEOUT
+        )
         if not request.ok:
             self.logger.info(request.text)
             raise RuntimeError(f"Post failure for {url}: {request}")
@@ -639,7 +643,7 @@ def save_build_logging(self, builtok, buildfolder):
 
         headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token}
 
-        request = requests.post(url, data=json.dumps(buildparameters_copy), headers=headers)
+        request = requests.post(url, data=json.dumps(buildparameters_copy), headers=headers, timeout=_TIMEOUT)
         if not request.ok:
             raise RuntimeError(f"Post failure for {url}: {request}")
 
@@ -650,7 +654,7 @@ def get_build_annotations(self, buildfolder):
 
         url = f"{conanserver_url}/annotations/{self.libname}/{self.target_name}/{conanhash}"
         with tempfile.TemporaryFile() as fd:
-            request = requests.get(url, stream=True)
+            request = requests.get(url, stream=True, timeout=_TIMEOUT)
             if not request.ok:
                 raise RuntimeError(f"Fetch failure for {url}: {request}")
             for chunk in request.iter_content(chunk_size=4 * 1024 * 1024):
@@ -678,7 +682,9 @@ def has_failed_before(self):
         headers = {"Content-Type": "application/json"}
 
         url = f"{conanserver_url}/hasfailedbefore"
-        request = requests.post(url, data=json.dumps(self.current_buildparameters_obj), headers=headers)
+        request = requests.post(
+            url, data=json.dumps(self.current_buildparameters_obj), headers=headers, timeout=_TIMEOUT
+        )
         if not request.ok:
             raise RuntimeError(f"Post failure for {url}: {request}")
         else:
@@ -722,7 +728,7 @@ def set_as_uploaded(self, buildfolder):
         headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token}
 
         url = f"{conanserver_url}/annotations/{self.libname}/{self.target_name}/{conanhash}"
-        request = requests.post(url, data=json.dumps(annotations), headers=headers)
+        request = requests.post(url, data=json.dumps(annotations), headers=headers, timeout=_TIMEOUT)
         if not request.ok:
             raise RuntimeError(f"Post failure for {url}: {request}")
 

bin/lib/rust_library_builder.py

@@ -22,6 +22,7 @@
 from lib.amazon_properties import get_properties_compilers_and_libraries
 from lib.library_build_config import LibraryBuildConfig
 
+_TIMEOUT = 600
 # min_compiler_version = version.parse('1.56.0')
 skip_compilers = ["nightly", "beta", "gccrs-snapshot", "mrustc-master", "rustccggcc-master"]
 
@@ -259,7 +260,9 @@ def conanproxy_login(self):
         login_body = defaultdict(lambda: [])
         login_body["password"] = get_ssm_param("/compiler-explorer/conanpwd")
 
-        request = requests.post(url, data=json.dumps(login_body), headers={"Content-Type": "application/json"})
+        request = requests.post(
+            url, data=json.dumps(login_body), headers={"Content-Type": "application/json"}, timeout=_TIMEOUT
+        )
         if not request.ok:
             self.logger.info(request.text)
             raise RuntimeError(f"Post failure for {url}: {request}")
@@ -292,7 +295,7 @@ def save_build_logging(self, builtok, build_folder, source_folder):
 
         headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token}
 
-        request = requests.post(url, data=json.dumps(buildparameters_copy), headers=headers)
+        request = requests.post(url, data=json.dumps(buildparameters_copy), headers=headers, timeout=_TIMEOUT)
         if not request.ok:
             raise RuntimeError(f"Post failure for {url}: {request}")
 
@@ -303,7 +306,7 @@ def get_build_annotations(self, buildfolder):
 
         url = f"{conanserver_url}/annotations/{self.libname}/{self.target_name}/{conanhash}"
         with tempfile.TemporaryFile() as fd:
-            request = requests.get(url, stream=True)
+            request = requests.get(url, stream=True, timeout=_TIMEOUT)
             if not request.ok:
                 raise RuntimeError(f"Fetch failure for {url}: {request}")
             for chunk in request.iter_content(chunk_size=4 * 1024 * 1024):
@@ -320,7 +323,9 @@ def has_failed_before(self):
         headers = {"Content-Type": "application/json"}
 
         url = f"{conanserver_url}/hasfailedbefore"
-        request = requests.post(url, data=json.dumps(self.current_buildparameters_obj), headers=headers)
+        request = requests.post(
+            url, data=json.dumps(self.current_buildparameters_obj), headers=headers, timeout=_TIMEOUT
+        )
         if not request.ok:
             raise RuntimeError(f"Post failure for {url}: {request}")
         else:
@@ -358,7 +363,7 @@ def set_as_uploaded(self, buildfolder, source_folder, build_method):
         headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token}
 
         url = f"{conanserver_url}/annotations/{self.libname}/{self.target_name}/{conanhash}"
-        request = requests.post(url, data=json.dumps(annotations), headers=headers)
+        request = requests.post(url, data=json.dumps(annotations), headers=headers, timeout=_TIMEOUT)
         if not request.ok:
             raise RuntimeError(f"Post failure for {url}: {request}")
 

lambda/cloudwatch_to_discord.py

@@ -16,7 +16,7 @@ def lambda_handler(event, _context):
 
         headers = {"content-type": "application/json"}
         logging.info(discord_data)
-        response = requests.post(webhook_url, data=json.dumps(discord_data), headers=headers)
+        response = requests.post(webhook_url, data=json.dumps(discord_data), headers=headers, timeout=30)
 
         logging.info("Discord response: %s", response.status_code)
         logging.info(response.content)

make_json.py

@@ -2,11 +2,17 @@
 import os
 import json
 
-obj = {}
-config = ConfigParser()
-config.read(os.path.join(os.getenv("HOME", ""), ".aws", "credentials"))
-obj["MY_ACCESS_KEY"] = config.get("default", "aws_access_key_id", fallback="")
-obj["MY_SECRET_KEY"] = config.get("default", "aws_secret_access_key", fallback="")
-
-with open("config.json", "w") as out:
-    json.dump(obj, out)
+
+def main():
+    obj = {}
+    config = ConfigParser()
+    config.read(os.path.join(os.getenv("HOME", ""), ".aws", "credentials"))
+    obj["MY_ACCESS_KEY"] = config.get("default", "aws_access_key_id", fallback="")
+    obj["MY_SECRET_KEY"] = config.get("default", "aws_secret_access_key", fallback="")
+
+    with open("config.json", "w", encoding="utf-8") as out:
+        json.dump(obj, out)
+
+
+if __name__ == "__main__":
+    main()