Django to 3.0

.circleci/config.yml

@@ -38,6 +38,9 @@ jobs:
             docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django python manage.py collectstatic --noinput
 
       - run: docker-compose exec django flake8 src/
+      - run: docker pull codalab/codalab-legacy:py37 # not available without "not e2e" tests as they pull ahead of time
+      - run: docker pull codalab/codalab-legacy:py3 # not available without "not e2e" tests as they pull ahead of time
+      - run: docker pull vergilgxw/autotable:v2 # not available without "not e2e" tests as they pull ahead of time
 
       - run:
           name: pytest
@@ -48,6 +51,13 @@ jobs:
           command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/tests/functional/ -m e2e
           no_output_timeout: 60m
 
+      # Example to run specific set of tests (for debugging individual tests from a batch of tests)
+      # - run:
+      #     name: e2e tests - competitions
+      #     command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/tests/functional/test_competitions.py -m e2e
+      #     no_output_timeout: 60m
+
+
       - store_artifacts:
           path: artifacts/
 

Caddyfile

@@ -2,7 +2,6 @@
   # HTTPS Options
   tls {$TLS_EMAIL}
 
-
   # Test HTTPS setup
   # tls {$TLS_EMAIL} {
   #   ca https://acme-staging-v02.api.letsencrypt.org/directory
@@ -20,23 +19,23 @@
     root /srv
     try_files maintenance.on
   }
-	handle @maintenanceModeActive {
+  handle @maintenanceModeActive {
     root * /srv
     redir @noRedirect /maintenance.html
-		file_server {
+    file_server {
       status 503
     }
-	}
+  }
 
   # Serves static files, should be the same as `STATIC_ROOT` setting:
-  root *  /var/www/django
+  root * /var/www/django
   file_server
-  
+
   @noStatic {
     not path /static/*
     not path /media/*
   }
-  
+
 
   # Serving dynamic requests:
   reverse_proxy @noStatic django:8000

compute_worker/compute_worker.py

@@ -280,7 +280,7 @@ async def watch_detailed_results(self):
             else:
                 logger.info(time.time() - start)
                 if time.time() - start > expiration_seconds:
-                    timeout_error_message = f"Detailed results not written to after {expiration_seconds} seconds, exiting!"
+                    timeout_error_message = f"WARNING: Detailed results not written before the execution."
                     logger.warning(timeout_error_message)
             await asyncio.sleep(5)
             file_path = self.get_detailed_results_file_path()
@@ -302,7 +302,9 @@ def get_detailed_results_file_path(self):
     async def send_detailed_results(self, file_path):
         logger.info(f"Updating detailed results {file_path} - {self.detailed_results_url}")
         self._put_file(self.detailed_results_url, file=file_path, content_type='text/html')
-        async with websockets.connect(self.websocket_url) as websocket:
+        websocket_url = f"{self.websocket_url}?kind=detailed_results"
+        logger.info(f"Connecting to {websocket_url} for detailed results")
+        async with websockets.connect(websocket_url) as websocket:
             await websocket.send(json.dumps({
                 "kind": 'detailed_result_update',
             }))
@@ -390,10 +392,14 @@ async def _send_data_through_socket(self, error_message):
         - Docker image pull failure logs
         - Execution time limit exceeded logs
         """
-        logger.info(f"Connecting to {self.websocket_url} to send docker image pull error")
+        # Create a unique websocket URL for error messages
+        websocket_url = f"{self.websocket_url}?kind=error_logs"
+        logger.info(f"Connecting to {websocket_url} to send error message")
+
+        logger.info(f"Connecting to {websocket_url} to send docker image pull error")
 
         # connect to web socket
-        websocket = await websockets.connect(self.websocket_url)
+        websocket = await websockets.connect(websocket_url)
 
         # define websocket errors
         websocket_errors = (socket.gaierror, websockets.WebSocketException, websockets.ConnectionClosedError, ConnectionRefusedError)
@@ -416,7 +422,7 @@ async def _send_data_through_socket(self, error_message):
             # no error in websocket message sending
             logger.info(f"Error sent successfully through websocket")
 
-        logger.info(f"Disconnecting from websocket {self.websocket_url}")
+        logger.info(f"Disconnecting from websocket {websocket_url}")
 
         # close websocket
         await websocket.close()
@@ -500,8 +506,11 @@ async def _run_container_engine_cmd(self, engine_cmd, kind):
         }
 
         # Start websocket, it will reconnect in the stdout/stderr listener loop below
-        logger.info(f"Connecting to {self.websocket_url}")
-        websocket = await websockets.connect(self.websocket_url)
+        # This ensures each task has its own independent WebSocket connection
+        websocket_url = f"{self.websocket_url}?kind={kind}"
+        logger.debug(f"WORKER_MARKER: Connecting to {websocket_url}")
+        websocket = await websockets.connect(websocket_url)
+        # websocket = await websockets.connect(self.websocket_url) # old BB
         websocket_errors = (socket.gaierror, websockets.WebSocketException, websockets.ConnectionClosedError, ConnectionRefusedError)
 
         # Function to read a line, if the line is larger than the buffer size we will
@@ -522,7 +531,7 @@ async def _readline_or_chunk(stream):
                 logs = [self.logs[kind][key] for key in ('stdout', 'stderr')]
                 for value in logs:
                     try:
-                        out = await asyncio.wait_for(_readline_or_chunk(value["stream"]), timeout=.1)
+                        out = await asyncio.wait_for(_readline_or_chunk(value["stream"]), timeout=0.1)
                         if out:
                             value["data"] += out
                             print("WS: " + str(out))
@@ -535,32 +544,36 @@ async def _readline_or_chunk(stream):
                     except asyncio.TimeoutError:
                         continue
             except websocket_errors:
+                logger.debug("\n\nWebsocket error (line 538)\n\n")
                 try:
                     # do we need to await websocket.close() on the old socket? before making a new one probably not?
                     await websocket.close()
                 except Exception as e:
                     logger.error(e)
-                    logger.info(e)
                     # TODO: catch proper exceptions here..! What can go wrong failing to close?
                     pass
 
                 # try to reconnect a few times
                 tries = 0
                 while tries < 3 and not websocket.open:
                     try:
-                        websocket = await websockets.connect(self.websocket_url)
+                        logger.debug(f"\n\nAttempting to reconnect in 2 seconds (attempt {tries+1}/3)")
+                        websocket = await websockets.connect(websocket_url)
+                        logger.debug(f"\n\nSuccessfully reconnected to {websocket_url}")
                     except websocket_errors:
+                        logger.error(f"\n\nReconnection attempt {tries+1} failed: {websocket_errors}")
                         await asyncio.sleep(2)
                         tries += 1
 
         self.logs[kind]["end"] = time.time()
 
-        logger.info(f"Process exited with {proc.returncode}")
-        logger.info(f"Disconnecting from websocket {self.websocket_url}")
+        logger.debug(f"Process exited with {proc.returncode}")
+        logger.debug(f"Disconnecting from websocket {websocket_url}")
 
         # Communicate that the program is closing
         self.completed_program_counter += 1
 
+        logger.debug(f"WORKER_MARKER: Disconnecting from {websocket_url}, program counter = {self.completed_program_counter}")
         await websocket.close()
 
     def _get_host_path(self, *paths):
@@ -583,7 +596,14 @@ def _get_host_path(self, *paths):
 
         return path
 
-    async def _run_program_directory(self, program_dir, kind, can_be_output=False):
+    async def _run_program_directory(self, program_dir, kind):
+        """
+        Function responsible for running program directory
+
+        Args:
+            - program_dir : can be either ingestion program or program/submission
+            - kind : either `program` or `ingestion`
+        """
         # If the directory doesn't even exist, move on
         if not os.path.exists(program_dir):
             logger.info(f"{program_dir} not found, no program to execute")
@@ -597,13 +617,11 @@ async def _run_program_directory(self, program_dir, kind, can_be_output=False):
         elif os.path.exists(os.path.join(program_dir, "metadata")):
             metadata_path = 'metadata'
         else:
-            if can_be_output:
+            # Display a warning in logs when there is no metadata file in submission/program dir
+            if kind == "program":
                 logger.info(
-                    "Program directory missing metadata, assuming it's going to be handled by ingestion "
-                    "program so move it to output"
+                    "Program directory missing metadata, assuming it's going to be handled by ingestion"
                 )
-                # Copying so that we don't move a code submission w/out a metadata command
-                shutil.copytree(program_dir, self.output_dir)
                 return
             else:
                 raise SubmissionException("Program directory missing 'metadata.yaml/metadata'")
@@ -815,7 +833,7 @@ def start(self):
         logger.info("Running scoring program, and then ingestion program")
         loop = asyncio.new_event_loop()
         gathered_tasks = asyncio.gather(
-            self._run_program_directory(program_dir, kind='program', can_be_output=True),
+            self._run_program_directory(program_dir, kind='program'),
             self._run_program_directory(ingestion_program_dir, kind='ingestion'),
             self.watch_detailed_results(),
             loop=loop,

docker-compose.selenium.yml

@@ -4,11 +4,12 @@ services:
     environment:
       - SELENIUM_HOSTNAME=selenium
       - SUBMISSIONS_API_URL=http://django:36475/api
+      - WEBSOCKET_ALLOWED_ORIGINS=*
     ports:
       - 36475:36475
 
   selenium:
-    image: selenium/standalone-firefox:124.0
+    image: selenium/standalone-firefox:120.0
     volumes:
       - ./src/tests/functional/test_files:/test_files/
       - ./artifacts:/artifacts/:z

docker-compose.yml

@@ -4,7 +4,7 @@ services:
   #   Web Services
   #-----------------------------------------------
   caddy:
-    image: caddy:2.8.4
+    image: caddy:2.10.0
     env_file: .env
     environment:
       - ACME_AGREE=true