optimize log

app/log.py

@@ -16,7 +16,7 @@ def _rotating_config(self):
         '''Rotating log'''
         handler = logging.handlers.TimedRotatingFileHandler(self._logfile, when='W0', interval=1)
         handler.setLevel(self._level)
-        formatter = logging.Formatter('[%(asctime)s] [%(levelname)s] %(name)s:%(lineno)s %(funcName)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+        formatter = logging.Formatter('[%(asctime)s] [%(levelname)s] %(name)s:%(lineno)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
         handler.setFormatter(formatter)
 
         logging.root.setLevel(self._level)
@@ -25,7 +25,7 @@ def _rotating_config(self):
     def _basic_config(self):
         '''Log to stdout. Mainly for development.'''
         kw = {
-            'format': '[seafevents] [%(asctime)s] [%(levelname)s] %(name)s:%(lineno)s %(funcName)s %(message)s',
+            'format': '[seafevents] [%(asctime)s] [%(levelname)s] %(name)s:%(lineno)s %(message)s',
             'datefmt': '%Y-%m-%d %H:%M:%S',
             'level': self._level,
             'stream': sys.stdout

repo_metadata/index_worker.py

@@ -109,7 +109,7 @@ def face_cluster_task_handler(self, mq, repo_id, should_stop, op_type, username=
             if mq.set(self._get_face_cluster_lock_key(repo_id), time.time(),
                       ex=self.LOCK_TIMEOUT, nx=True):
                 # get lock
-                face_recognition_logger.info('%s start face cluster repo %s' % (threading.currentThread().getName(), repo_id))
+                face_recognition_logger.info('%s start face cluster repo %s' % (threading.current_thread().name, repo_id))
                 lock_key = self._get_face_cluster_lock_key(repo_id)
                 self.locked_keys.add(lock_key)
                 self.update_face_cluster(repo_id, username)

repo_metadata/slow_task_handler.py

@@ -89,12 +89,12 @@ def slow_task_handler(self, repo_id, data):
             self.extract_file_info(repo_id, data)
 
     def extract_file_info(self, repo_id, data):
-        logger.info('%s start extract file info repo %s' % (threading.currentThread().getName(), repo_id))
+        logger.info('%s start extract file info repo %s' % (threading.current_thread().name, repo_id))
 
         try:
             obj_ids = data.get('obj_ids')
             add_file_details(repo_id, obj_ids, self.metadata_server_api, self.face_recognition_manager)
         except Exception as e:
             logger.exception('repo: %s, update metadata file info error: %s', repo_id, e)
 
-        logger.info('%s finish extract file info repo %s' % (threading.currentThread().getName(), repo_id))
+        logger.info('%s finish extract file info repo %s' % (threading.current_thread().name, repo_id))

requirements.txt

@@ -4,7 +4,6 @@ pyjwt==2.9.*
 pymysql==1.1.*
 gevent==24.2.*
 Flask==3.0.*
-apscheduler==3.10.*
 pyexiftool==0.4.*
 beautifulsoup4==4.12.3
 markdown-it-py==3.0.0

seasearch/index_task/filename_index_updater.py

@@ -1,9 +1,6 @@
 import logging
 import time
-from threading import Thread
-
-from apscheduler.triggers.interval import IntervalTrigger
-from apscheduler.schedulers.gevent import GeventScheduler
+from threading import Thread, Event
 from seafevents.seasearch.index_store.index_manager import IndexManager
 from seafevents.seasearch.index_store.repo_file_name_index import RepoFileNameIndex
 from seafevents.seasearch.index_store.repo_status_index import RepoStatusIndex
@@ -99,7 +96,7 @@ def clear_deleted_repo(repo_status_filename_index, repo_filename_index, index_ma
 
     repo_deleted = set(repo_all) - set(repos)
 
-    logger.info("filename index %d repos need to be deleted." % len(repo_deleted))
+    logger.info("%d repos of filename index need to be deleted." % len(repo_deleted))
     for repo_id in repo_deleted:
         index_manager.delete_repo_filename_index(repo_id, repo_filename_index, repo_status_filename_index)
         logger.info('Repo %s has been deleted from filename index.' % repo_id)
@@ -133,7 +130,7 @@ def update_repo_file_name_indexes(repo_status_filename_index, repo_filename_inde
 
             index_manager.update_library_filename_index(repo_id, commit_id, repo_filename_index, repo_status_filename_index, metadata_query_time)
 
-    logger.info("Finish update filename index")
+    logger.info("Finish updating filename index")
 
     clear_deleted_repo(repo_status_filename_index, repo_filename_index, index_manager, all_repos)
 
@@ -146,14 +143,17 @@ def __init__(self, repo_status_filename_index, repo_filename_index, index_manage
         self.index_manager = index_manager
         self.repo_data = repo_data
         self.interval = interval
+        self.finished = Event()
 
     def run(self):
-        sched = GeventScheduler()
-        logging.info('Start to update filename index...')
-        try:
-            sched.add_job(update_repo_file_name_indexes, IntervalTrigger(seconds=self.interval),
-                          args=(self.repo_status_filename_index, self.repo_filename_index, self.index_manager, self.repo_data))
-        except Exception as e:
-            logging.exception('periodical update filename index error: %s', e)
-
-        sched.start()
+        while not self.finished.is_set():
+            self.finished.wait(self.interval)
+            if not self.finished.is_set():
+                logging.info('starts to update filename index...')
+                try:
+                    update_repo_file_name_indexes(self.repo_status_filename_index, self.repo_filename_index, self.index_manager, self.repo_data)
+                except Exception as e:
+                    logging.exception('periodical update filename index error: %s', e)
+
+    def cancel(self):
+        self.finished.set()

seasearch/index_task/wiki_index_updater.py

@@ -1,8 +1,6 @@
 import logging
-from threading import Thread
+from threading import Thread, Event
 
-from apscheduler.triggers.interval import IntervalTrigger
-from apscheduler.schedulers.gevent import GeventScheduler
 from seafevents.seasearch.index_store.index_manager import IndexManager
 from seafevents.seasearch.index_store.wiki_index import WikiIndex
 from seafevents.seasearch.index_store.wiki_status_index import WikiStatusIndex
@@ -124,7 +122,7 @@ def update_wiki_indexes(wiki_status_index, wiki_index, index_manager, repo_data)
 
             index_manager.update_wiki_index(repo_id, commit_id, wiki_index, wiki_status_index)
 
-    logger.info("Finish update wiki index")
+    logger.info("Finish updating wiki index")
 
     clear_deleted_wiki(wiki_status_index, wiki_index, index_manager, all_wikis)
 
@@ -137,14 +135,17 @@ def __init__(self, wiki_status_index, wiki_index, index_manager, repo_data, inte
         self.index_manager = index_manager
         self.repo_data = repo_data
         self.interval = interval
+        self.finished = Event()
 
     def run(self):
-        sched = GeventScheduler()
-        logging.info('Start to update wiki index...')
-        try:
-            sched.add_job(update_wiki_indexes, IntervalTrigger(seconds=self.interval),
-                          args=(self.wiki_status_index, self.wiki_index, self.index_manager, self.repo_data))
-        except Exception as e:
-            logging.exception('periodical update wiki index error: %s', e)
-
-        sched.start()
+        while not self.finished.is_set():
+            self.finished.wait(self.interval)
+            if not self.finished.is_set():
+                logging.info('Start to update wiki index...')
+                try:
+                    update_wiki_indexes(self.wiki_status_index, self.wiki_index, self.index_manager, self.repo_data)
+                except Exception as e:
+                    logging.exception('periodical update wiki index error: %s', e)
+
+    def cancel(self):
+        self.finished.set()

seasearch/script/repo_filename_index_local.py

@@ -96,7 +96,7 @@ def thread_task(self, repos_queue):
                 if NO_TASKS:
                     logger.debug(
                         "Queue is empty, %s worker threads stop"
-                        % (threading.currentThread().getName())
+                        % (threading.current_thread().name)
                     )
                     break
                 else:
@@ -113,12 +113,12 @@ def thread_task(self, repos_queue):
 
         logger.info(
             "%s worker updated at %s time"
-            % (threading.currentThread().getName(),
+            % (threading.current_thread().name,
                time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time())))
         )
         logger.info(
             "%s worker get %s error"
-            % (threading.currentThread().getName(),
+            % (threading.current_thread().name,
                 str(self.error_counter))
         )
 

seasearch/script/wiki_index_local.py

@@ -88,7 +88,7 @@ def thread_task(self, wikis_queue):
                 if NO_TASKS:
                     logger.debug(
                         "Queue is empty, %s worker threads stop"
-                        % (threading.currentThread().getName())
+                        % (threading.current_thread().name)
                     )
                     break
                 else:
@@ -105,12 +105,12 @@ def thread_task(self, wikis_queue):
 
         logger.info(
             "%s worker updated at %s time"
-            % (threading.currentThread().getName(),
+            % (threading.current_thread().name,
                time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time())))
         )
         logger.info(
             "%s worker get %s error"
-            % (threading.currentThread().getName(),
+            % (threading.current_thread().name,
                 str(self.error_counter))
         )