feat: fix fetch rate

DB.py

@@ -69,6 +69,17 @@ def addTeacher(self, id: str, name: str):
     cur = self.con.cursor()
     cur.execute("INSERT OR REPLACE INTO TEACHER (id, name) VALUES (?, ?)", (id, name))
     self.con.commit()
+
+  def getTeachers(self):
+    cur = self.con.cursor()
+    request = cur.execute("SELECT * FROM TEACHER")
+    response = request.fetchall()
+
+    res = dict()
+    for x in response:
+      res[x[1]] = x[0]
+
+    return res
 
   def addCourse(self, courseData: dict, courseDataEn: dict, syllabus: str, description: str):
     if courseData["lmtKind"] == "必修":
@@ -129,7 +140,20 @@ def getCourse(self, y: str, s: str):
     response = request.fetchall()
 
     return [str(x[0]) for x in response]
+
+  def getThisSemesterCourse(self, y: str, s: str):
+    cur = self.con.cursor()
+    request = cur.execute('SELECT DISTINCT subNum FROM COURSE WHERE y = ? AND s = ?', [y, s])
+    response = request.fetchall()
 
+    return [str(x[0]) for x in response]
+
+  def isCourseExist(self, courseId: str):
+    cur = self.con.cursor()
+    request = cur.execute('SELECT COUNT(*) FROM COURSE WHERE id = ?', [courseId])
+    response = request.fetchone()
+    return response[0] > 0
 
 if __name__ == "__main__":
-  db = DB("test.db")
+  db = DB("test.db")
+  print(db.getThisSemesterCourse("111", "2"))

fetchRate.py

@@ -5,6 +5,10 @@ def fetchRate(url: str):
   res = requests.get(url.replace("https://", "http://"))
   res.raise_for_status()
   soup = BeautifulSoup(res.content, "html.parser")
-  rates = soup.find_all('table')[2].find_all('tr')
+  rates = soup.find('table', {"border": "1"}).find_all('tr')
 
-  return [x.find('td').get_text(strip=True) for x in rates]
+  return [x.find('td').get_text(strip=True) for x in rates]
+
+
+if __name__ == "__main__":
+  print(fetchRate("https://newdoc.nccu.edu.tw/teaschm/1011/statisticText.jsp-y=1002&tnum=101476&snum=000346021.htm"))

main.py

@@ -5,7 +5,7 @@
 from DB import DB
 
 from User import User
-from constant import YEAR_SEM
+from constant import YEAR_SEM, YEAR, SEM
 from fetchDescription import fetchDescription
 from fetchRate import fetchRate
 from translateRate import translateRate
@@ -102,18 +102,16 @@
           # Write to databse
           if not programOptions["skip_class_detail"]:
             for course in tqdm.tqdm(courses, leave=False):
-              detail = fetchDescription("{}{}".format(semester, course["subNum"]))
+              courseId = "{}{}".format(semester, course["subNum"])
+              if db.isCourseExist(courseId):
+                continue
+              detail = fetchDescription(courseId)
               db.addCourse(detail["qrysub"], detail["qrysubEn"], "".join(detail["description"]), "".join(detail["objectives"]))
         except Exception as e:
           logging.error(e)
 
     logging.debug(coursesList)
 
-    # Write courseList back to file
-    with open(os.path.join(dirPath, "_data", "classes.json"), "w+") as f:
-      f.write(json.dumps(coursesList))
-      f.close()
-
     print("Fetch Class done at {}".format(datetime.datetime.now()))
   else:
     print("Skipping Fetch Class")
@@ -124,8 +122,7 @@
 
   if not programOptions["skip_teacher"]:
     # Read course list
-    with open(os.path.join(dirPath, "_data", "classes.json"), "r") as f:
-      coursesList = json.loads(f.read())
+    coursesList = db.getThisSemesterCourse(YEAR, SEM)
 
     user = User()
 
@@ -182,11 +179,6 @@
       except Exception as e:
         logging.error(e)
         continue
-
-    # Write teacherIdDict back to file
-    with open(os.path.join(dirPath, "_data", "teachers.json"), "w+") as f:
-      f.write(json.dumps(teacherIdDict))
-      f.close()
 
     # Delete courses from track list
     tqdmCourses =  tqdm.tqdm(courses, leave=False)
@@ -210,8 +202,7 @@
 
   if not programOptions["skip_rate"]:
     # Read teacher list
-    with open(os.path.join(dirPath, "_data", "teachers.json"), "r") as f:
-      newTeacherList = json.loads(f.read())
+    newTeacherList = db.getTeachers()
     with open(os.path.join(dirPath, "old_data", "1111_teachers.json"), "r") as f:
       oldTeacherList = json.loads(f.read())
     teacherList = {**newTeacherList,**oldTeacherList}