diff --git a/DB.py b/DB.py index 220e5980..5c45f175 100644 --- a/DB.py +++ b/DB.py @@ -69,6 +69,17 @@ def addTeacher(self, id: str, name: str): cur = self.con.cursor() cur.execute("INSERT OR REPLACE INTO TEACHER (id, name) VALUES (?, ?)", (id, name)) self.con.commit() + + def getTeachers(self): + cur = self.con.cursor() + request = cur.execute("SELECT * FROM TEACHER") + response = request.fetchall() + + res = dict() + for x in response: + res[x[1]] = x[0] + + return res def addCourse(self, courseData: dict, courseDataEn: dict, syllabus: str, description: str): if courseData["lmtKind"] == "必修": @@ -129,7 +140,20 @@ def getCourse(self, y: str, s: str): response = request.fetchall() return [str(x[0]) for x in response] + + def getThisSemesterCourse(self, y: str, s: str): + cur = self.con.cursor() + request = cur.execute('SELECT DISTINCT subNum FROM COURSE WHERE y = ? AND s = ?', [y, s]) + response = request.fetchall() + return [str(x[0]) for x in response] + + def isCourseExist(self, courseId: str): + cur = self.con.cursor() + request = cur.execute('SELECT COUNT(*) FROM COURSE WHERE id = ?', [courseId]) + response = request.fetchone() + return response[0] > 0 if __name__ == "__main__": - db = DB("test.db") \ No newline at end of file + db = DB("test.db") + print(db.getThisSemesterCourse("111", "2")) \ No newline at end of file diff --git a/fetchRate.py b/fetchRate.py index 9c3de975..a97b8173 100644 --- a/fetchRate.py +++ b/fetchRate.py @@ -5,6 +5,10 @@ def fetchRate(url: str): res = requests.get(url.replace("https://", "http://")) res.raise_for_status() soup = BeautifulSoup(res.content, "html.parser") - rates = soup.find_all('table')[2].find_all('tr') + rates = soup.find('table', {"border": "1"}).find_all('tr') - return [x.find('td').get_text(strip=True) for x in rates] \ No newline at end of file + return [x.find('td').get_text(strip=True) for x in rates] + + +if __name__ == "__main__": + print(fetchRate("https://newdoc.nccu.edu.tw/teaschm/1011/statisticText.jsp-y=1002&tnum=101476&snum=000346021.htm")) \ No newline at end of file diff --git a/main.py b/main.py index b003855d..36b594c7 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ from DB import DB from User import User -from constant import YEAR_SEM +from constant import YEAR_SEM, YEAR, SEM from fetchDescription import fetchDescription from fetchRate import fetchRate from translateRate import translateRate @@ -102,18 +102,16 @@ # Write to databse if not programOptions["skip_class_detail"]: for course in tqdm.tqdm(courses, leave=False): - detail = fetchDescription("{}{}".format(semester, course["subNum"])) + courseId = "{}{}".format(semester, course["subNum"]) + if db.isCourseExist(courseId): + continue + detail = fetchDescription(courseId) db.addCourse(detail["qrysub"], detail["qrysubEn"], "".join(detail["description"]), "".join(detail["objectives"])) except Exception as e: logging.error(e) logging.debug(coursesList) - # Write courseList back to file - with open(os.path.join(dirPath, "_data", "classes.json"), "w+") as f: - f.write(json.dumps(coursesList)) - f.close() - print("Fetch Class done at {}".format(datetime.datetime.now())) else: print("Skipping Fetch Class") @@ -124,8 +122,7 @@ if not programOptions["skip_teacher"]: # Read course list - with open(os.path.join(dirPath, "_data", "classes.json"), "r") as f: - coursesList = json.loads(f.read()) + coursesList = db.getThisSemesterCourse(YEAR, SEM) user = User() @@ -182,11 +179,6 @@ except Exception as e: logging.error(e) continue - - # Write teacherIdDict back to file - with open(os.path.join(dirPath, "_data", "teachers.json"), "w+") as f: - f.write(json.dumps(teacherIdDict)) - f.close() # Delete courses from track list tqdmCourses = tqdm.tqdm(courses, leave=False) @@ -210,8 +202,7 @@ if not programOptions["skip_rate"]: # Read teacher list - with open(os.path.join(dirPath, "_data", "teachers.json"), "r") as f: - newTeacherList = json.loads(f.read()) + newTeacherList = db.getTeachers() with open(os.path.join(dirPath, "old_data", "1111_teachers.json"), "r") as f: oldTeacherList = json.loads(f.read()) teacherList = {**newTeacherList,**oldTeacherList}