indeed+so job scrapper (old)

Author: DeveloperYun

day2_5.py

@@ -1,9 +1,13 @@
-from indeed import extract_indeed_jobs, extract_indeed_pages
+from indeed import get_jobs as get_indeed_jobs
+from stack_over_flow import get_jobs as get_so_jobs
 
 # #range - 입력한 수 만큼의 크기의 배열을 생성
 # for n in range(max_page):
 #     print(f"start={n*50}")
 
-last_indeed_pages = extract_indeed_pages()
+# last_indeed_pages = extract_indeed_pages()
 
-extract_indeed_jobs(last_indeed_pages)
+# indeed_jobs = extract_indeed_jobs(last_indeed_pages)
+
+indeed_jobs = get_indeed_jobs()
+so_jobs = get_so_jobs()

indeed.py

@@ -4,7 +4,7 @@
 LIMIT = 50
 URL = f"https://www.indeed.com/jobs?q=Python&limit={LIMIT}"
 
-def extract_indeed_pages():
+def get_last_page():
     result = requests.get(URL)
 
     soup = BeautifulSoup(result.text, "html.parser")
@@ -17,11 +17,48 @@ def extract_indeed_pages():
 
     for link in links[:-1]: #마지막 요소는 읽지 않겠다는 뜻
         pages.append(int(link.string)) #string -> integer 변환
-
     max_page = pages[-1]
     return max_page
 
-def extract_indeed_jobs(last_page):
+def extract_job(html):
+    title = html.find("div", {"class": "title"}).find("a")["title"]
+    company = html.find("span", {"class":"company"})
+
+    if company:
+        company_anchor = company.find("a")
+        if company_anchor is not None:
+            company = str(company_anchor.string)
+        else:
+            company = str(company.string)
+        company = company.strip()
+    else:
+        company = None
+
+    location = html.find("div",{"class": "recJobLoc"})["data-rc-loc"] #div의 attribute를 가져옴
+    job_id = html["data-jk"]
+
+    return {
+        'title': title, 
+        'company': company, 
+        'location': location,
+        "link":f"https://www.indeed.com/viewjob?jk={job_id}"
+    }
+
+def extract_jobs(last_page):
+    jobs = []
     for page in range(last_page):
-        res = requests.get(f"{URL}&start={page*LIMIT}")
-        print(res.status_code)
+        res = requests.get(f"{URL}&start={last_page*LIMIT}")
+        soup = BeautifulSoup(res.text, "html.parser")
+        results = soup.find_all("div", {"class":"jobsearch-SerpJobCard"})
+
+        #print(results)
+        for res in results:
+            job = extract_job(res)
+            jobs.append(job)
+    return jobs
+
+
+def get_jobs():
+    last_page = get_last_page()
+    jobs = extract_jobs(last_page)
+    return jobs

stack_over_flow.py

@@ -0,0 +1,15 @@
+import requests
+from bs4 import BeautifulSoup
+
+
+URL = f"https://stackoverflow.com/jobs?q=python&pg=2"
+
+def get_last_page():
+    result = requests.get(URL)
+    soup = BeautifulSoup(result.text,"html.parser")
+    pages = soup.find("div", {"class":"pagination"}).find_all("a")
+
+
+def get_jobs():
+    last_page = get_last_page()
+    return []