Scraper created

Author: goranmrd

main/templates/main/index.html

@@ -1,5 +1,60 @@
 {% extends 'base.html' %}
-
+{% load static %}
 {% block content %}
-<h1>Inital setup done.</h1>
+<div class="container">
+
+    <div class="header">
+    
+        <h1 class="text-center">Make your search</h1>
+
+        <form id="labnol" action="" method="GET">
+            <div class="row">
+                <div class="col-xs-11 col-sm-8 col-md-6 col-lg-4 mx-auto speech">
+                    <input type="text" name="item" class="form-control" id="transcript" required placeholder="Search items">
+                    <img onclick="startDictation()" src="{% static 'images/mic.gif' %}" />
+                </div>
+            </div>
+            
+            <div class="row">
+                <div class="col-md-4 mx-auto text-center">Price range (optional):</div>
+            </div>
+            <div class="row">
+                    <div class="col-xs-2 col-sm-4 col-md-3 col-lg-2 ml-auto pt-2">
+                        <input type="text" class="form-control" name="from" id="price-low" placeholder="From">
+                    </div>
+                    <div class="col-xs-2 col-sm-4 col-md-3 col-lg-2 mr-auto pt-2">
+                        <input type="text" class="form-control" name="to" id="price-high" placeholder="To">
+                    </div>
+            </div>
+            <div class="row">
+                <div class="col-xs-11 col-sm-8 col-md-6 col-lg-4 mx-auto text-center">
+                        <button type="submit" class="btn btn-primary">Search</button>
+                </div>
+            </div>
+            
+        </form>
+    </div>
+    <hr>
+    {% if items %}
+        <table>
+        {% for item in items %}
+        <tr>
+            <td>
+                <a href="{{item.link}}">{{ item.name }}</a>
+            </td>
+            <td>
+                {{ item.condition }}
+            </td>
+            <td>
+                {{ item.price }}
+            </td>
+            <td>
+                <img src="{{ item.image }}" alt="">
+            </td>
+        </tr>
+            
+        {% endfor %}
+        </table>
+    {% endif %}
+</div>
 {% endblock %}

main/urls.py

@@ -2,5 +2,5 @@
 from . import views
 
 urlpatterns = [
-    path('', views.index, name="index")
+    path('', views.Index.as_view(), name="index")
 ]

main/views.py

@@ -1,6 +1,78 @@
 from django.shortcuts import render
+from django.views.generic import ListView
+import requests
+from bs4 import BeautifulSoup
+from multiprocessing import cpu_count, Pool
 
+class Index(ListView):
+    queryset = []
+    context_object_name = "items"
+    template_name = "main/index.html"
 
-# Create your views here.
-def index(request):
-    return render(request, "main/index.html")
+    def get_context_data(self, *args, **kwargs):
+        context = super(Index, self).get_context_data(*args, **kwargs)
+        context['queryset'] = self.queryset
+        return context    
+
+    def get_queryset(self):
+        base_url = "https://www.ebay.com/sch/parser.html?_from=R40&_nkw={item}&_ipg=25"
+        prices_url = "&_udlo={price_low}&_udhi={price_high}"
+        item = self.request.GET.get('item')
+        price_low = self.request.GET.get('from')
+        price_high = self.request.GET.get('to')
+        if self.request.method == 'GET' and item:
+            item = "+".join(item.split())
+            if price_low and price_high:
+                url = (base_url + prices_url).format(item=item,price_low=price_low,price_high=price_high)
+            else:
+                url = base_url.format(item=item)
+            scraper = Scraper(base_url=url)
+            app = scraper.run()
+        print(self.queryset)
+        return self.queryset
+
+
+class Scraper(Index):
+    def __init__(self, base_url=None):
+        super(Scraper, self).__init__()
+
+        self.base_url = base_url
+
+    def run(self):
+        
+        try:
+            bs = self.make_soup(self.base_url)
+            if not bs.get('error'):
+                rows = bs.find_all('div', class_="s-item__wrapper")[:10]
+                for parser in rows:
+                    self.parse_rows(parser)
+        except Exception as error:
+            print(error)
+        return self.queryset
+    
+    def parse_rows(self, parser):
+        name = parser.find('h3', class_="s-item__title").text
+        link = parser.find('a', class_="s-item__link").get('href')
+        condition = parser.find('span', class_="SECONDARY_INFO").text
+        price = parser.find('span', class_="s-item__price").text
+        image = parser.find('img', class_="s-item__image-img").get('src')
+        if image == 'https://ir.ebaystatic.com/cr/v/c1/s_1x2.gif':
+            soup = self.make_soup(link)
+            image = soup.find('img', {'id': "icImg"}).get('src')
+        print(self.queryset)
+        self.queryset.append(dict(name=name,link=link,condition=condition,price=price,image=image))
+
+    def make_soup(self, url):
+        headers = {'Accept': '*/*',
+                   'Accept-Encoding': 'gzip, deflate, sdch',
+                   'Accept-Language': 'en-US,en;q=0.8',
+                   'content-security-policy': "media-src 'self' *.ebaystatic.com; font-src 'self' *.ebaystatic.com",
+                   'Cache-Control': 'max-age=0',
+                   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'}
+        
+        page = requests.get(url, headers=headers, timeout=15)
+        if page.status_code == 200:
+            soup = BeautifulSoup(page.content, "lxml")
+        else:
+            soup = {'error': "We got status code %s" % page.status_code}
+        return soup

requirements.txt

@@ -1,3 +1,4 @@
 bs4==0.0.1
 Django==2.1.7
-requests==2.21.0
+requests==2.21.0
+lxml==4.3.3

static/css/style.css

@@ -0,0 +1,36 @@
+
+html {
+    height: 100%;
+}
+
+body {
+    margin: 0;
+    height: 100%;
+    background: rgb(12,17,98);
+    background: linear-gradient(0deg, rgba(12,17,98,0.8550770650056898) 0%, rgba(9,9,121,0.8130602582830007) 42%, rgba(0,212,255,0.9251050762101716) 100%);
+    background-repeat: no-repeat;
+    background-attachment: fixed;
+}
+
+.speech input {
+    height: 40px;
+}
+.speech img {
+    position: absolute;
+    right: 15px;
+    top: 0;
+    cursor: pointer;
+    width: 40px 
+}
+
+.btn-primary {
+    margin-top: 10px;
+    width: 70%;
+    -webkit-box-shadow: 0px 0px 11px 1px rgba(79,77,79,1);
+    -moz-box-shadow: 0px 0px 11px 1px rgba(79,77,79,1);
+    box-shadow: 0px 0px 11px 1px rgba(79,77,79,1);
+}
+
+img {
+    max-width: 225px;
+}

static/images/mic.gif

@@ -0,0 +1,25 @@
+function startDictation() {
+
+    if (window.hasOwnProperty('webkitSpeechRecognition')) {
+
+      var recognition = new webkitSpeechRecognition();
+
+      recognition.continuous = false;
+      recognition.interimResults = false;
+
+      recognition.lang = "en-US";
+      recognition.start();
+
+      recognition.onresult = function(e) {
+        document.getElementById('transcript').value
+                                 = e.results[0][0].transcript;
+        recognition.stop();
+        //document.getElementById('labnol').submit();
+      };
+
+      recognition.onerror = function(e) {
+        recognition.stop();
+      }
+
+    }
+  }

static/js/main.js

@@ -4,10 +4,10 @@
 <html>
   <head>
     <title>Django Scraper</title>
-    <link rel="icon" href="{% static 'images/favicon.png' %}" type="image/x-icon"/>
+    <link rel="icon" href="{% static 'images/favicon.png' %}" type="image/x-icon" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
-
+    <link rel="stylesheet" href="{% static 'css/style.css' %}" />
   </head>
   <body>
 
@@ -20,6 +20,8 @@
     <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js" integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous"></script>
 
     <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous"></script>
+    
+    <script src="{% static 'js/main.js' %}"></script>
     {% block script %}{% endblock %}
   </body>
 </html>