added new scrapers and made structure changes to folder

Author: Ubuntu

amazonin/amazonin.py amazonin.py

@@ -0,0 +1,51 @@
+#single page crawl
+import scrapy
+import logging
+from scrapy.contrib.spiders import Rule
+from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.selector import HtmlXPathSelector
+
+from scrapy import Request, Spider
+from scrapy.exceptions import CloseSpider
+from scrapy.selector import Selector
+
+
+from scrapy.item import Item, Field
+
+class ScrapySampleItem(Item):
+    title = Field()
+    link = Field()
+    desc = Field()
+    price =  Field()
+    image = Field()
+    
+    
+class StackOverflowSpider(scrapy.Spider): 
+
+        name = 'anhad' 
+        start_urls = ["https://www.theanhadshop.com/collections/home-textile","https://www.theanhadshop.com/collections/home-textile?page=2","https://www.theanhadshop.com/collections/home-textile?page=3","https://www.theanhadshop.com/collections/tableware"] 
+     
+		
+        def parse(self, response): 
+            for href in response.css('.ci a::attr(href)'): 
+              full_url = response.urljoin(href.extract()) 
+              logging.info(full_url)
+              yield scrapy.Request(full_url, callback=self.parse_product,dont_filter = True) 
+
+        def parse_product(self, response):
+           items = []
+           item = ScrapySampleItem()
+    
+           item['title'] =  response.css('h1::text').extract_first()
+           item['image'] =  response.css('.thumbnail::attr(src)').extract_first()
+           item['desc']  = response.css('.rte span::text').extract()
+           item['price'] = response.css('p[id="product-price"] .product-price .money::text').extract_first()
+           
+           if not item['desc']:
+               logging.info("EMPTY RECIEVED")
+               item['desc']  = response.css('h1::text').extract_first()
+           item['link']  = response.url
+           items.append(item)
+    		
+           for item in items:
+               yield item 

amazonin/amazonin.pyc amazonin.pyc

@@ -23,7 +23,7 @@ class ScrapySampleItem(Item):
 class StackOverflowSpider(scrapy.Spider): 
 
         name = 'green essence' 
-        start_urls = ["http://www.greenessencestore.com/Planters-depid-423-page-1.html"] 
+        start_urls = ["http://www.greenessencestore.com/Planters-depid-423-page-1.html","http://www.greenessencestore.com/Garden-Accessories-depid-431-page-1.html","http://www.greenessencestore.com/Planter-Stand---Shelves-depid-264466-page-1.html","http://www.greenessencestore.com/Wall-Pot-Holders---Brackets-depid-273823-page-1.html","http://www.greenessencestore.com/Potted-Plants-depid-16325-page-1.html"] 
 
 
         def parse(self, response): 
@@ -36,14 +36,14 @@ def parse_product(self, response):
            items = []
            item = ScrapySampleItem()
 
-           item['title'] =  map(unicode.strip, response.css('.title::text').extract())
+           item['title'] =  map(unicode.strip, response.css('.lht19::text').extract())
            item['image'] =  response.css('img[id="largeImage"]::attr(src)').extract_first()
            item['desc']  = response.css('.desc_shorttext').extract()
            item['price'] = response.css('.amt::text').extract_first()
 
            if not item['desc']:
                logging.info("EMPTY RECIEVED")
-               item['desc']  = response.css('.title::text').extract()
+               item['desc']  = response.css('.lht19::text').extract()
            item['link']  = response.url
            items.append(item)
 

anekdesign/anekdesigns_cushions.py anekdesigns_cushions.py

@@ -0,0 +1,51 @@
+#single page crawl
+import scrapy
+import logging
+from scrapy.contrib.spiders import Rule
+from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.selector import HtmlXPathSelector
+
+from scrapy import Request, Spider
+from scrapy.exceptions import CloseSpider
+from scrapy.selector import Selector
+
+
+from scrapy.item import Item, Field
+
+class ScrapySampleItem(Item):
+    title = Field()
+    link = Field()
+    desc = Field()
+    price =  Field()
+    image = Field()
+    
+    
+class StackOverflowSpider(scrapy.Spider): 
+
+        name = 'idamstore' 
+        start_urls = ["https://www.idamstore.com/collections/all?page=1","https://www.idamstore.com/collections/all?page=2","https://www.idamstore.com/collections/all?page=3"] 
+     
+		
+        def parse(self, response): 
+            for href in response.css('.ci a::attr(href)'): 
+              full_url = response.urljoin(href.extract()) 
+              logging.info(full_url)
+              yield scrapy.Request(full_url, callback=self.parse_product,dont_filter = True) 
+
+        def parse_product(self, response):
+           items = []
+           item = ScrapySampleItem()
+    
+           item['title'] =  response.css('.section-title h1::text').extract_first()
+           item['image'] =  response.css('.thumbnail::attr(src)').extract_first()
+           item['desc']  = response.css('.rte span::text').extract()
+           item['price'] = response.css('p[id="product-price"] .product-price::text').extract_first()
+           
+           if not item['desc']:
+               logging.info("EMPTY RECIEVED")
+               item['desc']  = response.css('.section-title h1::text').extract_first()
+           item['link']  = response.url
+           items.append(item)
+    		
+           for item in items:
+               yield item 

anekdesign/anekdesigns_cushions.pyc anekdesigns_cushions.pyc

@@ -0,0 +1,55 @@
+
+import scrapy
+import logging
+from scrapy.contrib.spiders import Rule
+from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.selector import HtmlXPathSelector
+
+from scrapy import Request, Spider
+from scrapy.exceptions import CloseSpider
+from scrapy.selector import Selector
+
+
+from scrapy.item import Item, Field
+
+class ScrapySampleItem(Item):
+    title = Field()
+    link = Field()
+    desc = Field()
+    price =  Field()
+    image = Field()
+    
+    
+class StackOverflowSpider(scrapy.Spider): 
+
+        name = 'lotushouse' 
+        start_urls = ["http://thelotushouse.com/paintings-wall-hangings.html","http://thelotushouse.com/showpieces-figurines.html","http://thelotushouse.com/fashion-accessories.html","http://thelotushouse.com/home-furnishings.html"] 
+
+
+        def parse(self, response): 
+             for href in response.css('.product-image::attr(href)'): 
+               full_url = response.urljoin(href.extract()) 
+               yield scrapy.Request(full_url, callback=self.parse_product) 
+              
+             next_page = response.css(".next::attr('href')")
+             if next_page:
+               url = response.urljoin(next_page[0].extract())
+               yield scrapy.Request(url, self.parse)
+
+        def parse_product(self, response):
+           items = []
+           item = ScrapySampleItem()
+    
+           item['title'] =  response.css('.product-name h1::text').extract_first()
+           item['image'] =  response.css('img[id="image"]::attr(src)').extract_first()
+           item['desc']  = response.css('.product-collateral').extract()
+           item['price'] = response.css('.add_to_cart .add-to-box .price_box .price-box .regular-price .price::text').extract()
+           
+           if not item['desc']:
+               logging.info("EMPTY RECIEVED")
+               item['desc']  = response.css('h1::text').extract_first()
+           item['link']  = response.url
+           items.append(item)
+    		
+           for item in items:
+               yield item 

anekdesign/anekdesigns_tablelinen.py anekdesigns_tablelinen.py

@@ -0,0 +1,51 @@
+#single page crawl
+import scrapy
+import logging
+from scrapy.contrib.spiders import Rule
+from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.selector import HtmlXPathSelector
+
+from scrapy import Request, Spider
+from scrapy.exceptions import CloseSpider
+from scrapy.selector import Selector
+
+
+from scrapy.item import Item, Field
+
+class ScrapySampleItem(Item):
+    title = Field()
+    link = Field()
+    desc = Field()
+    price =  Field()
+    image = Field()
+    
+    
+class StackOverflowSpider(scrapy.Spider): 
+
+        name = 'mudfingers' 
+        start_urls = ["http://www.mudfingers.com/AIR-PLANTS-depid-956101-page-1.html","http://www.mudfingers.com/COLORED-CLAYS-depid-932279-page-1.html","http://www.mudfingers.com/SUN-LOVERS-depid-421491-page-1.html","http://www.mudfingers.com/TERRARIUMS-depid-421269-page-1.html","http://www.mudfingers.com/AIR-PLANTS-depid-956101-page-1.html",] 
+     
+		
+        def parse(self, response): 
+            for href in response.css('.standard a::attr(href)'): 
+              full_url = response.urljoin(href.extract()) 
+              logging.info(full_url)
+              yield scrapy.Request(full_url, callback=self.parse_product,dont_filter = True) 
+
+        def parse_product(self, response):
+           items = []
+           item = ScrapySampleItem()
+    
+           item['title'] =  response.css('h1::text').extract_first()
+           item['image'] =  response.css('.z-product-thumbs img::attr(src)').extract_first()
+           item['desc']  = response.css('div[id="description"]').extract()
+           item['price'] = response.css('.prices .price_original span[id="price-standard"]::text').extract_first()
+           
+           if not item['desc']:
+               logging.info("EMPTY RECIEVED")
+               item['desc']  = response.css('h1::text').extract_first()
+           item['link']  = response.url
+           items.append(item)
+    		
+           for item in items:
+               yield item 

anekdesign/anekdesigns_tablelinen.pyc anekdesigns_tablelinen.pyc

@@ -0,0 +1,55 @@
+
+import scrapy
+import logging
+from scrapy.contrib.spiders import Rule
+from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.selector import HtmlXPathSelector
+
+from scrapy import Request, Spider
+from scrapy.exceptions import CloseSpider
+from scrapy.selector import Selector
+
+
+from scrapy.item import Item, Field
+
+class ScrapySampleItem(Item):
+    title = Field()
+    link = Field()
+    desc = Field()
+    price =  Field()
+    image = Field()
+    
+    
+class StackOverflowSpider(scrapy.Spider): 
+
+        name = 'olie' 
+        start_urls = ["http://www.olie.co.in/collections/lighting","http://www.olie.co.in/collections/cushion-covers","http://www.olie.co.in/collections/storage"] 
+
+
+        def parse(self, response): 
+             for href in response.css('.image a::attr(href)'): 
+               full_url = response.urljoin(href.extract()) 
+               yield scrapy.Request(full_url, callback=self.parse_product) 
+              
+             next_page = response.css(".next::attr('href')")
+             if next_page:
+               url = response.urljoin(next_page[0].extract())
+               yield scrapy.Request(url, self.parse)
+
+        def parse_product(self, response):
+           items = []
+           item = ScrapySampleItem()
+    
+           item['title'] =  response.css('.breadcrumb strong::text').extract_first()
+           item['image'] =  response.css('.zoomImg::attr(src)').extract()
+           item['desc']  = response.css('.description').extract()
+           item['price'] = response.css('h2[id="price-preview"] span::text').extract_first()
+           
+           if not item['desc']:
+               logging.info("EMPTY RECIEVED")
+               item['desc']  = response.css('.product-name h1::text').extract_first()
+           item['link']  = response.url
+           items.append(item)
+    		
+           for item in items:
+               yield item 

anhad.py

@@ -0,0 +1,51 @@
+#single page crawl
+import scrapy
+import logging
+from scrapy.contrib.spiders import Rule
+from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.selector import HtmlXPathSelector
+
+from scrapy import Request, Spider
+from scrapy.exceptions import CloseSpider
+from scrapy.selector import Selector
+
+
+from scrapy.item import Item, Field
+
+class ScrapySampleItem(Item):
+    title = Field()
+    link = Field()
+    desc = Field()
+    price =  Field()
+    image = Field()
+    
+    
+class StackOverflowSpider(scrapy.Spider): 
+
+        name = 'postbox' 
+        start_urls = ["https://www.thepostbox.in/collections/vibrant-funky-india-cotton-cushion-covers","https://www.thepostbox.in/collections/wall-art","https://www.thepostbox.in/collections/trays-by-kalakaari-haath","https://www.thepostbox.in/collections/terracotta-mugs","https://www.thepostbox.in/collections/handpainted-blue-pottery-the-postbox","https://www.thepostbox.in/collections/coasters-cork-board-city-themes-graphic-art","https://www.thepostbox.in/collections/vibrant-printed-ceramic-mugs-the-postbox"] 
+     
+		
+        def parse(self, response): 
+            for href in response.css('.ci a::attr(href)'): 
+              full_url = response.urljoin(href.extract()) 
+              logging.info(full_url)
+              yield scrapy.Request(full_url, callback=self.parse_product,dont_filter = True) 
+
+        def parse_product(self, response):
+           items = []
+           item = ScrapySampleItem()
+    
+           item['title'] =  response.css('.section-title h1::text').extract_first()
+           item['image'] =  response.css('.main-product-image img::attr(src)').extract_first()
+           item['desc']  = response.css('.rte').extract()
+           item['price'] = response.css('.product-price .money::text').extract_first()
+           
+           if not item['desc']:
+               logging.info("EMPTY RECIEVED")
+               item['desc']  = response.css('.section-title h1::text').extract_first()
+           item['link']  = response.url
+           items.append(item)
+    		
+           for item in items:
+               yield item