multi treading completed

Author: digitaldreams

main.py

@@ -1,10 +1,54 @@
-from Image import Download
-from functions import *
-from UrlFinder import UrlFinder
+from Image import Save
+from Models.Complete.Image import Image
+import os.path;
+from Crawler.Page import Page
+from Crawler.Image import Image
+from Image.Download import Download
+import functions
+import threading
+from queue import Queue
 
 if __name__ == '__main__':
-    PAGE_URL = 'http://www.indianexpress.com'
-    link = UrlFinder(page_url=PAGE_URL, tag='a', attr='href')
-    print(link.page_url)
-    #link.feed(link.html_string())
-    #print(link.get_values())
+    NUMBER_OF_THREADS = 4
+    queue = Queue()
+    page = Page('http://fdfashionbd.com')
+    links = page.fetch_links()
+
+
+    def create_jobs():
+        for link in sorted(links):
+            queue.put(link)
+        queue.join()
+
+
+    # Create worker threads (will die when main exits)
+    def create_workers():
+        for _ in range(NUMBER_OF_THREADS):
+            t = threading.Thread(target=downloading)
+            t.daemon = True
+            t.start()
+
+
+    def downloading():
+        while True:
+            try:
+                link = queue.get()
+                print(threading.current_thread().name + ' fetching ' + link)
+                img = Image(link)
+
+                images = img.fetch_links()
+                down = Download(links=images, path=functions.get_folder_name(link))
+                down.start()
+                queue.task_done()
+                print(threading.current_thread().name + ' Done ' + link)
+                if queue.empty():
+                    break
+            except:
+                queue.task_done()
+                continue
+
+
+
+
+    create_workers()
+    create_jobs()

test.py

@@ -1,19 +1,13 @@
-from Image import Save
-from Models.Complete.Image import Image
-import os.path;
-from Crawler.Page import Page
-from Crawler.Image import Image
-from Image.Download import Download
-import functions
+from queue import Queue
 
-if __name__ == '__main__':
-    page = Page('https://gopostie.com')
-    links = page.fetch_links()
-    for link in sorted(links):
-        print('Downloading ...' + link)
-        img = Image(link)
-        images = img.fetch_links()
-        down = Download(links=images,path=functions.get_folder_name(link))
-        down.start()
-        # down = Download()
-        # down.start()
+def do_stuff(q):
+  while not q.empty():
+    print(q.get())
+    q.task_done()
+
+q = Queue(maxsize=0)
+
+for x in range(20):
+  q.put(x)
+
+do_stuff(q)