Phyton integration on the system

Author: Mirdrack

README.md

@@ -20,4 +20,7 @@ The app will be pooling the selected threads, receiving the information comparin
 9. Develop a way to ask and storage the new data
 
 ## To do's
-Update this Read me with the steps to scrap the site
+Update this Read me with the steps to scrap the site
+
+### Palette
+http://paletton.com/#uid=1000u0kllllaFw0g0qFqFg0w0aF

python/scrapper.py

@@ -0,0 +1,5 @@
+import sys
+print "4Chan Scrapper"
+
+for arg in sys.argv:
+	print arg

routes.js

@@ -29,4 +29,25 @@ module.exports = function (app, fourChanService) {
 			res.render('thread.jade', { threadName : surl, resources : data });
 		});
 	});
+
+	app.get('/test', function (req, res) {
+
+		var spawn = require('child_process').spawn,
+		ls    = spawn('python', ['./python/scrapper.py', 'cadena', 1, 'otraCadena']);
+
+		ls.stdout.on('data', function (data) {
+		  console.log('stdout: ' + data);
+		});
+
+		ls.stderr.on('data', function (data) {
+		  console.log('stderr: ' + data);
+		});
+
+		ls.on('close', function (code) {
+		  console.log('child process exited with code ' + code);
+		});
+
+		res.render('test.jade');
+
+	});
 };

services/fourChanService.js

@@ -83,7 +83,7 @@ var fourChanService = {
 
 			var resources = results.resources;
 			console.log('There going to be downloaded ' + resources.length + ' resources.' );
-			for(var cont = 0; cont < resources.length; cont++)
+			for(var cont = 0; cont < resources.length && cont < 50; cont++)
 			{
 				var url = 'http:' + resources[cont];
 
@@ -100,6 +100,63 @@ var fourChanService = {
 					console.log(fileCounter + '::File ' + path + ' downloaded');
 				});
 			}
+
+			setTimeout(function () { console.log('Second batch...') }, 2000);
+			for(; cont < resources.length && cont < 100; cont++)
+			{
+				url = 'http:' + resources[cont];
+				
+				// We set the name for file
+				filename = resources[cont].split('/');
+				filename = filename[filename.length - 1];
+
+				// We set the full path '.files/boardName/semanticURL/filename'
+				path = './files/' + boardName + '/' + surl + '/' +filename;
+
+				// We download the file
+				fileCounter = cont + 1;
+				self.dowloadResource(url, boardName, surl, path, function () {
+					console.log(fileCounter + '::File ' + path + ' downloaded');
+				});
+			}
+
+			setTimeout(function () { console.log('Third batch...') }, 2000);
+			for(; cont < resources.length && cont < 150; cont++)
+			{
+				url = 'http:' + resources[cont];
+				
+				// We set the name for file
+				filename = resources[cont].split('/');
+				filename = filename[filename.length - 1];
+
+				// We set the full path '.files/boardName/semanticURL/filename'
+				path = './files/' + boardName + '/' + surl + '/' +filename;
+
+				// We download the file
+				fileCounter = cont + 1;
+				self.dowloadResource(url, boardName, surl, path, function () {
+					console.log(fileCounter + '::File ' + path + ' downloaded');
+				});
+			}
+
+			setTimeout(function () { console.log('Forth batch...') }, 2000);
+			for(; cont < resources.length && cont < 200; cont++)
+			{
+				url = 'http:' + resources[cont];
+				
+				// We set the name for file
+				filename = resources[cont].split('/');
+				filename = filename[filename.length - 1];
+
+				// We set the full path '.files/boardName/semanticURL/filename'
+				path = './files/' + boardName + '/' + surl + '/' +filename;
+
+				// We download the file
+				fileCounter = cont + 1;
+				self.dowloadResource(url, boardName, surl, path, function () {
+					console.log(fileCounter + '::File ' + path + ' downloaded');
+				});
+			}
 		    // After all, we call the callback
 			callback(resources);
 		});
@@ -132,7 +189,7 @@ var fourChanService = {
 		if( !fs.existsSync(filename) )
 		{
 			// We download the file
-			console.log('=== Dowloading::' + uri );
+			//console.log('=== Dowloading::' + uri );
 			/*request.head(uri, function(err, res, body) {
 				// console.log('content-type:', res.headers['content-type']);
 				// console.log('content-length:', res.headers['content-length']);
@@ -146,7 +203,15 @@ var fourChanService = {
 				})
 				.on('error', function (err) {
 					console.log(err)
-				});
+				})
+				.on('response', function(response) {
+					// unmodified http.IncomingMessage object
+					response.on('data', function(data) {
+						// compressed data as it is received
+						console.log('Received ' + data.length + ' bytes of compressed data')
+					});
+			    });
+
 
 		}
 

views/board.jade

@@ -16,8 +16,11 @@ block content
 					a(href='/thread/#{name}/#{thread.id}/#{thread.semantic_url}')
 						img(src='http://i.4cdn.org/#{name}/#{thread.imgurl}s.jpg' class="img-responsive")
 				div.description 
-					if thread.sub.length > 0
-						small.sub !{thread.sub} : 
-					small.teaser !{thread.teaser}
+					p
+						if thread.sub.length > 0
+							small.sub !{thread.sub} : 
+						small.teaser !{thread.teaser}
+					p 
+						small Images: #{thread.i}
 
 

views/test.jade

@@ -0,0 +1 @@
+h1 TEST!