Modifications to the download files, trying to get complete files when there are a lot of resources

Author: Mirdrack

.gitignore

@@ -3,4 +3,7 @@ npm-debug.log*
 
 # Dependency directory
 # https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git
-node_modules
+node_modules
+
+# Downloaded files
+files

routes.js

@@ -2,10 +2,11 @@ module.exports = function (app, fourChanService) {
 
 	app.get('/', function (req, res) {
 
-		fourChanService.getCategories(function (data) {
+		res.render('index.jade');
+		/*fourChanService.getCategories(function (data) {
 
 			res.render('index.jade', { categories : data });
-		}); 
+		}); */
 
 	});
 
@@ -25,7 +26,7 @@ module.exports = function (app, fourChanService) {
 		var surl = req.params.surl;
 		fourChanService.getThread(section, id, surl, function (data) {
 
-			res.render('thread.jade');
+			res.render('thread.jade', { threadName : surl, resources : data });
 		});
 	});
 };

services/fourChanService.js

@@ -21,68 +21,12 @@ var fourChanService = {
 		    categories.splice(-23, 23);
 		    callback(categories);
 		});
-
-
-		/*var noodle = require('noodlejs');
-		noodle.query({  
-		  url: 'http://www.4chan.org/',
-		  type: 'html',
-		  selector: 'a.boardlink',
-		  extract: 'href'
-		})
-		.then(function (data) {
-		  	console.log(typeof(data.results));
-			//data.results.splice(3 , 5, 'hola' );
-		  	console.log('========');
-		  	console.log(data.results);
-		  	console.log('========');
-		  	var keys = Object.keys(data.results);
-		  	console.log(keys);
-		});*/
-
-
-		/*var Xray = require('x-ray');
-		var x = Xray();
-
-		x('https://4chan.org/', {title:'.boardlink'})
-		(function (err, result) {
-			console.log(result)
-		});*/
-
-		/*[{
-			'title': '.boardlink a@title',
-		  	'href': '.boardlink a@href',
-		}]*/
-		
-
-		/*	var request = require("request");
-			var cheerio = require("cheerio");
-			var url = 'http://4chan.org/';
-			
-			request(url, function (error, response, body) {
-			if (!error) {
-				var $ = cheerio.load(body),
-					temperature = $('.boardlink').html();
-					
-					console.log("It’s " + temperature + " degrees Fahrenheit.");
-			} else {
-				console.log("We’ve encountered an error: " + error);
-			}
-		});*/
-
-		/*console.log('this.categories');
-		console.log(this.categories);
-		var data = {
-			categories : ['uno', 'dos']
-		};
-		return data;*/
 	},
 
 	getBoard : function (boardName, callback) {
 
 		// Proccess to scrap
 		var osmosis = require('osmosis');
-		console.log('http://boards.4chan.org/' + boardName + '/catalog');
 		osmosis.get('http://boards.4chan.org/' + boardName + '/catalog')
 			.set({
 		  		'threads': ['script']
@@ -127,6 +71,7 @@ var fourChanService = {
 	getThread : function(boardName, id, surl, callback) {
 
 		var self = this; // Saving the reference to the main object
+		
 		// Proccess to scrap
 		var osmosis = require('osmosis');
 		var url = 'http://boards.4chan.org/' + boardName + '/thread/' + id + '/' + surl;
@@ -137,6 +82,7 @@ var fourChanService = {
 		.data(function (results) {
 
 			var resources = results.resources;
+			console.log('There going to be downloaded ' + resources.length + ' resources.' );
 			for(var cont = 0; cont < resources.length; cont++)
 			{
 				var url = 'http:' + resources[cont];
@@ -148,17 +94,11 @@ var fourChanService = {
 				// We set the full path '.files/boardName/semanticURL/filename'
 				var path = './files/' + boardName + '/' + surl + '/' +filename;
 
-				console.log(url + '::' + path);
-
 				// We download the file
+				var fileCounter = cont + 1;
 				self.dowloadResource(url, boardName, surl, path, function () {
-					console.log('File downloaded');
+					console.log(fileCounter + '::File ' + path + ' downloaded');
 				});
-				
-				/*
-				self.dowloadResource(url, path, function () {
-					console.log('Dowload completed');
-				});*/
 			}
 		    // After all, we call the callback
 			callback(resources);
@@ -178,22 +118,38 @@ var fourChanService = {
 		// We check for the directories and we create them if they dont exists
 		var boardDirectory = './files/' + boardName
 		var surlDirectory = './files/' + boardName + '/' + surl;
-		if ( !fs.existsSync(boardDirectory) ) {
-		    
+		if ( !fs.existsSync(boardDirectory) ) {		    
+
 		    fs.mkdirSync(boardDirectory);
+		}
+		else {
+
 		    if( !fs.existsSync(surlDirectory) ) {
-		    	
 		    	fs.mkdirSync(surlDirectory);
 		    }
 		}
 
-		// We download the file
-		console.log('=== Dowloading::' + uri );
-		request.head(uri, function(err, res, body) {
-			// console.log('content-type:', res.headers['content-type']);
-			// console.log('content-length:', res.headers['content-length']);
-			request(uri).pipe(fs.createWriteStream(filename)).on('close', callback);
-		});
+		if( !fs.existsSync(filename) )
+		{
+			// We download the file
+			console.log('=== Dowloading::' + uri );
+			/*request.head(uri, function(err, res, body) {
+				// console.log('content-type:', res.headers['content-type']);
+				// console.log('content-length:', res.headers['content-length']);
+				request(uri).pipe(fs.createWriteStream(filename)).on('close', callback);
+			});*/
+
+			request(uri)
+				.pipe(fs.createWriteStream(filename))
+				.on('close', function () {
+					console.log(filename + ' downloaded.');	
+				})
+				.on('error', function (err) {
+					console.log(err)
+				});
+
+		}
+
 	}
 
 

views/thread.jade

@@ -3,4 +3,7 @@ extends layouts/default
 block content
 	div.row
 		div.col-md-12
-			h2 hola!
+			h2 #{threadName}
+			ul
+				each resource in resources
+					li #{resource}