Merge pull request #44 from layoutanalysis/master

getcss cli options to configure timeout, enable json output and stripping of wayback css

Author: johno

bin/getcss

@@ -6,13 +6,21 @@ var normalizeUrl = require('normalize-url');
 var getCss = require('..');
 
 program
-  .version('0.0.1')
-  .command('* <url>')
-  .action(function(url) {
+  .version('0.0.2')
+  .usage('[options] <url>')
+    .option("-j, --json","output downloaded html and css as json")
+    .option("-t, --timeout [ms]","timeout in ms to wait for responses", 30000)
+    .option("-s, --strip_wayback_css","remove wayback toolbar css from internet archive captures")
+  .action(function(url, options) {
     if(url) {
       url = normalizeUrl(url, { stripWWW: false });
-      getCss(url, { verbose: true }).then(function(css) {
-        console.log(css.css);
+      getCss(url, { verbose: true, timeout: options.timeout, stripWayback: options.strip_wayback_css }).then(function(css) {
+          if (options.json){
+              console.log(JSON.stringify(css));
+          }
+          else {
+              console.log(css.css);
+          }
       });
     }
   });

index.js

@@ -1,26 +1,26 @@
-"use strict";
-
 var q = require("q");
 var isCss = require("is-css");
 var isPresent = require("is-present");
 var isBlank = require("is-blank");
 var isUrl = require("is-url-superb");
-var request = require("request");
+var request = require("requestretry");
 var cheerio = require("cheerio");
 var normalizeUrl = require("normalize-url");
 var stripHtmlComments = require("strip-html-comments");
+var stripWaybackToolbar = require("strip-wayback-toolbar")
 var resolveCssImportUrls = require("resolve-css-import-urls");
 var ua = require("ua-string");
 
 var getLinkContents = require("./utils/get-link-contents");
 var createLink = require("./utils/create-link");
 
-module.exports = function(url, options, html) {
+module.exports = function(url, options, html){
   var deferred = q.defer();
   var options = options || {};
   options.headers = options.headers || {};
   options.headers["User-Agent"] = options.headers["User-Agent"] || ua;
   options.timeout = options.timeout || 5000;
+  options.stripWayback = options.stripWayback || false;
   options.gzip = true;
 
   if (typeof url !== "string" || isBlank(url) || !isUrl(url)) {
@@ -52,16 +52,20 @@ module.exports = function(url, options, html) {
   }
 
   function parseHtml(html) {
+    if (options.stripWayback) {
+      html = stripWaybackToolbar(html);
+    }
     var $ = cheerio.load(html);
     result.pageTitle = $("head > title").text();
+    result.html = html;
 
     $("[rel=stylesheet]").each(function() {
-      var link = $(this).attr("href");
-      if (isPresent(link)) {
-        result.links.push(createLink(link, url));
-      } else {
-        result.styles.push(stripHtmlComments($(this).text()));
-      }
+        var link = $(this).attr("href");
+        if (isPresent(link)) {
+            result.links.push(createLink(link, url));
+        } else{
+            result.styles.push(stripHtmlComments($(this).text()));
+        }
     });
 
     $("style").each(function() {

package.json

@@ -21,8 +21,10 @@
     "normalize-url": "^1.8.0",
     "q": "^1.4.1",
     "request": "^2.79.0",
+    "requestretry": "^1.13.0",
     "resolve-css-import-urls": "1.0.0",
     "strip-html-comments": "1.0.0",
+    "strip-wayback-toolbar": "^1.0.4",
     "ua-string": "^1.0.0"
   },
   "devDependencies": {