|
1 |
| -var q = require("q"); |
2 |
| -var isCss = require("is-css"); |
3 |
| -var isPresent = require("is-present"); |
4 |
| -var isBlank = require("is-blank"); |
5 |
| -var isUrl = require("is-url-superb"); |
6 |
| -var request = require("requestretry"); |
7 |
| -var cheerio = require("cheerio"); |
8 |
| -var normalizeUrl = require("normalize-url"); |
9 |
| -var stripHtmlComments = require("strip-html-comments"); |
10 |
| -var stripWaybackToolbar = require("strip-wayback-toolbar") |
11 |
| -var resolveCssImportUrls = require("resolve-css-import-urls"); |
12 |
| -var ua = require("ua-string"); |
13 |
| - |
14 |
| -var getLinkContents = require("./utils/get-link-contents"); |
15 |
| -var createLink = require("./utils/create-link"); |
16 |
| - |
17 |
| -module.exports = function(url, options, html){ |
18 |
| - var deferred = q.defer(); |
19 |
| - var options = options || {}; |
20 |
| - options.headers = options.headers || {}; |
21 |
| - options.headers["User-Agent"] = options.headers["User-Agent"] || ua; |
22 |
| - options.timeout = options.timeout || 5000; |
23 |
| - options.stripWayback = options.stripWayback || false; |
24 |
| - options.gzip = true; |
25 |
| - |
26 |
| - if (typeof url !== "string" || isBlank(url) || !isUrl(url)) { |
27 |
| - throw new TypeError("get-css expected a url as a string"); |
| 1 | +var q = require('q') |
| 2 | +var isCss = require('is-css') |
| 3 | +var isPresent = require('is-present') |
| 4 | +var isBlank = require('is-blank') |
| 5 | +var isUrl = require('is-url-superb') |
| 6 | +var request = require('requestretry') |
| 7 | +var cheerio = require('cheerio') |
| 8 | +var normalizeUrl = require('normalize-url') |
| 9 | +var stripHtmlComments = require('strip-html-comments') |
| 10 | +var stripWaybackToolbar = require('strip-wayback-toolbar') |
| 11 | +var resolveCssImportUrls = require('resolve-css-import-urls') |
| 12 | +var ua = require('ua-string') |
| 13 | + |
| 14 | +var getLinkContents = require('./utils/get-link-contents') |
| 15 | +var createLink = require('./utils/create-link') |
| 16 | + |
| 17 | +module.exports = function(url, options, html) { |
| 18 | + var deferred = q.defer() |
| 19 | + var options = options || {} |
| 20 | + options.headers = options.headers || {} |
| 21 | + options.headers['User-Agent'] = options.headers['User-Agent'] || ua |
| 22 | + options.timeout = options.timeout || 5000 |
| 23 | + options.stripWayback = options.stripWayback || false |
| 24 | + options.gzip = true |
| 25 | + |
| 26 | + if (typeof url !== 'string' || isBlank(url) || !isUrl(url)) { |
| 27 | + throw new TypeError('get-css expected a url as a string') |
28 | 28 | }
|
29 | 29 |
|
30 |
| - url = normalizeUrl(url, { stripWWW: false }); |
31 |
| - options.url = url; |
| 30 | + url = normalizeUrl(url, { stripWWW: false }) |
| 31 | + options.url = url |
32 | 32 |
|
33 | 33 | if (options.ignoreCerts) {
|
34 |
| - process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; |
| 34 | + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' |
35 | 35 | }
|
36 | 36 |
|
37 | 37 | var status = {
|
38 | 38 | parsed: 0,
|
39 | 39 | total: 0
|
40 |
| - }; |
| 40 | + } |
41 | 41 |
|
42 | 42 | var result = {
|
43 | 43 | links: [],
|
44 | 44 | styles: [],
|
45 |
| - css: "" |
46 |
| - }; |
| 45 | + css: '' |
| 46 | + } |
47 | 47 |
|
48 | 48 | function handleResolve() {
|
49 | 49 | if (status.parsed >= status.total) {
|
50 |
| - deferred.resolve(result); |
| 50 | + deferred.resolve(result) |
51 | 51 | }
|
52 | 52 | }
|
53 | 53 |
|
54 | 54 | function parseHtml(html) {
|
55 | 55 | if (options.stripWayback) {
|
56 |
| - html = stripWaybackToolbar(html); |
| 56 | + html = stripWaybackToolbar(html) |
57 | 57 | }
|
58 |
| - var $ = cheerio.load(html); |
59 |
| - result.pageTitle = $("head > title").text(); |
60 |
| - result.html = html; |
61 |
| - |
62 |
| - $("[rel=stylesheet]").each(function() { |
63 |
| - var link = $(this).attr("href"); |
64 |
| - if (isPresent(link)) { |
65 |
| - result.links.push(createLink(link, url)); |
66 |
| - } else{ |
67 |
| - result.styles.push(stripHtmlComments($(this).text())); |
68 |
| - } |
69 |
| - }); |
70 |
| - |
71 |
| - $("style").each(function() { |
72 |
| - result.styles.push(stripHtmlComments($(this).text())); |
73 |
| - }); |
74 |
| - |
75 |
| - status.total = result.links.length + result.styles.length; |
| 58 | + var $ = cheerio.load(html) |
| 59 | + result.pageTitle = $('head > title').text() |
| 60 | + result.html = html |
| 61 | + |
| 62 | + $('[rel=stylesheet]').each(function() { |
| 63 | + var link = $(this).attr('href') |
| 64 | + if (isPresent(link)) { |
| 65 | + result.links.push(createLink(link, url)) |
| 66 | + } else { |
| 67 | + result.styles.push(stripHtmlComments($(this).text())) |
| 68 | + } |
| 69 | + }) |
| 70 | + |
| 71 | + $('style').each(function() { |
| 72 | + result.styles.push(stripHtmlComments($(this).text())) |
| 73 | + }) |
| 74 | + |
| 75 | + status.total = result.links.length + result.styles.length |
76 | 76 | if (!status.total) {
|
77 |
| - deferred.resolve(false); |
| 77 | + deferred.resolve(false) |
78 | 78 | }
|
79 | 79 |
|
80 | 80 | result.links.forEach(function(link) {
|
81 | 81 | getLinkContents(link.url, options)
|
82 | 82 | .then(function(css) {
|
83 |
| - handleCssFromLink(link, css); |
| 83 | + handleCssFromLink(link, css) |
84 | 84 | })
|
85 | 85 | .catch(function(error) {
|
86 |
| - link.error = error; |
87 |
| - status.parsed++; |
88 |
| - handleResolve(); |
89 |
| - }); |
90 |
| - }); |
| 86 | + link.error = error |
| 87 | + status.parsed++ |
| 88 | + handleResolve() |
| 89 | + }) |
| 90 | + }) |
91 | 91 |
|
92 | 92 | result.styles.forEach(function(css) {
|
93 |
| - result.css += css; |
94 |
| - status.parsed++; |
95 |
| - handleResolve(); |
96 |
| - }); |
| 93 | + result.css += css |
| 94 | + status.parsed++ |
| 95 | + handleResolve() |
| 96 | + }) |
97 | 97 | }
|
98 | 98 |
|
99 | 99 | function handleCssFromLink(link, css) {
|
100 |
| - link.css += css; |
| 100 | + link.css += css |
101 | 101 |
|
102 |
| - parseCssForImports(link, css); |
| 102 | + parseCssForImports(link, css) |
103 | 103 |
|
104 |
| - status.parsed++; |
105 |
| - handleResolve(); |
| 104 | + status.parsed++ |
| 105 | + handleResolve() |
106 | 106 | }
|
107 | 107 |
|
108 | 108 | // Handle potential @import url(foo.css) statements in the CSS.
|
109 | 109 | function parseCssForImports(link, css) {
|
110 |
| - link.imports = resolveCssImportUrls(link.url, css); |
111 |
| - status.total += link.imports.length; |
112 |
| - result.css += css; |
| 110 | + link.imports = resolveCssImportUrls(link.url, css) |
| 111 | + status.total += link.imports.length |
| 112 | + result.css += css |
113 | 113 |
|
114 | 114 | link.imports.forEach(function(importUrl) {
|
115 |
| - var importLink = createLink(importUrl, importUrl); |
116 |
| - result.links.push(importLink); |
| 115 | + var importLink = createLink(importUrl, importUrl) |
| 116 | + result.links.push(importLink) |
117 | 117 |
|
118 | 118 | getLinkContents(importLink.url, options)
|
119 | 119 | .then(function(css) {
|
120 |
| - handleCssFromLink(importLink, css); |
| 120 | + handleCssFromLink(importLink, css) |
121 | 121 | })
|
122 | 122 | .catch(function(error) {
|
123 |
| - link.error = error; |
124 |
| - status.parsed++; |
125 |
| - handleResolve(); |
126 |
| - }); |
127 |
| - }); |
| 123 | + link.error = error |
| 124 | + status.parsed++ |
| 125 | + handleResolve() |
| 126 | + }) |
| 127 | + }) |
128 | 128 | }
|
129 | 129 |
|
130 | 130 | function handleBody(body) {
|
131 | 131 | if (isCss(url)) {
|
132 |
| - var link = createLink(url, url); |
133 |
| - result.links.push(link); |
134 |
| - handleCssFromLink(link, body); |
| 132 | + var link = createLink(url, url) |
| 133 | + result.links.push(link) |
| 134 | + handleCssFromLink(link, body) |
135 | 135 | } else {
|
136 |
| - parseHtml(body); |
| 136 | + parseHtml(body) |
137 | 137 | }
|
138 | 138 | }
|
139 | 139 |
|
140 | 140 | if (html) {
|
141 |
| - handleBody(html); |
| 141 | + handleBody(html) |
142 | 142 | } else {
|
143 | 143 | request(options, function(error, response, body) {
|
144 | 144 | if (error) {
|
145 |
| - if (options.verbose) console.log("Error from " + url + " " + error); |
146 |
| - deferred.reject(error); |
147 |
| - return; |
| 145 | + if (options.verbose) console.log('Error from ' + url + ' ' + error) |
| 146 | + deferred.reject(error) |
| 147 | + return |
148 | 148 | }
|
149 | 149 |
|
150 | 150 | if (response && response.statusCode != 200) {
|
151 | 151 | if (options.verbose)
|
152 |
| - console.log("Received a " + response.statusCode + " from: " + url); |
153 |
| - deferred.reject({ url: url, statusCode: response.code }); |
154 |
| - return; |
| 152 | + console.log('Received a ' + response.statusCode + ' from: ' + url) |
| 153 | + deferred.reject({ url: url, statusCode: response.code }) |
| 154 | + return |
155 | 155 | }
|
156 | 156 |
|
157 |
| - handleBody(body); |
158 |
| - }); |
| 157 | + handleBody(body) |
| 158 | + }) |
159 | 159 | }
|
160 | 160 |
|
161 |
| - return deferred.promise; |
162 |
| -}; |
| 161 | + return deferred.promise |
| 162 | +} |
0 commit comments