From fc1b2ce216355dc117e81d2fd5fd9678262a23da Mon Sep 17 00:00:00 2001 From: Denis Ah-Kang Date: Wed, 11 Dec 2024 15:54:15 +0400 Subject: [PATCH 1/2] replace request with superagent and add a user-agent --- lib/document-downloader.js | 26 +++++++++++--------------- lib/json-http-service.js | 24 ++++++++++-------------- lib/token-checker.js | 30 ++++++++++++++---------------- package.json | 2 +- 4 files changed, 36 insertions(+), 46 deletions(-) diff --git a/lib/document-downloader.js b/lib/document-downloader.js index 388e9ebc..6d692640 100644 --- a/lib/document-downloader.js +++ b/lib/document-downloader.js @@ -5,7 +5,7 @@ 'use strict'; import Fs from 'fs'; -import Request from 'request'; +import sua from 'superagent'; import pkg from 'immutable'; import Promise from 'promise'; import Url from 'url'; @@ -24,28 +24,24 @@ const DocumentDownloader = {}; DocumentDownloader.fetch = url => new Promise((resolve, reject) => { - Request.get( - url, - { - timeout: 30000, - encoding: null, // If not specified, utf8 by default - }, - (error, response, body) => { - if (error) { + sua + .get(url) + .set('User-Agent', 'W3C/Echidna') + .end((err, res) => { + if (err) { reject( new Error( - `Fetching ${url} triggered a network error: ${error.message}`, + `Fetching ${url} triggered a network error: ${err.message}`, ), ); - } else if (response.statusCode !== 200) { + } else if (res.statusCode !== 200) { reject( new Error( - `Fetching ${url} triggered an HTTP error: code ${response.statusCode}`, + `Fetching ${url} triggered an HTTP error: code ${res.statusCode}`, ), ); - } else resolve(body); - }, - ); + } else resolve(res.body); + }); }); DocumentDownloader.fetchAll = urls => diff --git a/lib/json-http-service.js b/lib/json-http-service.js index df3143f0..e8901ea6 100644 --- a/lib/json-http-service.js +++ b/lib/json-http-service.js @@ -6,7 +6,7 @@ 'use strict'; import Promise from 'promise'; -import Request from 'request'; +import sua from 'superagent'; /** * @exports lib/json-http-service @@ -45,19 +45,15 @@ JsonHttpService.prototype = { const self = this; return new Promise((resolve, reject) => { - Request.post( - { - url: self.url, - json: true, - auth: { user: self.user, pass: self.pass }, - headers: self.headers, - body, - }, - (error, response, bodyResult) => { - if (error) reject(error); - else resolve({ response, body: bodyResult }); - }, - ); + sua.post(self.url) + .set('Content-Type', 'application/json') + .auth(self.user, self.pass) + .set(self.headers) + .send(body) + .end((err, res) => { + if (err) reject(err); + else resolve({ response: res, body: res.body }); + }); }); }, }; diff --git a/lib/token-checker.js b/lib/token-checker.js index f6a6f023..fb0efd40 100644 --- a/lib/token-checker.js +++ b/lib/token-checker.js @@ -5,7 +5,7 @@ 'use strict'; import Promise from 'promise'; -import Request from 'request'; +import sua from 'superagent'; import pkg from 'immutable'; const { List } = pkg; @@ -23,25 +23,23 @@ const TokenChecker = {}; TokenChecker.check = (url, token) => // url shortlink new Promise((resolve, reject) => { - const options = { - uri: global.TOKEN_ENDPOINT, - headers: { 'X-W3C-Sso': 'bypass' }, - qs: { spec: url, token }, - // USERNAME & PASSWORD is a credential for https://www.w3.org/Web/publications/authorize? - auth: { user: global.USERNAME, pass: global.PASSWORD }, - }; - Request.get(options, (err, res, body) => { - if (err) - reject(new Error('There was an error while checking the token: ', err)); + sua + .get(global.TOKEN_ENDPOINT) + .set('User-Agent', 'W3C/Echidna') + .set('X-W3C-Sso', 'bypass') + .query({ spec: url, token }) + .auth(global.USERNAME, global.PASSWORD) + .end((err, res) => { + if (err) reject(new Error('There was an error while checking the token: ', err)); - const report = JSON.parse(body); - let errors = new List(); + const report = res.body; + let errors = new List(); - if (!report.authorized) errors = errors.push('not-authorized'); + if (!report.authorized) errors = errors.push('not-authorized'); - resolve(errors); - }); + resolve(errors); + }); }); export default TokenChecker; diff --git a/package.json b/package.json index c86c26ee..8659f616 100644 --- a/package.json +++ b/package.json @@ -33,8 +33,8 @@ "passport": "0.7.0", "passport-http": "0.3", "promise": "8.3.0", - "request": "2.88.2", "specberus": "11.4.3", + "superagent": "10.1.0", "tar-stream": "3.1.1", "uuid": "11.0.3" }, From 632e253130ecadbe4c4f7fd4d8813c930afe03d2 Mon Sep 17 00:00:00 2001 From: Denis Ah-Kang Date: Wed, 11 Dec 2024 16:42:32 +0400 Subject: [PATCH 2/2] change response type so we can still process different type of files --- lib/document-downloader.js | 1 + test/test.js | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/document-downloader.js b/lib/document-downloader.js index 6d692640..5f121919 100644 --- a/lib/document-downloader.js +++ b/lib/document-downloader.js @@ -27,6 +27,7 @@ DocumentDownloader.fetch = url => sua .get(url) .set('User-Agent', 'W3C/Echidna') + .responseType('blob') .end((err, res) => { if (err) { reject( diff --git a/test/test.js b/test/test.js index 82fe3599..7e9817b0 100644 --- a/test/test.js +++ b/test/test.js @@ -79,8 +79,7 @@ describe('DocumentDownloader', () => { const notFound = DocumentDownloader.fetch( `${server.location()}/et/si/tu/n/existais/pas`, ); - - return expect(notFound).to.eventually.be.rejectedWith(/code 404/); + return expect(notFound).to.eventually.be.rejectedWith(/Not Found/); }); it('should reject if the server is not reachable', () => { @@ -241,7 +240,7 @@ describe('DocumentDownloader', () => { '/tmp/whatever', ); - return expect(notFound).to.eventually.be.rejectedWith(/code 404/); + return expect(notFound).to.eventually.be.rejectedWith(/Not Found/); }); it('should reject if the server is not reachable', () => {