1 | 'use strict'
|
---|
2 | const Url = require('url')
|
---|
3 | const http = require('http')
|
---|
4 | const https = require('https')
|
---|
5 | const zlib = require('minizlib')
|
---|
6 | const Minipass = require('minipass')
|
---|
7 |
|
---|
8 | const Body = require('./body.js')
|
---|
9 | const { writeToStream, getTotalBytes } = Body
|
---|
10 | const Response = require('./response.js')
|
---|
11 | const Headers = require('./headers.js')
|
---|
12 | const { createHeadersLenient } = Headers
|
---|
13 | const Request = require('./request.js')
|
---|
14 | const { getNodeRequestOptions } = Request
|
---|
15 | const FetchError = require('./fetch-error.js')
|
---|
16 | const AbortError = require('./abort-error.js')
|
---|
17 |
|
---|
18 | const resolveUrl = Url.resolve
|
---|
19 |
|
---|
20 | const fetch = (url, opts) => {
|
---|
21 | if (/^data:/.test(url)) {
|
---|
22 | const request = new Request(url, opts)
|
---|
23 | try {
|
---|
24 | const split = url.split(',')
|
---|
25 | const data = Buffer.from(split[1], 'base64')
|
---|
26 | const type = split[0].match(/^data:(.*);base64$/)[1]
|
---|
27 | return Promise.resolve(new Response(data, {
|
---|
28 | headers: {
|
---|
29 | 'Content-Type': type,
|
---|
30 | 'Content-Length': data.length,
|
---|
31 | }
|
---|
32 | }))
|
---|
33 | } catch (er) {
|
---|
34 | return Promise.reject(new FetchError(`[${request.method}] ${
|
---|
35 | request.url} invalid URL, ${er.message}`, 'system', er))
|
---|
36 | }
|
---|
37 | }
|
---|
38 |
|
---|
39 | return new Promise((resolve, reject) => {
|
---|
40 | // build request object
|
---|
41 | const request = new Request(url, opts)
|
---|
42 | let options
|
---|
43 | try {
|
---|
44 | options = getNodeRequestOptions(request)
|
---|
45 | } catch (er) {
|
---|
46 | return reject(er)
|
---|
47 | }
|
---|
48 |
|
---|
49 | const send = (options.protocol === 'https:' ? https : http).request
|
---|
50 | const { signal } = request
|
---|
51 | let response = null
|
---|
52 | const abort = () => {
|
---|
53 | const error = new AbortError('The user aborted a request.')
|
---|
54 | reject(error)
|
---|
55 | if (Minipass.isStream(request.body) &&
|
---|
56 | typeof request.body.destroy === 'function') {
|
---|
57 | request.body.destroy(error)
|
---|
58 | }
|
---|
59 | if (response && response.body) {
|
---|
60 | response.body.emit('error', error)
|
---|
61 | }
|
---|
62 | }
|
---|
63 |
|
---|
64 | if (signal && signal.aborted)
|
---|
65 | return abort()
|
---|
66 |
|
---|
67 | const abortAndFinalize = () => {
|
---|
68 | abort()
|
---|
69 | finalize()
|
---|
70 | }
|
---|
71 |
|
---|
72 | const finalize = () => {
|
---|
73 | req.abort()
|
---|
74 | if (signal)
|
---|
75 | signal.removeEventListener('abort', abortAndFinalize)
|
---|
76 | clearTimeout(reqTimeout)
|
---|
77 | }
|
---|
78 |
|
---|
79 | // send request
|
---|
80 | const req = send(options)
|
---|
81 |
|
---|
82 | if (signal)
|
---|
83 | signal.addEventListener('abort', abortAndFinalize)
|
---|
84 |
|
---|
85 | let reqTimeout = null
|
---|
86 | if (request.timeout) {
|
---|
87 | req.once('socket', socket => {
|
---|
88 | reqTimeout = setTimeout(() => {
|
---|
89 | reject(new FetchError(`network timeout at: ${
|
---|
90 | request.url}`, 'request-timeout'))
|
---|
91 | finalize()
|
---|
92 | }, request.timeout)
|
---|
93 | })
|
---|
94 | }
|
---|
95 |
|
---|
96 | req.on('error', er => {
|
---|
97 | // if a 'response' event is emitted before the 'error' event, then by the
|
---|
98 | // time this handler is run it's too late to reject the Promise for the
|
---|
99 | // response. instead, we forward the error event to the response stream
|
---|
100 | // so that the error will surface to the user when they try to consume
|
---|
101 | // the body. this is done as a side effect of aborting the request except
|
---|
102 | // for in windows, where we must forward the event manually, otherwise
|
---|
103 | // there is no longer a ref'd socket attached to the request and the
|
---|
104 | // stream never ends so the event loop runs out of work and the process
|
---|
105 | // exits without warning.
|
---|
106 | // coverage skipped here due to the difficulty in testing
|
---|
107 | // istanbul ignore next
|
---|
108 | if (req.res)
|
---|
109 | req.res.emit('error', er)
|
---|
110 | reject(new FetchError(`request to ${request.url} failed, reason: ${
|
---|
111 | er.message}`, 'system', er))
|
---|
112 | finalize()
|
---|
113 | })
|
---|
114 |
|
---|
115 | req.on('response', res => {
|
---|
116 | clearTimeout(reqTimeout)
|
---|
117 |
|
---|
118 | const headers = createHeadersLenient(res.headers)
|
---|
119 |
|
---|
120 | // HTTP fetch step 5
|
---|
121 | if (fetch.isRedirect(res.statusCode)) {
|
---|
122 | // HTTP fetch step 5.2
|
---|
123 | const location = headers.get('Location')
|
---|
124 |
|
---|
125 | // HTTP fetch step 5.3
|
---|
126 | const locationURL = location === null ? null
|
---|
127 | : resolveUrl(request.url, location)
|
---|
128 |
|
---|
129 | // HTTP fetch step 5.5
|
---|
130 | switch (request.redirect) {
|
---|
131 | case 'error':
|
---|
132 | reject(new FetchError(`uri requested responds with a redirect, redirect mode is set to error: ${
|
---|
133 | request.url}`, 'no-redirect'))
|
---|
134 | finalize()
|
---|
135 | return
|
---|
136 |
|
---|
137 | case 'manual':
|
---|
138 | // node-fetch-specific step: make manual redirect a bit easier to
|
---|
139 | // use by setting the Location header value to the resolved URL.
|
---|
140 | if (locationURL !== null) {
|
---|
141 | // handle corrupted header
|
---|
142 | try {
|
---|
143 | headers.set('Location', locationURL)
|
---|
144 | } catch (err) {
|
---|
145 | /* istanbul ignore next: nodejs server prevent invalid
|
---|
146 | response headers, we can't test this through normal
|
---|
147 | request */
|
---|
148 | reject(err)
|
---|
149 | }
|
---|
150 | }
|
---|
151 | break
|
---|
152 |
|
---|
153 | case 'follow':
|
---|
154 | // HTTP-redirect fetch step 2
|
---|
155 | if (locationURL === null) {
|
---|
156 | break
|
---|
157 | }
|
---|
158 |
|
---|
159 | // HTTP-redirect fetch step 5
|
---|
160 | if (request.counter >= request.follow) {
|
---|
161 | reject(new FetchError(`maximum redirect reached at: ${
|
---|
162 | request.url}`, 'max-redirect'))
|
---|
163 | finalize()
|
---|
164 | return
|
---|
165 | }
|
---|
166 |
|
---|
167 | // HTTP-redirect fetch step 9
|
---|
168 | if (res.statusCode !== 303 &&
|
---|
169 | request.body &&
|
---|
170 | getTotalBytes(request) === null) {
|
---|
171 | reject(new FetchError(
|
---|
172 | 'Cannot follow redirect with body being a readable stream',
|
---|
173 | 'unsupported-redirect'
|
---|
174 | ))
|
---|
175 | finalize()
|
---|
176 | return
|
---|
177 | }
|
---|
178 |
|
---|
179 | // Update host due to redirection
|
---|
180 | request.headers.set('host', Url.parse(locationURL).host)
|
---|
181 |
|
---|
182 | // HTTP-redirect fetch step 6 (counter increment)
|
---|
183 | // Create a new Request object.
|
---|
184 | const requestOpts = {
|
---|
185 | headers: new Headers(request.headers),
|
---|
186 | follow: request.follow,
|
---|
187 | counter: request.counter + 1,
|
---|
188 | agent: request.agent,
|
---|
189 | compress: request.compress,
|
---|
190 | method: request.method,
|
---|
191 | body: request.body,
|
---|
192 | signal: request.signal,
|
---|
193 | timeout: request.timeout,
|
---|
194 | }
|
---|
195 |
|
---|
196 | // HTTP-redirect fetch step 11
|
---|
197 | if (res.statusCode === 303 || (
|
---|
198 | (res.statusCode === 301 || res.statusCode === 302) &&
|
---|
199 | request.method === 'POST'
|
---|
200 | )) {
|
---|
201 | requestOpts.method = 'GET'
|
---|
202 | requestOpts.body = undefined
|
---|
203 | requestOpts.headers.delete('content-length')
|
---|
204 | }
|
---|
205 |
|
---|
206 | // HTTP-redirect fetch step 15
|
---|
207 | resolve(fetch(new Request(locationURL, requestOpts)))
|
---|
208 | finalize()
|
---|
209 | return
|
---|
210 | }
|
---|
211 | } // end if(isRedirect)
|
---|
212 |
|
---|
213 |
|
---|
214 | // prepare response
|
---|
215 | res.once('end', () =>
|
---|
216 | signal && signal.removeEventListener('abort', abortAndFinalize))
|
---|
217 |
|
---|
218 | const body = new Minipass()
|
---|
219 | // exceedingly rare that the stream would have an error,
|
---|
220 | // but just in case we proxy it to the stream in use.
|
---|
221 | res.on('error', /* istanbul ignore next */ er => body.emit('error', er))
|
---|
222 | res.on('data', (chunk) => body.write(chunk))
|
---|
223 | res.on('end', () => body.end())
|
---|
224 |
|
---|
225 | const responseOptions = {
|
---|
226 | url: request.url,
|
---|
227 | status: res.statusCode,
|
---|
228 | statusText: res.statusMessage,
|
---|
229 | headers: headers,
|
---|
230 | size: request.size,
|
---|
231 | timeout: request.timeout,
|
---|
232 | counter: request.counter,
|
---|
233 | trailer: new Promise(resolve =>
|
---|
234 | res.on('end', () => resolve(createHeadersLenient(res.trailers))))
|
---|
235 | }
|
---|
236 |
|
---|
237 | // HTTP-network fetch step 12.1.1.3
|
---|
238 | const codings = headers.get('Content-Encoding')
|
---|
239 |
|
---|
240 | // HTTP-network fetch step 12.1.1.4: handle content codings
|
---|
241 |
|
---|
242 | // in following scenarios we ignore compression support
|
---|
243 | // 1. compression support is disabled
|
---|
244 | // 2. HEAD request
|
---|
245 | // 3. no Content-Encoding header
|
---|
246 | // 4. no content response (204)
|
---|
247 | // 5. content not modified response (304)
|
---|
248 | if (!request.compress ||
|
---|
249 | request.method === 'HEAD' ||
|
---|
250 | codings === null ||
|
---|
251 | res.statusCode === 204 ||
|
---|
252 | res.statusCode === 304) {
|
---|
253 | response = new Response(body, responseOptions)
|
---|
254 | resolve(response)
|
---|
255 | return
|
---|
256 | }
|
---|
257 |
|
---|
258 |
|
---|
259 | // Be less strict when decoding compressed responses, since sometimes
|
---|
260 | // servers send slightly invalid responses that are still accepted
|
---|
261 | // by common browsers.
|
---|
262 | // Always using Z_SYNC_FLUSH is what cURL does.
|
---|
263 | const zlibOptions = {
|
---|
264 | flush: zlib.constants.Z_SYNC_FLUSH,
|
---|
265 | finishFlush: zlib.constants.Z_SYNC_FLUSH,
|
---|
266 | }
|
---|
267 |
|
---|
268 | // for gzip
|
---|
269 | if (codings == 'gzip' || codings == 'x-gzip') {
|
---|
270 | const unzip = new zlib.Gunzip(zlibOptions)
|
---|
271 | response = new Response(
|
---|
272 | // exceedingly rare that the stream would have an error,
|
---|
273 | // but just in case we proxy it to the stream in use.
|
---|
274 | body.on('error', /* istanbul ignore next */ er => unzip.emit('error', er)).pipe(unzip),
|
---|
275 | responseOptions
|
---|
276 | )
|
---|
277 | resolve(response)
|
---|
278 | return
|
---|
279 | }
|
---|
280 |
|
---|
281 | // for deflate
|
---|
282 | if (codings == 'deflate' || codings == 'x-deflate') {
|
---|
283 | // handle the infamous raw deflate response from old servers
|
---|
284 | // a hack for old IIS and Apache servers
|
---|
285 | const raw = res.pipe(new Minipass())
|
---|
286 | raw.once('data', chunk => {
|
---|
287 | // see http://stackoverflow.com/questions/37519828
|
---|
288 | const decoder = (chunk[0] & 0x0F) === 0x08
|
---|
289 | ? new zlib.Inflate()
|
---|
290 | : new zlib.InflateRaw()
|
---|
291 | // exceedingly rare that the stream would have an error,
|
---|
292 | // but just in case we proxy it to the stream in use.
|
---|
293 | body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
|
---|
294 | response = new Response(decoder, responseOptions)
|
---|
295 | resolve(response)
|
---|
296 | })
|
---|
297 | return
|
---|
298 | }
|
---|
299 |
|
---|
300 |
|
---|
301 | // for br
|
---|
302 | if (codings == 'br') {
|
---|
303 | // ignoring coverage so tests don't have to fake support (or lack of) for brotli
|
---|
304 | // istanbul ignore next
|
---|
305 | try {
|
---|
306 | var decoder = new zlib.BrotliDecompress()
|
---|
307 | } catch (err) {
|
---|
308 | reject(err)
|
---|
309 | finalize()
|
---|
310 | return
|
---|
311 | }
|
---|
312 | // exceedingly rare that the stream would have an error,
|
---|
313 | // but just in case we proxy it to the stream in use.
|
---|
314 | body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
|
---|
315 | response = new Response(decoder, responseOptions)
|
---|
316 | resolve(response)
|
---|
317 | return
|
---|
318 | }
|
---|
319 |
|
---|
320 | // otherwise, use response as-is
|
---|
321 | response = new Response(body, responseOptions)
|
---|
322 | resolve(response)
|
---|
323 | })
|
---|
324 |
|
---|
325 | writeToStream(req, request)
|
---|
326 | })
|
---|
327 | }
|
---|
328 |
|
---|
329 | module.exports = fetch
|
---|
330 |
|
---|
331 | fetch.isRedirect = code =>
|
---|
332 | code === 301 ||
|
---|
333 | code === 302 ||
|
---|
334 | code === 303 ||
|
---|
335 | code === 307 ||
|
---|
336 | code === 308
|
---|
337 |
|
---|
338 | fetch.Headers = Headers
|
---|
339 | fetch.Request = Request
|
---|
340 | fetch.Response = Response
|
---|
341 | fetch.FetchError = FetchError
|
---|