[6a3a178] | 1 | 'use strict'
|
---|
| 2 | const Url = require('url')
|
---|
| 3 | const http = require('http')
|
---|
| 4 | const https = require('https')
|
---|
| 5 | const zlib = require('minizlib')
|
---|
| 6 | const Minipass = require('minipass')
|
---|
| 7 |
|
---|
| 8 | const Body = require('./body.js')
|
---|
| 9 | const { writeToStream, getTotalBytes } = Body
|
---|
| 10 | const Response = require('./response.js')
|
---|
| 11 | const Headers = require('./headers.js')
|
---|
| 12 | const { createHeadersLenient } = Headers
|
---|
| 13 | const Request = require('./request.js')
|
---|
| 14 | const { getNodeRequestOptions } = Request
|
---|
| 15 | const FetchError = require('./fetch-error.js')
|
---|
| 16 | const AbortError = require('./abort-error.js')
|
---|
| 17 |
|
---|
| 18 | const resolveUrl = Url.resolve
|
---|
| 19 |
|
---|
| 20 | const fetch = (url, opts) => {
|
---|
| 21 | if (/^data:/.test(url)) {
|
---|
| 22 | const request = new Request(url, opts)
|
---|
| 23 | try {
|
---|
| 24 | const split = url.split(',')
|
---|
| 25 | const data = Buffer.from(split[1], 'base64')
|
---|
| 26 | const type = split[0].match(/^data:(.*);base64$/)[1]
|
---|
| 27 | return Promise.resolve(new Response(data, {
|
---|
| 28 | headers: {
|
---|
| 29 | 'Content-Type': type,
|
---|
| 30 | 'Content-Length': data.length,
|
---|
| 31 | }
|
---|
| 32 | }))
|
---|
| 33 | } catch (er) {
|
---|
| 34 | return Promise.reject(new FetchError(`[${request.method}] ${
|
---|
| 35 | request.url} invalid URL, ${er.message}`, 'system', er))
|
---|
| 36 | }
|
---|
| 37 | }
|
---|
| 38 |
|
---|
| 39 | return new Promise((resolve, reject) => {
|
---|
| 40 | // build request object
|
---|
| 41 | const request = new Request(url, opts)
|
---|
| 42 | let options
|
---|
| 43 | try {
|
---|
| 44 | options = getNodeRequestOptions(request)
|
---|
| 45 | } catch (er) {
|
---|
| 46 | return reject(er)
|
---|
| 47 | }
|
---|
| 48 |
|
---|
| 49 | const send = (options.protocol === 'https:' ? https : http).request
|
---|
| 50 | const { signal } = request
|
---|
| 51 | let response = null
|
---|
| 52 | const abort = () => {
|
---|
| 53 | const error = new AbortError('The user aborted a request.')
|
---|
| 54 | reject(error)
|
---|
| 55 | if (Minipass.isStream(request.body) &&
|
---|
| 56 | typeof request.body.destroy === 'function') {
|
---|
| 57 | request.body.destroy(error)
|
---|
| 58 | }
|
---|
| 59 | if (response && response.body) {
|
---|
| 60 | response.body.emit('error', error)
|
---|
| 61 | }
|
---|
| 62 | }
|
---|
| 63 |
|
---|
| 64 | if (signal && signal.aborted)
|
---|
| 65 | return abort()
|
---|
| 66 |
|
---|
| 67 | const abortAndFinalize = () => {
|
---|
| 68 | abort()
|
---|
| 69 | finalize()
|
---|
| 70 | }
|
---|
| 71 |
|
---|
| 72 | const finalize = () => {
|
---|
| 73 | req.abort()
|
---|
| 74 | if (signal)
|
---|
| 75 | signal.removeEventListener('abort', abortAndFinalize)
|
---|
| 76 | clearTimeout(reqTimeout)
|
---|
| 77 | }
|
---|
| 78 |
|
---|
| 79 | // send request
|
---|
| 80 | const req = send(options)
|
---|
| 81 |
|
---|
| 82 | if (signal)
|
---|
| 83 | signal.addEventListener('abort', abortAndFinalize)
|
---|
| 84 |
|
---|
| 85 | let reqTimeout = null
|
---|
| 86 | if (request.timeout) {
|
---|
| 87 | req.once('socket', socket => {
|
---|
| 88 | reqTimeout = setTimeout(() => {
|
---|
| 89 | reject(new FetchError(`network timeout at: ${
|
---|
| 90 | request.url}`, 'request-timeout'))
|
---|
| 91 | finalize()
|
---|
| 92 | }, request.timeout)
|
---|
| 93 | })
|
---|
| 94 | }
|
---|
| 95 |
|
---|
| 96 | req.on('error', er => {
|
---|
| 97 | // if a 'response' event is emitted before the 'error' event, then by the
|
---|
| 98 | // time this handler is run it's too late to reject the Promise for the
|
---|
| 99 | // response. instead, we forward the error event to the response stream
|
---|
| 100 | // so that the error will surface to the user when they try to consume
|
---|
| 101 | // the body. this is done as a side effect of aborting the request except
|
---|
| 102 | // for in windows, where we must forward the event manually, otherwise
|
---|
| 103 | // there is no longer a ref'd socket attached to the request and the
|
---|
| 104 | // stream never ends so the event loop runs out of work and the process
|
---|
| 105 | // exits without warning.
|
---|
| 106 | // coverage skipped here due to the difficulty in testing
|
---|
| 107 | // istanbul ignore next
|
---|
| 108 | if (req.res)
|
---|
| 109 | req.res.emit('error', er)
|
---|
| 110 | reject(new FetchError(`request to ${request.url} failed, reason: ${
|
---|
| 111 | er.message}`, 'system', er))
|
---|
| 112 | finalize()
|
---|
| 113 | })
|
---|
| 114 |
|
---|
| 115 | req.on('response', res => {
|
---|
| 116 | clearTimeout(reqTimeout)
|
---|
| 117 |
|
---|
| 118 | const headers = createHeadersLenient(res.headers)
|
---|
| 119 |
|
---|
| 120 | // HTTP fetch step 5
|
---|
| 121 | if (fetch.isRedirect(res.statusCode)) {
|
---|
| 122 | // HTTP fetch step 5.2
|
---|
| 123 | const location = headers.get('Location')
|
---|
| 124 |
|
---|
| 125 | // HTTP fetch step 5.3
|
---|
| 126 | const locationURL = location === null ? null
|
---|
| 127 | : resolveUrl(request.url, location)
|
---|
| 128 |
|
---|
| 129 | // HTTP fetch step 5.5
|
---|
| 130 | switch (request.redirect) {
|
---|
| 131 | case 'error':
|
---|
| 132 | reject(new FetchError(`uri requested responds with a redirect, redirect mode is set to error: ${
|
---|
| 133 | request.url}`, 'no-redirect'))
|
---|
| 134 | finalize()
|
---|
| 135 | return
|
---|
| 136 |
|
---|
| 137 | case 'manual':
|
---|
| 138 | // node-fetch-specific step: make manual redirect a bit easier to
|
---|
| 139 | // use by setting the Location header value to the resolved URL.
|
---|
| 140 | if (locationURL !== null) {
|
---|
| 141 | // handle corrupted header
|
---|
| 142 | try {
|
---|
| 143 | headers.set('Location', locationURL)
|
---|
| 144 | } catch (err) {
|
---|
| 145 | /* istanbul ignore next: nodejs server prevent invalid
|
---|
| 146 | response headers, we can't test this through normal
|
---|
| 147 | request */
|
---|
| 148 | reject(err)
|
---|
| 149 | }
|
---|
| 150 | }
|
---|
| 151 | break
|
---|
| 152 |
|
---|
| 153 | case 'follow':
|
---|
| 154 | // HTTP-redirect fetch step 2
|
---|
| 155 | if (locationURL === null) {
|
---|
| 156 | break
|
---|
| 157 | }
|
---|
| 158 |
|
---|
| 159 | // HTTP-redirect fetch step 5
|
---|
| 160 | if (request.counter >= request.follow) {
|
---|
| 161 | reject(new FetchError(`maximum redirect reached at: ${
|
---|
| 162 | request.url}`, 'max-redirect'))
|
---|
| 163 | finalize()
|
---|
| 164 | return
|
---|
| 165 | }
|
---|
| 166 |
|
---|
| 167 | // HTTP-redirect fetch step 9
|
---|
| 168 | if (res.statusCode !== 303 &&
|
---|
| 169 | request.body &&
|
---|
| 170 | getTotalBytes(request) === null) {
|
---|
| 171 | reject(new FetchError(
|
---|
| 172 | 'Cannot follow redirect with body being a readable stream',
|
---|
| 173 | 'unsupported-redirect'
|
---|
| 174 | ))
|
---|
| 175 | finalize()
|
---|
| 176 | return
|
---|
| 177 | }
|
---|
| 178 |
|
---|
| 179 | // Update host due to redirection
|
---|
| 180 | request.headers.set('host', Url.parse(locationURL).host)
|
---|
| 181 |
|
---|
| 182 | // HTTP-redirect fetch step 6 (counter increment)
|
---|
| 183 | // Create a new Request object.
|
---|
| 184 | const requestOpts = {
|
---|
| 185 | headers: new Headers(request.headers),
|
---|
| 186 | follow: request.follow,
|
---|
| 187 | counter: request.counter + 1,
|
---|
| 188 | agent: request.agent,
|
---|
| 189 | compress: request.compress,
|
---|
| 190 | method: request.method,
|
---|
| 191 | body: request.body,
|
---|
| 192 | signal: request.signal,
|
---|
| 193 | timeout: request.timeout,
|
---|
| 194 | }
|
---|
| 195 |
|
---|
| 196 | // HTTP-redirect fetch step 11
|
---|
| 197 | if (res.statusCode === 303 || (
|
---|
| 198 | (res.statusCode === 301 || res.statusCode === 302) &&
|
---|
| 199 | request.method === 'POST'
|
---|
| 200 | )) {
|
---|
| 201 | requestOpts.method = 'GET'
|
---|
| 202 | requestOpts.body = undefined
|
---|
| 203 | requestOpts.headers.delete('content-length')
|
---|
| 204 | }
|
---|
| 205 |
|
---|
| 206 | // HTTP-redirect fetch step 15
|
---|
| 207 | resolve(fetch(new Request(locationURL, requestOpts)))
|
---|
| 208 | finalize()
|
---|
| 209 | return
|
---|
| 210 | }
|
---|
| 211 | } // end if(isRedirect)
|
---|
| 212 |
|
---|
| 213 |
|
---|
| 214 | // prepare response
|
---|
| 215 | res.once('end', () =>
|
---|
| 216 | signal && signal.removeEventListener('abort', abortAndFinalize))
|
---|
| 217 |
|
---|
| 218 | const body = new Minipass()
|
---|
| 219 | // exceedingly rare that the stream would have an error,
|
---|
| 220 | // but just in case we proxy it to the stream in use.
|
---|
| 221 | res.on('error', /* istanbul ignore next */ er => body.emit('error', er))
|
---|
| 222 | res.on('data', (chunk) => body.write(chunk))
|
---|
| 223 | res.on('end', () => body.end())
|
---|
| 224 |
|
---|
| 225 | const responseOptions = {
|
---|
| 226 | url: request.url,
|
---|
| 227 | status: res.statusCode,
|
---|
| 228 | statusText: res.statusMessage,
|
---|
| 229 | headers: headers,
|
---|
| 230 | size: request.size,
|
---|
| 231 | timeout: request.timeout,
|
---|
| 232 | counter: request.counter,
|
---|
| 233 | trailer: new Promise(resolve =>
|
---|
| 234 | res.on('end', () => resolve(createHeadersLenient(res.trailers))))
|
---|
| 235 | }
|
---|
| 236 |
|
---|
| 237 | // HTTP-network fetch step 12.1.1.3
|
---|
| 238 | const codings = headers.get('Content-Encoding')
|
---|
| 239 |
|
---|
| 240 | // HTTP-network fetch step 12.1.1.4: handle content codings
|
---|
| 241 |
|
---|
| 242 | // in following scenarios we ignore compression support
|
---|
| 243 | // 1. compression support is disabled
|
---|
| 244 | // 2. HEAD request
|
---|
| 245 | // 3. no Content-Encoding header
|
---|
| 246 | // 4. no content response (204)
|
---|
| 247 | // 5. content not modified response (304)
|
---|
| 248 | if (!request.compress ||
|
---|
| 249 | request.method === 'HEAD' ||
|
---|
| 250 | codings === null ||
|
---|
| 251 | res.statusCode === 204 ||
|
---|
| 252 | res.statusCode === 304) {
|
---|
| 253 | response = new Response(body, responseOptions)
|
---|
| 254 | resolve(response)
|
---|
| 255 | return
|
---|
| 256 | }
|
---|
| 257 |
|
---|
| 258 |
|
---|
| 259 | // Be less strict when decoding compressed responses, since sometimes
|
---|
| 260 | // servers send slightly invalid responses that are still accepted
|
---|
| 261 | // by common browsers.
|
---|
| 262 | // Always using Z_SYNC_FLUSH is what cURL does.
|
---|
| 263 | const zlibOptions = {
|
---|
| 264 | flush: zlib.constants.Z_SYNC_FLUSH,
|
---|
| 265 | finishFlush: zlib.constants.Z_SYNC_FLUSH,
|
---|
| 266 | }
|
---|
| 267 |
|
---|
| 268 | // for gzip
|
---|
| 269 | if (codings == 'gzip' || codings == 'x-gzip') {
|
---|
| 270 | const unzip = new zlib.Gunzip(zlibOptions)
|
---|
| 271 | response = new Response(
|
---|
| 272 | // exceedingly rare that the stream would have an error,
|
---|
| 273 | // but just in case we proxy it to the stream in use.
|
---|
| 274 | body.on('error', /* istanbul ignore next */ er => unzip.emit('error', er)).pipe(unzip),
|
---|
| 275 | responseOptions
|
---|
| 276 | )
|
---|
| 277 | resolve(response)
|
---|
| 278 | return
|
---|
| 279 | }
|
---|
| 280 |
|
---|
| 281 | // for deflate
|
---|
| 282 | if (codings == 'deflate' || codings == 'x-deflate') {
|
---|
| 283 | // handle the infamous raw deflate response from old servers
|
---|
| 284 | // a hack for old IIS and Apache servers
|
---|
| 285 | const raw = res.pipe(new Minipass())
|
---|
| 286 | raw.once('data', chunk => {
|
---|
| 287 | // see http://stackoverflow.com/questions/37519828
|
---|
| 288 | const decoder = (chunk[0] & 0x0F) === 0x08
|
---|
| 289 | ? new zlib.Inflate()
|
---|
| 290 | : new zlib.InflateRaw()
|
---|
| 291 | // exceedingly rare that the stream would have an error,
|
---|
| 292 | // but just in case we proxy it to the stream in use.
|
---|
| 293 | body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
|
---|
| 294 | response = new Response(decoder, responseOptions)
|
---|
| 295 | resolve(response)
|
---|
| 296 | })
|
---|
| 297 | return
|
---|
| 298 | }
|
---|
| 299 |
|
---|
| 300 |
|
---|
| 301 | // for br
|
---|
| 302 | if (codings == 'br') {
|
---|
| 303 | // ignoring coverage so tests don't have to fake support (or lack of) for brotli
|
---|
| 304 | // istanbul ignore next
|
---|
| 305 | try {
|
---|
| 306 | var decoder = new zlib.BrotliDecompress()
|
---|
| 307 | } catch (err) {
|
---|
| 308 | reject(err)
|
---|
| 309 | finalize()
|
---|
| 310 | return
|
---|
| 311 | }
|
---|
| 312 | // exceedingly rare that the stream would have an error,
|
---|
| 313 | // but just in case we proxy it to the stream in use.
|
---|
| 314 | body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
|
---|
| 315 | response = new Response(decoder, responseOptions)
|
---|
| 316 | resolve(response)
|
---|
| 317 | return
|
---|
| 318 | }
|
---|
| 319 |
|
---|
| 320 | // otherwise, use response as-is
|
---|
| 321 | response = new Response(body, responseOptions)
|
---|
| 322 | resolve(response)
|
---|
| 323 | })
|
---|
| 324 |
|
---|
| 325 | writeToStream(req, request)
|
---|
| 326 | })
|
---|
| 327 | }
|
---|
| 328 |
|
---|
| 329 | module.exports = fetch
|
---|
| 330 |
|
---|
| 331 | fetch.isRedirect = code =>
|
---|
| 332 | code === 301 ||
|
---|
| 333 | code === 302 ||
|
---|
| 334 | code === 303 ||
|
---|
| 335 | code === 307 ||
|
---|
| 336 | code === 308
|
---|
| 337 |
|
---|
| 338 | fetch.Headers = Headers
|
---|
| 339 | fetch.Request = Request
|
---|
| 340 | fetch.Response = Response
|
---|
| 341 | fetch.FetchError = FetchError
|
---|