1 | //////////////////////////////////////////
|
---|
2 | // Needle -- HTTP Client for Node.js
|
---|
3 | // Written by Tomás Pollak <tomas@forkhq.com>
|
---|
4 | // (c) 2012-2020 - Fork Ltd.
|
---|
5 | // MIT Licensed
|
---|
6 | //////////////////////////////////////////
|
---|
7 |
|
---|
8 | var fs = require('fs'),
|
---|
9 | http = require('http'),
|
---|
10 | https = require('https'),
|
---|
11 | url = require('url'),
|
---|
12 | stream = require('stream'),
|
---|
13 | debug = require('debug')('needle'),
|
---|
14 | stringify = require('./querystring').build,
|
---|
15 | multipart = require('./multipart'),
|
---|
16 | auth = require('./auth'),
|
---|
17 | cookies = require('./cookies'),
|
---|
18 | parsers = require('./parsers'),
|
---|
19 | decoder = require('./decoder');
|
---|
20 |
|
---|
21 | //////////////////////////////////////////
|
---|
22 | // variabilia
|
---|
23 |
|
---|
24 | var version = require('../package.json').version;
|
---|
25 |
|
---|
26 | var user_agent = 'Needle/' + version;
|
---|
27 | user_agent += ' (Node.js ' + process.version + '; ' + process.platform + ' ' + process.arch + ')';
|
---|
28 |
|
---|
29 | var tls_options = 'agent pfx key passphrase cert ca ciphers rejectUnauthorized secureProtocol checkServerIdentity family';
|
---|
30 |
|
---|
31 | // older versions of node (< 0.11.4) prevent the runtime from exiting
|
---|
32 | // because of connections in keep-alive state. so if this is the case
|
---|
33 | // we'll default new requests to set a Connection: close header.
|
---|
34 | var close_by_default = !http.Agent || http.Agent.defaultMaxSockets != Infinity;
|
---|
35 |
|
---|
36 | // see if we have Object.assign. otherwise fall back to util._extend
|
---|
37 | var extend = Object.assign ? Object.assign : require('util')._extend;
|
---|
38 |
|
---|
39 | // these are the status codes that Needle interprets as redirects.
|
---|
40 | var redirect_codes = [301, 302, 303, 307, 308];
|
---|
41 |
|
---|
42 | //////////////////////////////////////////
|
---|
43 | // decompressors for gzip/deflate/br bodies
|
---|
44 |
|
---|
45 | function bind_opts(fn, options) {
|
---|
46 | return fn.bind(null, options);
|
---|
47 | }
|
---|
48 |
|
---|
49 | var decompressors = {};
|
---|
50 |
|
---|
51 | try {
|
---|
52 |
|
---|
53 | var zlib = require('zlib');
|
---|
54 |
|
---|
55 | // Enable Z_SYNC_FLUSH to avoid Z_BUF_ERROR errors (Node PR #2595)
|
---|
56 | var zlib_options = {
|
---|
57 | flush: zlib.Z_SYNC_FLUSH,
|
---|
58 | finishFlush: zlib.Z_SYNC_FLUSH
|
---|
59 | };
|
---|
60 |
|
---|
61 | var br_options = {
|
---|
62 | flush: zlib.BROTLI_OPERATION_FLUSH,
|
---|
63 | finishFlush: zlib.BROTLI_OPERATION_FLUSH
|
---|
64 | };
|
---|
65 |
|
---|
66 | decompressors['x-deflate'] = bind_opts(zlib.Inflate, zlib_options);
|
---|
67 | decompressors['deflate'] = bind_opts(zlib.Inflate, zlib_options);
|
---|
68 | decompressors['x-gzip'] = bind_opts(zlib.Gunzip, zlib_options);
|
---|
69 | decompressors['gzip'] = bind_opts(zlib.Gunzip, zlib_options);
|
---|
70 | if (typeof zlib.BrotliDecompress === 'function') {
|
---|
71 | decompressors['br'] = bind_opts(zlib.BrotliDecompress, br_options);
|
---|
72 | }
|
---|
73 |
|
---|
74 | } catch(e) { /* zlib not available */ }
|
---|
75 |
|
---|
76 | //////////////////////////////////////////
|
---|
77 | // options and aliases
|
---|
78 |
|
---|
79 | var defaults = {
|
---|
80 | // data
|
---|
81 | boundary : '--------------------NODENEEDLEHTTPCLIENT',
|
---|
82 | encoding : 'utf8',
|
---|
83 | parse_response : 'all', // same as true. valid options: 'json', 'xml' or false/null
|
---|
84 | proxy : null,
|
---|
85 |
|
---|
86 | // headers
|
---|
87 | headers : {},
|
---|
88 | accept : '*/*',
|
---|
89 | user_agent : user_agent,
|
---|
90 |
|
---|
91 | // numbers
|
---|
92 | open_timeout : 10000,
|
---|
93 | response_timeout : 0,
|
---|
94 | read_timeout : 0,
|
---|
95 | follow_max : 0,
|
---|
96 | stream_length : -1,
|
---|
97 |
|
---|
98 | // booleans
|
---|
99 | compressed : false,
|
---|
100 | decode_response : true,
|
---|
101 | parse_cookies : true,
|
---|
102 | follow_set_cookies : false,
|
---|
103 | follow_set_referer : false,
|
---|
104 | follow_keep_method : false,
|
---|
105 | follow_if_same_host : false,
|
---|
106 | follow_if_same_protocol : false,
|
---|
107 | follow_if_same_location : false
|
---|
108 | }
|
---|
109 |
|
---|
110 | var aliased = {
|
---|
111 | options: {
|
---|
112 | decode : 'decode_response',
|
---|
113 | parse : 'parse_response',
|
---|
114 | timeout : 'open_timeout',
|
---|
115 | follow : 'follow_max'
|
---|
116 | },
|
---|
117 | inverted: {}
|
---|
118 | }
|
---|
119 |
|
---|
120 | // only once, invert aliased keys so we can get passed options.
|
---|
121 | Object.keys(aliased.options).map(function(k) {
|
---|
122 | var value = aliased.options[k];
|
---|
123 | aliased.inverted[value] = k;
|
---|
124 | });
|
---|
125 |
|
---|
126 | //////////////////////////////////////////
|
---|
127 | // helpers
|
---|
128 |
|
---|
129 | function keys_by_type(type) {
|
---|
130 | return Object.keys(defaults).map(function(el) {
|
---|
131 | if (defaults[el] !== null && defaults[el].constructor == type)
|
---|
132 | return el;
|
---|
133 | }).filter(function(el) { return el })
|
---|
134 | }
|
---|
135 |
|
---|
136 | function parse_content_type(header) {
|
---|
137 | if (!header || header === '') return {};
|
---|
138 |
|
---|
139 | var found, charset = 'utf8', arr = header.split(';');
|
---|
140 |
|
---|
141 | if (arr.length > 1 && (found = arr[1].match(/charset=(.+)/)))
|
---|
142 | charset = found[1];
|
---|
143 |
|
---|
144 | return { type: arr[0], charset: charset };
|
---|
145 | }
|
---|
146 |
|
---|
147 | function is_stream(obj) {
|
---|
148 | return typeof obj.pipe === 'function';
|
---|
149 | }
|
---|
150 |
|
---|
151 | function get_stream_length(stream, given_length, cb) {
|
---|
152 | if (given_length > 0)
|
---|
153 | return cb(given_length);
|
---|
154 |
|
---|
155 | if (stream.end !== void 0 && stream.end !== Infinity && stream.start !== void 0)
|
---|
156 | return cb((stream.end + 1) - (stream.start || 0));
|
---|
157 |
|
---|
158 | fs.stat(stream.path, function(err, stat) {
|
---|
159 | cb(stat ? stat.size - (stream.start || 0) : null);
|
---|
160 | });
|
---|
161 | }
|
---|
162 |
|
---|
163 | function resolve_url(href, base) {
|
---|
164 | if (url.URL)
|
---|
165 | return new url.URL(href, base);
|
---|
166 |
|
---|
167 | // older Node version (< v6.13)
|
---|
168 | return url.resolve(base, href);
|
---|
169 | }
|
---|
170 |
|
---|
171 | function pump_streams(streams, cb) {
|
---|
172 | if (stream.pipeline)
|
---|
173 | return stream.pipeline.apply(null, streams.concat(cb));
|
---|
174 |
|
---|
175 | var tmp = streams.shift();
|
---|
176 | while (streams.length) {
|
---|
177 | tmp = tmp.pipe(streams.shift());
|
---|
178 | tmp.once('error', function(e) {
|
---|
179 | cb && cb(e);
|
---|
180 | cb = null;
|
---|
181 | })
|
---|
182 | }
|
---|
183 | }
|
---|
184 |
|
---|
185 | //////////////////////////////////////////
|
---|
186 | // the main act
|
---|
187 |
|
---|
188 | function Needle(method, uri, data, options, callback) {
|
---|
189 | // if (!(this instanceof Needle)) {
|
---|
190 | // return new Needle(method, uri, data, options, callback);
|
---|
191 | // }
|
---|
192 |
|
---|
193 | if (typeof uri !== 'string')
|
---|
194 | throw new TypeError('URL must be a string, not ' + uri);
|
---|
195 |
|
---|
196 | this.method = method.toLowerCase();
|
---|
197 | this.uri = uri;
|
---|
198 | this.data = data;
|
---|
199 |
|
---|
200 | if (typeof options == 'function') {
|
---|
201 | this.callback = options;
|
---|
202 | this.options = {};
|
---|
203 | } else {
|
---|
204 | this.callback = callback;
|
---|
205 | this.options = options;
|
---|
206 | }
|
---|
207 |
|
---|
208 | }
|
---|
209 |
|
---|
210 | Needle.prototype.setup = function(uri, options) {
|
---|
211 |
|
---|
212 | function get_option(key, fallback) {
|
---|
213 | // if original is in options, return that value
|
---|
214 | if (typeof options[key] != 'undefined') return options[key];
|
---|
215 |
|
---|
216 | // otherwise, return value from alias or fallback/undefined
|
---|
217 | return typeof options[aliased.inverted[key]] != 'undefined'
|
---|
218 | ? options[aliased.inverted[key]] : fallback;
|
---|
219 | }
|
---|
220 |
|
---|
221 | function check_value(expected, key) {
|
---|
222 | var value = get_option(key),
|
---|
223 | type = typeof value;
|
---|
224 |
|
---|
225 | if (type != 'undefined' && type != expected)
|
---|
226 | throw new TypeError(type + ' received for ' + key + ', but expected a ' + expected);
|
---|
227 |
|
---|
228 | return (type == expected) ? value : defaults[key];
|
---|
229 | }
|
---|
230 |
|
---|
231 | //////////////////////////////////////////////////
|
---|
232 | // the basics
|
---|
233 |
|
---|
234 | var config = {
|
---|
235 | http_opts : {
|
---|
236 | localAddress: get_option('localAddress', undefined),
|
---|
237 | lookup: get_option('lookup', undefined)
|
---|
238 | }, // passed later to http.request() directly
|
---|
239 | headers : {},
|
---|
240 | output : options.output,
|
---|
241 | proxy : get_option('proxy', defaults.proxy),
|
---|
242 | parser : get_option('parse_response', defaults.parse_response),
|
---|
243 | encoding : options.encoding || (options.multipart ? 'binary' : defaults.encoding)
|
---|
244 | }
|
---|
245 |
|
---|
246 | keys_by_type(Boolean).forEach(function(key) {
|
---|
247 | config[key] = check_value('boolean', key);
|
---|
248 | })
|
---|
249 |
|
---|
250 | keys_by_type(Number).forEach(function(key) {
|
---|
251 | config[key] = check_value('number', key);
|
---|
252 | })
|
---|
253 |
|
---|
254 | // populate http_opts with given TLS options
|
---|
255 | tls_options.split(' ').forEach(function(key) {
|
---|
256 | if (typeof options[key] != 'undefined') {
|
---|
257 | config.http_opts[key] = options[key];
|
---|
258 | if (typeof options.agent == 'undefined')
|
---|
259 | config.http_opts.agent = false; // otherwise tls options are skipped
|
---|
260 | }
|
---|
261 | });
|
---|
262 |
|
---|
263 | //////////////////////////////////////////////////
|
---|
264 | // headers, cookies
|
---|
265 |
|
---|
266 | for (var key in defaults.headers)
|
---|
267 | config.headers[key] = defaults.headers[key];
|
---|
268 |
|
---|
269 | config.headers['accept'] = options.accept || defaults.accept;
|
---|
270 | config.headers['user-agent'] = options.user_agent || defaults.user_agent;
|
---|
271 |
|
---|
272 | if (options.content_type)
|
---|
273 | config.headers['content-type'] = options.content_type;
|
---|
274 |
|
---|
275 | // set connection header if opts.connection was passed, or if node < 0.11.4 (close)
|
---|
276 | if (options.connection || close_by_default)
|
---|
277 | config.headers['connection'] = options.connection || 'close';
|
---|
278 |
|
---|
279 | if ((options.compressed || defaults.compressed) && typeof zlib != 'undefined')
|
---|
280 | config.headers['accept-encoding'] = decompressors['br'] ? 'gzip, deflate, br' : 'gzip, deflate';
|
---|
281 |
|
---|
282 | if (options.cookies)
|
---|
283 | config.headers['cookie'] = cookies.write(options.cookies);
|
---|
284 |
|
---|
285 | //////////////////////////////////////////////////
|
---|
286 | // basic/digest auth
|
---|
287 |
|
---|
288 | if (uri.match(/[^\/]@/)) { // url contains user:pass@host, so parse it.
|
---|
289 | var parts = (url.parse(uri).auth || '').split(':');
|
---|
290 | options.username = parts[0];
|
---|
291 | options.password = parts[1];
|
---|
292 | }
|
---|
293 |
|
---|
294 | if (options.username) {
|
---|
295 | if (options.auth && (options.auth == 'auto' || options.auth == 'digest')) {
|
---|
296 | config.credentials = [options.username, options.password];
|
---|
297 | } else {
|
---|
298 | config.headers['authorization'] = auth.basic(options.username, options.password);
|
---|
299 | }
|
---|
300 | }
|
---|
301 |
|
---|
302 | // if proxy is present, set auth header from either url or proxy_user option.
|
---|
303 | if (config.proxy) {
|
---|
304 | if (config.proxy.indexOf('http') === -1)
|
---|
305 | config.proxy = 'http://' + config.proxy;
|
---|
306 |
|
---|
307 | if (config.proxy.indexOf('@') !== -1) {
|
---|
308 | var proxy = (url.parse(config.proxy).auth || '').split(':');
|
---|
309 | options.proxy_user = proxy[0];
|
---|
310 | options.proxy_pass = proxy[1];
|
---|
311 | }
|
---|
312 |
|
---|
313 | if (options.proxy_user)
|
---|
314 | config.headers['proxy-authorization'] = auth.basic(options.proxy_user, options.proxy_pass);
|
---|
315 | }
|
---|
316 |
|
---|
317 | // now that all our headers are set, overwrite them if instructed.
|
---|
318 | for (var h in options.headers)
|
---|
319 | config.headers[h.toLowerCase()] = options.headers[h];
|
---|
320 |
|
---|
321 | config.uri_modifier = get_option('uri_modifier', null);
|
---|
322 |
|
---|
323 | return config;
|
---|
324 | }
|
---|
325 |
|
---|
326 | Needle.prototype.start = function() {
|
---|
327 |
|
---|
328 | var out = new stream.PassThrough({ objectMode: false }),
|
---|
329 | uri = this.uri,
|
---|
330 | data = this.data,
|
---|
331 | method = this.method,
|
---|
332 | callback = (typeof this.options == 'function') ? this.options : this.callback,
|
---|
333 | options = this.options || {};
|
---|
334 |
|
---|
335 | // if no 'http' is found on URL, prepend it.
|
---|
336 | if (uri.indexOf('http') === -1)
|
---|
337 | uri = uri.replace(/^(\/\/)?/, 'http://');
|
---|
338 |
|
---|
339 | var self = this, body, waiting = false, config = this.setup(uri, options);
|
---|
340 |
|
---|
341 | // unless options.json was set to false, assume boss also wants JSON if content-type matches.
|
---|
342 | var json = options.json || (options.json !== false && config.headers['content-type'] == 'application/json');
|
---|
343 |
|
---|
344 | if (data) {
|
---|
345 |
|
---|
346 | if (options.multipart) { // boss says we do multipart. so we do it.
|
---|
347 | var boundary = options.boundary || defaults.boundary;
|
---|
348 |
|
---|
349 | waiting = true;
|
---|
350 | multipart.build(data, boundary, function(err, parts) {
|
---|
351 | if (err) throw(err);
|
---|
352 |
|
---|
353 | config.headers['content-type'] = 'multipart/form-data; boundary=' + boundary;
|
---|
354 | next(parts);
|
---|
355 | });
|
---|
356 |
|
---|
357 | } else if (is_stream(data)) {
|
---|
358 |
|
---|
359 | if (method == 'get')
|
---|
360 | throw new Error('Refusing to pipe() a stream via GET. Did you mean .post?');
|
---|
361 |
|
---|
362 | if (config.stream_length > 0 || (config.stream_length === 0 && data.path)) {
|
---|
363 | // ok, let's get the stream's length and set it as the content-length header.
|
---|
364 | // this prevents some servers from cutting us off before all the data is sent.
|
---|
365 | waiting = true;
|
---|
366 | get_stream_length(data, config.stream_length, function(length) {
|
---|
367 | data.length = length;
|
---|
368 | next(data);
|
---|
369 | })
|
---|
370 |
|
---|
371 | } else {
|
---|
372 | // if the boss doesn't want us to get the stream's length, or if it doesn't
|
---|
373 | // have a file descriptor for that purpose, then just head on.
|
---|
374 | body = data;
|
---|
375 | }
|
---|
376 |
|
---|
377 | } else if (Buffer.isBuffer(data)) {
|
---|
378 |
|
---|
379 | body = data; // use the raw buffer as request body.
|
---|
380 |
|
---|
381 | } else if (method == 'get' && !json) {
|
---|
382 |
|
---|
383 | // append the data to the URI as a querystring.
|
---|
384 | uri = uri.replace(/\?.*|$/, '?' + stringify(data));
|
---|
385 |
|
---|
386 | } else { // string or object data, no multipart.
|
---|
387 |
|
---|
388 | // if string, leave it as it is, otherwise, stringify.
|
---|
389 | body = (typeof(data) === 'string') ? data
|
---|
390 | : json ? JSON.stringify(data) : stringify(data);
|
---|
391 |
|
---|
392 | // ensure we have a buffer so bytecount is correct.
|
---|
393 | body = Buffer.from(body, config.encoding);
|
---|
394 | }
|
---|
395 |
|
---|
396 | }
|
---|
397 |
|
---|
398 | function next(body) {
|
---|
399 | if (body) {
|
---|
400 | if (body.length) config.headers['content-length'] = body.length;
|
---|
401 |
|
---|
402 | // if no content-type was passed, determine if json or not.
|
---|
403 | if (!config.headers['content-type']) {
|
---|
404 | config.headers['content-type'] = json
|
---|
405 | ? 'application/json; charset=utf-8'
|
---|
406 | : 'application/x-www-form-urlencoded'; // no charset says W3 spec.
|
---|
407 | }
|
---|
408 | }
|
---|
409 |
|
---|
410 | // unless a specific accept header was set, assume json: true wants JSON back.
|
---|
411 | if (options.json && (!options.accept && !(options.headers || {}).accept))
|
---|
412 | config.headers['accept'] = 'application/json';
|
---|
413 |
|
---|
414 | self.send_request(1, method, uri, config, body, out, callback);
|
---|
415 | }
|
---|
416 |
|
---|
417 | if (!waiting) next(body);
|
---|
418 | return out;
|
---|
419 | }
|
---|
420 |
|
---|
421 | Needle.prototype.get_request_opts = function(method, uri, config) {
|
---|
422 | var opts = config.http_opts,
|
---|
423 | proxy = config.proxy,
|
---|
424 | remote = proxy ? url.parse(proxy) : url.parse(uri);
|
---|
425 |
|
---|
426 | opts.protocol = remote.protocol;
|
---|
427 | opts.host = remote.hostname;
|
---|
428 | opts.port = remote.port || (remote.protocol == 'https:' ? 443 : 80);
|
---|
429 | opts.path = proxy ? uri : remote.pathname + (remote.search || '');
|
---|
430 | opts.method = method;
|
---|
431 | opts.headers = config.headers;
|
---|
432 |
|
---|
433 | if (!opts.headers['host']) {
|
---|
434 | // if using proxy, make sure the host header shows the final destination
|
---|
435 | var target = proxy ? url.parse(uri) : remote;
|
---|
436 | opts.headers['host'] = target.hostname;
|
---|
437 |
|
---|
438 | // and if a non standard port was passed, append it to the port header
|
---|
439 | if (target.port && [80, 443].indexOf(target.port) === -1) {
|
---|
440 | opts.headers['host'] += ':' + target.port;
|
---|
441 | }
|
---|
442 | }
|
---|
443 |
|
---|
444 | return opts;
|
---|
445 | }
|
---|
446 |
|
---|
447 | Needle.prototype.should_follow = function(location, config, original) {
|
---|
448 | if (!location) return false;
|
---|
449 |
|
---|
450 | // returns true if location contains matching property (host or protocol)
|
---|
451 | function matches(property) {
|
---|
452 | var property = original[property];
|
---|
453 | return location.indexOf(property) !== -1;
|
---|
454 | }
|
---|
455 |
|
---|
456 | // first, check whether the requested location is actually different from the original
|
---|
457 | if (!config.follow_if_same_location && location === original)
|
---|
458 | return false;
|
---|
459 |
|
---|
460 | if (config.follow_if_same_host && !matches('host'))
|
---|
461 | return false; // host does not match, so not following
|
---|
462 |
|
---|
463 | if (config.follow_if_same_protocol && !matches('protocol'))
|
---|
464 | return false; // procotol does not match, so not following
|
---|
465 |
|
---|
466 | return true;
|
---|
467 | }
|
---|
468 |
|
---|
469 | Needle.prototype.send_request = function(count, method, uri, config, post_data, out, callback) {
|
---|
470 |
|
---|
471 | if (typeof config.uri_modifier === 'function') {
|
---|
472 | var modified_uri = config.uri_modifier(uri);
|
---|
473 | debug('Modifying request URI', uri + ' => ' + modified_uri);
|
---|
474 | uri = modified_uri;
|
---|
475 | }
|
---|
476 |
|
---|
477 | var request,
|
---|
478 | timer,
|
---|
479 | returned = 0,
|
---|
480 | self = this,
|
---|
481 | request_opts = this.get_request_opts(method, uri, config),
|
---|
482 | protocol = request_opts.protocol == 'https:' ? https : http;
|
---|
483 |
|
---|
484 | function done(err, resp) {
|
---|
485 | if (returned++ > 0)
|
---|
486 | return debug('Already finished, stopping here.');
|
---|
487 |
|
---|
488 | if (timer) clearTimeout(timer);
|
---|
489 | request.removeListener('error', had_error);
|
---|
490 | out.done = true;
|
---|
491 |
|
---|
492 | if (callback)
|
---|
493 | return callback(err, resp, resp ? resp.body : undefined);
|
---|
494 |
|
---|
495 | // NOTE: this event used to be called 'end', but the behaviour was confusing
|
---|
496 | // when errors ocurred, because the stream would still emit an 'end' event.
|
---|
497 | out.emit('done', err);
|
---|
498 |
|
---|
499 | // trigger the 'done' event on streams we're being piped to, if any
|
---|
500 | var pipes = out._readableState.pipes || [];
|
---|
501 | if (!pipes.forEach) pipes = [pipes];
|
---|
502 | pipes.forEach(function(st) { st.emit('done', err); })
|
---|
503 | }
|
---|
504 |
|
---|
505 | function had_error(err) {
|
---|
506 | debug('Request error', err);
|
---|
507 | out.emit('err', err);
|
---|
508 | done(err || new Error('Unknown error when making request.'));
|
---|
509 | }
|
---|
510 |
|
---|
511 | function set_timeout(type, milisecs) {
|
---|
512 | if (timer) clearTimeout(timer);
|
---|
513 | if (milisecs <= 0) return;
|
---|
514 |
|
---|
515 | timer = setTimeout(function() {
|
---|
516 | out.emit('timeout', type);
|
---|
517 | request.abort();
|
---|
518 | // also invoke done() to terminate job on read_timeout
|
---|
519 | if (type == 'read') done(new Error(type + ' timeout'));
|
---|
520 | }, milisecs);
|
---|
521 | }
|
---|
522 |
|
---|
523 | // handle errors on the underlying socket, that may be closed while writing
|
---|
524 | // for an example case, see test/long_string_spec.js. we make sure this
|
---|
525 | // scenario ocurred by verifying the socket's writable & destroyed states.
|
---|
526 | function on_socket_end() {
|
---|
527 | if (returned && !this.writable && this.destroyed === false) {
|
---|
528 | this.destroy();
|
---|
529 | had_error(new Error('Remote end closed socket abruptly.'))
|
---|
530 | }
|
---|
531 | }
|
---|
532 |
|
---|
533 | debug('Making request #' + count, request_opts);
|
---|
534 | request = protocol.request(request_opts, function(resp) {
|
---|
535 |
|
---|
536 | var headers = resp.headers;
|
---|
537 | debug('Got response', resp.statusCode, headers);
|
---|
538 | out.emit('response', resp);
|
---|
539 |
|
---|
540 | set_timeout('read', config.read_timeout);
|
---|
541 |
|
---|
542 | // if we got cookies, parse them unless we were instructed not to. make sure to include any
|
---|
543 | // cookies that might have been set on previous redirects.
|
---|
544 | if (config.parse_cookies && (headers['set-cookie'] || config.previous_resp_cookies)) {
|
---|
545 | resp.cookies = extend(config.previous_resp_cookies || {}, cookies.read(headers['set-cookie']));
|
---|
546 | debug('Got cookies', resp.cookies);
|
---|
547 | }
|
---|
548 |
|
---|
549 | // if redirect code is found, determine if we should follow it according to the given options.
|
---|
550 | if (redirect_codes.indexOf(resp.statusCode) !== -1 && self.should_follow(headers.location, config, uri)) {
|
---|
551 | // clear timer before following redirects to prevent unexpected setTimeout consequence
|
---|
552 | clearTimeout(timer);
|
---|
553 |
|
---|
554 | if (count <= config.follow_max) {
|
---|
555 | out.emit('redirect', headers.location);
|
---|
556 |
|
---|
557 | // unless 'follow_keep_method' is true, rewrite the request to GET before continuing.
|
---|
558 | if (!config.follow_keep_method) {
|
---|
559 | method = 'GET';
|
---|
560 | post_data = null;
|
---|
561 | delete config.headers['content-length']; // in case the original was a multipart POST request.
|
---|
562 | }
|
---|
563 |
|
---|
564 | // if follow_set_cookies is true, insert cookies in the next request's headers.
|
---|
565 | // we set both the original request cookies plus any response cookies we might have received.
|
---|
566 | if (config.follow_set_cookies) {
|
---|
567 | var request_cookies = cookies.read(config.headers['cookie']);
|
---|
568 | config.previous_resp_cookies = resp.cookies;
|
---|
569 | if (Object.keys(request_cookies).length || Object.keys(resp.cookies || {}).length) {
|
---|
570 | config.headers['cookie'] = cookies.write(extend(request_cookies, resp.cookies));
|
---|
571 | }
|
---|
572 | } else if (config.headers['cookie']) {
|
---|
573 | debug('Clearing original request cookie', config.headers['cookie']);
|
---|
574 | delete config.headers['cookie'];
|
---|
575 | }
|
---|
576 |
|
---|
577 | if (config.follow_set_referer)
|
---|
578 | config.headers['referer'] = encodeURI(uri); // the original, not the destination URL.
|
---|
579 |
|
---|
580 | config.headers['host'] = null; // clear previous Host header to avoid conflicts.
|
---|
581 |
|
---|
582 | var redirect_url = resolve_url(headers.location, uri);
|
---|
583 | debug('Redirecting to ' + redirect_url.toString());
|
---|
584 | return self.send_request(++count, method, redirect_url.toString(), config, post_data, out, callback);
|
---|
585 | } else if (config.follow_max > 0) {
|
---|
586 | return done(new Error('Max redirects reached. Possible loop in: ' + headers.location));
|
---|
587 | }
|
---|
588 | }
|
---|
589 |
|
---|
590 | // if auth is requested and credentials were not passed, resend request, provided we have user/pass.
|
---|
591 | if (resp.statusCode == 401 && headers['www-authenticate'] && config.credentials) {
|
---|
592 | if (!config.headers['authorization']) { // only if authentication hasn't been sent
|
---|
593 | var auth_header = auth.header(headers['www-authenticate'], config.credentials, request_opts);
|
---|
594 |
|
---|
595 | if (auth_header) {
|
---|
596 | config.headers['authorization'] = auth_header;
|
---|
597 | return self.send_request(count, method, uri, config, post_data, out, callback);
|
---|
598 | }
|
---|
599 | }
|
---|
600 | }
|
---|
601 |
|
---|
602 | // ok, so we got a valid (non-redirect & authorized) response. let's notify the stream guys.
|
---|
603 | out.emit('header', resp.statusCode, headers);
|
---|
604 | out.emit('headers', headers);
|
---|
605 |
|
---|
606 | var pipeline = [],
|
---|
607 | mime = parse_content_type(headers['content-type']),
|
---|
608 | text_response = mime.type && (mime.type.indexOf('text/') != -1 || !!mime.type.match(/(\/|\+)(xml|json)$/));
|
---|
609 |
|
---|
610 | // To start, if our body is compressed and we're able to inflate it, do it.
|
---|
611 | if (headers['content-encoding'] && decompressors[headers['content-encoding']]) {
|
---|
612 |
|
---|
613 | var decompressor = decompressors[headers['content-encoding']]();
|
---|
614 |
|
---|
615 | // make sure we catch errors triggered by the decompressor.
|
---|
616 | decompressor.on('error', had_error);
|
---|
617 | pipeline.push(decompressor);
|
---|
618 | }
|
---|
619 |
|
---|
620 | // If parse is enabled and we have a parser for it, then go for it.
|
---|
621 | if (config.parser && parsers[mime.type]) {
|
---|
622 |
|
---|
623 | // If a specific parser was requested, make sure we don't parse other types.
|
---|
624 | var parser_name = config.parser.toString().toLowerCase();
|
---|
625 | if (['xml', 'json'].indexOf(parser_name) == -1 || parsers[mime.type].name == parser_name) {
|
---|
626 |
|
---|
627 | // OK, so either we're parsing all content types or the one requested matches.
|
---|
628 | out.parser = parsers[mime.type].name;
|
---|
629 | pipeline.push(parsers[mime.type].fn());
|
---|
630 |
|
---|
631 | // Set objectMode on out stream to improve performance.
|
---|
632 | out._writableState.objectMode = true;
|
---|
633 | out._readableState.objectMode = true;
|
---|
634 | }
|
---|
635 |
|
---|
636 | // If we're not parsing, and unless decoding was disabled, we'll try
|
---|
637 | // decoding non UTF-8 bodies to UTF-8, using the iconv-lite library.
|
---|
638 | } else if (text_response && config.decode_response && mime.charset) {
|
---|
639 | pipeline.push(decoder(mime.charset));
|
---|
640 | }
|
---|
641 |
|
---|
642 | // And `out` is the stream we finally push the decoded/parsed output to.
|
---|
643 | pipeline.push(out);
|
---|
644 |
|
---|
645 | // Now, release the kraken!
|
---|
646 | pump_streams([resp].concat(pipeline), function(err) {
|
---|
647 | if (err) debug(err)
|
---|
648 |
|
---|
649 | // on node v8.x, if an error ocurrs on the receiving end,
|
---|
650 | // then we want to abort the request to avoid having dangling sockets
|
---|
651 | if (err && err.message == 'write after end') request.destroy();
|
---|
652 | });
|
---|
653 |
|
---|
654 | // If the user has requested and output file, pipe the output stream to it.
|
---|
655 | // In stream mode, we will still get the response stream to play with.
|
---|
656 | if (config.output && resp.statusCode == 200) {
|
---|
657 |
|
---|
658 | // for some reason, simply piping resp to the writable stream doesn't
|
---|
659 | // work all the time (stream gets cut in the middle with no warning).
|
---|
660 | // so we'll manually need to do the readable/write(chunk) trick.
|
---|
661 | var file = fs.createWriteStream(config.output);
|
---|
662 | file.on('error', had_error);
|
---|
663 |
|
---|
664 | out.on('end', function() {
|
---|
665 | if (file.writable) file.end();
|
---|
666 | });
|
---|
667 |
|
---|
668 | file.on('close', function() {
|
---|
669 | delete out.file;
|
---|
670 | })
|
---|
671 |
|
---|
672 | out.on('readable', function() {
|
---|
673 | var chunk;
|
---|
674 | while ((chunk = this.read()) !== null) {
|
---|
675 | if (file.writable) file.write(chunk);
|
---|
676 |
|
---|
677 | // if callback was requested, also push it to resp.body
|
---|
678 | if (resp.body) resp.body.push(chunk);
|
---|
679 | }
|
---|
680 | })
|
---|
681 |
|
---|
682 | out.file = file;
|
---|
683 | }
|
---|
684 |
|
---|
685 | // Only aggregate the full body if a callback was requested.
|
---|
686 | if (callback) {
|
---|
687 | resp.raw = [];
|
---|
688 | resp.body = [];
|
---|
689 | resp.bytes = 0;
|
---|
690 |
|
---|
691 | // Gather and count the amount of (raw) bytes using a PassThrough stream.
|
---|
692 | var clean_pipe = new stream.PassThrough();
|
---|
693 |
|
---|
694 | clean_pipe.on('readable', function() {
|
---|
695 | var chunk;
|
---|
696 | while ((chunk = this.read()) != null) {
|
---|
697 | resp.bytes += chunk.length;
|
---|
698 | resp.raw.push(chunk);
|
---|
699 | }
|
---|
700 | })
|
---|
701 |
|
---|
702 | pump_streams([resp, clean_pipe], function(err) {
|
---|
703 | if (err) debug(err);
|
---|
704 | });
|
---|
705 |
|
---|
706 | // Listen on the 'readable' event to aggregate the chunks, but only if
|
---|
707 | // file output wasn't requested. Otherwise we'd have two stream readers.
|
---|
708 | if (!config.output || resp.statusCode != 200) {
|
---|
709 | out.on('readable', function() {
|
---|
710 | var chunk;
|
---|
711 | while ((chunk = this.read()) !== null) {
|
---|
712 | // We're either pushing buffers or objects, never strings.
|
---|
713 | if (typeof chunk == 'string') chunk = Buffer.from(chunk);
|
---|
714 |
|
---|
715 | // Push all chunks to resp.body. We'll bind them in resp.end().
|
---|
716 | resp.body.push(chunk);
|
---|
717 | }
|
---|
718 | })
|
---|
719 | }
|
---|
720 | }
|
---|
721 |
|
---|
722 | // And set the .body property once all data is in.
|
---|
723 | out.on('end', function() {
|
---|
724 | if (resp.body) { // callback mode
|
---|
725 |
|
---|
726 | // we want to be able to access to the raw data later, so keep a reference.
|
---|
727 | resp.raw = Buffer.concat(resp.raw);
|
---|
728 |
|
---|
729 | // if parse was successful, we should have an array with one object
|
---|
730 | if (resp.body[0] !== undefined && !Buffer.isBuffer(resp.body[0])) {
|
---|
731 |
|
---|
732 | // that's our body right there.
|
---|
733 | resp.body = resp.body[0];
|
---|
734 |
|
---|
735 | // set the parser property on our response. we may want to check.
|
---|
736 | if (out.parser) resp.parser = out.parser;
|
---|
737 |
|
---|
738 | } else { // we got one or several buffers. string or binary.
|
---|
739 | resp.body = Buffer.concat(resp.body);
|
---|
740 |
|
---|
741 | // if we're here and parsed is true, it means we tried to but it didn't work.
|
---|
742 | // so given that we got a text response, let's stringify it.
|
---|
743 | if (text_response || out.parser) {
|
---|
744 | resp.body = resp.body.toString();
|
---|
745 | }
|
---|
746 | }
|
---|
747 | }
|
---|
748 |
|
---|
749 | // if an output file is being written to, make sure the callback
|
---|
750 | // is triggered after all data has been written to it.
|
---|
751 | if (out.file) {
|
---|
752 | out.file.on('close', function() {
|
---|
753 | done(null, resp);
|
---|
754 | })
|
---|
755 | } else { // elvis has left the building.
|
---|
756 | done(null, resp);
|
---|
757 | }
|
---|
758 |
|
---|
759 | });
|
---|
760 |
|
---|
761 | // out.on('error', function(err) {
|
---|
762 | // had_error(err);
|
---|
763 | // if (err.code == 'ERR_STREAM_DESTROYED' || err.code == 'ERR_STREAM_PREMATURE_CLOSE') {
|
---|
764 | // request.abort();
|
---|
765 | // }
|
---|
766 | // })
|
---|
767 |
|
---|
768 | }); // end request call
|
---|
769 |
|
---|
770 | // unless open_timeout was disabled, set a timeout to abort the request.
|
---|
771 | set_timeout('open', config.open_timeout);
|
---|
772 |
|
---|
773 | // handle errors on the request object. things might get bumpy.
|
---|
774 | request.on('error', had_error);
|
---|
775 |
|
---|
776 | // make sure timer is cleared if request is aborted (issue #257)
|
---|
777 | request.once('abort', function() {
|
---|
778 | if (timer) clearTimeout(timer);
|
---|
779 | })
|
---|
780 |
|
---|
781 | // handle socket 'end' event to ensure we don't get delayed EPIPE errors.
|
---|
782 | request.once('socket', function(socket) {
|
---|
783 | if (socket.connecting) {
|
---|
784 | socket.once('connect', function() {
|
---|
785 | set_timeout('response', config.response_timeout);
|
---|
786 | })
|
---|
787 | } else {
|
---|
788 | set_timeout('response', config.response_timeout);
|
---|
789 | }
|
---|
790 |
|
---|
791 | // socket.once('close', function(e) {
|
---|
792 | // console.log('socket closed!', e);
|
---|
793 | // })
|
---|
794 |
|
---|
795 | if (!socket.on_socket_end) {
|
---|
796 | socket.on_socket_end = on_socket_end;
|
---|
797 | socket.once('end', function() { process.nextTick(on_socket_end.bind(socket)) });
|
---|
798 | }
|
---|
799 | })
|
---|
800 |
|
---|
801 | if (post_data) {
|
---|
802 | if (is_stream(post_data)) {
|
---|
803 | pump_streams([post_data, request], function(err) {
|
---|
804 | if (err) debug(err);
|
---|
805 | });
|
---|
806 | } else {
|
---|
807 | request.write(post_data, config.encoding);
|
---|
808 | request.end();
|
---|
809 | }
|
---|
810 | } else {
|
---|
811 | request.end();
|
---|
812 | }
|
---|
813 |
|
---|
814 | out.abort = function() { request.abort() }; // easier access
|
---|
815 | out.request = request;
|
---|
816 | return out;
|
---|
817 | }
|
---|
818 |
|
---|
819 | //////////////////////////////////////////
|
---|
820 | // exports
|
---|
821 |
|
---|
822 | if (typeof Promise !== 'undefined') {
|
---|
823 | module.exports = function() {
|
---|
824 | var verb, args = [].slice.call(arguments);
|
---|
825 |
|
---|
826 | if (args[0].match(/\.|\//)) // first argument looks like a URL
|
---|
827 | verb = (args.length > 2) ? 'post' : 'get';
|
---|
828 | else
|
---|
829 | verb = args.shift();
|
---|
830 |
|
---|
831 | if (verb.match(/get|head/i) && args.length == 2)
|
---|
832 | args.splice(1, 0, null); // assume no data if head/get with two args (url, options)
|
---|
833 |
|
---|
834 | return new Promise(function(resolve, reject) {
|
---|
835 | module.exports.request(verb, args[0], args[1], args[2], function(err, resp) {
|
---|
836 | return err ? reject(err) : resolve(resp);
|
---|
837 | });
|
---|
838 | })
|
---|
839 | }
|
---|
840 | }
|
---|
841 |
|
---|
842 | module.exports.version = version;
|
---|
843 |
|
---|
844 | module.exports.defaults = function(obj) {
|
---|
845 | for (var key in obj) {
|
---|
846 | var target_key = aliased.options[key] || key;
|
---|
847 |
|
---|
848 | if (defaults.hasOwnProperty(target_key) && typeof obj[key] != 'undefined') {
|
---|
849 | if (target_key != 'parse_response' && target_key != 'proxy') {
|
---|
850 | // ensure type matches the original, except for proxy/parse_response that can be null/bool or string
|
---|
851 | var valid_type = defaults[target_key].constructor.name;
|
---|
852 |
|
---|
853 | if (obj[key].constructor.name != valid_type)
|
---|
854 | throw new TypeError('Invalid type for ' + key + ', should be ' + valid_type);
|
---|
855 | }
|
---|
856 | defaults[target_key] = obj[key];
|
---|
857 | } else {
|
---|
858 | throw new Error('Invalid property for defaults:' + target_key);
|
---|
859 | }
|
---|
860 | }
|
---|
861 |
|
---|
862 | return defaults;
|
---|
863 | }
|
---|
864 |
|
---|
865 | 'head get'.split(' ').forEach(function(method) {
|
---|
866 | module.exports[method] = function(uri, options, callback) {
|
---|
867 | return new Needle(method, uri, null, options, callback).start();
|
---|
868 | }
|
---|
869 | })
|
---|
870 |
|
---|
871 | 'post put patch delete'.split(' ').forEach(function(method) {
|
---|
872 | module.exports[method] = function(uri, data, options, callback) {
|
---|
873 | return new Needle(method, uri, data, options, callback).start();
|
---|
874 | }
|
---|
875 | })
|
---|
876 |
|
---|
877 | module.exports.request = function(method, uri, data, opts, callback) {
|
---|
878 | return new Needle(method, uri, data, opts, callback).start();
|
---|
879 | };
|
---|