[6a3a178] | 1 | //////////////////////////////////////////
|
---|
| 2 | // Needle -- HTTP Client for Node.js
|
---|
| 3 | // Written by Tomás Pollak <tomas@forkhq.com>
|
---|
| 4 | // (c) 2012-2020 - Fork Ltd.
|
---|
| 5 | // MIT Licensed
|
---|
| 6 | //////////////////////////////////////////
|
---|
| 7 |
|
---|
| 8 | var fs = require('fs'),
|
---|
| 9 | http = require('http'),
|
---|
| 10 | https = require('https'),
|
---|
| 11 | url = require('url'),
|
---|
| 12 | stream = require('stream'),
|
---|
| 13 | debug = require('debug')('needle'),
|
---|
| 14 | stringify = require('./querystring').build,
|
---|
| 15 | multipart = require('./multipart'),
|
---|
| 16 | auth = require('./auth'),
|
---|
| 17 | cookies = require('./cookies'),
|
---|
| 18 | parsers = require('./parsers'),
|
---|
| 19 | decoder = require('./decoder');
|
---|
| 20 |
|
---|
| 21 | //////////////////////////////////////////
|
---|
| 22 | // variabilia
|
---|
| 23 |
|
---|
| 24 | var version = require('../package.json').version;
|
---|
| 25 |
|
---|
| 26 | var user_agent = 'Needle/' + version;
|
---|
| 27 | user_agent += ' (Node.js ' + process.version + '; ' + process.platform + ' ' + process.arch + ')';
|
---|
| 28 |
|
---|
| 29 | var tls_options = 'agent pfx key passphrase cert ca ciphers rejectUnauthorized secureProtocol checkServerIdentity family';
|
---|
| 30 |
|
---|
| 31 | // older versions of node (< 0.11.4) prevent the runtime from exiting
|
---|
| 32 | // because of connections in keep-alive state. so if this is the case
|
---|
| 33 | // we'll default new requests to set a Connection: close header.
|
---|
| 34 | var close_by_default = !http.Agent || http.Agent.defaultMaxSockets != Infinity;
|
---|
| 35 |
|
---|
| 36 | // see if we have Object.assign. otherwise fall back to util._extend
|
---|
| 37 | var extend = Object.assign ? Object.assign : require('util')._extend;
|
---|
| 38 |
|
---|
| 39 | // these are the status codes that Needle interprets as redirects.
|
---|
| 40 | var redirect_codes = [301, 302, 303, 307, 308];
|
---|
| 41 |
|
---|
| 42 | //////////////////////////////////////////
|
---|
| 43 | // decompressors for gzip/deflate/br bodies
|
---|
| 44 |
|
---|
| 45 | function bind_opts(fn, options) {
|
---|
| 46 | return fn.bind(null, options);
|
---|
| 47 | }
|
---|
| 48 |
|
---|
| 49 | var decompressors = {};
|
---|
| 50 |
|
---|
| 51 | try {
|
---|
| 52 |
|
---|
| 53 | var zlib = require('zlib');
|
---|
| 54 |
|
---|
| 55 | // Enable Z_SYNC_FLUSH to avoid Z_BUF_ERROR errors (Node PR #2595)
|
---|
| 56 | var zlib_options = {
|
---|
| 57 | flush: zlib.Z_SYNC_FLUSH,
|
---|
| 58 | finishFlush: zlib.Z_SYNC_FLUSH
|
---|
| 59 | };
|
---|
| 60 |
|
---|
| 61 | var br_options = {
|
---|
| 62 | flush: zlib.BROTLI_OPERATION_FLUSH,
|
---|
| 63 | finishFlush: zlib.BROTLI_OPERATION_FLUSH
|
---|
| 64 | };
|
---|
| 65 |
|
---|
| 66 | decompressors['x-deflate'] = bind_opts(zlib.Inflate, zlib_options);
|
---|
| 67 | decompressors['deflate'] = bind_opts(zlib.Inflate, zlib_options);
|
---|
| 68 | decompressors['x-gzip'] = bind_opts(zlib.Gunzip, zlib_options);
|
---|
| 69 | decompressors['gzip'] = bind_opts(zlib.Gunzip, zlib_options);
|
---|
| 70 | if (typeof zlib.BrotliDecompress === 'function') {
|
---|
| 71 | decompressors['br'] = bind_opts(zlib.BrotliDecompress, br_options);
|
---|
| 72 | }
|
---|
| 73 |
|
---|
| 74 | } catch(e) { /* zlib not available */ }
|
---|
| 75 |
|
---|
| 76 | //////////////////////////////////////////
|
---|
| 77 | // options and aliases
|
---|
| 78 |
|
---|
| 79 | var defaults = {
|
---|
| 80 | // data
|
---|
| 81 | boundary : '--------------------NODENEEDLEHTTPCLIENT',
|
---|
| 82 | encoding : 'utf8',
|
---|
| 83 | parse_response : 'all', // same as true. valid options: 'json', 'xml' or false/null
|
---|
| 84 | proxy : null,
|
---|
| 85 |
|
---|
| 86 | // headers
|
---|
| 87 | headers : {},
|
---|
| 88 | accept : '*/*',
|
---|
| 89 | user_agent : user_agent,
|
---|
| 90 |
|
---|
| 91 | // numbers
|
---|
| 92 | open_timeout : 10000,
|
---|
| 93 | response_timeout : 0,
|
---|
| 94 | read_timeout : 0,
|
---|
| 95 | follow_max : 0,
|
---|
| 96 | stream_length : -1,
|
---|
| 97 |
|
---|
| 98 | // booleans
|
---|
| 99 | compressed : false,
|
---|
| 100 | decode_response : true,
|
---|
| 101 | parse_cookies : true,
|
---|
| 102 | follow_set_cookies : false,
|
---|
| 103 | follow_set_referer : false,
|
---|
| 104 | follow_keep_method : false,
|
---|
| 105 | follow_if_same_host : false,
|
---|
| 106 | follow_if_same_protocol : false,
|
---|
| 107 | follow_if_same_location : false
|
---|
| 108 | }
|
---|
| 109 |
|
---|
| 110 | var aliased = {
|
---|
| 111 | options: {
|
---|
| 112 | decode : 'decode_response',
|
---|
| 113 | parse : 'parse_response',
|
---|
| 114 | timeout : 'open_timeout',
|
---|
| 115 | follow : 'follow_max'
|
---|
| 116 | },
|
---|
| 117 | inverted: {}
|
---|
| 118 | }
|
---|
| 119 |
|
---|
| 120 | // only once, invert aliased keys so we can get passed options.
|
---|
| 121 | Object.keys(aliased.options).map(function(k) {
|
---|
| 122 | var value = aliased.options[k];
|
---|
| 123 | aliased.inverted[value] = k;
|
---|
| 124 | });
|
---|
| 125 |
|
---|
| 126 | //////////////////////////////////////////
|
---|
| 127 | // helpers
|
---|
| 128 |
|
---|
| 129 | function keys_by_type(type) {
|
---|
| 130 | return Object.keys(defaults).map(function(el) {
|
---|
| 131 | if (defaults[el] !== null && defaults[el].constructor == type)
|
---|
| 132 | return el;
|
---|
| 133 | }).filter(function(el) { return el })
|
---|
| 134 | }
|
---|
| 135 |
|
---|
| 136 | function parse_content_type(header) {
|
---|
| 137 | if (!header || header === '') return {};
|
---|
| 138 |
|
---|
| 139 | var found, charset = 'utf8', arr = header.split(';');
|
---|
| 140 |
|
---|
| 141 | if (arr.length > 1 && (found = arr[1].match(/charset=(.+)/)))
|
---|
| 142 | charset = found[1];
|
---|
| 143 |
|
---|
| 144 | return { type: arr[0], charset: charset };
|
---|
| 145 | }
|
---|
| 146 |
|
---|
| 147 | function is_stream(obj) {
|
---|
| 148 | return typeof obj.pipe === 'function';
|
---|
| 149 | }
|
---|
| 150 |
|
---|
| 151 | function get_stream_length(stream, given_length, cb) {
|
---|
| 152 | if (given_length > 0)
|
---|
| 153 | return cb(given_length);
|
---|
| 154 |
|
---|
| 155 | if (stream.end !== void 0 && stream.end !== Infinity && stream.start !== void 0)
|
---|
| 156 | return cb((stream.end + 1) - (stream.start || 0));
|
---|
| 157 |
|
---|
| 158 | fs.stat(stream.path, function(err, stat) {
|
---|
| 159 | cb(stat ? stat.size - (stream.start || 0) : null);
|
---|
| 160 | });
|
---|
| 161 | }
|
---|
| 162 |
|
---|
| 163 | function resolve_url(href, base) {
|
---|
| 164 | if (url.URL)
|
---|
| 165 | return new url.URL(href, base);
|
---|
| 166 |
|
---|
| 167 | // older Node version (< v6.13)
|
---|
| 168 | return url.resolve(base, href);
|
---|
| 169 | }
|
---|
| 170 |
|
---|
| 171 | function pump_streams(streams, cb) {
|
---|
| 172 | if (stream.pipeline)
|
---|
| 173 | return stream.pipeline.apply(null, streams.concat(cb));
|
---|
| 174 |
|
---|
| 175 | var tmp = streams.shift();
|
---|
| 176 | while (streams.length) {
|
---|
| 177 | tmp = tmp.pipe(streams.shift());
|
---|
| 178 | tmp.once('error', function(e) {
|
---|
| 179 | cb && cb(e);
|
---|
| 180 | cb = null;
|
---|
| 181 | })
|
---|
| 182 | }
|
---|
| 183 | }
|
---|
| 184 |
|
---|
| 185 | //////////////////////////////////////////
|
---|
| 186 | // the main act
|
---|
| 187 |
|
---|
| 188 | function Needle(method, uri, data, options, callback) {
|
---|
| 189 | // if (!(this instanceof Needle)) {
|
---|
| 190 | // return new Needle(method, uri, data, options, callback);
|
---|
| 191 | // }
|
---|
| 192 |
|
---|
| 193 | if (typeof uri !== 'string')
|
---|
| 194 | throw new TypeError('URL must be a string, not ' + uri);
|
---|
| 195 |
|
---|
| 196 | this.method = method.toLowerCase();
|
---|
| 197 | this.uri = uri;
|
---|
| 198 | this.data = data;
|
---|
| 199 |
|
---|
| 200 | if (typeof options == 'function') {
|
---|
| 201 | this.callback = options;
|
---|
| 202 | this.options = {};
|
---|
| 203 | } else {
|
---|
| 204 | this.callback = callback;
|
---|
| 205 | this.options = options;
|
---|
| 206 | }
|
---|
| 207 |
|
---|
| 208 | }
|
---|
| 209 |
|
---|
| 210 | Needle.prototype.setup = function(uri, options) {
|
---|
| 211 |
|
---|
| 212 | function get_option(key, fallback) {
|
---|
| 213 | // if original is in options, return that value
|
---|
| 214 | if (typeof options[key] != 'undefined') return options[key];
|
---|
| 215 |
|
---|
| 216 | // otherwise, return value from alias or fallback/undefined
|
---|
| 217 | return typeof options[aliased.inverted[key]] != 'undefined'
|
---|
| 218 | ? options[aliased.inverted[key]] : fallback;
|
---|
| 219 | }
|
---|
| 220 |
|
---|
| 221 | function check_value(expected, key) {
|
---|
| 222 | var value = get_option(key),
|
---|
| 223 | type = typeof value;
|
---|
| 224 |
|
---|
| 225 | if (type != 'undefined' && type != expected)
|
---|
| 226 | throw new TypeError(type + ' received for ' + key + ', but expected a ' + expected);
|
---|
| 227 |
|
---|
| 228 | return (type == expected) ? value : defaults[key];
|
---|
| 229 | }
|
---|
| 230 |
|
---|
| 231 | //////////////////////////////////////////////////
|
---|
| 232 | // the basics
|
---|
| 233 |
|
---|
| 234 | var config = {
|
---|
| 235 | http_opts : {
|
---|
| 236 | localAddress: get_option('localAddress', undefined),
|
---|
| 237 | lookup: get_option('lookup', undefined)
|
---|
| 238 | }, // passed later to http.request() directly
|
---|
| 239 | headers : {},
|
---|
| 240 | output : options.output,
|
---|
| 241 | proxy : get_option('proxy', defaults.proxy),
|
---|
| 242 | parser : get_option('parse_response', defaults.parse_response),
|
---|
| 243 | encoding : options.encoding || (options.multipart ? 'binary' : defaults.encoding)
|
---|
| 244 | }
|
---|
| 245 |
|
---|
| 246 | keys_by_type(Boolean).forEach(function(key) {
|
---|
| 247 | config[key] = check_value('boolean', key);
|
---|
| 248 | })
|
---|
| 249 |
|
---|
| 250 | keys_by_type(Number).forEach(function(key) {
|
---|
| 251 | config[key] = check_value('number', key);
|
---|
| 252 | })
|
---|
| 253 |
|
---|
| 254 | // populate http_opts with given TLS options
|
---|
| 255 | tls_options.split(' ').forEach(function(key) {
|
---|
| 256 | if (typeof options[key] != 'undefined') {
|
---|
| 257 | config.http_opts[key] = options[key];
|
---|
| 258 | if (typeof options.agent == 'undefined')
|
---|
| 259 | config.http_opts.agent = false; // otherwise tls options are skipped
|
---|
| 260 | }
|
---|
| 261 | });
|
---|
| 262 |
|
---|
| 263 | //////////////////////////////////////////////////
|
---|
| 264 | // headers, cookies
|
---|
| 265 |
|
---|
| 266 | for (var key in defaults.headers)
|
---|
| 267 | config.headers[key] = defaults.headers[key];
|
---|
| 268 |
|
---|
| 269 | config.headers['accept'] = options.accept || defaults.accept;
|
---|
| 270 | config.headers['user-agent'] = options.user_agent || defaults.user_agent;
|
---|
| 271 |
|
---|
| 272 | if (options.content_type)
|
---|
| 273 | config.headers['content-type'] = options.content_type;
|
---|
| 274 |
|
---|
| 275 | // set connection header if opts.connection was passed, or if node < 0.11.4 (close)
|
---|
| 276 | if (options.connection || close_by_default)
|
---|
| 277 | config.headers['connection'] = options.connection || 'close';
|
---|
| 278 |
|
---|
| 279 | if ((options.compressed || defaults.compressed) && typeof zlib != 'undefined')
|
---|
| 280 | config.headers['accept-encoding'] = decompressors['br'] ? 'gzip, deflate, br' : 'gzip, deflate';
|
---|
| 281 |
|
---|
| 282 | if (options.cookies)
|
---|
| 283 | config.headers['cookie'] = cookies.write(options.cookies);
|
---|
| 284 |
|
---|
| 285 | //////////////////////////////////////////////////
|
---|
| 286 | // basic/digest auth
|
---|
| 287 |
|
---|
| 288 | if (uri.match(/[^\/]@/)) { // url contains user:pass@host, so parse it.
|
---|
| 289 | var parts = (url.parse(uri).auth || '').split(':');
|
---|
| 290 | options.username = parts[0];
|
---|
| 291 | options.password = parts[1];
|
---|
| 292 | }
|
---|
| 293 |
|
---|
| 294 | if (options.username) {
|
---|
| 295 | if (options.auth && (options.auth == 'auto' || options.auth == 'digest')) {
|
---|
| 296 | config.credentials = [options.username, options.password];
|
---|
| 297 | } else {
|
---|
| 298 | config.headers['authorization'] = auth.basic(options.username, options.password);
|
---|
| 299 | }
|
---|
| 300 | }
|
---|
| 301 |
|
---|
| 302 | // if proxy is present, set auth header from either url or proxy_user option.
|
---|
| 303 | if (config.proxy) {
|
---|
| 304 | if (config.proxy.indexOf('http') === -1)
|
---|
| 305 | config.proxy = 'http://' + config.proxy;
|
---|
| 306 |
|
---|
| 307 | if (config.proxy.indexOf('@') !== -1) {
|
---|
| 308 | var proxy = (url.parse(config.proxy).auth || '').split(':');
|
---|
| 309 | options.proxy_user = proxy[0];
|
---|
| 310 | options.proxy_pass = proxy[1];
|
---|
| 311 | }
|
---|
| 312 |
|
---|
| 313 | if (options.proxy_user)
|
---|
| 314 | config.headers['proxy-authorization'] = auth.basic(options.proxy_user, options.proxy_pass);
|
---|
| 315 | }
|
---|
| 316 |
|
---|
| 317 | // now that all our headers are set, overwrite them if instructed.
|
---|
| 318 | for (var h in options.headers)
|
---|
| 319 | config.headers[h.toLowerCase()] = options.headers[h];
|
---|
| 320 |
|
---|
| 321 | config.uri_modifier = get_option('uri_modifier', null);
|
---|
| 322 |
|
---|
| 323 | return config;
|
---|
| 324 | }
|
---|
| 325 |
|
---|
| 326 | Needle.prototype.start = function() {
|
---|
| 327 |
|
---|
| 328 | var out = new stream.PassThrough({ objectMode: false }),
|
---|
| 329 | uri = this.uri,
|
---|
| 330 | data = this.data,
|
---|
| 331 | method = this.method,
|
---|
| 332 | callback = (typeof this.options == 'function') ? this.options : this.callback,
|
---|
| 333 | options = this.options || {};
|
---|
| 334 |
|
---|
| 335 | // if no 'http' is found on URL, prepend it.
|
---|
| 336 | if (uri.indexOf('http') === -1)
|
---|
| 337 | uri = uri.replace(/^(\/\/)?/, 'http://');
|
---|
| 338 |
|
---|
| 339 | var self = this, body, waiting = false, config = this.setup(uri, options);
|
---|
| 340 |
|
---|
| 341 | // unless options.json was set to false, assume boss also wants JSON if content-type matches.
|
---|
| 342 | var json = options.json || (options.json !== false && config.headers['content-type'] == 'application/json');
|
---|
| 343 |
|
---|
| 344 | if (data) {
|
---|
| 345 |
|
---|
| 346 | if (options.multipart) { // boss says we do multipart. so we do it.
|
---|
| 347 | var boundary = options.boundary || defaults.boundary;
|
---|
| 348 |
|
---|
| 349 | waiting = true;
|
---|
| 350 | multipart.build(data, boundary, function(err, parts) {
|
---|
| 351 | if (err) throw(err);
|
---|
| 352 |
|
---|
| 353 | config.headers['content-type'] = 'multipart/form-data; boundary=' + boundary;
|
---|
| 354 | next(parts);
|
---|
| 355 | });
|
---|
| 356 |
|
---|
| 357 | } else if (is_stream(data)) {
|
---|
| 358 |
|
---|
| 359 | if (method == 'get')
|
---|
| 360 | throw new Error('Refusing to pipe() a stream via GET. Did you mean .post?');
|
---|
| 361 |
|
---|
| 362 | if (config.stream_length > 0 || (config.stream_length === 0 && data.path)) {
|
---|
| 363 | // ok, let's get the stream's length and set it as the content-length header.
|
---|
| 364 | // this prevents some servers from cutting us off before all the data is sent.
|
---|
| 365 | waiting = true;
|
---|
| 366 | get_stream_length(data, config.stream_length, function(length) {
|
---|
| 367 | data.length = length;
|
---|
| 368 | next(data);
|
---|
| 369 | })
|
---|
| 370 |
|
---|
| 371 | } else {
|
---|
| 372 | // if the boss doesn't want us to get the stream's length, or if it doesn't
|
---|
| 373 | // have a file descriptor for that purpose, then just head on.
|
---|
| 374 | body = data;
|
---|
| 375 | }
|
---|
| 376 |
|
---|
| 377 | } else if (Buffer.isBuffer(data)) {
|
---|
| 378 |
|
---|
| 379 | body = data; // use the raw buffer as request body.
|
---|
| 380 |
|
---|
| 381 | } else if (method == 'get' && !json) {
|
---|
| 382 |
|
---|
| 383 | // append the data to the URI as a querystring.
|
---|
| 384 | uri = uri.replace(/\?.*|$/, '?' + stringify(data));
|
---|
| 385 |
|
---|
| 386 | } else { // string or object data, no multipart.
|
---|
| 387 |
|
---|
| 388 | // if string, leave it as it is, otherwise, stringify.
|
---|
| 389 | body = (typeof(data) === 'string') ? data
|
---|
| 390 | : json ? JSON.stringify(data) : stringify(data);
|
---|
| 391 |
|
---|
| 392 | // ensure we have a buffer so bytecount is correct.
|
---|
| 393 | body = Buffer.from(body, config.encoding);
|
---|
| 394 | }
|
---|
| 395 |
|
---|
| 396 | }
|
---|
| 397 |
|
---|
| 398 | function next(body) {
|
---|
| 399 | if (body) {
|
---|
| 400 | if (body.length) config.headers['content-length'] = body.length;
|
---|
| 401 |
|
---|
| 402 | // if no content-type was passed, determine if json or not.
|
---|
| 403 | if (!config.headers['content-type']) {
|
---|
| 404 | config.headers['content-type'] = json
|
---|
| 405 | ? 'application/json; charset=utf-8'
|
---|
| 406 | : 'application/x-www-form-urlencoded'; // no charset says W3 spec.
|
---|
| 407 | }
|
---|
| 408 | }
|
---|
| 409 |
|
---|
| 410 | // unless a specific accept header was set, assume json: true wants JSON back.
|
---|
| 411 | if (options.json && (!options.accept && !(options.headers || {}).accept))
|
---|
| 412 | config.headers['accept'] = 'application/json';
|
---|
| 413 |
|
---|
| 414 | self.send_request(1, method, uri, config, body, out, callback);
|
---|
| 415 | }
|
---|
| 416 |
|
---|
| 417 | if (!waiting) next(body);
|
---|
| 418 | return out;
|
---|
| 419 | }
|
---|
| 420 |
|
---|
| 421 | Needle.prototype.get_request_opts = function(method, uri, config) {
|
---|
| 422 | var opts = config.http_opts,
|
---|
| 423 | proxy = config.proxy,
|
---|
| 424 | remote = proxy ? url.parse(proxy) : url.parse(uri);
|
---|
| 425 |
|
---|
| 426 | opts.protocol = remote.protocol;
|
---|
| 427 | opts.host = remote.hostname;
|
---|
| 428 | opts.port = remote.port || (remote.protocol == 'https:' ? 443 : 80);
|
---|
| 429 | opts.path = proxy ? uri : remote.pathname + (remote.search || '');
|
---|
| 430 | opts.method = method;
|
---|
| 431 | opts.headers = config.headers;
|
---|
| 432 |
|
---|
| 433 | if (!opts.headers['host']) {
|
---|
| 434 | // if using proxy, make sure the host header shows the final destination
|
---|
| 435 | var target = proxy ? url.parse(uri) : remote;
|
---|
| 436 | opts.headers['host'] = target.hostname;
|
---|
| 437 |
|
---|
| 438 | // and if a non standard port was passed, append it to the port header
|
---|
| 439 | if (target.port && [80, 443].indexOf(target.port) === -1) {
|
---|
| 440 | opts.headers['host'] += ':' + target.port;
|
---|
| 441 | }
|
---|
| 442 | }
|
---|
| 443 |
|
---|
| 444 | return opts;
|
---|
| 445 | }
|
---|
| 446 |
|
---|
| 447 | Needle.prototype.should_follow = function(location, config, original) {
|
---|
| 448 | if (!location) return false;
|
---|
| 449 |
|
---|
| 450 | // returns true if location contains matching property (host or protocol)
|
---|
| 451 | function matches(property) {
|
---|
| 452 | var property = original[property];
|
---|
| 453 | return location.indexOf(property) !== -1;
|
---|
| 454 | }
|
---|
| 455 |
|
---|
| 456 | // first, check whether the requested location is actually different from the original
|
---|
| 457 | if (!config.follow_if_same_location && location === original)
|
---|
| 458 | return false;
|
---|
| 459 |
|
---|
| 460 | if (config.follow_if_same_host && !matches('host'))
|
---|
| 461 | return false; // host does not match, so not following
|
---|
| 462 |
|
---|
| 463 | if (config.follow_if_same_protocol && !matches('protocol'))
|
---|
| 464 | return false; // procotol does not match, so not following
|
---|
| 465 |
|
---|
| 466 | return true;
|
---|
| 467 | }
|
---|
| 468 |
|
---|
| 469 | Needle.prototype.send_request = function(count, method, uri, config, post_data, out, callback) {
|
---|
| 470 |
|
---|
| 471 | if (typeof config.uri_modifier === 'function') {
|
---|
| 472 | var modified_uri = config.uri_modifier(uri);
|
---|
| 473 | debug('Modifying request URI', uri + ' => ' + modified_uri);
|
---|
| 474 | uri = modified_uri;
|
---|
| 475 | }
|
---|
| 476 |
|
---|
| 477 | var request,
|
---|
| 478 | timer,
|
---|
| 479 | returned = 0,
|
---|
| 480 | self = this,
|
---|
| 481 | request_opts = this.get_request_opts(method, uri, config),
|
---|
| 482 | protocol = request_opts.protocol == 'https:' ? https : http;
|
---|
| 483 |
|
---|
| 484 | function done(err, resp) {
|
---|
| 485 | if (returned++ > 0)
|
---|
| 486 | return debug('Already finished, stopping here.');
|
---|
| 487 |
|
---|
| 488 | if (timer) clearTimeout(timer);
|
---|
| 489 | request.removeListener('error', had_error);
|
---|
| 490 | out.done = true;
|
---|
| 491 |
|
---|
| 492 | if (callback)
|
---|
| 493 | return callback(err, resp, resp ? resp.body : undefined);
|
---|
| 494 |
|
---|
| 495 | // NOTE: this event used to be called 'end', but the behaviour was confusing
|
---|
| 496 | // when errors ocurred, because the stream would still emit an 'end' event.
|
---|
| 497 | out.emit('done', err);
|
---|
| 498 |
|
---|
| 499 | // trigger the 'done' event on streams we're being piped to, if any
|
---|
| 500 | var pipes = out._readableState.pipes || [];
|
---|
| 501 | if (!pipes.forEach) pipes = [pipes];
|
---|
| 502 | pipes.forEach(function(st) { st.emit('done', err); })
|
---|
| 503 | }
|
---|
| 504 |
|
---|
| 505 | function had_error(err) {
|
---|
| 506 | debug('Request error', err);
|
---|
| 507 | out.emit('err', err);
|
---|
| 508 | done(err || new Error('Unknown error when making request.'));
|
---|
| 509 | }
|
---|
| 510 |
|
---|
| 511 | function set_timeout(type, milisecs) {
|
---|
| 512 | if (timer) clearTimeout(timer);
|
---|
| 513 | if (milisecs <= 0) return;
|
---|
| 514 |
|
---|
| 515 | timer = setTimeout(function() {
|
---|
| 516 | out.emit('timeout', type);
|
---|
| 517 | request.abort();
|
---|
| 518 | // also invoke done() to terminate job on read_timeout
|
---|
| 519 | if (type == 'read') done(new Error(type + ' timeout'));
|
---|
| 520 | }, milisecs);
|
---|
| 521 | }
|
---|
| 522 |
|
---|
| 523 | // handle errors on the underlying socket, that may be closed while writing
|
---|
| 524 | // for an example case, see test/long_string_spec.js. we make sure this
|
---|
| 525 | // scenario ocurred by verifying the socket's writable & destroyed states.
|
---|
| 526 | function on_socket_end() {
|
---|
| 527 | if (returned && !this.writable && this.destroyed === false) {
|
---|
| 528 | this.destroy();
|
---|
| 529 | had_error(new Error('Remote end closed socket abruptly.'))
|
---|
| 530 | }
|
---|
| 531 | }
|
---|
| 532 |
|
---|
| 533 | debug('Making request #' + count, request_opts);
|
---|
| 534 | request = protocol.request(request_opts, function(resp) {
|
---|
| 535 |
|
---|
| 536 | var headers = resp.headers;
|
---|
| 537 | debug('Got response', resp.statusCode, headers);
|
---|
| 538 | out.emit('response', resp);
|
---|
| 539 |
|
---|
| 540 | set_timeout('read', config.read_timeout);
|
---|
| 541 |
|
---|
| 542 | // if we got cookies, parse them unless we were instructed not to. make sure to include any
|
---|
| 543 | // cookies that might have been set on previous redirects.
|
---|
| 544 | if (config.parse_cookies && (headers['set-cookie'] || config.previous_resp_cookies)) {
|
---|
| 545 | resp.cookies = extend(config.previous_resp_cookies || {}, cookies.read(headers['set-cookie']));
|
---|
| 546 | debug('Got cookies', resp.cookies);
|
---|
| 547 | }
|
---|
| 548 |
|
---|
| 549 | // if redirect code is found, determine if we should follow it according to the given options.
|
---|
| 550 | if (redirect_codes.indexOf(resp.statusCode) !== -1 && self.should_follow(headers.location, config, uri)) {
|
---|
| 551 | // clear timer before following redirects to prevent unexpected setTimeout consequence
|
---|
| 552 | clearTimeout(timer);
|
---|
| 553 |
|
---|
| 554 | if (count <= config.follow_max) {
|
---|
| 555 | out.emit('redirect', headers.location);
|
---|
| 556 |
|
---|
| 557 | // unless 'follow_keep_method' is true, rewrite the request to GET before continuing.
|
---|
| 558 | if (!config.follow_keep_method) {
|
---|
| 559 | method = 'GET';
|
---|
| 560 | post_data = null;
|
---|
| 561 | delete config.headers['content-length']; // in case the original was a multipart POST request.
|
---|
| 562 | }
|
---|
| 563 |
|
---|
| 564 | // if follow_set_cookies is true, insert cookies in the next request's headers.
|
---|
| 565 | // we set both the original request cookies plus any response cookies we might have received.
|
---|
| 566 | if (config.follow_set_cookies) {
|
---|
| 567 | var request_cookies = cookies.read(config.headers['cookie']);
|
---|
| 568 | config.previous_resp_cookies = resp.cookies;
|
---|
| 569 | if (Object.keys(request_cookies).length || Object.keys(resp.cookies || {}).length) {
|
---|
| 570 | config.headers['cookie'] = cookies.write(extend(request_cookies, resp.cookies));
|
---|
| 571 | }
|
---|
| 572 | } else if (config.headers['cookie']) {
|
---|
| 573 | debug('Clearing original request cookie', config.headers['cookie']);
|
---|
| 574 | delete config.headers['cookie'];
|
---|
| 575 | }
|
---|
| 576 |
|
---|
| 577 | if (config.follow_set_referer)
|
---|
| 578 | config.headers['referer'] = encodeURI(uri); // the original, not the destination URL.
|
---|
| 579 |
|
---|
| 580 | config.headers['host'] = null; // clear previous Host header to avoid conflicts.
|
---|
| 581 |
|
---|
| 582 | var redirect_url = resolve_url(headers.location, uri);
|
---|
| 583 | debug('Redirecting to ' + redirect_url.toString());
|
---|
| 584 | return self.send_request(++count, method, redirect_url.toString(), config, post_data, out, callback);
|
---|
| 585 | } else if (config.follow_max > 0) {
|
---|
| 586 | return done(new Error('Max redirects reached. Possible loop in: ' + headers.location));
|
---|
| 587 | }
|
---|
| 588 | }
|
---|
| 589 |
|
---|
| 590 | // if auth is requested and credentials were not passed, resend request, provided we have user/pass.
|
---|
| 591 | if (resp.statusCode == 401 && headers['www-authenticate'] && config.credentials) {
|
---|
| 592 | if (!config.headers['authorization']) { // only if authentication hasn't been sent
|
---|
| 593 | var auth_header = auth.header(headers['www-authenticate'], config.credentials, request_opts);
|
---|
| 594 |
|
---|
| 595 | if (auth_header) {
|
---|
| 596 | config.headers['authorization'] = auth_header;
|
---|
| 597 | return self.send_request(count, method, uri, config, post_data, out, callback);
|
---|
| 598 | }
|
---|
| 599 | }
|
---|
| 600 | }
|
---|
| 601 |
|
---|
| 602 | // ok, so we got a valid (non-redirect & authorized) response. let's notify the stream guys.
|
---|
| 603 | out.emit('header', resp.statusCode, headers);
|
---|
| 604 | out.emit('headers', headers);
|
---|
| 605 |
|
---|
| 606 | var pipeline = [],
|
---|
| 607 | mime = parse_content_type(headers['content-type']),
|
---|
| 608 | text_response = mime.type && (mime.type.indexOf('text/') != -1 || !!mime.type.match(/(\/|\+)(xml|json)$/));
|
---|
| 609 |
|
---|
| 610 | // To start, if our body is compressed and we're able to inflate it, do it.
|
---|
| 611 | if (headers['content-encoding'] && decompressors[headers['content-encoding']]) {
|
---|
| 612 |
|
---|
| 613 | var decompressor = decompressors[headers['content-encoding']]();
|
---|
| 614 |
|
---|
| 615 | // make sure we catch errors triggered by the decompressor.
|
---|
| 616 | decompressor.on('error', had_error);
|
---|
| 617 | pipeline.push(decompressor);
|
---|
| 618 | }
|
---|
| 619 |
|
---|
| 620 | // If parse is enabled and we have a parser for it, then go for it.
|
---|
| 621 | if (config.parser && parsers[mime.type]) {
|
---|
| 622 |
|
---|
| 623 | // If a specific parser was requested, make sure we don't parse other types.
|
---|
| 624 | var parser_name = config.parser.toString().toLowerCase();
|
---|
| 625 | if (['xml', 'json'].indexOf(parser_name) == -1 || parsers[mime.type].name == parser_name) {
|
---|
| 626 |
|
---|
| 627 | // OK, so either we're parsing all content types or the one requested matches.
|
---|
| 628 | out.parser = parsers[mime.type].name;
|
---|
| 629 | pipeline.push(parsers[mime.type].fn());
|
---|
| 630 |
|
---|
| 631 | // Set objectMode on out stream to improve performance.
|
---|
| 632 | out._writableState.objectMode = true;
|
---|
| 633 | out._readableState.objectMode = true;
|
---|
| 634 | }
|
---|
| 635 |
|
---|
| 636 | // If we're not parsing, and unless decoding was disabled, we'll try
|
---|
| 637 | // decoding non UTF-8 bodies to UTF-8, using the iconv-lite library.
|
---|
| 638 | } else if (text_response && config.decode_response && mime.charset) {
|
---|
| 639 | pipeline.push(decoder(mime.charset));
|
---|
| 640 | }
|
---|
| 641 |
|
---|
| 642 | // And `out` is the stream we finally push the decoded/parsed output to.
|
---|
| 643 | pipeline.push(out);
|
---|
| 644 |
|
---|
| 645 | // Now, release the kraken!
|
---|
| 646 | pump_streams([resp].concat(pipeline), function(err) {
|
---|
| 647 | if (err) debug(err)
|
---|
| 648 |
|
---|
| 649 | // on node v8.x, if an error ocurrs on the receiving end,
|
---|
| 650 | // then we want to abort the request to avoid having dangling sockets
|
---|
| 651 | if (err && err.message == 'write after end') request.destroy();
|
---|
| 652 | });
|
---|
| 653 |
|
---|
| 654 | // If the user has requested and output file, pipe the output stream to it.
|
---|
| 655 | // In stream mode, we will still get the response stream to play with.
|
---|
| 656 | if (config.output && resp.statusCode == 200) {
|
---|
| 657 |
|
---|
| 658 | // for some reason, simply piping resp to the writable stream doesn't
|
---|
| 659 | // work all the time (stream gets cut in the middle with no warning).
|
---|
| 660 | // so we'll manually need to do the readable/write(chunk) trick.
|
---|
| 661 | var file = fs.createWriteStream(config.output);
|
---|
| 662 | file.on('error', had_error);
|
---|
| 663 |
|
---|
| 664 | out.on('end', function() {
|
---|
| 665 | if (file.writable) file.end();
|
---|
| 666 | });
|
---|
| 667 |
|
---|
| 668 | file.on('close', function() {
|
---|
| 669 | delete out.file;
|
---|
| 670 | })
|
---|
| 671 |
|
---|
| 672 | out.on('readable', function() {
|
---|
| 673 | var chunk;
|
---|
| 674 | while ((chunk = this.read()) !== null) {
|
---|
| 675 | if (file.writable) file.write(chunk);
|
---|
| 676 |
|
---|
| 677 | // if callback was requested, also push it to resp.body
|
---|
| 678 | if (resp.body) resp.body.push(chunk);
|
---|
| 679 | }
|
---|
| 680 | })
|
---|
| 681 |
|
---|
| 682 | out.file = file;
|
---|
| 683 | }
|
---|
| 684 |
|
---|
| 685 | // Only aggregate the full body if a callback was requested.
|
---|
| 686 | if (callback) {
|
---|
| 687 | resp.raw = [];
|
---|
| 688 | resp.body = [];
|
---|
| 689 | resp.bytes = 0;
|
---|
| 690 |
|
---|
| 691 | // Gather and count the amount of (raw) bytes using a PassThrough stream.
|
---|
| 692 | var clean_pipe = new stream.PassThrough();
|
---|
| 693 |
|
---|
| 694 | clean_pipe.on('readable', function() {
|
---|
| 695 | var chunk;
|
---|
| 696 | while ((chunk = this.read()) != null) {
|
---|
| 697 | resp.bytes += chunk.length;
|
---|
| 698 | resp.raw.push(chunk);
|
---|
| 699 | }
|
---|
| 700 | })
|
---|
| 701 |
|
---|
| 702 | pump_streams([resp, clean_pipe], function(err) {
|
---|
| 703 | if (err) debug(err);
|
---|
| 704 | });
|
---|
| 705 |
|
---|
| 706 | // Listen on the 'readable' event to aggregate the chunks, but only if
|
---|
| 707 | // file output wasn't requested. Otherwise we'd have two stream readers.
|
---|
| 708 | if (!config.output || resp.statusCode != 200) {
|
---|
| 709 | out.on('readable', function() {
|
---|
| 710 | var chunk;
|
---|
| 711 | while ((chunk = this.read()) !== null) {
|
---|
| 712 | // We're either pushing buffers or objects, never strings.
|
---|
| 713 | if (typeof chunk == 'string') chunk = Buffer.from(chunk);
|
---|
| 714 |
|
---|
| 715 | // Push all chunks to resp.body. We'll bind them in resp.end().
|
---|
| 716 | resp.body.push(chunk);
|
---|
| 717 | }
|
---|
| 718 | })
|
---|
| 719 | }
|
---|
| 720 | }
|
---|
| 721 |
|
---|
| 722 | // And set the .body property once all data is in.
|
---|
| 723 | out.on('end', function() {
|
---|
| 724 | if (resp.body) { // callback mode
|
---|
| 725 |
|
---|
| 726 | // we want to be able to access to the raw data later, so keep a reference.
|
---|
| 727 | resp.raw = Buffer.concat(resp.raw);
|
---|
| 728 |
|
---|
| 729 | // if parse was successful, we should have an array with one object
|
---|
| 730 | if (resp.body[0] !== undefined && !Buffer.isBuffer(resp.body[0])) {
|
---|
| 731 |
|
---|
| 732 | // that's our body right there.
|
---|
| 733 | resp.body = resp.body[0];
|
---|
| 734 |
|
---|
| 735 | // set the parser property on our response. we may want to check.
|
---|
| 736 | if (out.parser) resp.parser = out.parser;
|
---|
| 737 |
|
---|
| 738 | } else { // we got one or several buffers. string or binary.
|
---|
| 739 | resp.body = Buffer.concat(resp.body);
|
---|
| 740 |
|
---|
| 741 | // if we're here and parsed is true, it means we tried to but it didn't work.
|
---|
| 742 | // so given that we got a text response, let's stringify it.
|
---|
| 743 | if (text_response || out.parser) {
|
---|
| 744 | resp.body = resp.body.toString();
|
---|
| 745 | }
|
---|
| 746 | }
|
---|
| 747 | }
|
---|
| 748 |
|
---|
| 749 | // if an output file is being written to, make sure the callback
|
---|
| 750 | // is triggered after all data has been written to it.
|
---|
| 751 | if (out.file) {
|
---|
| 752 | out.file.on('close', function() {
|
---|
| 753 | done(null, resp);
|
---|
| 754 | })
|
---|
| 755 | } else { // elvis has left the building.
|
---|
| 756 | done(null, resp);
|
---|
| 757 | }
|
---|
| 758 |
|
---|
| 759 | });
|
---|
| 760 |
|
---|
| 761 | // out.on('error', function(err) {
|
---|
| 762 | // had_error(err);
|
---|
| 763 | // if (err.code == 'ERR_STREAM_DESTROYED' || err.code == 'ERR_STREAM_PREMATURE_CLOSE') {
|
---|
| 764 | // request.abort();
|
---|
| 765 | // }
|
---|
| 766 | // })
|
---|
| 767 |
|
---|
| 768 | }); // end request call
|
---|
| 769 |
|
---|
| 770 | // unless open_timeout was disabled, set a timeout to abort the request.
|
---|
| 771 | set_timeout('open', config.open_timeout);
|
---|
| 772 |
|
---|
| 773 | // handle errors on the request object. things might get bumpy.
|
---|
| 774 | request.on('error', had_error);
|
---|
| 775 |
|
---|
| 776 | // make sure timer is cleared if request is aborted (issue #257)
|
---|
| 777 | request.once('abort', function() {
|
---|
| 778 | if (timer) clearTimeout(timer);
|
---|
| 779 | })
|
---|
| 780 |
|
---|
| 781 | // handle socket 'end' event to ensure we don't get delayed EPIPE errors.
|
---|
| 782 | request.once('socket', function(socket) {
|
---|
| 783 | if (socket.connecting) {
|
---|
| 784 | socket.once('connect', function() {
|
---|
| 785 | set_timeout('response', config.response_timeout);
|
---|
| 786 | })
|
---|
| 787 | } else {
|
---|
| 788 | set_timeout('response', config.response_timeout);
|
---|
| 789 | }
|
---|
| 790 |
|
---|
| 791 | // socket.once('close', function(e) {
|
---|
| 792 | // console.log('socket closed!', e);
|
---|
| 793 | // })
|
---|
| 794 |
|
---|
| 795 | if (!socket.on_socket_end) {
|
---|
| 796 | socket.on_socket_end = on_socket_end;
|
---|
| 797 | socket.once('end', function() { process.nextTick(on_socket_end.bind(socket)) });
|
---|
| 798 | }
|
---|
| 799 | })
|
---|
| 800 |
|
---|
| 801 | if (post_data) {
|
---|
| 802 | if (is_stream(post_data)) {
|
---|
| 803 | pump_streams([post_data, request], function(err) {
|
---|
| 804 | if (err) debug(err);
|
---|
| 805 | });
|
---|
| 806 | } else {
|
---|
| 807 | request.write(post_data, config.encoding);
|
---|
| 808 | request.end();
|
---|
| 809 | }
|
---|
| 810 | } else {
|
---|
| 811 | request.end();
|
---|
| 812 | }
|
---|
| 813 |
|
---|
| 814 | out.abort = function() { request.abort() }; // easier access
|
---|
| 815 | out.request = request;
|
---|
| 816 | return out;
|
---|
| 817 | }
|
---|
| 818 |
|
---|
| 819 | //////////////////////////////////////////
|
---|
| 820 | // exports
|
---|
| 821 |
|
---|
| 822 | if (typeof Promise !== 'undefined') {
|
---|
| 823 | module.exports = function() {
|
---|
| 824 | var verb, args = [].slice.call(arguments);
|
---|
| 825 |
|
---|
| 826 | if (args[0].match(/\.|\//)) // first argument looks like a URL
|
---|
| 827 | verb = (args.length > 2) ? 'post' : 'get';
|
---|
| 828 | else
|
---|
| 829 | verb = args.shift();
|
---|
| 830 |
|
---|
| 831 | if (verb.match(/get|head/i) && args.length == 2)
|
---|
| 832 | args.splice(1, 0, null); // assume no data if head/get with two args (url, options)
|
---|
| 833 |
|
---|
| 834 | return new Promise(function(resolve, reject) {
|
---|
| 835 | module.exports.request(verb, args[0], args[1], args[2], function(err, resp) {
|
---|
| 836 | return err ? reject(err) : resolve(resp);
|
---|
| 837 | });
|
---|
| 838 | })
|
---|
| 839 | }
|
---|
| 840 | }
|
---|
| 841 |
|
---|
| 842 | module.exports.version = version;
|
---|
| 843 |
|
---|
| 844 | module.exports.defaults = function(obj) {
|
---|
| 845 | for (var key in obj) {
|
---|
| 846 | var target_key = aliased.options[key] || key;
|
---|
| 847 |
|
---|
| 848 | if (defaults.hasOwnProperty(target_key) && typeof obj[key] != 'undefined') {
|
---|
| 849 | if (target_key != 'parse_response' && target_key != 'proxy') {
|
---|
| 850 | // ensure type matches the original, except for proxy/parse_response that can be null/bool or string
|
---|
| 851 | var valid_type = defaults[target_key].constructor.name;
|
---|
| 852 |
|
---|
| 853 | if (obj[key].constructor.name != valid_type)
|
---|
| 854 | throw new TypeError('Invalid type for ' + key + ', should be ' + valid_type);
|
---|
| 855 | }
|
---|
| 856 | defaults[target_key] = obj[key];
|
---|
| 857 | } else {
|
---|
| 858 | throw new Error('Invalid property for defaults:' + target_key);
|
---|
| 859 | }
|
---|
| 860 | }
|
---|
| 861 |
|
---|
| 862 | return defaults;
|
---|
| 863 | }
|
---|
| 864 |
|
---|
| 865 | 'head get'.split(' ').forEach(function(method) {
|
---|
| 866 | module.exports[method] = function(uri, options, callback) {
|
---|
| 867 | return new Needle(method, uri, null, options, callback).start();
|
---|
| 868 | }
|
---|
| 869 | })
|
---|
| 870 |
|
---|
| 871 | 'post put patch delete'.split(' ').forEach(function(method) {
|
---|
| 872 | module.exports[method] = function(uri, data, options, callback) {
|
---|
| 873 | return new Needle(method, uri, data, options, callback).start();
|
---|
| 874 | }
|
---|
| 875 | })
|
---|
| 876 |
|
---|
| 877 | module.exports.request = function(method, uri, data, opts, callback) {
|
---|
| 878 | return new Needle(method, uri, data, opts, callback).start();
|
---|
| 879 | };
|
---|