| 1 | 'use strict'
|
|---|
| 2 |
|
|---|
| 3 | const check = require('check-types')
|
|---|
| 4 | const eventify = require('./eventify')
|
|---|
| 5 | const events = require('./events')
|
|---|
| 6 | const JsonStream = require('./jsonstream')
|
|---|
| 7 | const Hoopy = require('hoopy')
|
|---|
| 8 | const promise = require('./promise')
|
|---|
| 9 | const tryer = require('tryer')
|
|---|
| 10 |
|
|---|
| 11 | const DEFAULT_BUFFER_LENGTH = 1024
|
|---|
| 12 |
|
|---|
| 13 | module.exports = streamify
|
|---|
| 14 |
|
|---|
| 15 | /**
|
|---|
| 16 | * Public function `streamify`.
|
|---|
| 17 | *
|
|---|
| 18 | * Asynchronously serialises a data structure to a stream of JSON
|
|---|
| 19 | * data. Sanely handles promises, buffers, maps and other iterables.
|
|---|
| 20 | *
|
|---|
| 21 | * @param data: The data to transform.
|
|---|
| 22 | *
|
|---|
| 23 | * @option space: Indentation string, or the number of spaces
|
|---|
| 24 | * to indent each nested level by.
|
|---|
| 25 | *
|
|---|
| 26 | * @option promises: 'resolve' or 'ignore', default is 'resolve'.
|
|---|
| 27 | *
|
|---|
| 28 | * @option buffers: 'toString' or 'ignore', default is 'toString'.
|
|---|
| 29 | *
|
|---|
| 30 | * @option maps: 'object' or 'ignore', default is 'object'.
|
|---|
| 31 | *
|
|---|
| 32 | * @option iterables: 'array' or 'ignore', default is 'array'.
|
|---|
| 33 | *
|
|---|
| 34 | * @option circular: 'error' or 'ignore', default is 'error'.
|
|---|
| 35 | *
|
|---|
| 36 | * @option yieldRate: The number of data items to process per timeslice,
|
|---|
| 37 | * default is 16384.
|
|---|
| 38 | *
|
|---|
| 39 | * @option bufferLength: The length of the buffer, default is 1024.
|
|---|
| 40 | *
|
|---|
| 41 | * @option highWaterMark: If set, will be passed to the readable stream constructor
|
|---|
| 42 | * as the value for the highWaterMark option.
|
|---|
| 43 | *
|
|---|
| 44 | * @option Promise: The promise constructor to use, defaults to bluebird.
|
|---|
| 45 | **/
|
|---|
| 46 | function streamify (data, options = {}) {
|
|---|
| 47 | const emitter = eventify(data, options)
|
|---|
| 48 | const json = new Hoopy(options.bufferLength || DEFAULT_BUFFER_LENGTH)
|
|---|
| 49 | const Promise = promise(options)
|
|---|
| 50 | const space = normaliseSpace(options)
|
|---|
| 51 | let streamOptions
|
|---|
| 52 | const { highWaterMark } = options
|
|---|
| 53 | if (highWaterMark) {
|
|---|
| 54 | streamOptions = { highWaterMark }
|
|---|
| 55 | }
|
|---|
| 56 | const stream = new JsonStream(read, streamOptions)
|
|---|
| 57 |
|
|---|
| 58 | let awaitPush = true
|
|---|
| 59 | let index = 0
|
|---|
| 60 | let indentation = ''
|
|---|
| 61 | let isEnded
|
|---|
| 62 | let isPaused = false
|
|---|
| 63 | let isProperty
|
|---|
| 64 | let length = 0
|
|---|
| 65 | let mutex = Promise.resolve()
|
|---|
| 66 | let needsComma
|
|---|
| 67 |
|
|---|
| 68 | emitter.on(events.array, noRacing(array))
|
|---|
| 69 | emitter.on(events.object, noRacing(object))
|
|---|
| 70 | emitter.on(events.property, noRacing(property))
|
|---|
| 71 | emitter.on(events.string, noRacing(string))
|
|---|
| 72 | emitter.on(events.number, noRacing(value))
|
|---|
| 73 | emitter.on(events.literal, noRacing(value))
|
|---|
| 74 | emitter.on(events.endArray, noRacing(endArray))
|
|---|
| 75 | emitter.on(events.endObject, noRacing(endObject))
|
|---|
| 76 | emitter.on(events.end, noRacing(end))
|
|---|
| 77 | emitter.on(events.error, noRacing(error))
|
|---|
| 78 | emitter.on(events.dataError, noRacing(dataError))
|
|---|
| 79 |
|
|---|
| 80 | return stream
|
|---|
| 81 |
|
|---|
| 82 | function read () {
|
|---|
| 83 | if (awaitPush) {
|
|---|
| 84 | awaitPush = false
|
|---|
| 85 |
|
|---|
| 86 | if (isEnded) {
|
|---|
| 87 | if (length > 0) {
|
|---|
| 88 | after()
|
|---|
| 89 | }
|
|---|
| 90 |
|
|---|
| 91 | return endStream()
|
|---|
| 92 | }
|
|---|
| 93 | }
|
|---|
| 94 |
|
|---|
| 95 | if (isPaused) {
|
|---|
| 96 | after()
|
|---|
| 97 | }
|
|---|
| 98 | }
|
|---|
| 99 |
|
|---|
| 100 | function after () {
|
|---|
| 101 | if (awaitPush) {
|
|---|
| 102 | return
|
|---|
| 103 | }
|
|---|
| 104 |
|
|---|
| 105 | let i
|
|---|
| 106 |
|
|---|
| 107 | for (i = 0; i < length && ! awaitPush; ++i) {
|
|---|
| 108 | if (! stream.push(json[i + index], 'utf8')) {
|
|---|
| 109 | awaitPush = true
|
|---|
| 110 | }
|
|---|
| 111 | }
|
|---|
| 112 |
|
|---|
| 113 | if (i === length) {
|
|---|
| 114 | index = length = 0
|
|---|
| 115 | } else {
|
|---|
| 116 | length -= i
|
|---|
| 117 | index += i
|
|---|
| 118 | }
|
|---|
| 119 | }
|
|---|
| 120 |
|
|---|
| 121 | function endStream () {
|
|---|
| 122 | if (! awaitPush) {
|
|---|
| 123 | stream.push(null)
|
|---|
| 124 | }
|
|---|
| 125 | }
|
|---|
| 126 |
|
|---|
| 127 | function noRacing (handler) {
|
|---|
| 128 | return eventData => mutex = mutex.then(() => handler(eventData))
|
|---|
| 129 | }
|
|---|
| 130 |
|
|---|
| 131 | function array () {
|
|---|
| 132 | return beforeScope()
|
|---|
| 133 | .then(() => addJson('['))
|
|---|
| 134 | .then(() => afterScope())
|
|---|
| 135 | }
|
|---|
| 136 |
|
|---|
| 137 | function beforeScope () {
|
|---|
| 138 | return before(true)
|
|---|
| 139 | }
|
|---|
| 140 |
|
|---|
| 141 | function before (isScope) {
|
|---|
| 142 | if (isProperty) {
|
|---|
| 143 | isProperty = false
|
|---|
| 144 |
|
|---|
| 145 | if (space) {
|
|---|
| 146 | return addJson(' ')
|
|---|
| 147 | }
|
|---|
| 148 |
|
|---|
| 149 | return Promise.resolve()
|
|---|
| 150 | }
|
|---|
| 151 |
|
|---|
| 152 | return Promise.resolve()
|
|---|
| 153 | .then(() => {
|
|---|
| 154 | if (needsComma) {
|
|---|
| 155 | if (isScope) {
|
|---|
| 156 | needsComma = false
|
|---|
| 157 | }
|
|---|
| 158 |
|
|---|
| 159 | return addJson(',')
|
|---|
| 160 | }
|
|---|
| 161 |
|
|---|
| 162 | if (! isScope) {
|
|---|
| 163 | needsComma = true
|
|---|
| 164 | }
|
|---|
| 165 | })
|
|---|
| 166 | .then(() => {
|
|---|
| 167 | if (space && indentation) {
|
|---|
| 168 | return indent()
|
|---|
| 169 | }
|
|---|
| 170 | })
|
|---|
| 171 | }
|
|---|
| 172 |
|
|---|
| 173 | function addJson (chunk) {
|
|---|
| 174 | if (length + 1 <= json.length) {
|
|---|
| 175 | json[index + length++] = chunk
|
|---|
| 176 | after()
|
|---|
| 177 | return Promise.resolve()
|
|---|
| 178 | }
|
|---|
| 179 |
|
|---|
| 180 | isPaused = true
|
|---|
| 181 | return new Promise(resolve => {
|
|---|
| 182 | const unpause = emitter.pause()
|
|---|
| 183 | tryer({
|
|---|
| 184 | interval: -10,
|
|---|
| 185 | until () {
|
|---|
| 186 | return length + 1 <= json.length
|
|---|
| 187 | },
|
|---|
| 188 | pass () {
|
|---|
| 189 | isPaused = false
|
|---|
| 190 | json[index + length++] = chunk
|
|---|
| 191 | resolve()
|
|---|
| 192 | setImmediate(unpause)
|
|---|
| 193 | }
|
|---|
| 194 | })
|
|---|
| 195 | })
|
|---|
| 196 | }
|
|---|
| 197 |
|
|---|
| 198 | function indent () {
|
|---|
| 199 | return addJson(`\n${indentation}`)
|
|---|
| 200 | }
|
|---|
| 201 |
|
|---|
| 202 | function afterScope () {
|
|---|
| 203 | needsComma = false
|
|---|
| 204 |
|
|---|
| 205 | if (space) {
|
|---|
| 206 | indentation += space
|
|---|
| 207 | }
|
|---|
| 208 | }
|
|---|
| 209 |
|
|---|
| 210 | function object () {
|
|---|
| 211 | return beforeScope()
|
|---|
| 212 | .then(() => addJson('{'))
|
|---|
| 213 | .then(() => afterScope())
|
|---|
| 214 | }
|
|---|
| 215 |
|
|---|
| 216 | function property (name) {
|
|---|
| 217 | return before()
|
|---|
| 218 | .then(() => addJson(`"${name}":`))
|
|---|
| 219 | .then(() => {
|
|---|
| 220 | isProperty = true
|
|---|
| 221 | })
|
|---|
| 222 | }
|
|---|
| 223 |
|
|---|
| 224 | function string (s) {
|
|---|
| 225 | return value(`"${s}"`)
|
|---|
| 226 | }
|
|---|
| 227 |
|
|---|
| 228 | function value (v) {
|
|---|
| 229 | return before()
|
|---|
| 230 | .then(() => addJson(`${v}`))
|
|---|
| 231 | }
|
|---|
| 232 |
|
|---|
| 233 | function endArray () {
|
|---|
| 234 | return beforeScopeEnd()
|
|---|
| 235 | .then(() => addJson(']'))
|
|---|
| 236 | .then(() => afterScopeEnd())
|
|---|
| 237 | }
|
|---|
| 238 |
|
|---|
| 239 | function beforeScopeEnd () {
|
|---|
| 240 | if (space) {
|
|---|
| 241 | indentation = indentation.substr(space.length)
|
|---|
| 242 |
|
|---|
| 243 | return indent()
|
|---|
| 244 | }
|
|---|
| 245 |
|
|---|
| 246 | return Promise.resolve()
|
|---|
| 247 | }
|
|---|
| 248 |
|
|---|
| 249 | function afterScopeEnd () {
|
|---|
| 250 | needsComma = true
|
|---|
| 251 | }
|
|---|
| 252 |
|
|---|
| 253 | function endObject () {
|
|---|
| 254 | return beforeScopeEnd()
|
|---|
| 255 | .then(() => addJson('}'))
|
|---|
| 256 | .then(() => afterScopeEnd())
|
|---|
| 257 | }
|
|---|
| 258 |
|
|---|
| 259 | function end () {
|
|---|
| 260 | after()
|
|---|
| 261 |
|
|---|
| 262 | isEnded = true
|
|---|
| 263 | endStream()
|
|---|
| 264 | }
|
|---|
| 265 |
|
|---|
| 266 | function error (err) {
|
|---|
| 267 | stream.emit('error', err)
|
|---|
| 268 | }
|
|---|
| 269 |
|
|---|
| 270 | function dataError (err) {
|
|---|
| 271 | stream.emit('dataError', err)
|
|---|
| 272 | }
|
|---|
| 273 | }
|
|---|
| 274 |
|
|---|
| 275 | function normaliseSpace (options) {
|
|---|
| 276 | if (check.positive(options.space)) {
|
|---|
| 277 | return new Array(options.space + 1).join(' ')
|
|---|
| 278 | }
|
|---|
| 279 |
|
|---|
| 280 | if (check.nonEmptyString(options.space)) {
|
|---|
| 281 | return options.space
|
|---|
| 282 | }
|
|---|
| 283 | }
|
|---|