var constants = require('../tokenizer/const'); var TYPE = constants.TYPE; var NAME = constants.NAME; var utils = require('../tokenizer/utils'); var cmpStr = utils.cmpStr; var EOF = TYPE.EOF; var WHITESPACE = TYPE.WhiteSpace; var COMMENT = TYPE.Comment; var OFFSET_MASK = 0x00FFFFFF; var TYPE_SHIFT = 24; var TokenStream = function() { this.offsetAndType = null; this.balance = null; this.reset(); }; TokenStream.prototype = { reset: function() { this.eof = false; this.tokenIndex = -1; this.tokenType = 0; this.tokenStart = this.firstCharOffset; this.tokenEnd = this.firstCharOffset; }, lookupType: function(offset) { offset += this.tokenIndex; if (offset < this.tokenCount) { return this.offsetAndType[offset] >> TYPE_SHIFT; } return EOF; }, lookupOffset: function(offset) { offset += this.tokenIndex; if (offset < this.tokenCount) { return this.offsetAndType[offset - 1] & OFFSET_MASK; } return this.source.length; }, lookupValue: function(offset, referenceStr) { offset += this.tokenIndex; if (offset < this.tokenCount) { return cmpStr( this.source, this.offsetAndType[offset - 1] & OFFSET_MASK, this.offsetAndType[offset] & OFFSET_MASK, referenceStr ); } return false; }, getTokenStart: function(tokenIndex) { if (tokenIndex === this.tokenIndex) { return this.tokenStart; } if (tokenIndex > 0) { return tokenIndex < this.tokenCount ? this.offsetAndType[tokenIndex - 1] & OFFSET_MASK : this.offsetAndType[this.tokenCount] & OFFSET_MASK; } return this.firstCharOffset; }, // TODO: -> skipUntilBalanced getRawLength: function(startToken, mode) { var cursor = startToken; var balanceEnd; var offset = this.offsetAndType[Math.max(cursor - 1, 0)] & OFFSET_MASK; var type; loop: for (; cursor < this.tokenCount; cursor++) { balanceEnd = this.balance[cursor]; // stop scanning on balance edge that points to offset before start token if (balanceEnd < startToken) { break loop; } type = this.offsetAndType[cursor] >> TYPE_SHIFT; // check token is stop type switch (mode(type, this.source, offset)) { case 1: break loop; case 2: cursor++; break loop; default: // fast forward to the end of balanced block if (this.balance[balanceEnd] === cursor) { cursor = balanceEnd; } offset = this.offsetAndType[cursor] & OFFSET_MASK; } } return cursor - this.tokenIndex; }, isBalanceEdge: function(pos) { return this.balance[this.tokenIndex] < pos; }, isDelim: function(code, offset) { if (offset) { return ( this.lookupType(offset) === TYPE.Delim && this.source.charCodeAt(this.lookupOffset(offset)) === code ); } return ( this.tokenType === TYPE.Delim && this.source.charCodeAt(this.tokenStart) === code ); }, getTokenValue: function() { return this.source.substring(this.tokenStart, this.tokenEnd); }, getTokenLength: function() { return this.tokenEnd - this.tokenStart; }, substrToCursor: function(start) { return this.source.substring(start, this.tokenStart); }, skipWS: function() { for (var i = this.tokenIndex, skipTokenCount = 0; i < this.tokenCount; i++, skipTokenCount++) { if ((this.offsetAndType[i] >> TYPE_SHIFT) !== WHITESPACE) { break; } } if (skipTokenCount > 0) { this.skip(skipTokenCount); } }, skipSC: function() { while (this.tokenType === WHITESPACE || this.tokenType === COMMENT) { this.next(); } }, skip: function(tokenCount) { var next = this.tokenIndex + tokenCount; if (next < this.tokenCount) { this.tokenIndex = next; this.tokenStart = this.offsetAndType[next - 1] & OFFSET_MASK; next = this.offsetAndType[next]; this.tokenType = next >> TYPE_SHIFT; this.tokenEnd = next & OFFSET_MASK; } else { this.tokenIndex = this.tokenCount; this.next(); } }, next: function() { var next = this.tokenIndex + 1; if (next < this.tokenCount) { this.tokenIndex = next; this.tokenStart = this.tokenEnd; next = this.offsetAndType[next]; this.tokenType = next >> TYPE_SHIFT; this.tokenEnd = next & OFFSET_MASK; } else { this.tokenIndex = this.tokenCount; this.eof = true; this.tokenType = EOF; this.tokenStart = this.tokenEnd = this.source.length; } }, forEachToken(fn) { for (var i = 0, offset = this.firstCharOffset; i < this.tokenCount; i++) { var start = offset; var item = this.offsetAndType[i]; var end = item & OFFSET_MASK; var type = item >> TYPE_SHIFT; offset = end; fn(type, start, end, i); } }, dump() { var tokens = new Array(this.tokenCount); this.forEachToken((type, start, end, index) => { tokens[index] = { idx: index, type: NAME[type], chunk: this.source.substring(start, end), balance: this.balance[index] }; }); return tokens; } }; module.exports = TokenStream;