diff --git a/magic-editor/src/console/src/scripts/editor/high-light.js b/magic-editor/src/console/src/scripts/editor/high-light.js index fe6d2db2..c1ed2611 100644 --- a/magic-editor/src/console/src/scripts/editor/high-light.js +++ b/magic-editor/src/console/src/scripts/editor/high-light.js @@ -1,7 +1,9 @@ export const HighLightOptions = { escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/, builtinFunctions: [], - digits: /\d+(_+\d+)*/, + digits: /[0-9_]+/, + binarydigits: /[0-1_]+/, + hexdigits: /[[0-9a-fA-F_]+/, regexpctl: /[(){}\[\]\$\^|\-*+?\.]/, regexpesc: /\\(?:[bBdDfnrstvwWn0\\\/]|@regexpctl|c[A-Z]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4})/, tokenizer: { @@ -15,13 +17,16 @@ export const HighLightOptions = { [/[a-zA-Z_$][\w$]*[\s]?/, { cases: { '@builtinFunctions': 'predefined', - "~(new|var|if|else|for|in|return|import|break|continue|as|null|true|false|try|catch|finally|async|while|exit|asc|desc|ASC|DESC|assert)[\\s]?": {token: "keywords"}, + "~(new|var|if|else|for|in|return|import|break|continue|as|null|true|false|try|catch|finally|async|while|exit|asc|desc|ASC|DESC|assert|let|const)[\\s]?": {token: "keywords"}, "~(select|from|left|join|on|and|or|order|by|where|group|having|SELECT|FROM|LEFT|JOIN|ON|AND|OR|ORDER|BY|WHERE|GROUP|HAVING)[\\s]{1}": {token: "keywords"}, "@default": "identifier" } }], [/::[a-zA-Z]+/, 'keywords'], [/[{}()[\]]/, '@brackets'], + [/(@digits)\.(@digits)/, 'number.float'], + [/0[xX](@hexdigits)n?/, 'number.hex'], + [/0[bB](@binarydigits)n?/, 'number.binary'], [/(@digits)[lLbBsSdDfFmM]?/, 'number'], [/\/\*/, 'comment', '@comment'], [/\/\//, 'comment', '@commentTodo'], @@ -35,6 +40,7 @@ export const HighLightOptions = { [/'([^'\\]|\\.)*$/, 'string.invalid'], [/"/, 'string', '@string_double'], [/'/, 'string', '@string_single'], + [/`/, 'string', '@string_backtick'] ], comment: [ [/((TODO)|(todo)|(fixme)|(FIXME))[ \t]+[^\n(?!\*\/)]+/, 'comment.todo'], @@ -108,5 +114,17 @@ export const HighLightOptions = { [/\\./, 'string.escape.invalid'], [/'/, 'string', '@pop'] ], + string_backtick: [ + [/\$\{/, { token: 'delimiter.bracket', next: '@bracketCounting' }], + [/[^\\`$]+/, 'string'], + [/@escapes/, 'string.escape'], + [/\\./, 'string.escape.invalid'], + [/`/, 'string', '@pop'] + ], + bracketCounting: [ + [/\{/, 'delimiter.bracket', '@bracketCounting'], + [/\}/, 'delimiter.bracket', '@pop'], + { include: 'root' } + ] } }; \ No newline at end of file diff --git a/magic-editor/src/console/src/scripts/parsing/ast.js b/magic-editor/src/console/src/scripts/parsing/ast.js index 8a048071..d04c4f2e 100644 --- a/magic-editor/src/console/src/scripts/parsing/ast.js +++ b/magic-editor/src/console/src/scripts/parsing/ast.js @@ -451,33 +451,35 @@ class BinaryOperation extends Node { async getJavaType(env) { let lType = await this.left.getJavaType(env); let rType = await this.right.getJavaType(env); - if (this.operator.type == TokenType.Plus || this.operator.type == TokenType.PlusEqual) { - if (lType == 'string' || rType == 'string' || lType == 'java.lang.String' || rType == 'java.lang.String') { + lType = lType.toLowerCase().substring(lType.lastIndexOf(".") + 1) + rType = rType.toLowerCase().substring(rType.lastIndexOf(".") + 1) + if (this.operator.type === TokenType.Plus || this.operator.type === TokenType.PlusEqual) { + if (lType === 'string' || rType === 'string') { return 'java.lang.String'; } } - if (this.operator.type == TokenType.Equal || (this.operator.type == TokenType.Assignment && this.linqLevel > 0)) { + if (this.operator.type === TokenType.Equal || (this.operator.type === TokenType.Assignment && this.linqLevel > 0)) { return 'java.lang.Boolean'; } - if (lType == 'BigDecimal' || rType == 'BigDecimal') { + if (lType === 'bigdecimal' || rType === 'bigdecimal') { return 'java.math.BigDecimal'; } - if (lType == 'double' || rType == 'double') { + if (lType === 'double' || rType === 'double') { return 'java.lang.Double'; } - if (lType == 'float' || rType == 'float') { + if (lType === 'float' || rType === 'float') { return 'java.lang.Float'; } - if (lType == 'long' || rType == 'long') { + if (lType === 'long' || rType === 'long') { return 'java.lang.Long'; } - if (lType == 'int' || rType == 'int') { + if (lType === 'integer' || rType === 'integer') { return 'java.lang.Integer'; } - if (lType == 'short' || rType == 'short') { + if (lType === 'short' || rType === 'short') { return 'java.lang.Short'; } - if (lType == 'byte' || rType == 'byte') { + if (lType === 'byte' || rType === 'byte') { return 'java.lang.Byte'; } return 'java.lang.Object'; diff --git a/magic-editor/src/console/src/scripts/parsing/index.js b/magic-editor/src/console/src/scripts/parsing/index.js index ec91b1a4..957a9995 100644 --- a/magic-editor/src/console/src/scripts/parsing/index.js +++ b/magic-editor/src/console/src/scripts/parsing/index.js @@ -156,12 +156,26 @@ const TokenType = { Or: {literal: '||', error: '||'}, Xor: {literal: '^', error: '^'}, Not: {literal: '!', error: '!'}, + BitAnd: {literal:'&', error: '&'}, + BitOr: {literal:'|', error: '|'}, + BitNot: {literal:'~', error: '~'}, + LShift: {literal:'<<', error: '<<'}, + RShift: {literal:'>>', error: '>>'}, + RShift2: {literal:'>>>', error: '>>>'}, + XorEqual: {literal:'^=', error: '^=', modifiable: true}, + BitAndEqual: {literal:'&=', error: '&=', modifiable: true}, + BitOrEqual: {literal:'|=', error: '|=', modifiable: true}, + LShiftEqual: {literal:'<<=', error: '<<=', modifiable: true}, + RShiftEqual: {literal:'>>=', error: '>>=', modifiable: true}, + RShift2Equal: {literal:'>>>=', error: '>>>=', modifiable: true}, + SqlAnd: {literal: 'and', error: 'and', inLinq: true}, SqlOr: {literal: 'or', error: 'or', inLinq: true}, SqlNotEqual: {literal: '<>', error: '<>', inLinq: true}, Questionmark: {literal: '?', error: '?'}, DoubleQuote: {literal: '"', error: '"'}, + TripleQuote: {literal: '"""', error: '"""'}, SingleQuote: {literal: '\'', error: '\''}, Lambda: {error: '=> 或 ->'}, BooleanLiteral: {error: 'true 或 false'}, @@ -200,15 +214,24 @@ TokenType.getSortedValues = function () { }; class Token { - constructor(tokenType, span) { + constructor(tokenType, span, valueOrTokenStream) { this.type = tokenType; this.span = span; + if(valueOrTokenStream instanceof TokenStream){ + this.tokenStream = valueOrTokenStream; + }else if(valueOrTokenStream){ + this.value = valueOrTokenStream; + } } getTokenType() { return this.type; } + getTokenStream() { + return this.tokenStream; + } + getSpan() { return this.span; } @@ -219,8 +242,8 @@ class Token { } class LiteralToken extends Token { - constructor(tokenType, span) { - super(tokenType, span) + constructor(tokenType, span, valueOrTokenStream) { + super(tokenType, span, valueOrTokenStream) } } @@ -258,6 +281,14 @@ class CharacterStream { this.index += needleLength; return true; } + matchAny(strs, consume) { + for(let i=0,len = strs.length; i < len;i++){ + if(this.match(strs[i], consume)){ + return true; + } + } + return false; + } matchDigit(consume) { if (this.index >= this.end) diff --git a/magic-editor/src/console/src/scripts/parsing/parser.js b/magic-editor/src/console/src/scripts/parsing/parser.js index 85639c77..07971a59 100644 --- a/magic-editor/src/console/src/scripts/parsing/parser.js +++ b/magic-editor/src/console/src/scripts/parsing/parser.js @@ -37,26 +37,34 @@ import { LanguageExpression } from './ast.js' -export const keywords = ["import", "as", "var", "return", "break", "continue", "if", "for", "in", "new", "true", "false", "null", "else", "try", "catch", "finally", "async", "while", "exit", "and", "or", /*"assert"*/]; +export const keywords = ["import", "as", "var", "let", "const", "return", "break", "continue", "if", "for", "in", "new", "true", "false", "null", "else", "try", "catch", "finally", "async", "while", "exit", "and", "or", /*"assert"*/]; export const linqKeywords = ["from", "join", "left", "group", "by", "as", "having", "and", "or", "in", "where", "on"]; const binaryOperatorPrecedence = [ [TokenType.Assignment], - [TokenType.PlusEqual, TokenType.MinusEqual, TokenType.AsteriskEqual, TokenType.ForwardSlashEqual, TokenType.PercentEqual], - [TokenType.Or, TokenType.And, TokenType.SqlOr, TokenType.SqlAnd, TokenType.Xor], + [TokenType.RShift2Equal, TokenType.RShiftEqual, TokenType.LShiftEqual, TokenType.XorEqual, TokenType.BitOrEqual, TokenType.BitAndEqual, TokenType.PercentEqual, TokenType.ForwardSlashEqual, TokenType.AsteriskEqual, TokenType.MinusEqual, TokenType.PlusEqual], + [TokenType.Or, TokenType.And, TokenType.SqlOr, TokenType.SqlAnd], + [TokenType.BitOr], + [TokenType.Xor], + [TokenType.BitAnd], [TokenType.EqualEqualEqual, TokenType.Equal, TokenType.NotEqualEqual, TokenType.NotEqual, TokenType.SqlNotEqual], [TokenType.Plus, TokenType.Minus], [TokenType.Less, TokenType.LessEqual, TokenType.Greater, TokenType.GreaterEqual], + [TokenType.LShift, TokenType.RShift, TokenType.RShift2], [TokenType.ForwardSlash, TokenType.Asterisk, TokenType.Percentage] ]; const linqBinaryOperatorPrecedence = [ - [TokenType.PlusEqual, TokenType.MinusEqual, TokenType.AsteriskEqual, TokenType.ForwardSlashEqual, TokenType.PercentEqual], + [TokenType.RShift2Equal, TokenType.RShiftEqual, TokenType.LShiftEqual, TokenType.XorEqual, TokenType.BitOrEqual, TokenType.BitAndEqual, TokenType.PercentEqual, TokenType.ForwardSlashEqual, TokenType.AsteriskEqual, TokenType.MinusEqual, TokenType.PlusEqual], [TokenType.Or, TokenType.And, TokenType.SqlOr, TokenType.SqlAnd, TokenType.Xor], + [TokenType.BitOr], + [TokenType.Xor], + [TokenType.BitAnd], [TokenType.Assignment, TokenType.EqualEqualEqual, TokenType.Equal, TokenType.NotEqualEqual, TokenType.Equal, TokenType.NotEqual, TokenType.SqlNotEqual], [TokenType.Plus, TokenType.Minus], [TokenType.Less, TokenType.LessEqual, TokenType.Greater, TokenType.GreaterEqual], + [TokenType.LShift, TokenType.RShift, TokenType.RShift2], [TokenType.ForwardSlash, TokenType.Asterisk, TokenType.Percentage] ] -const unaryOperators = [TokenType.Not, TokenType.PlusPlus, TokenType.MinusMinus, TokenType.Plus, TokenType.Minus]; +const unaryOperators = [TokenType.MinusMinus, TokenType.PlusPlus, TokenType.BitNot, TokenType.Minus, TokenType.Plus, TokenType.Not]; export class Parser { constructor(stream) { @@ -75,10 +83,10 @@ export class Parser { } } } catch (e) { + //console.error(e) if (ignoreError !== true) { throw e; } - //console.error(e) } return nodes; } @@ -93,7 +101,7 @@ export class Parser { let result = null; if (this.stream.match("import", false)) { result = this.parseImport(); - } else if (this.stream.match("var", false)) { + } else if (this.stream.match(["var", "let", "const"], false)) { result = this.parseVarDefine(); } else if (this.stream.match("if", false)) { result = this.parseIfStatement(); @@ -236,7 +244,7 @@ export class Parser { } parseNewExpression(opening) { - let identifier = this.stream.expect(TokenType.Identifier); + let expression = this.parseAccessOrCall(TokenType.Identifier, true); let args = this.parseArguments(); let closing = this.stream.expect(")").getSpan(); return this.parseConverterOrAccessOrCall(new NewStatement(new Span(opening, closing), identifier.getText(), args)); @@ -271,7 +279,7 @@ export class Parser { } parseVarDefine() { - let opening = this.stream.expect("var").getSpan(); + let opening = this.stream.consume().getSpan(); let token = this.stream.expect(TokenType.Identifier); this.checkKeyword(token.getSpan()); if (this.stream.match(TokenType.Assignment, true)) { @@ -441,7 +449,7 @@ export class Parser { return new Spread(new Span(spread.getSpan(), target.getSpan()), target); } - parseAccessOrCall(target) { + parseAccessOrCall(target, isNew) { if (target === TokenType.StringLiteral || target === TokenType.Identifier) { let identifier = this.stream.expect(target).getSpan(); if (target === TokenType.Identifier && "new" === identifier.getText()) { @@ -451,7 +459,7 @@ export class Parser { return this.parseLambdaBody(identifier, [identifier.getText()]); } let result = target === TokenType.StringLiteral ? new Literal(identifier, 'java.lang.String') : new VariableAccess(identifier, identifier.getText()); - return this.parseAccessOrCall(result); + return this.parseAccessOrCall(result, isNew); } else { while (this.stream.hasMore() && this.stream.match([TokenType.LeftParantheses, TokenType.LeftBracket, TokenType.Period, TokenType.QuestionPeriod], false)) { // function or method call @@ -465,6 +473,9 @@ export class Parser { } else { throw new ParseException("Expected a variable, field or method.", this.stream.hasMore() ? this.stream.consume().getSpan() : this.stream.getPrev().getSpan()); } + if(isNew){ + break; + } } // map or array access @@ -499,7 +510,7 @@ export class Parser { let key; if (this.stream.hasPrev()) { let prev = this.stream.getPrev(); - if (this.stream.match(TokenType.Spread, false) && (prev.getTokenType() == TokenType.LeftCurly || prev.getTokenType() == TokenType.Comma)) { + if (this.stream.match(TokenType.Spread, false) && (prev.getTokenType() === TokenType.LeftCurly || prev.getTokenType() === TokenType.Comma)) { let spread = this.stream.expect(TokenType.Spread); keys.push(spread); values.push(this.parseSpreadAccess(spread)); @@ -520,7 +531,7 @@ export class Parser { if (key.getTokenType() === TokenType.Identifier) { values.push(new VariableAccess(key.getSpan(), key.getText())); } else { - values.push(new StringLiteral(key.getSpan(), 'java.lang.String')); + values.push(new Literal(key.getSpan(), 'java.lang.String')); } } else { this.stream.expect(":"); @@ -719,7 +730,7 @@ export class Parser { let token = this.stream.consume(); let index = this.stream.makeIndex(); try { - if (token.type === TokenType.Identifier && token.getText() === 'var') { + if (token.type === TokenType.Identifier && ['var','let','const'].indexOf(token.getText()) > -1 ) { let varName = this.stream.consume().getText(); if (this.stream.match(TokenType.Assignment, true)) { let isAsync = this.stream.match("async", true); diff --git a/magic-editor/src/console/src/scripts/parsing/tokenizer.js b/magic-editor/src/console/src/scripts/parsing/tokenizer.js index bffe5111..be932159 100644 --- a/magic-editor/src/console/src/scripts/parsing/tokenizer.js +++ b/magic-editor/src/console/src/scripts/parsing/tokenizer.js @@ -1,9 +1,9 @@ -import {CharacterStream, LiteralToken, ParseException, Token, TokenType} from './index.js' +import {CharacterStream, LiteralToken, ParseException, Token, TokenStream, TokenType} from './index.js' -let regexpToken = (stream, tokens) => { +const regexpToken = (stream, tokens) => { if (tokens.length > 0) { let token = tokens[tokens.length - 1]; - if (token instanceof LiteralToken || token.getTokenType() == TokenType.Identifier) { + if (token instanceof LiteralToken || token.getTokenType() === TokenType.Identifier) { return false; } } @@ -27,7 +27,7 @@ let regexpToken = (stream, tokens) => { } else if (deep > 0 && stream.match("]", false)) { deep--; } else if (stream.match(TokenType.ForwardSlash.literal, true)) { - if (deep == 0) { + if (deep === 0) { if (stream.match("g", true)) { } if (stream.match("i", true)) { @@ -47,12 +47,12 @@ let regexpToken = (stream, tokens) => { } } let ch = stream.consume(); - if (ch == '\r' || ch == '\n') { + if (ch === '\r' || ch === '\n') { stream.reset(mark); return false; } } - if (deep != 0) { + if (deep !== 0) { throw new ParseException("Missing ']'", stream.getSpan(maybeMissForwardSlash, maybeMissForwardSlashEnd - 1)); } if (!matchedEndQuote) { @@ -67,9 +67,9 @@ let regexpToken = (stream, tokens) => { return false; } -let stringToken = (stream, tokens) => { +const tokenizerString = (stream, tokenType, tokens) => { // String literal - if (stream.match(TokenType.SingleQuote.literal, true)) { + if (stream.match(tokenType, true)) { stream.startSpan(); let matchedEndQuote = false; while (stream.hasMore()) { @@ -78,85 +78,196 @@ let stringToken = (stream, tokens) => { stream.consume(); continue; } - if (stream.match(TokenType.SingleQuote.literal, true)) { + if (stream.match(tokenType.literal, true)) { matchedEndQuote = true; break; } let ch = stream.consume(); - if (ch == '\r' || ch == '\n') { - throw new ParseException("''定义的字符串不能换行", stream.endSpan()); + if (tokenType !== TokenType.TripleQuote && (ch === '\r' || ch === '\n')) { + throw new ParseException(tokenType.getError() + tokenType.getError() + "定义的字符串不能换行", stream.endSpan()); } } if (!matchedEndQuote) { - throw new ParseException("字符串没有结束符\'", stream.endSpan()); + throw new ParseException("字符串没有结束符" + tokenType.error, stream.endSpan()); } let stringSpan = stream.endSpan(); - stringSpan = stream.getSpan(stringSpan.getStart() - 1, stringSpan.getEnd()); - tokens.push(new LiteralToken(TokenType.StringLiteral, stringSpan)); - return true; - } - - // String literal - if (stream.match('"""', true)) { - stream.startSpan(); - let matchedEndQuote = false; - while (stream.hasMore()) { - // Note: escape sequences like \n are parsed in StringLiteral - if (stream.match("\\", true)) { - stream.consume(); - continue; - } - if (stream.match('"""', true)) { - matchedEndQuote = true; - break; - } - stream.consume(); - } - if (!matchedEndQuote) { - throw new ParseException('多行字符串没有结束符"""', stream.endSpan()); - } - let stringSpan = stream.endSpan(); - stringSpan = stream.getSpan(stringSpan.getStart() - 1, stringSpan.getEnd() - 2); - tokens.push(new LiteralToken(TokenType.StringLiteral, stringSpan)); - return true; - } - - // String literal - if (stream.match(TokenType.DoubleQuote.literal, true)) { - stream.startSpan(); - let matchedEndQuote = false; - while (stream.hasMore()) { - // Note: escape sequences like \n are parsed in StringLiteral - if (stream.match("\\", true)) { - stream.consume(); - continue; - } - if (stream.match(TokenType.DoubleQuote.literal, true)) { - matchedEndQuote = true; - break; - } - let ch = stream.consume(); - if (ch === '\r' || ch === '\n') { - throw new ParseException("\"\"定义的字符串不能换行", stream.endSpan()); - } - } - if (!matchedEndQuote) { - throw new ParseException("字符串没有结束符\"", stream.endSpan()); - } - let stringSpan = stream.endSpan(); - stringSpan = stream.getSpan(stringSpan.getStart(), stringSpan.getEnd() - 1); + stringSpan = stream.getSpan(stringSpan.getStart(), stringSpan.getEnd() - tokenType.literal.length); tokens.push(new LiteralToken(TokenType.StringLiteral, stringSpan)); return true; } return false; }; -export default (source) => { - let stream = new CharacterStream(source, 0, source.length); - let tokens = []; +const autoNumberType = (span, radix) => { + let value = Number.parseInt(span.getText().substring(2).replace(/\_/g,''), radix) + if (value > 0x7fffffff || value < -0x80000000) { + return new LiteralToken(TokenType.LongLiteral, span, value); + } else if (value > 127 || value < -128) { + return new LiteralToken(TokenType.LongLiteral, span, value); + } + return new LiteralToken(TokenType.ByteLiteral, span, value); +} +const tokenizerNumber = (stream, tokens) => { + if (stream.match('0', false)) { + let index = stream.getPosition(); + stream.startSpan(); + stream.consume(); + if (stream.matchAny(['x', 'X'], true)) { + while (stream.matchDigit(true) || stream.matchAny(["A", "B", "C", "D", "E", "F", "a", "b", "c", "d", "e", "f", "_"], true)) { + ; + } + if (stream.matchAny(["L", "l"], true)) { + let span = stream.endSpan(); + let text = span.getText(); + tokens.push(new LiteralToken(TokenType.LongLiteral, span, parseInt(text.substring(2, text.length() - 1).replace(/\_/g,''), 16))); + return true; + } + tokens.push(autoNumberType(stream.endSpan(), 16)); + return true; + } else if (stream.matchAny(['b','B'], true)){ + while (stream.matchAny([ '0', '1', '_'], true)) { + ; + } + if (stream.matchAny([ "L", "l"], true)) { + let span = stream.endSpan(); + let text = span.getText(); + tokens.push(new LiteralToken(TokenType.LongLiteral, span, parseInt(text.substring(0, text.length() - 1).replace(/\_/g,''), 2))); + return true; + } + tokens.push(autoNumberType(stream.endSpan(), 2)); + return true; + } + stream.reset(index); + } + if (stream.matchDigit(false)) { + let type = TokenType.IntegerLiteral; + stream.startSpan(); + while (stream.matchDigit(true) || stream.match('_', true)) { + } + if (stream.match(TokenType.Period.literal, true)) { + type = TokenType.DoubleLiteral; + while (stream.matchDigit(true) || stream.match('_',true)) { + } + } + if (stream.matchAny(['b', 'B'], true)) { + if (type === TokenType.DoubleLiteral) { + throw new ParseException('Byte literal can not have a decimal point.', stream.endSpan()); + } + type = TokenType.ByteLiteral; + } else if (stream.matchAny(['s', 'S'], true)) { + if (type === TokenType.DoubleLiteral) { + throw new ParseException('Short literal can not have a decimal point.', stream.endSpan()); + } + type = TokenType.ShortLiteral; + } else if (stream.matchAny(['l', 'L'], true)) { + if (type === TokenType.DoubleLiteral) { + throw new ParseException('Long literal can not have a decimal point.', stream.endSpan()); + } + type = TokenType.LongLiteral; + } else if (stream.matchAny(['f', 'F'], true)) { + type = TokenType.FloatLiteral; + } else if (stream.matchAny(['d', 'D'], true)) { + type = TokenType.DoubleLiteral; + } else if (stream.matchAny(['m', 'M'], true)) { + type = TokenType.DecimalLiteral; + } + tokens.push(new LiteralToken(type, stream.endSpan())); + return true + } + return false; +} + +const tokenizerLanguage = (stream, tokens) => { + if (stream.match("```", true)) { + stream.startSpan(); + if (stream.matchIdentifierStart(true)) { + while (stream.matchIdentifierPart(true)) { + } + let language = stream.endSpan(); + tokens.push(new Token(TokenType.Language, language)); + stream.startSpan(); + if (!stream.skipUntil("```")) { + throw new ParseException('```需要以```结尾', stream.endSpan()); + } + tokens.push(new Token(TokenType.Language, stream.endSpan(-3))); + return true; + } else { + throw new ParseException('```后需要标识语言类型', stream.endSpan()); + } + } + return false; +} +const tokenizerIdentifier = (stream, tokens) => { + if (stream.matchIdentifierStart(true)) { + stream.startSpan(); + while (stream.matchIdentifierPart(true)) { + } + let identifierSpan = stream.endSpan(); + identifierSpan = stream.getSpan(identifierSpan.getStart() - 1, identifierSpan.getEnd()); + if ("true" === identifierSpan.getText() || "false" === identifierSpan.getText()) { + tokens.push(new LiteralToken(TokenType.BooleanLiteral, identifierSpan)); + } else if ("null" === identifierSpan.getText()) { + tokens.push(new LiteralToken(TokenType.NullLiteral, identifierSpan)); + } else if (TokenType.SqlAnd.literal === identifierSpan.getText()) { + tokens.push(new Token(TokenType.SqlAnd, identifierSpan)); + } else if (TokenType.SqlOr.literal === identifierSpan.getText()) { + tokens.push(new Token(TokenType.SqlOr, identifierSpan)); + } else { + tokens.push(new Token(TokenType.Identifier, identifierSpan)); + } + return true; + } + return false; +} + +const tokenizerTemplateString = (stream, tokens)=>{ + if (stream.match("`", true)) { + let begin = stream.getPosition(); + let start = begin; + let matchedEndQuote = false; + let subTokens = []; + while (stream.hasMore()) { + if (stream.match("\\", true)) { + stream.consume(); + continue; + } + if (stream.match("`", true)) { + matchedEndQuote = true; + break; + } + if (stream.match("${", true)) { + let end = stream.getPosition(); + if (start < end - 2) { + subTokens.push(new LiteralToken(TokenType.StringLiteral, stream.endSpan(start, end - 2))); + } + subTokens.push(...tokenizer(stream, [], "}")); + start = stream.getPosition(); + continue; + } + stream.consume(); + } + if (!matchedEndQuote) { + throw new ParseException("模板字符串没有结束符`", stream.endSpan()); + } + let stringSpan = stream.endSpan(begin, stream.getPosition()); + let end = stream.getPosition() - 1; + if (end - start > 0) { + subTokens.push(new LiteralToken(TokenType.StringLiteral, stream.endSpan(start, end))); + } + stringSpan = stream.getSpan(stringSpan.getStart() - 1, stringSpan.getEnd()); + tokens.push(new LiteralToken(TokenType.StringLiteral, stringSpan, new TokenStream(subTokens))); + return true; + } + return false; +} + +const tokenizer = (stream, tokens, except) => { let leftCount = 0; let rightCount = 0; while (stream.hasMore()) { stream.skipWhiteSpace(); + if (except && stream.match(except, true)) { + return tokens; + } if (stream.match("//", true)) { //注释 stream.skipLine(); continue; @@ -165,43 +276,12 @@ export default (source) => { stream.skipUntil("*/"); continue; } - if (stream.matchDigit(false)) { - let type = TokenType.IntegerLiteral; - stream.startSpan(); - while (stream.matchDigit(true)) { - } - if (stream.match(TokenType.Period.literal, true)) { - type = TokenType.DoubleLiteral; - while (stream.matchDigit(true)) { - } - } - if (stream.match("b", true) || stream.match("B", true)) { - if (type === TokenType.DoubleLiteral) { - throw new ParseException('Byte literal can not have a decimal point.', stream.endSpan()); - } - type = TokenType.ByteLiteral; - } else if (stream.match("s", true) || stream.match("S", true)) { - if (type === TokenType.DoubleLiteral) { - throw new ParseException('Short literal can not have a decimal point.', stream.endSpan()); - } - type = TokenType.ShortLiteral; - } else if (stream.match("l", true) || stream.match("L", true)) { - if (type === TokenType.DoubleLiteral) { - throw new ParseException('Long literal can not have a decimal point.', stream.endSpan()); - } - type = TokenType.LongLiteral; - } else if (stream.match("f", true) || stream.match("F", true)) { - type = TokenType.FloatLiteral; - } else if (stream.match("d", true) || stream.match("D", true)) { - type = TokenType.DoubleLiteral; - } else if (stream.match("m", true) || stream.match("M", true)) { - type = TokenType.DecimalLiteral; - } - tokens.push(new LiteralToken(type, stream.endSpan())); + // int short double long float byte decimal + if (tokenizerNumber(stream, tokens)) { continue; } - // string - if (stringToken(stream, tokens)) { + // '' "" """ """ + if (tokenizerString(stream, TokenType.SingleQuote, tokens) || tokenizerString(stream, TokenType.TripleQuote, tokens) || tokenizerString(stream, TokenType.DoubleQuote, tokens)) { continue; } @@ -209,45 +289,21 @@ export default (source) => { if (regexpToken(stream, tokens)) { continue; } - - // TODO exception - if (stream.match("```", true)) { - stream.startSpan(); - if (stream.matchIdentifierStart(true)) { - while (stream.matchIdentifierPart(true)) { - } - let language = stream.endSpan(); - tokens.push(new Token(TokenType.Language, language)); - stream.startSpan(); - if (!stream.skipUntil("```")) { - throw new ParseException('```需要以```结尾', stream.endSpan()); - } - tokens.push(new Token(TokenType.Language, stream.endSpan(-3))); - } else { - throw new ParseException('```后需要标识语言类型', stream.endSpan()); - } - } - // Identifier, keyword, boolean literal, or null literal - if (stream.matchIdentifierStart(true)) { - stream.startSpan(); - while (stream.matchIdentifierPart(true)) { - } - let identifierSpan = stream.endSpan(); - identifierSpan = stream.getSpan(identifierSpan.getStart() - 1, identifierSpan.getEnd()); - if ("true" === identifierSpan.getText() || "false" === identifierSpan.getText()) { - tokens.push(new LiteralToken(TokenType.BooleanLiteral, identifierSpan)); - } else if ("null" === identifierSpan.getText()) { - tokens.push(new LiteralToken(TokenType.NullLiteral, identifierSpan)); - } else if (TokenType.SqlAnd.literal === identifierSpan.getText()) { - tokens.push(new Token(TokenType.SqlAnd, identifierSpan)); - } else if (TokenType.SqlOr.literal === identifierSpan.getText()) { - tokens.push(new Token(TokenType.SqlOr, identifierSpan)); - } else { - tokens.push(new Token(TokenType.Identifier, identifierSpan)); - } + // ``` ``` + if(tokenizerLanguage(stream, tokens)){ continue; } - if (stream.match("=>", true) || stream.match("->", true)) { + // template string + if (tokenizerTemplateString(stream, tokens)) { + continue; + } + + // Identifier, keyword, boolean literal, or null literal + if(tokenizerIdentifier(stream, tokens)){ + continue; + } + // lambda + if (stream.matchAny(['=>','->'], true)) { tokens.push(new Token(TokenType.Lambda, stream.getSpan(stream.getPosition() - 2, stream.getPosition()))); continue; } @@ -280,4 +336,8 @@ export default (source) => { } } return tokens; +} + +export default (source) => { + return tokenizer(new CharacterStream(source, 0, source.length), []) } \ No newline at end of file