lywsvip/magic-api
1
0
Fork 0
mirror of https://gitee.com/ssssssss-team/magic-api.git synced 2026-06-09 18:32:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化代码提示、优化代码高亮，兼容asm分支

Browse Source
This commit is contained in:
mxd
2021-08-05 21:51:49 +08:00
parent d15ff3339a
commit 2c44fa47f7
5 changed files with 292 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
magic-editor/src/console/src/scripts/editor/high-light.js
View File

@@ -1,7 +1,9 @@
export const HighLightOptions = {
escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
builtinFunctions: [],
digits: /\d+(_+\d+)*/,
digits: /[0-9_]+/,
binarydigits: /[0-1_]+/,
hexdigits: /[[0-9a-fA-F_]+/,
regexpctl: /[(){}\[\]\$\^|\-*+?\.]/,
regexpesc: /\\(?:[bBdDfnrstvwWn0\\\/]|@regexpctl|c[A-Z]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4})/,
tokenizer: {
@@ -15,13 +17,16 @@ export const HighLightOptions = {
[/[a-zA-Z_$][\w$]*[\s]?/, {
cases: {
'@builtinFunctions': 'predefined',
"~(new|var|if|else|for|in|return|import|break|continue|as|null|true|false|try|catch|finally|async|while|exit|asc|desc|ASC|DESC|assert)[\\s]?": {token: "keywords"},
"~(new|var|if|else|for|in|return|import|break|continue|as|null|true|false|try|catch|finally|async|while|exit|asc|desc|ASC|DESC|assert|let|const)[\\s]?": {token: "keywords"},
"~(select|from|left|join|on|and|or|order|by|where|group|having|SELECT|FROM|LEFT|JOIN|ON|AND|OR|ORDER|BY|WHERE|GROUP|HAVING)[\\s]{1}": {token: "keywords"},
"@default": "identifier"
}
}],
[/::[a-zA-Z]+/, 'keywords'],
[/[{}()[\]]/, '@brackets'],
[/(@digits)\.(@digits)/, 'number.float'],
[/0[xX](@hexdigits)n?/, 'number.hex'],
[/0[bB](@binarydigits)n?/, 'number.binary'],
[/(@digits)[lLbBsSdDfFmM]?/, 'number'],
[/\/\*/, 'comment', '@comment'],
[/\/\//, 'comment', '@commentTodo'],
@@ -35,6 +40,7 @@ export const HighLightOptions = {
[/'([^'\\]|\\.)*$/, 'string.invalid'],
[/"/, 'string', '@string_double'],
[/'/, 'string', '@string_single'],
[/`/, 'string', '@string_backtick']
],
comment: [
[/((TODO)|(todo)|(fixme)|(FIXME))[ \t]+[^\n(?!\*\/)]+/, 'comment.todo'],
@@ -108,5 +114,17 @@ export const HighLightOptions = {
[/\\./, 'string.escape.invalid'],
[/'/, 'string', '@pop']
],
string_backtick: [
[/\$\{/, { token: 'delimiter.bracket', next: '@bracketCounting' }],
[/[^\\`$]+/, 'string'],
[/@escapes/, 'string.escape'],
[/\\./, 'string.escape.invalid'],
[/`/, 'string', '@pop']
],
bracketCounting: [
[/\{/, 'delimiter.bracket', '@bracketCounting'],
[/\}/, 'delimiter.bracket', '@pop'],
{ include: 'root' }
]
}
};

22
magic-editor/src/console/src/scripts/parsing/ast.js
View File

@@ -451,33 +451,35 @@ class BinaryOperation extends Node {
async getJavaType(env) {
let lType = await this.left.getJavaType(env);
let rType = await this.right.getJavaType(env);
if (this.operator.type == TokenType.Plus || this.operator.type == TokenType.PlusEqual) {
if (lType == 'string' || rType == 'string' || lType == 'java.lang.String' || rType == 'java.lang.String') {
lType = lType.toLowerCase().substring(lType.lastIndexOf(".") + 1)
rType = rType.toLowerCase().substring(rType.lastIndexOf(".") + 1)
if (this.operator.type === TokenType.Plus || this.operator.type === TokenType.PlusEqual) {
if (lType === 'string' || rType === 'string') {
return 'java.lang.String';
}
}
if (this.operator.type == TokenType.Equal || (this.operator.type == TokenType.Assignment && this.linqLevel > 0)) {
if (this.operator.type === TokenType.Equal || (this.operator.type === TokenType.Assignment && this.linqLevel > 0)) {
return 'java.lang.Boolean';
}
if (lType == 'BigDecimal' || rType == 'BigDecimal') {
if (lType === 'bigdecimal' || rType === 'bigdecimal') {
return 'java.math.BigDecimal';
}
if (lType == 'double' || rType == 'double') {
if (lType === 'double' || rType === 'double') {
return 'java.lang.Double';
}
if (lType == 'float' || rType == 'float') {
if (lType === 'float' || rType === 'float') {
return 'java.lang.Float';
}
if (lType == 'long' || rType == 'long') {
if (lType === 'long' || rType === 'long') {
return 'java.lang.Long';
}
if (lType == 'int' || rType == 'int') {
if (lType === 'integer' || rType === 'integer') {
return 'java.lang.Integer';
}
if (lType == 'short' || rType == 'short') {
if (lType === 'short' || rType === 'short') {
return 'java.lang.Short';
}
if (lType == 'byte' || rType == 'byte') {
if (lType === 'byte' || rType === 'byte') {
return 'java.lang.Byte';
}
return 'java.lang.Object';

37
magic-editor/src/console/src/scripts/parsing/index.js
View File

@@ -156,12 +156,26 @@ const TokenType = {
Or: {literal: '||', error: '||'},
Xor: {literal: '^', error: '^'},
Not: {literal: '!', error: '!'},
BitAnd: {literal:'&', error: '&'},
BitOr: {literal:'|', error: '|'},
BitNot: {literal:'~', error: '~'},
LShift: {literal:'<<', error: '<<'},
RShift: {literal:'>>', error: '>>'},
RShift2: {literal:'>>>', error: '>>>'},
XorEqual: {literal:'^=', error: '^=', modifiable: true},
BitAndEqual: {literal:'&=', error: '&=', modifiable: true},
BitOrEqual: {literal:'|=', error: '|=', modifiable: true},
LShiftEqual: {literal:'<<=', error: '<<=', modifiable: true},
RShiftEqual: {literal:'>>=', error: '>>=', modifiable: true},
RShift2Equal: {literal:'>>>=', error: '>>>=', modifiable: true},
SqlAnd: {literal: 'and', error: 'and', inLinq: true},
SqlOr: {literal: 'or', error: 'or', inLinq: true},
SqlNotEqual: {literal: '<>', error: '<>', inLinq: true},
Questionmark: {literal: '?', error: '?'},
DoubleQuote: {literal: '"', error: '"'},
TripleQuote: {literal: '"""', error: '"""'},
SingleQuote: {literal: '\'', error: '\''},
Lambda: {error: '=> 或 ->'},
BooleanLiteral: {error: 'true 或 false'},
@@ -200,15 +214,24 @@ TokenType.getSortedValues = function () {
};
class Token {
constructor(tokenType, span) {
constructor(tokenType, span, valueOrTokenStream) {
this.type = tokenType;
this.span = span;
if(valueOrTokenStream instanceof TokenStream){
this.tokenStream = valueOrTokenStream;
}else if(valueOrTokenStream){
this.value = valueOrTokenStream;
}
}
getTokenType() {
return this.type;
}
getTokenStream() {
return this.tokenStream;
}
getSpan() {
return this.span;
}
@@ -219,8 +242,8 @@ class Token {
}
class LiteralToken extends Token {
constructor(tokenType, span) {
super(tokenType, span)
constructor(tokenType, span, valueOrTokenStream) {
super(tokenType, span, valueOrTokenStream)
}
}
@@ -258,6 +281,14 @@ class CharacterStream {
this.index += needleLength;
return true;
}
matchAny(strs, consume) {
for(let i=0,len = strs.length; i < len;i++){
if(this.match(strs[i], consume)){
return true;
}
}
return false;
}
matchDigit(consume) {
if (this.index >= this.end)

39
magic-editor/src/console/src/scripts/parsing/parser.js
View File

@@ -37,26 +37,34 @@ import {
LanguageExpression
} from './ast.js'
export const keywords = ["import", "as", "var", "return", "break", "continue", "if", "for", "in", "new", "true", "false", "null", "else", "try", "catch", "finally", "async", "while", "exit", "and", "or", /*"assert"*/];
export const keywords = ["import", "as", "var", "let", "const", "return", "break", "continue", "if", "for", "in", "new", "true", "false", "null", "else", "try", "catch", "finally", "async", "while", "exit", "and", "or", /*"assert"*/];
export const linqKeywords = ["from", "join", "left", "group", "by", "as", "having", "and", "or", "in", "where", "on"];
const binaryOperatorPrecedence = [
[TokenType.Assignment],
[TokenType.PlusEqual, TokenType.MinusEqual, TokenType.AsteriskEqual, TokenType.ForwardSlashEqual, TokenType.PercentEqual],
[TokenType.Or, TokenType.And, TokenType.SqlOr, TokenType.SqlAnd, TokenType.Xor],
[TokenType.RShift2Equal, TokenType.RShiftEqual, TokenType.LShiftEqual, TokenType.XorEqual, TokenType.BitOrEqual, TokenType.BitAndEqual, TokenType.PercentEqual, TokenType.ForwardSlashEqual, TokenType.AsteriskEqual, TokenType.MinusEqual, TokenType.PlusEqual],
[TokenType.Or, TokenType.And, TokenType.SqlOr, TokenType.SqlAnd],
[TokenType.BitOr],
[TokenType.Xor],
[TokenType.BitAnd],
[TokenType.EqualEqualEqual, TokenType.Equal, TokenType.NotEqualEqual, TokenType.NotEqual, TokenType.SqlNotEqual],
[TokenType.Plus, TokenType.Minus],
[TokenType.Less, TokenType.LessEqual, TokenType.Greater, TokenType.GreaterEqual],
[TokenType.LShift, TokenType.RShift, TokenType.RShift2],
[TokenType.ForwardSlash, TokenType.Asterisk, TokenType.Percentage]
];
const linqBinaryOperatorPrecedence = [
[TokenType.PlusEqual, TokenType.MinusEqual, TokenType.AsteriskEqual, TokenType.ForwardSlashEqual, TokenType.PercentEqual],
[TokenType.RShift2Equal, TokenType.RShiftEqual, TokenType.LShiftEqual, TokenType.XorEqual, TokenType.BitOrEqual, TokenType.BitAndEqual, TokenType.PercentEqual, TokenType.ForwardSlashEqual, TokenType.AsteriskEqual, TokenType.MinusEqual, TokenType.PlusEqual],
[TokenType.Or, TokenType.And, TokenType.SqlOr, TokenType.SqlAnd, TokenType.Xor],
[TokenType.BitOr],
[TokenType.Xor],
[TokenType.BitAnd],
[TokenType.Assignment, TokenType.EqualEqualEqual, TokenType.Equal, TokenType.NotEqualEqual, TokenType.Equal, TokenType.NotEqual, TokenType.SqlNotEqual],
[TokenType.Plus, TokenType.Minus],
[TokenType.Less, TokenType.LessEqual, TokenType.Greater, TokenType.GreaterEqual],
[TokenType.LShift, TokenType.RShift, TokenType.RShift2],
[TokenType.ForwardSlash, TokenType.Asterisk, TokenType.Percentage]
]
const unaryOperators = [TokenType.Not, TokenType.PlusPlus, TokenType.MinusMinus, TokenType.Plus, TokenType.Minus];
const unaryOperators = [TokenType.MinusMinus, TokenType.PlusPlus, TokenType.BitNot, TokenType.Minus, TokenType.Plus, TokenType.Not];
export class Parser {
constructor(stream) {
@@ -75,10 +83,10 @@ export class Parser {
}
}
} catch (e) {
//console.error(e)
if (ignoreError !== true) {
throw e;
}
//console.error(e)
}
return nodes;
}
@@ -93,7 +101,7 @@ export class Parser {
let result = null;
if (this.stream.match("import", false)) {
result = this.parseImport();
} else if (this.stream.match("var", false)) {
} else if (this.stream.match(["var", "let", "const"], false)) {
result = this.parseVarDefine();
} else if (this.stream.match("if", false)) {
result = this.parseIfStatement();
@@ -236,7 +244,7 @@ export class Parser {
}
parseNewExpression(opening) {
let identifier = this.stream.expect(TokenType.Identifier);
let expression = this.parseAccessOrCall(TokenType.Identifier, true);
let args = this.parseArguments();
let closing = this.stream.expect(")").getSpan();
return this.parseConverterOrAccessOrCall(new NewStatement(new Span(opening, closing), identifier.getText(), args));
@@ -271,7 +279,7 @@ export class Parser {
}
parseVarDefine() {
let opening = this.stream.expect("var").getSpan();
let opening = this.stream.consume().getSpan();
let token = this.stream.expect(TokenType.Identifier);
this.checkKeyword(token.getSpan());
if (this.stream.match(TokenType.Assignment, true)) {
@@ -441,7 +449,7 @@ export class Parser {
return new Spread(new Span(spread.getSpan(), target.getSpan()), target);
}
parseAccessOrCall(target) {
parseAccessOrCall(target, isNew) {
if (target === TokenType.StringLiteral || target === TokenType.Identifier) {
let identifier = this.stream.expect(target).getSpan();
if (target === TokenType.Identifier && "new" === identifier.getText()) {
@@ -451,7 +459,7 @@ export class Parser {
return this.parseLambdaBody(identifier, [identifier.getText()]);
}
let result = target === TokenType.StringLiteral ? new Literal(identifier, 'java.lang.String') : new VariableAccess(identifier, identifier.getText());
return this.parseAccessOrCall(result);
return this.parseAccessOrCall(result, isNew);
} else {
while (this.stream.hasMore() && this.stream.match([TokenType.LeftParantheses, TokenType.LeftBracket, TokenType.Period, TokenType.QuestionPeriod], false)) {
// function or method call
@@ -465,6 +473,9 @@ export class Parser {
} else {
throw new ParseException("Expected a variable, field or method.", this.stream.hasMore() ? this.stream.consume().getSpan() : this.stream.getPrev().getSpan());
}
if(isNew){
break;
}
}
// map or array access
@@ -499,7 +510,7 @@ export class Parser {
let key;
if (this.stream.hasPrev()) {
let prev = this.stream.getPrev();
if (this.stream.match(TokenType.Spread, false) && (prev.getTokenType() == TokenType.LeftCurly || prev.getTokenType() == TokenType.Comma)) {
if (this.stream.match(TokenType.Spread, false) && (prev.getTokenType() === TokenType.LeftCurly || prev.getTokenType() === TokenType.Comma)) {
let spread = this.stream.expect(TokenType.Spread);
keys.push(spread);
values.push(this.parseSpreadAccess(spread));
@@ -520,7 +531,7 @@ export class Parser {
if (key.getTokenType() === TokenType.Identifier) {
values.push(new VariableAccess(key.getSpan(), key.getText()));
} else {
values.push(new StringLiteral(key.getSpan(), 'java.lang.String'));
values.push(new Literal(key.getSpan(), 'java.lang.String'));
}
} else {
this.stream.expect(":");
@@ -719,7 +730,7 @@ export class Parser {
let token = this.stream.consume();
let index = this.stream.makeIndex();
try {
if (token.type === TokenType.Identifier && token.getText() === 'var') {
if (token.type === TokenType.Identifier && ['var','let','const'].indexOf(token.getText()) > -1 ) {
let varName = this.stream.consume().getText();
if (this.stream.match(TokenType.Assignment, true)) {
let isAsync = this.stream.match("async", true);

342
magic-editor/src/console/src/scripts/parsing/tokenizer.js
View File

@@ -1,9 +1,9 @@
import {CharacterStream, LiteralToken, ParseException, Token, TokenType} from './index.js'
import {CharacterStream, LiteralToken, ParseException, Token, TokenStream, TokenType} from './index.js'
let regexpToken = (stream, tokens) => {
const regexpToken = (stream, tokens) => {
if (tokens.length > 0) {
let token = tokens[tokens.length - 1];
if (token instanceof LiteralToken || token.getTokenType() == TokenType.Identifier) {
if (token instanceof LiteralToken || token.getTokenType() === TokenType.Identifier) {
return false;
}
}
@@ -27,7 +27,7 @@ let regexpToken = (stream, tokens) => {
} else if (deep > 0 && stream.match("]", false)) {
deep--;
} else if (stream.match(TokenType.ForwardSlash.literal, true)) {
if (deep == 0) {
if (deep === 0) {
if (stream.match("g", true)) {
}
if (stream.match("i", true)) {
@@ -47,12 +47,12 @@ let regexpToken = (stream, tokens) => {
}
}
let ch = stream.consume();
if (ch == '\r' || ch == '\n') {
if (ch === '\r' || ch === '\n') {
stream.reset(mark);
return false;
}
}
if (deep != 0) {
if (deep !== 0) {
throw new ParseException("Missing ']'", stream.getSpan(maybeMissForwardSlash, maybeMissForwardSlashEnd - 1));
}
if (!matchedEndQuote) {
@@ -67,9 +67,9 @@ let regexpToken = (stream, tokens) => {
return false;
}
let stringToken = (stream, tokens) => {
const tokenizerString = (stream, tokenType, tokens) => {
// String literal
if (stream.match(TokenType.SingleQuote.literal, true)) {
if (stream.match(tokenType, true)) {
stream.startSpan();
let matchedEndQuote = false;
while (stream.hasMore()) {
@@ -78,85 +78,196 @@ let stringToken = (stream, tokens) => {
stream.consume();
continue;
}
if (stream.match(TokenType.SingleQuote.literal, true)) {
if (stream.match(tokenType.literal, true)) {
matchedEndQuote = true;
break;
}
let ch = stream.consume();
if (ch == '\r' || ch == '\n') {
throw new ParseException("''定义的字符串不能换行", stream.endSpan());
if (tokenType !== TokenType.TripleQuote && (ch === '\r' || ch === '\n')) {
throw new ParseException(tokenType.getError() + tokenType.getError() + "定义的字符串不能换行", stream.endSpan());
}
}
if (!matchedEndQuote) {
throw new ParseException("字符串没有结束符\'", stream.endSpan());
throw new ParseException("字符串没有结束符" + tokenType.error, stream.endSpan());
}
let stringSpan = stream.endSpan();
stringSpan = stream.getSpan(stringSpan.getStart() - 1, stringSpan.getEnd());
tokens.push(new LiteralToken(TokenType.StringLiteral, stringSpan));
return true;
}
// String literal
if (stream.match('"""', true)) {
stream.startSpan();
let matchedEndQuote = false;
while (stream.hasMore()) {
// Note: escape sequences like \n are parsed in StringLiteral
if (stream.match("\\", true)) {
stream.consume();
continue;
}
if (stream.match('"""', true)) {
matchedEndQuote = true;
break;
}
stream.consume();
}
if (!matchedEndQuote) {
throw new ParseException('多行字符串没有结束符"""', stream.endSpan());
}
let stringSpan = stream.endSpan();
stringSpan = stream.getSpan(stringSpan.getStart() - 1, stringSpan.getEnd() - 2);
tokens.push(new LiteralToken(TokenType.StringLiteral, stringSpan));
return true;
}
// String literal
if (stream.match(TokenType.DoubleQuote.literal, true)) {
stream.startSpan();
let matchedEndQuote = false;
while (stream.hasMore()) {
// Note: escape sequences like \n are parsed in StringLiteral
if (stream.match("\\", true)) {
stream.consume();
continue;
}
if (stream.match(TokenType.DoubleQuote.literal, true)) {
matchedEndQuote = true;
break;
}
let ch = stream.consume();
if (ch === '\r' || ch === '\n') {
throw new ParseException("\"\"定义的字符串不能换行", stream.endSpan());
}
}
if (!matchedEndQuote) {
throw new ParseException("字符串没有结束符\"", stream.endSpan());
}
let stringSpan = stream.endSpan();
stringSpan = stream.getSpan(stringSpan.getStart(), stringSpan.getEnd() - 1);
stringSpan = stream.getSpan(stringSpan.getStart(), stringSpan.getEnd() - tokenType.literal.length);
tokens.push(new LiteralToken(TokenType.StringLiteral, stringSpan));
return true;
}
return false;
};
export default (source) => {
let stream = new CharacterStream(source, 0, source.length);
let tokens = [];
const autoNumberType = (span, radix) => {
let value = Number.parseInt(span.getText().substring(2).replace(/\_/g,''), radix)
if (value > 0x7fffffff || value < -0x80000000) {
return new LiteralToken(TokenType.LongLiteral, span, value);
} else if (value > 127 || value < -128) {
return new LiteralToken(TokenType.LongLiteral, span, value);
}
return new LiteralToken(TokenType.ByteLiteral, span, value);
}
const tokenizerNumber = (stream, tokens) => {
if (stream.match('0', false)) {
let index = stream.getPosition();
stream.startSpan();
stream.consume();
if (stream.matchAny(['x', 'X'], true)) {
while (stream.matchDigit(true) || stream.matchAny(["A", "B", "C", "D", "E", "F", "a", "b", "c", "d", "e", "f", "_"], true)) {
;
}
if (stream.matchAny(["L", "l"], true)) {
let span = stream.endSpan();
let text = span.getText();
tokens.push(new LiteralToken(TokenType.LongLiteral, span, parseInt(text.substring(2, text.length() - 1).replace(/\_/g,''), 16)));
return true;
}
tokens.push(autoNumberType(stream.endSpan(), 16));
return true;
} else if (stream.matchAny(['b','B'], true)){
while (stream.matchAny([ '0', '1', '_'], true)) {
;
}
if (stream.matchAny([ "L", "l"], true)) {
let span = stream.endSpan();
let text = span.getText();
tokens.push(new LiteralToken(TokenType.LongLiteral, span, parseInt(text.substring(0, text.length() - 1).replace(/\_/g,''), 2)));
return true;
}
tokens.push(autoNumberType(stream.endSpan(), 2));
return true;
}
stream.reset(index);
}
if (stream.matchDigit(false)) {
let type = TokenType.IntegerLiteral;
stream.startSpan();
while (stream.matchDigit(true) || stream.match('_', true)) {
}
if (stream.match(TokenType.Period.literal, true)) {
type = TokenType.DoubleLiteral;
while (stream.matchDigit(true) || stream.match('_',true)) {
}
}
if (stream.matchAny(['b', 'B'], true)) {
if (type === TokenType.DoubleLiteral) {
throw new ParseException('Byte literal can not have a decimal point.', stream.endSpan());
}
type = TokenType.ByteLiteral;
} else if (stream.matchAny(['s', 'S'], true)) {
if (type === TokenType.DoubleLiteral) {
throw new ParseException('Short literal can not have a decimal point.', stream.endSpan());
}
type = TokenType.ShortLiteral;
} else if (stream.matchAny(['l', 'L'], true)) {
if (type === TokenType.DoubleLiteral) {
throw new ParseException('Long literal can not have a decimal point.', stream.endSpan());
}
type = TokenType.LongLiteral;
} else if (stream.matchAny(['f', 'F'], true)) {
type = TokenType.FloatLiteral;
} else if (stream.matchAny(['d', 'D'], true)) {
type = TokenType.DoubleLiteral;
} else if (stream.matchAny(['m', 'M'], true)) {
type = TokenType.DecimalLiteral;
}
tokens.push(new LiteralToken(type, stream.endSpan()));
return true
}
return false;
}
const tokenizerLanguage = (stream, tokens) => {
if (stream.match("```", true)) {
stream.startSpan();
if (stream.matchIdentifierStart(true)) {
while (stream.matchIdentifierPart(true)) {
}
let language = stream.endSpan();
tokens.push(new Token(TokenType.Language, language));
stream.startSpan();
if (!stream.skipUntil("```")) {
throw new ParseException('```需要以```结尾', stream.endSpan());
}
tokens.push(new Token(TokenType.Language, stream.endSpan(-3)));
return true;
} else {
throw new ParseException('```后需要标识语言类型', stream.endSpan());
}
}
return false;
}
const tokenizerIdentifier = (stream, tokens) => {
if (stream.matchIdentifierStart(true)) {
stream.startSpan();
while (stream.matchIdentifierPart(true)) {
}
let identifierSpan = stream.endSpan();
identifierSpan = stream.getSpan(identifierSpan.getStart() - 1, identifierSpan.getEnd());
if ("true" === identifierSpan.getText() || "false" === identifierSpan.getText()) {
tokens.push(new LiteralToken(TokenType.BooleanLiteral, identifierSpan));
} else if ("null" === identifierSpan.getText()) {
tokens.push(new LiteralToken(TokenType.NullLiteral, identifierSpan));
} else if (TokenType.SqlAnd.literal === identifierSpan.getText()) {
tokens.push(new Token(TokenType.SqlAnd, identifierSpan));
} else if (TokenType.SqlOr.literal === identifierSpan.getText()) {
tokens.push(new Token(TokenType.SqlOr, identifierSpan));
} else {
tokens.push(new Token(TokenType.Identifier, identifierSpan));
}
return true;
}
return false;
}
const tokenizerTemplateString = (stream, tokens)=>{
if (stream.match("`", true)) {
let begin = stream.getPosition();
let start = begin;
let matchedEndQuote = false;
let subTokens = [];
while (stream.hasMore()) {
if (stream.match("\\", true)) {
stream.consume();
continue;
}
if (stream.match("`", true)) {
matchedEndQuote = true;
break;
}
if (stream.match("${", true)) {
let end = stream.getPosition();
if (start < end - 2) {
subTokens.push(new LiteralToken(TokenType.StringLiteral, stream.endSpan(start, end - 2)));
}
subTokens.push(...tokenizer(stream, [], "}"));
start = stream.getPosition();
continue;
}
stream.consume();
}
if (!matchedEndQuote) {
throw new ParseException("模板字符串没有结束符`", stream.endSpan());
}
let stringSpan = stream.endSpan(begin, stream.getPosition());
let end = stream.getPosition() - 1;
if (end - start > 0) {
subTokens.push(new LiteralToken(TokenType.StringLiteral, stream.endSpan(start, end)));
}
stringSpan = stream.getSpan(stringSpan.getStart() - 1, stringSpan.getEnd());
tokens.push(new LiteralToken(TokenType.StringLiteral, stringSpan, new TokenStream(subTokens)));
return true;
}
return false;
}
const tokenizer = (stream, tokens, except) => {
let leftCount = 0;
let rightCount = 0;
while (stream.hasMore()) {
stream.skipWhiteSpace();
if (except && stream.match(except, true)) {
return tokens;
}
if (stream.match("//", true)) { //注释
stream.skipLine();
continue;
@@ -165,43 +276,12 @@ export default (source) => {
stream.skipUntil("*/");
continue;
}
if (stream.matchDigit(false)) {
let type = TokenType.IntegerLiteral;
stream.startSpan();
while (stream.matchDigit(true)) {
}
if (stream.match(TokenType.Period.literal, true)) {
type = TokenType.DoubleLiteral;
while (stream.matchDigit(true)) {
}
}
if (stream.match("b", true) || stream.match("B", true)) {
if (type === TokenType.DoubleLiteral) {
throw new ParseException('Byte literal can not have a decimal point.', stream.endSpan());
}
type = TokenType.ByteLiteral;
} else if (stream.match("s", true) || stream.match("S", true)) {
if (type === TokenType.DoubleLiteral) {
throw new ParseException('Short literal can not have a decimal point.', stream.endSpan());
}
type = TokenType.ShortLiteral;
} else if (stream.match("l", true) || stream.match("L", true)) {
if (type === TokenType.DoubleLiteral) {
throw new ParseException('Long literal can not have a decimal point.', stream.endSpan());
}
type = TokenType.LongLiteral;
} else if (stream.match("f", true) || stream.match("F", true)) {
type = TokenType.FloatLiteral;
} else if (stream.match("d", true) || stream.match("D", true)) {
type = TokenType.DoubleLiteral;
} else if (stream.match("m", true) || stream.match("M", true)) {
type = TokenType.DecimalLiteral;
}
tokens.push(new LiteralToken(type, stream.endSpan()));
// int short double long float byte decimal
if (tokenizerNumber(stream, tokens)) {
continue;
}
// string
if (stringToken(stream, tokens)) {
// '' "" """ """
if (tokenizerString(stream, TokenType.SingleQuote, tokens) || tokenizerString(stream, TokenType.TripleQuote, tokens) || tokenizerString(stream, TokenType.DoubleQuote, tokens)) {
continue;
}
@@ -209,45 +289,21 @@ export default (source) => {
if (regexpToken(stream, tokens)) {
continue;
}
// TODO exception
if (stream.match("```", true)) {
stream.startSpan();
if (stream.matchIdentifierStart(true)) {
while (stream.matchIdentifierPart(true)) {
}
let language = stream.endSpan();
tokens.push(new Token(TokenType.Language, language));
stream.startSpan();
if (!stream.skipUntil("```")) {
throw new ParseException('```需要以```结尾', stream.endSpan());
}
tokens.push(new Token(TokenType.Language, stream.endSpan(-3)));
} else {
throw new ParseException('```后需要标识语言类型', stream.endSpan());
}
}
// Identifier, keyword, boolean literal, or null literal
if (stream.matchIdentifierStart(true)) {
stream.startSpan();
while (stream.matchIdentifierPart(true)) {
}
let identifierSpan = stream.endSpan();
identifierSpan = stream.getSpan(identifierSpan.getStart() - 1, identifierSpan.getEnd());
if ("true" === identifierSpan.getText() || "false" === identifierSpan.getText()) {
tokens.push(new LiteralToken(TokenType.BooleanLiteral, identifierSpan));
} else if ("null" === identifierSpan.getText()) {
tokens.push(new LiteralToken(TokenType.NullLiteral, identifierSpan));
} else if (TokenType.SqlAnd.literal === identifierSpan.getText()) {
tokens.push(new Token(TokenType.SqlAnd, identifierSpan));
} else if (TokenType.SqlOr.literal === identifierSpan.getText()) {
tokens.push(new Token(TokenType.SqlOr, identifierSpan));
} else {
tokens.push(new Token(TokenType.Identifier, identifierSpan));
}
// ``` ```
if(tokenizerLanguage(stream, tokens)){
continue;
}
if (stream.match("=>", true) || stream.match("->", true)) {
// template string
if (tokenizerTemplateString(stream, tokens)) {
continue;
}
// Identifier, keyword, boolean literal, or null literal
if(tokenizerIdentifier(stream, tokens)){
continue;
}
// lambda
if (stream.matchAny(['=>','->'], true)) {
tokens.push(new Token(TokenType.Lambda, stream.getSpan(stream.getPosition() - 2, stream.getPosition())));
continue;
}
@@ -280,4 +336,8 @@ export default (source) => {
}
}
return tokens;
}
export default (source) => {
return tokenizer(new CharacterStream(source, 0, source.length), [])
}