index.js 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119
  1. "use strict";
  2. Object.defineProperty(exports, "__esModule", {
  3. value: true
  4. });
  5. exports.default = exports.Token = void 0;
  6. var _location = require("../util/location");
  7. var _comments = require("../parser/comments");
  8. var _identifier = require("../util/identifier");
  9. var _types = require("./types");
  10. var _parseError = require("../parse-error");
  11. var _whitespace = require("../util/whitespace");
  12. var _state = require("./state");
  13. var _helperStringParser = require("@babel/helper-string-parser");
  14. const _excluded = ["at"],
  15. _excluded2 = ["at"];
  16. function _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }
  17. function buildPosition(pos, lineStart, curLine) {
  18. return new _location.Position(curLine, pos - lineStart, pos);
  19. }
  20. const VALID_REGEX_FLAGS = new Set([103, 109, 115, 105, 121, 117, 100, 118]);
  21. class Token {
  22. constructor(state) {
  23. this.type = state.type;
  24. this.value = state.value;
  25. this.start = state.start;
  26. this.end = state.end;
  27. this.loc = new _location.SourceLocation(state.startLoc, state.endLoc);
  28. }
  29. }
  30. exports.Token = Token;
  31. class Tokenizer extends _comments.default {
  32. constructor(options, input) {
  33. super();
  34. this.isLookahead = void 0;
  35. this.tokens = [];
  36. this.errorHandlers_readInt = {
  37. invalidDigit: (pos, lineStart, curLine, radix) => {
  38. if (!this.options.errorRecovery) return false;
  39. this.raise(_parseError.Errors.InvalidDigit, {
  40. at: buildPosition(pos, lineStart, curLine),
  41. radix
  42. });
  43. return true;
  44. },
  45. numericSeparatorInEscapeSequence: this.errorBuilder(_parseError.Errors.NumericSeparatorInEscapeSequence),
  46. unexpectedNumericSeparator: this.errorBuilder(_parseError.Errors.UnexpectedNumericSeparator)
  47. };
  48. this.errorHandlers_readCodePoint = Object.assign({}, this.errorHandlers_readInt, {
  49. invalidEscapeSequence: this.errorBuilder(_parseError.Errors.InvalidEscapeSequence),
  50. invalidCodePoint: this.errorBuilder(_parseError.Errors.InvalidCodePoint)
  51. });
  52. this.errorHandlers_readStringContents_string = Object.assign({}, this.errorHandlers_readCodePoint, {
  53. strictNumericEscape: (pos, lineStart, curLine) => {
  54. this.recordStrictModeErrors(_parseError.Errors.StrictNumericEscape, {
  55. at: buildPosition(pos, lineStart, curLine)
  56. });
  57. },
  58. unterminated: (pos, lineStart, curLine) => {
  59. throw this.raise(_parseError.Errors.UnterminatedString, {
  60. at: buildPosition(pos - 1, lineStart, curLine)
  61. });
  62. }
  63. });
  64. this.errorHandlers_readStringContents_template = Object.assign({}, this.errorHandlers_readCodePoint, {
  65. strictNumericEscape: this.errorBuilder(_parseError.Errors.StrictNumericEscape),
  66. unterminated: (pos, lineStart, curLine) => {
  67. throw this.raise(_parseError.Errors.UnterminatedTemplate, {
  68. at: buildPosition(pos, lineStart, curLine)
  69. });
  70. }
  71. });
  72. this.state = new _state.default();
  73. this.state.init(options);
  74. this.input = input;
  75. this.length = input.length;
  76. this.isLookahead = false;
  77. }
  78. pushToken(token) {
  79. this.tokens.length = this.state.tokensLength;
  80. this.tokens.push(token);
  81. ++this.state.tokensLength;
  82. }
  83. next() {
  84. this.checkKeywordEscapes();
  85. if (this.options.tokens) {
  86. this.pushToken(new Token(this.state));
  87. }
  88. this.state.lastTokStart = this.state.start;
  89. this.state.lastTokEndLoc = this.state.endLoc;
  90. this.state.lastTokStartLoc = this.state.startLoc;
  91. this.nextToken();
  92. }
  93. eat(type) {
  94. if (this.match(type)) {
  95. this.next();
  96. return true;
  97. } else {
  98. return false;
  99. }
  100. }
  101. match(type) {
  102. return this.state.type === type;
  103. }
  104. createLookaheadState(state) {
  105. return {
  106. pos: state.pos,
  107. value: null,
  108. type: state.type,
  109. start: state.start,
  110. end: state.end,
  111. context: [this.curContext()],
  112. inType: state.inType,
  113. startLoc: state.startLoc,
  114. lastTokEndLoc: state.lastTokEndLoc,
  115. curLine: state.curLine,
  116. lineStart: state.lineStart,
  117. curPosition: state.curPosition
  118. };
  119. }
  120. lookahead() {
  121. const old = this.state;
  122. this.state = this.createLookaheadState(old);
  123. this.isLookahead = true;
  124. this.nextToken();
  125. this.isLookahead = false;
  126. const curr = this.state;
  127. this.state = old;
  128. return curr;
  129. }
  130. nextTokenStart() {
  131. return this.nextTokenStartSince(this.state.pos);
  132. }
  133. nextTokenStartSince(pos) {
  134. _whitespace.skipWhiteSpace.lastIndex = pos;
  135. return _whitespace.skipWhiteSpace.test(this.input) ? _whitespace.skipWhiteSpace.lastIndex : pos;
  136. }
  137. lookaheadCharCode() {
  138. return this.input.charCodeAt(this.nextTokenStart());
  139. }
  140. nextTokenInLineStart() {
  141. return this.nextTokenInLineStartSince(this.state.pos);
  142. }
  143. nextTokenInLineStartSince(pos) {
  144. _whitespace.skipWhiteSpaceInLine.lastIndex = pos;
  145. return _whitespace.skipWhiteSpaceInLine.test(this.input) ? _whitespace.skipWhiteSpaceInLine.lastIndex : pos;
  146. }
  147. lookaheadInLineCharCode() {
  148. return this.input.charCodeAt(this.nextTokenInLineStart());
  149. }
  150. codePointAtPos(pos) {
  151. let cp = this.input.charCodeAt(pos);
  152. if ((cp & 0xfc00) === 0xd800 && ++pos < this.input.length) {
  153. const trail = this.input.charCodeAt(pos);
  154. if ((trail & 0xfc00) === 0xdc00) {
  155. cp = 0x10000 + ((cp & 0x3ff) << 10) + (trail & 0x3ff);
  156. }
  157. }
  158. return cp;
  159. }
  160. setStrict(strict) {
  161. this.state.strict = strict;
  162. if (strict) {
  163. this.state.strictErrors.forEach(([toParseError, at]) => this.raise(toParseError, {
  164. at
  165. }));
  166. this.state.strictErrors.clear();
  167. }
  168. }
  169. curContext() {
  170. return this.state.context[this.state.context.length - 1];
  171. }
  172. nextToken() {
  173. this.skipSpace();
  174. this.state.start = this.state.pos;
  175. if (!this.isLookahead) this.state.startLoc = this.state.curPosition();
  176. if (this.state.pos >= this.length) {
  177. this.finishToken(137);
  178. return;
  179. }
  180. this.getTokenFromCode(this.codePointAtPos(this.state.pos));
  181. }
  182. skipBlockComment(commentEnd) {
  183. let startLoc;
  184. if (!this.isLookahead) startLoc = this.state.curPosition();
  185. const start = this.state.pos;
  186. const end = this.input.indexOf(commentEnd, start + 2);
  187. if (end === -1) {
  188. throw this.raise(_parseError.Errors.UnterminatedComment, {
  189. at: this.state.curPosition()
  190. });
  191. }
  192. this.state.pos = end + commentEnd.length;
  193. _whitespace.lineBreakG.lastIndex = start + 2;
  194. while (_whitespace.lineBreakG.test(this.input) && _whitespace.lineBreakG.lastIndex <= end) {
  195. ++this.state.curLine;
  196. this.state.lineStart = _whitespace.lineBreakG.lastIndex;
  197. }
  198. if (this.isLookahead) return;
  199. const comment = {
  200. type: "CommentBlock",
  201. value: this.input.slice(start + 2, end),
  202. start,
  203. end: end + commentEnd.length,
  204. loc: new _location.SourceLocation(startLoc, this.state.curPosition())
  205. };
  206. if (this.options.tokens) this.pushToken(comment);
  207. return comment;
  208. }
  209. skipLineComment(startSkip) {
  210. const start = this.state.pos;
  211. let startLoc;
  212. if (!this.isLookahead) startLoc = this.state.curPosition();
  213. let ch = this.input.charCodeAt(this.state.pos += startSkip);
  214. if (this.state.pos < this.length) {
  215. while (!(0, _whitespace.isNewLine)(ch) && ++this.state.pos < this.length) {
  216. ch = this.input.charCodeAt(this.state.pos);
  217. }
  218. }
  219. if (this.isLookahead) return;
  220. const end = this.state.pos;
  221. const value = this.input.slice(start + startSkip, end);
  222. const comment = {
  223. type: "CommentLine",
  224. value,
  225. start,
  226. end,
  227. loc: new _location.SourceLocation(startLoc, this.state.curPosition())
  228. };
  229. if (this.options.tokens) this.pushToken(comment);
  230. return comment;
  231. }
  232. skipSpace() {
  233. const spaceStart = this.state.pos;
  234. const comments = [];
  235. loop: while (this.state.pos < this.length) {
  236. const ch = this.input.charCodeAt(this.state.pos);
  237. switch (ch) {
  238. case 32:
  239. case 160:
  240. case 9:
  241. ++this.state.pos;
  242. break;
  243. case 13:
  244. if (this.input.charCodeAt(this.state.pos + 1) === 10) {
  245. ++this.state.pos;
  246. }
  247. case 10:
  248. case 8232:
  249. case 8233:
  250. ++this.state.pos;
  251. ++this.state.curLine;
  252. this.state.lineStart = this.state.pos;
  253. break;
  254. case 47:
  255. switch (this.input.charCodeAt(this.state.pos + 1)) {
  256. case 42:
  257. {
  258. const comment = this.skipBlockComment("*/");
  259. if (comment !== undefined) {
  260. this.addComment(comment);
  261. if (this.options.attachComment) comments.push(comment);
  262. }
  263. break;
  264. }
  265. case 47:
  266. {
  267. const comment = this.skipLineComment(2);
  268. if (comment !== undefined) {
  269. this.addComment(comment);
  270. if (this.options.attachComment) comments.push(comment);
  271. }
  272. break;
  273. }
  274. default:
  275. break loop;
  276. }
  277. break;
  278. default:
  279. if ((0, _whitespace.isWhitespace)(ch)) {
  280. ++this.state.pos;
  281. } else if (ch === 45 && !this.inModule && this.options.annexB) {
  282. const pos = this.state.pos;
  283. if (this.input.charCodeAt(pos + 1) === 45 && this.input.charCodeAt(pos + 2) === 62 && (spaceStart === 0 || this.state.lineStart > spaceStart)) {
  284. const comment = this.skipLineComment(3);
  285. if (comment !== undefined) {
  286. this.addComment(comment);
  287. if (this.options.attachComment) comments.push(comment);
  288. }
  289. } else {
  290. break loop;
  291. }
  292. } else if (ch === 60 && !this.inModule && this.options.annexB) {
  293. const pos = this.state.pos;
  294. if (this.input.charCodeAt(pos + 1) === 33 && this.input.charCodeAt(pos + 2) === 45 && this.input.charCodeAt(pos + 3) === 45) {
  295. const comment = this.skipLineComment(4);
  296. if (comment !== undefined) {
  297. this.addComment(comment);
  298. if (this.options.attachComment) comments.push(comment);
  299. }
  300. } else {
  301. break loop;
  302. }
  303. } else {
  304. break loop;
  305. }
  306. }
  307. }
  308. if (comments.length > 0) {
  309. const end = this.state.pos;
  310. const commentWhitespace = {
  311. start: spaceStart,
  312. end,
  313. comments,
  314. leadingNode: null,
  315. trailingNode: null,
  316. containingNode: null
  317. };
  318. this.state.commentStack.push(commentWhitespace);
  319. }
  320. }
  321. finishToken(type, val) {
  322. this.state.end = this.state.pos;
  323. this.state.endLoc = this.state.curPosition();
  324. const prevType = this.state.type;
  325. this.state.type = type;
  326. this.state.value = val;
  327. if (!this.isLookahead) {
  328. this.updateContext(prevType);
  329. }
  330. }
  331. replaceToken(type) {
  332. this.state.type = type;
  333. this.updateContext();
  334. }
  335. readToken_numberSign() {
  336. if (this.state.pos === 0 && this.readToken_interpreter()) {
  337. return;
  338. }
  339. const nextPos = this.state.pos + 1;
  340. const next = this.codePointAtPos(nextPos);
  341. if (next >= 48 && next <= 57) {
  342. throw this.raise(_parseError.Errors.UnexpectedDigitAfterHash, {
  343. at: this.state.curPosition()
  344. });
  345. }
  346. if (next === 123 || next === 91 && this.hasPlugin("recordAndTuple")) {
  347. this.expectPlugin("recordAndTuple");
  348. if (this.getPluginOption("recordAndTuple", "syntaxType") === "bar") {
  349. throw this.raise(next === 123 ? _parseError.Errors.RecordExpressionHashIncorrectStartSyntaxType : _parseError.Errors.TupleExpressionHashIncorrectStartSyntaxType, {
  350. at: this.state.curPosition()
  351. });
  352. }
  353. this.state.pos += 2;
  354. if (next === 123) {
  355. this.finishToken(7);
  356. } else {
  357. this.finishToken(1);
  358. }
  359. } else if ((0, _identifier.isIdentifierStart)(next)) {
  360. ++this.state.pos;
  361. this.finishToken(136, this.readWord1(next));
  362. } else if (next === 92) {
  363. ++this.state.pos;
  364. this.finishToken(136, this.readWord1());
  365. } else {
  366. this.finishOp(27, 1);
  367. }
  368. }
  369. readToken_dot() {
  370. const next = this.input.charCodeAt(this.state.pos + 1);
  371. if (next >= 48 && next <= 57) {
  372. this.readNumber(true);
  373. return;
  374. }
  375. if (next === 46 && this.input.charCodeAt(this.state.pos + 2) === 46) {
  376. this.state.pos += 3;
  377. this.finishToken(21);
  378. } else {
  379. ++this.state.pos;
  380. this.finishToken(16);
  381. }
  382. }
  383. readToken_slash() {
  384. const next = this.input.charCodeAt(this.state.pos + 1);
  385. if (next === 61) {
  386. this.finishOp(31, 2);
  387. } else {
  388. this.finishOp(56, 1);
  389. }
  390. }
  391. readToken_interpreter() {
  392. if (this.state.pos !== 0 || this.length < 2) return false;
  393. let ch = this.input.charCodeAt(this.state.pos + 1);
  394. if (ch !== 33) return false;
  395. const start = this.state.pos;
  396. this.state.pos += 1;
  397. while (!(0, _whitespace.isNewLine)(ch) && ++this.state.pos < this.length) {
  398. ch = this.input.charCodeAt(this.state.pos);
  399. }
  400. const value = this.input.slice(start + 2, this.state.pos);
  401. this.finishToken(28, value);
  402. return true;
  403. }
  404. readToken_mult_modulo(code) {
  405. let type = code === 42 ? 55 : 54;
  406. let width = 1;
  407. let next = this.input.charCodeAt(this.state.pos + 1);
  408. if (code === 42 && next === 42) {
  409. width++;
  410. next = this.input.charCodeAt(this.state.pos + 2);
  411. type = 57;
  412. }
  413. if (next === 61 && !this.state.inType) {
  414. width++;
  415. type = code === 37 ? 33 : 30;
  416. }
  417. this.finishOp(type, width);
  418. }
  419. readToken_pipe_amp(code) {
  420. const next = this.input.charCodeAt(this.state.pos + 1);
  421. if (next === code) {
  422. if (this.input.charCodeAt(this.state.pos + 2) === 61) {
  423. this.finishOp(30, 3);
  424. } else {
  425. this.finishOp(code === 124 ? 41 : 42, 2);
  426. }
  427. return;
  428. }
  429. if (code === 124) {
  430. if (next === 62) {
  431. this.finishOp(39, 2);
  432. return;
  433. }
  434. if (this.hasPlugin("recordAndTuple") && next === 125) {
  435. if (this.getPluginOption("recordAndTuple", "syntaxType") !== "bar") {
  436. throw this.raise(_parseError.Errors.RecordExpressionBarIncorrectEndSyntaxType, {
  437. at: this.state.curPosition()
  438. });
  439. }
  440. this.state.pos += 2;
  441. this.finishToken(9);
  442. return;
  443. }
  444. if (this.hasPlugin("recordAndTuple") && next === 93) {
  445. if (this.getPluginOption("recordAndTuple", "syntaxType") !== "bar") {
  446. throw this.raise(_parseError.Errors.TupleExpressionBarIncorrectEndSyntaxType, {
  447. at: this.state.curPosition()
  448. });
  449. }
  450. this.state.pos += 2;
  451. this.finishToken(4);
  452. return;
  453. }
  454. }
  455. if (next === 61) {
  456. this.finishOp(30, 2);
  457. return;
  458. }
  459. this.finishOp(code === 124 ? 43 : 45, 1);
  460. }
  461. readToken_caret() {
  462. const next = this.input.charCodeAt(this.state.pos + 1);
  463. if (next === 61 && !this.state.inType) {
  464. this.finishOp(32, 2);
  465. } else if (next === 94 && this.hasPlugin(["pipelineOperator", {
  466. proposal: "hack",
  467. topicToken: "^^"
  468. }])) {
  469. this.finishOp(37, 2);
  470. const lookaheadCh = this.input.codePointAt(this.state.pos);
  471. if (lookaheadCh === 94) {
  472. this.unexpected();
  473. }
  474. } else {
  475. this.finishOp(44, 1);
  476. }
  477. }
  478. readToken_atSign() {
  479. const next = this.input.charCodeAt(this.state.pos + 1);
  480. if (next === 64 && this.hasPlugin(["pipelineOperator", {
  481. proposal: "hack",
  482. topicToken: "@@"
  483. }])) {
  484. this.finishOp(38, 2);
  485. } else {
  486. this.finishOp(26, 1);
  487. }
  488. }
  489. readToken_plus_min(code) {
  490. const next = this.input.charCodeAt(this.state.pos + 1);
  491. if (next === code) {
  492. this.finishOp(34, 2);
  493. return;
  494. }
  495. if (next === 61) {
  496. this.finishOp(30, 2);
  497. } else {
  498. this.finishOp(53, 1);
  499. }
  500. }
  501. readToken_lt() {
  502. const {
  503. pos
  504. } = this.state;
  505. const next = this.input.charCodeAt(pos + 1);
  506. if (next === 60) {
  507. if (this.input.charCodeAt(pos + 2) === 61) {
  508. this.finishOp(30, 3);
  509. return;
  510. }
  511. this.finishOp(51, 2);
  512. return;
  513. }
  514. if (next === 61) {
  515. this.finishOp(49, 2);
  516. return;
  517. }
  518. this.finishOp(47, 1);
  519. }
  520. readToken_gt() {
  521. const {
  522. pos
  523. } = this.state;
  524. const next = this.input.charCodeAt(pos + 1);
  525. if (next === 62) {
  526. const size = this.input.charCodeAt(pos + 2) === 62 ? 3 : 2;
  527. if (this.input.charCodeAt(pos + size) === 61) {
  528. this.finishOp(30, size + 1);
  529. return;
  530. }
  531. this.finishOp(52, size);
  532. return;
  533. }
  534. if (next === 61) {
  535. this.finishOp(49, 2);
  536. return;
  537. }
  538. this.finishOp(48, 1);
  539. }
  540. readToken_eq_excl(code) {
  541. const next = this.input.charCodeAt(this.state.pos + 1);
  542. if (next === 61) {
  543. this.finishOp(46, this.input.charCodeAt(this.state.pos + 2) === 61 ? 3 : 2);
  544. return;
  545. }
  546. if (code === 61 && next === 62) {
  547. this.state.pos += 2;
  548. this.finishToken(19);
  549. return;
  550. }
  551. this.finishOp(code === 61 ? 29 : 35, 1);
  552. }
  553. readToken_question() {
  554. const next = this.input.charCodeAt(this.state.pos + 1);
  555. const next2 = this.input.charCodeAt(this.state.pos + 2);
  556. if (next === 63) {
  557. if (next2 === 61) {
  558. this.finishOp(30, 3);
  559. } else {
  560. this.finishOp(40, 2);
  561. }
  562. } else if (next === 46 && !(next2 >= 48 && next2 <= 57)) {
  563. this.state.pos += 2;
  564. this.finishToken(18);
  565. } else {
  566. ++this.state.pos;
  567. this.finishToken(17);
  568. }
  569. }
  570. getTokenFromCode(code) {
  571. switch (code) {
  572. case 46:
  573. this.readToken_dot();
  574. return;
  575. case 40:
  576. ++this.state.pos;
  577. this.finishToken(10);
  578. return;
  579. case 41:
  580. ++this.state.pos;
  581. this.finishToken(11);
  582. return;
  583. case 59:
  584. ++this.state.pos;
  585. this.finishToken(13);
  586. return;
  587. case 44:
  588. ++this.state.pos;
  589. this.finishToken(12);
  590. return;
  591. case 91:
  592. if (this.hasPlugin("recordAndTuple") && this.input.charCodeAt(this.state.pos + 1) === 124) {
  593. if (this.getPluginOption("recordAndTuple", "syntaxType") !== "bar") {
  594. throw this.raise(_parseError.Errors.TupleExpressionBarIncorrectStartSyntaxType, {
  595. at: this.state.curPosition()
  596. });
  597. }
  598. this.state.pos += 2;
  599. this.finishToken(2);
  600. } else {
  601. ++this.state.pos;
  602. this.finishToken(0);
  603. }
  604. return;
  605. case 93:
  606. ++this.state.pos;
  607. this.finishToken(3);
  608. return;
  609. case 123:
  610. if (this.hasPlugin("recordAndTuple") && this.input.charCodeAt(this.state.pos + 1) === 124) {
  611. if (this.getPluginOption("recordAndTuple", "syntaxType") !== "bar") {
  612. throw this.raise(_parseError.Errors.RecordExpressionBarIncorrectStartSyntaxType, {
  613. at: this.state.curPosition()
  614. });
  615. }
  616. this.state.pos += 2;
  617. this.finishToken(6);
  618. } else {
  619. ++this.state.pos;
  620. this.finishToken(5);
  621. }
  622. return;
  623. case 125:
  624. ++this.state.pos;
  625. this.finishToken(8);
  626. return;
  627. case 58:
  628. if (this.hasPlugin("functionBind") && this.input.charCodeAt(this.state.pos + 1) === 58) {
  629. this.finishOp(15, 2);
  630. } else {
  631. ++this.state.pos;
  632. this.finishToken(14);
  633. }
  634. return;
  635. case 63:
  636. this.readToken_question();
  637. return;
  638. case 96:
  639. this.readTemplateToken();
  640. return;
  641. case 48:
  642. {
  643. const next = this.input.charCodeAt(this.state.pos + 1);
  644. if (next === 120 || next === 88) {
  645. this.readRadixNumber(16);
  646. return;
  647. }
  648. if (next === 111 || next === 79) {
  649. this.readRadixNumber(8);
  650. return;
  651. }
  652. if (next === 98 || next === 66) {
  653. this.readRadixNumber(2);
  654. return;
  655. }
  656. }
  657. case 49:
  658. case 50:
  659. case 51:
  660. case 52:
  661. case 53:
  662. case 54:
  663. case 55:
  664. case 56:
  665. case 57:
  666. this.readNumber(false);
  667. return;
  668. case 34:
  669. case 39:
  670. this.readString(code);
  671. return;
  672. case 47:
  673. this.readToken_slash();
  674. return;
  675. case 37:
  676. case 42:
  677. this.readToken_mult_modulo(code);
  678. return;
  679. case 124:
  680. case 38:
  681. this.readToken_pipe_amp(code);
  682. return;
  683. case 94:
  684. this.readToken_caret();
  685. return;
  686. case 43:
  687. case 45:
  688. this.readToken_plus_min(code);
  689. return;
  690. case 60:
  691. this.readToken_lt();
  692. return;
  693. case 62:
  694. this.readToken_gt();
  695. return;
  696. case 61:
  697. case 33:
  698. this.readToken_eq_excl(code);
  699. return;
  700. case 126:
  701. this.finishOp(36, 1);
  702. return;
  703. case 64:
  704. this.readToken_atSign();
  705. return;
  706. case 35:
  707. this.readToken_numberSign();
  708. return;
  709. case 92:
  710. this.readWord();
  711. return;
  712. default:
  713. if ((0, _identifier.isIdentifierStart)(code)) {
  714. this.readWord(code);
  715. return;
  716. }
  717. }
  718. throw this.raise(_parseError.Errors.InvalidOrUnexpectedToken, {
  719. at: this.state.curPosition(),
  720. unexpected: String.fromCodePoint(code)
  721. });
  722. }
  723. finishOp(type, size) {
  724. const str = this.input.slice(this.state.pos, this.state.pos + size);
  725. this.state.pos += size;
  726. this.finishToken(type, str);
  727. }
  728. readRegexp() {
  729. const startLoc = this.state.startLoc;
  730. const start = this.state.start + 1;
  731. let escaped, inClass;
  732. let {
  733. pos
  734. } = this.state;
  735. for (;; ++pos) {
  736. if (pos >= this.length) {
  737. throw this.raise(_parseError.Errors.UnterminatedRegExp, {
  738. at: (0, _location.createPositionWithColumnOffset)(startLoc, 1)
  739. });
  740. }
  741. const ch = this.input.charCodeAt(pos);
  742. if ((0, _whitespace.isNewLine)(ch)) {
  743. throw this.raise(_parseError.Errors.UnterminatedRegExp, {
  744. at: (0, _location.createPositionWithColumnOffset)(startLoc, 1)
  745. });
  746. }
  747. if (escaped) {
  748. escaped = false;
  749. } else {
  750. if (ch === 91) {
  751. inClass = true;
  752. } else if (ch === 93 && inClass) {
  753. inClass = false;
  754. } else if (ch === 47 && !inClass) {
  755. break;
  756. }
  757. escaped = ch === 92;
  758. }
  759. }
  760. const content = this.input.slice(start, pos);
  761. ++pos;
  762. let mods = "";
  763. const nextPos = () => (0, _location.createPositionWithColumnOffset)(startLoc, pos + 2 - start);
  764. while (pos < this.length) {
  765. const cp = this.codePointAtPos(pos);
  766. const char = String.fromCharCode(cp);
  767. if (VALID_REGEX_FLAGS.has(cp)) {
  768. if (cp === 118) {
  769. if (mods.includes("u")) {
  770. this.raise(_parseError.Errors.IncompatibleRegExpUVFlags, {
  771. at: nextPos()
  772. });
  773. }
  774. } else if (cp === 117) {
  775. if (mods.includes("v")) {
  776. this.raise(_parseError.Errors.IncompatibleRegExpUVFlags, {
  777. at: nextPos()
  778. });
  779. }
  780. }
  781. if (mods.includes(char)) {
  782. this.raise(_parseError.Errors.DuplicateRegExpFlags, {
  783. at: nextPos()
  784. });
  785. }
  786. } else if ((0, _identifier.isIdentifierChar)(cp) || cp === 92) {
  787. this.raise(_parseError.Errors.MalformedRegExpFlags, {
  788. at: nextPos()
  789. });
  790. } else {
  791. break;
  792. }
  793. ++pos;
  794. mods += char;
  795. }
  796. this.state.pos = pos;
  797. this.finishToken(135, {
  798. pattern: content,
  799. flags: mods
  800. });
  801. }
  802. readInt(radix, len, forceLen = false, allowNumSeparator = true) {
  803. const {
  804. n,
  805. pos
  806. } = (0, _helperStringParser.readInt)(this.input, this.state.pos, this.state.lineStart, this.state.curLine, radix, len, forceLen, allowNumSeparator, this.errorHandlers_readInt, false);
  807. this.state.pos = pos;
  808. return n;
  809. }
  810. readRadixNumber(radix) {
  811. const startLoc = this.state.curPosition();
  812. let isBigInt = false;
  813. this.state.pos += 2;
  814. const val = this.readInt(radix);
  815. if (val == null) {
  816. this.raise(_parseError.Errors.InvalidDigit, {
  817. at: (0, _location.createPositionWithColumnOffset)(startLoc, 2),
  818. radix
  819. });
  820. }
  821. const next = this.input.charCodeAt(this.state.pos);
  822. if (next === 110) {
  823. ++this.state.pos;
  824. isBigInt = true;
  825. } else if (next === 109) {
  826. throw this.raise(_parseError.Errors.InvalidDecimal, {
  827. at: startLoc
  828. });
  829. }
  830. if ((0, _identifier.isIdentifierStart)(this.codePointAtPos(this.state.pos))) {
  831. throw this.raise(_parseError.Errors.NumberIdentifier, {
  832. at: this.state.curPosition()
  833. });
  834. }
  835. if (isBigInt) {
  836. const str = this.input.slice(startLoc.index, this.state.pos).replace(/[_n]/g, "");
  837. this.finishToken(133, str);
  838. return;
  839. }
  840. this.finishToken(132, val);
  841. }
  842. readNumber(startsWithDot) {
  843. const start = this.state.pos;
  844. const startLoc = this.state.curPosition();
  845. let isFloat = false;
  846. let isBigInt = false;
  847. let isDecimal = false;
  848. let hasExponent = false;
  849. let isOctal = false;
  850. if (!startsWithDot && this.readInt(10) === null) {
  851. this.raise(_parseError.Errors.InvalidNumber, {
  852. at: this.state.curPosition()
  853. });
  854. }
  855. const hasLeadingZero = this.state.pos - start >= 2 && this.input.charCodeAt(start) === 48;
  856. if (hasLeadingZero) {
  857. const integer = this.input.slice(start, this.state.pos);
  858. this.recordStrictModeErrors(_parseError.Errors.StrictOctalLiteral, {
  859. at: startLoc
  860. });
  861. if (!this.state.strict) {
  862. const underscorePos = integer.indexOf("_");
  863. if (underscorePos > 0) {
  864. this.raise(_parseError.Errors.ZeroDigitNumericSeparator, {
  865. at: (0, _location.createPositionWithColumnOffset)(startLoc, underscorePos)
  866. });
  867. }
  868. }
  869. isOctal = hasLeadingZero && !/[89]/.test(integer);
  870. }
  871. let next = this.input.charCodeAt(this.state.pos);
  872. if (next === 46 && !isOctal) {
  873. ++this.state.pos;
  874. this.readInt(10);
  875. isFloat = true;
  876. next = this.input.charCodeAt(this.state.pos);
  877. }
  878. if ((next === 69 || next === 101) && !isOctal) {
  879. next = this.input.charCodeAt(++this.state.pos);
  880. if (next === 43 || next === 45) {
  881. ++this.state.pos;
  882. }
  883. if (this.readInt(10) === null) {
  884. this.raise(_parseError.Errors.InvalidOrMissingExponent, {
  885. at: startLoc
  886. });
  887. }
  888. isFloat = true;
  889. hasExponent = true;
  890. next = this.input.charCodeAt(this.state.pos);
  891. }
  892. if (next === 110) {
  893. if (isFloat || hasLeadingZero) {
  894. this.raise(_parseError.Errors.InvalidBigIntLiteral, {
  895. at: startLoc
  896. });
  897. }
  898. ++this.state.pos;
  899. isBigInt = true;
  900. }
  901. if (next === 109) {
  902. this.expectPlugin("decimal", this.state.curPosition());
  903. if (hasExponent || hasLeadingZero) {
  904. this.raise(_parseError.Errors.InvalidDecimal, {
  905. at: startLoc
  906. });
  907. }
  908. ++this.state.pos;
  909. isDecimal = true;
  910. }
  911. if ((0, _identifier.isIdentifierStart)(this.codePointAtPos(this.state.pos))) {
  912. throw this.raise(_parseError.Errors.NumberIdentifier, {
  913. at: this.state.curPosition()
  914. });
  915. }
  916. const str = this.input.slice(start, this.state.pos).replace(/[_mn]/g, "");
  917. if (isBigInt) {
  918. this.finishToken(133, str);
  919. return;
  920. }
  921. if (isDecimal) {
  922. this.finishToken(134, str);
  923. return;
  924. }
  925. const val = isOctal ? parseInt(str, 8) : parseFloat(str);
  926. this.finishToken(132, val);
  927. }
  928. readCodePoint(throwOnInvalid) {
  929. const {
  930. code,
  931. pos
  932. } = (0, _helperStringParser.readCodePoint)(this.input, this.state.pos, this.state.lineStart, this.state.curLine, throwOnInvalid, this.errorHandlers_readCodePoint);
  933. this.state.pos = pos;
  934. return code;
  935. }
  936. readString(quote) {
  937. const {
  938. str,
  939. pos,
  940. curLine,
  941. lineStart
  942. } = (0, _helperStringParser.readStringContents)(quote === 34 ? "double" : "single", this.input, this.state.pos + 1, this.state.lineStart, this.state.curLine, this.errorHandlers_readStringContents_string);
  943. this.state.pos = pos + 1;
  944. this.state.lineStart = lineStart;
  945. this.state.curLine = curLine;
  946. this.finishToken(131, str);
  947. }
  948. readTemplateContinuation() {
  949. if (!this.match(8)) {
  950. this.unexpected(null, 8);
  951. }
  952. this.state.pos--;
  953. this.readTemplateToken();
  954. }
  955. readTemplateToken() {
  956. const opening = this.input[this.state.pos];
  957. const {
  958. str,
  959. firstInvalidLoc,
  960. pos,
  961. curLine,
  962. lineStart
  963. } = (0, _helperStringParser.readStringContents)("template", this.input, this.state.pos + 1, this.state.lineStart, this.state.curLine, this.errorHandlers_readStringContents_template);
  964. this.state.pos = pos + 1;
  965. this.state.lineStart = lineStart;
  966. this.state.curLine = curLine;
  967. if (firstInvalidLoc) {
  968. this.state.firstInvalidTemplateEscapePos = new _location.Position(firstInvalidLoc.curLine, firstInvalidLoc.pos - firstInvalidLoc.lineStart, firstInvalidLoc.pos);
  969. }
  970. if (this.input.codePointAt(pos) === 96) {
  971. this.finishToken(24, firstInvalidLoc ? null : opening + str + "`");
  972. } else {
  973. this.state.pos++;
  974. this.finishToken(25, firstInvalidLoc ? null : opening + str + "${");
  975. }
  976. }
  977. recordStrictModeErrors(toParseError, {
  978. at
  979. }) {
  980. const index = at.index;
  981. if (this.state.strict && !this.state.strictErrors.has(index)) {
  982. this.raise(toParseError, {
  983. at
  984. });
  985. } else {
  986. this.state.strictErrors.set(index, [toParseError, at]);
  987. }
  988. }
  989. readWord1(firstCode) {
  990. this.state.containsEsc = false;
  991. let word = "";
  992. const start = this.state.pos;
  993. let chunkStart = this.state.pos;
  994. if (firstCode !== undefined) {
  995. this.state.pos += firstCode <= 0xffff ? 1 : 2;
  996. }
  997. while (this.state.pos < this.length) {
  998. const ch = this.codePointAtPos(this.state.pos);
  999. if ((0, _identifier.isIdentifierChar)(ch)) {
  1000. this.state.pos += ch <= 0xffff ? 1 : 2;
  1001. } else if (ch === 92) {
  1002. this.state.containsEsc = true;
  1003. word += this.input.slice(chunkStart, this.state.pos);
  1004. const escStart = this.state.curPosition();
  1005. const identifierCheck = this.state.pos === start ? _identifier.isIdentifierStart : _identifier.isIdentifierChar;
  1006. if (this.input.charCodeAt(++this.state.pos) !== 117) {
  1007. this.raise(_parseError.Errors.MissingUnicodeEscape, {
  1008. at: this.state.curPosition()
  1009. });
  1010. chunkStart = this.state.pos - 1;
  1011. continue;
  1012. }
  1013. ++this.state.pos;
  1014. const esc = this.readCodePoint(true);
  1015. if (esc !== null) {
  1016. if (!identifierCheck(esc)) {
  1017. this.raise(_parseError.Errors.EscapedCharNotAnIdentifier, {
  1018. at: escStart
  1019. });
  1020. }
  1021. word += String.fromCodePoint(esc);
  1022. }
  1023. chunkStart = this.state.pos;
  1024. } else {
  1025. break;
  1026. }
  1027. }
  1028. return word + this.input.slice(chunkStart, this.state.pos);
  1029. }
  1030. readWord(firstCode) {
  1031. const word = this.readWord1(firstCode);
  1032. const type = _types.keywords.get(word);
  1033. if (type !== undefined) {
  1034. this.finishToken(type, (0, _types.tokenLabelName)(type));
  1035. } else {
  1036. this.finishToken(130, word);
  1037. }
  1038. }
  1039. checkKeywordEscapes() {
  1040. const {
  1041. type
  1042. } = this.state;
  1043. if ((0, _types.tokenIsKeyword)(type) && this.state.containsEsc) {
  1044. this.raise(_parseError.Errors.InvalidEscapedReservedWord, {
  1045. at: this.state.startLoc,
  1046. reservedWord: (0, _types.tokenLabelName)(type)
  1047. });
  1048. }
  1049. }
  1050. raise(toParseError, raiseProperties) {
  1051. const {
  1052. at
  1053. } = raiseProperties,
  1054. details = _objectWithoutPropertiesLoose(raiseProperties, _excluded);
  1055. const loc = at instanceof _location.Position ? at : at.loc.start;
  1056. const error = toParseError({
  1057. loc,
  1058. details
  1059. });
  1060. if (!this.options.errorRecovery) throw error;
  1061. if (!this.isLookahead) this.state.errors.push(error);
  1062. return error;
  1063. }
  1064. raiseOverwrite(toParseError, raiseProperties) {
  1065. const {
  1066. at
  1067. } = raiseProperties,
  1068. details = _objectWithoutPropertiesLoose(raiseProperties, _excluded2);
  1069. const loc = at instanceof _location.Position ? at : at.loc.start;
  1070. const pos = loc.index;
  1071. const errors = this.state.errors;
  1072. for (let i = errors.length - 1; i >= 0; i--) {
  1073. const error = errors[i];
  1074. if (error.loc.index === pos) {
  1075. return errors[i] = toParseError({
  1076. loc,
  1077. details
  1078. });
  1079. }
  1080. if (error.loc.index < pos) break;
  1081. }
  1082. return this.raise(toParseError, raiseProperties);
  1083. }
  1084. updateContext(prevType) {}
  1085. unexpected(loc, type) {
  1086. throw this.raise(_parseError.Errors.UnexpectedToken, {
  1087. expected: type ? (0, _types.tokenLabelName)(type) : null,
  1088. at: loc != null ? loc : this.state.startLoc
  1089. });
  1090. }
  1091. expectPlugin(pluginName, loc) {
  1092. if (this.hasPlugin(pluginName)) {
  1093. return true;
  1094. }
  1095. throw this.raise(_parseError.Errors.MissingPlugin, {
  1096. at: loc != null ? loc : this.state.startLoc,
  1097. missingPlugin: [pluginName]
  1098. });
  1099. }
  1100. expectOnePlugin(pluginNames) {
  1101. if (!pluginNames.some(name => this.hasPlugin(name))) {
  1102. throw this.raise(_parseError.Errors.MissingOneOfPlugins, {
  1103. at: this.state.startLoc,
  1104. missingPlugin: pluginNames
  1105. });
  1106. }
  1107. }
  1108. errorBuilder(error) {
  1109. return (pos, lineStart, curLine) => {
  1110. this.raise(error, {
  1111. at: buildPosition(pos, lineStart, curLine)
  1112. });
  1113. };
  1114. }
  1115. }
  1116. exports.default = Tokenizer;
  1117. //# sourceMappingURL=index.js.map