refactor(es/lexer): Split lexer (#10377)

We do not ensure each token is correct during the parsing of JS/TS code, so let's split `parse/lexer` into single crates, and this crate can ensure that the lexer is correct.

Author: bvanjoi

.changeset/tame-dots-own.md

@@ -0,0 +1,7 @@
+---
+swc_ecma_lexer: patch
+swc_ecma_parser: patch
+swc_core: patch
+---
+
+refactor(ecma/lexer): split lexer of parser

Cargo.lock

@@ -0,0 +1,66 @@
+[package]
+authors = ["강동윤 <kdy1997.dev@gmail.com>"]
+description = "Feature-complete es2019 parser."
+documentation = "https://rustdoc.swc.rs/swc_ecma_lexer/"
+edition = { workspace = true }
+include = ["Cargo.toml", "src/**/*.rs", "examples/**/*.rs"]
+license = { workspace = true }
+name = "swc_ecma_lexer"
+repository = { workspace = true }
+version = "11.1.2"
+
+ [package.metadata.docs.rs]
+ all-features = true
+ rustdoc-args = ["--cfg", "docsrs"]
+
+[lib]
+bench = false
+
+[features]
+# Used for debugging
+debug = ["tracing-spans"]
+default = ["typescript", "stacker"]
+tracing-spans = []
+typescript = []
+verify = ["swc_ecma_visit"]
+
+[dependencies]
+arrayvec = { workspace = true }
+bitflags = { workspace = true }
+either = { workspace = true }
+num-bigint = { workspace = true }
+num-traits = { workspace = true }
+rustc-hash = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+smallvec = { workspace = true }
+smartstring = { workspace = true }
+tracing = { workspace = true }
+typed-arena = { workspace = true }
+
+new_debug_unreachable = { workspace = true }
+phf = { workspace = true, features = ["macros"] }
+swc_atoms = { version = "5.0.0", path = "../swc_atoms" }
+swc_common = { version = "8.1.0", path = "../swc_common" }
+swc_ecma_ast = { version = "8.1.2", path = "../swc_ecma_ast" }
+swc_ecma_visit = { version = "8.0.0", path = "../swc_ecma_visit", optional = true }
+
+[target.'cfg(not(any(target_arch = "wasm32", target_arch = "arm")))'.dependencies]
+stacker = { version = "0.1.15", optional = true }
+
+[dev-dependencies]
+criterion = { workspace = true }
+pretty_assertions = { workspace = true }
+serde_json = { workspace = true }
+walkdir = { workspace = true }
+
+codspeed-criterion-compat = { workspace = true }
+swc_ecma_ast = { version = "8.1.2", path = "../swc_ecma_ast", features = [
+ "serde-impl",
+] }
+swc_ecma_visit = { version = "8.0.0", path = "../swc_ecma_visit" }
+swc_malloc = { version = "1.2.2", path = "../swc_malloc" }
+testing = { version = "9.0.0", path = "../testing" }
+
+[[bench]]
+harness = false
+name = "lexer"

crates/swc_ecma_lexer/Cargo.toml

@@ -1,8 +1,8 @@
 extern crate swc_malloc;
 
 use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
-use swc_common::FileName;
-use swc_ecma_parser::{lexer::Lexer, StringInput, Syntax, TsSyntax};
+use swc_common::{input::StringInput, FileName};
+use swc_ecma_lexer::{lexer::Lexer, Syntax, TsSyntax};
 
 fn bench_module(b: &mut Bencher, syntax: Syntax, src: &'static str) {
  let _ = ::testing::run_test(false, |cm, _| {
@@ -21,66 +21,74 @@ fn bench_module(b: &mut Bencher, syntax: Syntax, src: &'static str) {
 fn bench_files(c: &mut Criterion) {
  c.bench_function("es/lexer/colors", |b| {
  // Copied from ratel-rust
- bench_module(b, Default::default(), include_str!("../colors.js"))
+ bench_module(
+ b,
+ Default::default(),
+ include_str!("../../swc_ecma_parser/colors.js"),
+ )
  });
 
  c.bench_function("es/lexer/angular", |b| {
  bench_module(
  b,
  Default::default(),
- include_str!("./files/angular-1.2.5.js"),
+ include_str!("../../swc_ecma_parser/benches/files/angular-1.2.5.js"),
  )
  });
 
  c.bench_function("es/lexer/backbone", |b| {
  bench_module(
  b,
  Default::default(),
- include_str!("./files/backbone-1.1.0.js"),
+ include_str!("../../swc_ecma_parser/benches/files/backbone-1.1.0.js"),
  )
  });
 
  c.bench_function("es/lexer/jquery", |b| {
  bench_module(
  b,
  Default::default(),
- include_str!("./files/jquery-1.9.1.js"),
+ include_str!("../../swc_ecma_parser/benches/files/jquery-1.9.1.js"),
  )
  });
 
  c.bench_function("es/lexer/jquery mobile", |b| {
  bench_module(
  b,
  Default::default(),
- include_str!("./files/jquery.mobile-1.4.2.js"),
+ include_str!("../../swc_ecma_parser/benches/files/jquery.mobile-1.4.2.js"),
  )
  });
  c.bench_function("es/lexer/mootools", |b| {
  bench_module(
  b,
  Default::default(),
- include_str!("./files/mootools-1.4.5.js"),
+ include_str!("../../swc_ecma_parser/benches/files/mootools-1.4.5.js"),
  )
  });
 
  c.bench_function("es/lexer/underscore", |b| {
  bench_module(
  b,
  Default::default(),
- include_str!("./files/underscore-1.5.2.js"),
+ include_str!("../../swc_ecma_parser/benches/files/underscore-1.5.2.js"),
  )
  });
 
  c.bench_function("es/lexer/three", |b| {
  bench_module(
  b,
  Default::default(),
- include_str!("./files/three-0.138.3.js"),
+ include_str!("../../swc_ecma_parser/benches/files/three-0.138.3.js"),
  )
  });
 
  c.bench_function("es/lexer/yui", |b| {
- bench_module(b, Default::default(), include_str!("./files/yui-3.12.0.js"))
+ bench_module(
+ b,
+ Default::default(),
+ include_str!("../../swc_ecma_parser/benches/files/yui-3.12.0.js"),
+ )
  });
 
  c.bench_function("es/lexer/cal-com", |b| {
@@ -90,12 +98,16 @@ fn bench_files(c: &mut Criterion) {
  tsx: true,
  ..Default::default()
  }),
- include_str!("./files/cal.com.tsx"),
+ include_str!("../../swc_ecma_parser/benches/files/cal.com.tsx"),
  )
  });
 
  c.bench_function("es/lexer/typescript", |b| {
- bench_module(b, Default::default(), include_str!("./files/typescript.js"))
+ bench_module(
+ b,
+ Default::default(),
+ include_str!("../../swc_ecma_parser/benches/files/typescript.js"),
+ )
  });
 }
 

crates/swc_ecma_parser/benches/lexer.rs renamed to crates/swc_ecma_lexer/benches/lexer.rs

@@ -24,7 +24,7 @@ impl Spanned for Error {
 
 impl Error {
  #[cold]
- pub(crate) fn new(span: Span, error: SyntaxError) -> Self {
+ pub fn new(span: Span, error: SyntaxError) -> Self {
  Self {
  error: Box::new((span, error)),
  }