Conditionally enable tokenizing (#372)

* Enable 'vhdl_ls off' and 'vhdl_ls on' to conditionally enable / disable parsing * Add documentation about ignoring errors in the Readme.

Author: Schottkyc137

README.md

@@ -183,12 +183,37 @@ Using the `lint` table, you can configure the severity of diagnostics or turn of
 > [!WARNING]
 > You can overwrite every diagnostic error code including syntax or analysis errors using the lint table.
 > However, the intended use-case is for lints only.
-> Overwriting syntax or analysis errors (e.g., error codes `unused` or `syntax`) can cause unwanted side effects
+> Overwriting syntax or analysis errors (e.g., error codes `mismatched_kinds` or `syntax`) can cause unwanted side
+> effects
 
 Paths in the `vhdl_ls.toml` can contain glob patterns (i.e., `.../*/`).
 On Unix machines, they can contain environment variables using the `$NAME` or `${NAME}` syntax.
 On Windows machines, use the `%NAME%` syntax to substitute environment variables.
 
+## Ignoring errors
+
+You can use the comment-pair `-- vhdl_ls off` and `-- vhdl_ls on` to conditionally disable and re-enable parsing of
+source code. This can be helpful to ignore errors from correct code that vhdl_ls does not yet support, i.e., PSL
+statements or certain VHDL-2019 constructs.
+
+```vhdl
+library ieee;
+ use ieee.std_logic_1164.all;
+
+entity ent is
+ port (
+ clk : in std_logic
+ );
+end entity;
+
+architecture arch of ent is
+begin
+ -- vhdl_ls off
+ default clock is rising_edge(clk);
+ -- vhdl_ls on
+end architecture;
+```
+
 ## As an LSP-client developer how should I integrate VHDL-LS?
 
 I recommend that the `lsp-client` polls GitHub and downloads

vhdl_lang/src/syntax/tokens/tokenizer.rs

@@ -758,7 +758,18 @@ pub struct Comment {
  pub multi_line: bool,
 }
 
+impl Comment {
+ pub(crate) fn is_start_of_ignored_region(&self) -> bool {
+ self.value.trim() == "vhdl_ls off"
+ }
+
+ pub(crate) fn is_end_of_ignored_region(&self) -> bool {
+ self.value.trim() == "vhdl_ls on"
+ }
+}
+
 use crate::standard::VHDLStandard;
+use itertools::Itertools;
 use std::convert::AsRef;
 use std::fmt::{Debug, Display, Formatter};
 use std::ops::{Add, AddAssign, Sub};
@@ -889,6 +900,45 @@ impl Token {
  pub fn equal_format(&self, other: &Token) -> bool {
  self.kind == other.kind && self.value == other.value
  }
+
+ pub(crate) fn leading_is_start_of_ignored_region(&self) -> bool {
+ let Some(comments) = &self.comments else {
+ return false;
+ };
+ if let Some((index, _)) = comments
+ .leading
+ .iter()
+ .find_position(|comment| comment.is_start_of_ignored_region())
+ {
+ // This construct guards against the case where we have a `vhdl_ls on` in the same
+ // comment block after a a `vhdl_ls off`
+ !comments.leading[index..]
+ .iter()
+ .any(Comment::is_end_of_ignored_region)
+ } else {
+ false
+ }
+ }
+
+ pub(crate) fn leading_is_end_of_ignored_region(&self) -> bool {
+ let Some(comments) = &self.comments else {
+ return false;
+ };
+ comments
+ .leading
+ .iter()
+ .any(Comment::is_end_of_ignored_region)
+ }
+
+ pub(crate) fn trailing_is_end_of_ignored_region(&self) -> bool {
+ let Some(comments) = &self.comments else {
+ return false;
+ };
+ comments
+ .trailing
+ .as_ref()
+ .is_some_and(Comment::is_end_of_ignored_region)
+ }
 }
 
 impl Operator {
@@ -1973,7 +2023,38 @@ impl<'a> Tokenizer<'a> {
 
  pub fn pop(&mut self) -> DiagnosticResult<Option<Token>> {
  match self.pop_raw() {
- Ok(token) => Ok(token),
+ Ok(None) => Ok(None),
+ Ok(Some(token)) => {
+ if token.leading_is_start_of_ignored_region() {
+ if !token.trailing_is_end_of_ignored_region() {
+ loop {
+ match self.pop_raw() {
+ Ok(None) => {
+ // Note: we should probably emit an unterminated error here
+ // instead of silently failing.
+ return Ok(None);
+ }
+ Ok(Some(tok)) => {
+ if tok.trailing_is_end_of_ignored_region() {
+ break;
+ } else if tok.leading_is_end_of_ignored_region() {
+ // Note: to be pedantic, the 'vhdl_ls on' should be
+ // removed from the comments. However, because we have
+ // no public API and the comments don't really play
+ // a crucial role in the language server or binary,
+ // we just emit the token for simplicity.
+ return Ok(Some(tok));
+ }
+ }
+ Err(_) => {}
+ }
+ }
+ }
+ self.pop()
+ } else {
+ Ok(Some(token))
+ }
+ }
  Err(err) => {
  self.state.start = self.reader.state();
  Err(Diagnostic::syntax_error(
@@ -3106,4 +3187,63 @@ entity -- €
  vec![View, Default]
  );
  }
+
+ #[test]
+ fn ignore_code_in_between_explicitly_ignored_regions() {
+ let tokens = kinds_tokenize(
+ "\
+entity foo is
+--vhdl_ls off
+I am ignored cod€
+--vhdl_ls on
+end foo;
+ ",
+ );
+ assert_eq!(
+ &tokens,
+ &[Entity, Identifier, Is, End, Identifier, SemiColon]
+ );
+
+ let tokens = kinds_tokenize(
+ "\
+before Is
+--vhdl_ls off
+single_token
+--vhdl_ls on
+End after
+ ",
+ );
+ assert_eq!(&tokens, &[Identifier, Is, End, After]);
+ }
+
+ #[test]
+ fn ignore_code_without_on() {
+ // This should potentially emit a warning, but for now we just silently
+ // ignore that there should be a `vhdl_ls on` at some point.
+ let tokens = kinds_tokenize(
+ "\
+before Is
+--vhdl_ls off
+single_token
+End after
+ ",
+ );
+ assert_eq!(&tokens, &[Identifier, Is]);
+ }
+
+ #[test]
+ fn on_directly_after_off() {
+ // This should potentially emit a warning, but for now we just silently
+ // ignore that there should be a `vhdl_ls on` at some point.
+ let tokens = kinds_tokenize(
+ "\
+before Is
+--vhdl_ls off
+--vhdl_ls on
+single_token
+End after
+ ",
+ );
+ assert_eq!(&tokens, &[Identifier, Is, Identifier, End, After]);
+ }
 }