Conditionally enable tokenizing (#372)

* Enable 'vhdl_ls off' and 'vhdl_ls on' to conditionally enable / disable parsing

* Add documentation about ignoring errors in the Readme.

Author: Schottkyc137

README.md

@@ -183,12 +183,37 @@ Using the `lint` table, you can configure the severity of diagnostics or turn of
 > [!WARNING]
 > You can overwrite every diagnostic error code including syntax or analysis errors using the lint table.
 > However, the intended use-case is for lints only.
-> Overwriting syntax or analysis errors (e.g., error codes `unused` or `syntax`) can cause unwanted side effects
+> Overwriting syntax or analysis errors (e.g., error codes `mismatched_kinds` or `syntax`) can cause unwanted side
+> effects
 
 Paths in the `vhdl_ls.toml` can contain glob patterns (i.e., `.../*/`).
 On Unix machines, they can contain environment variables using the `$NAME` or `${NAME}` syntax.
 On Windows machines, use the `%NAME%` syntax to substitute environment variables.
 
+## Ignoring errors
+
+You can use the comment-pair `-- vhdl_ls off` and `-- vhdl_ls on` to conditionally disable and re-enable parsing of
+source code. This can be helpful to ignore errors from correct code that vhdl_ls does not yet support, i.e., PSL
+statements or certain VHDL-2019 constructs.
+
+```vhdl
+library ieee;
+    use ieee.std_logic_1164.all;
+
+entity ent is
+    port (
+       clk : in std_logic
+    );
+end entity;
+
+architecture arch of ent is
+begin
+    -- vhdl_ls off
+    default clock is rising_edge(clk);
+    -- vhdl_ls on
+end architecture;
+```
+
 ## As an LSP-client developer how should I integrate VHDL-LS?
 
 I recommend that the `lsp-client` polls GitHub and downloads

vhdl_lang/src/syntax/tokens/tokenizer.rs

@@ -758,7 +758,18 @@ pub struct Comment {
     pub multi_line: bool,
 }
 
+impl Comment {
+    pub(crate) fn is_start_of_ignored_region(&self) -> bool {
+        self.value.trim() == "vhdl_ls off"
+    }
+
+    pub(crate) fn is_end_of_ignored_region(&self) -> bool {
+        self.value.trim() == "vhdl_ls on"
+    }
+}
+
 use crate::standard::VHDLStandard;
+use itertools::Itertools;
 use std::convert::AsRef;
 use std::fmt::{Debug, Display, Formatter};
 use std::ops::{Add, AddAssign, Sub};
@@ -889,6 +900,45 @@ impl Token {
     pub fn equal_format(&self, other: &Token) -> bool {
         self.kind == other.kind && self.value == other.value
     }
+
+    pub(crate) fn leading_is_start_of_ignored_region(&self) -> bool {
+        let Some(comments) = &self.comments else {
+            return false;
+        };
+        if let Some((index, _)) = comments
+            .leading
+            .iter()
+            .find_position(|comment| comment.is_start_of_ignored_region())
+        {
+            // This construct guards against the case where we have a `vhdl_ls on` in the same
+            // comment block after a a `vhdl_ls off`
+            !comments.leading[index..]
+                .iter()
+                .any(Comment::is_end_of_ignored_region)
+        } else {
+            false
+        }
+    }
+
+    pub(crate) fn leading_is_end_of_ignored_region(&self) -> bool {
+        let Some(comments) = &self.comments else {
+            return false;
+        };
+        comments
+            .leading
+            .iter()
+            .any(Comment::is_end_of_ignored_region)
+    }
+
+    pub(crate) fn trailing_is_end_of_ignored_region(&self) -> bool {
+        let Some(comments) = &self.comments else {
+            return false;
+        };
+        comments
+            .trailing
+            .as_ref()
+            .is_some_and(Comment::is_end_of_ignored_region)
+    }
 }
 
 impl Operator {
@@ -1973,7 +2023,38 @@ impl<'a> Tokenizer<'a> {
 
     pub fn pop(&mut self) -> DiagnosticResult<Option<Token>> {
         match self.pop_raw() {
-            Ok(token) => Ok(token),
+            Ok(None) => Ok(None),
+            Ok(Some(token)) => {
+                if token.leading_is_start_of_ignored_region() {
+                    if !token.trailing_is_end_of_ignored_region() {
+                        loop {
+                            match self.pop_raw() {
+                                Ok(None) => {
+                                    // Note: we should probably emit an unterminated error here
+                                    // instead of silently failing.
+                                    return Ok(None);
+                                }
+                                Ok(Some(tok)) => {
+                                    if tok.trailing_is_end_of_ignored_region() {
+                                        break;
+                                    } else if tok.leading_is_end_of_ignored_region() {
+                                        // Note: to be pedantic, the 'vhdl_ls on' should be
+                                        // removed from the comments. However, because we have
+                                        // no public API and the comments don't really play
+                                        // a crucial role in the language server or binary,
+                                        // we just emit the token for simplicity.
+                                        return Ok(Some(tok));
+                                    }
+                                }
+                                Err(_) => {}
+                            }
+                        }
+                    }
+                    self.pop()
+                } else {
+                    Ok(Some(token))
+                }
+            }
             Err(err) => {
                 self.state.start = self.reader.state();
                 Err(Diagnostic::syntax_error(
@@ -3106,4 +3187,63 @@ entity -- €
             vec![View, Default]
         );
     }
+
+    #[test]
+    fn ignore_code_in_between_explicitly_ignored_regions() {
+        let tokens = kinds_tokenize(
+            "\
+entity foo is
+--vhdl_ls off
+I am ignored cod€
+--vhdl_ls on
+end foo;
+        ",
+        );
+        assert_eq!(
+            &tokens,
+            &[Entity, Identifier, Is, End, Identifier, SemiColon]
+        );
+
+        let tokens = kinds_tokenize(
+            "\
+before Is
+--vhdl_ls off
+single_token
+--vhdl_ls on
+End after
+        ",
+        );
+        assert_eq!(&tokens, &[Identifier, Is, End, After]);
+    }
+
+    #[test]
+    fn ignore_code_without_on() {
+        // This should potentially emit a warning, but for now we just silently
+        // ignore that there should be a `vhdl_ls on` at some point.
+        let tokens = kinds_tokenize(
+            "\
+before Is
+--vhdl_ls off
+single_token
+End after
+        ",
+        );
+        assert_eq!(&tokens, &[Identifier, Is]);
+    }
+
+    #[test]
+    fn on_directly_after_off() {
+        // This should potentially emit a warning, but for now we just silently
+        // ignore that there should be a `vhdl_ls on` at some point.
+        let tokens = kinds_tokenize(
+            "\
+before Is
+--vhdl_ls off
+--vhdl_ls on
+single_token
+End after
+        ",
+        );
+        assert_eq!(&tokens, &[Identifier, Is, Identifier, End, After]);
+    }
 }