wowlab_tidy/infra/
scanner.rs

1//! Shared code scanner for skipping string literals, char literals, and comments.
2
3#[cfg(test)]
4use googletest::prelude::*;
5use winnow::token::{any, take_till, take_while};
6
7use super::parse;
8
9/// Return the byte column where each source line's real `//` comment starts.
10// #t(fn: rust_cyclomatic_complexity) lexical state dispatch necessarily branches for each Rust token class
11pub(crate) fn line_comment_starts(source: &str) -> Vec<Option<usize>> {
12    const TWO_BYTE_TOKEN: usize = 2;
13
14    let bytes = source.as_bytes();
15    let mut starts = vec![None];
16    let mut line = 0;
17    let mut column = 0;
18    let mut index = 0;
19    let mut state = LexState::Code;
20
21    while let Some(&byte) = bytes.get(index) {
22        if byte == b'\n' {
23            starts.push(None);
24            line += 1;
25            column = 0;
26            index += 1;
27
28            if matches!(state, LexState::LineComment | LexState::Char { .. }) {
29                state = LexState::Code;
30            }
31
32            continue;
33        }
34
35        let next = bytes.get(index + 1).copied();
36
37        match state {
38            LexState::Code => match (byte, next) {
39                (b'/', Some(b'/')) => {
40                    if let Some(start) = starts.get_mut(line) {
41                        *start = Some(column);
42                    }
43
44                    state = LexState::LineComment;
45                    index += TWO_BYTE_TOKEN;
46                    column += TWO_BYTE_TOKEN;
47                }
48                (b'/', Some(b'*')) => {
49                    state = LexState::BlockComment { depth: 1 };
50                    index += TWO_BYTE_TOKEN;
51                    column += TWO_BYTE_TOKEN;
52                }
53                (b'"', _) => {
54                    state = LexState::String { escaped: false };
55                    index += 1;
56                    column += 1;
57                }
58                (b'\'', _) => {
59                    state = LexState::Char { escaped: false };
60                    index += 1;
61                    column += 1;
62                }
63                (b'r', _) => {
64                    let remaining = bytes.get(index..).unwrap_or_default();
65
66                    if let Some((hashes, consumed)) = raw_string_open(remaining) {
67                        state = LexState::RawString { hashes };
68                        index += consumed;
69                        column += consumed;
70                    } else {
71                        index += 1;
72                        column += 1;
73                    }
74                }
75                _ => {
76                    index += 1;
77                    column += 1;
78                }
79            },
80            LexState::LineComment => {
81                index += 1;
82                column += 1;
83            }
84            LexState::BlockComment { depth } => match (byte, next) {
85                (b'/', Some(b'*')) => {
86                    state = LexState::BlockComment { depth: depth + 1 };
87                    index += TWO_BYTE_TOKEN;
88                    column += TWO_BYTE_TOKEN;
89                }
90                (b'*', Some(b'/')) => {
91                    state = if depth == 1 {
92                        LexState::Code
93                    } else {
94                        LexState::BlockComment { depth: depth - 1 }
95                    };
96                    index += TWO_BYTE_TOKEN;
97                    column += TWO_BYTE_TOKEN;
98                }
99                _ => {
100                    index += 1;
101                    column += 1;
102                }
103            },
104            LexState::String { escaped } | LexState::Char { escaped } => {
105                let quote = if matches!(state, LexState::String { .. }) {
106                    b'"'
107                } else {
108                    b'\''
109                };
110
111                state = if escaped {
112                    if quote == b'"' {
113                        LexState::String { escaped: false }
114                    } else {
115                        LexState::Char { escaped: false }
116                    }
117                } else if byte == b'\\' {
118                    if quote == b'"' {
119                        LexState::String { escaped: true }
120                    } else {
121                        LexState::Char { escaped: true }
122                    }
123                } else if byte == quote {
124                    LexState::Code
125                } else {
126                    state
127                };
128                index += 1;
129                column += 1;
130            }
131            LexState::RawString { hashes } => {
132                let after_quote = bytes.get(index + 1..).unwrap_or_default();
133
134                if byte == b'"' && raw_string_close(after_quote, hashes) {
135                    let consumed = hashes + 1;
136
137                    state = LexState::Code;
138                    index += consumed;
139                    column += consumed;
140                } else {
141                    index += 1;
142                    column += 1;
143                }
144            }
145        }
146    }
147
148    starts
149}
150
151/// Hide directive-shaped text that is not a standalone source comment.
152pub(crate) fn directive_source_lines<'a>(source: &str, lines: &[&'a str]) -> Vec<&'a str> {
153    let comment_starts = line_comment_starts(source);
154
155    lines
156        .iter()
157        .enumerate()
158        .map(|(index, &line)| {
159            if parse::directive(line).is_none() {
160                return line;
161            }
162
163            let indentation = line.len() - line.trim_start().len();
164
165            if comment_starts.get(index) == Some(&Some(indentation)) {
166                line
167            } else {
168                ""
169            }
170        })
171        .collect()
172}
173
174#[derive(Clone, Copy)]
175enum LexState {
176    Code,
177    LineComment,
178    BlockComment { depth: usize },
179    String { escaped: bool },
180    Char { escaped: bool },
181    RawString { hashes: usize },
182}
183
184const RAW_DELIMITER_BYTES: usize = 2;
185
186fn raw_string_open(bytes: &[u8]) -> Option<(usize, usize)> {
187    let mut hashes = 0;
188
189    while bytes.get(hashes + 1) == Some(&b'#') {
190        hashes += 1;
191    }
192
193    (bytes.get(hashes + 1) == Some(&b'"')).then_some((hashes, hashes + RAW_DELIMITER_BYTES))
194}
195
196fn raw_string_close(bytes: &[u8], hashes: usize) -> bool {
197    bytes
198        .get(..hashes)
199        .is_some_and(|suffix| suffix.iter().all(|byte| *byte == b'#'))
200}
201
202/// Skip past a `"..."` string literal, assuming the opening `"` is already consumed.
203pub(crate) fn skip_string(input: &mut &str) {
204    loop {
205        let _ = parse::try_parse(input, take_till(0.., ['\\', '"']));
206
207        match parse::try_parse(input, any) {
208            Some('\\') => {
209                let _ = parse::try_parse(input, any);
210            }
211            _ => return,
212        }
213    }
214}
215
216/// Skip past a raw string literal with `hashes` closing `#`, opening `"` already consumed.
217pub(crate) fn skip_raw_string(input: &mut &str, hashes: usize) {
218    loop {
219        let _ = parse::try_parse(input, take_till(0.., '"'));
220
221        if parse::try_parse(input, any).is_none() {
222            return;
223        }
224
225        let mut matched = 0;
226
227        while matched < hashes && parse::try_parse(input, '#').is_some() {
228            matched += 1;
229        }
230
231        if matched == hashes {
232            return;
233        }
234    }
235}
236
237/// Skip a character literal, assuming the opening `'` is already consumed.
238pub(crate) fn skip_char_literal(input: &mut &str) {
239    let _ = parse::try_parse(input, '\\');
240    let _ = parse::try_parse(input, any);
241    let _ = parse::try_parse(input, '\'');
242}
243
244/// Consume a raw string prefix and body after an `r`, `true` if one was consumed.
245pub(crate) fn try_skip_raw_string(input: &mut &str) -> bool {
246    let hashes = parse::try_parse(input, take_while(0.., '#')).map_or(0, |s: &str| s.len());
247
248    if parse::try_parse(input, '"').is_some() {
249        skip_raw_string(input, hashes);
250
251        true
252    } else {
253        false
254    }
255}
256
257/// Return a copy of `line` with string literals, char literals, and trailing `//` comments stripped out.
258pub(crate) fn code_only(line: &str) -> String {
259    let mut code = String::with_capacity(line.len());
260    let mut input = line;
261
262    while let Some(ch) = parse::try_parse(&mut input, any) {
263        match ch {
264            '/' if parse::matches(input, '/') => break,
265            '"' => skip_string(&mut input),
266            'r' if parse::matches(input, '#') || parse::matches(input, '"') => {
267                if !try_skip_raw_string(&mut input) {
268                    code.push(ch);
269                }
270            }
271            '\'' => skip_char_literal(&mut input),
272            _ => code.push(ch),
273        }
274    }
275
276    code
277}
278
279/// Return the byte offsets of every occurrence of `target` in `line`, skipping string and char literals.
280pub(crate) fn char_positions(line: &str, target: char) -> Vec<usize> {
281    let mut out = Vec::new();
282    let mut input = line;
283    let mut pos = 0;
284
285    while let Some(ch) = parse::try_parse(&mut input, any) {
286        match ch {
287            '/' if parse::matches(input, '/') => break,
288            '"' => {
289                skip_string(&mut input);
290                pos = line.len() - input.len();
291                continue;
292            }
293            'r' if parse::matches(input, '#') || parse::matches(input, '"') => {
294                if try_skip_raw_string(&mut input) {
295                    pos = line.len() - input.len();
296                    continue;
297                }
298
299                if ch == target {
300                    out.push(pos);
301                }
302
303                pos += ch.len_utf8();
304                continue;
305            }
306            '\'' => {
307                skip_char_literal(&mut input);
308                pos = line.len() - input.len();
309                continue;
310            }
311            c if c == target => {
312                out.push(pos);
313            }
314            _ => {}
315        }
316
317        pos += ch.len_utf8();
318    }
319
320    out
321}
322
323#[cfg(test)]
324mod tests {
325    use super::*;
326
327    #[gtest]
328    fn code_only_strips_strings() -> Result<()> {
329        verify_eq!(code_only(r#"let x = "hello"; y"#), "let x = ; y")?;
330
331        Ok(())
332    }
333
334    #[gtest]
335    fn code_only_strips_comments() -> Result<()> {
336        verify_eq!(code_only("let x = 1; // comment"), "let x = 1; ")?;
337
338        Ok(())
339    }
340
341    #[gtest]
342    fn code_only_strips_char_literals() -> Result<()> {
343        verify_eq!(code_only("let c = '{'; rest"), "let c = ; rest")?;
344
345        Ok(())
346    }
347
348    #[gtest]
349    fn code_only_strips_raw_strings() -> Result<()> {
350        let input = "let s = r\"raw\"; rest";
351
352        verify_eq!(code_only(input), "let s = ; rest")?;
353
354        Ok(())
355    }
356
357    #[gtest]
358    fn char_positions_basic() -> Result<()> {
359        verify_eq!(char_positions("a, b, c", ','), vec![1, 4])?;
360
361        Ok(())
362    }
363
364    #[gtest]
365    fn char_positions_skips_strings() -> Result<()> {
366        let input = "call(a, \"hello, world\", b)";
367
368        verify_eq!(char_positions(input, ','), vec![6, 22])?;
369
370        Ok(())
371    }
372
373    #[gtest]
374    fn char_positions_skips_char_literals() -> Result<()> {
375        let input = "let c = '\\'' ; x, y";
376        let positions = char_positions(input, ',');
377
378        verify_eq!(positions, vec![16])?;
379
380        Ok(())
381    }
382
383    #[gtest]
384    fn char_positions_skips_comments() -> Result<()> {
385        verify_eq!(char_positions("a, b // c, d", ','), vec![1])?;
386
387        Ok(())
388    }
389
390    #[gtest]
391    fn comment_lines_exclude_multiline_string_lookalikes() -> Result<()> {
392        let source =
393            "let fixture = r#\"\n// #t(rust_panic) not source\n\"#;\n// #t(rust_dbg) source\n";
394        let starts = line_comment_starts(source);
395
396        verify_eq!(starts, vec![None, None, None, Some(0), None])?;
397
398        Ok(())
399    }
400
401    #[gtest]
402    fn comment_lines_exclude_nested_block_comment_lookalikes() -> Result<()> {
403        let source =
404            "/* outer\n/* nested */\n// #t(rust_panic) not source\n*/\n    // #t(rust_dbg) source";
405        let starts = line_comment_starts(source);
406
407        verify_eq!(starts, vec![None, None, None, None, Some(4)])?;
408
409        Ok(())
410    }
411
412    #[gtest]
413    fn directive_lines_exclude_non_comment_lookalikes() -> Result<()> {
414        let source = concat!(
415            "let fixture = r#\"\n",
416            "// #t(file: rust_panic) raw string\n",
417            "\"#;\n",
418            "/*\n",
419            "// #t(file: rust_dbg) block comment\n",
420            "*/\n",
421            "    // #t(rust_panic) real directive\n",
422            "panic!(\"boom\");",
423        );
424        let lines: Vec<_> = source.lines().collect();
425        let visible = directive_source_lines(source, &lines);
426
427        verify_eq!(visible[1], "")?;
428        verify_eq!(visible[4], "")?;
429        verify_eq!(visible[6], "    // #t(rust_panic) real directive")?;
430
431        Ok(())
432    }
433}
wowlab_tidy/infra/scanner.rs

wowlab_tidy/infra/
scanner.rs
