From 5346086a54970066524ab7d83bed61d0511705d4 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:26:10 -0700 Subject: [PATCH 1/2] Add backward movement and search to Scanner prev steps back one UTF-16 code unit, including across the synthetic newline between lines, and leaves the position unchanged at the start of the input. findPrev searches backward for a character or a CharMatcher, with overloads that take an inclusive lower position so the search can stop at a saved opener. The current character is included, the end sentinel is skipped and never matches, a hit returns how many code units were moved, and a miss returns -1 while staying at the start of the input or at the bound. A bound after the current position or outside the input throws IllegalArgumentException before the position changes. Fixes #395 --- .../org/commonmark/parser/beta/Scanner.java | 152 +++++- .../parser/InlineContentParserTest.java | 91 ++++ .../commonmark/parser/beta/ScannerTest.java | 467 ++++++++++++++++++ 3 files changed, 709 insertions(+), 1 deletion(-) diff --git a/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java b/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java index 1acf39ced..91aaa6015 100644 --- a/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java +++ b/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java @@ -13,7 +13,8 @@ public class Scanner { * "previous" methods). * *

Note that we can use NULL to represent this because CommonMark does not allow those in the - * input (we replace them in the beginning of parsing). + * input (we replace them in the beginning of parsing). It is not a searchable source character: + * {@link #find(char)} and {@link #findPrev(char)} skip it instead of matching it. */ public static final char END = '\0'; @@ -119,6 +120,30 @@ public void next() { } } + /** + * Move back by one UTF-16 code unit. + * + *

This reverses {@link #next()}. Lines yield a synthetic newline between lines and no + * newline after the final line: from index zero the position moves to the previous line's + * length (that newline), and from the end of input it moves to the last character, or across + * an empty final line to the preceding newline. At the beginning of the input, including an + * empty input, the position is left unchanged. + */ + public void prev() { + if (index > 0) { + index--; + } else if (lineIndex > 0) { + lineIndex--; + if (lineIndex < lines.size()) { + setLine(lines.get(lineIndex)); + } else { + // Landed past the end (after next() at EOF); same representation as next(). + setLine(SourceLine.of("", null)); + } + index = lineLength; + } + } + /** * Check if the specified char is next and advance the position. * @@ -221,6 +246,100 @@ public int find(CharMatcher matcher) { } } + /** + * Find the nearest occurrence of {@code c} at or before the current position. + * + *

The current character is included, matching {@link #find(char)}, so an immediate match + * returns {@code 0}. {@link #END} is skipped and never matches. From the end of input a match + * on the character immediately before the end returns {@code 1}. A newline between lines can + * be matched. If {@code c} does not occur, the scanner is left at the beginning of the input + * and {@code -1} is returned. + * + * @param c character to find (including a newline) + * @return the number of UTF-16 code units moved backward, or {@code -1} if not found + */ + public int findPrev(char c) { + return findPrev(c, null, 0, 0); + } + + /** + * Find the nearest character matched by {@code matcher} at or before the current position. + * + *

See {@link #findPrev(char)} for the current position, {@link #END}, and a search that + * reaches the beginning of the input. {@link #END} is not passed to {@code matcher}. + * + * @param matcher matcher applied to each character + * @return the number of UTF-16 code units moved backward, or {@code -1} if not found + */ + public int findPrev(CharMatcher matcher) { + return findPrev(END, matcher, 0, 0); + } + + /** + * Find the nearest occurrence of {@code c} at or before the current position and at or after + * {@code bound}. + * + *

{@code bound} is inclusive: the character there is examined, and the search never moves + * before it. If {@code bound} is the current position, only that character is checked. If there + * is no match, the scanner is left at {@code bound} and {@code -1} is returned. {@link #END} is + * skipped and never matches. On an empty input the search returns {@code -1}; the position at + * the beginning of that input is a valid bound. + * + *

The bound is checked before the scanner moves. It must have valid coordinates for this + * scanner and must not be after the current position. As with {@link #setPosition(Position)}, + * positions are expected to come from the same scanner. + * + * @param c character to find (including a newline) + * @param bound inclusive lower bound + * @return the number of UTF-16 code units moved backward, or {@code -1} if not found + * @throws IllegalArgumentException if {@code bound} is after the current position or outside + * the input + */ + public int findPrev(char c, Position bound) { + checkLowerBound(bound); + return findPrev(c, null, bound.lineIndex, bound.index); + } + + /** + * Find the nearest character matched by {@code matcher} at or before the current position and + * at or after {@code bound}. + * + *

See {@link #findPrev(char, Position)} for the bound, {@link #END}, and a search with no + * match. {@link #END} is not passed to {@code matcher}. + * + * @param matcher matcher applied to each character + * @param bound inclusive lower bound + * @return the number of UTF-16 code units moved backward, or {@code -1} if not found + * @throws IllegalArgumentException if {@code bound} is after the current position or outside + * the input + */ + public int findPrev(CharMatcher matcher, Position bound) { + checkLowerBound(bound); + return findPrev(END, matcher, bound.lineIndex, bound.index); + } + + // matcher == null matches c exactly, so the char overloads allocate nothing per character. + private int findPrev(char c, CharMatcher matcher, int boundLine, int boundIndex) { + int count = 0; + while (true) { + char cur = peek(); + if (cur != END && (matcher != null ? matcher.matches(cur) : cur == c)) { + return count; + } + if (lineIndex == boundLine && index == boundIndex) { + return -1; + } + int lineBefore = lineIndex; + int indexBefore = index; + prev(); + // Beginning of input: prev() is a no-op, so stop instead of spinning. + if (lineIndex == lineBefore && index == indexBefore) { + return -1; + } + count++; + } + } + // Don't expose the int index, because it would be good if we could switch input to a // List of lines later instead of one contiguous String. public Position position() { @@ -280,4 +399,35 @@ private void checkPosition(int lineIndex, int index) { "Index " + index + " out of range, line length: " + line.getContent().length()); } } + + private void checkLowerBound(Position position) { + // Empty input has no lines; its only position is the canonical (0, 0) cursor. + if (lines.isEmpty()) { + if (position.lineIndex != 0) { + throw new IllegalArgumentException( + "Line index " + + position.lineIndex + + " out of range, number of lines: " + + lines.size()); + } + if (position.index != 0) { + throw new IllegalArgumentException( + "Index " + position.index + " out of range, line length: 0"); + } + } else { + checkPosition(position.lineIndex, position.index); + } + if (position.lineIndex > lineIndex + || (position.lineIndex == lineIndex && position.index > index)) { + throw new IllegalArgumentException( + "Bound line " + + position.lineIndex + + " index " + + position.index + + " is after the current position line " + + lineIndex + + " index " + + index); + } + } } diff --git a/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java b/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java index 167f74f25..fe4d03a17 100644 --- a/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java +++ b/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java @@ -9,6 +9,8 @@ import org.commonmark.parser.beta.InlineParserState; import org.commonmark.parser.beta.ParsedInline; import org.commonmark.test.Nodes; +import org.commonmark.text.AsciiMatcher; +import org.commonmark.text.CharMatcher; import org.junit.jupiter.api.Test; class InlineContentParserTest { @@ -55,6 +57,46 @@ void bangInlineContentParser() { assertThat(((Text) image.getNext().getNext().getNext()).getLiteral()).isEqualTo("notimage"); } + @Test + void backwardSearchInlineContentParser() { + var parser = + Parser.builder() + .customInlineContentParserFactory(new DestinationInlineParser.Factory()) + .build(); + + var doc = parser.parse("img(not-a-link-)more-text) more text"); + var inline = Nodes.find(doc, DestinationInline.class); + assertThat(((Text) inline.getPrevious()).getLiteral()).isEqualTo("img"); + assertThat(inline.getDestination()).isEqualTo("not-a-link-)more-text"); + assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text"); + + doc = parser.parse("img(not-a-link-)more-text)"); + inline = Nodes.find(doc, DestinationInline.class); + assertThat(inline.getDestination()).isEqualTo("not-a-link-)more-text"); + assertThat(inline.getNext()).isNull(); + + doc = parser.parse("img(not-a-link) more text"); + inline = Nodes.find(doc, DestinationInline.class); + assertThat(inline.getDestination()).isEqualTo("not-a-link"); + assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text"); + + doc = parser.parse("img() more text"); + inline = Nodes.find(doc, DestinationInline.class); + assertThat(inline.getDestination()).isEqualTo(""); + assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text"); + + doc = parser.parse("img(not-a-link more text"); + assertThat(Nodes.tryFind(doc, DestinationInline.class)).isNull(); + assertThat(((Text) doc.getFirstChild().getFirstChild()).getLiteral()) + .isEqualTo("img(not-a-link more text"); + + // The ')' before the saved opening position is outside the inclusive bound. + doc = parser.parse("before)img(not-a-link more text"); + assertThat(Nodes.tryFind(doc, DestinationInline.class)).isNull(); + assertThat(((Text) doc.getFirstChild().getFirstChild()).getLiteral()) + .isEqualTo("before)img(not-a-link more text"); + } + private static class DollarInline extends CustomNode { private final String literal; private final int index; @@ -105,6 +147,55 @@ public InlineContentParser create() { } } + /** + * Custom image-like syntax {@code (destination)} whose destination may itself contain {@code + * )}. The closer is the last {@code )} before whitespace or the end of input, and it must not + * be before the opening parenthesis. + */ + private static class DestinationInline extends CustomNode { + private final String destination; + + public DestinationInline(String destination) { + this.destination = destination; + } + + public String getDestination() { + return destination; + } + } + + private static class DestinationInlineParser implements InlineContentParser { + + private static final CharMatcher WHITESPACE = + AsciiMatcher.builder().anyOf(" \t\n\u000B\f\r").build(); + + @Override + public ParsedInline tryParse(InlineParserState inlineParserState) { + var scanner = inlineParserState.scanner(); + scanner.next(); + var contentStart = scanner.position(); + scanner.find(WHITESPACE); + if (scanner.findPrev(')', contentStart) == -1) { + return ParsedInline.none(); + } + var destination = scanner.getSource(contentStart, scanner.position()).getContent(); + scanner.next(); + return ParsedInline.of(new DestinationInline(destination), scanner.position()); + } + + static class Factory implements InlineContentParserFactory { + @Override + public Set getTriggerCharacters() { + return Set.of('('); + } + + @Override + public InlineContentParser create() { + return new DestinationInlineParser(); + } + } + } + private static class BangInline extends CustomNode {} private static class BangInlineParser implements InlineContentParser { diff --git a/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java b/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java index 437812866..44f94c957 100644 --- a/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java +++ b/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java @@ -1,11 +1,13 @@ package org.commonmark.parser.beta; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.List; import org.commonmark.node.SourceSpan; import org.commonmark.parser.SourceLine; import org.commonmark.parser.SourceLines; +import org.commonmark.text.CharMatcher; import org.junit.jupiter.api.Test; class ScannerTest { @@ -144,6 +146,471 @@ private void assertSourceLines( assertThat(sourceLines.getSourceSpans()).isEqualTo(List.of(expectedSourceSpans)); } + @Test + void testPrev() { + Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("abc", null))); + Position start = scanner.position(); + scanner.next(); + Position b = scanner.position(); + assertThat(scanner.peek()).isEqualTo('b'); + scanner.next(); + Position c = scanner.position(); + scanner.next(); + Position eof = scanner.position(); + assertThat(scanner.peek()).isEqualTo('\0'); + assertThat(scanner.hasNext()).isFalse(); + + scanner.prev(); + assertThat(scanner.position()).isEqualTo(c); + assertThat(scanner.peek()).isEqualTo('c'); + scanner.prev(); + assertThat(scanner.position()).isEqualTo(b); + assertThat(scanner.peek()).isEqualTo('b'); + scanner.prev(); + assertThat(scanner.position()).isEqualTo(start); + assertThat(scanner.peek()).isEqualTo('a'); + + // No-op at the beginning. + scanner.prev(); + assertThat(scanner.position()).isEqualTo(start); + assertThat(scanner.peek()).isEqualTo('a'); + + scanner.next(); + scanner.prev(); + assertThat(scanner.position()).isEqualTo(start); + assertThat(scanner.peek()).isEqualTo('a'); + + scanner.setPosition(eof); + scanner.prev(); + scanner.next(); + assertThat(scanner.position()).isEqualTo(eof); + assertThat(scanner.peek()).isEqualTo('\0'); + + Scanner empty = Scanner.of(SourceLines.empty()); + Position emptyPos = empty.position(); + empty.prev(); + assertThat(empty.position()).isEqualTo(emptyPos); + assertThat(empty.peek()).isEqualTo('\0'); + assertThat(empty.hasNext()).isFalse(); + + Scanner emptyLine = Scanner.of(SourceLines.of(SourceLine.of("", null))); + Position emptyLinePos = emptyLine.position(); + assertThat(emptyLine.peek()).isEqualTo('\0'); + assertThat(emptyLine.hasNext()).isFalse(); + emptyLine.prev(); + assertThat(emptyLine.position()).isEqualTo(emptyLinePos); + assertThat(emptyLine.peek()).isEqualTo('\0'); + } + + @Test + void testPrevAcrossLines() { + Scanner scanner = + Scanner.of( + SourceLines.of( + List.of(SourceLine.of("ab", null), SourceLine.of("c", null)))); + Position start = scanner.position(); + scanner.next(); + assertThat(scanner.peek()).isEqualTo('b'); + scanner.next(); + Position newline = scanner.position(); + assertThat(scanner.peek()).isEqualTo('\n'); + scanner.next(); + assertThat(scanner.peek()).isEqualTo('c'); + scanner.next(); + assertThat(scanner.peek()).isEqualTo('\0'); + assertThat(scanner.hasNext()).isFalse(); + + scanner.prev(); + assertThat(scanner.peek()).isEqualTo('c'); + scanner.prev(); + assertThat(scanner.position()).isEqualTo(newline); + assertThat(scanner.peek()).isEqualTo('\n'); + scanner.prev(); + assertThat(scanner.peek()).isEqualTo('b'); + scanner.next(); + assertThat(scanner.position()).isEqualTo(newline); + assertThat(scanner.peek()).isEqualTo('\n'); + scanner.prev(); + scanner.prev(); + assertThat(scanner.position()).isEqualTo(start); + assertThat(scanner.peek()).isEqualTo('a'); + scanner.prev(); + assertThat(scanner.position()).isEqualTo(start); + + // Unequal lengths: cached line content/length must follow the line we step onto. + Scanner uneven = + Scanner.of( + SourceLines.of( + List.of(SourceLine.of("a", null), SourceLine.of("bbb", null)))); + uneven.next(); + uneven.next(); + assertThat(uneven.peek()).isEqualTo('b'); + uneven.prev(); + assertThat(uneven.peek()).isEqualTo('\n'); + uneven.prev(); + assertThat(uneven.peek()).isEqualTo('a'); + uneven.next(); + assertThat(uneven.peek()).isEqualTo('\n'); + uneven.next(); + uneven.next(); + uneven.next(); + uneven.next(); + assertThat(uneven.peek()).isEqualTo('\0'); + uneven.prev(); + assertThat(uneven.peek()).isEqualTo('b'); + + // Consecutive empty lines, including an empty final line: no newline after the last line. + Scanner blanks = + Scanner.of( + SourceLines.of( + List.of( + SourceLine.of("", null), + SourceLine.of("", null), + SourceLine.of("", null)))); + Position blanksStart = blanks.position(); + assertThat(blanks.peek()).isEqualTo('\n'); + blanks.next(); + assertThat(blanks.peek()).isEqualTo('\n'); + blanks.next(); + assertThat(blanks.peek()).isEqualTo('\0'); + assertThat(blanks.hasNext()).isFalse(); + blanks.prev(); + assertThat(blanks.peek()).isEqualTo('\n'); + blanks.prev(); + assertThat(blanks.position()).isEqualTo(blanksStart); + assertThat(blanks.peek()).isEqualTo('\n'); + blanks.next(); + blanks.next(); + assertThat(blanks.peek()).isEqualTo('\0'); + + // Empty final line steps back onto the preceding newline, not a made-up terminal one. + Scanner trailingBlank = + Scanner.of( + SourceLines.of( + List.of(SourceLine.of("a", null), SourceLine.of("", null)))); + Position trailingStart = trailingBlank.position(); + trailingBlank.next(); + Position precedingNewline = trailingBlank.position(); + assertThat(trailingBlank.peek()).isEqualTo('\n'); + trailingBlank.next(); + assertThat(trailingBlank.peek()).isEqualTo('\0'); + assertThat(trailingBlank.hasNext()).isFalse(); + trailingBlank.prev(); + assertThat(trailingBlank.position()).isEqualTo(precedingNewline); + assertThat(trailingBlank.peek()).isEqualTo('\n'); + trailingBlank.next(); + assertThat(trailingBlank.peek()).isEqualTo('\0'); + trailingBlank.prev(); + trailingBlank.prev(); + assertThat(trailingBlank.position()).isEqualTo(trailingStart); + assertThat(trailingBlank.peek()).isEqualTo('a'); + } + + @Test + void testFindPrev() { + Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("abXcdX", null))); + Position start = scanner.position(); + assertThat(scanner.findPrev('a')).isEqualTo(0); + assertThat(scanner.position()).isEqualTo(start); + assertThat(scanner.findPrev(c -> c == 'a')).isEqualTo(0); + assertThat(scanner.position()).isEqualTo(start); + + scanner.next(); + scanner.next(); + Position firstX = scanner.position(); + assertThat(scanner.peek()).isEqualTo('X'); + assertThat(scanner.findPrev('X')).isEqualTo(0); + assertThat(scanner.position()).isEqualTo(firstX); + + scanner.next(); + scanner.next(); + // Between the two matches: the nearer (earlier) X, not one after the cursor. + assertThat(scanner.peek()).isEqualTo('d'); + assertThat(scanner.findPrev('X')).isEqualTo(2); + assertThat(scanner.position()).isEqualTo(firstX); + + scanner.next(); + scanner.next(); + scanner.next(); + Position secondX = scanner.position(); + assertThat(scanner.peek()).isEqualTo('X'); + scanner.next(); + assertThat(scanner.peek()).isEqualTo('\0'); + // Match immediately before EOF counts the step off the sentinel. + assertThat(scanner.findPrev('X')).isEqualTo(1); + assertThat(scanner.position()).isEqualTo(secondX); + assertThat(scanner.peek()).isEqualTo('X'); + + scanner.next(); + assertThat(scanner.findPrev(c -> c == 'X')).isEqualTo(1); + assertThat(scanner.position()).isEqualTo(secondX); + + assertThat(scanner.findPrev('z')).isEqualTo(-1); + assertThat(scanner.position()).isEqualTo(start); + assertThat(scanner.peek()).isEqualTo('a'); + + scanner.setPosition(secondX); + scanner.next(); + assertThat(scanner.findPrev(Scanner.END)).isEqualTo(-1); + assertThat(scanner.position()).isEqualTo(start); + scanner.setPosition(secondX); + scanner.next(); + assertThat(scanner.findPrev(c -> c == Scanner.END)).isEqualTo(-1); + assertThat(scanner.position()).isEqualTo(start); + + CharMatcher any = c -> true; + scanner.setPosition(secondX); + assertThat(scanner.findPrev(any)).isEqualTo(0); + assertThat(scanner.position()).isEqualTo(secondX); + scanner.next(); + assertThat(scanner.findPrev(any)).isEqualTo(1); + assertThat(scanner.position()).isEqualTo(secondX); + assertThat(scanner.peek()).isEqualTo('X'); + + Scanner lines = + Scanner.of( + SourceLines.of( + List.of(SourceLine.of("ab", null), SourceLine.of("c", null)))); + lines.next(); + lines.next(); + Position newline = lines.position(); + lines.next(); + lines.next(); + assertThat(lines.peek()).isEqualTo('\0'); + assertThat(lines.findPrev('\n')).isEqualTo(2); + assertThat(lines.position()).isEqualTo(newline); + assertThat(lines.peek()).isEqualTo('\n'); + assertThat(lines.findPrev(c -> c == '\n')).isEqualTo(0); + assertThat(lines.position()).isEqualTo(newline); + + Scanner empty = Scanner.of(SourceLines.empty()); + Position emptyPos = empty.position(); + assertThat(empty.findPrev('a')).isEqualTo(-1); + assertThat(empty.findPrev(Scanner.END)).isEqualTo(-1); + assertThat(empty.findPrev(any)).isEqualTo(-1); + assertThat(empty.findPrev(c -> c == Scanner.END)).isEqualTo(-1); + assertThat(empty.position()).isEqualTo(emptyPos); + assertThat(empty.peek()).isEqualTo('\0'); + + Scanner text = Scanner.of(SourceLines.of(SourceLine.of("a", null))); + text.next(); + assertThat(text.findPrev(any)).isEqualTo(1); + assertThat(text.peek()).isEqualTo('a'); + } + + @Test + void testFindPrevBounds() { + Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("aXbYc", null))); + Position start = scanner.position(); + scanner.next(); + Position x = scanner.position(); + scanner.next(); + scanner.next(); + Position y = scanner.position(); + scanner.next(); + scanner.next(); + Position eof = scanner.position(); + assertThat(scanner.peek()).isEqualTo('\0'); + + // Inclusive bound on the same line, including a match sitting on the bound. + assertThat(scanner.findPrev('Y', x)).isEqualTo(2); + assertThat(scanner.position()).isEqualTo(y); + scanner.setPosition(eof); + assertThat(scanner.findPrev('Y', y)).isEqualTo(2); + assertThat(scanner.position()).isEqualTo(y); + assertThat(scanner.peek()).isEqualTo('Y'); + + // A match before the bound is excluded; a miss stays on the bound. + scanner.setPosition(eof); + assertThat(scanner.findPrev('X', y)).isEqualTo(-1); + assertThat(scanner.position()).isEqualTo(y); + assertThat(scanner.peek()).isEqualTo('Y'); + scanner.setPosition(eof); + assertThat(scanner.findPrev('z', y)).isEqualTo(-1); + assertThat(scanner.position()).isEqualTo(y); + + // Start and bound are the same position: only that character is checked. + assertThat(scanner.findPrev('Y', y)).isEqualTo(0); + assertThat(scanner.position()).isEqualTo(y); + assertThat(scanner.findPrev('X', y)).isEqualTo(-1); + assertThat(scanner.position()).isEqualTo(y); + scanner.setPosition(eof); + assertThat(scanner.findPrev('c', eof)).isEqualTo(-1); + assertThat(scanner.position()).isEqualTo(eof); + assertThat(scanner.peek()).isEqualTo('\0'); + + scanner.setPosition(eof); + assertThat(scanner.findPrev(c -> c == 'Y', x)).isEqualTo(2); + assertThat(scanner.position()).isEqualTo(y); + scanner.setPosition(eof); + assertThat(scanner.findPrev(c -> c == 'X', y)).isEqualTo(-1); + assertThat(scanner.position()).isEqualTo(y); + assertThat(scanner.findPrev(c -> c == 'Y', y)).isEqualTo(0); + + // Across lines the bound is still inclusive and stops the search. + Scanner lines = + Scanner.of( + SourceLines.of( + List.of(SourceLine.of("aXb", null), SourceLine.of("cYd", null)))); + Position lineStart = lines.position(); + lines.next(); + Position lineX = lines.position(); + lines.next(); + lines.next(); + Position newline = lines.position(); + assertThat(lines.peek()).isEqualTo('\n'); + lines.next(); + Position secondStart = lines.position(); + lines.next(); + Position lineY = lines.position(); + lines.next(); + lines.next(); + Position linesEof = lines.position(); + + assertThat(lines.findPrev('Y', newline)).isEqualTo(2); + assertThat(lines.position()).isEqualTo(lineY); + lines.setPosition(linesEof); + assertThat(lines.findPrev('\n', lineStart)).isEqualTo(4); + assertThat(lines.position()).isEqualTo(newline); + assertThat(lines.peek()).isEqualTo('\n'); + + lines.setPosition(linesEof); + assertThat(lines.findPrev('X', newline)).isEqualTo(-1); + assertThat(lines.position()).isEqualTo(newline); + assertThat(lines.peek()).isEqualTo('\n'); + lines.setPosition(linesEof); + assertThat(lines.findPrev('a', secondStart)).isEqualTo(-1); + assertThat(lines.position()).isEqualTo(secondStart); + + lines.setPosition(linesEof); + assertThat(lines.findPrev('a', lineStart)).isEqualTo(7); + assertThat(lines.position()).isEqualTo(lineStart); + lines.setPosition(linesEof); + assertThat(lines.findPrev(c -> c == '\n', newline)).isEqualTo(4); + assertThat(lines.position()).isEqualTo(newline); + lines.setPosition(linesEof); + assertThat(lines.findPrev(c -> c == 'X', secondStart)).isEqualTo(-1); + assertThat(lines.position()).isEqualTo(secondStart); + + // Future and out-of-range bounds throw before the cursor moves. + lines.setPosition(lineX); + assertThat(lines.peek()).isEqualTo('X'); + assertThatThrownBy(() -> lines.findPrev('a', lineY)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> lines.findPrev(c -> true, linesEof)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> lines.findPrev('a', new Position(-1, 0))) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> lines.findPrev('a', new Position(0, -1))) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> lines.findPrev('a', new Position(0, 4))) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> lines.findPrev('a', new Position(2, 0))) + .isInstanceOf(IllegalArgumentException.class); + assertThat(lines.position()).isEqualTo(lineX); + assertThat(lines.peek()).isEqualTo('X'); + + Scanner empty = Scanner.of(SourceLines.empty()); + Position origin = empty.position(); + assertThat(empty.findPrev('x', origin)).isEqualTo(-1); + assertThat(empty.findPrev(c -> true, origin)).isEqualTo(-1); + assertThat(empty.findPrev(Scanner.END, origin)).isEqualTo(-1); + assertThat(empty.position()).isEqualTo(origin); + assertThatThrownBy(() -> empty.findPrev('x', new Position(0, 1))) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> empty.findPrev(c -> true, new Position(1, 0))) + .isInstanceOf(IllegalArgumentException.class); + assertThat(empty.position()).isEqualTo(origin); + assertThat(empty.peek()).isEqualTo('\0'); + + // Same-line bound equal to the start still accepts the character there. + scanner.setPosition(start); + assertThat(scanner.findPrev('a', start)).isEqualTo(0); + assertThat(scanner.position()).isEqualTo(start); + } + + @Test + void testCodeUnitMovement() { + Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("A\uD83D\uDE0AZ", null))); + assertThat(scanner.peek()).isEqualTo('A'); + assertThat(scanner.peekCodePoint()).isEqualTo('A'); + assertThat(scanner.peekPreviousCodePoint()).isEqualTo('\0'); + + scanner.next(); + assertThat(scanner.peek()).isEqualTo('\uD83D'); + assertThat(scanner.peekCodePoint()).isEqualTo(128522); + assertThat(scanner.peekPreviousCodePoint()).isEqualTo('A'); + + scanner.next(); + assertThat(scanner.peek()).isEqualTo('\uDE0A'); + assertThat(scanner.peekCodePoint()).isEqualTo('\uDE0A'); + assertThat(scanner.peekPreviousCodePoint()).isEqualTo('\uD83D'); + + scanner.next(); + assertThat(scanner.peek()).isEqualTo('Z'); + // Previous code point reassembles the surrogate pair behind this code unit. + assertThat(scanner.peekPreviousCodePoint()).isEqualTo(128522); + scanner.next(); + assertThat(scanner.hasNext()).isFalse(); + assertThat(scanner.peek()).isEqualTo('\0'); + assertThat(scanner.peekCodePoint()).isEqualTo('\0'); + assertThat(scanner.peekPreviousCodePoint()).isEqualTo('Z'); + + scanner.prev(); + assertThat(scanner.peek()).isEqualTo('Z'); + scanner.prev(); + assertThat(scanner.peek()).isEqualTo('\uDE0A'); + assertThat(scanner.peekCodePoint()).isEqualTo('\uDE0A'); + assertThat(scanner.peekPreviousCodePoint()).isEqualTo('\uD83D'); + scanner.prev(); + assertThat(scanner.peek()).isEqualTo('\uD83D'); + assertThat(scanner.peekCodePoint()).isEqualTo(128522); + assertThat(scanner.peekPreviousCodePoint()).isEqualTo('A'); + + scanner.next(); + assertThat(scanner.peek()).isEqualTo('\uDE0A'); + scanner.prev(); + assertThat(scanner.peek()).isEqualTo('\uD83D'); + assertThat(scanner.peekCodePoint()).isEqualTo(128522); + + while (scanner.hasNext()) { + scanner.next(); + } + assertThat(scanner.findPrev('\uDE0A')).isEqualTo(2); + assertThat(scanner.peek()).isEqualTo('\uDE0A'); + assertThat(scanner.peekCodePoint()).isEqualTo('\uDE0A'); + while (scanner.hasNext()) { + scanner.next(); + } + assertThat(scanner.findPrev('\uD83D')).isEqualTo(3); + assertThat(scanner.peek()).isEqualTo('\uD83D'); + assertThat(scanner.peekCodePoint()).isEqualTo(128522); + assertThat(scanner.peekPreviousCodePoint()).isEqualTo('A'); + } + + @Test + void testFindPrevSource() { + Scanner scanner = + new Scanner( + List.of( + SourceLine.of("ab", SourceSpan.of(10, 3, 13, 2)), + SourceLine.of("cde", SourceSpan.of(11, 4, 20, 3))), + 0, + 0); + while (scanner.hasNext()) { + scanner.next(); + } + Position end = scanner.position(); + assertThat(scanner.peek()).isEqualTo('\0'); + assertThat(scanner.findPrev('b')).isEqualTo(5); + assertThat(scanner.peek()).isEqualTo('b'); + assertSourceLines( + scanner.getSource(scanner.position(), end), + "b\ncde", + SourceSpan.of(10, 4, 14, 1), + SourceSpan.of(11, 4, 20, 3)); + } + @Test void nextString() { Scanner scanner = From d63604862966dfbcf2c76fdf8eb4200082fd4a9d Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:11:30 -0700 Subject: [PATCH 2/2] Keep only bounded findPrev and document Scanner Remove the unbounded findPrev overloads and make prev private, so a backward search can't move before a caller-supplied bound. Add a class-level Javadoc to Scanner noting that positions and counts are in chars, and drop the repeated UTF-16 wording from method docs. Document the existing public Scanner methods. Rename the simple success case in InlineContentParserTest to a-link and move it first. --- .../org/commonmark/parser/beta/Scanner.java | 155 ++++++++++++------ .../parser/InlineContentParserTest.java | 17 +- .../commonmark/parser/beta/ScannerTest.java | 132 +++++++-------- 3 files changed, 175 insertions(+), 129 deletions(-) diff --git a/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java b/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java index 91aaa6015..adcc0a598 100644 --- a/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java +++ b/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java @@ -6,6 +6,13 @@ import org.commonmark.parser.SourceLines; import org.commonmark.text.CharMatcher; +/** + * A cursor over source lines for parsing inline content. + * + *

Lines are scanned as one sequence of characters with a {@code \n} yielded between lines but + * not after the last line. As usual for Java strings, positions and the counts returned by the + * methods are in {@code char}s. + */ public class Scanner { /** @@ -14,7 +21,7 @@ public class Scanner { * *

Note that we can use NULL to represent this because CommonMark does not allow those in the * input (we replace them in the beginning of parsing). It is not a searchable source character: - * {@link #find(char)} and {@link #findPrev(char)} skip it instead of matching it. + * {@link #find(char)} and {@link #findPrev(char, Position)} skip it instead of matching it. */ public static final char END = '\0'; @@ -41,10 +48,22 @@ public class Scanner { } } + /** + * Create a scanner positioned at the beginning of {@code lines}. + * + * @param lines the source lines to scan + * @return a new scanner + */ public static Scanner of(SourceLines lines) { return new Scanner(lines.getLines(), 0, 0); } + /** + * Return the character at the current position without advancing. + * + * @return the current character, {@code \n} at the end of a line that is not the last one, or + * {@link #END} at the end of input + */ public char peek() { if (index < lineLength) { return line.getContent().charAt(index); @@ -58,6 +77,13 @@ public char peek() { } } + /** + * Return the code point at the current position without advancing. If the current character is + * a high surrogate followed by a low surrogate on the same line, the pair is combined. + * + * @return the current code point, {@code \n} at the end of a line that is not the last one, or + * {@link #END} at the end of input + */ public int peekCodePoint() { if (index < lineLength) { char c = line.getContent().charAt(index); @@ -78,6 +104,13 @@ public int peekCodePoint() { } } + /** + * Return the code point before the current position without moving. If the previous character + * is a low surrogate preceded by a high surrogate on the same line, the pair is combined. + * + * @return the previous code point, {@code \n} at the beginning of a line that is not the first + * one, or {@link #END} at the beginning of input + */ public int peekPreviousCodePoint() { if (index > 0) { int prev = index - 1; @@ -98,6 +131,10 @@ public int peekPreviousCodePoint() { } } + /** + * @return true if there is a character at the current position (including a {@code \n} between + * lines), false at the end of input + */ public boolean hasNext() { if (index < lineLength) { return true; @@ -107,6 +144,7 @@ public boolean hasNext() { } } + /** Advance by one character, moving to the next line after the {@code \n} between lines. */ public void next() { index++; if (index > lineLength) { @@ -120,16 +158,10 @@ public void next() { } } - /** - * Move back by one UTF-16 code unit. - * - *

This reverses {@link #next()}. Lines yield a synthetic newline between lines and no - * newline after the final line: from index zero the position moves to the previous line's - * length (that newline), and from the end of input it moves to the last character, or across - * an empty final line to the preceding newline. At the beginning of the input, including an - * empty input, the position is left unchanged. - */ - public void prev() { + // Move back by one character, reversing next(). From index zero the position moves to the + // previous line's length (the newline between lines). At the beginning of the input, including + // an empty input, the position is left unchanged. + private void prev() { if (index > 0) { index--; } else if (lineIndex > 0) { @@ -181,6 +213,12 @@ public boolean next(String content) { } } + /** + * Advance past consecutive occurrences of {@code c}. + * + * @param c the character to match (including a newline) + * @return the number of characters matched, 0 if the current character is not {@code c} + */ public int matchMultiple(char c) { int count = 0; while (peek() == c) { @@ -190,6 +228,13 @@ public int matchMultiple(char c) { return count; } + /** + * Advance past consecutive characters matched by {@code matcher}. + * + * @param matcher matcher applied to each character, including {@code \n} between lines and + * {@link #END} at the end of input + * @return the number of characters matched + */ public int match(CharMatcher matcher) { int count = 0; while (matcher.matches(peek())) { @@ -199,6 +244,12 @@ public int match(CharMatcher matcher) { return count; } + /** + * Advance past consecutive whitespace characters: space, tab, newline, line tabulation, form + * feed and carriage return. + * + * @return the number of characters skipped + */ public int whitespace() { int count = 0; while (true) { @@ -218,6 +269,15 @@ public int whitespace() { } } + /** + * Find the next occurrence of {@code c} at or after the current position. + * + *

If found, the scanner is left at the match. Otherwise it is left at the end of input and + * {@code -1} is returned. {@link #END} is never matched. + * + * @param c character to find (including a newline) + * @return the number of characters moved forward, or {@code -1} if not found + */ public int find(char c) { int count = 0; while (true) { @@ -232,6 +292,15 @@ public int find(char c) { } } + /** + * Find the next character matched by {@code matcher} at or after the current position. + * + *

See {@link #find(char)} for where the scanner is left. {@link #END} is not passed to + * {@code matcher}. + * + * @param matcher matcher applied to each character + * @return the number of characters moved forward, or {@code -1} if not found + */ public int find(CharMatcher matcher) { int count = 0; while (true) { @@ -246,44 +315,20 @@ public int find(CharMatcher matcher) { } } - /** - * Find the nearest occurrence of {@code c} at or before the current position. - * - *

The current character is included, matching {@link #find(char)}, so an immediate match - * returns {@code 0}. {@link #END} is skipped and never matches. From the end of input a match - * on the character immediately before the end returns {@code 1}. A newline between lines can - * be matched. If {@code c} does not occur, the scanner is left at the beginning of the input - * and {@code -1} is returned. - * - * @param c character to find (including a newline) - * @return the number of UTF-16 code units moved backward, or {@code -1} if not found - */ - public int findPrev(char c) { - return findPrev(c, null, 0, 0); - } - - /** - * Find the nearest character matched by {@code matcher} at or before the current position. - * - *

See {@link #findPrev(char)} for the current position, {@link #END}, and a search that - * reaches the beginning of the input. {@link #END} is not passed to {@code matcher}. - * - * @param matcher matcher applied to each character - * @return the number of UTF-16 code units moved backward, or {@code -1} if not found - */ - public int findPrev(CharMatcher matcher) { - return findPrev(END, matcher, 0, 0); - } - /** * Find the nearest occurrence of {@code c} at or before the current position and at or after * {@code bound}. * + *

The current character is included, matching {@link #find(char)}, so an immediate match + * returns {@code 0}. From the end of input a match on the character immediately before the end + * returns {@code 1}. A newline between lines can be matched. {@link #END} is skipped and never + * matches. + * *

{@code bound} is inclusive: the character there is examined, and the search never moves * before it. If {@code bound} is the current position, only that character is checked. If there - * is no match, the scanner is left at {@code bound} and {@code -1} is returned. {@link #END} is - * skipped and never matches. On an empty input the search returns {@code -1}; the position at - * the beginning of that input is a valid bound. + * is no match, the scanner is left at {@code bound} and {@code -1} is returned. On an empty + * input the search returns {@code -1}; the position at the beginning of that input is a valid + * bound. * *

The bound is checked before the scanner moves. It must have valid coordinates for this * scanner and must not be after the current position. As with {@link #setPosition(Position)}, @@ -291,7 +336,7 @@ public int findPrev(CharMatcher matcher) { * * @param c character to find (including a newline) * @param bound inclusive lower bound - * @return the number of UTF-16 code units moved backward, or {@code -1} if not found + * @return the number of characters moved backward, or {@code -1} if not found * @throws IllegalArgumentException if {@code bound} is after the current position or outside * the input */ @@ -309,7 +354,7 @@ public int findPrev(char c, Position bound) { * * @param matcher matcher applied to each character * @param bound inclusive lower bound - * @return the number of UTF-16 code units moved backward, or {@code -1} if not found + * @return the number of characters moved backward, or {@code -1} if not found * @throws IllegalArgumentException if {@code bound} is after the current position or outside * the input */ @@ -318,7 +363,7 @@ public int findPrev(CharMatcher matcher, Position bound) { return findPrev(END, matcher, bound.lineIndex, bound.index); } - // matcher == null matches c exactly, so the char overloads allocate nothing per character. + // matcher == null matches c exactly, so the char overload allocates nothing per character. private int findPrev(char c, CharMatcher matcher, int boundLine, int boundIndex) { int count = 0; while (true) { @@ -342,10 +387,20 @@ private int findPrev(char c, CharMatcher matcher, int boundLine, int boundIndex) // Don't expose the int index, because it would be good if we could switch input to a // List of lines later instead of one contiguous String. + /** + * @return the current position, for use with {@link #setPosition(Position)}, {@link + * #getSource(Position, Position)} and {@link #findPrev(char, Position)} + */ public Position position() { return new Position(lineIndex, index); } + /** + * Move to a position previously returned by {@link #position()} on this scanner. + * + * @param position the position to move to + * @throws IllegalArgumentException if {@code position} is outside the input + */ public void setPosition(Position position) { checkPosition(position.lineIndex, position.index); this.lineIndex = position.lineIndex; @@ -355,6 +410,14 @@ public void setPosition(Position position) { // For cases where the caller appends the result to a StringBuilder, we could offer another // method to avoid some unnecessary copying. + /** + * Get the source between two positions of this scanner, with {@code begin} inclusive and {@code + * end} exclusive. A {@code \n} between lines is represented by splitting into separate lines. + * + * @param begin the start position + * @param end the end position, not before {@code begin} + * @return the source lines, including their source spans if available + */ public SourceLines getSource(Position begin, Position end) { if (begin.lineIndex == end.lineIndex) { // Shortcut for common case of text from a single line diff --git a/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java b/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java index fe4d03a17..8907adcd1 100644 --- a/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java +++ b/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java @@ -64,21 +64,22 @@ void backwardSearchInlineContentParser() { .customInlineContentParserFactory(new DestinationInlineParser.Factory()) .build(); - var doc = parser.parse("img(not-a-link-)more-text) more text"); + var doc = parser.parse("img(a-link) more text"); var inline = Nodes.find(doc, DestinationInline.class); assertThat(((Text) inline.getPrevious()).getLiteral()).isEqualTo("img"); - assertThat(inline.getDestination()).isEqualTo("not-a-link-)more-text"); + assertThat(inline.getDestination()).isEqualTo("a-link"); assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text"); - doc = parser.parse("img(not-a-link-)more-text)"); + doc = parser.parse("img(a-link-)more-text) more text"); inline = Nodes.find(doc, DestinationInline.class); - assertThat(inline.getDestination()).isEqualTo("not-a-link-)more-text"); - assertThat(inline.getNext()).isNull(); + assertThat(((Text) inline.getPrevious()).getLiteral()).isEqualTo("img"); + assertThat(inline.getDestination()).isEqualTo("a-link-)more-text"); + assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text"); - doc = parser.parse("img(not-a-link) more text"); + doc = parser.parse("img(a-link-)more-text)"); inline = Nodes.find(doc, DestinationInline.class); - assertThat(inline.getDestination()).isEqualTo("not-a-link"); - assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text"); + assertThat(inline.getDestination()).isEqualTo("a-link-)more-text"); + assertThat(inline.getNext()).isNull(); doc = parser.parse("img() more text"); inline = Nodes.find(doc, DestinationInline.class); diff --git a/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java b/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java index 44f94c957..5ee743972 100644 --- a/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java +++ b/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java @@ -147,12 +147,11 @@ private void assertSourceLines( } @Test - void testPrev() { + void testFindPrevSingleSteps() { Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("abc", null))); Position start = scanner.position(); scanner.next(); Position b = scanner.position(); - assertThat(scanner.peek()).isEqualTo('b'); scanner.next(); Position c = scanner.position(); scanner.next(); @@ -160,81 +159,60 @@ void testPrev() { assertThat(scanner.peek()).isEqualTo('\0'); assertThat(scanner.hasNext()).isFalse(); - scanner.prev(); + assertThat(scanner.findPrev('c', start)).isEqualTo(1); assertThat(scanner.position()).isEqualTo(c); - assertThat(scanner.peek()).isEqualTo('c'); - scanner.prev(); + assertThat(scanner.findPrev('b', start)).isEqualTo(1); assertThat(scanner.position()).isEqualTo(b); - assertThat(scanner.peek()).isEqualTo('b'); - scanner.prev(); + assertThat(scanner.findPrev('a', start)).isEqualTo(1); assertThat(scanner.position()).isEqualTo(start); - assertThat(scanner.peek()).isEqualTo('a'); - // No-op at the beginning. - scanner.prev(); - assertThat(scanner.position()).isEqualTo(start); - assertThat(scanner.peek()).isEqualTo('a'); - - scanner.next(); - scanner.prev(); + // A miss at the beginning stays there. + assertThat(scanner.findPrev('z', start)).isEqualTo(-1); assertThat(scanner.position()).isEqualTo(start); assertThat(scanner.peek()).isEqualTo('a'); + // Stepping back from the end and forward again returns to the end. scanner.setPosition(eof); - scanner.prev(); + assertThat(scanner.findPrev('c', start)).isEqualTo(1); scanner.next(); assertThat(scanner.position()).isEqualTo(eof); assertThat(scanner.peek()).isEqualTo('\0'); - Scanner empty = Scanner.of(SourceLines.empty()); - Position emptyPos = empty.position(); - empty.prev(); - assertThat(empty.position()).isEqualTo(emptyPos); - assertThat(empty.peek()).isEqualTo('\0'); - assertThat(empty.hasNext()).isFalse(); - Scanner emptyLine = Scanner.of(SourceLines.of(SourceLine.of("", null))); Position emptyLinePos = emptyLine.position(); assertThat(emptyLine.peek()).isEqualTo('\0'); assertThat(emptyLine.hasNext()).isFalse(); - emptyLine.prev(); + assertThat(emptyLine.findPrev(ch -> true, emptyLinePos)).isEqualTo(-1); assertThat(emptyLine.position()).isEqualTo(emptyLinePos); assertThat(emptyLine.peek()).isEqualTo('\0'); } @Test - void testPrevAcrossLines() { + void testFindPrevAcrossLines() { Scanner scanner = Scanner.of( SourceLines.of( List.of(SourceLine.of("ab", null), SourceLine.of("c", null)))); Position start = scanner.position(); scanner.next(); - assertThat(scanner.peek()).isEqualTo('b'); scanner.next(); Position newline = scanner.position(); assertThat(scanner.peek()).isEqualTo('\n'); scanner.next(); - assertThat(scanner.peek()).isEqualTo('c'); scanner.next(); assertThat(scanner.peek()).isEqualTo('\0'); assertThat(scanner.hasNext()).isFalse(); - scanner.prev(); - assertThat(scanner.peek()).isEqualTo('c'); - scanner.prev(); + assertThat(scanner.findPrev('c', start)).isEqualTo(1); + assertThat(scanner.findPrev('\n', start)).isEqualTo(1); assertThat(scanner.position()).isEqualTo(newline); - assertThat(scanner.peek()).isEqualTo('\n'); - scanner.prev(); + assertThat(scanner.findPrev('b', start)).isEqualTo(1); assertThat(scanner.peek()).isEqualTo('b'); scanner.next(); assertThat(scanner.position()).isEqualTo(newline); - assertThat(scanner.peek()).isEqualTo('\n'); - scanner.prev(); - scanner.prev(); + assertThat(scanner.findPrev('a', start)).isEqualTo(2); assertThat(scanner.position()).isEqualTo(start); - assertThat(scanner.peek()).isEqualTo('a'); - scanner.prev(); + assertThat(scanner.findPrev('z', start)).isEqualTo(-1); assertThat(scanner.position()).isEqualTo(start); // Unequal lengths: cached line content/length must follow the line we step onto. @@ -242,13 +220,12 @@ void testPrevAcrossLines() { Scanner.of( SourceLines.of( List.of(SourceLine.of("a", null), SourceLine.of("bbb", null)))); + Position unevenStart = uneven.position(); uneven.next(); uneven.next(); assertThat(uneven.peek()).isEqualTo('b'); - uneven.prev(); - assertThat(uneven.peek()).isEqualTo('\n'); - uneven.prev(); - assertThat(uneven.peek()).isEqualTo('a'); + assertThat(uneven.findPrev('\n', unevenStart)).isEqualTo(1); + assertThat(uneven.findPrev('a', unevenStart)).isEqualTo(1); uneven.next(); assertThat(uneven.peek()).isEqualTo('\n'); uneven.next(); @@ -256,7 +233,7 @@ void testPrevAcrossLines() { uneven.next(); uneven.next(); assertThat(uneven.peek()).isEqualTo('\0'); - uneven.prev(); + assertThat(uneven.findPrev('b', unevenStart)).isEqualTo(1); assertThat(uneven.peek()).isEqualTo('b'); // Consecutive empty lines, including an empty final line: no newline after the last line. @@ -270,13 +247,16 @@ void testPrevAcrossLines() { Position blanksStart = blanks.position(); assertThat(blanks.peek()).isEqualTo('\n'); blanks.next(); + Position secondNewline = blanks.position(); assertThat(blanks.peek()).isEqualTo('\n'); blanks.next(); + Position blanksEof = blanks.position(); assertThat(blanks.peek()).isEqualTo('\0'); assertThat(blanks.hasNext()).isFalse(); - blanks.prev(); - assertThat(blanks.peek()).isEqualTo('\n'); - blanks.prev(); + assertThat(blanks.findPrev('\n', blanksStart)).isEqualTo(1); + assertThat(blanks.position()).isEqualTo(secondNewline); + blanks.setPosition(blanksEof); + assertThat(blanks.findPrev('x', blanksStart)).isEqualTo(-1); assertThat(blanks.position()).isEqualTo(blanksStart); assertThat(blanks.peek()).isEqualTo('\n'); blanks.next(); @@ -286,8 +266,7 @@ void testPrevAcrossLines() { // Empty final line steps back onto the preceding newline, not a made-up terminal one. Scanner trailingBlank = Scanner.of( - SourceLines.of( - List.of(SourceLine.of("a", null), SourceLine.of("", null)))); + SourceLines.of(List.of(SourceLine.of("a", null), SourceLine.of("", null)))); Position trailingStart = trailingBlank.position(); trailingBlank.next(); Position precedingNewline = trailingBlank.position(); @@ -295,13 +274,12 @@ void testPrevAcrossLines() { trailingBlank.next(); assertThat(trailingBlank.peek()).isEqualTo('\0'); assertThat(trailingBlank.hasNext()).isFalse(); - trailingBlank.prev(); + assertThat(trailingBlank.findPrev(ch -> true, trailingStart)).isEqualTo(1); assertThat(trailingBlank.position()).isEqualTo(precedingNewline); assertThat(trailingBlank.peek()).isEqualTo('\n'); trailingBlank.next(); assertThat(trailingBlank.peek()).isEqualTo('\0'); - trailingBlank.prev(); - trailingBlank.prev(); + assertThat(trailingBlank.findPrev('a', trailingStart)).isEqualTo(2); assertThat(trailingBlank.position()).isEqualTo(trailingStart); assertThat(trailingBlank.peek()).isEqualTo('a'); } @@ -310,23 +288,23 @@ void testPrevAcrossLines() { void testFindPrev() { Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("abXcdX", null))); Position start = scanner.position(); - assertThat(scanner.findPrev('a')).isEqualTo(0); + assertThat(scanner.findPrev('a', start)).isEqualTo(0); assertThat(scanner.position()).isEqualTo(start); - assertThat(scanner.findPrev(c -> c == 'a')).isEqualTo(0); + assertThat(scanner.findPrev(c -> c == 'a', start)).isEqualTo(0); assertThat(scanner.position()).isEqualTo(start); scanner.next(); scanner.next(); Position firstX = scanner.position(); assertThat(scanner.peek()).isEqualTo('X'); - assertThat(scanner.findPrev('X')).isEqualTo(0); + assertThat(scanner.findPrev('X', start)).isEqualTo(0); assertThat(scanner.position()).isEqualTo(firstX); scanner.next(); scanner.next(); // Between the two matches: the nearer (earlier) X, not one after the cursor. assertThat(scanner.peek()).isEqualTo('d'); - assertThat(scanner.findPrev('X')).isEqualTo(2); + assertThat(scanner.findPrev('X', start)).isEqualTo(2); assertThat(scanner.position()).isEqualTo(firstX); scanner.next(); @@ -337,33 +315,33 @@ void testFindPrev() { scanner.next(); assertThat(scanner.peek()).isEqualTo('\0'); // Match immediately before EOF counts the step off the sentinel. - assertThat(scanner.findPrev('X')).isEqualTo(1); + assertThat(scanner.findPrev('X', start)).isEqualTo(1); assertThat(scanner.position()).isEqualTo(secondX); assertThat(scanner.peek()).isEqualTo('X'); scanner.next(); - assertThat(scanner.findPrev(c -> c == 'X')).isEqualTo(1); + assertThat(scanner.findPrev(c -> c == 'X', start)).isEqualTo(1); assertThat(scanner.position()).isEqualTo(secondX); - assertThat(scanner.findPrev('z')).isEqualTo(-1); + assertThat(scanner.findPrev('z', start)).isEqualTo(-1); assertThat(scanner.position()).isEqualTo(start); assertThat(scanner.peek()).isEqualTo('a'); scanner.setPosition(secondX); scanner.next(); - assertThat(scanner.findPrev(Scanner.END)).isEqualTo(-1); + assertThat(scanner.findPrev(Scanner.END, start)).isEqualTo(-1); assertThat(scanner.position()).isEqualTo(start); scanner.setPosition(secondX); scanner.next(); - assertThat(scanner.findPrev(c -> c == Scanner.END)).isEqualTo(-1); + assertThat(scanner.findPrev(c -> c == Scanner.END, start)).isEqualTo(-1); assertThat(scanner.position()).isEqualTo(start); CharMatcher any = c -> true; scanner.setPosition(secondX); - assertThat(scanner.findPrev(any)).isEqualTo(0); + assertThat(scanner.findPrev(any, start)).isEqualTo(0); assertThat(scanner.position()).isEqualTo(secondX); scanner.next(); - assertThat(scanner.findPrev(any)).isEqualTo(1); + assertThat(scanner.findPrev(any, start)).isEqualTo(1); assertThat(scanner.position()).isEqualTo(secondX); assertThat(scanner.peek()).isEqualTo('X'); @@ -371,30 +349,32 @@ void testFindPrev() { Scanner.of( SourceLines.of( List.of(SourceLine.of("ab", null), SourceLine.of("c", null)))); + Position linesStart = lines.position(); lines.next(); lines.next(); Position newline = lines.position(); lines.next(); lines.next(); assertThat(lines.peek()).isEqualTo('\0'); - assertThat(lines.findPrev('\n')).isEqualTo(2); + assertThat(lines.findPrev('\n', linesStart)).isEqualTo(2); assertThat(lines.position()).isEqualTo(newline); assertThat(lines.peek()).isEqualTo('\n'); - assertThat(lines.findPrev(c -> c == '\n')).isEqualTo(0); + assertThat(lines.findPrev(c -> c == '\n', linesStart)).isEqualTo(0); assertThat(lines.position()).isEqualTo(newline); Scanner empty = Scanner.of(SourceLines.empty()); Position emptyPos = empty.position(); - assertThat(empty.findPrev('a')).isEqualTo(-1); - assertThat(empty.findPrev(Scanner.END)).isEqualTo(-1); - assertThat(empty.findPrev(any)).isEqualTo(-1); - assertThat(empty.findPrev(c -> c == Scanner.END)).isEqualTo(-1); + assertThat(empty.findPrev('a', emptyPos)).isEqualTo(-1); + assertThat(empty.findPrev(Scanner.END, emptyPos)).isEqualTo(-1); + assertThat(empty.findPrev(any, emptyPos)).isEqualTo(-1); + assertThat(empty.findPrev(c -> c == Scanner.END, emptyPos)).isEqualTo(-1); assertThat(empty.position()).isEqualTo(emptyPos); assertThat(empty.peek()).isEqualTo('\0'); Scanner text = Scanner.of(SourceLines.of(SourceLine.of("a", null))); + Position textStart = text.position(); text.next(); - assertThat(text.findPrev(any)).isEqualTo(1); + assertThat(text.findPrev(any, textStart)).isEqualTo(1); assertThat(text.peek()).isEqualTo('a'); } @@ -532,6 +512,7 @@ void testFindPrevBounds() { @Test void testCodeUnitMovement() { Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("A\uD83D\uDE0AZ", null))); + Position start = scanner.position(); assertThat(scanner.peek()).isEqualTo('A'); assertThat(scanner.peekCodePoint()).isEqualTo('A'); assertThat(scanner.peekPreviousCodePoint()).isEqualTo('\0'); @@ -556,33 +537,33 @@ void testCodeUnitMovement() { assertThat(scanner.peekCodePoint()).isEqualTo('\0'); assertThat(scanner.peekPreviousCodePoint()).isEqualTo('Z'); - scanner.prev(); + assertThat(scanner.findPrev('Z', start)).isEqualTo(1); assertThat(scanner.peek()).isEqualTo('Z'); - scanner.prev(); + assertThat(scanner.findPrev('\uDE0A', start)).isEqualTo(1); assertThat(scanner.peek()).isEqualTo('\uDE0A'); assertThat(scanner.peekCodePoint()).isEqualTo('\uDE0A'); assertThat(scanner.peekPreviousCodePoint()).isEqualTo('\uD83D'); - scanner.prev(); + assertThat(scanner.findPrev('\uD83D', start)).isEqualTo(1); assertThat(scanner.peek()).isEqualTo('\uD83D'); assertThat(scanner.peekCodePoint()).isEqualTo(128522); assertThat(scanner.peekPreviousCodePoint()).isEqualTo('A'); scanner.next(); assertThat(scanner.peek()).isEqualTo('\uDE0A'); - scanner.prev(); + assertThat(scanner.findPrev('\uD83D', start)).isEqualTo(1); assertThat(scanner.peek()).isEqualTo('\uD83D'); assertThat(scanner.peekCodePoint()).isEqualTo(128522); while (scanner.hasNext()) { scanner.next(); } - assertThat(scanner.findPrev('\uDE0A')).isEqualTo(2); + assertThat(scanner.findPrev('\uDE0A', start)).isEqualTo(2); assertThat(scanner.peek()).isEqualTo('\uDE0A'); assertThat(scanner.peekCodePoint()).isEqualTo('\uDE0A'); while (scanner.hasNext()) { scanner.next(); } - assertThat(scanner.findPrev('\uD83D')).isEqualTo(3); + assertThat(scanner.findPrev('\uD83D', start)).isEqualTo(3); assertThat(scanner.peek()).isEqualTo('\uD83D'); assertThat(scanner.peekCodePoint()).isEqualTo(128522); assertThat(scanner.peekPreviousCodePoint()).isEqualTo('A'); @@ -597,12 +578,13 @@ void testFindPrevSource() { SourceLine.of("cde", SourceSpan.of(11, 4, 20, 3))), 0, 0); + Position start = scanner.position(); while (scanner.hasNext()) { scanner.next(); } Position end = scanner.position(); assertThat(scanner.peek()).isEqualTo('\0'); - assertThat(scanner.findPrev('b')).isEqualTo(5); + assertThat(scanner.findPrev('b', start)).isEqualTo(5); assertThat(scanner.peek()).isEqualTo('b'); assertSourceLines( scanner.getSource(scanner.position(), end),