use cursor marks to track line numbers more accurately and efficiently

- add Reader#mark method to mark position of cursor (store internally)
- add Reader#cursor_at_mark method to build Cursor from previously saved mark
- mark position of cursor after reading block attribute lines in Parser.next_block
- use Reader#cursor_at_mark to retrieve/record source location in Parser.next_block
- add cursor option to Reader#read_lines_until so unterminated block warning reports start location of block
- allow value of cursor option passed to Reader#read_lines_until to be :at_mark to read cursor from mark
- update test assertions

7 files changed, 54 insertions, 32 deletions

diff --git a/lib/asciidoctor/parser.rb b/lib/asciidoctor/parser.rb
index 0d5e2f23..335d1c8f 100644
--- a/lib/asciidoctor/parser.rb
+++ b/lib/asciidoctor/parser.rb
@@ -466,7 +466,8 @@ class Parser
     end
 
     # QUESTION should we introduce a parsing context object?
-    cursor_data, this_line, doc_attrs, style = reader.cursor_data, reader.read_line, document.attributes, attributes[1]
+    reader.mark
+    this_line, doc_attrs, style = reader.read_line, document.attributes, attributes[1]
     block = block_context = cloaked_context = terminator = nil
 
     if (delimited_block = is_delimited_block? this_line, true)
@@ -482,7 +483,7 @@ class Parser
         elsif block_extensions && extensions.registered_for_block?(style, block_context)
           block_context = style.to_sym
         else
-          logger.warn message_with_context %(invalid style for #{block_context} block: #{style}), :source_location => Reader::Cursor.new(*cursor_data)
+          logger.warn message_with_context %(invalid style for #{block_context} block: #{style}), :source_location => reader.cursor_at_mark
           style = block_context.to_s
         end
       end
@@ -693,7 +694,7 @@ class Parser
           # advance to block parsing =>
           break
         else
-          logger.warn message_with_context %(invalid style for paragraph: #{style}), :source_location => Reader::Cursor.new(*cursor_data)
+          logger.warn message_with_context %(invalid style for paragraph: #{style}), :source_location => reader.cursor_at_mark
           style = nil
           # continue to process paragraph
         end
@@ -851,7 +852,7 @@ class Parser
 
       when :table
         block_cursor = reader.cursor
-        block_reader = Reader.new reader.read_lines_until(:terminator => terminator, :skip_line_comments => true, :context => :table), block_cursor
+        block_reader = Reader.new reader.read_lines_until(:terminator => terminator, :skip_line_comments => true, :context => :table, :cursor => :at_mark), block_cursor
         # NOTE it's very rare that format is set when using a format hint char, so short-circuit
         unless terminator.start_with? '|', '!'
           # NOTE infer dsv once all other format hint chars are ruled out
@@ -888,7 +889,7 @@ class Parser
     end
 
     # FIXME we've got to clean this up, it's horrible!
-    block.source_location = Reader::Cursor.new(*cursor_data) if document.sourcemap
+    block.source_location = reader.cursor_at_mark if document.sourcemap
     # FIXME title should be assigned when block is constructed
     block.title = attributes.delete 'title' if attributes.key? 'title'
     # TODO eventually remove the style attribute from the attributes hash
@@ -896,7 +897,7 @@ class Parser
     block.style = attributes['style']
     if (block_id = (block.id ||= attributes['id']))
       unless document.register :refs, [block_id, block, attributes['reftext'] || (block.title? ? block.title : nil)]
-        logger.warn message_with_context %(id assigned to block already in use: #{block_id}), :source_location => Reader::Cursor.new(*cursor_data)
+        logger.warn message_with_context %(id assigned to block already in use: #{block_id}), :source_location => reader.cursor_at_mark
       end
     end
     # FIXME remove the need for this update!
@@ -1021,7 +1022,7 @@ class Parser
       end
       block_reader = nil
     elsif parse_as_content_model != :compound
-      lines = reader.read_lines_until :terminator => terminator, :skip_processing => skip_processing, :context => block_context
+      lines = reader.read_lines_until :terminator => terminator, :skip_processing => skip_processing, :context => block_context, :cursor => :at_mark
       block_reader = nil
     # terminator is false when reader has already been prepared
     elsif terminator == false
@@ -1030,7 +1031,7 @@ class Parser
     else
       lines = nil
       block_cursor = reader.cursor
-      block_reader = Reader.new reader.read_lines_until(:terminator => terminator, :skip_processing => skip_processing, :context => block_context), block_cursor
+      block_reader = Reader.new reader.read_lines_until(:terminator => terminator, :skip_processing => skip_processing, :context => block_context, :cursor => :at_mark), block_cursor
     end
 
     if content_model == :verbatim
@@ -1398,7 +1399,7 @@ class Parser
           buffer << this_line
           # grab all the lines in the block, leaving the delimiters in place
           # we're being more strict here about the terminator, but I think that's a good thing
-          buffer.concat reader.read_lines_until(:terminator => match.terminator, :read_last_line => true)
+          buffer.concat reader.read_lines_until(:terminator => match.terminator, :read_last_line => true, :context => nil)
           continuation = :inactive
         else
           break
diff --git a/lib/asciidoctor/reader.rb b/lib/asciidoctor/reader.rb
index c6844c0c..e8f7d19f 100644
--- a/lib/asciidoctor/reader.rb
+++ b/lib/asciidoctor/reader.rb
@@ -57,6 +57,7 @@ class Reader
     end
     @lines = data ? (prepare_lines data, opts) : []
     @source_lines = @lines.dup
+    @mark = nil
     @look_ahead = 0
     @process_lines = true
     @unescape_next_line = false
@@ -429,15 +430,12 @@ class Reader
   #   => ["First line", "Second line"]
   def read_lines_until options = {}
     result = []
-    shift if options[:skip_first_line]
     if @process_lines && options[:skip_processing]
       @process_lines = false
       restore_process_lines = true
-    else
-      restore_process_lines = false
     end
-
     if (terminator = options[:terminator])
+      start_cursor = options[:cursor] || cursor
       break_on_blank_lines = false
       break_on_list_continuation = false
     else
@@ -445,10 +443,8 @@ class Reader
       break_on_list_continuation = options[:break_on_list_continuation]
     end
     skip_comments = options[:skip_line_comments]
-    line_read = false
-    line_restored = false
-
-    complete = false
+    complete = line_read = line_restored = nil
+    shift if options[:skip_first_line]
     while !complete && (line = read_line)
       complete = while true
         break true if terminator && line == terminator
@@ -461,7 +457,6 @@ class Reader
         break true if block_given? && (yield line)
         break false
       end
-
       if complete
         if options[:read_last_line]
           result << line
@@ -482,8 +477,9 @@ class Reader
       @process_lines = true
       @look_ahead -= 1 if line_restored && !terminator
     end
-    if terminator && terminator != line
-      logger.warn message_with_context %(unterminated #{options[:context] || terminator} block), :source_location => prev_line_cursor
+    if terminator && terminator != line && (context = options.fetch :context, terminator)
+      start_cursor = cursor_at_mark if start_cursor == :at_mark
+      logger.warn message_with_context %(unterminated #{context} block), :source_location => start_cursor
       @unterminated = true
     end
     result
@@ -524,14 +520,22 @@ class Reader
     Cursor.new @file, @dir, @path, lineno
   end
 
-  def cursor_data
-    [@file, @dir, @path, @lineno]
+  def cursor_at_mark
+    @mark ? Cursor.new(*@mark) : cursor
   end
 
   def prev_line_cursor
     cursor_at @lineno - 1
   end
 
+  def cursor_data
+    [@file, @dir, @path, @lineno]
+  end
+
+  def mark
+    @mark = [@file, @dir, @path, @lineno]
+  end
+
   # Public: Get information about the last line read, including file name and line number.
   #
   # Returns A String summary of the last line read
diff --git a/test/attributes_test.rb b/test/attributes_test.rb
index a9f8c13f..7993c09d 100644
--- a/test/attributes_test.rb
+++ b/test/attributes_test.rb
@@ -762,7 +762,7 @@ content
       EOS
       doc = document_from_string input
       assert_nil doc.attr('hey')
-      assert_message @logger, :WARN, '<stdin>: line 6: unterminated comment block', Hash
+      assert_message @logger, :WARN, '<stdin>: line 3: unterminated comment block', Hash
     end
 
     test 'substitutes inside block title' do
diff --git a/test/blocks_test.rb b/test/blocks_test.rb
index 0703d1d3..04a9c2ab 100644
--- a/test/blocks_test.rb
+++ b/test/blocks_test.rb
@@ -202,7 +202,7 @@ supposed to be after comment block, except it got swallowed by block comment
       EOS
 
       render_embedded_string input
-      assert_message @logger, :WARN, '<stdin>: line 6: unterminated comment block', Hash
+      assert_message @logger, :WARN, '<stdin>: line 3: unterminated comment block', Hash
     end
 
     test 'should warn if unterminated comment block is detected inside another block' do
@@ -218,7 +218,7 @@ supposed to be after sidebar block, except it got swallowed by block comment
       EOS
 
       render_embedded_string input
-      assert_message @logger, :WARN, '<stdin>: line 5: unterminated comment block', Hash
+      assert_message @logger, :WARN, '<stdin>: line 4: unterminated comment block', Hash
     end
 
     # WARNING if first line of content is a directive, it will get interpretted before we know it's a comment block
@@ -840,7 +840,7 @@ eof
 
       output = render_embedded_string input
       assert_xpath '/*[@class="exampleblock"]', output, 1
-      assert_message @logger, :WARN, '<stdin>: line 8: unterminated example block', Hash
+      assert_message @logger, :WARN, '<stdin>: line 3: unterminated example block', Hash
     end
   end
 
@@ -3083,7 +3083,7 @@ eof
 
       output = render_embedded_string input
       assert_xpath '/*[@class="listingblock"]', output, 1
-      assert_message @logger, :WARN, '<stdin>: line 8: unterminated listing block', Hash
+      assert_message @logger, :WARN, '<stdin>: line 3: unterminated listing block', Hash
     end
   end
 
diff --git a/test/lists_test.rb b/test/lists_test.rb
index 2e0a8559..95d471ce 100644
--- a/test/lists_test.rb
+++ b/test/lists_test.rb
@@ -1540,6 +1540,23 @@ Item one, paragraph two
     end
 =end
 
+    test 'should warn if unterminated block is detected in list item' do
+      input = <<-EOS
+* item
++
+====
+example
+* swallowed item
+      EOS
+
+      using_memory_logger do |logger|
+        output = render_embedded_string input
+        assert_xpath '//ul/li', output, 1
+        assert_xpath '//ul/li/*[@class="exampleblock"]', output, 1
+        assert_xpath %(//p[text()="example\n* swallowed item"]), output, 1
+        assert_message logger, :WARN, '<stdin>: line 3: unterminated example block', Hash
+      end
+    end
   end
 end
 
diff --git a/test/reader_test.rb b/test/reader_test.rb
index bb1eca90..09fb3da6 100644
--- a/test/reader_test.rb
+++ b/test/reader_test.rb
@@ -389,7 +389,7 @@ not captured
         refute reader.unterminated
       end
 
-      test 'should mark reader as unterminated if reader reaches end of source without finding terminator' do
+      test 'should flag reader as unterminated if reader reaches end of source without finding terminator' do
         lines = <<-EOS.each_line.to_a
 ****
 captured
@@ -404,11 +404,11 @@ captured yet again
         using_memory_logger do |logger|
           doc = empty_safe_document :base_dir => DIRNAME
           reader = Asciidoctor::PreprocessorReader.new doc, lines, nil, :normalize => true
-          terminator = reader.read_line
-          result = reader.read_lines_until :terminator => terminator, :skip_processing => true
+          terminator = reader.peek_line
+          result = reader.read_lines_until :terminator => terminator, :skip_first_line => true, :skip_processing => true
           assert_equal expected, result
           assert reader.unterminated
-          assert_message logger, :WARN, '<stdin>: line 6: unterminated **** block', Hash
+          assert_message logger, :WARN, '<stdin>: line 1: unterminated **** block', Hash
         end
       end
     end
diff --git a/test/tables_test.rb b/test/tables_test.rb
index d0251e05..bd2c46b5 100644
--- a/test/tables_test.rb
+++ b/test/tables_test.rb
@@ -1399,7 +1399,7 @@ eof
       using_memory_logger do |logger|
         output = render_embedded_string input
         assert_xpath '/table', output, 1
-        assert_message logger, :WARN, '<stdin>: line 9: unterminated table block', Hash
+        assert_message logger, :WARN, '<stdin>: line 3: unterminated table block', Hash
       end
     end
   end