resolves #575, #572 and #581 refactor reader to track include stack

- split the Reader into Reader and PreprocessorReader
- maintain an internal stack of include contexts
- report file names with line numbers
- rework reader tests
- rename several reader methods to be more intuitive
- optimize operations in the readers

15 files changed, 1784 insertions, 1325 deletions

diff --git a/asciidoctor.gemspec b/asciidoctor.gemspec
index 2d5ede13..92772d4f 100644
--- a/asciidoctor.gemspec
+++ b/asciidoctor.gemspec
@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
   ## Rake build (see the validate task)
   s.name              = 'asciidoctor'
   s.version           = '0.1.4.preview.4'
-  s.date              = '2013-08-12'
+  s.date              = '2013-08-21'
   s.rubyforge_project = 'asciidoctor'
 
   s.summary     = 'A native Ruby AsciiDoc syntax processor and publishing toolchain'
@@ -63,6 +63,7 @@ EOS
     lib/asciidoctor/backends/_stylesheets.rb
     lib/asciidoctor/backends/base_template.rb
     lib/asciidoctor/backends/docbook45.rb
+    lib/asciidoctor/backends/docbook5.rb
     lib/asciidoctor/backends/html5.rb
     lib/asciidoctor/block.rb
     lib/asciidoctor/callouts.rb
diff --git a/lib/asciidoctor/document.rb b/lib/asciidoctor/document.rb
index 9d8966d0..99e3103b 100644
--- a/lib/asciidoctor/document.rb
+++ b/lib/asciidoctor/document.rb
@@ -25,9 +25,12 @@ class Document < AbstractBlock
     end
 
     def save_to(block_attributes)
-      block_attributes[:attribute_entries] ||= []
-      block_attributes[:attribute_entries] << self
+      (block_attributes[:attribute_entries] ||= []) << self
     end
+
+    #def save_to_next_block(document)
+    #  (document.attributes[:pending_attribute_entries] ||= []) << self
+    #end
   end
 
   # Public A read-only integer value indicating the level of security that
@@ -198,7 +201,7 @@ class Document < AbstractBlock
     @attribute_overrides['embedded'] = @options[:header_footer] ? nil : ''
 
     # the only way to set the include-depth attribute is via the document options
-    # 10 is the AsciiDoc default, though currently Asciidoctor only supports 1 level
+    # 10 is the AsciiDoc default
     @attribute_overrides['include-depth'] ||= 10
 
     # the only way to enable uri reads is via the document options, disabled by default
@@ -282,6 +285,9 @@ class Document < AbstractBlock
       @attributes['doctype'] ||= DEFAULT_DOCTYPE
       update_backend_attributes
 
+      @attributes['indir'] = @attributes['docdir']
+      @attributes['infile'] = @attributes['docfile']
+
       # dynamic intrinstic attribute values
       now = Time.new
       @attributes['localdate'] ||= now.strftime('%Y-%m-%d')
@@ -301,9 +307,11 @@ class Document < AbstractBlock
 
     if @parent_document
       # don't need to do the extra processing within our own document
-      @reader = Reader.new(data, self)
+      # FIXME how are line numbers being tracked in this case?!?!
+      @reader = Reader.new data
     else
-      @reader = Reader.new(data, self, true, &block)
+      # TODO review the docfile logic here
+      @reader = PreprocessorReader.new self, data, ((@attributes.has_key? 'docfile') ? File.basename(@attributes['docfile']) : nil)
     end
 
     # Now parse the lines in the reader into blocks
@@ -401,12 +409,12 @@ class Document < AbstractBlock
 
   # Make the raw source for the Document available.
   def source
-    @reader.source.join if @reader
+    @reader.source if @reader
   end
 
   # Make the raw source lines for the Document available.
   def source_lines
-    @reader.source if @reader
+    @reader.source_lines if @reader
   end
 
   def doctype
diff --git a/lib/asciidoctor/lexer.rb b/lib/asciidoctor/lexer.rb
index b13f78b0..f288d956 100644
--- a/lib/asciidoctor/lexer.rb
+++ b/lib/asciidoctor/lexer.rb
@@ -138,7 +138,7 @@ class Lexer
       document.attributes['mantitle'] = document.sub_attributes(m[1].rstrip.downcase)
       document.attributes['manvolnum'] = m[2].strip
     else
-      warn "asciidoctor: ERROR: line #{reader.lineno}: malformed manpage title"
+      warn "asciidoctor: ERROR: #{reader.prev_line_info}: malformed manpage title"
     end
 
     reader.skip_blank_lines
@@ -146,7 +146,7 @@ class Lexer
     if is_next_line_section?(reader, {})
       name_section = initialize_section(reader, document, {})
       if name_section.level == 1
-        name_section_buffer = reader.grab_lines_until(:break_on_blank_lines => true).join.tr_s("\n ", ' ')
+        name_section_buffer = reader.take_lines_until(:break_on_blank_lines => true).join.tr_s("\n ", ' ')
         if (m = name_section_buffer.match(REGEXP[:manname_manpurpose]))
           document.attributes['manname'] = m[1] 
           document.attributes['manpurpose'] = m[2] 
@@ -157,13 +157,13 @@ class Lexer
             document.attributes['outfilesuffix'] = ".#{document.attributes['manvolnum']}"
           end
         else
-          warn "asciidoctor: ERROR: line #{reader.lineno}: malformed name section body"
+          warn "asciidoctor: ERROR: #{reader.prev_line_info}: malformed name section body"
         end
       else
-        warn "asciidoctor: ERROR: line #{reader.lineno}: name section title must be at level 1"
+        warn "asciidoctor: ERROR: #{reader.prev_line_info}: name section title must be at level 1"
       end
     else
-      warn "asciidoctor: ERROR: line #{reader.lineno}: name section expected"
+      warn "asciidoctor: ERROR: #{reader.prev_line_info}: name section expected"
     end
   end
 
@@ -265,9 +265,9 @@ class Lexer
         doctype = parent.document.doctype
         if next_level > current_level || (section.is_a?(Document) && next_level == 0)
           if next_level == 0 && doctype != 'book'
-            warn "asciidoctor: ERROR: line #{reader.lineno + 1}: only book doctypes can contain level 0 sections"
+            warn "asciidoctor: ERROR: #{reader.line_info}: only book doctypes can contain level 0 sections"
           elsif !expected_next_levels.nil? && !expected_next_levels.include?(next_level)
-            warn "asciidoctor: WARNING: line #{reader.lineno + 1}: section title out of sequence: " +
+            warn "asciidoctor: WARNING: #{reader.line_info}: section title out of sequence: " +
                 "expected #{expected_next_levels.size > 1 ? 'levels' : 'level'} #{expected_next_levels * ' or '}, " +
                 "got level #{next_level}"
           end
@@ -276,7 +276,7 @@ class Lexer
           section << new_section
         else
           if next_level == 0 && doctype != 'book'
-            warn "asciidoctor: ERROR: line #{reader.lineno + 1}: only book doctypes can contain level 0 sections"
+            warn "asciidoctor: ERROR: #{reader.line_info}: only book doctypes can contain level 0 sections"
           end
           # close this section (and break out of the nesting) to begin a new one
           break
@@ -362,7 +362,7 @@ class Lexer
       end
 
       # QUESTION should we introduce a parsing context object?
-      this_line = reader.get_line
+      this_line = reader.read_line
       delimited_block = false
       block_context = nil
       terminator = nil
@@ -383,7 +383,7 @@ class Lexer
           elsif delimited_blk_match.masq.include?('admonition') && ADMONITION_STYLES.include?(style)
             block_context = :admonition
           else
-            warn "asciidoctor: WARNING: line #{reader.lineno}: invalid style for #{block_context} block: #{style}"
+            warn "asciidoctor: WARNING: #{reader.prev_line_info}: invalid style for #{block_context} block: #{style}"
             style = block_context.to_s
           end
         end
@@ -474,7 +474,7 @@ class Lexer
             begin
               # might want to move this check to a validate method
               if match[1].to_i != expected_index
-                warn "asciidoctor: WARNING: line #{reader.lineno + 1}: callout list item index: expected #{expected_index} got #{match[1]}"
+                warn "asciidoctor: WARNING: #{reader.path}: line #{reader.lineno - 2}: callout list item index: expected #{expected_index} got #{match[1]}"
               end
               list_item = next_list_item(reader, block, match)
               expected_index += 1
@@ -484,7 +484,7 @@ class Lexer
                 if !coids.empty?
                   list_item.attributes['coids'] = coids
                 else
-                  warn "asciidoctor: WARNING: line #{reader.lineno}: no callouts refer to list item #{block.items.size}"
+                  warn "asciidoctor: WARNING: #{reader.path}: line #{reader.lineno - 2}: no callouts refer to list item #{block.items.size}"
                 end
               end
             end while reader.has_more_lines? && match = reader.peek_line.match(REGEXP[:colist])
@@ -550,7 +550,7 @@ class Lexer
               # advance to block parsing =>
               break
             else
-              warn "asciidoctor: WARNING: line #{reader.lineno}: invalid style for paragraph: #{style}"
+              warn "asciidoctor: WARNING: #{reader.prev_line_info}: invalid style for paragraph: #{style}"
               style = nil
               # continue to process paragraph
             end
@@ -560,9 +560,9 @@ class Lexer
 
           # a literal paragraph is contiguous lines starting at least one space
           if style != 'normal' && this_line.match(REGEXP[:lit_par])
-            # So we need to actually include this one in the grab_lines group
+            # So we need to actually include this one in the take_lines group
             reader.unshift_line this_line
-            lines = reader.grab_lines_until(
+            lines = reader.take_lines_until(
                 :break_on_blank_lines => true,
                 :break_on_list_continuation => true,
                 :preserve_last_line => true) {|line|
@@ -584,7 +584,7 @@ class Lexer
           # a paragraph is contiguous nonblank/noncontinuation lines
           else
             reader.unshift_line this_line
-            lines = reader.grab_lines_until(
+            lines = reader.take_lines_until(
                 :break_on_blank_lines => true,
                 :break_on_list_continuation => true,
                 :preserve_last_line => true,
@@ -602,7 +602,7 @@ class Lexer
             # were the only lines found
             if lines.empty?
               # call get_line since the reader preserved the last line
-              reader.get_line
+              reader.read_line
               return nil
             end
 
@@ -638,6 +638,7 @@ class Lexer
               attributes['citetitle'] = citetitle unless citetitle.nil?
               # NOTE will only detect headings that are floating titles (not section titles)
               # TODO could assume a floating title when inside a block context
+              # FIXME Reader needs to be created w/ line info
               block = build_block(:quote, :compound, false, parent, Reader.new(lines), attributes)
             elsif !text_only && lines.size > 1 && first_line.start_with?('"') &&
                 lines.last.start_with?('-- ') && lines[-2].chomp.end_with?('"')
@@ -707,7 +708,10 @@ class Lexer
           block = build_block(block_context, :compound, terminator, parent, reader, attributes)
 
         when :table
-          block_reader = Reader.new reader.grab_lines_until(:terminator => terminator, :skip_line_comments => true)
+          file = reader.file
+          path = reader.path
+          lineno = reader.lineno
+          block_reader = Reader.new reader.take_lines_until(:terminator => terminator, :skip_line_comments => true), file, path, lineno
           case terminator[0..0]
             when ','
               attributes['format'] = 'csv'
@@ -722,7 +726,7 @@ class Lexer
 
         else
           # this should only happen if there is a misconfiguration
-          raise "Unsupported block type #{block_context} at line #{reader.lineno}"
+          raise "Unsupported block type #{block_context} at #{reader.line_info}"
         end
       end
     end
@@ -746,6 +750,12 @@ class Lexer
       end
       block.update_attributes(attributes)
 
+      #if document.attributes.has_key? :pending_attribute_entries
+      #  document.attributes.delete(:pending_attribute_entries).each do |entry|
+      #    entry.save_to block.attributes
+      #  end
+      #end
+
       # FIXME callout capabilities should be a setting on the block
       if block.context == :listing || block.context == :literal
         catalog_callouts(block.source, document)
@@ -820,10 +830,10 @@ class Lexer
 
     if terminator.nil?
       if parse_as_content_model == :verbatim
-        lines = reader.grab_lines_until(:break_on_blank_lines => true, :break_on_list_continuation => true)
+        lines = reader.take_lines_until(:break_on_blank_lines => true, :break_on_list_continuation => true)
       else
         content_model = :simple if content_model == :compound
-        lines = reader.grab_lines_until(
+        lines = reader.take_lines_until(
             :break_on_blank_lines => true,
             :break_on_list_continuation => true,
             :preserve_last_line => true,
@@ -834,7 +844,7 @@ class Lexer
       end
       block_reader = nil
     elsif parse_as_content_model != :compound
-      lines = reader.grab_lines_until(:terminator => terminator, :chomp_last_line => true)
+      lines = reader.take_lines_until(:terminator => terminator, :chomp_last_line => true)
       block_reader = nil
     # terminator is false when reader has already been prepared
     elsif terminator == false
@@ -842,7 +852,10 @@ class Lexer
       block_reader = reader
     else
       lines = nil
-      block_reader = Reader.new reader.grab_lines_until(:terminator => terminator)
+      file = reader.file
+      path = reader.path
+      lineno = reader.lineno
+      block_reader = Reader.new reader.take_lines_until(:terminator => terminator), file, path, lineno
     end
 
     if content_model == :skip
@@ -1050,15 +1063,18 @@ class Lexer
     end
 
     # first skip the line with the marker / term
-    reader.get_line
-    list_item_reader = Reader.new grab_lines_for_list_item(reader, list_type, sibling_trait, has_text)
+    reader.read_line
+    file = reader.file
+    path = reader.path
+    lineno = reader.lineno
+    list_item_reader = Reader.new take_lines_for_list_item(reader, list_type, sibling_trait, has_text), file, path, lineno
     if list_item_reader.has_more_lines?
       comment_lines = list_item_reader.consume_line_comments
       subsequent_line = list_item_reader.peek_line
       list_item_reader.unshift(*comment_lines) unless comment_lines.empty? 
 
       if !subsequent_line.nil?
-        continuation_connects_first_block = (subsequent_line == "\n")
+        continuation_connects_first_block = (subsequent_line == ::Asciidoctor::EOL)
         # if there's no continuation connecting the first block, then
         # treat the lines as paragraph text (activated when has_text = false)
         if !continuation_connects_first_block && list_type != :dlist
@@ -1108,7 +1124,7 @@ class Lexer
   # has_text        - Whether the list item has text defined inline (always true except for labeled lists)
   #
   # Returns an Array of lines belonging to the current list item.
-  def self.grab_lines_for_list_item(reader, list_type, sibling_trait = nil, has_text = true)
+  def self.take_lines_for_list_item(reader, list_type, sibling_trait = nil, has_text = true)
     buffer = []
 
     # three states for continuation: :inactive, :active & :frozen
@@ -1126,7 +1142,7 @@ class Lexer
     detached_continuation = nil
 
     while reader.has_more_lines?
-      this_line = reader.get_line
+      this_line = reader.read_line
 
       # if we've arrived at a sibling item in this list, we've captured
       # the complete list item and can begin processing it
@@ -1161,7 +1177,7 @@ class Lexer
           buffer << this_line
           # grab all the lines in the block, leaving the delimiters in place
           # we're being more strict here about the terminator, but I think that's a good thing
-          buffer.concat reader.grab_lines_until(:terminator => match.terminator, :grab_last_line => true)
+          buffer.concat reader.take_lines_until(:terminator => match.terminator, :take_last_line => true)
           continuation = :inactive
         else
           break
@@ -1178,7 +1194,7 @@ class Lexer
           # list item will throw off the exit from it
           if this_line.match(REGEXP[:lit_par])
             reader.unshift_line this_line
-            buffer.concat reader.grab_lines_until(
+            buffer.concat reader.take_lines_until(
                 :preserve_last_line => true,
                 :break_on_blank_lines => true,
                 :break_on_list_continuation => true) {|line|
@@ -1205,7 +1221,7 @@ class Lexer
           # advance to the next line of content
           if this_line.chomp.empty?
             reader.skip_blank_lines
-            this_line = reader.get_line 
+            this_line = reader.read_line 
             # if we hit eof or a sibling, stop reading
             break if this_line.nil? || is_sibling_list_item?(this_line, list_type, sibling_trait)
           end
@@ -1221,7 +1237,7 @@ class Lexer
               # slurp up any literal paragraph offset by blank lines
               if this_line.match(REGEXP[:lit_par])
                 reader.unshift_line this_line
-                buffer.concat reader.grab_lines_until(
+                buffer.concat reader.take_lines_until(
                     :preserve_last_line => true,
                     :break_on_blank_lines => true,
                     :break_on_list_continuation => true) {|line|
@@ -1279,8 +1295,8 @@ class Lexer
       buffer.pop
     end
 
-    #warn "BUFFER[#{list_type},#{sibling_trait}]>#{buffer.join}<BUFFER"
-    #warn "BUFFER[#{list_type},#{sibling_trait}]>#{buffer.inspect}<BUFFER"
+    #puts "BUFFER[#{list_type},#{sibling_trait}]>#{buffer.join}<BUFFER"
+    #puts "BUFFER[#{list_type},#{sibling_trait}]>#{buffer.inspect}<BUFFER"
 
     buffer
   end
@@ -1458,7 +1474,7 @@ class Lexer
   #--
   # NOTE for efficiency, we don't reuse methods that check for a section title
   def self.parse_section_title(reader, document)
-    line1 = reader.get_line
+    line1 = reader.read_line
     sect_id = nil
     sect_title = nil
     sect_level = -1
@@ -1483,7 +1499,7 @@ class Lexer
         end
         sect_level = section_level line2
         single_line = false
-        reader.get_line
+        reader.read_line
       end
     end
     if sect_level >= 0
@@ -1523,7 +1539,7 @@ class Lexer
     implicit_authors = nil
 
     if reader.has_more_lines? && !reader.next_line_empty?
-      author_metadata = process_authors reader.get_line
+      author_metadata = process_authors reader.read_line
 
       unless author_metadata.empty?
         # apply header subs and assign to document
@@ -1547,7 +1563,7 @@ class Lexer
       rev_metadata = {}
 
       if reader.has_more_lines? && !reader.next_line_empty?
-        rev_line = reader.get_line 
+        rev_line = reader.read_line 
         if match = rev_line.match(REGEXP[:revision_info])
           rev_metadata['revdate'] = match[2].strip
           rev_metadata['revnumber'] = match[1].rstrip unless match[1].nil?
@@ -1737,7 +1753,7 @@ class Lexer
     next_line = reader.peek_line
     if (commentish = next_line.start_with?('//')) && (match = next_line.match(REGEXP[:comment_blk]))
       terminator = match[0]
-      reader.grab_lines_until(:skip_first_line => true, :preserve_last_line => true, :terminator => terminator, :preprocess => false)
+      reader.take_lines_until(:skip_first_line => true, :preserve_last_line => true, :terminator => terminator, :preprocess => false)
     elsif commentish && next_line.match(REGEXP[:comment])
       # do nothing, we'll skip it
     elsif !options[:text] && (match = next_line.match(REGEXP[:attr_entry]))
@@ -1918,7 +1934,7 @@ class Lexer
     end
 
     if validate && expected != actual
-      # FIXME I need a reader reference or line number to report line number
+      # FIXME I need a reader reference or line number to report line number!!
       warn "asciidoctor: WARNING: list item index: expected #{expected}, got #{actual}"
     end
 
@@ -1979,7 +1995,7 @@ class Lexer
     loop_idx = -1
     while table_reader.has_more_lines?
       loop_idx += 1
-      line = table_reader.get_line
+      line = table_reader.read_line
 
       if skipped == 0 && loop_idx.zero? && !attributes.has_key?('options') &&
           !(next_line = table_reader.peek_line(false)).nil? && next_line.chomp.empty?
@@ -2211,7 +2227,7 @@ class Lexer
       save_current = lambda {
         if collector.empty?
           if type != :style
-            warn "asciidoctor: WARNING:#{reader.nil? ? nil : " line #{reader.lineno}:"} invalid empty #{type} detected in style attribute"
+            warn "asciidoctor: WARNING:#{reader.nil? ? nil : " #{reader.prev_line_info}:"} invalid empty #{type} detected in style attribute"
           end
         else
           case type
@@ -2220,7 +2236,7 @@ class Lexer
             parsed[type].push collector.join
           when :id
             if parsed.has_key? :id
-              warn "asciidoctor: WARNING:#{reader.nil? ? nil : " line #{reader.lineno}:"} multiple ids detected in style attribute"
+              warn "asciidoctor: WARNING:#{reader.nil? ? nil : " #{reader.prev_line_info}:"} multiple ids detected in style attribute"
             end
             parsed[type] = collector.join
           else
diff --git a/lib/asciidoctor/reader.rb b/lib/asciidoctor/reader.rb
index 572ac763..a51fbdb7 100644
--- a/lib/asciidoctor/reader.rb
+++ b/lib/asciidoctor/reader.rb
@@ -1,153 +1,250 @@
 module Asciidoctor
 # Public: Methods for retrieving lines from AsciiDoc source files
 class Reader
-
-  # Public: Get the document source as a String Array of lines.
-  attr_reader :source
+  attr_reader :lines
+  attr_reader :file
+  attr_reader :dir
+  attr_reader :path
 
   # Public: Get the 1-based offset of the current line.
   attr_reader :lineno
 
-  # Public: Initialize the Reader object.
-  #
-  # data       - The Array of Strings holding the Asciidoc source document. The
-  #              original instance of this Array is not modified (default: nil)
-  # document   - The document with which this reader is associated. Used to access
-  #              document attributes (default: nil)
-  # preprocess - A flag indicating whether to run the preprocessor on these lines.
-  #              Only enable for the outer-most Reader. If this argument is true,
-  #              a Document object must also be supplied.
-  #              (default: false)
-  # block      - A block that can be used to retrieve external Asciidoc
-  #              data to include in this document.
-  #
-  # Examples
-  #
-  #   data   = File.readlines(filename)
-  #   reader = Asciidoctor::Reader.new data
-  def initialize(data = nil, document = nil, preprocess = false, &block)
-    # TODO use Struct to track file/lineno info; track as file changes; offset for sub-readers
-    data = [] if data.nil?
-    @lineno = 0
-    @next_line_preprocessed = false
-    @unescape_next_line = false
-    @conditionals_stack = []
-    @skipping = false
-    @eof = false
+  # Public: Get the document source as a String Array of lines.
+  attr_reader :source_lines
 
-    if !preprocess
-      @lines = data.is_a?(String) ? data.lines.entries : data.dup
-      @preprocess_source = false
-      # document is not nil in the case of nested AsciiDoc document (in table cell)
-      unless document.nil?
-        @document = document
-        # preprocess first line, since we may not have hit it yet
-        # FIXME include handler (block) should be available to the peek_line call
-        @include_block = nil
-        peek_line true
-        @document = nil
-      end
-    elsif !data.empty?
-      # NOTE we assume document is not nil!
-      @document = document
-      @preprocess_source = true
-      @include_block = block_given? ? block : nil
-      normalize_data(data.is_a?(String) ? data.lines : data)
+  # Public: Initialize the Reader object
+  def initialize data = nil, file = nil, path = nil, lineno = 1
+    if file.nil?
+      @file = @dir = nil
+      @path = path
+    #elsif file.is_a? Reader
+    #  @file = file.file
+    #  @dir = File.dirname @file
+    #  @dir = nil if @dir == '.' # right?
+    #  @path = file.path
+    #  lineno = file.lineno
     else
-      @lines = []
-      @preprocess_source = false
+      @file = file
+      @dir = File.dirname @file
+      @dir = nil if @dir == '.' # right?
+      @path = path || File.basename(@file)
     end
+    @lineno = lineno # IMPORTANT lineno assignment must proceed prepare_lines call!
+    @lines = data.nil? ? [] : (prepare_lines data)
+    @source_lines = @lines.dup
+    @eof = @lines.empty?
+    @look_ahead = 0
+    @process_lines = true
+    @unescape_next_line = false
+  end
 
-    @source = @lines.dup
+  # Protected: Prepare the lines from the provided data
+  #
+  # This method strips whitespace from the end of every line of
+  # the source data and appends a LF (i.e., Unix endline). This
+  # whitespace substitution is very important to how Asciidoctor
+  # works.
+  #
+  # Any leading or trailing blank lines are also removed.
+  #
+  # The normalized lines are assigned to the @lines instance variable.
+  #
+  # data - A String Array of input data to be normalized
+  # opts - A Hash of options to control what cleansing is done
+  #
+  # Returns The String lines extracted from the data
+  def prepare_lines data, opts = {}
+    data.is_a?(String) ? data.each_line.to_a : data.dup
   end
 
-  # Public: Get a copy of the remaining Array of String lines parsed from the source
-  def lines
-    @lines.nil? ? nil : @lines.dup
+  # Protected: Processes a previously unvisited line
+  #
+  # By default, this method marks the line as processed
+  # by incrementing the look_ahead counter and returns
+  # the line unmodified.
+  #
+  # Returns The String line the Reader should make available to the next
+  # invocation of Reader#read_line or nil if the Reader should drop the line,
+  # advance to the next line and process it.
+  def process_line line
+    @look_ahead += 1 if @process_lines
+    line
   end
 
   # Public: Check whether there are any lines left to read.
   #
-  # If preprocessing is enabled for this Reader, and there are lines remaining,
-  # the next line is preprocessed before checking whether there are more lines. 
+  # If a previous call to this method resulted in a value of false,
+  # immediately returned the cached value. Otherwise, delegate to
+  # peek_line to determine if there is a next line available.
   #
-  # Returns true if @lines is empty, or false otherwise.
-  def has_more_lines?(preprocess = nil)
-    if @eof || (@eof = @lines.empty?)
-      false
-    elsif (preprocess.nil? && @preprocess_source || preprocess) && !@next_line_preprocessed
-      preprocess_next_line.nil? ? false : !@lines.empty?
+  # Returns True if there are more lines, False if there are not.
+  def has_more_lines?
+    !(@eof || (@eof = peek_line.nil?))
+  end
+
+  # Public: Peek at the next line and check if it's empty (i.e., whitespace only)
+  #
+  # This method Does not consume the line from the stack.
+  #
+  # Returns True if the there are no more lines or if the next line is empty
+  def next_line_empty?
+    (line = peek_line).nil? || line.chomp.empty?
+  end
+
+  # Public: Get the next line of source data. Does not consume the line returned.
+  #
+  # This method will probe the reader for more lines. If there is a next line
+  # that has not previously been visited, the line is passed to the
+  # Reader#preprocess_line method to be initialized. This call gives
+  # sub-classess the opportunity to do preprocessing. If the return value of
+  # the Reader#process_line is nil, the data is assumed to be changed and
+  # Reader#peek_line is invoked again to perform further processing.
+  #
+  # direct  - A Boolean flag to bypasses the check for more lines and immediately
+  #           returns the first element of the internal @lines Array. (default: false)
+  #
+  # Returns a String dup of the next line of the source data if data is present.
+  # Returns nil if there is no more data.
+  def peek_line direct = false
+    if @look_ahead > 0
+      @unescape_next_line ? @lines.first[1..-1].dup : @lines.first.dup
+    elsif @eof || @lines.empty?
+      @eof = true
+      @look_ahead = 0
+      nil
     else
-      true
+      # FIXME the problem with this approach is that we aren't
+      # retaining the modified line (hence the @unescape_next_line tweak)
+      # perhaps we need a stack of proxy lines
+      if (line = process_line @lines.first).nil?
+        peek_line
+      else
+        line.dup
+      end
     end
   end
 
-  # Public: Check whether this reader is empty (contains no lines)
+  # TODO document & test me!
+  def peek_lines num = 1
+    result = []
+    (1..num).each do
+      if has_more_lines?
+        result << read_line
+      else
+        break
+      end
+    end
+
+    restore_lines result unless result.empty?
+
+    result
+  end
+
+  # Public: Get the next line of source data. Consumes the line returned.
   #
-  # If preprocessing is enabled for this Reader, and there are lines remaining,
-  # the next line is preprocessed before checking whether there are more lines. 
+  # direct  - A Boolean flag to bypasses the check for more lines and immediately
+  #           returns the first element of the internal @lines Array. (default: false)
   #
-  # Returns true if @lines is empty, otherwise false.
-  def empty?
-    !has_more_lines?
+  # Returns the String of the next line of the source data if data is present.
+  # Returns nil if there is no more data.
+  def read_line direct = false
+    if direct || @look_ahead > 0 || has_more_lines?
+      shift
+    else
+      nil
+    end
   end
+  alias :get_line :read_line
 
-  # Private: Strip off leading blank lines in the Array of lines.
+  # Public: Get the remaining lines of source data.
   #
-  # Examples
+  # This method calls Reader#read_line repeatedly until all lines are consumed
+  # and returns the lines as a String Array. This method differs from
+  # Reader#lines in that it processes each line in turn, hence triggering
+  # any preprocessors implemented in sub-classes.
   #
-  #   @lines
-  #   => ["\n", "\t\n", "Foo\n", "Bar\n", "\n"]
+  # Returns the lines read as a String Array
+  def read_lines
+    lines = []
+    while has_more_lines?
+      lines << read_line
+    end
+    lines
+  end
+  alias :readlines :read_lines
+  alias :get_lines :read_lines
+
+  # Public: Get the remaining lines of source data joined as a String.
   #
-  #   skip_blank_lines
-  #   => 2
+  # Delegates to Reader#read_lines, then joins the result.
   #
-  #   @lines
-  #   => ["Foo\n", "Bar\n"]
+  # Returns the lines read joined as a String
+  def read
+    read_lines.join
+  end
+
+  # Public: Advance to the next line by discarding the line at the front of the stack
   #
-  # Returns an Integer of the number of lines skipped
-  def skip_blank_lines
-    skipped = 0
-    # optimized code for shortest execution path
-    while !(next_line = get_line).nil?
-      if next_line.chomp.empty?
-        skipped += 1
-      else
-        unshift_line next_line
-        break
-      end
-    end 
+  # direct  - A Boolean flag to bypasses the check for more lines and immediately
+  #           returns the first element of the internal @lines Array. (default: true)
+  #
+  # returns a Boolean indicating whether there was a line to discard.
+  def advance direct = true
+    !(read_line direct).nil?
+  end
 
-    skipped
+  # Public: Push the String line onto the beginning of the Array of source data.
+  #
+  # Since this line was (assumed to be) previously retrieved through the
+  # reader, it is marked as seen.
+  #
+  # returns nil
+  def unshift_line line_to_restore
+    unshift line_to_restore
+    nil
   end
+  alias :restore_line :unshift_line
 
-  # Public: Consume consecutive lines containing line- or block-level comments.
+  # Public: Push an Array of lines onto the front of the Array of source data.
   #
-  # Returns the Array of lines that were consumed
+  # Since these lines were (assumed to be) previously retrieved through the
+  # reader, they are marked as seen.
+  #
+  # Returns nil
+  def unshift_lines lines_to_restore
+    # QUESTION is it faster to use unshift(*lines_to_restore)?
+    lines_to_restore.reverse_each {|line| unshift line }
+    nil
+  end
+  alias :restore_lines :unshift_lines
+
+  # Public: Consume consecutive lines containing line comments.
   #
   # Examples
   #   @lines
-  #   => ["// foo\n", "////\n", "foo bar\n", "////\n", "actual text\n"]
+  #   => ["// foo\n", "bar\n"]
   #
   #   comment_lines = consume_comments
-  #   => ["// foo\n", "////\n", "foo bar\n", "////\n"]
+  #   => ["// foo\n"]
   #
   #   @lines
-  #   => ["actual text\n"]
-  def consume_comments(options = {})
+  #   => ["bar\n"]
+  #
+  # Returns the Array of lines that were consumed
+  def consume_comments opts = {}
+    return [] if eof?
+
     comment_lines = []
-    preprocess = options.fetch(:preprocess, true)
-    while !(next_line = get_line(preprocess)).nil?
-      if options[:include_blank_lines] && next_line.chomp.empty?
-        comment_lines << next_line
+    include_blank_lines = opts[:include_blank_lines]
+    while (next_line = peek_line)
+      if include_blank_lines && next_line.chomp.empty?
+        comment_lines << read_line
       elsif (commentish = next_line.start_with?('//')) && (match = next_line.match(REGEXP[:comment_blk]))
-        comment_lines << next_line
-        comment_lines.push(*(grab_lines_until(:terminator => match[0], :grab_last_line => true, :preprocess => false)))
+        comment_lines << read_line
+        comment_lines.push(*(take_lines_until(:terminator => match[0], :take_last_line => true, :raw => true)))
       elsif commentish && next_line.match(REGEXP[:comment])
-        comment_lines << next_line
+        comment_lines << read_line
       else
-        # throw it back
-        unshift_line next_line
         break
       end
     end
@@ -156,225 +253,323 @@ class Reader
   end
   alias :skip_comment_lines :consume_comments
 
-  # Ignore front-matter, commonly used in static site generators
-  def skip_front_matter(data, increment_linenos = true)
-    #if data.nil?
-    #  data = @lines
-    #  increment_linenos = true
-    #else
-    #  increment_linenos = false
-    #end
-
-    front_matter = nil
-    if data.size > 0 && data.first.chomp == '---'
-      original_data = data.dup
-      front_matter = []
-      data.shift
-      @lineno += 1 if increment_linenos
-      while !data.empty? && data.first.chomp != '---'
-        front_matter.push data.shift
-        @lineno += 1 if increment_linenos
-      end
+  def consume_line_comments
+    return [] if eof?
 
-      if data.empty?
-        data.unshift(*original_data)
-        @lineno = 0 if increment_linenos
-        front_matter = nil
+    comment_lines = []
+    # optimized code for shortest execution path
+    while (next_line = peek_line)
+      if next_line.match(REGEXP[:comment])
+        comment_lines << read_line
       else
-        data.shift
-        @lineno += 1 if increment_linenos
+        break
       end
     end
 
-    front_matter
+    comment_lines
   end
+  alias :skip_lines_comments :consume_comments
 
-  # Public: Consume consecutive lines containing line comments.
-  #
-  # Returns the Array of lines that were consumed
+  # Private: Strip off leading blank lines in the Array of lines.
   #
   # Examples
+  #
   #   @lines
-  #   => ["// foo\n", "bar\n"]
+  #   => ["\n", "\t\n", "Foo\n", "Bar\n", "\n"]
   #
-  #   comment_lines = consume_comments
-  #   => ["// foo\n"]
+  #   skip_blank_lines
+  #   => 2
   #
   #   @lines
-  #   => ["bar\n"]
-  def consume_line_comments
-    comment_lines = []
+  #   => ["Foo\n", "Bar\n"]
+  #
+  # Returns an Integer of the number of lines skipped
+  def skip_blank_lines
+    return 0 if eof?
+
+    num_skipped = 0
     # optimized code for shortest execution path
-    while !(next_line = get_line).nil?
-      if next_line.match(REGEXP[:comment])
-        comment_lines << next_line
+    while (next_line = peek_line)
+      if next_line.chomp.empty?
+        advance
+        num_skipped += 1
       else
-        unshift_line next_line
-        break
+        return num_skipped
       end
     end
 
-    comment_lines
+    num_skipped
   end
 
-  # Public: Get the next line of source data. Consumes the line returned.
-  #
-  # preprocess - A Boolean flag indicating whether to evaluate preprocessing
-  #              directives (macros) before reading line (default: true)
+  # Public: Advance to the end of the reader, consuming all remaining lines
   #
-  # Returns the String of the next line of the source data if data is present.
-  # Returns nil if there is no more data.
-  def get_line(preprocess = true)
-    if @eof || (@eof = @lines.empty?)
-      @next_line_preprocessed = true
-      nil
-    elsif preprocess && @preprocess_source &&
-        !@next_line_preprocessed && preprocess_next_line.nil?
-      @next_line_preprocessed = true
-      nil
-    else
-      @lineno += 1
-      @next_line_preprocessed = false
-      if @unescape_next_line
-        @unescape_next_line = false
-        @lines.shift[1..-1]
-      else
-        @lines.shift
-      end
-    end
+  # Returns nothing.
+  def terminate
+    @lineno += @lines.size
+    @lines.clear
+    @eof = true
+    @look_ahead = 0
+    nil
   end
 
-  # Public: Get the remaining lines of source data by calling
-  # Reader#get_line until all lines are consumed.
-  #
-  # preprocess - A Boolean flag indicating whether to evaluate preprocessing
-  #              directives (macros) before reading line (default: true)
+  # Public: Check whether this reader is empty (contains no lines)
   #
-  # Returns the lines read as a String Array
-  def get_lines(preprocess = true)
-    lines = nil
-    while has_more_lines? preprocess
-      (lines ||= []) << get_line(preprocess)
-    end
-    @eof = true
-    lines
+  # Returns true if there are no more lines to peek, otherwise false.
+  def eof?
+    !has_more_lines?
   end
+  alias :empty? :eof?
 
-  # Public: Advance to the next line by discarding the line at the front of the stack
+  # Public: Return all the lines from `@lines` until we (1) run out them,
+  #   (2) find a blank line with :break_on_blank_lines => true, or (3) find
+  #   a line for which the given block evals to true.
   #
-  # Removes the line at the front of the stack without any processing.
+  # options - an optional Hash of processing options:
+  #           * :break_on_blank_lines may be used to specify to break on
+  #               blank lines
+  #           * :skip_first_line may be used to tell the reader to advance
+  #               beyond the first line before beginning the scan
+  #           * :preserve_last_line may be used to specify that the String
+  #               causing the method to stop processing lines should be
+  #               pushed back onto the `lines` Array.
+  #           * :take_last_line may be used to specify that the String
+  #               causing the method to stop processing lines should be
+  #               included in the lines being returned
   #
-  # returns a boolean indicating whether there was a line to discard
-  def advance
-    @next_line_preprocessed = false
-    # we assume that we're advancing over a line of known content
-    if @eof || (@eof = @lines.empty?)
-      false
+  # Returns the Array of lines forming the next segment.
+  #
+  # Examples
+  #
+  #   reader = Reader.new ["First paragraph\n", "Second paragraph\n",
+  #                        "Open block\n", "\n", "Can have blank lines\n",
+  #                        "--\n", "\n", "In a different segment\n"]
+  #
+  #   reader.take_lines_until
+  #   => ["First paragraph\n", "Second paragraph\n", "Open block\n"]
+  def take_lines_until options = {}
+    result = []
+    advance if options[:skip_first_line]
+    if @process_lines && options[:raw]
+      @process_lines = false
+      reset_process_lines = true
     else
-      @lineno += 1
-      @lines.shift
-      true
+      reset_process_lines = false
     end
-  end
 
-  # Public: Advance to the end of the reader, consuming all remaining lines
-  #
-  # Returns nothing.
-  def terminate
-    while !@eof
-      advance
+    has_block = block_given?
+    if (terminator = (options.fetch :terminator, nil))
+      break_on_blank_lines = false
+      break_on_list_continuation = false
+      chomp_last_line = options.fetch :chomp_last_line, false
+    else
+      break_on_blank_lines = options[:break_on_blank_lines]
+      break_on_list_continuation = options[:break_on_list_continuation]
+      chomp_last_line = break_on_blank_lines
+    end
+    skip_line_comments = options[:skip_line_comments]
+    taken = false
+    
+    while (line = read_line)
+      finish = while true
+        break true if terminator && line.chomp == terminator
+        # QUESTION: can we get away with line.chomp.empty? here?
+        break true if break_on_blank_lines && line.chomp.empty?
+        if break_on_list_continuation && taken && line.chomp == LIST_CONTINUATION
+          options[:preserve_last_line] = true
+          break true
+        end
+        break true if has_block && (yield line)
+        break false
+      end
+
+      if finish
+        if options[:take_last_line]
+          result << line
+          taken = true
+        end
+        restore_line line if options[:preserve_last_line]
+        break
+      end
+
+      unless skip_line_comments && line.match(REGEXP[:comment])
+        result << line
+        taken = true
+      end
+    end
+
+    if chomp_last_line && taken
+      result[-1] = result.last.chomp
     end
+
+    @process_lines = true if reset_process_lines
+    result
+  end
+  alias :grab_lines_until :take_lines_until
+
+  # Protected: Shift the line off the stack and increment the lineno
+  def shift
+    @lineno += 1
+    @look_ahead -= 1 unless @look_ahead == 0
+    @lines.shift
   end
 
-  # Public: Return whether the next line is empty. Does not consume the line returned.
+  # Protected: Restore the line to the stack and decrement the lineno
+  def unshift line
+    @lineno -= 1
+    @look_ahead += 1
+    @eof = false
+    @lines.unshift line
+  end
+
+  # Public: Get information about the last line read, including file name and line number.
   #
-  # preprocess - A Boolean flag indicating whether to evaluate preprocessing
-  #              directives (macros) before reading line (default: true)
+  # Returns A String summary of the last line read
+  def line_info
+    %(#{@path}: line #{@lineno})
+  end
+  alias :next_line_info :line_info
+
+  def prev_line_info
+    %(#{@path}: line #{@lineno - 1})
+  end
+
+  # Public: Get a copy of the remaining Array of String lines managed by this Reader
   #
-  # Returns a Boolean indicating whether the next line of the source data is empty.
-  # Returns true if there is no more data.
-  def next_line_empty?(preprocess = true)
-    if !preprocess
-      @eof || (@eof = @lines.empty?) ? true : @lines.first.chomp.empty?
-    elsif has_more_lines? true
-      @lines.first.chomp.empty?
-    else
-      true
-    end
+  # Returns A copy of the String Array of lines remaining in this Reader
+  def lines
+    @lines.dup
   end
 
-  # Public: Get the next line of source data. Does not consume the line returned.
+  def source
+    @source_lines.join
+  end
+
+  # Public: Get a summary of this Reader.
   #
-  # preprocess - A Boolean flag indicating whether to evaluate preprocessing
-  #              directives (macros) before reading line (default: true)
   #
-  # Returns a String dup of the next line of the source data if data is present.
-  # Returns nil if there is no more data.
-  def peek_line(preprocess = true)
-    if !preprocess
-      # QUESTION do we need to dup?
-      @eof || (@eof = @lines.empty?) ? nil : @lines.first.dup
-    elsif has_more_lines? true
-      # QUESTION do we need to dup?
-      @lines.first.dup
+  # Returns A string summary of this reader, which contains the path and line information
+  def to_s
+    line_info
+  end
+end
+
+# Public: Methods for retrieving lines from AsciiDoc source files, evaluating preprocessor
+# directives as each line is read off the Array of lines.
+class PreprocessorReader < Reader
+  attr_reader :include_stack
+  attr_reader :includes
+
+  # Public: Initialize the PreprocessorReader object
+  def initialize document, data = nil, file = nil, path = nil
+    @document = document
+    super data, file, path
+    include_depth_default = document.attributes.fetch('include-depth', 10).to_i
+    include_depth_default = 0 if include_depth_default < 0
+    # track both absolute depth for comparing to size of include stack and relative depth for reporting
+    @maxdepth = {:abs => include_depth_default, :rel => include_depth_default}
+    @include_stack = []
+    @includes = (document.references[:includes] ||= [])
+    @skipping = false
+    @conditional_stack = []
+  end
+
+  def prepare_lines data, opts = {}
+    if data.is_a?(String)
+      if ::Asciidoctor::FORCE_ENCODING
+        result = data.each_line.map {|line| "#{line.rstrip.force_encoding ::Encoding::UTF_8}#{::Asciidoctor::EOL}" }
+      else
+        result = data.each_line.map {|line| "#{line.rstrip}#{::Asciidoctor::EOL}" }
+      end
     else
-      nil
+      if ::Asciidoctor::FORCE_ENCODING
+        result = data.map {|line| "#{line.rstrip.force_encoding ::Encoding::UTF_8}#{::Asciidoctor::EOL}" }
+      else
+        result = data.map {|line| "#{line.rstrip}#{::Asciidoctor::EOL}" }
+      end
     end
-  end
 
-  # TODO document & test me!
-  def peek_lines(number = 1)
-    lines = []
-    idx = 0
-    (1..number).each do
-      if @preprocess_source && !@next_line_preprocessed
-        advanced = preprocess_next_line
-        break if advanced.nil? || @eof || (@eof = @lines.empty?)
-        idx = 0 if advanced
+    # QUESTION should this work for AsciiDoc table cell content? Currently it does not.
+    unless @document.nil? || !(@document.attributes.has_key? 'skip-front-matter')
+      if (front_matter = skip_front_matter! result)
+        @document.attributes['front-matter'] = front_matter.join.chomp
       end
-      break if idx >= @lines.size
-      # QUESTION do we need to dup?
-      lines << @lines[idx].dup
-      idx += 1
     end
-    lines
+
+    # QUESTION should we chomp last line? (with or without the condense flag?)
+    if opts.fetch(:condense, true)
+      result.shift && @lineno += 1 while !(first = result.first).nil? && first == ::Asciidoctor::EOL
+      result.pop while !(last = result.last).nil? && last == ::Asciidoctor::EOL
+    end
+
+    if (indent = opts.fetch(:indent, nil))
+      Lexer.reset_block_indent! result, indent.to_i
+    end
+
+    result
   end
 
-  # Internal: Preprocess the next line until the cursor is at a line of content
-  #
-  # Evaluate preprocessor macros on the next line, continuing to do so until
-  # the cursor arrives at a line of included content. That line is marked as
-  # preprocessed so that preprocessing is not performed multiple times.
-  #
-  # returns a Boolean indicating whether the cursor advanced, or nil if there
-  # are no more lines available.
-  def preprocess_next_line
-    # this return could be happening from a recursive call
-    return nil if @eof || (next_line = @lines.first).nil?
-    if next_line.include?('::') && (next_line.include?('if') || next_line.include?('endif')) && (match = next_line.match(REGEXP[:ifdef_macro]))
-      if next_line.start_with? '\\'
-        @next_line_preprocessed = true
+  def process_line line
+    return line unless @process_lines
+
+    if line.chomp.empty?
+      @look_ahead += 1
+      return ''
+    end
+
+    found_colon = line.include?('::')
+    if found_colon && line.include?('if') && (match = line.match(REGEXP[:ifdef_macro]))
+      # if escaped, mark as processed and return line unescaped
+      if line.start_with? '\\'
         @unescape_next_line = true
-        false
+        @look_ahead += 1
+        line[1..-1]
       else
-        preprocess_conditional_inclusion(*match.captures)
+        if preprocess_conditional_inclusion(*match.captures)
+          # move the pointer past the conditional line
+          advance
+          # treat next line as uncharted territory
+          nil
+        else
+          # the line was not a valid conditional line
+          # mark it as visited and return it
+          @look_ahead += 1
+          line
+        end
       end
     elsif @skipping
       advance
-      # skip over comment blocks, we don't want to process directives in them
-      skip_comment_lines :include_blank_lines => true, :preprocess => false
-      preprocess_next_line.nil? ? nil : true
-    elsif next_line.include?('include::') && match = next_line.match(REGEXP[:include_macro])
-      if next_line.start_with? '\\'
-        @next_line_preprocessed = true
+      nil 
+    elsif found_colon && line.include?('include::') && (match = line.match(REGEXP[:include_macro]))
+      # if escaped, mark as processed and return line unescaped
+      if line.start_with? '\\'
         @unescape_next_line = true
-        false
+        @look_ahead += 1
+        line[1..-1]
       else
-        preprocess_include(match[1], match[2].strip)
+        # QUESTION should we strip whitespace from raw attributes in Substituters#parse_attributes? (check perf)
+        if preprocess_include match[1], match[2].strip
+          # peek again since the content has changed
+          nil
+        else
+          # the line was not a valid include line and is unchanged
+          # mark it as visited and return it
+          @look_ahead += 1
+          line
+        end
       end
     else
-      @next_line_preprocessed = true
-      false
+      super
+    end
+  end
+
+  def peek_line direct = false
+    if (line = super)
+      line
+    elsif !@include_stack.empty?
+      pop_include
+      peek_line direct
+    else
+      nil
     end
   end
 
@@ -397,37 +592,36 @@ class Reader
   #              Used for a single-line conditional block in the case of the ifdef or
   #              ifndef directives, and for the conditional expression for the ifeval directive.
   #
-  # returns a Boolean indicating whether the cursor advanced, or nil if there
-  # are no more lines available.
-  def preprocess_conditional_inclusion(directive, target, delimiter, text)
+  # returns a Boolean indicating whether the cursor should be advanced
+  def preprocess_conditional_inclusion directive, target, delimiter, text
     # must have a target before brackets if ifdef or ifndef
     # must not have text between brackets if endif
     # don't honor match if it doesn't meet this criteria
+    # QUESTION should we warn for these bogus declarations?
     if ((directive == 'ifdef' || directive == 'ifndef') && target.empty?) ||
         (directive == 'endif' && !text.nil?)
-      @next_line_preprocessed = true
       return false
     end
 
     if directive == 'endif'
-      stack_size = @conditionals_stack.size
+      stack_size = @conditional_stack.size
       if stack_size > 0
-        pair = @conditionals_stack.last
+        pair = @conditional_stack.last
         if target.empty? || target == pair[:target]
-          @conditionals_stack.pop
-          @skipping = @conditionals_stack.empty? ? false : @conditionals_stack.last[:skipping]
+          @conditional_stack.pop
+          @skipping = @conditional_stack.empty? ? false : @conditional_stack.last[:skipping]
         else
-          warn "asciidoctor: ERROR: line #{@lineno + 1}: mismatched macro: endif::#{target}[], expected endif::#{pair[:target]}[]"
+          warn "asciidoctor: ERROR: #{line_info}: mismatched macro: endif::#{target}[], expected endif::#{pair[:target]}[]"
         end
       else
-        warn "asciidoctor: ERROR: line #{@lineno + 1}: unmatched macro: endif::#{target}[]"
+        warn "asciidoctor: ERROR: #{line_info}: unmatched macro: endif::#{target}[]"
       end
-      advance
-      return preprocess_next_line.nil? ? nil : true
+      return true
     end
 
     skip = false
-    if !@skipping
+    unless @skipping
+      # QUESTION any way to wrap ifdef & ifndef logic up together?
       case directive
       when 'ifdef'
         case delimiter
@@ -457,32 +651,36 @@ class Reader
         # the text in brackets must match an expression
         # don't honor match if it doesn't meet this criteria
         if !target.empty? || !(expr_match = text.strip.match(REGEXP[:eval_expr]))
-          @next_line_preprocessed = true
           return false
         end
 
-        lhs = resolve_expr_val(expr_match[1])
+        lhs = resolve_expr_val expr_match[1]
+        # regex enforces a restrict set of math-related operations
         op = expr_match[2]
-        rhs = resolve_expr_val(expr_match[3])
+        rhs = resolve_expr_val expr_match[3]
 
-        skip = !lhs.send(op.to_sym, rhs)
+        skip = !(lhs.send op.to_sym, rhs)
       end
     end
-    advance
-    # single line conditional inclusion
-    if directive != 'ifeval' && !text.nil?
-      if !@skipping && !skip
-        unshift_line "#{text.rstrip}\n"
-        return true
-      end
+
     # conditional inclusion block
+    if directive == 'ifeval' || text.nil?
+      @skipping = true if skip
+      @conditional_stack << {:target => target, :skip => skip, :skipping => @skipping}
+    # single line conditional inclusion
     else
-      if !@skipping && skip
-        @skipping = true
+      unless @skipping || skip
+        # FIXME this should be simpler!
+        # slight hack to skip past conditional line
+        # but keep our synthetic line marked as processed
+        conditional_line = read_line true
+        unshift "#{text.rstrip}#{::Asciidoctor::EOL}"
+        unshift conditional_line
+        return true
       end
-      @conditionals_stack << {:target => target, :skip => skip, :skipping => @skipping}
     end
-    return preprocess_next_line.nil? ? nil : true
+
+    true
   end
 
   # Internal: Preprocess the directive (macro) to include the target document.
@@ -493,13 +691,12 @@ class Reader
   # If SafeMode is SECURE or greater, the directive is ignore and the include
   # directive line is emitted verbatim.
   #
-  # Otherwise, if an include handler is specified (currently controlled by a
-  # closure block), pass the target to that block and expect an Array of String
-  # lines in return.
+  # Otherwise, if an include processor is specified pass the target and
+  # attributes to that processor and expect an Array of String lines in return.
   #
-  # Otherwise, if the include-depth attribute is greater than 0, normalize the
-  # target path and read the lines onto the beginning of the Array of source
-  # data.
+  # Otherwise, if the max depth is greater than 0, and is not exceeded by the
+  # stack size, normalize the target path and read the lines onto the beginning
+  # of the Array of source data.
   #
   # If none of the above apply, emit the include directive line verbatim.
   #
@@ -508,50 +705,55 @@ class Reader
   #
   # returns a Boolean indicating whether the line under the cursor has changed.
   # -
-  # FIXME includes bork line numbers
-  def preprocess_include(target, raw_attributes)
-    target = @document.sub_attributes target
+  # TODO extensions need to be able to communicate line numbers
+  def preprocess_include target, raw_attributes
+    target = @document.sub_attributes target, :attribute_missing => 'drop-line'
     if target.empty?
-      advance
-      @next_line_preprocessed = false
-      false
+      if @document.attributes.fetch('attribute-missing', COMPLIANCE[:attribute_missing]) == 'skip'
+        false
+      else
+        advance
+        true
+      end
     # if running in SafeMode::SECURE or greater, don't process this directive
     # however, be friendly and at least make it a link to the source document
     elsif @document.safe >= SafeMode::SECURE
-      @lines[0] = "link:#{target}[#{target}]\n"
-      @next_line_preprocessed = true
-      false
-    # assume that if a block is given, the developer wants
-    # to handle when and how to process the include, even
-    # if the include-depth attribute is 0
-    elsif @include_block
-      advance
-      @lines.unshift(*normalize_include_data(@include_block.call(target, @document)))
-    # FIXME currently we're not checking the upper bound of the include depth
-    elsif @document.attributes.fetch('include-depth', 0).to_i > 0
       advance
+      unshift "link:#{target}[]#{::Asciidoctor::EOL}"
+      # TODO make creating the output target a helper method
+      #output_target = %(#{File.join(File.dirname(target), File.basename(target, File.extname(target)))}#{@document.attributes['outfilesuffix']})
+      #unshift "link:#{output_target}[]#{::Asciidoctor::EOL}"
+      true
+    elsif @maxdepth[:abs] > 0 && @include_stack.size >= @maxdepth[:abs]
+      warn %(asciidoctor: WARNING: #{line_info}: maximum include depth of #{@maxdepth[:rel]} exceeded)
+      false
+    # FIXME Insert IncludeProcessor logic here once merged
+    elsif @maxdepth[:abs] > 0
       if target.include?(':') && target.match(REGEXP[:uri_sniff])
         unless @document.attributes.has_key? 'allow-uri-read'
-          @lines[0] = "link:#{target}[#{target}]\n"
-          @next_line_preprocessed = true
-          return false
+          advance
+          unshift "link:#{target}[]#{::Asciidoctor::EOL}"
+          return true
         end
 
         target_type = :uri
-        include_file = target
+        include_file = path = target
         if @document.attributes.has_key? 'cache-uri'
-          # NOTE caching requires the open-uri-cached gem to be installed
+          # caching requires the open-uri-cached gem to be installed
+          # processing will be automatically aborted if these libraries can't be opened
           Helpers.require_library 'open-uri/cached', 'open-uri-cached'
         else
           Helpers.require_library 'open-uri'
         end
-        # FIXME should stop processing include if required library cannot be loaded; check for -1 return value?
       else
         target_type = :file
         @document.references[:includes] << Helpers.rootname(target)
-        include_file = @document.normalize_system_path(target, nil, nil, :target_name => 'include file')
+        # include file is resolved relative to current include context
+        include_file = @document.normalize_system_path(target, @dir, nil, :target_name => 'include file')
+        path = include_file[(@document.base_dir.length + 1)..-1]
         if !File.file?(include_file)
-          warn "asciidoctor: WARNING: line #{@lineno}: include file not found: #{include_file}"
+          warn "asciidoctor: WARNING: #{line_info}: include file not found: #{include_file}"
+          advance
           return true
         end
       end
@@ -560,6 +762,7 @@ class Reader
       tags = nil
       attributes = {}
       if !raw_attributes.empty?
+        # QUESTION should we use @document.parse_attribues?
         attributes = AttributeList.new(raw_attributes).parse
         if attributes.has_key? 'lines'
           inc_lines = []
@@ -584,40 +787,53 @@ class Reader
       if !inc_lines.nil?
         if !inc_lines.empty?
           selected = []
+          inc_line_offset = 0
+          inc_lineno = 0
           begin
             open(include_file) do |f|
               f.each_line do |l|
+                inc_lineno += 1
                 take = inc_lines.first
                 if take.is_a?(Float) && take.infinite?
                   selected.push l
+                  inc_line_offset = inc_lineno if inc_line_offset == 0
                 else
                   if f.lineno == take
                     selected.push l
-                    inc_lines.shift 
+                    inc_line_offset = inc_lineno if inc_line_offset == 0
+                    inc_lines.shift
                   end
                   break if inc_lines.empty?
                 end
               end
             end
           rescue
-            warn "asciidoctor: WARNING: line #{@lineno}: include #{target_type} not readable: #{include_file}"
+            warn "asciidoctor: WARNING: #{line_info}: include #{target_type} not readable: #{include_file}"
+            advance
             return true
           end
-          @lines.unshift(*normalize_include_data(selected, attributes['indent'])) unless selected.empty?
+          advance
+          # FIXME not accounting for skipped lines in reader line numbering
+          push_include selected, include_file, path, inc_line_offset, attributes
         end
       elsif !tags.nil?
         if !tags.empty?
           selected = []
+          inc_line_offset = 0
+          inc_lineno = 0
           active_tag = nil
           begin
             open(include_file) do |f|
               f.each_line do |l|
+                inc_lineno += 1
+                # must force encoding here since we're performing String operations on line
                 l.force_encoding(::Encoding::UTF_8) if ::Asciidoctor::FORCE_ENCODING
                 if !active_tag.nil?
                   if l.include?("end::#{active_tag}[]")
                     active_tag = nil
                   else
-                    selected.push "#{l.rstrip}\n"
+                    selected.push l
+                    inc_line_offset = inc_lineno if inc_line_offset == 0
                   end
                 else
                   tags.each do |tag|
@@ -630,184 +846,167 @@ class Reader
               end
             end
           rescue
-            warn "asciidoctor: WARNING: line #{@lineno}: include #{target_type} not readable: #{include_file}"
+            warn "asciidoctor: WARNING: #{line_info}: include #{target_type} not readable: #{include_file}"
+            advance
             return true
           end
-          #@lines.unshift(*selected) unless selected.empty?
-          @lines.unshift(*normalize_include_data(selected, attributes['indent'])) unless selected.empty?
+          advance
+          # FIXME not accounting for skipped lines in reader line numbering
+          push_include selected, include_file, path, inc_line_offset, attributes
         end
       else
         begin
-          @lines.unshift(*normalize_include_data(open(include_file) {|f| f.readlines }, attributes['indent']))
+          advance
+          push_include open(include_file) {|f| f.read }, include_file, path, 1, attributes
         rescue
-          warn "asciidoctor: WARNING: line #{@lineno}: include #{target_type} not readable: #{include_file}"
+          warn "asciidoctor: WARNING: #{line_info}: include #{target_type} not readable: #{include_file}"
+          advance
           return true
         end
       end
       true
     else
-      @next_line_preprocessed = true
       false
     end
   end
 
-  # Public: Push the String line onto the beginning of the Array of source data.
-  #
-  # Since this line was (assumed to be) previously retrieved through the
-  # reader, it is marked as preprocessed.
-  #
-  # returns nil
-  def unshift_line(line)
-    @lines.unshift line
-    @next_line_preprocessed = true
-    @eof = false
-    @lineno -= 1
-    nil
-  end
-
-  # Public: Push Array of lines onto the front of the Array of source data, unless `lines` has no non-nil values.
-  #
-  # Returns nil
-  def unshift(*new_lines)
-    size = new_lines.size
-    if size > 0
-      @lines.unshift(*new_lines)
-      # assume that what we are putting back on is already processed for directives
-      @next_line_preprocessed = true
+  def push_include data, file = nil, path = nil, lineno = 1, attributes = {}
+    # FIXME change this to File.basename(File.rootname(file)) or File.baserootname(file)
+    @includes << File.join(File.dirname(file), File.basename(file, File.extname(file)))
+    @include_stack << [@lines, @file, @path, @lineno, @maxdepth]
+    @file = file
+    @dir = File.dirname file
+    @path = path
+    @lineno = lineno
+    if attributes.has_key? 'depth'
+      depth = attributes['depth'].to_i
+      depth = 1 if depth <= 0
+      @maxdepth = {:abs => (@include_stack.size - 1) + depth, :rel => depth}
+    end
+    # effectively fill the buffer
+    @lines = prepare_lines data, :condense => false, :indent => attributes['indent']
+    # FIXME kind of a hack
+    #Document::AttributeEntry.new('infile', @file).save_to_next_block @document
+    #Document::AttributeEntry.new('indir', File.dirname(@file)).save_to_next_block @document
+    if @lines.empty?
+      pop_include
+    else
       @eof = false
-      @lineno -= size
+      @look_ahead = 0
     end
-    nil
   end
 
-  # Public: Chomp the String on the last line if this reader contains at least one line
-  #
-  # Delegates to chomp!
-  #
-  # Returns nil
-  def chomp_last!
-    @lines.last.chomp! unless @eof || (@eof = @lines.empty?)
-    nil
+  def pop_include
+    if @include_stack.size > 0
+      @lines, @file, @path, @lineno, @maxdepth = @include_stack.pop
+      @dir = @file.nil? ? nil : File.dirname(@file)
+      # FIXME kind of a hack
+      #Document::AttributeEntry.new('infile', @file).save_to_next_block @document
+      #Document::AttributeEntry.new('indir', File.dirname(@file)).save_to_next_block @document
+      @eof = @lines.empty?
+      @look_ahead = 0
+    end
   end
 
-  # Public: Return all the lines from `@lines` until we (1) run out them,
-  #   (2) find a blank line with :break_on_blank_lines => true, or (3) find
-  #   a line for which the given block evals to true.
-  #
-  # options - an optional Hash of processing options:
-  #           * :break_on_blank_lines may be used to specify to break on
-  #               blank lines
-  #           * :skip_first_line may be used to tell the reader to advance
-  #               beyond the first line before beginning the scan
-  #           * :preserve_last_line may be used to specify that the String
-  #               causing the method to stop processing lines should be
-  #               pushed back onto the `lines` Array.
-  #           * :grab_last_line may be used to specify that the String
-  #               causing the method to stop processing lines should be
-  #               included in the lines being returned
-  #
-  # Returns the Array of lines forming the next segment.
-  #
-  # Examples
-  #
-  #   reader = Reader.new ["First paragraph\n", "Second paragraph\n",
-  #                        "Open block\n", "\n", "Can have blank lines\n",
-  #                        "--\n", "\n", "In a different segment\n"]
-  #
-  #   reader.grab_lines_until
-  #   => ["First paragraph\n", "Second paragraph\n", "Open block\n"]
-  def grab_lines_until(options = {}, &block)
-    buffer = []
+  def include_depth
+    @include_stack.size 
+  end
 
-    advance if options[:skip_first_line]
-    # very hot code
-    # save options to locals for minor optimizations
-    if options.has_key? :terminator
-      terminator = options[:terminator]
-      break_on_blank_lines = false
-      break_on_list_continuation = false
-      chomp_last_line = options[:chomp_last_line] || false
+  # TODO Document this override
+  # also, we now have the field in the super class, so perhaps
+  # just implement the logic there?
+  def shift
+    if @unescape_next_line
+      @unescape_next_line = false
+      super[1..-1]
     else
-      terminator = nil
-      break_on_blank_lines = options[:break_on_blank_lines]
-      break_on_list_continuation = options[:break_on_list_continuation]
-      chomp_last_line = break_on_blank_lines
+      super
     end
-    skip_line_comments = options[:skip_line_comments]
-    preprocess = options.fetch(:preprocess, true)
-    buffer_empty = true
-    while !(this_line = get_line(preprocess)).nil?
-      # effectively a no-args lamba, but much faster
-      finish = while true
-        break true if terminator && this_line.chomp == terminator
-        break true if break_on_blank_lines && this_line.strip.empty?
-        if break_on_list_continuation && !buffer_empty && this_line.chomp == LIST_CONTINUATION
-          options[:preserve_last_line] = true
-          break true
-        end
-        break true if block && yield(this_line)
-        break false
-      end
+  end
 
-      if finish
-        if options[:grab_last_line]
-          buffer << this_line
-          buffer_empty = false
-        end
-        # QUESTION should we dup this_line when restoring??
-        unshift_line this_line if options[:preserve_last_line]
-        break
+  # Private: Ignore front-matter, commonly used in static site generators
+  def skip_front_matter! data, increment_linenos = true
+    #if data.nil?
+    #  data = @lines
+    #  increment_linenos = true
+    #else
+    #  increment_linenos = false
+    #end
+
+    front_matter = nil
+    if data.size > 0 && data.first.chomp == '---'
+      original_data = data.dup
+      front_matter = []
+      data.shift
+      @lineno += 1 if increment_linenos
+      while !data.empty? && data.first.chomp != '---'
+        front_matter.push data.shift
+        @lineno += 1 if increment_linenos
       end
 
-      unless skip_line_comments && this_line.match(REGEXP[:comment])
-        buffer << this_line
-        buffer_empty = false
+      if data.empty?
+        data.unshift(*original_data)
+        @lineno = 0 if increment_linenos
+        front_matter = nil
+      else
+        data.shift
+        @lineno += 1 if increment_linenos
       end
     end
 
-    # should we dup the line before chopping?
-    buffer.last.chomp! if chomp_last_line && !buffer_empty
-    buffer
+    front_matter
   end
 
-  # Public: Convert a string to a legal attribute name.
+  # Private: Resolve the value of one side of the expression
   #
-  # name  - The String holding the Asciidoc attribute name.
+  # Examples
   #
-  # Returns a String with the legal name.
+  #   expr = '"value"'
+  #   resolve_expr_val(expr)
+  #   # => "value"
   #
-  # Examples
+  #   expr = '"value'
+  #   resolve_expr_val(expr)
+  #   # => "\"value"
   #
-  #   sanitize_attribute_name('Foo Bar')
-  #   => 'foobar'
+  #   expr = '"{undefined}"'
+  #   resolve_expr_val(expr)
+  #   # => ""
   #
-  #   sanitize_attribute_name('foo')
-  #   => 'foo'
+  #   expr = '{undefined}'
+  #   resolve_expr_val(expr)
+  #   # => nil
   #
-  #   sanitize_attribute_name('Foo 3 #-Billy')
-  #   => 'foo3-billy'
-  def sanitize_attribute_name(name)
-    Lexer.sanitize_attribute_name(name)
-  end
-
-  # Private: Resolve the value of one side of the expression
+  #   expr = '2'
+  #   resolve_expr_val(expr)
+  #   # => 2
+  #
+  #   @document.attributes['name'] = 'value'
+  #   expr = '"{name}"'
+  #   resolve_expr_val(expr)
+  #   # => "value"
+  #
+  # Returns The value of the expression, coerced to the appropriate type
   def resolve_expr_val(str)
     val = str
     type = nil
 
     if val.start_with?('"') && val.end_with?('"') ||
         val.start_with?('\'') && val.end_with?('\'')
-      type = :s
-      val = val[1..-2]
+      type = :string
+      val = val[1...-1]
     end
 
+    # QUESTION should we substitute first?
     if val.include? '{'
       val = @document.sub_attributes val
     end
 
-    if type != :s
+    unless type == :string
       if val.empty?
         val = nil
+      elsif val.strip.empty?
+        val = ' '
       elsif val == 'true'
         val = true
       elsif val == 'false'
@@ -815,6 +1014,8 @@ class Reader
       elsif val.include?('.')
         val = val.to_f
       else
+        # fallback to coercing to integer, since we
+        # require string values to be explicitly quoted
         val = val.to_i
       end
     end
@@ -822,75 +1023,9 @@ class Reader
     val
   end
 
-  # Private: Normalize raw input read from an include directive
-  #
-  # This method strips whitespace from the end of every line of
-  # the source data and appends a LF (i.e., Unix endline). This
-  # whitespace substitution is very important to how Asciidoctor
-  # works.
-  #
-  # Any leading or trailing blank lines are also removed. (DISABLED)
-  #
-  # data - A String Array of input data to be normalized
-  #
-  # returns the processed lines
-  #-
-  # FIXME this shares too much in common w/ normalize_data; combine
-  # in a shared function
-  def normalize_include_data(data, indent = nil)
-    if ::Asciidoctor::FORCE_ENCODING
-      result = data.map {|line| "#{line.rstrip.force_encoding(::Encoding::UTF_8)}\n" }
-    else
-      result = data.map {|line| "#{line.rstrip}\n" }
-    end
-
-    # QUESTION: how should this be activated? (defering for now)
-    # QUESTION: should we save the front matter read from an include file somewhere?
-    #if !@document.nil? && @document.attributes.has_key?('skip-front-matter')
-    #  skip_front_matter result
-    #end
-
-    unless indent.nil?
-      Lexer.reset_block_indent! result, indent.to_i
-    end
-
-    result
-  end
-
-  # Private: Normalize raw input, used for the outermost Reader.
-  #
-  # This method strips whitespace from the end of every line of
-  # the source data and appends a LF (i.e., Unix endline). This
-  # whitespace substitution is very important to how Asciidoctor
-  # works.
-  #
-  # Any leading or trailing blank lines are also removed.
-  #
-  # The normalized lines are assigned to the @lines instance variable.
-  #
-  # data - A String Array of input data to be normalized
-  #
-  # returns nothing
-  def normalize_data(data)
-    # normalize line ending to LF (purging occurrences of CRLF)
-    # this rstrip is *very* important to how Asciidoctor works
-
-    if ::Asciidoctor::FORCE_ENCODING
-      @lines = data.map {|line| "#{line.rstrip.force_encoding(::Encoding::UTF_8)}\n" }
-    else
-      @lines = data.map {|line| "#{line.rstrip}\n" }
-    end
-
-    # QUESTION should this work for AsciiDoc table cell content? If so, it won't hit here
-    if !@document.nil? && @document.attributes.has_key?('skip-front-matter')
-      if (front_matter = skip_front_matter(@lines))
-        @document.attributes['front-matter'] = front_matter.join.chomp
-      end
-    end
-
-    @lines.shift && @lineno += 1 while !@lines.first.nil? && @lines.first.chomp.empty?
-    @lines.pop while !@lines.last.nil? && @lines.last.chomp.empty?
-    nil
+  def to_s
+    %(#{self.class.name} [path: #{@path}, line #: #{@lineno}, include depth: #{@include_stack.size}, include stack: [#{@include_stack.map {|include| include.to_s}.join ', '}]])
   end
 end
+
 end
diff --git a/lib/asciidoctor/table.rb b/lib/asciidoctor/table.rb
index d194fe50..bb14c98a 100644
--- a/lib/asciidoctor/table.rb
+++ b/lib/asciidoctor/table.rb
@@ -210,7 +210,19 @@ class Table::Cell < AbstractNode
       # FIXME hide doctitle from nested document; temporary workaround to fix
       # nested document seeing doctitle and assuming it has its own document title
       parent_doctitle = @document.attributes.delete('doctitle')
-      @inner_document = Document.new(@text, :header_footer => false, :parent => @document)
+      # NOTE we need to process the first line of content as it may not have been processed
+      # the included content cannot expect to match conditional terminators in the remaining
+      # lines of table cell content, it must be self-contained logic
+      inner_document_lines = @text.each_line.to_a
+      unless inner_document_lines.empty? || !inner_document_lines.first.include?('::')
+        unprocessed_lines = inner_document_lines[0..0]
+        processed_lines = PreprocessorReader.new(@document, unprocessed_lines).readlines
+        if processed_lines != unprocessed_lines
+          inner_document_lines.shift
+          inner_document_lines.unshift *processed_lines
+        end
+      end
+      @inner_document = Document.new(inner_document_lines, :header_footer => false, :parent => @document)
       @document.attributes['doctitle'] = parent_doctitle unless parent_doctitle.nil?
     end
   end
@@ -410,6 +422,7 @@ class Table::ParserContext
     if format == 'psv'
       cell_spec = take_cell_spec
       if cell_spec.nil?
+        # FIXME need a reader reference to report line info
         warn 'asciidoctor: ERROR: table missing leading separator, recovering automatically'
         cell_spec = {}
         repeat = 1
diff --git a/test/document_test.rb b/test/document_test.rb
index 148565a5..90a2c981 100644
--- a/test/document_test.rb
+++ b/test/document_test.rb
@@ -1390,4 +1390,28 @@ asciidoctor - converts AsciiDoc source files to HTML, DocBook and other formats
       assert_xpath '//*[@id="content"]/*[@class="sect1"]/h2[text()="SYNOPSIS"]', output, 1
     end
   end
+
+  context 'Secure Asset Path' do
+    test 'allows us to specify a path relative to the current dir' do
+      doc = Asciidoctor::Document.new
+      legit_path = Dir.pwd + '/foo'
+      assert_equal legit_path, doc.normalize_asset_path(legit_path)
+    end
+
+    test 'keeps naughty absolute paths from getting outside' do
+      naughty_path = "#{disk_root}etc/passwd"
+      doc = Asciidoctor::Document.new
+      secure_path = doc.normalize_asset_path(naughty_path)
+      assert naughty_path != secure_path
+      assert_match(/^#{doc.base_dir}/, secure_path)
+    end
+
+    test 'keeps naughty relative paths from getting outside' do
+      naughty_path = 'safe/ok/../../../../../etc/passwd'
+      doc = Asciidoctor::Document.new
+      secure_path = doc.normalize_asset_path(naughty_path)
+      assert naughty_path != secure_path
+      assert_match(/^#{doc.base_dir}/, secure_path)
+    end
+  end
 end
diff --git a/test/fixtures/basic-docinfo.xml b/test/fixtures/basic-docinfo.xml
index df66e54a..6181e12f 100644
--- a/test/fixtures/basic-docinfo.xml
+++ b/test/fixtures/basic-docinfo.xml
@@ -1,4 +1,4 @@
-<copyright>
-<year>2013</year>
-<holder>Acme, Inc.</holder>
+<copyright><!-- don't remove the indent! -->
+    <year>2013</year>
+    <holder>Acme, Inc.</holder>
 </copyright>
diff --git a/test/fixtures/child-include.adoc b/test/fixtures/child-include.adoc
new file mode 100644
index 00000000..d47011d3
--- /dev/null
+++ b/test/fixtures/child-include.adoc
@@ -0,0 +1,3 @@
+first line of child
+
+last line of child
diff --git a/test/fixtures/parent-include.adoc b/test/fixtures/parent-include.adoc
new file mode 100644
index 00000000..45aa82dc
--- /dev/null
+++ b/test/fixtures/parent-include.adoc
@@ -0,0 +1,5 @@
+first line of parent
+
+include::child-include.adoc[]
+
+last line of parent
diff --git a/test/lexer_test.rb b/test/lexer_test.rb
index df78aff1..da2c8159 100644
--- a/test/lexer_test.rb
+++ b/test/lexer_test.rb
@@ -7,6 +7,12 @@ context "Lexer" do
     assert Asciidoctor::Lexer.is_section_title?('=== AsciiDoc Home Page')
   end
 
+  test 'sanitize attribute name' do
+    assert_equal 'foobar', Asciidoctor::Lexer.sanitize_attribute_name("Foo Bar")
+    assert_equal 'foo', Asciidoctor::Lexer.sanitize_attribute_name("foo")
+    assert_equal 'foo3-bar', Asciidoctor::Lexer.sanitize_attribute_name("Foo 3^ # - Bar[")
+  end
+
   test "collect unnamed attribute" do
     attributes = {}
     line = 'quote'
diff --git a/test/lists_test.rb b/test/lists_test.rb
index 199a4603..7631007b 100644
--- a/test/lists_test.rb
+++ b/test/lists_test.rb
@@ -1089,7 +1089,7 @@ Lists
       assert_xpath '//ul/li[1]/*', output, 1
     end
 
-    test "consecutive blocks in list continuation attach to list item" do
+    test 'consecutive blocks in list continuation attach to list item' do
       input = <<-EOS
 Lists
 =====
@@ -1106,7 +1106,7 @@ ____
 +
 * Item two
       EOS
-      output = render_string input
+      output = render_embedded_string input
       assert_xpath '//ul', output, 1
       assert_xpath '//ul/li', output, 2
       assert_xpath '//ul/li[1]/p', output, 1
diff --git a/test/reader_test.rb b/test/reader_test.rb
index 1366c99a..975bfcb2 100644
--- a/test/reader_test.rb
+++ b/test/reader_test.rb
@@ -1,128 +1,257 @@
 require 'test_helper'
 
 class ReaderTest < Test::Unit::TestCase
-  # setup for test
-  def setup
-    @src_data = File.readlines(sample_doc_path(:asciidoc_index))
-    @reader = Asciidoctor::Reader.new @src_data
-  end
+  DIRNAME = File.dirname __FILE__
 
-  context "has_more_lines?" do
-    test "returns false for empty document" do
-      assert !Asciidoctor::Reader.new.has_more_lines?
-    end
+  SAMPLE_DATA = <<-EOS.each_line.to_a
+first line
+second line
+third line
+  EOS
+
+  context 'Reader' do
+    context 'Prepare lines' do
+      test 'should prepare lines from Array data' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert_equal SAMPLE_DATA, reader.lines
+      end
 
-    test "returns true with lines remaining" do
-      assert @reader.has_more_lines?, "Yo, didn't work"
+      test 'should prepare lines from String data' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert_equal SAMPLE_DATA, reader.lines
+      end
     end
-  end
 
-  context "with source data loaded" do
-    test "get_line returns next line" do
-      assert_equal @src_data[0], @reader.get_line
-    end
+    context 'With empty data' do
+      test 'has_more_lines? should return false with empty data' do
+        assert !Asciidoctor::Reader.new.has_more_lines?
+      end
 
-    test "get_line consumes the line it returns" do
-      reader = Asciidoctor::Reader.new(["foo", "bar"])
-      _ = reader.get_line
-      second = reader.get_line
-      assert_equal "bar", second
-    end
+      test 'empty? should return true with empty data' do
+        assert Asciidoctor::Reader.new.empty?
+        assert Asciidoctor::Reader.new.eof?
+      end
 
-    test "peek_line does not consume the line it returns" do
-      reader = Asciidoctor::Reader.new(["foo", "bar"])
-      _ = reader.peek_line
-      second = reader.peek_line
-      assert_equal "foo", second
-    end
+      test 'next_line_empty? should return true with empty data' do
+        assert Asciidoctor::Reader.new.next_line_empty?
+      end
 
-    test "unshift puts line onto Reader instance for the next get_line" do
-      reader = Asciidoctor::Reader.new(["foo"])
-      reader.unshift("bar")
-      assert_equal "bar", reader.get_line
-      assert_equal "foo", reader.get_line
+      test 'peek_line should return nil with empty data' do
+        assert_nil Asciidoctor::Reader.new.peek_line
+      end
+
+      test 'peek_lines should return empty Array with empty data' do
+        assert_equal [], Asciidoctor::Reader.new.peek_lines
+      end
+
+      test 'read_line should return nil with empty data' do
+        assert_nil Asciidoctor::Reader.new.read_line
+        assert_nil Asciidoctor::Reader.new.get_line
+      end
+
+      test 'read_lines should return empty Array with empty data' do
+        assert_equal [], Asciidoctor::Reader.new.read_lines
+        assert_equal [], Asciidoctor::Reader.new.get_lines
+      end
     end
 
-    test 'get lines consumes all remaining lines' do
-      input = <<-EOS
-line 1
-line 2
-line 3
-      EOS
-      input_lines = input.lines.entries
-      reader = Asciidoctor::Reader.new input_lines
-      assert_equal input_lines, reader.lines
-      assert_equal 'line 1', reader.get_line.chomp
-      assert_equal input_lines[1..-1], reader.get_lines
-      assert !reader.has_more_lines?
-      assert_equal 3, reader.lineno
+    context 'With data' do
+      test 'has_more_lines? should return true if there are lines remaining' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert reader.has_more_lines?
+      end
+
+      test 'empty? should return false if there are lines remaining' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert !reader.empty?
+        assert !reader.eof?
+      end
+
+      test 'next_line_empty? should return false if next line is not blank' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert !reader.next_line_empty?
+      end
+
+      test 'next_line_empty? should return true if next line is blank' do
+        reader = Asciidoctor::Reader.new ["\n", "second line\n"]
+        assert reader.next_line_empty?
+      end
+
+      test 'peek_line should return next line if there are lines remaining' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert_equal SAMPLE_DATA.first, reader.peek_line
+      end
+
+      test 'peek_line should not consume line or increment line number' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert_equal SAMPLE_DATA.first, reader.peek_line
+        assert_equal SAMPLE_DATA.first, reader.peek_line
+        assert_equal 1, reader.lineno
+      end
+
+      test 'peek_line should return next lines if there are lines remaining' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert_equal SAMPLE_DATA[0..1], reader.peek_lines(2)
+      end
+
+      test 'peek_lines should not consume lines or increment line number' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert_equal SAMPLE_DATA[0..1], reader.peek_lines(2)
+        assert_equal SAMPLE_DATA[0..1], reader.peek_lines(2)
+        assert_equal 1, reader.lineno
+      end
+
+      test 'peek_lines should not invert order of lines' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert_equal SAMPLE_DATA, reader.lines
+        reader.peek_lines 3
+        assert_equal SAMPLE_DATA, reader.lines
+      end
+
+      test 'read_line should return next line if there are lines remaining' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert_equal SAMPLE_DATA.first, reader.read_line
+      end
+
+      test 'read_line should consume next line and increment line number' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert_equal SAMPLE_DATA[0], reader.read_line
+        assert_equal SAMPLE_DATA[1], reader.read_line
+        assert_equal 3, reader.lineno
+      end
+
+      test 'advance should consume next line and return a Boolean indicating if a line was consumed' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert reader.advance
+        assert reader.advance
+        assert reader.advance
+        assert !reader.advance
+      end
+
+      test 'read_lines should return all lines' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert_equal SAMPLE_DATA, reader.read_lines
+      end
+
+      test 'read should return all lines joined as String' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        assert_equal SAMPLE_DATA.join, reader.read
+      end
+
+      test 'has_more_lines? should return false after read_lines is invoked' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        reader.read_lines
+        assert !reader.has_more_lines?
+      end
+
+      test 'unshift puts line onto Reader as next line to read' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        reader.unshift "line zero\n"
+        assert_equal "line zero\n", reader.peek_line
+        assert_equal "line zero\n", reader.get_line
+        assert_equal 1, reader.lineno
+      end
+
+      test 'terminate should consume all lines and update line number' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        reader.terminate
+        assert reader.eof?
+        assert_equal 4, reader.lineno
+      end
+
+      test 'skip_blank_lines should skip blank lines' do
+        reader = Asciidoctor::Reader.new ["", "\n"].concat(SAMPLE_DATA)
+        reader.skip_blank_lines
+        assert_equal SAMPLE_DATA.first, reader.peek_line
+      end
+
+      test 'lines should return remaining lines' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        reader.read_line
+        assert_equal SAMPLE_DATA[1..-1], reader.lines
+      end
+
+      test 'source_lines should return copy of original data Array' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        reader.read_lines
+        assert_equal SAMPLE_DATA, reader.source_lines 
+      end
+
+      test 'source should return original data Array joined as String' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA
+        reader.read_lines
+        assert_equal SAMPLE_DATA.join, reader.source
+      end
+
     end
 
-    test 'terminate should consume remaining lines' do
-      input = <<-EOS
-line 1
-line 2
-line 3
-      EOS
-      input_lines = input.lines.entries
-      reader = Asciidoctor::Reader.new input_lines
-      assert_equal input_lines, reader.lines
-      assert_equal 'line 1', reader.get_line.chomp
-      reader.terminate
-      assert !reader.has_more_lines?
-      assert_equal 3, reader.lineno
+    context 'Line context' do
+      test 'to_s should return file name and line number of current line' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA, 'sample.ad'
+        reader.read_line
+        assert_equal 'sample.ad: line 2', reader.to_s
+      end
+
+      test 'line_info should return file name and line number of current line' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA, 'sample.ad'
+        reader.read_line
+        assert_equal 'sample.ad: line 2', reader.line_info
+        assert_equal 'sample.ad: line 2', reader.next_line_info
+      end
+
+      test 'prev_line_info should return file name and line number of previous line read' do
+        reader = Asciidoctor::Reader.new SAMPLE_DATA, 'sample.ad'
+        reader.read_line
+        assert_equal 'sample.ad: line 1', reader.prev_line_info
+      end
     end
-  end
 
-  context "Grab lines" do
-    test "Grab until end" do
-      input = <<-EOS
+    context 'Take lines' do
+      test 'Take lines until end' do
+        lines = <<-EOS.each_line.to_a
 This is one paragraph.
 
 This is another paragraph.
-      EOS
-
-      lines = input.lines.entries
-      reader = Asciidoctor::Reader.new(lines)
-      result = reader.grab_lines_until
-      assert_equal 3, result.size
-      assert_equal lines, result
-      assert !reader.has_more_lines?
-      assert reader.empty?
-    end
+        EOS
+
+        reader = Asciidoctor::Reader.new lines
+        result = reader.take_lines_until
+        assert_equal 3, result.size
+        assert_equal lines, result
+        assert !reader.has_more_lines?
+        assert reader.eof?
+      end
 
-    test "Grab until blank line" do
-      input = <<-EOS
+      test 'Take lines until blank line' do
+        lines = <<-EOS.each_line.to_a
 This is one paragraph.
 
 This is another paragraph.
-      EOS
+        EOS
 
-      lines = input.lines.entries
-      reader = Asciidoctor::Reader.new(lines)
-      result = reader.grab_lines_until :break_on_blank_lines => true
-      assert_equal 1, result.size
-      assert_equal lines.first, result.first
-      assert_equal lines.last, reader.peek_line
-    end
+        reader = Asciidoctor::Reader.new lines
+        result = reader.take_lines_until :break_on_blank_lines => true
+        assert_equal 1, result.size
+        assert_equal lines.first.chomp, result.first
+        assert_equal lines.last, reader.peek_line
+      end
 
-    test "Grab until blank line preserving last line" do
-      input = <<-EOS
+      test 'Take lines until blank line preserving last line' do
+        lines = <<-EOS.each_line.to_a
 This is one paragraph.
 
 This is another paragraph.
-      EOS
+        EOS
 
-      lines = input.lines.entries
-      reader = Asciidoctor::Reader.new(lines)
-      result = reader.grab_lines_until :break_on_blank_lines => true, :preserve_last_line => true
-      assert_equal 1, result.size
-      assert_equal lines.first, result.first
-      assert_equal "\n", reader.peek_line
-    end
+        reader = Asciidoctor::Reader.new lines
+        result = reader.take_lines_until :break_on_blank_lines => true, :preserve_last_line => true
+        assert_equal 1, result.size
+        assert_equal lines.first.chomp, result.first
+        assert reader.next_line_empty?
+      end
 
-    test "Grab until condition" do
-      input = <<-EOS
+      test 'Take lines until condition is true' do
+        lines = <<-EOS.each_line.to_a
 --
 This is one paragraph inside the block.
 
@@ -130,19 +259,18 @@ This is another paragraph inside the block.
 --
 
 This is a paragraph outside the block.
-      EOS
-
-      lines = input.lines.entries
-      reader = Asciidoctor::Reader.new(lines)
-      reader.get_line
-      result = reader.grab_lines_until {|line| line.chomp == '--' }
-      assert_equal 3, result.size
-      assert_equal lines[1, 3], result
-      assert_equal "\n", reader.peek_line
-    end
+        EOS
+
+        reader = Asciidoctor::Reader.new lines
+        reader.read_line
+        result = reader.take_lines_until {|line| line.chomp == '--' }
+        assert_equal 3, result.size
+        assert_equal lines[1, 3], result
+        assert reader.next_line_empty?
+      end
 
-    test "Grab until condition with last line" do
-      input = <<-EOS
+      test 'Take lines until condition is true, taking last line' do
+        lines = <<-EOS.each_line.to_a
 --
 This is one paragraph inside the block.
 
@@ -150,19 +278,18 @@ This is another paragraph inside the block.
 --
 
 This is a paragraph outside the block.
-      EOS
-
-      lines = input.lines.entries
-      reader = Asciidoctor::Reader.new(lines)
-      reader.get_line
-      result = reader.grab_lines_until(:grab_last_line => true) {|line| line.chomp == '--' }
-      assert_equal 4, result.size
-      assert_equal lines[1, 4], result
-      assert_equal "\n", reader.peek_line
-    end
+        EOS
+
+        reader = Asciidoctor::Reader.new lines
+        reader.read_line
+        result = reader.take_lines_until(:take_last_line => true) {|line| line.chomp == '--' }
+        assert_equal 4, result.size
+        assert_equal lines[1, 4], result
+        assert reader.next_line_empty?
+      end
 
-    test "Grab until condition with last line and preserving last line" do
-      input = <<-EOS
+      test 'Take lines until condition is true, taking and preserving last line' do
+        lines = <<-EOS.each_line.to_a
 --
 This is one paragraph inside the block.
 
@@ -170,357 +297,530 @@ This is another paragraph inside the block.
 --
 
 This is a paragraph outside the block.
-      EOS
-
-      lines = input.lines.entries
-      reader = Asciidoctor::Reader.new(lines)
-      reader.get_line
-      result = reader.grab_lines_until(:grab_last_line => true, :preserve_last_line => true) {|line| line.chomp == '--' }
-      assert_equal 4, result.size
-      assert_equal lines[1, 4], result
-      assert_equal "--\n", reader.peek_line
+        EOS
+
+        reader = Asciidoctor::Reader.new lines
+        reader.read_line
+        result = reader.take_lines_until(:take_last_line => true, :preserve_last_line => true) {|line| line.chomp == '--' }
+        assert_equal 4, result.size
+        assert_equal lines[1, 4], result
+        assert_equal "--\n", reader.peek_line
+      end
     end
   end
 
-  context 'Include Macro' do
-    test 'include macro is disabled by default and becomes a link' do
-      input = <<-EOS
-include::include-file.asciidoc[]
-      EOS
-      para = block_from_string input, :attributes => { 'include-depth' => 0 }
-      assert_equal 1, para.lines.size
-      #assert_equal 'include::include-file.asciidoc[]', para.source
-      assert_equal 'link:include-file.asciidoc[include-file.asciidoc]', para.source
+  context 'PreprocessorReader' do
+    context 'Type hierarchy' do
+      test 'PreprocessorReader should extend from Reader' do
+        doc = Asciidoctor::Document.new
+        reader = Asciidoctor::PreprocessorReader.new doc
+        assert reader.is_a?(Asciidoctor::Reader)
+      end
+
+      test 'PreprocessorReader should invoke or emulate Reader initializer' do
+        doc = Asciidoctor::Document.new
+        reader = Asciidoctor::PreprocessorReader.new doc, SAMPLE_DATA
+        assert_equal SAMPLE_DATA, reader.lines
+        assert_equal 1, reader.lineno
+      end
     end
 
-    test 'include macro is enabled when safe mode is less than SECURE' do
-      input = <<-EOS
-include::fixtures/include-file.asciidoc[]
-      EOS
+    context 'Prepare lines' do
+      test 'should prepare and normalize lines from Array data' do
+        doc = Asciidoctor::Document.new
+        data = SAMPLE_DATA.map {|line| line.chomp}
+        data.unshift ''
+        data.push ''
+        reader = Asciidoctor::PreprocessorReader.new doc, data
+        assert_equal SAMPLE_DATA, reader.lines
+      end
 
-      doc = document_from_string input, :safe => Asciidoctor::SafeMode::SAFE, :attributes => {'docdir' => File.dirname(__FILE__)}
-      output = doc.render
-      assert_match(/included content/, output)
-    end
+      test 'should prepare and normalize lines from String data' do
+        doc = Asciidoctor::Document.new
+        data = SAMPLE_DATA.map {|line| line.chomp}
+        data.unshift ' '
+        data.push ' '
+        data_as_string = data * "\n"
+        reader = Asciidoctor::PreprocessorReader.new doc, data_as_string
+        assert_equal SAMPLE_DATA, reader.lines
+      end
 
-    test 'missing file referenced by include macro does not crash processor' do
-      input = <<-EOS
-include::fixtures/no-such-file.ad[]
+      test 'should clean CRLF from end of lines' do
+        input = <<-EOS
+source\r
+with\r
+CRLF\r
+endlines\r
       EOS
 
-      begin
-        doc = document_from_string input, :safe => Asciidoctor::SafeMode::SAFE, :attributes => {'docdir' => File.dirname(__FILE__)}
-        assert_equal 0, doc.blocks.size
-      rescue
-        flunk 'include macro should not raise exception on missing file'
+        doc = Asciidoctor::Document.new
+        [input, input.lines, input.split("\n"), input.split("\n").join].each do |lines|
+          reader = Asciidoctor::PreprocessorReader.new doc, lines
+          reader.lines.each do |line|
+            assert !line.end_with?("\r"), "CRLF not properly cleaned for source lines: #{lines.inspect}"
+            assert !line.end_with?("\r\n"), "CRLF not properly cleaned for source lines: #{lines.inspect}"
+            assert line.end_with?("\n"), "CRLF not properly cleaned for source lines: #{lines.inspect}"
+          end
+        end
       end
-    end
 
-    test 'include macro can retrieve data from uri' do
-      input = <<-EOS
-....
-include::https://raw.github.com/asciidoctor/asciidoctor/master/LICENSE[]
-....
-      EOS
+      test 'should not skip front matter by default' do
+        input = <<-EOS
+---
+layout: post
+title: Document Title
+author: username
+tags: [ first, second ]
+---
+= Document Title
+Author Name
+
+preamble
+        EOS
 
-      output = render_embedded_string input, :safe => :safe, :attributes => {'allow-uri-read' => ''}
-      assert_match(/MIT/, output)
+        doc = Asciidoctor::Document.new
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        assert_equal '---', reader.peek_line.chomp
     end
 
-    test 'inaccessible uri referenced by include macro does not crash processor' do
-      input = <<-EOS
-....
-include::http://127.0.0.1:0[]
-....
-      EOS
+    test 'should skip front matter if specified by skip-front-matter attribute' do
+        front_matter = %(layout: post
+title: Document Title
+author: username
+tags: [ first, second ])
+        input = <<-EOS
+---
+#{front_matter}
+---
+= Document Title
+Author Name
 
-      begin
-        output = render_embedded_string input, :safe => :safe, :attributes => {'allow-uri-read' => ''}
-        assert_css 'pre', output, 1
-        assert_css 'pre *', output, 0
-      rescue
-        flunk 'include macro should not raise exception on inaccessible uri'
+preamble
+        EOS
+
+        doc = Asciidoctor::Document.new [], :attributes => {'skip-front-matter' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        assert_equal '= Document Title', reader.peek_line.chomp
+        assert_equal front_matter, doc.attributes['front-matter']
       end
     end
 
-    test 'include macro supports line selection' do
-      input = <<-EOS
-include::fixtures/include-file.asciidoc[lines=1;3..4;6..-1]
-      EOS
+    context 'Include Macro' do
+      test 'include macro is disabled by default and becomes a link' do
+        input = <<-EOS
+include::include-file.asciidoc[]
+        EOS
+        doc = empty_document
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        assert_equal 'link:include-file.asciidoc[]', reader.read_line.chomp
+      end
+  
+      test 'include macro is enabled when safe mode is less than SECURE' do
+        input = <<-EOS
+include::fixtures/include-file.asciidoc[]
+        EOS
+  
+        doc = document_from_string input, :safe => :safe, :header_footer => false, :base_dir => DIRNAME
+        output = doc.render
+        assert_match(/included content/, output)
+      end
 
-      output = render_string input, :safe => Asciidoctor::SafeMode::SAFE, :header_footer => false, :attributes => {'docdir' => File.dirname(__FILE__)}
-      assert_match(/first line/, output)
-      assert_no_match(/second line/, output)
-      assert_match(/third line/, output)
-      assert_match(/fourth line/, output)
-      assert_no_match(/fifth line/, output)
-      assert_match(/sixth line/, output)
-      assert_match(/seventh line/, output)
-      assert_match(/eighth line/, output)
-      assert_match(/last line of included content/, output)
-    end
+      test 'include macro should resolve file relative to current include' do
+        input = <<-EOS
+include::fixtures/parent-include.adoc[]
+        EOS
 
-    test 'include macro supports line selection using quoted attribute value' do
-      input = <<-EOS
-include::fixtures/include-file.asciidoc[lines="1, 3..4 , 6 .. -1"]
-      EOS
+        pseudo_docfile = File.join DIRNAME, 'include-master.adoc'
+        fixtures_dir = File.join DIRNAME, 'fixtures'
+        parent_include_docfile = File.join fixtures_dir, 'parent-include.adoc'
+        child_include_docfile = File.join fixtures_dir, 'child-include.adoc'
 
-      output = render_string input, :safe => Asciidoctor::SafeMode::SAFE, :header_footer => false, :attributes => {'docdir' => File.dirname(__FILE__)}
-      assert_match(/first line/, output)
-      assert_no_match(/second line/, output)
-      assert_match(/third line/, output)
-      assert_match(/fourth line/, output)
-      assert_no_match(/fifth line/, output)
-      assert_match(/sixth line/, output)
-      assert_match(/seventh line/, output)
-      assert_match(/eighth line/, output)
-      assert_match(/last line of included content/, output)
-    end
+        doc = empty_safe_document :base_dir => DIRNAME
+        reader = Asciidoctor::PreprocessorReader.new doc, input, pseudo_docfile
 
-    test 'include macro supports tagged selection' do
-      input = <<-EOS
-include::fixtures/include-file.asciidoc[tags=snippetA;snippetB]
-      EOS
+        assert_equal pseudo_docfile, reader.file
+        assert_equal DIRNAME, reader.dir
+        assert_equal 'include-master.adoc', reader.path
 
-      output = render_string input, :safe => Asciidoctor::SafeMode::SAFE, :header_footer => false, :attributes => {'docdir' => File.dirname(__FILE__)}
-      assert_match(/snippetA content/, output)
-      assert_match(/snippetB content/, output)
-      assert_no_match(/non-tagged content/, output)
-      assert_no_match(/included content/, output)
-    end
+        assert_equal "first line of parent\n", reader.read_line
 
-    test 'lines attribute takes precedence over tags attribute in include macro' do
-      input = <<-EOS
-include::fixtures/include-file.asciidoc[lines=1, tags=snippetA;snippetB]
-      EOS
+        assert_equal 'fixtures/parent-include.adoc: line 1', reader.prev_line_info
+        assert_equal parent_include_docfile, reader.file
+        assert_equal fixtures_dir, reader.dir
+        assert_equal 'fixtures/parent-include.adoc', reader.path
 
-      output = render_string input, :safe => Asciidoctor::SafeMode::SAFE, :header_footer => false, :attributes => {'docdir' => File.dirname(__FILE__)}
-      assert_match(/first line of included content/, output)
-      assert_no_match(/snippetA content/, output)
-      assert_no_match(/snippetB content/, output)
-    end
+        reader.skip_blank_lines
+
+        assert_equal "first line of child\n", reader.read_line
 
-    test 'indent of included file can be reset to size of indent attribute' do
-      input = <<-EOS
+        assert_equal 'fixtures/child-include.adoc: line 1', reader.prev_line_info
+        assert_equal child_include_docfile, reader.file
+        assert_equal fixtures_dir, reader.dir
+        assert_equal 'fixtures/child-include.adoc', reader.path
+
+        reader.skip_blank_lines
+
+        assert_equal "last line of child\n", reader.read_line
+
+        reader.skip_blank_lines
+
+        assert_equal "last line of parent\n", reader.read_line
+
+        assert_equal 'fixtures/parent-include.adoc: line 5', reader.prev_line_info
+        assert_equal parent_include_docfile, reader.file
+        assert_equal fixtures_dir, reader.dir
+        assert_equal 'fixtures/parent-include.adoc', reader.path
+      end
+  
+      test 'missing file referenced by include macro does not crash processor' do
+        input = <<-EOS
+include::fixtures/no-such-file.ad[]
+        EOS
+  
+        begin
+          doc = document_from_string input, :safe => :safe, :base_dir => DIRNAME
+          assert_equal 0, doc.blocks.size
+        rescue
+          flunk 'include macro should not raise exception on missing file'
+        end
+      end
+  
+      test 'include macro can retrieve data from uri' do
+        input = <<-EOS
+....
+include::https://raw.github.com/asciidoctor/asciidoctor/master/LICENSE[]
+....
+        EOS
+  
+        output = render_embedded_string input, :safe => :safe, :attributes => {'allow-uri-read' => ''}
+        assert_match(/MIT/, output)
+      end
+  
+      test 'inaccessible uri referenced by include macro does not crash processor' do
+        input = <<-EOS
+....
+include::http://127.0.0.1:0[]
+....
+        EOS
+  
+        begin
+          output = render_embedded_string input, :safe => :safe, :attributes => {'allow-uri-read' => ''}
+          assert_css 'pre:empty', output, 1
+        rescue
+          flunk 'include macro should not raise exception on inaccessible uri'
+        end
+      end
+  
+      test 'include macro supports line selection' do
+        input = <<-EOS
+include::fixtures/include-file.asciidoc[lines=1;3..4;6..-1]
+        EOS
+  
+        output = render_string input, :safe => :safe, :header_footer => false, :base_dir => DIRNAME
+        assert_match(/first line/, output)
+        assert_no_match(/second line/, output)
+        assert_match(/third line/, output)
+        assert_match(/fourth line/, output)
+        assert_no_match(/fifth line/, output)
+        assert_match(/sixth line/, output)
+        assert_match(/seventh line/, output)
+        assert_match(/eighth line/, output)
+        assert_match(/last line of included content/, output)
+      end
+  
+      test 'include macro supports line selection using quoted attribute value' do
+        input = <<-EOS
+include::fixtures/include-file.asciidoc[lines="1, 3..4 , 6 .. -1"]
+        EOS
+  
+        output = render_string input, :safe => :safe, :header_footer => false, :base_dir => DIRNAME
+        assert_match(/first line/, output)
+        assert_no_match(/second line/, output)
+        assert_match(/third line/, output)
+        assert_match(/fourth line/, output)
+        assert_no_match(/fifth line/, output)
+        assert_match(/sixth line/, output)
+        assert_match(/seventh line/, output)
+        assert_match(/eighth line/, output)
+        assert_match(/last line of included content/, output)
+      end
+  
+      test 'include macro supports tagged selection' do
+        input = <<-EOS
+include::fixtures/include-file.asciidoc[tags=snippetA;snippetB]
+        EOS
+  
+        output = render_string input, :safe => :safe, :header_footer => false, :base_dir => DIRNAME
+        assert_match(/snippetA content/, output)
+        assert_match(/snippetB content/, output)
+        assert_no_match(/non-tagged content/, output)
+        assert_no_match(/included content/, output)
+      end
+  
+      test 'lines attribute takes precedence over tags attribute in include macro' do
+        input = <<-EOS
+include::fixtures/include-file.asciidoc[lines=1, tags=snippetA;snippetB]
+        EOS
+  
+        output = render_string input, :safe => :safe, :header_footer => false, :base_dir => DIRNAME
+        assert_match(/first line of included content/, output)
+        assert_no_match(/snippetA content/, output)
+        assert_no_match(/snippetB content/, output)
+      end
+  
+      test 'indent of included file can be reset to size of indent attribute' do
+        input = <<-EOS
 [source, xml]
 ----
 include::fixtures/basic-docinfo.xml[lines=2..3, indent=0]
 ----
-      EOS
-
-      output = render_string input, :safe => Asciidoctor::SafeMode::SAFE, :header_footer => false, :attributes => {'docdir' => File.dirname(__FILE__)}
-      result = xmlnodes_at_xpath('//pre', output, 1).text
-      assert_equal "<year>2013</year>\n<holder>Acme, Inc.</holder>", result
-    end
-
-    test "block is called to handle an include macro" do
-      input = <<-EOS
+        EOS
+  
+        output = render_string input, :safe => :safe, :header_footer => false, :base_dir => DIRNAME
+        result = xmlnodes_at_xpath('//pre', output, 1).text
+        assert_equal "<year>2013</year>\n<holder>Acme, Inc.</holder>", result
+      end
+  
+=begin
+      test 'block is called to handle an include macro' do
+        input = <<-EOS
 first line
 
 include::include-file.asciidoc[]
 
 last line
-      EOS
-      document = Asciidoctor::Document.new [], :safe => Asciidoctor::SafeMode::SAFE
-      reader = Asciidoctor::Reader.new(input.lines.entries, document, true) {|inc, doc|
-        ":includefile: #{inc}\n\nmiddle line".lines.entries
-      }
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
-      end
-      assert_match(/^:includefile: include-file.asciidoc$/, lines.join)
-    end
-
-    test 'attributes are substituted in target of include macro' do
-      input = <<-EOS
+        EOS
+        document = Asciidoctor::Document.new [], :safe => Asciidoctor::SafeMode::SAFE
+        reader = Asciidoctor::Reader.new(input.lines.entries, document, true) {|inc, doc|
+          ":includefile: #{inc}\n\nmiddle line".lines.entries
+        }
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.get_line
+        end
+        assert_match(/^:includefile: include-file.asciidoc$/, lines.join)
+      end
+=end
+  
+      test 'attributes are substituted in target of include macro' do
+        input = <<-EOS
 :fixturesdir: fixtures
 :ext: asciidoc
 
 include::{fixturesdir}/include-file.{ext}[]
-      EOS
-
-      doc = document_from_string input, :safe => Asciidoctor::SafeMode::SAFE, :attributes => {'docdir' => File.dirname(__FILE__)}
-      output = doc.render
-      assert_match(/included content/, output)
-    end
-
-    test 'line is dropped if target of include macro resolves to empty' do
-      input = <<-EOS
+        EOS
+  
+        doc = document_from_string input, :safe => :safe, :base_dir => DIRNAME
+        output = doc.render
+        assert_match(/included content/, output)
+      end
+  
+      test 'line is skipped by default if target of include macro resolves to empty' do
+        input = <<-EOS
 include::{foodir}/include-file.asciidoc[]
-      EOS
-
-      output = render_embedded_string input, :safe => Asciidoctor::SafeMode::SAFE, :attributes => {'docdir' => File.dirname(__FILE__)}
-      assert output.strip.empty?
-    end
+        EOS
+  
+        doc = empty_safe_document :base_dir => DIRNAME
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        assert_equal "include::{foodir}/include-file.asciidoc[]\n", reader.read_line
+      end
 
-    test 'line is dropped but not following line if target of include macro resolves to empty' do
-      input = <<-EOS
+      test 'line is dropped if target of include macro resolves to empty and attribute-missing attribute is not skip' do
+        input = <<-EOS
+include::{foodir}/include-file.asciidoc[]
+        EOS
+  
+        doc = empty_safe_document :base_dir => DIRNAME, :attributes => {'attribute-missing' => 'drop'}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        assert_nil reader.read_line
+      end
+  
+      test 'line following dropped include is not dropped' do
+        input = <<-EOS
 include::{foodir}/include-file.asciidoc[]
 yo
-      EOS
-
-      output = render_embedded_string input, :safe => Asciidoctor::SafeMode::SAFE, :attributes => {'docdir' => File.dirname(__FILE__)}
-      assert_xpath '//p', output, 1
-      assert_xpath '//p[text()="yo"]', output, 1
-    end
-
-    test 'escaped include macro is left unprocessed' do
-      input = <<-EOS
-\\include::include-file.asciidoc[]
-      EOS
-      para = block_from_string input
-      assert_equal 1, para.lines.size
-      assert_equal 'include::include-file.asciidoc[]', para.source
-    end
-
-    test 'include macro not at start of line is ignored' do
-      input = <<-EOS
+        EOS
+  
+        doc = empty_safe_document :base_dir => DIRNAME, :attributes => {'attribute-missing' => 'drop'}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        assert_equal "yo\n", reader.read_line
+      end
+  
+      test 'escaped include macro is left unprocessed' do
+        input = <<-EOS
+\\include::fixtures/include-file.asciidoc[]
+\\escape preserved here
+        EOS
+        doc = empty_safe_document :base_dir => DIRNAME
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        # we should be able to peek it multiple times and still have the backslash preserved
+        # this is the test for @unescape_next_line
+        assert_equal 'include::fixtures/include-file.asciidoc[]', reader.peek_line.chomp
+        assert_equal 'include::fixtures/include-file.asciidoc[]', reader.peek_line.chomp
+        assert_equal 'include::fixtures/include-file.asciidoc[]', reader.read_line.chomp
+        assert_equal '\\escape preserved here', reader.read_line.chomp
+      end
+  
+      test 'include macro not at start of line is ignored' do
+        input = <<-EOS
  include::include-file.asciidoc[]
-      EOS
-      para = block_from_string input
-      assert_equal 1, para.lines.size
-      # NOTE the space gets stripped because the line is treated as an inline literal
-      assert_equal :literal, para.context
-      assert_equal 'include::include-file.asciidoc[]', para.source
-    end
-
-    test 'include macro is disabled when include-depth attribute is 0' do
-      input = <<-EOS
+        EOS
+        para = block_from_string input
+        assert_equal 1, para.lines.size
+        # NOTE the space gets stripped because the line is treated as an inline literal
+        assert_equal :literal, para.context
+        assert_equal 'include::include-file.asciidoc[]', para.source
+      end
+  
+      test 'include macro is disabled when include-depth attribute is 0' do
+        input = <<-EOS
 include::include-file.asciidoc[]
-      EOS
-      para = block_from_string input, :safe => Asciidoctor::SafeMode::SAFE, :attributes => { 'include-depth' => 0 }
-      assert_equal 1, para.lines.size
-      assert_equal 'include::include-file.asciidoc[]', para.source
-    end
-
-    test 'include-depth cannot be set by document' do
-      input = <<-EOS
+        EOS
+        para = block_from_string input, :safe => :safe, :attributes => { 'include-depth' => 0 }
+        assert_equal 1, para.lines.size
+        assert_equal 'include::include-file.asciidoc[]', para.source
+      end
+  
+      test 'include-depth cannot be set by document' do
+        input = <<-EOS
 :include-depth: 1
-
+ 
 include::include-file.asciidoc[]
-      EOS
-      para = block_from_string input, :safe => Asciidoctor::SafeMode::SAFE, :attributes => { 'include-depth' => 0 }
-      assert_equal 1, para.lines.size
-      assert_equal 'include::include-file.asciidoc[]', para.source
-    end
-  end
+        EOS
+        para = block_from_string input, :safe => :safe, :attributes => { 'include-depth' => 0 }
+        assert_equal 1, para.lines.size
+        assert_equal 'include::include-file.asciidoc[]', para.source
+      end
 
-  context 'build secure asset path' do
-    test 'allows us to specify a path relative to the current dir' do
-      doc = Asciidoctor::Document.new
-      Asciidoctor::Reader.new([], doc, true)
-      legit_path = Dir.pwd + "/foo"
-      assert_equal legit_path, doc.normalize_asset_path(legit_path)
-    end
+      test 'include macro should be disabled if max include depth has been exceeded' do
+        input = <<-EOS
+include::fixtures/parent-include.adoc[depth=1]
+        EOS
 
-    test "keeps naughty absolute paths from getting outside" do
-      naughty_path = "#{disk_root}etc/passwd"
-      doc = Asciidoctor::Document.new
-      Asciidoctor::Reader.new([], doc, true)
-      secure_path = doc.normalize_asset_path(naughty_path)
-      assert naughty_path != secure_path
-      assert_match(/^#{doc.base_dir}/, secure_path)
-    end
+        pseudo_docfile = File.join DIRNAME, 'include-master.adoc'
+
+        doc = empty_safe_document :base_dir => DIRNAME
+        reader = Asciidoctor::PreprocessorReader.new doc, input, pseudo_docfile
 
-    test "keeps naughty relative paths from getting outside" do
-      naughty_path = "safe/ok/../../../../../etc/passwd"
-      doc = Asciidoctor::Document.new
-      Asciidoctor::Reader.new([], doc, true)
-      secure_path = doc.normalize_asset_path(naughty_path)
-      assert naughty_path != secure_path
-      assert_match(/^#{doc.base_dir}/, secure_path)
+        lines = reader.readlines
+        assert lines.include?("include::child-include.adoc[]\n")
+      end
     end
-  end
 
-  context 'Conditional Inclusions' do
-    test 'preprocess_next_line returns true if cursor advanced' do
-      input = <<-EOS
+    context 'Conditional Inclusions' do
+      test 'preprocess_line returns nil if cursor advanced' do
+        input = <<-EOS
 ifdef::asciidoctor[]
 Asciidoctor!
 endif::asciidoctor[]
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      assert reader.preprocess_next_line == true
-    end
+        EOS
+  
+        reader = Asciidoctor::PreprocessorReader.new empty_document, input
+        assert_nil reader.process_line(reader.lines.first)
+      end
 
-    test 'preprocess_next_line returns false if cursor not advanced' do
-      input = <<-EOS
+      test 'peek_line advances cursor to next conditional line of content' do
+        input = <<-EOS
+ifdef::asciidoctor[]
+Asciidoctor!
+endif::asciidoctor[]
+        EOS
+  
+        reader = Asciidoctor::PreprocessorReader.new empty_document, input
+        assert_equal 1, reader.lineno
+        assert_equal "Asciidoctor!\n", reader.peek_line
+        assert_equal 2, reader.lineno
+      end
+  
+      test 'process_line returns line if cursor not advanced' do
+        input = <<-EOS
 content
 ifdef::asciidoctor[]
 Asciidoctor!
 endif::asciidoctor[]
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      assert reader.preprocess_next_line == false
-    end
+        EOS
+  
+        reader = Asciidoctor::PreprocessorReader.new empty_document, input
+        assert_not_nil reader.process_line(reader.lines.first)
+      end
 
-    test 'preprocess_next_line returns nil if cursor advanced past end of source' do
-      input = <<-EOS
+      test 'peek_line does not advance cursor when on a regular content line' do
+        input = <<-EOS
+content
+ifdef::asciidoctor[]
+Asciidoctor!
+endif::asciidoctor[]
+        EOS
+  
+        reader = Asciidoctor::PreprocessorReader.new empty_document, input
+        assert_equal 1, reader.lineno
+        assert_equal "content\n", reader.peek_line
+        assert_equal 1, reader.lineno
+      end
+  
+      test 'peek_line returns nil if cursor advances past end of source' do
+        input = <<-EOS
 ifdef::foobar[]
 swallowed content
 endif::foobar[]
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      assert reader.preprocess_next_line.nil?
-    end
-
-    test 'ifdef with defined attribute includes block' do
-      input = <<-EOS
+        EOS
+  
+        reader = Asciidoctor::PreprocessorReader.new empty_document, input
+        assert_equal 1, reader.lineno
+        assert_nil reader.peek_line
+        assert_equal 4, reader.lineno
+      end
+  
+      test 'ifdef with defined attribute includes content' do
+        input = <<-EOS
 ifdef::holygrail[]
 There is a holy grail!
 endif::holygrail[]
-      EOS
-       
-      doc = Asciidoctor::Document.new [], :attributes => {'holygrail' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
-      end
-      assert_equal 'There is a holy grail!', lines.join.strip
-    end
-
-    test 'ifdef with defined attribute includes text in brackets' do
-      input = <<-EOS
+        EOS
+         
+        doc = empty_document :attributes => {'holygrail' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'There is a holy grail!', lines.join.chomp
+      end
+  
+      test 'ifdef with defined attribute includes text in brackets' do
+        input = <<-EOS
 On our quest we go...
 ifdef::holygrail[There is a holy grail!]
 There was much rejoicing.
-      EOS
-       
-      doc = Asciidoctor::Document.new [], :attributes => {'holygrail' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
-      end
-      assert_equal "On our quest we go...\nThere is a holy grail!\nThere was much rejoicing.", lines.join.strip
-    end
-
-    test 'ifndef with defined attribute does not include text in brackets' do
-      input = <<-EOS
+        EOS
+         
+        doc = empty_document :attributes => {'holygrail' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal "On our quest we go...\nThere is a holy grail!\nThere was much rejoicing.", lines.join.chomp
+      end
+  
+      test 'ifndef with defined attribute does not include text in brackets' do
+        input = <<-EOS
 On our quest we go...
 ifndef::hardships[There is a holy grail!]
 There was no rejoicing.
-      EOS
-       
-      doc = Asciidoctor::Document.new [], :attributes => {'hardships' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
-      end
-      assert_equal "On our quest we go...\nThere was no rejoicing.", lines.join.strip
-    end
-
-    test 'include with non-matching nested exclude' do
-      input = <<-EOS
+        EOS
+         
+        doc = empty_document :attributes => {'hardships' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal "On our quest we go...\nThere was no rejoicing.", lines.join.chomp
+      end
+  
+      test 'include with non-matching nested exclude' do
+        input = <<-EOS
 ifdef::grail[]
 holy
 ifdef::swallow[]
@@ -528,37 +828,37 @@ swallow
 endif::swallow[]
 grail
 endif::grail[]
-      EOS
-       
-      doc = Asciidoctor::Document.new [], :attributes => {'grail' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
-      end
-      assert_equal "holy\ngrail", lines.join.strip
-    end
-
-    test 'nested excludes with same condition' do
-      input = <<-EOS
+        EOS
+         
+        doc = empty_document :attributes => {'grail' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal "holy\ngrail", lines.join.chomp
+      end
+  
+      test 'nested excludes with same condition' do
+        input = <<-EOS
 ifndef::grail[]
 ifndef::grail[]
 not here
 endif::grail[]
 endif::grail[]
-      EOS
-       
-      doc = Asciidoctor::Document.new [], :attributes => {'grail' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
-      end
-      assert_equal '', lines.join.strip
-    end
-
-    test 'include with nested exclude of inverted condition' do
-      input = <<-EOS
+        EOS
+         
+        doc = empty_document :attributes => {'grail' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal '', lines.join.chomp
+      end
+  
+      test 'include with nested exclude of inverted condition' do
+        input = <<-EOS
 ifdef::grail[]
 holy
 ifndef::grail[]
@@ -566,19 +866,19 @@ not here
 endif::grail[]
 grail
 endif::grail[]
-      EOS
-       
-      doc = Asciidoctor::Document.new [], :attributes => {'grail' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
-      end
-      assert_equal "holy\ngrail", lines.join.strip
-    end
-
-    test 'exclude with matching nested exclude' do
-      input = <<-EOS
+        EOS
+         
+        doc = empty_document :attributes => {'grail' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal "holy\ngrail", lines.join.chomp
+      end
+  
+      test 'exclude with matching nested exclude' do
+        input = <<-EOS
 poof
 ifdef::swallow[]
 no
@@ -588,19 +888,19 @@ endif::swallow[]
 here
 endif::swallow[]
 gone
-      EOS
-       
-      doc = Asciidoctor::Document.new [], :attributes => {'grail' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
-      end
-      assert_equal "poof\ngone", lines.join.strip
-    end
-
-    test 'exclude with nested include using shorthand end' do
-      input = <<-EOS
+        EOS
+         
+        doc = empty_document :attributes => {'grail' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal "poof\ngone", lines.join.chomp
+      end
+  
+      test 'exclude with nested include using shorthand end' do
+        input = <<-EOS
 poof
 ifndef::grail[]
 no grail
@@ -610,364 +910,303 @@ endif::[]
 in here
 endif::[]
 gone
-      EOS
-       
-      doc = Asciidoctor::Document.new [], :attributes => {'grail' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
-      end
-      assert_equal "poof\ngone", lines.join.strip
-    end
-
-    test 'ifdef with one alternative attribute set includes content' do
-      input = <<-EOS
+        EOS
+         
+        doc = empty_document :attributes => {'grail' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal "poof\ngone", lines.join.chomp
+      end
+  
+      test 'ifdef with one alternative attribute set includes content' do
+        input = <<-EOS
 ifdef::holygrail,swallow[]
 Our quest is complete!
 endif::holygrail,swallow[]
-      EOS
-
-      doc = Asciidoctor::Document.new [], :attributes => {'swallow' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document :attributes => {'swallow' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'Our quest is complete!', lines.join.chomp
       end
-      assert_equal 'Our quest is complete!', lines.join.strip
-    end
-
-    test 'ifdef with no alternative attributes set does not include content' do
-      input = <<-EOS
+  
+      test 'ifdef with no alternative attributes set does not include content' do
+        input = <<-EOS
 ifdef::holygrail,swallow[]
 Our quest is complete!
 endif::holygrail,swallow[]
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal '', lines.join.chomp
       end
-      assert_equal '', lines.join.strip
-    end
-
-    test 'ifdef with all required attributes set includes content' do
-      input = <<-EOS
+  
+      test 'ifdef with all required attributes set includes content' do
+        input = <<-EOS
 ifdef::holygrail+swallow[]
 Our quest is complete!
 endif::holygrail+swallow[]
-      EOS
-
-      doc = Asciidoctor::Document.new [], :attributes => {'holygrail' => '', 'swallow' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document :attributes => {'holygrail' => '', 'swallow' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'Our quest is complete!', lines.join.chomp
       end
-      assert_equal 'Our quest is complete!', lines.join.strip
-    end
-
-    test 'ifdef with missing required attributes does not include content' do
-      input = <<-EOS
+  
+      test 'ifdef with missing required attributes does not include content' do
+        input = <<-EOS
 ifdef::holygrail+swallow[]
 Our quest is complete!
 endif::holygrail+swallow[]
-      EOS
-
-      doc = Asciidoctor::Document.new [], :attributes => {'holygrail' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document :attributes => {'holygrail' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal '', lines.join.chomp
       end
-      assert_equal '', lines.join.strip
-    end
-
-    test 'ifndef with undefined attribute includes block' do
-      input = <<-EOS
+  
+      test 'ifndef with undefined attribute includes block' do
+        input = <<-EOS
 ifndef::holygrail[]
 Our quest continues to find the holy grail!
 endif::holygrail[]
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'Our quest continues to find the holy grail!', lines.join.chomp
       end
-      assert_equal 'Our quest continues to find the holy grail!', lines.join.strip
-    end
-
-    test 'ifndef with one alternative attribute set includes content' do
-      input = <<-EOS
+  
+      test 'ifndef with one alternative attribute set includes content' do
+        input = <<-EOS
 ifndef::holygrail,swallow[]
 Our quest is complete!
 endif::holygrail,swallow[]
-      EOS
-
-      doc = Asciidoctor::Document.new [], :attributes => {'swallow' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document :attributes => {'swallow' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'Our quest is complete!', lines.join.chomp
       end
-      assert_equal 'Our quest is complete!', lines.join.strip
-    end
-
-    test 'ifndef with no alternative attributes set includes content' do
-      input = <<-EOS
+  
+      test 'ifndef with no alternative attributes set includes content' do
+        input = <<-EOS
 ifndef::holygrail,swallow[]
 Our quest is complete!
 endif::holygrail,swallow[]
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'Our quest is complete!', lines.join.chomp
       end
-      assert_equal 'Our quest is complete!', lines.join.strip
-    end
-
-    test 'ifndef with any required attributes set does not include content' do
-      input = <<-EOS
+  
+      test 'ifndef with any required attributes set does not include content' do
+        input = <<-EOS
 ifndef::holygrail+swallow[]
 Our quest is complete!
 endif::holygrail+swallow[]
-      EOS
-
-      doc = Asciidoctor::Document.new [], :attributes => {'swallow' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document :attributes => {'swallow' => ''}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal '', lines.join.chomp
       end
-      assert_equal '', lines.join.strip
-    end
-
-    test 'ifndef with no required attributes set includes content' do
-      input = <<-EOS
+  
+      test 'ifndef with no required attributes set includes content' do
+        input = <<-EOS
 ifndef::holygrail+swallow[]
 Our quest is complete!
 endif::holygrail+swallow[]
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'Our quest is complete!', lines.join.chomp
       end
-      assert_equal 'Our quest is complete!', lines.join.strip
-    end
-
-    test 'escaped ifdef is unescaped and ignored' do
-      input = <<-EOS
+  
+      test 'escaped ifdef is unescaped and ignored' do
+        input = <<-EOS
 \\ifdef::holygrail[]
 content
 \\endif::holygrail[]
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal "ifdef::holygrail[]\ncontent\nendif::holygrail[]", lines.join.chomp
       end
-      assert_equal "ifdef::holygrail[]\ncontent\nendif::holygrail[]", lines.join.strip
-    end
-
-    test 'ifeval comparing double-quoted attribute to matching string is included' do
-      input = <<-EOS
+  
+      test 'ifeval comparing double-quoted attribute to matching string is included' do
+        input = <<-EOS
 ifeval::["{gem}" == "asciidoctor"]
 Asciidoctor it is!
 endif::[]
-      EOS
-
-      doc = Asciidoctor::Document.new [], :attributes => {'gem' => 'asciidoctor'}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document :attributes => {'gem' => 'asciidoctor'}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'Asciidoctor it is!', lines.join.chomp
       end
-      assert_equal 'Asciidoctor it is!', lines.join.strip
-    end
-
-    test 'ifeval comparing single-quoted attribute to matching string is included' do
-      input = <<-EOS
+  
+      test 'ifeval comparing single-quoted attribute to matching string is included' do
+        input = <<-EOS
 ifeval::['{gem}' == 'asciidoctor']
 Asciidoctor it is!
 endif::[]
-      EOS
-
-      doc = Asciidoctor::Document.new [], :attributes => {'gem' => 'asciidoctor'}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document :attributes => {'gem' => 'asciidoctor'}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'Asciidoctor it is!', lines.join.chomp
       end
-      assert_equal 'Asciidoctor it is!', lines.join.strip
-    end
-
-    test 'ifeval comparing quoted attribute to non-matching string is ignored' do
-      input = <<-EOS
+  
+      test 'ifeval comparing quoted attribute to non-matching string is ignored' do
+        input = <<-EOS
 ifeval::['{gem}' == 'asciidoctor']
 Asciidoctor it is!
 endif::[]
-      EOS
-
-      doc = Asciidoctor::Document.new [], :attributes => {'gem' => 'tilt'}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document :attributes => {'gem' => 'tilt'}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal '', lines.join.chomp
       end
-      assert_equal '', lines.join.strip
-    end
-
-    test 'ifeval comparing attribute to lower version number is included' do
-      input = <<-EOS
+  
+      test 'ifeval comparing attribute to lower version number is included' do
+        input = <<-EOS
 ifeval::['{asciidoctor-version}' >= '0.1.0']
 That version will do!
 endif::[]
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'That version will do!', lines.join.chomp
       end
-      assert_equal 'That version will do!', lines.join.strip
-    end
-
-    test 'ifeval comparing attribute to self is included' do
-      input = <<-EOS
+  
+      test 'ifeval comparing attribute to self is included' do
+        input = <<-EOS
 ifeval::['{asciidoctor-version}' == '{asciidoctor-version}']
 Of course it's the same!
 endif::[]
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'Of course it\'s the same!', lines.join.chomp
       end
-      assert_equal 'Of course it\'s the same!', lines.join.strip
-    end
-
-    test 'ifeval arguments can be mirrored' do
-      input = <<-EOS
+  
+      test 'ifeval arguments can be transposed' do
+        input = <<-EOS
 ifeval::["0.1.0" <= "{asciidoctor-version}"]
 That version will do!
 endif::[]
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'That version will do!', lines.join.chomp
       end
-      assert_equal 'That version will do!', lines.join.strip
-    end
-
-    test 'ifeval matching numeric comparison is included' do
-      input = <<-EOS
+  
+      test 'ifeval matching numeric comparison is included' do
+        input = <<-EOS
 ifeval::[{rings} == 1]
 One ring to rule them all!
 endif::[]
-      EOS
-
-      doc = Asciidoctor::Document.new [], :attributes => {'rings' => 1}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
+        EOS
+  
+        doc = empty_document :attributes => {'rings' => 1}
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal 'One ring to rule them all!', lines.join.chomp
       end
-      assert_equal 'One ring to rule them all!', lines.join.strip
-    end
-
-    test 'ifdef with no target is ignored' do
-      input = <<-EOS
+  
+      test 'ifdef with no target is ignored' do
+        input = <<-EOS
 ifdef::[]
 content
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      lines = []
-      while reader.has_more_lines?
-        lines << reader.get_line
-      end
-      assert_equal "ifdef::[]\ncontent", lines.join.strip
-    end
-  end
-
-  context 'Text processing' do
-    test 'should clean CRLF from end of lines' do
-      input = <<-EOS
-source\r
-with\r
-CRLF\r
-endlines\r
-      EOS
-
-      reader = Asciidoctor::Reader.new(input.lines.entries, Asciidoctor::Document.new, true)
-      reader.lines.each do |line|
-        assert !line.end_with?("\r\n")
+        EOS
+  
+        doc = empty_document
+        reader = Asciidoctor::PreprocessorReader.new doc, input
+        lines = []
+        while reader.has_more_lines?
+          lines << reader.read_line
+        end
+        assert_equal "ifdef::[]\ncontent", lines.join.chomp
       end
     end
-
-    test 'sanitize attribute name' do
-      assert_equal 'foobar', @reader.sanitize_attribute_name("Foo Bar")
-      assert_equal 'foo', @reader.sanitize_attribute_name("foo")
-      assert_equal 'foo3-bar', @reader.sanitize_attribute_name("Foo 3^ # - Bar[")
-    end
-
-    test 'should not skip front matter by default' do
-      input = <<-EOS
----
-layout: post
-title: Document Title
-author: username
-tags: [ first, second ]
----
-= Document Title
-Author Name
-
-preamble
-      EOS
-
-      doc = Asciidoctor::Document.new
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      assert_equal '---', reader.peek_line.rstrip
-    end
-
-    test 'should skip front matter if specified by skip-front-matter attribute' do
-      front_matter = %(layout: post
-title: Document Title
-author: username
-tags: [ first, second ])
-      input = <<-EOS
----
-#{front_matter}
----
-= Document Title
-Author Name
-
-preamble
-      EOS
-
-      doc = Asciidoctor::Document.new nil, :attributes => {'skip-front-matter' => ''}
-      reader = Asciidoctor::Reader.new(input.lines.entries, doc, true)
-      assert_equal '= Document Title', reader.peek_line.rstrip
-      assert_equal front_matter, doc.attributes['front-matter']
-    end
   end
 end
diff --git a/test/tables_test.rb b/test/tables_test.rb
index 935f845d..8dcebe8a 100644
--- a/test/tables_test.rb
+++ b/test/tables_test.rb
@@ -556,7 +556,7 @@ a|include::fixtures/include-file.asciidoc[]
 |===
       EOS
 
-      output = render_embedded_string input, :safe => Asciidoctor::SafeMode::SAFE, :base_dir => File.dirname(__FILE__)
+      output = render_embedded_string input, :safe => :safe, :base_dir => File.dirname(__FILE__)
       assert_match(/included content/, output)
     end
 
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 8c658e9e..cba58e5d 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -22,6 +22,15 @@ class Test::Unit::TestCase
     "#{windows? ? File.expand_path(__FILE__).split('/').first : nil}/"
   end
 
+  def empty_document options = {}
+    Asciidoctor::Document.new [], options
+  end
+
+  def empty_safe_document options = {}
+    options[:safe] = :safe
+    Asciidoctor::Document.new [], options
+  end
+
   def sample_doc_path(name)
     name = name.to_s
     unless name.include?('.')
diff --git a/test/text_test.rb b/test/text_test.rb
index d5c13dbe..22e1825e 100644
--- a/test/text_test.rb
+++ b/test/text_test.rb
@@ -27,22 +27,22 @@ context "Text" do
   end
 
   # NOTE this test ensures we have the encoding line on block templates too
-  test "proper encoding to handle utf8 characters in arbitrary block" do
+  test 'proper encoding to handle utf8 characters in arbitrary block' do
     input = []
     input << "[verse]\n"
     input.concat(File.readlines(sample_doc_path(:encoding)))
-    doc = Asciidoctor::Document.new
-    reader = Asciidoctor::Reader.new(input, doc, true)
+    doc = empty_document
+    reader = Asciidoctor::PreprocessorReader.new doc, input
     block = Asciidoctor::Lexer.next_block(reader, doc)
     assert_xpath '//pre', block.render.gsub(/^\s*\n/, ''), 1
   end
 
-  test "proper encoding to handle utf8 characters from included file" do
+  test 'proper encoding to handle utf8 characters from included file' do
     input = <<-EOS
 include::fixtures/encoding.asciidoc[tags=romé]
     EOS
-    doc = Asciidoctor::Document.new [], :safe => Asciidoctor::SafeMode::SAFE, :base_dir => File.expand_path(File.dirname(__FILE__))
-    reader = Asciidoctor::Reader.new(input, doc, true)
+    doc = empty_safe_document :base_dir => File.expand_path(File.dirname(__FILE__))
+    reader = Asciidoctor::PreprocessorReader.new doc, input
     block = Asciidoctor::Lexer.next_block(reader, doc)
     output = block.render
     assert_css '.paragraph', output, 1