Merge pull request #476 from mojavelinux/include-uri

resolves #445 allow include macro to retrieve data from uri

9 files changed, 123 insertions, 44 deletions

diff --git a/lib/asciidoctor.rb b/lib/asciidoctor.rb
index bdf04126..f7be118c 100755
--- a/lib/asciidoctor.rb
+++ b/lib/asciidoctor.rb
@@ -546,7 +546,7 @@ module Asciidoctor
     # http://domain
     # https://domain
     # data:info
-    :uri_sniff        => /^[[:alpha:]][[:alnum:].+-]*:/i,
+    :uri_sniff        => %r{\A[[:alpha:]][[:alnum:].+-]*:},
 
     :uri_encode_chars => /[^\w\-.!~*';:@=+$,()\[\]]/
   }
diff --git a/lib/asciidoctor/backends/_stylesheets.rb b/lib/asciidoctor/backends/_stylesheets.rb
index 54af187d..af163d7b 100644
--- a/lib/asciidoctor/backends/_stylesheets.rb
+++ b/lib/asciidoctor/backends/_stylesheets.rb
@@ -4,7 +4,7 @@ module HTML5
   #
   # returns the default CodeRay stylesheet as a String
   def self.default_coderay_stylesheet
-    ::Asciidoctor::Helpers.require_library 'coderay'
+    ::Asciidoctor::Helpers.require_library 'coderay', true
     ::CodeRay::Encoders[:html]::CSS.new(:default).stylesheet
   end
 
diff --git a/lib/asciidoctor/document.rb b/lib/asciidoctor/document.rb
index a18cc603..20bbb90b 100644
--- a/lib/asciidoctor/document.rb
+++ b/lib/asciidoctor/document.rb
@@ -183,6 +183,11 @@ class Document < AbstractBlock
     # 10 is the AsciiDoc default, though currently Asciidoctor only supports 1 level
     @attribute_overrides['include-depth'] ||= 10
 
+    # the only way to enable uri reads is via the document options, disabled by default
+    unless !@attribute_overrides['allow-uri-read'].nil? || @attribute_overrides.has_key?('allow-uri-read!')
+      @attribute_overrides['allow-uri-read'] = nil
+    end
+
     # if the base_dir option is specified, it overrides docdir as the root for relative paths
     # otherwise, the base_dir is the directory of the source file (docdir) or the current
     # directory of the input is a string
diff --git a/lib/asciidoctor/helpers.rb b/lib/asciidoctor/helpers.rb
index 871d2036..b7f843bc 100644
--- a/lib/asciidoctor/helpers.rb
+++ b/lib/asciidoctor/helpers.rb
@@ -7,7 +7,7 @@ module Helpers
   #
   # returns false if the library is detected on the load path or the return
   # value of delegating to Kernel#require
-  def self.require_library(name)
+  def self.require_library(name, gem_name = nil)
     if Thread.list.size > 1
       main_script = "#{name}.rb"
       main_script_path_segment = "/#{name}.rb"
@@ -18,7 +18,17 @@ module Helpers
            "The use of an explicit require '#{name}' statement is recommended."
       end
     end
-    require name
+    begin
+      require name
+    rescue LoadError => e
+      if gem_name
+        puts "asciidoctor: FAILED: required gem '#{gem_name === true ? name : gem_name}' is not installed"
+      else
+        puts "asciidoctor: FAILED: #{e}"
+      end
+      # QUESTION: raise abort error or use exit code return value?
+      raise 'Processing aborted'
+    end
   end
 
   # Public: Encode a string for inclusion in a URI
diff --git a/lib/asciidoctor/reader.rb b/lib/asciidoctor/reader.rb
index cc0c0975..45da05e2 100644
--- a/lib/asciidoctor/reader.rb
+++ b/lib/asciidoctor/reader.rb
@@ -424,6 +424,8 @@ class Reader
   #          target slot of the include::[] macro
   #
   # returns a Boolean indicating whether the line under the cursor has changed.
+  # -
+  # FIXME includes bork line numbers
   def preprocess_include(target, raw_attributes)
     target = @document.sub_attributes target
     if target.empty?
@@ -441,16 +443,33 @@ class Reader
     # if the include-depth attribute is 0
     elsif @include_block
       advance
-      # FIXME this borks line numbers
       @lines.unshift(*normalize_include_data(@include_block.call(target, @document)))
     # FIXME currently we're not checking the upper bound of the include depth
     elsif @document.attributes.fetch('include-depth', 0).to_i > 0
       advance
-      # FIXME this borks line numbers
-      include_file = @document.normalize_system_path(target, nil, nil, :target_name => 'include file')
-      if !File.file?(include_file)
-        puts "asciidoctor: WARNING: line #{@lineno}: include file not found: #{include_file}"
-        return true
+      if target.include?(':') && target.match(REGEXP[:uri_sniff])
+        unless @document.attributes.has_key? 'allow-uri-read'
+          @lines[0] = "link:#{target}[#{target}]\n"
+          @next_line_preprocessed = true
+          return false
+        end
+
+        target_type = :uri
+        include_file = target
+        if @document.attributes.has_key? 'cache-uri'
+          # NOTE caching requires the open-uri-cached gem to be installed
+          Helpers.require_library 'open-uri/cached', 'open-uri-cached'
+        else
+          Helpers.require_library 'open-uri'
+        end
+        # FIXME should stop processing include if required library cannot be loaded; check for -1 return value?
+      else
+        target_type = :file
+        include_file = @document.normalize_system_path(target, nil, nil, :target_name => 'include file')
+        if !File.file?(include_file)
+          puts "asciidoctor: WARNING: line #{@lineno}: include file not found: #{include_file}"
+          return true
+        end
       end
 
       inc_lines = nil
@@ -481,18 +500,24 @@ class Reader
       if !inc_lines.nil?
         if !inc_lines.empty?
           selected = []
-          f = File.new(include_file)
-          f.each_line do |l|
-            take = inc_lines.first
-            if take.is_a?(Float) && take.infinite?
-              selected.push l
-            else
-              if f.lineno == take
-                selected.push l
-                inc_lines.shift 
+          begin
+            open(include_file) do |f|
+              f.each_line do |l|
+                take = inc_lines.first
+                if take.is_a?(Float) && take.infinite?
+                  selected.push l
+                else
+                  if f.lineno == take
+                    selected.push l
+                    inc_lines.shift 
+                  end
+                  break if inc_lines.empty?
+                end
               end
-              break if inc_lines.empty?
             end
+          rescue
+            puts "asciidoctor: WARNING: line #{@lineno}: include #{target_type} not readable: #{include_file}"
+            return true
           end
           @lines.unshift(*normalize_include_data(selected, attributes['indent'])) unless selected.empty?
         end
@@ -500,29 +525,40 @@ class Reader
         if !tags.empty?
           selected = []
           active_tag = nil
-          f = File.new(include_file)
-          f.each_line do |l|
-            l.force_encoding(::Encoding::UTF_8) if ::Asciidoctor::FORCE_ENCODING
-            if !active_tag.nil?
-              if l.include?("end::#{active_tag}[]")
-                active_tag = nil
-              else
-                selected.push "#{l.rstrip}\n"
-              end
-            else
-              tags.each do |tag|
-                if l.include?("tag::#{tag}[]")
-                  active_tag = tag
-                  break
+          begin
+            open(include_file) do |f|
+              f.each_line do |l|
+                l.force_encoding(::Encoding::UTF_8) if ::Asciidoctor::FORCE_ENCODING
+                if !active_tag.nil?
+                  if l.include?("end::#{active_tag}[]")
+                    active_tag = nil
+                  else
+                    selected.push "#{l.rstrip}\n"
+                  end
+                else
+                  tags.each do |tag|
+                    if l.include?("tag::#{tag}[]")
+                      active_tag = tag
+                      break
+                    end
+                  end
                 end
               end
             end
+          rescue
+            puts "asciidoctor: WARNING: line #{@lineno}: include #{target_type} not readable: #{include_file}"
+            return true
           end
           #@lines.unshift(*selected) unless selected.empty?
           @lines.unshift(*normalize_include_data(selected, attributes['indent'])) unless selected.empty?
         end
       else
-        @lines.unshift(*normalize_include_data(File.readlines(include_file), attributes['indent']))
+        begin
+          @lines.unshift(*normalize_include_data(open(include_file) {|f| f.readlines }, attributes['indent']))
+        rescue
+          puts "asciidoctor: WARNING: line #{@lineno}: include #{target_type} not readable: #{include_file}"
+          return true
+        end
       end
       true
     else
diff --git a/lib/asciidoctor/renderer.rb b/lib/asciidoctor/renderer.rb
index 08b0a3c5..c26b559d 100644
--- a/lib/asciidoctor/renderer.rb
+++ b/lib/asciidoctor/renderer.rb
@@ -36,7 +36,7 @@ class Renderer
 
     # If user passed in a template dir, let them override our base templates
     if (template_dirs = options.delete(:template_dirs))
-      Helpers.require_library 'tilt'
+      Helpers.require_library 'tilt', true
 
       if (template_cache = options[:template_cache]) === true
         # FIXME probably want to use our own cache object for more control
@@ -99,7 +99,7 @@ class Renderer
           ext_name = name_parts.last
           if ext_name == 'slim' && !slim_loaded
             # slim doesn't get loaded by Tilt
-            Helpers.require_library 'slim'
+            Helpers.require_library 'slim', true
           end
           next unless Tilt.registered? ext_name
           opts = view_opts[ext_name.to_sym]
@@ -147,11 +147,11 @@ class Renderer
       name = 'erb'
     end
 
-    Helpers.require_library name
-
     if name == 'erb'
+      Helpers.require_library 'erb'
       ::ERB
     elsif name == 'erubis'
+      Helpers.require_library 'erubis', true
       ::Erubis::FastEruby
     end
   end
diff --git a/lib/asciidoctor/substituters.rb b/lib/asciidoctor/substituters.rb
index 59e51a19..0befd26a 100644
--- a/lib/asciidoctor/substituters.rb
+++ b/lib/asciidoctor/substituters.rb
@@ -820,7 +820,7 @@ module Substituters
   # returns the highlighted source code, if a source highlighter is defined
   # on the document, otherwise the unprocessed text
   def highlight_source(source)
-    Helpers.require_library 'coderay'
+    Helpers.require_library 'coderay', true
     ::CodeRay::Duo[attr('language', 'text').to_sym, :html, {
         :css => @document.attributes.fetch('coderay-css', 'class').to_sym,
         :line_numbers => (attr?('linenums') ? @document.attributes.fetch('coderay-linenums-mode', 'table').to_sym : nil),
diff --git a/test/reader_test.rb b/test/reader_test.rb
index 23e2b5c7..d2d006ba 100644
--- a/test/reader_test.rb
+++ b/test/reader_test.rb
@@ -182,7 +182,34 @@ include::fixtures/no-such-file.ad[]
         doc = document_from_string input, :safe => Asciidoctor::SafeMode::SAFE, :attributes => {'docdir' => File.dirname(__FILE__)}
         assert_equal 0, doc.blocks.size
       rescue
-        flunk('include macro should not raise exception on missing file')
+        flunk 'include macro should not raise exception on missing file'
+      end
+    end
+
+    test 'include macro can retrieve data from uri' do
+      input = <<-EOS
+....
+include::https://raw.github.com/asciidoctor/asciidoctor/master/LICENSE[]
+....
+      EOS
+
+      output = render_embedded_string input, :safe => :safe, :attributes => {'allow-uri-read' => ''}
+      assert_match(/MIT/, output)
+    end
+
+    test 'inaccessible uri referenced by include macro does not crash processor' do
+      input = <<-EOS
+....
+include::http://localhost:0[]
+....
+      EOS
+
+      begin
+        output = render_embedded_string input, :safe => :safe, :attributes => {'allow-uri-read' => ''}
+        assert_css 'pre', output, 1
+        assert_css 'pre *', output, 0
+      rescue
+        flunk 'include macro should not raise exception on inaccessible uri'
       end
     end
 
diff --git a/test/test_helper.rb b/test/test_helper.rb
index e9b394f4..d65b6cfb 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -155,10 +155,11 @@ class Test::Unit::TestCase
     opts[:header_footer] = true unless opts.has_key?(:header_footer)
     if opts[:header_footer]
       # don't embed stylesheet unless test requests the default behavior
-      unless opts.has_key?(:linkcss_default)
-        opts[:attributes] ||= {}
-        opts[:attributes]['linkcss'] = nil
+      if opts.has_key? :linkcss_default
         opts.delete(:linkcss_default)
+      else
+        opts[:attributes] ||= {}
+        opts[:attributes]['linkcss'] = ''
       end
     end
     #opts[:template_dir] = File.join(File.dirname(__FILE__), '..', '..', 'asciidoctor-backends', 'slim')