blocking test

Author: ph

lib/logstash/outputs/file.rb

@@ -42,13 +42,27 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
  # Gzip the output stream before writing to disk.
  config :gzip, :validate => :boolean, :default => false
 
+ # If the event try to create a file outside of the extracted file root
+ # we will tag the message and save it to @filepath_error.
+ config :tag_on_failure, :validate => :array, :default => ["_filepath_failure"]
+
+ # If the generated path is invalid, the events will be saved
+ # into this file and inside the defined path.
+ config :filename_failure, :validate => :string, :default => '_filepath_failures'
+
  public
  def register
  require "fileutils" # For mkdir_p
 
  workers_not_supported
 
  @files = {}
+
+ if interpolated_path?
+ @file_root = extract_file_root
+ @failure_path = File.join(@file_root, @filename_failure)
+ end
+
  now = Time.now
  @last_flush_cycle = now
  @last_stale_cleanup_cycle = now
@@ -60,23 +74,61 @@ def register
  def receive(event)
  return unless output?(event)
 
- path = event.sprintf(@path)
- fd = open(path)
+ log_path = generate_filepath(event)
 
- # TODO(sissel): Check if we should rotate the file.
+ if interpolated_path? && !inside_file_root?(log_path)
+ tag_as_filepath_failure(event)
+ log_path = @failure_path
+ end
 
- if @message_format
- output = event.sprintf(@message_format)
- else
- output = event.to_json
+ output = format_message(event)
+ log_event(log_path, output)
+ end # def receive
+
+ def tag_as_filepath_failure(event)
+ event["tags"] ||= []
+ @tag_on_failure.each do |tag|
+ event["tags"] << tags unless event["tags"].include?(tag)
  end
+ end
+ 
+ def inside_file_root?(log_path)
+ target_file = File.expand_path(log_path)
+ return target_file.start_with?("#{@file_root.to_s}/")
+ end
+
+ def log_event(log_path, event)
+ @logger.debug("File, writing event to file.", :filename => log_path)
+ fd = open(log_path)
 
- fd.write(output)
+ # TODO(sissel): Check if we should rotate the file.
+
+ fd.write(event)
  fd.write("\n")
 
  flush(fd)
  close_stale_files
- end # def receive
+ end
+
+ def generate_filepath(event)
+ event.sprintf(@path)
+ end
+
+ def interpolated_path?
+ path =~ /%\{[^}]+\}/
+ end
+
+ def format_message(event)
+ if @message_format
+ event.sprintf(@message_format)
+ else
+ event.to_json
+ end
+ end
+ def extract_file_root
+ extracted_path = File.expand_path(path.gsub(/%{.+/, ''))
+ Pathname.new(extracted_path).expand_path
+ end
 
  def teardown
  @logger.debug("Teardown: closing files")
@@ -85,7 +137,7 @@ def teardown
  fd.close
  @logger.debug("Closed file #{path}", :fd => fd)
  rescue Exception => e
- @logger.error("Excpetion while flushing and closing files.", :exception => e)
+ @logger.error("Exception while flushing and closing files.", :exception => e)
  end
  end
  finished

spec/outputs/file_spec.rb

@@ -1,16 +1,16 @@
+# encoding: UTF-8
 require "spec_helper"
 require "logstash/outputs/file"
+require "logstash/event"
 require "logstash/json"
-require "tempfile"
+require "stud/temporary"
 
 describe LogStash::Outputs::File do
- 
-
  describe "ship lots of events to a file" do
- event_count = 10000 + rand(500)
- tmp_file = Tempfile.new('logstash-spec-output-file')
+ Stud::Temporary.file('logstash-spec-output-file') do |tmp_file|
+  event_count = 10000 + rand(500)
 
- config <<-CONFIG
+  config <<-CONFIG
  input {
  generator {
  message => "hello world"
@@ -23,51 +23,143 @@
  path => "#{tmp_file.path}"
  }
  }
- CONFIG
-
- agent do
- line_num = 0
- # Now check all events for order and correctness.
- File.foreach(tmp_file) do |line|
- event = LogStash::Event.new(LogStash::Json.load(line))
- insist {event["message"]} == "hello world"
- insist {event["sequence"]} == line_num
- line_num += 1
- end
- insist {line_num} == event_count
- end # agent
+ CONFIG
+
+ agent do
+ line_num = 0
+ # Now check all events for order and correctness.
+ tmp_file.each_line do |line|
+ # File.foreach(tmp_file) do |line|
+ event = LogStash::Event.new(LogStash::Json.load(line))
+ insist {event["message"]} == "hello world"
+ insist {event["sequence"]} == line_num
+ line_num += 1
+ end
+ insist {line_num} == event_count
+ end # agent
+ end
  end
 
  describe "ship lots of events to a file gzipped" do
- event_count = 10000 + rand(500)
- tmp_file = Tempfile.new('logstash-spec-output-file')
+ Stud::Temporary.file('logstash-spec-output-file') do |tmp_file|
+  event_count = 10000 + rand(500)
 
- config <<-CONFIG
- input {
- generator {
- message => "hello world"
- count => #{event_count}
- type => "generator"
+ config <<-CONFIG
+ input {
+ generator {
+ message => "hello world"
+ count => #{event_count}
+ type => "generator"
+ }
  }
- }
- output {
- file {
- path => "#{tmp_file.path}"
- gzip => true
+  output {
+  file {
+  path => "#{tmp_file.path}"
+  gzip => true
+ }
  }
- }
- CONFIG
-
- agent do
- line_num = 0
- # Now check all events for order and correctness.
- Zlib::GzipReader.open(tmp_file.path).each_line do |line|
- event = LogStash::Event.new(LogStash::Json.load(line))
- insist {event["message"]} == "hello world"
- insist {event["sequence"]} == line_num
- line_num += 1
+ CONFIG
+
+ agent do
+ line_num = 0
+ # Now check all events for order and correctness.
+ Zlib::GzipReader.open(tmp_file.path).each_line do |line|
+ event = LogStash::Event.new(LogStash::Json.load(line))
+ insist {event["message"]} == "hello world"
+ insist {event["sequence"]} == line_num
+ line_num += 1
+ end
+ insist {line_num} == event_count
+ end # agent
+ end
+ end
+
+ describe "receiving events" do
+ context "when using an interpolated path" do
+ it 'permits to write inside the file root of the defined path' do
+ event = LogStash::Event.new('@metadata' => { "name" => 'application', 'error' => )
  end
- insist {line_num} == event_count
- end # agent
+ end
  end
+
+ # describe '#generate_filepath' do
+ # let(:event) do
+ # event = LogStash::Event.new
+
+ # event["name"] = "name"
+ # event["type"] = "awesome"
+
+ # event
+ # end
+
+ # it 'uses the event data to generated the path' do
+ # path = '/tmp/%{type}/%{name}'
+
+ # output = LogStash::Outputs::File.new({ "path" => path })
+ 
+ # expect(output.generate_filepath(event)).to eq('/tmp/awesome/name')
+ # end
+
+ # it 'ignores relative path' do
+ # path = '/tmp/%{type}/%{name}/%{relative}/'
+ # event[:relative] = '../aaa/'
+
+ # output = LogStash::Outputs::File.new({ "path" => path })
+ 
+ # expect(output.generate_filepath(event)).to eq('/tmp/awesome/name/relative')
+ # end
+ # end
+
+# describe '#extract_file_root' do
+# context 'with interpolated strings in the path' do
+# it 'extracts the file root from the default path' do
+# path = '/tmp/%{type}/%{name}.txt'
+
+# output = LogStash::Outputs::File.new({ "path" => path })
+# expect(output.extract_file_root().to_s).to eq('/tmp')
+# end
+
+# it 'extracts to the file root down to the last concrete directory' do
+# path = '/tmp/down/%{type}/%{name}.txt'
+
+# output = LogStash::Outputs::File.new({ "path" => path })
+# expect(output.extract_file_root.to_s).to eq('/tmp/down')
+# end
+# end
+ 
+# context "without interpolated strings" do
+# it 'extracts the full path as the file root' do
+# path = '/tmp/down/log.txt'
+
+# output = LogStash::Outputs::File.new({ "path" => path })
+# expect(output.extract_file_root.to_s).to eq(path)
+# end
+# end
+# end
+
+# describe '#inside_file_root?' do
+# context 'when we follow relative paths' do
+# let(:path) { '/tmp/%{type}/%{name}.txt' }
+
+# it 'returns false if the target file is outside the file root' do
+# output = LogStash::Outputs::File.new({ 'path' => path })
+# output.register
+# expect(output.inside_file_root?('/tmp/../etc/eviluser/2004.txt')).to eq(false)
+# end
+
+# it 'returns true if the target file is inside the file root' do
+# output = LogStash::Outputs::File.new({ 'path' => path })
+# output.register
+# expect(output.inside_file_root?('/tmp/not/../etc/eviluser/2004.txt')).to eq(true)
+# end
+
+# it 'returns true if the target file is inside the file root' do
+# Stud::Temporary.file('logstash-spec-output-file') do |tmp_file|
+# output = LogStash::Outputs::File.new({ 'path' => tmp_file.path })
+# output.register
+# expect(output.inside_file_root?(tmp_file.path)).to eq(true)
+# end
+# end
+# end
+# end
 end