From ec64d639bc2c8ff7f35cdf981983b7a4e229a001 Mon Sep 17 00:00:00 2001 From: Patrick Christopher Date: Wed, 18 Nov 2015 14:15:00 -0800 Subject: [PATCH] initial commit: it works! --- Gemfile | 2 + Gemfile.lock | 117 ++++++++++++++++++++++++ Rakefile | 7 ++ lib/logstash/codecs/.fieldselect.rb.swp | Bin 0 -> 12288 bytes lib/logstash/codecs/fieldselect.rb | 62 +++++++++++++ logstash-codec-fieldselect.gemspec | 30 ++++++ spec/codecs/.fieldselect_spec.rb.swp | Bin 0 -> 12288 bytes spec/codecs/fieldselect_spec.rb | 55 +++++++++++ 8 files changed, 273 insertions(+) create mode 100644 Gemfile create mode 100644 Gemfile.lock create mode 100644 Rakefile create mode 100644 lib/logstash/codecs/.fieldselect.rb.swp create mode 100644 lib/logstash/codecs/fieldselect.rb create mode 100644 logstash-codec-fieldselect.gemspec create mode 100644 spec/codecs/.fieldselect_spec.rb.swp create mode 100644 spec/codecs/fieldselect_spec.rb diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..851fabc --- /dev/null +++ b/Gemfile @@ -0,0 +1,2 @@ +source 'https://rubygems.org' +gemspec diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..a5638b7 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,117 @@ +PATH + remote: . + specs: + logstash-codec-fieldselect (0.0.1) + logstash-core (>= 2.0.0.beta2, < 3.0.0) + logstash-mixin-aws + stud (~> 0.0.22) + +GEM + remote: https://rubygems.org/ + specs: + aws-sdk (2.1.36) + aws-sdk-resources (= 2.1.36) + aws-sdk-core (2.1.36) + jmespath (~> 1.0) + aws-sdk-resources (2.1.36) + aws-sdk-core (= 2.1.36) + aws-sdk-v1 (1.66.0) + json (~> 1.4) + nokogiri (>= 1.4.4) + cabin (0.7.2) + clamp (0.6.5) + coderay (1.1.0) + concurrent-ruby (0.9.2-java) + diff-lcs (1.2.5) + ffi (1.9.10-java) + filesize (0.0.4) + gem_publisher (1.5.0) + gems (0.8.3) + i18n (0.6.9) + insist (1.0.0) + jmespath (1.1.3) + jrjackson (0.3.7) + jruby-openssl (0.9.12-java) + json (1.8.3-java) + kramdown (1.9.0) + logstash-codec-line (2.0.2) + logstash-core (>= 2.0.0.beta2, < 3.0.0) + logstash-codec-plain (2.0.2) + logstash-core (>= 2.0.0.beta2, < 3.0.0) + logstash-core (2.1.0.snapshot2-java) + cabin (~> 0.7.0) + clamp (~> 0.6.5) + concurrent-ruby (= 0.9.2) + filesize (= 0.0.4) + gems (~> 0.8.3) + i18n (= 0.6.9) + jrjackson (~> 0.3.7) + jruby-openssl (>= 0.9.11) + minitar (~> 0.5.4) + pry (~> 0.10.1) + rubyzip (~> 1.1.7) + stud (~> 0.0.19) + thread_safe (~> 0.3.5) + treetop (< 1.5.0) + logstash-devutils (0.0.18-java) + gem_publisher + insist (= 1.0.0) + kramdown + minitar + rake + rspec (~> 3.1.0) + rspec-wait + stud (>= 0.0.20) + logstash-input-generator (2.0.2) + logstash-codec-plain + logstash-core (>= 2.0.0.beta2, < 3.0.0) + logstash-mixin-aws (2.0.2) + aws-sdk (~> 2.1.0) + aws-sdk-v1 (>= 1.61.0) + logstash-codec-plain + logstash-core (>= 2.0.0.beta2, < 3.0.0) + method_source (0.8.2) + minitar (0.5.4) + nokogiri (1.6.6.3-java) + polyglot (0.3.5) + pry (0.10.3-java) + coderay (~> 1.1.0) + method_source (~> 0.8.1) + slop (~> 3.4) + spoon (~> 0.0) + rake (10.4.2) + rspec (3.1.0) + rspec-core (~> 3.1.0) + rspec-expectations (~> 3.1.0) + rspec-mocks (~> 3.1.0) + rspec-core (3.1.7) + rspec-support (~> 3.1.0) + rspec-expectations (3.1.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.1.0) + rspec-mocks (3.1.3) + rspec-support (~> 3.1.0) + rspec-support (3.1.2) + rspec-wait (0.0.8) + rspec (>= 2.11, < 3.5) + rubyzip (1.1.7) + slop (3.6.0) + spoon (0.0.4) + ffi + stud (0.0.22) + thread_safe (0.3.5-java) + treetop (1.4.15) + polyglot + polyglot (>= 0.3.1) + +PLATFORMS + java + +DEPENDENCIES + logstash-codec-fieldselect! + logstash-codec-line + logstash-devutils + logstash-input-generator + +BUNDLED WITH + 1.10.6 diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..4f4b858 --- /dev/null +++ b/Rakefile @@ -0,0 +1,7 @@ +@files=[] + +task :default do + system("rake -T") +end + +require "logstash/devutils/rake" diff --git a/lib/logstash/codecs/.fieldselect.rb.swp b/lib/logstash/codecs/.fieldselect.rb.swp new file mode 100644 index 0000000000000000000000000000000000000000..a988ae192d8cc0fd7f1dc27c3cb5faeba1446969 GIT binary patch literal 12288 zcmeI2yK59d9LGls@f8&X!S0tmArX^{kDzjx_*R5ih*66e!rboUZeeft+?`7ZiQ0&T zrL~R!fLf_wqc%Pe1hufRQ3N{${{p`=vzxnU;-gf4M?QBqGr##gzB3yL`BitEJ9&!s zmIno18-%ES8#(vj+B$J`hY(GveKY;L*Pg*@s86U^r7}9Irj|6nY#a{wNuUOzq?T!! z)%(rJ@1G1+6l5w=z8;9e>in`b_OpSG^>SKWG5{~I0<6H|3bgcO|GqAumb<30=l_OTtQ{a^Xi84&t*S+=0pBq1y})A zfE8c`SOHdm6<`He0akz&U)A7z3xl9PnfA9=B zWmbR{U7wvy4WP+8NVuTB#f{9j95Pa=hJhDu`fex3#fz)!$cCL;aH?X$G2VFC3 zSxLW6K|*tOJhvbO4UuQ812su@c_I&<_8PAyYBLR|kfFfWB7B~^>!Ve!o??xtm@ZOU zkf4qJtIUes$ZYn~g3R1XvQ%a`0aJyFO{cF5lf)BhO2xYD%P8u#U;A={pb*qp2MG2< z_14(fw0$}UR+3gV3emA_70Kh`tc$p9DQ3*kcU*~MHk(*Y_ZDTkG}@xgXEI-H6voO~ z9W-CFvWw)-$DXU1HpgmG?OeoEvolkSiE<}?U8b4R=KRl&9`E0mUl|(#lRw|$5drS#!x~lG#p|QWl#D{$BSk~QRFP>aXE2HFvK1lpFpc#zRF=t_ z>KTWl5}mP~+3qYd(g`Ik-E8T;_H~~Qb)6)}jfTV){J5YKv1};vTx9FTR8%5!g-Qf{ zbpT&93v)a)rI)MN?TM0m%uq~)Y(ud44yuOmHUIzs literal 0 HcmV?d00001 diff --git a/lib/logstash/codecs/fieldselect.rb b/lib/logstash/codecs/fieldselect.rb new file mode 100644 index 0000000..9a807d2 --- /dev/null +++ b/lib/logstash/codecs/fieldselect.rb @@ -0,0 +1,62 @@ +# encoding: utf-8 +require "logstash/codecs/base" +require "logstash/codecs/line" + +class LogStash::Codecs::Fieldselect < LogStash::Codecs::Base + + # This codec will select only the fields listed or default + # to only selecting message + # + # input { + # stdin { codec => fieldselect { fields => [] } + # } + # + # or + # + # output { + # stdout { codec => fieldselect { fields => [] } + # } + config_name "fieldselect" + + # Select fields to output, concated with a comma + config :fields, :validate => :array, :default => ['message'] + + public + def register + @lines = LogStash::Codecs::Line.new + @lines.charset = "UTF-8" + end + + public + def decode(data) + @lines.decode(data) do |line| + selected = field_selector(line) + yield LogStash::Event.new(selected) + end + end # def decode + + public + def encode(event) + encoded = field_concater(event) + @on_event.call(event, encoded) + end # def encode + + private + def field_selector(data) + selected = {} + @fields.each do |field| + selected[field] = data[field] + end + return selected + end + + private + def field_concater(data) + concated = "" + @fields.each do |field| + concated += data[field] + ", " + end + concated.chomp!(", ") + return concated + "\n" + end +end # class LogStash::Codecs::Fieldselect diff --git a/logstash-codec-fieldselect.gemspec b/logstash-codec-fieldselect.gemspec new file mode 100644 index 0000000..294bf76 --- /dev/null +++ b/logstash-codec-fieldselect.gemspec @@ -0,0 +1,30 @@ +Gem::Specification.new do |s| + + s.name = 'logstash-codec-fieldselect' + s.version = '0.0.1' + s.licenses = ['Apache License (2.0)'] + s.summary = "This codec was created to select certain fields for output" + s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" + s.authors = ["Elastic"] + s.email = 'info@elastic.co' + s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" + s.require_paths = ["lib"] + + # Files + s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] + + # Tests + s.test_files = s.files.grep(%r{^(test|spec|features)/}) + + # Special flag to let us know this is actually a logstash plugin + s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } + + # Gem dependencies + s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0" + s.add_runtime_dependency 'logstash-mixin-aws' + s.add_runtime_dependency 'stud', '~> 0.0.22' + s.add_development_dependency 'logstash-devutils' + s.add_development_dependency 'logstash-input-generator' + s.add_development_dependency 'logstash-codec-line' +end + diff --git a/spec/codecs/.fieldselect_spec.rb.swp b/spec/codecs/.fieldselect_spec.rb.swp new file mode 100644 index 0000000000000000000000000000000000000000..f614f495575e7d46f4339767c0763379a1255045 GIT binary patch literal 12288 zcmeI2TZgC9XqeDX2igAXEp0!8t`2txcfBMKHbN8G;eWeM92+I(ZO*s|({sGJ)cj++VK2&K z?xp!en@~^J?5}8ToNyXzPB}6?y}{ln(!IIyp&joQqoupcgb)w{O9{-(c*up}Wv+Xas!-y$`(yoq&Gduhb9F7tp5= zL+j9fNI^fopwxHJJ?LBL8|Z83bLbA#ht{Dr=n!;oAI706M9?8CgI?Rh?l4N08Zju-Z^faHPF_xNHk_#z|DV?_I*O${T-o`!s`ut2#P@b8s*w zcZ8;xacR70KDU4`1=C!c;EG?8B^BX?ZDCcCtPF!Gr)f^t^CFF#!=K-{o9JP9rrn@> z(`HnMc50WXc6UGvzE(NpqiN4S3#r!eDb?gt74(3{VXCW;A5D=wOSi~O8IK)@v(%L7 z;%V2pQYS)vIxGGGZo)ZVoyQ{8%4?rbZI}K2&@Ibmjr~;}*z#I?9BS%+08=(9LX8<} z_kHNJ{m|L_La8&!R?7U*+SiNAMK@s>S&5OdYOU?HEwb;NV5j4As5zjy5?0oY2KIj| zU$ "hello world", "host" => "test", "leggo" => "my eggo"})} + + it "should return only the default select field, message" do + expect(subject).to receive(:on_event).once.and_call_original + subject.on_event do |e, d| + insist {d} == event['message'] + "\n" + end + subject.encode(event) + end + end + + context "#decode" do + it "should return only the default select field, message" do + decoded = false + subject.decode("hello world\n") do |e| + decoded = true + insist { e.is_a?(LogStash::Event) } + insist { e["message"] } == "hello world" + insist { e["host"] } == nil + end + insist { decoded } == true + end + + end + end + + context "select specific fields" do + subject do + next LogStash::Codecs::Fieldselect.new({'fields' => ['message', 'leggo']}) + end + + context "#encode" do + it "should return both selected fields, message and leggo" do + expect(subject).to receive(:on_event).once.and_call_original + subject.on_event do |e, d| + insist {d} == event['message'] + ", " + event['leggo'] + "\n" + end + end + end + end +end +