adding timezone field, header opc_meta_data property, and workload identity auth method (#19)

Co-authored-by: IMDADMI <[email protected]>

Author: paliwalparitosh

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change Log
 
+## 2.0.7 - 2024-10-10
+### Added
+- Support for timezone override for logs where timezone identifier is missing
+- Support for Workload Identity based authorization
+
 ## 2.0.6 - 2024-02-05
 ### Added
 - Support for endpoint override for instance principal auth mode.

Gemfile

@@ -1,7 +1,5 @@
-## Copyright (c) 2021, 2022  Oracle and/or its affiliates.
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
 ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 source "https://rubygems.org"
-
 gemspec
-

examples/apache.conf

@@ -33,7 +33,7 @@
     <record>
         oci_la_global_metadata ${{<key1>: <value1>, <key2>:<value2>}}
         oci_la_entity_id <LOGGING_ANALYTICS_ENTITY_OCID>              # If same across sources. Else keep this in individual filters
-        oci_la_entity_type <LOGGING_ANALYTICS_ENTITY_TYPE>            # If same across sources. Else keep this in individual filters
+        oci_la_entity_type <LOGGING_ANALYTICS_ENTITY_TYPE>         # If same across sources. Else keep this in individual filters
     </record>
 </filter>
 
@@ -45,21 +45,21 @@
         oci_la_log_source_name <LOGGING_ANALYTICS_SOURCENAME>
         oci_la_log_group_id <LOGGING_ANALYTICS_LOGGROUP_OCID>
         oci_la_log_path "${record['tailed_path']}"
+        oci_la_timezone <TIMEZONE>
         tag ${tag}
     </record>
 </filter>
 
 <match oci.**>
     @type oci-logging-analytics
     namespace                   <YOUR_OCI_TENANCY_NAMESPACE>
-    # Auth config file details
     config_file_location        ~/.oci/config
     profile_name                DEFAULT
     # Buffer Configuration
     <buffer>
           @type	file
-          path	/var/log
+          path  /var/log
           retry_forever	true
           disable_chunk_backup	true
     </buffer>
-</match>
+</match>

fluent-plugin-oci-logging-analytics.gemspec

@@ -1,12 +1,13 @@
-## Copyright (c) 2021, 2022  Oracle and/or its affiliates.
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
 ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+require_relative './lib/fluent/version/version'
 
 lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-oci-logging-analytics"
-  spec.version = "2.0.6"
+  spec.version = Version::VERSION
   spec.authors = ["Oracle","OCI Observability: Logging Analytics"]
   spec.email   = ["[email protected]"]
 

lib/fluent/dto/logEvents.rb

@@ -1,10 +1,10 @@
-## Copyright (c) 2021, 2022  Oracle and/or its affiliates.
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
 ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 class LogEvents
-  attr_accessor :entityId, :entityType, :logSourceName, :logPath, :logRecords , :metadata
+  attr_accessor :entityId, :entityType, :logSourceName, :logPath, :logRecords , :metadata, :timezone
   def initialize(lrpe_key, fluentd_records)
-    @metadata, @entityId, @entityType, @logSourceName, @logPath = lrpe_key
+    @metadata, @entityId, @entityType, @logSourceName, @logPath, @timezone = lrpe_key
     @logRecords = fluentd_records.map{ |record|
         record['message']
     }
@@ -17,8 +17,9 @@ def to_hash
       entityType: @entityType,
       logSourceName: @logSourceName,
       logPath: @logPath,
-      logRecords: @logRecords
+      logRecords: @logRecords,
+      timezone:@timezone
     }.compact
   end
-  
+
 end

lib/fluent/dto/logEventsJson.rb

@@ -1,4 +1,4 @@
-## Copyright (c) 2021, 2022  Oracle and/or its affiliates.
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
 ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 require_relative './logEvents'
@@ -20,5 +20,5 @@ def to_hash
       end
     }.compact
   end
-  
+
 end

lib/fluent/enums/source.rb

@@ -0,0 +1,7 @@
+## Copyright (c) 2024  Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+module Source
+  FLUENTD = :fluentd
+  KUBERNETES_SOLUTION = :kubernetes_solution
+end

lib/fluent/metrics/metricsLabels.rb

@@ -1,5 +1,8 @@
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
 class MetricsLabels
-  attr_accessor :worker_id, :tag, :logGroupId, :logSourceName, :logSet, :invalid_reason, :records_valid, :records_per_tag, :latency
+  attr_accessor :worker_id, :tag, :logGroupId, :logSourceName, :logSet, :invalid_reason, :records_valid, :records_per_tag, :latency,:timezone
   def initialize
       @worker_id = nil
       @tag = nil
@@ -10,5 +13,6 @@ def initialize
       @records_valid = 0
       @records_per_tag = 0
       @latency = 0
+      @timezone = nil
     end
-end
+end

lib/fluent/metrics/prometheusMetrics.rb

@@ -1,3 +1,6 @@
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
 require 'prometheus/client'
 require 'prometheus/client/registry'
 require 'prometheus/client/gauge'
@@ -35,4 +38,4 @@ def registerMetrics
     registry.register(@chunk_time_to_receive) unless registry.exist?('oci_la_fluentd_output_plugin_chunk_time_to_receive')
     registry.register(@chunk_time_to_upload) unless registry.exist?('oci_la_fluentd_output_plugin_chunk_time_to_post')
   end
-end
+end

lib/fluent/plugin/out_oci-logging-analytics.rb

@@ -1,4 +1,4 @@
-## Copyright (c) 2021, 2022  Oracle and/or its affiliates.
+## Copyright (c) 2021, 2024  Oracle and/or its affiliates.
 ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 require 'fluent/plugin/output'
@@ -7,11 +7,13 @@
 require 'yajl'
 require 'yajl/json_gem'
 
+# require 'tzinfo'
 require 'logger'
 require_relative '../dto/logEventsJson'
 require_relative '../dto/logEvents'
 require_relative '../metrics/prometheusMetrics'
 require_relative '../metrics/metricsLabels'
+require_relative '../enums/source'
 
 # Import only specific OCI modules to improve load times and reduce the memory requirements.
 require 'oci/auth/auth'
@@ -36,7 +38,6 @@
 require 'oci/waiter'
 require 'oci/retry/retry'
 require 'oci/object_storage/object_storage'
-
 module OCI
    class << self
      attr_accessor :sdk_name
@@ -97,7 +98,8 @@ class OutOracleOCILogAnalytics < Output
     config_param :zip_file_location,                    :string, :default => nil
     desc 'The kubernetes_metadata_keys_mapping.'
     config_param :kubernetes_metadata_keys_mapping,                    :hash, :default => {"container_name":"Container","namespace_name":"Namespace","pod_name":"Pod","container_image":"Container Image Name","host":"Node"}
-
+    desc 'opc-meta-properties'
+    config_param :collection_source, :string, :default => Source::FLUENTD
 
     #****************************************************************
     desc 'The http proxy to be used.'
@@ -256,6 +258,14 @@ def initialize_loganalytics_client()
             else
               @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config: OCI::Config.new, signer: instance_principals_signer)
             end
+          when "WorkloadIdentity"
+            workload_identity_signer = OCI::Auth::Signers::oke_workload_resource_principal_signer
+            if is_valid(@endpoint)
+              @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config: OCI::Config.new, endpoint: @endpoint, signer: workload_identity_signer)
+              @@logger.info {"loganalytics_client initialised with endpoint: #{@endpoint}"}
+            else
+              @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config: OCI::Config.new, signer: workload_identity_signer)
+            end
           when "ConfigFile"
             my_config = OCI::ConfigFileLoader.load_config(config_file_location: @config_file_location, profile_name: @profile_name)
             if is_valid(@endpoint)
@@ -628,6 +638,8 @@ def group_by_logGroupId(chunk)
          latency = 0
          records_per_tag = 0
 
+
+
          tag_metrics_set = Hash.new
          logGroup_labels_set = Hash.new
 
@@ -637,8 +649,8 @@ def group_by_logGroupId(chunk)
          tags_per_logGroupId = Hash.new
          tag_logSet_map = Hash.new
          tag_metadata_map = Hash.new
+         timezoneValuesByTag = Hash.new
          incoming_records = 0
-
          chunk.each do |time, record|
            incoming_records += 1
            metricsLabels = MetricsLabels.new
@@ -722,6 +734,8 @@ def group_by_logGroupId(chunk)
                     end
                     next
                   end
+
+                  # metricsLabels.timezone = record["oci_la_timezone"]
                   metricsLabels.logGroupId = record["oci_la_log_group_id"]
                   metricsLabels.logSourceName = record["oci_la_log_source_name"]
                   if record["oci_la_log_set"] != nil
@@ -770,6 +784,25 @@ def group_by_logGroupId(chunk)
                       tags_per_logGroupId[record["oci_la_log_group_id"]] = record["tag"]
                     end
                   end
+                  # validating the timezone field
+                   if !timezoneValuesByTag.has_key?(record["tag"])
+                     begin
+                       timezoneIdentifier = record["oci_la_timezone"]
+                       unless is_valid(timezoneIdentifier)
+                         record["oci_la_timezone"] = nil
+                       else
+                         isTimezoneExist = timezone_exist? timezoneIdentifier
+                         unless isTimezoneExist
+                           @@logger.warn { "Invalid timezone '#{timezoneIdentifier}', using default UTC." }
+                           record["oci_la_timezone"] = "UTC"
+                         end
+
+                       end
+                       timezoneValuesByTag[record["tag"]] = record["oci_la_timezone"]
+                     end
+                   else
+                     record["oci_la_timezone"] = timezoneValuesByTag[record["tag"]]
+                   end
 
                   records << record
               ensure
@@ -916,6 +949,14 @@ def write(chunk)
         end
       end
     end
+    def timezone_exist?(tz)
+      begin
+        TZInfo::Timezone.get(tz)
+        return true
+      rescue TZInfo::InvalidTimezoneIdentifier
+        return false
+      end
+    end
 
     # Each oci_la_log_set will correspond to a separate file in the zip
     # Only MAX_FILES_PER_ZIP files are allowed per zip.
@@ -958,6 +999,21 @@ def get_logSets_map_per_logGroupId(oci_la_log_group_id,records_per_logGroupId)
 
     # takes a fluentD chunk and converts it to an in-memory zipfile, populating metrics hash provided
     # Any exception raised is passed into the metrics hash, to be re-thrown from write()
+    def getCollectionSource(input)
+      collections_src = []
+      if !is_valid input
+        collections_src.unshift("source:#{Source::FLUENTD}")
+      else
+        if input == Source::FLUENTD.to_s or input == Source::KUBERNETES_SOLUTION.to_s
+          collections_src.unshift("source:#{input}")
+        else
+          # source not define ! using default source 'fluentd'
+          collections_src.unshift("source:#{Source::FLUENTD}")
+        end
+      end
+      collections_src
+    end
+
     def get_zipped_stream(oci_la_log_group_id,oci_la_global_metadata,records_per_logSet_map)
        begin
         current,  = Time.now
@@ -970,8 +1026,9 @@ def get_zipped_stream(oci_la_log_group_id,oci_la_global_metadata,records_per_log
                   record['oci_la_metadata'],
                   record['oci_la_entity_id'],
                   record['oci_la_entity_type'],
-                  record['oci_la_log_source_name'] ,
-                  record['oci_la_log_path']
+                  record['oci_la_log_source_name'],
+                  record['oci_la_log_path'],
+                  record['oci_la_timezone']
                 ]}.map { |lrpe_key, records_per_lrpe|
                   number_of_records += records_per_lrpe.length
                   LogEvents.new(lrpe_key, records_per_lrpe)
@@ -1021,9 +1078,10 @@ def save_zip_to_local(oci_la_log_group_id, zippedstream, current_s)
     # upload zipped stream to oci
     def upload_to_oci(oci_la_log_group_id, number_of_records, zippedstream, metricsLabels_array)
       begin
+        collection_src_prop = getCollectionSource @collection_source
         error_reason = nil
         error_code = nil
-        opts = {payload_type: "ZIP"}
+        opts = { payload_type: "ZIP", opc_meta_properties:collection_src_prop}
 
             response = @@loganalytics_client.upload_log_events_file(namespace_name=@namespace,
                                             logGroupId=oci_la_log_group_id ,

lib/fluent/version/version.rb

@@ -0,0 +1,8 @@
+## Copyright (c) 2024  Oracle and/or its affiliates.
+## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+# frozen_string_literal: true
+
+module Version
+  VERSION = "2.0.7".freeze
+end