Add plugin system

Author: hyfen

chronicle-etl.gemspec

@@ -49,7 +49,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "thor", "~> 1.2"
   spec.add_dependency "thor-hollaback", "~> 0.2"
   spec.add_dependency "tty-progressbar", "~> 0.17"
+  spec.add_dependency "tty-spinner"
   spec.add_dependency "tty-table", "~> 0.11"
+  spec.add_dependency "tty-prompt", "~> 0.23"
 
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "pry-byebug", "~> 3.9"

lib/chronicle/etl/cli.rb

@@ -5,4 +5,5 @@
 require 'chronicle/etl/cli/subcommand_base'
 require 'chronicle/etl/cli/connectors'
 require 'chronicle/etl/cli/jobs'
+require 'chronicle/etl/cli/plugins'
 require 'chronicle/etl/cli/main'

lib/chronicle/etl/cli/connectors.rb

@@ -8,11 +8,6 @@ class Connectors < SubcommandBase
         default_task 'list'
         namespace :connectors
 
-        desc "install NAME", "Installs connector NAME"
-        def install(name)
-          Chronicle::ETL::Registry.install_connector(name)
-        end
-
         desc "list", "Lists available connectors"
         # Display all available connectors that chronicle-etl has access to
         def list
@@ -44,21 +39,21 @@ def list
         desc "show PHASE IDENTIFIER", "Show information about a connector"
         def show(phase, identifier)
           unless ['extractor', 'transformer', 'loader'].include?(phase)
-            puts "phase argument must be one of: [extractor, transformer, loader]"
-            return
+            Chronicle::ETL::Logger.fatal("Phase argument must be one of: [extractor, transformer, loader]")
+            exit 1
           end
 
           begin
             connector = Chronicle::ETL::Registry.find_by_phase_and_identifier(phase.to_sym, identifier)
-          rescue Chronicle::ETL::ConnectorNotAvailableError
-            puts "Could not find #{phase} #{identifier}"
-            return
+          rescue Chronicle::ETL::ConnectorNotAvailableError, Chronicle::ETL::PluginError
+            Chronicle::ETL::Logger.fatal("Could not find #{phase} #{identifier}")
+            exit 1
           end
 
           puts connector.klass.to_s.bold
           puts "  #{connector.descriptive_phrase}"
           puts
-          puts "OPTIONS"
+          puts "Settings:"
 
           headers = ['name', 'default', 'required'].map{ |h| h.to_s.upcase.bold }
 

lib/chronicle/etl/cli/jobs.rb

@@ -1,12 +1,13 @@
 require 'pp'
+require 'tty-prompt'
 
 module Chronicle
   module ETL
     module CLI
       # CLI commands for working with ETL jobs
       class Jobs < SubcommandBase
         default_task "start"
-        namespace :jobs 
+        namespace :jobs
 
         class_option :name, aliases: '-j', desc: 'Job configuration name'
 
@@ -26,15 +27,9 @@ class Jobs < SubcommandBase
         class_option :output, aliases: '-o', desc: 'Output filename', type: 'string'
         class_option :fields, desc: 'Output only these fields', type: 'array', banner: 'field1 field2 ...'
 
-        class_option :log_level, desc: 'Log level (debug, info, warn, error, fatal)', default: 'info'
-        class_option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
-        class_option :silent, desc: 'Silence all output', type: :boolean
-
         # Thor doesn't like `run` as a command name
         map run: :start
         desc "run", "Start a job"
-        option :log_level, desc: 'Log level (debug, info, warn, error, fatal)', default: 'info'
-        option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
         option :dry_run, desc: 'Only run the extraction and transform steps, not the loading', type: :boolean
         long_desc <<-LONG_DESC
           This will run an ETL job. Each job needs three parts:
@@ -49,25 +44,39 @@ class Jobs < SubcommandBase
 LONG_DESC
         # Run an ETL job
         def start
-          setup_log_level
-          job_definition = build_job_definition(options)
-          job = Chronicle::ETL::Job.new(job_definition)
-          runner = Chronicle::ETL::Runner.new(job)
-          runner.run!
+          run_job(options)
+        rescue Chronicle::ETL::JobDefinitionError => e
+          missing_plugins = e.job_definition.errors
+            .select { |error| error.is_a?(Chronicle::ETL::PluginLoadError) }
+            .map(&:name)
+            .uniq
+
+          install_missing_plugins(missing_plugins)
+          run_job(options)
         end
 
         desc "create", "Create a job"
         # Create an ETL job
         def create
           job_definition = build_job_definition(options)
+          job_definition.validate!
+
           path = File.join('chronicle', 'etl', 'jobs', options[:name])
           Chronicle::ETL::Config.write(path, job_definition.definition)
+        rescue Chronicle::ETL::JobDefinitionError => e
+          Chronicle::ETL::Logger.debug(e.full_message)
+          Chronicle::ETL::Logger.fatal("Job definition error".red)
         end
 
         desc "show", "Show details about a job"
         # Show an ETL job
         def show
-          puts Chronicle::ETL::Job.new(build_job_definition(options))
+          job_definition = build_job_definition(options)
+          job_definition.validate!
+          puts Chronicle::ETL::Job.new(job_definition)
+        rescue Chronicle::ETL::JobDefinitionError => e
+          Chronicle::ETL::Logger.debug(e.full_message)
+          Chronicle::ETL::Logger.fatal("Job definition error".red)
         end
 
         desc "list", "List all available jobs"
@@ -87,21 +96,39 @@ def list
 
           headers = ['name', 'extractor', 'transformer', 'loader'].map { |h| h.upcase.bold }
 
+          puts "Available jobs:"
           table = TTY::Table.new(headers, job_details)
           puts table.render(indent: 0, padding: [0, 2])
         end
 
         private
 
-        def setup_log_level
-          if options[:silent]
-            Chronicle::ETL::Logger.log_level = Chronicle::ETL::Logger::SILENT
-          elsif options[:verbose]
-            Chronicle::ETL::Logger.log_level = Chronicle::ETL::Logger::DEBUG
-          elsif options[:log_level]
-            level = Chronicle::ETL::Logger.const_get(options[:log_level].upcase)
-            Chronicle::ETL::Logger.log_level = level 
+        def run_job(options)
+          job_definition = build_job_definition(options)
+          job = Chronicle::ETL::Job.new(job_definition)
+          runner = Chronicle::ETL::Runner.new(job)
+          runner.run!
+        end
+
+        # TODO: probably could merge this with something in cli/plugin
+        def install_missing_plugins(missing_plugins)
+          prompt = TTY::Prompt.new
+          message = "Plugin#{'s' if missing_plugins.count > 1} specified by job not installed.\n"
+          message += "Do you want to install "
+          message += missing_plugins.map { |name| "chronicle-#{name}".bold}.join(", ")
+          message += " and start the job?"
+          install = prompt.yes?(message)
+          return unless install
+
+          spinner = TTY::Spinner.new("[:spinner] Installing plugins...", format: :dots_2)
+          spinner.auto_spin
+          missing_plugins.each do |plugin|
+            Chronicle::ETL::Registry::PluginRegistry.install(plugin)
           end
+          spinner.success("(#{'successful'.green})")
+        rescue Chronicle::ETL::PluginNotAvailableError => e
+          spinner.error("Error".red)
+          Chronicle::ETL::Logger.fatal("Plugin '#{e.name}' could not be installed".red)
         end
 
         # Create job definition by reading config file and then overwriting with flag options

lib/chronicle/etl/cli/main.rb

@@ -21,6 +21,9 @@ class Main < ::Thor
         desc 'jobs:COMMAND', 'Configure and run jobs', hide: true
         subcommand 'jobs', Jobs
 
+        desc 'plugins:COMMAND', 'Configure plugins', hide: true
+        subcommand 'plugins', Plugins
+
         # Entrypoint for the CLI
         def self.start(given_args = ARGV, config = {})
           # take a subcommand:command and splits them so Thor knows how to hand off to the subcommand class

lib/chronicle/etl/cli/plugins.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "tty-prompt"
+require "tty-spinner"
+
+
+module Chronicle
+  module ETL
+    module CLI
+      # CLI commands for working with ETL plugins
+      class Plugins < SubcommandBase
+        default_task 'list'
+        namespace :plugins
+
+        desc "install", "Install a plugin"
+        def install(name)
+          spinner = TTY::Spinner.new("[:spinner] Installing plugin #{name}...", format: :dots_2)
+          spinner.auto_spin
+          Chronicle::ETL::Registry::PluginRegistry.install(name)
+          spinner.success("(#{'successful'.green})")
+        rescue Chronicle::ETL::PluginError => e
+          spinner.error("Error".red)
+          Chronicle::ETL::Logger.debug(e.full_message)
+          Chronicle::ETL::Logger.fatal("Plugin '#{name}' could not be installed".red)
+          exit 1
+        end
+
+        desc "uninstall", "Unintall a plugin"
+        def uninstall(name)
+          spinner = TTY::Spinner.new("[:spinner] Uninstalling plugin #{name}...", format: :dots_2)
+          spinner.auto_spin
+          Chronicle::ETL::Registry::PluginRegistry.uninstall(name)
+          spinner.success("(#{'successful'.green})")
+        rescue Chronicle::ETL::PluginError => e
+          spinner.error("Error".red)
+          Chronicle::ETL::Logger.debug(e.full_message)
+          Chronicle::ETL::Logger.fatal("Plugin '#{name}' could not be uninstalled (was it installed?)".red)
+          exit 1
+        end
+
+        desc "list", "Lists available plugins"
+        # Display all available plugins that chronicle-etl has access to
+        def list
+          plugins = Chronicle::ETL::Registry::PluginRegistry.all_installed_latest
+
+          info = plugins.map do |plugin|
+            {
+              name: plugin.name.sub("chronicle-", ""),
+              description: plugin.description,
+              version: plugin.version
+            }
+          end
+
+          headers = ['name', 'description', 'latest version'].map{ |h| h.to_s.upcase.bold }
+          table = TTY::Table.new(headers, info.map(&:values))
+          puts "Installed plugins:"
+          puts table.render(indent: 2, padding: [0, 0])
+        end
+      end
+    end
+  end
+end

lib/chronicle/etl/config.rb

@@ -23,6 +23,7 @@ def write(path, data)
       end
 
       # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
+      # TODO: raise error if we can't read directory
       def available_jobs
         job_directory = Runcom::Config.new('chronicle/etl/jobs').current
         Dir.glob(File.join(job_directory, "*.yml")).map do |filename|

lib/chronicle/etl/exceptions.rb

@@ -6,6 +6,26 @@ class ConfigurationError < Error; end
 
     class RunnerTypeError < Error; end
 
+    class JobDefinitionError < Error
+      attr_reader :job_definition
+
+      def initialize(job_definition)
+        @job_definition = job_definition
+        super
+      end
+    end
+
+    class PluginError < Error
+      attr_reader :name
+
+      def initialize(name)
+        @name = name
+      end
+    end
+
+    class PluginNotAvailableError < PluginError; end
+    class PluginLoadError < PluginError; end
+
     class ConnectorNotAvailableError < Error
       def initialize(message, provider: nil, name: nil)
         super(message)

lib/chronicle/etl/job.rb

@@ -1,6 +1,11 @@
 require 'forwardable'
+
 module Chronicle
   module ETL
+    # A runner job
+    #
+    # TODO: this can probably be merged with JobDefinition. Not clear
+    # where the boundaries are
     class Job
       extend Forwardable
 
@@ -12,7 +17,8 @@ class Job
                     :transformer_klass,
                     :transformer_options,
                     :loader_klass,
-                    :loader_options
+                    :loader_options,
+                    :job_definition
 
       # TODO: build a proper id system
       alias id name

lib/chronicle/etl/job_definition.rb

@@ -19,12 +19,31 @@ class JobDefinition
         }
       }.freeze
 
+      attr_reader :errors
       attr_accessor :definition
 
       def initialize()
         @definition = SKELETON_DEFINITION
       end
 
+      def validate
+        @errors = []
+
+        Chronicle::ETL::Registry::PHASES.each do |phase|
+          __send__("#{phase}_klass".to_sym)
+        rescue Chronicle::ETL::PluginError => e
+          @errors << e
+        end
+
+        @errors.empty?
+      end
+
+      def validate!
+        raise(Chronicle::ETL::JobDefinitionError.new(self), "Job definition is invalid") unless validate
+
+        true
+      end
+
       # Add config hash to this definition
       def add_config(config = {})
         @definition = @definition.deep_merge(config)
@@ -80,10 +99,6 @@ def load_credentials
           end
         end
       end
-
-      def validate
-        return true   # TODO
-      end
     end
   end
 end

lib/chronicle/etl/logger.rb

@@ -13,18 +13,21 @@ module Logger
       attr_accessor :log_level
 
       @log_level = INFO
-      @destination = $stderr
 
       def output message, level
         return unless level >= @log_level
 
         if @progress_bar
           @progress_bar.log(message)
         else
-          @destination.puts(message)
+          $stderr.puts(message)
         end
       end
 
+      def fatal(message)
+        output(message, FATAL)
+      end
+
       def error(message)
         output(message, ERROR)
       end

lib/chronicle/etl/registry/connector_registration.rb

@@ -44,6 +44,11 @@ def provider
           @provider || (built_in? ? 'chronicle' : '')
         end
 
+        # TODO: allow overriding here. Maybe through self-registration process
+        def plugin
+          @provider
+        end
+
         def descriptive_phrase
           prefix = case phase
           when :extractor