Ams gbh valkyrie support

app/factories/bulkrax/object_factory.rb

@@ -90,11 +90,14 @@ def update
     def find
       found = find_by_id if attributes[:id].present?
       return found if found.present?
+    rescue Valkyrie::Persistence::ObjectNotFoundError
+      false
+    ensure
       return search_by_identifier if attributes[work_identifier].present?
     end
 
     def find_by_id
-      klass.find(attributes[:id]) if klass.exists?(attributes[:id])
+      klass.find(attributes[:id]) if klass.exist?(attributes[:id])
     end
 
     def find_or_create

app/factories/bulkrax/valkyrie_object_factory.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ValkyrieObjectFactory < ObjectFactory
+    ##
+    # Retrieve properties from M3 model
+    # @param klass the model
+    # return Array<string>
+    def self.schema_properties(klass)
+      @schema_properties_map ||= {}
+
+      klass_key = klass.name
+      unless @schema_properties_map.has_key?(klass_key)
+        @schema_properties_map[klass_key] = klass.schema.map { |k| k.name.to_s }
+      end
+
+      @schema_properties_map[klass_key]
+    end
+
+    def run!
+      run
+      return object if object.persisted?
+
+      raise(RecordInvalid, object)
+    end
+
+    def find_by_id
+      Hyrax.query_service.find_by(id: attributes[:id]) if attributes.key? :id
+    end
+
+    def search_by_identifier
+      # Query can return partial matches (something6 matches both something6 and something68)
+      # so we need to weed out any that are not the correct full match. But other items might be
+      # in the multivalued field, so we have to go through them one at a time.
+      match = Hyrax.query_service.custom_queries.find_by_bulkrax_identifier(identifier: source_identifier_value)
+
+      return match if match
+    rescue => err
+      Hyrax.logger.error(err)
+      false
+    end
+
+    def create
+      attrs = transform_attributes
+        .merge(alternate_ids: [source_identifier_value])
+        .symbolize_keys
+
+      # temporary workaround just to see if we can get the import to work
+      attrs.merge!(title: ['']) if attrs[:title].blank?
+      attrs.merge!(creator: ['']) if attrs[:creator].blank?
+
+      cx = Hyrax::Forms::ResourceForm.for(klass.new).prepopulate!
+      cx.validate(attrs)
+
+      result = transaction
+        .with_step_args(
+          # "work_resource.add_to_parent" => {parent_id: @related_parents_parsed_mapping, user: @user},
+          "work_resource.add_bulkrax_files" => {files: get_s3_files(remote_files: attributes["remote_files"]), user: @user},
+          "change_set.set_user_as_depositor" => {user: @user},
+          "work_resource.change_depositor" => {user: @user},
+          'work_resource.save_acl' => { permissions_params: [attrs.try('visibility') || 'open'].compact }
+        )
+        .call(cx)
+
+      if result.failure?
+        msg = result.failure[0].to_s
+        msg += " - #{result.failure[1].full_messages.join(',')}" if result.failure[1].respond_to?(:full_messages)
+        raise StandardError, msg, result.trace
+      end
+
+      @object = result.value!
+
+      @object
+    end
+
+    def update
+      raise "Object doesn't exist" unless @object
+
+      destroy_existing_files if @replace_files && ![Collection, FileSet].include?(klass)
+
+      attrs = transform_attributes(update: true)
+
+      cx = Hyrax::Forms::ResourceForm.for(@object)
+      cx.validate(attrs)
+
+      result = update_transaction
+        .with_step_args(
+          "work_resource.add_bulkrax_files" => {files: get_s3_files(remote_files: attributes["remote_files"]), user: @user}
+
+          # TODO: uncomment when we upgrade Hyrax 4.x
+          # 'work_resource.save_acl' => { permissions_params: [attrs.try('visibility') || 'open'].compact }
+        )
+        .call(cx)
+
+      @object = result.value!
+    end
+
+    def get_s3_files(remote_files: {})
+      if remote_files.blank?
+        Hyrax.logger.info "No remote files listed for #{attributes["source_identifier"]}"
+        return []
+      end
+
+      s3_bucket_name = ENV.fetch("STAGING_AREA_S3_BUCKET", "comet-staging-area-#{Rails.env}")
+      s3_bucket = Rails.application.config.staging_area_s3_connection
+        .directories.get(s3_bucket_name)
+
+      remote_files.map { |r| r["url"] }.map do |key|
+        s3_bucket.files.get(key)
+      end.compact
+    end
+
+    ##
+    # TODO: What else fields are necessary: %i[id edit_users edit_groups read_groups work_members_attributes]?
+    # Regardless of what the Parser gives us, these are the properties we are prepared to accept.
+    def permitted_attributes
+      Bulkrax::ValkyrieObjectFactory.schema_properties(klass) +
+        %i[
+          admin_set_id
+          title
+          visibility
+        ]
+    end
+
+    def apply_depositor_metadata(object, user)
+      object.depositor = user.email
+      object = Hyrax.persister.save(resource: object)
+      Hyrax.publisher.publish("object.metadata.updated", object: object, user: @user)
+      object
+    end
+
+    # @Override remove branch for FileSets replace validation with errors
+    def new_remote_files
+      @new_remote_files ||= if @object.is_a? FileSet
+        parsed_remote_files.select do |file|
+          # is the url valid?
+          is_valid = file[:url]&.match(URI::ABS_URI)
+          # does the file already exist
+          is_existing = @object.import_url && @object.import_url == file[:url]
+          is_valid && !is_existing
+        end
+      else
+        parsed_remote_files.select do |file|
+          file[:url]&.match(URI::ABS_URI)
+        end
+      end
+    end
+
+    # @Override Destroy existing files with Hyrax::Transactions
+    def destroy_existing_files
+      existing_files = fetch_child_file_sets(resource: @object)
+
+      existing_files.each do |fs|
+        Hyrax::Transactions::Container["file_set.destroy"]
+          .with_step_args("file_set.remove_from_work" => {user: @user},
+            "file_set.delete" => {user: @user})
+          .call(fs)
+          .value!
+      end
+
+      @object.member_ids = @object.member_ids.reject { |m| existing_files.detect { |f| f.id == m } }
+      @object.rendering_ids = []
+      @object.representative_id = nil
+      @object.thumbnail_id = nil
+    end
+
+    private
+
+    def transaction
+      Hyrax::Transactions::Container["work_resource.create_with_bulk_behavior"]
+    end
+
+    # Customize Hyrax::Transactions::WorkUpdate transaction with bulkrax
+    def update_transaction
+      Hyrax::Transactions::Container["work_resource.update_with_bulk_behavior"]
+    end
+
+    # Query child FileSet in the resource/object
+    def fetch_child_file_sets(resource:)
+      Hyrax.custom_queries.find_child_file_sets(resource: resource)
+    end
+  end
+
+  class RecordInvalid < StandardError
+  end
+end

app/helpers/bulkrax/importers_helper.rb

@@ -6,7 +6,7 @@ module ImportersHelper
     def available_admin_sets
       # Restrict available_admin_sets to only those current user can deposit to.
       @available_admin_sets ||= Hyrax::Collections::PermissionsService.source_ids_for_deposit(ability: current_ability, source_type: 'admin_set').map do |admin_set_id|
-        [AdminSet.find(admin_set_id).title.first, admin_set_id]
+        [Hyrax.metadata_adapter.query_service.find_by(id: admin_set_id)&.title&.first || admin_set_id, admin_set_id]
       end
     end
   end

app/models/bulkrax/importer.rb

@@ -18,7 +18,7 @@ class Importer < ApplicationRecord # rubocop:disable Metrics/ClassLength
     validates :admin_set_id, presence: true if defined?(::Hyrax)
     validates :parser_klass, presence: true
 
-    delegate :valid_import?, :write_errored_entries_file, :visibility, to: :parser
+    delegate :create_parent_child_relationships, :valid_import?, :write_errored_entries_file, :visibility, to: :parser
 
     attr_accessor :only_updates, :file_style, :file
     attr_writer :current_run
@@ -123,6 +123,34 @@ def last_run
       @last_run ||= self.importer_runs.last
     end
 
+    def failed_statuses
+      @failed_statuses ||= Bulkrax::Status.latest_by_statusable
+        .includes(:statusable)
+        .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Failed')
+    end
+
+    def failed_entries
+      @failed_entries ||= failed_statuses.map(&:statusable)
+    end
+
+    def failed_messages
+      failed_statuses.inject({}) do |i, e|
+        i[e.error_message] ||= []
+        i[e.error_message] << e.id
+        i
+      end
+    end
+
+    def completed_statuses
+      @completed_statuses ||= Bulkrax::Status.latest_by_statusable
+        .includes(:statusable)
+        .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Complete')
+    end
+
+    def completed_entries
+      @completed_entries ||= completed_statuses.map(&:statusable)
+    end
+
     def seen
       @seen ||= {}
     end

app/models/concerns/bulkrax/has_matchers.rb

@@ -125,7 +125,9 @@ def field_supported?(field)
 
       return false if excluded?(field)
       return true if supported_bulkrax_fields.include?(field)
-      return factory_class.method_defined?(field) && factory_class.properties[field].present?
+      property_defined = factory_class.singleton_methods.include?(:properties) && factory_class.properties[field].present?
+
+      factory_class.method_defined?(field) && (Bulkrax::ValkyrieObjectFactory.schema_properties(factory_class).include?(field) || property_defined)
     end
 
     def supported_bulkrax_fields
@@ -155,7 +157,21 @@ def multiple?(field)
       return true if @multiple_bulkrax_fields.include?(field)
       return false if field == 'model'
 
-      field_supported?(field) && factory_class&.properties&.[](field)&.[]('multiple')
+      if factory.class.respond_to?(:schema)
+        field_supported?(field) && valkyrie_multiple?(field)
+      else
+        field_supported?(field) && ar_multiple?(field)
+      end
+    end
+
+    def ar_multiple?(field)
+      factory_class.singleton_methods.include?(:properties) && factory_class&.properties&.[](field)&.[]("multiple")
+    end
+
+    def valkyrie_multiple?(field)
+      # TODO there has got to be a better way. Only array types have 'of'
+      sym_field = field.to_sym
+      factory_class.schema.key(sym_field).respond_to?(:of) if factory_class.fields.include?(sym_field)
     end
 
     def get_object_name(field)

app/services/bulkrax/transactions/steps/add_files.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "dry/monads"
+
+module Bulkrax
+  module Transactions
+    module Steps
+      class AddFiles
+        include Dry::Monads[:result]
+
+        ##
+        # @param [Class] handler
+        def initialize(handler: Hyrax::WorkUploadsHandler)
+          @handler = handler
+        end
+
+        ##
+        # @param [Hyrax::Work] obj
+        # @param [Array<Fog::AWS::Storage::File>] file
+        # @param [User] user
+        #
+        # @return [Dry::Monads::Result]
+        def call(obj, files:, user:)
+          if files && user
+            begin
+              files.each do |file|
+                FileIngest.upload(
+                  content_type: file.content_type,
+                  file_body: StringIO.new(file.body),
+                  filename: Pathname.new(file.key).basename,
+                  last_modified: file.last_modified,
+                  permissions: Hyrax::AccessControlList.new(resource: obj),
+                  size: file.content_length,
+                  user: user,
+                  work: obj
+                )
+              end
+            rescue => e
+              Hyrax.logger.error(e)
+              return Failure[:failed_to_attach_file_sets, files]
+            end
+          end
+
+          Success(obj)
+        end
+      end
+    end
+  end
+end

bulkrax.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.files = Dir["{app,config,db,lib}/**/*", "LICENSE", "Rakefile", "README.md"]
 
   s.add_dependency 'rails', '>= 5.1.6'
-  s.add_dependency 'bagit', '~> 0.4'
+  s.add_dependency 'bagit', '~> 0.4.6'
   s.add_dependency 'coderay'
   s.add_dependency 'iso8601', '~> 0.9.0'
   s.add_dependency 'kaminari'

db/migrate/20230608153601_add_indices_to_bulkrax.rb

@@ -1,14 +1,27 @@
+# This migration comes from bulkrax (originally 20230608153601)
 class AddIndicesToBulkrax < ActiveRecord::Migration[5.1]
   def change
-    add_index :bulkrax_entries, :identifier unless index_exists?(:bulkrax_entries, :identifier)
-    add_index :bulkrax_entries, :type unless index_exists?(:bulkrax_entries, :type)
-    add_index :bulkrax_entries, [:importerexporter_id, :importerexporter_type], name: 'bulkrax_entries_importerexporter_idx' unless index_exists?(:bulkrax_entries, [:importerexporter_id, :importerexporter_type], name: 'bulkrax_entries_importerexporter_idx')
+    check_and_add_index :bulkrax_entries, :identifier 
+    check_and_add_index :bulkrax_entries, :type 
+    check_and_add_index :bulkrax_entries, [:importerexporter_id, :importerexporter_type], name: 'bulkrax_entries_importerexporter_idx' 
 
-    add_index :bulkrax_pending_relationships, :parent_id unless index_exists?(:bulkrax_pending_relationships, :parent_id)
-    add_index :bulkrax_pending_relationships, :child_id unless index_exists?(:bulkrax_pending_relationships, :child_id) 
+	check_and_add_index :bulkrax_pending_relationships, :parent_id 
+    check_and_add_index :bulkrax_pending_relationships, :child_id 
+
+    check_and_add_index :bulkrax_statuses, [:statusable_id, :statusable_type], name: 'bulkrax_statuses_statusable_idx' 
+    check_and_add_index :bulkrax_statuses, [:runnable_id, :runnable_type], name: 'bulkrax_statuses_runnable_idx' 
+    check_and_add_index :bulkrax_statuses, :error_class 
+  end
 
-    add_index :bulkrax_statuses, [:statusable_id, :statusable_type], name: 'bulkrax_statuses_statusable_idx' unless index_exists?(:bulkrax_statuses, [:statusable_id, :statusable_type], name: 'bulkrax_statuses_statusable_idx')
-    add_index :bulkrax_statuses, [:runnable_id, :runnable_type], name: 'bulkrax_statuses_runnable_idx' unless index_exists?(:bulkrax_statuses, [:runnable_id, :runnable_type], name: 'bulkrax_statuses_runnable_idx')
-    add_index :bulkrax_statuses, :error_class unless index_exists?(:bulkrax_statuses, :error_class)
+  if RUBY_VERSION =~ /^2/
+    def check_and_add_index(table_name, column_name, options = {})
+      add_index(table_name, column_name, options) unless index_exists?(table_name, column_name, options)
+    end
+  elsif RUBY_VERSION =~ /^3/
+    def check_and_add_index(table_name, column_name, **options)
+      add_index(table_name, column_name, **options) unless index_exists?(table_name, column_name, **options)
+    end
+  else
+    raise "Ruby version #{RUBY_VERSION} is unknown"
   end
 end