Add latest changes from gitlab-org/gitlab@master

Author: GitLab Bot

.rubocop_todo/capybara/testid_finders.yml

@@ -1,40 +1,6 @@
 ---
 Capybara/TestidFinders:
   Exclude:
-    - 'ee/spec/features/protected_branches_spec.rb'
-    - 'ee/spec/features/registrations/combined_registration_spec.rb'
-    - 'ee/spec/features/registrations/identity_verification_spec.rb'
-    - 'ee/spec/features/remote_development/workspaces_dropdown_group_spec.rb'
-    - 'ee/spec/features/remote_development/workspaces_spec.rb'
-    - 'ee/spec/features/search/elastic/group_search_spec.rb'
-    - 'ee/spec/features/search/zoekt/search_spec.rb'
-    - 'ee/spec/features/tanuki_bot_chat_spec.rb'
-    - 'ee/spec/features/trials/saas/creation_with_multiple_existing_namespace_flow_spec.rb'
-    - 'ee/spec/features/trials/show_trial_banner_spec.rb'
-    - 'spec/features/admin/admin_deploy_keys_spec.rb'
-    - 'spec/features/admin/admin_dev_ops_reports_spec.rb'
-    - 'spec/features/admin/admin_groups_spec.rb'
-    - 'spec/features/admin/admin_projects_spec.rb'
-    - 'spec/features/admin/admin_runners_spec.rb'
-    - 'spec/features/admin/admin_settings_spec.rb'
-    - 'spec/features/admin/admin_uses_repository_checks_spec.rb'
-    - 'spec/features/admin/broadcast_messages_spec.rb'
-    - 'spec/features/admin/users/user_spec.rb'
-    - 'spec/features/admin/users/users_spec.rb'
-    - 'spec/features/alert_management/alert_details_spec.rb'
-    - 'spec/features/boards/board_filters_spec.rb'
-    - 'spec/features/boards/boards_spec.rb'
-    - 'spec/features/boards/issue_ordering_spec.rb'
-    - 'spec/features/boards/new_issue_spec.rb'
-    - 'spec/features/boards/sidebar_assignee_spec.rb'
-    - 'spec/features/broadcast_messages_spec.rb'
-    - 'spec/features/callouts/registration_enabled_spec.rb'
-    - 'spec/features/clusters/create_agent_spec.rb'
-    - 'spec/features/commits_spec.rb'
-    - 'spec/features/dashboard/group_spec.rb'
-    - 'spec/features/dashboard/issues_spec.rb'
-    - 'spec/features/dashboard/merge_requests_spec.rb'
-    - 'spec/features/dashboard/milestones_spec.rb'
     - 'spec/features/dashboard/projects_spec.rb'
     - 'spec/features/dashboard/todos/todos_spec.rb'
     - 'spec/features/groups/board_sidebar_spec.rb'

app/helpers/sidebars_helper.rb

@@ -358,7 +358,9 @@ def super_sidebar_current_context(project: nil, group: nil)
   def context_switcher_links
     links = [
       ({ title: s_('Navigation|Your work'), link: root_path, icon: 'work' } if current_user),
-      { title: s_('Navigation|Explore'), link: explore_root_path, icon: 'compass' }
+      { title: s_('Navigation|Explore'), link: explore_root_path, icon: 'compass' },
+      ({ title: s_('Navigation|Profile'), link: profile_path, icon: 'profile' } if current_user),
+      ({ title: s_('Navigation|Preferences'), link: profile_preferences_path, icon: 'preferences' } if current_user)
     ]
 
     # Usually, using current_user.admin? is discouraged because it does not

app/models/service_desk/custom_email_credential.rb

@@ -2,6 +2,14 @@
 
 module ServiceDesk
   class CustomEmailCredential < ApplicationRecord
+    # Used to explicitly set the SMTP AUTH method.
+    # If nil Net::SMTP will choose one of methods listed by the SMTP server.
+    enum smtp_authentication: {
+      plain: 0,
+      login: 1,
+      cram_md5: 2
+    }
+
     attr_encrypted :smtp_username,
       mode: :per_attribute_iv,
       algorithm: 'aes-256-gcm',
@@ -44,7 +52,8 @@ def delivery_options
         password: smtp_password,
         address: smtp_address,
         domain: Mail::Address.new(service_desk_setting.custom_email).domain,
-        port: smtp_port || 587
+        port: smtp_port || 587,
+        authentication: smtp_authentication
       }
     end
 

app/workers/all_queues.yml

@@ -2557,7 +2557,7 @@
   :urgency: :low
   :resource_boundary: :unknown
   :weight: 1
-  :idempotent: false
+  :idempotent: true
   :tags: []
 - :name: bulk_imports_entity
   :worker_name: BulkImports::EntityWorker
@@ -2611,7 +2611,7 @@
   :urgency: :low
   :resource_boundary: :memory
   :weight: 1
-  :idempotent: false
+  :idempotent: true
   :tags: []
 - :name: bulk_imports_pipeline_batch
   :worker_name: BulkImports::PipelineBatchWorker
@@ -2620,7 +2620,7 @@
   :urgency: :low
   :resource_boundary: :memory
   :weight: 1
-  :idempotent: false
+  :idempotent: true
   :tags: []
 - :name: bulk_imports_relation_batch_export
   :worker_name: BulkImports::RelationBatchExportWorker

app/workers/bulk_import_worker.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
-class BulkImportWorker # rubocop:disable Scalability/IdempotentWorker
+class BulkImportWorker
   include ApplicationWorker
 
   data_consistency :always
   feature_category :importers
   sidekiq_options retry: false, dead: false
+  idempotent!
 
   def perform(bulk_import_id)
     bulk_import = BulkImport.find_by_id(bulk_import_id)

app/workers/bulk_imports/pipeline_batch_worker.rb

@@ -1,28 +1,43 @@
 # frozen_string_literal: true
 
 module BulkImports
-  class PipelineBatchWorker # rubocop:disable Scalability/IdempotentWorker
+  class PipelineBatchWorker
     include ApplicationWorker
     include ExclusiveLeaseGuard
 
     data_consistency :always # rubocop:disable SidekiqLoadBalancing/WorkerDataConsistency
     feature_category :importers
-    sidekiq_options retry: false, dead: false
+    sidekiq_options dead: false, retry: 3
     worker_has_external_dependencies!
     worker_resource_boundary :memory
+    idempotent!
+
+    sidekiq_retries_exhausted do |msg, exception|
+      new.perform_failure(msg['args'].first, exception)
+    end
 
     def perform(batch_id)
       @batch = ::BulkImports::BatchTracker.find(batch_id)
+
       @tracker = @batch.tracker
       @pending_retry = false
 
+      return unless process_batch?
+
       log_extra_metadata_on_done(:pipeline_class, @tracker.pipeline_name)
 
       try_obtain_lease { run }
     ensure
       ::BulkImports::FinishBatchedPipelineWorker.perform_async(tracker.id) unless pending_retry
     end
 
+    def perform_failure(batch_id, exception)
+      @batch = ::BulkImports::BatchTracker.find(batch_id)
+      @tracker = @batch.tracker
+
+      fail_batch(exception)
+    end
+
     private
 
     attr_reader :batch, :tracker, :pending_retry
@@ -36,8 +51,6 @@ def run
     rescue BulkImports::RetryPipelineError => e
       @pending_retry = true
       retry_batch(e)
-    rescue StandardError => e
-      fail_batch(e)
     end
 
     def fail_batch(exception)
@@ -59,6 +72,8 @@ def fail_batch(exception)
         exception_message: exception.message,
         correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id
       )
+
+      ::BulkImports::FinishBatchedPipelineWorker.perform_async(tracker.id)
     end
 
     def context
@@ -84,5 +99,9 @@ def re_enqueue(delay = FILE_EXTRACTION_PIPELINE_PERFORM_DELAY)
 
       self.class.perform_in(delay, batch.id)
     end
+
+    def process_batch?
+      batch.created? || batch.started?
+    end
   end
 end

app/workers/bulk_imports/pipeline_worker.rb

@@ -1,21 +1,26 @@
 # frozen_string_literal: true
 
 module BulkImports
-  class PipelineWorker # rubocop:disable Scalability/IdempotentWorker
+  class PipelineWorker
     include ApplicationWorker
     include ExclusiveLeaseGuard
 
     FILE_EXTRACTION_PIPELINE_PERFORM_DELAY = 10.seconds
 
     data_consistency :always
     feature_category :importers
-    sidekiq_options retry: false, dead: false
+    sidekiq_options dead: false, retry: 3
     worker_has_external_dependencies!
     deduplicate :until_executing
     worker_resource_boundary :memory
+    idempotent!
 
     version 2
 
+    sidekiq_retries_exhausted do |msg, exception|
+      new.perform_failure(msg['args'][0], msg['args'][2], exception)
+    end
+
     # Keep _stage parameter for backwards compatibility.
     def perform(pipeline_tracker_id, _stage, entity_id)
       @entity = ::BulkImports::Entity.find(entity_id)
@@ -24,20 +29,21 @@ def perform(pipeline_tracker_id, _stage, entity_id)
       log_extra_metadata_on_done(:pipeline_class, @pipeline_tracker.pipeline_name)
 
       try_obtain_lease do
-        if pipeline_tracker.enqueued?
+        if pipeline_tracker.enqueued? || pipeline_tracker.started?
           logger.info(log_attributes(message: 'Pipeline starting'))
 
           run
-        else
-          message = "Pipeline in #{pipeline_tracker.human_status_name} state instead of expected enqueued state"
-
-          logger.error(log_attributes(message: message))
-
-          fail_tracker(StandardError.new(message)) unless pipeline_tracker.finished? || pipeline_tracker.skipped?
         end
       end
     end
 
+    def perform_failure(pipeline_tracker_id, entity_id, exception)
+      @entity = ::BulkImports::Entity.find(entity_id)
+      @pipeline_tracker = ::BulkImports::Tracker.find(pipeline_tracker_id)
+
+      fail_tracker(exception)
+    end
+
     private
 
     attr_reader :pipeline_tracker, :entity
@@ -67,8 +73,6 @@ def run
       end
     rescue BulkImports::RetryPipelineError => e
       retry_tracker(e)
-    rescue StandardError => e
-      fail_tracker(e)
     end
 
     def source_version

db/migrate/20231031141439_add_smtp_authentication_to_service_desk_custom_email_credentials.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class AddSmtpAuthenticationToServiceDeskCustomEmailCredentials < Gitlab::Database::Migration[2.2]
+  milestone '16.6'
+
+  def change
+    add_column :service_desk_custom_email_credentials, :smtp_authentication, :integer,
+      limit: 2, null: true, default: nil
+  end
+end

db/post_migrate/20231023164908_async_drop_index_users_on_accepted_term_id.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class AsyncDropIndexUsersOnAcceptedTermId < Gitlab::Database::Migration[2.2]
+  milestone '16.6'
+  disable_ddl_transaction!
+
+  TABLE_NAME = 'users'
+  INDEX_NAME = 'index_users_on_accepted_term_id'
+  COLUMN = 'accepted_term_id'
+
+  # TODO: Index to be destroyed synchronously in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/135293
+  def up
+    prepare_async_index_removal TABLE_NAME, COLUMN, name: INDEX_NAME
+  end
+
+  def down
+    prepare_async_index_removal TABLE_NAME, COLUMN, name: INDEX_NAME
+  end
+end

db/post_migrate/20231027013210_remove_partial_index_deployments_for_legacy_successful_deployments.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class RemovePartialIndexDeploymentsForLegacySuccessfulDeployments < Gitlab::Database::Migration[2.2]
+  INDEX_NAME = 'partial_index_deployments_for_legacy_successful_deployments'
+
+  milestone '16.6'
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index_by_name :deployments, name: INDEX_NAME
+  end
+
+  def down
+    # This is based on the following `CREATE INDEX` command in db/init_structure.sql:
+    # CREATE INDEX partial_index_deployments_for_legacy_successful_deployments ON deployments
+    #   USING btree (id) WHERE ((finished_at IS NULL) AND (status = 2));
+    add_concurrent_index :deployments, :id, name: INDEX_NAME, where: '((finished_at IS NULL) AND (status = 2))'
+  end
+end

db/schema_migrations/20231023164908

@@ -0,0 +1 @@
+4d742e6f54307710370453fdd72313c0a0f6928bdf2e4812bc5c16ec1043dd3f

db/schema_migrations/20231027013210

@@ -0,0 +1 @@
+3ee898fd7593c7a300bdfc0dc6f041e2fb65f3600b85788521175449d250590f

db/schema_migrations/20231031141439

@@ -0,0 +1 @@
+568e7a227911f23e4285e1bbcc9dd516ecbd2013501a2add13e99e98880effc8

db/structure.sql

@@ -23217,6 +23217,7 @@ CREATE TABLE service_desk_custom_email_credentials (
     encrypted_smtp_username_iv bytea,
     encrypted_smtp_password bytea,
     encrypted_smtp_password_iv bytea,
+    smtp_authentication smallint,
     CONSTRAINT check_6dd11e956a CHECK ((char_length(smtp_address) <= 255))
 );
 
@@ -35020,8 +35021,6 @@ CREATE UNIQUE INDEX partial_index_bulk_import_exports_on_project_id_and_relation
 
 CREATE INDEX partial_index_ci_builds_on_scheduled_at_with_scheduled_jobs ON ci_builds USING btree (scheduled_at) WHERE ((scheduled_at IS NOT NULL) AND ((type)::text = 'Ci::Build'::text) AND ((status)::text = 'scheduled'::text));
 
-CREATE INDEX partial_index_deployments_for_legacy_successful_deployments ON deployments USING btree (id) WHERE ((finished_at IS NULL) AND (status = 2));
-
 CREATE INDEX partial_index_slack_integrations_with_bot_user_id ON slack_integrations USING btree (id) WHERE (bot_user_id IS NOT NULL);
 
 CREATE UNIQUE INDEX partial_index_sop_configs_on_namespace_id ON security_orchestration_policy_configurations USING btree (namespace_id) WHERE (namespace_id IS NOT NULL);

doc/development/database/loose_foreign_keys.md

@@ -251,8 +251,12 @@ When the loose foreign key definition is no longer needed (parent table is remov
 we need to remove the definition from the YAML file and ensure that we don't leave pending deleted
 records in the database.
 
-1. Remove the deletion tracking trigger from the parent table (if the parent table is still there).
 1. Remove the loose foreign key definition from the configuration (`config/gitlab_loose_foreign_keys.yml`).
+
+The deletion tracking trigger needs to be removed only when the parent table no longer uses loose foreign keys.
+If the model still has at least one `loose_foreign_key` definition remaining, then these steps can be skipped:
+
+1. Remove the trigger from the parent table (if the parent table is still there).
 1. Remove leftover deleted records from the `loose_foreign_keys_deleted_records` table.
 
 Migration for removing the trigger:

doc/development/database/understanding_explain_plans.md

@@ -352,7 +352,6 @@ Indexes:
     "index_users_on_static_object_token" UNIQUE, btree (static_object_token)
     "index_users_on_unlock_token" UNIQUE, btree (unlock_token)
     "index_on_users_name_lower" btree (lower(name::text))
-    "index_users_on_accepted_term_id" btree (accepted_term_id)
     "index_users_on_admin" btree (admin)
     "index_users_on_created_at" btree (created_at)
     "index_users_on_email_trigram" gin (email gin_trgm_ops)

doc/user/clusters/agent/user_access.md

@@ -202,7 +202,7 @@ glab cluster agent update-kubeconfig --agent '<agent-id>' --kubeconfig ~/gitlab.
 
 ### Configure local access manually using a personal access token
 
-You can configure access to a Kubernetes cluster using a long-lived personal access token following these steps:
+You can configure access to a Kubernetes cluster using a long-lived personal access token:
 
 1. On the left sidebar, select **Search or go to** and find your project.
 1. Select **Operate > Kubernetes clusters** and retrieve the numerical ID of the agent you want to access. You need the ID to construct the full API token.

locale/gitlab.pot

@@ -31344,9 +31344,15 @@ msgstr ""
 msgid "Navigation|Plan"
 msgstr ""
 
+msgid "Navigation|Preferences"
+msgstr ""
+
 msgid "Navigation|Primary navigation"
 msgstr ""
 
+msgid "Navigation|Profile"
+msgstr ""
+
 msgid "Navigation|Projects you visit often will appear here."
 msgstr ""