Add latest changes from gitlab-org/gitlab@master

Author: GitLab Bot

.gitlab/ci/rails/shared.gitlab-ci.yml

@@ -67,6 +67,7 @@ include:
     RECORD_DEPRECATIONS: "true"
     GEO_SECONDARY_PROXY: 0
     SUCCESSFULLY_RETRIED_TEST_EXIT_CODE: 137
+    EVENT_PROF: "sql.active_record"
   needs:
     - job: "setup-test-env"
     - job: "retrieve-tests-metadata"

app/controllers/oauth/authorized_applications_controller.rb

@@ -19,7 +19,7 @@ def destroy
       Doorkeeper::Application.revoke_tokens_and_grants_for(params[:id], current_resource_owner)
     end
 
-    redirect_to applications_profile_url,
+    redirect_to user_settings_applications_url,
       status: :found,
       notice: I18n.t(:notice, scope: [:doorkeeper, :flash, :authorized_applications, :destroy])
   end

app/controllers/projects/deploy_keys_controller.rb

@@ -26,7 +26,7 @@ def enabled_keys
     respond_to do |format|
       format.json do
         enabled_keys = find_keys(filter: :enabled_keys)
-        render json: serialize(enabled_keys)
+        render json: { keys: serialize(enabled_keys) }
       end
     end
   end
@@ -35,7 +35,7 @@ def available_project_keys
     respond_to do |format|
       format.json do
         available_project_keys = find_keys(filter: :available_project_keys)
-        render json: serialize(available_project_keys)
+        render json: { keys: serialize(available_project_keys) }
       end
     end
   end
@@ -45,7 +45,7 @@ def available_public_keys
       format.json do
         available_public_keys = find_keys(filter: :available_public_keys)
 
-        render json: serialize(available_public_keys)
+        render json: { keys: serialize(available_public_keys) }
       end
     end
   end

app/graphql/mutations/container_registry/protection/rule/create.rb

@@ -18,12 +18,12 @@ class Create < ::Mutations::BaseMutation
             required: true,
             description: 'Full path of the project where a protection rule is located.'
 
-          argument :container_path_pattern,
+          argument :repository_path_pattern,
             GraphQL::Types::String,
             required: true,
             description:
-              'ContainerRegistryname protected by the protection rule. For example `@my-scope/my-container-*`. ' \
-              'Wildcard character `*` allowed.'
+              'Container repository path pattern protected by the protection rule. ' \
+              'For example `my-project/my-container-*`. Wildcard character `*` allowed.'
 
           argument :push_protected_up_to_access_level,
             Types::ContainerRegistry::Protection::RuleAccessLevelEnum,

app/graphql/types/container_registry/protection/rule_type.rb

@@ -15,12 +15,12 @@ class RuleType < ::Types::BaseObject
           null: false,
           description: 'ID of the container registry protection rule.'
 
-        field :container_path_pattern,
+        field :repository_path_pattern,
           GraphQL::Types::String,
           null: false,
           description:
             'Container repository path pattern protected by the protection rule. ' \
-            'For example `@my-scope/my-container-*`. Wildcard character `*` allowed.'
+            'For example `my-project/my-container-*`. Wildcard character `*` allowed.'
 
         field :push_protected_up_to_access_level,
           Types::ContainerRegistry::Protection::RuleAccessLevelEnum,

app/models/bulk_import.rb

@@ -17,6 +17,7 @@ class BulkImport < ApplicationRecord
   enum source_type: { gitlab: 0 }
 
   scope :stale, -> { where('updated_at < ?', 24.hours.ago).where(status: [0, 1]) }
+  scope :order_by_updated_at_and_id, ->(direction) { order(updated_at: direction, id: :asc) }
   scope :order_by_created_at, ->(direction) { order(created_at: direction) }
 
   state_machine :status, initial: :created do

app/models/bulk_imports/entity.rb

@@ -57,6 +57,8 @@ class BulkImports::Entity < ApplicationRecord
   scope :stale, -> { where('updated_at < ?', 24.hours.ago).where(status: [0, 1]) }
   scope :by_bulk_import_id, ->(bulk_import_id) { where(bulk_import_id: bulk_import_id) }
   scope :order_by_created_at, ->(direction) { order(created_at: direction) }
+  scope :order_by_updated_at_and_id, ->(direction) { order(updated_at: direction, id: :asc) }
+  scope :with_trackers, -> { includes(:trackers) }
 
   alias_attribute :destination_slug, :destination_name
 

app/models/container_registry/protection/rule.rb

@@ -3,6 +3,10 @@
 module ContainerRegistry
   module Protection
     class Rule < ApplicationRecord
+      include IgnorableColumns
+
+      ignore_column :container_path_pattern, remove_with: '16.8', remove_after: '2023-12-22'
+
       enum delete_protected_up_to_access_level:
              Gitlab::Access.sym_options_with_owner.slice(:maintainer, :owner, :developer),
         _prefix: :delete_protected_up_to
@@ -12,7 +16,7 @@ class Rule < ApplicationRecord
 
       belongs_to :project, inverse_of: :container_registry_protection_rules
 
-      validates :container_path_pattern, presence: true, uniqueness: { scope: :project_id }, length: { maximum: 255 }
+      validates :repository_path_pattern, presence: true, uniqueness: { scope: :project_id }, length: { maximum: 255 }
       validates :delete_protected_up_to_access_level, presence: true
       validates :push_protected_up_to_access_level, presence: true
     end

app/serializers/deploy_keys/basic_deploy_key_entity.rb

@@ -2,6 +2,8 @@
 
 module DeployKeys
   class BasicDeployKeyEntity < Grape::Entity
+    include RequestAwareEntity
+
     expose :id
     expose :user_id
     expose :title
@@ -14,6 +16,17 @@ class BasicDeployKeyEntity < Grape::Entity
     expose :updated_at
     expose :can_edit
     expose :user, as: :owner, using: ::API::Entities::UserBasic, if: -> (_, opts) { can_read_owner?(opts) }
+    expose :edit_path, if: -> (_, opts) { opts[:project] } do |deploy_key|
+      edit_project_deploy_key_path(options[:project], deploy_key)
+    end
+
+    expose :enable_path, if: -> (_, opts) { opts[:project] } do |deploy_key|
+      enable_project_deploy_key_path(options[:project], deploy_key)
+    end
+
+    expose :disable_path, if: -> (_, opts) { opts[:project] } do |deploy_key|
+      disable_project_deploy_key_path(options[:project], deploy_key)
+    end
 
     private
 

app/services/container_registry/protection/create_rule_service.rb

@@ -4,7 +4,7 @@ module ContainerRegistry
   module Protection
     class CreateRuleService < BaseService
       ALLOWED_ATTRIBUTES = %i[
-        container_path_pattern
+        repository_path_pattern
         push_protected_up_to_access_level
         delete_protected_up_to_access_level
       ].freeze

app/services/import/github_service.rb

@@ -16,7 +16,7 @@ def execute(access_params, provider)
       track_access_level('github')
 
       if project.persisted?
-        store_import_settings(project, access_params)
+        store_import_settings(project)
         success(project)
       elsif project.errors[:import_source_disabled].present?
         error(project.errors[:import_source_disabled], :forbidden)
@@ -134,13 +134,12 @@ def log_and_return_error(message, translated_message, http_status)
       error(translated_message, http_status)
     end
 
-    def store_import_settings(project, access_params)
+    def store_import_settings(project)
       Gitlab::GithubImport::Settings
         .new(project)
         .write(
           timeout_strategy: params[:timeout_strategy] || ProjectImportData::PESSIMISTIC_TIMEOUT,
-          optional_stages: params[:optional_stages],
-          additional_access_tokens: access_params[:additional_access_tokens]
+          optional_stages: params[:optional_stages]
         )
     end
   end

app/workers/bulk_imports/stuck_import_worker.rb

@@ -12,22 +12,27 @@ class StuckImportWorker
 
     feature_category :importers
 
+    # Using Keyset pagination for scopes that involve timestamp indexes
     def perform
-      BulkImport.stale.find_each do |import|
-        logger.error(message: 'BulkImport stale', bulk_import_id: import.id)
-        import.cleanup_stale
+      Gitlab::Pagination::Keyset::Iterator.new(scope: bulk_import_scope).each_batch do |imports|
+        imports.each do |import|
+          logger.error(message: 'BulkImport stale', bulk_import_id: import.id)
+          import.cleanup_stale
+        end
       end
 
-      BulkImports::Entity.includes(:trackers).stale.find_each do |entity| # rubocop: disable CodeReuse/ActiveRecord
-        ApplicationRecord.transaction do
-          logger.with_entity(entity).error(
-            message: 'BulkImports::Entity stale'
-          )
+      Gitlab::Pagination::Keyset::Iterator.new(scope: entity_scope).each_batch do |entities|
+        entities.each do |entity|
+          ApplicationRecord.transaction do
+            logger.with_entity(entity).error(
+              message: 'BulkImports::Entity stale'
+            )
 
-          entity.cleanup_stale
+            entity.cleanup_stale
 
-          entity.trackers.find_each do |tracker|
-            tracker.cleanup_stale
+            entity.trackers.find_each do |tracker|
+              tracker.cleanup_stale
+            end
           end
         end
       end
@@ -36,5 +41,13 @@ def perform
     def logger
       @logger ||= Logger.build
     end
+
+    def bulk_import_scope
+      BulkImport.stale.order_by_updated_at_and_id(:asc)
+    end
+
+    def entity_scope
+      BulkImports::Entity.with_trackers.stale.order_by_updated_at_and_id(:asc)
+    end
   end
 end

app/workers/concerns/gitlab/github_import/object_importer.rb

@@ -57,12 +57,7 @@ def import(project, client, hash)
         end
 
         info(project.id, message: 'importer finished')
-      rescue NoMethodError => e
-        # This exception will be more useful in development when a new
-        # Representation is created but the developer forgot to add a
-        # `#github_identifiers` method.
-        track_and_raise_exception(project, e, fail_import: true)
-      rescue ActiveRecord::RecordInvalid, NotRetriableError => e
+      rescue ActiveRecord::RecordInvalid, NotRetriableError, NoMethodError => e
         # We do not raise exception to prevent job retry
         track_exception(project, e)
       rescue StandardError => e

config/feature_flags/development/custom_roles_in_members_page.yml

@@ -6,7 +6,7 @@
 resource :profile, only: [:show, :update] do
   member do
     get :audit_log, to: redirect('-/user_settings/authentication_log')
-    get :applications, to: 'oauth/applications#index'
+    get :applications, to: redirect('-/user_settings/applications')
 
     put :reset_incoming_email_token
     put :reset_feed_token

config/routes/profile.rb

@@ -3,6 +3,7 @@
 namespace :user_settings do
   scope module: 'user_settings' do
     get :authentication_log
+    get :applications, to: '/oauth/applications#index'
   end
   resources :active_sessions, only: [:index, :destroy]
 end

config/routes/user_settings.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class RenameContainerRegistryProtectionRulesContainerPathPattern < Gitlab::Database::Migration[2.2]
+  milestone '16.7'
+
+  disable_ddl_transaction!
+
+  def up
+    rename_column_concurrently :container_registry_protection_rules, :container_path_pattern, :repository_path_pattern
+  end
+
+  def down
+    undo_rename_column_concurrently :container_registry_protection_rules, :container_path_pattern,
+      :repository_path_pattern
+  end
+end

db/migrate/20231126200903_rename_container_registry_protection_rules_container_path_pattern.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class RenameIndexIContainerProtectionUniqueProjectIdContainerPathPattern < Gitlab::Database::Migration[2.2]
+  milestone '16.7'
+
+  disable_ddl_transaction!
+
+  def up
+    rename_index :container_registry_protection_rules, :idx_copy_d01a85dee8,
+      :i_container_protection_unique_project_repository_path_pattern
+  end
+
+  def down
+    rename_index :container_registry_protection_rules, :i_container_protection_unique_project_repository_path_pattern,
+      :idx_copy_d01a85dee8
+  end
+end

db/migrate/20231126200904_rename_index_i_container_protection_unique_project_id_container_path_pattern.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class CleanupContainerRegistryProtectionRulesContainerPathPatternAtRename < Gitlab::Database::Migration[2.2]
+  milestone '16.7'
+
+  disable_ddl_transaction!
+
+  def up
+    cleanup_concurrent_column_rename :container_registry_protection_rules, :container_path_pattern,
+      :repository_path_pattern
+  end
+
+  def down
+    undo_cleanup_concurrent_column_rename :container_registry_protection_rules, :container_path_pattern,
+      :repository_path_pattern
+
+    # Restoring the old index name `:i_container_protection_unique_project_id_container_path_pattern`
+    # that was changed in the following migrations:
+    # - `db/migrate/20231126200903_rename_container_registry_protection_rules_container_path_pattern.rb`
+    # - `db/migrate/20231126200904_rename_index_i_container_protection_unique_project_id_container_path_pattern.rb`
+    if index_exists?(:container_registry_protection_rules, [:project_id, :container_path_pattern],
+      name: :i_container_protection_unique_project_container_path_pattern)
+      rename_index :container_registry_protection_rules, :i_container_protection_unique_project_container_path_pattern,
+        :i_container_protection_unique_project_id_container_path_pattern
+    end
+  end
+end

db/post_migrate/20231126220000_cleanup_container_registry_protection_rules_container_path_pattern_at_rename.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class CleanupCiStagesPipelineIdBigintForSelfHost < Gitlab::Database::Migration[2.2]
+  disable_ddl_transaction!
+  milestone "16.7"
+
+  TABLE = :ci_stages
+  REFERENCING_TABLE = :ci_pipelines
+  COLUMN = :pipeline_id
+  OLD_COLUMN = :pipeline_id_convert_to_bigint
+  INDEXES = {
+    'index_ci_stages_on_pipeline_id_convert_to_bigint_and_name' => [
+      [:pipeline_id_convert_to_bigint, :name], { unique: true }
+    ],
+    'index_ci_stages_on_pipeline_id_convert_to_bigint' => [
+      [:pipeline_id_convert_to_bigint], {}
+    ],
+    'index_ci_stages_on_pipeline_id_convert_to_bigint_and_id' => [
+      [:pipeline_id_convert_to_bigint, :id], { where: 'status = ANY (ARRAY[0, 1, 2, 8, 9, 10])' }
+    ],
+    'index_ci_stages_on_pipeline_id_convert_to_bigint_and_position' => [
+      [:pipeline_id_convert_to_bigint, :position], {}
+    ]
+  }
+
+  def up
+    return unless column_exists?(TABLE, OLD_COLUMN)
+
+    with_lock_retries(raise_on_exhaustion: true) do
+      lock_tables(REFERENCING_TABLE, TABLE)
+      cleanup_conversion_of_integer_to_bigint(TABLE, [COLUMN])
+    end
+  end
+
+  def down
+    return if column_exists?(TABLE, OLD_COLUMN)
+    # See db/post_migrate/20231120070345_cleanup_ci_stages_pipeline_id_bigint.rb
+    # Both Gitlab.com and dev/test envinronments will be handled in that migration.
+    return if Gitlab.com_except_jh? || Gitlab.dev_or_test_env?
+
+    restore_conversion_of_integer_to_bigint(TABLE, [COLUMN])
+
+    INDEXES.each do |index_name, (columns, options)|
+      add_concurrent_index(TABLE, columns, name: index_name, **options)
+    end
+
+    add_concurrent_foreign_key(
+      TABLE, REFERENCING_TABLE,
+      column: OLD_COLUMN,
+      on_delete: :cascade, validate: true, reverse_lock_order: true
+    )
+  end
+end

db/post_migrate/20231207054819_cleanup_ci_stages_pipeline_id_bigint_for_self_host.rb

@@ -0,0 +1 @@
+89c33f31982aa26a63cdbd1fd35d51c984006d6ae66dc3cac8e88f3a8fadf461

db/schema_migrations/20231126200903

@@ -0,0 +1 @@
+9b1a9d983f5feebe9a7a64a653ff300dbb7b5d12520d751d875527755ca15c61

db/schema_migrations/20231126200904

@@ -0,0 +1 @@
+f841d351a89d3ed9b2fea8d386b528aff5f1a267c214f6f0a150281377522d44

db/schema_migrations/20231126220000

@@ -0,0 +1 @@
+b59e995833c187b21e561f3be24d53e1e6e56cee1f7a5933546b55f8a3a731c8