Add latest changes from gitlab-org/gitlab@master

Author: GitLab Bot

GITALY_SERVER_VERSION

@@ -1 +1 @@
-954816989a09915efa2cad385f92e53b25b01675
+d4daa7a154ccc86d1b57372e4265bb502c886537

app/models/concerns/cross_database_modification.rb

@@ -104,6 +104,8 @@ def gitlab_schema
         :gitlab_ci
       when 'MainClusterwide::ApplicationRecord'
         :gitlab_main_clusterwide
+      when 'PackageMetadata::ApplicationRecord'
+        :gitlab_pm
       else
         Gitlab::Database::GitlabSchema.table_schema(table_name) if table_name
       end

app/models/package_metadata/application_record.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module PackageMetadata
+  class ApplicationRecord < ::ApplicationRecord
+    self.abstract_class = true
+
+    def self.table_name_prefix
+      'pm_'
+    end
+  end
+end

app/services/merge_requests/refresh_service.rb

@@ -229,7 +229,7 @@ def comment_mr_branch_presence_changed(merge_request)
         :source, @push.branch_name, presence)
     end
 
-    # Add comment about pushing new commits to merge requests and send nofitication emails
+    # Add comment about pushing new commits to merge requests and send notification emails
     #
     def notify_about_push(merge_request)
       return unless @commits.present?

app/services/work_items/export_csv_service.rb

@@ -2,6 +2,14 @@
 
 module WorkItems
   class ExportCsvService < ExportCsv::BaseService
+    NotAvailableError = StandardError.new('This feature is currently behind a feature flag and it is not available.')
+
+    def csv_data
+      raise NotAvailableError unless Feature.enabled?(:import_export_work_items_csv, resource_parent)
+
+      super
+    end
+
     def email(mail_to_user)
       # TODO - will be implemented as part of https://gitlab.com/gitlab-org/gitlab/-/issues/379082
     end

config/feature_flags/development/import_export_work_items_csv.yml

@@ -0,0 +1,8 @@
+---
+name: import_export_work_items_csv
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111935
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/390330
+milestone: '15.9'
+type: development
+group: group::certify
+default_enabled: false

config/feature_flags/ops/database_async_foreign_key_validation.yml

@@ -0,0 +1,8 @@
+---
+name: database_async_foreign_key_validation
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112015
+rollout_issue_url:
+milestone: '15.9'
+type: ops
+group: group::database
+default_enabled: false

db/migrate/20230113195141_package_metadata_set_default_not_null.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class PackageMetadataSetDefaultNotNull < Gitlab::Database::Migration[2.1]
+  def change
+    change_column_null(:pm_package_versions, :pm_package_id, false)
+  end
+end

db/migrate/20230126023444_add_timestamps_to_package_metadata_tables.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddTimestampsToPackageMetadataTables < Gitlab::Database::Migration[2.1]
+  def up
+    add_timestamps_with_timezone(:pm_packages, null: false, default: -> { 'NOW()' })
+    add_timestamps_with_timezone(:pm_package_versions, null: false, default: -> { 'NOW()' })
+    add_timestamps_with_timezone(:pm_licenses, null: false, default: -> { 'NOW()' })
+    add_timestamps_with_timezone(:pm_package_version_licenses, null: false, default: -> { 'NOW()' })
+  end
+
+  def down
+    remove_timestamps(:pm_packages)
+    remove_timestamps(:pm_package_versions)
+    remove_timestamps(:pm_licenses)
+    remove_timestamps(:pm_package_version_licenses)
+  end
+end

db/migrate/20230127155217_add_id_column_to_package_metadata_join_table.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddIdColumnToPackageMetadataJoinTable < Gitlab::Database::Migration[2.1]
+  disable_ddl_transaction!
+
+  COMPOSITE_UNIQUE_INDEX = :i_pm_package_version_licenses_join_ids
+
+  def up
+    drop_constraint(:pm_package_version_licenses, :pm_package_version_licenses_pkey, cascade: true)
+    add_column(:pm_package_version_licenses, :id, :primary_key)
+    add_concurrent_index(:pm_package_version_licenses, [:pm_package_version_id, :pm_license_id], unique: true,
+      name: COMPOSITE_UNIQUE_INDEX)
+  end
+
+  def down
+    remove_column(:pm_package_version_licenses, :id)
+    add_primary_key_using_index(:pm_package_version_licenses, :pm_package_version_licenses_pkey, COMPOSITE_UNIQUE_INDEX)
+    remove_concurrent_index_by_name(:pm_package_version_licenses, COMPOSITE_UNIQUE_INDEX)
+  end
+end

db/schema_migrations/20230113195141

@@ -0,0 +1 @@
+3e69f6ab127ec01a005ad1de935500fd4bf5528745cec88139aa2917b2a07abf

db/schema_migrations/20230126023444

@@ -0,0 +1 @@
+76894f6bf574e61d078781f0dc914f77dc1bf82a6a6786a32adaf8f829261387

db/schema_migrations/20230127155217

@@ -0,0 +1 @@
+11645dc1986919fcc78d16b0a33033b95aee086b21620c7628e6bd7ec35cd9e5

db/structure.sql

@@ -19872,6 +19872,8 @@ CREATE TABLE pm_checkpoints (
 CREATE TABLE pm_licenses (
     id bigint NOT NULL,
     spdx_identifier text NOT NULL,
+    created_at timestamp with time zone DEFAULT now() NOT NULL,
+    updated_at timestamp with time zone DEFAULT now() NOT NULL,
     CONSTRAINT check_c1eb81d1ba CHECK ((char_length(spdx_identifier) <= 50))
 );
 
@@ -19886,13 +19888,27 @@ ALTER SEQUENCE pm_licenses_id_seq OWNED BY pm_licenses.id;
 
 CREATE TABLE pm_package_version_licenses (
     pm_package_version_id bigint NOT NULL,
-    pm_license_id bigint NOT NULL
+    pm_license_id bigint NOT NULL,
+    created_at timestamp with time zone DEFAULT now() NOT NULL,
+    updated_at timestamp with time zone DEFAULT now() NOT NULL,
+    id bigint NOT NULL
 );
 
+CREATE SEQUENCE pm_package_version_licenses_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE pm_package_version_licenses_id_seq OWNED BY pm_package_version_licenses.id;
+
 CREATE TABLE pm_package_versions (
     id bigint NOT NULL,
-    pm_package_id bigint,
+    pm_package_id bigint NOT NULL,
     version text NOT NULL,
+    created_at timestamp with time zone DEFAULT now() NOT NULL,
+    updated_at timestamp with time zone DEFAULT now() NOT NULL,
     CONSTRAINT check_2d8a88cfcc CHECK ((char_length(version) <= 255))
 );
 
@@ -19909,6 +19925,8 @@ CREATE TABLE pm_packages (
     id bigint NOT NULL,
     purl_type smallint NOT NULL,
     name text NOT NULL,
+    created_at timestamp with time zone DEFAULT now() NOT NULL,
+    updated_at timestamp with time zone DEFAULT now() NOT NULL,
     CONSTRAINT check_3a3aedb8ba CHECK ((char_length(name) <= 255))
 );
 
@@ -24880,6 +24898,8 @@ ALTER TABLE ONLY plans ALTER COLUMN id SET DEFAULT nextval('plans_id_seq'::regcl
 
 ALTER TABLE ONLY pm_licenses ALTER COLUMN id SET DEFAULT nextval('pm_licenses_id_seq'::regclass);
 
+ALTER TABLE ONLY pm_package_version_licenses ALTER COLUMN id SET DEFAULT nextval('pm_package_version_licenses_id_seq'::regclass);
+
 ALTER TABLE ONLY pm_package_versions ALTER COLUMN id SET DEFAULT nextval('pm_package_versions_id_seq'::regclass);
 
 ALTER TABLE ONLY pm_packages ALTER COLUMN id SET DEFAULT nextval('pm_packages_id_seq'::regclass);
@@ -27115,7 +27135,7 @@ ALTER TABLE ONLY pm_licenses
     ADD CONSTRAINT pm_licenses_pkey PRIMARY KEY (id);
 
 ALTER TABLE ONLY pm_package_version_licenses
-    ADD CONSTRAINT pm_package_version_licenses_pkey PRIMARY KEY (pm_package_version_id, pm_license_id);
+    ADD CONSTRAINT pm_package_version_licenses_pkey PRIMARY KEY (id);
 
 ALTER TABLE ONLY pm_package_versions
     ADD CONSTRAINT pm_package_versions_pkey PRIMARY KEY (id);
@@ -28701,6 +28721,8 @@ CREATE INDEX i_dast_scanner_profiles_tags_on_scanner_profiles_id ON dast_scanner
 
 CREATE UNIQUE INDEX i_pm_licenses_on_spdx_identifier ON pm_licenses USING btree (spdx_identifier);
 
+CREATE UNIQUE INDEX i_pm_package_version_licenses_join_ids ON pm_package_version_licenses USING btree (pm_package_version_id, pm_license_id);
+
 CREATE UNIQUE INDEX i_pm_package_versions_on_package_id_and_version ON pm_package_versions USING btree (pm_package_id, version);
 
 CREATE UNIQUE INDEX i_pm_packages_purl_type_and_name ON pm_packages USING btree (purl_type, name);

lib/gitlab/database/reindexing.rb

@@ -28,6 +28,7 @@ def self.invoke(database = nil)
           # Hack: Before we do actual reindexing work, create async indexes
           Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops)
           Gitlab::Database::AsyncIndexes.drop_pending_indexes!
+          Gitlab::Database::AsyncForeignKeys.validate_pending_entries! if Feature.enabled?(:database_async_foreign_key_validation, type: :ops)
 
           automatic_reindexing
         end

qa/qa/specs/features/browser_ui/3_create/web_ide/web_terminal_spec.rb

@@ -122,11 +122,11 @@ def destroy_key(path, action, confirmation_button)
         project.add_developer(user)
       end
 
-      it 'revoking the SSH key marks commits as unverified',
-        quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/390905' do
+      it 'revoking the SSH key marks commits as unverified' do
         visit project_commit_path(project, commit)
+        wait_for_all_requests
 
-        find('a.gpg-status-box', text: 'Verified').click
+        page.find('a.gpg-status-box', text: 'Verified').click
 
         within('.popover') do
           expect(page).to have_content("Verified commit")
@@ -136,8 +136,9 @@ def destroy_key(path, action, confirmation_button)
         destroy_key(profile_keys_path, 'Revoke', 'Revoke')
 
         visit project_commit_path(project, commit)
+        wait_for_all_requests
 
-        find('a.gpg-status-box', text: 'Unverified').click
+        page.find('a.gpg-status-box', text: 'Unverified').click
 
         within('.popover') do
           expect(page).to have_content("Unverified signature")

spec/features/profiles/keys_spec.rb

@@ -68,6 +68,25 @@
         end
       end
     end
+
+    context 'when async FK validation is enabled' do
+      it 'executes FK validation for each database prior to any reindexing actions' do
+        expect(Gitlab::Database::AsyncForeignKeys).to receive(:validate_pending_entries!).ordered.exactly(databases_count).times
+        expect(described_class).to receive(:automatic_reindexing).ordered.exactly(databases_count).times
+
+        described_class.invoke
+      end
+    end
+
+    context 'when async FK validation is disabled' do
+      it 'does not execute FK validation' do
+        stub_feature_flags(database_async_foreign_key_validation: false)
+
+        expect(Gitlab::Database::AsyncForeignKeys).not_to receive(:validate_pending_entries!)
+
+        described_class.invoke
+      end
+    end
   end
 
   describe '.automatic_reindexing' do

spec/lib/gitlab/database/reindexing_spec.rb

@@ -21,6 +21,14 @@
 
       expect(ApplicationRecord.gitlab_transactions_stack).to be_empty
 
+      PackageMetadata::ApplicationRecord.transaction do
+        expect(ApplicationRecord.gitlab_transactions_stack).to contain_exactly(:gitlab_pm)
+
+        Project.first
+      end
+
+      expect(ApplicationRecord.gitlab_transactions_stack).to be_empty
+
       Project.transaction do
         expect(ApplicationRecord.gitlab_transactions_stack).to contain_exactly(:gitlab_main)
 

spec/models/concerns/cross_database_modification_spec.rb

@@ -15,6 +15,16 @@ def csv
     CSV.parse(subject.csv_data, headers: true)
   end
 
+  context 'when import_export_work_items_csv flag is not enabled' do
+    before do
+      stub_feature_flags(import_export_work_items_csv: false)
+    end
+
+    it 'renders an error' do
+      expect { subject.csv_data }.to raise_error(described_class::NotAvailableError)
+    end
+  end
+
   it 'renders csv to string' do
     expect(subject.csv_data).to be_a String
   end