Add latest changes from gitlab-org/gitlab@master

Author: GitLab Bot

CHANGELOG.md

@@ -1 +1 @@
-58bf41c6a0d6024402551506cf1a40b340662453
+c45474cf95ee58efeeebe7adf6a6892b511db5ef

GITALY_SERVER_VERSION

@@ -1 +1 @@
-95b8e6a28986f6b4aaaa20283212259b1938e948
+b583acae369d671df2cb8c2a8b6fd42a2351e6bf

GITLAB_KAS_VERSION

@@ -132,7 +132,7 @@ export default {
   <gl-disclosure-dropdown :items="items" class="gl-block" placement="bottom" @shown="onShown">
     <template #toggle>
       <button
-        class="user-bar-button organization-switcher-button gl-flex gl-w-full gl-items-center gl-gap-3 gl-rounded-base gl-border-none gl-p-3 gl-text-left gl-leading-1"
+        class="user-bar-button organization-switcher-button gl-flex gl-w-full gl-items-center gl-gap-3 gl-rounded-base gl-border-none gl-p-2 gl-text-left gl-leading-1"
         data-testid="toggle-button"
       >
         <gl-avatar
@@ -150,7 +150,7 @@ export default {
     <template #list-item="{ item }">
       <gl-loading-icon v-if="item.id === $options.ITEM_LOADING.id" />
       <span v-else-if="item.id === $options.ITEM_EMPTY.id">{{ item.text }}</span>
-      <div v-else class="gl-flex gl-items-center gl-gap-3">
+      <div v-else class="-gl-m-2 gl-flex gl-items-center gl-gap-3">
         <gl-avatar
           :size="24"
           :shape="$options.AVATAR_SHAPE_OPTION_RECT"

app/assets/javascripts/super_sidebar/components/organization_switcher.vue

@@ -79,14 +79,11 @@ export default {
       }
       return '';
     },
-    mrFullPath() {
-      return `${this.itemContent.project.namespace.path}/${this.itemContent.project.path}`;
-    },
     showPath() {
-      return this.mrFullPath !== this.workItemFullPath;
+      return this.itemContent.project.fullPath !== this.workItemFullPath;
     },
     fullReferenceWithPath() {
-      return `${this.mrFullPath}${this.itemContent?.reference}`;
+      return `${this.itemContent.project.fullPath}${this.itemContent?.reference}`;
     },
     detailedStatus() {
       return this.itemContent?.headPipeline?.detailedStatus;

app/assets/javascripts/work_items/components/work_item_development/work_item_development_mr_item.vue

@@ -24,9 +24,7 @@ fragment MergeRequestFragment on MergeRequest {
     id
     name
     path
-    namespace {
-      path
-    }
+    fullPath
   }
   assignees {
     nodes {

app/assets/javascripts/work_items/graphql/merge_request.fragment.graphql

@@ -25,7 +25,10 @@ class TreeType < BaseObject
         calls_gitaly: true
 
       field :permalink_path, GraphQL::Types::String, null: true,
-        description: 'Web path to tree permalink.',
+        description: "Web path to tree permalink. " \
+          "The `permalinkPath` field returns a string that represents the web path to a specific version of " \
+          "a directory, identified by its commit SHA. Use this path to create permanent links to directories at " \
+          "specific points in your repository's history.",
         calls_gitaly: true,
         experiment: { milestone: '17.11' }
 

app/graphql/types/tree/tree_type.rb

@@ -8,6 +8,9 @@ class DiffNote < Note
   include DiffPositionableNote
   include Gitlab::Utils::StrongMemoize
 
+  # Duplicate ignore_column from Note to make the db:check-migrations job pass
+  ignore_column :attachment, remove_with: '18.1', remove_after: '2025-05-15'
+
   self.allow_legacy_sti_class = true
 
   def self.noteable_types

app/models/diff_note.rb

@@ -8,9 +8,11 @@ class ProjectGroupLink < ApplicationRecord
   belongs_to :project
   belongs_to :group
 
+  before_validation :prevent_concurrent_inserts
+
   validates :project_id, presence: true
   validates :group, presence: true
-  validates :group_id, uniqueness: { scope: [:project_id], message: N_("already shared with this group") }
+  validates :project_id, uniqueness: { scope: [:group_id], message: N_("already shared with this group") }
   validates :group_access, inclusion: { in: Gitlab::Access.all_values }, presence: true
   validate :different_group
 
@@ -35,6 +37,20 @@ def owner_access?
 
   private
 
+  # This method will block while another database transaction attempts to insert the same data.
+  # After the lock is released by the other transaction, the uniqueness validation may fail
+  # with record not unique validation error.
+
+  # Without this block the uniqueness validation wouldn't be able to detect duplicated
+  # records as transactions can't see each other's changes.
+  def prevent_concurrent_inserts
+    return if project_id.nil? || group_id.nil?
+
+    lock_key = ['project_group_links', project_id, group_id].join('-')
+    lock_expression = "hashtext(#{connection.quote(lock_key)})"
+    connection.execute("SELECT pg_advisory_xact_lock(#{lock_expression})")
+  end
+
   def different_group
     return unless self.group && self.project
 

app/models/project_group_link.rb

@@ -12,4 +12,7 @@
 
   config.verbose = false # this spams logfile a lot, set to true for debugging locally
   config.logger = Gitlab::AppLogger.primary_logger
+  # By default Puma starts in the tmp directory to mimic Omnibus GitLab, but
+  # this root path needs to be set for Coverband to identify application code.
+  config.root = Rails.root.to_s
 end

config/coverband.rb

@@ -7,3 +7,13 @@
 ENV["COVERBAND_CONFIG"] = Rails.root.join("config/coverband.rb").to_s
 
 require 'coverband'
+
+# Normally with a Rails Web app Coverband defines a Railtie, which
+# is responsible for inserting a middleware that launches a background
+# thread. However, since we dynamically require Coverband in this file,
+# it is too late for Railties to be defined now so we need to manually
+# start the background threads. In addition, each Puma process needs
+# its own background thread to report which code paths were hit.
+Gitlab::Cluster::LifecycleEvents.on_worker_start do
+  ::Coverband::Background.start
+end

config/initializers/coverband.rb

@@ -4,6 +4,11 @@
   announcement_milestone: "16.9"
   # Change breaking_change to false if needed.
   breaking_change: true
+  window: 3
+  scope: project
+  impact: medium
+  resolution_role: developer
+  manual_task: true
   # The stage and GitLab username of the person reporting the change,
   # and a link to the deprecation issue
   reporter: timofurrer

data/deprecations/16-9-deprecate-terraform-cicd-templates.yml

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# This index is being added so that we can backfill the namespace where project
+# id is NULL
+class AddTempIndexOnNotesForProjectsNullAndId < Gitlab::Database::Migration[2.2]
+  disable_ddl_transaction!
+  milestone '17.11'
+  NEW_INDEX_NAME = 'tmp_index_null_project_id_on_notes'
+
+  # Remove index issue: gitlab.com/gitlab-org/gitlab/-/issues/535969
+  # Make the index async issue: gitlab.com/gitlab-org/gitlab/-/issues/535970
+  def up
+    # rubocop:disable Migration/PreventIndexCreation -- Will be removed once the backfilling for note is complete
+    prepare_async_index(:notes,
+      :id,
+      name: NEW_INDEX_NAME,
+      where: 'project_id is NULL')
+    # rubocop:enable Migration/PreventIndexCreation
+  end
+
+  def down
+    unprepare_async_index :notes, NEW_INDEX_NAME
+  end
+end

db/post_migrate/20250411043427_add_temp_index_on_notes_for_projects_null_and_id.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class RemoveNotesAttachment < Gitlab::Database::Migration[2.2]
+  milestone '18.0'
+
+  def change
+    remove_column :notes, :attachment, :string
+  end
+end

db/post_migrate/20250414145659_remove_notes_attachment.rb

@@ -0,0 +1 @@
+db462f63722bba9d2bfdecd0a18ea3ecc0c6eba68a0ba4107752d8b133bcb487

db/schema_migrations/20250411043427

@@ -0,0 +1 @@
+f22e3a9f4396a13cf673acd69124c3cc8800c180f0669dc9abf04acbe9064116

db/schema_migrations/20250414145659

@@ -18348,7 +18348,6 @@ CREATE TABLE notes (
     created_at timestamp without time zone,
     updated_at timestamp without time zone,
     project_id bigint,
-    attachment character varying,
     line_code character varying,
     commit_id character varying,
     noteable_id bigint,

db/structure.sql

@@ -135,6 +135,43 @@ When the pack-object cache is enabled, pack-objects limiting kicks in only if th
 You can observe the behavior of this queue using Gitaly logs and Prometheus. For more information, see
 [Monitor Gitaly pack-objects concurrency limiting](monitoring.md#monitor-gitaly-pack-objects-concurrency-limiting).
 
+## Calibrating concurrency limits
+
+When setting concurrency limits, you should choose appropriate values based on your specific workload patterns. This section provides guidance on how to calibrate these limits effectively.
+
+### Using Prometheus metrics and logs for calibration
+
+Prometheus metrics provide quantitative insights into usage patterns and the impact of each type of RPC on Gitaly node resources. Several key metrics are particularly valuable for this analysis:
+
+- Resource consumption metrics per-RPC. Gitaly offloads most heavy operations to `git` processes and so the command usually shelled out to is the Git binary.
+  Gitaly exposes collected metrics from those commands as logs and Prometheus metrics.
+  - `gitaly_command_cpu_seconds_total` - Sum of CPU time spent by shelling out, with labels for `grpc_service`, `grpc_method`, `cmd`, and `subcmd`.
+  - `gitaly_command_real_seconds_total` - Sum of real time spent by shelling out, with similar labels.
+- Recent limiting metrics per-RPC:
+  - `gitaly_concurrency_limiting_in_progress` - Number of concurrent requests being processed.
+  - `gitaly_concurrency_limiting_queued` - Number of requests for an RPC for a given repository in waiting state.
+  - `gitaly_concurrency_limiting_acquiring_seconds` - Duration a request waits due to concurrency limits before processing.
+
+These metrics provide a high-level view of resource utilization at a given point in time. The `gitaly_command_cpu_seconds_total` metric is particularly effective for
+identifying specific RPCs that consume substantial CPU resources. Additional metrics are available for more detailed analysis as described in
+[Monitoring Gitaly](monitoring.md).
+
+While metrics capture overall resource usage patterns, they typically do not provide per-repository breakdowns. Therefore, logs serve as a complementary data source. To analyze logs:
+
+1. Filter logs by identified high-impact RPCs.
+1. Aggregate filtered logs by repository or project.
+1. Visualize aggregated results on a time-series graph.
+
+This combined approach of using both metrics and logs provides comprehensive visibility into both system-wide resource usage and repository-specific patterns. Analysis tools such as Kibana or similar log aggregation platforms can facilitate this process.
+
+### Adjusting limits
+
+If you find that your initial limits are not efficient enough, you might need to adjust them. With adaptive limiting, precise limits are less critical because the system
+automatically adjusts based on resource usage.
+
+Remember that concurrency limits are scoped by repository. A limit of 30 means allowing at most 30 simultaneous in-flight requests per repository. If the limit is reached,
+requests are queued and only rejected if the queue is full or the maximum waiting time is reached.
+
 ## Adaptive concurrency limiting
 
 {{< history >}}
@@ -181,7 +218,12 @@ The adaptive limiter calibrates the limits every 30 seconds and:
 Otherwise, the limits increase by one until reaching the upper bound. For more information about technical implementation
 of this system, refer to [the related design document](https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/gitaly_adaptive_concurrency_limit/).
 
-Adaptive limiting is enabled for each RPC or pack-objects cache individually. However, limits are calibrated at the same time.
+Adaptive limiting is enabled for each RPC or pack-objects cache individually. However, limits are calibrated at the same time. Adaptive limiting has the following configurations:
+
+- `adaptive` sets whether the adaptiveness is enabled.
+- `max_limit` is the maximum concurrency limit. Gitaly increases the current limit until it reaches this number. This should be a generous value that the system can fully support under typical conditions.
+- `min_limit` is the is the minimum concurrency limit of the configured RPC. When the host machine has a resource problem, Gitaly quickly reduces the limit until reaching this value. Setting `min_limit` to 0 could completely shut down processing, which is typically undesirable.
+- `initial_limit` provides a reasonable starting point between these extremes.
 
 ### Enable adaptiveness for RPC concurrency
 
@@ -224,15 +266,6 @@ gitaly['configuration'] = {
 }
 ```
 
-In this example:
-
-- `adaptive` sets whether the adaptiveness is enabled. If set, the `max_per_repo` value is ignored in favor of the following configuration.
-- `initial_limit` is the per-repository concurrency limit to use when Gitaly starts.
-- `max_limit` is the minimum per-repository concurrency limit of the configured RPC. Gitaly increases the current limit
-  until it reaches this number.
-- `min_limit` is the is the minimum per-repository concurrency limit of the configured RPC. When the host machine has a resource problem,
-  Gitaly quickly reduces the limit until reaching this value.
-
 For more information, see [RPC concurrency](#limit-rpc-concurrency).
 
 ### Enable adaptiveness for pack-objects concurrency
@@ -255,12 +288,75 @@ gitaly['pack_objects_limiting'] = {
 }
 ```
 
-In this example:
+For more information, see [pack-objects concurrency](#limit-pack-objects-concurrency).
+
+### Calibrating adaptive concurrency limits
 
-- `adaptive` sets whether the adaptiveness is enabled. If set, the value of `max_concurrency` is ignored in favor of the following configuration.
-- `initial_limit` is the per-IP concurrency limit to use when Gitaly starts.
-- `max_limit` is the minimum per-IP concurrency limit for pack-objects. Gitaly increases the current limit until it reaches this number.
-- `min_limit` is the is the minimum per-IP concurrency limit for pack-objects. When the host machine has a resources problem, Gitaly quickly
-  reduces the limit until it reaches this value.
+Adaptive concurrency limiting is very different from the usual way that GitLab protects Gitaly resources. Rather than relying on static thresholds that may be either too restrictive or too permissive, adaptive limiting intelligently responds to actual resource conditions in real-time.
 
-For more information, see [pack-objects concurrency](#limit-pack-objects-concurrency).
+This approach eliminates the need to find "perfect" threshold values through extensive calibration as described in
+[Calibrating concurrency limits](#calibrating-concurrency-limits). During failure scenarios, the adaptive limiter reduces limits exponentially (for example, 60 → 30 → 15 → 10)
+and then automatically recovers by incrementally raising limits when the system stabilizes.
+
+When calibrating adaptive limits, you can prioritize flexibility over precision.
+
+#### RPC categories and configuration examples
+
+Expensive Gitaly RPCs, which should be protected, can be categorized into two general types:
+
+- Pure Git data operations.
+- Time sensitive RPCs.
+
+Each type has distinct characteristics that influence how concurrency limits should be configured. The following examples illustrate the reasoning behind
+limit configuration. They can also be used as a starting point.
+
+##### Pure Git data operations
+
+These RPCs involve Git pull, push, and fetch operations, and possess the following characteristics:
+
+- Long-running processes.
+- Significant resource utilization.
+- Computationally expensive.
+- Not time-sensitive. Additional latency is generally acceptable.
+
+RPCs in `SmartHTTPService` and `SSHService` fall into the pure Git data operations category. A configuration example:
+
+```ruby
+{
+  rpc: "/gitaly.SmartHTTPService/PostUploadPackWithSidechannel", # or `/gitaly.SmartHTTPService/SSHUploadPackWithSidechannel`
+  adaptive: true,
+  min_limit: 10,  # Minimum concurrency to maintain even under extreme load
+  initial_limit: 40,  # Starting concurrency when service initializes
+  max_limit: 60,  # Maximum concurrency under ideal conditions
+  max_queue_wait: "60s",
+  max_queue_size: 300
+}
+```
+
+##### Time-sensitive RPCs
+
+These RPCs serve GitLab itself and other clients with different characteristics:
+
+- Typically part of online HTTP requests or Sidekiq background jobs.
+- Shorter latency profiles.
+- Generally less resource-intensive.
+
+For these RPCs, the timeout configuration in GitLab should inform the `max_queue_wait` parameter. For instance, `get_tree_entries` typically has a medium timeout of 30 seconds in GitLab:
+
+```ruby
+{
+  rpc: "/gitaly.CommitService/GetTreeEntries",
+  adaptive: true,
+  min_limit: 5,  # Minimum throughput maintained under resource pressure
+  initial_limit: 10,  # Initial concurrency setting
+  max_limit: 20,  # Maximum concurrency under optimal conditions
+  max_queue_size: 50,
+  max_queue_wait: "30s"
+}
+```
+
+### Monitoring adaptive limiting
+
+To observe how adaptive limits are behaving in production environments, refer to the monitoring tools and metrics described in
+[Monitor Gitaly adaptive concurrency limiting](monitoring.md#monitor-gitaly-adaptive-concurrency-limiting). Observing adaptive limit behavior helps confirm that limits
+are properly responding to resource pressures and adjusting as expected.