Merge branch 'master' into batch_read_5

bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java

@@ -1151,7 +1151,7 @@ public ClientConfiguration setRecoveryReadBatchSize(int batchSize) {
      * @return true if reorder read sequence is enabled, otherwise false.
      */
     public boolean isReorderReadSequenceEnabled() {
-        return getBoolean(REORDER_READ_SEQUENCE_ENABLED, false);
+        return getBoolean(REORDER_READ_SEQUENCE_ENABLED, true);
     }
 
     /**

bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java

@@ -96,6 +96,7 @@ public abstract class MockBookKeeperTestCase {
     protected BookieClient bookieClient;
     protected LedgerManager ledgerManager;
     protected LedgerIdGenerator ledgerIdGenerator;
+    protected EnsemblePlacementPolicy placementPolicy;
 
     private BookieWatcher bookieWatcher;
 
@@ -152,6 +153,7 @@ public void setup() throws Exception {
         scheduler = OrderedScheduler.newSchedulerBuilder().numThreads(4).name("bk-test").build();
         executor = OrderedExecutor.newBuilder().build();
         bookieWatcher = mock(BookieWatcher.class);
+        placementPolicy = new DefaultEnsemblePlacementPolicy();
 
         bookieClient = mock(BookieClient.class);
         ledgerManager = mock(LedgerManager.class);
@@ -194,7 +196,7 @@ public BookieWatcher getBookieWatcher() {
 
                 @Override
                 public EnsemblePlacementPolicy getPlacementPolicy() {
-                    return null;
+                    return placementPolicy;
                 }
 
                 @Override

bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ReadLastConfirmedAndEntryOpTest.java

@@ -85,7 +85,7 @@ public class ReadLastConfirmedAndEntryOpTest {
     private ScheduledExecutorService scheduler;
     private OrderedScheduler orderedScheduler;
     private ClientInternalConf internalConf;
-    private EnsemblePlacementPolicy mockPlacementPolicy;
+    private EnsemblePlacementPolicy placementPolicy;
     private LedgerMetadata ledgerMetadata;
     private DistributionSchedule distributionSchedule;
     private DigestManager digestManager;
@@ -121,10 +121,11 @@ public void setup() throws Exception {
             .build();
 
         this.mockBookieClient = mock(BookieClient.class);
-        this.mockPlacementPolicy = mock(EnsemblePlacementPolicy.class);
+        //this.mockPlacementPolicy = mock(EnsemblePlacementPolicy.class);
+        this.placementPolicy = new DefaultEnsemblePlacementPolicy();
         this.mockClientCtx = mock(ClientContext.class);
         when(mockClientCtx.getBookieClient()).thenReturn(mockBookieClient);
-        when(mockClientCtx.getPlacementPolicy()).thenReturn(mockPlacementPolicy);
+        when(mockClientCtx.getPlacementPolicy()).thenReturn(placementPolicy);
         when(mockClientCtx.getConf()).thenReturn(internalConf);
         when(mockClientCtx.getScheduler()).thenReturn(orderedScheduler);
         when(mockClientCtx.getMainWorkerPool()).thenReturn(orderedScheduler);

conf/bk_server.conf

@@ -1057,6 +1057,9 @@ statsProviderClass=org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvi
 # Enable/disable having read operations for a ledger to be sticky to a single bookie.
 stickyReadSEnabled=true
 
+# Enable/disable reordering read sequence on reading entries.
+reorderReadSequenceEnabled=true
+
 # The grace period, in milliseconds, that the replication worker waits before fencing and
 # replicating a ledger fragment that's still being written to upon bookie failure.
 # openLedgerRereplicationGracePeriod=30000

deploy/kubernetes/gke/bookkeeper.statefulset.yml

@@ -124,7 +124,6 @@ metadata:
     app: bookkeeper
     component: bookie
 spec:
-  ports:
   ports:
     - name: bookie
       port: 3181

site3/website/docusaurus.config.js

@@ -7,7 +7,7 @@ const baseUrl = process.env.BASE_URL || "/"
 const deployUrl = process.env.DEPLOY_URL || "https://bookkeeper.apache.org";
 const variables = {
   /** They are used in .md files*/
-  latest_release: "4.16.3",
+  latest_release: "4.16.4",
   stable_release: "4.14.8",
   github_repo: "https://github.com/apache/bookkeeper",
   github_master: "https://github.com/apache/bookkeeper/tree/master",

site3/website/src/pages/release-notes.md

@@ -1,6 +1,48 @@
 <!-- markdown-link-check-disable -->
 # Release notes
 
+## 4.16.4
+
+Release 4.16.4 includes multiple bug fixes and improvements, also we have a few dependency updates.
+
+Apache BookKeeper users are encouraged to upgrade to 4.16.4 if you are using 4.16.x.
+The technical details of this release are summarized below.
+
+### Highlights
+
+#### Bugs
+* Fix calculate checkSum when using Java9IntHash [PR #4140](https://github.com/apache/bookkeeper/pull/4140)
+* Fix the autorecovery failed replicate by add entry fenced error [PR #4163](https://github.com/apache/bookkeeper/pull/4163)
+* Fixing memory leak error when using DirectEntryLogger [PR #4135](https://github.com/apache/bookkeeper/pull/4135)
+* Fix bug of negative JournalQueueSize [PR #4077](https://github.com/apache/bookkeeper/pull/4077)
+* Fix NoSuchElementException when rereplicate empty ledgers [PR #4039](https://github.com/apache/bookkeeper/pull/4039)
+* Change the method getUnderreplicatedFragments to the package private [PR #4174](https://github.com/apache/bookkeeper/pull/4174)
+* Fix auditor elector executor block problem. [PR #4165](https://github.com/apache/bookkeeper/pull/4165)
+* Fix auditor thread leak problem. [PR #4162](https://github.com/apache/bookkeeper/pull/4162)
+* Use Flaky flag to skip testBookieServerZKSessionExpireBehaviour test [PR #4144](https://github.com/apache/bookkeeper/pull/4144)
+* Add ledgersCount.incrementAndGet in setExplicitLac function [PR #4138](https://github.com/apache/bookkeeper/pull/4138)
+* Fix no known bookies after reset racks for all BKs [PR #4128](https://github.com/apache/bookkeeper/pull/4128)
+* Fix a slow gc thread shutdown when compacting [PR #4127](https://github.com/apache/bookkeeper/pull/4127)
+* Remove the unused logs in the CleanupLedgerManager.recordPromise [PR #4121](https://github.com/apache/bookkeeper/pull/4121)
+* Fix Flaky-test: HandleFailuresTest.testHandleFailureBookieNotInWriteSet [PR #4110](https://github.com/apache/bookkeeper/pull/4110)
+* Ignore the empty `perRegionPlacement` when RegionAwareEnsemblePlacementPolicy#newEnsemble [PR #4106](https://github.com/apache/bookkeeper/pull/4106)
+* Fix LedgerHandle `ensembleChangeCounter` not used. [PR #4103](https://github.com/apache/bookkeeper/pull/4103)
+* Tune the TestReplicationWorker test. [PR #4093](https://github.com/apache/bookkeeper/pull/4093)
+* Make AuditorBookieTest#waitForNewAuditor stronger. [PR #4078](https://github.com/apache/bookkeeper/pull/4078)
+* Print compaction progress [PR #4071](https://github.com/apache/bookkeeper/pull/4071)
+* Fix readEntry parameter order [PR #4059](https://github.com/apache/bookkeeper/pull/4059)
+* Skip sync the RocksDB when no changes [PR #3904](https://github.com/apache/bookkeeper/pull/3904)
+* Try to use jdk api to create hardlink when rename file when compaction. [PR #3876](https://github.com/apache/bookkeeper/pull/3876)
+
+#### Dependency updates
+* Upgrade Zookeeper to 3.8.3 to address CVE-2023-44981 [PR #4112](https://github.com/apache/bookkeeper/pull/4112)
+* Update Jetty dependency [PR #4141](https://github.com/apache/bookkeeper/pull/4141)
+* Upgrade bc-fips to 1.0.2.4 to fix CVE-2022-45146 [PR #3915](https://github.com/apache/bookkeeper/pull/3915)
+
+#### Details
+
+https://github.com/apache/bookkeeper/pulls?q=is%3Apr+label%3Arelease%2F4.16.4+is%3Amerged+
+
 ## 4.15.5
 
 Release 4.15.5 includes multiple bug fixes and improvements, also we have a few dependency updates.

site3/website/versioned_docs/version-4.16.3/api/ledger-api.md → site3/website/versioned_docs/version-4.16.4/api/ledger-api.md

@@ -21,7 +21,7 @@ If you're using [Maven](https://maven.apache.org/), add this to your [`pom.xml`]
 
 ```xml
 <!-- in your <properties> block -->
-<bookkeeper.version>4.16.3</bookkeeper.version>
+<bookkeeper.version>4.16.4</bookkeeper.version>
 
 <!-- in your <dependencies> block -->
 <dependency>
@@ -37,7 +37,7 @@ shaded library, which relocate classes of protobuf and guava into a different na
 
 ```xml
 <!-- in your <properties> block -->
-<bookkeeper.version>4.16.3</bookkeeper.version>
+<bookkeeper.version>4.16.4</bookkeeper.version>
 
 <!-- in your <dependencies> block -->
 <dependency>
@@ -53,12 +53,12 @@ If you're using [Gradle](https://gradle.org/), add this to your [`build.gradle`]
 
 ```groovy
 dependencies {
-    compile group: 'org.apache.bookkeeper', name: 'bookkeeper-server', version: '4.16.3'
+    compile group: 'org.apache.bookkeeper', name: 'bookkeeper-server', version: '4.16.4'
 }
 
 // Alternatively:
 dependencies {
-    compile 'org.apache.bookkeeper:bookkeeper-server:4.16.3'
+    compile 'org.apache.bookkeeper:bookkeeper-server:4.16.4'
 }
 ```
 

site3/website/versioned_docs/version-4.16.3/overview/overview.md → site3/website/versioned_docs/version-4.16.4/overview/overview.md

@@ -1,6 +1,6 @@
 ---
 id: overview
-title: Apache BookKeeper 4.16.3-SNAPSHOT
+title: Apache BookKeeper 4.16.4-SNAPSHOT
 ---
 <!--
 Licensed to the Apache Software Foundation (ASF) under one
@@ -21,7 +21,7 @@ specific language governing permissions and limitations
 under the License.
 -->
 
-This documentation is for Apache BookKeeper&trade; version 4.16.3.
+This documentation is for Apache BookKeeper&trade; version 4.16.4.
 
 Apache BookKeeper&trade; is a scalable, fault-tolerant, low-latency storage service optimized for real-time workloads. It offers durability, replication, and strong consistency as essentials for building reliable real-time applications.
 
@@ -39,7 +39,7 @@ Object/[BLOB](https://en.wikipedia.org/wiki/Binary_large_object) storage | Stori
 
 Learn more about Apache BookKeeper&trade; and what it can do for your organization:
 
-- [Apache BookKeeper 4.16.3 Release Notes](/release-notes#4163)
+- [Apache BookKeeper 4.16.4 Release Notes](/release-notes#4164)
 - [Java API docs]({{ site.javadoc_base_url }})
 
 Or start [using](../getting-started/installation) Apache BookKeeper today.

site3/website/versioned_sidebars/version-4.16.3-sidebars.json → site3/website/versioned_sidebars/version-4.16.4-sidebars.json

@@ -2,7 +2,7 @@
   "docsSidebar": [
     {
       "type": "doc",
-      "id": "version-4.16.3/overview/overview",
+      "id": "version-4.16.4/overview/overview",
       "label": "Overview"
     },
     {
@@ -12,17 +12,17 @@
       "items": [
         {
           "type": "doc",
-          "id": "version-4.16.3/getting-started/installation",
+          "id": "version-4.16.4/getting-started/installation",
           "label": "Installation"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/getting-started/run-locally",
+          "id": "version-4.16.4/getting-started/run-locally",
           "label": "Run bookies locally"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/getting-started/concepts",
+          "id": "version-4.16.4/getting-started/concepts",
           "label": "Concepts and architecture"
         }
       ]
@@ -33,12 +33,12 @@
       "items": [
         {
           "type": "doc",
-          "id": "version-4.16.3/deployment/manual",
+          "id": "version-4.16.4/deployment/manual",
           "label": "Manual deployment"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/deployment/kubernetes",
+          "id": "version-4.16.4/deployment/kubernetes",
           "label": "BookKeeper on Kubernetes"
         }
       ]
@@ -49,32 +49,32 @@
       "items": [
         {
           "type": "doc",
-          "id": "version-4.16.3/admin/bookies",
+          "id": "version-4.16.4/admin/bookies",
           "label": "BookKeeper administration"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/admin/autorecovery",
+          "id": "version-4.16.4/admin/autorecovery",
           "label": "AutoRecovery"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/admin/metrics",
+          "id": "version-4.16.4/admin/metrics",
           "label": "Metrics collection"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/admin/upgrade",
+          "id": "version-4.16.4/admin/upgrade",
           "label": "Upgrade"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/admin/http",
+          "id": "version-4.16.4/admin/http",
           "label": "Admin REST API"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/admin/decomission",
+          "id": "version-4.16.4/admin/decomission",
           "label": "Decommissioning Bookies"
         }
       ]
@@ -85,22 +85,22 @@
       "items": [
         {
           "type": "doc",
-          "id": "version-4.16.3/api/overview",
+          "id": "version-4.16.4/api/overview",
           "label": "Overview"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/api/ledger-api",
+          "id": "version-4.16.4/api/ledger-api",
           "label": "Ledger API"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/api/ledger-adv-api",
+          "id": "version-4.16.4/api/ledger-adv-api",
           "label": "Advanced Ledger API"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/api/distributedlog-api",
+          "id": "version-4.16.4/api/distributedlog-api",
           "label": "DistributedLog"
         }
       ]
@@ -111,22 +111,22 @@
       "items": [
         {
           "type": "doc",
-          "id": "version-4.16.3/security/overview",
+          "id": "version-4.16.4/security/overview",
           "label": "Overview"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/security/tls",
+          "id": "version-4.16.4/security/tls",
           "label": "TLS Authentication"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/security/sasl",
+          "id": "version-4.16.4/security/sasl",
           "label": "SASL Authentication"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/security/zookeeper",
+          "id": "version-4.16.4/security/zookeeper",
           "label": "ZooKeeper Authentication"
         }
       ]
@@ -137,7 +137,7 @@
       "items": [
         {
           "type": "doc",
-          "id": "version-4.16.3/development/protocol",
+          "id": "version-4.16.4/development/protocol",
           "label": "BookKeeper protocol"
         }
       ]
@@ -148,12 +148,12 @@
       "items": [
         {
           "type": "doc",
-          "id": "version-4.16.3/reference/config",
+          "id": "version-4.16.4/reference/config",
           "label": "Configuration"
         },
         {
           "type": "doc",
-          "id": "version-4.16.3/reference/cli",
+          "id": "version-4.16.4/reference/cli",
           "label": "Command-line tools"
         }
       ]

site3/website/versions.json

@@ -1,5 +1,5 @@
 [
-  "4.16.3",
+  "4.16.4",
   "4.15.5",
   "4.14.8",
   "4.13.0",

tests/docker-images/statestore-image/Dockerfile

@@ -40,10 +40,10 @@ RUN set -x \
     && apt-get install -y --no-install-recommends python3 pip \
     && ln -s /usr/bin/python3 /usr/bin/python \
     && apt-get install -y --no-install-recommends gpg gpg-agent wget sudo \
-     && apt-get -y --purge autoremove \
-     && apt-get autoclean \
-     && apt-get clean \
-     && rm -rf /var/lib/apt/lists/*
+    && apt-get -y --purge autoremove \
+    && apt-get autoclean \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
     && pip install zk-shell \
     && mkdir -pv /opt \
     && cd /opt