Skip to content

HDFS-17772. Fix JournaledEditsCache int overflow while the maximum capacity to be Integer MAX_VALUE. #7617

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 21, 2025
Merged

HDFS-17772. Fix JournaledEditsCache int overflow while the maximum capacity to be Integer MAX_VALUE. #7617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a38286b
HDFS-17772. The JournaledEditsCache has an int overflow issue, causin…
Apr 16, 2025
b97c898
Merge branch 'apache:trunk' into HDFS-17772
gyz-web Apr 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions ...hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class JournaledEditsCache {
private static final long INVALID_TXN_ID = -1;

/** The capacity, in bytes, of this cache. */
private final int capacity;
private final long capacity;

/**
* Read/write lock pair wrapped in AutoCloseable; these refer to the same
Expand Down Expand Up @@ -117,7 +117,7 @@ class JournaledEditsCache {
*/
private long initialTxnId;
/** The current total size of all buffers in this cache. */
private int totalSize;
private long totalSize;

// ** End lock-protected fields **

Expand All @@ -128,8 +128,8 @@ class JournaledEditsCache {
String.format("Cache config %s is set at %f, it should be a positive float value, " +
"less than 1.0. The recommended value is less than 0.9.",
DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_FRACTION_KEY, fraction));
capacity = conf.getInt(DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY,
(int) (Runtime.getRuntime().maxMemory() * fraction));
capacity = conf.getLong(DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY,
(long) (Runtime.getRuntime().maxMemory() * fraction));
if (capacity > 0.9 * Runtime.getRuntime().maxMemory()) {
Journal.LOG.warn(String.format("Cache capacity is set at %d bytes but " +
"maximum JVM memory is only %d bytes. It is recommended that you " +
Expand Down Expand Up @@ -424,7 +424,7 @@ long getCacheMissAmount() {
}

@VisibleForTesting
int getCapacity() {
long getCapacity() {
return capacity;
}

Expand Down
4 changes: 2 additions & 2 deletions ...op-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournaledEditsCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ public void testCacheSizeConfigs() {
// Assert the default configs.
Configuration config = new Configuration();
cache = new JournaledEditsCache(config);
assertEquals((int) (Runtime.getRuntime().maxMemory() * 0.5f), cache.getCapacity());
assertEquals((long) (Runtime.getRuntime().maxMemory() * 0.5f), cache.getCapacity());

// Set dfs.journalnode.edit-cache-size.bytes.
Configuration config1 = new Configuration();
Expand All @@ -239,7 +239,7 @@ public void testCacheSizeConfigs() {
Configuration config2 = new Configuration();
config2.setFloat(DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_FRACTION_KEY, 0.1f);
cache = new JournaledEditsCache(config2);
assertEquals((int) (Runtime.getRuntime().maxMemory() * 0.1f), cache.getCapacity());
assertEquals((long) (Runtime.getRuntime().maxMemory() * 0.1f), cache.getCapacity());
}

private void storeEdits(int startTxn, int endTxn) throws Exception {
Expand Down