diff --git a/boms/geode-all-bom/src/test/resources/expected-pom.xml b/boms/geode-all-bom/src/test/resources/expected-pom.xml index e6d9b7401a9..4fe29c1ccce 100644 --- a/boms/geode-all-bom/src/test/resources/expected-pom.xml +++ b/boms/geode-all-bom/src/test/resources/expected-pom.xml @@ -475,22 +475,22 @@ com.fasterxml.jackson.core jackson-core - 2.21.6 + 2.21.8 com.fasterxml.jackson.core jackson-databind - 2.21.6 + 2.21.8 com.fasterxml.jackson.datatype jackson-datatype-joda - 2.21.6 + 2.21.8 com.fasterxml.jackson.datatype jackson-datatype-jsr310 - 2.21.6 + 2.21.8 com.jayway.jsonpath diff --git a/build-tools/geode-dependency-management/src/main/groovy/org/apache/geode/gradle/plugins/DependencyConstraints.groovy b/build-tools/geode-dependency-management/src/main/groovy/org/apache/geode/gradle/plugins/DependencyConstraints.groovy index 5d8f228089d..4e938d6d2f2 100644 --- a/build-tools/geode-dependency-management/src/main/groovy/org/apache/geode/gradle/plugins/DependencyConstraints.groovy +++ b/build-tools/geode-dependency-management/src/main/groovy/org/apache/geode/gradle/plugins/DependencyConstraints.groovy @@ -44,9 +44,9 @@ class DependencyConstraints { deps.put("shiro.version", "1.13.0") deps.put("slf4j-api.version", "1.7.36") deps.put("jboss-modules.version", "1.11.0.Final") - deps.put("jackson.version", "2.21.6") + deps.put("jackson.version", "2.21.8") deps.put("jackson.annotations.version", "2.21") - deps.put("jackson.databind.version", "2.21.6") + deps.put("jackson.databind.version", "2.21.8") deps.put("springshell.version", "1.2.0.RELEASE") deps.put("springframework.version", "5.3.20") diff --git a/geode-assembly/src/integrationTest/resources/assembly_content.txt b/geode-assembly/src/integrationTest/resources/assembly_content.txt index ef49276b907..2852932581a 100644 --- a/geode-assembly/src/integrationTest/resources/assembly_content.txt +++ b/geode-assembly/src/integrationTest/resources/assembly_content.txt @@ -1005,10 +1005,10 @@ lib/httpclient-4.5.13.jar lib/httpcore-4.4.15.jar lib/istack-commons-runtime-4.0.1.jar lib/jackson-annotations-2.21.jar -lib/jackson-core-2.21.6.jar -lib/jackson-databind-2.21.6.jar -lib/jackson-datatype-joda-2.21.6.jar -lib/jackson-datatype-jsr310-2.21.6.jar +lib/jackson-core-2.21.8.jar +lib/jackson-databind-2.21.8.jar +lib/jackson-datatype-joda-2.21.8.jar +lib/jackson-datatype-jsr310-2.21.8.jar lib/javax.activation-api-1.2.0.jar lib/javax.mail-api-1.6.2.jar lib/javax.resource-api-1.7.1.jar diff --git a/geode-assembly/src/integrationTest/resources/gfsh_dependency_classpath.txt b/geode-assembly/src/integrationTest/resources/gfsh_dependency_classpath.txt index 5b8e94562d2..bd4a60b51cc 100644 --- a/geode-assembly/src/integrationTest/resources/gfsh_dependency_classpath.txt +++ b/geode-assembly/src/integrationTest/resources/gfsh_dependency_classpath.txt @@ -21,10 +21,10 @@ spring-shell-1.2.0.RELEASE.jar spring-web-5.3.20.jar commons-lang3-3.18.0.jar rmiio-2.1.2.jar -jackson-datatype-joda-2.21.6.jar -jackson-core-2.21.6.jar -jackson-datatype-jsr310-2.21.6.jar -jackson-databind-2.21.6.jar +jackson-datatype-joda-2.21.8.jar +jackson-core-2.21.8.jar +jackson-datatype-jsr310-2.21.8.jar +jackson-databind-2.21.8.jar swagger-annotations-2.2.0.jar jopt-simple-5.0.4.jar log4j-slf4j-impl-2.25.5.jar diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java b/geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java index b544ca065fe..646876ace00 100644 --- a/geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java +++ b/geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java @@ -70,6 +70,7 @@ import it.unimi.dsi.fastutil.longs.LongOpenHashSet; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.TestOnly; import org.apache.geode.CancelCriterion; import org.apache.geode.CancelException; @@ -195,7 +196,7 @@ public class DiskStoreImpl implements DiskStore { GeodeGlossary.GEMFIRE_PREFIX + "disk.recoverLruValues"; static final long DRF_HASHMAP_OVERFLOW_THRESHOLD_DEFAULT = 805306368; - static final long DRF_HASHMAP_OVERFLOW_THRESHOLD = + final long DRF_HASHMAP_OVERFLOW_THRESHOLD = Long.getLong(DRF_HASHMAP_OVERFLOW_THRESHOLD_NAME, DRF_HASHMAP_OVERFLOW_THRESHOLD_DEFAULT); boolean RECOVER_VALUES = getBoolean(DiskStoreImpl.RECOVER_VALUE_PROPERTY_NAME, true); @@ -3507,18 +3508,26 @@ static class OplogEntryIdSet { private final List allLongs; private final AtomicReference currentInts; private final AtomicReference currentLongs; + private final long drfHashMapOverFlowThreashold; // For testing purposes only. @VisibleForTesting - OplogEntryIdSet(List allInts, List allLongs) { + OplogEntryIdSet(List allInts, List allLongs, + long drfHashMapOverflowThreshold) { this.allInts = allInts; currentInts = new AtomicReference<>(this.allInts.get(0)); this.allLongs = allLongs; currentLongs = new AtomicReference<>(this.allLongs.get(0)); + this.drfHashMapOverFlowThreashold = drfHashMapOverflowThreshold; } - public OplogEntryIdSet() { + @TestOnly + OplogEntryIdSet(List allInts, List allLongs) { + this(allInts, allLongs, DRF_HASHMAP_OVERFLOW_THRESHOLD_DEFAULT); + } + + public OplogEntryIdSet(long drfHashMapOverflowThreshold) { IntOpenHashSet intHashSet = new IntOpenHashSet((int) INVALID_ID); allInts = new ArrayList<>(); allInts.add(intHashSet); @@ -3528,6 +3537,11 @@ public OplogEntryIdSet() { allLongs = new ArrayList<>(); allLongs.add(longHashSet); currentLongs = new AtomicReference<>(longHashSet); + this.drfHashMapOverFlowThreashold = drfHashMapOverflowThreshold; + } + + public OplogEntryIdSet() { + this(DRF_HASHMAP_OVERFLOW_THRESHOLD_DEFAULT); } public void add(long id) { @@ -3555,14 +3569,14 @@ public void add(long id) { boolean shouldOverflow(final long id) { if (id > 0 && id <= 0x00000000FFFFFFFFL) { - return currentInts.get().size() == DRF_HASHMAP_OVERFLOW_THRESHOLD; + return currentInts.get().size() == drfHashMapOverFlowThreashold; } else { - return currentLongs.get().size() == DRF_HASHMAP_OVERFLOW_THRESHOLD; + return currentLongs.get().size() == drfHashMapOverFlowThreashold; } } void overflowToNewHashMap(final long id) { - if (DRF_HASHMAP_OVERFLOW_THRESHOLD == DRF_HASHMAP_OVERFLOW_THRESHOLD_DEFAULT) { + if (drfHashMapOverFlowThreashold == DRF_HASHMAP_OVERFLOW_THRESHOLD_DEFAULT) { logger.warn( "There is a large number of deleted entries within the disk-store, please execute an offline compaction."); } diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/PersistentOplogSet.java b/geode-core/src/main/java/org/apache/geode/internal/cache/PersistentOplogSet.java index 67291651a33..80a9378cfdc 100644 --- a/geode-core/src/main/java/org/apache/geode/internal/cache/PersistentOplogSet.java +++ b/geode-core/src/main/java/org/apache/geode/internal/cache/PersistentOplogSet.java @@ -440,7 +440,7 @@ void recoverRegionsThatAreReady() { } private long recoverOplogs(long byteCount) { - OplogEntryIdSet deletedIds = new OplogEntryIdSet(); + OplogEntryIdSet deletedIds = new OplogEntryIdSet(parent.DRF_HASHMAP_OVERFLOW_THRESHOLD); TreeSet oplogSet = getSortedOplogs(); if (!getAlreadyRecoveredOnce().get()) { diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/DiskStoreImplTest.java b/geode-core/src/test/java/org/apache/geode/internal/cache/DiskStoreImplTest.java new file mode 100644 index 00000000000..af025938e83 --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/internal/cache/DiskStoreImplTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.internal.cache; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junitpioneer.jupiter.SetSystemProperty; + +import org.apache.geode.Statistics; +import org.apache.geode.distributed.internal.InternalDistributedSystem; +import org.apache.geode.internal.statistics.StatisticsManager; + +class DiskStoreImplTest { + @Test + @SetSystemProperty(key = "gemfire.disk.drfHashMapOverflowThreshold", value = "10") + public void testDrfHashMapOverflowThresholdSystemPropertyIsUsed(@TempDir File dir1, + @TempDir File dir2) { + InternalCache cache = mock(InternalCache.class); + InternalDistributedSystem internalDistributedSystem = mock(InternalDistributedSystem.class); + DiskStoreAttributes diskStoreAttributes = mock(DiskStoreAttributes.class); + StatisticsManager statisticsManager = mock(StatisticsManager.class); + + when(internalDistributedSystem.getStatisticsManager()).thenReturn(statisticsManager); + when(cache.getInternalDistributedSystem()).thenReturn(internalDistributedSystem); + when(diskStoreAttributes.getDiskDirs()).thenReturn( + new File[] {dir1, dir2}); + when(diskStoreAttributes.getDiskDirSizes()).thenReturn(new int[] {1, 1}); + when(diskStoreAttributes.getDiskDirSizesUnit()).thenReturn(DiskDirSizesUnit.MEGABYTES); + when(statisticsManager.createStatistics(any(), any())).thenReturn(mock(Statistics.class)); + + DiskStoreImpl diskStore = new DiskStoreImpl(cache, diskStoreAttributes); + + Assertions.assertThat(diskStore.DRF_HASHMAP_OVERFLOW_THRESHOLD).isEqualTo(10); + } +} diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/OplogEntryIdSetDrfHashSetThresholdTest.java b/geode-core/src/test/java/org/apache/geode/internal/cache/OplogEntryIdSetDrfHashSetThresholdTest.java index ff7e43e7b54..e4b3c56f36c 100644 --- a/geode-core/src/test/java/org/apache/geode/internal/cache/OplogEntryIdSetDrfHashSetThresholdTest.java +++ b/geode-core/src/test/java/org/apache/geode/internal/cache/OplogEntryIdSetDrfHashSetThresholdTest.java @@ -25,7 +25,6 @@ import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import it.unimi.dsi.fastutil.longs.LongOpenHashSet; import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.SetSystemProperty; import org.apache.geode.internal.cache.DiskStoreImpl.OplogEntryIdSet; @@ -34,9 +33,7 @@ */ public class OplogEntryIdSetDrfHashSetThresholdTest { @Test - @SetSystemProperty(key = "gemfire.disk.drfHashMapOverflowThreshold", value = "10") public void addMethodOverflowBasedOnDrfOverflowThresholdParameters() { - int testEntries = 41; IntOpenHashSet intOpenHashSet = new IntOpenHashSet(); LongOpenHashSet longOpenHashSet = new LongOpenHashSet(); @@ -46,7 +43,7 @@ public void addMethodOverflowBasedOnDrfOverflowThresholdParameters() { List longOpenHashSets = new ArrayList<>(Collections.singletonList(longOpenHashSet)); - OplogEntryIdSet oplogEntryIdSet = new OplogEntryIdSet(intOpenHashSets, longOpenHashSets); + OplogEntryIdSet oplogEntryIdSet = new OplogEntryIdSet(intOpenHashSets, longOpenHashSets, 10); IntStream.range(1, testEntries).forEach(oplogEntryIdSet::add); LongStream.range(0x00000000FFFFFFFFL + 1, 0x00000000FFFFFFFFL + testEntries) .forEach(oplogEntryIdSet::add); diff --git a/geode-server-all/src/integrationTest/resources/dependency_classpath.txt b/geode-server-all/src/integrationTest/resources/dependency_classpath.txt index d220bdf79fa..79181d53489 100644 --- a/geode-server-all/src/integrationTest/resources/dependency_classpath.txt +++ b/geode-server-all/src/integrationTest/resources/dependency_classpath.txt @@ -9,7 +9,7 @@ spring-jcl-5.3.20.jar commons-codec-1.15.jar classgraph-4.8.146.jar commons-logging-1.3.5.jar -jackson-databind-2.21.6.jar +jackson-databind-2.21.8.jar geode-management-0.0.0.jar geode-core-0.0.0.jar javax.activation-api-1.2.0.jar @@ -44,7 +44,7 @@ rmiio-2.1.2.jar geode-tcp-server-0.0.0.jar log4j-jcl-2.25.5.jar geode-connectors-0.0.0.jar -jackson-core-2.21.6.jar +jackson-core-2.21.8.jar jetty-util-9.4.57.v20241219.jar log4j-slf4j-impl-2.25.5.jar lucene-analyzers-common-6.6.6.jar @@ -90,6 +90,6 @@ antlr-2.7.7.jar jetty-xml-9.4.57.v20241219.jar geode-rebalancer-0.0.0.jar jetty-server-9.4.57.v20241219.jar -jackson-datatype-jsr310-2.21.6.jar -jackson-datatype-joda-2.21.6.jar +jackson-datatype-jsr310-2.21.8.jar +jackson-datatype-joda-2.21.8.jar joda-time-2.12.7.jar