From 20e620709e3acab1335c87041e3c76fcbdf8b25e Mon Sep 17 00:00:00 2001 From: Manoel Campos Date: Tue, 20 Oct 2026 17:43:23 -0300 Subject: [PATCH 1/2] Add benchmark to measure VM and Host memory before and after cloudsimplus#586 ListenersMemoryBenchmark prints the heap retained by each VmSimple and HostSimple. scripts/listeners-memory-before-after.sh builds CloudSim Plus at the #586 commit and at its parent (using git worktrees) and runs the benchmark with each version. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 1 + scripts/listeners-memory-before-after.sh | 68 ++++++++ .../listeners/ListenersMemoryBenchmark.java | 145 ++++++++++++++++++ 3 files changed, 214 insertions(+) create mode 100755 scripts/listeners-memory-before-after.sh create mode 100644 src/main/java/org/cloudsimplus/listeners/ListenersMemoryBenchmark.java diff --git a/CLAUDE.md b/CLAUDE.md index 12cb355..2f49966 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,6 +30,7 @@ java -jar target/cloudsimplus-benchmarks-8.5.1-uber.jar -l Alternative entry points (run from the IDE or with `exec`/`java -cp`): - `org.cloudsimplus.benchmarks.Run` runs every JMH benchmark matching `org.cloudsimplus.` with preset options (1 fork, 5 warmup, 25 measurement iterations, ms, 1 thread). - `org.cloudsimplus.benchmarks.SimulationTimeBenchmark` is a plain `main` (not JMH): it runs a full simulation several times, growing Hosts/VMs/Cloudlets each run, and prints execution time. +- `org.cloudsimplus.listeners.ListenersMemoryBenchmark` is a plain `main` (not JMH): it prints the heap retained by each VM and Host (run it with `-XX:+UseSerialGC`). `scripts/listeners-memory-before-after.sh` builds CloudSim Plus before and after issue #586 (using git worktrees of `../cloudsimplus`) and runs it with each version. There are no unit tests in this project. diff --git a/scripts/listeners-memory-before-after.sh b/scripts/listeners-memory-before-after.sh new file mode 100755 index 0000000..51718f3 --- /dev/null +++ b/scripts/listeners-memory-before-after.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Compares the heap retained by each VM and Host before and after +# CloudSim Plus issue #586 (listener lists starting as the shared List.of()). +# +# It builds CloudSim Plus at the #586 commit and at its parent commit (using git worktrees, +# so the CloudSim Plus working copy isn't changed), then runs ListenersMemoryBenchmark with each JAR. +# The CloudSim Plus JAR comes first in the classpath, so its classes are used +# instead of the ones inside the benchmarks uber JAR. +# +# Usage: scripts/listeners-memory-before-after.sh [cloudsimplus-dir] [after-commit] +# cloudsimplus-dir: CloudSim Plus repository (default: ../cloudsimplus) +# after-commit: commit with the change (default: the #586 commit). The "before" commit is its parent. + +set -euo pipefail + +readonly BENCHMARKS_DIR="$(cd "$(dirname "$0")/.." && pwd)" +readonly CLOUDSIMPLUS_DIR="$(cd "${1:-$BENCHMARKS_DIR/../cloudsimplus}" && pwd)" +readonly AFTER_COMMIT="${2:-08134449e}" +readonly BEFORE_COMMIT="$AFTER_COMMIT~1" +readonly BENCHMARK_CLASS="org.cloudsimplus.listeners.ListenersMemoryBenchmark" +# A stop-the-world GC makes System.gc() do a full collection, so the used heap is accurate. +readonly JVM_OPTS=(-XX:+UseSerialGC -Xms2g -Xmx2g) + +WORK_DIR="$(mktemp -d)" +readonly WORK_DIR + +cleanup() { + git -C "$CLOUDSIMPLUS_DIR" worktree remove --force "$WORK_DIR/before" 2>/dev/null || true + git -C "$CLOUDSIMPLUS_DIR" worktree remove --force "$WORK_DIR/after" 2>/dev/null || true + rm -rf "$WORK_DIR" +} +trap cleanup EXIT + +# Builds the CloudSim Plus JAR for a given commit and prints its path. +# $1: commit; $2: name for the worktree and the JAR ("before" or "after") +build_cloudsimplus() { + local commit="$1" name="$2" + local worktree="$WORK_DIR/$name" + echo "Building CloudSim Plus $name #586 ($(git -C "$CLOUDSIMPLUS_DIR" rev-parse --short "$commit"))" >&2 + git -C "$CLOUDSIMPLUS_DIR" worktree add --quiet --detach "$worktree" "$commit" + (cd "$worktree" && ./mvnw -q -B package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true >&2) + + local jar + jar="$(find "$worktree/target" -maxdepth 1 -name 'cloudsimplus-*.jar' ! -name '*-sources.jar' ! -name '*-javadoc.jar' | head -1)" + cp "$jar" "$WORK_DIR/cloudsimplus-$name.jar" + echo "$WORK_DIR/cloudsimplus-$name.jar" +} + +build_benchmarks() { + echo "Building the benchmarks" >&2 + (cd "$BENCHMARKS_DIR" && ./mvnw -q -B package >&2) + find "$BENCHMARKS_DIR/target" -maxdepth 1 -name 'cloudsimplus-benchmarks-*-uber.jar' | head -1 +} + +# $1: CloudSim Plus JAR; $2: label printed in the results +run_benchmark() { + java "${JVM_OPTS[@]}" -cp "$1:$UBER_JAR" "$BENCHMARK_CLASS" "$2" + echo +} + +BEFORE_JAR="$(build_cloudsimplus "$BEFORE_COMMIT" before)" +AFTER_JAR="$(build_cloudsimplus "$AFTER_COMMIT" after)" +UBER_JAR="$(build_benchmarks)" +readonly BEFORE_JAR AFTER_JAR UBER_JAR + +echo +run_benchmark "$BEFORE_JAR" "before #586" +run_benchmark "$AFTER_JAR" "after #586" diff --git a/src/main/java/org/cloudsimplus/listeners/ListenersMemoryBenchmark.java b/src/main/java/org/cloudsimplus/listeners/ListenersMemoryBenchmark.java new file mode 100644 index 0000000..a2831ae --- /dev/null +++ b/src/main/java/org/cloudsimplus/listeners/ListenersMemoryBenchmark.java @@ -0,0 +1,145 @@ +/* + * CloudSim Plus: A modern, highly-extensible and easier-to-use Framework for + * Modeling and Simulation of Cloud Computing Infrastructures and Services. + * http://cloudsimplus.org + * + * Copyright (C) 2015-2021 Universidade da Beira Interior (UBI, Portugal) and + * the Instituto Federal de Educação Ciência e Tecnologia do Tocantins (IFTO, Brazil). + * + * This file is part of CloudSim Plus. + * + * CloudSim Plus is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CloudSim Plus is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CloudSim Plus. If not, see . + */ +package org.cloudsimplus.listeners; + +import org.cloudsimplus.hosts.HostSimple; +import org.cloudsimplus.resources.PeSimple; +import org.cloudsimplus.vms.VmSimple; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.ref.Reference; +import java.util.Arrays; +import java.util.List; +import java.util.function.IntFunction; + +/// Measures the heap retained by each VM and Host, to compare the memory used +/// before and after the listener lists became the shared `List.of()` (issue #586). +/// Before that, each VM created 6 empty `ArrayList`s and each Host created +/// 2 empty `ArrayList`s and 1 empty `HashSet`, even when no listener was added. +/// +/// It's a plain `main` (not JMH), since JMH measures time and allocation rate, +/// but here we need the memory that stays in the heap after the objects are created. +/// It creates [#OBJECTS] objects, keeps them in an array and measures the used heap +/// after a GC, before and after creating them. The difference divided by the number of objects +/// is the heap retained by each object. This is repeated [#ROUNDS] times and the median is shown. +/// +/// The results are only accurate with a stop-the-world GC, where [System#gc()] +/// does a full collection, so run it with `-XX:+UseSerialGC`. +/// The `scripts/listeners-memory-before-after.sh` script runs it +/// with CloudSim Plus built before and after issue #586. +/// +/// @author Manoel Campos da Silva Filho +/// @see CloudSim Plus issue #586 +/// @see CloudSim Plus issue #523 +public final class ListenersMemoryBenchmark { + /// Number of objects of each type created in each round. + private static final int OBJECTS = 100_000; + + /// Number of times each measurement is repeated. + private static final int ROUNDS = 5; + + /// Number of times [System#gc()] is called before reading the used heap, + /// so that objects waiting for finalization or in reference queues are also collected. + private static final int GC_CALLS = 3; + + private static final double VM_MIPS = 1000; + private static final long VM_PES = 1; + + private static final double PE_MIPS = 1000; + private static final long HOST_RAM = 4096; + private static final long HOST_BW = 10_000; + private static final long HOST_STORAGE = 100_000; + + private static final MemoryMXBean MEMORY = ManagementFactory.getMemoryMXBean(); + + /// A private constructor to avoid class instantiation. + private ListenersMemoryBenchmark(){} + + /// Runs the benchmark. + /// @param args an optional label to identify the CloudSim Plus version being measured (e.g., "before" or "after") + public static void main(final String[] args) { + final String label = args.length > 0 ? args[0] : "current"; + System.out.printf("%s (%s): %,d objects per round, median of %d rounds%n", + ListenersMemoryBenchmark.class.getSimpleName(), label, OBJECTS, ROUNDS); + + printResult("VmSimple", measure(ListenersMemoryBenchmark::newVm)); + printResult("HostSimple", measure(ListenersMemoryBenchmark::newHost)); + } + + private static VmSimple newVm(final int index) { + return new VmSimple(VM_MIPS, VM_PES); + } + + /// Creates a Host with a single PE. + /// The PE is included in the measured size, but it's the same before and after the change. + private static HostSimple newHost(final int index) { + return new HostSimple(HOST_RAM, HOST_BW, HOST_STORAGE, List.of(new PeSimple(PE_MIPS))); + } + + /// {@return the median heap (in bytes) retained by each object created by a given function} + /// @param factory a function that creates an object for a given index + private static double measure(final IntFunction factory) { + final double[] bytesPerObject = new double[ROUNDS]; + for (int round = 0; round < ROUNDS; round++) { + bytesPerObject[round] = measureRound(factory); + } + + return median(bytesPerObject); + } + + /// {@return the heap (in bytes) retained by each object created by a given function in a single round} + /// The array that keeps the objects alive is created before the first reading, + /// so it isn't included in the measured size. + private static double measureRound(final IntFunction factory) { + final Object[] objects = new Object[OBJECTS]; + final long usedBefore = usedHeapAfterGc(); + for (int i = 0; i < OBJECTS; i++) { + objects[i] = factory.apply(i); + } + + final long usedAfter = usedHeapAfterGc(); + Reference.reachabilityFence(objects); + return (usedAfter - usedBefore) / (double) OBJECTS; + } + + private static long usedHeapAfterGc() { + for (int i = 0; i < GC_CALLS; i++) { + System.gc(); + } + + return MEMORY.getHeapMemoryUsage().getUsed(); + } + + private static double median(final double[] values) { + final double[] sorted = values.clone(); + Arrays.sort(sorted); + final int middle = sorted.length / 2; + return sorted.length % 2 == 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; + } + + private static void printResult(final String className, final double bytesPerObject) { + System.out.printf(" %-10s %8.1f bytes per object%n", className, bytesPerObject); + } +} From a15baf9ade55b1e2b33e25f36f679962ee29ce83 Mon Sep 17 00:00:00 2001 From: Manoel Campos Date: Tue, 20 Oct 2026 18:05:30 -0300 Subject: [PATCH 2/2] Benchmark CloudSim Plus 8.5.1 using a Maven profile instead of a script The script that built CloudSim Plus at the #586 commit and its parent is removed. The cloudsimplus-8 profile sets the CloudSim Plus version to 8.5.1 (the new cloudsimplus.version property, which defaults to the project version), so the benchmarks can be built and run against both versions. ListenersNotificationBenchmark is excluded in that profile, since it uses the Listeners class from CloudSim Plus 9 (see #8). The pinned logback-classic 1.4.5 is removed. It was mixed with logback-core 1.5.34 from CloudSim Plus 9, failing when a VM was created. Each CloudSim Plus version already brings its matching logback. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- pom.xml | 40 +++++++++-- scripts/listeners-memory-before-after.sh | 68 ------------------- .../listeners/ListenersMemoryBenchmark.java | 12 +++- 4 files changed, 44 insertions(+), 78 deletions(-) delete mode 100755 scripts/listeners-memory-before-after.sh diff --git a/CLAUDE.md b/CLAUDE.md index 2f49966..a410410 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,7 +30,7 @@ java -jar target/cloudsimplus-benchmarks-8.5.1-uber.jar -l Alternative entry points (run from the IDE or with `exec`/`java -cp`): - `org.cloudsimplus.benchmarks.Run` runs every JMH benchmark matching `org.cloudsimplus.` with preset options (1 fork, 5 warmup, 25 measurement iterations, ms, 1 thread). - `org.cloudsimplus.benchmarks.SimulationTimeBenchmark` is a plain `main` (not JMH): it runs a full simulation several times, growing Hosts/VMs/Cloudlets each run, and prints execution time. -- `org.cloudsimplus.listeners.ListenersMemoryBenchmark` is a plain `main` (not JMH): it prints the heap retained by each VM and Host (run it with `-XX:+UseSerialGC`). `scripts/listeners-memory-before-after.sh` builds CloudSim Plus before and after issue #586 (using git worktrees of `../cloudsimplus`) and runs it with each version. +- `org.cloudsimplus.listeners.ListenersMemoryBenchmark` is a plain `main` (not JMH): it prints the heap retained by each VM and Host (run it with `-XX:+UseSerialGC`). To compare with CloudSim Plus 8.5.1 (before issue #586), build with `./mvnw clean package -Pcloudsimplus-8`, run it, then build without the profile and run it again (see its Javadoc). There are no unit tests in this project. diff --git a/pom.xml b/pom.xml index fd04e18..8a4ea56 100755 --- a/pom.xml +++ b/pom.xml @@ -78,15 +78,10 @@ org.cloudsimplus cloudsimplus - ${project.version} + ${cloudsimplus.version} + - - - ch.qos.logback - logback-classic - 1.4.5 - + ${project.version} 2.50.0 0.14.2 1.0.0 @@ -255,4 +252,33 @@ + + + + + cloudsimplus-8 + + 8.5.1 + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org/cloudsimplus/listeners/ListenersNotificationBenchmark.java + + + + + + + diff --git a/scripts/listeners-memory-before-after.sh b/scripts/listeners-memory-before-after.sh deleted file mode 100755 index 51718f3..0000000 --- a/scripts/listeners-memory-before-after.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# Compares the heap retained by each VM and Host before and after -# CloudSim Plus issue #586 (listener lists starting as the shared List.of()). -# -# It builds CloudSim Plus at the #586 commit and at its parent commit (using git worktrees, -# so the CloudSim Plus working copy isn't changed), then runs ListenersMemoryBenchmark with each JAR. -# The CloudSim Plus JAR comes first in the classpath, so its classes are used -# instead of the ones inside the benchmarks uber JAR. -# -# Usage: scripts/listeners-memory-before-after.sh [cloudsimplus-dir] [after-commit] -# cloudsimplus-dir: CloudSim Plus repository (default: ../cloudsimplus) -# after-commit: commit with the change (default: the #586 commit). The "before" commit is its parent. - -set -euo pipefail - -readonly BENCHMARKS_DIR="$(cd "$(dirname "$0")/.." && pwd)" -readonly CLOUDSIMPLUS_DIR="$(cd "${1:-$BENCHMARKS_DIR/../cloudsimplus}" && pwd)" -readonly AFTER_COMMIT="${2:-08134449e}" -readonly BEFORE_COMMIT="$AFTER_COMMIT~1" -readonly BENCHMARK_CLASS="org.cloudsimplus.listeners.ListenersMemoryBenchmark" -# A stop-the-world GC makes System.gc() do a full collection, so the used heap is accurate. -readonly JVM_OPTS=(-XX:+UseSerialGC -Xms2g -Xmx2g) - -WORK_DIR="$(mktemp -d)" -readonly WORK_DIR - -cleanup() { - git -C "$CLOUDSIMPLUS_DIR" worktree remove --force "$WORK_DIR/before" 2>/dev/null || true - git -C "$CLOUDSIMPLUS_DIR" worktree remove --force "$WORK_DIR/after" 2>/dev/null || true - rm -rf "$WORK_DIR" -} -trap cleanup EXIT - -# Builds the CloudSim Plus JAR for a given commit and prints its path. -# $1: commit; $2: name for the worktree and the JAR ("before" or "after") -build_cloudsimplus() { - local commit="$1" name="$2" - local worktree="$WORK_DIR/$name" - echo "Building CloudSim Plus $name #586 ($(git -C "$CLOUDSIMPLUS_DIR" rev-parse --short "$commit"))" >&2 - git -C "$CLOUDSIMPLUS_DIR" worktree add --quiet --detach "$worktree" "$commit" - (cd "$worktree" && ./mvnw -q -B package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true >&2) - - local jar - jar="$(find "$worktree/target" -maxdepth 1 -name 'cloudsimplus-*.jar' ! -name '*-sources.jar' ! -name '*-javadoc.jar' | head -1)" - cp "$jar" "$WORK_DIR/cloudsimplus-$name.jar" - echo "$WORK_DIR/cloudsimplus-$name.jar" -} - -build_benchmarks() { - echo "Building the benchmarks" >&2 - (cd "$BENCHMARKS_DIR" && ./mvnw -q -B package >&2) - find "$BENCHMARKS_DIR/target" -maxdepth 1 -name 'cloudsimplus-benchmarks-*-uber.jar' | head -1 -} - -# $1: CloudSim Plus JAR; $2: label printed in the results -run_benchmark() { - java "${JVM_OPTS[@]}" -cp "$1:$UBER_JAR" "$BENCHMARK_CLASS" "$2" - echo -} - -BEFORE_JAR="$(build_cloudsimplus "$BEFORE_COMMIT" before)" -AFTER_JAR="$(build_cloudsimplus "$AFTER_COMMIT" after)" -UBER_JAR="$(build_benchmarks)" -readonly BEFORE_JAR AFTER_JAR UBER_JAR - -echo -run_benchmark "$BEFORE_JAR" "before #586" -run_benchmark "$AFTER_JAR" "after #586" diff --git a/src/main/java/org/cloudsimplus/listeners/ListenersMemoryBenchmark.java b/src/main/java/org/cloudsimplus/listeners/ListenersMemoryBenchmark.java index a2831ae..560ab48 100644 --- a/src/main/java/org/cloudsimplus/listeners/ListenersMemoryBenchmark.java +++ b/src/main/java/org/cloudsimplus/listeners/ListenersMemoryBenchmark.java @@ -47,8 +47,16 @@ /// /// The results are only accurate with a stop-the-world GC, where [System#gc()] /// does a full collection, so run it with `-XX:+UseSerialGC`. -/// The `scripts/listeners-memory-before-after.sh` script runs it -/// with CloudSim Plus built before and after issue #586. +/// +/// To compare CloudSim Plus 8.5.1 (before #586) with the current version, build and run it once for each version: +/// +/// ```bash +/// ./mvnw clean package -Pcloudsimplus-8 # CloudSim Plus 8.5.1 +/// java -XX:+UseSerialGC -cp target/cloudsimplus-benchmarks-*-uber.jar org.cloudsimplus.listeners.ListenersMemoryBenchmark 8.5.1 +/// +/// ./mvnw clean package # current CloudSim Plus version +/// java -XX:+UseSerialGC -cp target/cloudsimplus-benchmarks-*-uber.jar org.cloudsimplus.listeners.ListenersMemoryBenchmark current +/// ``` /// /// @author Manoel Campos da Silva Filho /// @see CloudSim Plus issue #586