From fe6fd10b0b2b52351d50dad769f0693e746935ca Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Fri, 24 Jul 2026 10:04:55 +0200 Subject: [PATCH 1/3] test(#215): reproduce Netty ByteBuf.nioBuffer() returning stale data A value ByteBuf returned via ByteBufProxy.PROXY_NETTY reads correctly through the ByteBuf's own accessors, but ByteBuf.nioBuffer() does not reflect the stored data (it views Netty's separate, chunk-shared backing buffer, which the zero-copy read path never repoints). This test pins the current behaviour; the following commits add helpers to obtain a correct NIO buffer. Refs lmdbjava#215. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GdiZ3ABYsVHXBRNCkEizpE --- .../org/lmdbjava/ByteBufNioBufferTest.java | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 src/test/java/org/lmdbjava/ByteBufNioBufferTest.java diff --git a/src/test/java/org/lmdbjava/ByteBufNioBufferTest.java b/src/test/java/org/lmdbjava/ByteBufNioBufferTest.java new file mode 100644 index 00000000..8477825f --- /dev/null +++ b/src/test/java/org/lmdbjava/ByteBufNioBufferTest.java @@ -0,0 +1,99 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lmdbjava; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.lmdbjava.DbiFlags.MDB_CREATE; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.PooledByteBufAllocator; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Reproduces lmdbjava#215: {@code ByteBuf.nioBuffer()} on a value returned via {@link + * ByteBufProxy#PROXY_NETTY} does not reflect the stored data (it views Netty's separate, + * chunk-shared backing buffer, which the zero-copy read path never repoints). + */ +final class ByteBufNioBufferTest { + + private static final String VALUE = "Hello World"; + private static final byte[] VALUE_BYTES = VALUE.getBytes(UTF_8); + + private TempDir tempDir; + + @BeforeEach + void beforeEach() { + tempDir = new TempDir(); + } + + @AfterEach + void afterEach() { + tempDir.cleanup(); + } + + private Env openEnv() { + final Path dir = tempDir.createTempDir(); + return Env.create(ByteBufProxy.PROXY_NETTY).setMapSize(10_485_760).setMaxDbs(1).open(dir); + } + + private static Dbi openDb(final Env env) { + return env.createDbi().setDbName("db").withDefaultComparator().addDbiFlag(MDB_CREATE).open(); + } + + private static byte[] readable(final ByteBuf buffer) { + final byte[] dst = new byte[buffer.readableBytes()]; + buffer.getBytes(buffer.readerIndex(), dst); + return dst; + } + + private static byte[] drain(final ByteBuffer buffer) { + final byte[] dst = new byte[buffer.remaining()]; + buffer.get(dst); + return dst; + } + + /** + * Documents the lmdbjava#215 limitation: the raw {@link ByteBuf#nioBuffer()} does NOT reflect the + * LMDB data, even though the {@link ByteBuf}'s own accessors do. + */ + @Test + void rawByteBufNioBuffer_doesNotReflectStoredData() { + try (Env env = openEnv()) { + final Dbi db = openDb(env); + final ByteBuf key = PooledByteBufAllocator.DEFAULT.directBuffer(env.getMaxKeySize()); + final ByteBuf value = PooledByteBufAllocator.DEFAULT.directBuffer(64); + try { + key.writeCharSequence("greeting", UTF_8); + value.writeCharSequence(VALUE, UTF_8); + db.put(key, value); + try (Txn txn = env.txnRead()) { + final ByteBuf found = db.get(txn, key); + assertThat(found).isNotNull(); + assertThat(readable(found)).isEqualTo(VALUE_BYTES); // ByteBuf accessors are correct + assertThat(drain(found.nioBuffer())).isNotEqualTo(VALUE_BYTES); // nioBuffer() is not + } + } finally { + key.release(); + value.release(); + } + } + } +} From e35c628708f30452e6f974d65ae324ec244ac4ad Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Fri, 24 Jul 2026 10:07:15 +0200 Subject: [PATCH 2/3] feat(#215): add ByteBufProxy.nioBufferView (zero-copy NIO view) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a static helper that materialises a read-only NIO ByteBuffer aliasing the LMDB memory referenced by a PROXY_NETTY value ByteBuf, using the same Unsafe address/capacity technique ByteBufferProxy already uses for its own zero-copy NIO buffers. This lets Netty users hand a correct NIO buffer to NIO-based consumers (compression, hashing, ...) instead of getting the zeros that ByteBuf.nioBuffer() returns. The java.nio.Buffer field offsets are resolved lazily in a holder so a JVM lacking --add-opens java.base/java.nio only fails if nioBufferView is actually called, rather than breaking PROXY_NETTY initialisation for everyone. The returned buffer aliases LMDB-owned, read-only memory and is valid only while the owning read Txn is open — documented prominently, as misuse is a JVM crash. Refs lmdbjava#215. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GdiZ3ABYsVHXBRNCkEizpE --- src/main/java/org/lmdbjava/ByteBufProxy.java | 52 +++++++++++++++++++ .../org/lmdbjava/ByteBufNioBufferTest.java | 33 ++++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/lmdbjava/ByteBufProxy.java b/src/main/java/org/lmdbjava/ByteBufProxy.java index bcbb6ebf..88097304 100644 --- a/src/main/java/org/lmdbjava/ByteBufProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufProxy.java @@ -23,6 +23,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.PooledByteBufAllocator; import java.lang.reflect.Field; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Comparator; import jnr.ffi.Pointer; @@ -221,4 +222,55 @@ protected ByteBuf out(final ByteBuf buffer, final Pointer ptr) { buffer.clear().writerIndex((int) size); return buffer; } + + /** + * Returns a read-only NIO {@link ByteBuffer} view over the LMDB memory currently referenced by + * the readable region of {@code buffer} — typically a {@link ByteBuf} just returned from a read + * using {@link #PROXY_NETTY}. + * + *

Why this is needed. For zero copy, a {@code ByteBuf} returned from a read is + * repointed at LMDB's memory-mapped region by overwriting its {@code memoryAddress} (see {@link + * #out}). That makes the {@code ByteBuf}'s own accessors (e.g. {@link ByteBuf#getBytes(int, + * byte[])}) read the LMDB data correctly, but {@link ByteBuf#nioBuffer()} derives its buffer from + * Netty's separate, chunk-shared backing buffer, which was never repointed — so {@code + * buffer.nioBuffer()} yields zeros. (That chunk buffer cannot simply be repointed, as it is + * shared by every pooled buffer carved from the same chunk.) This method instead materialises a + * NIO buffer that actually aliases the LMDB region, so it can be handed to NIO-based consumers + * (compression, hashing, etc.). + * + *

Lifecycle — read carefully. The returned buffer aliases LMDB-owned, read-only memory. + * It is valid ONLY while the owning read {@link Txn} remains open and unmodified, exactly like + * {@link Txn#val()}. Using it after that transaction (or the {@link Env}) is closed, or after a + * write to the same slot, is undefined behaviour that may crash the JVM (SIGSEGV). If you need + * the bytes to outlive the transaction, copy them out (e.g. into a heap {@link ByteBuffer}). + * + * @param buffer a {@link ByteBuf} whose readable region points at LMDB memory (required) + * @return a read-only NIO view over the same memory (never null) + */ + public static ByteBuffer nioBufferView(final ByteBuf buffer) { + requireNonNull(buffer); + // Start from a real direct buffer, then repoint its single java.nio.Buffer address/capacity at + // the LMDB region — the same technique ByteBufferProxy uses for its own zero-copy NIO buffers. + // The original (zero-length) allocation stays referenced by the read-only view's attachment, so + // its Cleaner frees only that original base, never the LMDB memory we aliased. + final ByteBuffer view = ByteBuffer.allocateDirect(0); + UNSAFE.putLong( + view, NioBufferField.ADDRESS_OFFSET, buffer.memoryAddress() + buffer.readerIndex()); + UNSAFE.putInt(view, NioBufferField.CAPACITY_OFFSET, buffer.readableBytes()); + view.clear(); + return view.asReadOnlyBuffer(); + } + + /** + * Lazily-resolved offsets of {@link java.nio.Buffer}'s {@code address}/{@code capacity} fields. + * Kept in a holder (not a static field on {@link ByteBufProxy}) so that a JVM lacking the + * required {@code --add-opens java.base/java.nio} only fails if {@link #nioBufferView(ByteBuf)} + * is actually called, rather than breaking {@link #PROXY_NETTY} initialisation for every user. + */ + private static final class NioBufferField { + static final long ADDRESS_OFFSET = + UNSAFE.objectFieldOffset(findField("java.nio.Buffer", "address")); + static final long CAPACITY_OFFSET = + UNSAFE.objectFieldOffset(findField("java.nio.Buffer", "capacity")); + } } diff --git a/src/test/java/org/lmdbjava/ByteBufNioBufferTest.java b/src/test/java/org/lmdbjava/ByteBufNioBufferTest.java index 8477825f..4e01373e 100644 --- a/src/test/java/org/lmdbjava/ByteBufNioBufferTest.java +++ b/src/test/java/org/lmdbjava/ByteBufNioBufferTest.java @@ -28,9 +28,8 @@ import org.junit.jupiter.api.Test; /** - * Reproduces lmdbjava#215: {@code ByteBuf.nioBuffer()} on a value returned via {@link - * ByteBufProxy#PROXY_NETTY} does not reflect the stored data (it views Netty's separate, - * chunk-shared backing buffer, which the zero-copy read path never repoints). + * Reproduces lmdbjava#215 and covers {@link ByteBufProxy#nioBufferView(ByteBuf)}, a zero-copy NIO + * view over the LMDB memory of a value returned via {@link ByteBufProxy#PROXY_NETTY}. */ final class ByteBufNioBufferTest { @@ -70,6 +69,30 @@ private static byte[] drain(final ByteBuffer buffer) { return dst; } + /** Zero-copy view reflects the stored bytes (read inside the txn). */ + @Test + void nioBufferView_reflectsStoredData() { + try (Env env = openEnv()) { + final Dbi db = openDb(env); + final ByteBuf key = PooledByteBufAllocator.DEFAULT.directBuffer(env.getMaxKeySize()); + final ByteBuf value = PooledByteBufAllocator.DEFAULT.directBuffer(64); + try { + key.writeCharSequence("greeting", UTF_8); + value.writeCharSequence(VALUE, UTF_8); + db.put(key, value); + try (Txn txn = env.txnRead()) { + final ByteBuf found = db.get(txn, key); + assertThat(found).isNotNull(); + assertThat(readable(found)).isEqualTo(VALUE_BYTES); // sanity + assertThat(drain(ByteBufProxy.nioBufferView(found))).isEqualTo(VALUE_BYTES); + } + } finally { + key.release(); + value.release(); + } + } + } + /** * Documents the lmdbjava#215 limitation: the raw {@link ByteBuf#nioBuffer()} does NOT reflect the * LMDB data, even though the {@link ByteBuf}'s own accessors do. @@ -87,8 +110,8 @@ void rawByteBufNioBuffer_doesNotReflectStoredData() { try (Txn txn = env.txnRead()) { final ByteBuf found = db.get(txn, key); assertThat(found).isNotNull(); - assertThat(readable(found)).isEqualTo(VALUE_BYTES); // ByteBuf accessors are correct - assertThat(drain(found.nioBuffer())).isNotEqualTo(VALUE_BYTES); // nioBuffer() is not + assertThat(readable(found)).isEqualTo(VALUE_BYTES); + assertThat(drain(found.nioBuffer())).isNotEqualTo(VALUE_BYTES); } } finally { key.release(); From d54d275e0a910149bab6ed9fbc4a4b712627985c Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Fri, 24 Jul 2026 10:08:32 +0200 Subject: [PATCH 3/3] feat(#215): add ByteBufProxy.nioBufferCopy (safe copy) alternative MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a second static helper as an alternative to nioBufferView: instead of aliasing LMDB memory, it copies the readable bytes out via the ByteBuf's own (correct) accessor into a fresh direct NIO buffer. This uses no Unsafe/reflection, needs no --add-opens, and — unlike the view — remains valid after the owning transaction (or the Env) is closed. The two helpers are deliberately offered together so the maintainers can choose: keep the zero-copy view, the safe copy, or both. Dropping this commit removes the copy variant; dropping the previous commit removes the view. Recommendation: nioBufferCopy as the safe default, nioBufferView for zero-copy hot paths. Refs lmdbjava#215. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GdiZ3ABYsVHXBRNCkEizpE --- src/main/java/org/lmdbjava/ByteBufProxy.java | 31 +++++++++- .../org/lmdbjava/ByteBufNioBufferTest.java | 59 ++++++++++++++++++- 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/lmdbjava/ByteBufProxy.java b/src/main/java/org/lmdbjava/ByteBufProxy.java index 88097304..40e8fb26 100644 --- a/src/main/java/org/lmdbjava/ByteBufProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufProxy.java @@ -242,7 +242,7 @@ protected ByteBuf out(final ByteBuf buffer, final Pointer ptr) { * It is valid ONLY while the owning read {@link Txn} remains open and unmodified, exactly like * {@link Txn#val()}. Using it after that transaction (or the {@link Env}) is closed, or after a * write to the same slot, is undefined behaviour that may crash the JVM (SIGSEGV). If you need - * the bytes to outlive the transaction, copy them out (e.g. into a heap {@link ByteBuffer}). + * the bytes to outlive the transaction, use {@link #nioBufferCopy(ByteBuf)} instead. * * @param buffer a {@link ByteBuf} whose readable region points at LMDB memory (required) * @return a read-only NIO view over the same memory (never null) @@ -261,6 +261,35 @@ public static ByteBuffer nioBufferView(final ByteBuf buffer) { return view.asReadOnlyBuffer(); } + /** + * Returns a direct NIO {@link ByteBuffer} holding an independent copy of the readable + * region of {@code buffer} — typically a {@link ByteBuf} just returned from a read using {@link + * #PROXY_NETTY}. + * + *

This is the safe counterpart to {@link #nioBufferView(ByteBuf)}. It addresses the same + * lmdbjava#215 problem (a get-returned {@code ByteBuf} whose {@link ByteBuf#nioBuffer()} yields + * zeros), but by copying the bytes out via the {@code ByteBuf}'s own (correct) accessor rather + * than aliasing LMDB memory. The copy therefore uses no {@code Unsafe}/reflection, needs no + * {@code --add-opens}, and — crucially — remains valid after the transaction (or {@link Env}) + * is closed. Prefer this unless you specifically need zero copy and can guarantee the + * returned buffer is consumed before the owning transaction closes. + * + *

The returned buffer is a freshly allocated direct buffer, flipped and ready to read ({@code + * position=0}, {@code limit=size}). The caller owns it. + * + * @param buffer a {@link ByteBuf} whose readable region holds the bytes to copy (required) + * @return a new direct NIO buffer containing a copy of those bytes (never null) + */ + public static ByteBuffer nioBufferCopy(final ByteBuf buffer) { + requireNonNull(buffer); + final ByteBuffer copy = ByteBuffer.allocateDirect(buffer.readableBytes()); + // getBytes reads through the ByteBuf's (LMDB-repointed) memoryAddress, so it copies the real + // stored bytes — not the zeros seen via ByteBuf.nioBuffer(). + buffer.getBytes(buffer.readerIndex(), copy); + copy.flip(); + return copy; + } + /** * Lazily-resolved offsets of {@link java.nio.Buffer}'s {@code address}/{@code capacity} fields. * Kept in a holder (not a static field on {@link ByteBufProxy}) so that a JVM lacking the diff --git a/src/test/java/org/lmdbjava/ByteBufNioBufferTest.java b/src/test/java/org/lmdbjava/ByteBufNioBufferTest.java index 4e01373e..fb7321b0 100644 --- a/src/test/java/org/lmdbjava/ByteBufNioBufferTest.java +++ b/src/test/java/org/lmdbjava/ByteBufNioBufferTest.java @@ -28,8 +28,13 @@ import org.junit.jupiter.api.Test; /** - * Reproduces lmdbjava#215 and covers {@link ByteBufProxy#nioBufferView(ByteBuf)}, a zero-copy NIO - * view over the LMDB memory of a value returned via {@link ByteBufProxy#PROXY_NETTY}. + * Reproduces and fixes lmdbjava#215: {@code ByteBuf.nioBuffer()} on a value returned via {@link + * ByteBufProxy#PROXY_NETTY} yields zeros. Two complementary helpers are offered: + * + *

    + *
  • {@link ByteBufProxy#nioBufferView(ByteBuf)} — zero-copy, valid only within the txn. + *
  • {@link ByteBufProxy#nioBufferCopy(ByteBuf)} — an independent copy that outlives the txn. + *
*/ final class ByteBufNioBufferTest { @@ -93,9 +98,57 @@ void nioBufferView_reflectsStoredData() { } } + /** Copy reflects the stored bytes. */ + @Test + void nioBufferCopy_reflectsStoredData() { + try (Env env = openEnv()) { + final Dbi db = openDb(env); + final ByteBuf key = PooledByteBufAllocator.DEFAULT.directBuffer(env.getMaxKeySize()); + final ByteBuf value = PooledByteBufAllocator.DEFAULT.directBuffer(64); + try { + key.writeCharSequence("greeting", UTF_8); + value.writeCharSequence(VALUE, UTF_8); + db.put(key, value); + try (Txn txn = env.txnRead()) { + final ByteBuf found = db.get(txn, key); + assertThat(found).isNotNull(); + assertThat(drain(ByteBufProxy.nioBufferCopy(found))).isEqualTo(VALUE_BYTES); + } + } finally { + key.release(); + value.release(); + } + } + } + + /** The discriminator: a copy taken inside the txn is still valid after txn AND env are closed. */ + @Test + void nioBufferCopy_survivesTxnAndEnvClose() { + final Env env = openEnv(); + final ByteBuf key = PooledByteBufAllocator.DEFAULT.directBuffer(env.getMaxKeySize()); + final ByteBuf value = PooledByteBufAllocator.DEFAULT.directBuffer(64); + ByteBuffer copy = null; + try { + final Dbi db = openDb(env); + key.writeCharSequence("greeting", UTF_8); + value.writeCharSequence(VALUE, UTF_8); + db.put(key, value); + try (Txn txn = env.txnRead()) { + copy = ByteBufProxy.nioBufferCopy(db.get(txn, key)); + } + } finally { + key.release(); + value.release(); + env.close(); // both txn and env are now closed + } + assertThat(copy).isNotNull(); + assertThat(drain(copy)).isEqualTo(VALUE_BYTES); // copy is independent of LMDB memory + } + /** * Documents the lmdbjava#215 limitation: the raw {@link ByteBuf#nioBuffer()} does NOT reflect the - * LMDB data, even though the {@link ByteBuf}'s own accessors do. + * LMDB data (it views Netty's separate, never-repointed chunk buffer). This is why the two + * helpers exist. */ @Test void rawByteBufNioBuffer_doesNotReflectStoredData() {