◐ Shell
clean mode source ↗

Added support for aarch64 and ppc64le binaries by judovana · Pull Request #161 · lmdbjava/lmdbjava

judovana

Hello @judovana,

There are 2 major problem with the test suite and possibly with the Java API as such:

4096 is not the page size on all systems

The default expectation of LMDB Java is that the page size on the system is 4096. That does not hold. Ultimately my aarch64 and ppc64le (Power 9) systems running RHEL 8 report:

LMDB Java reads that as 1/2, i.e.

final Stat stat;
try (Txn<ByteBuffer> txn = env.txnRead()) {
  stat = db.stat(txn);
}

gives 32768 as the page size in stat.pageSize. Not 4096.

LMDB native API uses pages, not bytes

The LMDB Java code looks as if LIB.mdb_env_set_mapsize takes a number of bytes while in fact, the native API uses number of pages. e.g. (an example from a C app):

/** Maximum size of the DB in multiples of page size in bytes
 * e.g. on my system: getconf PAGESIZE is 4096 so setting 1048576 here gives 4GB
 * max LMDB DB size.*/
#define IPRANGER_MAX_MAP_SIZE_IN_PAGES 1048576

 E(mdb_env_set_mapsize(env, sysconf(_SC_PAGESIZE) *
                                 IPRANGER_MAX_MAP_SIZE_IN_PAGES));

So, in our case on these aarch64 and ppc64le systems, their 32768 is 8 times more than the expected default of 4096.
Introducing magic number 8 makes the test pass on these systems:

diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java
index 7634a34..c9c3397 100644
--- a/src/main/java/org/lmdbjava/Env.java
+++ b/src/main/java/org/lmdbjava/Env.java
@@ -486,7 +486,7 @@ public final class Env<T> implements AutoCloseable {
       checkRc(LIB.mdb_env_create(envPtr));
       final Pointer ptr = envPtr.getValue();
       try {
-        checkRc(LIB.mdb_env_set_mapsize(ptr, mapSize));
+        checkRc(LIB.mdb_env_set_mapsize(ptr, 8 * mapSize));
         checkRc(LIB.mdb_env_set_maxdbs(ptr, maxDbs));
         checkRc(LIB.mdb_env_set_maxreaders(ptr, maxReaders));
         final int flagsMask = mask(flags);
diff --git a/src/test/java/org/lmdbjava/DbiTest.java b/src/test/java/org/lmdbjava/DbiTest.java
index b9bf453..c35c440 100644
--- a/src/test/java/org/lmdbjava/DbiTest.java
+++ b/src/test/java/org/lmdbjava/DbiTest.java
@@ -457,7 +457,6 @@ public final class DbiTest {
     assertThat(stat.entries, is(3L));
     assertThat(stat.leafPages, is(1L));
     assertThat(stat.overflowPages, is(0L));
-    assertThat(stat.pageSize, is(4_096));
   }
 
   @Test(expected = MapFullException.class)
diff --git a/src/test/java/org/lmdbjava/EnvTest.java b/src/test/java/org/lmdbjava/EnvTest.java
index 3e9ece9..811b18c 100644
--- a/src/test/java/org/lmdbjava/EnvTest.java
+++ b/src/test/java/org/lmdbjava/EnvTest.java
@@ -26,6 +26,7 @@ import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.lmdbjava.CopyFlags.MDB_CP_COMPACT;
 import static org.lmdbjava.DbiFlags.MDB_CREATE;
@@ -68,7 +69,8 @@ public final class EnvTest {
         .setMapSize(MEBIBYTES.toBytes(1))
         .open(path, MDB_NOSUBDIR)) {
       final EnvInfo info = env.info();
-      assertThat(info.mapSize, is(MEBIBYTES.toBytes(1)));
+      // `is' does not make sense, one API uses bytes, the other uses pages...
+      assertThat(info.mapSize, greaterThanOrEqualTo(MEBIBYTES.toBytes(1)));
     }
   }
 
@@ -277,7 +279,7 @@ public final class EnvTest {
       assertThat(info.lastPageNumber, is(1L));
       assertThat(info.lastTransactionId, is(0L));
       assertThat(info.mapAddress, is(0L));
-      assertThat(info.mapSize, is(123_456L));
+      assertThat(info.mapSize, is(8 * 123_456L));
       assertThat(info.maxReaders, is(4));
       assertThat(info.numReaders, is(0));
       assertThat(info.toString(), containsString("maxReaders="));
@@ -355,7 +357,7 @@ public final class EnvTest {
       }
       assertThat(mapFullExThrown, is(true));
 
-      env.setMapSize(500_000);
+      env.setMapSize(8 * 500_000);
 
       try (Txn<ByteBuffer> roTxn = env.txnRead()) {
         assertThat(db.get(roTxn, bb(1)).getInt(), is(42));
@@ -390,7 +392,6 @@ public final class EnvTest {
       assertThat(stat.entries, is(0L));
       assertThat(stat.leafPages, is(0L));
       assertThat(stat.overflowPages, is(0L));
-      assertThat(stat.pageSize, is(4_096));
       assertThat(stat.toString(), containsString("pageSize="));
     }
   }
[INFO] -------------------------------------------------------
[INFO]  T E S T S
[INFO] -------------------------------------------------------
[INFO] Running org.lmdbjava.KeyRangeTest
[INFO] Tests run: 19, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.15 s - in org.lmdbjava.KeyRangeTest
[INFO] Running org.lmdbjava.CursorParamTest
[INFO] Tests run: 5, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 1.24 s - in org.lmdbjava.CursorParamTest
[INFO] Running org.lmdbjava.TxnTest
[INFO] Tests run: 21, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.092 s - in org.lmdbjava.TxnTest
[INFO] Running org.lmdbjava.ResultCodeMapperTest
[INFO] Tests run: 9, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.023 s - in org.lmdbjava.ResultCodeMapperTest
[INFO] Running org.lmdbjava.LibraryTest
[INFO] Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0 s - in org.lmdbjava.LibraryTest
[INFO] Running org.lmdbjava.MaskedFlagTest
[INFO] Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0 s - in org.lmdbjava.MaskedFlagTest
[INFO] Running org.lmdbjava.DbiTest
[INFO] Tests run: 24, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 34.76 s - in org.lmdbjava.DbiTest
[INFO] Running org.lmdbjava.CursorIterableTest
[INFO] Tests run: 24, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.192 s - in org.lmdbjava.CursorIterableTest
[INFO] Running org.lmdbjava.MetaTest
[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0 s - in org.lmdbjava.MetaTest
[INFO] Running org.lmdbjava.EnvTest
[INFO] Tests run: 24, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 36.082 s - in org.lmdbjava.EnvTest
[INFO] Running org.lmdbjava.ComparatorTest
[INFO] Tests run: 21, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.028 s - in org.lmdbjava.ComparatorTest
[INFO] Running org.lmdbjava.VerifierTest
[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 2.022 s - in org.lmdbjava.VerifierTest
[INFO] Running org.lmdbjava.CursorTest
[INFO] Tests run: 15, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.061 s - in org.lmdbjava.CursorTest
[INFO] Running org.lmdbjava.ByteBufferProxyTest
[INFO] Tests run: 10, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.01 s - in org.lmdbjava.ByteBufferProxyTest
[INFO] Running org.lmdbjava.TutorialTest
[INFO] Tests run: 7, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 3.138 s - in org.lmdbjava.TutorialTest
[INFO] 
[INFO] Results:
[INFO] 
[INFO] Tests run: 187, Failures: 0, Errors: 0, Skipped: 0

Fix

I don't know how to solve it elegantly at the moment. I propose dropping asserts for hardcoded 4096 and introduce calculating max size dynamically according to what LMDB C lib says is the page size on the system.

It would help to get a comment from maintainers. I can implement the fix if you hint what would be a good direction.