Upgrade OrientDB from 2.1.25 to 3.2.51

.gitignore

@@ -99,4 +99,7 @@ node_modules
 
 # ctags index
 tags
-test-output
+test-output
+
+# OrientDB test databases
+openidm-repo-orientdb/databases/

openidm-repo-orientdb/pom.xml

@@ -74,36 +74,134 @@
             <version>${project.version}</version>
         </dependency>
 
-        <!-- Commons -->
-
-        <!-- OrientDB --> 
+        <!-- OrientDB -->
+        <!--
+            OrientDB and its transitive dependencies (jnr-*, jffi, jaxb-*, ...) are embedded
+            into the openidm-repo-orientdb bundle (see Embed-Dependency below). They are
+            scoped 'provided' so they (and their transitives) are not also deployed as
+            standalone bundles by openidm-zip, where many of them either lack OSGi
+            metadata (jffi has no Bundle-SymbolicName) or have unsatisfied OSGi
+            requirements in the OpenIDM runtime (jnr-ffi → com.kenai.jffi,
+            jaxb-core → jakarta.xml.bind 3.x). 'provided' deps are still on the
+            compile/test classpath, so unit tests work.
+        -->
         <dependency>
             <groupId>com.orientechnologies</groupId>
             <artifactId>orientdb-core</artifactId>
             <version>${orientdb.version}</version>
+            <scope>provided</scope>
             <exclusions>
                 <exclusion>
                     <groupId>org.xerial.snappy</groupId>
                     <artifactId>snappy-java</artifactId>
                 </exclusion>
+                <!--
+                    OrientDB 3.x pulls in GraalVM/Truffle for SQL JS scripting via the java11+
+                    profile. These jars are not OSGi bundles and several of them
+                    (graal-sdk, regex, js-scriptengine, profiler, chromeinspector) do not even
+                    contain a META-INF/MANIFEST.MF entry, which causes Felix to abort with
+                    "FileNotFoundException: META-INF/MANIFEST.MF" during bundle install.
+                    OpenIDM does not use OrientDB's JS scripting, so they are excluded.
+                -->
+                <exclusion>
+                    <groupId>org.graalvm.sdk</groupId>
+                    <artifactId>graal-sdk</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.truffle</groupId>
+                    <artifactId>truffle-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.js</groupId>
+                    <artifactId>js</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.js</groupId>
+                    <artifactId>js-scriptengine</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.tools</groupId>
+                    <artifactId>profiler</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.tools</groupId>
+                    <artifactId>chromeinspector</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.regex</groupId>
+                    <artifactId>regex</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
             <groupId>com.orientechnologies</groupId>
             <artifactId>orientdb-server</artifactId>
             <version>${orientdb.version}</version>
+            <scope>provided</scope>
             <exclusions>
                 <exclusion>
                     <groupId>javax.activation</groupId>
                     <artifactId>activation</artifactId>
                 </exclusion>
+                <!-- See orientdb-core above: excluded GraalVM transitive dependencies. -->
+                <exclusion>
+                    <groupId>org.graalvm.sdk</groupId>
+                    <artifactId>graal-sdk</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.truffle</groupId>
+                    <artifactId>truffle-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.js</groupId>
+                    <artifactId>js</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.js</groupId>
+                    <artifactId>js-scriptengine</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.tools</groupId>
+                    <artifactId>profiler</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.tools</groupId>
+                    <artifactId>chromeinspector</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.graalvm.regex</groupId>
+                    <artifactId>regex</artifactId>
+                </exclusion>
+                <!--
+                    JAXB stack pulled in by orientdb-server for OServerConfiguration XML
+                    (de)serialization. OpenIDM's EmbeddedOServerService builds the OServer
+                    config programmatically (no XML), so JAXB is not needed at runtime.
+                    The jaxb bundles fail OSGi resolution because they require
+                    jakarta.xml.bind 3.0.x which is not provided by any other bundle in
+                    the OpenIDM runtime.
+                -->
+                <exclusion>
+                    <groupId>org.glassfish.jaxb</groupId>
+                    <artifactId>jaxb-runtime</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.glassfish.jaxb</groupId>
+                    <artifactId>jaxb-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.glassfish.jaxb</groupId>
+                    <artifactId>txw2</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.istack</groupId>
+                    <artifactId>istack-commons-runtime</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>jakarta.xml.bind</groupId>
+                    <artifactId>jakarta.xml.bind-api</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
-            <groupId>com.orientechnologies</groupId>
-            <artifactId>orientdb-enterprise</artifactId>
-            <version>${orientdb.version}</version>
-        </dependency>
         <dependency>
             <groupId>net.java.dev.jna</groupId>
             <artifactId>jna</artifactId>
@@ -158,10 +256,37 @@
                 <extensions>true</extensions>
                 <configuration>
                     <instructions>
-                        <Embed-Dependency>persistence-api;scope=provided</Embed-Dependency>
+                        <!--
+                            Embed only the OrientDB core/server jars and the JNR/jffi
+                            stack into this bundle. These artifacts are NOT OSGi bundles
+                            (OrientDB jars lack Bundle-SymbolicName; jffi lacks any OSGi
+                            metadata at all), so installing them as standalone bundles in
+                            openidm/bundle/ fails OSGi resolution. Embedding them here
+                            keeps the bundle self-contained.
+
+                            JNA is excluded because it is already a proper OSGi bundle and
+                            is deployed standalone via openidm-zip's runtime scope.
+
+                            Embed-Transitive is intentionally false to avoid accidentally
+                            embedding unrelated artifacts (e.g. osgi.core or other
+                            openidm-* bundles) which would cause ClassCastExceptions at
+                            runtime due to duplicate classes loaded by different
+                            classloaders.
+                        -->
+                        <Embed-Dependency>*;groupId=com.orientechnologies|com.github.jnr|com.googlecode.concurrentlinkedhashmap|at.yawk.lz4;inline=false</Embed-Dependency>
+                        <Embed-Transitive>true</Embed-Transitive>
                         <Export-Package>org.forgerock.openidm.repo.orientdb.metadata;version=${project.version}</Export-Package>
                         <Private-Package>org.forgerock.openidm.repo.orientdb.impl.*</Private-Package>
                         <Bundle-Activator>org.forgerock.openidm.repo.orientdb.impl.Activator</Bundle-Activator>
+                        <!--
+                            The embedded OrientDB and JNR code references many
+                            optional/platform-specific packages (graalvm, truffle,
+                            jakarta.xml.bind, com.sun.management, ...) that are not
+                            present in the OpenIDM runtime. Marking imports optional
+                            lets the bundle resolve and start without them.
+                        -->
+                        <Import-Package>*;resolution:=optional</Import-Package>
+                        <DynamicImport-Package>*</DynamicImport-Package>
                     </instructions>
                 </configuration>
             </plugin>

openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/DBHelper.java

@@ -52,7 +52,6 @@
 import com.orientechnologies.orient.core.metadata.security.OSecurity;
 import com.orientechnologies.orient.core.metadata.security.OUser;
 import com.orientechnologies.orient.core.record.impl.ODocument;
-import com.orientechnologies.orient.core.storage.OStorage;
 
 import java.util.Collection;
 import java.util.Set;
@@ -205,9 +204,6 @@ private static ODatabaseDocumentPool initPool(String dbURL, String user, String
         // Immediate disk sync for commit
         OGlobalConfiguration.TX_COMMIT_SYNCH.setValue(true);
 
-        // Have the storage closed when the DB is closed.
-        OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false);
-
         boolean success = false;
         int maxRetry = 10;
         int retryCount = 0;
@@ -512,8 +508,13 @@ private static void createOrUpdateOrientDBClass(ODatabaseDocumentTx db, OSchema
         OClass orientClass = schema.getClass(orientClassName);
         if (orientClass == null) {
             logger.info("OrientDB class {} does not exist and is being created.", orientClassName);
+            // OrientDB 3.x: createClass(String, int) treats the int as "number of
+            // clusters", NOT a cluster id (this was an API change from 2.x).
+            // Bind the class explicitly to the cluster we just created via the
+            // int[] overload to avoid creating N additional clusters per class
+            // (which causes cluster id overflow once it exceeds 32767).
             orientClass = schema.createClass(orientClassName,
-                    db.addCluster(orientClassName));
+                    new int[] { db.addCluster(orientClassName) });
         }
 
         List<String> indexProperties = new ArrayList<String>();
@@ -547,7 +548,7 @@ private static void createOrUpdateOrientDBClass(ODatabaseDocumentTx db, OSchema
             String[] propertyNames = propNamesList.toArray(new String[propNamesList.size()]);
             if (propertyNames.length > 0) {
                 String indexName = uniqueIndexName(orientClass.getName(), propertyNames);
-                OIndex<?> oIndex = orientClass.getClassIndex(indexName);
+                OIndex oIndex = orientClass.getClassIndex(indexName);
                 if (oIndex != null && !oIndex.getType().equalsIgnoreCase(indexType)) {
                     indexManager.dropIndex(indexName);
                     oIndex = null;
@@ -568,8 +569,8 @@ private static void createOrUpdateOrientDBClass(ODatabaseDocumentTx db, OSchema
             String propName = property.getName();
             if (!indexProperties.contains(propName))
             {
-                Set<OIndex<?>> propIndexes = indexManager.getClassInvolvedIndexes(orientClass.getName(), propName);
-                for (OIndex<?> propIndex : propIndexes) {
+                Set<OIndex> propIndexes = indexManager.getClassInvolvedIndexes(orientClass.getName(), propName);
+                for (OIndex propIndex : propIndexes) {
                     // Ensure that we only drop indexes which we created and
                     // match the OpenIDM index naming convention
                     String indexRegex = uniqueIndexName(orientClass.getName(), new String[]{".*"});

openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/EmbeddedOServerService.java

@@ -263,12 +263,14 @@ protected OServerConfiguration getOrientDBConfig(JsonValue config) {
                 new OServerUserConfiguration("guest", null, "server.listDatabases")
         };
         configuration.properties = new OServerEntryConfiguration[]{
-                new OServerEntryConfiguration("server.cache.staticResources", "false"),
-                new OServerEntryConfiguration("orientdb.www.path", "db/util/orientdb/studio"),
-                new OServerEntryConfiguration("orient.home", dbFolder.getAbsolutePath())
+                new OServerEntryConfiguration("server.cache.staticResources", "false")
+                // Legacy OrientDB 2.x server entries "orientdb.www.path" (OrientDB Studio
+                // is not shipped) and "orient.home" were removed: 3.x logs them as
+                // "Ignored storage configuration because not supported" on every start.
+                // ORIENTDB_HOME is set below as a system property which is the canonical
+                // way in 3.x.
         };
-        // OrientDB currently logs a warning if this is not set, 
-        // although it should be taking the setting from the config above instead.
+        // OrientDB expects ORIENTDB_HOME to be set as a system property.
         System.setProperty("ORIENTDB_HOME", dbFolder.getAbsolutePath());
 
         return configuration;

openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/OrientDBRepoService.java

@@ -88,7 +88,6 @@
 import com.orientechnologies.orient.core.index.OIndexException;
 import com.orientechnologies.orient.core.record.impl.ODocument;
 import com.orientechnologies.orient.core.storage.ORecordDuplicatedException;
-import com.orientechnologies.orient.core.version.OSimpleVersion;
 
 /**
  * Repository service implementation using OrientDB
@@ -418,7 +417,7 @@ public ResourceResponse delete(DeleteRequest request) throws ResourceException {
                 throw new NotFoundException("Object does not exist for delete on: " + request.getResourcePath());
             }
 
-            db.delete(existingDoc.getIdentity(), new OSimpleVersion(ver));
+            db.delete(existingDoc.getIdentity(), ver);
             logger.debug("delete for id succeeded: {} revision: {}", localId, request.getRevision());
             return DocumentUtil.toResource(existingDoc);
         } catch (ODatabaseException ex) {

openidm-repo-orientdb/src/test/java/org/forgerock/openidm/repo/orientdb/impl/DocumentUtilTest.java

@@ -29,7 +29,6 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.orientechnologies.common.log.OLogManager;
-import com.orientechnologies.orient.core.db.ODatabaseRecordThreadLocal;
 import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
 import com.orientechnologies.orient.core.metadata.schema.OType;
 import com.orientechnologies.orient.core.record.impl.ODocument;
@@ -72,6 +71,10 @@ public void init() throws Exception {
         } else {
             db.open("admin", "admin");
         }
+        // OrientDB 3.x requires the class to exist in the schema before newInstance() can be called
+        if (!db.getMetadata().getSchema().existsClass(orientDocClass)) {
+            db.getMetadata().getSchema().createClass(orientDocClass);
+        }
     }
 
     @AfterClass 
@@ -82,7 +85,7 @@ public void cleanup() {
     }
 
     public ODatabaseDocumentTx getDatabase() {
-        ODatabaseRecordThreadLocal.INSTANCE.set(db);
+        db.activateOnCurrentThread();
         return db;
     }
 

openidm-zip/src/main/resources/conf/boot/boot.properties

@@ -1,3 +1,10 @@
+# Maximum time (ms) HealthService waits after framework start before
+# checking that all required services are up and reporting startup failure.
+# Default in code is 15s, but on restart with an existing OrientDB database
+# the pool reopen + bundle reactivation can exceed 15s, causing a spurious
+# "SEVERE: OpenIDM failure during startup" on otherwise healthy restarts.
+openidm.healthservice.servicestartmax=900000
+
 openidm.port.http=8080
 openidm.port.https=8443
 openidm.port.mutualauth=8444

openidm-zip/src/main/resources/conf/logging.properties

@@ -73,6 +73,14 @@ org.identityconnectors.framework.impl.api.local.LocalConnectorInfoManagerImpl.le
 # Suppress warnings of failed error page model validation
 org.ops4j.pax.web.service.spi.model.elements.ErrorPageModel.level=SEVERE
 
+# OrientDB 3.x: suppress harmless WARNINGs that we cannot act on
+# - OScriptManager logs "ECMAScript engine not found" when no JSR-223 javascript
+#   engine is on the classpath (we don't ship one and don't use OrientDB JS).
+# - ONative logs "Error detecting block size ignoring" when the JNA-based
+#   filesystem block-size probe fails (e.g. on some Linux/container FS).
+com.orientechnologies.orient.core.command.script.OScriptManager.level=SEVERE
+com.orientechnologies.common.jna.ONative.level=SEVERE
+
 ############################################################
 # Handler specific properties.
 # Describes specific configuration info for Handlers.

openidm-zip/src/main/resources/shutdown.sh

@@ -50,6 +50,22 @@ EXISTING_START_RUNNING=`ps -p $START_PID -o command= | grep "./startup.sh"`
 if [ "$EXISTING_START_RUNNING" ]; then
     echo "Stopping OpenIDM ($START_PID)"
     pkill -P $START_PID
+    # Wait for the start script and its child JVM to actually exit before
+    # removing the PID file. Without this the caller (or the system)
+    # may immediately start a new instance while the previous JVM is still
+    # flushing on-disk state, which leaves the embedded OrientDB storage
+    # locked ("Database is locked by another process, please shutdown
+    # process and try again") on the next startup.
+    SHUTDOWN_WAIT=${OPENIDM_SHUTDOWN_WAIT:-120}
+    while [ "$SHUTDOWN_WAIT" -gt 0 ] && kill -0 "$START_PID" 2>/dev/null; do
+        sleep 1
+        SHUTDOWN_WAIT=$((SHUTDOWN_WAIT - 1))
+    done
+    if kill -0 "$START_PID" 2>/dev/null; then
+        echo "OpenIDM ($START_PID) did not stop within the timeout; sending SIGKILL"
+        pkill -KILL -P $START_PID
+        kill -KILL "$START_PID" 2>/dev/null
+    fi
     cleanupPidFile
     exit 0
 fi

openidm-zip/src/main/resources/startup.bat

@@ -37,6 +37,18 @@ set "ADD_OPENS_ARGS="
 if /I %JAVA_VERSION% GEQ 9 (
   set ADD_OPENS_ARGS=--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED
 )
+rem Silence JDK 17+ "restricted method" warning emitted by jffi (jnr-posix used
+rem by OrientDB) calling System::load to load its native stub.
+if /I %JAVA_VERSION% GEQ 17 (
+  set ADD_OPENS_ARGS=%ADD_OPENS_ARGS% --enable-native-access=ALL-UNNAMED
+)
+rem Silence JDK 23+ "terminally deprecated sun.misc.Unsafe" warning emitted by
+rem org.apache.felix.framework.util.SecureAction#staticFieldOffset. Remove once
+rem org.apache.felix.framework is upgraded to a release that no longer uses
+rem sun.misc.Unsafe::staticFieldOffset.
+if /I %JAVA_VERSION% GEQ 23 (
+  set ADD_OPENS_ARGS=%ADD_OPENS_ARGS% --sun-misc-unsafe-memory-access=allow
+)
 
 set "JPDA="
 

openidm-zip/src/main/resources/startup.sh

@@ -45,6 +45,18 @@ if [ $JAVA_VER -ge 90 ]; then
         --add-opens=java.base/java.util=ALL-UNNAMED
         --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED"
 fi
+# Silence JDK 17+ "restricted method" warning emitted by jffi (jnr-posix used by
+# OrientDB) calling System::load to load its native stub.
+if [ $JAVA_VER -ge 170 ]; then
+    ADD_OPENS_ARGS="$ADD_OPENS_ARGS --enable-native-access=ALL-UNNAMED"
+fi
+# Silence JDK 23+ "terminally deprecated sun.misc.Unsafe" warning emitted by
+# org.apache.felix.framework.util.SecureAction#staticFieldOffset. Remove once
+# org.apache.felix.framework is upgraded to a release that no longer uses
+# sun.misc.Unsafe::staticFieldOffset.
+if [ $JAVA_VER -ge 230 ]; then
+    ADD_OPENS_ARGS="$ADD_OPENS_ARGS --sun-misc-unsafe-memory-access=allow"
+fi
 
 
 # clean up left over pid files if necessary