We have a Neo4J test database with approximately 2.5 million nodes. We are attempting to update every node to include some additional properties. To accomplish this, we are using the REST API to update each node in turn.
About halfway through the process, we receive an out of memory error response from the server, and the following stack trace is returned in the REST response:
{
"message" : "java.io.IOException: Map failed",
"exception" : "RuntimeException",
"fullname" : "java.lang.RuntimeException",
"stacktrace" : [
"org.neo4j.index.impl.lucene.LuceneDataSource.refreshSearcher(LuceneDataSource.java:514)",
"org.neo4j.index.impl.lucene.LuceneDataSource.refreshSearcherIfNeeded(LuceneDataSource.java:631)",
"org.neo4j.index.impl.lucene.LuceneDataSource.getIndexSearcher(LuceneDataSource.java:575)",
"org.neo4j.index.impl.lucene.LuceneIndex.query(LuceneIndex.java:268)", "org.neo4j.index.impl.lucene.LuceneIndex.get(LuceneIndex.java:207)",
"org.neo4j.kernel.AbstractAutoIndexerImpl$ReadOnlyIndexToIndexAdapter.get(AbstractAutoIndexerImpl.java:236)",
"org.neo4j.cypher.internal.spi.gdsimpl.GDSBackedQueryContext$$anon$1.indexGet(GDSBackedQueryContext.scala:84)",
"org.neo4j.cypher.internal.executionplan.builders.IndexQueryBuilder$$anonfun$getNodeGetter$1.apply(IndexQueryBuilder.scala:76)",
"org.neo4j.cypher.internal.executionplan.builders.IndexQueryBuilder$$anonfun$getNodeGetter$1.apply(IndexQueryBuilder.scala:73)",
"org.neo4j.cypher.internal.pipes.StartPipe$$anonfun$internalCreateResults$1.apply(StartPipe.scala:36)",
"org.neo4j.cypher.internal.pipes.StartPipe$$anonfun$internalCreateResults$1.apply(StartPipe.scala:35)",
"scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)",
"scala.collection.Iterator$class.foreach(Iterator.scala:727)",
"scala.collection.AbstractIterator.foreach(Iterator.scala:1156)",
"scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)",
"scala.collection.mutable.ListBuffer.$plus$plus$eq(ListBuffer.scala:178)",
"scala.collection.mutable.ListBuffer.$plus$plus$eq(ListBuffer.scala:45)",
"scala.collection.TraversableOnce$class.to(TraversableOnce.scala:259)",
"scala.collection.AbstractIterator.to(Iterator.scala:1156)",
"scala.collection.TraversableOnce$class.toList(TraversableOnce.scala:243)",
"scala.collection.AbstractIterator.toList(Iterator.scala:1156)",
"org.neo4j.cypher.internal.pipes.EagerPipe.internalCreateResults(EagerPipe.scala:35)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:69)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:66)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:66)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:66)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl.org$neo4j$cypher$internal$executionplan$ExecutionPlanImpl$$prepareStateAndResult(ExecutionPlanImpl.scala:164)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl$$anonfun$6.apply(ExecutionPlanImpl.scala:146)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl$$anonfun$6.apply(ExecutionPlanImpl.scala:145)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl.execute(ExecutionPlanImpl.scala:38)",
"org.neo4j.cypher.ExecutionEngine.execute(ExecutionEngine.scala:72)",
"org.neo4j.cypher.ExecutionEngine.execute(ExecutionEngine.scala:76)",
"org.neo4j.cypher.javacompat.ExecutionEngine.execute(ExecutionEngine.java:79)",
"org.neo4j.server.rest.web.CypherService.cypher(CypherService.java:94)",
"java.lang.reflect.Method.invoke(Method.java:606)",
"org.neo4j.server.rest.security.SecurityFilter.doFilter(SecurityFilter.java:112)",
"org.neo4j.server.extension.auth.AuthenticationFilter.doFilter(AuthenticationFilter.java:57)"
],
"cause" : {
"message" : "Map failed",
"cause" : {
"message" : "Map failed",
"exception" : "OutOfMemoryError",
"stacktrace" : [
"sun.nio.ch.FileChannelImpl.map0(Native Method)",
"sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:846)",
"org.apache.lucene.store.MMapDirectory$MMapIndexInput.(MMapDirectory.java:270)",
"org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:220)",
"org.apache.lucene.store.FSDirectory.openInput(FSDirectory.java:345)",
"org.apache.lucene.index.CompoundFileWriter.copyFile(CompoundFileWriter.java:231)",
"org.apache.lucene.index.CompoundFileWriter.close(CompoundFileWriter.java:201)",
"org.apache.lucene.index.DocumentsWriter.flush(DocumentsWriter.java:604)",
"org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3587)",
"org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)",
"org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)",
"org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)",
"org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)",
"org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)",
"org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)",
"org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:392)",
"org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:634)",
"org.neo4j.index.impl.lucene.LuceneDataSource.refreshSearcher(LuceneDataSource.java:503)",
"org.neo4j.index.impl.lucene.LuceneDataSource.refreshSearcherIfNeeded(LuceneDataSource.java:631)",
"org.neo4j.index.impl.lucene.LuceneDataSource.getIndexSearcher(LuceneDataSource.java:575)",
"org.neo4j.index.impl.lucene.LuceneIndex.query(LuceneIndex.java:268)",
"org.neo4j.index.impl.lucene.LuceneIndex.get(LuceneIndex.java:207)",
"org.neo4j.kernel.AbstractAutoIndexerImpl$ReadOnlyIndexToIndexAdapter.get(AbstractAutoIndexerImpl.java:236)",
"org.neo4j.cypher.internal.spi.gdsimpl.GDSBackedQueryContext$$anon$1.indexGet(GDSBackedQueryContext.scala:84)",
"org.neo4j.cypher.internal.executionplan.builders.IndexQueryBuilder$$anonfun$getNodeGetter$1.apply(IndexQueryBuilder.scala:76)",
"org.neo4j.cypher.internal.executionplan.builders.IndexQueryBuilder$$anonfun$getNodeGetter$1.apply(IndexQueryBuilder.scala:73)",
"org.neo4j.cypher.internal.pipes.StartPipe$$anonfun$internalCreateResults$1.apply(StartPipe.scala:36)",
"org.neo4j.cypher.internal.pipes.StartPipe$$anonfun$internalCreateResults$1.apply(StartPipe.scala:35)", "scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)",
"scala.collection.Iterator$class.foreach(Iterator.scala:727)",
"scala.collection.AbstractIterator.foreach(Iterator.scala:1156)",
"scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)",
"scala.collection.mutable.ListBuffer.$plus$plus$eq(ListBuffer.scala:178)",
"scala.collection.mutable.ListBuffer.$plus$plus$eq(ListBuffer.scala:45)",
"scala.collection.TraversableOnce$class.to(TraversableOnce.scala:259)",
"scala.collection.AbstractIterator.to(Iterator.scala:1156)",
"scala.collection.TraversableOnce$class.toList(TraversableOnce.scala:243)",
"scala.collection.AbstractIterator.toList(Iterator.scala:1156)",
"org.neo4j.cypher.internal.pipes.EagerPipe.internalCreateResults(EagerPipe.scala:35)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:69)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:66)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:66)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:66)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl.org$neo4j$cypher$internal$executionplan$ExecutionPlanImpl$$prepareStateAndResult(ExecutionPlanImpl.scala:164)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl$$anonfun$6.apply(ExecutionPlanImpl.scala:146)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl$$anonfun$6.apply(ExecutionPlanImpl.scala:145)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl.execute(ExecutionPlanImpl.scala:38)", "org.neo4j.cypher.ExecutionEngine.execute(ExecutionEngine.scala:72)",
"org.neo4j.cypher.ExecutionEngine.execute(ExecutionEngine.scala:76)",
"org.neo4j.cypher.javacompat.ExecutionEngine.execute(ExecutionEngine.java:79)",
"org.neo4j.server.rest.web.CypherService.cypher(CypherService.java:94)",
"java.lang.reflect.Method.invoke(Method.java:606)",
"org.neo4j.server.rest.security.SecurityFilter.doFilter(SecurityFilter.java:112)",
"org.neo4j.server.extension.auth.AuthenticationFilter.doFilter(AuthenticationFilter.java:57)"
],
"fullname" : "java.lang.OutOfMemoryError"
},
"exception" : "IOException",
"stacktrace" : [
"sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:849)",
"org.apache.lucene.store.MMapDirectory$MMapIndexInput.(MMapDirectory.java:270)",
"org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:220)",
"org.apache.lucene.store.FSDirectory.openInput(FSDirectory.java:345)",
"org.apache.lucene.index.CompoundFileWriter.copyFile(CompoundFileWriter.java:231)",
"org.apache.lucene.index.CompoundFileWriter.close(CompoundFileWriter.java:201)",
"org.apache.lucene.index.DocumentsWriter.flush(DocumentsWriter.java:604)",
"org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3587)",
"org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)",
"org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)",
"org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)",
"org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)",
"org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)",
"org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)",
"org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:392)",
"org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:634)",
"org.neo4j.index.impl.lucene.LuceneDataSource.refreshSearcher(LuceneDataSource.java:503)",
"org.neo4j.index.impl.lucene.LuceneDataSource.refreshSearcherIfNeeded(LuceneDataSource.java:631)",
"org.neo4j.index.impl.lucene.LuceneDataSource.getIndexSearcher(LuceneDataSource.java:575)",
"org.neo4j.index.impl.lucene.LuceneIndex.query(LuceneIndex.java:268)",
"org.neo4j.index.impl.lucene.LuceneIndex.get(LuceneIndex.java:207)",
"org.neo4j.kernel.AbstractAutoIndexerImpl$ReadOnlyIndexToIndexAdapter.get(AbstractAutoIndexerImpl.java:236)",
"org.neo4j.cypher.internal.spi.gdsimpl.GDSBackedQueryContext$$anon$1.indexGet(GDSBackedQueryContext.scala:84)",
"org.neo4j.cypher.internal.executionplan.builders.IndexQueryBuilder$$anonfun$getNodeGetter$1.apply(IndexQueryBuilder.scala:76)",
"org.neo4j.cypher.internal.executionplan.builders.IndexQueryBuilder$$anonfun$getNodeGetter$1.apply(IndexQueryBuilder.scala:73)",
"org.neo4j.cypher.internal.pipes.StartPipe$$anonfun$internalCreateResults$1.apply(StartPipe.scala:36)",
"org.neo4j.cypher.internal.pipes.StartPipe$$anonfun$internalCreateResults$1.apply(StartPipe.scala:35)",
"scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)",
"scala.collection.Iterator$class.foreach(Iterator.scala:727)",
"scala.collection.AbstractIterator.foreach(Iterator.scala:1156)",
"scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)",
"scala.collection.mutable.ListBuffer.$plus$plus$eq(ListBuffer.scala:178)",
"scala.collection.mutable.ListBuffer.$plus$plus$eq(ListBuffer.scala:45)",
"scala.collection.TraversableOnce$class.to(TraversableOnce.scala:259)",
"scala.collection.AbstractIterator.to(Iterator.scala:1156)",
"scala.collection.TraversableOnce$class.toList(TraversableOnce.scala:243)",
"scala.collection.AbstractIterator.toList(Iterator.scala:1156)",
"org.neo4j.cypher.internal.pipes.EagerPipe.internalCreateResults(EagerPipe.scala:35)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:69)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:66)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:66)",
"org.neo4j.cypher.internal.pipes.PipeWithSource.createResults(Pipe.scala:66)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl.org$neo4j$cypher$internal$executionplan$ExecutionPlanImpl$$prepareStateAndResult(ExecutionPlanImpl.scala:164)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl$$anonfun$6.apply(ExecutionPlanImpl.scala:146)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl$$anonfun$6.apply(ExecutionPlanImpl.scala:145)",
"org.neo4j.cypher.internal.executionplan.ExecutionPlanImpl.execute(ExecutionPlanImpl.scala:38)", "org.neo4j.cypher.ExecutionEngine.execute(ExecutionEngine.scala:72)",
"org.neo4j.cypher.ExecutionEngine.execute(ExecutionEngine.scala:76)",
"org.neo4j.cypher.javacompat.ExecutionEngine.execute(ExecutionEngine.java:79)",
"org.neo4j.server.rest.web.CypherService.cypher(CypherService.java:94)",
"java.lang.reflect.Method.invoke(Method.java:606)",
"org.neo4j.server.rest.security.SecurityFilter.doFilter(SecurityFilter.java:112)",
"org.neo4j.server.extension.auth.AuthenticationFilter.doFilter(AuthenticationFilter.java:57)"
],
"fullname" : "java.io.IOException"
}
}
I have also included an example of our request content and headers:
JSON content:
{"query":"START myNode=node:node_auto_index(myIndex = {myIndex}) SET myNode.propOne = {propOne}, myNode.propTwo = {propTwo}, myNode.propThree = {propThree}","params":{"propOne":"valueOne","propTwo":"valueTwo","propThree":"valueThree","myIndex":"indexValue"}}
Request Headers:
POST /db/data/cypher HTTP/1.1
Host: localhost:7474
Accept: application/json
Content-Type: application/json
Content-Length:
neo4j-wrapper.conf
wrapper.java.additional=-Dorg.neo4j.server.properties=conf/neo4j-server.properties
wrapper.java.additional=-Djava.util.logging.config.file=conf/logging.properties
wrapper.java.additional=-Dlog4j.configuration=file:conf/log4j.properties
#********************************************************************
# JVM Parameters
#********************************************************************
wrapper.java.additional=-XX:+UseConcMarkSweepGC
wrapper.java.additional=-XX:+CMSClassUnloadingEnabled
# Uncomment the following lines to enable garbage collection logging
#wrapper.java.additional=-Xloggc:data/log/neo4j-gc.log
#wrapper.java.additional=-XX:+PrintGCDetails
#wrapper.java.additional=-XX:+PrintGCDateStamps
#wrapper.java.additional=-XX:+PrintGCApplicationStoppedTime
# Initial Java Heap Size (in MB)
wrapper.java.initmemory=2048
# Maximum Java Heap Size (in MB)
wrapper.java.maxmemory=8192
#********************************************************************
# Wrapper settings
#********************************************************************
# Override default pidfile and lockfile
#wrapper.pidfile=../data/neo4j-server.pid
#wrapper.lockfile=../data/neo4j-server.lck
#********************************************************************
# Wrapper Windows NT/2000/XP Service Properties
#********************************************************************
# WARNING - Do not modify any of these properties when an application
# using this configuration file has been installed as a service.
# Please uninstall the service before modifying this section. The
# service can then be reinstalled.
# Name of the service
wrapper.name=neo4j
# User account to be used for linux installs. Will default to current
# user if not set.
wrapper.user=neo4j
wrapper.user.created=false
neo4j.properties
# Default values for the low-level graph engine
neostore.nodestore.db.mapped_memory=100M
neostore.relationshipstore.db.mapped_memory=2G
neostore.propertystore.db.mapped_memory=90M
neostore.propertystore.db.strings.mapped_memory=250M
neostore.propertystore.db.arrays.mapped_memory=130M
# Enable this to be able to upgrade a store from 1.4 -> 1.5 or 1.4 -> 1.6
#allow_store_upgrade=true
# Enable this to specify a parser other than the default one. 1.5, 1.6, 1.7 are available
#cypher_parser_version=1.6
# Keep logical logs, helps debugging but uses more disk space, enabled for
# legacy reasons To limit space needed to store historical logs use values such
# as: "7 days" or "100M size" instead of "true"
keep_logical_logs=true
# Autoindexing
# Enable auto-indexing for nodes, default is false
node_auto_indexing=true
# The node property keys to be auto-indexed, if enabled
node_keys_indexable=user_id,group_id,program_id
# Enable auto-indexing for relationships, default is false
#relationship_auto_indexing=true
# The relationship property keys to be auto-indexed, if enabled
#relationship_keys_indexable=name,age
PLEASE READ VERY CAREFULLY:
I am not necessarily looking for alternative methods for updating our Node properties. I am actually very confident that we could easily come up with something that works around this error. My purpose in posting this question is to better understand the nature of this error. What exactly is causing it in the first place? My hope is that understanding the cause of this error would better help us to avoid it in a production environment.
Thank you in advance.