Hi,
we are running two instances of Geonetwork 2.11 in two different environments. We are experiencing relatively long upload and saving times of metadata, which worsens the more metadata is edited in a session.
Example:
- User creates a new metadata set
- User uploads a file as resource to the metadata. This runs reasonably fast. User subsequently uploads further files which get slower and slower
On top of that the saving of the metadata sometimes takes ages.
Now we are wondering if any of you has experienced a similar behavior and could give us a hint at what could go wrong here and how we might potentially resolve this situation. Is something clogging up the pipeline in Geonetwork?
A bit of information about the setup:
- Apache Tomcat 7
- Geonetwork 2.11.0
- JDK 1.8.0
- Nginx on one server, Apache Webserver on the other
- plenty of RAM and CPU on both machines
If you need further information, please let us know.
At the bottom is also our metrics report from within Geonetwork if that is of any help.
Best regards and thanks in advance,
Max
Metric report:
{
"jvm" : {
"vm" : {
"name" : "Java HotSpot(TM) 64-Bit Server VM",
"version" : "1.8.0_74-b02"
},
"memory" : {
"totalInit" : 2.150039552E9,
"totalUsed" : 2.23797416E9,
"totalMax" : 7.635730431E9,
"totalCommitted" : 3.352387584E9,
"heapInit" : 2.147483648E9,
"heapUsed" : 1.919411216E9,
"heapMax" : 7.635730432E9,
"heapCommitted" : 3.029336064E9,
"heap_usage" : 0.25137231245829295,
"non_heap_usage" : -3.18563264E8,
"memory_pool_usages" : {
"Code Cache" : 0.4138537089029948,
"Compressed Class Space" : 0.022932425141334534,
"Metaspace" : 0.9846254336311864,
"PS Eden Space" : 0.281616335591551,
"PS Old Gen" : 0.19324252505687897,
"PS Survivor Space" : 0.9827257044175092
}
},
"buffers" : {
"direct" : {
"count" : 26,
"memoryUsed" : 265366,
"totalCapacity" : 265366
},
"mapped" : {
"count" : 36,
"memoryUsed" : 36088917,
"totalCapacity" : 36088917
}
},
"daemon_thread_count" : 82,
"thread_count" : 100,
"current_time" : 1477517209866,
"uptime" : 82361,
"fd_usage" : 0.13427734375,
"thread-states" : {
"blocked" : 0.0,
"timed_waiting" : 0.43,
"terminated" : 0.0,
"new" : 0.0,
"runnable" : 0.23,
"waiting" : 0.34
},
"garbage-collectors" : {
"PS MarkSweep" : {
"runs" : 5,
"time" : 977
},
"PS Scavenge" : {
"runs" : 511,
"time" : 11439
}
}
},
"jeeves.server.dispatchers.ServiceManager" : {
"Gui_services_Timer" : {
"type" : "timer",
"duration" : {
"unit" : "milliseconds",
"min" : 4.838434,
"max" : 448.894587,
"mean" : 25.004167848434925,
"std_dev" : 37.10869062032913,
"median" : 12.319728,
"p75" : 21.179423,
"p95" : 108.213322,
"p98" : 114.799535,
"p99" : 121.936955,
"p999" : 448.894587
},
"rate" : {
"unit" : "seconds",
"count" : 607,
"mean" : 0.007370941214757464,
"m1" : 0.03797902818331702,
"m5" : 0.02187304540294613,
"m15" : 0.01512406078639581
}
},
"Output_Xsl_Transform_Timer" : {
"type" : "timer",
"duration" : {
"unit" : "milliseconds",
"min" : 10.444868,
"max" : 924.161326,
"mean" : 283.26581205436577,
"std_dev" : 349.8874509559093,
"median" : 23.100825,
"p75" : 752.98687,
"p95" : 839.928618,
"p98" : 864.563982,
"p99" : 884.883624,
"p999" : 924.161326
},
"rate" : {
"unit" : "seconds",
"count" : 607,
"mean" : 0.007370941121587527,
"m1" : 0.03797902818331702,
"m5" : 0.021873054206556865,
"m15" : 0.015124394822848974
}
},
"Service_Execution_Timer" : {
"type" : "timer",
"duration" : {
"unit" : "milliseconds",
"min" : 0.001179,
"max" : 11894.264669,
"mean" : 114.71661791410213,
"std_dev" : 769.1157542158502,
"median" : 2.186037,
"p75" : 3.623323,
"p95" : 104.5106529,
"p98" : 255.88512237999996,
"p99" : 954.6706109800001,
"p999" : 11892.888171501001
},
"rate" : {
"unit" : "seconds",
"count" : 5914,
"mean" : 0.07181506608763712,
"m1" : 0.06095795415860447,
"m5" : 0.07780154899432454,
"m15" : 0.09836982919717568
}
}
},
"jeeves.server.resources.ResourceManager" : {
"Free files descriptors" : {
"type" : "gauge",
"value" : 3546
},
"Idle_Connections_By_ResourceProvider" : {
"type" : "gauge",
"value" : 9
},
"Max_Connections_By_ResourceProvider" : {
"type" : "gauge",
"value" : 10
},
"Open_Connections_By_ResourceProvider" : {
"type" : "gauge",
"value" : 1
},
"Resource_is_Open_Timer" : {
"type" : "timer",
"duration" : {
"unit" : "milliseconds",
"min" : 0.140602,
"max" : 11881.680428,
"mean" : 129.24544458654378,
"std_dev" : 799.745523949467,
"median" : 2.2522575,
"p75" : 20.28366775,
"p95" : 179.08490134999997,
"p98" : 937.55594718,
"p99" : 9598.781007630007,
"p999" : 11879.998650981
},
"rate" : {
"unit" : "seconds",
"count" : 5425,
"mean" : 0.06587700152502249,
"m1" : 0.08058732155912506,
"m5" : 0.06429577809682865,
"m15" : 0.07805247448192335
}
},
"Wait_for_Resource_Timer" : {
"type" : "timer",
"duration" : {
"unit" : "milliseconds",
"min" : 0.141708,
"max" : 124.470939,
"mean" : 0.6357193142857143,
"std_dev" : 2.617135027027867,
"median" : 0.4251145,
"p75" : 0.50009025,
"p95" : 0.7590875,
"p98" : 1.3063188,
"p99" : 1.8415297300000018,
"p999" : 23.052160866000005
},
"rate" : {
"unit" : "seconds",
"count" : 5425,
"mean" : 0.06587700064561762,
"m1" : 0.08058732153194864,
"m5" : 0.06429418224657406,
"m15" : 0.07805058400680523
}
}
},
"org.apache.log4j.Appender" : {
"all" : {
"type" : "meter",
"event_type" : "statements",
"unit" : "seconds",
"count" : 0,
"mean" : 0.0,
"m1" : 0.0,
"m5" : 0.0,
"m15" : 0.0
},
"debug" : {
"type" : "meter",
"event_type" : "statements",
"unit" : "seconds",
"count" : 0,
"mean" : 0.0,
"m1" : 0.0,
"m5" : 0.0,
"m15" : 0.0
},
"error" : {
"type" : "meter",
"event_type" : "statements",
"unit" : "seconds",
"count" : 0,
"mean" : 0.0,
"m1" : 0.0,
"m5" : 0.0,
"m15" : 0.0
},
"fatal" : {
"type" : "meter",
"event_type" : "statements",
"unit" : "seconds",
"count" : 0,
"mean" : 0.0,
"m1" : 0.0,
"m5" : 0.0,
"m15" : 0.0
},
"info" : {
"type" : "meter",
"event_type" : "statements",
"unit" : "seconds",
"count" : 0,
"mean" : 0.0,
"m1" : 0.0,
"m5" : 0.0,
"m15" : 0.0
},
"trace" : {
"type" : "meter",
"event_type" : "statements",
"unit" : "seconds",
"count" : 0,
"mean" : 0.0,
"m1" : 0.0,
"m5" : 0.0,
"m15" : 0.0
},
"warn" : {
"type" : "meter",
"event_type" : "statements",
"unit" : "seconds",
"count" : 0,
"mean" : 0.0,
"m1" : 0.0,
"m5" : 0.0,
"m15" : 0.0
}
},
"org.fao.geonet.kernel.harvest.harvester.AbstractHarvester" : {
"HarvestingErrors" : {
"type" : "counter",
"count" : 0
}
}
}