[Geoserver-users] Too many open files

I figured out that before geoserver starts with the "out-of-memory" errors it
starts with "Too many open files" and then it gets "stuck" (both stacktraces
are below). GeoServer is deployed on my Weblogic server and I'm bombarding
my GeoServer with inserts from my ESB. In every transaction it inserts into
2 different postgres datastores and deletes from a postgres datastore I'm
new to Geoserver and Weblogic server, so it might be a configuration problem
on either side. Any help would be great.

1955649 [WARNING] org.vfny.geoserver.ServiceException - encountered error:
java.io.IOException: Possible file permission problem. Root cause:
java.io.IOException: Too many open files
StackTrace: java.io.IOException: Possible file permission problem. Root
cause:
java.io.IOException: Too many open files
        at org.geoserver.request.Dispatcher.post(Dispatcher.java:226)
        at org.geoserver.request.Dispatcher.dispatch(Dispatcher.java:181)
        at
org.geoserver.request.Dispatcher.handleRequestInternal(Dispatcher.java:52)
        at
org.springframework.web.servlet.mvc.AbstractController.handleRequest(AbstractController.java:139)
        at
org.springframework.web.servlet.mvc.SimpleControllerHandlerAdapter.handle(SimpleControllerHandlerAdapter.java:44)
        at
org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:684)
        at
org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:625)
        at
org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:392)
        at
org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:357)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:763)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:856)
        at
weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:223)
        at
weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:125)
        at
weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:283)
        at weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:26)
        at
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)
        at
org.vfny.geoserver.filters.SetCharacterEncodingFilter.doFilter(SetCharacterEncodingFilter.java:103)
        at
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)
        at
weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3243)
        at
weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
        at
weblogic.security.service.SecurityManager.runAs(SecurityManager.java:121)
        at
weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2003)
        at
weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:1909)
        at
weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1359)
        at weblogic.work.ExecuteThread.execute(ExecuteThread.java:209)
        at weblogic.work.ExecuteThread.run(ExecuteThread.java:181)

------------------------------------------------------------------------------------------------------

<Jan 3, 2008 8:16:36 AM PST> <Error> <WebLogicServer> <BEA-000337> <[STUCK]
ExecuteThread: '13' for queue: 'weblogic.kernel.Default (self-tuning)' has
been busy for "675" seconds working on the request "Http Request:
/geoserver/wfs", which is more than the configured time (StuckThreadMaxTime)
of "600" seconds. Stack trace:
Thread-150 "[STUCK] ExecuteThread: '13' for queue: 'weblogic.kernel.Default
(self-tuning)'" <alive, in native, blocked, priority=1, DAEMON> {
    -- Blocked trying to get lock: java.security.SecureRandom@anonymised.com[fat
lock]
    jrockit.vm.Threads.waitForSignal(Native Method)
    jrockit.vm.Locks.fatLockBlockOrSpin(Unknown Source)
    jrockit.vm.Locks.lockFat(Unknown Source)
    jrockit.vm.Locks.monitorEnterSecondStage(Unknown Source)
    java.security.SecureRandom.nextBytes(SecureRandom.java:413)
    java.security.SecureRandom.next(SecureRandom.java:435)
    java.util.Random.nextInt(Unknown Source)
   
weblogic.servlet.internal.session.SessionData.getNextId(SessionData.java:986)
   
weblogic.servlet.internal.session.SessionData.generateSessionId(SessionData.java:163)
   
weblogic.servlet.internal.session.SessionData.<init>(SessionData.java:120)
   
weblogic.servlet.internal.session.MemorySessionData.<init>(MemorySessionData.java:9)
   
weblogic.servlet.internal.session.MemorySessionContext.getNewSession(MemorySessionContext.java:34)
   
weblogic.servlet.internal.ServletRequestImpl$SessionHelper.getNewSession(ServletRequestImpl.java:2523)
   
weblogic.servlet.internal.ServletRequestImpl$SessionHelper.getSessionInternal(ServletRequestImpl.java:2103)
   
weblogic.servlet.internal.ServletRequestImpl$SessionHelper.getSession(ServletRequestImpl.java:2072)
   
weblogic.servlet.internal.ServletRequestImpl$SessionHelper.getSession(ServletRequestImpl.java:2068)
   
weblogic.servlet.internal.ServletRequestImpl.getSession(ServletRequestImpl.java:1200)
    org.vfny.geoserver.util.Requests.getBaseUrl(Requests.java:108)
    org.vfny.geoserver.util.Requests.getSchemaBaseUrl(Requests.java:224)
    org.vfny.geoserver.Request.getSchemaBaseUrl(Request.java:223)
   
org.vfny.geoserver.wfs.responses.WfsTransResponse.writeXmlResponse(WfsTransResponse.java:179)
   
org.vfny.geoserver.wfs.responses.TransactionResponse.writeTo(TransactionResponse.java:813)
   
org.vfny.geoserver.servlets.AbstractService.doService(AbstractService.java:592)
   
org.vfny.geoserver.servlets.AbstractService.doPost(AbstractService.java:447)
    org.geoserver.request.Dispatcher.post(Dispatcher.java:301)
    org.geoserver.request.Dispatcher.dispatch(Dispatcher.java:181)
   
org.geoserver.request.Dispatcher.handleRequestInternal(Dispatcher.java:66)
   
org.springframework.web.servlet.mvc.AbstractController.handleRequest(AbstractController.java:139)
   
org.springframework.web.servlet.mvc.SimpleControllerHandlerAdapter.handle(SimpleControllerHandlerAdapter.java:44)
   
org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:684)
   
org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:625)
   
org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:392)
   
org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:358)
    javax.servlet.http.HttpServlet.service(HttpServlet.java:763)
    javax.servlet.http.HttpServlet.service(HttpServlet.java:856)
   
weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:223)
   
weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:125)
   
weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:230)
    weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:26)

--
View this message in context: http://www.nabble.com/Too-many-open-files-tp14605905p14605905.html
Sent from the GeoServer - User mailing list archive at Nabble.com.

Hi Dominique,

I just realized something by looking at the stack trace. Which version
of GeoServer are you using? It must not be a 1.6.x version because it
seems that the old dispatch system is in use.

A major problem with the old dispatch system is that temporary files are
created when you do a post. On top of being very inefficient for a large
post the dispatcher does cleaning up after itself very well.

Is there any chance you can try out the latest 1.6 release (currently
RC2)? It employs a new dispatcher which remedies this behavior.

-Justin

dbhalsema wrote:

I figured out that before geoserver starts with the "out-of-memory" errors it
starts with "Too many open files" and then it gets "stuck" (both stacktraces
are below). GeoServer is deployed on my Weblogic server and I'm bombarding
my GeoServer with inserts from my ESB. In every transaction it inserts into
2 different postgres datastores and deletes from a postgres datastore I'm
new to Geoserver and Weblogic server, so it might be a configuration problem
on either side. Any help would be great.

1955649 [WARNING] org.vfny.geoserver.ServiceException - encountered error:
java.io.IOException: Possible file permission problem. Root cause:
java.io.IOException: Too many open files
StackTrace: java.io.IOException: Possible file permission problem. Root
cause:
java.io.IOException: Too many open files
        at org.geoserver.request.Dispatcher.post(Dispatcher.java:226)
        at org.geoserver.request.Dispatcher.dispatch(Dispatcher.java:181)
        at
org.geoserver.request.Dispatcher.handleRequestInternal(Dispatcher.java:52)
        at
org.springframework.web.servlet.mvc.AbstractController.handleRequest(AbstractController.java:139)
        at
org.springframework.web.servlet.mvc.SimpleControllerHandlerAdapter.handle(SimpleControllerHandlerAdapter.java:44)
        at
org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:684)
        at
org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:625)
        at
org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:392)
        at
org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:357)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:763)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:856)
        at
weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:223)
        at
weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:125)
        at
weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:283)
        at weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:26)
        at
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)
        at
org.vfny.geoserver.filters.SetCharacterEncodingFilter.doFilter(SetCharacterEncodingFilter.java:103)
        at
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)
        at
weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3243)
        at
weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
        at
weblogic.security.service.SecurityManager.runAs(SecurityManager.java:121)
        at
weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2003)
        at
weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:1909)
        at
weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1359)
        at weblogic.work.ExecuteThread.execute(ExecuteThread.java:209)
        at weblogic.work.ExecuteThread.run(ExecuteThread.java:181)

------------------------------------------------------------------------------------------------------

<Jan 3, 2008 8:16:36 AM PST> <Error> <WebLogicServer> <BEA-000337> <[STUCK]
ExecuteThread: '13' for queue: 'weblogic.kernel.Default (self-tuning)' has
been busy for "675" seconds working on the request "Http Request:
/geoserver/wfs", which is more than the configured time (StuckThreadMaxTime)
of "600" seconds. Stack trace:
Thread-150 "[STUCK] ExecuteThread: '13' for queue: 'weblogic.kernel.Default
(self-tuning)'" <alive, in native, blocked, priority=1, DAEMON> {
    -- Blocked trying to get lock: java.security.SecureRandom@anonymised.com[fat
lock]
    jrockit.vm.Threads.waitForSignal(Native Method)
    jrockit.vm.Locks.fatLockBlockOrSpin(Unknown Source)
    jrockit.vm.Locks.lockFat(Unknown Source)
    jrockit.vm.Locks.monitorEnterSecondStage(Unknown Source)
    java.security.SecureRandom.nextBytes(SecureRandom.java:413)
    java.security.SecureRandom.next(SecureRandom.java:435)
    java.util.Random.nextInt(Unknown Source)
   
weblogic.servlet.internal.session.SessionData.getNextId(SessionData.java:986)
   
weblogic.servlet.internal.session.SessionData.generateSessionId(SessionData.java:163)
   
weblogic.servlet.internal.session.SessionData.<init>(SessionData.java:120)
   
weblogic.servlet.internal.session.MemorySessionData.<init>(MemorySessionData.java:9)
   
weblogic.servlet.internal.session.MemorySessionContext.getNewSession(MemorySessionContext.java:34)
   
weblogic.servlet.internal.ServletRequestImpl$SessionHelper.getNewSession(ServletRequestImpl.java:2523)
   
weblogic.servlet.internal.ServletRequestImpl$SessionHelper.getSessionInternal(ServletRequestImpl.java:2103)
   
weblogic.servlet.internal.ServletRequestImpl$SessionHelper.getSession(ServletRequestImpl.java:2072)
   
weblogic.servlet.internal.ServletRequestImpl$SessionHelper.getSession(ServletRequestImpl.java:2068)
   
weblogic.servlet.internal.ServletRequestImpl.getSession(ServletRequestImpl.java:1200)
    org.vfny.geoserver.util.Requests.getBaseUrl(Requests.java:108)
    org.vfny.geoserver.util.Requests.getSchemaBaseUrl(Requests.java:224)
    org.vfny.geoserver.Request.getSchemaBaseUrl(Request.java:223)
   
org.vfny.geoserver.wfs.responses.WfsTransResponse.writeXmlResponse(WfsTransResponse.java:179)
   
org.vfny.geoserver.wfs.responses.TransactionResponse.writeTo(TransactionResponse.java:813)
   
org.vfny.geoserver.servlets.AbstractService.doService(AbstractService.java:592)
   
org.vfny.geoserver.servlets.AbstractService.doPost(AbstractService.java:447)
    org.geoserver.request.Dispatcher.post(Dispatcher.java:301)
    org.geoserver.request.Dispatcher.dispatch(Dispatcher.java:181)
   
org.geoserver.request.Dispatcher.handleRequestInternal(Dispatcher.java:66)
   
org.springframework.web.servlet.mvc.AbstractController.handleRequest(AbstractController.java:139)
   
org.springframework.web.servlet.mvc.SimpleControllerHandlerAdapter.handle(SimpleControllerHandlerAdapter.java:44)
   
org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:684)
   
org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:625)
   
org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:392)
   
org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:358)
    javax.servlet.http.HttpServlet.service(HttpServlet.java:763)
    javax.servlet.http.HttpServlet.service(HttpServlet.java:856)
   
weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:223)
   
weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:125)
   
weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:230)
    weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:26)

--
Justin Deoliveira
The Open Planning Project
http://topp.openplans.org