Description
GlassFish Version (and build number)
7.0.22
JDK version
21
OS
Red Hat
Database
Oracle
Problem Description
During a Load Test but also in Production when DB Connection got invalid the Connection Pool cannot create new Connection and it seems that invalid Connection are not removed from the Pool
In server.log we see many of these stacktraces
[2025-05-22T12:53:47.143000+02:00] [GF 7.0.22] [INFO] [] [oracle.jdbc] [tid: _ThreadID=540 _ThreadName=external-listener(14)] [levelValue: 800] [[
properties={LOCALE=en_US, DriverVersion=23.6.0.24.10, java.library.path: =/server/glassfish7/glassfish/lib:/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib, java.class.path: =/server/glassfish7/glassfish/modules/glassfish.jar, DatabaseProdu
ctVersion=19000, java.version: =21.0.6, URL=jdbc:oracle:thin:@(DESCRIPTION=(CONNECT_TIMEOUT=15)(RETRY_COUNT=20)(RETRY_DELAY=3)(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=mcuepdbsl2.lottery.co.at)(PORT=1529))(ADDRESS=(PROTOCOL=TCP)(HOST=rwuepdbsl2.lo
ttery.co.at)(PORT=1529))(FAILOVER=on)(LOAD_BALANCE=off))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=ueptpe2_rw)))}. ]]
[2025-05-22T12:53:47.144000+02:00] [GF 7.0.22] [INFO] [] [oracle.jdbc] [tid: _ThreadID=540 ThreadName=external-listener(14)] [levelValue: 800] [[
java.sql.SQLRecoverableException: ORA-17008: Closed connection
https://docs.oracle.com/error-help/db/ora-17008/
at oracle.jdbc.driver.OracleStatement.ensureOpen(OracleStatement.java:5318)
at oracle.jdbc.driver.OraclePreparedStatement.prepareForExecuteInternal(OraclePreparedStatement.java:4081)
at oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3940)
at oracle.jdbc.driver.OraclePreparedStatement.executeQuery(OraclePreparedStatement.java:4148)
at oracle.jdbc.driver.OraclePreparedStatementWrapper.executeQuery(OraclePreparedStatementWrapper.java:1117)
at com.sun.gjc.spi.jdbc40.PreparedStatementWrapper40.executeQuery(PreparedStatementWrapper40.java:656)
at org.eclipse.persistence.internal.databaseaccess.DatabasePlatform.wasFailureCommunicationBased(DatabasePlatform.java:3223)
at org.eclipse.persistence.platform.database.OraclePlatform.wasFailureCommunicationBased(OraclePlatform.java:1266)
at org.eclipse.persistence.platform.server.ServerPlatformBase.wasFailureCommunicationBased(ServerPlatformBase.java:580)
at org.eclipse.persistence.internal.databaseaccess.DatabaseAccessor.processExceptionForCommError(DatabaseAccessor.java:1801)
at org.eclipse.persistence.internal.databaseaccess.DatabaseAccessor.basicExecuteCall(DatabaseAccessor.java:694)
at org.eclipse.persistence.internal.databaseaccess.DatabaseAccessor.executeCall(DatabaseAccessor.java:569)
at org.eclipse.persistence.internal.sessions.AbstractSession.basicExecuteCall(AbstractSession.java:2048)
at org.eclipse.persistence.sessions.server.ServerSession.executeCall(ServerSession.java:611)
at org.eclipse.persistence.sessions.server.ClientSession.executeCall(ClientSession.java:263)
at org.eclipse.persistence.internal.queries.DatasourceCallQueryMechanism.executeCall(DatasourceCallQueryMechanism.java:280)
at org.eclipse.persistence.internal.queries.DatasourceCallQueryMechanism.executeCall(DatasourceCallQueryMechanism.java:266)
at org.eclipse.persistence.internal.queries.DatasourceCallQueryMechanism.executeSelectCall(DatasourceCallQueryMechanism.java:352)
at org.eclipse.persistence.internal.queries.DatasourceCallQueryMechanism.selectAllRows(DatasourceCallQueryMechanism.java:792)
at org.eclipse.persistence.internal.queries.ExpressionQueryMechanism.selectAllRowsFromTable(ExpressionQueryMechanism.java:2848)
at org.eclipse.persistence.internal.queries.ExpressionQueryMechanism.selectAllReportQueryRows(ExpressionQueryMechanism.java:2784)
at org.eclipse.persistence.queries.ReportQuery.executeDatabaseQuery(ReportQuery.java:891)
at org.eclipse.persistence.queries.DatabaseQuery.execute(DatabaseQuery.java:913)
at org.eclipse.persistence.queries.ObjectLevelReadQuery.execute(ObjectLevelReadQuery.java:1195)
at org.eclipse.persistence.queries.ReadAllQuery.execute(ReadAllQuery.java:485)
at org.eclipse.persistence.queries.ObjectLevelReadQuery.executeInUnitOfWork(ObjectLevelReadQuery.java:1283)
at org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.internalExecuteQuery(UnitOfWorkImpl.java:3031)
at org.eclipse.persistence.internal.sessions.AbstractSession.executeQuery(AbstractSession.java:1841)
at org.eclipse.persistence.internal.sessions.AbstractSession.retryQuery(AbstractSession.java:1912)
at org.eclipse.persistence.sessions.server.ClientSession.retryQuery(ClientSession.java:704)
at org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.retryQuery(UnitOfWorkImpl.java:5836)
at org.eclipse.persistence.internal.sessions.AbstractSession.executeQuery(AbstractSession.java:1878)
at org.eclipse.persistence.internal.sessions.AbstractSession.executeQuery(AbstractSession.java:1823)
at org.eclipse.persistence.internal.sessions.AbstractSession.executeQuery(AbstractSession.java:1788)
at org.eclipse.persistence.internal.jpa.QueryImpl.executeReadQuery(QueryImpl.java:263)
at org.eclipse.persistence.internal.jpa.QueryImpl.getSingleResult(QueryImpl.java:531)
at org.eclipse.persistence.internal.jpa.EJBQueryImpl.getSingleResult(EJBQueryImpl.java:400)
at uep.user.customer.external.business.CustomerExternalManager.getCustomerByPlayerUID(CustomerExternalManager.java:64)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at org.glassfish.exousia.AuthorizationService.lambda$invokeBeanMethod$1(AuthorizationService.java:506)
at org.glassfish.exousia.AuthorizationService.runInScope(AuthorizationService.java:586)
at org.glassfish.exousia.AuthorizationService.invokeBeanMethod(AuthorizationService.java:506)
at org.glassfish.ejb.security.application.EJBSecurityManager.invoke(EJBSecurityManager.java:297)
at com.sun.ejb.containers.BaseContainer.invokeBeanMethod(BaseContainer.java:4193)
at com.sun.ejb.EjbInvocation.invokeBeanMethod(EjbInvocation.java:604)
at com.sun.ejb.containers.interceptors.AroundInvokeChainImpl.invokeNext(InterceptorManager.java:706)
at com.sun.ejb.EjbInvocation.proceed(EjbInvocation.java:553)
at org.jboss.weld.module.ejb.AbstractEJBRequestScopeActivationInterceptor.aroundInvoke(AbstractEJBRequestScopeActivationInterceptor.java:78)
at org.jboss.weld.module.ejb.SessionBeanInterceptor.aroundInvoke(SessionBeanInterceptor.java:52)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.sun.ejb.containers.interceptors.AroundInvokeInterceptor.intercept(InterceptorManager.java:766)
at com.sun.ejb.containers.interceptors.AroundInvokeChainImpl.invokeNext(InterceptorManager.java:706)
at com.sun.ejb.EjbInvocation.proceed(EjbInvocation.java:553)
at com.sun.ejb.containers.interceptors.SystemInterceptorProxy.doCall(SystemInterceptorProxy.java:122)
at com.sun.ejb.containers.interceptors.SystemInterceptorProxy.aroundInvoke(SystemInterceptorProxy.java:99)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.sun.ejb.containers.interceptors.AroundInvokeInterceptor.intercept(InterceptorManager.java:766)
at com.sun.ejb.containers.interceptors.AroundInvokeChainImpl.invokeNext(InterceptorManager.java:706)
at com.sun.ejb.containers.interceptors.InterceptorManager.intercept(InterceptorManager.java:307)
at com.sun.ejb.containers.BaseContainer.__intercept(BaseContainer.java:4168)
at com.sun.ejb.containers.BaseContainer.intercept(BaseContainer.java:4158)
at com.sun.ejb.containers.EJBLocalObjectInvocationHandler.invoke(EJBLocalObjectInvocationHandler.java:169)
at com.sun.ejb.containers.EJBLocalObjectInvocationHandlerDelegate.invoke(EJBLocalObjectInvocationHandlerDelegate.java:61)
at jdk.proxy47/jdk.proxy47.$Proxy525.getCustomerByPlayerUID(Unknown Source)
at uep.user.customer.external.business.EJB31_Generated__CustomerExternalManager__Intf____Bean.getCustomerByPlayerUID(Unknown Source)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.sun.ejb.containers.interceptors.AroundInvokeInterceptor.intercept(InterceptorManager.java:766)
at com.sun.ejb.containers.interceptors.AroundInvokeChainImpl.invokeNext(InterceptorManager.java:706)
at com.sun.ejb.EjbInvocation.proceed(EjbInvocation.java:553)
at com.sun.ejb.containers.interceptors.SystemInterceptorProxy.doCall(SystemInterceptorProxy.java:122)
at com.sun.ejb.containers.interceptors.SystemInterceptorProxy.aroundInvoke(SystemInterceptorProxy.java:99)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.sun.ejb.containers.interceptors.AroundInvokeInterceptor.intercept(InterceptorManager.java:766)
at com.sun.ejb.containers.interceptors.AroundInvokeChainImpl.invokeNext(InterceptorManager.java:706)
at com.sun.ejb.containers.interceptors.InterceptorManager.intercept(InterceptorManager.java:307)
at com.sun.ejb.containers.BaseContainer.__intercept(BaseContainer.java:4168)
at com.sun.ejb.containers.BaseContainer.intercept(BaseContainer.java:4158)
at com.sun.ejb.containers.EJBLocalObjectInvocationHandler.invoke(EJBLocalObjectInvocationHandler.java:169)
at com.sun.ejb.containers.EJBLocalObjectInvocationHandlerDelegate.invoke(EJBLocalObjectInvocationHandlerDelegate.java:61)
at jdk.proxy47/jdk.proxy47.$Proxy525.getCustomerByPlayerUID(Unknown Source)
at uep.user.customer.external.business.EJB31_Generated__CustomerExternalManager__Intf____Bean.getCustomerByPlayerUID(Unknown Source)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at org.jboss.weld.util.reflection.Reflections.invokeAndUnwrap(Reflections.java:412)
at org.jboss.weld.module.ejb.EnterpriseBeanProxyMethodHandler.invoke(EnterpriseBeanProxyMethodHandler.java:133)
at org.jboss.weld.bean.proxy.EnterpriseTargetBeanInstance.invoke(EnterpriseTargetBeanInstance.java:54)
at org.jboss.weld.module.ejb.InjectionPointPropagatingEnterpriseTargetBeanInstance.invoke(InjectionPointPropagatingEnterpriseTargetBeanInstance.java:69)
at org.jboss.weld.bean.proxy.ProxyMethodHandler.invoke(ProxyMethodHandler.java:109)
at uep.user.customer.external.business.CustomerExternalManager$Proxy$$$Weld$EnterpriseProxy$.getCustomerByPlayerUID(Unknown Source)
at uep.user.customer.module.business.CustomerModuleManagerBean.getCustomerByPlayerUID(CustomerModuleManagerBean.java:268)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at org.glassfish.exousia.AuthorizationService.lambda$invokeBeanMethod$1(AuthorizationService.java:506)
at org.glassfish.exousia.AuthorizationService.runInScope(AuthorizationService.java:586)
at org.glassfish.exousia.AuthorizationService.invokeBeanMethod(AuthorizationService.java:506)
at org.glassfish.ejb.security.application.EJBSecurityManager.invoke(EJBSecurityManager.java:297)
at com.sun.ejb.containers.BaseContainer.invokeBeanMethod(BaseContainer.java:4193)
at com.sun.ejb.EjbInvocation.invokeBeanMethod(EjbInvocation.java:604)
at com.sun.ejb.containers.interceptors.AroundInvokeChainImpl.invokeNext(InterceptorManager.java:706)
at com.sun.ejb.EjbInvocation.proceed(EjbInvocation.java:553)
at org.jboss.weld.module.ejb.AbstractEJBRequestScopeActivationInterceptor.aroundInvoke(AbstractEJBRequestScopeActivationInterceptor.java:78)
at org.jboss.weld.module.ejb.SessionBeanInterceptor.aroundInvoke(SessionBeanInterceptor.java:52)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.sun.ejb.containers.interceptors.AroundInvokeInterceptor.intercept(InterceptorManager.java:766)
at com.sun.ejb.containers.interceptors.AroundInvokeChainImpl.invokeNext(InterceptorManager.java:706)
at com.sun.ejb.EjbInvocation.proceed(EjbInvocation.java:553)
at com.sun.ejb.containers.interceptors.SystemInterceptorProxy.doCall(SystemInterceptorProxy.java:122)
at com.sun.ejb.containers.interceptors.SystemInterceptorProxy.aroundInvoke(SystemInterceptorProxy.java:99)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.sun.ejb.containers.interceptors.AroundInvokeInterceptor.intercept(InterceptorManager.java:766)
at com.sun.ejb.containers.interceptors.AroundInvokeChainImpl.invokeNext(InterceptorManager.java:706)
at com.sun.ejb.containers.interceptors.InterceptorManager.intercept(InterceptorManager.java:307)
at com.sun.ejb.containers.BaseContainer.__intercept(BaseContainer.java:4168)
at com.sun.ejb.containers.BaseContainer.intercept(BaseContainer.java:4158)
at com.sun.ejb.containers.EJBLocalObjectInvocationHandler.invoke(EJBLocalObjectInvocationHandler.java:169)
at com.sun.ejb.containers.EJBLocalObjectInvocationHandlerDelegate.invoke(EJBLocalObjectInvocationHandlerDelegate.java:61)
at jdk.proxy47/jdk.proxy47.$Proxy345.getCustomerByPlayerUID(Unknown Source)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at org.jboss.weld.util.reflection.Reflections.invokeAndUnwrap(Reflections.java:412)
at org.jboss.weld.module.ejb.EnterpriseBeanProxyMethodHandler.invoke(EnterpriseBeanProxyMethodHandler.java:133)
at org.jboss.weld.bean.proxy.EnterpriseTargetBeanInstance.invoke(EnterpriseTargetBeanInstance.java:54)
at org.jboss.weld.module.ejb.InjectionPointPropagatingEnterpriseTargetBeanInstance.invoke(InjectionPointPropagatingEnterpriseTargetBeanInstance.java:69)
at org.jboss.weld.bean.proxy.ProxyMethodHandler.invoke(ProxyMethodHandler.java:109)
at uep.core.interfaces.api.UepModuleInfo$CustomerModuleManager$1233244708$Proxy$$$Weld$EnterpriseProxy$.getCustomerByPlayerUID(Unknown Source)
at uep.game.external.evolution.frontend.EvolutionOSSController.getBalanceForGame(EvolutionOSSController.java:67)
at uep.game.external.evolution.frontend.EvolutionOSSService.getBalance(EvolutionOSSService.java:69)
at uep.game.external.evolution.frontend.EvolutionOSSService$Proxy$$$_WeldClientProxy.getBalance(Unknown Source)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory.lambda$static$0(ResourceMethodInvocationHandlerFactory.java:52)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:146)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:189)
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:176)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:93)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:478)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:400)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:81)
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:274)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244)
at org.glassfish.jersey.internal.Errors.process(Errors.java:292)
at org.glassfish.jersey.internal.Errors.process(Errors.java:274)
at org.glassfish.jersey.internal.Errors.process(Errors.java:244)
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:266)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:253)
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:696)
at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:397)
at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:349)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:358)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:312)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:205)
at org.apache.catalina.core.StandardWrapper.service(StandardWrapper.java:1368)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:302)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:172)
at uep.core.web.ServletLoggingFilter.doFilter(ServletLoggingFilter.java:127)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:214)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:172)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:123)
Steps to reproduce
We are still testing how to reproduce it but seems to only happen during load tests