@Ryan.Paredez  can you help me  ... View more

ERROR [2024-01-06 15:37:24,915] com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage: Failed to reach URL [http://welcomeB:9080/v1/store/report] ! java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/2001:0:2851:782c:1451:2eab:3b33:ff4d:9080 ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:285) ! at okhttp3.internal.connection.RealConnection.connect(RealConnection.kt:195) ! at okhttp3.internal.connection.ExchangeFinder.findConnection(ExchangeFinder.kt:249) ! at okhttp3.internal.connection.ExchangeFinder.findHealthyConnection(ExchangeFinder.kt:108) ! at okhttp3.internal.connection.ExchangeFinder.find(ExchangeFinder.kt:76) ! at okhttp3.internal.connection.RealCall.initExchange$okhttp(RealCall.kt:245) ! at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.kt:32) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.kt:82) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.kt:83) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.kt:76) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.connection.RealCall.getResponseWithInterceptorChain$okhttp(RealCall.kt:197) ! at okhttp3.internal.connection.RealCall.execute(RealCall.kt:148) ! at com.appdynamics.platformadmin.util.HealthCheckWrapper.healthCheckRequestOkHttp3(HealthCheckWrapper.java:99) ! ... 13 common frames omitted ! Suppressed: java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/192.168.27.207:9080 ! ... 30 common frames omitted ! Suppressed: java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/fe80:0:0:0:280e:a326:367e:6dfb%5:9080 ! ... 30 common frames omitted ! Suppressed: java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/fe80:0:0:0:1451:2eab:3b33:ff4d%11:9080 ! ... 30 common frames omitted ! Causing: java.lang.RuntimeException: Health check request failed to connect with exception ! at com.appdynamics.platformadmin.util.HealthCheckWrapper.healthCheckRequestOkHttp3(HealthCheckWrapper.java:102) ! at com.appdynamics.platformadmin.util.HealthCheckWrapper.healthCheckRequest(HealthCheckWrapper.java:71) ! at com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage.getClusterHealth(ESClusterHealthCheckStage.java:164) ! at com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage.getDetailedClusterHealth(ESClusterHealthCheckStage.java:102) ! at com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage.execute(ESClusterHealthCheckStage.java:76) ! at com.appdynamics.platformadmin.core.job.JobProcessor.run(JobProcessor.java:260) ! at com.appdynamics.platformadmin.core.job.JobProcessor.call(JobProcessor.java:114) ! at com.appdynamics.platformadmin.core.service.JobExecutionServiceImpl.lambda$registerRecurringJobHelper$0(JobExecutionServiceImpl.java:204) ! at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604) ! at java.util.concurrent.CompletableFuture$AsyncSupply.exec(CompletableFuture.java:1596) ! at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ! at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) ! at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) ! at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) ERROR [2024-01-06 15:37:38,924] com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage: Failed to reach URL [http://welcomeB:9080/v1/store/report] ! java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/2001:0:2851:782c:1451:2eab:3b33:ff4d:9080 ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:285) ! at okhttp3.internal.connection.RealConnection.connect(RealConnection.kt:195) ! at okhttp3.internal.connection.ExchangeFinder.findConnection(ExchangeFinder.kt:249) ! at okhttp3.internal.connection.ExchangeFinder.findHealthyConnection(ExchangeFinder.kt:108) ! at okhttp3.internal.connection.ExchangeFinder.find(ExchangeFinder.kt:76) ! at okhttp3.internal.connection.RealCall.initExchange$okhttp(RealCall.kt:245) ! at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.kt:32) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.kt:82) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.kt:83) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.kt:76) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.connection.RealCall.getResponseWithInterceptorChain$okhttp(RealCall.kt:197) ! at okhttp3.internal.connection.RealCall.execute(RealCall.kt:148) ! at com.appdynamics.platformadmin.util.HealthCheckWrapper.healthCheckRequestOkHttp3(HealthCheckWrapper.java:99) ! ... 13 common frames omitted ! Suppressed: java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/192.168.27.207:9080 ! ... 30 common frames omitted ! Suppressed: java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/fe80:0:0:0:280e:a326:367e:6dfb%5:9080 ! ... 30 common frames omitted ! Suppressed: java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/fe80:0:0:0:1451:2eab:3b33:ff4d%11:9080 ! ... 30 common frames omitted ! Causing: java.lang.RuntimeException: Health check request failed to connect with exception ! at com.appdynamics.platformadmin.util.HealthCheckWrapper.healthCheckRequestOkHttp3(HealthCheckWrapper.java:102) ! at com.appdynamics.platformadmin.util.HealthCheckWrapper.healthCheckRequest(HealthCheckWrapper.java:71) ! at com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage.getClusterHealth(ESClusterHealthCheckStage.java:164) ! at com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage.getDetailedClusterHealth(ESClusterHealthCheckStage.java:102) ! at com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage.execute(ESClusterHealthCheckStage.java:76) ! at com.appdynamics.platformadmin.core.job.JobProcessor.run(JobProcessor.java:260) ! at com.appdynamics.platformadmin.core.job.JobProcessor.call(JobProcessor.java:114) ! at com.appdynamics.platformadmin.core.service.JobExecutionServiceImpl.lambda$registerRecurringJobHelper$0(JobExecutionServiceImpl.java:204) ! at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604) ! at java.util.concurrent.CompletableFuture$AsyncSupply.exec(CompletableFuture.java:1596) ! at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ! at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) ! at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) ! at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) INFO [2024-01-06 15:37:45,028] com.appdynamics.orcha.core.OrchaRunnerImpl: Running playbook K:\Platform\platform-admin\conf\..\archives\platform-configuration\..\controller\23.9.0-10017\playbooks\check-single-controller-appserver-processes.orcha with variables file K:\Platform\platform-admin\conf\..\archives\platform-configuration\..\controller\23.9.0-10017\playbooks\controller.groovy INFO [2024-01-06 15:37:45,032] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Uploading the diagnosis script if necessary INFO [2024-01-06 15:37:45,063] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Uploading the diagnosis script if necessary] completed executing in [14] ms. INFO [2024-01-06 15:37:45,074] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Setting execute permissions for the script INFO [2024-01-06 15:37:45,091] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Setting execute permissions for the script] completed executing in [11] ms. INFO [2024-01-06 15:37:45,101] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Running diagnosis scripts INFO [2024-01-06 15:37:45,705] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Running diagnosis scripts] completed executing in [596] ms. INFO [2024-01-06 15:37:45,719] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Stringify appserver diagnosis result INFO [2024-01-06 15:37:45,736] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Stringify appserver diagnosis result] completed executing in [11] ms. INFO [2024-01-06 15:37:47,208] com.appdynamics.orcha.core.OrchaRunnerImpl: Running playbook K:\Platform\platform-admin\conf\..\archives\platform-configuration\..\controller\23.9.0-10017\playbooks\check-single-controller-mysql-process.orcha with variables file K:\Platform\platform-admin\conf\..\archives\platform-configuration\..\controller\23.9.0-10017\playbooks\controller.groovy INFO [2024-01-06 15:37:47,211] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Uploading the diagnosis script if necessary INFO [2024-01-06 15:37:47,242] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Uploading the diagnosis script if necessary] completed executing in [13] ms. INFO [2024-01-06 15:37:47,254] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Setting execute permissions for the script INFO [2024-01-06 15:37:47,271] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Setting execute permissions for the script] completed executing in [11] ms. INFO [2024-01-06 15:37:47,282] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Running diagnosis scripts INFO [2024-01-06 15:37:48,170] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Running diagnosis scripts] completed executing in [879] ms. INFO [2024-01-06 15:37:48,183] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Stringify MySQL diagnosis result INFO [2024-01-06 15:37:48,202] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Stringify MySQL diagnosis result] completed executing in [12] ms. INFO [2024-01-06 15:37:48,231] com.appdynamics.platformadmin.controller.job.stages.ControllerHealthCheckStage: Discovered Primary controller health status on host 'welcomeB': UNHEALTHY. INFO [2024-01-06 15:37:48,231] com.appdynamics.platformadmin.core.job.JobProcessor: Platform/Job [1/e85fa4e1-fd2e-4ebc-aa0d-e3d77ff13258]: Ignored stage [controller_health_check_stage]'s error: [Health check result: Controller status check failed for the url http://welcomeB:8090/controller/rest/serverstatus with message: Health check request failed with HTTP 404: <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><title>GlassFish v4 - Error report</title><style type="text/css"><!--H1 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:22px;} H2 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:16px;} H3 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:14px;} BODY {font-family:Tahoma,Arial,sans-serif;color:black;background-color:white;} B {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;} P{font-family:Tahoma,Arial,sans-serif;background:white;color:black;font-size:12px;}A {color : black;}HR {color : #525D76;}--></style> </head><body><h1>HTTP Status 404 - </h1><hr/><p><b>type</b> Status report</p><p><b>message</b></p><p><b>description</b>The requested resource is not available.</p><hr/><h3>GlassFish Server Open Source Edition 4.1.1 </h3></body></html>] INFO [2024-01-06 15:37:48,245] com.appdynamics.platformadmin.controller.listener.ControllerRecurringJobListener: Controller listener expected health status [Critical] and received real time status [Critical] in platform 1 ERROR [2024-01-06 15:37:52,934] com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage: Failed to reach URL [http://welcomeB:9080/v1/store/report] ! java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/2001:0:2851:782c:1451:2eab:3b33:ff4d:9080 ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:285) ! at okhttp3.internal.connection.RealConnection.connect(RealConnection.kt:195) ! at okhttp3.internal.connection.ExchangeFinder.findConnection(ExchangeFinder.kt:249) ! at okhttp3.internal.connection.ExchangeFinder.findHealthyConnection(ExchangeFinder.kt:108) ! at okhttp3.internal.connection.ExchangeFinder.find(ExchangeFinder.kt:76) ! at okhttp3.internal.connection.RealCall.initExchange$okhttp(RealCall.kt:245) ! at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.kt:32) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.kt:82) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.kt:83) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.kt:76) ! at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.kt:100) ! at okhttp3.internal.connection.RealCall.getResponseWithInterceptorChain$okhttp(RealCall.kt:197) ! at okhttp3.internal.connection.RealCall.execute(RealCall.kt:148) ! at com.appdynamics.platformadmin.util.HealthCheckWrapper.healthCheckRequestOkHttp3(HealthCheckWrapper.java:99) ! ... 13 common frames omitted ! Suppressed: java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/192.168.27.207:9080 ! ... 30 common frames omitted ! Suppressed: java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/fe80:0:0:0:280e:a326:367e:6dfb%5:9080 ! ... 30 common frames omitted ! Suppressed: java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ! at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ! at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ! at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ! at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ! at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ! at java.net.Socket.connect(Socket.java:613) ! at okhttp3.internal.platform.Platform.connectSocket(Platform.kt:117) ! at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.kt:283) ! ... 29 common frames omitted ! Causing: java.net.ConnectException: Failed to connect to welcomeb/fe80:0:0:0:1451:2eab:3b33:ff4d%11:9080 ! ... 30 common frames omitted ! Causing: java.lang.RuntimeException: Health check request failed to connect with exception ! at com.appdynamics.platformadmin.util.HealthCheckWrapper.healthCheckRequestOkHttp3(HealthCheckWrapper.java:102) ! at com.appdynamics.platformadmin.util.HealthCheckWrapper.healthCheckRequest(HealthCheckWrapper.java:71) ! at com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage.getClusterHealth(ESClusterHealthCheckStage.java:164) ! at com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage.getDetailedClusterHealth(ESClusterHealthCheckStage.java:102) ! at com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage.execute(ESClusterHealthCheckStage.java:76) ! at com.appdynamics.platformadmin.core.job.JobProcessor.run(JobProcessor.java:260) ! at com.appdynamics.platformadmin.core.job.JobProcessor.call(JobProcessor.java:114) ! at com.appdynamics.platformadmin.core.service.JobExecutionServiceImpl.lambda$registerRecurringJobHelper$0(JobExecutionServiceImpl.java:204) ! at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604) ! at java.util.concurrent.CompletableFuture$AsyncSupply.exec(CompletableFuture.java:1596) ! at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ! at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) ! at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) ! at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) INFO [2024-01-06 15:37:56,850] com.appdynamics.orcha.extensions.es.health.StoreNodeHealth: Connection failure while checking node health status on the host: welcomeB ERROR [2024-01-06 15:37:56,850] com.appdynamics.orcha.extensions.es.health.StoreNodeHealth: Failed to connect to welcomeb/2001:0:2851:782c:1451:2eab:3b33:ff4d:9080 ERROR [2024-01-06 15:38:00,860] com.appdynamics.orcha.extensions.es.health.StoreNodeHealth: Connection failure while checking for number of data nodes on the host: welcomeB ERROR [2024-01-06 15:38:02,936] com.appdynamics.platformadmin.core.job.JobProcessor: Platform/Job [1/a6631ee1-e56a-4689-8d9a-4da5127b1e01]: Stage [es_cluster_health_stage] failed due to [Unable to check health of Events Service hosts [welcomeB] through port 9080.] INFO [2024-01-06 15:38:15,225] com.appdynamics.platformadmin.resources.VersionResource: Found Enterprise Console version 23.9.0-10017, build INFO [2024-01-06 15:38:45,045] com.appdynamics.orcha.core.OrchaRunnerImpl: Running playbook K:\Platform\platform-admin\conf\..\archives\platform-configuration\..\controller\23.9.0-10017\playbooks\check-single-controller-appserver-processes.orcha with variables file K:\Platform\platform-admin\conf\..\archives\platform-configuration\..\controller\23.9.0-10017\playbooks\controller.groovy INFO [2024-01-06 15:38:45,049] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Uploading the diagnosis script if necessary INFO [2024-01-06 15:38:45,080] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Uploading the diagnosis script if necessary] completed executing in [13] ms. INFO [2024-01-06 15:38:45,093] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Setting execute permissions for the script INFO [2024-01-06 15:38:45,109] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Setting execute permissions for the script] completed executing in [10] ms. INFO [2024-01-06 15:38:45,120] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Running diagnosis scripts INFO [2024-01-06 15:38:45,719] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Running diagnosis scripts] completed executing in [592] ms. INFO [2024-01-06 15:38:45,733] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Stringify appserver diagnosis result INFO [2024-01-06 15:38:45,751] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Stringify appserver diagnosis result] completed executing in [11] ms. INFO [2024-01-06 15:38:47,335] com.appdynamics.orcha.core.OrchaRunnerImpl: Running playbook K:\Platform\platform-admin\conf\..\archives\platform-configuration\..\controller\23.9.0-10017\playbooks\check-single-controller-mysql-process.orcha with variables file K:\Platform\platform-admin\conf\..\archives\platform-configuration\..\controller\23.9.0-10017\playbooks\controller.groovy INFO [2024-01-06 15:38:47,338] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Uploading the diagnosis script if necessary INFO [2024-01-06 15:38:47,377] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Uploading the diagnosis script if necessary] completed executing in [20] ms. INFO [2024-01-06 15:38:47,389] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Setting execute permissions for the script INFO [2024-01-06 15:38:47,407] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Setting execute permissions for the script] completed executing in [11] ms. INFO [2024-01-06 15:38:47,418] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Running diagnosis scripts INFO [2024-01-06 15:38:48,303] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Running diagnosis scripts] completed executing in [878] ms. INFO [2024-01-06 15:38:48,317] com.appdynamics.orcha.core.executor.DefaultPlaybookExecutor: Executing : Stringify MySQL diagnosis result INFO [2024-01-06 15:38:48,335] com.appdynamics.orcha.core.executor.TaskExecutor: Task [Stringify MySQL diagnosis result] completed executing in [11] ms. INFO [2024-01-06 15:38:48,364] com.appdynamics.platformadmin.controller.job.stages.ControllerHealthCheckStage: Discovered Primary controller health status on host 'welcomeB': UNHEALTHY. INFO [2024-01-06 15:38:48,364] com.appdynamics.platformadmin.core.job.JobProcessor: Platform/Job [1/e85fa4e1-fd2e-4ebc-aa0d-e3d77ff13258]: Ignored stage [controller_health_check_stage]'s error: [Health check result: Controller status check failed for the url http://welcomeB:8090/controller/rest/serverstatus with message: Health check request failed with HTTP 404: <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><title>GlassFish v4 - Error report</title><style type="text/css"><!--H1 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:22px;} H2 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:16px;} H3 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:14px;} BODY {font-family:Tahoma,Arial,sans-serif;color:black;background-color:white;} B {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;} P{font-family:Tahoma,Arial,sans-serif;background:white;color:black;font-size:12px;}A {color : black;}HR {color : #525D76;}--></style> </head><body><h1>HTTP Status 404 - </h1><hr/><p><b>type</b> Status report</p><p><b>message</b></p><p><b>description</b>The requested resource is not available.</p><hr/><h3>GlassFish Server Open Source Edition 4.1.1 </h3></body></html>] INFO [2024-01-06 15:38:48,379] com.appdynamics.platformadmin.controller.listener.ControllerRecurringJobListener: Controller listener expected health status [Critical] and received real time status [Critical] in platform 1 INFO [2024-01-06 15:38:56,856] com.appdynamics.orcha.extensions.es.health.StoreNodeHealth: Connection failure while checking node health status on the host: welcomeB ERROR [2024-01-06 15:38:56,856] com.appdynamics.orcha.extensions.es.health.StoreNodeHealth: Failed to connect to welcomeb/2001:0:2851:782c:1451:2eab:3b33:ff4d:9080 ERROR [2024-01-06 15:38:56,878] com.appdynamics.platformadmin.es.job.stages.ESClusterHealthCheckStage: Failed to reach URL [http://welcomeB:9080/v1/store/report] ! java.net.ConnectException: Connection refused: connect ! at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) ! at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) ... View more