Hi
The issue happens multiple snapshots, I have run a test with
scheduled hourly volume snapshots for a test, here are the logs:
2021-03-05 10:03:43,277 DEBUG [c.c.v.VmWorkJobHandlerProxy]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142 ctx-43a8373c)
(logid:fc7d7320) Execute VM work job:
com.cloud.vm.VmWorkTakeVolumeSnapshot{"volumeId":225,"policyId":8,"snapshotId":861,"quiesceVm":false,"asyncBackup":false,"userId":1,"accountId":9,"vmId":156,"handlerName":"VolumeApiServiceImpl"}
2021-03-05 10:03:43,342 DEBUG [c.c.a.t.Request]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142 ctx-43a8373c)
(logid:fc7d7320) Seq 6-5629781009189976396: Sending { Cmd , MgmtId:
345050527765, via: 6(vxtr-ln1-chst02.vxtream.com), Ver: v1, Flags: 100011,
[{"org.apache.cloudstack.storage.command.CreateObjectCommand":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"volume":{"uuid":"3c6bbe64-e924-4c94-a0eb-4b18e5786d58","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"658c4937-b7bc-3aa7-a0d3-aa144224dd52","id":"7","poolType":"RBD","host":"ceph-mon.cloudstack","path":"cloudstack2019","port":"6789","url":"RBD://ceph-mon.cloudstack/cloudstack2019/?ROLE=Primary&STOREUUID=658c4937-b7bc-3aa7-a0d3-aa144224dd52","isManaged":"false"}},"name":"ROOT-156","size":"(100.00
GB)
107374182400","path":"3c6bbe64-e924-4c94-a0eb-4b18e5786d58","volumeId":"225","vmName":"i-9-156-VM","accountId":"9","format":"QCOW2","provisioningType":"THIN","id":"225","deviceId":"0","hypervisorType":"KVM","directDownload":"false","deployAsIs":"false"},"dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"658c4937-b7bc-3aa7-a0d3-aa144224dd52","id":"7","poolType":"RBD","host":"ceph-mon.cloudstack","path":"cloudstack2019","port":"6789","url":"RBD://ceph-mon.cloudstack/cloudstack2019/?ROLE=Primary&STOREUUID=658c4937-b7bc-3aa7-a0d3-aa144224dd52","isManaged":"false"}},"vmName":"i-9-156-VM","name":"ESP-LN1-VOIP01_ROOT-156_20210305100342","hypervisorType":"KVM","id":"861","quiescevm":"false","physicalSize":"0"}},"wait":"0"}}]
}
2021-03-05 10:03:44,670 DEBUG [c.c.a.t.Request]
(AgentManager-Handler-9:null) (logid:) Seq 6-5629781009189976396:
Processing: { Ans: , MgmtId: 345050527765, via: 6, Ver: v1, Flags: 10,
[{"org.apache.cloudstack.storage.command.CreateObjectAnswer":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"path":"cloudstack2019/3c6bbe64-e924-4c94-a0eb-4b18e5786d58/a00ba130-19f0-454c-9b74-55f95a64bfb5","id":"0","quiescevm":"false","physicalSize":"0"}},"result":"true","wait":"0"}}]
}
2021-03-05 10:03:44,757 DEBUG [o.a.c.s.m.AncientDataMotionStrategy]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142 ctx-43a8373c)
(logid:fc7d7320) copyAsync inspecting src type SNAPSHOT copyAsync
inspecting dest type SNAPSHOT
2021-03-05 10:03:44,888 ERROR [o.a.c.s.e.DefaultEndPointSelector]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142 ctx-43a8373c)
(logid:fc7d7320) IR24 select BACKUPSNAPSHOT from primary to secondary 861
dest=861
2021-03-05 10:03:44,898 DEBUG [c.c.a.t.Request]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142 ctx-43a8373c)
(logid:fc7d7320) Seq 6-5629781009189976397: Sending { Cmd , MgmtId:
345050527765, via: 6(vxtr-ln1-chst02.vxtream.com), Ver: v1, Flags: 100011,
[{"org.apache.cloudstack.storage.command.CopyCommand":{"srcTO":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"path":"cloudstack2019/3c6bbe64-e924-4c94-a0eb-4b18e5786d58/a00ba130-19f0-454c-9b74-55f95a64bfb5","volume":{"uuid":"3c6bbe64-e924-4c94-a0eb-4b18e5786d58","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"658c4937-b7bc-3aa7-a0d3-aa144224dd52","id":"7","poolType":"RBD","host":"ceph-mon.cloudstack","path":"cloudstack2019","port":"6789","url":"RBD://ceph-mon.cloudstack/cloudstack2019/?ROLE=Primary&STOREUUID=658c4937-b7bc-3aa7-a0d3-aa144224dd52","isManaged":"false"}},"name":"ROOT-156","size":"(100.00
GB)
107374182400","path":"3c6bbe64-e924-4c94-a0eb-4b18e5786d58","volumeId":"225","vmName":"i-9-156-VM","accountId":"9","format":"QCOW2","provisioningType":"THIN","id":"225","deviceId":"0","hypervisorType":"KVM","directDownload":"false","deployAsIs":"false"},"dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"658c4937-b7bc-3aa7-a0d3-aa144224dd52","id":"7","poolType":"RBD","host":"ceph-mon.cloudstack","path":"cloudstack2019","port":"6789","url":"RBD://ceph-mon.cloudstack/cloudstack2019/?ROLE=Primary&STOREUUID=658c4937-b7bc-3aa7-a0d3-aa144224dd52","isManaged":"false"}},"vmName":"i-9-156-VM","name":"ESP-LN1-VOIP01_ROOT-156_20210305100342","hypervisorType":"KVM","id":"861","quiescevm":"false","physicalSize":"0"}},"destTO":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"path":"snapshots/9/225","volume":{"uuid":"3c6bbe64-e924-4c94-a0eb-4b18e5786d58","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"658c4937-b7bc-3aa7-a0d3-aa144224dd52","id":"7","poolType":"RBD","host":"ceph-mon.cloudstack","path":"cloudstack2019","port":"6789","url":"RBD://ceph-mon.cloudstack/cloudstack2019/?ROLE=Primary&STOREUUID=658c4937-b7bc-3aa7-a0d3-aa144224dd52","isManaged":"false"}},"name":"ROOT-156","size":"(100.00
GB)
107374182400","path":"3c6bbe64-e924-4c94-a0eb-4b18e5786d58","volumeId":"225","vmName":"i-9-156-VM","accountId":"9","format":"QCOW2","provisioningType":"THIN","id":"225","deviceId":"0","hypervisorType":"KVM","directDownload":"false","deployAsIs":"false"},"dataStore":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://
10.100.7.51/export/secondary","_role":"Image"}},"vmName":"i-9-156-VM","name":"ESP-LN1-VOIP01_ROOT-156_20210305100342","hypervisorType":"KVM","id":"861","quiescevm":"false","physicalSize":"0"}},"executeInSequence":"false","options":{"fullSnapshot":"true"},"options2":{},"wait":"21600"}}]
}
2021-03-05 10:08:42,314 DEBUG [c.c.s.s.SnapshotSchedulerImpl]
(SnapshotPollTask:ctx-357a9dcc) (logid:7175331e) Snapshot scheduler.poll is
being called at 2021-03-05 10:08:42 GMT
2021-03-05 10:08:42,315 DEBUG [c.c.s.s.SnapshotSchedulerImpl]
(SnapshotPollTask:ctx-357a9dcc) (logid:7175331e) Got 0 snapshots to be
executed at 2021-03-05 10:08:42 GMT
2021-03-05 10:09:24,952 DEBUG [c.c.a.t.Request]
(AgentManager-Handler-2:null) (logid:) Seq 6-5629781009189976397:
Processing: { Ans: , MgmtId: 345050527765, via: 6, Ver: v1, Flags: 10,
[{"org.apache.cloudstack.storage.command.CopyCmdAnswer":{"newData":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"path":"snapshots/9/225/a00ba130-19f0-454c-9b74-55f95a64bfb5","id":"0","quiescevm":"false","physicalSize":"16888233984"}},"result":"true","wait":"0"}}]
}
2021-03-05 10:09:24,976 DEBUG [c.c.s.s.SnapshotManagerImpl]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142 ctx-43a8373c)
(logid:fc7d7320) Max snaps: 1 exceeded for snapshot policy with Id: 8.
Deleting oldest snapshot: 860
2021-03-05 10:09:24,978 DEBUG [c.c.u.AccountManagerImpl]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142 ctx-43a8373c)
(logid:fc7d7320) Access to
com.cloud.storage.SnapshotVO$$EnhancerByCGLIB$$43460565@642f5e4f granted to
Acct[2ad78538-d6d9-4f24-967a-90701ff3a35f-edward] by DomainChecker
2021-03-05 10:09:25,037 DEBUG [c.c.a.t.Request]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142 ctx-43a8373c)
(logid:fc7d7320) Seq 16-5311714285507142014: Sending { Cmd , MgmtId:
345050527765, via: 16(vxtr-ln1-chst03.vxtream.com), Ver: v1, Flags: 100011,
[{"org.apache.cloudstack.storage.command.DeleteCommand":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"path":"cloudstack2019/3c6bbe64-e924-4c94-a0eb-4b18e5786d58/b8e3403e-7b6c-4bee-8b27-5ad9843353cd","volume":{"uuid":"3c6bbe64-e924-4c94-a0eb-4b18e5786d58","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"658c4937-b7bc-3aa7-a0d3-aa144224dd52","id":"7","poolType":"RBD","host":"ceph-mon.cloudstack","path":"cloudstack2019","port":"6789","url":"RBD://ceph-mon.cloudstack/cloudstack2019/?ROLE=Primary&STOREUUID=658c4937-b7bc-3aa7-a0d3-aa144224dd52","isManaged":"false"}},"name":"ROOT-156","size":"(100.00
GB)
107374182400","path":"3c6bbe64-e924-4c94-a0eb-4b18e5786d58","volumeId":"225","vmName":"i-9-156-VM","accountId":"9","format":"QCOW2","provisioningType":"THIN","id":"225","deviceId":"0","hypervisorType":"KVM","directDownload":"false","deployAsIs":"false"},"dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"658c4937-b7bc-3aa7-a0d3-aa144224dd52","id":"7","poolType":"RBD","host":"ceph-mon.cloudstack","path":"cloudstack2019","port":"6789","url":"RBD://ceph-mon.cloudstack/cloudstack2019/?ROLE=Primary&STOREUUID=658c4937-b7bc-3aa7-a0d3-aa144224dd52","isManaged":"false"}},"vmName":"i-9-156-VM","name":"ESP-LN1-VOIP01_ROOT-156_20210305090342","hypervisorType":"KVM","id":"860","quiescevm":"false","physicalSize":"0"}},"wait":"0"}}]
}
2021-03-05 10:09:25,574 DEBUG [c.c.a.t.Request]
(AgentManager-Handler-9:null) (logid:) Seq 16-5311714285507142014:
Processing: { Ans: , MgmtId: 345050527765, via: 16, Ver: v1, Flags: 10,
[{"com.cloud.agent.api.Answer":{"result":"true","details":"Snapshot
3c6bbe64-e924-4c94-a0eb-4b18e5786d58@b8e3403e-7b6c-4bee-8b27-5ad9843353cd
removed successfully.","wait":"0"}}] }
2021-03-05 10:09:25,588 DEBUG [c.c.r.ResourceLimitManagerImpl]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142 ctx-43a8373c)
(logid:fc7d7320) Updating resource Type = snapshot count for Account = 9
Operation = decreasing Amount = 1
2021-03-05 10:09:25,620 DEBUG [c.c.v.VmWorkJobHandlerProxy]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142 ctx-43a8373c)
(logid:fc7d7320) Done executing VM work job:
com.cloud.vm.VmWorkTakeVolumeSnapshot{"volumeId":225,"policyId":8,"snapshotId":861,"quiesceVm":false,"asyncBackup":false,"userId":1,"accountId":9,"vmId":156,"handlerName":"VolumeApiServiceImpl"}
2021-03-05 10:09:25,628 DEBUG [c.c.v.VmWorkJobDispatcher]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142) (logid:fc7d7320) Done
with run of VM work job: com.cloud.vm.VmWorkTakeVolumeSnapshot for VM 156,
job origin: 6141
2021-03-05 10:09:25,628 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
(Work-Job-Executor-26:ctx-65993d1d job-6141/job-6142) (logid:fc7d7320) Done
executing com.cloud.vm.VmWorkTakeVolumeSnapshot for job-6142
2021-03-05 10:09:25,666 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
(API-Job-Executor-19:ctx-3956f4cc job-6141 ctx-d69f2a1d) (logid:fc7d7320)
Complete async job-6141, jobStatus: SUCCEEDED, resultCode: 0, result:
org.apache.cloudstack.api.response.SnapshotResponse/snapshot/{"id":"076c3f3e-4875-419d-beed-398e6d938f41","account":"edward","domainid":"76fcd53c-c1bc-47cf-946c-336c30a53ca1","domain":"ESP","snapshottype":"HOURLY","volumeid":"3c6bbe64-e924-4c94-a0eb-4b18e5786d58","volumename":"ROOT-156","volumetype":"ROOT","created":"2021-03-05T10:03:42+0000","name":"ESP-LN1-VOIP01_ROOT-156_20210305100342","intervaltype":"HOURLY","state":"BackedUp","physicalsize":"16888233984","zoneid":"9000b853-f5e0-4451-ad49-64dfae95db84","revertable":"true","ostypeid":"2cd2aa56-ee6a-11e9-8f54-0050569d3815","osdisplayname":"CentOS
7","virtualsize":"107374182400","tags":[]}
2021-03-05 10:09:25,670 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
(API-Job-Executor-19:ctx-3956f4cc job-6141) (logid:fc7d7320) Done executing
org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd for
job-6141
Here are the actual snapshots on secondary storage:
-rw-r--r--. 1 root root 16888233984 Mar 5 10:09
a00ba130-19f0-454c-9b74-55f95a64bfb5
-rw-r--r--. 1 root root 16888233984 Mar 5 09:09
b8e3403e-7b6c-4bee-8b27-5ad9843353cd
Here are the entries in the DB relating to the snapshots:
select * from snapshots order by created desc limit 2;
+-----+----------------+------------+-----------+-----------+------------------+-----------+------+----------------------------------------+--------------------------------------+---------------+------------------+--------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+----------+----------+---------------+
| id | data_center_id | account_id | domain_id | volume_id |
disk_offering_id | status | path | name
| uuid | snapshot_type | type_description
| size | created | removed | backup_snap_id | swift_id
| sechost_id | prev_snap_id | hypervisor_type | version | s3_id | min_iops
| max_iops | location_type |
+-----+----------------+------------+-----------+-----------+------------------+-----------+------+----------------------------------------+--------------------------------------+---------------+------------------+--------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+----------+----------+---------------+
| 861 | 1 | 9 | 6 | 225 |
5 | BackedUp | NULL | ESP-LN1-VOIP01_ROOT-156_20210305100342 |
076c3f3e-4875-419d-beed-398e6d938f41 | 3 | HOURLY |
107374182400 | 2021-03-05 10:03:42 | NULL | NULL | NULL |
NULL | NULL | KVM | 2.2 | NULL | NULL |
NULL | NULL |
| 860 | 1 | 9 | 6 | 225 |
5 | Destroyed | NULL | ESP-LN1-VOIP01_ROOT-156_20210305090342 |
2998a50e-7282-45c5-bab6-8231e6343783 | 3 | HOURLY |
107374182400 | 2021-03-05 09:03:42 | NULL | NULL | NULL |
NULL | NULL | KVM | 2.2 | NULL | NULL |
NULL | NULL |
+-----+----------------+------------+-----------+-----------+------------------+-----------+------+----------------------------------------+--------------------------------------+---------------+------------------+--------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+----------+----------+---------------+
Regards
Ed
On Fri, 5 Mar 2021 at 02:20, Gabriel Bräscher <gabrasc...@gmail.com> wrote:
> There was an open issue indeed but a PR solved it for 4.15. The issue
> reported on this email looks a bit different than the ones I have seen.
>
> Background on a recent issue:
> - Issue #4498 "RBD Snapshot fails when snapshot.backup.to.secondary: true"
> has been fixed by PR #4568 "kvm: Fix double-escape issue while creating rbd
> disk options"
>
> Reading the log:
> - In order for such a null pointer to happen the volume from the snapshot
> has been deleted on DB. Thus, it is either a bug or an inconsistency on the
> DB; e.g.: deleting the volume prior to its Snapshots.
>
> Is this occurrence an isolated one or does it happen with multiple
> snapshots that have been deleted?
>
> #4498: https://github.com/apache/cloudstack/issues/4498
> #4568: https://github.com/apache/cloudstack/pull/4568
>
> Em qui., 4 de mar. de 2021 às 18:55, Rakesh v <www.rakeshv....@gmail.com>
> escreveu:
>
> > If I remember properly there is already an issue raised for it in github
> >
> > Sent from my iPhone
> >
> > > On Mar 4, 2021, at 9:48 PM, Andrija Panic <andrija.pa...@gmail.com>
> > wrote:
> > >
> > > @Gabriel Beims Bräscher <gabrasc...@gmail.com> does this rings any
> > bells
> > > for you? I haven't played with Ceph / snap clean-up issue on 4.15
> myself.
> > >
> > > Best,
> > >
> > >> On Tue, 2 Mar 2021 at 17:16, Edward St Pierre <
> > edward.stpie...@gmail.com>
> > >> wrote:
> > >>
> > >> Hi All,
> > >>
> > >> I Wonder if someone could help me.
> > >>
> > >> Currently using ceph for primary storage and have
> > >> 'snapshot.backup.to.secondary' enabled.
> > >>
> > >> Since upgrading to 4.15 the volume snapshots do not seem to be getting
> > >> deleted from the secondary storage. Could this be a bug?
> > >>
> > >> Also when logged in as the main admin account when navigating to
> > 'Storage /
> > >> Snapshots' an Error 500 is returned with the following error in the
> > logs:
> > >>
> > >> 2021-03-02 16:11:01,018 ERROR [c.c.a.ApiServer]
> > >> (qtp1762902523-19:ctx-60162a51 ctx-75a1b986) (logid:368908bb)
> unhandled
> > >> exception executing api command: [Ljava.lang.String;@7ac2d249
> > >> java.lang.NullPointerException
> > >> at
> > >>
> > >>
> >
> org.apache.cloudstack.storage.snapshot.StorageSystemSnapshotStrategy.canHandle(StorageSystemSnapshotStrategy.java:944)
> > >> at
> > >>
> > >>
> >
> org.apache.cloudstack.storage.helper.StorageStrategyFactoryImpl$3.canHandle(StorageStrategyFactoryImpl.java:72)
> > >> at
> > >>
> > >>
> >
> org.apache.cloudstack.storage.helper.StorageStrategyFactoryImpl$3.canHandle(StorageStrategyFactoryImpl.java:69)
> > >> at
> > >>
> > >>
> >
> org.apache.cloudstack.storage.helper.StorageStrategyFactoryImpl.bestMatch(StorageStrategyFactoryImpl.java:95)
> > >> at
> > >>
> > >>
> >
> org.apache.cloudstack.storage.helper.StorageStrategyFactoryImpl.getSnapshotStrategy(StorageStrategyFactoryImpl.java:69)
> > >> at
> > >>
> > >>
> >
> org.apache.cloudstack.storage.snapshot.SnapshotObject.isRevertable(SnapshotObject.java:153)
> > >> at
> > >>
> > >>
> >
> com.cloud.api.ApiResponseHelper.createSnapshotResponse(ApiResponseHelper.java:569)
> > >> at
> > >>
> > >>
> >
> org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd.execute(ListSnapshotsCmd.java:117)
> > >> at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:156)
> > >> at com.cloud.api.ApiServer.queueCommand(ApiServer.java:764)
> > >> at com.cloud.api.ApiServer.handleRequest(ApiServer.java:588)
> > >> at
> > >> com.cloud.api.ApiServlet.processRequestInContext(ApiServlet.java:321)
> > >> at com.cloud.api.ApiServlet$1.run(ApiServlet.java:134)
> > >> at
> > >>
> > >>
> >
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55)
> > >> at
> > >>
> > >>
> >
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:102)
> > >> at
> > >>
> > >>
> >
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52)
> > >> at com.cloud.api.ApiServlet.processRequest(ApiServlet.java:131)
> > >> at com.cloud.api.ApiServlet.doGet(ApiServlet.java:93)
> > >> at javax.servlet.http.HttpServlet.service(HttpServlet.java:645)
> > >> at javax.servlet.http.HttpServlet.service(HttpServlet.java:750)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.servlet.ServletHolder$NotAsyncServlet.service(ServletHolder.java:1386)
> > >> at
> > >> org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:755)
> > >> at
> > >>
> >
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:547)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
> > >> at
> > >>
> >
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:590)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
> > >> at
> > >>
> > org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:767)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
> > >> at org.eclipse.jetty.server.Server.handle(Server.java:500)
> > >> at
> > >>
> >
> org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
> > >> at
> > >> org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
> > >> at
> > >> org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
> > >> at
> > >>
> >
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273)
> > >> at
> > >> org.eclipse.jetty.io
> > >>
> .AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
> > >> at org.eclipse.jetty.io
> > >> .FillInterest.fillable(FillInterest.java:103)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:543)
> > >> at
> > >>
> > org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:398)
> > >> at
> > >>
> >
> org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:161)
> > >> at org.eclipse.jetty.io
> > >> .FillInterest.fillable(FillInterest.java:103)
> > >> at
> > >> org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:375)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
> > >> at
> > >>
> > >>
> >
> org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
> > >> at java.base/java.lang.Thread.run(Thread.java:834)
> > >>
> > >
> > >
> > > --
> > >
> > > Andrija Panić
> >
>
