[
https://issues.apache.org/jira/browse/MESOS-4715?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Guillermo Rodriguez updated MESOS-4715:
---------------------------------------
Description:
When the allocation and deallocation of CPUs comes very close to 0, Mesos will
not consider it 0. It can keep it as a very small float like
2.22044604925031e-16 but not 0.
This is the output of the slave status. Look at the used resources data.
http://mesos-master:5050/master/slaves
{code}
{"slaves":[{"active":true,"attributes":{},"hostname":"172.31.41.90","id":"4943165a-6025-4496-a16b-68bbefb637ca-S1","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.41.90:5051","registered_time":1455828323.77997,"reregistered_time":1455828323.77999,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.22044604925031e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.163","id":"4943165a-6025-4496-a16b-68bbefb637ca-S5","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.163:5051","registered_time":1455828323.58233,"reregistered_time":1455828323.58236,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":6.66133814775094e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.47.7","id":"4943165a-6025-4496-a16b-68bbefb637ca-S13","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.47.7:5051","registered_time":1455828323.52293,"reregistered_time":1455828323.5231,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.162","id":"4943165a-6025-4496-a16b-68bbefb637ca-S6","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.162:5051","registered_time":1455828322.8728,"reregistered_time":1455828322.87282,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.22044604925031e-15,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.32.165","id":"4943165a-6025-4496-a16b-68bbefb637ca-S3","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.32.165:5051","registered_time":1455828323.80057,"reregistered_time":1455828323.8006,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":3.33066907387547e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.36.13","id":"4943165a-6025-4496-a16b-68bbefb637ca-S11","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.36.13:5051","registered_time":1455828323.05929,"reregistered_time":1455828323.05933,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":6.66133814775094e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.35.20","id":"e6611503-e2df-4924-bb76-7b4a9bcd0e34-S1","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.35.20:5051","registered_time":1455828323.28517,"reregistered_time":1455828323.47201,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":0.700000000000002,"disk":0,"mem":2560.0,"ports":"[4400-4400,
9090-9090,
9093-9093]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.33.174","id":"4943165a-6025-4496-a16b-68bbefb637ca-S9","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.33.174:5051","registered_time":1455828323.3524,"reregistered_time":1455828323.35244,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":1.11022302462516e-15,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.1","id":"4943165a-6025-4496-a16b-68bbefb637ca-S12","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.1:5051","registered_time":1455828323.56901,"reregistered_time":1455828323.56903,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.43.161","id":"4943165a-6025-4496-a16b-68bbefb637ca-S4","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.43.161:5051","registered_time":1455828322.84732,"reregistered_time":1455828322.84735,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.16493489801906e-15,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.35.21","id":"e6611503-e2df-4924-bb76-7b4a9bcd0e34-S0","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.35.21:5051","registered_time":1455828323.3962,"reregistered_time":1455828323.40149,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":1.0,"disk":0,"mem":2048.0,"ports":"[9090-9090,
9093-9093]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.41.60","id":"4943165a-6025-4496-a16b-68bbefb637ca-S8","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.41.60:5051","registered_time":1455828323.4388,"reregistered_time":1455828323.43883,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.46.222","id":"4943165a-6025-4496-a16b-68bbefb637ca-S7","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.46.222:5051","registered_time":1455828323.44999,"reregistered_time":1455828323.45021,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.0,"disk":0,"mem":1024.0,"ports":"[2376-2376,
3375-3375]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.43.53","id":"e6611503-e2df-4924-bb76-7b4a9bcd0e34-S2","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.43.53:5051","registered_time":1455828323.48255,"reregistered_time":1455828323.48439,"reserved_resources":{"system":{"cpus":4.0,"disk":91727.0,"mem":15027.0,"ports":"[80-32000]"}},"resources":{"cpus":4.0,"disk":91727.0,"mem":15027.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":0,"disk":0,"mem":0},"used_resources":{"cpus":1.6,"disk":0,"mem":3328.0,"ports":"[2003-2003,
3000-3000, 8080-8080, 9091-9091, 9093-9093,
9106-9106]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.34.27","id":"4943165a-6025-4496-a16b-68bbefb637ca-S10","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.34.27:5051","registered_time":1455828323.5514,"reregistered_time":1455828323.55143,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.97","id":"4943165a-6025-4496-a16b-68bbefb637ca-S2","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.97:5051","registered_time":1455828323.48525,"reregistered_time":1455828323.48532,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":6.66133814775094e-16,"disk":0,"mem":0},"version":"0.26.0"}]}
{code}
Now, maybe this can cause the engine to crash like this. This are the last
lines of a log before the engine crashed. Look that at 18:21:35.566262 it
recovered one of those tiny amounts of CPU.
{code}
I0218 18:21:35.565951 10 master.cpp:6066] Updating the state of task
effective_spread_generate-paris^paris.29bfd60c1e91 of framework
4943165a-6025-4496-a16b-68bbefb637ca-0011 (latest state: TASK_KILLED, status
update state: TASK_KILLED)
I0218 18:21:35.566107 10 master.cpp:6132] Removing task
effective_spread_generate-paris^paris.29bfd60c1e91 with resources cpus(*):0.9;
mem(*):878 of framework 4943165a-6025-4496-a16b-68bbefb637ca-0011 on slave
4943165a-6025-4496-a16b-68bbefb637ca-S1 at slave(1)@172.31.41.90:5051
(172.31.41.90)
I0218 18:21:35.566264 10 master.cpp:6066] Updating the state of task
realised_spread_generate-paris^paris.e1cd3d6753bc of framework
4943165a-6025-4496-a16b-68bbefb637ca-0011 (latest state: TASK_KILLED, status
update state: TASK_KILLED)
I0218 18:21:35.566262 8 hierarchical.cpp:744] Recovered cpus(*):0.9;
mem(*):878 (total: ports(*):[80-32000]; cpus(*):8; mem(*):31155;
disk(*):111570, allocated: cpus(*):1.44329e-15) on slave
4943165a-6025-4496-a16b-68bbefb637ca-S1 from framework
4943165a-6025-4496-a16b-68bbefb637ca-0011
I0218 18:21:35.566471 8 hierarchical.cpp:744] Recovered cpus(*):0.9;
mem(*):888 (total: ports(*):[80-32000]; cpus(*):8; mem(*):31155;
disk(*):111570, allocated: cpus(*):0.8; mem(*):24612; ports(*):[80-32000];
disk(*):111570) on slave 4943165a-6025-4496-a16b-68bbefb637ca-S8 from framework
4943165a-6025-4496-a16b-68bbefb637ca-0011
I0218 18:21:35.566390 10 master.cpp:6132] Removing task
realised_spread_generate-paris^paris.e1cd3d6753bc with resources cpus(*):0.9;
mem(*):888 of framework 4943165a-6025-4496-a16b-68bbefb637ca-0011 on slave
4943165a-6025-4496-a16b-68bbefb637ca-S8 at slave(1)@172.31.41.60:5051
(172.31.41.60)
F0218 18:21:35.566793 10 sorter.cpp:233] Check failed:
total.resources.contains(slaveId)
*** Check failure stack trace: ***
@ 0x7fa7a51ac9ad google::LogMessage::Fail()
@ 0x7fa7a51ae7ed google::LogMessage::SendToLog()
@ 0x7fa7a51ac59c google::LogMessage::Flush()
@ 0x7fa7a51af0e9 google::LogMessageFatal::~LogMessageFatal()
@ 0x7fa7a4b2335b
mesos::internal::master::allocator::DRFSorter::remove()
@ 0x7fa7a4b13833
mesos::internal::master::allocator::internal::HierarchicalAllocatorProcess::removeFramework()
@ 0x7fa7a515d8e1 process::ProcessManager::resume()
@ 0x7fa7a515dbe7
_ZNSt6thread5_ImplISt12_Bind_simpleIFSt5_BindIFZN7process14ProcessManager12init_threadsEvEUlRKSt11atomic_boolE_St17reference_wrapperIS6_EEEvEEE6_M_runEv
@ 0x7fa7a3e0ba40 (unknown)
@ 0x7fa7a3628182 start_thread
@ 0x7fa7a335547d (unknown)
*** Aborted at 1455819695 (unix time) try "date -d @1455819695" if you are
using GNU date ***
PC: @ 0x7fa7a3295227 (unknown)
*** SIGSEGV (@0x0) received by PID 1 (TID 0x7fa79b57e700) from PID 0; stack
trace: ***
@ 0x7fa7a3630340 (unknown)
@ 0x7fa7a3295227 (unknown)
@ 0x7fa7a51b4c69 google::DumpStackTraceAndExit()
@ 0x7fa7a51ac9ad google::LogMessage::Fail()
@ 0x7fa7a51ae7ed google::LogMessage::SendToLog()
@ 0x7fa7a51ac59c google::LogMessage::Flush()
@ 0x7fa7a51af0e9 google::LogMessageFatal::~LogMessageFatal()
@ 0x7fa7a4b2335b mesos::internal::master::allocator::DRFSorter::remove()
@ 0x7fa7a4b13833
mesos::internal::master::allocator::internal::HierarchicalAllocatorProcess::removeFramework()
@ 0x7fa7a515d8e1 process::ProcessManager::resume()
@ 0x7fa7a515dbe7
_ZNSt6thread5_ImplISt12_Bind_simpleIFSt5_BindIFZN7process14ProcessManager12init_threadsEvEUlRKSt11atomic_boolE_St17reference_wrapperIS6_EEEvEEE6_M_runEv
@ 0x7fa7a3e0ba40 (unknown)
@ 0x7fa7a3628182 start_thread
@ 0x7fa7a335547d (unknown)
{code}
was:
When the allocation and deallocation of CPUs comes very close to 0, Mesos will
not consider it 0. It can keep it as a very small float like
2.22044604925031e-16 but not 0.
This is the output of the slave status:
http://mesos-master:5050/master/slaves
{code}
{"slaves":[{"active":true,"attributes":{},"hostname":"172.31.41.90","id":"4943165a-6025-4496-a16b-68bbefb637ca-S1","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.41.90:5051","registered_time":1455828323.77997,"reregistered_time":1455828323.77999,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.22044604925031e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.163","id":"4943165a-6025-4496-a16b-68bbefb637ca-S5","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.163:5051","registered_time":1455828323.58233,"reregistered_time":1455828323.58236,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":6.66133814775094e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.47.7","id":"4943165a-6025-4496-a16b-68bbefb637ca-S13","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.47.7:5051","registered_time":1455828323.52293,"reregistered_time":1455828323.5231,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.162","id":"4943165a-6025-4496-a16b-68bbefb637ca-S6","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.162:5051","registered_time":1455828322.8728,"reregistered_time":1455828322.87282,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.22044604925031e-15,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.32.165","id":"4943165a-6025-4496-a16b-68bbefb637ca-S3","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.32.165:5051","registered_time":1455828323.80057,"reregistered_time":1455828323.8006,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":3.33066907387547e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.36.13","id":"4943165a-6025-4496-a16b-68bbefb637ca-S11","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.36.13:5051","registered_time":1455828323.05929,"reregistered_time":1455828323.05933,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":6.66133814775094e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.35.20","id":"e6611503-e2df-4924-bb76-7b4a9bcd0e34-S1","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.35.20:5051","registered_time":1455828323.28517,"reregistered_time":1455828323.47201,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":0.700000000000002,"disk":0,"mem":2560.0,"ports":"[4400-4400,
9090-9090,
9093-9093]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.33.174","id":"4943165a-6025-4496-a16b-68bbefb637ca-S9","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.33.174:5051","registered_time":1455828323.3524,"reregistered_time":1455828323.35244,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":1.11022302462516e-15,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.1","id":"4943165a-6025-4496-a16b-68bbefb637ca-S12","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.1:5051","registered_time":1455828323.56901,"reregistered_time":1455828323.56903,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.43.161","id":"4943165a-6025-4496-a16b-68bbefb637ca-S4","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.43.161:5051","registered_time":1455828322.84732,"reregistered_time":1455828322.84735,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.16493489801906e-15,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.35.21","id":"e6611503-e2df-4924-bb76-7b4a9bcd0e34-S0","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.35.21:5051","registered_time":1455828323.3962,"reregistered_time":1455828323.40149,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":1.0,"disk":0,"mem":2048.0,"ports":"[9090-9090,
9093-9093]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.41.60","id":"4943165a-6025-4496-a16b-68bbefb637ca-S8","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.41.60:5051","registered_time":1455828323.4388,"reregistered_time":1455828323.43883,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.46.222","id":"4943165a-6025-4496-a16b-68bbefb637ca-S7","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.46.222:5051","registered_time":1455828323.44999,"reregistered_time":1455828323.45021,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.0,"disk":0,"mem":1024.0,"ports":"[2376-2376,
3375-3375]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.43.53","id":"e6611503-e2df-4924-bb76-7b4a9bcd0e34-S2","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.43.53:5051","registered_time":1455828323.48255,"reregistered_time":1455828323.48439,"reserved_resources":{"system":{"cpus":4.0,"disk":91727.0,"mem":15027.0,"ports":"[80-32000]"}},"resources":{"cpus":4.0,"disk":91727.0,"mem":15027.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":0,"disk":0,"mem":0},"used_resources":{"cpus":1.6,"disk":0,"mem":3328.0,"ports":"[2003-2003,
3000-3000, 8080-8080, 9091-9091, 9093-9093,
9106-9106]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.34.27","id":"4943165a-6025-4496-a16b-68bbefb637ca-S10","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.34.27:5051","registered_time":1455828323.5514,"reregistered_time":1455828323.55143,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.97","id":"4943165a-6025-4496-a16b-68bbefb637ca-S2","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.97:5051","registered_time":1455828323.48525,"reregistered_time":1455828323.48532,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":6.66133814775094e-16,"disk":0,"mem":0},"version":"0.26.0"}]}
{code}
Now, maybe this can cause the engine to crash like this. This are the last
lines of a log before the engine crashed. Look that at 18:21:35.566262 it
recovered one of those tiny amounts of CPU.
{code}
I0218 18:21:35.565951 10 master.cpp:6066] Updating the state of task
effective_spread_generate-paris^paris.29bfd60c1e91 of framework
4943165a-6025-4496-a16b-68bbefb637ca-0011 (latest state: TASK_KILLED, status
update state: TASK_KILLED)
I0218 18:21:35.566107 10 master.cpp:6132] Removing task
effective_spread_generate-paris^paris.29bfd60c1e91 with resources cpus(*):0.9;
mem(*):878 of framework 4943165a-6025-4496-a16b-68bbefb637ca-0011 on slave
4943165a-6025-4496-a16b-68bbefb637ca-S1 at slave(1)@172.31.41.90:5051
(172.31.41.90)
I0218 18:21:35.566264 10 master.cpp:6066] Updating the state of task
realised_spread_generate-paris^paris.e1cd3d6753bc of framework
4943165a-6025-4496-a16b-68bbefb637ca-0011 (latest state: TASK_KILLED, status
update state: TASK_KILLED)
I0218 18:21:35.566262 8 hierarchical.cpp:744] Recovered cpus(*):0.9;
mem(*):878 (total: ports(*):[80-32000]; cpus(*):8; mem(*):31155;
disk(*):111570, allocated: cpus(*):1.44329e-15) on slave
4943165a-6025-4496-a16b-68bbefb637ca-S1 from framework
4943165a-6025-4496-a16b-68bbefb637ca-0011
I0218 18:21:35.566471 8 hierarchical.cpp:744] Recovered cpus(*):0.9;
mem(*):888 (total: ports(*):[80-32000]; cpus(*):8; mem(*):31155;
disk(*):111570, allocated: cpus(*):0.8; mem(*):24612; ports(*):[80-32000];
disk(*):111570) on slave 4943165a-6025-4496-a16b-68bbefb637ca-S8 from framework
4943165a-6025-4496-a16b-68bbefb637ca-0011
I0218 18:21:35.566390 10 master.cpp:6132] Removing task
realised_spread_generate-paris^paris.e1cd3d6753bc with resources cpus(*):0.9;
mem(*):888 of framework 4943165a-6025-4496-a16b-68bbefb637ca-0011 on slave
4943165a-6025-4496-a16b-68bbefb637ca-S8 at slave(1)@172.31.41.60:5051
(172.31.41.60)
F0218 18:21:35.566793 10 sorter.cpp:233] Check failed:
total.resources.contains(slaveId)
*** Check failure stack trace: ***
@ 0x7fa7a51ac9ad google::LogMessage::Fail()
@ 0x7fa7a51ae7ed google::LogMessage::SendToLog()
@ 0x7fa7a51ac59c google::LogMessage::Flush()
@ 0x7fa7a51af0e9 google::LogMessageFatal::~LogMessageFatal()
@ 0x7fa7a4b2335b
mesos::internal::master::allocator::DRFSorter::remove()
@ 0x7fa7a4b13833
mesos::internal::master::allocator::internal::HierarchicalAllocatorProcess::removeFramework()
@ 0x7fa7a515d8e1 process::ProcessManager::resume()
@ 0x7fa7a515dbe7
_ZNSt6thread5_ImplISt12_Bind_simpleIFSt5_BindIFZN7process14ProcessManager12init_threadsEvEUlRKSt11atomic_boolE_St17reference_wrapperIS6_EEEvEEE6_M_runEv
@ 0x7fa7a3e0ba40 (unknown)
@ 0x7fa7a3628182 start_thread
@ 0x7fa7a335547d (unknown)
*** Aborted at 1455819695 (unix time) try "date -d @1455819695" if you are
using GNU date ***
PC: @ 0x7fa7a3295227 (unknown)
*** SIGSEGV (@0x0) received by PID 1 (TID 0x7fa79b57e700) from PID 0; stack
trace: ***
@ 0x7fa7a3630340 (unknown)
@ 0x7fa7a3295227 (unknown)
@ 0x7fa7a51b4c69 google::DumpStackTraceAndExit()
@ 0x7fa7a51ac9ad google::LogMessage::Fail()
@ 0x7fa7a51ae7ed google::LogMessage::SendToLog()
@ 0x7fa7a51ac59c google::LogMessage::Flush()
@ 0x7fa7a51af0e9 google::LogMessageFatal::~LogMessageFatal()
@ 0x7fa7a4b2335b mesos::internal::master::allocator::DRFSorter::remove()
@ 0x7fa7a4b13833
mesos::internal::master::allocator::internal::HierarchicalAllocatorProcess::removeFramework()
@ 0x7fa7a515d8e1 process::ProcessManager::resume()
@ 0x7fa7a515dbe7
_ZNSt6thread5_ImplISt12_Bind_simpleIFSt5_BindIFZN7process14ProcessManager12init_threadsEvEUlRKSt11atomic_boolE_St17reference_wrapperIS6_EEEvEEE6_M_runEv
@ 0x7fa7a3e0ba40 (unknown)
@ 0x7fa7a3628182 start_thread
@ 0x7fa7a335547d (unknown)
{code}
> Almost 0 resources are not cleaned and cause a crash
> ----------------------------------------------------
>
> Key: MESOS-4715
> URL: https://issues.apache.org/jira/browse/MESOS-4715
> Project: Mesos
> Issue Type: Bug
> Components: allocation
> Affects Versions: 0.26.0
> Environment: CoreOS 899.7
> Docker 1.9.1
> Mesos 0.26
> Marathon 0.15.2
> Reporter: Guillermo Rodriguez
>
> When the allocation and deallocation of CPUs comes very close to 0, Mesos
> will not consider it 0. It can keep it as a very small float like
> 2.22044604925031e-16 but not 0.
> This is the output of the slave status. Look at the used resources data.
> http://mesos-master:5050/master/slaves
> {code}
> {"slaves":[{"active":true,"attributes":{},"hostname":"172.31.41.90","id":"4943165a-6025-4496-a16b-68bbefb637ca-S1","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.41.90:5051","registered_time":1455828323.77997,"reregistered_time":1455828323.77999,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.22044604925031e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.163","id":"4943165a-6025-4496-a16b-68bbefb637ca-S5","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.163:5051","registered_time":1455828323.58233,"reregistered_time":1455828323.58236,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":6.66133814775094e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.47.7","id":"4943165a-6025-4496-a16b-68bbefb637ca-S13","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.47.7:5051","registered_time":1455828323.52293,"reregistered_time":1455828323.5231,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.162","id":"4943165a-6025-4496-a16b-68bbefb637ca-S6","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.162:5051","registered_time":1455828322.8728,"reregistered_time":1455828322.87282,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.22044604925031e-15,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.32.165","id":"4943165a-6025-4496-a16b-68bbefb637ca-S3","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.32.165:5051","registered_time":1455828323.80057,"reregistered_time":1455828323.8006,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":3.33066907387547e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.36.13","id":"4943165a-6025-4496-a16b-68bbefb637ca-S11","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.36.13:5051","registered_time":1455828323.05929,"reregistered_time":1455828323.05933,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":6.66133814775094e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.35.20","id":"e6611503-e2df-4924-bb76-7b4a9bcd0e34-S1","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.35.20:5051","registered_time":1455828323.28517,"reregistered_time":1455828323.47201,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":0.700000000000002,"disk":0,"mem":2560.0,"ports":"[4400-4400,
> 9090-9090,
> 9093-9093]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.33.174","id":"4943165a-6025-4496-a16b-68bbefb637ca-S9","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.33.174:5051","registered_time":1455828323.3524,"reregistered_time":1455828323.35244,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":1.11022302462516e-15,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.1","id":"4943165a-6025-4496-a16b-68bbefb637ca-S12","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.1:5051","registered_time":1455828323.56901,"reregistered_time":1455828323.56903,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.43.161","id":"4943165a-6025-4496-a16b-68bbefb637ca-S4","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.43.161:5051","registered_time":1455828322.84732,"reregistered_time":1455828322.84735,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.16493489801906e-15,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.35.21","id":"e6611503-e2df-4924-bb76-7b4a9bcd0e34-S0","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.35.21:5051","registered_time":1455828323.3962,"reregistered_time":1455828323.40149,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":1.0,"disk":0,"mem":2048.0,"ports":"[9090-9090,
>
> 9093-9093]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.41.60","id":"4943165a-6025-4496-a16b-68bbefb637ca-S8","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.41.60:5051","registered_time":1455828323.4388,"reregistered_time":1455828323.43883,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.46.222","id":"4943165a-6025-4496-a16b-68bbefb637ca-S7","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.46.222:5051","registered_time":1455828323.44999,"reregistered_time":1455828323.45021,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":2.0,"disk":0,"mem":1024.0,"ports":"[2376-2376,
>
> 3375-3375]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.43.53","id":"e6611503-e2df-4924-bb76-7b4a9bcd0e34-S2","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.43.53:5051","registered_time":1455828323.48255,"reregistered_time":1455828323.48439,"reserved_resources":{"system":{"cpus":4.0,"disk":91727.0,"mem":15027.0,"ports":"[80-32000]"}},"resources":{"cpus":4.0,"disk":91727.0,"mem":15027.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":0,"disk":0,"mem":0},"used_resources":{"cpus":1.6,"disk":0,"mem":3328.0,"ports":"[2003-2003,
> 3000-3000, 8080-8080, 9091-9091, 9093-9093,
> 9106-9106]"},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.34.27","id":"4943165a-6025-4496-a16b-68bbefb637ca-S10","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.34.27:5051","registered_time":1455828323.5514,"reregistered_time":1455828323.55143,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":4.44089209850063e-16,"disk":0,"mem":0},"version":"0.26.0"},{"active":true,"attributes":{},"hostname":"172.31.39.97","id":"4943165a-6025-4496-a16b-68bbefb637ca-S2","offered_resources":{"cpus":0,"disk":0,"mem":0},"pid":"slave(1)@172.31.39.97:5051","registered_time":1455828323.48525,"reregistered_time":1455828323.48532,"reserved_resources":{},"resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"unreserved_resources":{"cpus":8.0,"disk":111570.0,"mem":31155.0,"ports":"[80-32000]"},"used_resources":{"cpus":6.66133814775094e-16,"disk":0,"mem":0},"version":"0.26.0"}]}
> {code}
> Now, maybe this can cause the engine to crash like this. This are the last
> lines of a log before the engine crashed. Look that at 18:21:35.566262 it
> recovered one of those tiny amounts of CPU.
> {code}
> I0218 18:21:35.565951 10 master.cpp:6066] Updating the state of task
> effective_spread_generate-paris^paris.29bfd60c1e91 of framework
> 4943165a-6025-4496-a16b-68bbefb637ca-0011 (latest state: TASK_KILLED, status
> update state: TASK_KILLED)
> I0218 18:21:35.566107 10 master.cpp:6132] Removing task
> effective_spread_generate-paris^paris.29bfd60c1e91 with resources
> cpus(*):0.9; mem(*):878 of framework
> 4943165a-6025-4496-a16b-68bbefb637ca-0011 on slave
> 4943165a-6025-4496-a16b-68bbefb637ca-S1 at slave(1)@172.31.41.90:5051
> (172.31.41.90)
> I0218 18:21:35.566264 10 master.cpp:6066] Updating the state of task
> realised_spread_generate-paris^paris.e1cd3d6753bc of framework
> 4943165a-6025-4496-a16b-68bbefb637ca-0011 (latest state: TASK_KILLED, status
> update state: TASK_KILLED)
> I0218 18:21:35.566262 8 hierarchical.cpp:744] Recovered cpus(*):0.9;
> mem(*):878 (total: ports(*):[80-32000]; cpus(*):8; mem(*):31155;
> disk(*):111570, allocated: cpus(*):1.44329e-15) on slave
> 4943165a-6025-4496-a16b-68bbefb637ca-S1 from framework
> 4943165a-6025-4496-a16b-68bbefb637ca-0011
> I0218 18:21:35.566471 8 hierarchical.cpp:744] Recovered cpus(*):0.9;
> mem(*):888 (total: ports(*):[80-32000]; cpus(*):8; mem(*):31155;
> disk(*):111570, allocated: cpus(*):0.8; mem(*):24612; ports(*):[80-32000];
> disk(*):111570) on slave 4943165a-6025-4496-a16b-68bbefb637ca-S8 from
> framework 4943165a-6025-4496-a16b-68bbefb637ca-0011
> I0218 18:21:35.566390 10 master.cpp:6132] Removing task
> realised_spread_generate-paris^paris.e1cd3d6753bc with resources cpus(*):0.9;
> mem(*):888 of framework 4943165a-6025-4496-a16b-68bbefb637ca-0011 on slave
> 4943165a-6025-4496-a16b-68bbefb637ca-S8 at slave(1)@172.31.41.60:5051
> (172.31.41.60)
> F0218 18:21:35.566793 10 sorter.cpp:233] Check failed:
> total.resources.contains(slaveId)
> *** Check failure stack trace: ***
> @ 0x7fa7a51ac9ad google::LogMessage::Fail()
> @ 0x7fa7a51ae7ed google::LogMessage::SendToLog()
> @ 0x7fa7a51ac59c google::LogMessage::Flush()
> @ 0x7fa7a51af0e9 google::LogMessageFatal::~LogMessageFatal()
> @ 0x7fa7a4b2335b
> mesos::internal::master::allocator::DRFSorter::remove()
> @ 0x7fa7a4b13833
> mesos::internal::master::allocator::internal::HierarchicalAllocatorProcess::removeFramework()
> @ 0x7fa7a515d8e1 process::ProcessManager::resume()
> @ 0x7fa7a515dbe7
> _ZNSt6thread5_ImplISt12_Bind_simpleIFSt5_BindIFZN7process14ProcessManager12init_threadsEvEUlRKSt11atomic_boolE_St17reference_wrapperIS6_EEEvEEE6_M_runEv
> @ 0x7fa7a3e0ba40 (unknown)
> @ 0x7fa7a3628182 start_thread
> @ 0x7fa7a335547d (unknown)
> *** Aborted at 1455819695 (unix time) try "date -d @1455819695" if you are
> using GNU date ***
> PC: @ 0x7fa7a3295227 (unknown)
> *** SIGSEGV (@0x0) received by PID 1 (TID 0x7fa79b57e700) from PID 0; stack
> trace: ***
> @ 0x7fa7a3630340 (unknown)
> @ 0x7fa7a3295227 (unknown)
> @ 0x7fa7a51b4c69 google::DumpStackTraceAndExit()
> @ 0x7fa7a51ac9ad google::LogMessage::Fail()
> @ 0x7fa7a51ae7ed google::LogMessage::SendToLog()
> @ 0x7fa7a51ac59c google::LogMessage::Flush()
> @ 0x7fa7a51af0e9 google::LogMessageFatal::~LogMessageFatal()
> @ 0x7fa7a4b2335b
> mesos::internal::master::allocator::DRFSorter::remove()
> @ 0x7fa7a4b13833
> mesos::internal::master::allocator::internal::HierarchicalAllocatorProcess::removeFramework()
> @ 0x7fa7a515d8e1 process::ProcessManager::resume()
> @ 0x7fa7a515dbe7
> _ZNSt6thread5_ImplISt12_Bind_simpleIFSt5_BindIFZN7process14ProcessManager12init_threadsEvEUlRKSt11atomic_boolE_St17reference_wrapperIS6_EEEvEEE6_M_runEv
> @ 0x7fa7a3e0ba40 (unknown)
> @ 0x7fa7a3628182 start_thread
> @ 0x7fa7a335547d (unknown)
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
