[ https://issues.apache.org/jira/browse/AURORA-163?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Bill Farner resolved AURORA-163. -------------------------------- Resolution: Cannot Reproduce > investigate the cause of "Cannot take control of a task in terminal state" > -------------------------------------------------------------------------- > > Key: AURORA-163 > URL: https://issues.apache.org/jira/browse/AURORA-163 > Project: Aurora > Issue Type: Task > Components: Executor, Thermos > Reporter: brian wickman > Priority: Minor > > {noformat} > Uncaught exception: > Traceback (most recent call last): > File "/usr/lib64/python2.6/runpy.py", line 122, in _run_module_as_main > "__main__", fname, loader, pkg_name) > File "/usr/lib64/python2.6/runpy.py", line 34, in _run_code > exec code in run_globals > File "./thermos_runner.pex/__main__.py", line 28, in > PEX(__entry_point__).execute() > File > "/var/lib/mesos/slaves/201211292101-1968906762-5050-60720-432/frameworks/201104070004-0000002563-0000/executors/thermos-1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a/runs/0/thermos_runner.pex/.bootstrap/twitter/common/python/pex.py", > line 110, in execute > self.execute_entry(entry_point, args) > File > "/var/lib/mesos/slaves/201211292101-1968906762-5050-60720-432/frameworks/201104070004-0000002563-0000/executors/thermos-1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a/runs/0/thermos_runner.pex/.bootstrap/twitter/common/python/pex.py", > line 138, in execute_entry > runner(entry_point) > File > "/var/lib/mesos/slaves/201211292101-1968906762-5050-60720-432/frameworks/201104070004-0000002563-0000/executors/thermos-1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a/runs/0/thermos_runner.pex/.bootstrap/twitter/common/python/pex.py", > line 143, in execute_module > runpy.run_module(module_name, run_name='__main__') > File "/usr/lib64/python2.6/runpy.py", line 140, in run_module > fname, loader, pkg_name) > File "/usr/lib64/python2.6/runpy.py", line 34, in _run_code > exec code in run_globals > File > "/var/lib/mesos/slaves/201211292101-1968906762-5050-60720-432/frameworks/201104070004-0000002563-0000/executors/thermos-1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a/runs/0/thermos_runner.pex/thermos_runner.py", > line 68, in > File "twitter/common/app/__init__.py", line 59, in proxy_function > return bound_method(*args, **kwargs) > File "twitter/common/app/application.py", line 764, in main > rc = self._run_main(main_method, *args) > File "twitter/common/app/ > application.py", line 685, in _run_main > rc = main_method(*args, **kwargs) > File > "/var/lib/mesos/slaves/201211292101-1968906762-5050-60720-432/frameworks/201104070004-0000002563-0000/executors/thermos-1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a/runs/0/thermos_runner.pex/thermos_runner.py", > line 65, in main > File "twitter/thermos/runner/runner.py", line 817, in kill > with self.control(force): > File "/usr/lib64/python2.6/contextlib.py", line 16, in __enter__ > return self.gen.next() > File "twitter/thermos/runner/runner.py", line 483, in control > raise TaskRunner.StateError('Cannot take control of a task in terminal > state.') > StateError: Cannot take control of a task in terminal state. > {noformat} > looks like a race condition: > {noformat} > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_runner] Log file created at: 2012/12/21 00:47:24 > thermos_runner] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_runner] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_runner] Command line: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_executor] Log file created at: 2012/12/21 00:47:23 > thermos_executor] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_executor] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_executor] Command line: ./thermos_executor.pex > thermos_executor] Log file created at: 2012/12/21 00:47:23 > thermos_executor] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_executor] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_executor] Command line: ./thermos_executor.pex > thermos_executor] Log file created at: 2012/12/21 00:47:23 > thermos_executor] Running on machine: smf1-afl-23-sr3.prod.twitter.com > thermos_executor] [DIWEF]mmdd hh:mm:ss.uuuuuu pid file:line] msg > thermos_executor] Command line: ./thermos_executor.pex > thermos_executor] I1221 00:47:23.422457 61597 executor_base.py:43] > Executor [None]: registered() called with: > thermos_executor] I1221 00:47:23.423259 61597 executor_base.py:43] > Executor [None]: ExecutorInfo: executor_id { > thermos_executor] value: > "thermos-1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a" > thermos_executor] } > thermos_executor] resources { > thermos_executor] name: "cpus" > thermos_executor] type: SCALAR > thermos_executor] scalar { > thermos_executor] value: 0.25 > thermos_executor] } > thermos_executor] } > thermos_executor] resources { > thermos_executor] name: "mem" > thermos_executor] type: SCALAR > thermos_executor] scalar { > thermos_executor] value: 128.0 > thermos_executor] } > thermos_executor] } > thermos_executor] command { > thermos_executor] uris { > thermos_executor] value: > "hftp://hadoop-nn.local.twitter.com:50070/mesos/pkg/mesos/bin/prod/thermos_executor.pex" > thermos_executor] executable: true > thermos_executor] } > thermos_executor] value: "./thermos_executor.pex" > thermos_executor] } > thermos_executor] > thermos_executor] I1221 00:47:23.423513 61597 executor_base.py:43] > Executor [None]: FrameworkInfo: user: "root" > thermos_executor] name: "TwitterScheduler" > thermos_executor] id { > thermos_executor] value: "201104070004-0000002563-0000" > thermos_executor] } > thermos_executor] failover_timeout: 1814400.0 > thermos_executor] > thermos_executor] I1221 00:47:23.424176 61597 executor_base.py:43] > Executor [None]: SlaveInfo: hostname: > "smf1-afl-23-sr3.prod.twitter.com" > thermos_executor] webui_hostname: "smf1-afl-23-sr3.prod.twitter.com" > thermos_executor] resources { > thermos_executor] name: "cpus" > thermos_executor] type: SCALAR > thermos_executor] scalar { > thermos_executor] value: 14.0 > thermos_executor] } > thermos_executor] } > thermos_executor] resources { > thermos_executor] name: "mem" > thermos_executor] type: SCALAR > thermos_executor] scalar { > thermos_executor] value: 22088.0 > thermos_executor] } > thermos_executor] } > thermos_executor] resources { > thermos_executor] name: "ports" > thermos_executor] type: RANGES > thermos_executor] ranges { > thermos_executor] range { > thermos_executor] begin: 31000 > thermos_executor] end: 32000 > thermos_executor] } > thermos_executor] } > thermos_executor] } > thermos_executor] resources { > thermos_executor] name: "disk" > thermos_executor] type: SCALAR > thermos_executor] scalar { > thermos_executor] value: 400000.0 > thermos_executor] } > thermos_executor] } > thermos_executor] webui_port: 8081 > thermos_executor] attributes { > thermos_executor] name: "rack" > thermos_executor] type: TEXT > thermos_executor] text { > thermos_executor] value: "afl" > thermos_executor] } > thermos_executor] } > thermos_executor] attributes { > thermos_executor] name: "host" > thermos_executor] type: TEXT > thermos_executor] text { > thermos_executor] value: "smf1-afl-23-sr3" > thermos_executor] } > thermos_executor] } > thermos_executor] attributes { > thermos_executor] name: "executor" > thermos_executor] type: TEXT > thermos_executor] text { > thermos_executor] value: "legacy" > thermos_executor] } > thermos_executor] } > thermos_executor] > thermos_executor] I1221 00:47:23.424945 61597 executor_base.py:43] > Executor [None]: launchTask got task: > cabraham/hraven_rest_api_test:1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > thermos_executor] I1221 00:47:23.427172 61597 executor_base.py:43] > Executor [201211292101-1968906762-5050-60720-432]: Updating > 1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > => STARTING > thermos_executor] I1221 00:47:23.427336 61597 executor_base.py:43] > Executor [201211292101-1968906762-5050-60720-432]: Reason: Initializing > sandbox. > thermos_executor] W1221 00:47:23.484622 61597 monitor.py:64] Could not > read from discovered task > 1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a. > thermos_executor] I1221 00:47:23.484857 61597 task_runner_wrapper.py:91] > Creating sandbox. > thermos_executor] D1221 00:47:23.486375 61597 sandbox_manager.py:57] > DirectorySandbox: mkdir > /var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > thermos_executor] D1221 00:47:23.487433 61597 sandbox_manager.py:62] > DirectorySandbox: chown cabraham:employee > /var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > thermos_executor] D1221 00:47:23.487602 61597 sandbox_manager.py:64] > DirectorySandbox: chmod 700 > /var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > thermos_executor] I1221 00:47:23.487854 61597 task_runner_wrapper.py:114] > Forking off runner with cmdline: ./thermos_runner.pex --setuid=cabraham > --thermos_json=./task.json > --sandbox=/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --log_dir=/var/log/mesos > --task_id=1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > --checkpoint_root=/var/run/thermos --port=health:31181 > thermos_executor] D1221 00:47:23.491122 61597 thermos_executor.py:113] > Waiting for task to start. > thermos_executor] D1221 00:47:23.491385 61597 thermos_executor.py:117] > Task started. > thermos_executor] I1221 00:47:23.491657 61597 executor_base.py:43] > Executor [201211292101-1968906762-5050-60720-432]: Updating > 1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a > => RUNNING > thermos_executor] I1221 00:47:23.491827 61597 executor_base.py:43] > Executor [201211292101-1968906762-5050-60720-432]: Reason: None > thermos_executor] D1221 00:47:23.538803 61597 resource_manager.py:300] > Sampled CPU percentage: 0.00 > thermos_executor] D1221 00:47:23.539340 61597 resource_manager.py:304] > Sampled disk bytes: 0.0MB > thermos_runner] I1221 00:47:26.552576 61719 runner.py:720] Forking > Process(installer) > thermos_executor] D1221 00:47:43.551332 61597 resource_manager.py:300] > Sampled CPU percentage: 4.25 > thermos_executor] D1221 00:48:03.555243 61597 resource_manager.py:300] > Sampled CPU percentage: 0.00 > thermos_executor] D1221 00:48:23.533051 61597 resource_manager.py:304] > Sampled disk bytes: 0.0MB > thermos_executor] D1221 00:48:23.563019 61597 resource_manager.py:300] > Sampled CPU percentage: 0.00 > thermos_runner] I1221 00:48:29.516884 61719 runner.py:87] > Process(installer) finished successfully [rc=0] > thermos_runner] I1221 00:48:29.527692 61719 helper.py:131] Process > installer [pid: 61726] completed. > thermos_runner] I1221 00:48:29.663256 61719 runner.py:720] Forking > Process(hraven_rest_api_test) > thermos_executor] D1221 00:48:43.585268 61597 resource_manager.py:300] > Sampled CPU percentage: 12.89 > thermos_executor] D1221 00:49:03.588412 61597 resource_manager.py:300] > Sampled CPU percentage: 0.00 > thermos_executor] D1221 00:49:23.554073 61597 resource_manager.py:304] > Sampled disk bytes: 39.6MB > thermos_executor] I1221 00:49:23.554346 61597 resource_manager.py:309] > Resource manager triggering kill reason: FailureReason('RAM limit exceeded. > Reserved 134217728 bytes vs resident 195518464 bytes') > thermos_executor] D1221 00:49:23.586950 61597 resource_manager.py:300] > Sampled CPU percentage: 0.00 > thermos_executor] I1221 00:49:23.660841 61597 status_manager.py:61] Got > force_status=3, failure_reason=RAM limit exceeded. Reserved 134217728 bytes > vs resident 195518464 bytes from <ResourceManager(Thread-5, started daemon > 1259678016)> > thermos_executor] I1221 00:49:23.661031 61597 status_manager.py:66] > Executor polling thread detected termination condition. > thermos_runner] I1221 00:49:27.220036 61719 runner.py:87] > Process(hraven_rest_api_test) finished successfully [rc=0] > thermos_runner] I1221 00:49:27.230494 61719 helper.py:131] Process > hraven_rest_api_test [pid: 61920] completed. > thermos_runner] I1221 00:49:27.232523 61719 runner.py:254] Regular plan > complete. > thermos_runner] I1221 00:49:27.420850 61719 helper.py:121] > Coordinator installer [pid: 61725] completed. > thermos_runner] I1221 00:49:27.421155 61719 helper.py:131] Process > installer [pid: 61726] completed. > thermos_runner] I1221 00:49:27.421375 61719 helper.py:121] > Coordinator hraven_rest_api_test [pid: 61918] completed. > thermos_runner] I1221 00:49:27.421581 61719 helper.py:131] Process > hraven_rest_api_test [pid: 61920] completed. > thermos_runner] I1221 00:49:28.520051 61719 helper.py:121] > Coordinator installer [pid: 61725] completed. > thermos_runner] I1221 00:49:28.520376 61719 helper.py:131] Process > installer [pid: 61726] completed. > thermos_runner] I1221 00:49:28.520610 61719 helper.py:121] > Coordinator hraven_rest_api_test [pid: 61918] completed. > thermos_runner] I1221 00:49:28.520883 61719 helper.py:131] Process > hraven_rest_api_test [pid: 61920] completed. > thermos_runner] I1221 00:49:28.535448 61719 helper.py:121] > Coordinator installer [pid: 61725] completed. > thermos_runner] I1221 00:49:28.535794 61719 helper.py:131] Process > installer [pid: 61726] completed. > thermos_runner] I1221 00:49:28.550539 61719 helper.py:121] > Coordinator hraven_rest_api_test [pid: 61918] completed. > thermos_runner] I1221 00:49:28.550875 61719 helper.py:131] Process > hraven_rest_api_test [pid: 61920] completed. > thermos_runner] I1221 00:49:28.953026 61719 helper.py:121] > Coordinator installer [pid: 61725] completed. > thermos_runner] I1221 00:49:33.860250 61719 helper.py:131] Process > installer [pid: 61726] completed. > thermos_runner] I1221 00:49:33.860564 61719 helper.py:121] > Coordinator hraven_rest_api_test [pid: 61918] completed. > thermos_runner] I1221 00:49:33.860826 61719 helper.py:131] Process > hraven_rest_api_test [pid: 61920] completed. > thermos_runner] I1221 00:49:33.876190 61719 helper.py:121] > Coordinator installer [pid: 61725] completed. > thermos_executor] I1221 00:49:33.894011 61597 task_runner_wrapper.py:166] > Runner is alive, sending SIGINT > thermos_executor] D1221 00:49:36.452985 61597 ckpt.py:304] Flipping task > state from (undefined) to ACTIVE > thermos_executor] D1221 00:49:36.453316 61597 ckpt.py:284] Initializing > TaskRunner header to RunnerHeader(launch_time_ms=1356050845210, > task_id='1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a', > hostname='smf1-afl-23-sr3.prod.twitter.com', > sandbox='/var/lib/thermos/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a', > user='cabraham', ports={'health': 31181}) > thermos_executor] D1221 00:49:36.453536 61597 ckpt.py:335] Running state > machine for process=installer/seq=0 > thermos_executor] D1221 00:49:36.453788 61597 ckpt.py:335] Running state > machine for process=installer/seq=1 > thermos_executor] D1221 00:49:36.454011 61597 ckpt.py:335] Running state > machine for process=installer/seq=2 > thermos_executor] D1221 00:49:36.454236 61597 ckpt.py:335] Running state > machine for process=installer/seq=3 > thermos_executor] D1221 00:49:36.454444 61597 ckpt.py:335] Running state > machine for process=hraven_rest_api_test/seq=0 > thermos_executor] D1221 00:49:36.454667 61597 ckpt.py:335] Running state > machine for process=hraven_rest_api_test/seq=1 > thermos_executor] D1221 00:49:36.454905 61597 ckpt.py:335] Running state > machine for process=hraven_rest_api_test/seq=2 > thermos_executor] D1221 00:49:36.455130 61597 ckpt.py:335] Running state > machine for process=hraven_rest_api_test/seq=3 > thermos_executor] D1221 00:49:36.455341 61597 ckpt.py:304] Flipping task > state from ACTIVE to CLEANING > thermos_executor] D1221 00:49:36.455540 61597 ckpt.py:304] Flipping task > state from CLEANING to FINALIZING > thermos_executor] D1221 00:49:36.455754 61597 ckpt.py:304] Flipping task > state from FINALIZING to SUCCESS > thermos_executor] D1221 00:49:36.455897 61597 recordio.py:143] > /var/run/thermos/checkpoints/1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a/runner > has no data (cur offset = 956) > thermos_executor] I1221 00:49:36.456615 61597 status_manager.py:106] > Waiting for terminal state, current state: SUCCESS > thermos_executor] I1221 00:49:36.456764 61597 status_manager.py:108] > Terminal reached, breaking > thermos_executor] I1221 00:49:36.457376 61597 status_manager.py:124] > State we've accepted: Thermos(SUCCESS) [force_status=Mesos(TASK_FAILED), > force_message=RAM limit exceeded. Reserved 134217728 bytes vs resident > 195518464 bytes] > thermos_executor] I1221 00:49:36.457601 61597 status_manager.py:153] > Sending terminal state update: TASK_FAILED > thermos_executor] I1221 00:49:36.457916 61597 status_manager.py:157] > Stopping executor. > thermos_executor] I1221 00:49:36.458097 61597 status_manager.py:161] > Sleeping briefly to mitigate https://issues.apache.org/jira/browse/MESOS-243 > thermos_runner] I1221 00:49:36.462757 61719 thermos_runner.py:63] > Thermos runner got SIGINT, shutting down. > thermos_runner] E1221 00:49:36.654937 61719 __init__.py:42] Uncaught > exception: > thermos_runner] Traceback (most recent call last): > thermos_runner] File "/usr/lib64/python2.6/runpy.py", line 122, in > _run_module_as_main > thermos_runner] "__main__", fname, loader, pkg_name) > thermos_runner] File "/usr/lib64/python2.6/runpy.py", line 34, in > _run_code > thermos_runner] exec code in run_globals > thermos_runner] File "./thermos_runner.pex/__main__.py", line 28, in > <module> > thermos_runner] PEX(__entry_point__).execute() > thermos_runner] File > "/var/lib/mesos/slaves/201211292101-1968906762-5050-60720-432/frameworks/201104070004-0000002563-0000/executors/thermos-1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a/runs/0/thermos_runner.pex/.bootstrap/twitter/common/python/pex.py", > line 110, in execute > thermos_runner] self.execute_entry(entry_point, args) > thermos_runner] File > "/var/lib/mesos/slaves/201211292101-1968906762-5050-60720-432/frameworks/201104070004-0000002563-0000/executors/thermos-1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a/runs/0/thermos_runner.pex/.bootstrap/twitter/common/python/pex.py", > line 138, in execute_entry > thermos_runner] runner(entry_point) > thermos_runner] File > "/var/lib/mesos/slaves/201211292101-1968906762-5050-60720-432/frameworks/201104070004-0000002563-0000/executors/thermos-1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a/runs/0/thermos_runner.pex/.bootstrap/twitter/common/python/pex.py", > line 143, in execute_module > thermos_runner] runpy.run_module(module_name, run_name='__main__') > thermos_runner] File "/usr/lib64/python2.6/runpy.py", line 140, in > run_module > thermos_runner] fname, loader, pkg_name) > thermos_runner] File "/usr/lib64/python2.6/runpy.py", line 34, in > _run_code > thermos_runner] exec code in run_globals > thermos_runner] File > "/var/lib/mesos/slaves/201211292101-1968906762-5050-60720-432/frameworks/201104070004-0000002563-0000/executors/thermos-1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a/runs/0/thermos_runner.pex/thermos_runner.py", > line 68, in <module> > thermos_runner] File "twitter/common/app/__init__.py", line 59, in > proxy_function > thermos_runner] return bound_method(*args, **kwargs) > thermos_runner] File "twitter/common/app/application.py", line 764, > in main > thermos_runner] rc = self._run_main(main_method, *args) > thermos_runner] File "twitter/common/app/application.py", line 685, > in _run_main > thermos_runner] rc = main_method(*args, **kwargs) > thermos_runner] File > "/var/lib/mesos/slaves/201211292101-1968906762-5050-60720-432/frameworks/201104070004-0000002563-0000/executors/thermos-1356050822533-cabraham-hraven_rest_api_test-0-d0a75e07-ca5f-4040-9937-91162ccdf95a/runs/0/thermos_runner.pex/thermos_runner.py", > line 65, in main > thermos_runner] File "twitter/thermos/runner/runner.py", line 817, in > kill > thermos_runner] with self.control(force): > thermos_runner] File "/usr/lib64/python2.6/contextlib.py", line 16, > in __enter__ > thermos_runner] return self.gen.next() > thermos_runner] File "twitter/thermos/runner/runner.py", line 483, in > control > thermos_runner] raise TaskRunner.StateError('Cannot take control of > a task in terminal state.') > thermos_runner] StateError: Cannot take control of a task in terminal > state. > thermos_runner] > thermos_executor] I1221 00:49:41.462981 61597 > thermos_executor_main.py:32] MesosExecutorDriver.run() has finished. > {noformat} -- This message was sent by Atlassian JIRA (v6.3.4#6332)