[ https://issues.apache.org/jira/browse/SLIDER-1034?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15062677#comment-15062677 ]
Steve Loughran commented on SLIDER-1034: ---------------------------------------- {code} assert cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1) == 0 | | | | | | 49 echo role.pending.aa.instances false { "version" : "1.0", "name" : "testaaecho", "type" : "agent", "state" : 3, "createTime" : 1450211128303, "updateTime" : 1450211150344, "originConfigurationPath" : "file:/home/jenkins/.slider/cluster/testaaecho/snapshot", "generatedConfigurationPath" : "file:/home/jenkins/.slider/cluster/testaaecho/generated", "dataPath" : "file:/home/jenkins/.slider/cluster/testaaecho/database", "options" : { "site.global.security_enabled" : "false", "internal.appdef.dir.path" : "file:/home/jenkins/.slider/cluster/testaaecho/tmp/appdef", "internal.application.home" : "/", "agent.package.root" : "/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python", "slider.cluster.directory.permissions" : "0770", "application.name" : "testaaecho", "internal.addons.dir.path" : "file:/home/jenkins/.slider/cluster/testaaecho/tmp/addons", "internal.tmp.dir" : "file:/home/jenkins/.slider/cluster/testaaecho/tmp", "internal.snapshot.conf.path" : "file:/home/jenkins/.slider/cluster/testaaecho/snapshot", "env.MALLOC_ARENA_MAX" : "4", "zookeeper.path" : "/services/slider/users/jenkins/testaaecho", "internal.container.failure.shortlife" : "60000", "internal.application.image.path" : null, "internal.generated.conf.path" : "file:/home/jenkins/.slider/cluster/testaaecho/generated", "site.fs.default.name" : "file:///", "agent.version" : "file:/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/version", "agent.conf" : "file:/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/agent.ini", "zookeeper.hosts" : "localhost", "internal.provider.name" : "agent", "internal.data.dir.path" : "file:/home/jenkins/.slider/cluster/testaaecho/database", "site.fs.defaultFS" : "file:///", "slider.data.directory.permissions" : "0770", "zookeeper.quorum" : "localhost:43906", "internal.am.tmp.dir" : "file:/home/jenkins/.slider/cluster/testaaecho/tmp/appmaster", "internal.container.failure.threshold" : "5", "application.def" : "file:/tmp/junit4103973573779864736/testpkg/appdef_1.zip" }, "info" : { "info.am.container.id" : "container_1450211118134_0001_01_000001", "info.am.rpc.port" : "1024", "info.am.attempt.id" : "appattempt_1450211118134_0001_000001", "yarn.memory" : "8192", "info.am.web.port" : "1025", "info.am.web.url" : "http://priapus.apache.org:1025/", "info.am.app.id" : "application_1450211118134_0001", "yarn.vcores" : "32", "info.am.hostname" : "priapus.apache.org", "status.application.build.info" : "Slider Core-0.90.0-incubating-SNAPSHOT Built against commit# 593460970a on Java 1.7.0_80 by jenkins", "status.hadoop.build.info" : "2.7.1", "status.hadoop.deployed.info" : "branch-2.7.1 @fc0a1a23fc1868e4d5ee7fa2b28a58a", "live.time" : "15 Dec 2015 20:25:28 GMT", "live.time.millis" : "1450211128303", "create.time" : "15 Dec 2015 20:25:28 GMT", "create.time.millis" : "1450211128303", "containers.at.am-restart" : "0", "status.time" : "15 Dec 2015 20:25:50 GMT", "status.time.millis" : "1450211150344", "info.am.agent.status.url" : "https://priapus.apache.org:37655/", "info.am.agent.status.port" : "37655", "info.am.agent.ops.url" : "https://priapus.apache.org:41282/", "info.am.agent.ops.port" : "41282" }, "statistics" : { "slider-appmaster" : { "containers.unknown.completed" : 0, "containers.start.started" : 1, "containers.live" : 2, "containers.start.failed" : 0, "containers.failed" : 0, "containers.completed" : 0, "containers.surplus" : 0 }, "echo" : { "containers.failed.node" : 0, "containers.start.started" : 0, "containers.live" : 1, "containers.start.failed" : 0, "containers.failed.preempted" : 0, "containers.active.requests" : 0, "containers.failed.recently" : 0, "containers.failed" : 0, "containers.completed" : 0, "containers.anti-affine.pending" : 49, "containers.desired" : 1, "containers.requested" : 1 } }, "instances" : { "slider-appmaster" : [ "container_1450211118134_0001_01_000001" ], "echo" : [ "container_1450211118134_0001_01_000002" ] }, "roles" : { "slider-appmaster" : { "role.failed.recently.instances" : "0", "yarn.memory" : "1024", "role.requested.instances" : "0", "role.failed.starting.instances" : "0", "role.actual.instances" : "1", "yarn.vcores" : "1", "yarn.component.instances" : "1", "role.releasing.instances" : "0", "role.failed.node.instances" : "0", "role.failed.instances" : "0", "role.failed.preempted.instances" : "0" }, "echo" : { "role.failed.recently.instances" : "0", "yarn.component.placement.policy" : "4", "role.requested.instances" : "0", "test.relax.validation" : "true", "role.failed.starting.instances" : "0", "role.failed.node.instances" : "0", "role.failed.preempted.instances" : "0", "app.name" : "Agent", "yarn.role.priority" : "1", "role.actual.instances" : "1", "agent.script" : "echo.py", "yarn.component.instances" : "1", "role.releasing.instances" : "0", "role.failed.instances" : "0", "role.pending.aa.instances" : "49" } }, "clientProperties" : { }, "status" : { "live" : { "slider-appmaster" : { "container_1450211118134_0001_01_000001" : { "name" : "container_1450211118134_0001_01_000001", "role" : "slider-appmaster", "roleId" : 0, "createTime" : 1450211128324, "startTime" : 1450211128324, "released" : false, "host" : "priapus.apache.org", "hostUrl" : "http://priapus.apache.org:1025", "state" : 3, "exitCode" : 0 } }, "echo" : { "container_1450211118134_0001_01_000002" : { "name" : "container_1450211118134_0001_01_000002", "role" : "echo", "roleId" : 1, "createTime" : 1450211131904, "startTime" : 1450211131985, "released" : false, "host" : "priapus.apache.org", "hostUrl" : "http://priapus.apache.org:40388", "state" : 3, "exitCode" : 0, "command" : "python /home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/agent/main.py --label container_1450211118134_0001_01_000002___echo --zk-quorum localhost:43906 --zk-reg-path /registry/users/jenkins/services/org-apache-slider/testaaecho > <LOG_DIR>/slider-agent.out 2>&1 ; ", "environment" : [ "LANGUAGE=\"en_US.UTF-8\"", "AGENT_WORK_ROOT=\"$PWD\"", "HADOOP_USER_NAME=\"jenkins\"", "AGENT_LOG_ROOT=\"<LOG_DIR>\"", "PYTHONPATH=\"./infra/agent/slider-agent/\"", "LC_ALL=\"en_US.UTF-8\"", "SLIDER_PASSPHRASE=\"3qheI7PwP7dfe7S0lno5B6ed0jqWCsr7n1MjkIXxT7ouyxptK3\"", "MALLOC_ARENA_MAX=\"4\"", "LANG=\"en_US.UTF-8\"" ] } } } }, "liveness" : { "allRequestsSatisfied" : true, "requestsOutstanding" : 0, "activeRequests" : 0 } } Stacktrace org.codehaus.groovy.runtime.powerassert.PowerAssertionError: assert cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1) == 0 | | | | | | 49 echo role.pending.aa.instances false { "version" : "1.0", "name" : "testaaecho", "type" : "agent", "state" : 3, "createTime" : 1450211128303, "updateTime" : 1450211150344, "originConfigurationPath" : "file:/home/jenkins/.slider/cluster/testaaecho/snapshot", "generatedConfigurationPath" : "file:/home/jenkins/.slider/cluster/testaaecho/generated", "dataPath" : "file:/home/jenkins/.slider/cluster/testaaecho/database", "options" : { "site.global.security_enabled" : "false", "internal.appdef.dir.path" : "file:/home/jenkins/.slider/cluster/testaaecho/tmp/appdef", "internal.application.home" : "/", "agent.package.root" : "/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python", "slider.cluster.directory.permissions" : "0770", "application.name" : "testaaecho", "internal.addons.dir.path" : "file:/home/jenkins/.slider/cluster/testaaecho/tmp/addons", "internal.tmp.dir" : "file:/home/jenkins/.slider/cluster/testaaecho/tmp", "internal.snapshot.conf.path" : "file:/home/jenkins/.slider/cluster/testaaecho/snapshot", "env.MALLOC_ARENA_MAX" : "4", "zookeeper.path" : "/services/slider/users/jenkins/testaaecho", "internal.container.failure.shortlife" : "60000", "internal.application.image.path" : null, "internal.generated.conf.path" : "file:/home/jenkins/.slider/cluster/testaaecho/generated", "site.fs.default.name" : "file:///", "agent.version" : "file:/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/version", "agent.conf" : "file:/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/agent.ini", "zookeeper.hosts" : "localhost", "internal.provider.name" : "agent", "internal.data.dir.path" : "file:/home/jenkins/.slider/cluster/testaaecho/database", "site.fs.defaultFS" : "file:///", "slider.data.directory.permissions" : "0770", "zookeeper.quorum" : "localhost:43906", "internal.am.tmp.dir" : "file:/home/jenkins/.slider/cluster/testaaecho/tmp/appmaster", "internal.container.failure.threshold" : "5", "application.def" : "file:/tmp/junit4103973573779864736/testpkg/appdef_1.zip" }, "info" : { "info.am.container.id" : "container_1450211118134_0001_01_000001", "info.am.rpc.port" : "1024", "info.am.attempt.id" : "appattempt_1450211118134_0001_000001", "yarn.memory" : "8192", "info.am.web.port" : "1025", "info.am.web.url" : "http://priapus.apache.org:1025/", "info.am.app.id" : "application_1450211118134_0001", "yarn.vcores" : "32", "info.am.hostname" : "priapus.apache.org", "status.application.build.info" : "Slider Core-0.90.0-incubating-SNAPSHOT Built against commit# 593460970a on Java 1.7.0_80 by jenkins", "status.hadoop.build.info" : "2.7.1", "status.hadoop.deployed.info" : "branch-2.7.1 @fc0a1a23fc1868e4d5ee7fa2b28a58a", "live.time" : "15 Dec 2015 20:25:28 GMT", "live.time.millis" : "1450211128303", "create.time" : "15 Dec 2015 20:25:28 GMT", "create.time.millis" : "1450211128303", "containers.at.am-restart" : "0", "status.time" : "15 Dec 2015 20:25:50 GMT", "status.time.millis" : "1450211150344", "info.am.agent.status.url" : "https://priapus.apache.org:37655/", "info.am.agent.status.port" : "37655", "info.am.agent.ops.url" : "https://priapus.apache.org:41282/", "info.am.agent.ops.port" : "41282" }, "statistics" : { "slider-appmaster" : { "containers.unknown.completed" : 0, "containers.start.started" : 1, "containers.live" : 2, "containers.start.failed" : 0, "containers.failed" : 0, "containers.completed" : 0, "containers.surplus" : 0 }, "echo" : { "containers.failed.node" : 0, "containers.start.started" : 0, "containers.live" : 1, "containers.start.failed" : 0, "containers.failed.preempted" : 0, "containers.active.requests" : 0, "containers.failed.recently" : 0, "containers.failed" : 0, "containers.completed" : 0, "containers.anti-affine.pending" : 49, "containers.desired" : 1, "containers.requested" : 1 } }, "instances" : { "slider-appmaster" : [ "container_1450211118134_0001_01_000001" ], "echo" : [ "container_1450211118134_0001_01_000002" ] }, "roles" : { "slider-appmaster" : { "role.failed.recently.instances" : "0", "yarn.memory" : "1024", "role.requested.instances" : "0", "role.failed.starting.instances" : "0", "role.actual.instances" : "1", "yarn.vcores" : "1", "yarn.component.instances" : "1", "role.releasing.instances" : "0", "role.failed.node.instances" : "0", "role.failed.instances" : "0", "role.failed.preempted.instances" : "0" }, "echo" : { "role.failed.recently.instances" : "0", "yarn.component.placement.policy" : "4", "role.requested.instances" : "0", "test.relax.validation" : "true", "role.failed.starting.instances" : "0", "role.failed.node.instances" : "0", "role.failed.preempted.instances" : "0", "app.name" : "Agent", "yarn.role.priority" : "1", "role.actual.instances" : "1", "agent.script" : "echo.py", "yarn.component.instances" : "1", "role.releasing.instances" : "0", "role.failed.instances" : "0", "role.pending.aa.instances" : "49" } }, "clientProperties" : { }, "status" : { "live" : { "slider-appmaster" : { "container_1450211118134_0001_01_000001" : { "name" : "container_1450211118134_0001_01_000001", "role" : "slider-appmaster", "roleId" : 0, "createTime" : 1450211128324, "startTime" : 1450211128324, "released" : false, "host" : "priapus.apache.org", "hostUrl" : "http://priapus.apache.org:1025", "state" : 3, "exitCode" : 0 } }, "echo" : { "container_1450211118134_0001_01_000002" : { "name" : "container_1450211118134_0001_01_000002", "role" : "echo", "roleId" : 1, "createTime" : 1450211131904, "startTime" : 1450211131985, "released" : false, "host" : "priapus.apache.org", "hostUrl" : "http://priapus.apache.org:40388", "state" : 3, "exitCode" : 0, "command" : "python /home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/agent/main.py --label container_1450211118134_0001_01_000002___echo --zk-quorum localhost:43906 --zk-reg-path /registry/users/jenkins/services/org-apache-slider/testaaecho > <LOG_DIR>/slider-agent.out 2>&1 ; ", "environment" : [ "LANGUAGE=\"en_US.UTF-8\"", "AGENT_WORK_ROOT=\"$PWD\"", "HADOOP_USER_NAME=\"jenkins\"", "AGENT_LOG_ROOT=\"<LOG_DIR>\"", "PYTHONPATH=\"./infra/agent/slider-agent/\"", "LC_ALL=\"en_US.UTF-8\"", "SLIDER_PASSPHRASE=\"3qheI7PwP7dfe7S0lno5B6ed0jqWCsr7n1MjkIXxT7ouyxptK3\"", "MALLOC_ARENA_MAX=\"4\"", "LANG=\"en_US.UTF-8\"" ] } } } }, "liveness" : { "allRequestsSatisfied" : true, "requestsOutstanding" : 0, "activeRequests" : 0 } } at org.codehaus.groovy.runtime.InvokerHelper.assertFailed(InvokerHelper.java:402) at org.codehaus.groovy.runtime.ScriptBytecodeAdapter.assertFailed(ScriptBytecodeAdapter.java:650) at org.apache.slider.providers.agent.TestAgentAAEcho.postLaunchActions(TestAgentAAEcho.groovy:186) at org.apache.slider.providers.agent.TestAgentAAEcho.testAgentEcho(TestAgentAAEcho.groovy:88) {code} > TestAgentAAEcho failed with #of pending instances == 49 > ------------------------------------------------------- > > Key: SLIDER-1034 > URL: https://issues.apache.org/jira/browse/SLIDER-1034 > Project: Slider > Issue Type: Bug > Components: appmaster, test > Affects Versions: Slider 0.90 > Environment: Jenkins > Reporter: Steve Loughran > Assignee: Steve Loughran > Fix For: Slider 0.91 > > > A transient jenkins failure had an as > {code} > ssert cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1) == 0 > | | | | | > | 49 echo role.pending.aa.instances > false > {code} > And yes, that's what is in the JSON > {code} > "role.pending.aa.instances" : "49" > {code} > This hints at some race condition in the test, reporting, or appstate -- This message was sent by Atlassian JIRA (v6.3.4#6332)