[ 
https://issues.apache.org/jira/browse/SLIDER-1034?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15062677#comment-15062677
 ] 

Steve Loughran commented on SLIDER-1034:
----------------------------------------

{code}
assert cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1) == 0
       |  |             |                  |                              |
       |  49            echo               role.pending.aa.instances      false
       {
         "version" : "1.0",
         "name" : "testaaecho",
         "type" : "agent",
         "state" : 3,
         "createTime" : 1450211128303,
         "updateTime" : 1450211150344,
         "originConfigurationPath" : 
"file:/home/jenkins/.slider/cluster/testaaecho/snapshot",
         "generatedConfigurationPath" : 
"file:/home/jenkins/.slider/cluster/testaaecho/generated",
         "dataPath" : "file:/home/jenkins/.slider/cluster/testaaecho/database",
         "options" : {
           "site.global.security_enabled" : "false",
           "internal.appdef.dir.path" : 
"file:/home/jenkins/.slider/cluster/testaaecho/tmp/appdef",
           "internal.application.home" : "/",
           "agent.package.root" : 
"/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python",
           "slider.cluster.directory.permissions" : "0770",
           "application.name" : "testaaecho",
           "internal.addons.dir.path" : 
"file:/home/jenkins/.slider/cluster/testaaecho/tmp/addons",
           "internal.tmp.dir" : 
"file:/home/jenkins/.slider/cluster/testaaecho/tmp",
           "internal.snapshot.conf.path" : 
"file:/home/jenkins/.slider/cluster/testaaecho/snapshot",
           "env.MALLOC_ARENA_MAX" : "4",
           "zookeeper.path" : "/services/slider/users/jenkins/testaaecho",
           "internal.container.failure.shortlife" : "60000",
           "internal.application.image.path" : null,
           "internal.generated.conf.path" : 
"file:/home/jenkins/.slider/cluster/testaaecho/generated",
           "site.fs.default.name" : "file:///",
           "agent.version" : 
"file:/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/version",
           "agent.conf" : 
"file:/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/agent.ini",
           "zookeeper.hosts" : "localhost",
           "internal.provider.name" : "agent",
           "internal.data.dir.path" : 
"file:/home/jenkins/.slider/cluster/testaaecho/database",
           "site.fs.defaultFS" : "file:///",
           "slider.data.directory.permissions" : "0770",
           "zookeeper.quorum" : "localhost:43906",
           "internal.am.tmp.dir" : 
"file:/home/jenkins/.slider/cluster/testaaecho/tmp/appmaster",
           "internal.container.failure.threshold" : "5",
           "application.def" : 
"file:/tmp/junit4103973573779864736/testpkg/appdef_1.zip"
         },
         "info" : {
           "info.am.container.id" : "container_1450211118134_0001_01_000001",
           "info.am.rpc.port" : "1024",
           "info.am.attempt.id" : "appattempt_1450211118134_0001_000001",
           "yarn.memory" : "8192",
           "info.am.web.port" : "1025",
           "info.am.web.url" : "http://priapus.apache.org:1025/";,
           "info.am.app.id" : "application_1450211118134_0001",
           "yarn.vcores" : "32",
           "info.am.hostname" : "priapus.apache.org",
           "status.application.build.info" : "Slider 
Core-0.90.0-incubating-SNAPSHOT Built against commit# 593460970a on Java 
1.7.0_80 by jenkins",
           "status.hadoop.build.info" : "2.7.1",
           "status.hadoop.deployed.info" : "branch-2.7.1 
@fc0a1a23fc1868e4d5ee7fa2b28a58a",
           "live.time" : "15 Dec 2015 20:25:28 GMT",
           "live.time.millis" : "1450211128303",
           "create.time" : "15 Dec 2015 20:25:28 GMT",
           "create.time.millis" : "1450211128303",
           "containers.at.am-restart" : "0",
           "status.time" : "15 Dec 2015 20:25:50 GMT",
           "status.time.millis" : "1450211150344",
           "info.am.agent.status.url" : "https://priapus.apache.org:37655/";,
           "info.am.agent.status.port" : "37655",
           "info.am.agent.ops.url" : "https://priapus.apache.org:41282/";,
           "info.am.agent.ops.port" : "41282"
         },
         "statistics" : {
           "slider-appmaster" : {
             "containers.unknown.completed" : 0,
             "containers.start.started" : 1,
             "containers.live" : 2,
             "containers.start.failed" : 0,
             "containers.failed" : 0,
             "containers.completed" : 0,
             "containers.surplus" : 0
           },
           "echo" : {
             "containers.failed.node" : 0,
             "containers.start.started" : 0,
             "containers.live" : 1,
             "containers.start.failed" : 0,
             "containers.failed.preempted" : 0,
             "containers.active.requests" : 0,
             "containers.failed.recently" : 0,
             "containers.failed" : 0,
             "containers.completed" : 0,
             "containers.anti-affine.pending" : 49,
             "containers.desired" : 1,
             "containers.requested" : 1
           }
         },
         "instances" : {
           "slider-appmaster" : [ "container_1450211118134_0001_01_000001" ],
           "echo" : [ "container_1450211118134_0001_01_000002" ]
         },
         "roles" : {
           "slider-appmaster" : {
             "role.failed.recently.instances" : "0",
             "yarn.memory" : "1024",
             "role.requested.instances" : "0",
             "role.failed.starting.instances" : "0",
             "role.actual.instances" : "1",
             "yarn.vcores" : "1",
             "yarn.component.instances" : "1",
             "role.releasing.instances" : "0",
             "role.failed.node.instances" : "0",
             "role.failed.instances" : "0",
             "role.failed.preempted.instances" : "0"
           },
           "echo" : {
             "role.failed.recently.instances" : "0",
             "yarn.component.placement.policy" : "4",
             "role.requested.instances" : "0",
             "test.relax.validation" : "true",
             "role.failed.starting.instances" : "0",
             "role.failed.node.instances" : "0",
             "role.failed.preempted.instances" : "0",
             "app.name" : "Agent",
             "yarn.role.priority" : "1",
             "role.actual.instances" : "1",
             "agent.script" : "echo.py",
             "yarn.component.instances" : "1",
             "role.releasing.instances" : "0",
             "role.failed.instances" : "0",
             "role.pending.aa.instances" : "49"
           }
         },
         "clientProperties" : { },
         "status" : {
           "live" : {
             "slider-appmaster" : {
               "container_1450211118134_0001_01_000001" : {
                 "name" : "container_1450211118134_0001_01_000001",
                 "role" : "slider-appmaster",
                 "roleId" : 0,
                 "createTime" : 1450211128324,
                 "startTime" : 1450211128324,
                 "released" : false,
                 "host" : "priapus.apache.org",
                 "hostUrl" : "http://priapus.apache.org:1025";,
                 "state" : 3,
                 "exitCode" : 0
               }
             },
             "echo" : {
               "container_1450211118134_0001_01_000002" : {
                 "name" : "container_1450211118134_0001_01_000002",
                 "role" : "echo",
                 "roleId" : 1,
                 "createTime" : 1450211131904,
                 "startTime" : 1450211131985,
                 "released" : false,
                 "host" : "priapus.apache.org",
                 "hostUrl" : "http://priapus.apache.org:40388";,
                 "state" : 3,
                 "exitCode" : 0,
                 "command" : "python 
/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/agent/main.py
 --label container_1450211118134_0001_01_000002___echo --zk-quorum 
localhost:43906 --zk-reg-path 
/registry/users/jenkins/services/org-apache-slider/testaaecho > 
<LOG_DIR>/slider-agent.out 2>&1 ; ",
                 "environment" : [ "LANGUAGE=\"en_US.UTF-8\"", 
"AGENT_WORK_ROOT=\"$PWD\"", "HADOOP_USER_NAME=\"jenkins\"", 
"AGENT_LOG_ROOT=\"<LOG_DIR>\"", "PYTHONPATH=\"./infra/agent/slider-agent/\"", 
"LC_ALL=\"en_US.UTF-8\"", 
"SLIDER_PASSPHRASE=\"3qheI7PwP7dfe7S0lno5B6ed0jqWCsr7n1MjkIXxT7ouyxptK3\"", 
"MALLOC_ARENA_MAX=\"4\"", "LANG=\"en_US.UTF-8\"" ]
               }
             }
           }
         },
         "liveness" : {
           "allRequestsSatisfied" : true,
           "requestsOutstanding" : 0,
           "activeRequests" : 0
         }
       }
Stacktrace

org.codehaus.groovy.runtime.powerassert.PowerAssertionError: assert 
cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1) == 0
       |  |             |                  |                              |
       |  49            echo               role.pending.aa.instances      false
       {
         "version" : "1.0",
         "name" : "testaaecho",
         "type" : "agent",
         "state" : 3,
         "createTime" : 1450211128303,
         "updateTime" : 1450211150344,
         "originConfigurationPath" : 
"file:/home/jenkins/.slider/cluster/testaaecho/snapshot",
         "generatedConfigurationPath" : 
"file:/home/jenkins/.slider/cluster/testaaecho/generated",
         "dataPath" : "file:/home/jenkins/.slider/cluster/testaaecho/database",
         "options" : {
           "site.global.security_enabled" : "false",
           "internal.appdef.dir.path" : 
"file:/home/jenkins/.slider/cluster/testaaecho/tmp/appdef",
           "internal.application.home" : "/",
           "agent.package.root" : 
"/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python",
           "slider.cluster.directory.permissions" : "0770",
           "application.name" : "testaaecho",
           "internal.addons.dir.path" : 
"file:/home/jenkins/.slider/cluster/testaaecho/tmp/addons",
           "internal.tmp.dir" : 
"file:/home/jenkins/.slider/cluster/testaaecho/tmp",
           "internal.snapshot.conf.path" : 
"file:/home/jenkins/.slider/cluster/testaaecho/snapshot",
           "env.MALLOC_ARENA_MAX" : "4",
           "zookeeper.path" : "/services/slider/users/jenkins/testaaecho",
           "internal.container.failure.shortlife" : "60000",
           "internal.application.image.path" : null,
           "internal.generated.conf.path" : 
"file:/home/jenkins/.slider/cluster/testaaecho/generated",
           "site.fs.default.name" : "file:///",
           "agent.version" : 
"file:/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/version",
           "agent.conf" : 
"file:/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/agent.ini",
           "zookeeper.hosts" : "localhost",
           "internal.provider.name" : "agent",
           "internal.data.dir.path" : 
"file:/home/jenkins/.slider/cluster/testaaecho/database",
           "site.fs.defaultFS" : "file:///",
           "slider.data.directory.permissions" : "0770",
           "zookeeper.quorum" : "localhost:43906",
           "internal.am.tmp.dir" : 
"file:/home/jenkins/.slider/cluster/testaaecho/tmp/appmaster",
           "internal.container.failure.threshold" : "5",
           "application.def" : 
"file:/tmp/junit4103973573779864736/testpkg/appdef_1.zip"
         },
         "info" : {
           "info.am.container.id" : "container_1450211118134_0001_01_000001",
           "info.am.rpc.port" : "1024",
           "info.am.attempt.id" : "appattempt_1450211118134_0001_000001",
           "yarn.memory" : "8192",
           "info.am.web.port" : "1025",
           "info.am.web.url" : "http://priapus.apache.org:1025/";,
           "info.am.app.id" : "application_1450211118134_0001",
           "yarn.vcores" : "32",
           "info.am.hostname" : "priapus.apache.org",
           "status.application.build.info" : "Slider 
Core-0.90.0-incubating-SNAPSHOT Built against commit# 593460970a on Java 
1.7.0_80 by jenkins",
           "status.hadoop.build.info" : "2.7.1",
           "status.hadoop.deployed.info" : "branch-2.7.1 
@fc0a1a23fc1868e4d5ee7fa2b28a58a",
           "live.time" : "15 Dec 2015 20:25:28 GMT",
           "live.time.millis" : "1450211128303",
           "create.time" : "15 Dec 2015 20:25:28 GMT",
           "create.time.millis" : "1450211128303",
           "containers.at.am-restart" : "0",
           "status.time" : "15 Dec 2015 20:25:50 GMT",
           "status.time.millis" : "1450211150344",
           "info.am.agent.status.url" : "https://priapus.apache.org:37655/";,
           "info.am.agent.status.port" : "37655",
           "info.am.agent.ops.url" : "https://priapus.apache.org:41282/";,
           "info.am.agent.ops.port" : "41282"
         },
         "statistics" : {
           "slider-appmaster" : {
             "containers.unknown.completed" : 0,
             "containers.start.started" : 1,
             "containers.live" : 2,
             "containers.start.failed" : 0,
             "containers.failed" : 0,
             "containers.completed" : 0,
             "containers.surplus" : 0
           },
           "echo" : {
             "containers.failed.node" : 0,
             "containers.start.started" : 0,
             "containers.live" : 1,
             "containers.start.failed" : 0,
             "containers.failed.preempted" : 0,
             "containers.active.requests" : 0,
             "containers.failed.recently" : 0,
             "containers.failed" : 0,
             "containers.completed" : 0,
             "containers.anti-affine.pending" : 49,
             "containers.desired" : 1,
             "containers.requested" : 1
           }
         },
         "instances" : {
           "slider-appmaster" : [ "container_1450211118134_0001_01_000001" ],
           "echo" : [ "container_1450211118134_0001_01_000002" ]
         },
         "roles" : {
           "slider-appmaster" : {
             "role.failed.recently.instances" : "0",
             "yarn.memory" : "1024",
             "role.requested.instances" : "0",
             "role.failed.starting.instances" : "0",
             "role.actual.instances" : "1",
             "yarn.vcores" : "1",
             "yarn.component.instances" : "1",
             "role.releasing.instances" : "0",
             "role.failed.node.instances" : "0",
             "role.failed.instances" : "0",
             "role.failed.preempted.instances" : "0"
           },
           "echo" : {
             "role.failed.recently.instances" : "0",
             "yarn.component.placement.policy" : "4",
             "role.requested.instances" : "0",
             "test.relax.validation" : "true",
             "role.failed.starting.instances" : "0",
             "role.failed.node.instances" : "0",
             "role.failed.preempted.instances" : "0",
             "app.name" : "Agent",
             "yarn.role.priority" : "1",
             "role.actual.instances" : "1",
             "agent.script" : "echo.py",
             "yarn.component.instances" : "1",
             "role.releasing.instances" : "0",
             "role.failed.instances" : "0",
             "role.pending.aa.instances" : "49"
           }
         },
         "clientProperties" : { },
         "status" : {
           "live" : {
             "slider-appmaster" : {
               "container_1450211118134_0001_01_000001" : {
                 "name" : "container_1450211118134_0001_01_000001",
                 "role" : "slider-appmaster",
                 "roleId" : 0,
                 "createTime" : 1450211128324,
                 "startTime" : 1450211128324,
                 "released" : false,
                 "host" : "priapus.apache.org",
                 "hostUrl" : "http://priapus.apache.org:1025";,
                 "state" : 3,
                 "exitCode" : 0
               }
             },
             "echo" : {
               "container_1450211118134_0001_01_000002" : {
                 "name" : "container_1450211118134_0001_01_000002",
                 "role" : "echo",
                 "roleId" : 1,
                 "createTime" : 1450211131904,
                 "startTime" : 1450211131985,
                 "released" : false,
                 "host" : "priapus.apache.org",
                 "hostUrl" : "http://priapus.apache.org:40388";,
                 "state" : 3,
                 "exitCode" : 0,
                 "command" : "python 
/home/jenkins/jenkins-slave/workspace/Slider-develop/slider-core/./src/test/python/agent/main.py
 --label container_1450211118134_0001_01_000002___echo --zk-quorum 
localhost:43906 --zk-reg-path 
/registry/users/jenkins/services/org-apache-slider/testaaecho > 
<LOG_DIR>/slider-agent.out 2>&1 ; ",
                 "environment" : [ "LANGUAGE=\"en_US.UTF-8\"", 
"AGENT_WORK_ROOT=\"$PWD\"", "HADOOP_USER_NAME=\"jenkins\"", 
"AGENT_LOG_ROOT=\"<LOG_DIR>\"", "PYTHONPATH=\"./infra/agent/slider-agent/\"", 
"LC_ALL=\"en_US.UTF-8\"", 
"SLIDER_PASSPHRASE=\"3qheI7PwP7dfe7S0lno5B6ed0jqWCsr7n1MjkIXxT7ouyxptK3\"", 
"MALLOC_ARENA_MAX=\"4\"", "LANG=\"en_US.UTF-8\"" ]
               }
             }
           }
         },
         "liveness" : {
           "allRequestsSatisfied" : true,
           "requestsOutstanding" : 0,
           "activeRequests" : 0
         }
       }
        at 
org.codehaus.groovy.runtime.InvokerHelper.assertFailed(InvokerHelper.java:402)
        at 
org.codehaus.groovy.runtime.ScriptBytecodeAdapter.assertFailed(ScriptBytecodeAdapter.java:650)
        at 
org.apache.slider.providers.agent.TestAgentAAEcho.postLaunchActions(TestAgentAAEcho.groovy:186)
        at 
org.apache.slider.providers.agent.TestAgentAAEcho.testAgentEcho(TestAgentAAEcho.groovy:88)
{code}

> TestAgentAAEcho failed with #of pending instances == 49
> -------------------------------------------------------
>
>                 Key: SLIDER-1034
>                 URL: https://issues.apache.org/jira/browse/SLIDER-1034
>             Project: Slider
>          Issue Type: Bug
>          Components: appmaster, test
>    Affects Versions: Slider 0.90
>         Environment: Jenkins
>            Reporter: Steve Loughran
>            Assignee: Steve Loughran
>             Fix For: Slider 0.91
>
>
> A transient jenkins failure had an as
> {code}
> ssert cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1) == 0
>        |  |             |                  |                              |
>        |  49            echo               role.pending.aa.instances      
> false
> {code}
> And yes, that's what is in the JSON
> {code}
> "role.pending.aa.instances" : "49"
> {code}
> This hints at some race condition in the test, reporting, or appstate



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Reply via email to