aurora git commit: Add @Timed annotations to cron stores.

2016-04-18 Thread zmanji 
Repository: aurora
Updated Branches:
  refs/heads/master a6197316c -> 96d5b648e


Add @Timed annotations to cron stores.

Both cron stores don't have @Timed annoations. This adds them so operators can
look at the throughput of storage.

Bugs closed: AURORA-1665

Reviewed at https://reviews.apache.org/r/46346/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/96d5b648
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/96d5b648
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/96d5b648

Branch: refs/heads/master
Commit: 96d5b648ea250c1bf91c9149a3f5a156df1f1805
Parents: a619731
Author: Zameer Manji 
Authored: Mon Apr 18 14:52:30 2016 -0700
Committer: Zameer Manji 
Committed: Mon Apr 18 14:52:30 2016 -0700

--
 .../org/apache/aurora/scheduler/storage/db/DbCronJobStore.java | 6 ++
 .../apache/aurora/scheduler/storage/mem/MemCronJobStore.java   | 6 ++
 2 files changed, 12 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/96d5b648/src/main/java/org/apache/aurora/scheduler/storage/db/DbCronJobStore.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/db/DbCronJobStore.java 
b/src/main/java/org/apache/aurora/scheduler/storage/db/DbCronJobStore.java
index 6a9c73e..e48a982 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbCronJobStore.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/db/DbCronJobStore.java
@@ -18,6 +18,7 @@ import javax.inject.Inject;
 import com.google.common.base.Optional;
 import com.google.common.collect.FluentIterable;
 
+import org.apache.aurora.common.inject.TimedInterceptor.Timed;
 import org.apache.aurora.scheduler.storage.CronJobStore;
 import org.apache.aurora.scheduler.storage.db.views.DbJobConfiguration;
 import org.apache.aurora.scheduler.storage.entities.IJobConfiguration;
@@ -44,6 +45,7 @@ class DbCronJobStore implements CronJobStore.Mutable {
 this.taskConfigManager = requireNonNull(taskConfigManager);
   }
 
+  @Timed("db_storage_cron_save_accepted_job")
   @Override
   public void saveAcceptedJob(IJobConfiguration jobConfig) {
 requireNonNull(jobConfig);
@@ -51,17 +53,20 @@ class DbCronJobStore implements CronJobStore.Mutable {
 cronJobMapper.merge(jobConfig, 
taskConfigManager.insert(jobConfig.getTaskConfig()));
   }
 
+  @Timed("db_storage_cron_remove_job")
   @Override
   public void removeJob(IJobKey jobKey) {
 requireNonNull(jobKey);
 cronJobMapper.delete(jobKey);
   }
 
+  @Timed("db_storage_cron_delete_jobs")
   @Override
   public void deleteJobs() {
 cronJobMapper.truncate();
   }
 
+  @Timed("db_storage_cron_fetch_jobs")
   @Override
   public Iterable fetchJobs() {
 return FluentIterable.from(cronJobMapper.selectAll())
@@ -69,6 +74,7 @@ class DbCronJobStore implements CronJobStore.Mutable {
 .toList();
   }
 
+  @Timed("db_storage_cron_fetch_job")
   @Override
   public Optional fetchJob(IJobKey jobKey) {
 requireNonNull(jobKey);

http://git-wip-us.apache.org/repos/asf/aurora/blob/96d5b648/src/main/java/org/apache/aurora/scheduler/storage/mem/MemCronJobStore.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemCronJobStore.java 
b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemCronJobStore.java
index 365494c..50cfb35 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemCronJobStore.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemCronJobStore.java
@@ -19,6 +19,7 @@ import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 
+import org.apache.aurora.common.inject.TimedInterceptor.Timed;
 import org.apache.aurora.scheduler.base.JobKeys;
 import org.apache.aurora.scheduler.storage.CronJobStore;
 import org.apache.aurora.scheduler.storage.entities.IJobConfiguration;
@@ -30,27 +31,32 @@ import org.apache.aurora.scheduler.storage.entities.IJobKey;
 class MemCronJobStore implements CronJobStore.Mutable {
   private final Map jobs = Maps.newConcurrentMap();
 
+  @Timed("mem_storage_cron_save_accepted_job")
   @Override
   public void saveAcceptedJob(IJobConfiguration jobConfig) {
 IJobKey key = JobKeys.assertValid(jobConfig.getKey());
 jobs.put(key, jobConfig);
   }
 
+  @Timed("mem_storage_cron_remove_job")
   @Override
   public void removeJob(IJobKey jobKey) {
 jobs.remove(jobKey);
   }
 
+  @Timed("mem_storage_cron_delete_jobs")
   @Override
   public void deleteJobs() {
 jobs.clear();
   }
 
+  @Timed("mem_storage_cron_fetch_jobs")
   @Override
   public Iterable fetchJobs() {

aurora git commit: Adding a flag to control whether the executor runs health checks as the job role's user

2016-04-18 Thread wfarner 
Repository: aurora
Updated Branches:
  refs/heads/master f4a08459c -> a6197316c


Adding a flag to control whether the executor runs health checks as the job 
role's user

Bugs closed: AURORA-1666

Reviewed at https://reviews.apache.org/r/46290/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/a6197316
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/a6197316
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/a6197316

Branch: refs/heads/master
Commit: a6197316c20f4b759453a6f2015fef7a1956de5f
Parents: f4a0845
Author: Dmitriy Shirchenko 
Authored: Mon Apr 18 12:23:11 2016 -0700
Committer: Bill Farner 
Committed: Mon Apr 18 12:23:11 2016 -0700

--
 RELEASE-NOTES.md|  3 ++
 .../executor/bin/thermos_executor_main.py   | 11 +-
 .../aurora/executor/common/health_checker.py| 18 ++
 .../executor/common/test_health_checker.py  | 36 
 4 files changed, 61 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/a6197316/RELEASE-NOTES.md
--
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 99d261b..2068d9c 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -5,6 +5,9 @@
 
 - Added a new optional [Apache Curator](https://curator.apache.org/) backend 
for performing
   scheduler leader election. You can enable this with the new 
`-zk_use_curator` scheduler argument.
+- Adding --setuid-health-checks flag to control whether the executor runs 
health checks as the job's
+  role's user.
+
 
 0.13.0
 --

http://git-wip-us.apache.org/repos/asf/aurora/blob/a6197316/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
--
diff --git 
a/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py 
b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
index 40a0cd6..0b3c38c 100644
--- a/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
+++ b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
@@ -145,6 +145,15 @@ app.add_option(
 
 
 app.add_option(
+'--setuid-health-checks',
+dest='setuid_health_checks',
+action="store_true",
+help='If False, does not run shell health checks as job\'s role\'s user',
+default=True
+)
+
+
+app.add_option(
  "--preserve_env",
  dest="preserve_env",
  default=False,
@@ -181,7 +190,7 @@ def initialize(options):
 
   # status providers:
   status_providers = [
-  HealthCheckerProvider(),
+  HealthCheckerProvider(setuid_health_checks=options.setuid_health_checks),
   ResourceManagerProvider(checkpoint_root=checkpoint_root)
   ]
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/a6197316/src/main/python/apache/aurora/executor/common/health_checker.py
--
diff --git a/src/main/python/apache/aurora/executor/common/health_checker.py 
b/src/main/python/apache/aurora/executor/common/health_checker.py
index d8675be..88b629e 100644
--- a/src/main/python/apache/aurora/executor/common/health_checker.py
+++ b/src/main/python/apache/aurora/executor/common/health_checker.py
@@ -208,6 +208,9 @@ class HealthChecker(StatusChecker):
 
 class HealthCheckerProvider(StatusCheckerProvider):
 
+  def __init__(self, setuid_health_checks=True):
+self.setuid_health_checks = setuid_health_checks
+
   @staticmethod
   def interpolate_cmd(task, cmd):
 """
@@ -244,14 +247,17 @@ class HealthCheckerProvider(StatusCheckerProvider):
 task=assigned_task,
 cmd=shell_command
   )
-
-  pw_entry = pwd.getpwnam(assigned_task.task.job.role)
-  def demote_to_user():
-os.setgid(pw_entry.pw_gid)
-os.setuid(pw_entry.pw_uid)
+  # If we do not want user which is job's role to execute the health shell 
check.
+  if self.setuid_health_checks:
+pw_entry = pwd.getpwnam(assigned_task.task.job.role)
+def demote_to_job_role_user():
+  os.setgid(pw_entry.pw_gid)
+  os.setuid(pw_entry.pw_uid)
+  else:
+demote_to_job_role_user = None
 
   shell_signaler = ShellHealthCheck(cmd=interpolated_command,
-preexec_fn=demote_to_user,
+preexec_fn=demote_to_job_role_user,
 timeout_secs=timeout_secs)
   a_health_checker = lambda: shell_signaler()
 else:

http://git-wip-us.apache.org/repos/asf/aurora/blob/a6197316/src/test/python/apache/aurora/executor/common/test_health_checker.py
--
diff --git 
a/src/test/python/apache/aurora/executor/common/test_health_checker.py

aurora git commit: Plumb Curator discovery as an option.

2016-04-18 Thread jsirois 
Repository: aurora
Updated Branches:
  refs/heads/master ae051f3b9 -> f4a08459c


Plumb Curator discovery as an option.

This Adds a Guice module for the Curator discovery implementations and
re-works the `ServiceDiscoveryModule` to optionally bind it when the new
`-zk_use_curator` flag is set.

Bugs closed: AURORA-1468, AURORA-1669

Reviewed at https://reviews.apache.org/r/46286/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/f4a08459
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/f4a08459
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/f4a08459

Branch: refs/heads/master
Commit: f4a08459c2e03ba4ab51b1b90812812f076a5472
Parents: ae051f3
Author: John Sirois 
Authored: Mon Apr 18 12:26:39 2016 -0600
Committer: John Sirois 
Committed: Mon Apr 18 12:26:39 2016 -0600

--
 RELEASE-NOTES.md|  10 +-
 config/legacy_untested_classes.txt  |   5 +-
 examples/vagrant/upstart/aurora-scheduler.conf  |   1 +
 .../aurora/scheduler/app/SchedulerMain.java |   6 +-
 .../discovery/CommonsServerGroupMonitor.java|  59 ---
 .../CommonsServiceDiscoveryModule.java  | 102 +++
 .../discovery/CommonsServiceGroupMonitor.java   |  59 +++
 .../CuratorServiceDiscoveryModule.java  | 172 +++
 .../discovery/FlaggedZooKeeperConfig.java   |   6 +
 .../discovery/ServiceDiscoveryBindings.java |  59 +++
 .../discovery/ServiceDiscoveryModule.java   | 150 ++--
 .../discovery/ZooKeeperClientModule.java| 144 
 .../scheduler/discovery/ZooKeeperConfig.java|  44 -
 .../log/mesos/MesosLogStreamModule.java |  14 +-
 .../aurora/scheduler/app/SchedulerIT.java   |  13 +-
 .../discovery/AbstractDiscoveryModuleTest.java  |  77 +
 .../discovery/CommonsDiscoveryModuleTest.java   |  29 
 .../CommonsServerGroupMonitorTest.java  | 137 ---
 .../CommonsServiceGroupMonitorTest.java | 137 +++
 .../discovery/CuratorDiscoveryModuleTest.java   |  67 
 .../discovery/ZooKeeperConfigTest.java  |  23 +--
 21 files changed, 886 insertions(+), 428 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/f4a08459/RELEASE-NOTES.md
--
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index a0536ec..99d261b 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -1,4 +1,12 @@
-0.13.0 (Not yet released)
+0.14.0 (Not yet released)
+--
+
+### New/updated:
+
+- Added a new optional [Apache Curator](https://curator.apache.org/) backend 
for performing
+  scheduler leader election. You can enable this with the new 
`-zk_use_curator` scheduler argument.
+
+0.13.0
 --
 
 ### New/updated:

http://git-wip-us.apache.org/repos/asf/aurora/blob/f4a08459/config/legacy_untested_classes.txt
--
diff --git a/config/legacy_untested_classes.txt 
b/config/legacy_untested_classes.txt
index 30875da..10e8b77 100644
--- a/config/legacy_untested_classes.txt
+++ b/config/legacy_untested_classes.txt
@@ -16,9 +16,8 @@ 
org/apache/aurora/scheduler/configuration/executor/ExecutorModule$1
 org/apache/aurora/scheduler/cron/quartz/CronSchedulerImpl
 org/apache/aurora/scheduler/cron/quartz/CronSchedulerImpl$1
 org/apache/aurora/scheduler/discovery/FlaggedZooKeeperConfig
-org/apache/aurora/scheduler/discovery/ZooKeeperClientModule$1
-org/apache/aurora/scheduler/discovery/ZooKeeperClientModule$LocalClientProvider
-org/apache/aurora/scheduler/discovery/ZooKeeperClientModule$TestServerService
+org/apache/aurora/scheduler/discovery/ServiceDiscoveryModule$TestServerService
+org/apache/aurora/scheduler/discovery/ServiceDiscoveryModule$LocalZooKeeperClusterProvider
 org/apache/aurora/scheduler/http/AbortCallback
 org/apache/aurora/scheduler/http/api/security/Kerberos5Realm
 org/apache/aurora/scheduler/http/JerseyTemplateServlet

http://git-wip-us.apache.org/repos/asf/aurora/blob/f4a08459/examples/vagrant/upstart/aurora-scheduler.conf
--
diff --git a/examples/vagrant/upstart/aurora-scheduler.conf 
b/examples/vagrant/upstart/aurora-scheduler.conf
index b9732d2..084016a 100644
--- a/examples/vagrant/upstart/aurora-scheduler.conf
+++ b/examples/vagrant/upstart/aurora-scheduler.conf
@@ -33,6 +33,7 @@ exec bin/aurora-scheduler \
   -hostname=aurora.local \
   -http_port=8081 \
   -native_log_quorum_size=1 \
+  -zk_use_curator \
   -zk_endpoints=localhost:2181 \
   -mesos_master_address=zk://localhost:2181/mesos \
   -serverset_path=/aurora/scheduler \

aurora git commit: Make MyBatis connection pool size configurable.

2016-04-18 Thread zmanji 
Repository: aurora
Updated Branches:
  refs/heads/master e543415d2 -> 96c990875


Make MyBatis connection pool size configurable.

This adds two arguments which enables operators to configure the internal
MyBatis connection pool size. Increasing the size will allow for greater
concurrency in larger clusters.

Reviewed at https://reviews.apache.org/r/46289/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/96c99087
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/96c99087
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/96c99087

Branch: refs/heads/master
Commit: 96c99087503599e2c686e83027be411b26ded8a5
Parents: e543415
Author: Zameer Manji 
Authored: Mon Apr 18 11:04:03 2016 -0700
Committer: Zameer Manji 
Committed: Mon Apr 18 11:04:03 2016 -0700

--
 .../aurora/scheduler/storage/db/DbModule.java   | 22 
 1 file changed, 22 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/96c99087/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java 
b/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java
index 743993c..e7287ce 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java
@@ -34,6 +34,7 @@ import com.google.inject.util.Modules;
 
 import org.apache.aurora.common.args.Arg;
 import org.apache.aurora.common.args.CmdLine;
+import org.apache.aurora.common.args.constraints.Positive;
 import org.apache.aurora.common.inject.Bindings.KeyFactory;
 import org.apache.aurora.common.quantity.Amount;
 import org.apache.aurora.common.quantity.Time;
@@ -87,6 +88,17 @@ public final class DbModule extends PrivateModule {
   private static final Arg> H2_LOCK_TIMEOUT =
   Arg.create(Amount.of(1L, Time.MINUTES));
 
+  // Flags to configure the PooledDataSource from mybatis.
+  @Positive
+  @CmdLine(name = "db_max_active_connection_count",
+  help = "Max number of connections to use with database via MyBatis")
+  private static final Arg MYBATIS_MAX_ACTIVE_CONNECTION_COUNT = 
Arg.create();
+
+  @Positive
+  @CmdLine(name = "db_max_idle_connection_count",
+  help = "Max number of idle connections to the database via MyBatis")
+  private static final Arg MYBATIS_MAX_IDLE_CONNECTION_COUNT = 
Arg.create();
+
   private static final Set MAPPER_CLASSES = 
ImmutableSet.builder()
   .add(AttributeMapper.class)
   .add(CronJobMapper.class)
@@ -260,6 +272,16 @@ public final class DbModule extends PrivateModule {
 bindProperties(binder(), ImmutableMap.of("mybatis.pooled.pingEnabled", 
"true"));
 bindProperties(binder(), ImmutableMap.of("mybatis.pooled.pingQuery", 
"SELECT 1;"));
 
+if (MYBATIS_MAX_ACTIVE_CONNECTION_COUNT.hasAppliedValue()) {
+  String val = MYBATIS_MAX_ACTIVE_CONNECTION_COUNT.get().toString();
+  bindProperties(binder(), 
ImmutableMap.of("mybatis.pooled.maximumActiveConnections", val));
+}
+
+if (MYBATIS_MAX_IDLE_CONNECTION_COUNT.hasAppliedValue()) {
+  String val = MYBATIS_MAX_IDLE_CONNECTION_COUNT.get().toString();
+  bindProperties(binder(), 
ImmutableMap.of("mybatis.pooled.maximumIdleConnections", val));
+}
+
 // Exposed for unit tests.
 bind(TaskConfigManager.class);
 expose(TaskConfigManager.class);

aurora git commit: Add an explicit box version to Vagrantfile.

2016-04-18 Thread jcohen 
Repository: aurora
Updated Branches:
  refs/heads/master 81f52e4ae -> e543415d2


Add an explicit box version to Vagrantfile.

Bugs closed: AURORA-1667

Reviewed at https://reviews.apache.org/r/46335/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/e543415d
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/e543415d
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/e543415d

Branch: refs/heads/master
Commit: e543415d276a6ac769545ab4ce4d88ddfbe9a746
Parents: 81f52e4
Author: Joshua Cohen 
Authored: Mon Apr 18 10:41:02 2016 -0500
Committer: Joshua Cohen 
Committed: Mon Apr 18 10:41:02 2016 -0500

--
 Vagrantfile | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/e543415d/Vagrantfile
--
diff --git a/Vagrantfile b/Vagrantfile
index 4ebf7d9..606dfa4 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -25,6 +25,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.hostname = "aurora.local"
   # See build-support/packer/README.md for instructions on updating this box.
   config.vm.box = "apache-aurora/dev-environment"
+  config.vm.box_version = "0.0.4"
 
   config.vm.define "devcluster" do |dev|
 dev.vm.network :private_network, ip: "192.168.33.7"