coren has submitted this change and it was merged.
Change subject: inserted a package structure for jobutils
......................................................................
inserted a package structure for jobutils
Change-Id: I4c92ba0cede54616caf037a99ca79247b2c189cf
---
A packages/jobutils/DEBIAN/control
A packages/jobutils/usr/local/bin/jstart
A packages/jobutils/usr/local/bin/jstop
A packages/jobutils/usr/local/bin/jsub
A packages/jobutils/usr/share/man/man1/jstart.1
A packages/jobutils/usr/share/man/man1/jstop.1
A packages/jobutils/usr/share/man/man1/jsub.1
7 files changed, 350 insertions(+), 0 deletions(-)
Approvals:
coren: Verified; Looks good to me, approved
diff --git a/packages/jobutils/DEBIAN/control b/packages/jobutils/DEBIAN/control
new file mode 100644
index 0000000..c2ac5e9
--- /dev/null
+++ b/packages/jobutils/DEBIAN/control
@@ -0,0 +1,11 @@
+Package: Jobutils
+Version: 1.0.0
+Section: universe/utils
+Priority: optional
+Architecture: all
+Essential: no
+Depends: perl, gridengine-common
+Installed-Size: 9
+Maintainer: Marc-André Pelletier <m...@wikimedia.org>, Petr Bena
<benap...@gmail.com>
+Description: Set of utilities to use on wikimedia bots and tools cluster
+ This package will install jstart (jsub) and jstop, which can be used with GE
diff --git a/packages/jobutils/usr/local/bin/jstart
b/packages/jobutils/usr/local/bin/jstart
new file mode 100755
index 0000000..756ee1d
--- /dev/null
+++ b/packages/jobutils/usr/local/bin/jstart
@@ -0,0 +1,126 @@
+#! /usr/bin/perl
+
+my %qsubargs = (
+ '-a' => 1, '-b' => 1, '-cwd' => 0, '-e' => 1, '-hard' => 0, '-i' => 1,
'-j' => 1,
+ '-l' => 1, '-now' => 1, '-N' => 1, '-o' => 1, '-p' => 1, '-q' => 1,
'-soft' => 0,
+ '-sync' => 1, '-v' => 1, '-wd' => 1,
+);
+my %qsargs;
+my @args;
+my $memory = '256m';
+my $once = 0;
+my $stderr = 0;
+my $continuous = 0;
+my $script = $0;
+my $queue = 'task';
+$script = $1 if $script =~ m{/([^/]+)$};
+
+$continuous = 1 if $script eq 'jstart';
+
+while($#ARGV > 0) {
+ if(defined $qsubargs{$ARGV[0]}) {
+ $arg = shift;
+ $qsargs{$arg} = $qsubargs{$arg}? $ARGV[0]: '';
+ if($arg eq '-N' or $arg eq '-q') {
+ shift if $qsubargs{$arg};
+ next;
+ }
+ push @args, $arg;
+ push @args, shift if $qsubargs{$arg};
+ } elsif ($ARGV[0] eq '-mem') {
+ shift;
+ $memory = shift;
+ die "$script: memory value must be an integer followed by 'k', 'm' or
'g'\n" unless $memory =~ m/^([1-9][0-9]*[mMgGkK])$/;
+ } elsif ($ARGV[0] eq '-once') {
+ $once = 1;
+ shift;
+ } elsif ($ARGV[0] eq '-stderr') {
+ $stderr = 1;
+ shift;
+ } elsif ($ARGV[0] eq '-continuous') {
+ $continuous = 1;
+ $once = 1;
+ shift;
+ } else {
+ last;
+ }
+}
+
+if($#ARGV<0 or $ARGV[0] =~ m/^-/) {
+ print STDERR <<"END"
+
+usage: $script [options...] program [arg...]
+
+Options include many (but not all) qsub options, along with:
+ -stderr Send errors to stderr rather than the error
+ output file.
+ -mem <value> Request <value> amount of memory for the job.
+ (number prefixed by 'k', 'm' or 'g')
+ -once Only start one job with that name, fail if
+ another is already started or queued.
+ -continuous Start a self-restarting job on the continuous
+ queue (default if invoked as 'jstart')
+
+Unlike qsub, if you do not specify output and error files, output is appended
+by default to files named <job>.err and <job>.out in your home directory
+(without job numbers), including errors from starting the job itself.
+
+Additionally, you need to provide an executable on the command line,
+$script will not read a script from standard input.
+
+END
+ ;
+ exit 1;
+}
+
+my $exe = shift;
+my $prog = `/usr/bin/which $exe`;
+chomp $prog;
+$prog = readlink $prog while -l $prog;
+my $cwd = `pwd`;
+chomp $cwd;
+$prog = "$cwd/$exe" unless -f $prog and -x $prog;
+
+my $jobname = 'unknown';
+$jobname = $1 if $prog =~ m{([^/.]+)(\.[^/]*)?$};
+$jobname = $qsargs{'-N'} if defined $qsargs{'-N'};
+
+my $err = "$ENV{'HOME'}/$jobname.err";
+my $out = "$ENV{'HOME'}/$jobname.out";
+
+$err = $qsargs{'-e'} if defined $qsargs{'-e'};
+$out = $qsargs{'-o'} if defined $qsargs{'-o'};
+$err = $out if defined $qsargs{'-j'} and $qsargs{'-j'} =~ m/^[yY]/;
+
+open STDERR, ">>$err" unless $stderr;
+
+my $now = localtime;
+
+die "\[$now\] $prog: not an executable file\n" unless -f $prog and -x $prog;
+
+if($once) {
+ my $running = system "/usr/local/bin/job", '-q', $jobname;
+ die "\[$now\] unable to get job status\n" if $running & 127;
+ $running >>= 8;
+ die "\[$now\] there is a job named '$jobname' already active\n" unless
$running==0;
+}
+
+push @args, '-e', $err unless defined $qsargs{'-e'};
+push @args, '-o', $out unless defined $qsargs{'-o'};
+push @args, '-N', $jobname, '-hard', '-l', "h_vmem=$memory";
+
+if($continuous) {
+ push @args, '-q', 'continuous';
+ open QSUB, "|/usr/bin/qsub '".join("' '", @args)."'" or die "\[$now\] unable
to start qsub: $!\n";
+ print QSUB "#! /bin/bash\n";
+ print QSUB "while ! '$prog' '".join("' '", @ARGV)."' ; do\n";
+ print QSUB " sleep 5\n";
+ print QSUB "done\n";
+ close QSUB;
+} else {
+ $queue = $qsargs{'-q'} if defined $qsargs{'-q'};
+ push @args, '-q', $queue, '-b', 'y', $prog, @ARGV;
+ exec '/usr/bin/qsub', @args;
+ die "\[$now\] qsub failed: $!\n";
+}
+
diff --git a/packages/jobutils/usr/local/bin/jstop
b/packages/jobutils/usr/local/bin/jstop
new file mode 100755
index 0000000..f0c5305
--- /dev/null
+++ b/packages/jobutils/usr/local/bin/jstop
@@ -0,0 +1,9 @@
+#! /bin/bash
+
+id=$(/usr/local/bin/job "$1")
+if [ $? != 0 ]; then
+ qdel $id
+else
+ echo "No job '$1' is currently queued or running" >&2
+ exit 1
+fi
diff --git a/packages/jobutils/usr/local/bin/jsub
b/packages/jobutils/usr/local/bin/jsub
new file mode 100755
index 0000000..756ee1d
--- /dev/null
+++ b/packages/jobutils/usr/local/bin/jsub
@@ -0,0 +1,126 @@
+#! /usr/bin/perl
+
+my %qsubargs = (
+ '-a' => 1, '-b' => 1, '-cwd' => 0, '-e' => 1, '-hard' => 0, '-i' => 1,
'-j' => 1,
+ '-l' => 1, '-now' => 1, '-N' => 1, '-o' => 1, '-p' => 1, '-q' => 1,
'-soft' => 0,
+ '-sync' => 1, '-v' => 1, '-wd' => 1,
+);
+my %qsargs;
+my @args;
+my $memory = '256m';
+my $once = 0;
+my $stderr = 0;
+my $continuous = 0;
+my $script = $0;
+my $queue = 'task';
+$script = $1 if $script =~ m{/([^/]+)$};
+
+$continuous = 1 if $script eq 'jstart';
+
+while($#ARGV > 0) {
+ if(defined $qsubargs{$ARGV[0]}) {
+ $arg = shift;
+ $qsargs{$arg} = $qsubargs{$arg}? $ARGV[0]: '';
+ if($arg eq '-N' or $arg eq '-q') {
+ shift if $qsubargs{$arg};
+ next;
+ }
+ push @args, $arg;
+ push @args, shift if $qsubargs{$arg};
+ } elsif ($ARGV[0] eq '-mem') {
+ shift;
+ $memory = shift;
+ die "$script: memory value must be an integer followed by 'k', 'm' or
'g'\n" unless $memory =~ m/^([1-9][0-9]*[mMgGkK])$/;
+ } elsif ($ARGV[0] eq '-once') {
+ $once = 1;
+ shift;
+ } elsif ($ARGV[0] eq '-stderr') {
+ $stderr = 1;
+ shift;
+ } elsif ($ARGV[0] eq '-continuous') {
+ $continuous = 1;
+ $once = 1;
+ shift;
+ } else {
+ last;
+ }
+}
+
+if($#ARGV<0 or $ARGV[0] =~ m/^-/) {
+ print STDERR <<"END"
+
+usage: $script [options...] program [arg...]
+
+Options include many (but not all) qsub options, along with:
+ -stderr Send errors to stderr rather than the error
+ output file.
+ -mem <value> Request <value> amount of memory for the job.
+ (number prefixed by 'k', 'm' or 'g')
+ -once Only start one job with that name, fail if
+ another is already started or queued.
+ -continuous Start a self-restarting job on the continuous
+ queue (default if invoked as 'jstart')
+
+Unlike qsub, if you do not specify output and error files, output is appended
+by default to files named <job>.err and <job>.out in your home directory
+(without job numbers), including errors from starting the job itself.
+
+Additionally, you need to provide an executable on the command line,
+$script will not read a script from standard input.
+
+END
+ ;
+ exit 1;
+}
+
+my $exe = shift;
+my $prog = `/usr/bin/which $exe`;
+chomp $prog;
+$prog = readlink $prog while -l $prog;
+my $cwd = `pwd`;
+chomp $cwd;
+$prog = "$cwd/$exe" unless -f $prog and -x $prog;
+
+my $jobname = 'unknown';
+$jobname = $1 if $prog =~ m{([^/.]+)(\.[^/]*)?$};
+$jobname = $qsargs{'-N'} if defined $qsargs{'-N'};
+
+my $err = "$ENV{'HOME'}/$jobname.err";
+my $out = "$ENV{'HOME'}/$jobname.out";
+
+$err = $qsargs{'-e'} if defined $qsargs{'-e'};
+$out = $qsargs{'-o'} if defined $qsargs{'-o'};
+$err = $out if defined $qsargs{'-j'} and $qsargs{'-j'} =~ m/^[yY]/;
+
+open STDERR, ">>$err" unless $stderr;
+
+my $now = localtime;
+
+die "\[$now\] $prog: not an executable file\n" unless -f $prog and -x $prog;
+
+if($once) {
+ my $running = system "/usr/local/bin/job", '-q', $jobname;
+ die "\[$now\] unable to get job status\n" if $running & 127;
+ $running >>= 8;
+ die "\[$now\] there is a job named '$jobname' already active\n" unless
$running==0;
+}
+
+push @args, '-e', $err unless defined $qsargs{'-e'};
+push @args, '-o', $out unless defined $qsargs{'-o'};
+push @args, '-N', $jobname, '-hard', '-l', "h_vmem=$memory";
+
+if($continuous) {
+ push @args, '-q', 'continuous';
+ open QSUB, "|/usr/bin/qsub '".join("' '", @args)."'" or die "\[$now\] unable
to start qsub: $!\n";
+ print QSUB "#! /bin/bash\n";
+ print QSUB "while ! '$prog' '".join("' '", @ARGV)."' ; do\n";
+ print QSUB " sleep 5\n";
+ print QSUB "done\n";
+ close QSUB;
+} else {
+ $queue = $qsargs{'-q'} if defined $qsargs{'-q'};
+ push @args, '-q', $queue, '-b', 'y', $prog, @ARGV;
+ exec '/usr/bin/qsub', @args;
+ die "\[$now\] qsub failed: $!\n";
+}
+
diff --git a/packages/jobutils/usr/share/man/man1/jstart.1
b/packages/jobutils/usr/share/man/man1/jstart.1
new file mode 100644
index 0000000..33cd924
--- /dev/null
+++ b/packages/jobutils/usr/share/man/man1/jstart.1
@@ -0,0 +1,31 @@
+.\" Man page for jstart
+.\" Licensed under BSD-like License.
+.\" Created by Marc-André Pelletier
+.\"
+.TH Jobutils "jstart" "User Manual"
+.SH NAME
+usage: jstart [options...] program [arg...]
+
+Options include many (but not all) qsub options, along with:
+ \-stderr Send errors to stderr rather than the error
+ output file.
+ \-mem <value> Request <value> amount of memory for the job.
+ (number prefixed by 'k', 'm' or 'g')
+ \-once Only start one job with that name, fail if
+ another is already started or queued.
+ \-continuous Start a self-restarting job on the continuous
+ queue (default if invoked as 'jstart')
+
+Unlike qsub, if you do not specify output and error files, output is appended
+by default to files named <job>.err and <job>.out in your home directory
+(without job numbers), including errors from starting the job itself.
+
+Additionally, you need to provide an executable on the command line,
+jstart will not read a script from standard input.
+
+.SH DESCRIPTION
+jobutils will insert extra commands to grid engine, these tools were created
for wikimedia tools project
+.PP
+.SH "AUTHOR"
+Marc-André Pelletier, Petr Bena
+.br
diff --git a/packages/jobutils/usr/share/man/man1/jstop.1
b/packages/jobutils/usr/share/man/man1/jstop.1
new file mode 100644
index 0000000..50bea5e
--- /dev/null
+++ b/packages/jobutils/usr/share/man/man1/jstop.1
@@ -0,0 +1,16 @@
+.\" Man page for jstart
+.\" Licensed under CC-BY-SA
+.\" Created by Marc-André Pelletier
+.\"
+.TH Jobutils "jstop" "User Manual"
+.SH NAME
+usage: jstop [jobname]
+
+Delete a job
+
+.SH DESCRIPTION
+jobutils will insert extra commands to grid engine, these tools were created
for wikimedia tools project
+.PP
+.SH "AUTHOR"
+Marc-André Pelletier, Petr Bena
+.br
diff --git a/packages/jobutils/usr/share/man/man1/jsub.1
b/packages/jobutils/usr/share/man/man1/jsub.1
new file mode 100644
index 0000000..a3ae4d9
--- /dev/null
+++ b/packages/jobutils/usr/share/man/man1/jsub.1
@@ -0,0 +1,31 @@
+.\" Man page for jstart
+.\" Licensed under CC-BY-SA
+.\" Created by Marc-André Pelletier
+.\"
+.TH Jobutils "jsub" "User Manual"
+.SH NAME
+usage: jsub [options...] program [arg...]
+
+Options include many (but not all) qsub options, along with:
+ \-stderr Send errors to stderr rather than the error
+ output file.
+ \-mem <value> Request <value> amount of memory for the job.
+ (number prefixed by 'k', 'm' or 'g')
+ \-once Only start one job with that name, fail if
+ another is already started or queued.
+ \-continuous Start a self-restarting job on the continuous
+ queue (default if invoked as 'jstart')
+
+Unlike qsub, if you do not specify output and error files, output is appended
+by default to files named <job>.err and <job>.out in your home directory
+(without job numbers), including errors from starting the job itself.
+
+Additionally, you need to provide an executable on the command line,
+jstart will not read a script from standard input.
+
+.SH DESCRIPTION
+jobutils will insert extra commands to grid engine, these tools were created
for wikimedia tools project
+.PP
+.SH "AUTHOR"
+Marc-André Pelletier, Petr Bena
+.br
--
To view, visit https://gerrit.wikimedia.org/r/57740
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I4c92ba0cede54616caf037a99ca79247b2c189cf
Gerrit-PatchSet: 2
Gerrit-Project: labs/toollabs
Gerrit-Branch: master
Gerrit-Owner: Petrb <benap...@gmail.com>
Gerrit-Reviewer: coren <mpellet...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
