Author: daijy
Date: Thu Oct 31 20:02:30 2013
New Revision: 1537610
URL: http://svn.apache.org/r1537610
Log:
HIVE-5510: [WebHCat] GET job/queue return wrong job information
Modified:
hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf
hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/QueueStatusBean.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
Modified: hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1537610&r1=1537609&r2=1537610&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
(original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm Thu
Oct 31 20:02:30 2013
@@ -647,19 +647,16 @@ sub compare
foreach my $key (keys %$json_matches) {
my $regex_expected_value = $json_matches->{$key};
my $path = JSON::Path->new($key);
- my $value;
- # when filter_job_status is defined
- if (defined $testCmd->{'filter_job_status'}) {
- # decode $testResult->{'body'} to an array of hash
- my $body = decode_json $testResult->{'body'};
- # in the tests, we run this case with jobName =
"PigLatin:loadstore.pig"
- # filter $body to leave only records with this jobName
- my @filtered_body = grep {($_->{detail}{profile}{jobName} eq
"PigLatin:loadstore.pig")} @$body;
- my @sorted_filtered_body = sort { $a->{id} cmp $b->{id}
} @filtered_body;
- $value = $path->value(\@sorted_filtered_body);
+
+ # decode $testResult->{'body'} to an array of hash
+ my $body = decode_json $testResult->{'body'};
+ my @sorted_body;
+ if (ref @$body[0] eq 'HASH') {
+ @sorted_body = sort { $a->{id} cmp $b->{id} } @$body;
} else {
- $value = $path->value($testResult->{'body'});
+ @sorted_body = sort { $a cmp $b } @$body;
}
+ my $value = $path->value(\@sorted_body);
if ($value !~ /$regex_expected_value/s) {
print $log "$0::$subName INFO check failed:"
Modified: hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf?rev=1537610&r1=1537609&r2=1537610&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf Thu Oct 31
20:02:30 2013
@@ -89,7 +89,6 @@ $cfg =
'json_path' => {'$[-1:].detail.status.username' => ':UNAME_OTHER:',
'$[-2:].detail.status.username' => ':UNAME_OTHER:',
'$[-3:].detail.status.username' => ':UNAME:',
'$[-4:].detail.status.username' => ':UNAME:',
'$[-5:].detail.status.username' => ':UNAME_OTHER:',
'$[-6:].detail.status.username' => ':UNAME_OTHER:'},
'status_code' => 200,
- 'filter_job_status' => 1,
},
{
# GET jobs?user.name=UNAME_OTHER&fields=*, should get only jobs launched
as UNAME_OTHER
@@ -134,11 +133,11 @@ $cfg =
'$[-1:].detail.profile.jobId' => 'job_.*',
'$[-1:].detail.id' => 'job_.*',
'$[-1:].detail.parentId' => 'job_.*',
- '$[-1:].detail.percentComplete' => '100%',
- '$[-1:].detail.exitValue' => '0',
- '$[-1:].detail.user' => ':UNAME_OTHER:',
- '$[-1:].detail.callback' => '^.+$',
- '$[-1:].detail.completed' => 'done',
+ '$[-2:].detail.exitValue' => '0',
+ '$[-2:].detail.user' => ':UNAME_OTHER:',
+ '$[-2:].detail.callback' => '^.+$',
+ '$[-2:].detail.completed' => 'done',
+ '$[-2:].detail.percentComplete' => '100%',
},
'status_code' => 200,
},
Modified:
hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf?rev=1537610&r1=1537609&r2=1537610&view=diff
==============================================================================
---
hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf
(original)
+++
hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf
Thu Oct 31 20:02:30 2013
@@ -65,7 +65,7 @@ $cfg =
#results
'status_code' => 200,
'check_job_created' => 1,
- 'check_job_complete' => 'FAILURE',
+ 'check_job_complete' => 'SUCCESS',
'check_job_exit_value' => 1,
'check_call_back' => 1,
},
@@ -102,7 +102,7 @@ $cfg =
#results
'status_code' => 200,
'check_job_created' => 1,
- 'check_job_complete' => 'FAILURE',
+ 'check_job_complete' => 'SUCCESS',
'check_logs' => { 'job_num' => '1' },
'check_job_exit_value' => 1,
'check_call_back' => 1,
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java?rev=1537610&r1=1537609&r2=1537610&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java
Thu Oct 31 20:02:30 2013
@@ -19,7 +19,10 @@
package org.apache.hive.hcatalog.templeton;
import java.io.IOException;
+import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.JobID;
@@ -30,6 +33,7 @@ import org.apache.hive.hcatalog.templeto
* Delete a job
*/
public class DeleteDelegator extends TempletonDelegator {
+ private static final Log LOG = LogFactory.getLog(DeleteDelegator.class);
public DeleteDelegator(AppConfig appConf) {
super(appConf);
}
@@ -47,9 +51,16 @@ public class DeleteDelegator extends Tem
throw new BadParam("Invalid jobid: " + id);
tracker.killJob(jobid);
state = new JobState(id, Main.getAppConfigInstance());
- String childid = state.getChildId();
- if (childid != null)
- tracker.killJob(StatusDelegator.StringToJobID(childid));
+ List<JobState> children = state.getChildren();
+ if (children != null) {
+ for (JobState child : children) {
+ try {
+ tracker.killJob(StatusDelegator.StringToJobID(child.getId()));
+ } catch (IOException e) {
+ LOG.warn("templeton: fail to kill job " + child.getId());
+ }
+ }
+ }
return StatusDelegator.makeStatus(tracker, jobid, state);
} catch (IllegalStateException e) {
throw new BadParam(e.getMessage());
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java?rev=1537610&r1=1537609&r2=1537610&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java
Thu Oct 31 20:02:30 2013
@@ -26,7 +26,6 @@ import org.apache.hadoop.hive.shims.Hado
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.hcatalog.templeton.tool.JobState;
/**
* List jobs owned by a user.
@@ -50,17 +49,9 @@ public class ListDelegator extends Templ
if (jobs != null) {
for (JobStatus job : jobs) {
- JobState state = null;
- try {
- String id = job.getJobID().toString();
- state = new JobState(id, Main.getAppConfigInstance());
- if (showall || user.equals(state.getUser()))
- ids.add(id);
- } finally {
- if (state != null) {
- state.close();
- }
- }
+ String id = job.getJobID().toString();
+ if (showall || user.equals(job.getUsername()))
+ ids.add(id);
}
}
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/QueueStatusBean.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/QueueStatusBean.java?rev=1537610&r1=1537609&r2=1537610&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/QueueStatusBean.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/QueueStatusBean.java
Thu Oct 31 20:02:30 2013
@@ -57,12 +57,10 @@ public class QueueStatusBean {
this.profile = profile;
id = profile.getJobID().toString();
- parentId = state.getId();
- if (id.equals(parentId))
- parentId = null;
+ parentId = state.getParent();
percentComplete = state.getPercentComplete();
exitValue = state.getExitValue();
- user = state.getUser();
+ user = profile.getUser();
callback = state.getCallback();
completed = state.getCompleteStatus();
userargs = state.getUserArgs();
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java?rev=1537610&r1=1537609&r2=1537610&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java
Thu Oct 31 20:02:30 2013
@@ -31,7 +31,13 @@ import org.apache.hadoop.mapred.JobStatu
import org.apache.hive.hcatalog.templeton.tool.JobState;
/**
- * Fetch the status of a given job id in the queue.
+ * Fetch the status of a given job id in the queue. There are three sources of
the info
+ * 1. Query result from JobTracker
+ * 2. JobState saved by TempletonControllerJob when monitoring the
TempletonControllerJob
+ * 3. TempletonControllerJob put a JobState for every job it launches, so
child job can
+ * retrieve its parent job by its JobState
+ *
+ * Currently there is no permission restriction, any user can query any job
*/
public class StatusDelegator extends TempletonDelegator {
private static final Log LOG = LogFactory.getLog(StatusDelegator.class);
@@ -63,40 +69,19 @@ public class StatusDelegator extends Tem
}
}
- public static QueueStatusBean makeStatus(WebHCatJTShim tracker,
+ static QueueStatusBean makeStatus(WebHCatJTShim tracker,
JobID jobid,
- String childid,
JobState state)
throws BadParam, IOException {
- JobID bestid = jobid;
- if (childid != null)
- bestid = StatusDelegator.StringToJobID(childid);
-
- JobStatus status = tracker.getJobStatus(bestid);
- JobProfile profile = tracker.getJobProfile(bestid);
-
- if (status == null || profile == null) {
- if (bestid != jobid) { // Corrupt childid, retry.
- LOG.error("Corrupt child id " + childid + " for " + jobid);
- bestid = jobid;
- status = tracker.getJobStatus(bestid);
- profile = tracker.getJobProfile(bestid);
- }
- }
+ JobStatus status = tracker.getJobStatus(jobid);
+ JobProfile profile = tracker.getJobProfile(jobid);
if (status == null || profile == null) // No such job.
- throw new BadParam("Could not find job " + bestid);
+ throw new BadParam("Could not find job " + jobid);
return new QueueStatusBean(state, status, profile);
}
- public static QueueStatusBean makeStatus(WebHCatJTShim tracker,
- JobID jobid,
- JobState state)
- throws BadParam, IOException {
- return makeStatus(tracker, jobid, state.getChildId(), state);
- }
-
/**
* A version of JobID.forName with our app specific error handling.
*/
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java?rev=1537610&r1=1537609&r2=1537610&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java
Thu Oct 31 20:02:30 2013
@@ -126,19 +126,6 @@ public class JobState {
}
/**
- * The child id of TempletonControllerJob
- */
- public String getChildId()
- throws IOException {
- return getField("childid");
- }
-
- public void setChildId(String childid)
- throws IOException {
- setField("childid", childid);
- }
-
- /**
* Add a jobid to the list of children of this job.
*
* @param jobid
@@ -151,6 +138,9 @@ public class JobState {
} catch (Exception e) {
// There are none or they're not readable.
}
+ if (jobids==null) {
+ jobids = "";
+ }
if (!jobids.equals("")) {
jobids += ",";
}
@@ -159,6 +149,18 @@ public class JobState {
}
/**
+ * Set parent job of this job
+ * @param id
+ */
+ public void setParent(String id) throws IOException {
+ setField("parent", id);
+ }
+
+ public String getParent() throws IOException {
+ return getField("parent");
+ }
+
+ /**
* Get a list of jobstates for jobs that are children of this job.
* @throws IOException
*/
@@ -171,31 +173,6 @@ public class JobState {
}
/**
- * Save a comma-separated list of jobids that are children
- * of this job.
- * @param jobids
- * @throws IOException
- */
- public void setChildren(String jobids) throws IOException {
- setField("children", jobids);
- }
-
- /**
- * Set the list of child jobs of this job
- * @param children
- */
- public void setChildren(List<JobState> children) throws IOException {
- String val = "";
- for (JobState jobstate : children) {
- if (!val.equals("")) {
- val += ",";
- }
- val += jobstate.getId();
- }
- setField("children", val);
- }
-
- /**
* The system exit value of the job.
*/
public Long getExitValue()
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1537610&r1=1537609&r2=1537610&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
Thu Oct 31 20:02:30 2013
@@ -261,8 +261,15 @@ public class LaunchMapper extends Mapper
if (percent != null || childid != null) {
state = new JobState(jobid.toString(), conf);
- state.setPercentComplete(percent);
- state.setChildId(childid);
+ if (percent != null) {
+ state.setPercentComplete(percent);
+ }
+ if (childid != null) {
+ JobState childState = new JobState(childid, conf);
+ childState.setParent(jobid.toString());
+ state.addChild(childid);
+ state.close();
+ }
}
} catch (IOException e) {
LOG.error("templeton: state error: ", e);
