[hadoop] branch trunk updated: YARN-11158. Support (Create/Renew/Cancel) DelegationToken API's for Federation. (#5104)
This is an automated email from the ASF dual-hosted git repository. inigoiri pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 4af4997e112 YARN-11158. Support (Create/Renew/Cancel) DelegationToken API's for Federation. (#5104) 4af4997e112 is described below commit 4af4997e1129d3b9bd7fde7ec8731d6e79093fd8 Author: slfan1989 <55643692+slfan1...@users.noreply.github.com> AuthorDate: Fri Dec 2 05:20:21 2022 +0800 YARN-11158. Support (Create/Renew/Cancel) DelegationToken API's for Federation. (#5104) --- .../hadoop/yarn/server/router/RouterMetrics.java | 97 - .../yarn/server/router/RouterServerUtil.java | 24 .../clientrm/FederationClientInterceptor.java | 97 - .../yarn/server/router/TestRouterMetrics.java | 80 ++- .../clientrm/TestFederationClientInterceptor.java | 156 - .../TestableFederationClientInterceptor.java | 28 6 files changed, 468 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java index b03aeda38b4..31d838d1b3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java @@ -127,6 +127,12 @@ public final class RouterMetrics { private MutableGaugeInt numGetRMNodeLabelsFailedRetrieved; @Metric("# of checkUserAccessToQueue failed to be retrieved") private MutableGaugeInt numCheckUserAccessToQueueFailedRetrieved; + @Metric("# of getDelegationToken failed to be retrieved") + private MutableGaugeInt numGetDelegationTokenFailedRetrieved; + @Metric("# of renewDelegationToken failed to be retrieved") + private MutableGaugeInt numRenewDelegationTokenFailedRetrieved; + @Metric("# of renewDelegationToken failed to be retrieved") + private MutableGaugeInt numCancelDelegationTokenFailedRetrieved; // Aggregate metrics are shared, and don't have to be looked up per call @Metric("Total number of successful Submitted apps and latency(ms)") @@ -215,6 +221,12 @@ public final class RouterMetrics { private MutableRate totalSucceededGetRMNodeLabelsRetrieved; @Metric("Total number of successful Retrieved CheckUserAccessToQueue and latency(ms)") private MutableRate totalSucceededCheckUserAccessToQueueRetrieved; + @Metric("Total number of successful Retrieved GetDelegationToken and latency(ms)") + private MutableRate totalSucceededGetDelegationTokenRetrieved; + @Metric("Total number of successful Retrieved RenewDelegationToken and latency(ms)") + private MutableRate totalSucceededRenewDelegationTokenRetrieved; + @Metric("Total number of successful Retrieved CancelDelegationToken and latency(ms)") + private MutableRate totalSucceededCancelDelegationTokenRetrieved; /** * Provide quantile counters for all latencies. @@ -262,6 +274,9 @@ public final class RouterMetrics { private MutableQuantiles getRefreshQueuesLatency; private MutableQuantiles getRMNodeLabelsLatency; private MutableQuantiles checkUserAccessToQueueLatency; + private MutableQuantiles getDelegationTokenLatency; + private MutableQuantiles renewDelegationTokenLatency; + private MutableQuantiles cancelDelegationTokenLatency; private static volatile RouterMetrics instance = null; private static MetricsRegistry registry; @@ -423,6 +438,15 @@ public final class RouterMetrics { checkUserAccessToQueueLatency = registry.newQuantiles("checkUserAccessToQueueLatency", "latency of get apptimeouts timeouts", "ops", "latency", 10); + +getDelegationTokenLatency = registry.newQuantiles("getDelegationTokenLatency", +"latency of get delegation token timeouts", "ops", "latency", 10); + +renewDelegationTokenLatency = registry.newQuantiles("renewDelegationTokenLatency", + "latency of renew delegation token timeouts", "ops", "latency", 10); + +cancelDelegationTokenLatency = registry.newQuantiles("cancelDelegationTokenLatency", +"latency of cancel delegation token timeouts", "ops", "latency", 10); } public static RouterMetrics getMetrics() { @@ -655,10 +679,25 @@ public final class RouterMetrics { } @VisibleForTesting - public long getNumSucceededCheckUserAccessToQueueRetrievedRetrieved() { + public long getNumSucceededCheckUserAccessToQueueRetrieved() { return totalSucceededCheckUserAccessToQueueRetrieved.lastStat().numSamples(); } + @VisibleForTesting + public long getNumS
svn commit: r58388 [2/3] - /dev/hadoop/hadoop-3.3.5-RC0/
Added: dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt == --- dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt (added) +++ dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt Thu Dec 1 16:22:33 2022 @@ -0,0 +1,16715 @@ + +* +Summary +--- +Generated at: 2022-11-30T15:52:29+00:00 + +Notes: 0 +Binaries: 0 +Archives: 0 +Standards: 3 + +Apache Licensed: 3 +Generated Documents: 0 + +JavaDocs are generated, thus a license header is optional. +Generated files do not require license headers. + +0 Unknown Licenses + +* + Files with Apache License headers will be marked AL + Binary files (which do not require any license headers) will be marked B + Compressed archives will be marked A + Notices, licenses etc. will be marked N + AL/build/source/hadoop-tools/hadoop-kafka/pom.xml + AL /build/source/hadoop-tools/hadoop-kafka/src/main/java/org/apache/hadoop/metrics2/sink/KafkaSink.java + AL /build/source/hadoop-tools/hadoop-kafka/src/test/java/org/apache/hadoop/metrics2/impl/TestKafkaMetrics.java + +* + +* +Summary +--- +Generated at: 2022-11-30T15:52:29+00:00 + +Notes: 0 +Binaries: 0 +Archives: 0 +Standards: 1 + +Apache Licensed: 1 +Generated Documents: 0 + +JavaDocs are generated, thus a license header is optional. +Generated files do not require license headers. + +0 Unknown Licenses + +* + Files with Apache License headers will be marked AL + Binary files (which do not require any license headers) will be marked B + Compressed archives will be marked A + Notices, licenses etc. will be marked N + AL/build/source/hadoop-tools/hadoop-openstack/pom.xml + +* + +* +Summary +--- +Generated at: 2022-11-30T15:52:28+00:00 + +Notes: 1 +Binaries: 0 +Archives: 0 +Standards: 13 + +Apache Licensed: 13 +Generated Documents: 0 + +JavaDocs are generated, thus a license header is optional. +Generated files do not require license headers. + +0 Unknown Licenses + +* + Files with Apache License headers will be marked AL + Binary files (which do not require any license headers) will be marked B + Compressed archives will be marked A + Notices, licenses etc. will be marked N + AL/build/source/hadoop-tools/hadoop-datajoin/pom.xml + AL /build/source/hadoop-tools/hadoop-datajoin/dev-support/findbugs-exclude.xml + AL /build/source/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinMapperBase.java + AL /build/source/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinReducerBase.java + AL /build/source/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/TaggedMapOutput.java + AL /build/source/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/ArrayListBackedIterator.java + AL /build/source/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/JobBase.java + AL /build/source/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java + AL /build/source/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/ResetableIterator.java + AL /build/source/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/TestDataJoin.java + AL /build/source/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/SampleTaggedMapOutput.java + AL /build/source/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/SampleDataJoinReducer.java + N /build/source/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt + AL /build/source/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/SampleDataJoinMapper.java + +* + +* +Summary +--- +Generated at: 2022-11-30T15:52:29+00:00 + +Notes: 0 +Binaries: 0 +Archives: 0 +Standards: 49 + +Apache Licensed: 49 +Generated Documents: 0 + +JavaDocs are generated, thus a license header is optional. +Generated files do not require license headers. + +0 Unknown Licenses + +* + Files with Apache License headers will be marked AL + Binary files (which do not require any license headers) will be marked B + Compressed archives will be marked A + Notices, licenses etc. will be marked N + AL/build/source/hadoop-tools/hadoop-aliyun/pom
svn commit: r58388 [1/3] - /dev/hadoop/hadoop-3.3.5-RC0/
Author: stevel Date: Thu Dec 1 16:22:33 2022 New Revision: 58388 Log: HADOOP-18470. Release Hadoop 3.3.5 RC0 Added: dev/hadoop/hadoop-3.3.5-RC0/ dev/hadoop/hadoop-3.3.5-RC0/CHANGELOG.md dev/hadoop/hadoop-3.3.5-RC0/CHANGELOG.md.asc dev/hadoop/hadoop-3.3.5-RC0/CHANGELOG.md.sha512 dev/hadoop/hadoop-3.3.5-RC0/RELEASENOTES.md dev/hadoop/hadoop-3.3.5-RC0/RELEASENOTES.md.asc dev/hadoop/hadoop-3.3.5-RC0/RELEASENOTES.md.sha512 dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt.asc dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt.sha512 dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-site.tar.gz (with props) dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-site.tar.gz.asc dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-site.tar.gz.sha512 dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-src.tar.gz (with props) dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-src.tar.gz.asc dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-src.tar.gz.sha512 dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5.tar.gz (with props) dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5.tar.gz.asc dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5.tar.gz.sha512 Added: dev/hadoop/hadoop-3.3.5-RC0/CHANGELOG.md == --- dev/hadoop/hadoop-3.3.5-RC0/CHANGELOG.md (added) +++ dev/hadoop/hadoop-3.3.5-RC0/CHANGELOG.md Thu Dec 1 16:22:33 2022 @@ -0,0 +1,311 @@ + + +# Apache Hadoop Changelog + +## Release 3.3.5 - 2022-11-30 + + + +### IMPORTANT ISSUES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|: |: | :--- |: |: |: | +| [HADOOP-17956](https://issues.apache.org/jira/browse/HADOOP-17956) | Replace all default Charset usage with UTF-8 | Major | . | Viraj Jasani | Viraj Jasani | + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|: |: | :--- |: |: |: | +| [HDFS-16331](https://issues.apache.org/jira/browse/HDFS-16331) | Make dfs.blockreport.intervalMsec reconfigurable | Major | . | Tao Li | Tao Li | +| [HDFS-16371](https://issues.apache.org/jira/browse/HDFS-16371) | Exclude slow disks when choosing volume | Major | . | Tao Li | Tao Li | +| [HDFS-16400](https://issues.apache.org/jira/browse/HDFS-16400) | Reconfig DataXceiver parameters for datanode | Major | . | Tao Li | Tao Li | +| [HDFS-16399](https://issues.apache.org/jira/browse/HDFS-16399) | Reconfig cache report parameters for datanode | Major | . | Tao Li | Tao Li | +| [HDFS-16398](https://issues.apache.org/jira/browse/HDFS-16398) | Reconfig block report parameters for datanode | Major | . | Tao Li | Tao Li | +| [HDFS-16396](https://issues.apache.org/jira/browse/HDFS-16396) | Reconfig slow peer parameters for datanode | Major | . | Tao Li | Tao Li | +| [HDFS-16397](https://issues.apache.org/jira/browse/HDFS-16397) | Reconfig slow disk parameters for datanode | Major | . | Tao Li | Tao Li | +| [MAPREDUCE-7341](https://issues.apache.org/jira/browse/MAPREDUCE-7341) | Add a task-manifest output committer for Azure and GCS | Major | client | Steve Loughran | Steve Loughran | +| [HADOOP-18163](https://issues.apache.org/jira/browse/HADOOP-18163) | hadoop-azure support for the Manifest Committer of MAPREDUCE-7341 | Major | fs/azure | Steve Loughran | Steve Loughran | +| [HDFS-16413](https://issues.apache.org/jira/browse/HDFS-16413) | Reconfig dfs usage parameters for datanode | Major | . | Tao Li | Tao Li | +| [HDFS-16521](https://issues.apache.org/jira/browse/HDFS-16521) | DFS API to retrieve slow datanodes | Major | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16568](https://issues.apache.org/jira/browse/HDFS-16568) | dfsadmin -reconfig option to start/query reconfig on all live datanodes | Major | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16582](https://issues.apache.org/jira/browse/HDFS-16582) | Expose aggregate latency of slow node as perceived by the reporting node | Major | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16595](https://issues.apache.org/jira/browse/HDFS-16595) | Slow peer metrics - add median, mad and upper latency limits | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-18345](https://issues.apache.org/jira/browse/HADOOP-18345) | Enhance client protocol to propagate last seen state IDs for multiple nameservices. | Major | . | Simbarashe Dzinamarira | Simbarashe Dzinamarira | +| [YARN-11241](https://issues.apache.org/jira/browse/YARN-11241) | Add uncleaning option for local app log file with log-aggregation enabled | Major | log-aggregation | Ashutosh Gupta | Ashutosh Gupta | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|: |: | :--- |: |: |: | +| [HDFS-15745](https://issues.apache.org/jira/browse/HDFS-15745) | Make DataNodePeerMetrics#LOW\_THRESHOLD\_MS and MIN\_OUTLIER\_DETECTION\_NODES configurable | Major | . | Haibin Huang | Haibin Hua
svn commit: r58388 [3/3] - /dev/hadoop/hadoop-3.3.5-RC0/
Added: dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt.asc == --- dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt.asc (added) +++ dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt.asc Thu Dec 1 16:22:33 2022 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEOCN+5CUFAoUHfbV60iz4RtuxYqAFAmOHlYcACgkQ0iz4Rtux +YqB1ZRAAkUJdDvBlXgCFDvRa9ZteD6S9BwS9/cFAB4L1GqvYyu2n+x2fgcF612cm +K0tgJPkD7DPnb05YcB9O4Jb488qxOM5jdvvivkBUBIm3JwUI0CixhFAEUz1U2BL2 +bxCGNZPVdKtp2lxGekc2VSeHXhnVAz6838bhKM9u2EN0ky1ToILC11fNMRQIAxFo +VS89rW4MQi3VUpSyBbZ+q/MPx98ppFMEPvtVkEXXWo5BGZ5sZRPnhPlnMU002dQZ +U7d+izJBTjNt0PmJUqdl3G8OG6mSM+zsI+tkZA0MCN1FvEj/2dxJvk+vQ7Wy5WO9 +tlbx2tNnS2+szQ0wRJ455Hxoqb+lHEKynw4nobCR38e2jGtOI9INsqEnLfj+wD8X +JD+0PBAjVAToUrL8UIu0HOXQ9OWoSjfTFDOzUdn4T8i9Qg/J14w7Cjv6lPHdOvp8 +90otHHKVxYd9VQ+Vg7BAC9tQY8QbkSo4IY8n/7WN/3ottoMIF+ZCkAs15yetC2c3 +Zl2D4k6RB1MIAep5N/oAp2vwateFWx8pGVvuM37H31sbJKZ+enf2mw6aNyBCQdx+ +86XLppFb2Fu7IC4s5s5VDyaULnWUGLFoK//jb5S0YrugsSdVFJxjqmLcFhtqnJMh +5iRu9JJEuLZEcLIW+nv+wpC8bOmkZpuIuyxQi4Te/DN1QAVSNvw= +=hEiY +-END PGP SIGNATURE- Added: dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt.sha512 == --- dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt.sha512 (added) +++ dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-rat.txt.sha512 Thu Dec 1 16:22:33 2022 @@ -0,0 +1 @@ +SHA512 (hadoop-3.3.5-rat.txt) = 3c8343a912e2b852dcd2eaa1f17af9eddbc31d425184821870535bb0adcea7155b13db26c1c243020b406b2dcee0eba101a81627bbfb82c16b90348b0e637377 Added: dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-site.tar.gz == Binary file - no diff available. Propchange: dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-site.tar.gz -- svn:mime-type = application/octet-stream Added: dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-site.tar.gz.asc == --- dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-site.tar.gz.asc (added) +++ dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-site.tar.gz.asc Thu Dec 1 16:22:33 2022 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEOCN+5CUFAoUHfbV60iz4RtuxYqAFAmOHlYgACgkQ0iz4Rtux +YqB6hxAAtNDp/AXu1UmR68ihumzbRzXGT7E2+MFQzpU1LIxfuvci6LrkV9r9Qd4e +Nvc7LHeGLSegi2ADzBfYM4lCnm/WfkqMDk56uVL6hAERsVnj0AfQY/lQonC+qpln +hdDPZPb7yWY563FnO7DfYH1X2FaQ4i+Bk+PUnq/KOiscUy6uo5YbZLSeuMPdpNY2 +m78cBaiytPgcyU1TqrArsiSR7P67Qdha9RbWJ2OfpN1OdxvKuImGoSz6c+4fG6fU +DOURHBMRUZd7f2qg+cOYdvLKD/tDfr5eBWK9wHvMMb72N3q1jr9uCGq5yl3PMgrm +LRdiedyrKBXpD5I1JKAD5Ed3gOWZnquVMqi1bJsTongpebYe+l9uMuiHhSNYqhOP +2PU8uP0MfJG1yuFqD4wS8YDOJ5oj2f18Cli8LhnDPQJ3rI96Z2zY0/Gs8YAlYMui +JDcgHhslQA42GnZMkv7OIC8mjOgpFj13GA66u9wV2QZSH32h3DjUN9X4NxtgD6io +mmcAFOiulMGZw6E3khNCCGMJ3VkR0JQDSMP1MDzWueKX9KZGSdUmNmxz0RvPJQAo +IGU5bI2YahZZxnTt/l1YgEFwvFvdcnNPTRFhkGugfgDwMTsnOmlKORfxcUGsj/OL +0nzMgJeV0olZ34AM8rLJYG8C9okmGRtxBvO+3POtSongC31nTYw= +=NIbt +-END PGP SIGNATURE- Added: dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-site.tar.gz.sha512 == --- dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-site.tar.gz.sha512 (added) +++ dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-site.tar.gz.sha512 Thu Dec 1 16:22:33 2022 @@ -0,0 +1 @@ +SHA512 (hadoop-3.3.5-site.tar.gz) = 413679247c2a5886185a6cd0c893f39456dc2ff9c2364a71c828501bc9d3eb3ae75f14c2ad6071a06ef0910f9c9735c45e64992fdfc25d4515421c76a7eeebcb Added: dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-src.tar.gz == Binary file - no diff available. Propchange: dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-src.tar.gz -- svn:mime-type = application/octet-stream Added: dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-src.tar.gz.asc == --- dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-src.tar.gz.asc (added) +++ dev/hadoop/hadoop-3.3.5-RC0/hadoop-3.3.5-src.tar.gz.asc Thu Dec 1 16:22:33 2022 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEOCN+5CUFAoUHfbV60iz4RtuxYqAFAmOHlYgACgkQ0iz4Rtux +YqD8Rg/+MaTZ6ofA4ah6N98n6ZVqPViGXCGdEEyryPobquGXymSes7GikD2vmARK +E05lQHJbdBo/1fv/LEotwyV3An+3L2lq3HjCMRgoM6v2vCqRDZdLxpoWugCtTezM +h8+utWO9trCqrDjmPFXrbK+i4CbFlZcmCzSo1ePH+mSZ0kjW5WpCdpIG6b/TsPmL +fmLRcn8odGBVXWDwB7Mjo3kS1F2tWFLmnMmoBomAveaiou1phPtXxm+uI0+9UUEx +o/stKuX2GHRZaYjNWJQFe5IqSKgWWx6kiykH0vsjpG2HRan9tf7TuEgz0LhO9i9g +WSGBTrxQeC/L74uXafu1KyWn1BtjxkAQq2GI/ikszPRQAzHqNCFZYxzslk4b+pvJ +OhXvwaqDRvfHPfaYz35bIDngJN78M+GJIACyS+vrbjn+ko63vCoV+wXYS4Sqk3kv +KVtkilGmQvB55zmfs+BFPHZNgxOm9UlWtb5M7Pa4c5SiYBakZVFihTC6RZ1O23QA +7QmAhSlOqa0YccxuJyDX2ixZ3IG/yef1wVNcT+N6tr05eMzECa
[hadoop] annotated tag release-3.3.5-RC0 created (now 9157563c5b4)
This is an automated email from the ASF dual-hosted git repository. stevel pushed a change to annotated tag release-3.3.5-RC0 in repository https://gitbox.apache.org/repos/asf/hadoop.git at 9157563c5b4 (tag) tagging 99c5802280b9ac6cc8dcd6eecf0b9fb3ea309402 (commit) replaces remove-ozone by Steve Loughran on Thu Dec 1 15:57:22 2022 + - Log - Release candidate 3.3.5-RC0 -BEGIN PGP SIGNATURE- iQIzBAABCgAdFiEEOCN+5CUFAoUHfbV60iz4RtuxYqAFAmOIzuIACgkQ0iz4Rtux YqDsVxAAsIIR+e4f2D43enGqpLXAWBvKk+pFk0v+9kM7ksVGinfbzGTlo3ZOFTbG 9knETyv0aFmGazmSFMt2LCqGt0wITaBhflK1XqPK6jLVQbE5NQoptGEblInM/Qbu jnICy+FDL84atXB/YK82eqvzgavrQLOyPgAgtWG6xxV3tYwl4Hxb/D3fs2dz7nTS Y+Q0q6u7u0xT3FhwG6FL8O1nm3HOzO3paxNK9MU4PQPXVfWlBeNcttdJ/zW4+NLK VcWl3vm+YWe6zHKVbJPVoLi2cndfPdc8gQLAf5uc/p3knjG0U6EiXJd4uEPvG6Wl 4B7CjzTzgMD24jPjT4tZihzAhZn13CBnJoETVSo49b1ias5R+UPOQL++sNyN4K3g mmgfhpdjtfVr3PUcPfBR12GMdpPZBFOveBaYeV0vw/dDCNffMwNCVOihfFpYOoR9 4Wk8aiboxi3k11CpfW9Z/dTEMenzMBX7EAKuNP/O1E+fNodujNObWc4lT/xvg7i5 360YDRrldQHHZZvGLu4zdLGVATABfotDUjVp3SJ/RUxhssCQONsKY+Pi4DejgTPK llW3zWlfXeyMywJokZkUCkcV9hnn2I9zkZZ+cI6w1tGzexnhBPe1YLjdJqyMMXnS SpQJncVQkQzcZoVTsZrXLJ3gH6g4ADaL2ZyvA0sequ5MNRhTHEM= =QlyD -END PGP SIGNATURE- --- No new revisions were added by this update. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10946. AbstractCSQueue: Create separate class for constructing Queue API objects. Contributed by Peter Szucs
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 5440c75c4ab YARN-10946. AbstractCSQueue: Create separate class for constructing Queue API objects. Contributed by Peter Szucs 5440c75c4ab is described below commit 5440c75c4abee87aeff6da3c7e3e586978cd27dc Author: Szilard Nemeth AuthorDate: Thu Dec 1 15:11:58 2022 +0100 YARN-10946. AbstractCSQueue: Create separate class for constructing Queue API objects. Contributed by Peter Szucs --- .../scheduler/capacity/AbstractCSQueue.java| 80 +- .../scheduler/capacity/CSQueueInfoProvider.java| 117 + 2 files changed, 118 insertions(+), 79 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 1a5a1ce0fd4..7a3ec4f6f4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -29,10 +29,8 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; -import org.apache.hadoop.yarn.api.records.QueueConfigurations; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueState; -import org.apache.hadoop.yarn.api.records.QueueStatistics; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -584,83 +582,7 @@ public abstract class AbstractCSQueue implements CSQueue { // from schedulerApplicationAttempt, to avoid deadlock, sacrifice // consistency here. // TODO, improve this -QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); -queueInfo.setQueueName(queuePath.getLeafName()); -queueInfo.setQueuePath(queuePath.getFullPath()); - queueInfo.setAccessibleNodeLabels(queueNodeLabelsSettings.getAccessibleNodeLabels()); -queueInfo.setCapacity(queueCapacities.getCapacity()); -queueInfo.setMaximumCapacity(queueCapacities.getMaximumCapacity()); -queueInfo.setQueueState(getState()); - queueInfo.setDefaultNodeLabelExpression(queueNodeLabelsSettings.getDefaultLabelExpression()); -queueInfo.setCurrentCapacity(getUsedCapacity()); -queueInfo.setQueueStatistics(getQueueStatistics()); -queueInfo.setPreemptionDisabled(preemptionSettings.isPreemptionDisabled()); -queueInfo.setIntraQueuePreemptionDisabled( -getIntraQueuePreemptionDisabled()); -queueInfo.setQueueConfigurations(getQueueConfigurations()); -queueInfo.setWeight(queueCapacities.getWeight()); - queueInfo.setMaxParallelApps(queueAppLifetimeSettings.getMaxParallelApps()); -return queueInfo; - } - - public QueueStatistics getQueueStatistics() { -// Deliberately doesn't use lock here, because this method will be invoked -// from schedulerApplicationAttempt, to avoid deadlock, sacrifice -// consistency here. -// TODO, improve this -QueueStatistics stats = recordFactory.newRecordInstance( -QueueStatistics.class); -stats.setNumAppsSubmitted(getMetrics().getAppsSubmitted()); -stats.setNumAppsRunning(getMetrics().getAppsRunning()); -stats.setNumAppsPending(getMetrics().getAppsPending()); -stats.setNumAppsCompleted(getMetrics().getAppsCompleted()); -stats.setNumAppsKilled(getMetrics().getAppsKilled()); -stats.setNumAppsFailed(getMetrics().getAppsFailed()); -stats.setNumActiveUsers(getMetrics().getActiveUsers()); -stats.setAvailableMemoryMB(getMetrics().getAvailableMB()); -stats.setAllocatedMemoryMB(getMetrics().getAllocatedMB()); -stats.setPendingMemoryMB(getMetrics().getPendingMB()); -stats.setReservedMemoryMB(getMetrics().getReservedMB()); -stats.setAvailableVCores(getMetrics().getAvailableVirtualCores()); -stats.setAllocatedVCores(getMetrics().getAllocatedVirtualCores()); -stats.setPendingVCores(getMetrics().getPendingVirtualCores()); -stats.setReservedVCores(getMetrics().getReservedVirtualCores()); -stats.setPendingContainers(getMetrics().getPendingContainers()); -stats.setAllocatedContainers(getMetrics().getA