[1/3] hbase-site git commit: Published site at 93498ddc599fe15f4357acf742a2dcb8b2c84b9c.
Repository: hbase-site Updated Branches: refs/heads/asf-site 57e2bad6f -> 8bbf0a2dc http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8bbf0a2d/book.html -- diff --git a/book.html b/book.html index 6c9b14f..09ab4cb 100644 --- a/book.html +++ b/book.html @@ -28110,11 +28110,14 @@ If a slave cluster does run out of room, or is inaccessible for other reasons, i Consistency Across Replicated Clusters -How your application builds on top of the HBase API matters when replication is in play. HBase’s replication system provides at-least-once delivery of client edits for an enabled column family to each configured destination cluster. In the event of failure to reach a given destination, the replication system will retry sending edits in a way that might repeat a given message. Further more, there is not a guaranteed order of delivery for client edits. In the event of a RegionServer failing, recovery of the replication queue happens independent of recovery of the individual regions that server was previously handling. This means that it is possible for the not-yet-replicated edits to be serviced by a RegionServer that is currently slower to replicate than the one that handles edits from after the failure. +How your application builds on top of the HBase API matters when replication is in play. HBase’s replication system provides at-least-once delivery of client edits for an enabled column family to each configured destination cluster. In the event of failure to reach a given destination, the replication system will retry sending edits in a way that might repeat a given message. HBase provides two ways of replication, one is the original replication and the other is serial replication. In the previous way of replication, there is not a guaranteed order of delivery for client edits. In the event of a RegionServer failing, recovery of the replication queue happens independent of recovery of the individual regions that server was previously handling. This means that it is possible for the not-yet-replicated edits to be serviced by a RegionServer that is currently slower to replicate than the one that handles edits from after the failure. The combination of these two properties (at-least-once delivery and the lack of message ordering) means that some destination clusters may end up in a different state if your application makes use of operations that are not idempotent, e.g. Increments. + +To solve the problem, HBase now supports serial replication, which sends edits to destination cluster as the order of requests from client. See Serial Replication. + @@ -28197,6 +28200,10 @@ Create tables with the same names and column families on both the source and des LOG.info("Replicating "+clusterId + " -> " + peerClusterId); + +Serial Replication Configuration +See Serial Replication + Cluster Management Commands @@ -28248,7 +28255,48 @@ replication as long as peers exist. -151.3. Verifying Replicated Data +151.3. Serial Replication + +Note: this feature is introduced in HBase 2.1 + + +Function of serial replication +Serial replication supports to push logs to the destination cluster in the same order as logs reach to the source cluster. + + +Why need serial replication? +In replication of HBase, we push mutations to destination cluster by reading WAL in each region server. We have a queue for WAL files so we can read them in order of creation time. However, when region-move or RS failure occurs in source cluster, the hlog entries that are not pushed before region-move or RS-failure will be pushed by original RS(for region move) or another RS which takes over the remained hlog of dead RS(for RS failure), and the new entries for the same region(s) will be pushed by the RS which now serves the region(s), but they push the hlog entries of a same region concurrently without coordination. + + +This treatment can possibly lead to data inconsistency between source and destination clusters: + + + + +there are put and then delete written to source cluster. + + +due to region-move / RS-failure, they are pushed by different replication-source threads to peer cluster. + + +if delete is pushed to peer cluster before put, and flush and major-compact occurs in peer cluster before put is pushed to peer cluster, the delete is collected and the put remains in peer cluster, but in source cluster the put is masked by the delete, hence data inconsistency between source and destination clusters. + + + + +Serial replication configuration + + +Set the serial flag to true for a repliation peer. You can either set it to true when creating a replication peer, or change it to true later. + + + + +The serial replication feature had been done firstly in https://issues.apache.org/jira/browse/HBASE-9465";>HBASE-9465 and then reverted and redone in https://issues.apache.org/jira/browse/HBASE-20046";>HBASE-200
[3/3] hbase-site git commit: Published site at 93498ddc599fe15f4357acf742a2dcb8b2c84b9c.
Published site at 93498ddc599fe15f4357acf742a2dcb8b2c84b9c. Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/8bbf0a2d Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/8bbf0a2d Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/8bbf0a2d Branch: refs/heads/asf-site Commit: 8bbf0a2dcdb359780485576aa6c6a9190e632252 Parents: 57e2bad Author: jenkins Authored: Tue Apr 10 14:10:49 2018 + Committer: jenkins Committed: Tue Apr 10 14:10:49 2018 + -- acid-semantics.html | 4 +- apache_hbase_reference_guide.pdf| 22707 + book.html |76 +- bulk-loads.html | 4 +- checkstyle-aggregate.html | 4 +- coc.html| 4 +- dependencies.html | 4 +- dependency-convergence.html | 4 +- dependency-info.html| 4 +- dependency-management.html | 4 +- devapidocs/constant-values.html | 4 +- .../org/apache/hadoop/hbase/Version.html| 4 +- export_control.html | 4 +- index.html | 6 +- integration.html| 4 +- issue-tracking.html | 4 +- license.html| 4 +- mail-lists.html | 4 +- metrics.html| 4 +- old_news.html | 4 +- plugin-management.html | 4 +- plugins.html| 4 +- poweredbyhbase.html | 4 +- project-info.html | 4 +- project-reports.html| 4 +- project-summary.html| 4 +- pseudo-distributed.html | 4 +- replication.html| 4 +- resources.html | 4 +- source-repository.html | 4 +- sponsors.html | 4 +- supportingprojects.html | 4 +- team-list.html | 4 +- 33 files changed, 11809 insertions(+), 11100 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8bbf0a2d/acid-semantics.html -- diff --git a/acid-semantics.html b/acid-semantics.html index 1f5f2f6..b89c9ad 100644 --- a/acid-semantics.html +++ b/acid-semantics.html @@ -7,7 +7,7 @@ - + Apache HBase – Apache HBase (TM) ACID Properties @@ -601,7 +601,7 @@ under the License. --> https://www.apache.org/";>The Apache Software Foundation. All rights reserved. - Last Published: 2018-04-09 + Last Published: 2018-04-10
hbase-site git commit: INFRA-10751 Empty commit
Repository: hbase-site Updated Branches: refs/heads/asf-site 8bbf0a2dc -> 5cfc2e823 INFRA-10751 Empty commit Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/5cfc2e82 Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/5cfc2e82 Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/5cfc2e82 Branch: refs/heads/asf-site Commit: 5cfc2e82305236cd66413b07cf52ea40646ada60 Parents: 8bbf0a2 Author: jenkins Authored: Tue Apr 10 14:11:00 2018 + Committer: jenkins Committed: Tue Apr 10 14:11:00 2018 + -- --
[2/3] hbase-site git commit: Published site at 93498ddc599fe15f4357acf742a2dcb8b2c84b9c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8bbf0a2d/apache_hbase_reference_guide.pdf -- diff --git a/apache_hbase_reference_guide.pdf b/apache_hbase_reference_guide.pdf index 6be0202..2a5ee85 100644 --- a/apache_hbase_reference_guide.pdf +++ b/apache_hbase_reference_guide.pdf @@ -5,16 +5,16 @@ /Author (Apache HBase Team) /Creator (Asciidoctor PDF 1.5.0.alpha.15, based on Prawn 2.2.2) /Producer (Apache HBase Team) -/ModDate (D:20180409172027+00'00') -/CreationDate (D:20180409172027+00'00') +/ModDate (D:20180410140832+00'00') +/CreationDate (D:20180410140832+00'00') >> endobj 2 0 obj << /Type /Catalog /Pages 3 0 R /Names 26 0 R -/Outlines 4588 0 R -/PageLabels 4814 0 R +/Outlines 4595 0 R +/PageLabels 4821 0 R /PageMode /UseOutlines /OpenAction [7 0 R /FitH 842.89] /ViewerPreferences << /DisplayDocTitle true @@ -23,8 +23,8 @@ endobj endobj 3 0 obj << /Type /Pages -/Count 717 -/Kids [7 0 R 12 0 R 14 0 R 16 0 R 18 0 R 20 0 R 22 0 R 24 0 R 44 0 R 47 0 R 50 0 R 54 0 R 61 0 R 63 0 R 67 0 R 69 0 R 71 0 R 78 0 R 81 0 R 83 0 R 89 0 R 92 0 R 94 0 R 96 0 R 103 0 R 109 0 R 114 0 R 116 0 R 132 0 R 137 0 R 144 0 R 153 0 R 161 0 R 170 0 R 181 0 R 185 0 R 187 0 R 191 0 R 200 0 R 209 0 R 217 0 R 226 0 R 231 0 R 240 0 R 248 0 R 257 0 R 270 0 R 277 0 R 287 0 R 295 0 R 303 0 R 310 0 R 318 0 R 324 0 R 330 0 R 337 0 R 345 0 R 356 0 R 365 0 R 377 0 R 385 0 R 393 0 R 400 0 R 409 0 R 417 0 R 427 0 R 435 0 R 442 0 R 451 0 R 463 0 R 472 0 R 479 0 R 487 0 R 495 0 R 504 0 R 511 0 R 516 0 R 520 0 R 525 0 R 529 0 R 545 0 R 556 0 R 560 0 R 575 0 R 580 0 R 585 0 R 587 0 R 589 0 R 592 0 R 594 0 R 596 0 R 604 0 R 610 0 R 615 0 R 620 0 R 627 0 R 637 0 R 645 0 R 649 0 R 653 0 R 655 0 R 665 0 R 679 0 R 687 0 R 694 0 R 706 0 R 714 0 R 730 0 R 744 0 R 750 0 R 756 0 R 759 0 R 763 0 R 767 0 R 770 0 R 773 0 R 775 0 R 778 0 R 782 0 R 784 0 R 788 0 R 794 0 R 799 0 R 803 0 R 806 0 R 812 0 R 814 0 R 818 0 R 826 0 R 828 0 R 831 0 R 834 0 R 837 0 R 840 0 R 854 0 R 862 0 R 872 0 R 883 0 R 889 0 R 899 0 R 910 0 R 913 0 R 917 0 R 920 0 R 925 0 R 934 0 R 942 0 R 946 0 R 950 0 R 955 0 R 959 0 R 961 0 R 976 0 R 987 0 R 992 0 R 999 0 R 1002 0 R 1010 0 R 1018 0 R 1023 0 R 1028 0 R 1033 0 R 1035 0 R 1037 0 R 1039 0 R 1049 0 R 1057 0 R 1061 0 R 1068 0 R 1075 0 R 1083 0 R 1087 0 R 1093 0 R 1098 0 R 1106 0 R 1110 0 R 1115 0 R 1117 0 R 1123 0 R 1131 0 R 1138 0 R 1145 0 R 1156 0 R 1160 0 R 1162 0 R 1164 0 R 1168 0 R 1171 0 R 1176 0 R 1179 0 R 1191 0 R 1195 0 R 1201 0 R 1209 0 R 1214 0 R 1218 0 R 1222 0 R 1224 0 R 1227 0 R 1230 0 R 1233 0 R 1237 0 R 1241 0 R 1245 0 R 1250 0 R 1254 0 R 1257 0 R 1259 0 R 1269 0 R 1272 0 R 1280 0 R 1289 0 R 1295 0 R 1299 0 R 1301 0 R 1312 0 R 1315 0 R 1321 0 R 1329 0 R 1332 0 R 1339 0 R 1347 0 R 1349 0 R 1351 0 R 1360 0 R 1362 0 R 1364 0 R 1367 0 R 1369 0 R 1371 0 R 1373 0 R 1375 0 R 1378 0 R 1382 0 R 1387 0 R 1389 0 R 1391 0 R 1393 0 R 1398 0 R 1405 0 R 1411 0 R 1414 0 R 1416 0 R 1419 0 R 1423 0 R 1427 0 R 1430 0 R 1432 0 R 1434 0 R 1437 0 R 1442 0 R 1448 0 R 1456 0 R 1470 0 R 1484 0 R 1487 0 R 1492 0 R 1505 0 R 1510 0 R 1525 0 R 1533 0 R 1537 0 R 1546 0 R 1561 0 R 1575 0 R 1587 0 R 1592 0 R 1598 0 R 1608 0 R 1613 0 R 1618 0 R 1626 0 R 1629 0 R 1638 0 R 1644 0 R 1649 0 R 1661 0 R 1666 0 R 1672 0 R 1674 0 R 1680 0 R 1688 0 R 1696 0 R 1700 0 R 1702 0 R 1704 0 R 1716 0 R 1722 0 R 1731 0 R 1737 0 R 1750 0 R 1756 0 R 1762 0 R 1773 0 R 1779 0 R 1784 0 R 1789 0 R 1792 0 R 1795 0 R 1800 0 R 1805 0 R 1812 0 R 1816 0 R 1821 0 R 1830 0 R 1835 0 R 1840 0 R 1842 0 R 1851 0 R 1858 0 R 1864 0 R 1869 0 R 1873 0 R 1876 0 R 1881 0 R 1886 0 R 1892 0 R 1894 0 R 1896 0 R 1899 0 R 1910 0 R 1913 0 R 1920 0 R 1928 0 R 1933 0 R 1937 0 R 1942 0 R 1944 0 R 1947 0 R 1952 0 R 1955 0 R 1957 0 R 1960 0 R 1963 0 R 1966 0 R 1976 0 R 1981 0 R 1986 0 R 1988 0 R 1996 0 R 2003 0 R 2010 0 R 2016 0 R 2021 0 R 2023 0 R 2032 0 R 2042 0 R 2052 0 R 2058 0 R 2065 0 R 2067 0 R 2072 0 R 2074 0 R 2076 0 R 2080 0 R 2083 0 R 2086 0 R 2091 0 R 2095 0 R 2106 0 R 2109 0 R 2114 0 R 2117 0 R 2119 0 R 2124 0 R 2134 0 R 2136 0 R 2138 0 R 2140 0 R 2142 0 R 2145 0 R 2147 0 R 2149 0 R 2152 0 R 2154 0 R 2156 0 R 2161 0 R 2166 0 R 2175 0 R 2177 0 R 2179 0 R 2185 0 R 2187 0 R 2192 0 R 2194 0 R 2196 0 R 2203 0 R 2208 0 R 2212 0 R 2217 0 R 2221 0 R 2223 0 R 2225 0 R 2229 0 R 2232 0 R 2234 0 R 2236 0 R 2240 0 R 2242 0 R 2245 0 R 2247 0 R 2249 0 R 2251 0 R 2258 0 R 2261 0 R 2266 0 R 2268 0 R 2270 0 R 2272 0 R 2274 0 R 2282 0 R 2293 0 R 2307 0 R 2318 0 R 2322 0 R 2327 0 R 2331 0 R 2334 0 R 2339 0 R 2345 0 R 2347 0 R 2350 0 R 2352 0 R 2354 0 R 2356 0 R 2361 0 R 2363 0 R 2376 0 R 2379 0 R 2387 0 R 2393 0 R 2405 0 R 2419 0 R 2432 0 R 2449 0 R 2453 0 R 2455 0 R 2459 0 R 2477 0 R 2483 0 R 2495 0 R 2499 0 R 2503 0 R 2512 0 R 2524 0 R 2529 0 R 2539 0 R 2552 0 R 2571 0 R 2580 0 R 2583 0 R 2592 0 R 2609 0 R 2616 0 R 2619 0 R 2624 0 R 2628 0 R 2631 0 R 2640 0 R 2648 0
[1/2] hbase git commit: HBASE-17553 Put up 2.0.0RC0; Update CHANGELOG and RELEASENOTES
Repository: hbase Updated Branches: refs/heads/branch-2.0 9ef75b96d -> 5d85e7d78 HBASE-17553 Put up 2.0.0RC0; Update CHANGELOG and RELEASENOTES Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1f01327f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1f01327f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1f01327f Branch: refs/heads/branch-2.0 Commit: 1f01327fcba8c0f31d3804338b0a4b0c520ffbde Parents: 9ef75b9 Author: Michael Stack Authored: Tue Apr 10 09:49:25 2018 -0700 Committer: Michael Stack Committed: Tue Apr 10 09:49:25 2018 -0700 -- CHANGES.md | 1 + RELEASENOTES.md | 20 2 files changed, 21 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/1f01327f/CHANGES.md -- diff --git a/CHANGES.md b/CHANGES.md index 5c7f90f..7da3031 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -47,6 +47,7 @@ comment to preserve continuity of the CHANGELOG. | JIRA | Summary | Priority | Component | |: |: | :--- |: | +| [HBASE-20276](https://issues.apache.org/jira/browse/HBASE-20276) | [shell] Revert shell REPL change and document | Blocker | documentation, shell | | [HBASE-16459](https://issues.apache.org/jira/browse/HBASE-16459) | Remove unused hbase shell --format option | Trivial | shell | | [HBASE-19128](https://issues.apache.org/jira/browse/HBASE-19128) | Purge Distributed Log Replay from codebase, configurations, text; mark the feature as unsupported, broken. | Major | documentation | | [HBASE-19504](https://issues.apache.org/jira/browse/HBASE-19504) | Add TimeRange support into checkAndMutate | Major | . | http://git-wip-us.apache.org/repos/asf/hbase/blob/1f01327f/RELEASENOTES.md -- diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 41240ce..709cd84 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -48,6 +48,26 @@ These release notes cover new developer and user-facing incompatibilities, impor --- +* [HBASE-18828](https://issues.apache.org/jira/browse/HBASE-18828) | *Blocker* | **[2.0] Generate CHANGES.txt** + +Moves us over to yetus releasedocmaker tooling generating CHANGES. CHANGES is not markdown (CHANGES.md) as opposed to CHANGES.txt. We've also added a new RELEASENOTES.md that lists JIRA release notes (courtesy of releasedocmaker). + +CHANGES/RELEASENOTES are current as of now. Will need a 'freshening' when we cut the RC. + + +--- + +* [HBASE-20276](https://issues.apache.org/jira/browse/HBASE-20276) | *Blocker* | **[shell] Revert shell REPL change and document** + + +The HBase shell now behaves as it did prior to the changes that started in HBASE-15965. Namely, some shell commands return values that may be further manipulated within the shell's IRB session. + +The command line option `--return-values` is no longer acted on by the shell since it now always behaves as it did when passed this parameter. Passing the option results in a harmless warning about this change. + +Users who wish to maintain the behavior seen in the 1.4.0-1.4.2 releases of the HBase shell should refer to the section _irbrc_ in the reference guide for how to configure their IRB session to avoid echoing expression results to the console. + +--- + * [HBASE-14175](https://issues.apache.org/jira/browse/HBASE-14175) | *Critical* | **Adopt releasedocmaker for better generated release notes** We will use yetus releasedocmaker to make our changes doc from here on out. A CHANGELOG.md will replace our current CHANGES.txt. Adjacent, we'll keep up a RELEASENOTES.md doc courtesy of releasedocmaker.
[2/2] hbase git commit: HBASE-20149 Purge dev javadoc from bin tarball (or make a separate tarball of javadoc)
HBASE-20149 Purge dev javadoc from bin tarball (or make a separate tarball of javadoc) Instead point back to the website where version-specific dev apis are hosted. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5d85e7d7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5d85e7d7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5d85e7d7 Branch: refs/heads/branch-2.0 Commit: 5d85e7d78c27b92c253039349c966a6118c7f237 Parents: 1f01327 Author: Michael Stack Authored: Fri Apr 6 14:25:07 2018 -0700 Committer: Michael Stack Committed: Tue Apr 10 09:56:03 2018 -0700 -- hbase-assembly/src/main/assembly/components.xml | 5 + src/main/site/site.xml | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/5d85e7d7/hbase-assembly/src/main/assembly/components.xml -- diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml index fe57078..2eb16e7 100644 --- a/hbase-assembly/src/main/assembly/components.xml +++ b/hbase-assembly/src/main/assembly/components.xml @@ -26,6 +26,11 @@ ${project.basedir}/../target/site docs + + +**/*devapidocs/** + http://git-wip-us.apache.org/repos/asf/hbase/blob/5d85e7d7/src/main/site/site.xml -- diff --git a/src/main/site/site.xml b/src/main/site/site.xml index 400ae8c..f036702 100644 --- a/src/main/site/site.xml +++ b/src/main/site/site.xml @@ -100,8 +100,8 @@ - - + https://hbase.apache.org/2.0/devapidocs/index.html"; target="_blank" /> + https://hbase.apache.org/2.0/testdevapidocs/index.html"; target="_blank" /> http://abloz.com/hbase/book.html"; target="_blank" />
hbase git commit: HBASE-20149 Purge dev javadoc from bin tarball (or make a separate tarball of javadoc)
Repository: hbase Updated Branches: refs/heads/master 93498ddc5 -> 95ca38a53 HBASE-20149 Purge dev javadoc from bin tarball (or make a separate tarball of javadoc) Instead point back to the website where version-specific dev apis are hosted. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/95ca38a5 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/95ca38a5 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/95ca38a5 Branch: refs/heads/master Commit: 95ca38a539e983a59d08187bf36e7fadbbf0 Parents: 93498dd Author: Michael Stack Authored: Fri Apr 6 14:25:07 2018 -0700 Committer: Michael Stack Committed: Tue Apr 10 10:04:22 2018 -0700 -- hbase-assembly/src/main/assembly/components.xml | 5 + src/site/site.xml | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/95ca38a5/hbase-assembly/src/main/assembly/components.xml -- diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml index e6c45e0..59502d6 100644 --- a/hbase-assembly/src/main/assembly/components.xml +++ b/hbase-assembly/src/main/assembly/components.xml @@ -26,6 +26,11 @@ ${project.basedir}/../target/site docs + + +**/*devapidocs/** + http://git-wip-us.apache.org/repos/asf/hbase/blob/95ca38a5/src/site/site.xml -- diff --git a/src/site/site.xml b/src/site/site.xml index ce7f8ae..808801b 100644 --- a/src/site/site.xml +++ b/src/site/site.xml @@ -100,8 +100,8 @@ - - + https://hbase.apache.org/2.0/devapidocs/index.html"; target="_blank" /> + https://hbase.apache.org/2.0/testdevapidocs/index.html"; target="_blank" /> http://abloz.com/hbase/book.html"; target="_blank" />
hbase git commit: HBASE-17553 Put up 2.0.0RC0; Update CHANGELOG and RELEASENOTES; ADDENDUM Add last commit
Repository: hbase Updated Branches: refs/heads/branch-2.0 5d85e7d78 -> 011dd2dae HBASE-17553 Put up 2.0.0RC0; Update CHANGELOG and RELEASENOTES; ADDENDUM Add last commit Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/011dd2da Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/011dd2da Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/011dd2da Branch: refs/heads/branch-2.0 Commit: 011dd2dae33456b3a2bcc2513e9fdd29de23be46 Parents: 5d85e7d Author: Michael Stack Authored: Tue Apr 10 10:03:24 2018 -0700 Committer: Michael Stack Committed: Tue Apr 10 10:03:24 2018 -0700 -- CHANGES.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/011dd2da/CHANGES.md -- diff --git a/CHANGES.md b/CHANGES.md index 7da3031..44e5068 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -47,6 +47,7 @@ comment to preserve continuity of the CHANGELOG. | JIRA | Summary | Priority | Component | |: |: | :--- |: | +| [HBASE-20149](https://issues.apache.org/jira/browse/HBASE-20149) | Purge dev javadoc from bin tarball (or make a separate tarball of javadoc) | Critical | build, community, documentation | | [HBASE-20276](https://issues.apache.org/jira/browse/HBASE-20276) | [shell] Revert shell REPL change and document | Blocker | documentation, shell | | [HBASE-16459](https://issues.apache.org/jira/browse/HBASE-16459) | Remove unused hbase shell --format option | Trivial | shell | | [HBASE-19128](https://issues.apache.org/jira/browse/HBASE-19128) | Purge Distributed Log Replay from codebase, configurations, text; mark the feature as unsupported, broken. | Major | documentation |
hbase git commit: HBASE-20149 Purge dev javadoc from bin tarball (or make a separate tarball of javadoc)
Repository: hbase Updated Branches: refs/heads/branch-2 1f674e2b2 -> 37227b6bb HBASE-20149 Purge dev javadoc from bin tarball (or make a separate tarball of javadoc) Instead point back to the website where version-specific dev apis are hosted. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/37227b6b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/37227b6b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/37227b6b Branch: refs/heads/branch-2 Commit: 37227b6bb3a9debbfaadb6934184c2b35f9404e3 Parents: 1f674e2 Author: Michael Stack Authored: Fri Apr 6 14:25:07 2018 -0700 Committer: Michael Stack Committed: Tue Apr 10 10:03:58 2018 -0700 -- hbase-assembly/src/main/assembly/components.xml | 5 + src/main/site/site.xml | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/37227b6b/hbase-assembly/src/main/assembly/components.xml -- diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml index e6c45e0..59502d6 100644 --- a/hbase-assembly/src/main/assembly/components.xml +++ b/hbase-assembly/src/main/assembly/components.xml @@ -26,6 +26,11 @@ ${project.basedir}/../target/site docs + + +**/*devapidocs/** + http://git-wip-us.apache.org/repos/asf/hbase/blob/37227b6b/src/main/site/site.xml -- diff --git a/src/main/site/site.xml b/src/main/site/site.xml index 400ae8c..f036702 100644 --- a/src/main/site/site.xml +++ b/src/main/site/site.xml @@ -100,8 +100,8 @@ - - + https://hbase.apache.org/2.0/devapidocs/index.html"; target="_blank" /> + https://hbase.apache.org/2.0/testdevapidocs/index.html"; target="_blank" /> http://abloz.com/hbase/book.html"; target="_blank" />
[hbase] Git Push Summary
Repository: hbase Updated Tags: refs/tags/2.0.0RC0 [created] e6f4356cd
svn commit: r26275 - in /dev/hbase: hbase-2.0.0-beta-2.RC0/ hbase-2.0.0RC0/
Author: stack Date: Tue Apr 10 19:18:32 2018 New Revision: 26275 Log: Add hbase-2.0.0RC0 and remove hbase-2.0.0-beta-2.RC0 Added: dev/hbase/hbase-2.0.0RC0/ dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-bin.tar.gz (with props) dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-bin.tar.gz.asc dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-bin.tar.gz.sha512 dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-src.tar.gz (with props) dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-src.tar.gz.asc dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-src.tar.gz.sha512 Removed: dev/hbase/hbase-2.0.0-beta-2.RC0/ Added: dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-bin.tar.gz == Binary file - no diff available. Propchange: dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-bin.tar.gz -- svn:mime-type = application/octet-stream Added: dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-bin.tar.gz.asc == --- dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-bin.tar.gz.asc (added) +++ dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-bin.tar.gz.asc Tue Apr 10 19:18:32 2018 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIcBAABCAAGBQJazQPuAAoJEJgWx/yKzJPS34cQAKb+O9OFLr0+1Kbsw4rJ2g7U +x3V0uzPMJOoLufYw9tAdMsh24KQfA+4ZPQeFOJl1FdnNVXlaNn/4kRdDPGCXqITM +rXOKdP36JY9BtLbI5d9hWOc4451Mm2C4yPuZ1jZUZUBCDf6nm9N2Ya8kw2EXeq9k +MbCUCVHpWaYddTUDQ8Vgb4rRFD/lGAqQL33l42/HAw3vE8ivsDtdQcY4uz51lNnF +GyK+ixLASoAn3XdcNM5wga3RgpxJ20EIGX0DnQf8GvIBRqSWaEEPqYtZFSTpvwfn +vPPOEccZxTCNSy8h+10EGk6yuLKNlYM42+HHuIcahnHkDg8xI/fM5KPfDjdPgU6p +WNIzGh5LPJ1MfDq72M11Rk4sVxXC++t+nYUTpeuYybyRAbBXKaTLMZ83y5sFH2oP +MlnrQ/Z7E9rTjQlvvE/F3Pp5WuwL//vnlAYNmoqgPq8vRveh5s5I3zisKvCNOu4q +oln0Bd1TVr2bZB67mQMclA9jaN4fNIzji/1DJ0JoPmQ1ZdMLLJJOJYk30AtvrMr9 +rHOoYB88Vtr/LebkdVCxI72TbB7Ii299+xy10KfOlB2DC2hNp5Q3wOrH9QAyE947 +2/Uj/rMqBt6wWQcZsUQCnNjIlE3WSFfumAc3GvWTcThpxUJNAfcJtVUT6JUTPsLP +4dPZEiuC+CRJEGqy2Xne +=6Trk +-END PGP SIGNATURE- Added: dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-bin.tar.gz.sha512 == --- dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-bin.tar.gz.sha512 (added) +++ dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-bin.tar.gz.sha512 Tue Apr 10 19:18:32 2018 @@ -0,0 +1,3 @@ +hbase-2.0.0-bin.tar.gz: 98D62B05 2F014C82 C8A7B76D 5B25FEBB C5A3853C DD92EC19 +231808E1 8CBAA068 20F18D5B 754303EB ADF98964 B15FA61D +9A784339 6C9DBAFA 4E4FFBBD 8096403B Added: dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-src.tar.gz == Binary file - no diff available. Propchange: dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-src.tar.gz -- svn:mime-type = application/octet-stream Added: dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-src.tar.gz.asc == --- dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-src.tar.gz.asc (added) +++ dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-src.tar.gz.asc Tue Apr 10 19:18:32 2018 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIcBAABCAAGBQJazQPvAAoJEJgWx/yKzJPSdsUP/A54IbnRYBbIS7dsBzoyqwhL +Qts0LqBzII/Bw4PX5kamzGSBzY3KPOx6fJ6nSthpJ6/X01uNtAi94RjV6wWminLc +RriYNr0g5v2YNEk5BhVWKvbAIVWyJSKRjC2eHmM+dTTZpCY3pET0PikZZyvnPBD1 +E6ZUca14KSTJwB28AOXpJrPMWYSowqTQRYL55OlHyPvDgSXCa5KyDjvUm1b3Y61R +qEOgLMIcCJoot0nR/y3nEMfyOI5XEolgqmd5Z9tF7yrBWK3hrPHIpfSmZHsKiinL +pwHX1LiRrXBsYRExHhEQyuVBHjZOKLL+ZHfxyk4NlXZn3YkPGXpAGX6wu3FlBvzq +gdLTh/0loQuedH99g8DPL670VOAcjpV+q0ssJKhhDGSjqkFAv3TmmzgUEge9/5kx +KlcZ3+nB3P9UAP1XD2asJF5/yYASgR29fJQ5xjHcQXST3jVbnn8ymIF4KTS8q7k3 +UaQ0s9+TIVgPB7IHWGWfxFCoG/E127F8jwj4fY7xrg3OOf2WIfxxAMCk7lgB2ogh +du29HY02eLIdcBV01oxCsV9i5Z0oEfxqaWPqnBG/+tIFHcdCo1qBEPFSFYaduwpn +PqONhaICH6EFEKxJVHhPgw+vAXT5G1xPtaFM2PglFMrCyQ4suPD6g3XAghNhnVVg +YHH1MaLyFwgkQlORBvOL +=cRiz +-END PGP SIGNATURE- Added: dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-src.tar.gz.sha512 == --- dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-src.tar.gz.sha512 (added) +++ dev/hbase/hbase-2.0.0RC0/hbase-2.0.0-src.tar.gz.sha512 Tue Apr 10 19:18:32 2018 @@ -0,0 +1,3 @@ +hbase-2.0.0-src.tar.gz: 15892A83 38B47779 B5451545 F02F3C4A 857E2F4D AB091C78 +C7F2CFCD 32B13D69 D4E210D9 B9AF5BB0 97350E79 9E6CC528 +5F648A7A 7150F7F7 694A1079 98DD403B
svn commit: r26276 [1/3] - in /dev/hbase/hbase-2.0.0RC0: CHANGES.md RELEASENOTES.md
Author: stack Date: Tue Apr 10 19:27:36 2018 New Revision: 26276 Log: Add CHANGELOG and RELEASENOTES Added: dev/hbase/hbase-2.0.0RC0/CHANGES.md dev/hbase/hbase-2.0.0RC0/RELEASENOTES.md
svn commit: r26276 [3/3] - in /dev/hbase/hbase-2.0.0RC0: CHANGES.md RELEASENOTES.md
Added: dev/hbase/hbase-2.0.0RC0/RELEASENOTES.md == --- dev/hbase/hbase-2.0.0RC0/RELEASENOTES.md (added) +++ dev/hbase/hbase-2.0.0RC0/RELEASENOTES.md Tue Apr 10 19:27:36 2018 @@ -0,0 +1,8249 @@ +# HBASE 2.0.0 Release Notes + + + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [HBASE-18828](https://issues.apache.org/jira/browse/HBASE-18828) | *Blocker* | **[2.0] Generate CHANGES.txt** + +Moves us over to yetus releasedocmaker tooling generating CHANGES. CHANGES is not markdown (CHANGES.md) as opposed to CHANGES.txt. We've also added a new RELEASENOTES.md that lists JIRA release notes (courtesy of releasedocmaker). + +CHANGES/RELEASENOTES are current as of now. Will need a 'freshening' when we cut the RC. + + +--- + +* [HBASE-20276](https://issues.apache.org/jira/browse/HBASE-20276) | *Blocker* | **[shell] Revert shell REPL change and document** + + +The HBase shell now behaves as it did prior to the changes that started in HBASE-15965. Namely, some shell commands return values that may be further manipulated within the shell's IRB session. + +The command line option `--return-values` is no longer acted on by the shell since it now always behaves as it did when passed this parameter. Passing the option results in a harmless warning about this change. + +Users who wish to maintain the behavior seen in the 1.4.0-1.4.2 releases of the HBase shell should refer to the section _irbrc_ in the reference guide for how to configure their IRB session to avoid echoing expression results to the console. + +--- + +* [HBASE-14175](https://issues.apache.org/jira/browse/HBASE-14175) | *Critical* | **Adopt releasedocmaker for better generated release notes** + +We will use yetus releasedocmaker to make our changes doc from here on out. A CHANGELOG.md will replace our current CHANGES.txt. Adjacent, we'll keep up a RELEASENOTES.md doc courtesy of releasedocmaker. + +Over in HBASE-18828 is where we are working through steps for the RM integrating this new tooling. + + +--- + +* [HBASE-16499](https://issues.apache.org/jira/browse/HBASE-16499) | *Critical* | **slow replication for small HBase clusters** + +Changed the default value for replication.source.ratio from 0.1 to 0.5. Which means now by default 50% of the total RegionServers in peer cluster(s) will participate in replication. + + +--- + +* [HBASE-16459](https://issues.apache.org/jira/browse/HBASE-16459) | *Trivial* | **Remove unused hbase shell --format option** + + + +The HBase `shell` command no longer recognizes the option `--format`. Previously this option only recognized the default value of 'console'. The default value is now always used. + + +--- + +* [HBASE-20259](https://issues.apache.org/jira/browse/HBASE-20259) | *Critical* | **Doc configs for in-memory-compaction and add detail to in-memory-compaction logging** + +Disables in-memory compaction as default. + +Adds logging of in-memory compaction configuration on creation. + +Adds a chapter to the refguide on this new feature. + + +--- + +* [HBASE-20282](https://issues.apache.org/jira/browse/HBASE-20282) | *Major* | **Provide short name invocations for useful tools** + +\`hbase regionsplitter\` is a new short invocation for \`hbase org.apache.hadoop.hbase.util.RegionSplitter\` + + +--- + +* [HBASE-20314](https://issues.apache.org/jira/browse/HBASE-20314) | *Major* | **Precommit build for master branch fails because of surefire fork fails** + +Upgrade surefire plugin to 2.21.0. + + +--- + +* [HBASE-20130](https://issues.apache.org/jira/browse/HBASE-20130) | *Critical* | **Use defaults (16020 & 16030) as base ports when the RS is bound to localhost** + + +When region servers bind to localhost (mostly in pseudo distributed mode), default ports (16020 & 16030) are used as base ports. This will support up to 9 instances of region servers by default with `local-regionservers.sh` script. If additional instances are needed, see the reference guide on how to deploy with a different range using the environment variables `HBASE_RS_BASE_PORT` and `HBASE_RS_INFO_BASE_PORT`. + + +--- + +* [HBASE-20111](https://issues.apache.org/jira/browse/HBASE-20111) | *Critical* | **Able to split region explicitly even on shouldSplit return false from split policy** + +When a split is requested on a Region, the RegionServer hosting that Region will now consult the configured SplitPolicy for that table when determining if a split of that Region is allowed. When a split is disallowed (due to the Region not being OPEN or the SplitPolicy denying the request), the operation will \*not\* be implicitly retried as it has previously done. Users will need to guard against and explicitly retry region split requests which are denied by the system. + + +--- + +* [HBASE-20223](https://issues.apache.org/jira/browse/HBASE-20223) | *Blocker* | **
svn commit: r26276 [2/3] - in /dev/hbase/hbase-2.0.0RC0: CHANGES.md RELEASENOTES.md
Added: dev/hbase/hbase-2.0.0RC0/CHANGES.md == --- dev/hbase/hbase-2.0.0RC0/CHANGES.md (added) +++ dev/hbase/hbase-2.0.0RC0/CHANGES.md Tue Apr 10 19:27:36 2018 @@ -0,0 +1,6060 @@ +# HBASE Changelog + + +## Release 2.0.0 - Unreleased (as of 2018-04-06) + +### INCOMPATIBLE CHANGES: + +| JIRA | Summary | Priority | Component | +|: |: | :--- |: | +| [HBASE-20149](https://issues.apache.org/jira/browse/HBASE-20149) | Purge dev javadoc from bin tarball (or make a separate tarball of javadoc) | Critical | build, community, documentation | +| [HBASE-20276](https://issues.apache.org/jira/browse/HBASE-20276) | [shell] Revert shell REPL change and document | Blocker | documentation, shell | +| [HBASE-16459](https://issues.apache.org/jira/browse/HBASE-16459) | Remove unused hbase shell --format option | Trivial | shell | +| [HBASE-19128](https://issues.apache.org/jira/browse/HBASE-19128) | Purge Distributed Log Replay from codebase, configurations, text; mark the feature as unsupported, broken. | Major | documentation | +| [HBASE-19504](https://issues.apache.org/jira/browse/HBASE-19504) | Add TimeRange support into checkAndMutate | Major | . | +| [HBASE-20119](https://issues.apache.org/jira/browse/HBASE-20119) | Introduce a pojo class to carry coprocessor information in order to make TableDescriptorBuilder accept multiple cp at once | Minor | . | +| [HBASE-19437](https://issues.apache.org/jira/browse/HBASE-19437) | Batch operation can't handle the null result for Append/Increment | Critical | Usability | +| [HBASE-19873](https://issues.apache.org/jira/browse/HBASE-19873) | Add a CategoryBasedTimeout ClassRule for all UTs | Major | . | +| [HBASE-19783](https://issues.apache.org/jira/browse/HBASE-19783) | Change replication peer cluster key/endpoint from a not-null value to null is not allowed | Minor | Replication | +| [HBASE-19483](https://issues.apache.org/jira/browse/HBASE-19483) | Add proper privilege check for rsgroup commands | Major | rsgroup, security | +| [HBASE-19492](https://issues.apache.org/jira/browse/HBASE-19492) | Add EXCLUDE\_NAMESPACE and EXCLUDE\_TABLECFS support to replication peer config | Major | . | +| [HBASE-19357](https://issues.apache.org/jira/browse/HBASE-19357) | Bucket cache no longer L2 for LRU cache | Major | . | +| [HBASE-19359](https://issues.apache.org/jira/browse/HBASE-19359) | Revisit the default config of hbase client retries number | Major | . | +| [HBASE-19092](https://issues.apache.org/jira/browse/HBASE-19092) | Make Tag IA.LimitedPrivate and expose for CPs | Critical | Coprocessors | +| [HBASE-19187](https://issues.apache.org/jira/browse/HBASE-19187) | Remove option to create on heap bucket cache | Minor | regionserver | +| [HBASE-19033](https://issues.apache.org/jira/browse/HBASE-19033) | Allow CP users to change versions and TTL before opening StoreScanner | Blocker | Coprocessors | +| [HBASE-19047](https://issues.apache.org/jira/browse/HBASE-19047) | CP exposed Scanner types should not extend Shipper | Critical | Coprocessors | +| [HBASE-18905](https://issues.apache.org/jira/browse/HBASE-18905) | Allow CPs to request flush on Region and know the completion of the requested flush | Major | Coprocessors | +| [HBASE-18410](https://issues.apache.org/jira/browse/HBASE-18410) | FilterList Improvement. | Major | Filters | +| [HBASE-18893](https://issues.apache.org/jira/browse/HBASE-18893) | Remove Add/Modify/DeleteColumnFamilyProcedure in favor of using ModifyTableProcedure | Major | Coprocessors, master | +| [HBASE-19067](https://issues.apache.org/jira/browse/HBASE-19067) | Do not expose getHDFSBlockDistribution in StoreFile | Major | Coprocessors | +| [HBASE-18989](https://issues.apache.org/jira/browse/HBASE-18989) | Polish the compaction related CP hooks | Major | Compaction, Coprocessors | +| [HBASE-19046](https://issues.apache.org/jira/browse/HBASE-19046) | RegionObserver#postCompactSelection Avoid passing shaded ImmutableList param | Major | Coprocessors | +| [HBASE-19001](https://issues.apache.org/jira/browse/HBASE-19001) | Remove the hooks in RegionObserver which are designed to construct a StoreScanner which is marked as IA.Private | Major | Coprocessors | +| [HBASE-14247](https://issues.apache.org/jira/browse/HBASE-14247) | Separate the old WALs into different regionserver directories | Critical | wal | +| [HBASE-18183](https://issues.apache.org/jira/browse/HBASE-18183) | Region interface cleanup for CP expose | Major | Coprocessors | +| [HBASE-18878](https://issues.apache.org/jira/browse/HBASE-18878) | Use Optional\ return types when T can be null | Major | Coprocessors | +| [HBASE-18649](https://issues.apache.org/jira/browse/HBASE-18649) | Deprecate KV Usage in MR to move to Cells in 3.0 | Major | API, mapreduce | +| [HBASE-18897](https://issues.apache.org/jira/browse/HBASE-18897) | Substitute MemStore for Memstore |
svn commit: r26277 - in /dev/hbase/hbase-2.0.0RC0: CHANGES.md RELEASENOTES.md
Author: stack Date: Tue Apr 10 19:44:10 2018 New Revision: 26277 Log: Removed changes and release notes. Not needed. Removed: dev/hbase/hbase-2.0.0RC0/CHANGES.md dev/hbase/hbase-2.0.0RC0/RELEASENOTES.md
hbase-site git commit: Add registration link for HBaseCon 2018
Repository: hbase-site Updated Branches: refs/heads/asf-site 5cfc2e823 -> e5b323a78 Add registration link for HBaseCon 2018 Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/e5b323a7 Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/e5b323a7 Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/e5b323a7 Branch: refs/heads/asf-site Commit: e5b323a787da5d42ccb3536e8ec84776e1a800e5 Parents: 5cfc2e8 Author: Josh Elser Authored: Tue Apr 10 18:08:14 2018 -0400 Committer: Josh Elser Committed: Tue Apr 10 18:08:14 2018 -0400 -- hbasecon-2018/feed.xml | 2 +- hbasecon-2018/index.html | 8 +++- 2 files changed, 4 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase-site/blob/e5b323a7/hbasecon-2018/feed.xml -- diff --git a/hbasecon-2018/feed.xml b/hbasecon-2018/feed.xml index 4db31e4..605fd42 100644 --- a/hbasecon-2018/feed.xml +++ b/hbasecon-2018/feed.xml @@ -4,7 +4,7 @@ https://jekyllrb.com/"; version="3.7.3">Jekyll https://hbase.apache.org/hbasecon-2018//hbasecon-2018/feed.xml"; rel="self" type="application/atom+xml" /> https://hbase.apache.org/hbasecon-2018//hbasecon-2018/"; rel="alternate" type="text/html" /> - 2018-03-23T11:32:05-04:00 + 2018-04-10T18:02:48-04:00 https://hbase.apache.org/hbasecon-2018//hbasecon-2018/ http://git-wip-us.apache.org/repos/asf/hbase-site/blob/e5b323a7/hbasecon-2018/index.html -- diff --git a/hbasecon-2018/index.html b/hbasecon-2018/index.html index a286d38..834011f 100644 --- a/hbasecon-2018/index.html +++ b/hbasecon-2018/index.html @@ -149,7 +149,7 @@ - + @@ -161,10 +161,8 @@ - Registration for the event will be handled through the DataWorks Summit registration system. This is not yet available. - Presently, the cost for both events is planned to be $200 per individual. This includes meals/snacks throughout the day. - As an additional benefit, attendees of the event will also have access to a 25% discount on the DataWorks Summit San Jose - registration fee. More details on this discount will be made available in the future. + Registration for HBaseCon 2018 is now open! Please use the following page to register: https://hbasecon2018.hortonworksevents.com/";>Register Here + Registration for HBaseCon includes registration to PhoenixCon which is running at concurrently at the same venue.
hbase git commit: HBASE-20352 [Chore] Backport HBASE-18309 (Support multi threads in CleanerChore) to branch-1
Repository: hbase Updated Branches: refs/heads/branch-1 de0dd9e82 -> 427c95e2e HBASE-20352 [Chore] Backport HBASE-18309 (Support multi threads in CleanerChore) to branch-1 Signed-off-by: Yu Li Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/427c95e2 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/427c95e2 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/427c95e2 Branch: refs/heads/branch-1 Commit: 427c95e2e555e76329286d0a8596693bf567cbf7 Parents: de0dd9e Author: Reid Chan Authored: Wed Apr 11 14:16:08 2018 +0800 Committer: Yu Li Committed: Wed Apr 11 14:17:51 2018 +0800 -- .../org/apache/hadoop/hbase/master/HMaster.java | 4 + .../hbase/master/cleaner/CleanerChore.java | 401 ++- .../hadoop/hbase/master/cleaner/LogCleaner.java | 181 + .../TestZooKeeperTableArchiveClient.java| 3 + .../hbase/master/cleaner/TestCleanerChore.java | 99 - .../hbase/master/cleaner/TestHFileCleaner.java | 1 + .../master/cleaner/TestHFileLinkCleaner.java| 1 + .../hbase/master/cleaner/TestLogsCleaner.java | 57 +++ 8 files changed, 647 insertions(+), 100 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/427c95e2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 6951098..85d5e23 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -97,6 +97,7 @@ import org.apache.hadoop.hbase.master.balancer.BalancerChore; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore; import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; +import org.apache.hadoop.hbase.master.cleaner.CleanerChore; import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; import org.apache.hadoop.hbase.master.cleaner.LogCleaner; import org.apache.hadoop.hbase.master.cleaner.ReplicationZKLockCleanerChore; @@ -893,6 +894,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { (System.currentTimeMillis() - masterActiveTime) / 1000.0f)); this.masterFinishedInitializationTime = System.currentTimeMillis(); configurationManager.registerObserver(this.balancer); +configurationManager.registerObserver(this.logCleaner); // Set master as 'initialized'. setInitialized(true); @@ -1215,6 +1217,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server { this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1); startProcedureExecutor(); +// Initial cleaner chore +CleanerChore.initChorePool(conf); // Start log cleaner thread int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000); this.logCleaner = http://git-wip-us.apache.org/repos/asf/hbase/blob/427c95e2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java index 743d9a4..50d8da3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java @@ -20,6 +20,10 @@ package org.apache.hadoop.hbase.master.cleaner; import java.io.IOException; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinTask; +import java.util.concurrent.RecursiveTask; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; @@ -28,12 +32,16 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.RemoteExceptionHandler; import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.conf.ConfigurationObserver; import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.FileStatusFilter; +import org.apache.hadoop.ipc.RemoteException; import com.google.common.annotations.VisibleForTesting; +impo
[01/12] hbase git commit: HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven.
Repository: hbase Updated Branches: refs/heads/branch-1 427c95e2e -> ff3e56293 refs/heads/branch-1.2 c20214459 -> 1ea3a8bc8 refs/heads/branch-1.3 1197ecaa3 -> fe8bd3ff0 refs/heads/branch-1.4 65a51fa28 -> a5c456de9 refs/heads/branch-2 37227b6bb -> 9bf087d28 refs/heads/branch-2.0 011dd2dae -> 388764805 HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven. Signed-off-by: Josh Elser Signed-off-by: Reid Chan Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1ef2450c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1ef2450c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1ef2450c Branch: refs/heads/branch-2 Commit: 1ef2450ca9a8f1e9e285799455a36ca5622ea511 Parents: 37227b6 Author: Sean Busbey Authored: Sun Apr 8 21:27:58 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:30:46 2018 -0500 -- dev-support/hbase-personality.sh | 16 1 file changed, 12 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/1ef2450c/dev-support/hbase-personality.sh -- diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index d34f761..90786f2 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -277,8 +277,10 @@ function refguide_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean site --batch-mode \ +$(maven_executor) clean site --batch-mode \ -pl . \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -352,8 +354,10 @@ function shadedjars_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean verify -fae --batch-mode \ +$(maven_executor) clean verify -fae --batch-mode \ -pl hbase-shaded/hbase-shaded-check-invariants -am \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -459,8 +463,10 @@ function hadoopcheck_rebuild export MAVEN_OPTS="${MAVEN_OPTS}" for hadoopver in ${hbase_hadoop2_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-two.version="${hadoopver}" count=$(${GREP} -c '\[ERROR\]' "${logfile}") @@ -473,8 +479,10 @@ function hadoopcheck_rebuild for hadoopver in ${hbase_hadoop3_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-three.version="${hadoopver}" \ -Dhadoop.profile=3.0
hbase git commit: HBASE-15291 FileSystem not closed in secure bulkLoad
Repository: hbase Updated Branches: refs/heads/master 95ca38a53 -> 828a1c76c HBASE-15291 FileSystem not closed in secure bulkLoad Signed-off-by: Ashish Singhi Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/828a1c76 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/828a1c76 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/828a1c76 Branch: refs/heads/master Commit: 828a1c76c71b0179bd9709e3da5d988b18fea631 Parents: 95ca38a Author: Ashish Singhi Authored: Wed Apr 11 12:01:28 2018 +0530 Committer: Ashish Singhi Committed: Wed Apr 11 12:01:28 2018 +0530 -- .../regionserver/SecureBulkLoadManager.java | 82 +--- 1 file changed, 54 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/828a1c76/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java index 264d985..a4ee517 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java @@ -145,15 +145,26 @@ public class SecureBulkLoadManager { public void cleanupBulkLoad(final HRegion region, final CleanupBulkLoadRequest request) throws IOException { -region.getCoprocessorHost().preCleanupBulkLoad(getActiveUser()); +try { + region.getCoprocessorHost().preCleanupBulkLoad(getActiveUser()); -Path path = new Path(request.getBulkToken()); -if (!fs.delete(path, true)) { - if (fs.exists(path)) { -throw new IOException("Failed to clean up " + path); + Path path = new Path(request.getBulkToken()); + if (!fs.delete(path, true)) { +if (fs.exists(path)) { + throw new IOException("Failed to clean up " + path); +} + } + LOG.info("Cleaned up " + path + " successfully."); +} finally { + UserGroupInformation ugi = getActiveUser().getUGI(); + try { +if (!UserGroupInformation.getLoginUser().equals(ugi)) { + FileSystem.closeAllForUGI(ugi); +} + } catch (IOException e) { +LOG.error("Failed to close FileSystem for: " + ugi, e); } } -LOG.info("Cleaned up " + path + " successfully."); } public Map> secureBulkLoadHFiles(final HRegion region, @@ -304,7 +315,7 @@ public class SecureBulkLoadManager { } if (srcFs == null) { -srcFs = FileSystem.get(p.toUri(), conf); +srcFs = FileSystem.newInstance(p.toUri(), conf); } if(!isFile(p)) { @@ -334,34 +345,49 @@ public class SecureBulkLoadManager { @Override public void doneBulkLoad(byte[] family, String srcPath) throws IOException { LOG.debug("Bulk Load done for: " + srcPath); + closeSrcFs(); +} + +private void closeSrcFs() throws IOException { + if (srcFs != null) { +srcFs.close(); +srcFs = null; + } } @Override public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException { - if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) { -// files are copied so no need to move them back -return; - } - Path p = new Path(srcPath); - Path stageP = new Path(stagingDir, - new Path(Bytes.toString(family), p.getName())); + try { +Path p = new Path(srcPath); +if (srcFs == null) { + srcFs = FileSystem.newInstance(p.toUri(), conf); +} +if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) { + // files are copied so no need to move them back + return; +} +Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName())); - // In case of Replication for bulk load files, hfiles are not renamed by end point during - // prepare stage, so no need of rename here again - if (p.equals(stageP)) { -LOG.debug(p.getName() + " is already available in source directory. Skipping rename."); -return; - } +// In case of Replication for bulk load files, hfiles are not renamed by end point during +// prepare stage, so no need of rename here again +if (p.equals(stageP)) { + LOG.debug(p.getName() + " is already available in source directory. Skipping rename."); + return; +} - LOG.debug("Moving " + stageP + " back to " + p); - if(!fs.rename(stageP, p)) -throw new IOException("Failed to move HFile: " + stageP + " to " + p); +
[02/12] hbase git commit: HBASE-20335 nightly job bash cleanup.
HBASE-20335 nightly job bash cleanup. * Ensure Jenkins steps that invoke bash inline set -e * machine stats script should check that passed directory will work Signed-off-by: Michael Stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9bf087d2 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9bf087d2 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9bf087d2 Branch: refs/heads/branch-2 Commit: 9bf087d28b599020ff2825107cd6bca8df609a38 Parents: 1ef2450 Author: Sean Busbey Authored: Tue Apr 3 12:08:05 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:30:49 2018 -0500 -- dev-support/Jenkinsfile | 11 +++ dev-support/gather_machine_environment.sh | 5 + 2 files changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/9bf087d2/dev-support/Jenkinsfile -- diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index c91a2b1..238d0e6 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -63,6 +63,7 @@ pipeline { stage ('yetus install') { steps { sh '''#!/usr/bin/env bash +set -e echo "Ensure we have a copy of Apache Yetus." if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" @@ -101,6 +102,7 @@ fi // Set up the file we need at PERSONALITY_FILE location dir ("tools") { sh """#!/usr/bin/env bash +set -e echo "Downloading Project personality." curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" """ @@ -144,12 +146,14 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' // TODO roll this into the hbase_nightly_yetus script sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -202,11 +206,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -273,11 +279,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -351,11 +359,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -425,6 +435,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine" ''' http://git-wip-us.apache.org/repos/asf/hbase/blob/9bf087d2/dev-support/gather_machine_environment.sh
[05/12] hbase git commit: HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven.
HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven. Signed-off-by: Josh Elser Signed-off-by: Reid Chan Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/358d451d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/358d451d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/358d451d Branch: refs/heads/branch-1 Commit: 358d451d96ae174d40525a54914e8e61af2a38e3 Parents: 427c95e Author: Sean Busbey Authored: Sun Apr 8 21:27:58 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:31:02 2018 -0500 -- dev-support/hbase-personality.sh | 16 1 file changed, 12 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/358d451d/dev-support/hbase-personality.sh -- diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 9664095..819beee 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -277,8 +277,10 @@ function refguide_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean site --batch-mode \ +$(maven_executor) clean site --batch-mode \ -pl . \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -352,8 +354,10 @@ function shadedjars_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean verify -fae --batch-mode \ +$(maven_executor) clean verify -fae --batch-mode \ -pl hbase-shaded/hbase-shaded-check-invariants -am \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -459,8 +463,10 @@ function hadoopcheck_rebuild export MAVEN_OPTS="${MAVEN_OPTS}" for hadoopver in ${hbase_hadoop2_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-two.version="${hadoopver}" count=$(${GREP} -c '\[ERROR\]' "${logfile}") @@ -473,8 +479,10 @@ function hadoopcheck_rebuild for hadoopver in ${hbase_hadoop3_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-three.version="${hadoopver}" \ -Dhadoop.profile=3.0
[10/12] hbase git commit: HBASE-20335 nightly job bash cleanup.
HBASE-20335 nightly job bash cleanup. * Ensure Jenkins steps that invoke bash inline set -e * machine stats script should check that passed directory will work Signed-off-by: Michael Stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fe8bd3ff Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fe8bd3ff Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fe8bd3ff Branch: refs/heads/branch-1.3 Commit: fe8bd3ff0bc7d871b71050a110fe539a8b8f487b Parents: dfc264a Author: Sean Busbey Authored: Tue Apr 3 12:08:05 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:31:20 2018 -0500 -- dev-support/Jenkinsfile | 11 +++ dev-support/gather_machine_environment.sh | 5 + 2 files changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/fe8bd3ff/dev-support/Jenkinsfile -- diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index bb1da7a..a8356b4 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -63,6 +63,7 @@ pipeline { stage ('yetus install') { steps { sh '''#!/usr/bin/env bash +set -e echo "Ensure we have a copy of Apache Yetus." if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" @@ -101,6 +102,7 @@ fi // Set up the file we need at PERSONALITY_FILE location dir ("tools") { sh """#!/usr/bin/env bash +set -e echo "Downloading Project personality." curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" """ @@ -144,12 +146,14 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' // TODO roll this into the hbase_nightly_yetus script sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -202,11 +206,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -273,11 +279,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -351,11 +359,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -425,6 +435,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine" ''' http://git-wip-us.apache.org/repos/asf/hbase/blob/fe8bd3ff/dev-support/gather_machine_environment.sh --
[12/12] hbase git commit: HBASE-20335 nightly job bash cleanup.
HBASE-20335 nightly job bash cleanup. * Ensure Jenkins steps that invoke bash inline set -e * machine stats script should check that passed directory will work Signed-off-by: Michael Stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1ea3a8bc Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1ea3a8bc Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1ea3a8bc Branch: refs/heads/branch-1.2 Commit: 1ea3a8bc86a4132cb3f5df85c60e939ca9c77a92 Parents: a69e879 Author: Sean Busbey Authored: Tue Apr 3 12:08:05 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:31:28 2018 -0500 -- dev-support/Jenkinsfile | 11 +++ dev-support/gather_machine_environment.sh | 5 + 2 files changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/1ea3a8bc/dev-support/Jenkinsfile -- diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index eeeda83..ffd8c18 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -63,6 +63,7 @@ pipeline { stage ('yetus install') { steps { sh '''#!/usr/bin/env bash +set -e echo "Ensure we have a copy of Apache Yetus." if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" @@ -101,6 +102,7 @@ fi // Set up the file we need at PERSONALITY_FILE location dir ("tools") { sh """#!/usr/bin/env bash +set -e echo "Downloading Project personality." curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" """ @@ -144,12 +146,14 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' // TODO roll this into the hbase_nightly_yetus script sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -202,11 +206,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -273,11 +279,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -351,11 +359,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -425,6 +435,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine" ''' http://git-wip-us.apache.org/repos/asf/hbase/blob/1ea3a8bc/dev-support/gather_machine_environment.sh --
[06/12] hbase git commit: HBASE-20335 nightly job bash cleanup.
HBASE-20335 nightly job bash cleanup. * Ensure Jenkins steps that invoke bash inline set -e * machine stats script should check that passed directory will work Signed-off-by: Michael Stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ff3e5629 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ff3e5629 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ff3e5629 Branch: refs/heads/branch-1 Commit: ff3e56293966f2eb8c5124b22689287ccf2c3d69 Parents: 358d451 Author: Sean Busbey Authored: Tue Apr 3 12:08:05 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:31:05 2018 -0500 -- dev-support/Jenkinsfile | 11 +++ dev-support/gather_machine_environment.sh | 5 + 2 files changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/ff3e5629/dev-support/Jenkinsfile -- diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index bb1da7a..a8356b4 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -63,6 +63,7 @@ pipeline { stage ('yetus install') { steps { sh '''#!/usr/bin/env bash +set -e echo "Ensure we have a copy of Apache Yetus." if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" @@ -101,6 +102,7 @@ fi // Set up the file we need at PERSONALITY_FILE location dir ("tools") { sh """#!/usr/bin/env bash +set -e echo "Downloading Project personality." curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" """ @@ -144,12 +146,14 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' // TODO roll this into the hbase_nightly_yetus script sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -202,11 +206,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -273,11 +279,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -351,11 +359,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -425,6 +435,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine" ''' http://git-wip-us.apache.org/repos/asf/hbase/blob/ff3e5629/dev-support/gather_machine_environment.sh
[04/12] hbase git commit: HBASE-20335 nightly job bash cleanup.
HBASE-20335 nightly job bash cleanup. * Ensure Jenkins steps that invoke bash inline set -e * machine stats script should check that passed directory will work Signed-off-by: Michael Stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/38876480 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/38876480 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/38876480 Branch: refs/heads/branch-2.0 Commit: 3887648057e30fbaad95395aba433184d41d5fd4 Parents: 86f4686 Author: Sean Busbey Authored: Tue Apr 3 12:08:05 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:30:56 2018 -0500 -- dev-support/Jenkinsfile | 11 +++ dev-support/gather_machine_environment.sh | 5 + 2 files changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/38876480/dev-support/Jenkinsfile -- diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index c91a2b1..238d0e6 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -63,6 +63,7 @@ pipeline { stage ('yetus install') { steps { sh '''#!/usr/bin/env bash +set -e echo "Ensure we have a copy of Apache Yetus." if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" @@ -101,6 +102,7 @@ fi // Set up the file we need at PERSONALITY_FILE location dir ("tools") { sh """#!/usr/bin/env bash +set -e echo "Downloading Project personality." curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" """ @@ -144,12 +146,14 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' // TODO roll this into the hbase_nightly_yetus script sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -202,11 +206,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -273,11 +279,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -351,11 +359,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -425,6 +435,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine" ''' http://git-wip-us.apache.org/repos/asf/hbase/blob/38876480/dev-support/gather_machine_environment.sh --
[07/12] hbase git commit: HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven.
HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven. Signed-off-by: Josh Elser Signed-off-by: Reid Chan Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a5b1c940 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a5b1c940 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a5b1c940 Branch: refs/heads/branch-1.4 Commit: a5b1c940961b8c894cfcfe14db8b194718e12fde Parents: 65a51fa Author: Sean Busbey Authored: Sun Apr 8 21:27:58 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:31:09 2018 -0500 -- dev-support/hbase-personality.sh | 16 1 file changed, 12 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a5b1c940/dev-support/hbase-personality.sh -- diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 9664095..819beee 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -277,8 +277,10 @@ function refguide_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean site --batch-mode \ +$(maven_executor) clean site --batch-mode \ -pl . \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -352,8 +354,10 @@ function shadedjars_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean verify -fae --batch-mode \ +$(maven_executor) clean verify -fae --batch-mode \ -pl hbase-shaded/hbase-shaded-check-invariants -am \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -459,8 +463,10 @@ function hadoopcheck_rebuild export MAVEN_OPTS="${MAVEN_OPTS}" for hadoopver in ${hbase_hadoop2_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-two.version="${hadoopver}" count=$(${GREP} -c '\[ERROR\]' "${logfile}") @@ -473,8 +479,10 @@ function hadoopcheck_rebuild for hadoopver in ${hbase_hadoop3_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-three.version="${hadoopver}" \ -Dhadoop.profile=3.0
[08/12] hbase git commit: HBASE-20335 nightly job bash cleanup.
HBASE-20335 nightly job bash cleanup. * Ensure Jenkins steps that invoke bash inline set -e * machine stats script should check that passed directory will work Signed-off-by: Michael Stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a5c456de Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a5c456de Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a5c456de Branch: refs/heads/branch-1.4 Commit: a5c456de9c31074232c20bc92e2d34880ef12051 Parents: a5b1c94 Author: Sean Busbey Authored: Tue Apr 3 12:08:05 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:31:12 2018 -0500 -- dev-support/Jenkinsfile | 11 +++ dev-support/gather_machine_environment.sh | 5 + 2 files changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a5c456de/dev-support/Jenkinsfile -- diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index bb1da7a..a8356b4 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -63,6 +63,7 @@ pipeline { stage ('yetus install') { steps { sh '''#!/usr/bin/env bash +set -e echo "Ensure we have a copy of Apache Yetus." if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" @@ -101,6 +102,7 @@ fi // Set up the file we need at PERSONALITY_FILE location dir ("tools") { sh """#!/usr/bin/env bash +set -e echo "Downloading Project personality." curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" """ @@ -144,12 +146,14 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' // TODO roll this into the hbase_nightly_yetus script sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -202,11 +206,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -273,11 +279,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -351,11 +359,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -425,6 +435,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine" ''' http://git-wip-us.apache.org/repos/asf/hbase/blob/a5c456de/dev-support/gather_machine_environment.sh --
[03/12] hbase git commit: HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven.
HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven. Signed-off-by: Josh Elser Signed-off-by: Reid Chan Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/86f4686b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/86f4686b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/86f4686b Branch: refs/heads/branch-2.0 Commit: 86f4686bc655590cfd1d20d19ce640a07475c2f0 Parents: 011dd2d Author: Sean Busbey Authored: Sun Apr 8 21:27:58 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:30:53 2018 -0500 -- dev-support/hbase-personality.sh | 16 1 file changed, 12 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/86f4686b/dev-support/hbase-personality.sh -- diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index d34f761..90786f2 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -277,8 +277,10 @@ function refguide_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean site --batch-mode \ +$(maven_executor) clean site --batch-mode \ -pl . \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -352,8 +354,10 @@ function shadedjars_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean verify -fae --batch-mode \ +$(maven_executor) clean verify -fae --batch-mode \ -pl hbase-shaded/hbase-shaded-check-invariants -am \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -459,8 +463,10 @@ function hadoopcheck_rebuild export MAVEN_OPTS="${MAVEN_OPTS}" for hadoopver in ${hbase_hadoop2_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-two.version="${hadoopver}" count=$(${GREP} -c '\[ERROR\]' "${logfile}") @@ -473,8 +479,10 @@ function hadoopcheck_rebuild for hadoopver in ${hbase_hadoop3_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-three.version="${hadoopver}" \ -Dhadoop.profile=3.0
[09/12] hbase git commit: HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven.
HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven. Signed-off-by: Josh Elser Signed-off-by: Reid Chan Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dfc264a6 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dfc264a6 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dfc264a6 Branch: refs/heads/branch-1.3 Commit: dfc264a6a7fd0c3a5e9d11680ed9dd6d710ca0ea Parents: 1197eca Author: Sean Busbey Authored: Sun Apr 8 21:27:58 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:31:17 2018 -0500 -- dev-support/hbase-personality.sh | 16 1 file changed, 12 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/dfc264a6/dev-support/hbase-personality.sh -- diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 23a834c..19e7e97 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -273,8 +273,10 @@ function refguide_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean site --batch-mode \ +$(maven_executor) clean site --batch-mode \ -pl . \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -348,8 +350,10 @@ function shadedjars_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean verify -fae --batch-mode \ +$(maven_executor) clean verify -fae --batch-mode \ -pl hbase-shaded/hbase-shaded-check-invariants -am \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -455,8 +459,10 @@ function hadoopcheck_rebuild export MAVEN_OPTS="${MAVEN_OPTS}" for hadoopver in ${hbase_hadoop2_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-two.version="${hadoopver}" count=$(${GREP} -c '\[ERROR\]' "${logfile}") @@ -469,8 +475,10 @@ function hadoopcheck_rebuild for hadoopver in ${hbase_hadoop3_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-three.version="${hadoopver}" \ -Dhadoop.profile=3.0
[11/12] hbase git commit: HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven.
HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven. Signed-off-by: Josh Elser Signed-off-by: Reid Chan Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a69e879c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a69e879c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a69e879c Branch: refs/heads/branch-1.2 Commit: a69e879c6972ef7a955835a7951e422345034292 Parents: c202144 Author: Sean Busbey Authored: Sun Apr 8 21:27:58 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:31:25 2018 -0500 -- dev-support/hbase-personality.sh | 16 1 file changed, 12 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a69e879c/dev-support/hbase-personality.sh -- diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 23a834c..19e7e97 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -273,8 +273,10 @@ function refguide_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean site --batch-mode \ +$(maven_executor) clean site --batch-mode \ -pl . \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -348,8 +350,10 @@ function shadedjars_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean verify -fae --batch-mode \ +$(maven_executor) clean verify -fae --batch-mode \ -pl hbase-shaded/hbase-shaded-check-invariants -am \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -455,8 +459,10 @@ function hadoopcheck_rebuild export MAVEN_OPTS="${MAVEN_OPTS}" for hadoopver in ${hbase_hadoop2_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-two.version="${hadoopver}" count=$(${GREP} -c '\[ERROR\]' "${logfile}") @@ -469,8 +475,10 @@ function hadoopcheck_rebuild for hadoopver in ${hbase_hadoop3_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-three.version="${hadoopver}" \ -Dhadoop.profile=3.0
[1/3] hbase git commit: HBASE-20335 nightly job bash cleanup.
Repository: hbase Updated Branches: refs/heads/master 828a1c76c -> a2c1be9a7 HBASE-20335 nightly job bash cleanup. * Ensure Jenkins steps that invoke bash inline set -e * machine stats script should check that passed directory will work Signed-off-by: Michael Stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a2c1be9a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a2c1be9a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a2c1be9a Branch: refs/heads/master Commit: a2c1be9a7674e5d18892b72b2984a524c7dc64c9 Parents: 83ee82d Author: Sean Busbey Authored: Tue Apr 3 12:08:05 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:33:14 2018 -0500 -- dev-support/Jenkinsfile | 11 +++ dev-support/gather_machine_environment.sh | 5 + 2 files changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a2c1be9a/dev-support/Jenkinsfile -- diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index e8bf8c9..3f3066b 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -63,6 +63,7 @@ pipeline { stage ('yetus install') { steps { sh '''#!/usr/bin/env bash +set -e echo "Ensure we have a copy of Apache Yetus." if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" @@ -101,6 +102,7 @@ fi // Set up the file we need at PERSONALITY_FILE location dir ("tools") { sh """#!/usr/bin/env bash +set -e echo "Downloading Project personality." curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" """ @@ -144,12 +146,14 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' // TODO roll this into the hbase_nightly_yetus script sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -202,11 +206,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -273,11 +279,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -351,11 +359,13 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" ''' sh '''#!/usr/bin/env bash + set -e rm -rf "${OUTPUT_DIR}/commentfile}" declare -i status=0 if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then @@ -425,6 +435,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" checkout scm } sh '''#!/usr/bin/env bash + set -e rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine" "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine" ''' http://git-wip-us.apache.org/repos/asf/hbase/blob/a2c1
[2/3] hbase git commit: HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven.
HBASE-20068 personality tests for Apache Yetus should use the maven plugin to exec maven. Signed-off-by: Josh Elser Signed-off-by: Reid Chan Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/83ee82d6 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/83ee82d6 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/83ee82d6 Branch: refs/heads/master Commit: 83ee82d6a02f2b4b0f8e490483fd7a7b0ace73d8 Parents: 1efd3ac Author: Sean Busbey Authored: Sun Apr 8 21:27:58 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:33:14 2018 -0500 -- dev-support/hbase-personality.sh | 16 1 file changed, 12 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/83ee82d6/dev-support/hbase-personality.sh -- diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index d34f761..90786f2 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -277,8 +277,10 @@ function refguide_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean site --batch-mode \ +$(maven_executor) clean site --batch-mode \ -pl . \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -352,8 +354,10 @@ function shadedjars_rebuild start_clock + # disabled because "maven_executor" needs to return both command and args + # shellcheck disable=2046 echo_and_redirect "${logfile}" \ -"${MAVEN}" "${MAVEN_ARGS[@]}" clean verify -fae --batch-mode \ +$(maven_executor) clean verify -fae --batch-mode \ -pl hbase-shaded/hbase-shaded-check-invariants -am \ -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \ -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true @@ -459,8 +463,10 @@ function hadoopcheck_rebuild export MAVEN_OPTS="${MAVEN_OPTS}" for hadoopver in ${hbase_hadoop2_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-two.version="${hadoopver}" count=$(${GREP} -c '\[ERROR\]' "${logfile}") @@ -473,8 +479,10 @@ function hadoopcheck_rebuild for hadoopver in ${hbase_hadoop3_versions}; do logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" +# disabled because "maven_executor" needs to return both command and args +# shellcheck disable=2046 echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ + $(maven_executor) clean install \ -DskipTests -DHBasePatchProcess \ -Dhadoop-three.version="${hadoopver}" \ -Dhadoop.profile=3.0
[3/3] hbase git commit: HBASE-20372 [website] move stuff from more than 2 years ago to old news
HBASE-20372 [website] move stuff from more than 2 years ago to old news Signed-off-by: Josh Elser Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1efd3ac2 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1efd3ac2 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1efd3ac2 Branch: refs/heads/master Commit: 1efd3ac2402233d5cb1d36ef764c34d2dba1f1a5 Parents: 828a1c7 Author: Sean Busbey Authored: Mon Apr 9 11:34:57 2018 -0500 Committer: Sean Busbey Committed: Wed Apr 11 01:33:14 2018 -0500 -- src/site/xdoc/index.xml| 13 - src/site/xdoc/old_news.xml | 13 + 2 files changed, 13 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/1efd3ac2/src/site/xdoc/index.xml -- diff --git a/src/site/xdoc/index.xml b/src/site/xdoc/index.xml index 49cb6f7..e1440ab 100644 --- a/src/site/xdoc/index.xml +++ b/src/site/xdoc/index.xml @@ -90,19 +90,6 @@ Apache HBase is an open-source, distributed, versioned, non-relational database December 8th, 2016 https://www.meetup.com/hbaseusergroup/events/235542241/";>Meetup@Splice in San Francisco September 26th, 2016 http://www.meetup.com/HBase-NYC/events/233024937/";>HBaseConEast2016 at Google in Chelsea, NYC May 24th, 2016 http://www.hbasecon.com/";>HBaseCon2016 at The Village, 969 Market, San Francisco - June 25th, 2015 http://www.zusaar.com/event/14057003";>HBase Summer Meetup 2015 in Tokyo - May 7th, 2015 http://hbasecon.com/";>HBaseCon2015 in San Francisco - February 17th, 2015 http://www.meetup.com/hbaseusergroup/events/219260093/";>HBase meetup around Strata+Hadoop World in San Jose - January 15th, 2015 http://www.meetup.com/hbaseusergroup/events/218744798/";>HBase meetup @ AppDynamics in San Francisco - November 20th, 2014 http://www.meetup.com/hbaseusergroup/events/205219992/";>HBase meetup @ WANdisco in San Ramon - October 27th, 2014 http://www.meetup.com/hbaseusergroup/events/207386102/";>HBase Meetup @ Apple in Cupertino - October 15th, 2014 http://www.meetup.com/HBase-NYC/events/20762/";>HBase Meetup @ Google on the night before Strata/HW in NYC - September 25th, 2014 http://www.meetup.com/hbaseusergroup/events/203173692/";>HBase Meetup @ Continuuity in Palo Alto - August 28th, 2014 http://www.meetup.com/hbaseusergroup/events/197773762/";>HBase Meetup @ Sift Science in San Francisco - July 17th, 2014 http://www.meetup.com/hbaseusergroup/events/190994082/";>HBase Meetup @ HP in Sunnyvale - June 5th, 2014 http://www.meetup.com/Hadoop-Summit-Community-San-Jose/events/179081342/";>HBase BOF at Hadoop Summit, San Jose Convention Center - May 5th, 2014 http://www.hbasecon.com/";>HBaseCon2014 at the Hilton San Francisco on Union Square - March 12th, 2014 http://www.meetup.com/hbaseusergroup/events/160757912/";>HBase Meetup @ Ancestry.com in San Francisco Old News http://git-wip-us.apache.org/repos/asf/hbase/blob/1efd3ac2/src/site/xdoc/old_news.xml -- diff --git a/src/site/xdoc/old_news.xml b/src/site/xdoc/old_news.xml index 705d69d..2203e4e 100644 --- a/src/site/xdoc/old_news.xml +++ b/src/site/xdoc/old_news.xml @@ -31,6 +31,19 @@ under the License. + June 25th, 2015 http://www.zusaar.com/event/14057003";>HBase Summer Meetup 2015 in Tokyo + May 7th, 2015 http://hbasecon.com/";>HBaseCon2015 in San Francisco + February 17th, 2015 http://www.meetup.com/hbaseusergroup/events/219260093/";>HBase meetup around Strata+Hadoop World in San Jose + January 15th, 2015 http://www.meetup.com/hbaseusergroup/events/218744798/";>HBase meetup @ AppDynamics in San Francisco + November 20th, 2014 http://www.meetup.com/hbaseusergroup/events/205219992/";>HBase meetup @ WANdisco in San Ramon + October 27th, 2014 http://www.meetup.com/hbaseusergroup/events/207386102/";>HBase Meetup @ Apple in Cupertino + October 15th, 2014 http://www.meetup.com/HBase-NYC/events/20762/";>HBase Meetup @ Google on the night before Strata/HW in NYC + September 25th, 2014 http://www.meetup.com/hbaseusergroup/events/203173692/";>HBase Meetup @ Continuuity in Palo Alto + August 28th, 2014 http://www.meetup.com/hbaseusergroup/events/197773762/";>HBase Meetup @ Sift Science in San Francisco + July 17th, 2014 http://www.meetup.com/hbaseusergroup/events/190994082/";>HBase Meetup @ HP in Sunnyvale + June 5th, 2014 http://www.meetup.com/Hadoop-Summit-Community-San-Jose/events/179081342/";>HBase BOF at Hadoop Summit, San Jose Convention Center + May 5th, 2014 http:/
hbase git commit: HBASE-15291 FileSystem not closed in secure bulkLoad
Repository: hbase Updated Branches: refs/heads/branch-2 9bf087d28 -> 4bcb560e2 HBASE-15291 FileSystem not closed in secure bulkLoad Signed-off-by: Ashish Singhi Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4bcb560e Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4bcb560e Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4bcb560e Branch: refs/heads/branch-2 Commit: 4bcb560e226088b036ef935768ed3b7ec6986789 Parents: 9bf087d Author: Ashish Singhi Authored: Wed Apr 11 12:11:41 2018 +0530 Committer: Ashish Singhi Committed: Wed Apr 11 12:11:41 2018 +0530 -- .../regionserver/SecureBulkLoadManager.java | 82 +--- 1 file changed, 54 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/4bcb560e/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java index 264d985..a4ee517 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java @@ -145,15 +145,26 @@ public class SecureBulkLoadManager { public void cleanupBulkLoad(final HRegion region, final CleanupBulkLoadRequest request) throws IOException { -region.getCoprocessorHost().preCleanupBulkLoad(getActiveUser()); +try { + region.getCoprocessorHost().preCleanupBulkLoad(getActiveUser()); -Path path = new Path(request.getBulkToken()); -if (!fs.delete(path, true)) { - if (fs.exists(path)) { -throw new IOException("Failed to clean up " + path); + Path path = new Path(request.getBulkToken()); + if (!fs.delete(path, true)) { +if (fs.exists(path)) { + throw new IOException("Failed to clean up " + path); +} + } + LOG.info("Cleaned up " + path + " successfully."); +} finally { + UserGroupInformation ugi = getActiveUser().getUGI(); + try { +if (!UserGroupInformation.getLoginUser().equals(ugi)) { + FileSystem.closeAllForUGI(ugi); +} + } catch (IOException e) { +LOG.error("Failed to close FileSystem for: " + ugi, e); } } -LOG.info("Cleaned up " + path + " successfully."); } public Map> secureBulkLoadHFiles(final HRegion region, @@ -304,7 +315,7 @@ public class SecureBulkLoadManager { } if (srcFs == null) { -srcFs = FileSystem.get(p.toUri(), conf); +srcFs = FileSystem.newInstance(p.toUri(), conf); } if(!isFile(p)) { @@ -334,34 +345,49 @@ public class SecureBulkLoadManager { @Override public void doneBulkLoad(byte[] family, String srcPath) throws IOException { LOG.debug("Bulk Load done for: " + srcPath); + closeSrcFs(); +} + +private void closeSrcFs() throws IOException { + if (srcFs != null) { +srcFs.close(); +srcFs = null; + } } @Override public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException { - if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) { -// files are copied so no need to move them back -return; - } - Path p = new Path(srcPath); - Path stageP = new Path(stagingDir, - new Path(Bytes.toString(family), p.getName())); + try { +Path p = new Path(srcPath); +if (srcFs == null) { + srcFs = FileSystem.newInstance(p.toUri(), conf); +} +if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) { + // files are copied so no need to move them back + return; +} +Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName())); - // In case of Replication for bulk load files, hfiles are not renamed by end point during - // prepare stage, so no need of rename here again - if (p.equals(stageP)) { -LOG.debug(p.getName() + " is already available in source directory. Skipping rename."); -return; - } +// In case of Replication for bulk load files, hfiles are not renamed by end point during +// prepare stage, so no need of rename here again +if (p.equals(stageP)) { + LOG.debug(p.getName() + " is already available in source directory. Skipping rename."); + return; +} - LOG.debug("Moving " + stageP + " back to " + p); - if(!fs.rename(stageP, p)) -throw new IOException("Failed to move HFile: " + stageP + " to " + p);
hbase git commit: HBASE-20182 Addendum throw IOException instead of NoServerForRegionException because it is a DoNotRetryRegionException
Repository: hbase Updated Branches: refs/heads/master a2c1be9a7 -> a2b917277 HBASE-20182 Addendum throw IOException instead of NoServerForRegionException because it is a DoNotRetryRegionException Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a2b91727 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a2b91727 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a2b91727 Branch: refs/heads/master Commit: a2b91727715e10afa29c759cef20540f29a79228 Parents: a2c1be9 Author: zhangduo Authored: Wed Apr 11 14:37:56 2018 +0800 Committer: zhangduo Committed: Wed Apr 11 14:47:06 2018 +0800 -- .../org/apache/hadoop/hbase/HRegionInfo.java| 8 --- .../hbase/client/AsyncNonMetaRegionLocator.java | 23 ++-- .../hbase/client/ConnectionImplementation.java | 4 ++-- .../hadoop/hbase/client/RegionInfoBuilder.java | 8 --- 4 files changed, 24 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a2b91727/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index e2982bd..fc03926 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -159,9 +159,11 @@ public class HRegionInfo implements RegionInfo, Comparable { } private byte [] endKey = HConstants.EMPTY_BYTE_ARRAY; - // This flag is in the parent of a split while the parent is still referenced - // by daughter regions. We USED to set this flag when we disabled a table - // but now table state is kept up in zookeeper as of 0.90.0 HBase. + // This flag is in the parent of a split while the parent is still referenced by daughter regions. + // We USED to set this flag when we disabled a table but now table state is kept up in zookeeper + // as of 0.90.0 HBase. And now in DisableTableProcedure, finally we will create bunch of + // UnassignProcedures and at the last of the procedure we will set the region state to CLOSED, and + // will not change the offLine flag. private boolean offLine = false; private long regionId = -1; private transient byte [] regionName = HConstants.EMPTY_BYTE_ARRAY; http://git-wip-us.apache.org/repos/asf/hbase/blob/a2b91727/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java index c30de9a..7634b10 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java @@ -269,12 +269,7 @@ class AsyncNonMetaRegionLocator { } // return whether we should stop the scan - private boolean onScanNext(TableName tableName, LocateRequest req, Result result, - Throwable error) { -if (error != null) { - complete(tableName, req, null, error); - return true; -} + private boolean onScanNext(TableName tableName, LocateRequest req, Result result) { RegionLocations locs = MetaTableAccessor.getRegionLocations(result); LOG.debug("The fetched location of '{}', row='{}', locateType={} is {}", tableName, Bytes.toStringBinary(req.row), req.locateType, locs); @@ -298,7 +293,7 @@ class AsyncNonMetaRegionLocator { } if (loc.getServerName() == null) { complete(tableName, req, null, -new NoServerForRegionException( +new IOException( String.format("No server address listed for region '%s', row='%s', locateType=%s", info.getRegionNameAsString(), Bytes.toStringBinary(req.row), req.locateType))); return true; @@ -370,22 +365,28 @@ class AsyncNonMetaRegionLocator { private boolean completeNormally = false; + private boolean tableNotFound = true; + @Override public void onError(Throwable error) { -onScanNext(tableName, req, null, error); +complete(tableName, req, null, error); } @Override public void onComplete() { -if (!completeNormally) { - onScanNext(tableName, req, null, new TableNotFoundException(tableName)); +if (tableNotFound) { + complete(tableName, req, null, new TableNotFoundException(tableName)); +} else if (!completeN
hbase git commit: HBASE-20182 Addendum throw IOException instead of NoServerForRegionException because it is a DoNotRetryRegionException
Repository: hbase Updated Branches: refs/heads/branch-2 4bcb560e2 -> 28d09cedc HBASE-20182 Addendum throw IOException instead of NoServerForRegionException because it is a DoNotRetryRegionException Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/28d09ced Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/28d09ced Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/28d09ced Branch: refs/heads/branch-2 Commit: 28d09cedcfac5b9b6e07ea54fd352541cc9a6f14 Parents: 4bcb560 Author: zhangduo Authored: Wed Apr 11 14:37:56 2018 +0800 Committer: zhangduo Committed: Wed Apr 11 14:47:41 2018 +0800 -- .../org/apache/hadoop/hbase/HRegionInfo.java| 8 --- .../hbase/client/AsyncNonMetaRegionLocator.java | 23 ++-- .../hbase/client/ConnectionImplementation.java | 4 ++-- .../hadoop/hbase/client/RegionInfoBuilder.java | 8 --- 4 files changed, 24 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/28d09ced/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index e2982bd..fc03926 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -159,9 +159,11 @@ public class HRegionInfo implements RegionInfo, Comparable { } private byte [] endKey = HConstants.EMPTY_BYTE_ARRAY; - // This flag is in the parent of a split while the parent is still referenced - // by daughter regions. We USED to set this flag when we disabled a table - // but now table state is kept up in zookeeper as of 0.90.0 HBase. + // This flag is in the parent of a split while the parent is still referenced by daughter regions. + // We USED to set this flag when we disabled a table but now table state is kept up in zookeeper + // as of 0.90.0 HBase. And now in DisableTableProcedure, finally we will create bunch of + // UnassignProcedures and at the last of the procedure we will set the region state to CLOSED, and + // will not change the offLine flag. private boolean offLine = false; private long regionId = -1; private transient byte [] regionName = HConstants.EMPTY_BYTE_ARRAY; http://git-wip-us.apache.org/repos/asf/hbase/blob/28d09ced/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java index c30de9a..7634b10 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java @@ -269,12 +269,7 @@ class AsyncNonMetaRegionLocator { } // return whether we should stop the scan - private boolean onScanNext(TableName tableName, LocateRequest req, Result result, - Throwable error) { -if (error != null) { - complete(tableName, req, null, error); - return true; -} + private boolean onScanNext(TableName tableName, LocateRequest req, Result result) { RegionLocations locs = MetaTableAccessor.getRegionLocations(result); LOG.debug("The fetched location of '{}', row='{}', locateType={} is {}", tableName, Bytes.toStringBinary(req.row), req.locateType, locs); @@ -298,7 +293,7 @@ class AsyncNonMetaRegionLocator { } if (loc.getServerName() == null) { complete(tableName, req, null, -new NoServerForRegionException( +new IOException( String.format("No server address listed for region '%s', row='%s', locateType=%s", info.getRegionNameAsString(), Bytes.toStringBinary(req.row), req.locateType))); return true; @@ -370,22 +365,28 @@ class AsyncNonMetaRegionLocator { private boolean completeNormally = false; + private boolean tableNotFound = true; + @Override public void onError(Throwable error) { -onScanNext(tableName, req, null, error); +complete(tableName, req, null, error); } @Override public void onComplete() { -if (!completeNormally) { - onScanNext(tableName, req, null, new TableNotFoundException(tableName)); +if (tableNotFound) { + complete(tableName, req, null, new TableNotFoundException(tableName)); +} else if (!compl
hbase git commit: HBASE-20182 Addendum throw IOException instead of NoServerForRegionException because it is a DoNotRetryRegionException
Repository: hbase Updated Branches: refs/heads/branch-2.0 388764805 -> 2e6eff085 HBASE-20182 Addendum throw IOException instead of NoServerForRegionException because it is a DoNotRetryRegionException Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2e6eff08 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2e6eff08 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2e6eff08 Branch: refs/heads/branch-2.0 Commit: 2e6eff085b60bcc07ce2b44df03a12d1cd0656f9 Parents: 3887648 Author: zhangduo Authored: Wed Apr 11 14:37:56 2018 +0800 Committer: zhangduo Committed: Wed Apr 11 14:48:13 2018 +0800 -- .../org/apache/hadoop/hbase/HRegionInfo.java| 8 --- .../hbase/client/AsyncNonMetaRegionLocator.java | 23 ++-- .../hbase/client/ConnectionImplementation.java | 4 ++-- .../hadoop/hbase/client/RegionInfoBuilder.java | 8 --- 4 files changed, 24 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/2e6eff08/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index e2982bd..fc03926 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -159,9 +159,11 @@ public class HRegionInfo implements RegionInfo, Comparable { } private byte [] endKey = HConstants.EMPTY_BYTE_ARRAY; - // This flag is in the parent of a split while the parent is still referenced - // by daughter regions. We USED to set this flag when we disabled a table - // but now table state is kept up in zookeeper as of 0.90.0 HBase. + // This flag is in the parent of a split while the parent is still referenced by daughter regions. + // We USED to set this flag when we disabled a table but now table state is kept up in zookeeper + // as of 0.90.0 HBase. And now in DisableTableProcedure, finally we will create bunch of + // UnassignProcedures and at the last of the procedure we will set the region state to CLOSED, and + // will not change the offLine flag. private boolean offLine = false; private long regionId = -1; private transient byte [] regionName = HConstants.EMPTY_BYTE_ARRAY; http://git-wip-us.apache.org/repos/asf/hbase/blob/2e6eff08/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java index c30de9a..7634b10 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java @@ -269,12 +269,7 @@ class AsyncNonMetaRegionLocator { } // return whether we should stop the scan - private boolean onScanNext(TableName tableName, LocateRequest req, Result result, - Throwable error) { -if (error != null) { - complete(tableName, req, null, error); - return true; -} + private boolean onScanNext(TableName tableName, LocateRequest req, Result result) { RegionLocations locs = MetaTableAccessor.getRegionLocations(result); LOG.debug("The fetched location of '{}', row='{}', locateType={} is {}", tableName, Bytes.toStringBinary(req.row), req.locateType, locs); @@ -298,7 +293,7 @@ class AsyncNonMetaRegionLocator { } if (loc.getServerName() == null) { complete(tableName, req, null, -new NoServerForRegionException( +new IOException( String.format("No server address listed for region '%s', row='%s', locateType=%s", info.getRegionNameAsString(), Bytes.toStringBinary(req.row), req.locateType))); return true; @@ -370,22 +365,28 @@ class AsyncNonMetaRegionLocator { private boolean completeNormally = false; + private boolean tableNotFound = true; + @Override public void onError(Throwable error) { -onScanNext(tableName, req, null, error); +complete(tableName, req, null, error); } @Override public void onComplete() { -if (!completeNormally) { - onScanNext(tableName, req, null, new TableNotFoundException(tableName)); +if (tableNotFound) { + complete(tableName, req, null, new TableNotFoundException(tableName)); +} else if (!c
hbase git commit: HBASE-15291 FileSystem not closed in secure bulkLoad
Repository: hbase Updated Branches: refs/heads/branch-2.0 2e6eff085 -> b3ec5f0ab HBASE-15291 FileSystem not closed in secure bulkLoad Signed-off-by: Ashish Singhi Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b3ec5f0a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b3ec5f0a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b3ec5f0a Branch: refs/heads/branch-2.0 Commit: b3ec5f0ab4e086b90a14df8ebda8849122ac7a70 Parents: 2e6eff0 Author: Ashish Singhi Authored: Wed Apr 11 12:22:46 2018 +0530 Committer: Ashish Singhi Committed: Wed Apr 11 12:24:16 2018 +0530 -- .../regionserver/SecureBulkLoadManager.java | 82 +--- 1 file changed, 54 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/b3ec5f0a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java index 264d985..a4ee517 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java @@ -145,15 +145,26 @@ public class SecureBulkLoadManager { public void cleanupBulkLoad(final HRegion region, final CleanupBulkLoadRequest request) throws IOException { -region.getCoprocessorHost().preCleanupBulkLoad(getActiveUser()); +try { + region.getCoprocessorHost().preCleanupBulkLoad(getActiveUser()); -Path path = new Path(request.getBulkToken()); -if (!fs.delete(path, true)) { - if (fs.exists(path)) { -throw new IOException("Failed to clean up " + path); + Path path = new Path(request.getBulkToken()); + if (!fs.delete(path, true)) { +if (fs.exists(path)) { + throw new IOException("Failed to clean up " + path); +} + } + LOG.info("Cleaned up " + path + " successfully."); +} finally { + UserGroupInformation ugi = getActiveUser().getUGI(); + try { +if (!UserGroupInformation.getLoginUser().equals(ugi)) { + FileSystem.closeAllForUGI(ugi); +} + } catch (IOException e) { +LOG.error("Failed to close FileSystem for: " + ugi, e); } } -LOG.info("Cleaned up " + path + " successfully."); } public Map> secureBulkLoadHFiles(final HRegion region, @@ -304,7 +315,7 @@ public class SecureBulkLoadManager { } if (srcFs == null) { -srcFs = FileSystem.get(p.toUri(), conf); +srcFs = FileSystem.newInstance(p.toUri(), conf); } if(!isFile(p)) { @@ -334,34 +345,49 @@ public class SecureBulkLoadManager { @Override public void doneBulkLoad(byte[] family, String srcPath) throws IOException { LOG.debug("Bulk Load done for: " + srcPath); + closeSrcFs(); +} + +private void closeSrcFs() throws IOException { + if (srcFs != null) { +srcFs.close(); +srcFs = null; + } } @Override public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException { - if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) { -// files are copied so no need to move them back -return; - } - Path p = new Path(srcPath); - Path stageP = new Path(stagingDir, - new Path(Bytes.toString(family), p.getName())); + try { +Path p = new Path(srcPath); +if (srcFs == null) { + srcFs = FileSystem.newInstance(p.toUri(), conf); +} +if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) { + // files are copied so no need to move them back + return; +} +Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName())); - // In case of Replication for bulk load files, hfiles are not renamed by end point during - // prepare stage, so no need of rename here again - if (p.equals(stageP)) { -LOG.debug(p.getName() + " is already available in source directory. Skipping rename."); -return; - } +// In case of Replication for bulk load files, hfiles are not renamed by end point during +// prepare stage, so no need of rename here again +if (p.equals(stageP)) { + LOG.debug(p.getName() + " is already available in source directory. Skipping rename."); + return; +} - LOG.debug("Moving " + stageP + " back to " + p); - if(!fs.rename(stageP, p)) -throw new IOException("Failed to move HFile: " + stageP + " to " +