Repository: hbase-site Updated Branches: refs/heads/asf-site 350ed3878 -> a71e7848d
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a71e7848/testdevapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.html index 2bb994d..f34435c 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.html @@ -330,9 +330,9 @@ <span class="sourceLineNo">322</span> public void testOnConfigurationChange() throws Exception {<a name="line.322"></a> <span class="sourceLineNo">323</span> // constants<a name="line.323"></a> <span class="sourceLineNo">324</span> final int ORIGINAL_THROTTLE_POINT = 512 * 1024;<a name="line.324"></a> -<span class="sourceLineNo">325</span> final int ORIGINAL_QUEUE_SIZE = 512;<a name="line.325"></a> +<span class="sourceLineNo">325</span> final int ORIGINAL_QUEUE_INIT_SIZE = 512;<a name="line.325"></a> <span class="sourceLineNo">326</span> final int UPDATE_THROTTLE_POINT = 1024;// small enough to change large/small check<a name="line.326"></a> -<span class="sourceLineNo">327</span> final int UPDATE_QUEUE_SIZE = 1024;<a name="line.327"></a> +<span class="sourceLineNo">327</span> final int UPDATE_QUEUE_INIT_SIZE = 1024;<a name="line.327"></a> <span class="sourceLineNo">328</span> final int LARGE_FILE_NUM = 5;<a name="line.328"></a> <span class="sourceLineNo">329</span> final int SMALL_FILE_NUM = 20;<a name="line.329"></a> <span class="sourceLineNo">330</span><a name="line.330"></a> @@ -340,8 +340,8 @@ <span class="sourceLineNo">332</span> // no cleaner policies = delete all files<a name="line.332"></a> <span class="sourceLineNo">333</span> conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");<a name="line.333"></a> <span class="sourceLineNo">334</span> conf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, ORIGINAL_THROTTLE_POINT);<a name="line.334"></a> -<span class="sourceLineNo">335</span> conf.setInt(HFileCleaner.LARGE_HFILE_DELETE_QUEUE_SIZE, ORIGINAL_QUEUE_SIZE);<a name="line.335"></a> -<span class="sourceLineNo">336</span> conf.setInt(HFileCleaner.SMALL_HFILE_DELETE_QUEUE_SIZE, ORIGINAL_QUEUE_SIZE);<a name="line.336"></a> +<span class="sourceLineNo">335</span> conf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE);<a name="line.335"></a> +<span class="sourceLineNo">336</span> conf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, ORIGINAL_QUEUE_INIT_SIZE);<a name="line.336"></a> <span class="sourceLineNo">337</span> Server server = new DummyServer();<a name="line.337"></a> <span class="sourceLineNo">338</span> Path archivedHfileDir =<a name="line.338"></a> <span class="sourceLineNo">339</span> new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);<a name="line.339"></a> @@ -350,8 +350,8 @@ <span class="sourceLineNo">342</span> FileSystem fs = UTIL.getDFSCluster().getFileSystem();<a name="line.342"></a> <span class="sourceLineNo">343</span> final HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir);<a name="line.343"></a> <span class="sourceLineNo">344</span> Assert.assertEquals(ORIGINAL_THROTTLE_POINT, cleaner.getThrottlePoint());<a name="line.344"></a> -<span class="sourceLineNo">345</span> Assert.assertEquals(ORIGINAL_QUEUE_SIZE, cleaner.getLargeQueueSize());<a name="line.345"></a> -<span class="sourceLineNo">346</span> Assert.assertEquals(ORIGINAL_QUEUE_SIZE, cleaner.getSmallQueueSize());<a name="line.346"></a> +<span class="sourceLineNo">345</span> Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize());<a name="line.345"></a> +<span class="sourceLineNo">346</span> Assert.assertEquals(ORIGINAL_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize());<a name="line.346"></a> <span class="sourceLineNo">347</span><a name="line.347"></a> <span class="sourceLineNo">348</span> // clean up archive directory and create files for testing<a name="line.348"></a> <span class="sourceLineNo">349</span> fs.delete(archivedHfileDir, true);<a name="line.349"></a> @@ -367,60 +367,62 @@ <span class="sourceLineNo">359</span> };<a name="line.359"></a> <span class="sourceLineNo">360</span> t.setDaemon(true);<a name="line.360"></a> <span class="sourceLineNo">361</span> t.start();<a name="line.361"></a> -<span class="sourceLineNo">362</span> // let the cleaner run for some while<a name="line.362"></a> -<span class="sourceLineNo">363</span> Thread.sleep(20);<a name="line.363"></a> -<span class="sourceLineNo">364</span><a name="line.364"></a> -<span class="sourceLineNo">365</span> // trigger configuration change<a name="line.365"></a> -<span class="sourceLineNo">366</span> Configuration newConf = new Configuration(conf);<a name="line.366"></a> -<span class="sourceLineNo">367</span> newConf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, UPDATE_THROTTLE_POINT);<a name="line.367"></a> -<span class="sourceLineNo">368</span> newConf.setInt(HFileCleaner.LARGE_HFILE_DELETE_QUEUE_SIZE, UPDATE_QUEUE_SIZE);<a name="line.368"></a> -<span class="sourceLineNo">369</span> newConf.setInt(HFileCleaner.SMALL_HFILE_DELETE_QUEUE_SIZE, UPDATE_QUEUE_SIZE);<a name="line.369"></a> -<span class="sourceLineNo">370</span> cleaner.onConfigurationChange(newConf);<a name="line.370"></a> -<span class="sourceLineNo">371</span> LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()<a name="line.371"></a> -<span class="sourceLineNo">372</span> + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());<a name="line.372"></a> -<span class="sourceLineNo">373</span><a name="line.373"></a> -<span class="sourceLineNo">374</span> // check values after change<a name="line.374"></a> -<span class="sourceLineNo">375</span> Assert.assertEquals(UPDATE_THROTTLE_POINT, cleaner.getThrottlePoint());<a name="line.375"></a> -<span class="sourceLineNo">376</span> Assert.assertEquals(UPDATE_QUEUE_SIZE, cleaner.getLargeQueueSize());<a name="line.376"></a> -<span class="sourceLineNo">377</span> Assert.assertEquals(UPDATE_QUEUE_SIZE, cleaner.getSmallQueueSize());<a name="line.377"></a> -<span class="sourceLineNo">378</span> Assert.assertEquals(2, cleaner.getCleanerThreads().size());<a name="line.378"></a> -<span class="sourceLineNo">379</span><a name="line.379"></a> -<span class="sourceLineNo">380</span> // wait until clean done and check<a name="line.380"></a> -<span class="sourceLineNo">381</span> t.join();<a name="line.381"></a> -<span class="sourceLineNo">382</span> LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()<a name="line.382"></a> -<span class="sourceLineNo">383</span> + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());<a name="line.383"></a> -<span class="sourceLineNo">384</span> Assert.assertTrue("Should delete more than " + LARGE_FILE_NUM<a name="line.384"></a> -<span class="sourceLineNo">385</span> + " files from large queue but actually " + cleaner.getNumOfDeletedLargeFiles(),<a name="line.385"></a> -<span class="sourceLineNo">386</span> cleaner.getNumOfDeletedLargeFiles() > LARGE_FILE_NUM);<a name="line.386"></a> -<span class="sourceLineNo">387</span> Assert.assertTrue("Should delete less than " + SMALL_FILE_NUM<a name="line.387"></a> -<span class="sourceLineNo">388</span> + " files from small queue but actually " + cleaner.getNumOfDeletedSmallFiles(),<a name="line.388"></a> -<span class="sourceLineNo">389</span> cleaner.getNumOfDeletedSmallFiles() < SMALL_FILE_NUM);<a name="line.389"></a> -<span class="sourceLineNo">390</span> }<a name="line.390"></a> -<span class="sourceLineNo">391</span><a name="line.391"></a> -<span class="sourceLineNo">392</span> private void createFilesForTesting(int largeFileNum, int smallFileNum, FileSystem fs,<a name="line.392"></a> -<span class="sourceLineNo">393</span> Path archivedHfileDir) throws IOException {<a name="line.393"></a> -<span class="sourceLineNo">394</span> final Random rand = new Random();<a name="line.394"></a> -<span class="sourceLineNo">395</span> final byte[] large = new byte[1024 * 1024];<a name="line.395"></a> -<span class="sourceLineNo">396</span> for (int i = 0; i < large.length; i++) {<a name="line.396"></a> -<span class="sourceLineNo">397</span> large[i] = (byte) rand.nextInt(128);<a name="line.397"></a> -<span class="sourceLineNo">398</span> }<a name="line.398"></a> -<span class="sourceLineNo">399</span> final byte[] small = new byte[1024];<a name="line.399"></a> -<span class="sourceLineNo">400</span> for (int i = 0; i < small.length; i++) {<a name="line.400"></a> -<span class="sourceLineNo">401</span> small[i] = (byte) rand.nextInt(128);<a name="line.401"></a> -<span class="sourceLineNo">402</span> }<a name="line.402"></a> -<span class="sourceLineNo">403</span> // create large and small files<a name="line.403"></a> -<span class="sourceLineNo">404</span> for (int i = 1; i <= largeFileNum; i++) {<a name="line.404"></a> -<span class="sourceLineNo">405</span> FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "large-file-" + i));<a name="line.405"></a> -<span class="sourceLineNo">406</span> out.write(large);<a name="line.406"></a> -<span class="sourceLineNo">407</span> out.close();<a name="line.407"></a> -<span class="sourceLineNo">408</span> }<a name="line.408"></a> -<span class="sourceLineNo">409</span> for (int i = 1; i <= smallFileNum; i++) {<a name="line.409"></a> -<span class="sourceLineNo">410</span> FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "small-file-" + i));<a name="line.410"></a> -<span class="sourceLineNo">411</span> out.write(small);<a name="line.411"></a> -<span class="sourceLineNo">412</span> out.close();<a name="line.412"></a> -<span class="sourceLineNo">413</span> }<a name="line.413"></a> -<span class="sourceLineNo">414</span> }<a name="line.414"></a> -<span class="sourceLineNo">415</span>}<a name="line.415"></a> +<span class="sourceLineNo">362</span> // wait until file clean started<a name="line.362"></a> +<span class="sourceLineNo">363</span> while (cleaner.getNumOfDeletedSmallFiles() == 0) {<a name="line.363"></a> +<span class="sourceLineNo">364</span> Thread.yield();<a name="line.364"></a> +<span class="sourceLineNo">365</span> }<a name="line.365"></a> +<span class="sourceLineNo">366</span><a name="line.366"></a> +<span class="sourceLineNo">367</span> // trigger configuration change<a name="line.367"></a> +<span class="sourceLineNo">368</span> Configuration newConf = new Configuration(conf);<a name="line.368"></a> +<span class="sourceLineNo">369</span> newConf.setInt(HFileCleaner.HFILE_DELETE_THROTTLE_THRESHOLD, UPDATE_THROTTLE_POINT);<a name="line.369"></a> +<span class="sourceLineNo">370</span> newConf.setInt(HFileCleaner.LARGE_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE);<a name="line.370"></a> +<span class="sourceLineNo">371</span> newConf.setInt(HFileCleaner.SMALL_HFILE_QUEUE_INIT_SIZE, UPDATE_QUEUE_INIT_SIZE);<a name="line.371"></a> +<span class="sourceLineNo">372</span> LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()<a name="line.372"></a> +<span class="sourceLineNo">373</span> + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());<a name="line.373"></a> +<span class="sourceLineNo">374</span> cleaner.onConfigurationChange(newConf);<a name="line.374"></a> +<span class="sourceLineNo">375</span><a name="line.375"></a> +<span class="sourceLineNo">376</span> // check values after change<a name="line.376"></a> +<span class="sourceLineNo">377</span> Assert.assertEquals(UPDATE_THROTTLE_POINT, cleaner.getThrottlePoint());<a name="line.377"></a> +<span class="sourceLineNo">378</span> Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getLargeQueueInitSize());<a name="line.378"></a> +<span class="sourceLineNo">379</span> Assert.assertEquals(UPDATE_QUEUE_INIT_SIZE, cleaner.getSmallQueueInitSize());<a name="line.379"></a> +<span class="sourceLineNo">380</span> Assert.assertEquals(2, cleaner.getCleanerThreads().size());<a name="line.380"></a> +<span class="sourceLineNo">381</span><a name="line.381"></a> +<span class="sourceLineNo">382</span> // wait until clean done and check<a name="line.382"></a> +<span class="sourceLineNo">383</span> t.join();<a name="line.383"></a> +<span class="sourceLineNo">384</span> LOG.debug("File deleted from large queue: " + cleaner.getNumOfDeletedLargeFiles()<a name="line.384"></a> +<span class="sourceLineNo">385</span> + "; from small queue: " + cleaner.getNumOfDeletedSmallFiles());<a name="line.385"></a> +<span class="sourceLineNo">386</span> Assert.assertTrue("Should delete more than " + LARGE_FILE_NUM<a name="line.386"></a> +<span class="sourceLineNo">387</span> + " files from large queue but actually " + cleaner.getNumOfDeletedLargeFiles(),<a name="line.387"></a> +<span class="sourceLineNo">388</span> cleaner.getNumOfDeletedLargeFiles() > LARGE_FILE_NUM);<a name="line.388"></a> +<span class="sourceLineNo">389</span> Assert.assertTrue("Should delete less than " + SMALL_FILE_NUM<a name="line.389"></a> +<span class="sourceLineNo">390</span> + " files from small queue but actually " + cleaner.getNumOfDeletedSmallFiles(),<a name="line.390"></a> +<span class="sourceLineNo">391</span> cleaner.getNumOfDeletedSmallFiles() < SMALL_FILE_NUM);<a name="line.391"></a> +<span class="sourceLineNo">392</span> }<a name="line.392"></a> +<span class="sourceLineNo">393</span><a name="line.393"></a> +<span class="sourceLineNo">394</span> private void createFilesForTesting(int largeFileNum, int smallFileNum, FileSystem fs,<a name="line.394"></a> +<span class="sourceLineNo">395</span> Path archivedHfileDir) throws IOException {<a name="line.395"></a> +<span class="sourceLineNo">396</span> final Random rand = new Random();<a name="line.396"></a> +<span class="sourceLineNo">397</span> final byte[] large = new byte[1024 * 1024];<a name="line.397"></a> +<span class="sourceLineNo">398</span> for (int i = 0; i < large.length; i++) {<a name="line.398"></a> +<span class="sourceLineNo">399</span> large[i] = (byte) rand.nextInt(128);<a name="line.399"></a> +<span class="sourceLineNo">400</span> }<a name="line.400"></a> +<span class="sourceLineNo">401</span> final byte[] small = new byte[1024];<a name="line.401"></a> +<span class="sourceLineNo">402</span> for (int i = 0; i < small.length; i++) {<a name="line.402"></a> +<span class="sourceLineNo">403</span> small[i] = (byte) rand.nextInt(128);<a name="line.403"></a> +<span class="sourceLineNo">404</span> }<a name="line.404"></a> +<span class="sourceLineNo">405</span> // create large and small files<a name="line.405"></a> +<span class="sourceLineNo">406</span> for (int i = 1; i <= largeFileNum; i++) {<a name="line.406"></a> +<span class="sourceLineNo">407</span> FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "large-file-" + i));<a name="line.407"></a> +<span class="sourceLineNo">408</span> out.write(large);<a name="line.408"></a> +<span class="sourceLineNo">409</span> out.close();<a name="line.409"></a> +<span class="sourceLineNo">410</span> }<a name="line.410"></a> +<span class="sourceLineNo">411</span> for (int i = 1; i <= smallFileNum; i++) {<a name="line.411"></a> +<span class="sourceLineNo">412</span> FSDataOutputStream out = fs.create(new Path(archivedHfileDir, "small-file-" + i));<a name="line.412"></a> +<span class="sourceLineNo">413</span> out.write(small);<a name="line.413"></a> +<span class="sourceLineNo">414</span> out.close();<a name="line.414"></a> +<span class="sourceLineNo">415</span> }<a name="line.415"></a> +<span class="sourceLineNo">416</span> }<a name="line.416"></a> +<span class="sourceLineNo">417</span>}<a name="line.417"></a> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a71e7848/testdevapidocs/src-html/org/apache/hadoop/hbase/util/TestStealJobQueue.TestTask.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/util/TestStealJobQueue.TestTask.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/util/TestStealJobQueue.TestTask.html index d1f3510..0a7fae6 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/util/TestStealJobQueue.TestTask.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/util/TestStealJobQueue.TestTask.html @@ -46,7 +46,7 @@ <span class="sourceLineNo">038</span>public class TestStealJobQueue {<a name="line.38"></a> <span class="sourceLineNo">039</span><a name="line.39"></a> <span class="sourceLineNo">040</span> StealJobQueue<Integer> stealJobQueue;<a name="line.40"></a> -<span class="sourceLineNo">041</span> BlockingQueue stealFromQueue;<a name="line.41"></a> +<span class="sourceLineNo">041</span> BlockingQueue<Integer> stealFromQueue;<a name="line.41"></a> <span class="sourceLineNo">042</span><a name="line.42"></a> <span class="sourceLineNo">043</span> @Before<a name="line.43"></a> <span class="sourceLineNo">044</span> public void setup() {<a name="line.44"></a> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a71e7848/testdevapidocs/src-html/org/apache/hadoop/hbase/util/TestStealJobQueue.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/util/TestStealJobQueue.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/util/TestStealJobQueue.html index d1f3510..0a7fae6 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/util/TestStealJobQueue.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/util/TestStealJobQueue.html @@ -46,7 +46,7 @@ <span class="sourceLineNo">038</span>public class TestStealJobQueue {<a name="line.38"></a> <span class="sourceLineNo">039</span><a name="line.39"></a> <span class="sourceLineNo">040</span> StealJobQueue<Integer> stealJobQueue;<a name="line.40"></a> -<span class="sourceLineNo">041</span> BlockingQueue stealFromQueue;<a name="line.41"></a> +<span class="sourceLineNo">041</span> BlockingQueue<Integer> stealFromQueue;<a name="line.41"></a> <span class="sourceLineNo">042</span><a name="line.42"></a> <span class="sourceLineNo">043</span> @Before<a name="line.43"></a> <span class="sourceLineNo">044</span> public void setup() {<a name="line.44"></a>
