http://git-wip-us.apache.org/repos/asf/hbase-site/blob/dab57116/apidocs/src-html/org/apache/hadoop/hbase/quotas/SpaceLimitingException.html ---------------------------------------------------------------------- diff --git a/apidocs/src-html/org/apache/hadoop/hbase/quotas/SpaceLimitingException.html b/apidocs/src-html/org/apache/hadoop/hbase/quotas/SpaceLimitingException.html new file mode 100644 index 0000000..7a52c0e --- /dev/null +++ b/apidocs/src-html/org/apache/hadoop/hbase/quotas/SpaceLimitingException.html @@ -0,0 +1,167 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> +<html lang="en"> +<head> +<title>Source code</title> +<link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> +</head> +<body> +<div class="sourceContainer"> +<pre><span class="sourceLineNo">001</span>/*<a name="line.1"></a> +<span class="sourceLineNo">002</span> * Licensed to the Apache Software Foundation (ASF) under one or more<a name="line.2"></a> +<span class="sourceLineNo">003</span> * contributor license agreements. See the NOTICE file distributed with<a name="line.3"></a> +<span class="sourceLineNo">004</span> * this work for additional information regarding copyright ownership.<a name="line.4"></a> +<span class="sourceLineNo">005</span> * The ASF licenses this file to you under the Apache License, Version 2.0<a name="line.5"></a> +<span class="sourceLineNo">006</span> * (the "License"); you may not use this file except in compliance with<a name="line.6"></a> +<span class="sourceLineNo">007</span> * the License. You may obtain a copy of the License at<a name="line.7"></a> +<span class="sourceLineNo">008</span> *<a name="line.8"></a> +<span class="sourceLineNo">009</span> * http://www.apache.org/licenses/LICENSE-2.0<a name="line.9"></a> +<span class="sourceLineNo">010</span> *<a name="line.10"></a> +<span class="sourceLineNo">011</span> * Unless required by applicable law or agreed to in writing, software<a name="line.11"></a> +<span class="sourceLineNo">012</span> * distributed under the License is distributed on an "AS IS" BASIS,<a name="line.12"></a> +<span class="sourceLineNo">013</span> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.<a name="line.13"></a> +<span class="sourceLineNo">014</span> * See the License for the specific language governing permissions and<a name="line.14"></a> +<span class="sourceLineNo">015</span> * limitations under the License.<a name="line.15"></a> +<span class="sourceLineNo">016</span> */<a name="line.16"></a> +<span class="sourceLineNo">017</span>package org.apache.hadoop.hbase.quotas;<a name="line.17"></a> +<span class="sourceLineNo">018</span><a name="line.18"></a> +<span class="sourceLineNo">019</span><a name="line.19"></a> +<span class="sourceLineNo">020</span>import org.apache.commons.logging.Log;<a name="line.20"></a> +<span class="sourceLineNo">021</span>import org.apache.commons.logging.LogFactory;<a name="line.21"></a> +<span class="sourceLineNo">022</span>import org.apache.hadoop.hbase.classification.InterfaceAudience;<a name="line.22"></a> +<span class="sourceLineNo">023</span>import org.apache.hadoop.hbase.classification.InterfaceStability;<a name="line.23"></a> +<span class="sourceLineNo">024</span><a name="line.24"></a> +<span class="sourceLineNo">025</span>/**<a name="line.25"></a> +<span class="sourceLineNo">026</span> * An Exception that is thrown when a space quota is in violation.<a name="line.26"></a> +<span class="sourceLineNo">027</span> */<a name="line.27"></a> +<span class="sourceLineNo">028</span>@InterfaceAudience.Public<a name="line.28"></a> +<span class="sourceLineNo">029</span>@InterfaceStability.Evolving<a name="line.29"></a> +<span class="sourceLineNo">030</span>public class SpaceLimitingException extends QuotaExceededException {<a name="line.30"></a> +<span class="sourceLineNo">031</span> private static final long serialVersionUID = 2319438922387583600L;<a name="line.31"></a> +<span class="sourceLineNo">032</span> private static final Log LOG = LogFactory.getLog(SpaceLimitingException.class);<a name="line.32"></a> +<span class="sourceLineNo">033</span> private static final String MESSAGE_PREFIX = SpaceLimitingException.class.getName() + ": ";<a name="line.33"></a> +<span class="sourceLineNo">034</span><a name="line.34"></a> +<span class="sourceLineNo">035</span> private final String policyName;<a name="line.35"></a> +<span class="sourceLineNo">036</span><a name="line.36"></a> +<span class="sourceLineNo">037</span> public SpaceLimitingException(String msg) {<a name="line.37"></a> +<span class="sourceLineNo">038</span> super(parseMessage(msg));<a name="line.38"></a> +<span class="sourceLineNo">039</span><a name="line.39"></a> +<span class="sourceLineNo">040</span> // Hack around ResponseConverter expecting to invoke a single-arg String constructor<a name="line.40"></a> +<span class="sourceLineNo">041</span> // on this class<a name="line.41"></a> +<span class="sourceLineNo">042</span> if (msg != null) {<a name="line.42"></a> +<span class="sourceLineNo">043</span> for (SpaceViolationPolicy definedPolicy : SpaceViolationPolicy.values()) {<a name="line.43"></a> +<span class="sourceLineNo">044</span> if (msg.indexOf(definedPolicy.name()) != -1) {<a name="line.44"></a> +<span class="sourceLineNo">045</span> policyName = definedPolicy.name();<a name="line.45"></a> +<span class="sourceLineNo">046</span> return;<a name="line.46"></a> +<span class="sourceLineNo">047</span> }<a name="line.47"></a> +<span class="sourceLineNo">048</span> }<a name="line.48"></a> +<span class="sourceLineNo">049</span> }<a name="line.49"></a> +<span class="sourceLineNo">050</span> policyName = null;<a name="line.50"></a> +<span class="sourceLineNo">051</span> }<a name="line.51"></a> +<span class="sourceLineNo">052</span><a name="line.52"></a> +<span class="sourceLineNo">053</span> public SpaceLimitingException(String policyName, String msg) {<a name="line.53"></a> +<span class="sourceLineNo">054</span> super(msg);<a name="line.54"></a> +<span class="sourceLineNo">055</span> this.policyName = policyName;<a name="line.55"></a> +<span class="sourceLineNo">056</span> }<a name="line.56"></a> +<span class="sourceLineNo">057</span><a name="line.57"></a> +<span class="sourceLineNo">058</span> public SpaceLimitingException(String policyName, String msg, Throwable e) {<a name="line.58"></a> +<span class="sourceLineNo">059</span> super(msg, e);<a name="line.59"></a> +<span class="sourceLineNo">060</span> this.policyName = policyName;<a name="line.60"></a> +<span class="sourceLineNo">061</span> }<a name="line.61"></a> +<span class="sourceLineNo">062</span><a name="line.62"></a> +<span class="sourceLineNo">063</span> /**<a name="line.63"></a> +<span class="sourceLineNo">064</span> * Returns the violation policy in effect.<a name="line.64"></a> +<span class="sourceLineNo">065</span> *<a name="line.65"></a> +<span class="sourceLineNo">066</span> * @return The violation policy in effect.<a name="line.66"></a> +<span class="sourceLineNo">067</span> */<a name="line.67"></a> +<span class="sourceLineNo">068</span> public String getViolationPolicy() {<a name="line.68"></a> +<span class="sourceLineNo">069</span> return this.policyName;<a name="line.69"></a> +<span class="sourceLineNo">070</span> }<a name="line.70"></a> +<span class="sourceLineNo">071</span><a name="line.71"></a> +<span class="sourceLineNo">072</span> private static String parseMessage(String originalMessage) {<a name="line.72"></a> +<span class="sourceLineNo">073</span> // Serialization of the exception places a duplicate class name. Try to strip that off if it<a name="line.73"></a> +<span class="sourceLineNo">074</span> // exists. Best effort... Looks something like:<a name="line.74"></a> +<span class="sourceLineNo">075</span> // "org.apache.hadoop.hbase.quotas.SpaceLimitingException: NO_INSERTS A Put is disallowed due<a name="line.75"></a> +<span class="sourceLineNo">076</span> // to a space quota."<a name="line.76"></a> +<span class="sourceLineNo">077</span> if (originalMessage != null && originalMessage.startsWith(MESSAGE_PREFIX)) {<a name="line.77"></a> +<span class="sourceLineNo">078</span> // If it starts with the class name, rip off the policy too.<a name="line.78"></a> +<span class="sourceLineNo">079</span> try {<a name="line.79"></a> +<span class="sourceLineNo">080</span> int index = originalMessage.indexOf(' ', MESSAGE_PREFIX.length());<a name="line.80"></a> +<span class="sourceLineNo">081</span> return originalMessage.substring(index + 1);<a name="line.81"></a> +<span class="sourceLineNo">082</span> } catch (Exception e) {<a name="line.82"></a> +<span class="sourceLineNo">083</span> if (LOG.isTraceEnabled()) {<a name="line.83"></a> +<span class="sourceLineNo">084</span> LOG.trace("Failed to trim exception message", e);<a name="line.84"></a> +<span class="sourceLineNo">085</span> }<a name="line.85"></a> +<span class="sourceLineNo">086</span> }<a name="line.86"></a> +<span class="sourceLineNo">087</span> }<a name="line.87"></a> +<span class="sourceLineNo">088</span> return originalMessage;<a name="line.88"></a> +<span class="sourceLineNo">089</span> }<a name="line.89"></a> +<span class="sourceLineNo">090</span><a name="line.90"></a> +<span class="sourceLineNo">091</span> @Override<a name="line.91"></a> +<span class="sourceLineNo">092</span> public String getMessage() {<a name="line.92"></a> +<span class="sourceLineNo">093</span> return (policyName == null ? "(unknown policy)" : policyName) + " " + super.getMessage();<a name="line.93"></a> +<span class="sourceLineNo">094</span> }<a name="line.94"></a> +<span class="sourceLineNo">095</span>}<a name="line.95"></a> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +</pre> +</div> +</body> +</html>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/dab57116/apidocs/src-html/org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.html ---------------------------------------------------------------------- diff --git a/apidocs/src-html/org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.html b/apidocs/src-html/org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.html new file mode 100644 index 0000000..1b919df --- /dev/null +++ b/apidocs/src-html/org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.html @@ -0,0 +1,117 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> +<html lang="en"> +<head> +<title>Source code</title> +<link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> +</head> +<body> +<div class="sourceContainer"> +<pre><span class="sourceLineNo">001</span>/*<a name="line.1"></a> +<span class="sourceLineNo">002</span> * Licensed to the Apache Software Foundation (ASF) under one or more<a name="line.2"></a> +<span class="sourceLineNo">003</span> * contributor license agreements. See the NOTICE file distributed with<a name="line.3"></a> +<span class="sourceLineNo">004</span> * this work for additional information regarding copyright ownership.<a name="line.4"></a> +<span class="sourceLineNo">005</span> * The ASF licenses this file to you under the Apache License, Version 2.0<a name="line.5"></a> +<span class="sourceLineNo">006</span> * (the "License"); you may not use this file except in compliance with<a name="line.6"></a> +<span class="sourceLineNo">007</span> * the License. You may obtain a copy of the License at<a name="line.7"></a> +<span class="sourceLineNo">008</span> *<a name="line.8"></a> +<span class="sourceLineNo">009</span> * http://www.apache.org/licenses/LICENSE-2.0<a name="line.9"></a> +<span class="sourceLineNo">010</span> *<a name="line.10"></a> +<span class="sourceLineNo">011</span> * Unless required by applicable law or agreed to in writing, software<a name="line.11"></a> +<span class="sourceLineNo">012</span> * distributed under the License is distributed on an "AS IS" BASIS,<a name="line.12"></a> +<span class="sourceLineNo">013</span> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.<a name="line.13"></a> +<span class="sourceLineNo">014</span> * See the License for the specific language governing permissions and<a name="line.14"></a> +<span class="sourceLineNo">015</span> * limitations under the License.<a name="line.15"></a> +<span class="sourceLineNo">016</span> */<a name="line.16"></a> +<span class="sourceLineNo">017</span>package org.apache.hadoop.hbase.quotas;<a name="line.17"></a> +<span class="sourceLineNo">018</span><a name="line.18"></a> +<span class="sourceLineNo">019</span>import org.apache.hadoop.hbase.classification.InterfaceAudience;<a name="line.19"></a> +<span class="sourceLineNo">020</span><a name="line.20"></a> +<span class="sourceLineNo">021</span>/**<a name="line.21"></a> +<span class="sourceLineNo">022</span> * Enumeration that represents the action HBase will take when a space quota is violated.<a name="line.22"></a> +<span class="sourceLineNo">023</span> *<a name="line.23"></a> +<span class="sourceLineNo">024</span> * The target for a violation policy is either an HBase table or namespace. In the case of a<a name="line.24"></a> +<span class="sourceLineNo">025</span> * namespace, it is treated as a collection of tables (all tables are subject to the same policy).<a name="line.25"></a> +<span class="sourceLineNo">026</span> */<a name="line.26"></a> +<span class="sourceLineNo">027</span>@InterfaceAudience.Public<a name="line.27"></a> +<span class="sourceLineNo">028</span>public enum SpaceViolationPolicy {<a name="line.28"></a> +<span class="sourceLineNo">029</span> /**<a name="line.29"></a> +<span class="sourceLineNo">030</span> * Disables the table(s).<a name="line.30"></a> +<span class="sourceLineNo">031</span> */<a name="line.31"></a> +<span class="sourceLineNo">032</span> DISABLE,<a name="line.32"></a> +<span class="sourceLineNo">033</span> /**<a name="line.33"></a> +<span class="sourceLineNo">034</span> * Disallows any mutations or compactions on the table(s).<a name="line.34"></a> +<span class="sourceLineNo">035</span> */<a name="line.35"></a> +<span class="sourceLineNo">036</span> NO_WRITES_COMPACTIONS,<a name="line.36"></a> +<span class="sourceLineNo">037</span> /**<a name="line.37"></a> +<span class="sourceLineNo">038</span> * Disallows any mutations (but allows compactions) on the table(s).<a name="line.38"></a> +<span class="sourceLineNo">039</span> */<a name="line.39"></a> +<span class="sourceLineNo">040</span> NO_WRITES,<a name="line.40"></a> +<span class="sourceLineNo">041</span> /**<a name="line.41"></a> +<span class="sourceLineNo">042</span> * Disallows any updates (but allows deletes and compactions) on the table(s).<a name="line.42"></a> +<span class="sourceLineNo">043</span> */<a name="line.43"></a> +<span class="sourceLineNo">044</span> NO_INSERTS;<a name="line.44"></a> +<span class="sourceLineNo">045</span>}<a name="line.45"></a> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +</pre> +</div> +</body> +</html> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/dab57116/book.html ---------------------------------------------------------------------- diff --git a/book.html b/book.html index 1769ade..e3950ab 100644 --- a/book.html +++ b/book.html @@ -26273,7 +26273,7 @@ handling multiple workloads: <div class="sect2"> <h3 id="quota"><a class="anchor" href="#quota"></a>136.1. Quotas</h3> <div class="paragraph"> -<p>HBASE-11598 introduces quotas, which allow you to throttle requests based on +<p>HBASE-11598 introduces RPC quotas, which allow you to throttle requests based on the following limits:</p> </div> <div class="olist arabic"> @@ -26491,6 +26491,103 @@ at the same time and that fewer scans can be executed at the same time. A value increase and the number of gets will decrease.</p> </div> </div> +<div class="sect2"> +<h3 id="space-quotas"><a class="anchor" href="#space-quotas"></a>136.4. Space Quotas</h3> +<div class="paragraph"> +<p><a href="https://issues.apache.org/jira/browse/HBASE-16961";>HBASE-16961</a> introduces a new type of +quotas for HBase to leverage: filesystem quotas. These "space" quotas limit the amount of space +on the filesystem that HBase namespaces and tables can consume. If a user, malicious or ignorant, +has the ability to write data into HBase, with enough time, that user can effectively crash HBase +(or worse HDFS) by consuming all available space. When there is no filesystem space available, +HBase crashes because it can no longer create/sync data to the write-ahead log.</p> +</div> +<div class="paragraph"> +<p>This feature allows a for a limit to be set on the size of a table or namespace. When a space quota is set +on a namespace, the quota’s limit applies to the sum of usage of all tables in that namespace. +When a table with a quota exists in a namespace with a quota, the table quota takes priority +over the namespace quota. This allows for a scenario where a large limit can be placed on +a collection of tables, but a single table in that collection can have a fine-grained limit set.</p> +</div> +<div class="paragraph"> +<p>The existing <code>set_quota</code> and <code>list_quota</code> HBase shell commands can be used to interact with +space quotas. Space quotas are quotas with a <code>TYPE</code> of <code>SPACE</code> and have <code>LIMIT</code> and <code>POLICY</code> +attributes. The <code>LIMIT</code> is a string that refers to the amount of space on the filesystem +that the quota subject (e.g. the table or namespace) may consume. For example, valid values +of <code>LIMIT</code> are <code>'10G'</code>, <code>'2T'</code>, or <code>'256M'</code>. The <code>POLICY</code> refers to the action that HBase will +take when the quota subject’s usage exceeds the <code>LIMIT</code>. The following are valid <code>POLICY</code> values.</p> +</div> +<div class="ulist"> +<ul> +<li> +<p><code>NO_INSERTS</code> - No new data may be written (e.g. <code>Put</code>, <code>Increment</code>, <code>Append</code>).</p> +</li> +<li> +<p><code>NO_WRITES</code> - Same as <code>NO_INSERTS</code> but <code>Deletes</code> are also disallowed.</p> +</li> +<li> +<p><code>NO_WRITES_COMPACTIONS</code> - Same as <code>NO_WRITES</code> but compactions are also disallowed.</p> +</li> +<li> +<p><code>DISABLE</code> - The table(s) are disabled, preventing all read/write access.</p> +</li> +</ul> +</div> +<div class="listingblock"> +<div class="title">Setting simple space quotas</div> +<div class="content"> +<pre># Sets a quota on the table 't1' with a limit of 1GB, disallowing Puts/Increments/Appends when the table exceeds 1GB +hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => '1G', POLICY => NO_INSERTS + +# Sets a quota on the namespace 'ns1' with a limit of 50TB, disallowing Puts/Increments/Appends/Deletes +hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '50T', POLICY => NO_WRITES + +# Sets a quota on the table 't3' with a limit of 2TB, disallowing any writes and compactions when the table exceeds 2TB. +hbase> set_quota TYPE => SPACE, TABLE => 't3', LIMIT => '2T', POLICY => NO_WRITES_COMPACTIONS + +# Sets a quota on the table 't2' with a limit of 50GB, disabling the table when it exceeds 50GB +hbase> set_quota TYPE => SPACE, TABLE => 't2', LIMIT => '50G', POLICY => DISABLE</pre> +</div> +</div> +<div class="paragraph"> +<p>Consider the following scenario to set up quotas on a namespace, overriding the quota on tables in that namespace</p> +</div> +<div class="listingblock"> +<div class="title">Table and Namespace space quotas</div> +<div class="content"> +<pre>hbase> create_namespace 'ns1' +hbase> create 'ns1:t1' +hbase> create 'ns1:t2' +hbase> create 'ns1:t3' +hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '100T', POLICY => NO_INSERTS +hbase> set_quota TYPE => SPACE, TABLE => 'ns1:t2', LIMIT => '200G', POLICY => NO_WRITES +hbase> set_quota TYPE => SPACE, TABLE => 'ns1:t3', LIMIT => '20T', POLICY => NO_WRITES</pre> +</div> +</div> +<div class="paragraph"> +<p>In the above scenario, the tables in the namespace <code>ns1</code> will not be allowed to consume more than +100TB of space on the filesystem among each other. The table 'ns1:t2' is only allowed to be 200GB in size, and will +disallow all writes when the usage exceeds this limit. The table 'ns1:t3' is allowed to grow to 20TB in size +and also will disallow all writes then the usage exceeds this limit. Because there is no table quota +on 'ns1:t1', this table can grow up to 100TB, but only if 'ns1:t2' and 'ns1:t3' have a usage of zero bytes. +Practically, it’s limit is 100TB less the current usage of 'ns1:t2' and 'ns1:t3'.</p> +</div> +</div> +<div class="sect2"> +<h3 id="ops.space.quota.deletion"><a class="anchor" href="#ops.space.quota.deletion"></a>136.5. Disabling Automatic Space Quota Deletion</h3> +<div class="paragraph"> +<p>By default, if a table or namespace is deleted that has a space quota, the quota itself is +also deleted. In some cases, it may be desirable for the space quota to not be automatically deleted. +In these cases, the user may configure the system to not delete any space quota automatically via hbase-site.xml.</p> +</div> +<div class="listingblock"> +<div class="content"> +<pre class="CodeRay highlight"><code data-lang="java"> <property> + <name>hbase.master.quota.observer.ignore</name> + <value><span class="predefined-constant">true</span></value> + </property></code></pre> +</div> +</div> +</div> </div> </div> <div class="sect1"> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/dab57116/bulk-loads.html ---------------------------------------------------------------------- diff --git a/bulk-loads.html b/bulk-loads.html index a8c1298..6bb5ba3 100644 --- a/bulk-loads.html +++ b/bulk-loads.html @@ -7,7 +7,7 @@ <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170522" /> + <meta name="Date-Revision-yyyymmdd" content="20170523" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache HBase – Bulk Loads in Apache HBase (TM) @@ -311,7 +311,7 @@ under the License. --> <a href="https://www.apache.org/";>The Apache Software Foundation</a>. All rights reserved. - <li id="publishDate" class="pull-right">Last Published: 2017-05-22</li> + <li id="publishDate" class="pull-right">Last Published: 2017-05-23</li> </p> </div>
