http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/c782b73a/content/css/group.css ---------------------------------------------------------------------- diff --git a/content/css/group.css b/content/css/group.css deleted file mode 100644 index b03d722..0000000 --- a/content/css/group.css +++ /dev/null @@ -1,215 +0,0 @@ -/* Space out content a bit */ - html { - overflow:scroll; -} - -body { - padding-top: 20px; - padding-bottom: 20px; -} - -/* Everything but the jumbotron gets side spacing for mobile first views */ -.header, -.footer { - padding-left: 15px; - padding-right: 15px; -} - -/* Custom page header */ -.header { - border-bottom: 1px solid #e5e5e5; -} - -/* Custom page footer */ -.footer { - padding-top: 19px; - color: #777; - border-top: 1px solid #e5e5e5; - text-align: center; - font-size: 0.8em; - clear: both; -} -.footer p { - margin: 0.1em 0; -} -.footer a:link, .footer a:visited { - color: #777; -} - -/* Customize container */ -@media (min-width: 768px) { - .container { - max-width: 767px; - } -} -.container-narrow > hr { - margin: 30px 0; -} - -/* Jumbotron */ -.jumbotron { - text-align: center; - border-bottom: 1px solid #e5e5e5; -} -.jumbotron p { - font-size: 15px; -} -.jumbotron { - padding-top: 18px; - padding-bottom: 10px; -} - -/* Group logo */ -#logo { - height: 35px; - margin: 2px 0 8px 0; -} - -/* Responsive: Portrait tablets and up */ -@media screen and (min-width: 768px) { - /* Remove the padding we set earlier */ - .header, - .footer { - clear: both; - padding-left: 0; - padding-right: 0; - } - /* Space out the masthead */ - .header { - margin-bottom: 30px; - } - /* Remove the bottom border on the jumbotron for visual effect */ - .jumbotron { - border-bottom: 0; - } -} - -/* Slightly better-looking header on mobile. */ -@media screen and (max-width: 767px) { - .nav-pills { - display: block; - margin: 0 auto 1em; - } - #logo { - display: block; - clear: both; - margin: 1em auto; - } -} - -/* Photo */ -.inset-image { - width: 60%; - margin: 0 auto; -} - -/* People list */ - -dl.people dl { - width: 100%; - clear: both; - overflow: hidden; - margin: 0; -} - -dl.people dt { - clear: both; - float: left; - content: " "; - width: 15%; - margin-bottom: 2em; - margin: 0; -} - -dl.people dd { - width: 85%; - min-height: 6.5em; - margin-bottom: 1em; -} - - - -/* Front page news. */ -ul.news .date { - color: #999; - font-weight: bold; - display: block; -} -ul.news > li { - margin-top: 1em; -} -ul.news .shortnews .date { - float: left; - margin-right: 1em; -} -ul.news .bloglink a { - font-size: 1.4em; -} - -.illustration { - float: right; - margin: 0 0 1em 1em; -} - -ul.double-col { - columns: 2; - -webkit-columns: 2; - -moz-columns: 2; - overflow:hidden; - width: 800px; -} - -div.bio { - margin: 0 0 1em 1em; -} - -ul.double-col li { - clear: both; - height: auto; - display: inline; - vertical-align: middle; - width: 100%; - margin: .5rem 0rem; - float: left; -} - -.col-md-8 { -} - -.col-md-4 { -} - -table tr:nth-child(odd) { - background-color: #FFFFFF; -} - -table tr:nth-child(even) { - background-color: #F5F5F5; -} - -table tr:first-child th { - border-top: 1; -} - -table tr:last-child td { - border-bottom: 1; -} - -table tr td:first-child, table tr th:first-child { - border-left: 1; -} - -table tr td:last-child, table tr th:last-child { - border-right: 1; -} -table, th { - background-color: #F5F5F5; - border-color: lightgrey; -} - -th, td { - border: 0.75px solid grey; - padding: 7px; - line-height: 24px; - border-color: lightgrey; -}
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/c782b73a/content/css/print.css ---------------------------------------------------------------------- diff --git a/content/css/print.css b/content/css/print.css deleted file mode 100644 index e95929e..0000000 --- a/content/css/print.css +++ /dev/null @@ -1,36 +0,0 @@ -.container { - -moz-column-count: 2; - -webkit-column-count: 2; - column-count: 2; - - font-size: 0.8em; -} - -.header { - display: none; -} - -h2 { - font-size: 1.3em; - margin: 0; -} -h3 { - font-size: 1.2em; - margin-top: 0; -} - -.noprint { - display: none; -} - -body { - padding: -2em 0 0 0; -} - -/* Disable Bootstrap's link display. */ -@media print { - a:link:after, - a:visited:after { - content: "" !important; - } -} http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/c782b73a/content/documentation/index.html ---------------------------------------------------------------------- diff --git a/content/documentation/index.html b/content/documentation/index.html deleted file mode 100644 index 68ee8c8..0000000 --- a/content/documentation/index.html +++ /dev/null @@ -1,358 +0,0 @@ -<!DOCTYPE html> -<html> - <head> - <meta charset="utf-8"> - <title>The Apache Crail (Incubating) Project: Documentation</title> - <meta name="viewport" content="width=device-width, initial-scale=1.0"> - <link href="http://crail.incubator.apache.org/css/bootstrap.min.css"; rel="stylesheet"> - <link href="http://crail.incubator.apache.org/css/group.css"; rel="stylesheet"> - <link rel="alternate" type="application/atom+xml" title="Atom" - href="http://crail.incubator.apache.org/blog/blog.xml";> - - <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/documentation-summary.png"; /> - <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/documentation-summary.png"; /> - </head> - - <body> - <div class="container"> - <div class="header"> - <ul class="nav nav-pills pull-right"> - - - - <li > - <a href="http://crail.incubator.apache.org/";> - Home - </a> - </li> - - - <li > - <a href="http://crail.incubator.apache.org/overview/";> - Overview - </a> - </li> - - - <li > - <a href="http://crail.incubator.apache.org/blog/";> - Blog - </a> - </li> - - - <li > - <a href="http://crail.incubator.apache.org/community/";> - Community - </a> - </li> - - - <li class="active"> - <a href="http://crail.incubator.apache.org/documentation/";> - Documentation - </a> - </li> - - </ul> - <a href="http://crail.incubator.apache.org/";> - <img src="http://crail.incubator.apache.org/img/crail_logo.png"; - srcset="http://crail.incubator.apache.org/img/crail_logo.png"; - alt="Crail" id="logo"> - </a> - </div> - - - - <h2>Documentation</h2> - - - <p>Apache Crail (Incubating) is a fast multi-tiered distributed storage system designed from ground up for high-performance network and storage hardware. The unique features of Crail include:</p> - -<ul> - <li>Zero-copy network access from userspace</li> - <li>Integration of multiple storage tiers such DRAM, flash and disaggregated shared storage</li> - <li>Ultra-low latencies for both meta data and data operations. For instance: opening, reading and closing a small file residing in the distributed DRAM tier less than 10 microseconds, which is in the same ballpark as some of the fastest RDMA-based key/value stores</li> - <li>High-performance sequential read/write operations: For instance: read operations on large files residing in the distributed DRAM tier are typically limited only by the performance of the network</li> - <li>Very low CPU consumption: a single core sharing both application and file system client can drive sequential read/write operations at the speed of up to 100Gbps and more</li> - <li>Asynchronous API leveraging the asynchronous nature of RDMA-based networking hardware</li> - <li>Extensible plugin architecture: new storage tiers tailored to specific hardware can be added easily</li> -</ul> - -<p>Crail is implemented in Java offering a Java API which integrates directly with the Java off-heap memory. Crail is designed for performance critical temporary data within a scope of a rack or two.</p> - -<h2 id="requirements">Requirements</h2> - -<ul> - <li>Java 8 or higher</li> - <li>RDMA-based network, e.g., Infiniband, iWARP, RoCE. There are two options to run Crail without RDMA networking hardware: (a) use SoftiWARP, (b) us the TCP/DRAM storage tier</li> - <li>Libdisni.so, available as part of <a href="https://github.com/zrlio/disni";>DiSNI</a></li> -</ul> - -<h2 id="building">Building</h2> - -<p>To build Crail from source using <a href="http://maven.apache.org/";>Apache Maven</a> execute the following steps:</p> - -<ol> - <li>Obtain a copy of <a href="https://github.com/apache/incubator-crail";>Crail</a> from Github</li> - <li>Run: mvn -DskipTests install</li> - <li>Copy tarball to the cluster and unpack it using tar xvfz crail-1.0-bin.tar.gz</li> -</ol> - -<p>Note: later, when deploying Crail, make sure libdisni.so is part of your LD_LIBRARY_PATH. The easiest way to make it work is to copy libdisni.so into crail-1.0/lib</p> - -<h2 id="configuration">Configuration</h2> - -<p>To configure Crail use crail-site.conf.template as a basis and modify it to match your environment.</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>cd crail-1.0/conf -mv crail-site.conf.template crail-site.conf -</code></pre></div></div> - -<p>There are a general file system properties and specific properties for the different storage tiers. A typical configuration for the general file system section may look as follows:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>crail.namenode.address crail://namenode:9060 -crail.storage.types org.apache.crail.storage.rdma.RdmaStorageTier -crail.cachepath /dev/hugepages/cache -crail.cachelimit 12884901888 -crail.blocksize 1048576 -crail.buffersize 1048576 -</code></pre></div></div> - -<p>In this configuration the namenode is configured to run using port 9060 on host ânamenodeâ, which must be a valid host in the cluster. We further configure a single storage tier, in this case the RDMA-based DRAM tier. The cachepath property needs to point to a directory that is used by the file system to allocate memory for the client cache. Up to cachelimit size, all the memory that is used by Crail will be allocated via mmap from this location. Ideally, the directory specified in cachepath points to a hugetlbfs mountpoint. Aside from the general properties, each storage tier needs to be configured separately.</p> - -<h3 id="rdmadram-storage">RDMA/DRAM Storage</h3> - -<p>For the RDMA/DRAM tier we need to specify the interface that should be used by the storage nodes.</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>crail.storage.rdma.interface eth0 -</code></pre></div></div> - -<p>The datapath property specifies a path from which the storage nodes will allocate blocks of memory via mmap. Again, that path best points to a hugetlbfs mountpoint.</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>crail.storage.rdma.datapath /memory/data -</code></pre></div></div> - -<p>You want to specify how much DRAM each datanode should donate into the file system pool using the <code class="highlighter-rouge">storagelimit</code> property. DRAM is allocated in chunks of <code class="highlighter-rouge">allocationsize</code>, which needs to be a multiple of <code class="highlighter-rouge">crail.blocksize</code>.</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>crail.storage.rdma.allocationsize 1073741824 -crail.storage.rdma.storagelimit 75161927680 -</code></pre></div></div> - -<p>Crail supports optimized local operations via memcpy (instead of RDMA) in case a given file operation is backed by a local storage node. The indexpath specifies where Crail will store the necessary metadata that make these optimizations possible. Important: the indexpath must NOT point to a hugetlbfs mountpoint because index files will be updated which not possible in hugetlbfs.</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>crail.storage.rdma.localmap true -crail.storage.rdma.indexpath /index -</code></pre></div></div> - -<h3 id="nvmfflash-storage">NVMf/Flash Storage</h3> - -<p>Crail is a multi-tiered storage system. Additinoal tiers can be enabled by adding them to the configuration as follows.</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>crail.storage.types org.apache.crail.storage.rdma.RdmaStorageTier,org.apache.crail.storage.nvmf.NvmfStorageTier -</code></pre></div></div> - -<p>For the NVMf storage tier we need to configure the server IP that is used when listening for new connections. We also need to configure the PCI address of the flash device we want to use, as well as the huge page mount point to be used for allocating memory.</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>crail.storage.nvmf.bindip 10.40.0.XX -crail.storage.nvmf.pcieaddr 0000:11:00.0 -crail.storage.nvmf.hugedir /dev/hugepages -crail.storage.nvmf.servermempool 512 -crail.storage.nvmf.clientmempool 512 -</code></pre></div></div> - -<h2 id="deploying">Deploying</h2> - -<p>For all deployments, make sure you define CRAIL_HOME on each machine to point to the top level Crail directory.</p> - -<h3 id="starting-crail-manually">Starting Crail manually</h3> - -<p>The simplest way to run Crail is to start it manually on just a handful nodes. You will need to start the Crail namenode, plus at least one datanode. To start the namenode execute the following command on the host that is configured to be the namenode:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>cd crail-1.0/ -./bin/crail namenode -</code></pre></div></div> - -<p>To start a datanode run the following command on a host in the cluster (ideally this is a different physical machine than the one running the namenode):</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail datanode -</code></pre></div></div> - -<p>Now you should have a small deployment up with just one datanode. In this case the datanode is of type RDMA/DRAM, which is the default datnode. If you want to start a different storage tier you can do so by passing a specific datanode class as follows:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail datanode -t org.apache.crail.storage.nvmf.NvmfStorageTier -</code></pre></div></div> - -<p>This would start the shared storage datanode. Note that configuration in crail-site.conf needs to have the specific properties set of this type of datanode, in order for this to work.</p> - -<h3 id="larger-deployments">Larger deployments</h3> - -<p>To run larger deployments start Crail using</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/start-crail.sh -</code></pre></div></div> - -<p>Similarly, Crail can be stopped by using</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/stop-crail.sh -</code></pre></div></div> - -<p>For this to work include the list of machines to start datanodes in conf/slaves. You can start multiple datanode of different types on the same host as follows:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>host02-ib -host02-ib -t org.apache.crail.storage.nvmf.NvmfStorageTier -host03-ib -</code></pre></div></div> - -<p>In this example, we are configuring a Crail cluster with 2 physical hosts but 3 datanodes and two different storage tiers.</p> - -<h2 id="crail-shell">Crail Shell</h2> - -<p>Crail provides an contains an HDFS adaptor, thus, you can interact with Crail using the HDFS shell:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail fs -</code></pre></div></div> - -<p>Crail, however, does not implement the full HDFS shell functionality. The basic commands to copy file to/from Crail, or to move and delete files, will work.</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail fs -mkdir /test -./bin/crail fs -ls / -./bin/crail fs -copyFromLocal <path-to-local-file> /test -./bin/crail fs -cat /test/<file-name> -</code></pre></div></div> - -<p>For the Crail shell to work properly, the HDFS configuration in crail-1.0/conf/core-site.xml needs to be configured accordingly:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code><configuration> - <property> - <name>fs.crail.impl</name> - <value>org.apache.crail.hdfs.CrailHadoopFileSystem</value> - </property> - <property> - <name>fs.defaultFS</name> - <value>crail://namenode:9060</value> - </property> - <property> - <name>fs.AbstractFileSystem.crail.impl</name> - <value>org.apache.crail.hdfs.CrailHDFS</value> - </property> - </configuration> -</code></pre></div></div> - -<p>Note that the Crail HDFS interface currently cannot provide the full performance of Crail due to limitations of the HDFS API. In particular, the HDFS <code class="highlighter-rouge">FSDataOutputStream</code> API only support heap-based <code class="highlighter-rouge">byte[]</code> arrays which requires a data copy. Moreover, HDFS operations are synchronous preventing efficient pipelining of operations. Instead, applications that seek the best performance should use the Crail interface directly, as shown next.</p> - -<h2 id="programming-against-crail">Programming against Crail</h2> - -<p>The best way to program against Crail is to use Maven. Make sure you have the Crail dependency specified in your application pom.xml file:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code><dependency> - <groupId>org.apache.crail</groupId> - <artifactId>crail-client</artifactId> - <version>1.0</version> -</dependency> -</code></pre></div></div> - -<p>Then, create a Crail client as follows:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>CrailConfiguration conf = new CrailConfiguration(); -CrailStore store = CrailStore.newInstance(conf); -</code></pre></div></div> - -<p>Make sure the crail-1.0/conf directory is part of the classpath.</p> - -<p>Crail supports different file types. The simplest way to create a file in Crail is as follows:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>CrailFile file = store.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get().syncDir(); -</code></pre></div></div> - -<p>Aside from the actual filename, the âcreate()â call takes as input the storage and location classes which are preferences for the storage tier and physical location that this file should be created in. Crail tries to satisfy these preferences later when the file is written. In the example we do not request any particular storage or location affinity.</p> - -<p>This âcreate()â command is non-blocking, calling âget()â on the returning future object awaits the completion of the call. At that time, the file has been created, but its directory entry may not be visible. Therefore, the file may not yet show up in a file enumeration of the given parent directory. Calling âsyncDir()â waits to for the directory entry to be completed. Both the âget()â and the âsyncDir()â operation can be deffered to a later time at which they may become non-blocking operations.</p> - -<p>Once the file is created, a file stream can be obtained for writing:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>CrailBufferedOutputStream outstream = file.getBufferedOutputStream(1024); -</code></pre></div></div> - -<p>Here, we create a buffered stream so that we can pass heap byte arrays as well. We could also create a non-buffered stream using</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>CrailOutputStream outstream = file.getDirectOutputStream(1024); -</code></pre></div></div> - -<p>In both cases, we pass a write hint (1024 in the example) that indicates to Crail how much data we are intending to write. This allows Crail to optimize metadatanode lookups. Crail never prefetches data, but it may fetch the metadata of the very next operation concurrently with the current data operation if the write hint allows to do so.</p> - -<p>Once the stream has been obtained, there exist various ways to write a file. The code snippet below shows the use of the asynchronous interface:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>CrailBuffer dataBuf = fs.allocateBuffer(); -Future<DataResult> future = outputStream.write(dataBuf); -... -future.get(); -</code></pre></div></div> - -<p>Reading files works very similar to writing. There exist various examples in org.apache.crail.tools.CrailBenchmark.</p> - -<h2 id="tcp-storage-tiers-and-rpc-binding">TCP Storage Tiers and RPC binding</h2> - -<p>Crail is designed for user-level networking and storage. It does, however, also provide plain TCP-based storage backends for storage and RPC and, thus, can be run easily on any machine without requiring spspecial hardware support. The TCP storage backend can be enabled as follows:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>crail.storage.types org.apache.crail.storage.tcp.TcpStorageTier -</code></pre></div></div> - -<p>The TCP RPC binding can be enabled as follows:</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>crail.namenode.rpctype org.apache.crail.namenode.rpc.tcp.TcpNameNode -</code></pre></div></div> - -<h2 id="benchmarks">Benchmarks</h2> - -<p>Crail provides a set of benchmark tools to measure the performance. Type</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -</code></pre></div></div> - -<p>to get an overview of the available benchmarks. For instance, to benchmark the sequential write performance, type</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -t write -s 1048576 -k 102400 -f /tmp.dat -</code></pre></div></div> - -<p>This will create a file of size 100G, written sequentially in a sequence of 1MB operations.</p> - -<p>To read a file sequentially, type</p> - -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -t read -s 1048576 -k 102400 -f /tmp.dat -</code></pre></div></div> - -<p>This command issues 102400 read operations of 1MB each.</p> - -<p>The tool also contains benchmarks to read files randomly, or to measure the performance of opening files, etc.</p> - -<h2 id="applications">Applications</h2> - -<p>Crail is used by <a href="https://github.com/zrlio/crail-spark-io";>Crail-Spark-IO</a>, a high-performance shuffle engine for Spark. <a href="https://github.com/zrlio/crail-terasort";>Crail-Terasort</a> is a fast sorting benchmark for Spark based on Crail.</p> - -<h2 id="contributions">Contributions</h2> - -<p>PRs are always welcome. Please fork, and make necessary modifications -you propose, and let us know.</p> - -<h2 id="contact">Contact</h2> - -<p>Please join the Crail developer mailing list for discussions and notifications. The list is at:</p> - -<p>d...@crail.incubator.apache.org.</p> - - - <br> - <br> - <div class="footer"> - <p>Apache Crail is an effort undergoing <a href="https://incubator.apache.org/";>incubation</a> at <a href="https://www.apache.org/";>The Apache Software Foundation (ASF)</a>, sponsored by the Apache Incubator PMC. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF. - </p> - </div> - - </div> <!-- /container --> - - <!-- Support retina images. --> - <script type="text/javascript" - src="http://crail.incubator.apache.org/js/srcset-polyfill.js";></script> - </body> -</html> http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/c782b73a/content/feed.xml ---------------------------------------------------------------------- diff --git a/content/feed.xml b/content/feed.xml index 9ec960d..247c017 100644 --- a/content/feed.xml +++ b/content/feed.xml @@ -1,670 +1 @@ -<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom"; ><generator uri="https://jekyllrb.com/"; version="3.6.2">Jekyll</generator><link href="http://crail.incubator.apache.org//feed.xml"; rel="self" type="application/atom+xml" /><link href="http://crail.incubator.apache.org//"; rel="alternate" type="text/html" /><updated>2018-06-05T11:03:53+02:00</updated><id>http://crail.incubator.apache.org//</id><title type="html">The Apache Crail (Incubating) Project</title><entry><title type="html">Sparksummit</title><link href="http://crail.incubator.apache.org//blog/2018/06/sparksummit.html"; rel="alternate" type="text/html" title="Sparksummit" /><published>2018-06-05T00:00:00+02:00</published><updated>2018-06-05T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/06/sparksummit</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/06/sparksummit.html";><p>A Spark serverless architecture powered by Crail will be presente d today at the <a href="https://databricks.com/session/serverless-machine-learning-on-modern-hardware-using-apache-spark">Spark Summit</a></p></content><author><name></name></author><category term="news" /><summary type="html">A Spark serverless architecture powered by Crail will be presented today at the Spark Summit</summary></entry><entry><title type="html">Dataworks</title><link href="http://crail.incubator.apache.org//blog/2018/06/dataworks.html"; rel="alternate" type="text/html" title="Dataworks" /><published>2018-06-05T00:00:00+02:00</published><updated>2018-06-05T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/06/dataworks</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/06/dataworks.html";><p>Apache Crail (incubating) to feature in the <a href="https://dataworkssummit.com/san-jose-2018/session/data-processing-at-the-speed-of-100-gbpsapache-crail-incubating/">DataWorks Sum mit</a> on June 21st</p></content><author><name></name></author><category term="news" /><summary type="html">Apache Crail (incubating) to feature in the DataWorks Summit on June 21st</summary></entry><entry><title type="html">Apache Release</title><link href="http://crail.incubator.apache.org//blog/2018/06/apache-release.html"; rel="alternate" type="text/html" title="Apache Release" /><published>2018-06-04T00:00:00+02:00</published><updated>2018-06-04T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/06/apache-release</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/06/apache-release.html";><p>Apache Crail 1.0 incubator <a href="https://dist.apache.org/repos/dist/release/incubator/crail/1.0-incubating/">release</a></p></content><author><name></name></author><category term="news" /><summary type="html">Apache Crail 1.0 incubator release</summary></entry><entry><title type="html">Apache</ title><link href="http://crail.incubator.apache.org//blog/2018/01/apache.html"; rel="alternate" type="text/html" title="Apache" /><published>2018-01-22T00:00:00+01:00</published><updated>2018-01-22T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2018/01/apache</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/01/apache.html";><p>Crail is now an Apache Incubator project!</p></content><author><name></name></author><category term="news" /><summary type="html">Crail is now an Apache Incubator project!</summary></entry><entry><title type="html">Iops</title><link href="http://crail.incubator.apache.org//blog/2017/11/iops.html"; rel="alternate" type="text/html" title="Iops" /><published>2017-11-23T00:00:00+01:00</published><updated>2017-11-23T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/iops</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/iops.html";><p>New blog <a href="http://crail.incubator.apache.org/blog/2017/11/crail-metadata.html">post</a> about Crailâs metadata performance and scalability</p></content><author><name></name></author><category term="news" /><summary type="html">New blog post about Crailâs metadata performance and scalability</summary></entry><entry><title type="html">Crail Storage Performance â Part III: Metadata</title><link href="http://crail.incubator.apache.org//blog/2017/11/crail-metadata.html"; rel="alternate" type="text/html" title="Crail Storage Performance -- Part III: Metadata" /><published>2017-11-21T00:00:00+01:00</published><updated>2017-11-21T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/crail-metadata</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/crail-metadata.html";><div style="text-align: justify"> -<p> -This is part III of our series of posts discussing Crail's raw storage performance. This part is about Crail's metadata performance and scalability. -</p> -</div> - -<h3 id="hardware-configuration">Hardware Configuration</h3> - -<p>The specific cluster configuration used for the experiments in this blog:</p> - -<ul> - <li>Cluster - <ul> - <li>8 node x86_64 cluster</li> - </ul> - </li> - <li>Node configuration - <ul> - <li>CPU: 2 x Intel(R) Xeon(R) CPU E5-2690 0 @ 2.90GHz</li> - <li>DRAM: 96GB DDR3</li> - <li>Network: 1x100Gbit/s Mellanox ConnectX-5</li> - </ul> - </li> - <li>Software - <ul> - <li>Ubuntu 16.04.3 LTS (Xenial Xerus) with Linux kernel version 4.10.0-33-generic</li> - <li>Crail 1.0, internal version 2993</li> - </ul> - </li> -</ul> - -<h3 id="crail-metadata-operation-overview">Crail Metadata Operation Overview</h3> - -<div style="text-align: justify"> -<p> -As described in <a href="http://crail.incubator.apache.org/blog/2017/08/crail-memory.html">part I</a>, Crail data operations are composed of actual data transfers and metadata operations. Examples of metadata operations are operations for creating or modifying the state of a file, or operations to lookup the storage server that stores a particular range (block) of a file. In Crail, all the metadata is managed by the namenode(s) (as opposed to the data which is managed by the storage nodes). Clients interact with Crail namenodes via Remote Procedure Calls (RPCs). Crail supports multiple RPC protocols for different types of networks and also offers a pluggable RPC interface so that new RPC bindings can be implemented easily. On RDMA networks, the default DaRPC (<a href="https://dl.acm.org/citation.cfm?id=2670994">DaRPC paper</a>, <a href="http://github.com/zrlio/darpc">DaRPC GitHub</a>) based RPC binding provides the be st performance. The figure below gives an overview of the Crail metadata processing in a DaRPC configuration. -</p> -</div> - -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/rpc.png" width="480" /></div> -<p><br /></p> - -<div style="text-align: justify"> -<p> -Crail supports partitioning of metadata across several namenods. Thereby, metadata operations issued by clients are hashed to a particular namenode depending on the name of object the operation attempts to create or retrieve. With the DaRPC binding, RPC messages are exchanged using RDMA send/recv operations. At the server, RPC processing is parallelized across different cores. To minimize locking and cache contention, each core handles a disjoint set of client connections. Connections assigned to the same core share the same RDMA completion queue which is processed exclusively by that given core. All the network queues, including send-, recv- and completion queues are mapped into user-space and accessed directly from within the JVM process. Since Crail offers a hierarchical storage namespace, metadata operations to create, delete or rename new storage resources effectively result in modifications to a tree-like data structure at the namenode. These structural operations require a so mewhat more expensive locking than the more lightweight operations used to lookup the file status or to extend a file with a new storage block. Consequently, Crail namenodes use two separate data structures to manage metadata: (a) a basic tree data structure that requires directory-based locking, and (b) a fast lock-free map to lookup of storage resources that are currently being read or written. -</p> -</div> - -<h3 id="experimental-setup">Experimental Setup</h3> - -<div style="text-align: justify"> -<p> -In two of the previous blogs (<a href="http://crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a>, <a href="http://crail.incubator.apache.org/blog/2017/08/crail-nvme-fabrics-v1.html">NVMf</a>) we have already shown that Crail metadata operations are very low latency. Essentially a single metadata operation issued by a remote client takes 5-6 microseconds, which is only slightly more than the raw network latency of the RDMA network fabric. In this blog, we want to explore the scalability of Crail's metadata management, that is, the number of clients Crail can support, or how Crail scales as the cluster size increases. The level of scalability of Crail is mainly determined by the number of metadata operations Crail can process concurrently, a metric that is often referred to as IOPS. The higher the number of IOPS the system can handle, the more clients can concurrently use Crail without performance loss. -</p> -<p> -An important metadata operation is ''getFile()'', which is used by clients to lookup the status of a file (whether the file exists, what size it has, etc.). The ''getFile()'' operation is served by Crail's fast lock-free map and in spirit is very similar to the ''getBlock()'' metadata operation (used by clients to query which storage nodes holds a particular block). In a typical Crail use case, ''getFile()'' and ''getBlock()'' are responsible for the peak metadata load at a namenode. In this experiment, we measure the achievable IOPS on the server side in an artificial configuration with many clients distributed across the cluster issuing ''getFile()'' in a tight loop. Note that the client side RPC interface in Crail is asynchronous, thus, clients can issue multiple metadata operations without blocking while asynchronously waiting for the result. In the experiments below, each client may have a maximum of 128 ''getFile()'' operations outstanding at any point in time. In a practical scenario, Crail clients may also have multiple metadata operations in flight either because clients are shared by different cores, or because Crail interleaves metadata and data operations (see <a href="http://crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a>). What makes the benchmark artificial is that clients exclusively focus on generating load for the namenode and thereby are neither performing data operations nor are they doing any compute. The basic command of the benchmark as executed by each of the individual clients is given by the following command: -</p> -</div> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -t getMultiFileAsync -f / -k 10000000 -b 128 -</code></pre></div></div> -<div style="text-align: justify"> -<p> -Where ''-t'' specifies the benchmark to run, ''-f'' specifies the path on the -Crail file system to be used for the benchmark, ''-k'' specifies the number of -iterations to be performed by the benchmark -(how many times will the benchmark execute ''getFile()'') and -''-b'' specifies the maximum number of requests in flight. -</p> -</div> - -<h3 id="single-namenode-scalability">Single Namenode Scalability</h3> - -<div style="text-align: justify"> -<p> -In the first experiment, we measure the aggregated number of metadata operations a single Crail namenode can handle per second. The namenode runs on 8 physical cores with hyper-threading disabled. The result is shown in the first graph below, labeled ''Namenode IOPS''. The namenode only gets saturated with more than 16 clients. The graph shows that the namenode can handle close to 10 million ''getFile()'' operations per second. With significantly more clients, the overall number of IOPS drops slightly, as more resources are being allocated on the single RDMA card, which basically creates a contention on hardware resources. -</p> -<p> -As comparison, we measure the raw number of IOPS, which can be executed on the RDMA network. We measure the raw number using ib_send_bw. We configured ib_send_bw with the same parameters in terms of RDMA configuration as the namenode. This means, we instructed ib_send_bw not to do CQ moderation, and to use a receive queue and a send queue of length 32, which equals the length of the namenode queues. Note that the default configuration of ib_send_bw uses CQ moderation and does preposting of send operations, which can only be done, if the operation is known in advance. This is not the case in a real system, like crail's namenode. The basic ib_send_bw command is given below: -</p> -</div> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>ib_send_bw -s 1 -Q 1 -r 32 -t 32 -n 10000000 -</code></pre></div></div> -<div style="text-align: justify"> -<p> -Where ''-s 1'' specifies to send packets with a payload of 1 (we don't want to -measure the transmission time of data, just the number of I/O operations), -''-Q 1'' specifies not to do CQ moderation, ''-r 32'' specifies the receive -queue length to be 32, ''-t 32'' specifies the send queue length to be 32 -and ''-n'' specifies the number of -iterations to be performed by ib_send_bw. -</p> -</div> -<div style="text-align: justify"> -<p> -The line of the raw number of IOPS, labeled ''ib send'' is shown in the same graph. With this measurement we show that Crail's namenode IOPS are similar to the raw ib_send_bw IOPS with the same configuration. -</p> -</div> -<p><br /></p> -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_ibsend_iops64.svg" width="550" /></div> -<p><br /></p> -<div style="text-align: justify"> -<p> -If one starts ib_send_bw without specifying the queue sizes or whether or not to use CQ moderation, the raw number of IOPS might be higher. This is due to the fact, that the default values of ib_send_bw use a receive queue of 512, a send queue of 128 and CQ moderation of 100, meaning that a new completion is generated only after 100 sends. As comparison, we did this -measurement too and show the result, labeled 'ib_send CQ mod', in the same graph. Fine tuning of receive and send queue sizes, CQ moderation size, postlists etc might lead to a higher number of IOPS. -</p> -</div> - -<h3 id="multiple-namenode-scalability">Multiple Namenode Scalability</h3> - -<div style="text-align: justify"> -<p> -To increase the number of IOPS the overall system can handle, we allow starting multiple namenode instances. Hot metadata operations, such as ''getFile()'', are distributed over all running instances of the namenode. ''getFile()'' is implemented such that no synchronization among the namenodes is required. As such, we expect good scalability. The graph below compares the overall IOPS of a system with one namenode to a system with two namenodes and four namenodes. -</p> -</div> -<p><br /></p> -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_multi64.svg" width="550" /></div> -<p><br /></p> - -<div style="text-align: justify"> -<p> -We show in this graph that the system can handle around 17Mio IOPS with two namenodes and 28Mio IOPS with four namenodes (with more than 64 clients we measured the number of IOPS to be slightly higher than 30Mio IOPS). Having multiple namenode instances matters especially with a higher number of clients. In the graph we see that the more clients we have the more we can benefit from a second namenode instance or even more instances. -</p> -</div> - -<div style="text-align: justify"> -<p> -We only have 7 physical nodes available to run the client processes. This -means, after 7 client processes, processes start sharing a physical machine. -With 64 client processes, each machine runs 9 (10 in one case) client -instances, which share the cores and the resources of the RDMA hardware. -We believe this is the reason, why the graphs appear not to scale linearly. -The number of total IOPS is client-bound, not namenode-bound. -With more physical machines, we believe that scalability could be shown -much better. Again, there is absolutely no communication among the -namenodes happening, which should lead to linear scalability. -</p> -</div> - -<h3 id="cluster-sizes">Cluster sizes</h3> - -<div style="text-align: justify"> -<p> -Let us look at a concrete application, which ideally runs on a large cluster: -TeraSort. In a previous blog, <a href="http://crail.incubator.apache.org/blog/2017/01/sorting.html">sorting</a>, -we analyze performance characteristics of TeraSort on Crail on a big cluster -of 128 nodes, where we run 384 executors in total. This already proves that -Crail can at least handle 384 clients. Now we analyze the theoretical number -of clients without performance loss at the namenode. Still this theoretical -number is not a hard limit on the number of clients. Just adding more -clients would start dropping the number of IOPS per client (not at the -namenode). -</p> -</div> - -<div style="text-align: justify"> -<p> -In contrast to the benchmarks above, a real-world application, like TeraSort, -does not issue RPC requests in a tight loop. It rather does sorting -(computation), file reading and writing and and of course a certain amount of -RPCs to manage the files. -</p> -</div> - -<div style="text-align: justify"> -<p> -We would like to know how many RPCs a run of TeraSort generates and therefore -how big the load in terms of number of IOPS is at the namenode for a -real-world application. -We run TeraSort on a data set of 200GB and measured the -number of IOPS at the namenode with 4 executors, 8 executors and 12 executors. -Every executor runs 12 cores. For this experiment, we use a single namenode -instance. We plot the distribution of the number of IOPS measured at the -namenode over the elapsed runtime of the TeraSort application. -</p> -</div> - -<p><br /></p> -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/terasort_iops.svg" width="550" /></div> -<p><br /></p> - -<div style="text-align: justify"> -<p> -From the graph we pick the peak number of IOPS measured -throughout the execution time for all three cases. The following table -shows the three peak IOPS numbers: -</p> -</div> - -<p><br /></p> -<center> -<table> - <thead> - <tr> - <th>#Executor nodes</th> - <th>Measured IOPS</th> - <th>% of single namenode</th> - </tr> - </thead> - <tbody> - <tr> - <td align="right">4</td> - <td align="right">32k</td> - <td align="right">0.32%</td> - </tr> - <tr> - <td align="right">8</td> - <td align="right">67k</td> - <td align="right">0.67%</td> - </tr> - <tr> - <td align="right">12</td> - <td align="right">107k</td> - <td align="right">1.07%</td> - </tr> - </tbody> -</table> -</center> -<p><br /></p> - -<div style="text-align: justify"> -<p> -From this table we see that it scales linearly. Even more important, -we notice that with 12 nodes we still use only around 1% of the -number of IOPS a single namenode can handle. -If we extrapolate this to a -100%, we can handle a cluster size of almost 1200 nodes (1121 clients being just -below 10Mio IOPS at the namenode). The -extrapolated numbers would look like this: -</p> -</div> - -<p><br /></p> -<center> -<table> - <thead> - <tr> - <th>#Namenodes</th> - <th>Max IOPS by namenodes</th> - <th>#Executor nodes</th> - <th>Extrapolated IOPS</th> - <th>% of all namenodes</th> - </tr> - </thead> - <tbody> - <tr> - <td align="right">1</td> - <td align="right">10000k</td> - <td align="right">1121</td> - <td align="right">9996k</td> - <td align="right">99.96%</td> - </tr> - <tr> - <td align="right">1</td> - <td align="right">10000k</td> - <td align="right">1200</td> - <td align="right">10730k</td> - <td align="right">107.3%</td> - </tr> - <tr> - <td align="right">2</td> - <td align="right">17000k</td> - <td align="right">1906</td> - <td align="right">16995k</td> - <td align="right">99.97%</td> - </tr> - <tr> - <td align="right">4</td> - <td align="right">30000k</td> - <td align="right">3364</td> - <td align="right">29995k</td> - <td align="right">99.98%</td> - </tr> -</tbody> -</table> -</center> -<p><br /></p> - -<div style="text-align: justify"> -<p> -Of course we know that there is no system with perfect linear scalability. -But even if we would loose 50% of the number of IOPS (compared to the -theoretical maximum) on a big cluster, Crail could still handle a cluster size -of 600 nodes and a single namenode without any performance loss at the -namenode. -Should we still want to run an application like TeraSort on a bigger cluster, -we can add a second namenode or have even more instances of namenodes -to ensure that clients do not suffer from contention in terms of IOPS at -the namenode. -</p> -</div> - -<div style="text-align: justify"> -<p> -We believe that the combination of benchmarks above, the scalability -experiments and the real-world -application of TeraSort shows clearly that Crail and Crail's namenode can handle -a big cluster of at least several hundreds of nodes, theoretically up to -1200 nodes with a single namenode and even more with multiple namenodes. -</p> -</div> - -<h3 id="system-comparison">System comparison</h3> -<div style="text-align: justify"> -<p> -In this section we compare the number of IOPS Crail can handle to -two other systems: -<a href="http://hadoop.apache.org/">Hadoop's HDFS namenode</a> and -<a href="https://ramcloud.atlassian.net/wiki/spaces/RAM/overview">RAMCloud</a>. -</p> -</div> - -<div style="text-align: justify"> -<p> -HDFS is a well known distributed file system. Like Crail, HDFS runs -a namenode and several datanodes. The namenode implements similar functionality -as Crail's namenode, while HDFS's datanodes provide additional functionality, -like replication, for example. We are interested in the -number of IOPS the namenode can handle. As such, the datanode's functionality -is not relevant for this experiment. HDFS is implemented in Java like Crail. -Due to this high similarity in terms of functionality and language used to -implement the system, HDFS is a good candidate to compare Crail to. -</p> -</div> - -<div style="text-align: justify"> -<p> -HDFS does not use RDMA to send RPCs. Instead, RPCs are sent over a regular -IP network. In our case, it is the same 100Gbit/s ethernet-based RoCE network. -</p> -</div> - -<div style="text-align: justify"> -<p> -To measure the number of IOPS HDFS's namenode can handle, we run the same -experiment as for Crail. The clients issue a ''getFile()'' RPC to the -namenode and we vary the number of clients from 1 to 64. The following -plot shows the number of IOPS relative to the number of clients. -</p> -</div> - -<p><br /></p> -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_hdfs_iops.svg" width="550" /></div> -<p><br /></p> - -<div style="text-align: justify"> -<p> -The graph shows that the namenode can handle around 200000 IOPS. One reason -for the difference to the number of IOPS of Crail is surely that HDFS does not -use the capabilities offered by the RDMA network, while Crail does. However -this cannot be the only reason, why the namenode cannot handle more than -200000 IOPS. We would need to analyze more deeply where the bottleneck is -to find an answer. We believe that the amount of code which -gets executed at probably various layers of the software stack -is too big to achieve high performance in terms of IOPS. -</p> -</div> - -<div style="text-align: justify"> -<p> -RAMCloud is a fast key-value store, which makes use of the RDMA network -to reach low latency and high throughput. It runs one master coordinator and -and optionally several slave coordinators, which can take over, if the master -coordinator fails. Coordinator persistence can be achieved -by external persistent storage, like Zookeeper or LogCabin. -RAMCloud runs several storage servers, which -store key-value pairs in RAM. Optionally, replicas can be stored on secondary -storage, which provides persistence. RAMCloud is implemented in C++. Therefore -it is natively compiled code. -</p> -</div> - -<div style="text-align: justify"> -<p> -We are interested in the number of IOPS RAMCloud can handle. We decided -to run the readThroughput benchmark of RAMCloud's ClusterPerf program, which -measures the number of object reads per second. This is probably the closest -benchmark to the RPC benchmark of Crail and HDFS. -</p> -</div> - -<div style="text-align: justify"> -<p> -For a fair comparison, we run RAMCloud without any persistence, so without -Zookeeper and without replicas to secondary storage. We run one coordinator -and one storage server, which is somewhat similar to running one namenode -in the Crail and HDFS cases. Also, we wanted to vary the number of clients -from 1 to 64. At the moment we can only get results for up to 16 clients. -We asked the RAMCloud developers for possible reasons and got to know that the -reason is a starvation bug in the benchmark (not in the RAMCloud system -itself). The RAMCloud developers are looking into this issue. We will update -the blog with the latest numbers as soon as the bug is fixed. -</p> -</div> - -<p><br /></p> -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/ramcloud_iops.svg" width="550" /></div> -<p><br /></p> - -<div style="text-align: justify"> -<p> -RAMCloud reaches a peak of 1.12Mio IOPS with 14 clients. The utilization of the -dispatcher thread is at 100% already with 10 clients. Even with more clients, -the number of IOPS won't get higher than 1.12Mio, because the -dispatcher thread is the bottleneck, as can be seen in the graph. -In addition, we got a confirmation from the developers that more than -10 clients will not increase the number of IOPS. -So we think that the measurements are not unfair, even if we do not have -results for more than 16 clients. Again, we we will update the blog -with a higher number of clients, as soon as the bug is fixed. -</p> -</div> - -<div style="text-align: justify"> -<p> -Let us now summarize the number of IOPS of all three systems in one plot -below. For a fair comparison, Crail runs only one namenode for this -experiments and we compare the results to RAMCloud with one coordinator and -one storage server (without replication as described above) and the one -namenode instance of HDFS. We see that Crail's single namenode can handle -a much bigger number of RPCs compared to the other two systems (remember -that Crail can run multiple namenodes and we measured a number of IOPS -of 30Mio/s with 4 namenodes). -</p> -</div> - -<p><br /></p> -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/max_iops_crail_hdfs_ramcloud.svg" width="550" /></div> -<p><br /></p> - -<div style="text-align: justify"> -<p> -HDFS is deployed on production clusters and handles real workloads -with roughly 200000 IOPS. We believe that Crail, which can handle a much -bigger number of IOPS, is able to run real workloads on very large -clusters. A common assumption is that Java-based implementations suffer from -performance loss. We show that a Java-based system can handle a high amount -of operations even compared to a C++-based system like RAMCloud. -</p> -</div> - -<h3 id="summary">Summary</h3> - -<div style="text-align: justify"> -<p> -In this blog we show three key points of Crail: First, Crail's namenode performs the same as ib_send_bw with realistic parameters in terms of IOPS. This shows that the actual processing of the RPC is implemented efficiently. Second, with only one namenode, Crail performs 10x to 50x better than RAMCloud and HDFS, two popular systems, where RAMCloud is RDMA-based and implemented natively. Third, Crail's metadata service can be scaled out to serve large number of clients. We have shown that Crail offers near linear scaling with up to 4 namenodes, offering a performance that is sufficient to serve several 1000s of clients. -</p> -</div></content><author><name>Adrian Schuepbach and Patrick Stuedi</name></author><category term="blog" /><summary type="html">This is part III of our series of posts discussing Crail's raw storage performance. This part is about Crail's metadata performance and scalability.</summary></entry><entry><title type="html">Floss</title><link href="http://crail.incubator.apache.org//blog/2017/11/floss.html"; rel="alternate" type="text/html" title="Floss" /><published>2017-11-17T00:00:00+01:00</published><updated>2017-11-17T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/floss</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/floss.html";><p>Crail features in the <a href="https://twit.tv/shows/floss-weekly/episodes/458?autostart=false">FLOSS weekly podcast</a></p></content><author><name></name></author><category term="news" /><summary type="html">Crail features in the FLOSS weekly podcast</sum mary></entry><entry><title type="html">Blog</title><link href="http://crail.incubator.apache.org//blog/2017/11/blog.html"; rel="alternate" type="text/html" title="Blog" /><published>2017-11-17T00:00:00+01:00</published><updated>2017-11-17T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/blog</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/blog.html";><p>New blog <a href="http://crail.incubator.apache.org/blog/2017/11/rdmashuffle.html">post</a> about SparkRDMA and Crail shuffle plugins</p></content><author><name></name></author><category term="news" /><summary type="html">New blog post about SparkRDMA and Crail shuffle plugins</summary></entry><entry><title type="html">Spark Shuffle: SparkRDMA vs Crail</title><link href="http://crail.incubator.apache.org//blog/2017/11/rdmashuffle.html"; rel="alternate" type="text/html" title="Spark Shuffle: SparkRDMA vs Crail" /><published>2017-11-17T00:00:00 +01:00</published><updated>2017-11-17T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/rdmashuffle</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/rdmashuffle.html";><div style="text-align: justify"> -<p> -This blog is comparing the shuffle performance of Crail with SparkRDMA, an alternative RDMA-based shuffle plugin for Spark. -</p> -</div> - -<h3 id="hardware-configuration">Hardware Configuration</h3> - -<p>The specific cluster configuration used for the experiments in this blog:</p> - -<ul> - <li>Cluster - <ul> - <li>8 compute + 1 management node x86_64 cluster</li> - </ul> - </li> - <li>Node configuration - <ul> - <li>CPU: 2 x Intel(R) Xeon(R) CPU E5-2690 0 @ 2.90GHz</li> - <li>DRAM: 96GB DDR3</li> - <li>Network: 1x100Gbit/s Mellanox ConnectX-5</li> - </ul> - </li> - <li>Software - <ul> - <li>Ubuntu 16.04.3 LTS (Xenial Xerus) with Linux kernel version 4.10.0-33-generic</li> - <li><a href="https://github.com/zrlio/crail">Crail 1.0</a>, commit a45c8382050f471e9342e1c6cf25f9f2001af6b5</li> - <li><a href="">Crail Shuffle plugin</a>, commit 2273b5dd53405cab3389f5c1fc2ee4cd30f02ae6</li> - <li><a href="https://github.com/Mellanox/SparkRDMA">SparkRDMA</a>, commit d95ce3e370a8e3b5146f4e0ab5e67a19c6f405a5 (latest master on 8th of November 2017)</li> - </ul> - </li> -</ul> - -<h3 id="overview">Overview</h3> -<div style="text-align: justify"> -<p> -Lately there has been an increasing interest in the community to include RDMA networking into data processing frameworks like Spark and Hadoop. One natural spot to integrate RDMA is in the shuffle operation that involves all-to-all network communication pattern. Naturally, due to its performance requirements the shuffle operation is of interest to us as well, and we have developed a Spark plugin for shuffle. In our previous blog posts, we have already shown that the Crail Shuffler achieves great workload-level speedups compared to vanilla Spark. In this blog post, we take a look at another recently proposed design called <a href="https://github.com/Mellanox/SparkRDMA">SparkRDMA</a> (<a href="https://issues.apache.org/jira/browse/SPARK-22229">SPARK-22229 JIRA</a>). SparkRDMA proposes to improve the shuffle performance of Spark by performing data transfers over RDMA. For this, the code manages its own off-heap memory which needs to be regist ered with the NIC for RDMA use. It supports two ways to store shuffle data between the stages: (1) shuffle data is stored in regular files (just like vanilla Spark) but the data transfer is implemented via RDMA, (2) data is stored in memory (allocated and registered for RDMA transfer) and the data transfer is implemented via RDMA. We call it the "last-mile" approach where just the networking operations are replaced by the RDMA operations. -</p> -<p> -In contrast, the Crail shuffler plugin takes a more holistic approach and leverages the high performance of Crail distributed data store to deliver gains. It uses Crail store to efficiently manage I/O resources, storage and networking devices, memory registrations, client sessions, data distribution, etc. Consequently, the shuffle operation becomes as simple as writing and reading files. And recall that Crail store is designed as a fast data bus for the intermediate data. The shuffle operation is just one of many operations that can be accelerated using Crail store. Beyond these operations, the modular architecture of Crail store enables us to seamlessly leverage different storage types (DRAM, NVMe, and more), perform tiering, support disaggregation, share inter-job data, jointly optimize I/O resources for various workloads, etc. These capabilities and performance gains give us confidence in the design choices we made for the Crail project. -</p> -</div> - -<h3 id="performance-comparison">Performance comparison</h3> -<div style="text-align: justify"> -<p>Lets start by quantitatively assessing performance gains from the Crail shuffle plugin and SparkRDMA. As described above, SparkRDMA can be operated in two different modes. Users decide which mode to use by selecting a particular type of shuffle writer (spark.shuffle.rdma.shuffleWriterMethod). The Wrapper shuffle writer writes shuffle data to files between the stages, the Chunked shuffle writer stores shuffle data in memory. We evaluate both writer methods for terasort and SQL equijoin. -</p> -</div> -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/rdma-shuffle/terasort.svg" width="550" /></div> -<p><br /></p> -<div style="text-align: justify"> -<p> -First we run <a href="https://github.com/zrlio/crail-spark-terasort">terasort</a> on our 8+1 machine cluster (see above). We sort 200GB, thus, each node gets 25GB of data (equal distribution). We further did a basic search of the parameter space for each of the systems to find the best possible configuration. In all the experiments we use 8 executors with 12 cores each. Note that in a typical Spark run more CPU cores than assigned are engaged because of garbabge collection, etc. In our test runs assigning 12 cores lead to the best performance. -</p> -<p> -The plot above shows runtimes of the various configuration we run with terasort. SparkRDMA with the Wrapper shuffle writer performance slightly better (3-4%) than vanilla Spark whereas the Chunked shuffle writer shows a 30% overhead. On a quick inspection we found that this overhead stems from memory allocation and registration for the shuffle data that is kept in memory between the stages. Compared to vanilla Spark, Crail's shuffle plugin shows performance improvement of around 235%. -</p> -</div> -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/rdma-shuffle/sql.svg" width="550" /></div> -<p><br /></p> - -<div style="text-align: justify"> -<p> -For our second workload we choose the <a href="https://github.com/zrlio/sql-benchmarks">SQL equijoin</a> with a <a href="https://github.com/zrlio/spark-nullio-fileformat">special fileformat</a> that allows data to be generated on the fly. By generating data on the fly we eliminate any costs for reading data from storage and focus entirely on the shuffle performance. The shuffle data size is around 148GB. Here the Wrapper shuffle writer is slightly slower than vanilla Spark but instead the Chunked shuffle writer is roughly the same amount faster. The Crail shuffle plugin again delivers a great performance increase over vanilla Spark. -</p> -</div> - -<div style="text-align: justify"> -<p>Please let us know if your have recommendations about how these experiments can be improved.</p> -</div> - -<h3 id="summary">Summary</h3> - -<div style="text-align: justify"> -<p> -These benchmarks validate our belief that a "last-mile" integration cannot deliver the same performance gains as a holistic approach, i.e. one has to look at the whole picture in how to integrate RDMA into Spark applications (and for that matter any framework or application). Only replacing the data transfer alone does not lead to the anticipated performance increase. We learned this the hard way when we intially started working on Crail. -</p> - -</div></content><author><name>Jonas Pfefferle, Patrick Stuedi, Animesh Trivedi, Bernard Metzler, Adrian Schuepbach</name></author><category term="blog" /><summary type="html">This blog is comparing the shuffle performance of Crail with SparkRDMA, an alternative RDMA-based shuffle plugin for Spark.</summary></entry><entry><title type="html">Crail Storage Performance â Part II: NVMf</title><link href="http://crail.incubator.apache.org//blog/2017/08/crail-nvme-fabrics-v1.html"; rel="alternate" type="text/html" title="Crail Storage Performance -- Part II: NVMf" /><published>2017-08-22T00:00:00+02:00</published><updated>2017-08-22T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2017/08/crail-nvme-fabrics-v1</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/08/crail-nvme-fabrics-v1.html";><div style="text-align: justify"> -<p> -This is part II of our series of posts discussing Crail's raw storage performance. This part is about Crail's NVMe storage tier, a low-latency flash storage backend for Crail completely based on user-level storage access. -</p> -</div> - -<h3 id="hardware-configuration">Hardware Configuration</h3> - -<p>The specific cluster configuration used for the experiments in this blog:</p> - -<ul> - <li>Cluster - <ul> - <li>8 node OpenPower cluster</li> - </ul> - </li> - <li>Node configuration - <ul> - <li>CPU: 2x OpenPOWER Power8 10-core @2.9Ghz</li> - <li>DRAM: 512GB DDR4</li> - <li>4x 512 GB Samsung 960Pro NVMe SSDs (512Byte sector size, no metadata)</li> - <li>Network: 1x100Gbit/s Mellanox ConnectX-4 IB</li> - </ul> - </li> - <li>Software - <ul> - <li>RedHat 7.3 with Linux kernel version 3.10</li> - <li>Crail 1.0, internal version 2843</li> - <li>SPDK git commit 5109f56ea5e85b99207556c4ff1d48aa638e7ceb with patches for POWER support</li> - <li>DPDK git commit bb7927fd2179d7482de58d87352ecc50c69da427</li> - </ul> - </li> -</ul> - -<h3 id="the-crail-nvmf-storage-tier">The Crail NVMf Storage Tier</h3> - -<div style="text-align: justify"> -<p> -Crail is a framework that allows arbitrary storage backends to be added by implementing the Crail storage interface. A storage backend manages the point-to-point data transfers on a per block granularity between a Crail client and a set of storage servers. The Crail storage interface essentially consists of three virtual functions, which simplified look like this: -</p> -</div> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>//Server-side interface: donate storage resources to Crail -StorageResource allocateResource(); -//Client-side interface: read/write remote/local storage resources -writeBlock(BlockInfo, ByteBuffer); -readBlock(BlockInfo, ByteBuffer); -</code></pre></div></div> -<div style="text-align: justify"> -<p> -A specific implementation of this interface provides an efficient mapping of Crail storage operations to the actual storage and network hardware the backend is exporting. Crail comes with two native storage backends, an RDMA-based DRAM backend and an RDMA-based NVMe backend, but other storage backends are available as well (e.g., Netty) and we plan to provide more custom backends in the future as new storage and network technologies are emerging. -</p> -<p> -The Crail NVMf storage backend we evaluate in this blog provides user-level access to local and remote flash through the NVMe over Fabrics protocol. Crail NVMf is implemented using <a href="https://github.com/zrlio/disni">DiSNI</a>, a user-level network and storage interface for Java offering both RDMA and NVMf APIs. DiSNI itself is based on <a href="http://www.spdk.io">SPDK</a> for its NVMf APIs. -</p> -<p> -The server side of the NVMf backend is designed in a way that each server process manages exactly one NVMe drive. On hosts with multiple NVMe drives one may start several Crail NVMf servers. A server is setting up an NVMf target through DiSNI and implements the allocateResource() storage interface by allocating storage regions from the NVMe drive (basically splits up the NVMe namespace into smaller segments). The Crail storage runtime makes information about storage regions available to the Crail namenode, from where regions are further broken down into smaller units called blocks that make up files in Crail. -</p> -<p> -The Crail client runtime invokes the NVMf client interface during file read/write operations for all data transfers on NVMf blocks. Using the block information provided by the namenode, the NVMf storage client implementation is able to connect to the appropriate NVMf target and perform the data operations using DiSNI's NVMf API. -</p> -<p> -One downside of the NVMe interface is that byte level access is prohibited. Instead data operations have to be issued for entire drive sectors which are typically 512Byte or 4KB large (we used 512Byte sector size in all the experiments shown in this blog). As we wanted to use the standard NVMf protocol (and Crail has a client driven philosophy) we needed to implement byte level access at the client side. For reads this can be achieved in a straight forward way by reading the whole sector and copying out the requested part. For writes that modify a certain subrange of a sector that has already been written before we need to do a read modify write operation. -</p> -</div> - -<h3 id="performance-comparison-to-native-spdk-nvmf">Performance comparison to native SPDK NVMf</h3> - -<div style="text-align: justify"> -<p> -We perform latency and throughput measurement of our Crail NVMf storage tier against a native SPDK NVMf benchmark to determine how much overhead our implementation adds. The first plot shows random read latency on a single 512GB Samsung 960Pro accessed remotely through SPDK. For Crail we also show the time it takes to perform a metadata operations. You can run the Crail benchmark from the command line like this: -</p> -</div> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -t readRandom -b false -s &lt;size&gt; -k &lt;iterations&gt; -w 32 -f /tmp.dat -</code></pre></div></div> -<p>and SPDK:</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./perf -q 1 -s &lt;size&gt; -w randread -r 'trtype:RDMA adrfam:IPv4 traddr:&lt;ip&gt; trsvcid:&lt;port&gt;' -t &lt;time in seconds&gt; -</code></pre></div></div> -<div style="text-align: justify"> -<p> -The main take away from this plot is that the time it takes to perform a random read operation on a NVMe-backed file in Crail takes only about 7 microseconds more time than fetching the same amount of data over a point-to-point SPDK connection. This is impressive because it shows that using Crail a bunch of NVMe drives can be turned into a fully distributed storage space at almost no extra cost. The 7 microseconds are due to Crail having to look up the specific NVMe storage node that holdes the data -- an operation which requires one extra network roundtrip (client to namenode). The experiment represents an extreme case where no metadata is cached at the client. In practice, file blocks are often accessed multiple times in which case the read latency is further reduced. Also note that unlike SPDK which is a native library, Crail delivers data directly into Java off-heap memory. -</p> -</div> - -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/latency.svg" width="550" /></div> -<p><br /></p> - -<div style="text-align: justify"> -<p> -The second plot shows sequential read and write throughput with a transfer size of 64KB and 128 outstanding operations. The Crail throughput benchmark can be run like this: -</p> -</div> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -t readAsync -s 65536 -k &lt;iterations&gt; -b 128 -w 32 -f /tmp.dat -</code></pre></div></div> -<p>and SPDK:</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./perf -q 128 -s 65536 -w read -r 'trtype:RDMA adrfam:IPv4 traddr:&lt;ip&gt; trsvcid:&lt;port&gt;' -t &lt;time in seconds&gt; -</code></pre></div></div> -<div style="text-align: justify"> -<p> -For sequential operations in Crail, metadata fetching is inlined with data operations as described in the <a href="http://crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a> blog. This is possible as long as the data transfer has a lower latency than the metadata RPC, which is typically the case. As a consequence, our NVMf storage tier reaches the same throughput as the native SPDK benchmark (device limit). -</p> -</div> -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/throughput.svg" width="550" /></div> - -<h3 id="sequential-throughput">Sequential Throughput</h3> - -<div style="text-align: justify"> -<p> -Let us look at the sequential read and write throughput for buffered and direct streams and compare them to a buffered Crail stream on DRAM. All benchmarks are single thread/client performed against 8 storage nodes with 4 drives each, cf. configuration above. In this benchmark we use 32 outstanding operations for the NVMf storage tier buffered stream experiments by using a buffer size of 16MB and a slice size of 512KB, cf. <a href="http://crail.incubator.apache.org/blog/2017/07/crail-memory.html">part I</a>. The buffered stream reaches line speed at a transfer size of around 1KB and shows only slightly slower performance when compared to the DRAM tier buffered stream. However we are only using 2 outstanding operations with the DRAM tier to achieve these results. Basically for sizes smaller than 1KB the buffered stream is limited by the copy speed to fill the application buffer. The direct stream reaches line speed at around 128KB with 128 outstanding operations . Here no copy operation is performed for transfer size greater than 512Byte (sector size). The command to run the Crail buffered stream benchmark: -</p> -</div> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -t read -s &lt;size&gt; -k &lt;iterations&gt; -w 32 -f /tmp.dat -</code></pre></div></div> -<p>The direct stream benchmark:</p> -<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -t readAsync -s &lt;size&gt; -k &lt;iterations&gt; -b 128 -w 32 -f /tmp.dat -</code></pre></div></div> - -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/throughput2.svg" width="550" /></div> - -<h3 id="random-read-latency">Random Read Latency</h3> - -<div style="text-align: justify"> -<p> -Random read latency is limited by the flash technology and we currently see around 70us when performing sector size accesses to the device with the Crail NVMf backend. In comparison, remote DRAM latencies with Crail are about 7-8x faster. However, we believe that this will change in the near future with new technologies like PCM. Intel's Optane drives already can deliver random read latencies of around 10us. Considering that there is an overhead of around 10us to access a drive with Crail from anywhere in the cluster, using such a device would put random read latencies somewhere around 20us which is only half the performance of our DRAM tier. -</p> -</div> - -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/latency2.svg" width="550" /></div> - -<h3 id="tiering-dram---nvmf">Tiering DRAM - NVMf</h3> - -<div style="text-align: justify"> -<p> -In this paragraph we show how Crail can leverage flash memory when there is not sufficient DRAM available in the cluster to hold all the data. As described in the <a href="http://crail.incubator.apache.org/overview/">overview</a> section, if you have multiple storage tiers deployed in Crail, e.g. the DRAM tier and the NVMf tier, Crail by default first uses up all available resources of the faster tier. Basically a remote resource of a faster tier (e.g. remote DRAM) is preferred over a slower local resource (e.g., local flash), motivated by the fast network. This is what we call horizontal tiering. -</p> -</div> -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/crail_tiering.png" width="500" vspace="10" /></div> -<p><br /></p> -<div style="text-align: justify"> -<p> -In the following 200G Terasort experiment we gradually limit the DRAM resources in Crail while adding more flash to the Crail NVMf storage tier. Note that here Crail is used for both input/output as well as shuffle data. The figure shows that by putting all the data in flash we only increase the sorting time by around 48% compared to the configuration where all the data resides in DRAM. Considering the cost of DRAM and the advances in technology described above we believe cheaper NVM storage can replace DRAM for most of the applications with only a minor performance decrease. Also, note that even with 100% of the data in NVMe, Spark/Crail is still faster than vanilla Spark with all the data in memory. The vanilla Spark experiment uses Alluxio for input/output and RamFS for the shuffle data. -</p> -</div> - -<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/tiering.svg" width="550" /></div> - -<p>To summarize, in this blog we have shown that the NVMf storage backend for Crail â due to its efficient user-level implementation â offers latencies and throughput very close to the hardware speed. The Crail NVMf storage tier can be used conveniently in combination with the Crail DRAM tier to either save cost or to handle situations where the available DRAM is not sufficient to store the working set of a data processing workload.</p></content><author><name>Jonas Pfefferle</name></author><category term="blog" /><summary type="html">This is part II of our series of posts discussing Crail's raw storage performance. This part is about Crail's NVMe storage tier, a low-latency flash storage backend for Crail completely based on user-level storage access.</summary></entry></feed> \ No newline at end of file +<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom"; ><generator uri="https://jekyllrb.com/"; version="3.8.3">Jekyll</generator><link href="http://localhost:4000/feed.xml"; rel="self" type="application/atom+xml" /><link href="http://localhost:4000/"; rel="alternate" type="text/html" /><updated>2018-06-07T13:27:00+02:00</updated><id>http://localhost:4000/</id><title type="html">The Apache Crail (Incubating) Project</title></feed> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/c782b73a/content/img/apache_incubator.png ---------------------------------------------------------------------- diff --git a/content/img/apache_incubator.png b/content/img/apache_incubator.png deleted file mode 100644 index 987c79e..0000000 Binary files a/content/img/apache_incubator.png and /dev/null differ
![http://crail.incubator.apache.org/img/blog/preview/documentation-summary.png"](http://crail.incubator.apache.org/img/blog/preview/documentation-summary.png"){kind=link}
![http://crail.incubator.apache.org/img/crail_logo.png"](http://crail.incubator.apache.org/img/crail_logo.png"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-metadata/rpc.png"](http://crail.incubator.apache.org/img/blog/crail-metadata/rpc.png"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_ibsend_iops64.svg"](http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_ibsend_iops64.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_multi64.svg"](http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_multi64.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-metadata/terasort_iops.svg"](http://crail.incubator.apache.org/img/blog/crail-metadata/terasort_iops.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_hdfs_iops.svg"](http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_hdfs_iops.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-metadata/ramcloud_iops.svg"](http://crail.incubator.apache.org/img/blog/crail-metadata/ramcloud_iops.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-metadata/max_iops_crail_hdfs_ramcloud.svg"](http://crail.incubator.apache.org/img/blog/crail-metadata/max_iops_crail_hdfs_ramcloud.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/rdma-shuffle/terasort.svg"](http://crail.incubator.apache.org/img/blog/rdma-shuffle/terasort.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/rdma-shuffle/sql.svg"](http://crail.incubator.apache.org/img/blog/rdma-shuffle/sql.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-nvmf/latency.svg"](http://crail.incubator.apache.org/img/blog/crail-nvmf/latency.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-nvmf/throughput.svg"](http://crail.incubator.apache.org/img/blog/crail-nvmf/throughput.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-nvmf/throughput2.svg"](http://crail.incubator.apache.org/img/blog/crail-nvmf/throughput2.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-nvmf/latency2.svg"](http://crail.incubator.apache.org/img/blog/crail-nvmf/latency2.svg"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-nvmf/crail_tiering.png"](http://crail.incubator.apache.org/img/blog/crail-nvmf/crail_tiering.png"){kind=link}
![http://crail.incubator.apache.org/img/blog/crail-nvmf/tiering.svg"](http://crail.incubator.apache.org/img/blog/crail-nvmf/tiering.svg"){kind=link}
