Author: tommaso
Date: Sun Jan 8 17:16:00 2012
New Revision: 1228898
URL: http://svn.apache.org/viewvc?rev=1228898&view=rev
Log:
[DIRECTMEMORY-59] - fixed statistics issue and added unit testing for it
Added:
incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
Modified:
incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
Modified:
incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
URL:
http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java?rev=1228898&r1=1228897&r2=1228898&view=diff
==============================================================================
---
incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
(original)
+++
incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
Sun Jan 8 17:16:00 2012
@@ -1,5 +1,3 @@
-package org.apache.directmemory.examples.solr;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -18,8 +16,11 @@ package org.apache.directmemory.examples
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.directmemory.examples.solr;
import org.apache.directmemory.cache.Cache;
+import org.apache.directmemory.cache.CacheService;
+import org.apache.directmemory.measures.Monitor;
import org.apache.directmemory.measures.Ram;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
@@ -37,11 +38,9 @@ import java.util.concurrent.atomic.Atomi
* A {@link SolrCache} based on Apache DirectMemory
*/
public class SolrOffHeapCache<K, V>
- implements SolrCache<K, V>
-{
+ implements SolrCache<K, V> {
- private static class CumulativeStats
- {
+ private static class CumulativeStats {
AtomicLong lookups = new AtomicLong();
AtomicLong hits = new AtomicLong();
@@ -74,32 +73,32 @@ public class SolrOffHeapCache<K, V>
private String description = "DM Cache";
@Override
- public Object init( Map args, Object persistence, CacheRegenerator
regenerator )
- {
- Object buffers = args.get( "buffers" );
- String size = String.valueOf( args.get( "size" ) );
- Cache.init( buffers != null ? Integer.valueOf( String.valueOf( buffers
) ) : 1,
- Ram.Mb( Double.valueOf( size ) / 512 ) );
+ public Object init(Map args, Object persistence, CacheRegenerator
regenerator) {
+ Object buffers = args.get("buffers");
+ String size = String.valueOf(args.get("size"));
+ Integer capacity =
Integer.parseInt(String.valueOf(args.get("initialSize")));
+ Cache.init(buffers != null ? Integer.valueOf(String.valueOf(buffers))
: 1,
+ Ram.Mb(Double.valueOf(size) / 512),
+ Ram.Mb(Double.valueOf(capacity) / 512),
+ CacheService.DEFAULT_CONCURRENCY_LEVEL);
state = State.CREATED;
this.regenerator = regenerator;
- name = (String) args.get( "name" );
+ name = (String) args.get("name");
String str = size;
- final int limit = str == null ? 1024 : Integer.parseInt( str );
- str = (String) args.get( "initialSize" );
- final int initialSize = Math.min( str == null ? 1024 :
Integer.parseInt( str ), limit );
- str = (String) args.get( "autowarmCount" );
- autowarmCount = str == null ? 0 : Integer.parseInt( str );
+ final int limit = str == null ? 1024 : Integer.parseInt(str);
+ str = (String) args.get("initialSize");
+ final int initialSize = Math.min(str == null ? 1024 :
Integer.parseInt(str), limit);
+ str = (String) args.get("autowarmCount");
+ autowarmCount = str == null ? 0 : Integer.parseInt(str);
description = "Solr OffHeap Cache(maxSize=" + limit + ", initialSize="
+ initialSize;
- if ( autowarmCount > 0 )
- {
+ if (autowarmCount > 0) {
description += ", autowarmCount=" + autowarmCount + ",
regenerator=" + regenerator;
}
description += ')';
- if ( persistence == null )
- {
+ if (persistence == null) {
// must be the first time a cache of this type is being created
persistence = new CumulativeStats();
}
@@ -110,138 +109,140 @@ public class SolrOffHeapCache<K, V>
}
@Override
- public String name()
- {
+ public String name() {
return name;
}
@Override
- public int size()
- {
- return Long.valueOf( Cache.entries() ).intValue();
+ public int size() {
+ return Long.valueOf(Cache.entries()).intValue();
}
@Override
- public V put( K key, V value )
- {
- return (V) Cache.put( String.valueOf( key ), value );
+ public V put(K key, V value) {
+ synchronized (this) {
+ if (state == State.LIVE) {
+ stats.inserts.incrementAndGet();
+ }
+
+ inserts++;
+ return (V) Cache.put(String.valueOf(key), value);
+ }
}
@Override
- public V get( K key )
- {
- return (V) Cache.retrieve( String.valueOf( key ) );
+ public V get(K key) {
+ synchronized (this) {
+ V val = (V) Cache.retrieve(String.valueOf(key));
+ if (state == State.LIVE) {
+ // only increment lookups and hits if we are live.
+ lookups++;
+ stats.lookups.incrementAndGet();
+ if (val != null) {
+ hits++;
+ stats.hits.incrementAndGet();
+ }
+ }
+ return val;
+ }
}
@Override
- public void clear()
- {
- synchronized ( this )
- {
+ public void clear() {
+ synchronized (this) {
Cache.clear();
}
}
@Override
- public void setState( State state )
- {
+ public void setState(State state) {
this.state = state;
}
@Override
- public State getState()
- {
+ public State getState() {
return state;
}
@Override
- public void warm( SolrIndexSearcher searcher, SolrCache<K, V> old )
- throws IOException
- {
+ public void warm(SolrIndexSearcher searcher, SolrCache<K, V> old)
+ throws IOException {
// it looks like there is no point in warming an off heap item
}
@Override
- public void close()
- {
+ public void close() {
+ Cache.dump();
+ Monitor.dump();
}
@Override
- public String getName()
- {
+ public String getName() {
return SolrOffHeapCache.class.getName();
}
@Override
- public String getVersion()
- {
+ public String getVersion() {
return SolrCore.version;
}
- public String getDescription()
- {
+ public String getDescription() {
return description;
}
- public Category getCategory()
- {
+ public Category getCategory() {
return Category.CACHE;
}
@Override
- public String getSourceId()
- {
+ public String getSourceId() {
return null;
}
@Override
- public String getSource()
- {
+ public String getSource() {
return null;
}
@Override
- public URL[] getDocs()
- {
+ public URL[] getDocs() {
return new URL[0];
}
- public NamedList getStatistics()
- {
+ public NamedList getStatistics() {
NamedList lst = new SimpleOrderedMap();
- synchronized ( this )
- {
- lst.add( "lookups", lookups );
- lst.add( "hits", hits );
- lst.add( "hitratio", calcHitRatio( lookups, hits ) );
- lst.add( "inserts", inserts );
- lst.add( "evictions", evictions );
- lst.add( "size", Cache.entries() );
+ synchronized (this) {
+ lst.add("lookups", lookups);
+ lst.add("hits", hits);
+ lst.add("hitratio", calcHitRatio(lookups, hits));
+ lst.add("inserts", inserts);
+ lst.add("evictions", evictions);
+ lst.add("size", Cache.entries());
}
- lst.add( "warmupTime", warmupTime );
+ lst.add("warmupTime", warmupTime);
long clookups = stats.lookups.get();
long chits = stats.hits.get();
- lst.add( "cumulative_lookups", clookups );
- lst.add( "cumulative_hits", chits );
- lst.add( "cumulative_hitratio", calcHitRatio( clookups, chits ) );
- lst.add( "cumulative_inserts", stats.inserts.get() );
- lst.add( "cumulative_evictions", stats.evictions.get() );
+ lst.add("cumulative_lookups", clookups);
+ lst.add("cumulative_hits", chits);
+ lst.add("cumulative_hitratio", calcHitRatio(clookups, chits));
+ lst.add("cumulative_inserts", stats.inserts.get());
+ lst.add("cumulative_evictions", stats.evictions.get());
return lst;
}
- private Object calcHitRatio( long clookups, long chits )
- {
- int ones = (int) ( hits * 100 / lookups );
- int tenths = (int) ( hits * 1000 / lookups ) - ones * 10;
- return Integer.toString( ones ) + '.' + tenths;
+ private Object calcHitRatio(long clookups, long chits) {
+ if (lookups == 0) return "0.00";
+ if (lookups == hits) return "1.00";
+ int hundredths = (int) (hits * 100 / lookups); // rounded down
+ if (hundredths < 10) return "0.0" + hundredths;
+ return "0." + hundredths;
}
@Override
- public String toString()
- {
+ public String toString() {
return name + getStatistics().toString();
}
}
Added:
incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
URL:
http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java?rev=1228898&view=auto
==============================================================================
---
incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
(added)
+++
incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
Sun Jan 8 17:16:00 2012
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.directmemory.examples.solr;
+
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.search.QueryResultKey;
+import org.apache.solr.search.function.DocValues;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+/**
+ * Testcase for {@link SolrOffHeapCache}
+ */
+public class SolrOffHeapCacheTest {
+
+ private SolrOffHeapCache solrOffHeapCache;
+
+ @Before
+ public void setUp() {
+ solrOffHeapCache = new SolrOffHeapCache();
+ Map<String, String> args = new HashMap<String, String>();
+ args.put("size", "10000");
+ args.put("initialSize", "1000");
+ solrOffHeapCache.init(args, null, null);
+ }
+
+ @After
+ public void tearDown() {
+ solrOffHeapCache.clear();
+ solrOffHeapCache.close();
+ }
+
+ @Test
+ public void testStatisticsWhenCacheNotUsedYet() {
+ try {
+ NamedList stats = solrOffHeapCache.getStatistics();
+ assertNotNull(stats);
+ assertEquals(0l, stats.get("lookups"));
+ assertEquals(0l, stats.get("evictions"));
+ assertEquals(0l, stats.get("hits"));
+ assertEquals(0l, stats.get("inserts"));
+ } catch (Exception e) {
+ fail(e.getLocalizedMessage());
+ }
+ }
+
+ @Test
+ public void testPut() {
+ try {
+ QueryResultKey queryResultKey = new QueryResultKey(new
MatchAllDocsQuery(), new ArrayList<Query>(), new Sort(), 1);
+ DocValues docValues = new DocValues() {
+ @Override
+ public String toString(int doc) {
+ return doc + "asd";
+ }
+ };
+ solrOffHeapCache.put(queryResultKey, docValues);
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getLocalizedMessage());
+ }
+ }
+}
Modified:
incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
URL:
http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java?rev=1228898&r1=1228897&r2=1228898&view=diff
==============================================================================
---
incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
(original)
+++
incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
Sun Jan 8 17:16:00 2012
@@ -19,7 +19,8 @@ package org.apache.directmemory.examples
import org.apache.directmemory.cache.Cache;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.util.TestHarness;
-import org.junit.Before;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
@@ -27,41 +28,39 @@ import static org.junit.Assert.fail;
/**
*/
-public class SolrOffHeapIntegrationTest
-{
- private TestHarness h;
-
- @Before
- public void setUp()
- {
+public class SolrOffHeapIntegrationTest {
+ private static TestHarness h;
+
+ @BeforeClass
+ public static void setUp() {
String data = "target/data/expand";
- String config = this.getClass().getResource(
"/solr/config/solrconfig.xml" ).getFile();
- String schema = this.getClass().getResource( "/solr/config/schema.xml"
).getFile();
- this.h = new TestHarness( data, config, schema );
+ String config =
SolrOffHeapCache.class.getResource("/solr/config/solrconfig.xml").getFile();
+ String schema =
SolrOffHeapCache.class.getResource("/solr/config/schema.xml").getFile();
+ h = new TestHarness(data, config, schema);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ Cache.clear();
}
@Test
- public void testSimpleQuery()
- {
- try
- {
+ public void testSimpleQuery() {
+ try {
// add a doc to Solr
- h.validateAddDoc( "id", "1", "text", "something is happening here"
);
+ h.validateAddDoc("id", "1", "text", "something is happening here");
// make the query
- LocalSolrQueryRequest request = h.getRequestFactory( "standard",
0, 100 ).makeRequest( "q", "something" );
- String response = h.query( "standard", request );
- assertTrue( response != null );
- assertTrue( !response.contains( "error" ) );
+ LocalSolrQueryRequest request = h.getRequestFactory("standard", 0,
10).makeRequest("q", "something");
+ String response = h.query("standard", request);
+ assertTrue(response != null);
+ assertTrue(!response.contains("error"));
// check the cache has been hit
- assertTrue( Cache.entries() > 0 );
+ assertTrue(Cache.entries() > 0);
- }
- catch ( Throwable e )
- {
- e.printStackTrace();
- fail( e.getLocalizedMessage() );
+ } catch (Throwable e) {
+ fail(e.getLocalizedMessage());
}
}
}
Modified:
incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
URL:
http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml?rev=1228898&r1=1228897&r2=1228898&view=diff
==============================================================================
---
incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
(original)
+++
incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
Sun Jan 8 17:16:00 2012
@@ -66,7 +66,7 @@
cause a serious error to be logged if it can't be loaded. <lib
path="../a-jar-that-does-not-exist.jar" />
-->
-
+ <luceneMatchVersion>LUCENE_35</luceneMatchVersion>
<!--
Used to specify an alternate directory to hold all index data other
@@ -360,8 +360,8 @@
for each document). Since Lucene internal document ids are
transient, this cache will not be autowarmed.
-->
- <queryResultCache
class="org.apache.directmemory.examples.solr.SolrOffHeapCache" size="512"
- initialSize="512" autowarmCount="0" buffers="10"/>
+ <queryResultCache
class="org.apache.directmemory.examples.solr.SolrOffHeapCache" size="10000"
+ initialSize="1000" autowarmCount="0" buffers="2"/>
<!--
If true, stored fields that are not requested will be loaded
