This is an automated email from the ASF dual-hosted git repository.
pdallig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new d5eee9f160 [ZEPPELIN-5903] Improve LRU NoteCache - Add a cleanup
(#4590)
d5eee9f160 is described below
commit d5eee9f160fbb2030574f2cfcdf651a3918abb79
Author: Philipp Dallig <philipp.dal...@gmail.com>
AuthorDate: Tue May 2 08:39:06 2023 +0200
[ZEPPELIN-5903] Improve LRU NoteCache - Add a cleanup (#4590)
* Improve LRU NoteCache - Add a cleanup
* Remove debug code
Co-authored-by: Guanhua Li <guanhua...@foxmail.com>
---------
Co-authored-by: Guanhua Li <guanhua...@foxmail.com>
---
.../org/apache/zeppelin/notebook/NoteManager.java | 26 +++++++++++++++++++---
.../apache/zeppelin/notebook/NoteManagerTest.java | 12 ++++++++--
2 files changed, 33 insertions(+), 5 deletions(-)
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java
index e2d393af54..69c83030d6 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java
@@ -21,7 +21,7 @@ package org.apache.zeppelin.notebook;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -736,7 +736,6 @@ public class NoteManager {
if (size() <= NoteCache.this.threshold) {
return false;
}
-
final Note eldestNote = eldest.getValue();
final Lock lock = eldestNote.getLock().writeLock();
if (lock.tryLock()) { // avoid eviction in case the note is in use
@@ -748,11 +747,32 @@ public class NoteManager {
} else {
LOGGER.info("Can not evict note {}, because the write lock can not
be acquired. {} notes currently loaded.",
eldestNote.getId(), size());
+ cleanupCache();
return false;
}
}
- }
+ private void cleanupCache() {
+ Iterator<Map.Entry<String, Note>> iterator =
this.entrySet().iterator();
+ int count = 0;
+ // if size >= shrinked_size and have next() try remove
+ while ((this.size() - 1) >= NoteCache.this.threshold &&
iterator.hasNext()) {
+ Map.Entry<String, Note> noteEntry = iterator.next();
+ final Note note = noteEntry.getValue();
+ final Lock lock = note.getLock().writeLock();
+ if (lock.tryLock()) { // avoid eviction in case the note is in use
+ try {
+ iterator.remove(); // remove LRU element from LinkedHashMap
+ LOGGER.debug("Remove note {} from LRU Cache", note.getId());
+ ++count;
+ } finally {
+ lock.unlock();
+ }
+ }
+ }
+ LOGGER.info("The cache cleanup removes {} entries", count);
+ }
+ }
}
diff --git
a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteManagerTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteManagerTest.java
index f8d58dd272..f825a97b37 100644
---
a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteManagerTest.java
+++
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteManagerTest.java
@@ -45,7 +45,6 @@ public class NoteManagerTest {
@Rule
public ExpectedException thrown = ExpectedException.none();
-
@Before
public void setUp() throws IOException {
conf = ZeppelinConfiguration.create();
@@ -151,8 +150,9 @@ public class NoteManagerTest {
}
assertEquals(cacheThreshold, noteManager.getCacheSize());
- // add cache + 1
+ // add cache + 1 with read flag
Note noteNew2 = createNote("/prod/notenew2");
+ noteNew2.getLock().readLock().lock();
noteManager.addNote(noteNew2, AuthenticationInfo.ANONYMOUS);
// since all notes in the cache are with a read lock, the cache grows
@@ -162,6 +162,14 @@ public class NoteManagerTest {
noteManager.removeNote(noteNew2.getId(), AuthenticationInfo.ANONYMOUS);
assertFalse(noteManager.containsNote(noteNew2.getPath()));
assertEquals(cacheThreshold, noteManager.getCacheSize());
+
+ // add cache + 1 without read flag
+ Note noteNew3 = createNote("/prod/notenew3");
+ noteManager.addNote(noteNew3, AuthenticationInfo.ANONYMOUS);
+
+ // since all dirty notes in the cache are with a read flag, the cache
removes noteNew3, because it has no read flag
+ assertEquals(cacheThreshold, noteManager.getCacheSize());
+ assertTrue(noteManager.containsNote(noteNew3.getPath()));
}
@Test
