I'm not convinced about this. Reaching 8MB less than the max memory is
normal - for example if you use the -server VM, it will always happen, and
soon after startup, because the VM doesn't GC until it reaches the limit.
IMHO we should only execute the low memory hooks if we either get an OOM, or
if we have less than 8MB left *after a garbage collection*. Note that there
are command line options to make explicit garbage collection a no-op, so the
latter isn't easy; and explicitly GCing on a regular schedule is generally
regarded as a *bad* thing.
On Friday 02 May 2008 11:10, j16sdiz at freenetproject.org wrote:
> Author: j16sdiz
> Date: 2008-05-02 10:10:12 +0000 (Fri, 02 May 2008)
> New Revision: 19675
>
> Modified:
> trunk/freenet/src/freenet/crypt/SHA256.java
> trunk/freenet/src/freenet/node/MemoryChecker.java
> trunk/freenet/src/freenet/node/Node.java
> trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
> trunk/freenet/src/freenet/support/OOMHandler.java
> trunk/freenet/src/freenet/support/OOMHook.java
> Log:
> alternative low memory / out of memory handling
>
>
> Modified: trunk/freenet/src/freenet/crypt/SHA256.java
> ===================================================================
> --- trunk/freenet/src/freenet/crypt/SHA256.java 2008-05-02 10:09:13 UTC
> (rev
19674)
> +++ trunk/freenet/src/freenet/crypt/SHA256.java 2008-05-02 10:10:12 UTC
> (rev
19675)
> @@ -44,6 +44,8 @@
> import freenet.node.Node;
> import freenet.node.NodeInitException;
> import freenet.support.Logger;
> +import freenet.support.OOMHandler;
> +import freenet.support.OOMHook;
> import freenet.support.io.Closer;
>
> /**
> @@ -105,7 +107,7 @@
> String algo = md256.getAlgorithm();
> if(!(algo.equals("SHA-256") || algo.equals("SHA256")))
> throw new IllegalArgumentException("Should be SHA-256
> but is " + algo);
> - if (digests.size() > 16) // don't cache too many of them
> + if (digests.size() > 16 || noCache) // don't cache too many of
> them
> return;
> md256.reset();
> digests.add(md256);
> @@ -121,4 +123,20 @@
> public static int getDigestLength() {
> return HASH_SIZE;
> }
> +
> + private static boolean noCache = false;
> +
> + static {
> + OOMHandler.addOOMHook(new OOMHook() {
> + public void handleLowMemory() throws Exception {
> + digests.clear();
> + noCache = true;
> + }
> +
> + public void handleOutOfMemory() throws Exception {
> + digests.clear();
> + noCache = true;
> + }
> + });
> + }
> }
>
> Modified: trunk/freenet/src/freenet/node/MemoryChecker.java
> ===================================================================
> --- trunk/freenet/src/freenet/node/MemoryChecker.java 2008-05-02 10:09:13
UTC (rev 19674)
> +++ trunk/freenet/src/freenet/node/MemoryChecker.java 2008-05-02 10:10:12
UTC (rev 19675)
> @@ -4,6 +4,7 @@
> package freenet.node;
>
> import freenet.support.Logger;
> +import freenet.support.OOMHandler;
> import freenet.support.SizeUtil;
>
> public class MemoryChecker implements Runnable {
> @@ -42,6 +43,11 @@
>
> Logger.normal(this, "Memory in
use: "+SizeUtil.formatSize((r.totalMemory()-r.freeMemory())));
>
> + if (r.freeMemory() < 4096 * 1024 * 1024) { // free memory < 8 MB
> + Logger.error(this, "memory too low, trying to free
> some");
> + OOMHandler.lowMemory();
> + }
> +
> int sleeptime = aggressiveGCModificator;
> if(sleeptime <= 0) { // We are done
> ps.queueTimedJob(this, 120 * 250); // 30 sec
>
> Modified: trunk/freenet/src/freenet/node/Node.java
> ===================================================================
> --- trunk/freenet/src/freenet/node/Node.java 2008-05-02 10:09:13 UTC (rev
19674)
> +++ trunk/freenet/src/freenet/node/Node.java 2008-05-02 10:10:12 UTC (rev
19675)
> @@ -124,7 +124,7 @@
> /**
> * @author amphibian
> */
> -public class Node implements TimeSkewDetectorCallback, GetPubkey, OOMHook {
> +public class Node implements TimeSkewDetectorCallback, GetPubkey {
>
> private static boolean logMINOR;
>
> @@ -1604,8 +1604,6 @@
> e.printStackTrace();
> throw new
NodeInitException(NodeInitException.EXIT_COULD_NOT_START_UPDATER, "Could not
create Updater: "+e);
> }
> -
> - OOMHandler.addOOMHook(this);
>
> Logger.normal(this, "Node constructor completed");
> System.out.println("Node constructor completed");
> @@ -3306,23 +3304,4 @@
> public void setDispatcherHook(NodeDispatcherCallback cb) {
> this.dispatcher.setHook(cb);
> }
> -
> - /**
> - * Free some memory
> - */
> - public void handleOOM() throws Exception {
> - if (cachedPubKeys != null) {
> - Object value;
> - do {
> - value = cachedPubKeys.popKey();
> - } while (value != null);
> - }
> - if (recentlyCompletedIDs != null) {
> - synchronized (recentlyCompletedIDs) {
> - // half it size
> - while (recentlyCompletedIDs.size() >
> MAX_RECENTLY_COMPLETED_IDS / 2)
> - recentlyCompletedIDs.pop();
> - }
> - }
> - }
> }
>
> Modified: trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
> ===================================================================
> --- trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
> 2008-05-02
10:09:13 UTC (rev 19674)
> +++ trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
> 2008-05-02
10:10:12 UTC (rev 19675)
> @@ -2239,15 +2239,30 @@
> bf.get(data);
> }
>
> - public void handleOOM() throws Exception {
> - if (storeRAF != null)
> - storeRAF.getFD().sync();
> - if (keysRAF != null)
> - keysRAF.getFD().sync();
> - if (lruRAF != null)
> - lruRAF.getFD().sync();
> + public void handleLowMemory() throws Exception {
> + // Flush all
> + if (storeFC != null)
> + storeFC.force(true);
> + if (keysFC != null)
> + keysFC.force(true);
> + if (lruFC != null)
> + lruFC.force(true);
> }
>
> + public void handleOutOfMemory() throws Exception {
> + // database likely to be corrupted,
> + // reconstruct it just in case
> + reconstructFile.createNewFile();
> +
> + // Flush all
> + if (storeFC != null)
> + storeFC.force(true);
> + if (keysFC != null)
> + keysFC.force(true);
> + if (lruFC != null)
> + lruFC.force(true);
> + }
> +
> /**
> * @return
> */
>
> Modified: trunk/freenet/src/freenet/support/OOMHandler.java
> ===================================================================
> --- trunk/freenet/src/freenet/support/OOMHandler.java 2008-05-02 10:09:13
UTC (rev 19674)
> +++ trunk/freenet/src/freenet/support/OOMHandler.java 2008-05-02 10:10:12
UTC (rev 19675)
> @@ -33,6 +33,30 @@
> }
> }
>
> + /**
> + * Call this when running low of memory
> + */
> + public static void lowMemory() {
> + System.gc();
> + System.runFinalization();
> +
> + // iterate all oom hooks
> + Iterator it = oomHooks.iterator();
> + while (it.hasNext()) {
> + OOMHook hook = ((OOMHook) it.next());
> + if (hook != null) {
> + try {
> + hook.handleLowMemory();
> + } catch (Throwable t) {
> + //ignore
> + }
> + }
> + }
> +
> + System.gc();
> + System.runFinalization();
> + }
> +
> public static void handleOOM(OutOfMemoryError e) {
> if (isOOM) {
> Logger.error(null, "Double OOM", e);
> @@ -51,20 +75,18 @@
>
> System.gc();
> System.runFinalization();
> -
> +
> // iterate all oom hooks
> Iterator it = oomHooks.iterator();
> while (it.hasNext()) {
> OOMHook hook = ((OOMHook) it.next());
> if (hook != null) {
> try {
> - hook.handleOOM();
> + hook.handleOutOfMemory();
> } catch (Throwable t) {
> //ignore
> }
> }
> -
> - System.gc();
> }
>
> System.gc();
>
> Modified: trunk/freenet/src/freenet/support/OOMHook.java
> ===================================================================
> --- trunk/freenet/src/freenet/support/OOMHook.java 2008-05-02 10:09:13 UTC
(rev 19674)
> +++ trunk/freenet/src/freenet/support/OOMHook.java 2008-05-02 10:10:12 UTC
(rev 19675)
> @@ -5,9 +5,14 @@
> */
> public interface OOMHook {
> /**
> - * Handle OutOfMemoryError
> + * Handle running low of memory
> *
> * (try to free some cache, save the files, etc).
> */
> - void handleOOM() throws Exception;
> + void handleLowMemory() throws Exception;
> +
> + /**
> + * Handle running out of memory
> + */
> + void handleOutOfMemory() throws Exception;
> }
>
> _______________________________________________
> cvs mailing list
> cvs at freenetproject.org
> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
>
>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
URL:
<https://emu.freenetproject.org/pipermail/devl/attachments/20080503/08e0a449/attachment.pgp>
