Author: burton
Date: Wed Mar 9 11:37:12 2005
New Revision: 156674
URL: http://svn.apache.org/viewcvs?view=rev&rev=156674
Log:
new rrd logging and graphing infra
Added:
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/BenchmarkSource.java
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/ExampleRandomSource.java
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/GraphTask.java
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/GraphTaskRunner.java
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/Main.java
(with props)
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/Source.java
Added:
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/BenchmarkSource.java
URL:
http://svn.apache.org/viewcvs/jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/BenchmarkSource.java?view=auto&rev=156674
==============================================================================
---
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/BenchmarkSource.java
(added)
+++
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/BenchmarkSource.java
Wed Mar 9 11:37:12 2005
@@ -0,0 +1,37 @@
+/*
+ * Copyright 1999,2004 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.benchmark.rrd;
+
+/**
+ * Represents an abstract datasource for an source for logging.
+ *
+ * @author <a href="mailto:[EMAIL PROTECTED]">Kevin A. Burton</a>
+ * @version $Id: $
+ */
+public abstract class BenchmarkSource {
+
+ String title = null;
+ String description = null;
+
+ /**
+ * Get the current value for this counter.
+ *
+ * @author <a href="mailto:[EMAIL PROTECTED]">Kevin A. Burton</a>
+ */
+ public abstract long getValue() throws Exception;
+
+}
\ No newline at end of file
Added:
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/ExampleRandomSource.java
URL:
http://svn.apache.org/viewcvs/jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/ExampleRandomSource.java?view=auto&rev=156674
==============================================================================
---
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/ExampleRandomSource.java
(added)
+++
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/ExampleRandomSource.java
Wed Mar 9 11:37:12 2005
@@ -0,0 +1,50 @@
+/*
+ * Copyright 1999,2004 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.benchmark.rrd;
+
+import java.util.*;
+
+/**
+ * Source which logs a random number between 0 and 100 for testing purposes.
+ *
+ * @author <a href="mailto:[EMAIL PROTECTED]">Kevin A. Burton</a>
+ * @version $Id: $
+ */
+public class ExampleRandomSource extends Source {
+
+ private String foo = null;
+
+ /**
+ * Get the current value for this counter.
+ *
+ * @author <a href="mailto:[EMAIL PROTECTED]">Kevin A. Burton</a>
+ */
+ public long getValue() throws Exception {
+
+ Random r = new Random();
+ return (long)(r.nextFloat() * 100F);
+ }
+
+ public void setFoo( String foo ) {
+ this.foo = foo;
+ }
+
+ public String getFoo() {
+ return foo;
+ }
+
+}
\ No newline at end of file
Added:
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/GraphTask.java
URL:
http://svn.apache.org/viewcvs/jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/GraphTask.java?view=auto&rev=156674
==============================================================================
---
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/GraphTask.java
(added)
+++
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/GraphTask.java
Wed Mar 9 11:37:12 2005
@@ -0,0 +1,354 @@
+/*
+ * Copyright 1999,2004 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.benchmark.rrd;
+
+import java.io.*;
+import java.util.*;
+import java.net.*;
+import java.text.*;
+
+import org.jrobin.core.*;
+import org.jrobin.graph.*;
+
+import java.awt.Color;
+
+/**
+ * Handles updating Sources and updating the RRD information. This is done so
+ * that we can then generate human readable histographs from the data. Note
+ * that this is a Thread which is a bit heavy when all the datasources are
local
+ * benchmarks. The problem is that we have NO idea the complexity of the
Source
+ * and it might block on database resources, filesystems, etc and we need to
+ * make sure all the benchmarks complete at once.
+ */
+public class GraphTask extends Thread {
+
+ /**
+ * True when we should generate tasks within the current process.
+ */
+ public static boolean ENABLE_GENERATE_GRAPHS=true;
+
+ /**
+ * Used to store RRDs and graphs.
+ */
+ public static String ROOT = "/var/benchmark";
+
+ public long lastUpdatedSeconds = currentTimeSeconds();
+
+ long begin = lastUpdatedSeconds;
+
+ private String name, title = null;
+
+ /**
+ * Path to the RRD file.
+ */
+ private String rrd_path = null;
+
+ /**
+ * The current RRD sample...
+ */
+ private Sample sample = null;
+
+ /**
+ * Unit title and description.
+ */
+ private String unit_desc, unit_name = null;
+
+ /**
+ * The given source for this task which provides getValue()
+ */
+ Source source = null;
+
+ /**
+ * True when this task is waiting to run again.
+ */
+ boolean isWaiting = false;
+
+ /**
+ *
+ * @param unit_desc The description of this unit. Example: "Feeds Per
Minute (FPM)"
+ * @param unit_name The name of this unit. Example: "FPM"
+ * @author <a href="mailto:[EMAIL PROTECTED]">Kevin A. Burton</a>
+ */
+ public GraphTask( String name,
+ String title,
+ String unit_desc,
+ String unit_name,
+ Source source ) {
+
+ this.name = name;
+ this.title = title;
+ this.unit_desc = unit_desc;
+ this.unit_name = unit_name;
+ this.source = source;
+
+ rrd_path = new File( ROOT, name + ".rrd" ).getPath();
+
+ }
+
+ /**
+ * Get the current time in seconds for use with RRD.
+ */
+ public static long currentTimeSeconds() {
+ return System.currentTimeMillis() / 1000;
+ }
+
+ /**
+ * Simple time format.
+ */
+ public String formatTime( long v ) {
+
+ return new SimpleDateFormat().format( new java.util.Date( v * 1000 ) );
+ }
+
+ /**
+ * Generate all graphs necessary to view this RRD.
+ */
+ void generateGraphs() throws Exception {
+
+ //FIXME: ideally we wouldn't have to generate these each time because
+ //they waste a LOT of CPU.
+
+ generateGraph( name + "-1-hour.png",
+ 60 * 60,
+ title + " - 1 hour" );
+
+ generateGraph( name + "-6-hours.png",
+ 6 * 60 * 60,
+ title + " - 6 hours" );
+
+ generateGraph( name + "-12-hours.png",
+ 12 * 60 * 60,
+ title + " - 12 hours" );
+
+ generateGraph( name + "-1-day.png",
+ 24 * 60 * 60,
+ title + " - 1 day" );
+
+ generateGraph( name + "-1-week.png",
+ 7 * 24 * 60 * 60,
+ title + " - 1 week" );
+
+ generateGraph( name + "-1-month.png",
+ 30 * 24 * 60 * 60,
+ title + " - 1 month" );
+
+ generateGraph( name + "-1-year.png",
+ 365 * 24 * 60 * 60,
+ title + " - 1 year" );
+
+ }
+
+ void generateGraph( String path,
+ long interval,
+ String title ) throws Exception {
+
+ if ( ENABLE_GENERATE_GRAPHS = false )
+ return;
+
+ path = new File( ROOT, path ).getPath();
+
+ System.out.println( "Generating graph: " + path + " ..." );
+
+ //FIXME: only generate the last HOURS worth of data.
+
+ RrdGraphDef graphDef = new RrdGraphDef();
+
+ long startTime, endTime = lastUpdatedSeconds;
+
+ startTime = endTime - interval;
+
+ System.out.println( "startTime: " + formatTime( startTime ) );
+ System.out.println( "endTime: " + formatTime( endTime ) );
+
+ graphDef.setShowSignature( false );
+
+ graphDef.setTimePeriod( startTime, endTime );
+ graphDef.setTitle( title );
+ graphDef.datasource( "myspeed", rrd_path, "speed", "AVERAGE" );
+
+ //graphDef.line( "myspeed",
+ // new Color.RED,
+ // "Feeds Per Minute (FPM)",
+ // 1 );
+
+ graphDef.area( "myspeed", Color.GREEN, unit_desc );
+
+ //FIXME: don't include 3 decimal places of precision. We don't need
+ //this.
+ graphDef.gprint( "myspeed", "AVERAGE", "Average " + unit_name + ":
@[EMAIL PROTECTED]");
+ graphDef.gprint( "myspeed", "MAX", "Max " + unit_name + ": @[EMAIL
PROTECTED]");
+ graphDef.gprint( "myspeed", "MIN", "Min " + unit_name + ": @[EMAIL
PROTECTED]");
+
+ //Sat Dec 25 2004 03:04 PM ([EMAIL PROTECTED]): LAST doesn't work and
+ //returns NaN. Why is this?
+ //graphDef.gprint( "myspeed", "LAST", "Current " + unit_name + ":
@[EMAIL PROTECTED]");
+
+ //FIXME: would be NICE to have STDDEV here as well as MEDIAN as well as
+ //LAST. I could ADD this to JRobin but I'd need to do it in
+ //FetchData.java and add getStandardDeviation and getMedian methods.
+ //Should be easy though.cn
+
+ //NOTE: total doesn't seem to make much sense at all times.
+ //graphDef.gprint( "myspeed", "TOTAL", "Total " + unit_name + ":
@[EMAIL PROTECTED]");
+
+ graphDef.comment( "Last updated on: " + formatTime( endTime ) );
+
+ RrdGraph graph = new RrdGraph( graphDef );
+
+ //graph.saveAsGIF( path );
+ graph.saveAsPNG( path );
+
+ System.out.println( "Generating graph: " + path + " ...done" );
+
+ }
+
+ /**
+ * Create the RRD when it doesn't exist including setting up archive
+ * functions, adding datasources, etc.
+ *
+ * @author <a href="mailto:[EMAIL PROTECTED]">Kevin A. Burton</a>
+ */
+ private void doCreateRRD() throws Exception {
+
+ new File( ROOT ).mkdirs();
+
+ System.out.println( "Creating rrd file: " + rrd_path );
+
+ RrdDef rrdDef = new RrdDef( rrd_path );
+ rrdDef.setStartTime( begin );
+
+ rrdDef.addDatasource( "speed",
+ "GAUGE",
+ GraphTaskRunner.COUNTER_INTERVAL_SECONDS,
+ Double.NaN,
+ Double.NaN );
+
+ //set the step to 1 minute (the default is 5 minutes).
+ rrdDef.setStep( GraphTaskRunner.COUNTER_INTERVAL_SECONDS );
+
+ //keep one measurement for every single poll.
+
+ //rrdDef.addArchive( "AVERAGE", 0.5, 1, 3600 );
+
+ //rrdDef.addArchive( "AVERAGE", 0.5, 1, 60 );
+
+ //FIXME: increase this to about 400... because this would yield a much
+ //higher density image.
+
+ //6 hours (6 minute intervals... with 60 measurements which yields 360
+ //minutes which is 6 hours).
+
+ rrdDef.addArchive( "AVERAGE", 0.5, 1, 360 );
+
+ //12 hours
+ rrdDef.addArchive( "AVERAGE", 0.5, 2, 360 );
+
+ //24 hours
+ rrdDef.addArchive( "AVERAGE", 0.5, 4, 360 );
+
+ //1 week
+ rrdDef.addArchive( "AVERAGE", 0.5, 28, 360 );
+
+ //1 month
+ rrdDef.addArchive( "AVERAGE", 0.5, 120, 360 );
+
+ //1 year
+ rrdDef.addArchive( "AVERAGE", 0.5, 1460, 360 );
+
+ //FIXME: don't we need additional archives here?
+
+ RrdDb rrdDb = new RrdDb( rrdDef );
+ rrdDb.close();
+
+ }
+
+ /**
+ * Exec this task in the current thread.
+ */
+ public void exec() throws Exception {
+
+ File file = new File( rrd_path );
+
+ if ( file.exists() == false ) {
+ doCreateRRD();
+ }
+
+ RrdDb rrdDb = new RrdDb( rrd_path );
+ sample = rrdDb.createSample();
+
+ try {
+
+ data( source.getValue() );
+ //value += 10;
+ //data( value );
+ rrdDb.sync();
+ rrdDb.close();
+
+ } finally {
+
+ rrdDb.sync();
+ rrdDb.close();
+ }
+
+ }
+
+ /**
+ * Run this task as a dedicated thread waiting to be notified by the
runner.
+ */
+ public void run() {
+
+ while ( true ) {
+ synchronized( this ) {
+
+ try {
+
+ isWaiting = true;
+ wait();
+ isWaiting = false;
+
+ exec();
+ generateGraphs();
+
+ } catch ( Exception e ) {
+ e.printStackTrace();
+ }
+
+ }
+ }
+ }
+
+ /**
+ * Take the RRD and call setAndUpdate on it with the given value.
+ */
+ public void data( long v ) throws Exception {
+
+ //NOTE: this is the correct mechanism but it might be off by a bit due
+ //to skew with running getValue() from diff impls...
+ //lastUpdatedSeconds = currentTimeSeconds();
+
+ lastUpdatedSeconds += GraphTaskRunner.COUNTER_INTERVAL_SECONDS;
+
+ String time = formatTime( lastUpdatedSeconds );
+
+ System.out.println( "time: " + time + " , " + unit_name + " value: " +
v );
+ System.out.println( "lastUpdated: " + lastUpdatedSeconds );
+
+ sample.setAndUpdate( lastUpdatedSeconds + ":" + v );
+
+ }
+
+}
\ No newline at end of file
Added:
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/GraphTaskRunner.java
URL:
http://svn.apache.org/viewcvs/jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/GraphTaskRunner.java?view=auto&rev=156674
==============================================================================
---
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/GraphTaskRunner.java
(added)
+++
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/GraphTaskRunner.java
Wed Mar 9 11:37:12 2005
@@ -0,0 +1,105 @@
+/*
+ * Copyright 1999,2004 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.benchmark.rrd;
+
+import java.util.*;
+
+/**
+ * Handles processing all tasks and updating graphs if enabled and necessary.
+ *
+ * @author <a href="mailto:[EMAIL PROTECTED]">Kevin A. Burton</a>
+ */
+public class GraphTaskRunner {
+
+ public static final long SLEEP_INTERVAL_MILLIS = 60L * 1000L;
+ public static final long COUNTER_INTERVAL_SECONDS = 60L;
+
+ public static List tasks = new LinkedList();
+
+ /**
+ * Runs all given tasks. Designed to be used within an infinite loop.
+ */
+ public void run() throws Exception {
+
+ Iterator it = tasks.iterator();
+
+ long lastUpdatedSeconds = GraphTask.currentTimeSeconds();
+
+ while ( it.hasNext() ) {
+
+ try {
+
+ GraphTask t = (GraphTask)it.next();
+
+ if ( t.isAlive() == false ) {
+ t.start();
+ }
+
+ while ( t.isWaiting == false ) {
+ Thread.sleep( 50 );
+ }
+
+ t.lastUpdatedSeconds = lastUpdatedSeconds;
+
+ synchronized( t ) {
+ t.notifyAll();
+ }
+
+ } catch ( Throwable t ) {
+
+ //FIXME: what happens here when we fail too many times..? This
+ //would be bad and we wouldn't really handle it well.
+ t.printStackTrace();
+ }
+
+ }
+
+ }
+
+ public void runForever() throws Exception {
+
+ // ********************
+ while ( true ) {
+
+ //FIXME: modularize this more.
+
+ long begin = System.currentTimeMillis();
+
+ run();
+
+ long duration = System.currentTimeMillis() - begin;
+
+ long sleep_interval = GraphTaskRunner.SLEEP_INTERVAL_MILLIS -
duration;
+
+ if ( sleep_interval > 0 ) {
+
+ System.out.println( "Sleeping for millis: " + sleep_interval );
+
+ //spin until the next read interval...
+ Thread.sleep( sleep_interval );
+
+ } else {
+
+ System.out.println( "Took too long to perform tasks. Not
sleeping..." );
+
+ }
+
+ }
+
+ }
+
+}
Added:
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/Main.java
URL:
http://svn.apache.org/viewcvs/jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/Main.java?view=auto&rev=156674
==============================================================================
---
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/Main.java
(added)
+++
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/Main.java
Wed Mar 9 11:37:12 2005
@@ -0,0 +1,44 @@
+/*
+ * Copyright 1999,2004 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.benchmark.rrd;
+
+import org.apache.commons.benchmark.config.*;
+
+/**
+ * Simple commandline rrd grapher based on GraphTaskRunner. This can be used
by
+ * itself or with another package.
+ *
+ * @author <a href="mailto:[EMAIL PROTECTED]">Kevin A. Burton</a>
+ * @version $Id: $
+ */
+public class Main {
+
+ /**
+ * Default path to the configuration file for launching the RRD daemon.
+ */
+ public static String CONFIG = "/etc/commons-benchmark/benchmark.xml";
+
+ public static void main( String[] args ) throws Exception {
+
+ XMLConfigurator.configure( CONFIG );
+
+ GraphTaskRunner runner = new GraphTaskRunner();
+ runner.runForever();
+
+ }
+
+}
Propchange:
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/Main.java
------------------------------------------------------------------------------
svn:executable = *
Added:
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/Source.java
URL:
http://svn.apache.org/viewcvs/jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/Source.java?view=auto&rev=156674
==============================================================================
---
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/Source.java
(added)
+++
jakarta/commons/sandbox/benchmark/trunk/src/java/org/apache/commons/benchmark/rrd/Source.java
Wed Mar 9 11:37:12 2005
@@ -0,0 +1,37 @@
+/*
+ * Copyright 1999,2004 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.benchmark.rrd;
+
+/**
+ * Represents an abstract datasource for an source for logging.
+ *
+ * @author <a href="mailto:[EMAIL PROTECTED]">Kevin A. Burton</a>
+ * @version $Id: $
+ */
+public abstract class Source {
+
+ String title = null;
+ String description = null;
+
+ /**
+ * Get the current value for this counter.
+ *
+ * @author <a href="mailto:[EMAIL PROTECTED]">Kevin A. Burton</a>
+ */
+ public abstract long getValue() throws Exception;
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
