This is an automated email from the ASF dual-hosted git repository.
liurui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new aae9a57 [IOTDB-23] Add monitor statistics of file size. Phase 1.1
(#88)
aae9a57 is described below
commit aae9a573f023017ff42c45013c3d10f0f71ee9a5
Author: Rui Liu <[email protected]>
AuthorDate: Fri Mar 22 09:29:13 2019 +0800
[IOTDB-23] Add monitor statistics of file size. Phase 1.1 (#88)
* add monitor statistics for file size
* fix conflict in DataPoint
* fix config
* fix config usage in UT
* remove unenable memContorller in FileSizeTest
* register file size stats series with FileNodeManager
* defaultly disable stat monitor
* format code
* add doc about the file size monitor statistics
* format ServiceType
---
.../UserGuideV0.7.0/4-Deployment and Management.md | 77 +++++++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../iotdb/db/engine/filenode/FileNodeManager.java | 87 +++++------
.../db/engine/filenode/FileNodeProcessor.java | 20 +--
.../org/apache/iotdb/db/monitor/IStatistic.java | 5 +-
.../apache/iotdb/db/monitor/MonitorConstants.java | 67 ++++++--
.../org/apache/iotdb/db/monitor/StatMonitor.java | 160 ++++++++++++-------
.../iotdb/db/monitor/collector/FileSize.java | 173 +++++++++++++++++++++
.../org/apache/iotdb/db/service/ServiceType.java | 18 ++-
.../org/apache/iotdb/db/monitor/MonitorTest.java | 66 +++++---
.../iotdb/db/monitor/collector/FileSizeTest.java | 93 +++++++++++
.../tsfile/write/record/datapoint/DataPoint.java | 9 +-
12 files changed, 612 insertions(+), 165 deletions(-)
diff --git a/docs/Documentation/UserGuideV0.7.0/4-Deployment and Management.md
b/docs/Documentation/UserGuideV0.7.0/4-Deployment and Management.md
index 655bd89..53066f8 100644
--- a/docs/Documentation/UserGuideV0.7.0/4-Deployment and Management.md
+++ b/docs/Documentation/UserGuideV0.7.0/4-Deployment and Management.md
@@ -39,6 +39,7 @@
- [Data Status Monitoring](#data-status-monitoring)
- [Writing Data Monitor](#writing-data-monitor)
- [Example](#example)
+ - [File Size Monitor](#file-size-monitor)
- [System log](#system-log)
- [Dynamic System Log Configuration](#dynamic-system-log-configuration)
- [Connect JMX](#connect-jmx)
@@ -813,6 +814,82 @@ If you want to know the current timeseries point in the
system, you can use `MAX
select MAX_VALUE(TOTAL_POINTS_SUCCESS) from root.stats.write.root_ln
```
+#### File Size Monitor
+
+Sometimes we are concerned about how the data file size of IoTDB is changing,
maybe to help calculate how much disk space is left or the data ingestion
speed. The File Size Monitor provides several statistics to show how different
types of file-sizes change.
+
+The file size monitor defaults to collect file size data every 5 seconds using
the same shared parameter ```back_loop_period_sec```,
+
+Unlike Writing Data Monitor, currently File Size Monitor will not delete
statistic data at regular intervals.
+
+You can also use `select` clause to get the file size statistics like other
time series.
+
+Here are the file size statistics:
+
+* DATA
+
+|Name| DATA |
+|:---:|:---|
+|Description| Calculate the sum of all the files's sizes under the data
directory (```data/data``` by default) in byte.|
+|Type| File size statistics |
+|Timeseries Name| root.stats.file\_size.DATA |
+|Reset After Restarting System| No |
+|Example| select DATA from root.stats.file\_size.DATA|
+
+* SETTLED
+
+|Name| SETTLED |
+|:---:|:---|
+|Description| Calculate the sum of all the ```TsFile``` size (under
```data/data/settled``` by default) in byte. If there are multiple ```TsFile```
directories like ```{data/data/settled1, data/data/settled2}```, this statistic
is the sum of their size.|
+|Type| File size statistics |
+|Timeseries Name| root.stats.file\_size.SETTLED |
+|Reset After Restarting System| No |
+|Example| select SETTLED from root.stats.file\_size.SETTLED|
+
+* OVERFLOW
+
+|Name| OVERFLOW |
+|:---:|:---|
+|Description| Calculate the sum of all the ```out-of-order data file``` size
(under ```data/data/overflow``` by default) in byte.|
+|Type| File size statistics |
+|Timeseries Name| root.stats.file\_size.OVERFLOW |
+|Reset After Restarting System| No |
+|Example| select OVERFLOW from root.stats.file\_size.OVERFLOW|
+
+
+* WAL
+
+|Name| WAL |
+|:---:|:---|
+|Description| Calculate the sum of all the ```Write-Ahead-Log file``` size
(under ```data/wal``` by default) in byte.|
+|Type| File size statistics |
+|Timeseries Name| root.stats.file\_size.WAL |
+|Reset After Restarting System| No |
+|Example| select WAL from root.stats.file\_size.WAL|
+
+
+* INFO
+
+|Name| INFO|
+|:---:|:---|
+|Description| Calculate the sum of all the ```.restore```, etc. file size
(under ```data/system/info```) in byte.|
+|Type| File size statistics |
+|Timeseries Name| root.stats.file\_size.INFO |
+|Reset After Restarting System| No |
+|Example| select INFO from root.stats.file\_size.INFO|
+
+* SCHEMA
+
+|Name| SCHEMA |
+|:---:|:---|
+|Description| Calculate the sum of all the ```metadata file``` size (under
```data/system/metadata```) in byte.|
+|Type| File size statistics |
+|Timeseries Name| root.stats.file\_size.SCHEMA |
+|Reset After Restarting System| No |
+|Example| select SCHEMA from root.stats.file\_size.SCHEMA|
+
+
+
## System log
IoTDB allows users to configure IoTDB system logs (such as log output level)
by modifying the log configuration file. The default location of the system log
configuration file is in \$IOTDB_HOME/conf folder.
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 22f17ae..dfe4c13 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -93,7 +93,7 @@ public class IoTDBConfig {
/**
* Data directory of bufferWrite data.
- * It can be setted as bufferWriteDirs = {"settled1", "settled2",
"settled3"};
+ * It can be settled as bufferWriteDirs = {"settled1", "settled2",
"settled3"};
*/
private String[] bufferWriteDirs = {DEFAULT_TSFILE_DIR};
diff --git
a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
index eec4942..76e1f2f 100644
---
a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
+++
b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
@@ -115,8 +115,8 @@ public class FileNodeManager implements IStatistic,
IService {
//TODO merge this with label A
if (TsFileDBConf.isEnableStatMonitor()) {
StatMonitor statMonitor = StatMonitor.getInstance();
- registStatMetadata();
- statMonitor.registStatistics(MonitorConstants.STAT_STORAGE_DELTA_NAME,
this);
+ registerStatMetadata();
+ statMonitor.registerStatistics(MonitorConstants.STAT_STORAGE_DELTA_NAME,
this);
}
}
@@ -146,7 +146,7 @@ public class FileNodeManager implements IStatistic,
IService {
List<String> list = new ArrayList<>();
for (MonitorConstants.FileNodeManagerStatConstants statConstant :
MonitorConstants.FileNodeManagerStatConstants.values()) {
- list.add(MonitorConstants.STAT_STORAGE_DELTA_NAME +
MonitorConstants.MONITOR_PATH_SEPERATOR
+ list.add(MonitorConstants.STAT_STORAGE_DELTA_NAME +
MonitorConstants.MONITOR_PATH_SEPARATOR
+ statConstant.name());
}
return list;
@@ -164,18 +164,18 @@ public class FileNodeManager implements IStatistic,
IService {
}
/**
- * Init Stat MetaDta
+ * Init Stat MetaDta.
*/
@Override
- public void registStatMetadata() {
+ public void registerStatMetadata() {
Map<String, String> hashMap = new HashMap<>();
for (MonitorConstants.FileNodeManagerStatConstants statConstant :
MonitorConstants.FileNodeManagerStatConstants.values()) {
hashMap
- .put(MonitorConstants.STAT_STORAGE_DELTA_NAME +
MonitorConstants.MONITOR_PATH_SEPERATOR +
- statConstant.name(), MonitorConstants.DATA_TYPE);
+ .put(MonitorConstants.STAT_STORAGE_DELTA_NAME +
MonitorConstants.MONITOR_PATH_SEPARATOR
+ + statConstant.name(), MonitorConstants.DATA_TYPE_INT64);
}
- StatMonitor.getInstance().registStatStorageGroup(hashMap);
+ StatMonitor.getInstance().registerStatStorageGroup(hashMap);
}
/**
@@ -533,9 +533,9 @@ public class FileNodeManager implements IStatistic,
IService {
}
try {
overflowProcessor.getLogNode().write(new DeletePlan(timestamp,
- new Path(deviceId + "." + measurementId)));
+ new Path(deviceId + "." + measurementId)));
bufferWriteProcessor.getLogNode().write(new DeletePlan(timestamp,
- new Path(deviceId + "." + measurementId)));
+ new Path(deviceId + "." + measurementId)));
} catch (IOException e) {
throw new FileNodeManagerException(e);
}
@@ -556,6 +556,39 @@ public class FileNodeManager implements IStatistic,
IService {
}
}
+ private void delete(String processorName,
+ Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator)
+ throws FileNodeManagerException {
+ if (!processorMap.containsKey(processorName)) {
+ LOGGER.warn("The processorMap doesn't contain the filenode processor
{}.", processorName);
+ return;
+ }
+ LOGGER.info("Try to delete the filenode processor {}.", processorName);
+ FileNodeProcessor processor = processorMap.get(processorName);
+ if (!processor.tryWriteLock()) {
+ LOGGER.warn("Can't get the write lock of the filenode processor {}.",
processorName);
+ return;
+ }
+
+ try {
+ if (!processor.canBeClosed()) {
+ LOGGER.warn("The filenode processor {} can't be deleted.",
processorName);
+ return;
+ }
+
+ try {
+ LOGGER.info("Delete the filenode processor {}.", processorName);
+ processor.delete();
+ processorIterator.remove();
+ } catch (ProcessorException e) {
+ LOGGER.error("Delete the filenode processor {} by iterator error.",
processorName, e);
+ throw new FileNodeManagerException(e);
+ }
+ } finally {
+ processor.writeUnlock();
+ }
+ }
+
/**
* Similar to delete(), but only deletes data in BufferWrite. Only used by
WAL recovery.
*/
@@ -592,40 +625,6 @@ public class FileNodeManager implements IStatistic,
IService {
fileNodeProcessor.setOverflowed(true);
}
-
- private void delete(String processorName,
- Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator)
- throws FileNodeManagerException {
- if (!processorMap.containsKey(processorName)) {
- LOGGER.warn("The processorMap doesn't contain the filenode processor
{}.", processorName);
- return;
- }
- LOGGER.info("Try to delete the filenode processor {}.", processorName);
- FileNodeProcessor processor = processorMap.get(processorName);
- if (!processor.tryWriteLock()) {
- LOGGER.warn("Can't get the write lock of the filenode processor {}.",
processorName);
- return;
- }
-
- try {
- if (!processor.canBeClosed()) {
- LOGGER.warn("The filenode processor {} can't be deleted.",
processorName);
- return;
- }
-
- try {
- LOGGER.info("Delete the filenode processor {}.", processorName);
- processor.delete();
- processorIterator.remove();
- } catch (ProcessorException e) {
- LOGGER.error("Delete the filenode processor {} by iterator error.",
processorName, e);
- throw new FileNodeManagerException(e);
- }
- } finally {
- processor.writeUnlock();
- }
- }
-
/**
* begin query.
* @param deviceId queried deviceId
diff --git
a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
index 6f56ebc..2c770a5 100644
---
a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
+++
b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
@@ -229,8 +229,8 @@ public class FileNodeProcessor extends Processor implements
IStatistic {
statParamsHashMap.put(statConstant.name(), new AtomicLong(0));
}
statStorageDeltaName =
- MonitorConstants.STAT_STORAGE_GROUP_PREFIX +
MonitorConstants.MONITOR_PATH_SEPERATOR
- + MonitorConstants.FILE_NODE_PATH +
MonitorConstants.MONITOR_PATH_SEPERATOR
+ MonitorConstants.STAT_STORAGE_GROUP_PREFIX +
MonitorConstants.MONITOR_PATH_SEPARATOR
+ + MonitorConstants.FILE_NODE_PATH +
MonitorConstants.MONITOR_PATH_SEPARATOR
+ processorName.replaceAll("\\.", "_");
this.parameters = new HashMap<>();
@@ -287,8 +287,8 @@ public class FileNodeProcessor extends Processor implements
IStatistic {
// RegistStatService
if (TsFileDBConf.isEnableStatMonitor()) {
StatMonitor statMonitor = StatMonitor.getInstance();
- registStatMetadata();
- statMonitor.registStatistics(statStorageDeltaName, this);
+ registerStatMetadata();
+ statMonitor.registerStatistics(statStorageDeltaName, this);
}
try {
versionController = new SimpleFileVersionController(fileNodeDirPath);
@@ -303,15 +303,15 @@ public class FileNodeProcessor extends Processor
implements IStatistic {
}
@Override
- public void registStatMetadata() {
+ public void registerStatMetadata() {
Map<String, String> hashMap = new HashMap<>();
for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
MonitorConstants.FileNodeProcessorStatConstants.values()) {
hashMap
- .put(statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPERATOR
+ statConstant.name(),
- MonitorConstants.DATA_TYPE);
+ .put(statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPARATOR
+ statConstant.name(),
+ MonitorConstants.DATA_TYPE_INT64);
}
- StatMonitor.getInstance().registStatStorageGroup(hashMap);
+ StatMonitor.getInstance().registerStatStorageGroup(hashMap);
}
@Override
@@ -320,7 +320,7 @@ public class FileNodeProcessor extends Processor implements
IStatistic {
for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
MonitorConstants.FileNodeProcessorStatConstants.values()) {
list.add(
- statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPERATOR +
statConstant.name());
+ statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPARATOR +
statConstant.name());
}
return list;
}
@@ -1765,7 +1765,7 @@ public class FileNodeProcessor extends Processor
implements IStatistic {
if (TsFileDBConf.isEnableStatMonitor()) {
// remove the monitor
LOGGER.info("Deregister the filenode processor: {} from monitor.",
getProcessorName());
- StatMonitor.getInstance().deregistStatistics(statStorageDeltaName);
+ StatMonitor.getInstance().deregisterStatistics(statStorageDeltaName);
}
closeBufferWrite();
closeOverflow();
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/IStatistic.java
b/iotdb/src/main/java/org/apache/iotdb/db/monitor/IStatistic.java
index aeb0953..36c83da 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/IStatistic.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/IStatistic.java
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.iotdb.db.monitor;
import java.util.List;
@@ -34,9 +35,9 @@ public interface IStatistic {
Map<String, TSRecord> getAllStatisticsValue();
/**
- * registStatMetadata registers statistics info to the manager.
+ * registerStatMetadata registers statistics info to the manager.
*/
- void registStatMetadata();
+ void registerStatMetadata();
/**
* Get all module's statistics parameters as a time-series seriesPath.
diff --git
a/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
b/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
index 21042ee..ec899c8 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
@@ -16,35 +16,46 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.iotdb.db.monitor;
+import java.io.File;
import java.util.HashMap;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.monitor.collector.FileSize;
+import org.apache.iotdb.db.service.Monitor;
public class MonitorConstants {
- public static final String DATA_TYPE = "INT64";
- public static final String FILENODE_PROCESSOR_CONST =
"FILENODE_PROCESSOR_CONST";
- public static final String FILENODE_MANAGER_CONST = "FILENODE_MANAGER_CONST";
- public static final String MONITOR_PATH_SEPERATOR = ".";
+ private static IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
+ public static final String DATA_TYPE_INT64 = "INT64";
public static final String STAT_STORAGE_GROUP_PREFIX = "root.stats";
-
+ static final String FILENODE_PROCESSOR_CONST = "FILENODE_PROCESSOR_CONST";
+ private static final String FILENODE_MANAGER_CONST =
"FILENODE_MANAGER_CONST";
+ static final String FILE_SIZE_CONST = "FILE_SIZE_CONST";
+ public static final String MONITOR_PATH_SEPARATOR = ".";
+ // statistic for file size statistic module
+ private static final String FILE_SIZE = "file_size";
+ public static final String FILE_SIZE_STORAGE_GROUP_NAME =
STAT_STORAGE_GROUP_PREFIX
+ + MONITOR_PATH_SEPARATOR + FILE_SIZE;
// statistic for write module
- public static final String FILE_NODE_MANAGER_PATH = "write.global";
+ static final String FILE_NODE_MANAGER_PATH = "write.global";
public static final String FILE_NODE_PATH = "write";
/**
* Stat information.
*/
public static final String STAT_STORAGE_DELTA_NAME =
STAT_STORAGE_GROUP_PREFIX
- + MONITOR_PATH_SEPERATOR + FILE_NODE_MANAGER_PATH;
+ + MONITOR_PATH_SEPARATOR + FILE_NODE_MANAGER_PATH;
/**
- * function for initing values.
+ * function for initializing stats values.
*
* @param constantsType produce initialization values for Statistics Params
* @return HashMap contains all the Statistics Params
*/
- public static HashMap<String, AtomicLong> initValues(String constantsType) {
+ static HashMap<String, AtomicLong> initValues(String constantsType) {
HashMap<String, AtomicLong> hashMap = new HashMap<>();
switch (constantsType) {
case FILENODE_PROCESSOR_CONST:
@@ -54,23 +65,49 @@ public class MonitorConstants {
}
break;
case FILENODE_MANAGER_CONST:
- for (FileNodeManagerStatConstants statConstant :
FileNodeManagerStatConstants.values()) {
- hashMap.put(statConstant.name(), new AtomicLong(0));
+ hashMap = (HashMap<String, AtomicLong>)
FileSize.getInstance().getStatParamsHashMap();
+ break;
+ case FILE_SIZE_CONST:
+ for (FileSizeConstants kinds : FileSizeConstants.values()) {
+ hashMap.put(kinds.name(), new AtomicLong(0));
}
break;
default:
- // TODO: throws some errors
+
break;
}
return hashMap;
}
public enum FileNodeManagerStatConstants {
- TOTAL_POINTS, TOTAL_REQ_SUCCESS, TOTAL_REQ_FAIL, TOTAL_POINTS_SUCCESS,
TOTAL_POINTS_FAIL,
-
+ TOTAL_POINTS, TOTAL_REQ_SUCCESS, TOTAL_REQ_FAIL, TOTAL_POINTS_SUCCESS,
TOTAL_POINTS_FAIL
}
public enum FileNodeProcessorStatConstants {
- TOTAL_REQ_SUCCESS, TOTAL_REQ_FAIL, TOTAL_POINTS_SUCCESS, TOTAL_POINTS_FAIL,
+ TOTAL_REQ_SUCCESS, TOTAL_REQ_FAIL, TOTAL_POINTS_SUCCESS, TOTAL_POINTS_FAIL
+ }
+
+ public enum OsStatConstants {
+ NETWORK_REC, NETWORK_SEND, CPU_USAGE, MEM_USAGE, IOTDB_MEM_SIZE,
DISK_USAGE, DISK_READ_SPEED,
+ DISK_WRITE_SPEED, DISK_TPS
+ }
+
+ public enum FileSizeConstants {
+ DATA(Monitor.INSTANCE.getBaseDirectory()),
+ OVERFLOW(new File(config.getOverflowDataDir()).getAbsolutePath()),
+ SETTLED(Monitor.INSTANCE.getBaseDirectory() + File.separatorChar +
"settled"),
+ WAL(new File(config.getWalFolder()).getAbsolutePath()),
+ INFO(new File(config.getFileNodeDir()).getAbsolutePath()),
+ SCHEMA(new File(config.getMetadataDir()).getAbsolutePath());
+
+ public String getPath() {
+ return path;
+ }
+
+ private String path;
+
+ FileSizeConstants(String path) {
+ this.path = path;
+ }
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
index fdb4d84..a9a5355 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.iotdb.db.monitor;
import java.io.IOException;
@@ -26,7 +27,6 @@ import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.iotdb.db.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.db.concurrent.ThreadName;
import org.apache.iotdb.db.conf.IoTDBConfig;
@@ -37,6 +37,9 @@ import
org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.metadata.MManager;
+import
org.apache.iotdb.db.monitor.MonitorConstants.FileNodeManagerStatConstants;
+import
org.apache.iotdb.db.monitor.MonitorConstants.FileNodeProcessorStatConstants;
+import org.apache.iotdb.db.monitor.collector.FileSize;
import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.ServiceType;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
@@ -44,7 +47,6 @@ import
org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
import org.apache.iotdb.tsfile.write.record.datapoint.LongDataPoint;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,11 +58,12 @@ public class StatMonitor implements IService {
private final int statMonitorDetectFreqSec;
private final int statMonitorRetainIntervalSec;
private long runningTimeMillis = System.currentTimeMillis();
+ private static final ArrayList<String> temporaryStatList = new ArrayList<>();
/**
* key: is the statistics store seriesPath Value: is an interface that
implements statistics
* function.
*/
- private HashMap<String, IStatistic> statisticMap;
+ private final HashMap<String, IStatistic> statisticMap;
private ScheduledExecutorService service;
/**
@@ -72,6 +75,7 @@ public class StatMonitor implements IService {
private AtomicLong numInsertError = new AtomicLong(0);
private StatMonitor() {
+ initTemporaryStatList();
MManager mmanager = MManager.getInstance();
statisticMap = new HashMap<>();
IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
@@ -90,6 +94,15 @@ public class StatMonitor implements IService {
}
}
+ private void initTemporaryStatList() {
+ for (FileNodeManagerStatConstants constants :
FileNodeManagerStatConstants.values()) {
+ temporaryStatList.add(constants.name());
+ }
+ for (FileNodeProcessorStatConstants constants :
FileNodeProcessorStatConstants.values()) {
+ temporaryStatList.add(constants.name());
+ }
+ }
+
public static StatMonitor getInstance() {
return StatMonitorHolder.INSTANCE;
}
@@ -99,20 +112,17 @@ public class StatMonitor implements IService {
*
* @param hashMap key is statParams name, values is AtomicLong type
* @param statGroupDeltaName is the deviceId seriesPath of this module
- * @param curTime TODO need to be fixed because it may contain overflow
+ * @param curTime current time stamp
* @return TSRecord contains the DataPoints of a statGroupDeltaName
*/
public static TSRecord convertToTSRecord(Map<String, AtomicLong> hashMap,
- String statGroupDeltaName, long
curTime) {
+ String statGroupDeltaName, long curTime) {
TSRecord tsRecord = new TSRecord(curTime, statGroupDeltaName);
- tsRecord.dataPointList = new ArrayList<DataPoint>() {
- {
- for (Map.Entry<String, AtomicLong> entry : hashMap.entrySet()) {
- AtomicLong value = entry.getValue();
- add(new LongDataPoint(entry.getKey(), value.get()));
- }
- }
- };
+ tsRecord.dataPointList = new ArrayList<>();
+ for (Map.Entry<String, AtomicLong> entry : hashMap.entrySet()) {
+ AtomicLong value = entry.getValue();
+ tsRecord.dataPointList.add(new LongDataPoint(entry.getKey(),
value.get()));
+ }
return tsRecord;
}
@@ -128,7 +138,7 @@ public class StatMonitor implements IService {
return numInsertError.get();
}
- public void registStatStorageGroup() {
+ void registerStatStorageGroup() {
MManager mManager = MManager.getInstance();
String prefix = MonitorConstants.STAT_STORAGE_GROUP_PREFIX;
try {
@@ -140,12 +150,17 @@ public class StatMonitor implements IService {
}
}
- public synchronized void registStatStorageGroup(Map<String, String> hashMap)
{
+ /**
+ * register monitor statistics time series metadata into MManager.
+ *
+ * @param hashMap series path and data type pair, for example:
[root.stat.file.size.DATA, INT64]
+ */
+ public synchronized void registerStatStorageGroup(Map<String, String>
hashMap) {
MManager mManager = MManager.getInstance();
try {
for (Map.Entry<String, String> entry : hashMap.entrySet()) {
- if (entry.getKey() == null) {
- LOGGER.error("Registering metadata but {} is null", entry.getKey());
+ if (entry.getValue() == null) {
+ LOGGER.error("Registering metadata but data type of {} is null",
entry.getKey());
}
if (!mManager.pathExist(entry.getKey())) {
@@ -163,15 +178,14 @@ public class StatMonitor implements IService {
// // restore the FildeNode Manager TOTAL_POINTS statistics info
}
- public void activate() {
-
+ void activate() {
service = IoTDBThreadPoolFactory.newScheduledThreadPool(1,
- ThreadName.STAT_MONITOR.getName());
+ ThreadName.STAT_MONITOR.getName());
service.scheduleAtFixedRate(
new StatBackLoop(), 1, backLoopPeriod, TimeUnit.SECONDS);
}
- public void clearIStatisticMap() {
+ void clearIStatisticMap() {
statisticMap.clear();
}
@@ -179,7 +193,13 @@ public class StatMonitor implements IService {
return numBackLoop.get();
}
- public void registStatistics(String path, IStatistic iStatistic) {
+ /**
+ * register class which implemented IStatistic interface into statisticMap
+ *
+ * @param path the stat series prefix path, like root.stat.file.size
+ * @param iStatistic instance of class which implemented IStatistic interface
+ */
+ public void registerStatistics(String path, IStatistic iStatistic) {
synchronized (statisticMap) {
LOGGER.debug("Register {} to StatMonitor for statistics service", path);
this.statisticMap.put(path, iStatistic);
@@ -187,9 +207,9 @@ public class StatMonitor implements IService {
}
/**
- * deregist statistics.
+ * deregister statistics.
*/
- public void deregistStatistics(String path) {
+ public void deregisterStatistics(String path) {
LOGGER.debug("Deregister {} in StatMonitor for stopping statistics
service", path);
synchronized (statisticMap) {
if (statisticMap.containsKey(path)) {
@@ -199,7 +219,7 @@ public class StatMonitor implements IService {
}
/**
- * TODO: need to complete the query key concept.
+ * This function is not used and need to complete the query key concept.
*
* @return TSRecord, query statistics params
*/
@@ -208,8 +228,9 @@ public class StatMonitor implements IService {
// or FileNodeManager seriesPath:FileNodeManager
String queryPath;
if (key.contains("\\.")) {
- queryPath = MonitorConstants.STAT_STORAGE_GROUP_PREFIX +
MonitorConstants.MONITOR_PATH_SEPERATOR
- + key.replaceAll("\\.", "_");
+ queryPath =
+ MonitorConstants.STAT_STORAGE_GROUP_PREFIX +
MonitorConstants.MONITOR_PATH_SEPARATOR
+ + key.replaceAll("\\.", "_");
} else {
queryPath = key;
}
@@ -219,8 +240,8 @@ public class StatMonitor implements IService {
long currentTimeMillis = System.currentTimeMillis();
HashMap<String, TSRecord> hashMap = new HashMap<>();
TSRecord tsRecord = convertToTSRecord(
-
MonitorConstants.initValues(MonitorConstants.FILENODE_PROCESSOR_CONST),
queryPath,
- currentTimeMillis);
+
MonitorConstants.initValues(MonitorConstants.FILENODE_PROCESSOR_CONST),
queryPath,
+ currentTimeMillis);
hashMap.put(queryPath, tsRecord);
return hashMap;
}
@@ -229,21 +250,31 @@ public class StatMonitor implements IService {
/**
* get all statistics.
*/
- public HashMap<String, TSRecord> gatherStatistics() {
+ public Map<String, TSRecord> gatherStatistics() {
synchronized (statisticMap) {
long currentTimeMillis = System.currentTimeMillis();
HashMap<String, TSRecord> tsRecordHashMap = new HashMap<>();
for (Map.Entry<String, IStatistic> entry : statisticMap.entrySet()) {
if (entry.getValue() == null) {
- tsRecordHashMap.put(entry.getKey(),
+ switch (entry.getKey()) {
+ case MonitorConstants.STAT_STORAGE_DELTA_NAME:
+ tsRecordHashMap.put(entry.getKey(),
+ convertToTSRecord(
+
MonitorConstants.initValues(MonitorConstants.FILENODE_PROCESSOR_CONST),
+ entry.getKey(), currentTimeMillis));
+ break;
+ case MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME:
+ tsRecordHashMap.put(entry.getKey(),
convertToTSRecord(
-
MonitorConstants.initValues(MonitorConstants.FILENODE_PROCESSOR_CONST),
- entry.getKey(), currentTimeMillis));
+
MonitorConstants.initValues(MonitorConstants.FILE_SIZE_CONST),
+ entry.getKey(), currentTimeMillis));
+ break;
+ default:
+ }
} else {
tsRecordHashMap.putAll(entry.getValue().getAllStatisticsValue());
}
}
- LOGGER.debug("Values of tsRecordHashMap is : {}",
tsRecordHashMap.toString());
for (TSRecord value : tsRecordHashMap.values()) {
value.time = currentTimeMillis;
}
@@ -251,23 +282,6 @@ public class StatMonitor implements IService {
}
}
- private void insert(HashMap<String, TSRecord> tsRecordHashMap) {
- FileNodeManager fManager = FileNodeManager.getInstance();
- int count = 0;
- int pointNum;
- for (Map.Entry<String, TSRecord> entry : tsRecordHashMap.entrySet()) {
- try {
- fManager.insert(entry.getValue(), true);
- numInsert.incrementAndGet();
- pointNum = entry.getValue().dataPointList.size();
- numPointsInsert.addAndGet(pointNum);
- count += pointNum;
- } catch (FileNodeManagerException e) {
- numInsertError.incrementAndGet();
- LOGGER.error("Inserting stat points error.", e);
- }
- }
- }
/**
* close statistic service.
@@ -283,6 +297,8 @@ public class StatMonitor implements IService {
service.awaitTermination(10, TimeUnit.SECONDS);
} catch (InterruptedException e) {
LOGGER.error("StatMonitor timing service could not be shutdown.", e);
+ // Restore interrupted state...
+ Thread.currentThread().interrupt();
}
}
@@ -294,8 +310,8 @@ public class StatMonitor implements IService {
}
} catch (Exception e) {
String errorMessage = String
- .format("Failed to start %s because of %s",
this.getID().getName(),
- e.getMessage());
+ .format("Failed to start %s because of %s", this.getID().getName(),
+ e.getMessage());
throw new StartupException(errorMessage);
}
}
@@ -313,40 +329,64 @@ public class StatMonitor implements IService {
}
private static class StatMonitorHolder {
- private StatMonitorHolder(){
+
+ private StatMonitorHolder() {
//allowed do nothing
}
+
private static final StatMonitor INSTANCE = new StatMonitor();
}
class StatBackLoop implements Runnable {
+
+ FileSize fileSize = FileSize.getInstance();
+
@Override
public void run() {
try {
long currentTimeMillis = System.currentTimeMillis();
long seconds = (currentTimeMillis - runningTimeMillis) / 1000;
- if (seconds - statMonitorDetectFreqSec >= 0) {
+ if (seconds >= statMonitorDetectFreqSec) {
runningTimeMillis = currentTimeMillis;
// delete time-series data
FileNodeManager fManager = FileNodeManager.getInstance();
try {
for (Map.Entry<String, IStatistic> entry :
statisticMap.entrySet()) {
for (String statParamName :
entry.getValue().getStatParamsHashMap().keySet()) {
- fManager.delete(entry.getKey(), statParamName,
- currentTimeMillis - statMonitorRetainIntervalSec * 1000);
+ if (temporaryStatList.contains(statParamName)) {
+ fManager.delete(entry.getKey(), statParamName,
+ currentTimeMillis - statMonitorRetainIntervalSec * 1000);
+ }
}
}
} catch (FileNodeManagerException e) {
- LOGGER.error("Error occurred when deleting statistics information
periodically, because",
- e);
+ LOGGER
+ .error("Error occurred when deleting statistics information
periodically, because",
+ e);
}
}
- HashMap<String, TSRecord> tsRecordHashMap = gatherStatistics();
+ Map<String, TSRecord> tsRecordHashMap = gatherStatistics();
insert(tsRecordHashMap);
numBackLoop.incrementAndGet();
} catch (Exception e) {
LOGGER.error("Error occurred in Stat Monitor thread", e);
}
}
+
+ public void insert(Map<String, TSRecord> tsRecordHashMap) {
+ FileNodeManager fManager = FileNodeManager.getInstance();
+ int pointNum;
+ for (Map.Entry<String, TSRecord> entry : tsRecordHashMap.entrySet()) {
+ try {
+ fManager.insert(entry.getValue(), true);
+ numInsert.incrementAndGet();
+ pointNum = entry.getValue().dataPointList.size();
+ numPointsInsert.addAndGet(pointNum);
+ } catch (FileNodeManagerException e) {
+ numInsertError.incrementAndGet();
+ LOGGER.error("Inserting stat points error.", e);
+ }
+ }
+ }
}
}
diff --git
a/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
b/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
new file mode 100644
index 0000000..f3e3ffe
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.monitor.collector;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.monitor.IStatistic;
+import org.apache.iotdb.db.monitor.MonitorConstants;
+import org.apache.iotdb.db.monitor.MonitorConstants.FileSizeConstants;
+import org.apache.iotdb.db.monitor.StatMonitor;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class is to collect some file size statistics.
+ */
+public class FileSize implements IStatistic {
+
+ private static IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
+ private static final Logger LOGGER = LoggerFactory.getLogger(FileSize.class);
+ private static final long ABNORMAL_VALUE = -1L;
+ private static final long INIT_VALUE_IF_FILE_NOT_EXIST = 0L;
+ private FileNodeManager fileNodeManager;
+
+ @Override
+ public Map<String, TSRecord> getAllStatisticsValue() {
+ long curTime = System.currentTimeMillis();
+ TSRecord tsRecord = StatMonitor
+ .convertToTSRecord(getStatParamsHashMap(),
MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME,
+ curTime);
+ HashMap<String, TSRecord> ret = new HashMap<>();
+ ret.put(MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME, tsRecord);
+ return ret;
+ }
+
+ @Override
+ public void registerStatMetadata() {
+ Map<String, String> hashMap = new HashMap<>();
+ for (FileSizeConstants kind : FileSizeConstants.values()) {
+ String seriesPath = MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME
+ + MonitorConstants.MONITOR_PATH_SEPARATOR
+ + kind.name();
+ hashMap.put(seriesPath, MonitorConstants.DATA_TYPE_INT64);
+ Path path = new Path(seriesPath);
+ try {
+ fileNodeManager.addTimeSeries(path,
TSDataType.valueOf(MonitorConstants.DATA_TYPE_INT64),
+ TSEncoding.valueOf("RLE"),
CompressionType.valueOf(TSFileConfig.compressor),
+ Collections.emptyMap());
+ } catch (FileNodeManagerException e) {
+ LOGGER.error("Register File Size Stats into fileNodeManager Failed.",
e);
+ }
+ }
+ StatMonitor.getInstance().registerStatStorageGroup(hashMap);
+ }
+
+ @Override
+ public List<String> getAllPathForStatistic() {
+ List<String> list = new ArrayList<>();
+ for (FileSizeConstants kind : MonitorConstants.FileSizeConstants.values())
{
+ list.add(
+ MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME +
MonitorConstants.MONITOR_PATH_SEPARATOR
+ + kind.name());
+ }
+ return list;
+ }
+
+ @Override
+ public Map<String, AtomicLong> getStatParamsHashMap() {
+ Map<FileSizeConstants, Long> fileSizeMap = getFileSizesInByte();
+ Map<String, AtomicLong> statParamsMap = new HashMap<>();
+ for (FileSizeConstants kind : MonitorConstants.FileSizeConstants.values())
{
+ statParamsMap.put(kind.name(), new AtomicLong(fileSizeMap.get(kind)));
+ }
+ return statParamsMap;
+ }
+
+ private static class FileSizeHolder {
+
+ private static final FileSize INSTANCE = new FileSize();
+ }
+
+ private FileSize() {
+ fileNodeManager = FileNodeManager.getInstance();
+ if (config.isEnableStatMonitor()) {
+ StatMonitor statMonitor = StatMonitor.getInstance();
+ registerStatMetadata();
+
statMonitor.registerStatistics(MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME,
this);
+ }
+ }
+
+ public static FileSize getInstance() {
+ return FileSizeHolder.INSTANCE;
+ }
+
+ /**
+ * Return a map[FileSizeConstants, Long]. The key is the dir type and the
value is the dir size in
+ * byte.
+ *
+ * @return a map[FileSizeConstants, Long] with the dir type and the dir size
in byte
+ */
+ public Map<FileSizeConstants, Long> getFileSizesInByte() {
+ EnumMap<FileSizeConstants, Long> fileSizes = new
EnumMap<>(FileSizeConstants.class);
+ for (FileSizeConstants kinds :
MonitorConstants.FileSizeConstants.values()) {
+ if (kinds.equals(MonitorConstants.FileSizeConstants.SETTLED)) {
+ //sum bufferWriteDirs size
+ long settledSize = INIT_VALUE_IF_FILE_NOT_EXIST;
+ for (String bufferWriteDir : config.getBufferWriteDirs()) {
+ File settledFile = new File(bufferWriteDir);
+ if (settledFile.exists()) {
+ try {
+ settledSize += FileUtils.sizeOfDirectory(settledFile);
+ } catch (Exception e) {
+ LOGGER.error("Meet error while trying to get {} size with dir {}
.", kinds,
+ bufferWriteDir, e);
+ fileSizes.put(kinds, ABNORMAL_VALUE);
+ }
+ }
+ }
+ fileSizes.put(kinds, settledSize);
+ } else {
+ File file = new File(kinds.getPath());
+ if (file.exists()) {
+ try {
+ fileSizes.put(kinds, FileUtils.sizeOfDirectory(file));
+ } catch (Exception e) {
+ LOGGER
+ .error("Meet error while trying to get {} size with dir {} .",
kinds,
+ kinds.getPath(),
+ e);
+ fileSizes.put(kinds, ABNORMAL_VALUE);
+ }
+ } else {
+ fileSizes.put(kinds, INIT_VALUE_IF_FILE_NOT_EXIST);
+ }
+ }
+ }
+ return fileSizes;
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/service/ServiceType.java
b/iotdb/src/main/java/org/apache/iotdb/db/service/ServiceType.java
index 61ae722..e6c58ea 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/ServiceType.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/ServiceType.java
@@ -16,16 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.iotdb.db.service;
public enum ServiceType {
- FILE_NODE_SERVICE("File Node ServerService", ""), JMX_SERVICE("JMX
ServerService",
- "JMX ServerService"), JDBC_SERVICE("JDBC ServerService", "JDBCService"),
MONITOR_SERVICE(
- "Monitor ServerService", "Monitor"), STAT_MONITOR_SERVICE("Statistics
ServerService",
- ""), WAL_SERVICE("WAL ServerService", ""),
CLOSE_MERGE_SERVICE("Close&Merge ServerService",
- ""), JVM_MEM_CONTROL_SERVICE("Memory Controller", ""),
AUTHORIZATION_SERVICE(
- "Authorization ServerService",
- ""), FILE_READER_MANAGER_SERVICE("File reader manager ServerService",
"");
+ FILE_NODE_SERVICE("File Node ServerService", ""),
+ JMX_SERVICE("JMX ServerService", "JMX ServerService"),
+ JDBC_SERVICE("JDBC ServerService", "JDBCService"),
+ MONITOR_SERVICE("Monitor ServerService", "Monitor"),
+ STAT_MONITOR_SERVICE("Statistics ServerService", ""),
+ WAL_SERVICE("WAL ServerService", ""),
+ CLOSE_MERGE_SERVICE("Close&Merge ServerService", ""),
+ JVM_MEM_CONTROL_SERVICE("Memory Controller", ""),
+ AUTHORIZATION_SERVICE("Authorization ServerService", ""),
+ FILE_READER_MANAGER_SERVICE("File reader manager ServerService", "");
private String name;
private String jmxName;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java
b/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java
index 48fa61a..1f959e1 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java
@@ -16,16 +16,23 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.iotdb.db.monitor;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.exception.FileNodeManagerException;
import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.monitor.MonitorConstants.FileSizeConstants;
+import org.apache.iotdb.db.monitor.collector.FileSize;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
@@ -33,17 +40,9 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.*;
-
-/**
- * @author Liliang
- */
-
public class MonitorTest {
- private IoTDBConfig tsdbconfig = IoTDBDescriptor.getInstance().getConfig();
-
- private FileNodeManager fManager = null;
+ private IoTDBConfig ioTDBConfig = IoTDBDescriptor.getInstance().getConfig();
private StatMonitor statMonitor;
@Before
@@ -52,35 +51,45 @@ public class MonitorTest {
// modify stat parameter
EnvironmentUtils.closeMemControl();
EnvironmentUtils.envSetUp();
- tsdbconfig.setEnableStatMonitor(true);
- tsdbconfig.setBackLoopPeriodSec(1);
+ ioTDBConfig.setEnableStatMonitor(true);
+ ioTDBConfig.setBackLoopPeriodSec(1);
}
@After
public void tearDown() throws Exception {
- tsdbconfig.setEnableStatMonitor(false);
+ ioTDBConfig.setEnableStatMonitor(false);
statMonitor.close();
EnvironmentUtils.cleanEnv();
}
@Test
public void testFileNodeManagerMonitorAndAddMetadata() {
- fManager = FileNodeManager.getInstance();
+ FileNodeManager fManager = FileNodeManager.getInstance();
+ FileSize fileSize = FileSize.getInstance();
statMonitor = StatMonitor.getInstance();
- statMonitor.registStatStorageGroup();
+ statMonitor.registerStatStorageGroup();
fManager.getStatParamsHashMap().forEach((key, value) -> value.set(0));
+ fileSize.getStatParamsHashMap().forEach((key, value) -> value.set(0));
statMonitor.clearIStatisticMap();
- statMonitor.registStatistics(fManager.getClass().getSimpleName(),
fManager);
+ statMonitor.registerStatistics(fManager.getClass().getSimpleName(),
fManager);
+ statMonitor
+ .registerStatistics(MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME,
FileSize.getInstance());
// add metadata
MManager mManager = MManager.getInstance();
- fManager.registStatMetadata();
+ fManager.registerStatMetadata();
+ fileSize.registerStatMetadata();
Map<String, AtomicLong> statParamsHashMap =
fManager.getStatParamsHashMap();
+ Map<String, AtomicLong> fileSizeStatsHashMap =
fileSize.getStatParamsHashMap();
for (String statParam : statParamsHashMap.keySet()) {
- assertEquals(true,
- mManager.pathExist(
- MonitorConstants.STAT_STORAGE_GROUP_PREFIX +
MonitorConstants.MONITOR_PATH_SEPERATOR
- + MonitorConstants.FILE_NODE_MANAGER_PATH +
MonitorConstants.MONITOR_PATH_SEPERATOR
- + statParam));
+ assertTrue(mManager.pathExist(
+ MonitorConstants.STAT_STORAGE_GROUP_PREFIX +
MonitorConstants.MONITOR_PATH_SEPARATOR
+ + MonitorConstants.FILE_NODE_MANAGER_PATH +
MonitorConstants.MONITOR_PATH_SEPARATOR
+ + statParam));
+ }
+ for (String statParam : fileSizeStatsHashMap.keySet()) {
+ assertTrue(mManager.pathExist(
+ MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME +
MonitorConstants.MONITOR_PATH_SEPARATOR
+ + statParam));
}
statMonitor.activate();
// wait for time second
@@ -95,16 +104,20 @@ public class MonitorTest {
// Get stat data and test right
Map<String, TSRecord> statHashMap = fManager.getAllStatisticsValue();
+ Map<String, TSRecord> fileSizeStatMap = fileSize.getAllStatisticsValue();
String path = fManager.getAllPathForStatistic().get(0);
+ String fileSizeStatPath = fileSize.getAllPathForStatistic().get(0);
int pos = path.lastIndexOf('.');
+ int fileSizeStatPos = fileSizeStatPath.lastIndexOf('.');
TSRecord fTSRecord = statHashMap.get(path.substring(0, pos));
+ TSRecord fileSizeRecord =
fileSizeStatMap.get(fileSizeStatPath.substring(0, fileSizeStatPos));
assertNotEquals(null, fTSRecord);
+ assertNotEquals(null, fileSizeRecord);
for (DataPoint dataPoint : fTSRecord.dataPointList) {
String m = dataPoint.getMeasurementId();
Long v = (Long) dataPoint.getValue();
-
if (m.equals("TOTAL_REQ_SUCCESS")) {
assertEquals(v, new Long(0));
}
@@ -116,6 +129,13 @@ public class MonitorTest {
assertEquals(v, new Long(0));
}
}
+ for (DataPoint dataPoint : fileSizeRecord.dataPointList) {
+ String m = dataPoint.getMeasurementId();
+ Long v = (Long) dataPoint.getValue();
+ if (m.equals(FileSizeConstants.OVERFLOW.name())) {
+ assertEquals(v, new Long(0));
+ }
+ }
try {
fManager.deleteAll();
diff --git
a/iotdb/src/test/java/org/apache/iotdb/db/monitor/collector/FileSizeTest.java
b/iotdb/src/test/java/org/apache/iotdb/db/monitor/collector/FileSizeTest.java
new file mode 100644
index 0000000..dc7a47c
--- /dev/null
+++
b/iotdb/src/test/java/org/apache/iotdb/db/monitor/collector/FileSizeTest.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.monitor.collector;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import org.apache.iotdb.db.monitor.MonitorConstants.FileSizeConstants;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FileSizeTest {
+
+ private static final String TEST_FILE_CONTENT = "FileSize UT test file";
+ private static final String TEST_FILE_PATH =
+ FileSizeConstants.DATA.getPath() + File.separatorChar + "testFile";
+
+ @Before
+ public void setUp() throws Exception {
+ EnvironmentUtils.envSetUp();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ }
+
+ @Test
+ public void testGetFileSizesInByte() {
+ long dataSizeBefore;
+ long dataSizeAfter;
+ boolean isWriteSuccess = true;
+ File testFile = new File(TEST_FILE_PATH);
+ if (testFile.exists()) {
+ try {
+ Files.delete(testFile.toPath());
+ } catch (IOException e) {
+ isWriteSuccess = false;
+ e.printStackTrace();
+ }
+ }
+ try {
+ if (!testFile.createNewFile()) {
+ isWriteSuccess = false;
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ dataSizeBefore =
FileSize.getInstance().getFileSizesInByte().get(FileSizeConstants.DATA);
+ byte[] contentInBytes = TEST_FILE_CONTENT.getBytes();
+ // write something into the test file under data dir
+ try (FileOutputStream fileOutputStream = new FileOutputStream(testFile)) {
+ fileOutputStream.write(contentInBytes);
+ fileOutputStream.flush();
+ } catch (IOException e) {
+ isWriteSuccess = false;
+ e.printStackTrace();
+ }
+ // calculate the delta of data dir file size
+ dataSizeAfter =
FileSize.getInstance().getFileSizesInByte().get(FileSizeConstants.DATA);
+ long deltaSize = dataSizeAfter - dataSizeBefore;
+
+ if (isWriteSuccess) {
+ //check if the the delta of data dir file size is equal to the written
content size in byte
+ assertEquals(contentInBytes.length, deltaSize);
+ } else {
+ assertEquals(0, deltaSize);
+ }
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/datapoint/DataPoint.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/datapoint/DataPoint.java
index dc13faf..8c7e633 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/datapoint/DataPoint.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/datapoint/DataPoint.java
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.iotdb.tsfile.write.record.datapoint;
import java.io.IOException;
@@ -86,10 +87,12 @@ public abstract class DataPoint {
dataPoint = new StringDataPoint(measurementId, new Binary(value));
break;
default:
- throw new UnSupportedDataTypeException(String.format("Data type %s
is not supported.", dataType));
+ throw new UnSupportedDataTypeException(
+ String.format("Data type %s is not supported.", dataType));
}
- } catch (Exception e){
- throw new UnSupportedDataTypeException(String.format("Data type of %s is
%s, but input value is %s", measurementId,
+ } catch (Exception e) {
+ throw new UnSupportedDataTypeException(
+ String.format("Data type of %s is %s, but input value is %s",
measurementId,
dataType, value));
}
