This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 17e9647 [IOTDB-200] Support creating TsFileWriter with config of
storage file system (#388)
17e9647 is described below
commit 17e9647e077603ab092d7871c409804e26810c5b
Author: Zesong Sun <[email protected]>
AuthorDate: Thu Sep 12 17:32:06 2019 +0800
[IOTDB-200] Support creating TsFileWriter with config of storage file
system (#388)
* Add TSFileFactory to create files according to FS in TsFile module
* Add getBufferedReader and getBufferedWriter in TSFileFactory
---
docs/Documentation/UserGuide/8-TsFile/2-Usage.md | 13 +
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 3 +-
.../iotdb/tsfile/TsFileWriteWithRowBatch.java | 4 +-
.../iotdb/tsfile/TsFileWriteWithTSRecord.java | 5 +-
server/pom.xml | 19 --
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 39 ++-
.../db/conf/directories/DirectoryManager.java | 9 +-
.../iotdb/db/engine/fileSystem/FileFactory.java | 20 +-
.../iotdb/db/engine/fileSystem/HdfsFile.java | 156 ---------
.../db/engine/merge/manage/MergeResource.java | 4 +-
.../db/engine/modification/ModificationFile.java | 5 +-
.../io/LocalTextModificationAccessor.java | 25 +-
.../engine/storagegroup/StorageGroupProcessor.java | 23 +-
.../db/engine/storagegroup/TsFileResource.java | 12 +-
.../org/apache/iotdb/db/utils/CommonUtils.java | 10 +-
.../writelog/recover/TsFileRecoverPerformer.java | 3 +-
.../db/engine/memtable/MemTableFlushTaskTest.java | 4 +-
.../db/engine/modification/DeletionQueryTest.java | 1 -
spark-tsfile/pom.xml | 14 +-
.../apache/iotdb/tsfile/io/TsFileRecordWriter.java | 1 +
.../org/apache/iotdb/tsfile/DefaultSource.scala | 2 +-
.../org/apache/iotdb/tsfile/NarrowConverter.scala | 2 +-
.../org/apache/iotdb/tsfile/WideConverter.scala | 2 +-
.../scala/org/apache/iotdb/tool/TsFileExample.java | 3 +-
.../org/apache/iotdb/tsfile/ConverterTest.scala | 2 +-
.../org/apache/iotdb/tsfile/HDFSInputTest.java | 2 +-
.../scala/org/apache/iotdb/tsfile/TSFileSuit.scala | 2 +-
tsfile/pom.xml | 18 +
.../iotdb/tsfile/common/conf/TSFileConfig.java | 43 ++-
.../org/apache/iotdb/tsfile/fileSystem/FSType.java | 24 ++
.../iotdb/tsfile/fileSystem/FileInputFactory.java | 51 +++
.../iotdb/tsfile/fileSystem/FileOutputFactory.java | 51 +++
.../apache/iotdb/tsfile/fileSystem/HDFSFile.java | 376 +++++++++++++++++++++
.../apache/iotdb/tsfile/fileSystem}/HDFSInput.java | 13 +-
.../iotdb/tsfile/fileSystem}/HDFSOutput.java | 22 +-
.../iotdb/tsfile/fileSystem/TSFileFactory.java | 104 ++++++
.../iotdb/tsfile/read/TsFileRestorableReader.java | 5 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 11 +-
.../apache/iotdb/tsfile/write/TsFileWriter.java | 3 +
.../write/writer/RestorableTsFileIOWriter.java | 3 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 25 +-
.../tsfile/read/TsFileRestorableReaderTest.java | 5 +-
.../org/apache/iotdb/tsfile/utils/FileUtils.java | 3 +-
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 15 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 33 +-
45 files changed, 864 insertions(+), 326 deletions(-)
diff --git a/docs/Documentation/UserGuide/8-TsFile/2-Usage.md
b/docs/Documentation/UserGuide/8-TsFile/2-Usage.md
index 329ba40..a07f092 100644
--- a/docs/Documentation/UserGuide/8-TsFile/2-Usage.md
+++ b/docs/Documentation/UserGuide/8-TsFile/2-Usage.md
@@ -92,11 +92,24 @@ A TsFile can be generated by following three steps and the
complete code will be
```
public TsFileWriter(TsFileOutput output, Schema schema) throws
IOException
```
+
+ If you want to set some TSFile configuration on your own, you could use
param `config`. For example:
+ ```
+ TSFileConfig conf = new TSFileConfig();
+ conf.setTSFileStorageFs("HDFS");
+ TsFileWriter tsFileWriter = new TsFileWriter(file, schema, conf);
+ ```
+ In this example, data files will be stored in HDFS, instead of local file
system. If you'd like to store data files in local file system, you can use
`conf.setTSFileStorageFs("LOCAL")`, which is also the default config.
+
+ You can also config the ip and port of your HDFS by
`config.setHdfsIp(...)` and `config.setHdfsPort(...)`. The default ip is
`localhost` and default port is `9000`.
+
**Parameters:**
* file : The TsFile to write
* schema : The file schemas, will be introduced in next part.
+
+ * config : The config of TsFile.
* Second, add measurements
diff --git
a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
index 363ab07..83c7f63 100644
---
a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
+++
b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
@@ -36,6 +36,7 @@ import
org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
@@ -48,7 +49,7 @@ public class TsFileSequenceRead {
filename = args[0];
}
TsFileSequenceReader reader = new TsFileSequenceReader(filename);
- System.out.println("file length: " + new File(filename).length());
+ System.out.println("file length: " +
TSFileFactory.INSTANCE.getFile(filename).length());
System.out.println("file magic head: " + reader.readHeadMagic());
System.out.println("file magic tail: " + reader.readTailMagic());
System.out.println("Level 1 metadata position: " +
reader.getFileMetadataPos());
diff --git
a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java
b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java
index bb9fc32..292c268 100644
---
a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java
+++
b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java
@@ -23,6 +23,8 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import java.io.File;
+
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -36,7 +38,7 @@ public class TsFileWriteWithRowBatch {
public static void main(String[] args) {
try {
String path = "test.tsfile";
- File f = new File(path);
+ File f = TSFileFactory.INSTANCE.getFile(path);
if (f.exists()) {
f.delete();
}
diff --git
a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithTSRecord.java
b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithTSRecord.java
index f664086..a12c8c9 100644
---
a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithTSRecord.java
+++
b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithTSRecord.java
@@ -22,11 +22,10 @@ package org.apache.iotdb.tsfile;
import java.io.File;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
-import org.apache.iotdb.tsfile.write.record.datapoint.FloatDataPoint;
-import org.apache.iotdb.tsfile.write.record.datapoint.IntDataPoint;
import org.apache.iotdb.tsfile.write.record.datapoint.LongDataPoint;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
/**
@@ -39,7 +38,7 @@ public class TsFileWriteWithTSRecord {
public static void main(String args[]) {
try {
String path = "test.tsfile";
- File f = new File(path);
+ File f = TSFileFactory.INSTANCE.getFile(path);
if (f.exists()) {
f.delete();
}
diff --git a/server/pom.xml b/server/pom.xml
index 1fab01f..9055d28 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -85,25 +85,6 @@
<artifactId>powermock-api-mockito2</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
</dependencies>
<build>
<plugins>
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index e464e5d..1b71c8e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.conf;
-import org.apache.iotdb.db.engine.fileSystem.FSType;
import java.io.File;
import java.time.ZoneId;
import java.util.ArrayList;
@@ -29,6 +28,8 @@ import java.util.regex.Pattern;
import org.apache.iotdb.db.engine.merge.selector.MergeFileStrategy;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.service.TSServiceImpl;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.fileSystem.FSType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -356,17 +357,21 @@ public class IoTDBConfig {
dirs.add(indexFileDir);
dirs.addAll(Arrays.asList(dataDirs));
- String homeDir = System.getProperty(IoTDBConstant.IOTDB_HOME, null);
- for (int i = 0; i < dirs.size(); i++) {
- String dir = dirs.get(i);
- if (!new File(dir).isAbsolute() && homeDir != null && homeDir.length() >
0) {
- if (!homeDir.endsWith(File.separator)) {
- dir = homeDir + File.separatorChar + dir;
- } else {
- dir = homeDir + dir;
- }
+ for (int i = 0; i < 4; i++) {
+ addHomeDir(dirs, i);
+ }
+
+ if (TSFileConfig.getTSFileStorageFs().equals(FSType.HDFS)) {
+ String hdfsDir = "hdfs://" + TSFileConfig.getHdfsIp() + ":" +
TSFileConfig.getHdfsPort();
+ for (int i = 5; i < dirs.size(); i++) {
+ String dir = dirs.get(i);
+ dir = hdfsDir + File.separatorChar + dir;
dirs.set(i, dir);
}
+ } else {
+ for (int i = 5; i < dirs.size(); i++) {
+ addHomeDir(dirs, i);
+ }
}
baseDir = dirs.get(0);
systemDir = dirs.get(1);
@@ -378,6 +383,18 @@ public class IoTDBConfig {
}
}
+ private void addHomeDir(List<String> dirs, int i) {
+ String dir = dirs.get(i);
+ String homeDir = System.getProperty(IoTDBConstant.IOTDB_HOME, null);
+ if (!new File(dir).isAbsolute() && homeDir != null && homeDir.length() >
0) {
+ if (!homeDir.endsWith(File.separator)) {
+ dir = homeDir + File.separatorChar + dir;
+ } else {
+ dir = homeDir + dir;
+ }
+ dirs.set(i, dir);
+ }
+ }
private void confirmMultiDirStrategy() {
if (getMultiDirStrategyClassName() == null) {
@@ -767,7 +784,7 @@ public class IoTDBConfig {
public void setMemtableSizeThreshold(long memtableSizeThreshold) {
this.memtableSizeThreshold = memtableSizeThreshold;
}
-
+
public MergeFileStrategy getMergeFileStrategy() {
return mergeFileStrategy;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryManager.java
b/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryManager.java
index 9b4b58d..be70416 100644
---
a/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryManager.java
+++
b/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryManager.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.strategy.DirectoryStrategy;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,14 +47,14 @@ public class DirectoryManager {
for (int i = 0; i < sequenceFileFolders.size(); i++) {
sequenceFileFolders.set(i, sequenceFileFolders.get(i) + File.separator +
"sequence");
}
- mkDirs(sequenceFileFolders);
+ mkDataDirs(sequenceFileFolders);
unsequenceFileFolders =
new
ArrayList<>(Arrays.asList(IoTDBDescriptor.getInstance().getConfig().getDataDirs()));
for (int i = 0; i < unsequenceFileFolders.size(); i++) {
unsequenceFileFolders.set(i, unsequenceFileFolders.get(i) +
File.separator + "unsequence");
}
- mkDirs(unsequenceFileFolders);
+ mkDataDirs(unsequenceFileFolders);
String strategyName = "";
try {
@@ -72,9 +73,9 @@ public class DirectoryManager {
return DirectoriesHolder.INSTANCE;
}
- private void mkDirs(List<String> folders) {
+ private void mkDataDirs(List<String> folders) {
for (String folder : folders) {
- File file = new File(folder);
+ File file = TSFileFactory.INSTANCE.getFile(folder);
if (file.mkdirs()) {
logger.info("folder {} doesn't exist, create it", file.getPath());
} else {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FileFactory.java
b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FileFactory.java
index 5240f8c..32e8121 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FileFactory.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/FileFactory.java
@@ -20,6 +20,8 @@
package org.apache.iotdb.db.engine.fileSystem;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.tsfile.fileSystem.FSType;
+import org.apache.iotdb.tsfile.fileSystem.HDFSFile;
import java.io.File;
import java.net.URI;
@@ -28,35 +30,35 @@ public enum FileFactory {
INSTANCE;
- private static FSType FSType =
IoTDBDescriptor.getInstance().getConfig().getStorageFs();
+ private static FSType fsType =
IoTDBDescriptor.getInstance().getConfig().getStorageFs();
public File getFile(String pathname) {
- if (FSType.equals(FSType.HDFS)) {
- return new HdfsFile(pathname);
+ if (fsType.equals(FSType.HDFS)) {
+ return new HDFSFile(pathname);
} else {
return new File(pathname);
}
}
public File getFile(String parent, String child) {
- if (FSType.equals(FSType.HDFS)) {
- return new HdfsFile(parent, child);
+ if (fsType.equals(FSType.HDFS)) {
+ return new HDFSFile(parent, child);
} else {
return new File(parent, child);
}
}
public File getFile(File parent, String child) {
- if (FSType.equals(FSType.HDFS)) {
- return new HdfsFile(parent, child);
+ if (fsType.equals(FSType.HDFS)) {
+ return new HDFSFile(parent, child);
} else {
return new File(parent, child);
}
}
public File getFile(URI uri) {
- if (FSType.equals(FSType.HDFS)) {
- return new HdfsFile(uri);
+ if (fsType.equals(FSType.HDFS)) {
+ return new HDFSFile(uri);
} else {
return new File(uri);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/HdfsFile.java
b/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/HdfsFile.java
deleted file mode 100644
index 3378954..0000000
--- a/server/src/main/java/org/apache/iotdb/db/engine/fileSystem/HdfsFile.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.fileSystem;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.*;
-import org.apache.iotdb.tsfile.write.TsFileWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-
-public class HdfsFile extends File {
-
- private Path hdfsPath;
-
- private static final Logger logger =
LoggerFactory.getLogger(TsFileWriter.class);
-
-
- public HdfsFile(String pathname) {
- super(pathname);
- hdfsPath = new Path(pathname);
- }
-
- public HdfsFile(String parent, String child) {
- super(parent, child);
- }
-
- public HdfsFile(File parent, String child) {
- super(parent, child);
- }
-
- public HdfsFile(URI uri) {
- super(uri);
- }
-
- @Override
- public String getAbsolutePath() {
- return hdfsPath.toUri().toString();
- }
-
- @Override
- public long length() {
- try {
- FileSystem fs = hdfsPath.getFileSystem(new Configuration());
- return fs.getFileStatus(hdfsPath).getLen();
- } catch (IOException e) {
- logger.error("Fail to get length of the file. ", e);
- return 0;
- }
- }
-
- @Override
- public boolean exists() {
- try {
- FileSystem fs = hdfsPath.getFileSystem(new Configuration());
- return fs.exists(hdfsPath);
- } catch (IOException e) {
- logger.error("Fail to check whether the file or directory exists. ", e);
- return false;
- }
- }
-
- @Override
- public File[] listFiles() {
- ArrayList<HdfsFile> files = new ArrayList<>();
- try {
- FileSystem fs = hdfsPath.getFileSystem(new Configuration());
- RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(hdfsPath,
true);
- while (iterator.hasNext()) {
- LocatedFileStatus fileStatus = iterator.next();
- Path fullPath = fileStatus.getPath();
- files.add(new HdfsFile(fullPath.toUri()));
- }
- return files.toArray(new HdfsFile[files.size()]);
- } catch (IOException e) {
- logger.error("Fail to list files. ", e);
- return null;
- }
- }
-
- @Override
- public File[] listFiles(FileFilter filter) {
- ArrayList<HdfsFile> files = new ArrayList<>();
- try {
- PathFilter pathFilter = new GlobFilter(filter.toString()); // TODO
- FileSystem fs = hdfsPath.getFileSystem(new Configuration());
- RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(hdfsPath,
true);
- while (iterator.hasNext()) {
- LocatedFileStatus fileStatus = iterator.next();
- Path fullPath = fileStatus.getPath();
- if (pathFilter.accept(fullPath)) {
- files.add(new HdfsFile(fullPath.toUri()));
- }
- }
- return files.toArray(new HdfsFile[files.size()]);
- } catch (IOException e) {
- logger.error("Fail to list files. ", e);
- return null;
- }
- }
-
- @Override
- public File getParentFile() {
- return new HdfsFile(hdfsPath.getParent().toUri());
- }
-
- @Override
- public boolean createNewFile() throws IOException {
- FileSystem fs = hdfsPath.getFileSystem(new Configuration());
- return fs.createNewFile(hdfsPath);
- }
-
- @Override
- public boolean delete() {
- try {
- FileSystem fs = hdfsPath.getFileSystem(new Configuration());
- return fs.delete(hdfsPath, true);
- } catch (IOException e) {
- logger.error("Fail to delete file. ", e);
- return false;
- }
- }
-
- @Override
- public boolean mkdirs() {
- try {
- FileSystem fs = hdfsPath.getFileSystem(new Configuration());
- return fs.mkdirs(hdfsPath);
- } catch (IOException e) {
- logger.error("Fail to create directory. ", e);
- return false;
- }
- }
-}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
index d8b6c4c..3a5262d 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
@@ -27,7 +27,7 @@ import org.apache.iotdb.db.utils.MergeUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -98,7 +98,7 @@ public class MergeResource {
public RestorableTsFileIOWriter getMergeFileWriter(TsFileResource resource)
throws IOException {
RestorableTsFileIOWriter writer = fileWriterCache.get(resource);
if (writer == null) {
- writer = new RestorableTsFileIOWriter(FileFactory.INSTANCE
+ writer = new RestorableTsFileIOWriter(TSFileFactory.INSTANCE
.getFile(resource.getFile().getPath() + MERGE_SUFFIX));
fileWriterCache.put(resource, writer);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java
b/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java
index 26438a8..12316a2 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java
@@ -19,14 +19,15 @@
package org.apache.iotdb.db.engine.modification;
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+
import
org.apache.iotdb.db.engine.modification.io.LocalTextModificationAccessor;
import org.apache.iotdb.db.engine.modification.io.ModificationReader;
import org.apache.iotdb.db.engine.modification.io.ModificationWriter;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
/**
* ModificationFile stores the Modifications of a TsFile or unseq file in
another file in the same
@@ -121,7 +122,7 @@ public class ModificationFile {
public void remove() throws IOException {
close();
- new File(filePath).delete();
+ TSFileFactory.INSTANCE.getFile(filePath).delete();
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/modification/io/LocalTextModificationAccessor.java
b/server/src/main/java/org/apache/iotdb/db/engine/modification/io/LocalTextModificationAccessor.java
index 01f151e..274e8a6 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/modification/io/LocalTextModificationAccessor.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/modification/io/LocalTextModificationAccessor.java
@@ -19,20 +19,19 @@
package org.apache.iotdb.db.engine.modification.io;
+import org.apache.iotdb.db.engine.modification.Deletion;
+import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.BufferedReader;
import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import org.apache.iotdb.db.engine.modification.Deletion;
-import org.apache.iotdb.db.engine.modification.Modification;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* LocalTextModificationAccessor uses a file on local file system to store the
modifications
@@ -58,14 +57,14 @@ public class LocalTextModificationAccessor implements
ModificationReader, Modifi
@Override
public Collection<Modification> read() {
- if (!new File(filePath).exists()) {
+ if (!TSFileFactory.INSTANCE.getFile(filePath).exists()) {
logger.debug("No modification has been written to this file");
return new ArrayList<>();
}
String line;
List<Modification> modificationList = new ArrayList<>();
- try (BufferedReader reader = new BufferedReader(new FileReader(filePath))){
+ try(BufferedReader reader =
TSFileFactory.INSTANCE.getBufferedReader(filePath)) {
while ((line = reader.readLine()) != null) {
if (line.equals(ABORT_MARK) && !modificationList.isEmpty()) {
modificationList.remove(modificationList.size() - 1);
@@ -75,7 +74,7 @@ public class LocalTextModificationAccessor implements
ModificationReader, Modifi
}
} catch (IOException e) {
logger.error("An error occurred when reading modifications, and the
remaining modifications "
- + "were ignored.", e);
+ + "were ignored.", e);
}
return modificationList;
}
@@ -91,7 +90,7 @@ public class LocalTextModificationAccessor implements
ModificationReader, Modifi
@Override
public void abort() throws IOException {
if (writer == null) {
- writer = new BufferedWriter(new FileWriter(filePath, true));
+ writer = TSFileFactory.INSTANCE.getBufferedWriter(filePath, true);
}
writer.write(ABORT_MARK);
writer.newLine();
@@ -101,7 +100,7 @@ public class LocalTextModificationAccessor implements
ModificationReader, Modifi
@Override
public void write(Modification mod) throws IOException {
if (writer == null) {
- writer = new BufferedWriter(new FileWriter(filePath, true));
+ writer = TSFileFactory.INSTANCE.getBufferedWriter(filePath, true);
}
writer.write(encodeModification(mod));
writer.newLine();
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 389de7e..8ceae5b 100755
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -24,7 +24,6 @@ import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFF
import java.io.File;
import java.io.IOException;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -75,6 +74,7 @@ import
org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.Schema;
@@ -185,7 +185,7 @@ public class StorageGroupProcessor {
this.schema = constructSchema(storageGroupName);
try {
- storageGroupSysDir = new File(systemInfoDir, storageGroupName);
+ storageGroupSysDir = FileFactory.INSTANCE.getFile(systemInfoDir,
storageGroupName);
if (storageGroupSysDir.mkdirs()) {
logger.info("Storage Group system Directory {} doesn't exist, create
it",
storageGroupSysDir.getPath());
@@ -215,7 +215,7 @@ public class StorageGroupProcessor {
recoverUnseqFiles(unseqTsFiles);
String taskName = storageGroupName + "-" + System.currentTimeMillis();
- File mergingMods = new File(storageGroupSysDir,
MERGING_MODIFICAITON_FILE_NAME);
+ File mergingMods = FileFactory.INSTANCE.getFile(storageGroupSysDir,
MERGING_MODIFICAITON_FILE_NAME);
if (mergingMods.exists()) {
mergingModification = new ModificationFile(mergingMods.getPath());
}
@@ -240,7 +240,7 @@ public class StorageGroupProcessor {
private List<TsFileResource> getAllFiles(List<String> folders) throws
IOException {
List<File> tsFiles = new ArrayList<>();
for (String baseDir : folders) {
- File fileFolder = FileFactory.INSTANCE.getFile(baseDir,
storageGroupName);
+ File fileFolder = TSFileFactory.INSTANCE.getFile(baseDir,
storageGroupName);
if (!fileFolder.exists()) {
continue;
}
@@ -265,7 +265,7 @@ public class StorageGroupProcessor {
File[] files = fileFolder.listFiles(file ->
file.getName().endsWith(suffix));
if (files != null) {
for (File tempResource : files) {
- File originResource = new File(tempResource.getPath().replace(suffix,
""));
+ File originResource =
TSFileFactory.INSTANCE.getFile(tempResource.getPath().replace(suffix, ""));
if (originResource.exists()) {
tempResource.delete();
} else {
@@ -488,18 +488,17 @@ public class StorageGroupProcessor {
} else {
baseDir =
DirectoryManager.getInstance().getNextFolderForUnSequenceFile();
}
- new File(baseDir, storageGroupName).mkdirs();
+ TSFileFactory.INSTANCE.getFile(baseDir, storageGroupName).mkdirs();
- String filePath = Paths.get(baseDir, storageGroupName,
- System.currentTimeMillis() + "-" +
versionController.nextVersion()).toString()
- + TSFILE_SUFFIX;
+ String filePath = baseDir + File.separator + storageGroupName +
File.separator +
+ System.currentTimeMillis() + "-" + versionController.nextVersion() +
TSFILE_SUFFIX;
if (sequence) {
- return new TsFileProcessor(storageGroupName,
FileFactory.INSTANCE.getFile(filePath),
+ return new TsFileProcessor(storageGroupName,
TSFileFactory.INSTANCE.getFile(filePath),
schema, versionController, this::closeUnsealedTsFileProcessor,
this::updateLatestFlushTimeCallback, sequence);
} else {
- return new TsFileProcessor(storageGroupName,
FileFactory.INSTANCE.getFile(filePath),
+ return new TsFileProcessor(storageGroupName,
TSFileFactory.INSTANCE.getFile(filePath),
schema, versionController, this::closeUnsealedTsFileProcessor,
() -> true, sequence);
}
@@ -537,7 +536,7 @@ public class StorageGroupProcessor {
List<String> folder =
DirectoryManager.getInstance().getAllSequenceFileFolders();
folder.addAll(DirectoryManager.getInstance().getAllUnSequenceFileFolders());
for (String tsfilePath : folder) {
- File storageGroupFolder = new File(tsfilePath, storageGroupName);
+ File storageGroupFolder = TSFileFactory.INSTANCE.getFile(tsfilePath,
storageGroupName);
if (storageGroupFolder.exists()) {
try {
FileUtils.deleteDirectory(storageGroupFolder);
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 6dfdf57..1e614e3 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -30,7 +30,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
public class TsFileResource {
@@ -118,8 +118,8 @@ public class TsFileResource {
ReadWriteIOUtils.write(entry.getValue(), outputStream);
}
}
- File src = FileFactory.INSTANCE.getFile(file + RESOURCE_SUFFIX +
TEMP_SUFFIX);
- File dest = FileFactory.INSTANCE.getFile(file + RESOURCE_SUFFIX);
+ File src = TSFileFactory.INSTANCE.getFile(file + RESOURCE_SUFFIX +
TEMP_SUFFIX);
+ File dest = TSFileFactory.INSTANCE.getFile(file + RESOURCE_SUFFIX);
dest.delete();
FileUtils.moveFile(src, dest);
}
@@ -161,7 +161,7 @@ public class TsFileResource {
}
public boolean fileExists() {
- return FileFactory.INSTANCE.getFile(file + RESOURCE_SUFFIX).exists();
+ return TSFileFactory.INSTANCE.getFile(file + RESOURCE_SUFFIX).exists();
}
public void forceUpdateEndTime(String device, long time) {
@@ -236,8 +236,8 @@ public class TsFileResource {
public void remove() {
file.delete();
- FileFactory.INSTANCE.getFile(file.getPath() + RESOURCE_SUFFIX).delete();
- FileFactory.INSTANCE.getFile(file.getPath() +
ModificationFile.FILE_SUFFIX).delete();
+ TSFileFactory.INSTANCE.getFile(file.getPath() + RESOURCE_SUFFIX).delete();
+ TSFileFactory.INSTANCE.getFile(file.getPath() +
ModificationFile.FILE_SUFFIX).delete();
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
b/server/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
index 1c40886..36f10fb 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
@@ -18,7 +18,7 @@
*/
package org.apache.iotdb.db.utils;
-import java.io.File;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -42,8 +42,14 @@ public class CommonUtils {
}
}
+ /**
+ * NOTICE: This method is currently used only for data dir, thus using
TSFileFactory to get file
+ *
+ * @param dir directory path
+ * @return
+ */
public static long getUsableSpace(String dir) {
- return new File(dir).getFreeSpace();
+ return TSFileFactory.INSTANCE.getFile(dir).getFreeSpace();
}
public static boolean hasSpace(String dir) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index 3039195..2fa3ec1 100644
---
a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++
b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
@@ -82,7 +83,7 @@ public class TsFileRecoverPerformer {
this.logReplayer = new LogReplayer(logNodePrefix, insertFilePath,
tsFileResource.getModFile(),
versionController,
tsFileResource, schema, recoverMemTable, acceptUnseq);
- File insertFile = FileFactory.INSTANCE.getFile(insertFilePath);
+ File insertFile = TSFileFactory.INSTANCE.getFile(insertFilePath);
if (!insertFile.exists()) {
logger.error("TsFile {} is missing, will skip its recovery.",
insertFilePath);
return;
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
index 5586144..852be52 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
@@ -26,7 +26,7 @@ import org.apache.iotdb.db.engine.MetadataManagerHelper;
import org.apache.iotdb.db.engine.flush.MemTableFlushTask;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.db.engine.fileSystem.FileFactory;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import org.junit.After;
import org.junit.Before;
@@ -45,7 +45,7 @@ public class MemTableFlushTaskTest {
public void setUp() throws Exception {
MetadataManagerHelper.initMetadata();
EnvironmentUtils.envSetUp();
- writer = new
RestorableTsFileIOWriter(FileFactory.INSTANCE.getFile(filePath));
+ writer = new
RestorableTsFileIOWriter(TSFileFactory.INSTANCE.getFile(filePath));
memTable = new PrimitiveMemTable();
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
index 3eb681a..ee15ca9 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
@@ -188,7 +188,6 @@ public class DeletionQueryTest {
while (dataSet.hasNext()) {
RowRecord record = dataSet.next();
count++;
- System.out.println(record.getTimestamp());
}
assertEquals(150, count);
}
diff --git a/spark-tsfile/pom.xml b/spark-tsfile/pom.xml
index 4613de1..1a922bd 100644
--- a/spark-tsfile/pom.xml
+++ b/spark-tsfile/pom.xml
@@ -37,13 +37,19 @@
<version>0.9.0-SNAPSHOT</version>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
diff --git
a/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileRecordWriter.java
b/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileRecordWriter.java
index b9806c2..9a28013 100644
---
a/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileRecordWriter.java
+++
b/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileRecordWriter.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.fileSystem.HDFSOutput;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.schema.Schema;
diff --git
a/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/DefaultSource.scala
b/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/DefaultSource.scala
index fa1418d..37868a4 100755
--- a/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/DefaultSource.scala
+++ b/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/DefaultSource.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.fs.{FileStatus, Path}
import org.apache.hadoop.mapreduce.Job
import org.apache.iotdb.tsfile.DefaultSource.SerializableConfiguration
import org.apache.iotdb.tsfile.common.constant.QueryConstant
-import org.apache.iotdb.tsfile.io.HDFSInput
+import org.apache.iotdb.tsfile.fileSystem.HDFSInput
import org.apache.iotdb.tsfile.qp.Executor
import org.apache.iotdb.tsfile.read.common.Field
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet
diff --git
a/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/NarrowConverter.scala
b/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/NarrowConverter.scala
index 28c1b1f..3284866 100644
--- a/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/NarrowConverter.scala
+++ b/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/NarrowConverter.scala
@@ -25,7 +25,7 @@ import org.apache.hadoop.fs.FileStatus
import org.apache.iotdb.tsfile.common.constant.QueryConstant
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData
import org.apache.iotdb.tsfile.file.metadata.enums.{TSDataType, TSEncoding}
-import org.apache.iotdb.tsfile.io.HDFSInput
+import org.apache.iotdb.tsfile.fileSystem.HDFSInput
import org.apache.iotdb.tsfile.qp.QueryProcessor
import org.apache.iotdb.tsfile.qp.common.{BasicOperator, FilterOperator,
SQLConstant, TSQueryPlan}
import org.apache.iotdb.tsfile.read.TsFileSequenceReader
diff --git
a/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/WideConverter.scala
b/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/WideConverter.scala
index 748c3e6..66c06d1 100755
--- a/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/WideConverter.scala
+++ b/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/WideConverter.scala
@@ -25,7 +25,7 @@ import org.apache.hadoop.fs.FileStatus
import org.apache.iotdb.tsfile.common.constant.QueryConstant
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData
import org.apache.iotdb.tsfile.file.metadata.enums.{TSDataType, TSEncoding}
-import org.apache.iotdb.tsfile.io.HDFSInput
+import org.apache.iotdb.tsfile.fileSystem.HDFSInput
import org.apache.iotdb.tsfile.read.TsFileSequenceReader
import org.apache.iotdb.tsfile.read.common.Path
import org.apache.iotdb.tsfile.read.expression.impl.{BinaryExpression,
GlobalTimeExpression, SingleSeriesExpression}
diff --git
a/spark-tsfile/src/test/scala/org/apache/iotdb/tool/TsFileExample.java
b/spark-tsfile/src/test/scala/org/apache/iotdb/tool/TsFileExample.java
index 1da7b77..ce8d0a4 100644
--- a/spark-tsfile/src/test/scala/org/apache/iotdb/tool/TsFileExample.java
+++ b/spark-tsfile/src/test/scala/org/apache/iotdb/tool/TsFileExample.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.tool;
import java.io.File;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
@@ -36,7 +37,7 @@ import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
public class TsFileExample {
public static void create(String tsfilePath) throws Exception {
- File f = new File(tsfilePath);
+ File f = TSFileFactory.INSTANCE.getFile(tsfilePath);
if (f.exists()) {
f.delete();
}
diff --git
a/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/ConverterTest.scala
b/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/ConverterTest.scala
index b0f3499..c881360 100644
--- a/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/ConverterTest.scala
+++ b/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/ConverterTest.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
import org.apache.iotdb.tool.TsFileWriteTool
import org.apache.iotdb.tsfile.common.constant.QueryConstant
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType
-import org.apache.iotdb.tsfile.io.HDFSInput
+import org.apache.iotdb.tsfile.fileSystem.HDFSInput
import org.apache.iotdb.tsfile.read.TsFileSequenceReader
import org.apache.iotdb.tsfile.read.common.Field
import org.apache.iotdb.tsfile.utils.Binary
diff --git
a/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java
b/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java
index 9aab7f8..c27e00a 100644
--- a/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java
+++ b/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java
@@ -22,7 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.iotdb.tool.TsFileWriteTool;
-import org.apache.iotdb.tsfile.io.HDFSInput;
+import org.apache.iotdb.tsfile.fileSystem.HDFSInput;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
diff --git
a/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/TSFileSuit.scala
b/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/TSFileSuit.scala
index 8aa1a8c..f86f5d4 100644
--- a/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/TSFileSuit.scala
+++ b/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/TSFileSuit.scala
@@ -25,7 +25,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.iotdb.tool.TsFileWriteTool
import org.apache.iotdb.tsfile.common.constant.QueryConstant
-import org.apache.iotdb.tsfile.io.HDFSInput
+import org.apache.iotdb.tsfile.fileSystem.HDFSInput
import org.apache.iotdb.tsfile.read.TsFileSequenceReader
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types._
diff --git a/tsfile/pom.xml b/tsfile/pom.xml
index cab5fb5..8991eac 100644
--- a/tsfile/pom.xml
+++ b/tsfile/pom.xml
@@ -53,6 +53,24 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 647b610..79b6622 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -18,6 +18,8 @@
*/
package org.apache.iotdb.tsfile.common.conf;
+import org.apache.iotdb.tsfile.fileSystem.FSType;
+
/**
* TSFileConfig is a configure class. Every variables is public and has
default value.
*
@@ -140,9 +142,46 @@ public class TSFileConfig {
public static String endian = "BIG_ENDIAN";
/**
- * only can be used by TsFileDescriptor.
+ * Default storage is in local file system
+ */
+ public static FSType TSFileStorageFs = FSType.LOCAL;
+
+ /**
+ * Default hdfs ip is localhost
*/
- protected TSFileConfig() {
+ public static String hdfsIp = "localhost";
+
+ /**
+ * Default hdfs port is 9000
+ */
+ public static String hdfsPort = "9000";
+
+ public TSFileConfig() {
+
+ }
+
+
+ public static FSType getTSFileStorageFs() {
+ return TSFileStorageFs;
+ }
+
+ public static void setTSFileStorageFs(String TSFileStorageFs) {
+ TSFileConfig.TSFileStorageFs = FSType.valueOf(TSFileStorageFs);
+ }
+
+ public static String getHdfsIp() {
+ return hdfsIp;
+ }
+
+ public static void setHdfsIp(String hdfsIp) {
+ TSFileConfig.hdfsIp = hdfsIp;
+ }
+
+ public static String getHdfsPort() {
+ return hdfsPort;
+ }
+ public static void setHdfsPort(String hdfsPort) {
+ TSFileConfig.hdfsPort = hdfsPort;
}
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/FSType.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/FSType.java
new file mode 100644
index 0000000..0dda437
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/FSType.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.fileSystem;
+
+public enum FSType {
+ LOCAL, HDFS
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/FileInputFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/FileInputFactory.java
new file mode 100644
index 0000000..edb2858
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/FileInputFactory.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.fileSystem;
+
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.read.reader.DefaultTsFileInput;
+import org.apache.iotdb.tsfile.read.reader.TsFileInput;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public enum FileInputFactory {
+
+ INSTANCE;
+
+ private static FSType fsType =
TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs();
+ private static final Logger logger =
LoggerFactory.getLogger(TsFileIOWriter.class);
+
+ public TsFileInput getTsFileInput(String filePath) {
+ try {
+ if (fsType.equals(FSType.HDFS)) {
+ return new HDFSInput(filePath);
+ } else {
+ return new DefaultTsFileInput(Paths.get(filePath));
+ }
+ } catch (IOException e) {
+ logger.error("Failed to get TsFile input of file: {}, ", filePath, e);
+ return null;
+ }
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/FileOutputFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/FileOutputFactory.java
new file mode 100644
index 0000000..0853d20
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/FileOutputFactory.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.fileSystem;
+
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.write.writer.DefaultTsFileOutput;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
+import org.apache.iotdb.tsfile.write.writer.TsFileOutput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+public enum FileOutputFactory {
+
+ INSTANCE;
+
+ private static FSType fsType =
TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs();
+ private static final Logger logger =
LoggerFactory.getLogger(TsFileIOWriter.class);
+
+ public TsFileOutput getTsFileOutput(String filePath, boolean append) {
+ try {
+ if (fsType.equals(FSType.HDFS)) {
+ return new HDFSOutput(filePath, append);
+ } else {
+ return new DefaultTsFileOutput(new FileOutputStream(filePath, append));
+ }
+ } catch (IOException e) {
+ logger.error("Failed to get TsFile Output: ", e);
+ return null;
+ }
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/HDFSFile.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/HDFSFile.java
new file mode 100644
index 0000000..599365d
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/HDFSFile.java
@@ -0,0 +1,376 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.fileSystem;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.*;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URL;
+import java.util.ArrayList;
+
+public class HDFSFile extends File {
+
+ private Path hdfsPath;
+ private FileSystem fs;
+ private static final Logger logger =
LoggerFactory.getLogger(TsFileWriter.class);
+
+
+ public HDFSFile(String pathname) {
+ super(pathname);
+ hdfsPath = new Path(pathname);
+ setConfAndGetFS();
+ }
+
+ public HDFSFile(String parent, String child) {
+ super(parent, child);
+ hdfsPath = new Path(parent + child);
+ setConfAndGetFS();
+ }
+
+ public HDFSFile(File parent, String child) {
+ super(parent, child);
+ hdfsPath = new Path(parent.getAbsolutePath() + child);
+ setConfAndGetFS();
+ }
+
+ public HDFSFile(URI uri) {
+ super(uri);
+ hdfsPath = new Path(uri);
+ setConfAndGetFS();
+ }
+
+ private void setConfAndGetFS() {
+ Configuration conf = new Configuration();
+ conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
+ try {
+ fs = hdfsPath.getFileSystem(conf);
+ } catch (IOException e) {
+ logger.error("Fail to get HDFS! ", e);
+ }
+ }
+
+ @Override
+ public String getAbsolutePath() {
+ return hdfsPath.toUri().toString();
+ }
+
+ @Override
+ public String getPath() {
+ return hdfsPath.toUri().toString();
+ }
+
+ @Override
+ public long length() {
+ try {
+ return fs.getFileStatus(hdfsPath).getLen();
+ } catch (IOException e) {
+ logger.error("Fail to get length of the file {}, ",
hdfsPath.toUri().toString(), e);
+ return 0;
+ }
+ }
+
+ @Override
+ public boolean exists() {
+ try {
+ return fs.exists(hdfsPath);
+ } catch (IOException e) {
+ logger.error("Fail to check whether the file {} exists. ",
hdfsPath.toUri().toString(), e);
+ return false;
+ }
+ }
+
+ @Override
+ public File[] listFiles() {
+ ArrayList<HDFSFile> files = new ArrayList<>();
+ try {
+ RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(hdfsPath,
true);
+ while (iterator.hasNext()) {
+ LocatedFileStatus fileStatus = iterator.next();
+ Path fullPath = fileStatus.getPath();
+ files.add(new HDFSFile(fullPath.toUri()));
+ }
+ return files.toArray(new HDFSFile[files.size()]);
+ } catch (IOException e) {
+ logger.error("Fail to list files in {}. ", hdfsPath.toUri().toString(),
e);
+ return null;
+ }
+ }
+
+ @Override
+ public File[] listFiles(FileFilter filter) {
+ ArrayList<HDFSFile> files = new ArrayList<>();
+ try {
+ PathFilter pathFilter = new GlobFilter(filter.toString()); // TODO test
this filter in the future
+ RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(hdfsPath,
true);
+ while (iterator.hasNext()) {
+ LocatedFileStatus fileStatus = iterator.next();
+ Path fullPath = fileStatus.getPath();
+ if (pathFilter.accept(fullPath)) {
+ files.add(new HDFSFile(fullPath.toUri()));
+ }
+ }
+ return files.toArray(new HDFSFile[files.size()]);
+ } catch (IOException e) {
+ logger.error("Fail to list files in {}. ", hdfsPath.toUri().toString(),
e);
+ return null;
+ }
+ }
+
+ @Override
+ public File getParentFile() {
+ return new HDFSFile(hdfsPath.getParent().toUri());
+ }
+
+ @Override
+ public boolean createNewFile() throws IOException {
+ return fs.createNewFile(hdfsPath);
+ }
+
+ @Override
+ public boolean delete() {
+ try {
+ return fs.delete(hdfsPath, true);
+ } catch (IOException e) {
+ logger.error("Fail to delete file {}. ", hdfsPath.toUri().toString(), e);
+ return false;
+ }
+ }
+
+ @Override
+ public boolean mkdirs() {
+ try {
+ return fs.mkdirs(hdfsPath);
+ } catch (IOException e) {
+ logger.error("Fail to create directory {}. ",
hdfsPath.toUri().toString(), e);
+ return false;
+ }
+ }
+
+ @Override
+ public boolean isDirectory() {
+ try {
+ return fs.getFileStatus(hdfsPath).isDirectory();
+ } catch (IOException e) {
+ logger.error("Fail to judge whether {} is a directory. ",
hdfsPath.toUri().toString(), e);
+ return false;
+ }
+ }
+
+ @Override
+ public long getFreeSpace() {
+ try {
+ return fs.getStatus().getRemaining();
+ } catch (IOException e) {
+ logger.error("Fail to get free space of {}. ",
hdfsPath.toUri().toString(), e);
+ return 0L;
+ }
+ }
+
+ @Override
+ public String getName() {
+ return hdfsPath.getName();
+ }
+
+ @Override
+ public String toString() {
+ return hdfsPath.toUri().toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return hdfsPath.hashCode();
+ }
+
+ @Override
+ public int compareTo(File pathname) {
+ if(pathname instanceof HDFSFile) {
+ return hdfsPath.toUri().toString().compareTo(pathname.getPath());
+ } else {
+ logger.error("File {} is not HDFS file. ", pathname.getPath());
+ throw new IllegalArgumentException("Compare file is not HDFS file.");
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if ((obj != null) && (obj instanceof HDFSFile)) {
+ return compareTo((HDFSFile)obj) == 0;
+ }
+ return false;
+ }
+
+
+ @Override
+ public String getParent() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean isAbsolute() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public File getAbsoluteFile() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public String getCanonicalPath() throws IOException {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public File getCanonicalFile() throws IOException {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public URL toURL() throws MalformedURLException {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public URI toURI() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean canRead() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean canWrite() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean isFile() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean isHidden() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public long lastModified() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public void deleteOnExit() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public String[] list() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public String[] list(FilenameFilter filter) {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public File[] listFiles(FilenameFilter filter) {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean mkdir() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean renameTo(File dest) {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean setLastModified(long time) {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean setReadOnly() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean setWritable(boolean writable, boolean ownerOnly) {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean setWritable(boolean writable) {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean setReadable(boolean readable, boolean ownerOnly) {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean setReadable(boolean readable) {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean setExecutable(boolean executable, boolean ownerOnly) {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean setExecutable(boolean executable) {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public boolean canExecute() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public long getTotalSpace() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public long getUsableSpace() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+
+ @Override
+ public java.nio.file.Path toPath() {
+ throw new UnsupportedOperationException("Unsupported operation.");
+ }
+}
diff --git
a/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/HDFSInput.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/HDFSInput.java
similarity index 93%
rename from spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/HDFSInput.java
rename to tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/HDFSInput.java
index fc14fad..ddcc948 100644
--- a/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/HDFSInput.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/HDFSInput.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.tsfile.io;
+package org.apache.iotdb.tsfile.fileSystem;
import java.io.IOException;
import java.io.InputStream;
@@ -99,16 +99,7 @@ public class HDFSInput implements TsFileInput {
long srcPosition = fsDataInputStream.getPos();
fsDataInputStream.seek(position);
-
- int res;
- if (byteBufferReadable) {
- res = fsDataInputStream.read(dst);
- } else {
- byte[] bytes = new byte[dst.remaining()];
- res = fsDataInputStream.read(bytes);
- dst.put(bytes);
- }
-
+ int res = read(dst);
fsDataInputStream.seek(srcPosition);
return res;
diff --git
a/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/HDFSOutput.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/HDFSOutput.java
similarity index 83%
rename from
spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/HDFSOutput.java
rename to
tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/HDFSOutput.java
index 9a18e4a..7244758 100644
--- a/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/HDFSOutput.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/HDFSOutput.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.tsfile.io;
+package org.apache.iotdb.tsfile.fileSystem;
import java.io.IOException;
import java.io.OutputStream;
@@ -35,44 +35,44 @@ import org.apache.iotdb.tsfile.write.writer.TsFileOutput;
public class HDFSOutput implements TsFileOutput {
private FSDataOutputStream fsDataOutputStream;
+ private FileSystem fs;
+ private Path path;
- public HDFSOutput(String filePath, boolean overwriter) throws IOException {
-
- this(filePath, new Configuration(), overwriter);
+ public HDFSOutput(String filePath, boolean overwrite) throws IOException {
+ this(filePath, new Configuration(), overwrite);
+ path = new Path(filePath);
}
public HDFSOutput(String filePath, Configuration configuration, boolean
overwriter)
throws IOException {
-
this(new Path(filePath), configuration, overwriter);
+ path = new Path(filePath);
}
public HDFSOutput(Path path, Configuration configuration, boolean overwriter)
throws IOException {
- FileSystem fs = path.getFileSystem(configuration);
+ fs = path.getFileSystem(configuration);
fsDataOutputStream = fs.create(path, overwriter);
+ this.path = path;
}
@Override
public void write(byte[] b) throws IOException {
-
fsDataOutputStream.write(b);
}
public void write(ByteBuffer b) throws IOException {
- throw new IOException("Not support");
+ throw new UnsupportedOperationException("Unsupported operation.");
}
@Override
public long getPosition() throws IOException {
-
return fsDataOutputStream.getPos();
}
@Override
public void close() throws IOException {
-
fsDataOutputStream.close();
}
@@ -88,6 +88,6 @@ public class HDFSOutput implements TsFileOutput {
@Override
public void truncate(long position) throws IOException {
- throw new IOException("Not support");
+ fs.truncate(path, position);
}
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/TSFileFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/TSFileFactory.java
new file mode 100644
index 0000000..6c8fd04
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/TSFileFactory.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.fileSystem;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.net.URI;
+
+public enum TSFileFactory {
+
+ INSTANCE;
+
+ private static FSType fSType =
TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs();
+ private static final Logger logger =
LoggerFactory.getLogger(TsFileWriter.class);
+ private FileSystem fs;
+ private Configuration conf = new Configuration();
+
+ public File getFile(String pathname) {
+ if (fSType.equals(fSType.HDFS)) {
+ return new HDFSFile(pathname);
+ } else {
+ return new File(pathname);
+ }
+ }
+
+ public File getFile(String parent, String child) {
+ if (fSType.equals(fSType.HDFS)) {
+ return new HDFSFile(parent, child);
+ } else {
+ return new File(parent, child);
+ }
+ }
+
+ public File getFile(File parent, String child) {
+ if (fSType.equals(fSType.HDFS)) {
+ return new HDFSFile(parent, child);
+ } else {
+ return new File(parent, child);
+ }
+ }
+
+ public File getFile(URI uri) {
+ if (fSType.equals(fSType.HDFS)) {
+ return new HDFSFile(uri);
+ } else {
+ return new File(uri);
+ }
+ }
+
+ public BufferedReader getBufferedReader(String filePath) {
+ try {
+ if (fSType.equals(fSType.HDFS)) {
+ Path path = new Path(filePath);
+ fs = path.getFileSystem(conf);
+ return new BufferedReader(new InputStreamReader(fs.open(path)));
+ } else {
+ return new BufferedReader(new FileReader(filePath));
+ }
+ } catch (IOException e) {
+ logger.error("Fail to get buffered reader. ", e);
+ return null;
+ }
+ }
+
+ public BufferedWriter getBufferedWriter(String filePath, boolean append) {
+ try {
+ if (fSType.equals(fSType.HDFS)) {
+ Path path = new Path(filePath);
+ fs = path.getFileSystem(conf);
+ return new BufferedWriter(new OutputStreamWriter(fs.create(path)));
+ } else {
+ return new BufferedWriter(new FileWriter(filePath, append));
+ }
+ } catch (IOException e) {
+ logger.error("Fail to get buffered writer. ", e);
+ return null;
+ }
+ }
+
+}
\ No newline at end of file
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
index 7b43a9d..b998c03 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileRestorableReader.java
@@ -19,8 +19,9 @@
package org.apache.iotdb.tsfile.read;
-import java.io.File;
import java.io.IOException;
+
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import org.slf4j.Logger;
@@ -59,7 +60,7 @@ public class TsFileRestorableReader extends
TsFileSequenceReader {
if (!isComplete()) {
// Try to close it
logger.info("File {} has no correct tail magic, try to repair...", file);
- RestorableTsFileIOWriter rWriter = new RestorableTsFileIOWriter(new
File(file));
+ RestorableTsFileIOWriter rWriter = new
RestorableTsFileIOWriter(TSFileFactory.INSTANCE.getFile(file));
TsFileWriter writer = new TsFileWriter(rWriter);
// This writes the right magic string
writer.close();
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 9dd8a08..0461666 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -31,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.compress.IUnCompressor;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
@@ -46,9 +47,10 @@ import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+import org.apache.iotdb.tsfile.fileSystem.FileInputFactory;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.reader.DefaultTsFileInput;
import org.apache.iotdb.tsfile.read.reader.TsFileInput;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -58,6 +60,8 @@ import org.slf4j.LoggerFactory;
public class TsFileSequenceReader implements AutoCloseable {
private static final Logger logger =
LoggerFactory.getLogger(TsFileSequenceReader.class);
+ protected static final TSFileConfig config =
TSFileDescriptor.getInstance().getConfig();
+
protected String file;
private TsFileInput tsFileInput;
private long fileMetadataPos;
@@ -89,8 +93,7 @@ public class TsFileSequenceReader implements AutoCloseable {
*/
public TsFileSequenceReader(String file, boolean loadMetadataSize) throws
IOException {
this.file = file;
- final java.nio.file.Path path = Paths.get(file);
- tsFileInput = new DefaultTsFileInput(path);
+ tsFileInput = FileInputFactory.INSTANCE.getTsFileInput(file);
try {
if (loadMetadataSize) {
loadMetadataSize();
@@ -505,7 +508,7 @@ public class TsFileSequenceReader implements AutoCloseable {
*/
public long selfCheck(Map<String, MeasurementSchema> newSchema,
List<ChunkGroupMetaData> newMetaData, boolean fastFinish) throws
IOException {
- File checkFile = new File(this.file);
+ File checkFile = TSFileFactory.INSTANCE.getFile(this.file);
long fileSize;
if (!checkFile.exists()) {
return TsFileCheckStatus.FILE_NOT_FOUND;
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
index 1796d5f..227b66b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
@@ -50,6 +50,8 @@ import org.slf4j.LoggerFactory;
public class TsFileWriter implements AutoCloseable{
private static final Logger LOG =
LoggerFactory.getLogger(TsFileWriter.class);
+ protected static final TSFileConfig config =
TSFileDescriptor.getInstance().getConfig();
+
/**
* schema of this TsFile.
**/
@@ -140,6 +142,7 @@ public class TsFileWriter implements AutoCloseable{
this.schema.registerMeasurements(fileWriter.getKnownSchema());
this.pageSize = TSFileConfig.pageSizeInByte;
this.chunkGroupSizeThreshold = TSFileConfig.groupSizeInByte;
+ config.setTSFileStorageFs(conf.getTSFileStorageFs().name());
if (this.pageSize >= chunkGroupSizeThreshold) {
LOG.warn(
"TsFile's page size {} is greater than chunk group size {}, please
enlarge the chunk group"
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
index 9fd10ea..cdf6f67 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
@@ -34,6 +34,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.fileSystem.FileOutputFactory;
import org.apache.iotdb.tsfile.read.TsFileCheckStatus;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -71,7 +72,7 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
*/
public RestorableTsFileIOWriter(File file) throws IOException {
this.file = file;
- this.out = new DefaultTsFileOutput(file, true);
+ this.out = FileOutputFactory.INSTANCE.getTsFileOutput(file.getPath(),
true);
// file doesn't exist
if (file.length() == 0) {
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 14253ba..8a0bda1 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -31,6 +31,7 @@ import java.util.Map;
import java.util.TreeMap;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
@@ -45,6 +46,7 @@ import
org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+import org.apache.iotdb.tsfile.fileSystem.FileOutputFactory;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BytesUtils;
@@ -63,7 +65,8 @@ import org.slf4j.LoggerFactory;
public class TsFileIOWriter {
public static final byte[] magicStringBytes;
- private static final Logger LOG =
LoggerFactory.getLogger(TsFileIOWriter.class);
+ private static final Logger logger =
LoggerFactory.getLogger(TsFileIOWriter.class);
+ protected static final TSFileConfig config =
TSFileDescriptor.getInstance().getConfig();
static {
magicStringBytes = BytesUtils.stringToBytes(TSFileConfig.MAGIC_STRING);
@@ -120,7 +123,7 @@ public class TsFileIOWriter {
*/
public TsFileIOWriter(TsFileOutput out, List<ChunkGroupMetaData>
chunkGroupMetaDataList)
throws IOException {
- this.out = out;
+ this.out = FileOutputFactory.INSTANCE.getTsFileOutput(file.getPath(),
false); //NOTE overwrite false here
this.chunkGroupMetaDataList = chunkGroupMetaDataList;
if (chunkGroupMetaDataList.isEmpty()) {
startFile();
@@ -148,7 +151,7 @@ public class TsFileIOWriter {
* @param deviceId device id
*/
public void startChunkGroup(String deviceId) throws IOException {
- LOG.debug("start chunk group:{}, file position {}", deviceId,
out.getPosition());
+ logger.debug("start chunk group:{}, file position {}", deviceId,
out.getPosition());
currentChunkGroupMetaData = new ChunkGroupMetaData(deviceId, new
ArrayList<>(),
out.getPosition());
}
@@ -165,7 +168,7 @@ public class TsFileIOWriter {
currentChunkGroupMetaData.setEndOffsetOfChunkGroup(out.getPosition());
currentChunkGroupMetaData.setVersion(version);
chunkGroupMetaDataList.add(currentChunkGroupMetaData);
- LOG.debug("end chunk group:{}", currentChunkGroupMetaData);
+ logger.debug("end chunk group:{}", currentChunkGroupMetaData);
currentChunkGroupMetaData = null;
}
@@ -186,7 +189,7 @@ public class TsFileIOWriter {
TSDataType tsDataType, TSEncoding encodingType, Statistics<?>
statistics, long maxTime,
long minTime,
int dataSize, int numOfPages) throws IOException {
- LOG.debug("start series chunk:{}, file position {}", descriptor,
out.getPosition());
+ logger.debug("start series chunk:{}, file position {}", descriptor,
out.getPosition());
currentChunkMetaData = new ChunkMetaData(descriptor.getMeasurementId(),
tsDataType,
out.getPosition(), minTime, maxTime);
@@ -195,7 +198,7 @@ public class TsFileIOWriter {
compressionCodecName,
encodingType, numOfPages);
header.serializeTo(out.wrapAsStream());
- LOG.debug("finish series chunk:{} header, file position {}", header,
out.getPosition());
+ logger.debug("finish series chunk:{} header, file position {}", header,
out.getPosition());
// TODO add your statistics
ByteBuffer[] statisticsArray = new
ByteBuffer[StatisticType.getTotalTypeNum()];
@@ -239,7 +242,7 @@ public class TsFileIOWriter {
public void endChunk(long totalValueCount) {
currentChunkMetaData.setNumOfPoints(totalValueCount);
currentChunkGroupMetaData.addTimeSeriesChunkMetaData(currentChunkMetaData);
- LOG.debug("end series chunk:{},totalvalue:{}", currentChunkMetaData,
totalValueCount);
+ logger.debug("end series chunk:{},totalvalue:{}", currentChunkMetaData,
totalValueCount);
currentChunkMetaData = null;
totalChunkNum ++;
}
@@ -257,7 +260,7 @@ public class TsFileIOWriter {
// get all measurementSchema of this TsFile
Map<String, MeasurementSchema> schemaDescriptors =
schema.getMeasurementSchemaMap();
- LOG.debug("get time series list:{}", schemaDescriptors);
+ logger.debug("get time series list:{}", schemaDescriptors);
Map<String, TsDeviceMetadataIndex> tsDeviceMetadataIndexMap =
flushTsDeviceMetaDataAndGetIndex(
this.chunkGroupMetaDataList);
@@ -268,11 +271,11 @@ public class TsFileIOWriter {
tsFileMetaData.setInvalidChunkNum(invalidChunkNum);
long footerIndex = out.getPosition();
- LOG.debug("start to flush the footer,file pos:{}", footerIndex);
+ logger.debug("start to flush the footer,file pos:{}", footerIndex);
// write TsFileMetaData
int size = tsFileMetaData.serializeTo(out.wrapAsStream());
- LOG.debug("finish flushing the footer {}, file pos:{}", tsFileMetaData,
out.getPosition());
+ logger.debug("finish flushing the footer {}, file pos:{}", tsFileMetaData,
out.getPosition());
// write TsFileMetaData size
ReadWriteIOUtils.write(size, out.wrapAsStream());// write the size of the
file metadata.
@@ -283,7 +286,7 @@ public class TsFileIOWriter {
// close file
out.close();
canWrite = false;
- LOG.info("output stream is closed");
+ logger.info("output stream is closed");
}
/**
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileRestorableReaderTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileRestorableReaderTest.java
index df3d03b..ac95527 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileRestorableReaderTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileRestorableReaderTest.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.utils.TsFileGeneratorForTest;
import org.apache.iotdb.tsfile.utils.IncompleteFileTestUtil;
import org.junit.Test;
@@ -35,7 +36,7 @@ public class TsFileRestorableReaderTest {
@Test
public void testToReadDamagedFileAndRepair() throws IOException {
- File file = new File(FILE_PATH);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_PATH);
IncompleteFileTestUtil.writeFileWithOneIncompleteChunkHeader(file);
@@ -50,7 +51,7 @@ public class TsFileRestorableReaderTest {
@Test(expected = IllegalArgumentException.class)
public void testToReadDamagedFileNoRepair() throws IOException {
- File file = new File(FILE_PATH);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_PATH);
IncompleteFileTestUtil.writeFileWithOneIncompleteChunkHeader(file);
// This should throw an Illegal Argument Exception
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileUtils.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileUtils.java
index 4bc03ec..81a49d7 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileUtils.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.tsfile.utils;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import java.io.File;
/**
@@ -29,7 +30,7 @@ import java.io.File;
public class FileUtils {
public static double getLocalFileByte(String filePath, Unit unit) {
- File f = new File(filePath);
+ File f = TSFileFactory.INSTANCE.getFile(filePath);
return getLocalFileByte(f, unit);
}
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
index a067dc0..e745953 100755
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
@@ -34,6 +34,7 @@ import
org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.schema.Schema;
@@ -77,22 +78,22 @@ public class TsFileGeneratorForTest {
}
public static void after() {
- File file = new File(inputDataFile);
+ File file = TSFileFactory.INSTANCE.getFile(inputDataFile);
if (file.exists()) {
Assert.assertTrue(file.delete());
}
- file = new File(outputDataFile);
+ file = TSFileFactory.INSTANCE.getFile(outputDataFile);
if (file.exists()) {
Assert.assertTrue(file.delete());
}
- file = new File(errorOutputDataFile);
+ file = TSFileFactory.INSTANCE.getFile(errorOutputDataFile);
if (file.exists()) {
Assert.assertTrue(file.delete());
}
}
static private void generateSampleInputDataFile(int minRowCount, int
maxRowCount) throws IOException {
- File file = new File(inputDataFile);
+ File file = TSFileFactory.INSTANCE.getFile(inputDataFile);
if (file.exists()) {
Assert.assertTrue(file.delete());
}
@@ -145,8 +146,8 @@ public class TsFileGeneratorForTest {
}
static public void write() throws IOException {
- File file = new File(outputDataFile);
- File errorFile = new File(errorOutputDataFile);
+ File file = TSFileFactory.INSTANCE.getFile(outputDataFile);
+ File errorFile = TSFileFactory.INSTANCE.getFile(errorOutputDataFile);
if (file.exists()) {
Assert.assertTrue(file.delete());
}
@@ -161,7 +162,7 @@ public class TsFileGeneratorForTest {
innerWriter = new TsFileWriter(file, schema,
TSFileDescriptor.getInstance().getConfig());
// write
- try (Scanner in = new Scanner(new File(inputDataFile))) {
+ try (Scanner in = new
Scanner(TSFileFactory.INSTANCE.getFile(inputDataFile))) {
assert in != null;
while (in.hasNextLine()) {
String str = in.nextLine();
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
index f6a5231..22b390c 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
@@ -19,11 +19,6 @@
package org.apache.iotdb.tsfile.write.writer;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertTrue;
-
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
@@ -35,6 +30,7 @@ import
org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.file.metadata.statistics.FloatStatistics;
+import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.read.ReadOnlyTsFile;
import org.apache.iotdb.tsfile.read.TsFileCheckStatus;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -49,6 +45,8 @@ import
org.apache.iotdb.tsfile.write.record.datapoint.FloatDataPoint;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.junit.Test;
+import static org.junit.Assert.*;
+
@SuppressWarnings("squid:S4042") // Suppress use java.nio.Files#delete warning
public class RestorableTsFileIOWriterTest {
@@ -56,7 +54,7 @@ public class RestorableTsFileIOWriterTest {
@Test(expected = IOException.class)
public void testBadHeadMagic() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
FileWriter fWriter = new FileWriter(file);
fWriter.write("Tsfile");
fWriter.close();
@@ -69,7 +67,7 @@ public class RestorableTsFileIOWriterTest {
@Test
public void testOnlyHeadMagic() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
TsFileWriter writer = new TsFileWriter(file);
writer.getIOWriter().close();
@@ -88,12 +86,11 @@ public class RestorableTsFileIOWriterTest {
@Test
public void testOnlyFirstMask() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
TsFileWriter writer = new TsFileWriter(file);
//we have to flush using inner API.
writer.getIOWriter().out.write(new byte[] {MetaMarker.CHUNK_HEADER});
writer.getIOWriter().close();
- assertEquals(TsFileIOWriter.magicStringBytes.length + 1, file.length());
RestorableTsFileIOWriter rWriter = new RestorableTsFileIOWriter(file);
writer = new TsFileWriter(rWriter);
writer.close();
@@ -103,7 +100,7 @@ public class RestorableTsFileIOWriterTest {
@Test
public void testOnlyOneIncompleteChunkHeader() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
IncompleteFileTestUtil.writeFileWithOneIncompleteChunkHeader(file);
@@ -133,7 +130,7 @@ public class RestorableTsFileIOWriterTest {
@Test
public void testOnlyOneChunkHeaderAndSomePage() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
TsFileWriter writer = new TsFileWriter(file);
writer.addMeasurement(new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
writer.addMeasurement(new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
@@ -156,7 +153,7 @@ public class RestorableTsFileIOWriterTest {
@Test
public void testOnlyOneChunkGroup() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
TsFileWriter writer = new TsFileWriter(file);
writer.addMeasurement(new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
writer.addMeasurement(new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
@@ -192,7 +189,7 @@ public class RestorableTsFileIOWriterTest {
@Test
public void testOnlyOneChunkGroupAndOneMask() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
TsFileWriter writer = new TsFileWriter(file);
writer.addMeasurement(new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
writer.addMeasurement(new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
@@ -217,7 +214,7 @@ public class RestorableTsFileIOWriterTest {
@Test
public void testTwoChunkGroupAndMore() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
TsFileWriter writer = new TsFileWriter(file);
writer.addMeasurement(new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
writer.addMeasurement(new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
@@ -244,7 +241,7 @@ public class RestorableTsFileIOWriterTest {
@Test
public void testNoSeperatorMask() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
TsFileWriter writer = new TsFileWriter(file);
writer.addMeasurement(new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
writer.addMeasurement(new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
@@ -275,7 +272,7 @@ public class RestorableTsFileIOWriterTest {
@Test
public void testHavingSomeFileMetadata() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
TsFileWriter writer = new TsFileWriter(file);
writer.addMeasurement(new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
writer.addMeasurement(new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
@@ -306,7 +303,7 @@ public class RestorableTsFileIOWriterTest {
@Test
public void testOpenCompleteFile() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
TsFileWriter writer = new TsFileWriter(file);
writer.addMeasurement(new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
writer.addMeasurement(new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
@@ -331,7 +328,7 @@ public class RestorableTsFileIOWriterTest {
@Test
public void testAppendDataOnCompletedFile() throws Exception {
- File file = new File(FILE_NAME);
+ File file = TSFileFactory.INSTANCE.getFile(FILE_NAME);
TsFileWriter writer = new TsFileWriter(file);
writer.addMeasurement(new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
writer.addMeasurement(new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
