This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch load-pri in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c7f312e1bc0f998144897fd57f189c8caacb9bce Author: Caideyipi <[email protected]> AuthorDate: Sat May 9 12:24:20 2026 +0800 Load pri --- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 7 ++- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 18 ++++++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 21 +++++++ .../protocol/legacy/loader/TsFileLoader.java | 2 +- .../protocol/thrift/IoTDBDataNodeReceiver.java | 2 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 29 ++++++++-- .../load/TreeSchemaAutoCreatorAndVerifier.java | 36 +++++++----- .../relational/analyzer/StatementAnalyzer.java | 5 ++ .../relational/security/TableModelPrivilege.java | 6 ++ .../security/TreeAccessCheckVisitor.java | 11 +++- .../plan/relational/sql/ast/LoadTsFile.java | 17 +++++- .../plan/scheduler/load/LoadTsFileScheduler.java | 4 +- .../plan/statement/crud/LoadTsFileStatement.java | 66 ++++++++++++++++++++-- .../load/active/ActiveLoadTsFileLoader.java | 3 +- .../org/apache/iotdb/db/auth/TreeAccessTest.java | 37 ++++++++++++ .../relational/sql/parser/AuthorStatementTest.java | 4 ++ .../statement/crud/LoadTsFileStatementTest.java | 35 ++++++++++++ .../conf/iotdb-system.properties.template | 6 ++ .../iotdb/commons/auth/entity/PrivilegeType.java | 6 +- .../org/apache/iotdb/commons/utils/AuthUtils.java | 5 ++ .../db/relational/grammar/sql/RelationalSql.g4 | 4 +- 21 files changed, 288 insertions(+), 36 deletions(-) diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index 5c5cbe4a186..b131e403266 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -1129,6 +1129,7 @@ PRIVILEGE_VALUE | EXTEND_TEMPLATE | MANAGE_DATABASE | MAINTAIN + | LOAD_TSFILE ; READ_DATA @@ -1187,6 +1188,10 @@ MAINTAIN : M A I N T A I N ; +LOAD_TSFILE + : L O A D '_' T S F I L E + ; + SECURITY : S E C U R I T Y ; @@ -1399,4 +1404,4 @@ fragment V: [vV]; fragment W: [wW]; fragment X: [xX]; fragment Y: [yY]; -fragment Z: [zZ]; \ No newline at end of file +fragment Z: [zZ]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 9b177cffcfa..8613bc318da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -298,6 +298,8 @@ public class IoTDBConfig { tierDataDirs[0][0] + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME }; + private String[] loadTsFileAllowedDirs = new String[0]; + /** Strategy of multiple directories. */ private String multiDirStrategyClassName = null; @@ -1355,6 +1357,9 @@ public class IoTDBConfig { for (int i = 0; i < loadActiveListeningDirs.length; i++) { loadActiveListeningDirs[i] = addDataHomeDir(loadActiveListeningDirs[i]); } + for (int i = 0; i < loadTsFileAllowedDirs.length; i++) { + loadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]); + } loadActiveListeningPipeDir = addDataHomeDir(loadActiveListeningPipeDir); loadActiveListeningFailDir = addDataHomeDir(loadActiveListeningFailDir); udfDir = addDataHomeDir(udfDir); @@ -1560,6 +1565,19 @@ public class IoTDBConfig { return this.loadTsFileDirs; } + public String[] getLoadTsFileAllowedDirs() { + return this.loadTsFileAllowedDirs.length == 0 + ? getLoadTsFileDirs() + : this.loadTsFileAllowedDirs; + } + + public void setLoadTsFileAllowedDirs(String[] loadTsFileAllowedDirs) { + for (int i = 0; i < loadTsFileAllowedDirs.length; i++) { + loadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]); + } + this.loadTsFileAllowedDirs = loadTsFileAllowedDirs; + } + public void formulateLoadTsFileDirs(String[][] tierDataDirs) { if (tierDataDirs.length < 1) { logger.warn("No data directory is set. loadTsFileDirs is kept as the default value."); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 7193fd27c58..beceafc6494 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -2456,6 +2456,15 @@ public class IoTDBDescriptor { "load_write_throughput_bytes_per_second", String.valueOf(conf.getLoadWriteThroughputBytesPerSecond())))); + conf.setLoadTsFileAllowedDirs( + Arrays.stream( + properties + .getProperty("load_tsfile_allowed_dirs", "") + .trim() + .split(",")) + .filter(dir -> !dir.isEmpty()) + .toArray(String[]::new)); + conf.setLoadTabletConversionThresholdBytes( Long.parseLong( properties.getProperty( @@ -2573,6 +2582,18 @@ public class IoTDBDescriptor { ConfigurationFileUtils.getConfigurationDefaultValue( "load_write_throughput_bytes_per_second")))); + conf.setLoadTsFileAllowedDirs( + Arrays.stream( + properties + .getProperty( + "load_tsfile_allowed_dirs", + ConfigurationFileUtils.getConfigurationDefaultValue( + "load_tsfile_allowed_dirs")) + .trim() + .split(",")) + .filter(dir -> !dir.isEmpty()) + .toArray(String[]::new)); + conf.setLoadActiveListeningEnable( Boolean.parseBoolean( properties.getProperty( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java index 1c95b574b54..92139942b0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java @@ -55,7 +55,7 @@ public class TsFileLoader implements ILoader { @Override public void load() { try { - LoadTsFileStatement statement = new LoadTsFileStatement(tsFile.getAbsolutePath()); + LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath()); statement.setDeleteAfterLoad(true); statement.setConvertOnTypeMismatch(true); statement.setDatabaseLevel(parseSgLevel()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java index c10fdbc4f67..dd6e684031c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java @@ -585,7 +585,7 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver { private TSStatus loadTsFileSync(final String dataBaseName, final String fileAbsolutePath) throws FileNotFoundException { - final LoadTsFileStatement statement = new LoadTsFileStatement(fileAbsolutePath); + final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(fileAbsolutePath); statement.setDeleteAfterLoad(true); statement.setConvertOnTypeMismatch(true); statement.setVerifySchema(validateTsFile.get()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index e1e6d597191..0538ecbc576 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -53,6 +53,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.io.FileUtils; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TableSchema; @@ -443,14 +444,15 @@ public class LoadTsFileAnalyzer implements AutoCloseable { isTableModelTsFile.get(i) ? loadTsFileDataTypeConverter .convertForTableModel( - new LoadTsFile(null, tsFiles.get(i).getPath(), Collections.emptyMap()) + LoadTsFile.createUnchecked( + null, tsFiles.get(i).getPath(), Collections.emptyMap()) .setDatabase(databaseForTableData) .setDeleteAfterLoad(isDeleteAfterLoad) .setConvertOnTypeMismatch(isConvertOnTypeMismatch)) .orElse(null) : loadTsFileDataTypeConverter .convertForTreeModel( - new LoadTsFileStatement(tsFiles.get(i).getPath()) + LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath()) .setDeleteAfterLoad(isDeleteAfterLoad) .setConvertOnTypeMismatch(isConvertOnTypeMismatch)) .orElse(null); @@ -505,6 +507,8 @@ public class LoadTsFileAnalyzer implements AutoCloseable { if (isAutoCreateSchemaOrVerifySchemaEnabled) { getOrCreateTreeSchemaVerifier().autoCreateAndVerify(reader, device2TimeseriesMetadata); + } else { + checkTreeWritePermission(device2TimeseriesMetadata); } // TODO: how to get the correct write point count when @@ -522,6 +526,22 @@ public class LoadTsFileAnalyzer implements AutoCloseable { addWritePointCount(writePointCount); } + private void checkTreeWritePermission( + final Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadataList) + throws AuthException { + for (final Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry : + device2TimeseriesMetadataList.entrySet()) { + final IDeviceID device = entry.getKey(); + for (final TimeseriesMetadata timeseriesMetadata : entry.getValue()) { + if (!TSDataType.VECTOR.equals(timeseriesMetadata.getTsDataType()) + && !timeseriesMetadata.getMeasurementId().isEmpty()) { + TreeSchemaAutoCreatorAndVerifier.checkWriteDataPermission( + this, device, timeseriesMetadata); + } + } + } + } + private void doAnalyzeSingleTableFile( final File tsFile, final TsFileSequenceReader reader, @@ -711,14 +731,15 @@ public class LoadTsFileAnalyzer implements AutoCloseable { isTableModelTsFile.get(i) ? loadTsFileDataTypeConverter .convertForTableModel( - new LoadTsFile(null, tsFiles.get(i).getPath(), Collections.emptyMap()) + LoadTsFile.createUnchecked( + null, tsFiles.get(i).getPath(), Collections.emptyMap()) .setDatabase(databaseForTableData) .setDeleteAfterLoad(isDeleteAfterLoad) .setConvertOnTypeMismatch(isConvertOnTypeMismatch)) .orElse(null) : loadTsFileDataTypeConverter .convertForTreeModel( - new LoadTsFileStatement(tsFiles.get(i).getPath()) + LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath()) .setDeleteAfterLoad(isDeleteAfterLoad) .setConvertOnTypeMismatch(isConvertOnTypeMismatch)) .orElse(null); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java index de56a8603c5..d8fbb9c80d7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java @@ -149,21 +149,7 @@ public class TreeSchemaAutoCreatorAndVerifier { // not a timeseries, skip } else { - // check WRITE_DATA permission of timeseries - long startTime = System.nanoTime(); - try { - UserEntity userEntity = loadTsFileAnalyzer.context.getSession().getUserEntity(); - TSStatus status = - AuthorityChecker.getAccessControl() - .checkFullPathWriteDataPermission( - userEntity, device, timeseriesMetadata.getMeasurementId()); - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new AuthException( - TSStatusCode.representOf(status.getCode()), status.getMessage()); - } - } finally { - PerformanceOverviewMetrics.getInstance().recordAuthCost(System.nanoTime() - startTime); - } + checkWriteDataPermission(loadTsFileAnalyzer, device, timeseriesMetadata); final Pair<CompressionType, TSEncoding> compressionEncodingPair = reader.readTimeseriesCompressionTypeAndEncoding(timeseriesMetadata); schemaCache.addTimeSeries( @@ -187,6 +173,26 @@ public class TreeSchemaAutoCreatorAndVerifier { } } + static void checkWriteDataPermission( + final LoadTsFileAnalyzer loadTsFileAnalyzer, + final IDeviceID device, + final TimeseriesMetadata timeseriesMetadata) + throws AuthException { + final long startTime = System.nanoTime(); + try { + final UserEntity userEntity = loadTsFileAnalyzer.context.getSession().getUserEntity(); + final TSStatus status = + AuthorityChecker.getAccessControl() + .checkFullPathWriteDataPermission( + userEntity, device, timeseriesMetadata.getMeasurementId()); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new AuthException(TSStatusCode.representOf(status.getCode()), status.getMessage()); + } + } finally { + PerformanceOverviewMetrics.getInstance().recordAuthCost(System.nanoTime() - startTime); + } + } + /** * This can only be invoked after all timeseries in the current tsfile have been processed. * Otherwise, the isAligned status may be wrong. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 6d3326dd402..87803d70d79 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer; import org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.common.SessionInfo; @@ -830,6 +831,10 @@ public class StatementAnalyzer { @Override public Scope visitLoadTsFile(final LoadTsFile node, final Optional<Scope> scope) { queryContext.setQueryType(QueryType.OTHER); + accessControl.checkMissingPrivileges( + sessionContext.getUserName(), + Collections.singletonList(PrivilegeType.LOAD_TSFILE), + queryContext); try (final LoadTsFileAnalyzer loadTsFileAnalyzer = new LoadTsFileAnalyzer(node, node.isGeneratedByPipe(), queryContext)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java index d06a08ef026..3fc99e55f41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java @@ -32,6 +32,7 @@ public enum TableModelPrivilege { SYSTEM, SECURITY, AUDIT, + LOAD_TSFILE, // scope privilege CREATE, @@ -65,6 +66,8 @@ public enum TableModelPrivilege { return PrivilegeType.SECURITY; case AUDIT: return PrivilegeType.AUDIT; + case LOAD_TSFILE: + return PrivilegeType.LOAD_TSFILE; default: throw new IllegalStateException("Unexpected value:" + this); } @@ -94,6 +97,8 @@ public enum TableModelPrivilege { return TableModelPrivilege.SECURITY; case AUDIT: return TableModelPrivilege.AUDIT; + case LOAD_TSFILE: + return TableModelPrivilege.LOAD_TSFILE; default: throw new IllegalStateException("Unexpected value:" + privilegeType); } @@ -109,6 +114,7 @@ public enum TableModelPrivilege { return AuditLogOperation.QUERY; case INSERT: case DELETE: + case LOAD_TSFILE: return AuditLogOperation.DML; case MANAGE_ROLE: case MANAGE_USER: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java index 7b4655c79a8..e525a5e061d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java @@ -169,6 +169,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.collect.ImmutableList; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -1149,8 +1150,14 @@ public class TreeAccessCheckVisitor extends StatementVisitor<TSStatus, TreeAcces @Override public TSStatus visitLoadFile(LoadTsFileStatement statement, TreeAccessCheckContext context) { - // no need to check here, it will be checked in process phase - return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + return checkGlobalAuth( + context.setAuditLogOperation(AuditLogOperation.DML), + PrivilegeType.LOAD_TSFILE, + () -> + statement.getTsFiles().stream() + .map(File::getPath) + .collect(Collectors.toList()) + .toString()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java index 03de9a9ccb1..dc1a549f636 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java @@ -72,6 +72,19 @@ public class LoadTsFile extends Statement { private boolean needDecode4TimeColumn; public LoadTsFile(NodeLocation location, String filePath, Map<String, String> loadAttributes) { + this(location, filePath, loadAttributes, true); + } + + public static LoadTsFile createUnchecked( + NodeLocation location, String filePath, Map<String, String> loadAttributes) { + return new LoadTsFile(location, filePath, loadAttributes, false); + } + + private LoadTsFile( + NodeLocation location, + String filePath, + Map<String, String> loadAttributes, + boolean validateSourcePath) { super(location); this.filePath = requireNonNull(filePath, "filePath is null"); @@ -89,7 +102,7 @@ public class LoadTsFile extends Statement { try { this.tsFiles = org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement.processTsFile( - new File(filePath)); + new File(filePath), validateSourcePath); this.resources = new ArrayList<>(); this.writePointCountList = new ArrayList<>(); this.isTableModel = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), true)); @@ -283,7 +296,7 @@ public class LoadTsFile extends Statement { final Map<String, String> properties = this.loadAttributes; final LoadTsFile subStatement = - new LoadTsFile(getLocation().orElse(null), filePath, properties); + LoadTsFile.createUnchecked(getLocation().orElse(null), filePath, properties); // Copy all configuration properties subStatement.databaseLevel = this.databaseLevel; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java index bc8446fd3da..6671dad3548 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java @@ -586,14 +586,14 @@ public class LoadTsFileScheduler implements IScheduler { failedNode.isTableModel() ? loadTsFileDataTypeConverter .convertForTableModel( - new LoadTsFile(null, filePath, Collections.emptyMap()) + LoadTsFile.createUnchecked(null, filePath, Collections.emptyMap()) .setDatabase(failedNode.getDatabase()) .setDeleteAfterLoad(failedNode.isDeleteAfterLoad()) .setConvertOnTypeMismatch(true)) .orElse(null) : loadTsFileDataTypeConverter .convertForTreeModel( - new LoadTsFileStatement(filePath) + LoadTsFileStatement.createUnchecked(filePath) .setDeleteAfterLoad(failedNode.isDeleteAfterLoad()) .setConvertOnTypeMismatch(true)) .orElse(null); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java index 512a62fdae7..a846c7feb43 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java @@ -35,7 +35,9 @@ import org.apache.tsfile.common.constant.TsFileConstant; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -71,6 +73,15 @@ public class LoadTsFileStatement extends Statement { private boolean needDecode4TimeColumn; public LoadTsFileStatement(String filePath) throws FileNotFoundException { + this(filePath, true); + } + + public static LoadTsFileStatement createUnchecked(String filePath) throws FileNotFoundException { + return new LoadTsFileStatement(filePath, false); + } + + private LoadTsFileStatement(String filePath, boolean validateSourcePath) + throws FileNotFoundException { this.file = new File(filePath).getAbsoluteFile(); this.databaseLevel = IoTDBDescriptor.getInstance().getConfig().getDefaultDatabaseLevel(); this.verifySchema = true; @@ -80,7 +91,7 @@ public class LoadTsFileStatement extends Statement { IoTDBDescriptor.getInstance().getConfig().getLoadTabletConversionThresholdBytes(); this.autoCreateDatabase = IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled(); - this.tsFiles = processTsFile(file); + this.tsFiles = processTsFile(file, validateSourcePath); this.resources = new ArrayList<>(); this.writePointCountList = new ArrayList<>(); this.isTableModel = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), false)); @@ -88,6 +99,15 @@ public class LoadTsFileStatement extends Statement { } public static List<File> processTsFile(final File file) throws FileNotFoundException { + return processTsFile(file, true); + } + + public static List<File> processTsFile(final File file, final boolean validateSourcePath) + throws FileNotFoundException { + if (validateSourcePath) { + validateLoadSourcePath(file); + } + final List<File> tsFiles = new ArrayList<>(); if (file.isFile()) { tsFiles.add(file); @@ -98,7 +118,7 @@ public class LoadTsFileStatement extends Statement { "Can not find %s on this machine, notice that load can only handle files on this machine.", file.getPath())); } - tsFiles.addAll(findAllTsFile(file)); + tsFiles.addAll(findAllTsFile(file, validateSourcePath)); } sortTsFiles(tsFiles); return tsFiles; @@ -120,7 +140,8 @@ public class LoadTsFileStatement extends Statement { this.statementType = StatementType.MULTI_BATCH_INSERT; } - private static List<File> findAllTsFile(File file) { + private static List<File> findAllTsFile(File file, boolean validateSourcePath) + throws FileNotFoundException { final File[] files = file.listFiles(); if (files == null) { return Collections.emptyList(); @@ -128,15 +149,50 @@ public class LoadTsFileStatement extends Statement { final List<File> tsFiles = new ArrayList<>(); for (File nowFile : files) { + if (validateSourcePath) { + validateLoadSourcePath(nowFile); + } if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) { tsFiles.add(nowFile); } else if (nowFile.isDirectory()) { - tsFiles.addAll(findAllTsFile(nowFile)); + tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath)); } } return tsFiles; } + public static void validateLoadSourcePath(final String filePath) throws FileNotFoundException { + validateLoadSourcePath(new File(filePath)); + } + + private static void validateLoadSourcePath(final File file) throws FileNotFoundException { + final Path sourcePath = canonicalPath(file); + final String[] allowedDirs = + IoTDBDescriptor.getInstance().getConfig().getLoadTsFileAllowedDirs(); + + for (final String allowedDir : allowedDirs) { + if (sourcePath.startsWith(canonicalPath(new File(allowedDir)))) { + return; + } + } + + throw new FileNotFoundException( + String.format( + "Load TsFile source path %s is outside allowed directories %s.", + sourcePath, Arrays.toString(allowedDirs))); + } + + private static Path canonicalPath(final File file) throws FileNotFoundException { + try { + return file.getCanonicalFile().toPath(); + } catch (final IOException e) { + throw new FileNotFoundException( + String.format( + "Failed to resolve canonical path for Load TsFile source %s: %s", + file.getPath(), e.getMessage())); + } + } + private static void sortTsFiles(List<File> files) { files.sort( (o1, o2) -> { @@ -389,7 +445,7 @@ public class LoadTsFileStatement extends Statement { loadAttributes.put(PIPE_GENERATED_KEY, String.valueOf(true)); } - return new LoadTsFile(null, file.getAbsolutePath(), loadAttributes); + return LoadTsFile.createUnchecked(null, file.getAbsolutePath(), loadAttributes); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java index d0be2ead5cb..67fa0fff300 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java @@ -219,7 +219,8 @@ public class ActiveLoadTsFileLoader { final ActiveLoadPendingQueue.ActiveLoadEntry entry, final IClientSession session) throws FileNotFoundException { final File tsFile = new File(entry.getFile()); - final LoadTsFileStatement statement = new LoadTsFileStatement(tsFile.getAbsolutePath()); + final LoadTsFileStatement statement = + LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath()); final List<File> files = statement.getTsFiles(); statement.setDeleteAfterLoad(true); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java index af8746fae3e..6eff04b0e47 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.auth.entity.User; import org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckContext; import org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckVisitor; import org.apache.iotdb.db.queryengine.plan.statement.AuthorType; +import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.AuthorStatement; import org.apache.iotdb.rpc.TSStatusCode; @@ -33,6 +34,9 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; +import java.nio.file.Files; +import java.nio.file.Path; + public class TreeAccessTest { @Before @@ -81,4 +85,37 @@ public class TreeAccessTest { .visitAuthor(authorStatement, new TreeAccessCheckContext(10000L, "user1", "")) .getCode()); } + + @Test + public void testLoadTsFileRequiresLoadTsFilePrivilege() throws Exception { + final User mockUser = Mockito.mock(User.class); + Mockito.when(mockUser.getName()).thenReturn("loadUser"); + Mockito.when(mockUser.getUserId()).thenReturn(10002L); + Mockito.when(mockUser.checkSysPrivilege(PrivilegeType.LOAD_TSFILE)).thenReturn(false); + Mockito.when(mockUser.checkSysPrivilege(PrivilegeType.SYSTEM)).thenReturn(false); + AuthorityChecker.getAuthorityFetcher() + .getAuthorCache() + .putUserCache(mockUser.getName(), mockUser); + + final Path tsFile = Files.createTempFile("load-auth", ".tsfile"); + try { + final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.toString()); + final TreeAccessCheckVisitor treeAccessCheckVisitor = new TreeAccessCheckVisitor(); + + Assert.assertEquals( + TSStatusCode.NO_PERMISSION.getStatusCode(), + treeAccessCheckVisitor + .visitLoadFile(statement, new TreeAccessCheckContext(10002L, "loadUser", "")) + .getCode()); + + Mockito.when(mockUser.checkSysPrivilege(PrivilegeType.LOAD_TSFILE)).thenReturn(true); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + treeAccessCheckVisitor + .visitLoadFile(statement, new TreeAccessCheckContext(10002L, "loadUser", "")) + .getCode()); + } finally { + Files.deleteIfExists(tsFile); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java index 509c529b36d..bb43f35f316 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java @@ -238,6 +238,8 @@ public class AuthorStatementTest { checkGrantPrivileges(true, Collections.singletonList(PrivilegeType.SECURITY), "", false, false); checkGrantPrivileges( true, Arrays.asList(PrivilegeType.SYSTEM, PrivilegeType.SECURITY), "", true, false); + checkGrantPrivileges( + true, Collections.singletonList(PrivilegeType.LOAD_TSFILE), "", true, false); checkGrantPrivileges( true, Collections.singletonList(PrivilegeType.SECURITY), "ANY", true, true); @@ -301,6 +303,8 @@ public class AuthorStatementTest { true, Arrays.asList(PrivilegeType.SYSTEM, PrivilegeType.SECURITY), "", false, false); checkRevokePrivileges( true, Arrays.asList(PrivilegeType.SYSTEM, PrivilegeType.SECURITY), "", true, false); + checkRevokePrivileges( + true, Collections.singletonList(PrivilegeType.LOAD_TSFILE), "", false, false); // Illegal privileges combination checkRevokePrivileges( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java index 941794bb074..25644e97c71 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.junit.Assert; import org.junit.Test; +import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -38,10 +39,12 @@ public class LoadTsFileStatementTest { public void testSubStatementsKeepDatabase() throws Exception { final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); final int originalBatchSize = config.getLoadTsFileSubStatementBatchSize(); + final String[] originalAllowedDirs = config.getLoadTsFileAllowedDirs().clone(); final Path tempDir = Files.createTempDirectory("load-tsfile-sub-statements"); try { config.setLoadTsFileSubStatementBatchSize(1); + config.setLoadTsFileAllowedDirs(new String[] {tempDir.toString()}); Files.createFile(tempDir.resolve("a.tsfile")); Files.createFile(tempDir.resolve("b.tsfile")); @@ -54,10 +57,42 @@ public class LoadTsFileStatementTest { subStatement -> Assert.assertEquals("test_db", subStatement.getDatabase())); } finally { config.setLoadTsFileSubStatementBatchSize(originalBatchSize); + config.setLoadTsFileAllowedDirs(originalAllowedDirs); deleteRecursively(tempDir); } } + @Test + public void testLoadSourcePathMustBeInAllowedDirs() throws Exception { + final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + final String[] originalAllowedDirs = config.getLoadTsFileAllowedDirs().clone(); + final Path allowedDir = Files.createTempDirectory("load-tsfile-allowed"); + final Path deniedDir = Files.createTempDirectory("load-tsfile-denied"); + + try { + config.setLoadTsFileAllowedDirs(new String[] {allowedDir.toString()}); + final Path deniedTsFile = Files.createFile(deniedDir.resolve("denied.tsfile")); + final Path traversalTsFile = + allowedDir.resolve("..").resolve(deniedDir.getFileName()).resolve("denied.tsfile"); + + assertLoadSourcePathRejected(deniedTsFile); + assertLoadSourcePathRejected(traversalTsFile); + } finally { + config.setLoadTsFileAllowedDirs(originalAllowedDirs); + deleteRecursively(allowedDir); + deleteRecursively(deniedDir); + } + } + + private static void assertLoadSourcePathRejected(final Path sourcePath) { + try { + new LoadTsFileStatement(sourcePath.toString()); + Assert.fail("Expected disallowed LOAD TSFILE source path to be rejected."); + } catch (final FileNotFoundException e) { + Assert.assertTrue(e.getMessage().contains("outside allowed directories")); + } + } + private static void deleteRecursively(final Path path) throws IOException { if (path == null || !Files.exists(path)) { return; diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 69bcd275250..7606909a039 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -2182,6 +2182,12 @@ load_clean_up_task_execution_delay_time_seconds=1800 # Datatype: int load_write_throughput_bytes_per_second=-1 +# Comma-separated list of directories from which user-issued LOAD TSFILE statements can read. +# If empty, IoTDB only permits LOAD sources under the internal load TsFile directories. +# effectiveMode: hot_reload +# Datatype: String +load_tsfile_allowed_dirs= + # Whether to enable the active listening mode for tsfile loading. # effectiveMode: hot_reload # Datatype: Boolean diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java index 29dc5301a84..5d0b4359840 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java @@ -54,7 +54,8 @@ public enum PrivilegeType { SYSTEM(PrivilegeModelType.SYSTEM), SECURITY(PrivilegeModelType.SYSTEM), - AUDIT(PrivilegeModelType.SYSTEM); + AUDIT(PrivilegeModelType.SYSTEM), + LOAD_TSFILE(PrivilegeModelType.SYSTEM); private final PrivilegeModelType modelType; @@ -134,6 +135,7 @@ public enum PrivilegeType { switch (this) { case MANAGE_USER: case MANAGE_ROLE: + case LOAD_TSFILE: case SYSTEM: case SECURITY: case AUDIT: @@ -160,6 +162,7 @@ public enum PrivilegeType { case USE_PIPE: case MANAGE_DATABASE: case EXTEND_TEMPLATE: + case LOAD_TSFILE: return Arrays.asList(this, PrivilegeType.SYSTEM); default: return Collections.singletonList(this); @@ -208,6 +211,7 @@ public enum PrivilegeType { case WRITE_SCHEMA: case INSERT: case DELETE: + case LOAD_TSFILE: return AuditLogOperation.DML; case MANAGE_USER: case MANAGE_ROLE: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java index 1aae1909ef2..f6c636251e7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java @@ -568,6 +568,8 @@ public class AuthUtils { return PrivilegeType.SECURITY; case 12: return PrivilegeType.AUDIT; + case 13: + return PrivilegeType.LOAD_TSFILE; default: // Not reach here. LOGGER.warn("Not support position"); @@ -603,6 +605,8 @@ public class AuthUtils { return 11; case AUDIT: return 12; + case LOAD_TSFILE: + return 13; default: return -1; } @@ -621,6 +625,7 @@ public class AuthUtils { case USE_PIPE: case MANAGE_DATABASE: case EXTEND_TEMPLATE: + case LOAD_TSFILE: return Arrays.asList(priv, PrivilegeType.SYSTEM); default: return Collections.singletonList(priv); diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index ca766464686..8e94f92ed69 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -829,6 +829,7 @@ systemPrivilege | MANAGE_ROLE | SYSTEM | SECURITY + | LOAD_TSFILE ; objectPrivilege @@ -1493,7 +1494,7 @@ nonReserved | INDEX | INDEXES | IF | IGNORE | IMMEDIATE | INCLUDING | INITIAL | INPUT | INTERVAL | INVOKER | IO | ITERATE | ISOLATION | JSON | KEEP | KEY | KEYS | KILL - | LANGUAGE | LAST | LATERAL | LEADING | LEAVE | LEVEL | LIMIT | LINEAR | LOAD | LOCAL | LOGICAL | LOOP + | LANGUAGE | LAST | LATERAL | LEADING | LEAVE | LEVEL | LIMIT | LINEAR | LOAD | LOAD_TSFILE | LOCAL | LOGICAL | LOOP | MANAGE_ROLE | MANAGE_USER | MAP | MATCH | MATCHED | MATCHES | MATCH_RECOGNIZE | MATERIALIZED | MEASURES | MEMORY_THRESHOLD | METHOD | MERGE | MICROSECOND | MIGRATE | MILLISECOND | MINUTE | MODEL | MODELS | MODIFY | MONTH | NANOSECOND | NESTED | NEXT | NFC | NFD | NFKC | NFKD | NO | NODEID | NONE | NULLIF | NULLS | OBJECT | OF | OFFSET | OMIT | ONE | ONLY | OPTION | ORDINALITY | OUTPUT | OVER | OVERFLOW @@ -1700,6 +1701,7 @@ LINEAR: 'LINEAR'; LIST: 'LIST'; LISTAGG: 'LISTAGG'; LOAD: 'LOAD'; +LOAD_TSFILE: 'LOAD_TSFILE'; LOADED: 'LOADED'; LOCAL: 'LOCAL'; LOCALTIME: 'LOCALTIME';
