This is an automated email from the ASF dual-hosted git repository. liyang-kylin pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit afe87e28c92a86652648fb48f774e83bec133b2e Author: Li Yang <[email protected]> AuthorDate: Fri Oct 2 21:35:03 2026 +0800 KYLIN-6110 harden metadata dump URL handling --- .../org/apache/kylin/util/MetadataDumpUtil.java | 81 ++++++++++++++++++++-- .../org/apache/kylin/rest/service/JobService.java | 1 + .../apache/kylin/rest/service/JobServiceTest.java | 26 +++++++ 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/src/core-metadata/src/main/java/org/apache/kylin/util/MetadataDumpUtil.java b/src/core-metadata/src/main/java/org/apache/kylin/util/MetadataDumpUtil.java index a67de9a777..cc94aa08dc 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/util/MetadataDumpUtil.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/util/MetadataDumpUtil.java @@ -18,17 +18,25 @@ package org.apache.kylin.util; +import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_PARAMETER; + import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.Map; import java.util.Objects; import java.util.Properties; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.Path; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.StorageURL; +import org.apache.kylin.common.exception.KylinException; import org.apache.kylin.common.persistence.RawResource; import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.persistence.metadata.FileSystemMetadataStore; import org.apache.kylin.common.persistence.metadata.MetadataStore; import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams; import org.apache.kylin.guava30.shaded.common.collect.Maps; @@ -43,12 +51,9 @@ public class MetadataDumpUtil { public static void dumpMetadata(DumpInfo info) throws Exception { KylinConfig config = KylinConfig.getInstanceFromEnv(); + validateMetadataStoreUrl(config, info); String metaDumpUrl = info.getDistMetaUrl(); - if (StringUtils.isEmpty(metaDumpUrl)) { - throw new RuntimeException("Missing metaUrl"); - } - final Properties props = config.exportToProperties(); props.setProperty("kylin.metadata.url", metaDumpUrl); @@ -63,6 +68,74 @@ public class MetadataDumpUtil { log.debug("Dump metadata finished."); } + public static void validateMetadataStoreUrl(KylinConfig config, DumpInfo info) { + validateMetadataStoreUrl(config, info.getProject(), info.getDistMetaUrl()); + } + + public static void validateMetadataStoreUrl(KylinConfig config, String project, String metaDumpUrl) { + if (StringUtils.isBlank(metaDumpUrl)) { + throw invalidMetadataStoreUrl(); + } + + StorageURL storageUrl; + try { + storageUrl = StorageURL.valueOf(metaDumpUrl); + } catch (RuntimeException e) { + throw invalidMetadataStoreUrl(); + } + String scheme = storageUrl.getScheme(); + if (!FileSystemMetadataStore.HDFS_SCHEME.equalsIgnoreCase(scheme) + && !FileSystemMetadataStore.FILE_SCHEME.equalsIgnoreCase(scheme)) { + throw invalidMetadataStoreUrl(); + } + if (!config.getMetadataUrlPrefix().equals(storageUrl.getIdentifier())) { + throw invalidMetadataStoreUrl(); + } + if (storageUrl.getAllParameters().size() != 1 || !storageUrl.containsParameter("path") + || StringUtils.isBlank(storageUrl.getParameter("path"))) { + throw invalidMetadataStoreUrl(); + } + + Path metadataStorePath = new Path(storageUrl.getParameter("path")); + Path allowedRoot = new Path(new Path(config.getWorkingDirectoryWithConfiguredFs(project)), "job_tmp"); + if (!isStrictDescendant(metadataStorePath, allowedRoot)) { + throw invalidMetadataStoreUrl(); + } + } + + private static boolean isStrictDescendant(Path candidate, Path root) { + try { + URI candidateUri = normalize(candidate.toUri()); + URI rootUri = normalize(root.toUri()); + String candidateScheme = StringUtils.defaultIfBlank(candidateUri.getScheme(), rootUri.getScheme()); + String candidateAuthority = StringUtils.defaultIfBlank(candidateUri.getAuthority(), rootUri.getAuthority()); + if (!StringUtils.equalsIgnoreCase(candidateScheme, rootUri.getScheme()) + || !StringUtils.equalsIgnoreCase(candidateAuthority, rootUri.getAuthority())) { + return false; + } + + String candidatePath = candidateUri.getPath(); + String rootPath = rootUri.getPath(); + if (StringUtils.isBlank(candidatePath) || StringUtils.isBlank(rootPath)) { + return false; + } + if (!rootPath.endsWith("/")) { + rootPath += "/"; + } + return candidatePath.startsWith(rootPath); + } catch (IllegalArgumentException | URISyntaxException e) { + return false; + } + } + + private static URI normalize(URI uri) throws URISyntaxException { + return new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), null, null).normalize(); + } + + private static KylinException invalidMetadataStoreUrl() { + return new KylinException(INVALID_PARAMETER, "Invalid metadata dump URL."); + } + private static void dumpMetadataViaTmpDir(KylinConfig config, MetadataStore dstMetadataStore, DumpInfo info) throws IOException { File tmpDir = File.createTempFile("kylin_job_meta", EMPTY); diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java index 589d11b2c1..294dc10b23 100644 --- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java +++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java @@ -295,6 +295,7 @@ public class JobService extends BasicService { Set<String> dumpList = new LinkedHashSet<>(); KylinConfig config = KylinConfig.getInstanceFromEnv(); + MetadataDumpUtil.validateMetadataStoreUrl(config, project, distMetaUrl); NDataflow df = NDataflowManager.getInstance(config, project).getDataflow(modelId); dumpList.addAll(df.collectPrecalculationResource()); dumpList.addAll(getLogicalViewMetaDumpList(config, project, viewTable, modelId)); diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java index 011246ff95..b3c83987e7 100644 --- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java +++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java @@ -82,6 +82,8 @@ import org.apache.kylin.rest.response.ExecutableResponse; import org.apache.kylin.rest.response.JobStatisticsResponse; import org.apache.kylin.rest.util.AclEvaluate; import org.apache.kylin.rest.util.AclUtil; +import org.apache.kylin.util.DumpInfo; +import org.apache.kylin.util.MetadataDumpUtil; import org.awaitility.Duration; import org.junit.After; import org.junit.Assert; @@ -695,6 +697,30 @@ public class JobServiceTest extends NLocalFileMetadataTestCase { Assert.assertThrows(AccessDeniedException.class, () -> jobService.updateDumpedMetadata(request)); } + @Test + public void testUpdateDumpedMetadataRejectsUntrustedMetadataStore() { + ReplaceMetaRequest request = new ReplaceMetaRequest(); + request.setProject(project); + request.setDistMetaUrl("kylin_metadata@jdbc,driverClassName=org.h2.Driver,url=jdbc:h2:mem:test"); + Assert.assertThrows(KylinException.class, () -> jobService.updateDumpedMetadata(request)); + } + + @Test + public void testUpdateDumpedMetadataRejectsMetadataStoreOutsideJobTemp() { + ReplaceMetaRequest request = new ReplaceMetaRequest(); + request.setProject(project); + request.setDistMetaUrl(getTestConfig().getMetadataUrlPrefix() + "@hdfs,path=/tmp/not-job-temp/meta"); + Assert.assertThrows(KylinException.class, () -> jobService.updateDumpedMetadata(request)); + } + + @Test + public void testValidateGeneratedJobTempMetadataStore() { + DumpInfo dumpInfo = new DumpInfo(project, + getTestConfig().getJobTmpMetaStoreUrl(project, RandomUtil.randomUUIDStr()).toString(), Sets.newHashSet(), + DumpInfo.DumpType.DATA_LOADING); + MetadataDumpUtil.validateMetadataStoreUrl(getTestConfig(), dumpInfo); + } + @Test public void testGetJobStats() throws ParseException { JobStatisticsResponse jobStats = jobService.getJobStats("default", Long.MIN_VALUE, Long.MAX_VALUE);
