This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 79b7b13a3859052765c81471140545a2abc10378
Author: binbin.zheng <binbin.zh...@kyligence.io>
AuthorDate: Tue Nov 15 21:34:22 2022 +0800
KYLIN-5413 disable sample data when has no data query permission
---
.../apache/kylin/rest/service/TableService.java | 12 ++++-
.../rest/service/StreamingTableServiceTest.java | 24 ++++++----
.../kylin/streaming/StreamingMergeEntryTest.java | 23 +++++++--
.../kylin/rest/service/TableServiceTest.java | 56 +++++++++++++++++++++-
4 files changed, 101 insertions(+), 14 deletions(-)
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
index 00379246e7..75ea7362b5 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -152,6 +152,7 @@ import org.apache.kylin.rest.response.TableNameResponse;
import org.apache.kylin.rest.response.TableRefresh;
import org.apache.kylin.rest.response.TableRefreshAll;
import org.apache.kylin.rest.response.TablesAndColumnsResponse;
+import org.apache.kylin.rest.security.ExternalAclProvider;
import org.apache.kylin.rest.security.KerberosLoginManager;
import org.apache.kylin.rest.source.DataSourceState;
import org.apache.kylin.rest.util.AclEvaluate;
@@ -211,6 +212,9 @@ public class TableService extends BasicService {
@Autowired
private AclEvaluate aclEvaluate;
+ @Autowired
+ private AccessService accessService;
+
@Autowired
@Qualifier("kafkaService")
private KafkaService kafkaService;
@@ -451,6 +455,8 @@ public class TableService extends BasicService {
final boolean isAclGreen =
AclPermissionUtil.canUseACLGreenChannel(project, groups);
FileSystem fs = HadoopUtil.getWorkingFileSystem();
List<NDataModel> healthyModels =
projectManager.listHealthyModels(project);
+ Set<String> extPermissionSet =
accessService.getUserNormalExtPermissions(project);
+ boolean hasDataQueryPermission =
extPermissionSet.contains(ExternalAclProvider.DATA_QUERY);
for (val originTable : tables) {
TableDesc table = getAuthorizedTableDesc(project, isAclGreen,
originTable, aclTCRS);
if (Objects.isNull(table)) {
@@ -471,9 +477,11 @@ public class TableService extends BasicService {
TableExtDesc tableExtDesc =
getManager(NTableMetadataManager.class, project).getTableExtIfExists(table);
if (tableExtDesc != null) {
tableDescResponse.setTotalRecords(tableExtDesc.getTotalRows());
-
tableDescResponse.setSamplingRows(tableExtDesc.getSampleRows());
tableDescResponse.setJodID(tableExtDesc.getJodID());
- filterSamplingRows(project, tableDescResponse, isAclGreen,
aclTCRS);
+ if (hasDataQueryPermission) {
+
tableDescResponse.setSamplingRows(tableExtDesc.getSampleRows());
+ filterSamplingRows(project, tableDescResponse, isAclGreen,
aclTCRS);
+ }
}
if (CollectionUtils.isNotEmpty(modelsUsingRootTable)) {
diff --git
a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
index 3494c29a46..42ed55b721 100644
---
a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
+++
b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
@@ -67,8 +67,14 @@ public class StreamingTableServiceTest extends
NLocalFileMetadataTestCase {
@Mock
private AclEvaluate aclEvaluate = Mockito.spy(AclEvaluate.class);
- // @Mock
- // private AclTCRService aclTCRService =
Mockito.spy(AclTCRService.class);
+ @Mock
+ private UserService userService = Mockito.spy(UserService.class);
+
+ @Mock
+ private UserAclService userAclService = Mockito.spy(UserAclService.class);
+
+ @InjectMocks
+ private AccessService accessService = Mockito.spy(new AccessService());
@InjectMocks
private StreamingTableService streamingTableService = Mockito.spy(new
StreamingTableService());
@@ -95,21 +101,24 @@ public class StreamingTableServiceTest extends
NLocalFileMetadataTestCase {
projectManager.forceDropProject("broken_test");
projectManager.forceDropProject("bad_query_test");
- SystemPropertiesCache.setProperty("HADOOP_USER_NAME", "root");
+ System.setProperty("HADOOP_USER_NAME", "root");
ReflectionTestUtils.setField(aclEvaluate, "aclUtil", aclUtil);
ReflectionTestUtils.setField(streamingTableService, "aclEvaluate",
aclEvaluate);
- //ReflectionTestUtils.setField(streamingTableService, "aclTCRService",
aclTCRService);
- //ReflectionTestUtils.setField(streamingTableService,
"userGroupService", userGroupService);
-
//ReflectionTestUtils.setField(streamingTableService,"tableSupporters",
Arrays.asList(tableService));
ReflectionTestUtils.setField(tableService, "aclEvaluate", aclEvaluate);
- //ReflectionTestUtils.setField(tableService, "aclTCRService",
aclTCRService);
ReflectionTestUtils.setField(tableService, "userGroupService",
userGroupService);
+ ReflectionTestUtils.setField(tableService, "accessService",
accessService);
+ ReflectionTestUtils.setField(userAclService, "userService",
userService);
+ ReflectionTestUtils.setField(accessService, "userAclService",
userAclService);
+ ReflectionTestUtils.setField(accessService, "userService",
userService);
val prjManager = NProjectManager.getInstance(getTestConfig());
val prj = prjManager.getProject(PROJECT);
val copy = prjManager.copyForWrite(prj);
prjManager.updateProject(copy);
+
Mockito.when(userService.listSuperAdminUsers()).thenReturn(Arrays.asList("admin"));
+
Mockito.when(userAclService.hasUserAclPermissionInProject(Mockito.anyString(),
Mockito.anyString()))
+ .thenReturn(false);
try {
new JdbcRawRecStore(getTestConfig());
@@ -147,7 +156,6 @@ public class StreamingTableServiceTest extends
NLocalFileMetadataTestCase {
public void testReloadTable() {
val database = "DEFAULT";
- val config = getTestConfig();
try {
val tableDescList = tableService.getTableDesc(PROJECT, true, "",
database, true);
Assert.assertEquals(2, tableDescList.size());
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java
index 59a6912cac..9e6072b010 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java
@@ -376,7 +376,9 @@ public class StreamingMergeEntryTest extends
StreamingTestCase {
@Test
public void testRemoveLastL0Segment_EmptySegment() {
val entry = Mockito.spy(new StreamingMergeEntry());
- ReflectionTestUtils.invokeMethod(entry, "removeLastL0Segment", new
Segments<NDataSegment>());
+ val segments = new Segments<NDataSegment>();
+ ReflectionTestUtils.invokeMethod(entry, "removeLastL0Segment",
segments);
+ Assert.assertTrue(segments.isEmpty());
}
@Test
@@ -402,6 +404,7 @@ public class StreamingMergeEntryTest extends
StreamingTestCase {
}
ReflectionTestUtils.invokeMethod(entry, "removeLastL0Segment",
segments);
+ Assert.assertEquals(3, segments.size());
}
@Test
@@ -429,6 +432,7 @@ public class StreamingMergeEntryTest extends
StreamingTestCase {
}
ReflectionTestUtils.invokeMethod(entry, "removeLastL0Segment",
segments);
+ Assert.assertEquals(2, segments.size());
}
@Test
@@ -456,6 +460,7 @@ public class StreamingMergeEntryTest extends
StreamingTestCase {
}
ReflectionTestUtils.invokeMethod(entry, "removeLastL0Segment",
segments);
+ Assert.assertEquals(3, segments.size());
}
@Test
@@ -465,7 +470,16 @@ public class StreamingMergeEntryTest extends
StreamingTestCase {
val args = new String[] { PROJECT, DATAFLOW_ID + "-err", "5k", "5",
"xx" };
try {
createSparkSession();
- StreamingMergeEntry.main(args);
+ val entry = new StreamingMergeEntry() {
+ public RestSupport createRestSupport(KylinConfig config) {
+ return new RestSupport(config) {
+ public RestResponse<String> execute(HttpRequestBase
httpReqBase, Object param) {
+ return RestResponse.ok("001");
+ }
+ };
+ }
+ };
+ entry.execute(args);
} catch (Exception e) {
Assert.assertTrue(e instanceof ExecuteException);
}
@@ -506,6 +520,8 @@ public class StreamingMergeEntryTest extends
StreamingTestCase {
val seg = NDataSegment.empty();
entry.parseParams(new String[] { PROJECT, DATAFLOW_ID, "32m", "3",
"xx" });
entry.removeSegment(PROJECT, DATAFLOW_ID, seg);
+ val dataflow = NDataflowManager.getInstance(config,
PROJECT).getDataflow(DATAFLOW_ID);
+ Assert.assertNull(dataflow.getSegment(seg.getId()));
}
@Test
@@ -550,6 +566,7 @@ public class StreamingMergeEntryTest extends
StreamingTestCase {
Mockito.doNothing().when(entry).process(PROJECT, DATAFLOW_ID);
Mockito.doReturn(true).when(entry).isGracefulShutdown(PROJECT, jobId);
entry.doExecute();
+ Assert.assertTrue(entry.getStopFlag());
}
@Test
@@ -729,7 +746,7 @@ public class StreamingMergeEntryTest extends
StreamingTestCase {
public void tryReplaceHostAddress() {
val url = "http://localhost:8080";;
StreamingMergeEntry entry = new StreamingMergeEntry();
- val host = entry.tryReplaceHostAddress(url);
+ String host = entry.tryReplaceHostAddress(url);
Assert.assertEquals("http://127.0.0.1:8080";, host);
val url1 = "http://unknow-host-9345:8080";;
diff --git
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
index 4d1c15734c..1b6a97b776 100644
---
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
+++
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
@@ -136,6 +136,15 @@ public class TableServiceTest extends CSVSourceTestCase {
@Mock
private final AclEvaluate aclEvaluate = Mockito.spy(AclEvaluate.class);
+ @InjectMocks
+ private final UserAclService userAclService = Mockito.spy(new
UserAclService());
+
+ @Mock
+ private final UserService userService = Mockito.spy(UserService.class);
+
+ @InjectMocks
+ private AccessService accessService = Mockito.spy(new AccessService());
+
@Rule
public ExpectedException thrown = ExpectedException.none();
@@ -165,6 +174,10 @@ public class TableServiceTest extends CSVSourceTestCase {
ReflectionTestUtils.setField(fusionModelService, "modelService",
modelService);
ReflectionTestUtils.setField(tableService, "fusionModelService",
fusionModelService);
ReflectionTestUtils.setField(tableService, "jobService", jobService);
+ ReflectionTestUtils.setField(userAclService, "userService",
userService);
+ ReflectionTestUtils.setField(accessService, "userAclService",
userAclService);
+ ReflectionTestUtils.setField(accessService, "userService",
userService);
+ ReflectionTestUtils.setField(tableService, "accessService",
accessService);
NProjectManager projectManager =
NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
ProjectInstance projectInstance = projectManager.getProject("default");
LinkedHashMap<String, String> overrideKylinProps =
projectInstance.getOverrideKylinProps();
@@ -174,6 +187,7 @@ public class TableServiceTest extends CSVSourceTestCase {
projectInstance.getOwner(), projectInstance.getDescription(),
overrideKylinProps);
projectManager.updateProject(projectInstance,
projectInstanceUpdate.getName(),
projectInstanceUpdate.getDescription(),
projectInstanceUpdate.getOverrideKylinProps());
+
Mockito.when(userService.listSuperAdminUsers()).thenReturn(Arrays.asList("admin"));
try {
new JdbcRawRecStore(getTestConfig());
} catch (Exception e) {
@@ -191,7 +205,6 @@ public class TableServiceTest extends CSVSourceTestCase {
@Test
public void testGetTableDesc() throws IOException {
-
List<TableDesc> tableDesc = tableService.getTableDesc("default", true,
"", "DEFAULT", true);
Assert.assertEquals(12, tableDesc.size());
List<TableDesc> tableDesc2 = tableService.getTableDesc("default",
true, "TEST_COUNTRY", "DEFAULT", false);
@@ -254,6 +267,9 @@ public class TableServiceTest extends CSVSourceTestCase {
col1.setMaxValue("Zimbabwe");
col1.setNullCount(0);
tableExt.setColumnStats(Lists.newArrayList(col1));
+ List<String[]> sampleRows = new ArrayList<>();
+ sampleRows.add(new String[] { "America" });
+ tableExt.setSampleRows(sampleRows);
tableMgr.mergeAndUpdateTableExt(oldExtDesc, tableExt);
// verify the column stats update successfully
@@ -270,6 +286,44 @@ public class TableServiceTest extends CSVSourceTestCase {
Assert.assertEquals("America", extColumns[0].getMinValue());
Assert.assertEquals("Zimbabwe", extColumns[0].getMaxValue());
Assert.assertEquals(0L, extColumns[0].getNullCount().longValue());
+
+ // check sample rows
+ Assert.assertEquals(1, t.getSamplingRows().size());
+ }
+
+ @Test
+ public void testGetSamplingRows() throws IOException {
+ final String tableIdentity = "DEFAULT.TEST_COUNTRY";
+ final NTableMetadataManager tableMgr =
NTableMetadataManager.getInstance(getTestConfig(), "newten");
+ final TableDesc tableDesc = tableMgr.getTableDesc(tableIdentity);
+ final TableExtDesc oldExtDesc =
tableMgr.getOrCreateTableExt(tableDesc);
+
+ // mock table ext desc
+ TableExtDesc tableExt = new TableExtDesc(oldExtDesc);
+ tableExt.setIdentity(tableIdentity);
+ TableExtDesc.ColumnStats col1 = new TableExtDesc.ColumnStats();
+ col1.setCardinality(100);
+ col1.setTableExtDesc(tableExt);
+ col1.setColumnName(tableDesc.getColumns()[0].getName());
+ col1.setMinValue("America");
+ col1.setMaxValue("Zimbabwe");
+ col1.setNullCount(0);
+ tableExt.setColumnStats(Lists.newArrayList(col1));
+ List<String[]> sampleRows = new ArrayList<>();
+ sampleRows.add(new String[] { "America" });
+ tableExt.setSampleRows(sampleRows);
+ tableMgr.mergeAndUpdateTableExt(oldExtDesc, tableExt);
+
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("test",
"test", Constant.ROLE_MODELER));
+
Mockito.when(userService.isGlobalAdmin(Mockito.anyString())).thenReturn(true);
+
Mockito.when(userAclService.hasUserAclPermissionInProject(Mockito.anyString(),
Mockito.anyString()))
+ .thenReturn(false);
+
+ List<TableDesc> tableExtList = tableService.getTableDesc("newten",
true, "TEST_COUNTRY", "DEFAULT", true);
+ Assert.assertEquals(0, ((TableDescResponse)
tableExtList.get(0)).getSamplingRows().size());
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("ADMIN",
"ADMIN", Constant.ROLE_ADMIN));
}
@Test
