This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 8d59e2f7 [ISSUE-442] Fix Flaky Test:
QuotaManagerTest#testDetectUserResource (#453)
8d59e2f7 is described below
commit 8d59e2f7dc9cc9f9abe5ec5a4d9c8e984ffffb25
Author: jokercurry <[email protected]>
AuthorDate: Fri Dec 30 17:17:25 2022 +0800
[ISSUE-442] Fix Flaky Test: QuotaManagerTest#testDetectUserResource (#453)
### What changes were proposed in this pull request?
`rss.coordinator.quota.default.path` changes the hdfs path to a local path
to avoid accidental loss of files.
### Why are the changes needed?
For #442
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Fixed ut.
---
.../apache/uniffle/coordinator/QuotaManager.java | 9 ++--
.../uniffle/coordinator/QuotaManagerTest.java | 58 ++++------------------
.../src/test/resources/quotaFile.properties | 20 ++++++++
3 files changed, 35 insertions(+), 52 deletions(-)
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/QuotaManager.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/QuotaManager.java
index 69deefee..c9bb635a 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/QuotaManager.java
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/QuotaManager.java
@@ -103,9 +103,12 @@ public class QuotaManager {
try (BufferedReader bufferedReader =
new BufferedReader(new InputStreamReader(fsDataInputStream,
StandardCharsets.UTF_8))) {
while ((content = bufferedReader.readLine()) != null) {
- String user = content.split(Constants.EQUAL_SPLIT_CHAR)[0].trim();
- Integer appNum =
Integer.valueOf(content.split(Constants.EQUAL_SPLIT_CHAR)[1].trim());
- defaultUserApps.put(user, appNum);
+ // to avoid reading comments
+ if (!content.startsWith("#") && !content.isEmpty()) {
+ String user = content.split(Constants.EQUAL_SPLIT_CHAR)[0].trim();
+ Integer appNum =
Integer.valueOf(content.split(Constants.EQUAL_SPLIT_CHAR)[1].trim());
+ defaultUserApps.put(user, appNum);
+ }
}
} catch (Exception e) {
LOG.error("Error occur when parsing file {}", quotaFilePath, e);
diff --git
a/coordinator/src/test/java/org/apache/uniffle/coordinator/QuotaManagerTest.java
b/coordinator/src/test/java/org/apache/uniffle/coordinator/QuotaManagerTest.java
index 734e8d32..feb2b21f 100644
---
a/coordinator/src/test/java/org/apache/uniffle/coordinator/QuotaManagerTest.java
+++
b/coordinator/src/test/java/org/apache/uniffle/coordinator/QuotaManagerTest.java
@@ -17,25 +17,16 @@
package org.apache.uniffle.coordinator;
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.awaitility.Awaitility;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
-import org.junit.jupiter.api.io.TempDir;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -45,47 +36,20 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
* QuotaManager is a manager for resource restriction.
*/
public class QuotaManagerTest {
- private static final Configuration hdfsConf = new Configuration();
- private static MiniDFSCluster cluster;
- @TempDir
- private static File remotePath = new File("hdfs://rss");
- @BeforeEach
- public void setUp() throws IOException {
- hdfsConf.set("fs.defaultFS", remotePath.getAbsolutePath());
- hdfsConf.set("dfs.nameservices", "rss");
- hdfsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR,
remotePath.getAbsolutePath());
- cluster = (new MiniDFSCluster.Builder(hdfsConf)).build();
- }
+ private final String quotaFile =
+
Objects.requireNonNull(this.getClass().getClassLoader().getResource(fileName)).getFile();
+ private static final String fileName = "quotaFile.properties";
- @AfterAll
- public static void clear() {
- cluster.close();
- }
-
@Timeout(value = 10)
@Test
- public void testDetectUserResource() throws Exception {
- final String quotaFile =
- new
Path(remotePath.getAbsolutePath()).getFileSystem(hdfsConf).getName() +
"/quotaFile.properties";
- final FSDataOutputStream fsDataOutputStream =
- new Path(remotePath.toString()).getFileSystem(hdfsConf).create(new
Path(quotaFile));
- String quota1 = "user1 =10";
- String quota2 = "user2= 20";
- String quota3 = "user3 = 30";
- fsDataOutputStream.write(quota1.getBytes(StandardCharsets.UTF_8));
- fsDataOutputStream.write("\n".getBytes(StandardCharsets.UTF_8));
- fsDataOutputStream.write(quota2.getBytes(StandardCharsets.UTF_8));
- fsDataOutputStream.write("\n".getBytes(StandardCharsets.UTF_8));
- fsDataOutputStream.write(quota3.getBytes(StandardCharsets.UTF_8));
- fsDataOutputStream.flush();
- fsDataOutputStream.close();
+ public void testDetectUserResource() {
CoordinatorConf conf = new CoordinatorConf();
conf.set(CoordinatorConf.COORDINATOR_QUOTA_DEFAULT_PATH,
quotaFile);
ApplicationManager applicationManager = new ApplicationManager(conf);
Awaitility.await().timeout(5, TimeUnit.SECONDS).until(
- () -> applicationManager.getDefaultUserApps().size() > 0);
+ () -> applicationManager.getDefaultUserApps().size() > 2);
Integer user1 = applicationManager.getDefaultUserApps().get("user1");
Integer user2 = applicationManager.getDefaultUserApps().get("user2");
@@ -97,8 +61,6 @@ public class QuotaManagerTest {
@Test
public void testQuotaManagerWithoutAccessQuotaChecker() throws Exception {
- final String quotaFile =
- new
Path(remotePath.getAbsolutePath()).getFileSystem(hdfsConf).getName() +
"/quotaFile.properties";
CoordinatorConf conf = new CoordinatorConf();
conf.set(CoordinatorConf.COORDINATOR_QUOTA_DEFAULT_PATH,
quotaFile);
@@ -112,8 +74,6 @@ public class QuotaManagerTest {
@Test
public void testCheckQuota() throws Exception {
- final String quotaFile =
- new
Path(remotePath.getAbsolutePath()).getFileSystem(hdfsConf).getName() +
"/quotaFile.properties";
CoordinatorConf conf = new CoordinatorConf();
conf.set(CoordinatorConf.COORDINATOR_QUOTA_DEFAULT_PATH,
quotaFile);
@@ -127,15 +87,15 @@ public class QuotaManagerTest {
final int i1 = uuid.incrementAndGet();
uuidAndTime.put(String.valueOf(i1), System.currentTimeMillis());
Map<String, Long> appAndTime =
applicationManager.getQuotaManager().getCurrentUserAndApp()
- .computeIfAbsent("user1", x -> uuidAndTime);
+ .computeIfAbsent("user4", x -> uuidAndTime);
// This thread may remove the uuid and put the appId in.
final Thread registerThread = new Thread(() ->
applicationManager.getQuotaManager().registerApplicationInfo("application_test_"
+ i1, appAndTime));
registerThread.start();
final boolean icCheck = applicationManager.getQuotaManager()
- .checkQuota("user1", String.valueOf(i1));
+ .checkQuota("user4", String.valueOf(i1));
registerThread.join();
assertTrue(icCheck);
-
assertEquals(applicationManager.getQuotaManager().getCurrentUserAndApp().get("user1").size(),
5);
+
assertEquals(applicationManager.getQuotaManager().getCurrentUserAndApp().get("user4").size(),
5);
}
}
diff --git a/coordinator/src/test/resources/quotaFile.properties
b/coordinator/src/test/resources/quotaFile.properties
new file mode 100644
index 00000000..7d411028
--- /dev/null
+++ b/coordinator/src/test/resources/quotaFile.properties
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+user1 =10
+user2= 20
+user3 = 30
