This is an automated email from the ASF dual-hosted git repository.
jermy pushed a commit to branch master
in repository
https://gitbox.apache.org/repos/asf/incubator-hugegraph-computer.git
The following commit(s) were added to refs/heads/master by this push:
new ff85e34e feat(k8s): Add MinIO as internal(default) storage (#272)
ff85e34e is described below
commit ff85e34efaecb2a14ae72c0048a427a0aeb61bf3
Author: Aaron Wang <[email protected]>
AuthorDate: Sat Oct 28 18:09:41 2023 +0800
feat(k8s): Add MinIO as internal(default) storage (#272)
---
.../computer/core/config/ComputerOptions.java | 4 +-
.../computer/core/snapshot/SnapshotManager.java | 23 ++++-
.../crd-generate/config/default/kustomization.yaml | 1 +
.../crd-generate/config/manager/manager.yaml | 2 +
.../crd-generate/config/minio/kustomization.yaml | 27 ++++++
.../crd-generate/config/minio/minio_server.yaml | 108 +++++++++++++++++++++
.../manifest/hugegraph-computer-operator.yaml | 107 ++++++++++++++++++++
.../k8s/operator/config/OperatorOptions.java | 8 ++
.../operator/controller/ComputerJobDeployer.java | 4 +
9 files changed, 279 insertions(+), 5 deletions(-)
diff --git
a/computer-api/src/main/java/org/apache/hugegraph/computer/core/config/ComputerOptions.java
b/computer-api/src/main/java/org/apache/hugegraph/computer/core/config/ComputerOptions.java
index 43f3b6ca..6978bf08 100644
---
a/computer-api/src/main/java/org/apache/hugegraph/computer/core/config/ComputerOptions.java
+++
b/computer-api/src/main/java/org/apache/hugegraph/computer/core/config/ComputerOptions.java
@@ -235,7 +235,7 @@ public class ComputerOptions extends OptionHolder {
"snapshot.minio_access_key",
"The access key of MinIO.",
null,
- ""
+ "minioadmin"
);
public static final ConfigOption<String> SNAPSHOT_MINIO_SECRET_KEY =
@@ -243,7 +243,7 @@ public class ComputerOptions extends OptionHolder {
"snapshot.minio_secret_key",
"The secret key of MinIO.",
null,
- ""
+ "minioadmin"
);
public static final ConfigOption<String> SNAPSHOT_MINIO_BUCKET_NAME =
diff --git
a/computer-core/src/main/java/org/apache/hugegraph/computer/core/snapshot/SnapshotManager.java
b/computer-core/src/main/java/org/apache/hugegraph/computer/core/snapshot/SnapshotManager.java
index 4734ee5a..674d95d7 100644
---
a/computer-core/src/main/java/org/apache/hugegraph/computer/core/snapshot/SnapshotManager.java
+++
b/computer-core/src/main/java/org/apache/hugegraph/computer/core/snapshot/SnapshotManager.java
@@ -17,8 +17,10 @@
package org.apache.hugegraph.computer.core.snapshot;
+import io.minio.BucketExistsArgs;
import io.minio.DownloadObjectArgs;
import io.minio.ListObjectsArgs;
+import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
import io.minio.RemoveObjectsArgs;
import io.minio.Result;
@@ -89,14 +91,29 @@ public class SnapshotManager implements Manager {
@Override
public void init(Config config) {
String endpoint = config.get(ComputerOptions.SNAPSHOT_MINIO_ENDPOINT);
- String accessKey =
config.get(ComputerOptions.SNAPSHOT_MINIO_ACCESS_KEY);
- String secretKey =
config.get(ComputerOptions.SNAPSHOT_MINIO_SECRET_KEY);
this.bucketName =
config.get(ComputerOptions.SNAPSHOT_MINIO_BUCKET_NAME);
- if (StringUtils.isNotEmpty(endpoint)) {
+
+ if (StringUtils.isNotEmpty(endpoint) &&
StringUtils.isNotEmpty(this.bucketName)) {
+ String accessKey =
config.get(ComputerOptions.SNAPSHOT_MINIO_ACCESS_KEY);
+ String secretKey =
config.get(ComputerOptions.SNAPSHOT_MINIO_SECRET_KEY);
this.minioClient = MinioClient.builder()
.endpoint(endpoint)
.credentials(accessKey, secretKey)
.build();
+
+ try {
+ boolean bucketExist = this.minioClient.bucketExists(
+ BucketExistsArgs.builder()
+ .bucket(this.bucketName)
+ .build());
+ if (!bucketExist) {
+ this.minioClient.makeBucket(MakeBucketArgs.builder()
+
.bucket(this.bucketName)
+ .build());
+ }
+ } catch (Exception e) {
+ throw new ComputerException("Failed to initialize bucket %s",
this.bucketName, e);
+ }
}
}
diff --git
a/computer-k8s-operator/crd-generate/config/default/kustomization.yaml
b/computer-k8s-operator/crd-generate/config/default/kustomization.yaml
index aa3037f2..cb4e1459 100644
--- a/computer-k8s-operator/crd-generate/config/default/kustomization.yaml
+++ b/computer-k8s-operator/crd-generate/config/default/kustomization.yaml
@@ -35,6 +35,7 @@ resources:
- ../rbac
- ../etcd
- ../manager
+- ../minio
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK]
prefix including the one in
# crd/kustomization.yaml
- ../webhook
diff --git a/computer-k8s-operator/crd-generate/config/manager/manager.yaml
b/computer-k8s-operator/crd-generate/config/manager/manager.yaml
index ec67232f..3d5cc1ae 100644
--- a/computer-k8s-operator/crd-generate/config/manager/manager.yaml
+++ b/computer-k8s-operator/crd-generate/config/manager/manager.yaml
@@ -60,6 +60,8 @@ spec:
value: "6"
- name: INTERNAL_ETCD_URL
value:
"http://hugegraph-computer-operator-etcd.hugegraph-computer-operator-system:2379";
+ - name: INTERNAL_MINIO_URL
+ value:
"http://hugegraph-computer-operator-minio.hugegraph-computer-operator-system:9000";
- name: LOG_LEVEL
value: "INFO"
- name: AUTO_DESTROY_POD
diff --git a/computer-k8s-operator/crd-generate/config/minio/kustomization.yaml
b/computer-k8s-operator/crd-generate/config/minio/kustomization.yaml
new file mode 100644
index 00000000..6776474a
--- /dev/null
+++ b/computer-k8s-operator/crd-generate/config/minio/kustomization.yaml
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+resources:
+ - minio_server.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+ - name: minio
+ newName: quay.io/minio/minio
+ newTag: RELEASE.2023-10-16T04-13-43Z
+commonLabels:
+ app.kubernetes.io/name: hugegraph-computer-operator-minio
+ app.kubernetes.io/version: v1
diff --git a/computer-k8s-operator/crd-generate/config/minio/minio_server.yaml
b/computer-k8s-operator/crd-generate/config/minio/minio_server.yaml
new file mode 100644
index 00000000..02bc642d
--- /dev/null
+++ b/computer-k8s-operator/crd-generate/config/minio/minio_server.yaml
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ name: local-storage
+ namespace: system
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ service.app: hugegraph-computer-operator-minio
+ name: minio
+ namespace: system
+ annotations:
+ service.alpha.kubernetes.io/tolerate-unready-endpoints: 'true'
+spec:
+ ports:
+ - name: client
+ port: 9000
+ protocol: TCP
+ targetPort: 9090
+ selector:
+ service.app: hugegraph-computer-operator-minio
+ clusterIP: None
+ type: ClusterIP
+ sessionAffinity: None
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: minio
+ namespace: system
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ service.app: hugegraph-computer-operator-minio
+ template:
+ metadata:
+ labels:
+ service.app: hugegraph-computer-operator-minio
+ spec:
+ hostname: hugegraph-computer-operator-minio-0
+ subdomain: hugegraph-computer-operator-minio
+ containers:
+ - name: hugegraph-computer-operator-minio
+ image: minio:latest
+ command:
+ - /bin/bash
+ - -c
+ args:
+ - minio server /data --console-address :9090
+ env:
+ # MinIO access key and secret key
+ - name: MINIO_ACCESS_KEY
+ value: "minioadmin"
+ - name: MINIO_SECRET_KEY
+ value: "minioadmin"
+ ports:
+ - name: client
+ containerPort: 9090
+ volumeMounts:
+ - mountPath: /data
+ name: localvolume # Corresponds to the `spec.volumes` Persistent
Volume
+ livenessProbe:
+ httpGet:
+ path: /minio/health/live
+ port: 9000
+ scheme: HTTP
+ initialDelaySeconds: 120
+ periodSeconds: 30
+ timeoutSeconds: 10
+ successThreshold: 1
+ failureThreshold: 3
+ readinessProbe:
+ httpGet:
+ path: /minio/health/ready
+ # TODO: check the port
+ port: 9000
+ scheme: HTTP
+ initialDelaySeconds: 120
+ periodSeconds: 15
+ timeoutSeconds: 10
+ successThreshold: 1
+ failureThreshold: 3
+ volumes:
+ - name: localvolume
+ hostPath: # MinIO generally recommends using locally-attached volumes
+ path: /mnt/disk1/data # Specify a path to a local drive or volume
on the Kubernetes worker node
+ type: DirectoryOrCreate # The path to the last directory must exist
diff --git a/computer-k8s-operator/manifest/hugegraph-computer-operator.yaml
b/computer-k8s-operator/manifest/hugegraph-computer-operator.yaml
index aebc27be..9949da76 100644
--- a/computer-k8s-operator/manifest/hugegraph-computer-operator.yaml
+++ b/computer-k8s-operator/manifest/hugegraph-computer-operator.yaml
@@ -23,6 +23,17 @@ metadata:
control-plane: controller-manager
name: hugegraph-computer-operator-system
---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ labels:
+ app.kubernetes.io/name: hugegraph-computer-operator-minio
+ app.kubernetes.io/version: v1
+ name: hugegraph-computer-operator-local-storage
+ namespace: system
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+---
apiVersion: v1
kind: ServiceAccount
metadata:
@@ -436,6 +447,31 @@ spec:
---
apiVersion: v1
kind: Service
+metadata:
+ annotations:
+ service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+ labels:
+ app.kubernetes.io/name: hugegraph-computer-operator-minio
+ app.kubernetes.io/version: v1
+ service.app: hugegraph-computer-operator-minio
+ name: hugegraph-computer-operator-minio
+ namespace: hugegraph-computer-operator-system
+spec:
+ clusterIP: None
+ ports:
+ - name: client
+ port: 9000
+ protocol: TCP
+ targetPort: 9090
+ selector:
+ app.kubernetes.io/name: hugegraph-computer-operator-minio
+ app.kubernetes.io/version: v1
+ service.app: hugegraph-computer-operator-minio
+ sessionAffinity: None
+ type: ClusterIP
+---
+apiVersion: v1
+kind: Service
metadata:
name: hugegraph-computer-operator-webhook-service
namespace: hugegraph-computer-operator-system
@@ -525,6 +561,8 @@ spec:
value: "6"
- name: INTERNAL_ETCD_URL
value:
http://hugegraph-computer-operator-etcd.hugegraph-computer-operator-system:2379
+ - name: INTERNAL_MINIO_URL
+ value:
http://hugegraph-computer-operator-minio.hugegraph-computer-operator-system:9000
- name: LOG_LEVEL
value: INFO
- name: AUTO_DESTROY_POD
@@ -620,6 +658,75 @@ spec:
hostname: hugegraph-computer-operator-etcd-0
subdomain: hugegraph-computer-operator-etcd
---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+ app.kubernetes.io/name: hugegraph-computer-operator-minio
+ app.kubernetes.io/version: v1
+ name: hugegraph-computer-operator-minio
+ namespace: hugegraph-computer-operator-system
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: hugegraph-computer-operator-minio
+ app.kubernetes.io/version: v1
+ service.app: hugegraph-computer-operator-minio
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: hugegraph-computer-operator-minio
+ app.kubernetes.io/version: v1
+ service.app: hugegraph-computer-operator-minio
+ spec:
+ containers:
+ - args:
+ - minio server /data --console-address :9090
+ command:
+ - /bin/bash
+ - -c
+ env:
+ - name: MINIO_ACCESS_KEY
+ value: minioadmin
+ - name: MINIO_SECRET_KEY
+ value: minioadmin
+ image: quay.io/minio/minio:RELEASE.2023-10-16T04-13-43Z
+ livenessProbe:
+ failureThreshold: 3
+ httpGet:
+ path: /minio/health/live
+ port: 9000
+ scheme: HTTP
+ initialDelaySeconds: 120
+ periodSeconds: 30
+ successThreshold: 1
+ timeoutSeconds: 10
+ name: hugegraph-computer-operator-minio
+ ports:
+ - containerPort: 9090
+ name: client
+ readinessProbe:
+ failureThreshold: 3
+ httpGet:
+ path: /minio/health/ready
+ port: 9000
+ scheme: HTTP
+ initialDelaySeconds: 120
+ periodSeconds: 15
+ successThreshold: 1
+ timeoutSeconds: 10
+ volumeMounts:
+ - mountPath: /data
+ name: localvolume
+ hostname: hugegraph-computer-operator-minio-0
+ subdomain: hugegraph-computer-operator-minio
+ volumes:
+ - hostPath:
+ path: /mnt/disk1/data
+ type: DirectoryOrCreate
+ name: localvolume
+---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
diff --git
a/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/config/OperatorOptions.java
b/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/config/OperatorOptions.java
index 677c7cc1..f9ac6610 100644
---
a/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/config/OperatorOptions.java
+++
b/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/config/OperatorOptions.java
@@ -134,6 +134,14 @@ public class OperatorOptions extends OptionHolder {
"http://127.0.0.1:2379";
);
+ public static final ConfigOption<String> INTERNAL_MINIO_URL =
+ new ConfigOption<>(
+ "k8s.internal_minio_url",
+ "The internal minio url for operator system.",
+ disallowEmpty(),
+ "http://127.0.0.1:9000";
+ );
+
public static final ConfigOption<Boolean> AUTO_DESTROY_POD =
new ConfigOption<>(
"k8s.auto_destroy_pod",
diff --git
a/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java
b/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java
index 9d3aa61e..08775088 100644
---
a/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java
+++
b/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java
@@ -101,12 +101,14 @@ public class ComputerJobDeployer {
private static final Long TERMINATION_GRACE_PERIOD = 180L;
private final String internalEtcdUrl;
+ private final String internalMinIOUrl;
private final String timezone;
public ComputerJobDeployer(NamespacedKubernetesClient kubeClient,
HugeConfig config) {
this.kubeClient = kubeClient;
this.internalEtcdUrl = config.get(OperatorOptions.INTERNAL_ETCD_URL);
+ this.internalMinIOUrl = config.get(OperatorOptions.INTERNAL_MINIO_URL);
this.timezone = config.get(OperatorOptions.TIMEZONE);
}
@@ -186,6 +188,8 @@ public class ComputerJobDeployer {
config.put(ComputerOptions.RPC_SERVER_HOST_NAME, ip);
config.putIfAbsent(ComputerOptions.BSP_ETCD_ENDPOINTS.name(),
this.internalEtcdUrl);
+ config.putIfAbsent(ComputerOptions.SNAPSHOT_MINIO_ENDPOINT.name(),
+ this.internalMinIOUrl);
String transportPort = config.get(
ComputerOptions.TRANSPORT_SERVER_PORT.name());
