This is an automated email from the ASF dual-hosted git repository.
HTHou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb-extras.git
The following commit(s) were added to refs/heads/master by this push:
new b478305 Add metric scrape module (#111)
b478305 is described below
commit b478305d072bf7cb4b9a0f3f4e1b5cc86c1bd23e
Author: Haonan <[email protected]>
AuthorDate: Wed Jul 22 09:25:24 2026 +0800
Add metric scrape module (#111)
---
README-zh.md | 7 +-
README.md | 7 +-
metric-scrape/Dockerfile | 26 ++
metric-scrape/README.md | 146 ++++++++
metric-scrape/conf/metric-scrape.yml | 61 ++++
metric-scrape/k8s/configmap.yaml | 60 ++++
metric-scrape/k8s/deployment.yaml | 64 ++++
metric-scrape/pom.xml | 112 ++++++
.../apache/iotdb/metricscrape/IdentifierUtils.java | 50 +++
.../iotdb/metricscrape/MetricScrapeConfig.java | 195 +++++++++++
.../metricscrape/MetricScrapeConfigLoader.java | 256 ++++++++++++++
.../iotdb/metricscrape/MetricScrapeHttpClient.java | 59 ++++
.../iotdb/metricscrape/MetricScrapeMain.java | 63 ++++
.../iotdb/metricscrape/MetricScrapePlanner.java | 73 ++++
.../iotdb/metricscrape/MetricScrapeService.java | 119 +++++++
.../iotdb/metricscrape/MetricScrapeTarget.java | 62 ++++
.../iotdb/metricscrape/MetricTableWriter.java | 211 +++++++++++
.../iotdb/metricscrape/PrometheusSample.java | 69 ++++
.../iotdb/metricscrape/PrometheusTextParser.java | 389 +++++++++++++++++++++
.../iotdb/metricscrape/ScheduledExecutorUtil.java | 53 +++
.../metricscrape/MetricScrapeConfigLoaderTest.java | 41 +++
.../metricscrape/PrometheusTextParserTest.java | 112 ++++++
pom.xml | 1 +
23 files changed, 2232 insertions(+), 4 deletions(-)
diff --git a/README-zh.md b/README-zh.md
index eaea49f..ffe425f 100644
--- a/README-zh.md
+++ b/README-zh.md
@@ -61,7 +61,9 @@ IoTDB (物联网数据库) 是一个专为物联网 (IoT) 场景设计的时序
5. **MyBatis 生成器**：数据库访问代码生成工具
-6. **示例**：展示 IoTDB 与各种技术结合使用的示例应用程序和代码示例
+6. **Metric Scrape**：用于 IoTDB 表模型的独立 Prometheus 文本格式指标抓取工具
+
+7. **示例**：展示 IoTDB 与各种技术结合使用的示例应用程序和代码示例
## 环境要求
@@ -83,7 +85,7 @@ IoTDB (物联网数据库) 是一个专为物联网 (IoT) 场景设计的时序
2. 使用 Maven 构建项目：
```bash
- # 构建整个项目（包括 distributions、iotdb-collector、mybatis-generator）
+ # 构建整个项目（包括 distributions、iotdb-collector、metric-scrape、mybatis-generator）
mvn clean package -DskipTests
# 或者构建所有组件
@@ -310,6 +312,7 @@ mvn clean package
-Pwith-all-connectors,with-examples,with-springboot -DskipTest
- [Flink IoTDB 连接器](/connectors/flink-iotdb-connector/README.md)
- [Grafana 插件](/connectors/grafana-plugin/README.md)
- [IoTDB Spring Boot Starter](/iotdb-spring-boot-starter/README.md)
+- [Metric Scrape](/metric-scrape/README.md)
- [Kubernetes Helm Charts](/helm/README.md)
- [IoTDB Operator](/iotdb-operator/README.md)
diff --git a/README.md b/README.md
index 4e2ad89..270ead1 100644
--- a/README.md
+++ b/README.md
@@ -61,7 +61,9 @@ This repository includes:
5. **MyBatis Generator**: Code generation tools for database access
-6. **Examples**: Sample applications and code examples demonstrating the use
of IoTDB with various technologies
+6. **Metric Scrape**: Standalone Prometheus text exposition scraper for the
IoTDB table model
+
+7. **Examples**: Sample applications and code examples demonstrating the use
of IoTDB with various technologies
## Prerequisites
@@ -83,7 +85,7 @@ To build the project from source, follow these steps:
2. Build the project with Maven:
```bash
- # Build the entire project (includes
distributions,iotdb-collector,mybatis-generator)
+ # Build the entire project (includes
distributions,iotdb-collector,metric-scrape,mybatis-generator)
mvn clean package -DskipTests
# Or build all
@@ -312,6 +314,7 @@ You can also refer to module-specific documentation:
- [Flink IoTDB Connector](/connectors/flink-iotdb-connector/README.md)
- [Grafana Plugin](/connectors/grafana-plugin/README.md)
- [IoTDB Spring Boot Starter](/iotdb-spring-boot-starter/README.md)
+- [Metric Scrape](/metric-scrape/README.md)
- [Kubernetes Helm Charts](/helm/README.md)
- [IoTDB Operator](/iotdb-operator/README.md)
diff --git a/metric-scrape/Dockerfile b/metric-scrape/Dockerfile
new file mode 100644
index 0000000..4df24f6
--- /dev/null
+++ b/metric-scrape/Dockerfile
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+FROM eclipse-temurin:17-jre-jammy
+
+WORKDIR /app
+
+COPY target/metric-scrape-2.0.4-SNAPSHOT.jar /app/metric-scrape.jar
+
+ENTRYPOINT ["java", "-jar", "/app/metric-scrape.jar", "-c",
"/app/conf/metric-scrape.yml"]
diff --git a/metric-scrape/README.md b/metric-scrape/README.md
new file mode 100644
index 0000000..13cab17
--- /dev/null
+++ b/metric-scrape/README.md
@@ -0,0 +1,146 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# Metric Scrape
+
+Metric Scrape is a standalone service that periodically scrapes Prometheus
text exposition
+endpoints and writes metrics into the Apache IoTDB table model through the
+`org.apache.iotdb:iotdb-session` client.
+
+It is a pull scraper, not a Prometheus remote write receiver.
+
+## Build
+
+```bash
+mvn -pl metric-scrape -am clean package -DskipTests
+```
+
+The runnable jar is generated at:
+
+```text
+metric-scrape/target/metric-scrape-2.0.4-SNAPSHOT.jar
+```
+
+## Run
+
+```bash
+java -jar metric-scrape/target/metric-scrape-2.0.4-SNAPSHOT.jar \
+ -c metric-scrape/conf/metric-scrape.yml
+```
+
+If `-c` is omitted, the service reads `conf/metric-scrape.yml` from the
current working
+directory.
+
+## Configuration
+
+```yaml
+global:
+ scrape_interval: 15s
+ scrape_timeout: 10s
+ iotdb_database: 'metrics'
+ iotdb_username: 'root'
+ iotdb_password: 'root'
+ iotdb_urls: ['127.0.0.1:6667']
+
+scrape_configs:
+ - job_name: 'iotdbModel'
+ static_configs:
+ - targets: ['localhost:9090']
+ metrics_path: /metrics
+ relabel_configs:
+ - target_label: k8s_cluster
+ replacement: example-cluster
+ - target_label: k8s_namespace
+ replacement: default
+ - target_label: instance_id
+ replacement: model-service-local
+```
+
+Supported fields:
+
+| Field | Description |
+| --- | --- |
+| `global.scrape_interval` | Default scrape interval. Supports `ms`, `s`, `m`,
`h`; no suffix means seconds. |
+| `global.scrape_timeout` | Default HTTP timeout. Supports the same duration
format. |
+| `global.iotdb_database` | Target IoTDB table-model database. |
+| `global.iotdb_username` | Session username. |
+| `global.iotdb_password` | Session password. |
+| `global.iotdb_urls` | IoTDB node URLs, for example `127.0.0.1:6667`. |
+| `global.write_batch_size` | Optional tablet batch size. Default is `1000`. |
+| `scrape_configs[].job_name` | Scrape job name. It is written as tag column
`job_name`. |
+| `scrape_configs[].scheme` | Optional target scheme. Default is `http`. |
+| `scrape_configs[].metrics_path` | Metrics path. Default is `/metrics`. |
+| `scrape_configs[].scrape_interval` | Optional interval override for this
job. |
+| `scrape_configs[].scrape_timeout` | Optional timeout override for this job. |
+| `scrape_configs[].static_configs[].targets` | Target host list. |
+| `scrape_configs[].static_configs[].labels` | Extra labels for targets. They
are written as tag columns. |
+| `scrape_configs[].relabel_configs[].target_label` | Constant target label
name. |
+| `scrape_configs[].relabel_configs[].replacement` | Constant target label
value. |
+
+Only constant relabeling with `target_label` and `replacement` is supported
for now.
+
+## Data Model
+
+Metric Scrape writes into the IoTDB table model:
+
+| Prometheus content | Table model mapping |
+| --- | --- |
+| `global.iotdb_database` | Database name. |
+| `# HELP` metric family name | Table name. |
+| Sample metric name | Field column name. |
+| Sample value | Field column value, `DOUBLE`. |
+| `job_name` | Tag column. |
+| Target address | Tag column `instance`. |
+| Sample labels, static labels, relabel replacement labels | Tag columns,
`STRING`. |
+| Sample timestamp | Row time. If missing, the current scrape time in
milliseconds is used. |
+
+For histogram and summary samples, the longest matching `# HELP` metric family
is used as the
+table name. For example, `request_duration_seconds_sum` and
+`request_duration_seconds_count` are written into table
`request_duration_seconds` if the text
+contains `# HELP request_duration_seconds ...`.
+
+Metric names and label names are normalized before being used as table or
column identifiers:
+characters other than letters, digits, and `_` are replaced by `_`, and
identifiers starting with
+a digit get a leading `_`.
+
+## Docker And Kubernetes
+
+The module includes a `Dockerfile` and example Kubernetes manifests under
`k8s/`.
+
+```bash
+docker build -t metric-scrape:latest metric-scrape
+kubectl apply -f metric-scrape/k8s/configmap.yaml
+kubectl apply -f metric-scrape/k8s/deployment.yaml
+```
+
+Update the image name, IoTDB URLs, credentials, and scrape targets before
deploying to a real
+cluster.
+
+## Example Query
+
+```sql
+USE metrics;
+
+SELECT time, job_name, instance, env, http_server_requests_seconds_count
+FROM http_server_requests_seconds
+ORDER BY time DESC
+LIMIT 10;
+```
diff --git a/metric-scrape/conf/metric-scrape.yml
b/metric-scrape/conf/metric-scrape.yml
new file mode 100644
index 0000000..645d62d
--- /dev/null
+++ b/metric-scrape/conf/metric-scrape.yml
@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+global:
+ scrape_interval: 15s
+ scrape_timeout: 10s
+ iotdb_database: 'metrics'
+ iotdb_username: 'root'
+ iotdb_password: 'root'
+ iotdb_urls: ['127.0.0.1:6667']
+
+scrape_configs:
+ - job_name: 'iotdbModel'
+ static_configs:
+ - targets: ['localhost:9090']
+ metrics_path: /metrics
+ relabel_configs:
+ - target_label: k8s_cluster
+ replacement: example-cluster
+ - target_label: k8s_namespace
+ replacement: default
+ - target_label: instance_id
+ replacement: model-service-local
+
+ - job_name: 'iotdbWeb'
+ metrics_path: /actuator/prometheus
+ static_configs:
+ - targets: ['localhost:8080']
+ labels: {env: local, instance_id: web-service-local}
+ relabel_configs:
+ - target_label: k8s_cluster
+ replacement: example-cluster
+ - target_label: k8s_namespace
+ replacement: default
+
+ - job_name: 'iotdbAdmin'
+ metrics_path: /actuator/prometheus
+ static_configs:
+ - targets: ['localhost:8081']
+ labels: {env: local, instance_id: admin-service-local}
+ relabel_configs:
+ - target_label: k8s_cluster
+ replacement: example-cluster
+ - target_label: k8s_namespace
+ replacement: default
diff --git a/metric-scrape/k8s/configmap.yaml b/metric-scrape/k8s/configmap.yaml
new file mode 100644
index 0000000..00fcd59
--- /dev/null
+++ b/metric-scrape/k8s/configmap.yaml
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: metric-scrape-config
+ namespace: default
+ labels:
+ app.kubernetes.io/name: metric-scrape
+ app.kubernetes.io/part-of: iotdb-monitoring
+data:
+ metric-scrape.yml: |
+ global:
+ scrape_interval: 30s
+ scrape_timeout: 10s
+ iotdb_database: 'metrics'
+ iotdb_username: 'root'
+ iotdb_password: 'root'
+ iotdb_urls: ['iotdb-service.default.svc.cluster.local:6667']
+
+ scrape_configs:
+ - job_name: 'iotdbModel'
+ metrics_path: /metrics
+ static_configs:
+ - targets: ['model-service.default.svc.cluster.local:9090']
+ relabel_configs:
+ - target_label: k8s_cluster
+ replacement: example-cluster
+ - target_label: k8s_namespace
+ replacement: default
+ - target_label: instance_id
+ replacement: model-service
+
+ - job_name: 'iotdbWeb'
+ metrics_path: /actuator/prometheus
+ static_configs:
+ - targets: ['web-service.default.svc.cluster.local:8080']
+ labels: {env: example, instance_id: web-service}
+ relabel_configs:
+ - target_label: k8s_cluster
+ replacement: example-cluster
+ - target_label: k8s_namespace
+ replacement: default
diff --git a/metric-scrape/k8s/deployment.yaml
b/metric-scrape/k8s/deployment.yaml
new file mode 100644
index 0000000..cbd23bd
--- /dev/null
+++ b/metric-scrape/k8s/deployment.yaml
@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: metric-scrape
+ namespace: default
+ labels:
+ app.kubernetes.io/name: metric-scrape
+ app.kubernetes.io/part-of: iotdb-monitoring
+spec:
+ replicas: 1
+ strategy:
+ type: Recreate
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: metric-scrape
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: metric-scrape
+ app.kubernetes.io/part-of: iotdb-monitoring
+ annotations:
+ configmap/checksum: "v1"
+ spec:
+ containers:
+ - name: metric-scrape
+ image: metric-scrape:latest
+ imagePullPolicy: IfNotPresent
+ resources:
+ requests:
+ cpu: 100m
+ memory: 256Mi
+ limits:
+ cpu: 500m
+ memory: 768Mi
+ volumeMounts:
+ - mountPath: /app/conf
+ name: config
+ readOnly: true
+ volumes:
+ - name: config
+ configMap:
+ name: metric-scrape-config
+ items:
+ - key: metric-scrape.yml
+ path: metric-scrape.yml
diff --git a/metric-scrape/pom.xml b/metric-scrape/pom.xml
new file mode 100644
index 0000000..b527546
--- /dev/null
+++ b/metric-scrape/pom.xml
@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-extras-parent</artifactId>
+ <version>2.0.4-SNAPSHOT</version>
+ </parent>
+ <artifactId>metric-scrape</artifactId>
+ <name>IoTDB Extras: Metric Scrape</name>
+ <description>Standalone Prometheus text exposition scraper for the IoTDB
table model.</description>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>isession</artifactId>
+ <version>${iotdb.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-session</artifactId>
+ <version>${iotdb.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>service-rpc</artifactId>
+ <version>${iotdb.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tsfile</groupId>
+ <artifactId>common</artifactId>
+ <version>${tsfile.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tsfile</groupId>
+ <artifactId>tsfile</artifactId>
+ <version>${tsfile.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>2.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+
<createDependencyReducedPom>false</createDependencyReducedPom>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+
<mainClass>org.apache.iotdb.metricscrape.MetricScrapeMain</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <ignoredUnusedDeclaredDependencies>
+
<ignoredUnusedDeclaredDependency>org.slf4j:slf4j-simple</ignoredUnusedDeclaredDependency>
+ </ignoredUnusedDeclaredDependencies>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/IdentifierUtils.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/IdentifierUtils.java
new file mode 100644
index 0000000..21dc5fb
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/IdentifierUtils.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+public class IdentifierUtils {
+
+ private IdentifierUtils() {}
+
+ public static String sanitize(String identifier) {
+ if (identifier == null || identifier.trim().isEmpty()) {
+ return "unnamed";
+ }
+ StringBuilder builder = new StringBuilder(identifier.length());
+ for (int i = 0; i < identifier.length(); i++) {
+ char c = identifier.charAt(i);
+ if (Character.isLetterOrDigit(c) || c == '_') {
+ builder.append(c);
+ } else {
+ builder.append('_');
+ }
+ }
+ if (builder.length() == 0) {
+ return "unnamed";
+ }
+ if (Character.isDigit(builder.charAt(0))) {
+ builder.insert(0, '_');
+ }
+ return builder.toString();
+ }
+
+ public static String quoteSqlIdentifier(String identifier) {
+ return "\"" + identifier.replace("\"", "\"\"") + "\"";
+ }
+}
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfig.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfig.java
new file mode 100644
index 0000000..116afe9
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfig.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class MetricScrapeConfig {
+
+ private final GlobalConfig global;
+ private final List<ScrapeConfig> scrapeConfigs;
+
+ public MetricScrapeConfig(GlobalConfig global, List<ScrapeConfig>
scrapeConfigs) {
+ this.global = global;
+ this.scrapeConfigs = copyList(scrapeConfigs);
+ }
+
+ public GlobalConfig getGlobal() {
+ return global;
+ }
+
+ public List<ScrapeConfig> getScrapeConfigs() {
+ return scrapeConfigs;
+ }
+
+ public static class GlobalConfig {
+ private final Duration scrapeInterval;
+ private final Duration scrapeTimeout;
+ private final String databaseName;
+ private final String username;
+ private final String password;
+ private final List<String> nodeUrls;
+ private final int writeBatchSize;
+
+ public GlobalConfig(
+ Duration scrapeInterval,
+ Duration scrapeTimeout,
+ String databaseName,
+ String username,
+ String password,
+ List<String> nodeUrls,
+ int writeBatchSize) {
+ this.scrapeInterval = scrapeInterval;
+ this.scrapeTimeout = scrapeTimeout;
+ this.databaseName = databaseName;
+ this.username = username;
+ this.password = password;
+ this.nodeUrls = copyList(nodeUrls);
+ this.writeBatchSize = writeBatchSize;
+ }
+
+ public Duration getScrapeInterval() {
+ return scrapeInterval;
+ }
+
+ public Duration getScrapeTimeout() {
+ return scrapeTimeout;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public List<String> getNodeUrls() {
+ return nodeUrls;
+ }
+
+ public int getWriteBatchSize() {
+ return writeBatchSize;
+ }
+ }
+
+ public static class ScrapeConfig {
+ private final String jobName;
+ private final String scheme;
+ private final String metricsPath;
+ private final Duration scrapeInterval;
+ private final Duration scrapeTimeout;
+ private final List<StaticConfig> staticConfigs;
+ private final List<RelabelConfig> relabelConfigs;
+
+ public ScrapeConfig(
+ String jobName,
+ String scheme,
+ String metricsPath,
+ Duration scrapeInterval,
+ Duration scrapeTimeout,
+ List<StaticConfig> staticConfigs,
+ List<RelabelConfig> relabelConfigs) {
+ this.jobName = jobName;
+ this.scheme = scheme;
+ this.metricsPath = metricsPath;
+ this.scrapeInterval = scrapeInterval;
+ this.scrapeTimeout = scrapeTimeout;
+ this.staticConfigs = copyList(staticConfigs);
+ this.relabelConfigs = copyList(relabelConfigs);
+ }
+
+ public String getJobName() {
+ return jobName;
+ }
+
+ public String getScheme() {
+ return scheme;
+ }
+
+ public String getMetricsPath() {
+ return metricsPath;
+ }
+
+ public Duration getScrapeInterval() {
+ return scrapeInterval;
+ }
+
+ public Duration getScrapeTimeout() {
+ return scrapeTimeout;
+ }
+
+ public List<StaticConfig> getStaticConfigs() {
+ return staticConfigs;
+ }
+
+ public List<RelabelConfig> getRelabelConfigs() {
+ return relabelConfigs;
+ }
+ }
+
+ public static class StaticConfig {
+ private final List<String> targets;
+ private final Map<String, String> labels;
+
+ public StaticConfig(List<String> targets, Map<String, String> labels) {
+ this.targets = copyList(targets);
+ this.labels = Collections.unmodifiableMap(new LinkedHashMap<>(labels));
+ }
+
+ public List<String> getTargets() {
+ return targets;
+ }
+
+ public Map<String, String> getLabels() {
+ return labels;
+ }
+ }
+
+ public static class RelabelConfig {
+ private final String targetLabel;
+ private final String replacement;
+
+ public RelabelConfig(String targetLabel, String replacement) {
+ this.targetLabel = targetLabel;
+ this.replacement = replacement;
+ }
+
+ public String getTargetLabel() {
+ return targetLabel;
+ }
+
+ public String getReplacement() {
+ return replacement;
+ }
+ }
+
+ private static <T> List<T> copyList(List<T> values) {
+ return Collections.unmodifiableList(new ArrayList<>(values));
+ }
+}
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoader.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoader.java
new file mode 100644
index 0000000..ea0d5e6
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoader.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import org.apache.iotdb.metricscrape.MetricScrapeConfig.GlobalConfig;
+import org.apache.iotdb.metricscrape.MetricScrapeConfig.RelabelConfig;
+import org.apache.iotdb.metricscrape.MetricScrapeConfig.ScrapeConfig;
+import org.apache.iotdb.metricscrape.MetricScrapeConfig.StaticConfig;
+
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+public class MetricScrapeConfigLoader {
+
+ private static final Duration DEFAULT_SCRAPE_INTERVAL =
Duration.ofSeconds(15);
+ private static final Duration DEFAULT_SCRAPE_TIMEOUT =
Duration.ofSeconds(10);
+ private static final String DEFAULT_METRICS_PATH = "/metrics";
+ private static final String DEFAULT_SCHEME = "http";
+ private static final int DEFAULT_WRITE_BATCH_SIZE = 1000;
+
+ private MetricScrapeConfigLoader() {}
+
+ public static MetricScrapeConfig load(Path path) throws IOException {
+ try (InputStream inputStream = Files.newInputStream(path)) {
+ Object loaded = new Yaml().load(inputStream);
+ if (!(loaded instanceof Map)) {
+ throw new IllegalArgumentException("YAML root should be a map");
+ }
+ return parseConfig(castMap(loaded, "root"));
+ }
+ }
+
+ private static MetricScrapeConfig parseConfig(Map<String, Object> root) {
+ Map<String, Object> globalMap = castMap(required(root, "global"),
"global");
+ GlobalConfig global = parseGlobal(globalMap);
+ List<ScrapeConfig> scrapeConfigs = parseScrapeConfigs(required(root,
"scrape_configs"), global);
+ if (scrapeConfigs.isEmpty()) {
+ throw new IllegalArgumentException("scrape_configs should not be empty");
+ }
+ return new MetricScrapeConfig(global, scrapeConfigs);
+ }
+
+ private static GlobalConfig parseGlobal(Map<String, Object> globalMap) {
+ Duration interval =
+ parseDuration(optional(globalMap, "scrape_interval"),
DEFAULT_SCRAPE_INTERVAL);
+ Duration timeout = parseDuration(optional(globalMap, "scrape_timeout"),
DEFAULT_SCRAPE_TIMEOUT);
+ String databaseName = requiredString(globalMap, "iotdb_database");
+ String username = stringValue(optional(globalMap, "iotdb_username"),
"root");
+ String password = stringValue(optional(globalMap, "iotdb_password"),
"root");
+ List<String> nodeUrls = parseStringList(required(globalMap, "iotdb_urls"),
"iotdb_urls");
+ int batchSize =
+ parsePositiveInt(optional(globalMap, "write_batch_size"),
DEFAULT_WRITE_BATCH_SIZE);
+ return new GlobalConfig(
+ interval, timeout, databaseName, username, password, nodeUrls,
batchSize);
+ }
+
+ private static List<ScrapeConfig> parseScrapeConfigs(Object value,
GlobalConfig global) {
+ List<Object> configs = castList(value, "scrape_configs");
+ List<ScrapeConfig> result = new ArrayList<>();
+ for (int i = 0; i < configs.size(); i++) {
+ Map<String, Object> map = castMap(configs.get(i), "scrape_configs[" + i
+ "]");
+ String jobName = requiredString(map, "job_name");
+ String scheme = stringValue(optional(map, "scheme"), DEFAULT_SCHEME);
+ String metricsPath =
+ normalizeMetricsPath(stringValue(optional(map, "metrics_path"),
DEFAULT_METRICS_PATH));
+ Duration interval =
+ parseDuration(optional(map, "scrape_interval"),
global.getScrapeInterval());
+ Duration timeout = parseDuration(optional(map, "scrape_timeout"),
global.getScrapeTimeout());
+ List<StaticConfig> staticConfigs = parseStaticConfigs(required(map,
"static_configs"));
+ List<RelabelConfig> relabelConfigs = parseRelabelConfigs(optional(map,
"relabel_configs"));
+ result.add(
+ new ScrapeConfig(
+ jobName, scheme, metricsPath, interval, timeout, staticConfigs,
relabelConfigs));
+ }
+ return result;
+ }
+
+ private static List<StaticConfig> parseStaticConfigs(Object value) {
+ List<Object> configs = castList(value, "static_configs");
+ List<StaticConfig> result = new ArrayList<>();
+ for (int i = 0; i < configs.size(); i++) {
+ Map<String, Object> map = castMap(configs.get(i), "static_configs[" + i
+ "]");
+ List<String> targets = parseStringList(required(map, "targets"),
"targets");
+ Map<String, String> labels = parseStringMap(optional(map, "labels"));
+ result.add(new StaticConfig(targets, labels));
+ }
+ return result;
+ }
+
+ private static List<RelabelConfig> parseRelabelConfigs(Object value) {
+ if (value == null) {
+ return Collections.emptyList();
+ }
+ List<Object> configs = castList(value, "relabel_configs");
+ List<RelabelConfig> result = new ArrayList<>();
+ for (int i = 0; i < configs.size(); i++) {
+ Map<String, Object> map = castMap(configs.get(i), "relabel_configs[" + i
+ "]");
+ String targetLabel = requiredString(map, "target_label");
+ String replacement = stringValue(optional(map, "replacement"), "");
+ result.add(new RelabelConfig(targetLabel, replacement));
+ }
+ return result;
+ }
+
+ private static Object required(Map<String, Object> map, String key) {
+ Object value = map.get(key);
+ if (value == null) {
+ throw new IllegalArgumentException("Missing required config: " + key);
+ }
+ return value;
+ }
+
+ private static Object optional(Map<String, Object> map, String key) {
+ return map.get(key);
+ }
+
+ private static String requiredString(Map<String, Object> map, String key) {
+ String value = stringValue(required(map, key), null);
+ if (isBlank(value)) {
+ throw new IllegalArgumentException("Config " + key + " should not be
empty");
+ }
+ return value;
+ }
+
+ private static String stringValue(Object value, String defaultValue) {
+ return value == null ? defaultValue : String.valueOf(value);
+ }
+
+ private static String normalizeMetricsPath(String path) {
+ if (isBlank(path)) {
+ return DEFAULT_METRICS_PATH;
+ }
+ return path.startsWith("/") ? path : "/" + path;
+ }
+
+ private static Duration parseDuration(Object value, Duration defaultValue) {
+ if (value == null) {
+ return defaultValue;
+ }
+ String text = String.valueOf(value).trim().toLowerCase(Locale.ROOT);
+ if (text.isEmpty()) {
+ return defaultValue;
+ }
+ long multiplier;
+ String number;
+ if (text.endsWith("ms")) {
+ multiplier = 1;
+ number = text.substring(0, text.length() - 2);
+ } else if (text.endsWith("s")) {
+ multiplier = 1000;
+ number = text.substring(0, text.length() - 1);
+ } else if (text.endsWith("m")) {
+ multiplier = 60_000;
+ number = text.substring(0, text.length() - 1);
+ } else if (text.endsWith("h")) {
+ multiplier = 3_600_000;
+ number = text.substring(0, text.length() - 1);
+ } else {
+ multiplier = 1000;
+ number = text;
+ }
+ try {
+ long amount = Long.parseLong(number);
+ if (amount <= 0) {
+ throw new IllegalArgumentException("Duration should be positive: " +
value);
+ }
+ return Duration.ofMillis(amount * multiplier);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Illegal duration: " + value, e);
+ }
+ }
+
+ private static int parsePositiveInt(Object value, int defaultValue) {
+ if (value == null) {
+ return defaultValue;
+ }
+ int parsed = Integer.parseInt(String.valueOf(value));
+ if (parsed <= 0) {
+ throw new IllegalArgumentException("Value should be positive: " + value);
+ }
+ return parsed;
+ }
+
+ private static List<String> parseStringList(Object value, String name) {
+ List<Object> rawList = castList(value, name);
+ List<String> result = new ArrayList<>();
+ for (Object item : rawList) {
+ String text = String.valueOf(item);
+ if (isBlank(text)) {
+ throw new IllegalArgumentException(name + " contains empty value");
+ }
+ result.add(text);
+ }
+ return result;
+ }
+
+ private static Map<String, String> parseStringMap(Object value) {
+ if (value == null) {
+ return Collections.emptyMap();
+ }
+ Map<String, Object> rawMap = castMap(value, "labels");
+ Map<String, String> result = new LinkedHashMap<>();
+ for (Map.Entry<String, Object> entry : rawMap.entrySet()) {
+ result.put(entry.getKey(), String.valueOf(entry.getValue()));
+ }
+ return result;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<String, Object> castMap(Object value, String name) {
+ if (!(value instanceof Map)) {
+ throw new IllegalArgumentException(name + " should be a map");
+ }
+ Map<String, Object> result = new LinkedHashMap<>();
+ ((Map<?, ?>) value).forEach((key, item) -> result.put(String.valueOf(key),
item));
+ return result;
+ }
+
+ private static List<Object> castList(Object value, String name) {
+ if (!(value instanceof List)) {
+ throw new IllegalArgumentException(name + " should be a list");
+ }
+ return new ArrayList<>((List<?>) value);
+ }
+
+ private static boolean isBlank(String value) {
+ return value == null || value.trim().isEmpty();
+ }
+}
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeHttpClient.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeHttpClient.java
new file mode 100644
index 0000000..267c55b
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeHttpClient.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+
+public class MetricScrapeHttpClient {
+
+ public String get(String url, int timeoutMs) throws IOException {
+ HttpURLConnection connection = (HttpURLConnection) new
URL(url).openConnection();
+ connection.setRequestMethod("GET");
+ connection.setConnectTimeout(timeoutMs);
+ connection.setReadTimeout(timeoutMs);
+ connection.setRequestProperty(
+ "Accept", "text/plain; version=0.0.4, application/openmetrics-text,
*/*");
+ connection.setRequestProperty("User-Agent",
"Apache-IoTDB-MetricScrape/1.0");
+
+ int statusCode = connection.getResponseCode();
+ if (statusCode < 200 || statusCode >= 300) {
+ throw new IOException("Metric scrape target " + url + " returns HTTP " +
statusCode);
+ }
+ try (InputStream inputStream = connection.getInputStream()) {
+ return readToString(inputStream);
+ } finally {
+ connection.disconnect();
+ }
+ }
+
+ private String readToString(InputStream inputStream) throws IOException {
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ byte[] buffer = new byte[8192];
+ int length;
+ while ((length = inputStream.read(buffer)) != -1) {
+ outputStream.write(buffer, 0, length);
+ }
+ return new String(outputStream.toByteArray(), StandardCharsets.UTF_8);
+ }
+}
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeMain.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeMain.java
new file mode 100644
index 0000000..2885f05
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeMain.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.CountDownLatch;
+
+public class MetricScrapeMain {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(MetricScrapeMain.class);
+ private static final String DEFAULT_CONFIG = "conf/metric-scrape.yml";
+
+ public static void main(String[] args) throws Exception {
+ Path configPath = parseConfigPath(args);
+ MetricScrapeConfig config = MetricScrapeConfigLoader.load(configPath);
+ MetricScrapeService service = new MetricScrapeService(config);
+ CountDownLatch stopLatch = new CountDownLatch(1);
+
+ Runtime.getRuntime()
+ .addShutdownHook(
+ new Thread(
+ () -> {
+ LOGGER.info("Stopping metric scrape service.");
+ service.stop();
+ stopLatch.countDown();
+ },
+ "metric-scrape-shutdown"));
+
+ service.start();
+ LOGGER.info("Metric scrape service started with config {}.", configPath);
+ stopLatch.await();
+ }
+
+ private static Path parseConfigPath(String[] args) {
+ if (args.length == 0) {
+ return Paths.get(DEFAULT_CONFIG);
+ }
+ if (args.length == 2 && ("-c".equals(args[0]) ||
"--config".equals(args[0]))) {
+ return Paths.get(args[1]);
+ }
+ throw new IllegalArgumentException("Usage: java -jar metric-scrape.jar [-c
config.yml]");
+ }
+}
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapePlanner.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapePlanner.java
new file mode 100644
index 0000000..42e48af
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapePlanner.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import org.apache.iotdb.metricscrape.MetricScrapeConfig.RelabelConfig;
+import org.apache.iotdb.metricscrape.MetricScrapeConfig.ScrapeConfig;
+import org.apache.iotdb.metricscrape.MetricScrapeConfig.StaticConfig;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class MetricScrapePlanner {
+
+ public List<MetricScrapeTarget> buildTargets(MetricScrapeConfig config) {
+ List<MetricScrapeTarget> result = new ArrayList<>();
+ for (ScrapeConfig scrapeConfig : config.getScrapeConfigs()) {
+ for (StaticConfig staticConfig : scrapeConfig.getStaticConfigs()) {
+ for (String target : staticConfig.getTargets()) {
+ Map<String, String> labels = new LinkedHashMap<>();
+ labels.put("job_name", scrapeConfig.getJobName());
+ labels.put("instance", target);
+ labels.putAll(staticConfig.getLabels());
+ for (RelabelConfig relabelConfig : scrapeConfig.getRelabelConfigs())
{
+ labels.put(relabelConfig.getTargetLabel(),
relabelConfig.getReplacement());
+ }
+ result.add(
+ new MetricScrapeTarget(
+ scrapeConfig.getJobName(),
+ buildUrl(scrapeConfig, target),
+ scrapeConfig.getScrapeInterval(),
+ scrapeConfig.getScrapeTimeout(),
+ labels));
+ }
+ }
+ }
+ return result;
+ }
+
+ private String buildUrl(ScrapeConfig scrapeConfig, String target) {
+ if (target.startsWith("http://";) || target.startsWith("https://";)) {
+ return appendMetricsPath(target, scrapeConfig.getMetricsPath());
+ }
+ return scrapeConfig.getScheme() + "://" + target +
scrapeConfig.getMetricsPath();
+ }
+
+ private String appendMetricsPath(String target, String metricsPath) {
+ if (target.endsWith(metricsPath)) {
+ return target;
+ }
+ if (target.endsWith("/")) {
+ return target.substring(0, target.length() - 1) + metricsPath;
+ }
+ return target + metricsPath;
+ }
+}
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeService.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeService.java
new file mode 100644
index 0000000..466469b
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeService.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class MetricScrapeService {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(MetricScrapeService.class);
+
+ private final MetricScrapeConfig config;
+ private final MetricScrapeHttpClient httpClient = new
MetricScrapeHttpClient();
+ private final PrometheusTextParser parser = new PrometheusTextParser();
+ private final MetricScrapePlanner planner = new MetricScrapePlanner();
+ private MetricTableWriter writer;
+ private ScheduledExecutorService executorService;
+
+ public MetricScrapeService(MetricScrapeConfig config) {
+ this.config = config;
+ }
+
+ public synchronized void start() throws Exception {
+ List<MetricScrapeTarget> targets = planner.buildTargets(config);
+ if (targets.isEmpty()) {
+ throw new IllegalArgumentException("No metric scrape target configured");
+ }
+ writer = new MetricTableWriter(config.getGlobal());
+ writer.initializeDatabase();
+ AtomicInteger threadIndex = new AtomicInteger();
+ executorService =
+ Executors.newScheduledThreadPool(
+ Math.min(targets.size(),
Runtime.getRuntime().availableProcessors() * 2),
+ runnable -> {
+ Thread thread =
+ new Thread(runnable, "metric-scrape-" +
threadIndex.incrementAndGet());
+ thread.setDaemon(false);
+ return thread;
+ });
+ for (MetricScrapeTarget target : targets) {
+ ScheduledExecutorUtil.safelyScheduleWithFixedDelay(
+ executorService,
+ () -> scrapeTarget(target),
+ 0,
+ target.getInterval().toMillis(),
+ TimeUnit.MILLISECONDS);
+ LOGGER.info(
+ "Scheduled metric target {} with interval {}.", target.getUrl(),
target.getInterval());
+ }
+ }
+
+ public synchronized void stop() {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ executorService = null;
+ }
+ if (writer != null) {
+ writer.close();
+ writer = null;
+ }
+ }
+
+ private void scrapeTarget(MetricScrapeTarget target) {
+ try {
+ String text =
+ httpClient.get(target.getUrl(),
Math.toIntExact(target.getTimeout().toMillis()));
+ List<PrometheusSample> samples = parser.parse(text,
System.currentTimeMillis());
+ writer.write(enrichSamples(samples, target));
+ LOGGER.debug("Scraped {} samples from {}.", samples.size(),
target.getUrl());
+ } catch (IOException | RuntimeException e) {
+ LOGGER.warn("Failed to scrape metric target {}.", target.getUrl(), e);
+ } catch (Exception e) {
+ LOGGER.warn("Failed to write metric samples from {}.", target.getUrl(),
e);
+ }
+ }
+
+ private List<PrometheusSample> enrichSamples(
+ List<PrometheusSample> samples, MetricScrapeTarget target) {
+ List<PrometheusSample> result = new ArrayList<>(samples.size());
+ for (PrometheusSample sample : samples) {
+ result.add(sample.withLabels(mergeLabels(target.getLabels(),
sample.getLabels())));
+ }
+ return result;
+ }
+
+ private Map<String, String> mergeLabels(
+ Map<String, String> targetLabels, Map<String, String> sampleLabels) {
+ java.util.LinkedHashMap<String, String> labels = new
java.util.LinkedHashMap<>(targetLabels);
+ for (Map.Entry<String, String> entry : sampleLabels.entrySet()) {
+ labels.putIfAbsent(entry.getKey(), entry.getValue());
+ }
+ return labels;
+ }
+}
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeTarget.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeTarget.java
new file mode 100644
index 0000000..2519987
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeTarget.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import java.time.Duration;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class MetricScrapeTarget {
+
+ private final String jobName;
+ private final String url;
+ private final Duration interval;
+ private final Duration timeout;
+ private final Map<String, String> labels;
+
+ public MetricScrapeTarget(
+ String jobName, String url, Duration interval, Duration timeout,
Map<String, String> labels) {
+ this.jobName = jobName;
+ this.url = url;
+ this.interval = interval;
+ this.timeout = timeout;
+ this.labels = Collections.unmodifiableMap(new LinkedHashMap<>(labels));
+ }
+
+ public String getJobName() {
+ return jobName;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public Duration getInterval() {
+ return interval;
+ }
+
+ public Duration getTimeout() {
+ return timeout;
+ }
+
+ public Map<String, String> getLabels() {
+ return labels;
+ }
+}
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricTableWriter.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricTableWriter.java
new file mode 100644
index 0000000..a0df465
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricTableWriter.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import org.apache.iotdb.isession.ITableSession;
+import org.apache.iotdb.metricscrape.MetricScrapeConfig.GlobalConfig;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.TableSessionBuilder;
+
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.write.record.Tablet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class MetricTableWriter {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(MetricTableWriter.class);
+
+ private final GlobalConfig config;
+ private final ITableSession session;
+
+ public MetricTableWriter(GlobalConfig config) throws
IoTDBConnectionException {
+ this.config = config;
+ this.session =
+ new TableSessionBuilder()
+ .nodeUrls(config.getNodeUrls())
+ .username(config.getUsername())
+ .password(config.getPassword())
+ .database(config.getDatabaseName())
+ .build();
+ }
+
+ public synchronized void initializeDatabase()
+ throws IoTDBConnectionException, StatementExecutionException {
+ session.executeNonQueryStatement(
+ "CREATE DATABASE IF NOT EXISTS "
+ + IdentifierUtils.quoteSqlIdentifier(config.getDatabaseName()));
+ }
+
+ public synchronized void write(List<PrometheusSample> samples)
+ throws IoTDBConnectionException, StatementExecutionException {
+ if (samples.isEmpty()) {
+ return;
+ }
+ Map<TabletKey, TabletBuilder> builders = new LinkedHashMap<>();
+ for (PrometheusSample sample : samples) {
+ TabletKey key = TabletKey.from(sample);
+ TabletBuilder builder =
+ builders.computeIfAbsent(
+ key, ignored -> new TabletBuilder(key,
config.getWriteBatchSize()));
+ builder.add(sample);
+ if (builder.isFull()) {
+ flush(builder);
+ builder.reset();
+ }
+ }
+ for (TabletBuilder builder : builders.values()) {
+ if (!builder.isEmpty()) {
+ flush(builder);
+ }
+ }
+ }
+
+ public void close() {
+ try {
+ session.close();
+ } catch (IoTDBConnectionException e) {
+ LOGGER.warn("Failed to close IoTDB session.", e);
+ }
+ }
+
+ private void flush(TabletBuilder builder)
+ throws IoTDBConnectionException, StatementExecutionException {
+ session.insert(builder.tablet);
+ }
+
+ private static class TabletKey {
+ private final String table;
+ private final String field;
+ private final List<String> labelKeys;
+ private final Map<String, String> sanitizedToOriginalLabel;
+
+ private static TabletKey from(PrometheusSample sample) {
+ Map<String, String> sanitizedToOriginalLabel = sanitizeLabels(sample);
+ List<String> labelKeys = new
ArrayList<>(sanitizedToOriginalLabel.keySet());
+ return new TabletKey(
+ IdentifierUtils.sanitize(sample.getMetricFamilyName()),
+ IdentifierUtils.sanitize(sample.getMetricName()),
+ labelKeys,
+ sanitizedToOriginalLabel);
+ }
+
+ private static Map<String, String> sanitizeLabels(PrometheusSample sample)
{
+ List<String> originalLabels = new
ArrayList<>(sample.getLabels().keySet());
+ Collections.sort(originalLabels);
+ Map<String, String> result = new LinkedHashMap<>();
+ for (String originalLabel : originalLabels) {
+ String sanitizedLabel = IdentifierUtils.sanitize(originalLabel);
+ String previous = result.put(sanitizedLabel, originalLabel);
+ if (previous != null && !previous.equals(originalLabel)) {
+ throw new IllegalArgumentException(
+ "Prometheus labels "
+ + previous
+ + " and "
+ + originalLabel
+ + " map to the same IoTDB column "
+ + sanitizedLabel);
+ }
+ }
+ return result;
+ }
+
+ private TabletKey(
+ String table,
+ String field,
+ List<String> labelKeys,
+ Map<String, String> sanitizedToOriginalLabel) {
+ this.table = table;
+ this.field = field;
+ this.labelKeys = Collections.unmodifiableList(new
ArrayList<>(labelKeys));
+ this.sanitizedToOriginalLabel =
+ Collections.unmodifiableMap(new
LinkedHashMap<>(sanitizedToOriginalLabel));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof TabletKey)) {
+ return false;
+ }
+ TabletKey tabletKey = (TabletKey) o;
+ return Objects.equals(table, tabletKey.table)
+ && Objects.equals(field, tabletKey.field)
+ && Objects.equals(labelKeys, tabletKey.labelKeys);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(table, field, labelKeys);
+ }
+ }
+
+ private static class TabletBuilder {
+ private final TabletKey key;
+ private final Tablet tablet;
+
+ private TabletBuilder(TabletKey key, int maxRows) {
+ this.key = key;
+ List<String> columnNames = new ArrayList<>(key.labelKeys);
+ columnNames.add(key.field);
+ List<TSDataType> dataTypes = new ArrayList<>(columnNames.size());
+ List<ColumnCategory> columnCategories = new
ArrayList<>(columnNames.size());
+ for (int i = 0; i < key.labelKeys.size(); i++) {
+ dataTypes.add(TSDataType.STRING);
+ columnCategories.add(ColumnCategory.TAG);
+ }
+ dataTypes.add(TSDataType.DOUBLE);
+ columnCategories.add(ColumnCategory.FIELD);
+ this.tablet = new Tablet(key.table, columnNames, dataTypes,
columnCategories, maxRows);
+ }
+
+ private boolean isFull() {
+ return tablet.getRowSize() == tablet.getMaxRowNumber();
+ }
+
+ private boolean isEmpty() {
+ return tablet.getRowSize() == 0;
+ }
+
+ private void reset() {
+ tablet.reset();
+ }
+
+ private void add(PrometheusSample sample) {
+ int row = tablet.getRowSize();
+ tablet.addTimestamp(row, sample.getTimestamp());
+ for (String labelKey : key.labelKeys) {
+ String originalLabel = key.sanitizedToOriginalLabel.get(labelKey);
+ tablet.addValue(labelKey, row, sample.getLabels().get(originalLabel));
+ }
+ tablet.addValue(key.field, row, sample.getValue());
+ }
+ }
+}
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusSample.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusSample.java
new file mode 100644
index 0000000..7e8a5f8
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusSample.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class PrometheusSample {
+
+ private final String metricFamilyName;
+ private final String metricName;
+ private final Map<String, String> labels;
+ private final double value;
+ private final long timestamp;
+
+ public PrometheusSample(
+ String metricFamilyName,
+ String metricName,
+ Map<String, String> labels,
+ double value,
+ long timestamp) {
+ this.metricFamilyName = metricFamilyName;
+ this.metricName = metricName;
+ this.labels = Collections.unmodifiableMap(new LinkedHashMap<>(labels));
+ this.value = value;
+ this.timestamp = timestamp;
+ }
+
+ public String getMetricFamilyName() {
+ return metricFamilyName;
+ }
+
+ public String getMetricName() {
+ return metricName;
+ }
+
+ public Map<String, String> getLabels() {
+ return labels;
+ }
+
+ public double getValue() {
+ return value;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public PrometheusSample withLabels(Map<String, String> newLabels) {
+ return new PrometheusSample(metricFamilyName, metricName, newLabels,
value, timestamp);
+ }
+}
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusTextParser.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusTextParser.java
new file mode 100644
index 0000000..c328dd1
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusTextParser.java
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+public class PrometheusTextParser {
+
+ private static final String[] METRIC_FAMILY_SAMPLE_SUFFIXES = {
+ "_sum", "_count", "_bucket", "_created"
+ };
+
+ public List<PrometheusSample> parse(String text, long defaultTimestamp) {
+ if (text == null) {
+ throw new IllegalArgumentException("Prometheus text should not be null");
+ }
+ List<PrometheusSample> samples = new ArrayList<>();
+ String[] lines = text.split("\\r\\n|\\n|\\r");
+ Set<String> helpMetricNames = collectHelpMetricNames(lines);
+ for (int i = 0; i < lines.length; i++) {
+ String line = lines[i].trim();
+ if (line.isEmpty() || line.startsWith("#")) {
+ continue;
+ }
+ samples.add(parseSample(line, i + 1, defaultTimestamp, helpMetricNames));
+ }
+ return samples;
+ }
+
+ private PrometheusSample parseSample(
+ String line, int lineNumber, long defaultTimestamp, Set<String>
helpMetricNames) {
+ Parser parser = new Parser(line, lineNumber);
+ String metricName = parser.parseMetricName();
+ Map<String, String> labels = parser.parseLabels();
+ parser.skipWhitespace();
+ double value = parser.parseValue();
+ parser.skipWhitespace();
+ long timestamp = parser.parseOptionalTimestamp(defaultTimestamp);
+ parser.skipWhitespace();
+ parser.expectEnd();
+ return new PrometheusSample(
+ resolveMetricFamilyName(metricName, helpMetricNames), metricName,
labels, value, timestamp);
+ }
+
+ private static Set<String> collectHelpMetricNames(String[] lines) {
+ Set<String> helpMetricNames = new LinkedHashSet<>();
+ for (String rawLine : lines) {
+ String line = rawLine.trim();
+ if (!line.startsWith("# HELP")) {
+ continue;
+ }
+ int position = "# HELP".length();
+ while (position < line.length() &&
Character.isWhitespace(line.charAt(position))) {
+ position++;
+ }
+ int start = position;
+ while (position < line.length() &&
!Character.isWhitespace(line.charAt(position))) {
+ position++;
+ }
+ if (start < position) {
+ helpMetricNames.add(line.substring(start, position));
+ }
+ }
+ return helpMetricNames;
+ }
+
+ private static String resolveMetricFamilyName(String metricName, Set<String>
helpMetricNames) {
+ if (helpMetricNames.contains(metricName)) {
+ return metricName;
+ }
+
+ String result = null;
+ for (String helpMetricName : helpMetricNames) {
+ if (isMetricFamilyComponent(metricName, helpMetricName)
+ && (result == null || helpMetricName.length() > result.length())) {
+ result = helpMetricName;
+ }
+ }
+ return result == null ? metricName : result;
+ }
+
+ private static boolean isMetricFamilyComponent(String metricName, String
helpMetricName) {
+ for (String suffix : METRIC_FAMILY_SAMPLE_SUFFIXES) {
+ if (metricName.equals(helpMetricName + suffix)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static double parsePrometheusDouble(String token, int lineNumber) {
+ String lowerToken = token.toLowerCase(Locale.ROOT);
+ switch (lowerToken) {
+ case "inf":
+ case "+inf":
+ case "infinity":
+ case "+infinity":
+ return Double.POSITIVE_INFINITY;
+ case "-inf":
+ case "-infinity":
+ return Double.NEGATIVE_INFINITY;
+ case "nan":
+ case "+nan":
+ case "-nan":
+ return Double.NaN;
+ default:
+ break;
+ }
+ if (token.endsWith("d") || token.endsWith("D") || token.endsWith("f") ||
token.endsWith("F")) {
+ throw new IllegalArgumentException(
+ "Illegal Prometheus sample value " + token + " at line " +
lineNumber);
+ }
+ try {
+ return Double.parseDouble(removeValidNumericUnderscores(token,
lineNumber));
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(
+ "Illegal Prometheus sample value " + token + " at line " +
lineNumber, e);
+ }
+ }
+
+ private static String removeValidNumericUnderscores(String token, int
lineNumber) {
+ if (token.indexOf('_') < 0) {
+ return token;
+ }
+ for (int i = 0; i < token.length(); i++) {
+ if (token.charAt(i) == '_') {
+ if (i == 0
+ || i == token.length() - 1
+ || !Character.isDigit(token.charAt(i - 1))
+ || !Character.isDigit(token.charAt(i + 1))) {
+ throw new IllegalArgumentException(
+ "Illegal Prometheus sample value " + token + " at line " +
lineNumber);
+ }
+ }
+ }
+ return token.replace("_", "");
+ }
+
+ private static class Parser {
+ private final String line;
+ private final int lineNumber;
+ private int position;
+
+ private Parser(String line, int lineNumber) {
+ this.line = line;
+ this.lineNumber = lineNumber;
+ }
+
+ private String parseMetricName() {
+ if (position >= line.length() ||
Character.isWhitespace(line.charAt(position))) {
+ throw new IllegalArgumentException("Illegal Prometheus metric name at
line " + lineNumber);
+ }
+ int start = position;
+ while (position < line.length()
+ && !Character.isWhitespace(line.charAt(position))
+ && line.charAt(position) != '{') {
+ position++;
+ }
+ if (start == position) {
+ throw new IllegalArgumentException("Illegal Prometheus metric name at
line " + lineNumber);
+ }
+ return line.substring(start, position);
+ }
+
+ private Map<String, String> parseLabels() {
+ Map<String, String> labels = new LinkedHashMap<>();
+ if (position >= line.length() || line.charAt(position) != '{') {
+ return labels;
+ }
+ position++;
+ skipWhitespace();
+ if (position < line.length() && line.charAt(position) == '}') {
+ position++;
+ return labels;
+ }
+
+ while (position < line.length()) {
+ String name = parseLabelName();
+ skipWhitespace();
+ expect('=');
+ skipWhitespace();
+ String value = parseLabelValue();
+ if (labels.put(name, value) != null) {
+ throw new IllegalArgumentException(
+ "Duplicate Prometheus label " + name + " at line " + lineNumber);
+ }
+ skipWhitespace();
+ if (position < line.length() && line.charAt(position) == ',') {
+ position++;
+ skipWhitespace();
+ if (position < line.length() && line.charAt(position) == '}') {
+ position++;
+ return labels;
+ }
+ continue;
+ }
+ expect('}');
+ return labels;
+ }
+ throw new IllegalArgumentException("Unclosed Prometheus label set at
line " + lineNumber);
+ }
+
+ private String parseLabelName() {
+ int start = position;
+ while (position < line.length() && line.charAt(position) != '=') {
+ if (line.charAt(position) == ',' || line.charAt(position) == '}') {
+ break;
+ }
+ position++;
+ }
+ String name = line.substring(start, position).trim();
+ if (name.isEmpty()) {
+ throw new IllegalArgumentException("Illegal Prometheus label name at
line " + lineNumber);
+ }
+ return name;
+ }
+
+ private String parseLabelValue() {
+ expect('"');
+ StringBuilder builder = new StringBuilder();
+ int delimiterDepth = 0;
+ while (position < line.length()) {
+ char c = line.charAt(position++);
+ if (c == '"') {
+ if (delimiterDepth == 0 && isLabelValueTerminator()) {
+ return builder.toString();
+ }
+ builder.append(c);
+ continue;
+ }
+ if (c == '\\') {
+ if (position >= line.length()) {
+ throw new IllegalArgumentException(
+ "Illegal Prometheus label escape at line " + lineNumber);
+ }
+ char escaped = line.charAt(position++);
+ switch (escaped) {
+ case 'n':
+ builder.append('\n');
+ break;
+ case '\\':
+ case '"':
+ builder.append(escaped);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Illegal Prometheus label escape at line " + lineNumber);
+ }
+ } else {
+ builder.append(c);
+ delimiterDepth = updateDelimiterDepth(delimiterDepth, c);
+ }
+ }
+ throw new IllegalArgumentException("Unclosed Prometheus label value at
line " + lineNumber);
+ }
+
+ private int updateDelimiterDepth(int delimiterDepth, char c) {
+ switch (c) {
+ case '{':
+ case '[':
+ case '(':
+ return delimiterDepth + 1;
+ case '}':
+ case ']':
+ case ')':
+ return Math.max(0, delimiterDepth - 1);
+ default:
+ return delimiterDepth;
+ }
+ }
+
+ private boolean isLabelValueTerminator() {
+ int next = skipWhitespace(position);
+ if (next >= line.length()) {
+ return true;
+ }
+ if (line.charAt(next) == '}') {
+ return next + 1 >= line.length() ||
Character.isWhitespace(line.charAt(next + 1));
+ }
+ if (line.charAt(next) != ',') {
+ return false;
+ }
+ int afterComma = skipWhitespace(next + 1);
+ return afterComma >= line.length()
+ || line.charAt(afterComma) == '}'
+ || isLabelAssignmentStart(afterComma);
+ }
+
+ private boolean isLabelAssignmentStart(int start) {
+ if (start >= line.length() || line.charAt(start) == '"') {
+ return false;
+ }
+ int scan = start;
+ while (scan < line.length()) {
+ char c = line.charAt(scan);
+ if (c == '=') {
+ String labelName = line.substring(start, scan).trim();
+ int valueStart = skipWhitespace(scan + 1);
+ return !labelName.isEmpty()
+ && valueStart < line.length()
+ && line.charAt(valueStart) == '"';
+ }
+ if (c == ',' || c == '}') {
+ return false;
+ }
+ scan++;
+ }
+ return false;
+ }
+
+ private double parseValue() {
+ return parsePrometheusDouble(parseToken("sample value"), lineNumber);
+ }
+
+ private long parseOptionalTimestamp(long defaultTimestamp) {
+ if (position >= line.length()) {
+ return defaultTimestamp;
+ }
+ String token = parseToken("sample timestamp");
+ try {
+ return Long.parseLong(token);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(
+ "Illegal Prometheus sample timestamp " + token + " at line " +
lineNumber, e);
+ }
+ }
+
+ private String parseToken(String name) {
+ if (position >= line.length() ||
Character.isWhitespace(line.charAt(position))) {
+ throw new IllegalArgumentException("Missing Prometheus " + name + " at
line " + lineNumber);
+ }
+ int start = position;
+ while (position < line.length() &&
!Character.isWhitespace(line.charAt(position))) {
+ position++;
+ }
+ return line.substring(start, position);
+ }
+
+ private void skipWhitespace() {
+ while (position < line.length() &&
Character.isWhitespace(line.charAt(position))) {
+ position++;
+ }
+ }
+
+ private int skipWhitespace(int from) {
+ while (from < line.length() &&
Character.isWhitespace(line.charAt(from))) {
+ from++;
+ }
+ return from;
+ }
+
+ private void expect(char expected) {
+ if (position >= line.length() || line.charAt(position) != expected) {
+ throw new IllegalArgumentException(
+ "Expected '" + expected + "' in Prometheus sample at line " +
lineNumber);
+ }
+ position++;
+ }
+
+ private void expectEnd() {
+ if (position != line.length()) {
+ throw new IllegalArgumentException(
+ "Unexpected Prometheus sample content at line " + lineNumber);
+ }
+ }
+ }
+}
diff --git
a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/ScheduledExecutorUtil.java
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/ScheduledExecutorUtil.java
new file mode 100644
index 0000000..fbbd6d9
--- /dev/null
+++
b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/ScheduledExecutorUtil.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+final class ScheduledExecutorUtil {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(ScheduledExecutorUtil.class);
+
+ private ScheduledExecutorUtil() {}
+
+ @SuppressWarnings("unsafeThreadSchedule")
+ static ScheduledFuture<?> safelyScheduleWithFixedDelay(
+ ScheduledExecutorService executor,
+ Runnable command,
+ long initialDelay,
+ long delay,
+ TimeUnit unit) {
+ return executor.scheduleWithFixedDelay(
+ () -> {
+ try {
+ command.run();
+ } catch (Throwable t) {
+ LOGGER.error("Scheduled metric scrape task failed.", t);
+ }
+ },
+ initialDelay,
+ delay,
+ unit);
+ }
+}
diff --git
a/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoaderTest.java
b/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoaderTest.java
new file mode 100644
index 0000000..648c5fc
--- /dev/null
+++
b/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoaderTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import org.junit.Test;
+
+import java.nio.file.Paths;
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+
+public class MetricScrapeConfigLoaderTest {
+
+ @Test
+ public void loadExampleConfig() throws Exception {
+ MetricScrapeConfig config =
MetricScrapeConfigLoader.load(Paths.get("conf/metric-scrape.yml"));
+
+ assertEquals(Duration.ofSeconds(15),
config.getGlobal().getScrapeInterval());
+ assertEquals(Duration.ofSeconds(10),
config.getGlobal().getScrapeTimeout());
+ assertEquals("metrics", config.getGlobal().getDatabaseName());
+ assertEquals(3, config.getScrapeConfigs().size());
+ assertEquals("iotdbModel", config.getScrapeConfigs().get(0).getJobName());
+ assertEquals("/metrics",
config.getScrapeConfigs().get(0).getMetricsPath());
+ }
+}
diff --git
a/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/PrometheusTextParserTest.java
b/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/PrometheusTextParserTest.java
new file mode 100644
index 0000000..3677f94
--- /dev/null
+++
b/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/PrometheusTextParserTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.metricscrape;
+
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+public class PrometheusTextParserTest {
+
+ @Test
+ public void parseSummarySamplesIntoHelpFamily() {
+ String text =
+ "# HELP request_duration_seconds Request duration.\n"
+ + "# TYPE request_duration_seconds summary\n"
+ + "request_duration_seconds_sum{method=\"query\",status=\"ok\"}
10.5\n"
+ + "request_duration_seconds_count{method=\"query\",status=\"ok\"}
3 1000\n";
+
+ List<PrometheusSample> samples = new PrometheusTextParser().parse(text,
123);
+
+ assertEquals(2, samples.size());
+ assertEquals("request_duration_seconds",
samples.get(0).getMetricFamilyName());
+ assertEquals("request_duration_seconds_sum",
samples.get(0).getMetricName());
+ assertEquals(10.5, samples.get(0).getValue(), 0.0);
+ assertEquals(123, samples.get(0).getTimestamp());
+ assertEquals(1000, samples.get(1).getTimestamp());
+ }
+
+ @Test
+ public void parseLabelValueContainingQueryText() {
+ String text = "statement_cost{type=\"select * from root where a=\\\"b\\\",
x=1\"} 1.0";
+
+ List<PrometheusSample> samples = new PrometheusTextParser().parse(text,
123);
+
+ assertEquals(1, samples.size());
+ assertEquals("select * from root where a=\"b\", x=1",
samples.get(0).getLabels().get("type"));
+ }
+
+ @Test
+ public void parseUnescapedQuoteInNestedLabelValueWithoutGhostLabels() {
+ String text = "m{type=\"Filter{predicate=\"x\",
op=\"AND\"}\",quantile=\"0.5\"} 1";
+
+ List<PrometheusSample> samples = new PrometheusTextParser().parse(text,
123);
+
+ assertEquals(1, samples.size());
+ assertEquals("Filter{predicate=\"x\", op=\"AND\"}",
samples.get(0).getLabels().get("type"));
+ assertEquals("0.5", samples.get(0).getLabels().get("quantile"));
+ assertFalse(samples.get(0).getLabels().containsKey("op"));
+ }
+
+ @Test
+ public void onlyRouteKnownFamilyComponentSuffixesToHelpFamily() {
+ String text =
+ "# HELP http_requests Total requests.\n"
+ + "http_requests 5\n"
+ + "http_requests_duration_seconds 0.3\n"
+ + "http_requests_sum 7\n";
+
+ List<PrometheusSample> samples = new PrometheusTextParser().parse(text,
123);
+
+ assertEquals("http_requests", samples.get(0).getMetricFamilyName());
+ assertEquals("http_requests_duration_seconds",
samples.get(1).getMetricFamilyName());
+ assertEquals("http_requests", samples.get(2).getMetricFamilyName());
+ }
+
+ @Test
+ public void parsePrometheusFloatTokens() {
+ List<PrometheusSample> samples =
+ new PrometheusTextParser().parse("a inf\nb -inf\nc nan\nd 1_000\n",
123);
+
+ assertTrue(Double.isInfinite(samples.get(0).getValue()));
+ assertTrue(Double.isInfinite(samples.get(1).getValue()));
+ assertTrue(Double.isNaN(samples.get(2).getValue()));
+ assertEquals(1000.0, samples.get(3).getValue(), 0.0);
+ }
+
+ @Test
+ public void rejectJavaSpecificFloatSuffixes() {
+ assertThrows(
+ IllegalArgumentException.class, () -> new
PrometheusTextParser().parse("a 1d\n", 123));
+ assertThrows(
+ IllegalArgumentException.class, () -> new
PrometheusTextParser().parse("a 2f\n", 123));
+ }
+
+ @Test
+ public void rejectUnknownLabelEscapes() {
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> new PrometheusTextParser().parse("a{label=\"\\t\"} 1\n", 123));
+ }
+}
diff --git a/pom.xml b/pom.xml
index f22711d..b155418 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,6 +36,7 @@
<module>connectors</module>
<module>distributions</module>
<module>iotdb-collector</module>
+ <module>metric-scrape</module>
<module>mybatis-generator</module>
</modules>
<properties>
