This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a commit to branch object_type_tiff
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/object_type_tiff by this push:
new 5b9bab3229f 更新跨行逻辑
5b9bab3229f is described below
commit 5b9bab3229f1f0c4f4fcea00c6ee976cfde051c0
Author: spricoder <[email protected]>
AuthorDate: Mon Sep 8 12:49:43 2025 +0800
更新跨行逻辑
---
.../db/utils/model/CompressedTiffModelReader.java | 160 +++++++++++----------
1 file changed, 81 insertions(+), 79 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTiffModelReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTiffModelReader.java
index 9cbd42b130f..9b0c0f35049 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTiffModelReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTiffModelReader.java
@@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -73,34 +74,29 @@ public class CompressedTiffModelReader extends ModelReader {
return out;
}
+ static class SearchReq {
+ final int idx; // 原始顺序
+ final int startPix; // 像元索引
+ final int endPix;
+ final int row;
+ final int col0, col1;
+
+ SearchReq(int idx, int startPix, int endPix, int row, int col0, int col1) {
+ this.idx = idx;
+ this.startPix = startPix;
+ this.endPix = endPix;
+ this.row = row;
+ this.col0 = col0;
+ this.col1 = col1;
+ }
+ }
+
@Override
public List<float[]> penetrate(String filePath, List<List<Integer>>
startAndEndTimeArray) {
if (startAndEndTimeArray == null || startAndEndTimeArray.isEmpty()) {
return Collections.emptyList();
}
- // 为保证返回顺序与输入一致：先把输入解析成条目列表（带原始 index）
- class Req {
- final int idx; // 原始顺序
- final int startPix; // 像元索引（0..total-1）
- final int endPix; // 像元索引（>=startPix）
- int row, col0, col1; // 解析后的行号与列范围
-
- Req(int idx, int s, int e) {
- this.idx = idx;
- this.startPix = Math.min(s, e);
- this.endPix = Math.max(s, e);
- }
- }
- List<Req> reqs = new ArrayList<>(startAndEndTimeArray.size());
- for (int i = 0; i < startAndEndTimeArray.size(); i++) {
- List<Integer> r = startAndEndTimeArray.get(i);
- if (r == null || r.size() < 2) {
- throw new IllegalArgumentException("Each range must be [start, end].");
- }
- reqs.add(new Req(i, r.get(0), r.get(1)));
- }
-
Dataset ds = gdal.OpenShared(filePath, gdalconstConstants.GA_ReadOnly);
if (ds == null) {
throw new RuntimeException("Failed to open: " + gdal.GetLastErrorMsg());
@@ -112,62 +108,65 @@ public class CompressedTiffModelReader extends
ModelReader {
final int height = band.getYSize();
final long total = (long) width * (long) height;
- // 解析行/列 & 校验仅同一行
- for (Req q : reqs) {
- if (q.startPix < 0 || (long) q.endPix >= total) {
+ // ---- step1: 解析输入，允许跨行 ----
+ List<SearchReq> reqs = new ArrayList<>();
+ for (int i = 0; i < startAndEndTimeArray.size(); i++) {
+ List<Integer> r = startAndEndTimeArray.get(i);
+ if (r == null || r.size() < 2) {
+ throw new IllegalArgumentException("Each range must be [start,
end].");
+ }
+ int sPix = Math.min(r.get(0), r.get(1));
+ int ePix = Math.max(r.get(0), r.get(1));
+ if (sPix < 0 || ePix >= total) {
throw new IndexOutOfBoundsException(
- String.format("Range [%d,%d] out of bounds [0,%d).", q.startPix,
q.endPix, total));
+ String.format("Range [%d,%d] out of bounds [0,%d).", sPix, ePix,
total));
}
- int sRow = q.startPix / width, sCol = q.startPix % width;
- int eRow = q.endPix / width, eCol = q.endPix % width;
- if (sRow != eRow) {
- throw new IllegalArgumentException(
- "Range crosses rows: [" + q.startPix + "," + q.endPix + "]");
+ int sRow = sPix / width, sCol = sPix % width;
+ int eRow = ePix / width, eCol = ePix % width;
+
+ if (sRow == eRow) {
+ // 单行：直接保留
+ reqs.add(new SearchReq(i, sPix, ePix, sRow, sCol, eCol));
+ } else {
+ // 跨行：拆分
+ // 首行片段
+ reqs.add(new SearchReq(i, sPix, (sRow + 1) * width - 1, sRow, sCol,
width - 1));
+ // 中间整行
+ for (int row = sRow + 1; row < eRow; row++) {
+ reqs.add(new SearchReq(i, row * width, (row + 1) * width - 1, row,
0, width - 1));
+ }
+ // 末行片段
+ reqs.add(new SearchReq(i, eRow * width, ePix, eRow, 0, eCol));
}
- q.row = sRow;
- q.col0 = sCol;
- q.col1 = eCol;
}
- // NoData 配置
+ // ---- step2: NoData 配置 ----
Double[] nd = new Double[1];
band.GetNoDataValue(nd);
final boolean needMapNoData = nd[0] != null && !Double.isNaN(nd[0]);
final float nodata = needMapNoData ? nd[0].floatValue() : Float.NaN;
- // 行 -> （合并前的列区间列表）
+ // ---- step3: 行 -> 区间列表 ----
Map<Integer, List<int[]>> perRow = new LinkedHashMap<>();
- for (Req q : reqs) {
+ Map<Integer, List<SearchReq>> rowReqs = new LinkedHashMap<>();
+ for (SearchReq q : reqs) {
perRow.computeIfAbsent(q.row, k -> new ArrayList<>()).add(new int[]
{q.col0, q.col1});
+ rowReqs.computeIfAbsent(q.row, k -> new ArrayList<>()).add(q);
}
- // 为每个请求预先分配目标数组，最后按 idx 顺序收集
- float[][] outputs = new float[reqs.size()][];
- for (Req q : reqs) {
- outputs[q.idx] = new float[q.col1 - q.col0 + 1];
- }
-
- // mergeGap 可按需要调大，如 4/8（允许读取少量“间隙像元”换更少的 ReadRaster 次数）
- final int mergeGap = 0;
-
- // 行内再建立“原始区间列表”（保持输入顺序），用于把窗口数据拆回去
- Map<Integer, List<Req>> rowReqs = new LinkedHashMap<>();
- for (Req q : reqs) rowReqs.computeIfAbsent(q.row, k -> new
ArrayList<>()).add(q);
+ // 每个原始 idx -> 拼接结果
+ Map<Integer, List<float[]>> grouped = new HashMap<>();
- // 逐行处理
+ // ---- step4: 按行读取，保留原优化 ----
for (Map.Entry<Integer, List<int[]>> e : perRow.entrySet()) {
int row = e.getKey();
- List<int[]> ranges = e.getValue();
+ List<int[]> ranges = mergeRanges(e.getValue(), 0);
- // 合并区间 -> 更少的读取窗口
- List<int[]> merged = mergeRanges(ranges, mergeGap);
-
- // 按起点排序，便于窗口覆盖
- List<Req> rowList = rowReqs.get(row);
+ List<SearchReq> rowList = rowReqs.get(row);
rowList.sort(Comparator.comparingInt(a -> a.col0));
int p = 0;
- for (int[] win : merged) {
+ for (int[] win : ranges) {
int c0 = win[0], c1 = win[1];
int winLen = c1 - c0 + 1;
@@ -185,46 +184,49 @@ public class CompressedTiffModelReader extends
ModelReader {
+ gdal.GetLastErrorMsg());
}
- // 把窗口分发给所有落在其中的原始区间
while (p < rowList.size()) {
- Req q = rowList.get(p);
+ SearchReq q = rowList.get(p);
if (q.col1 < c0) {
p++;
continue;
- } // 在窗口左侧，跳过
- if (q.col0 > c1) {
- break;
- } // 窗口右侧，进入下一个窗口
+ }
+ if (q.col0 > c1) break;
int from = Math.max(q.col0, c0);
int to = Math.min(q.col1, c1);
int len = to - from + 1;
+ float[] seg = new float[len];
+ System.arraycopy(scratch, from - c0, seg, 0, len);
- float[] dst = outputs[q.idx];
- System.arraycopy(scratch, from - c0, dst, from - q.col0, len);
+ grouped.computeIfAbsent(q.idx, k -> new ArrayList<>()).add(seg);
- // 注意：若 mergeGap>0，极端情况下一个 req 可能跨两个合并窗口；
- // 这里不 p++，而是仅当整个 req 覆盖完才前移指针
if (to == q.col1) {
p++;
- } // 该 req 已经完全覆盖
- else {
- break;
- } // 仍有剩余，等待下一窗口补齐
+ } else break;
}
}
}
- // NoData -> NaN
- if (needMapNoData) {
- for (float[] arr : outputs) {
- for (int i = 0; i < arr.length; i++) if (arr[i] == nodata) arr[i] =
Float.NaN;
+ // ---- step5: 拼接 & 替换 NoData ----
+ List<float[]> result = new ArrayList<>(startAndEndTimeArray.size());
+ for (int i = 0; i < startAndEndTimeArray.size(); i++) {
+ List<float[]> parts = grouped.get(i);
+ if (parts == null) {
+ result.add(new float[0]);
+ continue;
+ }
+ int totalLen = parts.stream().mapToInt(a -> a.length).sum();
+ float[] merged = new float[totalLen];
+ int pos = 0;
+ for (float[] seg : parts) {
+ System.arraycopy(seg, 0, merged, pos, seg.length);
+ pos += seg.length;
}
+ if (needMapNoData) {
+ for (int j = 0; j < merged.length; j++) if (merged[j] == nodata)
merged[j] = Float.NaN;
+ }
+ result.add(merged);
}
-
- // 按原始顺序返回
- List<float[]> result = new ArrayList<>(outputs.length);
- for (int i = 0; i < outputs.length; i++) result.add(outputs[i]);
return result;
} finally {
ds.delete();
