This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch record-ttl
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 27ca9e04c3f1a9ffee6c3d06cfa1bb484bb91979
Author: Wu Sheng <wu.sh...@foxmail.com>
AuthorDate: Tue Aug 20 11:29:23 2019 +0800
Fix TTL issue.
---
.../server/core/storage/ttl/GeneralStorageTTL.java | 6 +++++-
.../{StorageTTL.java => RecordTTLCalculator.java} | 17 ++++++++++++-----
.../oap/server/core/storage/ttl/StorageTTL.java | 4 +++-
.../elasticsearch/base/HistoryDeleteEsDAO.java | 9 ++++++++-
.../elasticsearch/ttl/ElasticsearchStorageTTL.java | 6 +++++-
.../elasticsearch/ttl/EsRecordTTLCalculator.java} | 20 ++++++++++++++------
.../plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java | 12 ++++++++++--
7 files changed, 57 insertions(+), 17 deletions(-)
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/GeneralStorageTTL.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/GeneralStorageTTL.java
index aefaa53..971bcb7 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/GeneralStorageTTL.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/GeneralStorageTTL.java
@@ -24,7 +24,7 @@ import
org.apache.skywalking.oap.server.core.analysis.Downsampling;
*/
public class GeneralStorageTTL implements StorageTTL {
- @Override public TTLCalculator calculator(Downsampling downsampling) {
+ @Override public TTLCalculator metricsCalculator(Downsampling
downsampling) {
switch (downsampling) {
case Hour:
return new HourTTLCalculator();
@@ -38,4 +38,8 @@ public class GeneralStorageTTL implements StorageTTL {
return new MinuteTTLCalculator();
}
}
+
+ @Override public TTLCalculator recordCalculator() {
+ return new RecordTTLCalculator();
+ }
}
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/RecordTTLCalculator.java
similarity index 67%
copy from
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java
copy to
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/RecordTTLCalculator.java
index 0f38659..e64582e 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/RecordTTLCalculator.java
@@ -13,15 +13,22 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
+ *
*/
package org.apache.skywalking.oap.server.core.storage.ttl;
-import org.apache.skywalking.oap.server.core.analysis.Downsampling;
+import org.apache.skywalking.oap.server.core.DataTTLConfig;
+import org.joda.time.DateTime;
/**
- * @author peng-yongsheng
+ * Calculate TTL for record.
+ *
+ * @author wusheng
*/
-public interface StorageTTL {
- TTLCalculator calculator(Downsampling downsampling);
-}
+public class RecordTTLCalculator implements TTLCalculator {
+
+ @Override public long timeBefore(DateTime currentTime, DataTTLConfig
dataTTLConfig) {
+ return Long.valueOf(currentTime.plusMinutes(0 -
dataTTLConfig.getRecordDataTTL()).toString("yyyyMMddHHmm"));
+ }
+}
\ No newline at end of file
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java
index 0f38659..7a0b4e9 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java
@@ -23,5 +23,7 @@ import
org.apache.skywalking.oap.server.core.analysis.Downsampling;
* @author peng-yongsheng
*/
public interface StorageTTL {
- TTLCalculator calculator(Downsampling downsampling);
+ TTLCalculator metricsCalculator(Downsampling downsampling);
+
+ TTLCalculator recordCalculator();
}
diff --git
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java
index 1299cc6..e780e1f 100644
---
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java
@@ -25,6 +25,7 @@ import
org.apache.skywalking.oap.server.core.config.ConfigService;
import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO;
import org.apache.skywalking.oap.server.core.storage.model.Model;
import org.apache.skywalking.oap.server.core.storage.ttl.StorageTTL;
+import org.apache.skywalking.oap.server.core.storage.ttl.TTLCalculator;
import
org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder;
import org.joda.time.DateTime;
@@ -51,7 +52,13 @@ public class HistoryDeleteEsDAO extends EsDAO implements
IHistoryDeleteDAO {
ConfigService configService =
moduleDefineHolder.find(CoreModule.NAME).provider().getService(ConfigService.class);
ElasticSearchClient client = getClient();
- long timeBefore =
storageTTL.calculator(model.getDownsampling()).timeBefore(new DateTime(),
configService.getDataTTLConfig());
+ TTLCalculator ttlCalculator;
+ if (model.isRecord()) {
+ ttlCalculator = storageTTL.recordCalculator();
+ } else {
+ ttlCalculator =
storageTTL.metricsCalculator(model.getDownsampling());
+ }
+ long timeBefore = ttlCalculator.timeBefore(new DateTime(),
configService.getDataTTLConfig());
if (model.isCapableOfTimeSeries()) {
List<String> indexes =
client.retrievalIndexByAliases(model.getName());
diff --git
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/ElasticsearchStorageTTL.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/ElasticsearchStorageTTL.java
index 5779de4..46af52d 100644
---
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/ElasticsearchStorageTTL.java
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/ElasticsearchStorageTTL.java
@@ -25,7 +25,7 @@ import org.apache.skywalking.oap.server.core.storage.ttl.*;
*/
public class ElasticsearchStorageTTL implements StorageTTL {
- @Override public TTLCalculator calculator(Downsampling downsampling) {
+ @Override public TTLCalculator metricsCalculator(Downsampling
downsampling) {
switch (downsampling) {
case Month:
return new MonthTTLCalculator();
@@ -37,4 +37,8 @@ public class ElasticsearchStorageTTL implements StorageTTL {
return new DayTTLCalculator();
}
}
+
+ @Override public TTLCalculator recordCalculator() {
+ return new EsRecordTTLCalculator();
+ }
}
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/EsRecordTTLCalculator.java
similarity index 58%
copy from
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java
copy to
oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/EsRecordTTLCalculator.java
index 0f38659..0154038 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/EsRecordTTLCalculator.java
@@ -13,15 +13,23 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
+ *
*/
-package org.apache.skywalking.oap.server.core.storage.ttl;
+package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ttl;
-import org.apache.skywalking.oap.server.core.analysis.Downsampling;
+import org.apache.skywalking.oap.server.core.DataTTLConfig;
+import org.apache.skywalking.oap.server.core.storage.ttl.TTLCalculator;
+import org.joda.time.DateTime;
/**
- * @author peng-yongsheng
+ * Calculate TTL for record.
+ *
+ * @author wusheng
*/
-public interface StorageTTL {
- TTLCalculator calculator(Downsampling downsampling);
-}
+public class EsRecordTTLCalculator implements TTLCalculator {
+
+ @Override public long timeBefore(DateTime currentTime, DataTTLConfig
dataTTLConfig) {
+ return Long.valueOf(currentTime.plusMinutes(0 -
dataTTLConfig.getRecordDataTTL()).toString("yyyyMMdd"));
+ }
+}
\ No newline at end of file
diff --git
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java
b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java
index 69fcd40..f709566 100644
---
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java
+++
b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java
@@ -19,12 +19,14 @@
package org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao;
import java.io.IOException;
-import java.sql.*;
+import java.sql.Connection;
+import java.sql.SQLException;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.config.ConfigService;
import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO;
import org.apache.skywalking.oap.server.core.storage.model.Model;
import org.apache.skywalking.oap.server.core.storage.ttl.StorageTTL;
+import org.apache.skywalking.oap.server.core.storage.ttl.TTLCalculator;
import
org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException;
import
org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient;
import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder;
@@ -53,7 +55,13 @@ public class H2HistoryDeleteDAO implements IHistoryDeleteDAO
{
SQLBuilder dataDeleteSQL = new SQLBuilder("delete from " +
model.getName() + " where ").append(timeBucketColumnName).append("<= ?");
try (Connection connection = client.getConnection()) {
- long timeBefore =
storageTTL.calculator(model.getDownsampling()).timeBefore(new DateTime(),
configService.getDataTTLConfig());
+ TTLCalculator ttlCalculator;
+ if (model.isRecord()) {
+ ttlCalculator = storageTTL.recordCalculator();
+ } else {
+ ttlCalculator =
storageTTL.metricsCalculator(model.getDownsampling());
+ }
+ long timeBefore = ttlCalculator.timeBefore(new DateTime(),
configService.getDataTTLConfig());
client.execute(connection, dataDeleteSQL.toString(), timeBefore);
} catch (JDBCClientException | SQLException e) {
throw new IOException(e.getMessage(), e);
