dsmiley commented on code in PR #3682:
URL: https://github.com/apache/solr/pull/3682#discussion_r2373362021
##########
solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java:
##########
@@ -387,12 +239,192 @@ public Map<String, Object> getRunningMerges() {
@Override
protected void doAfterFlush() throws IOException {
- if (flushes != null) { // this is null when writer is used only for
snapshot cleanup
- flushes.inc(); // or if mergeTotals == false
+ if (flushesCounter != null) { // this is null when writer is used only for
snapshot cleanup
+ flushesCounter.inc(); // or if mergeTotals == false
}
super.doAfterFlush();
}
+ private void initMetrics(final SolrCore core) {
+ if (solrMetricsContext == null) {
+ solrMetricsContext = core.getSolrMetricsContext().getChildContext(this);
+ }
+ var coreName = core.getName();
+ var baseAttributesBuilder =
+ Attributes.builder()
+ .put(CATEGORY_ATTR, SolrInfoBean.Category.INDEX.toString())
+ .put(CORE_ATTR, coreName);
+ if (core.getCoreContainer().isZooKeeperAware()) {
+ String collectionName = core.getCoreDescriptor().getCollectionName();
+ baseAttributesBuilder
+ .put(COLLECTION_ATTR, collectionName)
+ .put(SHARD_ATTR,
core.getCoreDescriptor().getCloudDescriptor().getShardId())
+ .put(REPLICA_ATTR, Utils.parseMetricsReplicaName(collectionName,
coreName));
+ }
+ var baseAttributes = baseAttributesBuilder.build();
+
+ var mergeTimerBaseMetric =
+ solrMetricsContext.longHistogram(
+ "solr_indexwriter_merge_time", "Time spent merging segments",
OtelUnit.MILLISECONDS);
+
+ majorMergeTimer =
+ new AttributedLongTimer(
+ mergeTimerBaseMetric,
baseAttributes.toBuilder().put(MERGE_TYPE_ATTR, "major").build());
+ minorMergeTimer =
+ new AttributedLongTimer(
+ mergeTimerBaseMetric,
baseAttributes.toBuilder().put(MERGE_TYPE_ATTR, "minor").build());
+
+ mergeErrorsCounter =
+ new AttributedLongCounter(
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_merge_errors", "Number of merge errors"),
+ baseAttributes);
+
+ flushesCounter =
+ new AttributedLongCounter(
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_flushes", "Number of times documents have
been flushed to disk"),
+ baseAttributes);
+
+ var mergesCountBaseMetric =
+ solrMetricsContext.longCounter("solr_indexwriter_merges", "Number of
merge operations");
+ var docsMergedCountBaseMetric =
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_docs_merged", "Number of documents involved in
merge");
+ var docsDeletedCountBasedMetric =
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_docs_deleted", "Number of documents deleted in
merge");
+ var segmentsCountBaseMetric =
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_segments_merged", "Number of segments involved
in merge");
Review Comment:
similar, could be "create" vs "merge" as operations in an attribute
##########
solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java:
##########
@@ -387,12 +239,192 @@ public Map<String, Object> getRunningMerges() {
@Override
protected void doAfterFlush() throws IOException {
- if (flushes != null) { // this is null when writer is used only for
snapshot cleanup
- flushes.inc(); // or if mergeTotals == false
+ if (flushesCounter != null) { // this is null when writer is used only for
snapshot cleanup
+ flushesCounter.inc(); // or if mergeTotals == false
}
super.doAfterFlush();
}
+ private void initMetrics(final SolrCore core) {
+ if (solrMetricsContext == null) {
+ solrMetricsContext = core.getSolrMetricsContext().getChildContext(this);
+ }
+ var coreName = core.getName();
+ var baseAttributesBuilder =
+ Attributes.builder()
+ .put(CATEGORY_ATTR, SolrInfoBean.Category.INDEX.toString())
+ .put(CORE_ATTR, coreName);
+ if (core.getCoreContainer().isZooKeeperAware()) {
+ String collectionName = core.getCoreDescriptor().getCollectionName();
+ baseAttributesBuilder
+ .put(COLLECTION_ATTR, collectionName)
+ .put(SHARD_ATTR,
core.getCoreDescriptor().getCloudDescriptor().getShardId())
+ .put(REPLICA_ATTR, Utils.parseMetricsReplicaName(collectionName,
coreName));
+ }
+ var baseAttributes = baseAttributesBuilder.build();
+
+ var mergeTimerBaseMetric =
+ solrMetricsContext.longHistogram(
+ "solr_indexwriter_merge_time", "Time spent merging segments",
OtelUnit.MILLISECONDS);
+
+ majorMergeTimer =
+ new AttributedLongTimer(
+ mergeTimerBaseMetric,
baseAttributes.toBuilder().put(MERGE_TYPE_ATTR, "major").build());
+ minorMergeTimer =
+ new AttributedLongTimer(
+ mergeTimerBaseMetric,
baseAttributes.toBuilder().put(MERGE_TYPE_ATTR, "minor").build());
+
+ mergeErrorsCounter =
+ new AttributedLongCounter(
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_merge_errors", "Number of merge errors"),
+ baseAttributes);
+
+ flushesCounter =
+ new AttributedLongCounter(
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_flushes", "Number of times documents have
been flushed to disk"),
+ baseAttributes);
+
+ var mergesCountBaseMetric =
+ solrMetricsContext.longCounter("solr_indexwriter_merges", "Number of
merge operations");
+ var docsMergedCountBaseMetric =
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_docs_merged", "Number of documents involved in
merge");
+ var docsDeletedCountBasedMetric =
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_docs_deleted", "Number of documents deleted in
merge");
Review Comment:
I'm thinking maybe we can have stats on docs and use an operation attribute
like "merged", "deleted", "created" (the lucene word there would be "flush" but
its too confusing for users). The fact that the first 2 happen on merge is a
detail and not fundamental characteristic of a metric that must be in the name.
##########
solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java:
##########
@@ -387,12 +239,192 @@ public Map<String, Object> getRunningMerges() {
@Override
protected void doAfterFlush() throws IOException {
- if (flushes != null) { // this is null when writer is used only for
snapshot cleanup
- flushes.inc(); // or if mergeTotals == false
+ if (flushesCounter != null) { // this is null when writer is used only for
snapshot cleanup
+ flushesCounter.inc(); // or if mergeTotals == false
}
super.doAfterFlush();
}
+ private void initMetrics(final SolrCore core) {
+ if (solrMetricsContext == null) {
+ solrMetricsContext = core.getSolrMetricsContext().getChildContext(this);
+ }
+ var coreName = core.getName();
+ var baseAttributesBuilder =
+ Attributes.builder()
+ .put(CATEGORY_ATTR, SolrInfoBean.Category.INDEX.toString())
+ .put(CORE_ATTR, coreName);
+ if (core.getCoreContainer().isZooKeeperAware()) {
+ String collectionName = core.getCoreDescriptor().getCollectionName();
+ baseAttributesBuilder
+ .put(COLLECTION_ATTR, collectionName)
+ .put(SHARD_ATTR,
core.getCoreDescriptor().getCloudDescriptor().getShardId())
+ .put(REPLICA_ATTR, Utils.parseMetricsReplicaName(collectionName,
coreName));
+ }
+ var baseAttributes = baseAttributesBuilder.build();
+
+ var mergeTimerBaseMetric =
+ solrMetricsContext.longHistogram(
+ "solr_indexwriter_merge_time", "Time spent merging segments",
OtelUnit.MILLISECONDS);
+
+ majorMergeTimer =
+ new AttributedLongTimer(
+ mergeTimerBaseMetric,
baseAttributes.toBuilder().put(MERGE_TYPE_ATTR, "major").build());
+ minorMergeTimer =
+ new AttributedLongTimer(
+ mergeTimerBaseMetric,
baseAttributes.toBuilder().put(MERGE_TYPE_ATTR, "minor").build());
+
+ mergeErrorsCounter =
+ new AttributedLongCounter(
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_merge_errors", "Number of merge errors"),
+ baseAttributes);
+
+ flushesCounter =
+ new AttributedLongCounter(
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_flushes", "Number of times documents have
been flushed to disk"),
+ baseAttributes);
+
+ var mergesCountBaseMetric =
+ solrMetricsContext.longCounter("solr_indexwriter_merges", "Number of
merge operations");
+ var docsMergedCountBaseMetric =
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_docs_merged", "Number of documents involved in
merge");
+ var docsDeletedCountBasedMetric =
+ solrMetricsContext.longCounter(
+ "solr_indexwriter_docs_deleted", "Number of documents deleted in
merge");
Review Comment:
BTW, the _write amplification factor_ = 1 + docs_merged / docs_created
WAF is a classic metric in assessing the index costs of a log structured
index, like Lucene.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
