dsmiley commented on code in PR #882:
URL: https://github.com/apache/solr/pull/882#discussion_r888042243
##########
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java:
##########
@@ -278,48 +279,72 @@ private void parseError(SolrInputDocument lineRecord,
String line, String trace)
if (this.cause != null) {
lineRecord.setField("root_cause_t", cause.replace("Caused by:",
"").trim());
}
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
}
- private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("qtime_i", parseQTime(line));
- lineRecord.setField("status_s", parseStatus(line));
-
- String path = parsePath(line);
- lineRecord.setField("path_s", path);
-
- if (line.contains("hits=")) {
- lineRecord.setField("hits_l", parseHits(line));
+ private Map<String, Object> extractJSONFormattedMessage(String line) {
+ if (line.contains("o.a.s.c.S.Request")
+ || (line.contains("o.a.s.s.HttpSolrCall") &&
line.contains("\"prefix\""))) {
+ int startPos = 0; // '{' starts at
+ while (startPos < line.length() && line.charAt(startPos) != '{') {
+ startPos++;
+ }
Review Comment:
See `String.indexOf` :-)
##########
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java:
##########
@@ -278,48 +279,72 @@ private void parseError(SolrInputDocument lineRecord,
String line, String trace)
if (this.cause != null) {
lineRecord.setField("root_cause_t", cause.replace("Caused by:",
"").trim());
}
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
}
- private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("qtime_i", parseQTime(line));
- lineRecord.setField("status_s", parseStatus(line));
-
- String path = parsePath(line);
- lineRecord.setField("path_s", path);
-
- if (line.contains("hits=")) {
- lineRecord.setField("hits_l", parseHits(line));
+ private Map<String, Object> extractJSONFormattedMessage(String line) {
+ if (line.contains("o.a.s.c.S.Request")
+ || (line.contains("o.a.s.s.HttpSolrCall") &&
line.contains("\"prefix\""))) {
+ int startPos = 0; // '{' starts at
+ while (startPos < line.length() && line.charAt(startPos) != '{') {
+ startPos++;
+ }
+ int endPos = startPos + 1; // '}' ends at
+ while (endPos < line.length() && line.charAt(endPos) != '}') {
+ endPos++;
+ }
+ String json = line.substring(startPos, endPos + 1);
+ @SuppressWarnings("unchecked")
+ Map<String, Object> fromJSON = (Map<String, Object>)
Utils.fromJSONString(json);
+ return fromJSON;
}
+ return null;
+ }
- String params = parseParams(line);
- lineRecord.setField("params_t", params);
- addParams(lineRecord, params);
+ private void parseRecord(SolrInputDocument lineRecord, String line) {
+ Map<String, Object> keyValuePairs = extractJSONFormattedMessage(line);
+ if (keyValuePairs != null) {
+ // query request record
+ if (keyValuePairs.containsKey("QTime")) {
+ lineRecord.setField("qtime_i", keyValuePairs.get("QTime"));
+ lineRecord.setField("status_s", keyValuePairs.get("status"));
+ lineRecord.setField("path_s", keyValuePairs.get("path"));
+ if (keyValuePairs.containsKey("hits")) {
+ lineRecord.setField("hits_l", keyValuePairs.get("hits"));
+ }
+ lineRecord.setField("params_t", keyValuePairs.get("params"));
+ addParams(lineRecord, keyValuePairs.get("params").toString());
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("node_s", parseNode(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
+ lineRecord.setField("node_s", keyValuePairs.get("node_name"));
- if (path != null && path.contains("/admin")) {
- lineRecord.setField("type_s", "admin");
- } else if (path != null && params.contains("/replication")) {
- lineRecord.setField("type_s", "replication");
- } else if (path != null && path.contains("/get")) {
- lineRecord.setField("type_s", "get");
- } else {
- lineRecord.setField("type_s", "query");
+ if (keyValuePairs.containsKey("prefix") &&
keyValuePairs.get("prefix").equals("admin")) {
+ lineRecord.setField("type_s", "admin");
+ } else if
(keyValuePairs.get("params").toString().contains("/replication")) {
+ lineRecord.setField("type_s", "replication");
+ } else if (keyValuePairs.get("path").equals("/get")) {
+ lineRecord.setField("type_s", "get");
+ } else {
+ lineRecord.setField("type_s", "query");
+ }
+ }
+
+ // update request record
+ if (keyValuePairs.get("path").equals("/update")) {
+ if (keyValuePairs.containsKey("deleteByQuery")) {
+ lineRecord.setField("type_s", "deleteByQuery");
+ } else if (keyValuePairs.containsKey("delete")) {
+ lineRecord.setField("type_s", "delete");
+ } else if (keyValuePairs.containsKey("commit") && (Boolean)
keyValuePairs.get("commit")) {
+ lineRecord.setField("type_s", "commit");
+ } else {
+ lineRecord.setField("type_s", "update");
+ }
+ }
}
}
- private void parseNewSearch(SolrInputDocument lineRecord, String line) {
+ private void setMDCFields(SolrInputDocument lineRecord, String line) {
+ // x:value format
lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("type_s", "newSearcher");
Review Comment:
why remove?
##########
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java:
##########
@@ -278,48 +279,72 @@ private void parseError(SolrInputDocument lineRecord,
String line, String trace)
if (this.cause != null) {
lineRecord.setField("root_cause_t", cause.replace("Caused by:",
"").trim());
}
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
}
- private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("qtime_i", parseQTime(line));
- lineRecord.setField("status_s", parseStatus(line));
-
- String path = parsePath(line);
- lineRecord.setField("path_s", path);
-
- if (line.contains("hits=")) {
- lineRecord.setField("hits_l", parseHits(line));
+ private Map<String, Object> extractJSONFormattedMessage(String line) {
+ if (line.contains("o.a.s.c.S.Request")
+ || (line.contains("o.a.s.s.HttpSolrCall") &&
line.contains("\"prefix\""))) {
+ int startPos = 0; // '{' starts at
+ while (startPos < line.length() && line.charAt(startPos) != '{') {
+ startPos++;
+ }
+ int endPos = startPos + 1; // '}' ends at
+ while (endPos < line.length() && line.charAt(endPos) != '}') {
+ endPos++;
+ }
Review Comment:
Can we assume the message ends with '}'? I think so.
##########
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java:
##########
@@ -278,48 +279,72 @@ private void parseError(SolrInputDocument lineRecord,
String line, String trace)
if (this.cause != null) {
lineRecord.setField("root_cause_t", cause.replace("Caused by:",
"").trim());
}
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
}
- private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("qtime_i", parseQTime(line));
- lineRecord.setField("status_s", parseStatus(line));
-
- String path = parsePath(line);
- lineRecord.setField("path_s", path);
-
- if (line.contains("hits=")) {
- lineRecord.setField("hits_l", parseHits(line));
+ private Map<String, Object> extractJSONFormattedMessage(String line) {
+ if (line.contains("o.a.s.c.S.Request")
Review Comment:
Maybe we shouldn't even bother looking for the specific classes we know do
this; just look for valid JSON? This will future-proof other use of JSON and
allow someone to format the logger names more fully and have this tool still
work.
##########
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java:
##########
@@ -278,48 +279,72 @@ private void parseError(SolrInputDocument lineRecord,
String line, String trace)
if (this.cause != null) {
lineRecord.setField("root_cause_t", cause.replace("Caused by:",
"").trim());
}
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
}
- private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("qtime_i", parseQTime(line));
- lineRecord.setField("status_s", parseStatus(line));
-
- String path = parsePath(line);
- lineRecord.setField("path_s", path);
-
- if (line.contains("hits=")) {
- lineRecord.setField("hits_l", parseHits(line));
+ private Map<String, Object> extractJSONFormattedMessage(String line) {
Review Comment:
As a general rule, when creating variable/method/class names, don't use
all-caps for acronyms. Yes, you will see many cases where existing code
doesn't do this but most developers find it best to just lowercase them. Thus
this here use "Json" not "JSON". Same for "Mdc" not "MDC".
##########
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java:
##########
@@ -278,48 +279,72 @@ private void parseError(SolrInputDocument lineRecord,
String line, String trace)
if (this.cause != null) {
lineRecord.setField("root_cause_t", cause.replace("Caused by:",
"").trim());
}
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
}
- private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("qtime_i", parseQTime(line));
- lineRecord.setField("status_s", parseStatus(line));
-
- String path = parsePath(line);
- lineRecord.setField("path_s", path);
-
- if (line.contains("hits=")) {
- lineRecord.setField("hits_l", parseHits(line));
+ private Map<String, Object> extractJSONFormattedMessage(String line) {
+ if (line.contains("o.a.s.c.S.Request")
+ || (line.contains("o.a.s.s.HttpSolrCall") &&
line.contains("\"prefix\""))) {
+ int startPos = 0; // '{' starts at
+ while (startPos < line.length() && line.charAt(startPos) != '{') {
+ startPos++;
+ }
+ int endPos = startPos + 1; // '}' ends at
+ while (endPos < line.length() && line.charAt(endPos) != '}') {
+ endPos++;
+ }
Review Comment:
Also, note `String.lastIndexOf` :-)
##########
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java:
##########
@@ -278,48 +279,72 @@ private void parseError(SolrInputDocument lineRecord,
String line, String trace)
if (this.cause != null) {
lineRecord.setField("root_cause_t", cause.replace("Caused by:",
"").trim());
}
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
}
- private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("qtime_i", parseQTime(line));
- lineRecord.setField("status_s", parseStatus(line));
-
- String path = parsePath(line);
- lineRecord.setField("path_s", path);
-
- if (line.contains("hits=")) {
- lineRecord.setField("hits_l", parseHits(line));
+ private Map<String, Object> extractJSONFormattedMessage(String line) {
+ if (line.contains("o.a.s.c.S.Request")
+ || (line.contains("o.a.s.s.HttpSolrCall") &&
line.contains("\"prefix\""))) {
+ int startPos = 0; // '{' starts at
+ while (startPos < line.length() && line.charAt(startPos) != '{') {
+ startPos++;
+ }
+ int endPos = startPos + 1; // '}' ends at
+ while (endPos < line.length() && line.charAt(endPos) != '}') {
+ endPos++;
+ }
+ String json = line.substring(startPos, endPos + 1);
+ @SuppressWarnings("unchecked")
+ Map<String, Object> fromJSON = (Map<String, Object>)
Utils.fromJSONString(json);
+ return fromJSON;
}
+ return null;
+ }
- String params = parseParams(line);
- lineRecord.setField("params_t", params);
- addParams(lineRecord, params);
+ private void parseRecord(SolrInputDocument lineRecord, String line) {
+ Map<String, Object> keyValuePairs = extractJSONFormattedMessage(line);
+ if (keyValuePairs != null) {
+ // query request record
+ if (keyValuePairs.containsKey("QTime")) {
+ lineRecord.setField("qtime_i", keyValuePairs.get("QTime"));
+ lineRecord.setField("status_s", keyValuePairs.get("status"));
+ lineRecord.setField("path_s", keyValuePairs.get("path"));
+ if (keyValuePairs.containsKey("hits")) {
+ lineRecord.setField("hits_l", keyValuePairs.get("hits"));
+ }
+ lineRecord.setField("params_t", keyValuePairs.get("params"));
+ addParams(lineRecord, keyValuePairs.get("params").toString());
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("node_s", parseNode(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
+ lineRecord.setField("node_s", keyValuePairs.get("node_name"));
- if (path != null && path.contains("/admin")) {
- lineRecord.setField("type_s", "admin");
- } else if (path != null && params.contains("/replication")) {
- lineRecord.setField("type_s", "replication");
- } else if (path != null && path.contains("/get")) {
- lineRecord.setField("type_s", "get");
- } else {
- lineRecord.setField("type_s", "query");
+ if (keyValuePairs.containsKey("prefix") &&
keyValuePairs.get("prefix").equals("admin")) {
+ lineRecord.setField("type_s", "admin");
+ } else if
(keyValuePairs.get("params").toString().contains("/replication")) {
+ lineRecord.setField("type_s", "replication");
+ } else if (keyValuePairs.get("path").equals("/get")) {
+ lineRecord.setField("type_s", "get");
+ } else {
+ lineRecord.setField("type_s", "query");
+ }
+ }
+
+ // update request record
+ if (keyValuePairs.get("path").equals("/update")) {
+ if (keyValuePairs.containsKey("deleteByQuery")) {
+ lineRecord.setField("type_s", "deleteByQuery");
+ } else if (keyValuePairs.containsKey("delete")) {
+ lineRecord.setField("type_s", "delete");
+ } else if (keyValuePairs.containsKey("commit") && (Boolean)
keyValuePairs.get("commit")) {
+ lineRecord.setField("type_s", "commit");
+ } else {
Review Comment:
These else-if's surprise me; I think a log line could have all of these?
@joel-bernstein I see Nazerke merely updated the same logic so this decision I
think was written by you. Should the "type" of the log line be, in effect,
multi-valued because there are multiple update payloads (delete (id and/or
query), docs, commits)?
##########
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java:
##########
@@ -278,48 +279,72 @@ private void parseError(SolrInputDocument lineRecord,
String line, String trace)
if (this.cause != null) {
lineRecord.setField("root_cause_t", cause.replace("Caused by:",
"").trim());
}
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
}
- private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("qtime_i", parseQTime(line));
- lineRecord.setField("status_s", parseStatus(line));
-
- String path = parsePath(line);
- lineRecord.setField("path_s", path);
-
- if (line.contains("hits=")) {
- lineRecord.setField("hits_l", parseHits(line));
+ private Map<String, Object> extractJSONFormattedMessage(String line) {
+ if (line.contains("o.a.s.c.S.Request")
+ || (line.contains("o.a.s.s.HttpSolrCall") &&
line.contains("\"prefix\""))) {
+ int startPos = 0; // '{' starts at
+ while (startPos < line.length() && line.charAt(startPos) != '{') {
+ startPos++;
+ }
+ int endPos = startPos + 1; // '}' ends at
+ while (endPos < line.length() && line.charAt(endPos) != '}') {
+ endPos++;
+ }
+ String json = line.substring(startPos, endPos + 1);
+ @SuppressWarnings("unchecked")
+ Map<String, Object> fromJSON = (Map<String, Object>)
Utils.fromJSONString(json);
+ return fromJSON;
}
+ return null;
+ }
- String params = parseParams(line);
- lineRecord.setField("params_t", params);
- addParams(lineRecord, params);
+ private void parseRecord(SolrInputDocument lineRecord, String line) {
+ Map<String, Object> keyValuePairs = extractJSONFormattedMessage(line);
+ if (keyValuePairs != null) {
+ // query request record
+ if (keyValuePairs.containsKey("QTime")) {
+ lineRecord.setField("qtime_i", keyValuePairs.get("QTime"));
+ lineRecord.setField("status_s", keyValuePairs.get("status"));
+ lineRecord.setField("path_s", keyValuePairs.get("path"));
+ if (keyValuePairs.containsKey("hits")) {
+ lineRecord.setField("hits_l", keyValuePairs.get("hits"));
+ }
+ lineRecord.setField("params_t", keyValuePairs.get("params"));
+ addParams(lineRecord, keyValuePairs.get("params").toString());
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("node_s", parseNode(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
+ lineRecord.setField("node_s", keyValuePairs.get("node_name"));
- if (path != null && path.contains("/admin")) {
- lineRecord.setField("type_s", "admin");
- } else if (path != null && params.contains("/replication")) {
- lineRecord.setField("type_s", "replication");
- } else if (path != null && path.contains("/get")) {
- lineRecord.setField("type_s", "get");
- } else {
- lineRecord.setField("type_s", "query");
+ if (keyValuePairs.containsKey("prefix") &&
keyValuePairs.get("prefix").equals("admin")) {
+ lineRecord.setField("type_s", "admin");
+ } else if
(keyValuePairs.get("params").toString().contains("/replication")) {
+ lineRecord.setField("type_s", "replication");
+ } else if (keyValuePairs.get("path").equals("/get")) {
+ lineRecord.setField("type_s", "get");
+ } else {
+ lineRecord.setField("type_s", "query");
+ }
+ }
+
+ // update request record
+ if (keyValuePairs.get("path").equals("/update")) {
Review Comment:
there used to be a method `parseUpdate` and I think it makes sense to
continue to have that structure
##########
solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java:
##########
@@ -278,48 +279,72 @@ private void parseError(SolrInputDocument lineRecord,
String line, String trace)
if (this.cause != null) {
lineRecord.setField("root_cause_t", cause.replace("Caused by:",
"").trim());
}
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
}
- private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("qtime_i", parseQTime(line));
- lineRecord.setField("status_s", parseStatus(line));
-
- String path = parsePath(line);
- lineRecord.setField("path_s", path);
-
- if (line.contains("hits=")) {
- lineRecord.setField("hits_l", parseHits(line));
+ private Map<String, Object> extractJSONFormattedMessage(String line) {
+ if (line.contains("o.a.s.c.S.Request")
Review Comment:
If we can assume the pertinent log lines end with a "}", this is a very
cheap condition to add as well.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@solr.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@solr.apache.org
For additional commands, e-mail: issues-h...@solr.apache.org
