GitHub user swaroopchenchu added a comment to the discussion: Pipeline
processing speeds become slower as output grown.
MongoDB logs mentions slow query log, i think it takes around 36 secs to place
10000 records
and currently collection have 122 million records and still pipeline is running
{"t":{"$date":"2025-11-25T07:02:39.487+00:00"},"s":"I", "c":"COMMAND",
"id":51803, "ctx":"conn1021180","msg":"Slow
query","attr":{"type":"command","ns":"cms-db.optiva/410/confirmationDetail","command":{"insert":"optiva/410/confirmationDetail","ordered":true,"writeConcern":{"w":1},"txnNumber":12487,"$db":"cms-db","$clusterTime":{"clusterTime":{"$timestamp":{"t":1764054122,"i":502}},"signature":{"hash":{"$binary":{"base64":"xd6DBArksU6TWTnUC/1HIzxlhek=","subType":"0"}},"keyId":7574110960332308487}},"lsid":{"id":{"$uuid":"2ff318f1-592d-4dc6-9efc-14e947cd7961"}}},"ninserted":10000,"keysInserted":10000,"numYields":0,"reslen":230,"locks":{"ParallelBatchWriterMode":{"acquireCount":{"r":40}},"FeatureCompatibilityVersion":{"acquireCount":{"w":40}},"ReplicationStateTransition":{"acquireCount":{"w":41}},"Global":{"acquireCount":{"w":40}},"Database":{"acquireCount":{"w":40}},"Collection":{"acquireCount":{"w":40}},"Mutex":{"acquireCount":{"r":40}}},"authorization":{"startedUserCacheAcquisitio
nAttempts":1,"completedUserCacheAcquisitionAttempts":1,"userCacheWaitTimeMicros":6},"flowControl":{"acquireCount":20,"timeAcquiringMicros":70},"readConcern":{"level":"local","provenance":"implicitDefault"},"writeConcern":{"w":1,"wtimeout":0,"provenance":"clientSupplied"},"storage":{"data":{"bytesRead":82242929,"timeReadingMicros":35481980}},"remote":"10.200.2.159:37842","protocol":"op_msg","durationMillis":35920}}
GitHub link:
https://github.com/apache/hop/discussions/6070#discussioncomment-15071172
----
This is an automatically sent email for [email protected].
To unsubscribe, please send an email to: [email protected]
