[
https://issues.apache.org/jira/browse/SPARK-37391?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17447529#comment-17447529
]
Danny Guinther commented on SPARK-37391:
----------------------------------------
Here's an example stacktrace for one of the blocked threads:
{{org.apache.spark.sql.execution.datasources.jdbc.connection.ConnectionProviderBase.create(ConnectionProvider.scala:92)}}
{{org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$createConnectionFactory$1(JdbcUtils.scala:63)}}
{{org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$$Lambda$6294/1994845663.apply(Unknown
Source)}}
{{org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:56)}}
{{org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation$.getSchema(JDBCRelation.scala:226)}}
{{org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:35)}}
{{org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:390)}}
{{org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:444)}}
{{org.apache.spark.sql.DataFrameReader.$anonfun$load$3(DataFrameReader.scala:400)}}
{{org.apache.spark.sql.DataFrameReader$$Lambda$6224/1118373872.apply(Unknown
Source)}}
{{scala.Option.getOrElse(Option.scala:189)}}
{{org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:400)}}
{{org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:273)}}
{{<application specific stack frames removed>}}
{{scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)}}
{{scala.concurrent.Future$.$anonfun$apply$1(Future.scala:659)}}
{{scala.concurrent.Future$$$Lambda$442/341778327.apply(Unknown Source)}}
{{scala.util.Success.$anonfun$map$1(Try.scala:255)}}
{{scala.util.Success.map(Try.scala:213)}}
{{scala.concurrent.Future.$anonfun$map$1(Future.scala:292)}}
{{scala.concurrent.Future$$Lambda$443/424848797.apply(Unknown Source)}}
{{scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)}}
{{scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)}}
{{scala.concurrent.impl.Promise$$Lambda$444/1710905079.apply(Unknown Source)}}
{{scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)}}
{{java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)}}
{{java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)}}
{{java.lang.Thread.run(Thread.java:748)}}
The stacktrace from the thread that is holding the lock looks like so:
{{java.net.SocketInputStream.socketRead0(Native Method)}}
{{java.net.SocketInputStream.socketRead(SocketInputStream.java:116)}}
{{java.net.SocketInputStream.read(SocketInputStream.java:171)}}
{{java.net.SocketInputStream.read(SocketInputStream.java:141)}}
{{com.microsoft.sqlserver.jdbc.TDSChannel$ProxyInputStream.readInternal(IOBuffer.java:1019)}}
{{com.microsoft.sqlserver.jdbc.TDSChannel$ProxyInputStream.read(IOBuffer.java:1009)}}
{{sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:476)}}
{{sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:470)}}
{{sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(SSLSocketInputRecord.java:70)}}
{{sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1364)}}
{{sun.security.ssl.SSLSocketImpl.access$300(SSLSocketImpl.java:73)}}
{{sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:973)}}
{{com.microsoft.sqlserver.jdbc.TDSChannel.read(IOBuffer.java:2058)}}
{{com.microsoft.sqlserver.jdbc.TDSReader.readPacket(IOBuffer.java:6617) =>
holding Monitor(com.microsoft.sqlserver.jdbc.TDSReader@1035497922})}}
{{com.microsoft.sqlserver.jdbc.TDSCommand.startResponse(IOBuffer.java:7805)}}
{{com.microsoft.sqlserver.jdbc.TDSCommand.startResponse(IOBuffer.java:7768)}}
{{com.microsoft.sqlserver.jdbc.SQLServerConnection.sendLogon(SQLServerConnection.java:5332)}}
{{com.microsoft.sqlserver.jdbc.SQLServerConnection.logon(SQLServerConnection.java:4066)}}
{{com.microsoft.sqlserver.jdbc.SQLServerConnection.access$000(SQLServerConnection.java:85)}}
{{com.microsoft.sqlserver.jdbc.SQLServerConnection$LogonCommand.doExecute(SQLServerConnection.java:4004)}}
{{com.microsoft.sqlserver.jdbc.TDSCommand.execute(IOBuffer.java:7418)}}
{{com.microsoft.sqlserver.jdbc.SQLServerConnection.executeCommand(SQLServerConnection.java:3272)
=> holding Monitor(java.lang.Object@564746804})}}
{{com.microsoft.sqlserver.jdbc.SQLServerConnection.connectHelper(SQLServerConnection.java:2768)}}
{{com.microsoft.sqlserver.jdbc.SQLServerConnection.login(SQLServerConnection.java:2418)}}
{{com.microsoft.sqlserver.jdbc.SQLServerConnection.connectInternal(SQLServerConnection.java:2265)}}
{{com.microsoft.sqlserver.jdbc.SQLServerConnection.connect(SQLServerConnection.java:1291)}}
{{com.microsoft.sqlserver.jdbc.SQLServerDriver.connect(SQLServerDriver.java:881)}}
{{org.apache.spark.sql.execution.datasources.jdbc.connection.BasicConnectionProvider.getConnection(BasicConnectionProvider.scala:49)}}
{{org.apache.spark.sql.execution.datasources.jdbc.connection.ConnectionProviderBase.create(ConnectionProvider.scala:94)
=> holding
Monitor(org.apache.spark.security.SecurityConfigurationLock$@404826344})}}
{{org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$createConnectionFactory$1(JdbcUtils.scala:63)}}
{{org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$$Lambda$6294/1994845663.apply(Unknown
Source)}}
{{org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:56)}}
{{org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation$.getSchema(JDBCRelation.scala:226)}}
{{org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:35)}}
{{org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:390)}}
{{org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:444)}}
{{org.apache.spark.sql.DataFrameReader.$anonfun$load$3(DataFrameReader.scala:400)}}
{{org.apache.spark.sql.DataFrameReader$$Lambda$6224/1118373872.apply(Unknown
Source)}}
{{scala.Option.getOrElse(Option.scala:189)}}
{{org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:400)}}
{{org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:273)}}
{{<application specific stack frames removed>}}
{{scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)}}
{{scala.concurrent.Future$.$anonfun$apply$1(Future.scala:659)}}
{{scala.concurrent.Future$$$Lambda$442/341778327.apply(Unknown Source)}}
{{scala.util.Success.$anonfun$map$1(Try.scala:255)}}
{{scala.util.Success.map(Try.scala:213)}}
{{scala.concurrent.Future.$anonfun$map$1(Future.scala:292)}}
{{scala.concurrent.Future$$Lambda$443/424848797.apply(Unknown Source)}}
{{scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)}}
{{scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)}}
{{scala.concurrent.impl.Promise$$Lambda$444/1710905079.apply(Unknown Source)}}
{{scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)}}
{{java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)}}
{{java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)}}
{{java.lang.Thread.run(Thread.java:748)}}
> SIGNIFICANT bottleneck introduced by fix for SPARK-32001
> --------------------------------------------------------
>
> Key: SPARK-37391
> URL: https://issues.apache.org/jira/browse/SPARK-37391
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 3.1.0, 3.1.1, 3.1.2, 3.2.0
> Environment: N/A
> Reporter: Danny Guinther
> Priority: Major
> Attachments: so-much-blocking.jpg, spark-regression-dashes.jpg
>
>
> The fix for https://issues.apache.org/jira/browse/SPARK-32001 (
> [https://github.com/apache/spark/pull/29024/files#diff-345beef18081272d77d91eeca2d9b5534ff6e642245352f40f4e9c9b8922b085R58]
> ) does not seem to have consider the reality that some apps may rely on
> being able to establish many JDBC connections simultaneously for performance
> reasons.
> The fix forces concurrency to 1 when establishing database connections and
> that strikes me as a *significant* user impacting change and a *significant*
> bottleneck.
> Can anyone propose a workaround for this? I have an app that makes
> connections to thousands of databases and I can't upgrade to any version
> >3.1.x because of this significant bottleneck.
>
> Thanks in advance for your help!
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
