Yes that is the log for one of the consumers, but it is same for the other consumers too that we have running for this queue. We have 4 consumers running with 300 streams each to match the partitions on the brokers.
And yes the backlog is growing and offsets not moving. Following is the sampling that we took at 48 intervals and the offsets are stuck. +----+--------+------------+ | count | partition name | offset | +----+--------+------------+ | 48 | ,1-0 | 1898883492 | | 48 | ,1-1 | 1915507531 | | 48 | ,1-10 | 1918332917 | | 48 | ,1-100 | 236006405 | | 48 | ,1-101 | 236582777 | | 48 | ,1-102 | 235905931 | | 48 | ,1-103 | 236231929 | | 48 | ,1-104 | 237221764 | | 48 | ,1-105 | 238297312 | | 47 | ,1-106 | 235628377 | | 48 | ,1-107 | 234600170 | | 48 | ,1-108 | 238394669 | | 45 | ,1-109 | 239396416 | | 48 | ,1-11 | 1896022577 | | 48 | ,1-110 | 237924875 | | 48 | ,1-111 | 235374233 | | 48 | ,1-112 | 213809615 | | 48 | ,1-113 | 237226354 | | 48 | ,1-114 | 237396165 | | 48 | ,1-115 | 236191104 | | 48 | ,1-116 | 238128466 | | 48 | ,1-117 | 237335134 | | 48 | ,1-118 | 237350820 | | 48 | ,1-119 | 236527060 | | 18 | ,1-12 | 1917295864 | | 48 | ,1-120 | 235306049 | | 48 | ,1-121 | 235316391 | | 48 | ,1-122 | 235329292 | | 48 | ,1-123 | 236826301 | | 46 | ,1-124 | 236710764 | | 48 | ,1-125 | 236215243 | | 48 | ,1-126 | 234641764 | | 48 | ,1-127 | 238293647 | | 48 | ,1-128 | 236609422 | | 47 | ,1-129 | 236212259 | | 47 | ,1-13 | 1919322217 | | 48 | ,1-130 | 234732966 | | 46 | ,1-131 | 235335704 | | 48 | ,1-132 | 235608449 | | 48 | ,1-133 | 235595652 | | 48 | ,1-134 | 235550350 | | 48 | ,1-135 | 238191384 | | 48 | ,1-136 | 236527843 | | 48 | ,1-137 | 235194210 | | 48 | ,1-138 | 238141134 | | 48 | ,1-139 | 237072855 | | 47 | ,1-14 | 1921374677 | | 48 | ,1-140 | 219005350 | | 48 | ,1-141 | 236705945 | | 48 | ,1-142 | 235678986 | | 48 | ,1-143 | 217891350 | | 47 | ,1-144 | 238338081 | | 48 | ,1-145 | 237575489 | | 48 | ,1-146 | 236203967 | | 48 | ,1-147 | 237968410 | | 48 | ,1-148 | 233984418 | | 48 | ,1-149 | 238502295 | | 48 | ,1-15 | 1903106145 | | 48 | ,1-150 | 13499853 | | 48 | ,1-151 | 13666233 | | 48 | ,1-152 | 13670468 | | 48 | ,1-153 | 13340356 | | 44 | ,1-154 | 13406522 | | 48 | ,1-155 | 13226218 | | 48 | ,1-156 | 13779615 | | 48 | ,1-157 | 13516640 | | 48 | ,1-158 | 13523236 | | 48 | ,1-159 | 13705191 | | 48 | ,1-16 | 1919070142 | | 47 | ,1-160 | 13397958 | | 48 | ,1-161 | 13504273 | | 48 | ,1-162 | 13475059 | | 47 | ,1-163 | 13452042 | | 46 | ,1-164 | 13857996 | | 48 | ,1-165 | 13613007 | | 48 | ,1-166 | 13577619 | | 48 | ,1-167 | 13369409 | | 47 | ,1-168 | 13459302 | | 48 | ,1-169 | 14036491 | | 48 | ,1-17 | 1921745019 | | 48 | ,1-170 | 13896516 | | 48 | ,1-171 | 13693589 | | 48 | ,1-172 | 13600061 | | 48 | ,1-173 | 13283785 | | 48 | ,1-174 | 13871786 | | 48 | ,1-175 | 13330547 | | 48 | ,1-176 | 13843534 | | 48 | ,1-177 | 14018109 | | 48 | ,1-178 | 13818216 | | 48 | ,1-179 | 13401970 | | 46 | ,1-18 | 1919449120 | | 48 | ,1-180 | 13464534 | | 48 | ,1-181 | 13508706 | | 48 | ,1-182 | 13510167 | | 48 | ,1-183 | 13646266 | | 48 | ,1-184 | 13529613 | | 48 | ,1-185 | 13547768 | | 48 | ,1-186 | 13429748 | | 48 | ,1-187 | 13514664 | | 48 | ,1-188 | 13891603 | | 48 | ,1-189 | 13637978 | | 48 | ,1-19 | 1920930653 | | 48 | ,1-190 | 13645878 | | 48 | ,1-191 | 13839054 | | 48 | ,1-192 | 13616829 | | 48 | ,1-193 | 13963633 | | 48 | ,1-194 | 13815109 | | 48 | ,1-195 | 13540978 | | 48 | ,1-196 | 13424007 | | 48 | ,1-197 | 13601153 | | 48 | ,1-198 | 13743523 | | 48 | ,1-199 | 13646540 | | 48 | ,1-2 | 1922014731 | | 47 | ,1-20 | 1917994809 | | 48 | ,1-200 | 13563564 | | 48 | ,1-201 | 13830668 | | 48 | ,1-202 | 13807384 | | 48 | ,1-203 | 13600923 | | 48 | ,1-204 | 13710967 | | 48 | ,1-205 | 13474445 | | 48 | ,1-206 | 13774855 | | 48 | ,1-207 | 13228583 | | 48 | ,1-208 | 13462556 | | 48 | ,1-209 | 13434872 | | 48 | ,1-21 | 1921349977 | | 47 | ,1-210 | 13885902 | | 48 | ,1-211 | 13602042 | | 47 | ,1-212 | 13410148 | | 47 | ,1-213 | 13624848 | | 48 | ,1-214 | 13575902 | | 48 | ,1-215 | 13662290 | | 48 | ,1-216 | 14028721 | | 48 | ,1-217 | 13779944 | | 48 | ,1-218 | 13708732 | | 48 | ,1-219 | 13301453 | | 48 | ,1-22 | 1916682254 | | 48 | ,1-220 | 13573933 | | 48 | ,1-221 | 13905591 | | 48 | ,1-222 | 13377735 | | 48 | ,1-223 | 13459744 | | 48 | ,1-224 | 13193531 | | 48 | ,1-225 | 13947733 | | 48 | ,1-226 | 13308468 | | 48 | ,1-227 | 13863137 | | 48 | ,1-228 | 13641330 | | 48 | ,1-229 | 13461964 | | 47 | ,1-23 | 1924089008 | | 48 | ,1-230 | 13558746 | | 48 | ,1-231 | 13518064 | | 48 | ,1-232 | 13521781 | | 48 | ,1-233 | 13620913 | | 48 | ,1-234 | 13343532 | | 47 | ,1-235 | 13163686 | | 48 | ,1-236 | 13419783 | | 48 | ,1-237 | 13791752 | | 48 | ,1-238 | 13303617 | | 48 | ,1-239 | 13606714 | | 48 | ,1-24 | 1919970966 | | 48 | ,1-240 | 14122025 | | 48 | ,1-241 | 13154760 | | 48 | ,1-242 | 13563069 | | 48 | ,1-243 | 13477994 | | 48 | ,1-244 | 13495815 | | 48 | ,1-245 | 13874971 | | 46 | ,1-246 | 13377323 | | 48 | ,1-247 | 13324760 | | 48 | ,1-248 | 13345922 | | 48 | ,1-249 | 13353271 | | 48 | ,1-25 | 1916662117 | | 48 | ,1-250 | 13513409 | | 48 | ,1-251 | 13905710 | | 48 | ,1-252 | 13555490 | | 48 | ,1-253 | 13696026 | | 48 | ,1-254 | 13177106 | | 48 | ,1-255 | 13325701 | | 48 | ,1-256 | 13415022 | | 48 | ,1-257 | 13725258 | | 48 | ,1-258 | 13835500 | | 47 | ,1-259 | 13851656 | | 48 | ,1-26 | 1919766628 | | 48 | ,1-260 | 13539873 | | 48 | ,1-261 | 13696018 | | 48 | ,1-262 | 13443699 | | 48 | ,1-263 | 13621764 | | 48 | ,1-264 | 13408958 | | 48 | ,1-265 | 13361606 | | 48 | ,1-266 | 13473269 | | 48 | ,1-267 | 13699374 | | 48 | ,1-268 | 13851366 | | 48 | ,1-269 | 13929745 | | 48 | ,1-27 | 1922590212 | | 48 | ,1-270 | 13663613 | | 48 | ,1-271 | 13369617 | | 48 | ,1-272 | 13804602 | | 47 | ,1-273 | 13369766 | | 48 | ,1-274 | 13950843 | | 48 | ,1-275 | 13433980 | | 48 | ,1-276 | 13712829 | | 48 | ,1-277 | 13745697 | | 48 | ,1-278 | 13564556 | | 48 | ,1-279 | 13528017 | | 48 | ,1-28 | 1923726826 | | 48 | ,1-280 | 13346676 | | 48 | ,1-281 | 13439931 | | 48 | ,1-282 | 13505038 | | 48 | ,1-283 | 13603863 | | 48 | ,1-284 | 13439779 | | 47 | ,1-285 | 13363454 | | 48 | ,1-286 | 13535972 | | 48 | ,1-287 | 13348086 | | 48 | ,1-288 | 13562673 | | 48 | ,1-289 | 13671791 | | 48 | ,1-29 | 1923490085 | | 48 | ,1-290 | 13643033 | | 47 | ,1-291 | 13527694 | | 48 | ,1-292 | 13593083 | | 48 | ,1-293 | 14019680 | | 48 | ,1-294 | 13641352 | | 48 | ,1-295 | 13601707 | | 48 | ,1-296 | 13611321 | | 48 | ,1-297 | 13679345 | | 48 | ,1-298 | 13724448 | | 48 | ,1-299 | 14056772 | | 48 | ,1-3 | 1902071981 | | 48 | ,1-30 | 1898316395 | | 48 | ,1-31 | 1916914106 | | 48 | ,1-32 | 1903454914 | | 48 | ,1-33 | 1922191092 | | 48 | ,1-34 | 1896666349 | | 48 | ,1-35 | 1918309703 | | 48 | ,1-36 | 1926191484 | | 48 | ,1-37 | 1918677957 | | 48 | ,1-38 | 1919942719 | | 48 | ,1-39 | 1917354696 | | 48 | ,1-4 | 1903677351 | | 48 | ,1-40 | 1903464952 | | 48 | ,1-41 | 1921505084 | | 48 | ,1-42 | 1922341150 | | 48 | ,1-43 | 1921512403 | | 48 | ,1-44 | 1920228115 | | 48 | ,1-45 | 1922545516 | | 48 | ,1-46 | 1920130042 | | 48 | ,1-47 | 1919020489 | | 48 | ,1-48 | 1920486605 | | 48 | ,1-49 | 1920764659 | | 48 | ,1-5 | 1897315538 | | 48 | ,1-50 | 1922238952 | | 48 | ,1-51 | 1920585832 | | 48 | ,1-52 | 1917846287 | | 48 | ,1-53 | 1923259957 | | 48 | ,1-54 | 1918026253 | | 48 | ,1-55 | 1924061736 | | 48 | ,1-56 | 1920195695 | | 48 | ,1-57 | 1922090006 | | 48 | ,1-58 | 1916859221 | | 48 | ,1-59 | 1903359842 | | 48 | ,1-6 | 1916854092 | | 48 | ,1-60 | 1899539195 | | 48 | ,1-61 | 1921778909 | | 47 | ,1-62 | 1921255890 | | 48 | ,1-63 | 1920129152 | | 48 | ,1-64 | 1921408493 | | 48 | ,1-65 | 1915728691 | | 48 | ,1-66 | 1921231584 | | 48 | ,1-67 | 1921420027 | | 48 | ,1-68 | 1920888770 | | 48 | ,1-69 | 1900286679 | | 48 | ,1-7 | 1923654032 | | 48 | ,1-70 | 1918407344 | | 48 | ,1-71 | 1922290360 | | 48 | ,1-72 | 1920232052 | | 48 | ,1-73 | 1919931750 | | 48 | ,1-74 | 1920917825 | | 48 | ,1-75 | 237775530 | | 48 | ,1-76 | 237858953 | | 48 | ,1-77 | 236398382 | | 48 | ,1-78 | 215678878 | | 48 | ,1-79 | 234905264 | | 48 | ,1-8 | 1920937857 | | 47 | ,1-80 | 237832055 | | 48 | ,1-81 | 237282551 | | 48 | ,1-82 | 237380238 | | 48 | ,1-83 | 237495932 | | 48 | ,1-84 | 236855214 | | 48 | ,1-85 | 238333628 | | 48 | ,1-86 | 237086930 | | 47 | ,1-87 | 236138666 | | 48 | ,1-88 | 236603937 | | 48 | ,1-89 | 233845116 | | 48 | ,1-9 | 1921222399 | | 48 | ,1-90 | 237669932 | | 48 | ,1-91 | 237745665 | | 48 | ,1-92 | 236063363 | | 48 | ,1-93 | 236397979 | | 48 | ,1-94 | 235888624 | | 48 | ,1-95 | 235458093 | | 48 | ,1-96 | 239433538 | | 47 | ,1-97 | 237923224 | | 48 | ,1-98 | 235832581 | | 48 | ,1-99 | 235122322 | | 47 | ,2-0 | 1921555162 | | 48 | ,2-1 | 1921553567 | | 48 | ,2-10 | 1919967620 | | 48 | ,2-100 | 217746066 | | 48 | ,2-101 | 235007585 | | 48 | ,2-102 | 235063667 | | 48 | ,2-103 | 235070522 | | 48 | ,2-104 | 214975553 | | 48 | ,2-105 | 236466519 | | 48 | ,2-106 | 236734262 | | 48 | ,2-107 | 212217628 | | 47 | ,2-108 | 235672246 | | 48 | ,2-109 | 238174360 | | 48 | ,2-11 | 1921142965 | | 48 | ,2-110 | 235803792 | | 48 | ,2-111 | 214183430 | | 48 | ,2-114 | 234925019 | | 48 | ,2-115 | 218549783 | | 48 | ,2-116 | 235448904 | | 48 | ,2-117 | 237535782 | | 48 | ,2-118 | 235683042 | | 48 | ,2-119 | 217255832 | | 48 | ,2-12 | 1922407896 | | 48 | ,2-120 | 234944710 | | 48 | ,2-121 | 235470275 | | 48 | ,2-122 | 235164724 | | 48 | ,2-123 | 215208119 | | 47 | ,2-124 | 235808112 | | 47 | ,2-125 | 235131936 | | 48 | ,2-126 | 235438940 | | 47 | ,2-127 | 235246249 | | 48 | ,2-128 | 234776897 | | 48 | ,2-13 | 1922345257 | | 48 | ,2-130 | 222129489 | | 48 | ,2-131 | 238856065 | | 48 | ,2-133 | 217357068 | | 48 | ,2-134 | 235889229 | | 48 | ,2-135 | 235486102 | | 48 | ,2-136 | 235819003 | | 48 | ,2-137 | 235108303 | | 48 | ,2-138 | 237208068 | | 48 | ,2-139 | 212818320 | | 48 | ,2-14 | 1924091721 | | 48 | ,2-140 | 235130148 | | 48 | ,2-141 | 218884515 | | 48 | ,2-142 | 234460354 | | 48 | ,2-144 | 212856457 | | 48 | ,2-145 | 234995619 | | 48 | ,2-146 | 233688413 | | 48 | ,2-147 | 215035632 | | 48 | ,2-148 | 236883390 | | 48 | ,2-149 | 237710610 | | 48 | ,2-15 | 1908247656 | | 48 | ,2-150 | 13569356 | | 48 | ,2-151 | 13674690 | | 48 | ,2-152 | 13641372 | | 48 | ,2-153 | 13863526 | | 48 | ,2-154 | 13427902 | | 48 | ,2-155 | 13295541 | | 48 | ,2-156 | 13862579 | | 48 | ,2-157 | 13354624 | | 48 | ,2-158 | 13648101 | | 48 | ,2-159 | 13354610 | | 48 | ,2-16 | 1923235422 | | 48 | ,2-160 | 13660657 | | 48 | ,2-161 | 13787051 | | 48 | ,2-162 | 13299796 | | 48 | ,2-163 | 13549754 | | 48 | ,2-166 | 13368468 | | 48 | ,2-167 | 13556569 | | 48 | ,2-168 | 13526648 | | 47 | ,2-169 | 13311148 | | 48 | ,2-17 | 1924119782 | | 48 | ,2-170 | 13485162 | | 48 | ,2-171 | 13598806 | | 47 | ,2-172 | 13487402 | | 47 | ,2-173 | 13392131 | | 48 | ,2-174 | 13825566 | | 48 | ,2-175 | 13597102 | | 48 | ,2-177 | 13352130 | | 48 | ,2-178 | 13373764 | | 48 | ,2-179 | 13352943 | | 48 | ,2-18 | 1899863036 | | 48 | ,2-180 | 13675261 | | 48 | ,2-181 | 13336614 | | 48 | ,2-182 | 13671981 | | 48 | ,2-183 | 13380311 | | 48 | ,2-184 | 13423949 | | 48 | ,2-185 | 13291246 | | 48 | ,2-186 | 13477032 | | 46 | ,2-187 | 13899639 | | 48 | ,2-188 | 13676399 | | 48 | ,2-189 | 13735126 | | 48 | ,2-190 | 13727021 | | 48 | ,2-191 | 13746144 | | 48 | ,2-192 | 13382921 | | 48 | ,2-193 | 13608259 | | 48 | ,2-194 | 13404529 | | 47 | ,2-195 | 13592500 | | 48 | ,2-196 | 13404613 | | 48 | ,2-197 | 13463431 | | 48 | ,2-198 | 13642621 | | 48 | ,2-2 | 1922848102 | | 48 | ,2-20 | 1925885629 | | 48 | ,2-200 | 13635443 | | 48 | ,2-201 | 13872944 | | 48 | ,2-202 | 13329970 | | 48 | ,2-203 | 13550987 | | 48 | ,2-204 | 14011123 | | 48 | ,2-205 | 13584327 | | 48 | ,2-206 | 13901794 | | 48 | ,2-207 | 13660148 | | 48 | ,2-209 | 13488331 | | 48 | ,2-21 | 1924592862 | | 48 | ,2-210 | 13388688 | | 48 | ,2-211 | 13486135 | | 48 | ,2-212 | 13630579 | | 48 | ,2-213 | 13651976 | | 48 | ,2-214 | 13496385 | | 48 | ,2-215 | 13439274 | | 47 | ,2-216 | 13392169 | | 48 | ,2-217 | 13813623 | | 48 | ,2-218 | 13672190 | | 48 | ,2-219 | 13494167 | | 48 | ,2-220 | 13682289 | | 48 | ,2-221 | 13506001 | | 48 | ,2-222 | 13695210 | | 48 | ,2-223 | 13294876 | | 48 | ,2-224 | 13314729 | | 47 | ,2-225 | 13630389 | | 47 | ,2-226 | 13202324 | | 48 | ,2-227 | 13681701 | | 48 | ,2-229 | 13712679 | | 48 | ,2-23 | 1922212647 | | 48 | ,2-230 | 13576035 | | 48 | ,2-231 | 13430479 | | 47 | ,2-232 | 14161316 | | 48 | ,2-233 | 13315780 | | 48 | ,2-234 | 13726915 | | 48 | ,2-235 | 13908401 | | 48 | ,2-236 | 13408963 | | 48 | ,2-237 | 13346480 | | 48 | ,2-238 | 13994929 | | 48 | ,2-239 | 13398339 | | 48 | ,2-24 | 1902884280 | | 48 | ,2-240 | 13347625 | | 48 | ,2-241 | 13529775 | | 48 | ,2-242 | 13574503 | | 48 | ,2-243 | 13259841 | | 48 | ,2-244 | 13316580 | | 48 | ,2-245 | 13604713 | | 48 | ,2-247 | 13703693 | | 48 | ,2-248 | 13469603 | | 48 | ,2-25 | 1902134443 | | 48 | ,2-250 | 13741928 | | 48 | ,2-251 | 13250037 | | 48 | ,2-252 | 13553447 | | 48 | ,2-253 | 13645615 | | 48 | ,2-254 | 13474829 | | 48 | ,2-255 | 13607477 | | 48 | ,2-258 | 13822931 | | 48 | ,2-259 | 13996837 | | 48 | ,2-26 | 1921689871 | | 48 | ,2-260 | 13428391 | | 47 | ,2-261 | 13324477 | | 48 | ,2-262 | 13993343 | | 48 | ,2-263 | 13628551 | | 48 | ,2-264 | 13674508 | | 48 | ,2-265 | 13539591 | | 48 | ,2-266 | 13968220 | | 48 | ,2-267 | 13719377 | | 48 | ,2-268 | 13574679 | | 48 | ,2-269 | 13392025 | | 48 | ,2-27 | 1922207745 | | 48 | ,2-270 | 13637786 | | 48 | ,2-273 | 13539174 | | 48 | ,2-274 | 13519610 | | 48 | ,2-275 | 13315322 | | 48 | ,2-276 | 13538470 | | 48 | ,2-278 | 14174820 | | 48 | ,2-279 | 13655976 | | 48 | ,2-28 | 1923576769 | | 48 | ,2-280 | 13423049 | | 48 | ,2-281 | 13750488 | | 48 | ,2-282 | 13246028 | | 48 | ,2-283 | 13643324 | | 48 | ,2-284 | 13318295 | | 48 | ,2-285 | 13398653 | | 48 | ,2-287 | 13411118 | | 48 | ,2-288 | 13428818 | | 48 | ,2-29 | 1922504929 | | 48 | ,2-290 | 13541099 | | 48 | ,2-291 | 13793119 | | 48 | ,2-292 | 13865221 | | 48 | ,2-293 | 13850260 | | 48 | ,2-295 | 13409159 | | 48 | ,2-297 | 13547241 | | 48 | ,2-298 | 13679457 | | 48 | ,2-299 | 13790240 | | 48 | ,2-30 | 1928036163 | | 48 | ,2-31 | 1922824392 | | 48 | ,2-32 | 1920541886 | | 47 | ,2-33 | 1922352725 | | 48 | ,2-34 | 1924853181 | | 48 | ,2-35 | 1919326535 | | 48 | ,2-36 | 1901090328 | | 48 | ,2-37 | 1926055212 | | 48 | ,2-39 | 1903555940 | | 48 | ,2-4 | 1903595717 | | 48 | ,2-40 | 1921923931 | | 48 | ,2-41 | 1923037373 | | 48 | ,2-42 | 1921936149 | | 48 | ,2-44 | 1927025213 | | 48 | ,2-45 | 1924006013 | | 48 | ,2-46 | 1928123113 | | 48 | ,2-48 | 1924655801 | | 48 | ,2-49 | 1923917536 | | 48 | ,2-5 | 1924577343 | | 48 | ,2-50 | 1898465648 | | 48 | ,2-51 | 1919080125 | | 48 | ,2-52 | 1920232969 | | 48 | ,2-53 | 1919657055 | | 48 | ,2-54 | 1918150833 | | 48 | ,2-55 | 1922479339 | | 48 | ,2-56 | 1919158263 | | 48 | ,2-57 | 1916093150 | | 48 | ,2-58 | 1920541908 | | 48 | ,2-59 | 1915805833 | | 48 | ,2-6 | 1922739509 | | 48 | ,2-60 | 1922217631 | | 48 | ,2-61 | 1916732709 | | 48 | ,2-62 | 1902693010 | | 48 | ,2-64 | 1919963759 | | 48 | ,2-66 | 1917114515 | | 48 | ,2-67 | 1918194912 | | 48 | ,2-68 | 1915494468 | | 48 | ,2-69 | 1917558232 | | 48 | ,2-7 | 1899946683 | | 48 | ,2-70 | 1918809285 | | 48 | ,2-72 | 1915488518 | | 48 | ,2-73 | 1892995372 | | 48 | ,2-74 | 1915184772 | | 48 | ,2-75 | 234420058 | | 48 | ,2-79 | 236680061 | | 48 | ,2-8 | 1920441903 | | 48 | ,2-80 | 236019844 | | 48 | ,2-81 | 233862801 | | 48 | ,2-84 | 213241057 | | 48 | ,2-85 | 237818562 | | 48 | ,2-86 | 235954485 | | 48 | ,2-87 | 235351814 | | 48 | ,2-88 | 238929337 | | 48 | ,2-89 | 236270541 | | 48 | ,2-9 | 1923139464 | | 48 | ,2-90 | 235214482 | | 48 | ,2-91 | 212552706 | | 48 | ,2-92 | 236646051 | | 48 | ,2-93 | 235559538 | | 48 | ,2-94 | 236574974 | | 48 | ,2-95 | 235714462 | | 48 | ,2-97 | 235843502 | | 48 | ,3-0 | 1919866863 | | 48 | ,3-1 | 1919151143 | | 48 | ,3-10 | 1917860156 | | 48 | ,3-100 | 212044048 | | 48 | ,3-101 | 213435917 | | 48 | ,3-102 | 237260918 | | 48 | ,3-103 | 236530056 | | 48 | ,3-104 | 233921061 | | 48 | ,3-105 | 232718467 | | 48 | ,3-106 | 234353103 | | 48 | ,3-107 | 235497250 | | 48 | ,3-108 | 216824422 | | 48 | ,3-109 | 234645803 | | 48 | ,3-11 | 1922596633 | | 48 | ,3-110 | 232714885 | | 48 | ,3-111 | 235675809 | | 48 | ,3-112 | 237255559 | | 46 | ,3-113 | 235711462 | | 48 | ,3-114 | 236345464 | | 48 | ,3-115 | 234582254 | | 48 | ,3-116 | 215143605 | | 48 | ,3-117 | 235287145 | | 48 | ,3-118 | 235739286 | | 48 | ,3-119 | 233561004 | | 48 | ,3-12 | 1922854599 | | 47 | ,3-120 | 235426103 | | 48 | ,3-121 | 212533829 | | 48 | ,3-122 | 237355342 | | 48 | ,3-123 | 237428339 | | 48 | ,3-124 | 233165519 | | 48 | ,3-125 | 236024973 | | 48 | ,3-127 | 238200053 | | 48 | ,3-128 | 212348165 | | 48 | ,3-129 | 235790282 | | 48 | ,3-13 | 1922699849 | | 47 | ,3-130 | 236594240 | | 48 | ,3-131 | 235380856 | | 48 | ,3-132 | 236652638 | | 48 | ,3-133 | 234499777 | | 48 | ,3-134 | 235241824 | | 48 | ,3-135 | 234276689 | | 48 | ,3-136 | 235104552 | | 48 | ,3-137 | 236340322 | | 48 | ,3-138 | 235260580 | | 48 | ,3-139 | 234485471 | | 48 | ,3-140 | 236184219 | | 48 | ,3-141 | 234051708 | | 48 | ,3-142 | 211077915 | | 48 | ,3-143 | 234568427 | | 48 | ,3-144 | 234171657 | | 48 | ,3-145 | 237757772 | | 48 | ,3-146 | 237302491 | | 48 | ,3-147 | 237535991 | | 48 | ,3-148 | 235015318 | | 48 | ,3-149 | 236862998 | | 48 | ,3-15 | 1903670266 | | 48 | ,3-150 | 13552414 | | 48 | ,3-151 | 13572335 | | 48 | ,3-152 | 13439661 | | 48 | ,3-153 | 13495416 | | 48 | ,3-154 | 13316469 | | 48 | ,3-155 | 13269479 | | 48 | ,3-156 | 13726665 | | 48 | ,3-157 | 13365508 | | 48 | ,3-158 | 13589573 | | 48 | ,3-159 | 13427873 | | 48 | ,3-16 | 1925859601 | | 48 | ,3-160 | 13476481 | | 48 | ,3-161 | 13867102 | | 48 | ,3-163 | 13627922 | | 48 | ,3-164 | 13286911 | | 48 | ,3-165 | 13783610 | | 48 | ,3-166 | 13284148 | | 48 | ,3-167 | 13801206 | | 48 | ,3-168 | 13862410 | | 48 | ,3-169 | 13477681 | | 48 | ,3-17 | 1918277223 | | 48 | ,3-170 | 13593463 | | 48 | ,3-171 | 13970171 | | 48 | ,3-172 | 13477958 | | 48 | ,3-173 | 13279296 | | 48 | ,3-174 | 13702769 | | 48 | ,3-175 | 13660253 | | 48 | ,3-176 | 14045267 | | 48 | ,3-177 | 13345262 | | 48 | ,3-178 | 13404696 | | 48 | ,3-179 | 13842418 | | 48 | ,3-18 | 1922854240 | | 48 | ,3-180 | 13435005 | | 48 | ,3-181 | 13518162 | | 48 | ,3-182 | 13708633 | | 48 | ,3-183 | 13345713 | | 48 | ,3-184 | 13612129 | | 48 | ,3-185 | 13509103 | | 48 | ,3-186 | 13747421 | | 48 | ,3-187 | 13336087 | | 48 | ,3-188 | 13767406 | | 48 | ,3-189 | 14001069 | | 48 | ,3-19 | 1919114696 | | 48 | ,3-190 | 13329497 | | 48 | ,3-192 | 13595162 | | 48 | ,3-193 | 13696732 | | 48 | ,3-194 | 13462080 | | 48 | ,3-195 | 13434859 | | 48 | ,3-196 | 13400375 | | 48 | ,3-197 | 13600322 | | 48 | ,3-198 | 13733118 | | 48 | ,3-199 | 13421166 | | 48 | ,3-2 | 1920171534 | | 48 | ,3-20 | 1924856688 | | 48 | ,3-200 | 13919037 | | 48 | ,3-201 | 13198393 | | 48 | ,3-202 | 13518364 | | 48 | ,3-203 | 13940036 | | 48 | ,3-204 | 13245129 | | 48 | ,3-205 | 13235959 | | 48 | ,3-206 | 13568888 | | 48 | ,3-207 | 13636441 | | 48 | ,3-208 | 13578365 | | 48 | ,3-209 | 13397212 | | 48 | ,3-21 | 1917361051 | | 48 | ,3-210 | 13898491 | | 48 | ,3-211 | 13682146 | | 47 | ,3-212 | 13712146 | | 48 | ,3-213 | 13762496 | | 48 | ,3-214 | 13126242 | | 48 | ,3-215 | 13551785 | | 48 | ,3-216 | 13279022 | | 48 | ,3-217 | 13404716 | | 48 | ,3-218 | 13438331 | | 48 | ,3-219 | 13919615 | | 48 | ,3-22 | 1920940102 | | 48 | ,3-220 | 13564777 | | 48 | ,3-221 | 13435122 | | 48 | ,3-222 | 13581029 | | 48 | ,3-223 | 13298838 | | 48 | ,3-224 | 13685588 | | 48 | ,3-225 | 13576111 | | 48 | ,3-226 | 13761397 | | 48 | ,3-227 | 13550589 | | 48 | ,3-228 | 13770800 | | 48 | ,3-229 | 13444835 | | 48 | ,3-23 | 1924343168 | | 48 | ,3-230 | 13640712 | | 48 | ,3-231 | 13358078 | | 48 | ,3-232 | 13341311 | | 48 | ,3-233 | 13390279 | | 48 | ,3-234 | 13102756 | | 48 | ,3-235 | 13349077 | | 48 | ,3-236 | 13617807 | | 48 | ,3-237 | 13624337 | | 48 | ,3-238 | 13525050 | | 48 | ,3-239 | 13730907 | | 48 | ,3-24 | 1923314823 | | 48 | ,3-240 | 13310582 | | 48 | ,3-241 | 13606322 | | 48 | ,3-242 | 13420494 | | 48 | ,3-243 | 13492297 | | 48 | ,3-244 | 13619899 | | 48 | ,3-245 | 13559757 | | 48 | ,3-246 | 13206250 | | 48 | ,3-247 | 13698073 | | 48 | ,3-248 | 13314358 | | 48 | ,3-249 | 13465063 | | 48 | ,3-25 | 1920775177 | | 48 | ,3-250 | 13488609 | | 48 | ,3-251 | 13882306 | | 48 | ,3-252 | 13249079 | | 48 | ,3-253 | 13643123 | | 48 | ,3-254 | 13559264 | | 48 | ,3-255 | 13754242 | | 48 | ,3-256 | 13682424 | | 48 | ,3-257 | 13404576 | | 48 | ,3-258 | 13448792 | | 48 | ,3-259 | 13641309 | | 48 | ,3-26 | 1922961970 | | 48 | ,3-260 | 13592281 | | 48 | ,3-261 | 13629470 | | 48 | ,3-262 | 13950389 | | 48 | ,3-263 | 13402762 | | 48 | ,3-264 | 13538192 | | 48 | ,3-265 | 13788523 | | 48 | ,3-266 | 13792254 | | 48 | ,3-267 | 13385495 | | 48 | ,3-268 | 13686644 | | 48 | ,3-269 | 13370661 | | 48 | ,3-27 | 1897442832 | | 48 | ,3-270 | 13352098 | | 48 | ,3-271 | 13720187 | | 48 | ,3-272 | 13522694 | | 48 | ,3-273 | 13342608 | | 48 | ,3-274 | 13749635 | | 48 | ,3-275 | 13186309 | | 48 | ,3-276 | 13683289 | | 48 | ,3-277 | 13192781 | | 48 | ,3-278 | 13347552 | | 48 | ,3-279 | 13604370 | | 48 | ,3-28 | 1922137215 | | 48 | ,3-280 | 13467206 | | 48 | ,3-281 | 13831904 | | 48 | ,3-282 | 13573833 | | 48 | ,3-283 | 13735218 | | 48 | ,3-284 | 13493708 | | 48 | ,3-285 | 13490341 | | 48 | ,3-286 | 13534560 | | 48 | ,3-287 | 13503301 | | 47 | ,3-288 | 13460448 | | 48 | ,3-289 | 13779303 | | 48 | ,3-29 | 1923662741 | | 31 | ,3-290 | 14080878 | | 3 | ,3-290 | 14078050 | | 5 | ,3-290 | 14071421 | | 48 | ,3-291 | 13219161 | | 48 | ,3-292 | 13379878 | | 48 | ,3-293 | 13663608 | | 48 | ,3-294 | 13337833 | | 48 | ,3-295 | 13581186 | | 48 | ,3-296 | 13347249 | | 48 | ,3-297 | 13295343 | | 48 | ,3-298 | 13745349 | | 48 | ,3-299 | 13551045 | | 48 | ,3-3 | 1924057585 | | 48 | ,3-30 | 1923831275 | | 48 | ,3-31 | 1922044191 | | 48 | ,3-32 | 1922012371 | | 48 | ,3-33 | 1917094903 | | 48 | ,3-34 | 1923219806 | | 48 | ,3-35 | 1920885071 | | 48 | ,3-36 | 1921713634 | | 48 | ,3-37 | 1907257074 | | 48 | ,3-38 | 1922725060 | | 48 | ,3-39 | 1922481938 | | 48 | ,3-4 | 1920419870 | | 48 | ,3-40 | 1924301238 | | 48 | ,3-41 | 1921891742 | | 48 | ,3-42 | 1923132065 | | 48 | ,3-43 | 1923643157 | | 48 | ,3-44 | 1917622219 | | 48 | ,3-45 | 1922182380 | | 48 | ,3-46 | 1922607064 | | 48 | ,3-47 | 1921661469 | | 48 | ,3-48 | 1921121013 | | 48 | ,3-49 | 1919925008 | | 48 | ,3-5 | 1919165712 | | 48 | ,3-50 | 1923075353 | | 48 | ,3-51 | 1921993380 | | 48 | ,3-52 | 1919726241 | | 48 | ,3-53 | 1895421836 | | 48 | ,3-54 | 1918637092 | | 48 | ,3-55 | 1919237643 | | 48 | ,3-56 | 1893677888 | | 48 | ,3-57 | 1918855049 | | 48 | ,3-58 | 1917528388 | | 48 | ,3-59 | 1919139446 | | 48 | ,3-6 | 1899861815 | | 48 | ,3-60 | 1895500532 | | 48 | ,3-61 | 1920506052 | | 48 | ,3-62 | 1916835294 | | 48 | ,3-63 | 1916310705 | | 48 | ,3-64 | 1918845138 | | 48 | ,3-65 | 1919184769 | | 48 | ,3-66 | 1915402616 | | 48 | ,3-67 | 1918927052 | | 48 | ,3-68 | 1922536381 | | 48 | ,3-69 | 1918520761 | | 48 | ,3-7 | 1899777610 | | 48 | ,3-70 | 1898374346 | | 48 | ,3-71 | 1916479446 | | 48 | ,3-72 | 1920524043 | | 48 | ,3-73 | 1902550900 | | 48 | ,3-74 | 1919570397 | | 48 | ,3-75 | 235093210 | | 48 | ,3-76 | 237441227 | | 48 | ,3-77 | 233264782 | | 48 | ,3-78 | 234434258 | | 48 | ,3-79 | 237687837 | | 48 | ,3-8 | 1910563638 | | 48 | ,3-80 | 233832198 | | 48 | ,3-81 | 237367418 | | 48 | ,3-82 | 236374522 | | 48 | ,3-83 | 235825432 | | 48 | ,3-84 | 233379724 | | 48 | ,3-85 | 211380548 | | 48 | ,3-86 | 237043268 | | 48 | ,3-87 | 236007286 | | 48 | ,3-88 | 233813190 | | 48 | ,3-89 | 236040062 | | 48 | ,3-9 | 1921863014 | | 48 | ,3-90 | 237954693 | | 48 | ,3-91 | 236756075 | | 48 | ,3-92 | 234548828 | | 48 | ,3-93 | 233403831 | | 48 | ,3-94 | 233540976 | | 48 | ,3-95 | 211704970 | | 48 | ,3-96 | 238110137 | | 48 | ,3-97 | 236124272 | | 48 | ,3-98 | 235344711 | | 48 | ,3-99 | 235066526 | +----+--------+------------+ On Thu, Aug 15, 2013 at 5:30 AM, Joel Koshy <jjkosh...@gmail.com> wrote: > So those are the logs for one consumer right? How many consumer > instances do you have? Can you run the offset checker tool at an > interval greater than the offset commit interval and verify that > offsets are not moving and that log sizes are growing? > > > https://cwiki.apache.org/confluence/display/KAFKA/FAQ#FAQ-Myconsumerseemstohavestopped%2Cwhy%3F > > Thanks, > > Joel > > On Wed, Aug 14, 2013 at 4:29 PM, Ramakrishna Mallya > <rmal...@sprinklr.com> wrote: > > Hi, > > > > We are using kafka-0.7.2 with zookeeper (3.4.5) > > > > Our cluster configuration: > > 3 brokers on 3 different machines. Each broker machine has a zookeeper > > instance running as well. > > > > We are currently facing a problem with messages not being consumed by the > > kafka high level consumers. This is a dedicated topic that gets a very > high > > number of messages. Consumer had no Full GCs. > > > > Api called def > > createMessageStreams(topicCountMap: Map[String,Int]): Map[String, > > List[KafkaStream[Array[Byte],Array[Byte]]]] > > > > We have already handled the exceptions at the consumers level and we have > > ensured and checked that our threads are not dying. > > > > All our consumer thread stack > > > > java.lang.Thread.State: WAITING (parking) > > at sun.misc.Unsafe.park(Native Method) > > - parking to wait for <0x000000065d104ee8> (a > > java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) > > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:156) > > at > > > java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) > > at > > > java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) > > at > > > java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) > > at > > > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) > > at java.lang.Thread.run(Thread.java:662) > > > > Kafka consumer auto commit thread stack > > > > "Kafka-consumer-autocommit-0" prio=10 tid=0x00007f99ad3a2000 nid=0x330b > in > > Object.wait() [0x00007f9989084000] > > java.lang.Thread.State: WAITING (on object monitor) > > at java.lang.Object.wait(Native Method) > > at java.lang.Object.wait(Object.java:485) > > at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1309) > > - locked <0x0000000786bdd6c0> (a org.apache.zookeeper.ClientCnxn$Packet) > > at org.apache.zookeeper.ZooKeeper.setData(ZooKeeper.java:1264) > > at org.I0Itec.zkclient.ZkConnection.writeData(ZkConnection.java:111) > > at org.I0Itec.zkclient.ZkClient$10.call(ZkClient.java:813) > > at org.I0Itec.zkclient.ZkClient.retryUntilConnected(ZkClient.java:675) > > at org.I0Itec.zkclient.ZkClient.writeData(ZkClient.java:809) > > at org.I0Itec.zkclient.ZkClient.writeData(ZkClient.java:777) > > at kafka.utils.ZkUtils$.updatePersistentPath(ZkUtils.scala:103) > > at > > > kafka.consumer.ZookeeperConsumerConnector$$anonfun$commitOffsets$2$$anonfun$apply$4.apply(ZookeeperConsumerConnector.scala:251) > > at > > > kafka.consumer.ZookeeperConsumerConnector$$anonfun$commitOffsets$2$$anonfun$apply$4.apply(ZookeeperConsumerConnector.scala:248) > > at scala.collection.Iterator$class.foreach(Iterator.scala:631) > > at > > > scala.collection.JavaConversions$JIteratorWrapper.foreach(JavaConversions.scala:474) > > at scala.collection.IterableLike$class.foreach(IterableLike.scala:79) > > at > > > scala.collection.JavaConversions$JCollectionWrapper.foreach(JavaConversions.scala:495) > > at > > > kafka.consumer.ZookeeperConsumerConnector$$anonfun$commitOffsets$2.apply(ZookeeperConsumerConnector.scala:248) > > at > > > kafka.consumer.ZookeeperConsumerConnector$$anonfun$commitOffsets$2.apply(ZookeeperConsumerConnector.scala:246) > > at scala.collection.Iterator$class.foreach(Iterator.scala:631) > > at kafka.utils.Pool$$anon$1.foreach(Pool.scala:53) > > at scala.collection.IterableLike$class.foreach(IterableLike.scala:79) > > at kafka.utils.Pool.foreach(Pool.scala:24) > > at > > > kafka.consumer.ZookeeperConsumerConnector.commitOffsets(ZookeeperConsumerConnector.scala:246) > > at > > > kafka.consumer.ZookeeperConsumerConnector.autoCommit(ZookeeperConsumerConnector.scala:232) > > at > > > kafka.consumer.ZookeeperConsumerConnector$$anonfun$1.apply$mcV$sp(ZookeeperConsumerConnector.scala:126) > > at kafka.utils.Utils$$anon$2.run(Utils.scala:58) > > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441) > > at > > > java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317) > > at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150) > > at > > > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98) > > at > > > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180) > > at > > > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204) > > at > > > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) > > at > > > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) > > at java.lang.Thread.run(Thread.java:662) > > > > Fetcher Runnable stack > > > > "FetchRunnable-0" prio=10 tid=0x00007f997000c000 nid=0x38b0 > > sleeping[0x00007f9948da2000] > > java.lang.Thread.State: TIMED_WAITING (sleeping) > > at java.lang.Thread.sleep(Native Method) > > at kafka.consumer.FetcherRunnable.run(FetcherRunnable.scala:99) > > > > > > > > > > We have defined 300 partitions per broker, and the fetch requests is not > > processing 857 of those partitions, they are blocked, even though data is > > being added to those partitions. > > > > We are not seeing any unusual logging taking place in either kakfa > brokers > > or the zookeeper clusters. > > > > > > Following is the detail of the fetcher runnable in trace mode. > > > > > > 2013-08-14 23:28:13 TRACE FetcherRunnable:35 - recevied response from > fetch > > request: List(FetchRequest(topic:event_queue, part:290 offset:13858793 > > maxSize:3145728), FetchRequest(topic:event_queue, part:71 > offset:1919388040 > > maxSize:3145728), FetchRequest(topic:event_queue, part:14 > offset:1924428035 > > maxSize:3145728), FetchRequest(topic:event_queue, part:287 > offset:13753665 > > maxSize:3145728), FetchRequest(topic:event_queue, part:21 > offset:1924929744 > > maxSize:3145728), FetchRequest(topic:event_queue, part:148 > offset:237226086 > > maxSize:3145728), FetchRequest(topic:event_queue, part:220 > offset:14046778 > > maxSize:3145728), FetchRequest(topic:event_queue, part:66 > offset:1917452262 > > maxSize:3145728), FetchRequest(topic:event_queue, part:248 > offset:13820576 > > maxSize:3145728), FetchRequest(topic:event_queue, part:213 > offset:14009238 > > maxSize:3145728), FetchRequest(topic:event_queue, part:176 > offset:13889675 > > maxSize:3145728), FetchRequest(topic:event_queue, part:24 > offset:1922108511 > > maxSize:3145728), FetchRequest(topic:event_queue, part:276 > offset:13860230 > > maxSize:3145728), FetchRequest(topic:event_queue, part:57 > offset:1916430767 > > maxSize:3145728), FetchRequest(topic:event_queue, part:73 > offset:1915483219 > > maxSize:3145728), FetchRequest(topic:event_queue, part:85 > offset:238172050 > > maxSize:3145728), FetchRequest(topic:event_queue, part:150 > offset:13902607 > > maxSize:3145728), FetchRequest(topic:event_queue, part:31 > offset:1923158067 > > maxSize:3145728), FetchRequest(topic:event_queue, part:252 > offset:13880253 > > maxSize:3145728), FetchRequest(topic:event_queue, part:185 > offset:13632601 > > maxSize:3145728), FetchRequest(topic:event_queue, part:241 > offset:13870809 > > maxSize:3145728), FetchRequest(topic:event_queue, part:169 > offset:13653452 > > maxSize:3145728), FetchRequest(topic:event_queue, part:157 > offset:13698490 > > maxSize:3145728), FetchRequest(topic:event_queue, part:84 > offset:235590608 > > maxSize:3145728), FetchRequest(topic:event_queue, part:259 > offset:14340519 > > maxSize:3145728), FetchRequest(topic:event_queue, part:139 > offset:235498949 > > maxSize:3145728), FetchRequest(topic:event_queue, part:112 > offset:235697262 > > maxSize:3145728), FetchRequest(topic:event_queue, part:202 > offset:13646959 > > maxSize:3145728), FetchRequest(topic:event_queue, part:192 > offset:13743839 > > maxSize:3145728), FetchRequest(topic:event_queue, part:42 > offset:1922290453 > > maxSize:3145728), FetchRequest(topic:event_queue, part:122 > offset:235463836 > > maxSize:3145728), FetchRequest(topic:event_queue, part:94 > offset:236971652 > > maxSize:3145728), FetchRequest(topic:event_queue, part:101 > offset:235374917 > > maxSize:3145728), FetchRequest(topic:event_queue, part:280 > offset:13739293 > > maxSize:3145728), FetchRequest(topic:event_queue, part:298 > offset:14022233 > > maxSize:3145728), FetchRequest(topic:event_queue, part:119 > offset:235026837 > > maxSize:3145728), FetchRequest(topic:event_queue, part:49 > offset:1924256307 > > maxSize:3145728), FetchRequest(topic:event_queue, part:168 > offset:13845729 > > maxSize:3145728), FetchRequest(topic:event_queue, part:23 > offset:1922510880 > > maxSize:3145728), FetchRequest(topic:event_queue, part:195 > offset:13936005 > > maxSize:3145728), FetchRequest(topic:event_queue, part:277 > offset:13648569 > > maxSize:3145728), FetchRequest(topic:event_queue, part:230 > offset:13905437 > > maxSize:3145728), FetchRequest(topic:event_queue, part:91 > offset:234971350 > > maxSize:3145728), FetchRequest(topic:event_queue, part:41 > offset:1923355011 > > maxSize:3145728), FetchRequest(topic:event_queue, part:279 > offset:14000137 > > maxSize:3145728), FetchRequest(topic:event_queue, part:65 > offset:1914859800 > > maxSize:3145728), FetchRequest(topic:event_queue, part:221 > offset:13826817 > > maxSize:3145728), FetchRequest(topic:event_queue, part:32 > offset:1920897120 > > maxSize:3145728), FetchRequest(topic:event_queue, part:251 > offset:13587337 > > maxSize:3145728), FetchRequest(topic:event_queue, part:177 > offset:13673153 > > maxSize:3145728), FetchRequest(topic:event_queue, part:288 > offset:13738515 > > maxSize:3145728), FetchRequest(topic:event_queue, part:212 > offset:13972659 > > maxSize:3145728), FetchRequest(topic:event_queue, part:249 > offset:13687993 > > maxSize:3145728), FetchRequest(topic:event_queue, part:270 > offset:13973886 > > maxSize:3145728), FetchRequest(topic:event_queue, part:158 > offset:13961991 > > maxSize:3145728), FetchRequest(topic:event_queue, part:260 > offset:13749477 > > maxSize:3145728), FetchRequest(topic:event_queue, part:13 > offset:1922663539 > > maxSize:3145728), FetchRequest(topic:event_queue, part:56 > offset:1919536166 > > maxSize:3145728), FetchRequest(topic:event_queue, part:223 > offset:13634407 > > maxSize:3145728), FetchRequest(topic:event_queue, part:194 > offset:13726728 > > maxSize:3145728), FetchRequest(topic:event_queue, part:63 > offset:1920874310 > > maxSize:3145728), FetchRequest(topic:event_queue, part:129 > offset:238085303 > > maxSize:3145728), FetchRequest(topic:event_queue, part:4 > offset:1926372785 > > maxSize:3145728), FetchRequest(topic:event_queue, part:120 > offset:235299544 > > maxSize:3145728), FetchRequest(topic:event_queue, part:111 > offset:236673617 > > maxSize:3145728), FetchRequest(topic:event_queue, part:36 > offset:1923256197 > > maxSize:3145728), FetchRequest(topic:event_queue, part:167 > offset:13864323 > > maxSize:3145728), FetchRequest(topic:event_queue, part:278 > offset:14512652 > > maxSize:3145728), FetchRequest(topic:event_queue, part:22 > offset:1921366567 > > maxSize:3145728), FetchRequest(topic:event_queue, part:50 > offset:1921299677 > > maxSize:3145728), FetchRequest(topic:event_queue, part:102 > offset:235378466 > > maxSize:3145728), FetchRequest(topic:event_queue, part:203 > offset:13909878 > > maxSize:3145728), FetchRequest(topic:event_queue, part:269 > offset:13730961 > > maxSize:3145728), FetchRequest(topic:event_queue, part:43 > offset:1923207679 > > maxSize:3145728), FetchRequest(topic:event_queue, part:29 > offset:1922837141 > > maxSize:3145728), FetchRequest(topic:event_queue, part:93 > offset:235942183 > > maxSize:3145728), FetchRequest(topic:event_queue, part:231 > offset:13740564 > > maxSize:3145728), FetchRequest(topic:event_queue, part:15 > offset:1924837631 > > maxSize:3145728), FetchRequest(topic:event_queue, part:174 > offset:14167227 > > maxSize:3145728), FetchRequest(topic:event_queue, part:184 > offset:13792167 > > maxSize:3145728), FetchRequest(topic:event_queue, part:138 > offset:237532346 > > maxSize:3145728), FetchRequest(topic:event_queue, part:222 > offset:14037383 > > maxSize:3145728), FetchRequest(topic:event_queue, part:240 > offset:13699592 > > maxSize:3145728), FetchRequest(topic:event_queue, part:74 > offset:1915585918 > > maxSize:3145728), FetchRequest(topic:event_queue, part:159 > offset:13656050 > > maxSize:3145728), FetchRequest(topic:event_queue, part:145 > offset:235316997 > > maxSize:3145728), FetchRequest(topic:event_queue, part:83 > offset:239701975 > > maxSize:3145728), FetchRequest(topic:event_queue, part:55 > offset:1922797177 > > maxSize:3145728), FetchRequest(topic:event_queue, part:297 > offset:13892607 > > maxSize:3145728), FetchRequest(topic:event_queue, part:186 > offset:13827509 > > maxSize:3145728), FetchRequest(topic:event_queue, part:152 > offset:13992704 > > maxSize:3145728), FetchRequest(topic:event_queue, part:257 > offset:13974765 > > maxSize:3145728), FetchRequest(topic:event_queue, part:130 > offset:238644589 > > maxSize:3145728), FetchRequest(topic:event_queue, part:89 > offset:236607948 > > maxSize:3145728), FetchRequest(topic:event_queue, part:264 > offset:14004638 > > maxSize:3145728), FetchRequest(topic:event_queue, part:64 > offset:1920319726 > > maxSize:3145728), FetchRequest(topic:event_queue, part:103 > offset:235407571 > > maxSize:3145728), FetchRequest(topic:event_queue, part:250 > offset:14072244 > > maxSize:3145728), FetchRequest(topic:event_queue, part:92 > offset:236982568 > > maxSize:3145728), FetchRequest(topic:event_queue, part:166 > offset:13706618 > > maxSize:3145728), FetchRequest(topic:event_queue, part:8 > offset:1920802613 > > maxSize:3145728), FetchRequest(topic:event_queue, part:271 > offset:14091149 > > maxSize:3145728), FetchRequest(topic:event_queue, part:193 > offset:13919759 > > maxSize:3145728), FetchRequest(topic:event_queue, part:200 > offset:13964577 > > maxSize:3145728), FetchRequest(topic:event_queue, part:1 > offset:1921914028 > > maxSize:3145728), FetchRequest(topic:event_queue, part:179 > offset:13693807 > > maxSize:3145728), FetchRequest(topic:event_queue, part:2 > offset:1923189002 > > maxSize:3145728), FetchRequest(topic:event_queue, part:160 > offset:13987352 > > maxSize:3145728), FetchRequest(topic:event_queue, part:82 > offset:234913477 > > maxSize:3145728), FetchRequest(topic:event_queue, part:96 > offset:236478691 > > maxSize:3145728), FetchRequest(topic:event_queue, part:175 > offset:13985981 > > maxSize:3145728), FetchRequest(topic:event_queue, part:30 > offset:1928338809 > > maxSize:3145728), FetchRequest(topic:event_queue, part:44 > offset:1927344365 > > maxSize:3145728), FetchRequest(topic:event_queue, part:61 > offset:1917076449 > > maxSize:3145728), FetchRequest(topic:event_queue, part:54 > offset:1918544576 > > maxSize:3145728), FetchRequest(topic:event_queue, part:137 > offset:235429924 > > maxSize:3145728), FetchRequest(topic:event_queue, part:146 > offset:234024621 > > maxSize:3145728), FetchRequest(topic:event_queue, part:68 > offset:1915824648 > > maxSize:3145728), FetchRequest(topic:event_queue, part:207 > offset:13983596 > > maxSize:3145728), FetchRequest(topic:event_queue, part:214 > offset:13843173 > > maxSize:3145728), FetchRequest(topic:event_queue, part:131 > offset:239218119 > > maxSize:3145728), FetchRequest(topic:event_queue, part:75 > offset:234730864 > > maxSize:3145728), FetchRequest(topic:event_queue, part:242 > offset:13924920 > > maxSize:3145728), FetchRequest(topic:event_queue, part:272 > offset:14129560 > > maxSize:3145728), FetchRequest(topic:event_queue, part:16 > offset:1923561885 > > maxSize:3145728), FetchRequest(topic:event_queue, part:124 > offset:236152284 > > maxSize:3145728), FetchRequest(topic:event_queue, part:187 > offset:14249552 > > maxSize:3145728), FetchRequest(topic:event_queue, part:81 > offset:234178502 > > maxSize:3145728), FetchRequest(topic:event_queue, part:228 > offset:14033290 > > maxSize:3145728), FetchRequest(topic:event_queue, part:165 > offset:14133744 > > maxSize:3145728), FetchRequest(topic:event_queue, part:110 > offset:236192852 > > maxSize:3145728), FetchRequest(topic:event_queue, part:258 > offset:14139932 > > maxSize:3145728), FetchRequest(topic:event_queue, part:151 > offset:14008504 > > maxSize:3145728), FetchRequest(topic:event_queue, part:117 > offset:237862268 > > maxSize:3145728), FetchRequest(topic:event_queue, part:172 > offset:13823876 > > maxSize:3145728), FetchRequest(topic:event_queue, part:215 > offset:13807059 > > maxSize:3145728), FetchRequest(topic:event_queue, part:243 > offset:13579687 > > maxSize:3145728), FetchRequest(topic:event_queue, part:299 > offset:14130546 > > maxSize:3145728), FetchRequest(topic:event_queue, part:17 > offset:1924461681 > > maxSize:3145728), FetchRequest(topic:event_queue, part:45 > offset:1924311439 > > maxSize:3145728), FetchRequest(topic:event_queue, part:132 > offset:238453123 > > maxSize:3145728), FetchRequest(topic:event_queue, part:285 > offset:13712821 > > maxSize:3145728), FetchRequest(topic:event_queue, part:154 > offset:13729543 > > maxSize:3145728), FetchRequest(topic:event_queue, part:229 > offset:14063729 > > maxSize:3145728), FetchRequest(topic:event_queue, part:201 > offset:14202133 > > maxSize:3145728), FetchRequest(topic:event_queue, part:161 > offset:14083985 > > maxSize:3145728), FetchRequest(topic:event_queue, part:62 > offset:1920871581 > > maxSize:3145728), FetchRequest(topic:event_queue, part:95 > offset:236078056 > > maxSize:3145728), FetchRequest(topic:event_queue, part:76 > offset:235057152 > > maxSize:3145728), FetchRequest(topic:event_queue, part:67 > offset:1918530968 > > maxSize:3145728), FetchRequest(topic:event_queue, part:118 > offset:236009547 > > maxSize:3145728), FetchRequest(topic:event_queue, part:10 > offset:1920294682 > > maxSize:3145728), FetchRequest(topic:event_queue, part:286 > offset:14140007 > > maxSize:3145728), FetchRequest(topic:event_queue, part:90 > offset:235531660 > > maxSize:3145728), FetchRequest(topic:event_queue, part:164 > offset:13949421 > > maxSize:3145728), FetchRequest(topic:event_queue, part:234 > offset:14068535 > > maxSize:3145728), FetchRequest(topic:event_queue, part:38 > offset:1921910104 > > maxSize:3145728), FetchRequest(topic:event_queue, part:181 > offset:13684468 > > maxSize:3145728), FetchRequest(topic:event_queue, part:7 > offset:1922581694 > > maxSize:3145728), FetchRequest(topic:event_queue, part:35 > offset:1919656737 > > maxSize:3145728), FetchRequest(topic:event_queue, part:109 > offset:238515986 > > maxSize:3145728), FetchRequest(topic:event_queue, part:188 > offset:14030914 > > maxSize:3145728), FetchRequest(topic:event_queue, part:104 > offset:237177303 > > maxSize:3145728), FetchRequest(topic:event_queue, part:123 > offset:235236776 > > maxSize:3145728), FetchRequest(topic:event_queue, part:0 > offset:1921886214 > > maxSize:3145728), FetchRequest(topic:event_queue, part:245 > offset:13954109 > > maxSize:3145728), FetchRequest(topic:event_queue, part:136 > offset:236161678 > > maxSize:3145728), FetchRequest(topic:event_queue, part:199 > offset:14224604 > > maxSize:3145728), FetchRequest(topic:event_queue, part:227 > offset:14037136 > > maxSize:3145728), FetchRequest(topic:event_queue, part:143 > offset:235733339 > > maxSize:3145728), FetchRequest(topic:event_queue, part:206 > offset:14198134 > > maxSize:3145728), FetchRequest(topic:event_queue, part:216 > offset:13736453 > > maxSize:3145728), FetchRequest(topic:event_queue, part:18 > offset:1921683680 > > maxSize:3145728), FetchRequest(topic:event_queue, part:237 > offset:13669627 > > maxSize:3145728), FetchRequest(topic:event_queue, part:126 > offset:235772779 > > maxSize:3145728), FetchRequest(topic:event_queue, part:209 > offset:13803127 > > maxSize:3145728), FetchRequest(topic:event_queue, part:294 > offset:13930318 > > maxSize:3145728), FetchRequest(topic:event_queue, part:153 > offset:14247805 > > maxSize:3145728), FetchRequest(topic:event_queue, part:59 > offset:1916170361 > > maxSize:3145728), FetchRequest(topic:event_queue, part:80 > offset:236311660 > > maxSize:3145728), FetchRequest(topic:event_queue, part:87 > offset:235675888 > > maxSize:3145728), FetchRequest(topic:event_queue, part:273 > offset:13894364 > > maxSize:3145728), FetchRequest(topic:event_queue, part:291 > offset:14120121 > > maxSize:3145728), FetchRequest(topic:event_queue, part:37 > offset:1926437324 > > maxSize:3145728), FetchRequest(topic:event_queue, part:28 > offset:1923884304 > > maxSize:3145728), FetchRequest(topic:event_queue, part:244 > offset:13639986 > > maxSize:3145728), FetchRequest(topic:event_queue, part:162 > offset:13631619 > > maxSize:3145728), FetchRequest(topic:event_queue, part:266 > offset:14276657 > > maxSize:3145728), FetchRequest(topic:event_queue, part:133 > offset:234190625 > > maxSize:3145728), FetchRequest(topic:event_queue, part:173 > offset:13718276 > > maxSize:3145728), FetchRequest(topic:event_queue, part:284 > offset:13634149 > > maxSize:3145728), FetchRequest(topic:event_queue, part:88 > offset:239251670 > > maxSize:3145728), FetchRequest(topic:event_queue, part:256 > offset:13963549 > > maxSize:3145728), FetchRequest(topic:event_queue, part:235 > offset:14268066 > > maxSize:3145728), FetchRequest(topic:event_queue, part:180 > offset:13994435 > > maxSize:3145728), FetchRequest(topic:event_queue, part:77 > offset:237671159 > > maxSize:3145728), FetchRequest(topic:event_queue, part:108 > offset:236023907 > > maxSize:3145728), FetchRequest(topic:event_queue, part:263 > offset:13953716 > > maxSize:3145728), FetchRequest(topic:event_queue, part:70 > offset:1919192226 > > maxSize:3145728), FetchRequest(topic:event_queue, part:142 > offset:234821981 > > maxSize:3145728), FetchRequest(topic:event_queue, part:105 > offset:236803009 > > maxSize:3145728), FetchRequest(topic:event_queue, part:156 > offset:14194305 > > maxSize:3145728), FetchRequest(topic:event_queue, part:144 > offset:235378461 > > maxSize:3145728), FetchRequest(topic:event_queue, part:46 > offset:1928419689 > > maxSize:3145728), FetchRequest(topic:event_queue, part:198 > offset:13974447 > > maxSize:3145728), FetchRequest(topic:event_queue, part:149 > offset:238015271 > > maxSize:3145728), FetchRequest(topic:event_queue, part:115 > offset:235811382 > > maxSize:3145728), FetchRequest(topic:event_queue, part:98 > offset:235100030 > > maxSize:3145728), FetchRequest(topic:event_queue, part:265 > offset:13907349 > > maxSize:3145728), FetchRequest(topic:event_queue, part:274 > offset:13864687 > > maxSize:3145728), FetchRequest(topic:event_queue, part:226 > offset:13537827 > > maxSize:3145728), FetchRequest(topic:event_queue, part:9 > offset:1923454590 > > maxSize:3145728), FetchRequest(topic:event_queue, part:163 > offset:13882392 > > maxSize:3145728), FetchRequest(topic:event_queue, part:255 > offset:13946694 > > maxSize:3145728), FetchRequest(topic:event_queue, part:60 > offset:1922506946 > > maxSize:3145728), FetchRequest(topic:event_queue, part:134 > offset:236241772 > > maxSize:3145728), FetchRequest(topic:event_queue, part:293 > offset:14174207 > > maxSize:3145728), FetchRequest(topic:event_queue, part:69 > offset:1917935964 > > maxSize:3145728), FetchRequest(topic:event_queue, part:53 > offset:1920040686 > > maxSize:3145728), FetchRequest(topic:event_queue, part:189 > offset:14081801 > > maxSize:3145728), FetchRequest(topic:event_queue, part:125 > offset:235462272 > > maxSize:3145728), FetchRequest(topic:event_queue, part:27 > offset:1922583519 > > maxSize:3145728), FetchRequest(topic:event_queue, part:107 > offset:235043816 > > maxSize:3145728), FetchRequest(topic:event_queue, part:292 > offset:14225910 > > maxSize:3145728), FetchRequest(topic:event_queue, part:217 > offset:14155810 > > maxSize:3145728), FetchRequest(topic:event_queue, part:208 > offset:13869168 > > maxSize:3145728), FetchRequest(topic:event_queue, part:79 > offset:237019821 > > maxSize:3145728), FetchRequest(topic:event_queue, part:171 > offset:13921149 > > maxSize:3145728), FetchRequest(topic:event_queue, part:218 > offset:14012275 > > maxSize:3145728), FetchRequest(topic:event_queue, part:52 > offset:1920570629 > > maxSize:3145728), FetchRequest(topic:event_queue, part:283 > offset:13970846 > > maxSize:3145728), FetchRequest(topic:event_queue, part:47 > offset:1924652182 > > maxSize:3145728), FetchRequest(topic:event_queue, part:232 > offset:14499312 > > maxSize:3145728), FetchRequest(topic:event_queue, part:211 > offset:13847224 > > maxSize:3145728), FetchRequest(topic:event_queue, part:141 > offset:237422517 > > maxSize:3145728), FetchRequest(topic:event_queue, part:106 > offset:237027686 > > maxSize:3145728), FetchRequest(topic:event_queue, part:26 > offset:1921991622 > > maxSize:3145728), FetchRequest(topic:event_queue, part:236 > offset:13700792 > > maxSize:3145728), FetchRequest(topic:event_queue, part:190 > offset:14048595 > > maxSize:3145728), FetchRequest(topic:event_queue, part:225 > offset:13975669 > > maxSize:3145728), FetchRequest(topic:event_queue, part:40 > offset:1922256327 > > maxSize:3145728), FetchRequest(topic:event_queue, part:116 > offset:235790080 > > maxSize:3145728), FetchRequest(topic:event_queue, part:33 > offset:1922681776 > > maxSize:3145728), FetchRequest(topic:event_queue, part:197 > offset:13786691 > > maxSize:3145728), FetchRequest(topic:event_queue, part:5 > offset:1924902525 > > maxSize:3145728), FetchRequest(topic:event_queue, part:178 > offset:13756870 > > maxSize:3145728), FetchRequest(topic:event_queue, part:204 > offset:14337005 > > maxSize:3145728), FetchRequest(topic:event_queue, part:97 > offset:236192027 > > maxSize:3145728), FetchRequest(topic:event_queue, part:239 > offset:13759979 > > maxSize:3145728), FetchRequest(topic:event_queue, part:78 > offset:233418728 > > maxSize:3145728), FetchRequest(topic:event_queue, part:121 > offset:235819188 > > maxSize:3145728), FetchRequest(topic:event_queue, part:12 > offset:1922738965 > > maxSize:3145728), FetchRequest(topic:event_queue, part:114 > offset:235220219 > > maxSize:3145728), FetchRequest(topic:event_queue, part:289 > offset:14041944 > > maxSize:3145728), FetchRequest(topic:event_queue, part:261 > offset:13667852 > > maxSize:3145728), FetchRequest(topic:event_queue, part:183 > offset:13697688 > > maxSize:3145728), FetchRequest(topic:event_queue, part:254 > offset:13772026 > > maxSize:3145728), FetchRequest(topic:event_queue, part:246 > offset:13545843 > > maxSize:3145728), FetchRequest(topic:event_queue, part:19 > offset:1925964016 > > maxSize:3145728), FetchRequest(topic:event_queue, part:25 > offset:1924976308 > > maxSize:3145728), FetchRequest(topic:event_queue, part:282 > offset:13591865 > > maxSize:3145728), FetchRequest(topic:event_queue, part:275 > offset:13649678 > > maxSize:3145728), FetchRequest(topic:event_queue, part:155 > offset:13665620 > > maxSize:3145728), FetchRequest(topic:event_queue, part:128 > offset:235160241 > > maxSize:3145728), FetchRequest(topic:event_queue, part:268 > offset:13889869 > > maxSize:3145728), FetchRequest(topic:event_queue, part:135 > offset:235842584 > > maxSize:3145728), FetchRequest(topic:event_queue, part:72 > offset:1915802630 > > maxSize:3145728), FetchRequest(topic:event_queue, part:296 > offset:13958874 > > maxSize:3145728), FetchRequest(topic:event_queue, part:39 > offset:1923280812 > > maxSize:3145728), FetchRequest(topic:event_queue, part:51 > offset:1919445372 > > maxSize:3145728), FetchRequest(topic:event_queue, part:20 > offset:1926233046 > > maxSize:3145728), FetchRequest(topic:event_queue, part:6 > offset:1923071675 > > maxSize:3145728), FetchRequest(topic:event_queue, part:34 > offset:1925185060 > > maxSize:3145728), FetchRequest(topic:event_queue, part:267 > offset:14059111 > > maxSize:3145728), FetchRequest(topic:event_queue, part:11 > offset:1921467904 > > maxSize:3145728), FetchRequest(topic:event_queue, part:48 > offset:1924981816 > > maxSize:3145728), FetchRequest(topic:event_queue, part:219 > offset:13847303 > > maxSize:3145728), FetchRequest(topic:event_queue, part:295 > offset:13759737 > > maxSize:3145728), FetchRequest(topic:event_queue, part:170 > offset:13809787 > > maxSize:3145728), FetchRequest(topic:event_queue, part:58 > offset:1920889286 > > maxSize:3145728), FetchRequest(topic:event_queue, part:205 > offset:13929926 > > maxSize:3145728), FetchRequest(topic:event_queue, part:247 > offset:14051637 > > maxSize:3145728), FetchRequest(topic:event_queue, part:224 > offset:13651775 > > maxSize:3145728), FetchRequest(topic:event_queue, part:147 > offset:235945537 > > maxSize:3145728), FetchRequest(topic:event_queue, part:233 > offset:13648633 > > maxSize:3145728), FetchRequest(topic:event_queue, part:100 > offset:235475945 > > maxSize:3145728), FetchRequest(topic:event_queue, part:86 > offset:236263448 > > maxSize:3145728), FetchRequest(topic:event_queue, part:113 > offset:234976306 > > maxSize:3145728), FetchRequest(topic:event_queue, part:99 > offset:235273075 > > maxSize:3145728), FetchRequest(topic:event_queue, part:210 > offset:13753055 > > maxSize:3145728), FetchRequest(topic:event_queue, part:127 > offset:235577543 > > maxSize:3145728), FetchRequest(topic:event_queue, part:262 > offset:14314531 > > maxSize:3145728), FetchRequest(topic:event_queue, part:182 > offset:13947552 > > maxSize:3145728), FetchRequest(topic:event_queue, part:191 > offset:14075203 > > maxSize:3145728), FetchRequest(topic:event_queue, part:238 > offset:14341058 > > maxSize:3145728), FetchRequest(topic:event_queue, part:253 > offset:13961191 > > maxSize:3145728), FetchRequest(topic:event_queue, part:3 > offset:1921808922 > > maxSize:3145728), FetchRequest(topic:event_queue, part:140 > offset:235461115 > > maxSize:3145728), FetchRequest(topic:event_queue, part:281 > offset:14095956 > > maxSize:3145728), FetchRequest(topic:event_queue, part:196 > offset:13767437 > > maxSize:3145728)) > > 2013-08-14 23:28:13 TRACE FetcherRunnable:35 - fetched bytes: 0 > > 2013-08-14 23:28:13 DEBUG FetcherRunnable:48 - backing off 10 ms > > > > -- > > Best Regards, > > Ramakrishna Mallya > -- Best Regards, Ramakrishna Mallya
