{"id":223331,"date":"2022-05-01T14:51:22","date_gmt":"2022-05-01T06:51:22","guid":{"rendered":"https:\/\/www.idc.net\/help\/223331\/"},"modified":"2022-05-01T14:51:22","modified_gmt":"2022-05-01T06:51:22","slug":"%e4%b8%80%e7%af%87%e5%b8%a6%e7%bb%99%e4%bd%a0kafka%e6%a0%b8%e5%bf%83%e7%9f%a5%e8%af%86%e6%80%bb%e7%bb%93%ef%bc%81","status":"publish","type":"post","link":"https:\/\/idc.net\/help\/223331\/","title":{"rendered":"\u4e00\u7bc7\u5e26\u7ed9\u4f60Kafka\u6838\u5fc3\u77e5\u8bc6\u603b\u7ed3\uff01"},"content":{"rendered":"<h1>\u4e00\u7bc7\u5e26\u7ed9\u4f60Kafka\u6838\u5fc3\u77e5\u8bc6\u603b\u7ed3\uff01<\/h1>\n<p><span>\u4f5c\u8005\uff1a\u65e5\u5e38\u52a0\u6cb9\u7ad9<\/span> 2021-09-06 08:31:11<br \/>\n<i><\/i><br \/>\n<span>\u5f00\u53d1<\/span><br \/>\n<i><\/i><br \/>\n<span>\u67b6\u6784<\/span><br \/>\n<i><\/i><br \/>\n<span>Kafka<\/span> Apache Kafka\u662f\u7531LinkedIn\u91c7\u7528Scala\u548cJava\u5f00\u53d1\u7684\u5f00\u6e90\u6d41\u5904\u7406\u8f6f\u4ef6\u5e73\u53f0\uff0c\u5e76\u6350\u8d60\u7ed9\u4e86Apache Software Foundation\u3002 <\/p>\n<h2><span>[[421913]]<\/span><\/h2>\n<h2>\u57fa\u672c\u7b80\u4ecb<\/h2>\n<p>Apache Kafka\u662f\u7531LinkedIn\u91c7\u7528Scala\u548cJava\u5f00\u53d1\u7684\u5f00\u6e90\u6d41\u5904\u7406\u8f6f\u4ef6\u5e73\u53f0\uff0c\u5e76\u6350\u8d60\u7ed9\u4e86Apache Software Foundation\u3002<\/p>\n<p>\u8be5\u9879\u76ee\u65e8\u5728\u63d0\u4f9b\u7edf\u4e00\u7684\u3001\u9ad8\u541e\u5410\u91cf\u3001\u4f4e\u5ef6\u8fdf\u7684\u5e73\u53f0\u6765\u5904\u7406\u5b9e\u65f6\u6570\u636e\u6d41\u3002<\/p>\n<p>Kafka\u53ef\u4ee5\u901a\u8fc7Kafka Connect\u8fde\u63a5\u5230\u5916\u90e8\u7cfb\u7edf\uff0c\u5e76\u63d0\u4f9b\u4e86Kafka Streams\u3002<\/p>\n<\/p>\n<h3>\u300cKafka\u7684\u7279\u6027\u300d<\/h3>\n<p>Kafka\u662f\u4e00\u79cd\u5206\u5e03\u5f0f\u7684\uff0c\u57fa\u4e8e\u53d1\u5e03\/\u8ba2\u9605\u7684\u6d88\u606f\u7cfb\u7edf\uff0c\u4e3b\u8981\u7279\u6027\u5982\u4e0b\uff1a<\/p>\n<p style=\"text-align: center\">\n<h2>\u7248\u672c\u53f7<\/h2>\n<h3>\u300cKafka\u7248\u672c\u547d\u540d\u300d<\/h3>\n<p>\u6211\u4eec\u5728\u5b98\u7f51\u4e0a\u4e0b\u8f7dKafka\u65f6\uff0c\u4f1a\u770b\u5230\u8fd9\u6837\u7684\u7248\u672c\uff1a<\/p>\n<p style=\"text-align: center\">\n<p>\u524d\u9762\u7684\u7248\u672c\u53f7\u662f\u7f16\u8bd1Kafka\u6e90\u4ee3\u7801\u7684Scala\u7f16\u8bd1\u5668\u7248\u672c\u3002<\/p>\n<p>Kafka\u670d\u52a1\u5668\u7aef\u7684\u4ee3\u7801\u5b8c\u5168\u7531Scala\u8bed\u8a00\u7f16\u5199\uff0cScala\u540c\u65f6\u652f\u6301\u9762\u5411\u5bf9\u8c61\u7f16\u7a0b\u548c\u51fd\u6570\u5f0f\u7f16\u7a0b\uff0c\u7528Scala\u5199\u6210\u7684\u6e90\u4ee3\u7801\u7f16\u8bd1\u4e4b\u540e\u4e5f\u662f\u666e\u901a\u7684.class\u6587\u4ef6\uff0c\u56e0\u6b64\u6211\u4eec\u8bf4Scala\u662fJVM\u7cfb\u7684\u8bed\u8a00\u3002<\/p>\n<p>\u771f\u6b63\u7684Kafka\u7248\u672c\u53f7\u5b9e\u9645\u4e0a\u662f2.1.1\u3002<\/p>\n<ul>\n<li>\u90a3\u4e48\u8fd9\u4e2a2.1.1\u53c8\u8868\u793a\u4ec0\u4e48\u5462?<\/li>\n<\/ul>\n<p>\u524d\u9762\u76842\u8868\u793a\u5927\u7248\u672c\u53f7\uff0c\u5373Major Version;\u4e2d\u95f4\u76841\u8868\u793a\u5c0f\u7248\u672c\u53f7\u6216\u6b21\u7248\u672c\u53f7\uff0c\u5373Minor Version;\u6700\u540e\u76841\u8868\u793a\u4fee\u8ba2\u7248\u672c\u53f7\uff0c\u4e5f\u5c31\u662fPatch\u53f7\u3002<\/p>\n<p>Kafka\u793e\u533a\u5728\u53d1\u5e031.0.0\u7248\u672c\u540e\u5199\u8fc7\u4e00\u7bc7\u6587\u7ae0\uff0c\u5ba3\u5e03Kafka\u7248\u672c\u547d\u540d\u89c4\u5219\u6b63\u5f0f\u4ece4\u4f4d\u6f14\u8fdb\u52303\u4f4d\uff0c\u6bd4\u59820.11.0.0\u7248\u672c\u5c31\u662f4\u4f4d\u7248\u672c\u53f7\u3002<\/p>\n<p>\u6709\u4e2a\u5efa\u8bae\uff0c\u4e0d\u8bba\u7528\u7684\u662f\u54ea\u4e2a\u7248\u672c\uff0c\u90fd\u8bf7\u5c3d\u91cf\u4fdd\u6301\u670d\u52a1\u5668\u7aef\u7248\u672c\u548c\u5ba2\u6237\u7aef\u7248\u672c\u4e00\u81f4\uff0c\u5426\u5219\u4f60\u5c06\u635f\u5931\u5f88\u591aKafka\u4e3a\u4f60\u63d0\u4f9b\u7684\u6027\u80fd\u4f18\u5316\u6536\u76ca\u3002<\/p>\n<h3>\u300c\u7248\u672c\u6f14\u8fdb\u300d<\/h3>\n<p>0.7\u7248\u672c\uff1a\u53ea\u63d0\u4f9b\u4e86\u6700\u57fa\u7840\u7684\u6d88\u606f\u961f\u5217\u529f\u80fd\u3002<\/p>\n<p>0.8\u7248\u672c\uff1a\u5f15\u5165\u4e86\u526f\u672c\u673a\u5236\uff0c\u81f3\u6b64kafka\u6210\u4e3a\u4e86\u4e00\u4e2a\u6574\u6574\u610f\u4e49\u4e0a\u5b8c\u5907\u7684\u5206\u5e03\u5f0f\u53ef\u9760\u6d88\u606f\u961f\u5217\u89e3\u51b3\u65b9\u6848<\/p>\n<p>0.9.0.0\u7248\u672c\uff1a\u589e\u52a0\u4e86\u57fa\u7840\u7684\u5b89\u5168\u8ba4\u8bc1\/\u6743\u9650\u529f\u80fd;\u4f7f\u7528Java\u91cd\u65b0\u4e86\u65b0\u7248\u672c\u6d88\u8d39\u8005API;\u5f15\u5165\u4e86Kafka Connect\u7ec4\u4ef6\u3002<\/p>\n<p>0.11.0.0\u7248\u672c\uff1a\u63d0\u4f9b\u4e86\u5e42\u7b49\u6027Producer API\u4ee5\u53ca\u4e8b\u52a1API;\u5bf9Kafka\u6d88\u606f\u683c\u5f0f\u505a\u4e86\u91cd\u6784\u3002<\/p>\n<p>1.0\u548c2.0\u7248\u672c\uff1a\u4e3b\u8981\u8fd8\u662fKafka Streams\u7684\u5404\u79cd\u6539\u8fdb<\/p>\n<h2>\u57fa\u672c\u6982\u5ff5<\/h2>\n<p style=\"text-align: center\">\n<h3>\u300c\u4e3b\u9898\u300d<\/h3>\n<p>\u53d1\u5e03\u8ba2\u9605\u7684\u5bf9\u8c61\u662f\u4e3b\u9898(Topic)\uff0c\u53ef\u4ee5\u4e3a\u6bcf \u4e2a\u4e1a\u52a1\u3001\u6bcf\u4e2a\u5e94\u7528\u751a\u81f3\u662f\u6bcf\u7c7b\u6570\u636e\u90fd\u521b\u5efa\u4e13\u5c5e\u7684\u4e3b\u9898<\/p>\n<h3>\u300c\u751f\u4ea7\u8005\u548c\u6d88\u8d39\u8005\u300d<\/h3>\n<p>\u5411\u4e3b\u9898\u53d1\u5e03\u6d88\u606f\u7684\u5ba2\u6237\u7aef\u5e94\u7528\u7a0b\u5e8f\u79f0\u4e3a\u751f\u4ea7\u8005\uff0c\u751f\u4ea7\u8005\u7a0b\u5e8f\u901a\u5e38\u6301\u7eed\u4e0d\u65ad\u5730 \u5411\u4e00\u4e2a\u6216\u591a\u4e2a\u4e3b\u9898\u53d1\u9001\u6d88\u606f<\/p>\n<p>\u8ba2\u9605\u8fd9\u4e9b\u4e3b\u9898\u6d88\u606f\u7684\u5ba2\u6237\u7aef\u5e94\u7528\u7a0b\u5e8f\u5c31\u88ab\u79f0\u4e3a\u6d88\u8d39\u8005\uff0c\u6d88\u8d39\u8005\u4e5f\u80fd\u591f\u540c\u65f6\u8ba2\u9605\u591a\u4e2a\u4e3b\u9898\u7684\u6d88\u606f<\/p>\n<h3>\u300cBroker\u300d<\/h3>\n<p>\u96c6\u7fa4\u7531\u591a\u4e2a Broker \u7ec4\u6210\uff0cBroker \u8d1f\u8d23\u63a5\u6536\u548c\u5904\u7406\u5ba2\u6237\u7aef\u53d1\u9001\u8fc7\u6765\u7684\u8bf7\u6c42\uff0c\u4ee5\u53ca\u5bf9\u6d88\u606f\u8fdb\u884c\u6301\u4e45\u5316<\/p>\n<p>\u867d\u7136\u591a\u4e2a Broker \u8fdb\u7a0b\u80fd\u591f\u8fd0\u884c\u5728\u540c\u4e00\u53f0\u673a\u5668\u4e0a\uff0c\u4f46\u66f4\u5e38\u89c1\u7684\u505a\u6cd5\u662f\u5c06 \u4e0d\u540c\u7684 Broker \u5206\u6563\u8fd0\u884c\u5728\u4e0d\u540c\u7684\u673a\u5668\u4e0a\uff0c\u8fd9\u6837\u5982\u679c\u96c6\u7fa4\u4e2d\u67d0\u4e00\u53f0\u673a\u5668\u5b95\u673a\uff0c\u5373\u4f7f\u5728\u5b83\u4e0a\u9762 \u8fd0\u884c\u7684\u6240\u6709 Broker \u8fdb\u7a0b\u90fd\u6302\u6389\u4e86\uff0c\u5176\u4ed6\u673a\u5668\u4e0a\u7684 Broker \u4e5f\u4f9d\u7136\u80fd\u591f\u5bf9\u5916\u63d0\u4f9b\u670d\u52a1<\/p>\n<h3>\u300c\u5907\u4efd\u673a\u5236\u300d<\/h3>\n<p>\u5907\u4efd\u7684\u601d\u60f3\u5f88\u7b80\u5355\uff0c\u5c31\u662f\u628a\u76f8\u540c\u7684\u6570\u636e\u62f7\u8d1d\u5230\u591a\u53f0\u673a\u5668\u4e0a\uff0c\u800c\u8fd9\u4e9b\u76f8\u540c\u7684\u6570\u636e\u62f7\u8d1d\u88ab\u79f0\u4e3a\u526f\u672c<\/p>\n<p>\u5b9a\u4e49\u4e86\u4e24\u7c7b\u526f\u672c\uff1a\u9886\u5bfc\u8005\u526f\u672c\u548c\u8ffd\u968f\u8005\u526f\u672c<\/p>\n<p>\u524d\u8005\u5bf9\u5916\u63d0\u4f9b\u670d\u52a1\uff0c\u8fd9\u91cc\u7684\u5bf9\u5916\u6307\u7684\u662f\u4e0e \u5ba2\u6237\u7aef\u7a0b\u5e8f\u8fdb\u884c\u4ea4\u4e92;\u800c\u540e\u8005\u53ea\u662f\u88ab\u52a8\u5730\u8ffd\u968f\u9886\u5bfc\u8005\u526f\u672c\u800c\u5df2\uff0c\u4e0d\u80fd\u4e0e\u5916\u754c\u8fdb\u884c\u4ea4\u4e92<\/p>\n<h3>\u300c\u5206\u533a\u300d<\/h3>\n<p>\u5206\u533a\u673a\u5236\u6307\u7684\u662f\u5c06\u6bcf\u4e2a\u4e3b\u9898\u5212\u5206\u6210\u591a\u4e2a\u5206\u533a\uff0c\u6bcf\u4e2a\u5206\u533a\u662f\u4e00\u7ec4\u6709\u5e8f\u7684\u6d88\u606f\u65e5\u5fd7<\/p>\n<p>\u751f\u4ea7\u8005\u751f\u4ea7\u7684\u6bcf\u6761\u6d88\u606f\u53ea\u4f1a\u88ab\u53d1\u9001\u5230\u4e00\u4e2a\u5206\u533a\u4e2d\uff0c\u4e5f\u5c31\u662f\u8bf4\u5982\u679c\u5411\u4e00\u4e2a\u53cc\u5206\u533a\u7684\u4e3b\u9898\u53d1\u9001\u4e00\u6761\u6d88\u606f\uff0c\u8fd9\u6761\u6d88\u606f\u8981\u4e48\u5728\u5206\u533a 0 \u4e2d\uff0c\u8981\u4e48\u5728\u5206\u533a 1 \u4e2d<\/p>\n<p>\u6bcf\u4e2a\u5206\u533a\u4e0b\u53ef\u4ee5\u914d\u7f6e\u82e5\u5e72\u4e2a\u526f\u672c\uff0c\u5176\u4e2d\u53ea\u80fd\u6709 1 \u4e2a\u9886 \u5bfc\u8005\u526f\u672c\u548c N-1 \u4e2a\u8ffd\u968f\u8005\u526f\u672c<\/p>\n<p>\u751f\u4ea7\u8005\u5411\u5206\u533a\u5199\u5165\u6d88\u606f\uff0c\u6bcf\u6761\u6d88\u606f\u5728\u5206\u533a\u4e2d\u7684\u4f4d\u7f6e\u4fe1\u606f\u53eb\u4f4d\u79fb<\/p>\n<h3>\u300c\u6d88\u8d39\u8005\u7ec4\u300d<\/h3>\n<p>\u591a\u4e2a\u6d88\u8d39\u8005\u5b9e\u4f8b\u5171\u540c\u7ec4\u6210\u4e00\u4e2a\u7ec4\u6765 \u6d88\u8d39\u4e00\u7ec4\u4e3b\u9898<\/p>\n<p>\u8fd9\u7ec4\u4e3b\u9898\u4e2d\u7684\u6bcf\u4e2a\u5206\u533a\u90fd\u53ea\u4f1a\u88ab\u7ec4\u5185\u7684\u4e00\u4e2a\u6d88\u8d39\u8005\u5b9e\u4f8b\u6d88\u8d39\uff0c\u5176\u4ed6\u6d88\u8d39\u8005\u5b9e\u4f8b\u4e0d\u80fd\u6d88\u8d39\u5b83<\/p>\n<h3>\u540c\u65f6\u5b9e\u73b0\u4e86\u4f20\u7edf\u6d88\u606f\u5f15\u64ce\u7cfb\u7edf\u7684\u4e24\u5927\u6a21\u578b\uff1a<\/h3>\n<p>\u5982\u679c\u6240\u6709\u5b9e\u4f8b\u90fd\u5c5e\u4e8e\u540c\u4e00\u4e2a Group\uff0c \u90a3\u4e48\u5b83\u5b9e\u73b0\u7684\u5c31\u662f\u6d88\u606f\u961f\u5217\u6a21\u578b;<\/p>\n<p>\u5982\u679c\u6240\u6709\u5b9e\u4f8b\u5206\u522b\u5c5e\u4e8e\u4e0d \u540c\u7684 Group\uff0c\u90a3\u4e48\u5b83\u5b9e\u73b0\u7684\u5c31\u662f\u53d1\u5e03\/\u8ba2\u9605\u6a21\u578b<\/p>\n<h3>\u300cCoordinator\uff1a\u534f\u8c03\u8005\u300d<\/h3>\n<p>\u6240\u8c13\u534f\u8c03\u8005\uff0c\u5b83\u4e13\u95e8\u4e3aConsumer Group\u670d\u52a1\uff0c\u8d1f\u8d23\u4e3aGroup\u6267\u884cRebalance\u4ee5\u53ca\u63d0\u4f9b\u4f4d\u79fb\u7ba1\u7406\u548c\u7ec4\u6210\u5458\u7ba1\u7406\u7b49\u3002<\/p>\n<p>\u5177\u4f53\u6765\u8bb2\uff0cConsumer\u7aef\u5e94\u7528\u7a0b\u5e8f\u5728\u63d0\u4ea4\u4f4d\u79fb\u65f6\uff0c\u5176\u5b9e\u662f\u5411Coordinator\u6240\u5728\u7684Broker\u63d0\u4ea4\u4f4d\u79fb\uff0c\u540c\u6837\u5730\uff0c\u5f53Consumer\u5e94\u7528\u542f\u52a8\u65f6\uff0c\u4e5f\u662f\u5411Coordinator\u6240\u5728\u7684Broker\u53d1\u9001\u5404\u79cd\u8bf7\u6c42\uff0c\u7136\u540e\u7531Coordinator\u8d1f\u8d23\u6267\u884c\u6d88\u8d39\u8005\u7ec4\u7684\u6ce8\u518c\u3001\u6210\u5458\u7ba1\u7406\u8bb0\u5f55\u7b49\u5143\u6570\u636e\u7ba1\u7406\u64cd\u4f5c\u3002<\/p>\n<p>\u6240\u6709Broker\u5728\u542f\u52a8\u65f6\uff0c\u90fd\u4f1a\u521b\u5efa\u548c\u5f00\u542f\u76f8\u5e94\u7684Coordinator\u7ec4\u4ef6\u3002<\/p>\n<p>\u4e5f\u5c31\u662f\u8bf4\uff0c\u300c\u6240\u6709Broker\u90fd\u6709\u5404\u81ea\u7684Coordinator\u7ec4\u4ef6\u300d\u3002<\/p>\n<p>\u90a3\u4e48\uff0cConsumer Group\u5982\u4f55\u786e\u5b9a\u4e3a\u5b83\u670d\u52a1\u7684Coordinator\u5728\u54ea\u53f0Broker\u4e0a\u5462?<\/p>\n<p>\u901a\u8fc7Kafka\u5185\u90e8\u4e3b\u9898__consumer_offsets\u3002<\/p>\n<p>\u76ee\u524d\uff0cKafka\u4e3a\u67d0\u4e2aConsumer Group\u786e\u5b9aCoordinator\u6240\u5728\u7684Broker\u7684\u7b97\u6cd5\u67092\u4e2a\u6b65\u9aa4\u3002<\/p>\n<ul>\n<li>\u7b2c1\u6b65\uff1a\u786e\u5b9a\u7531__consumer_offsets\u4e3b\u9898\u7684\u54ea\u4e2a\u5206\u533a\u6765\u4fdd\u5b58\u8be5Group\u6570\u636e\uff1apartitionId=Math.abs(groupId.hashCode() % offsetsTopicPartitionCount)\u3002<\/li>\n<li>\u7b2c2\u6b65\uff1a\u627e\u51fa\u8be5\u5206\u533aLeader\u526f\u672c\u6240\u5728\u7684Broker\uff0c\u8be5Broker\u5373\u4e3a\u5bf9\u5e94\u7684Coordinator\u3002<\/li>\n<\/ul>\n<p>\u9996\u5148\uff0cKafka\u4f1a\u8ba1\u7b97\u8be5Group\u7684group.id\u53c2\u6570\u7684\u54c8\u5e0c\u503c\u3002<\/p>\n<p>\u6bd4\u5982\u4f60\u6709\u4e2aGroup\u7684group.id\u8bbe\u7f6e\u6210\u4e86test-group\uff0c\u90a3\u4e48\u5b83\u7684hashCode\u503c\u5c31\u5e94\u8be5\u662f627841412\u3002<\/p>\n<p>\u5176\u6b21\uff0cKafka\u4f1a\u8ba1\u7b97__consumer_offsets\u7684\u5206\u533a\u6570\uff0c\u901a\u5e38\u662f50\u4e2a\u5206\u533a\uff0c\u4e4b\u540e\u5c06\u521a\u624d\u90a3\u4e2a\u54c8\u5e0c\u503c\u5bf9\u5206\u533a\u6570\u8fdb\u884c\u53d6\u6a21\u52a0\u6c42\u7edd\u5bf9\u503c\u8ba1\u7b97\uff0c\u5373abs(627841412 % 50) = 12\u3002<\/p>\n<p>\u6b64\u65f6\uff0c\u6211\u4eec\u5c31\u77e5\u9053\u4e86__consumer_offsets\u4e3b\u9898\u7684\u5206\u533a12\u8d1f\u8d23\u4fdd\u5b58\u8fd9\u4e2aGroup\u7684\u6570\u636e\u3002<\/p>\n<p>\u6709\u4e86\u5206\u533a\u53f7\uff0c\u6211\u4eec\u53ea\u9700\u8981\u627e\u51fa__consumer_offsets\u4e3b\u9898\u5206\u533a12\u7684Leader\u526f\u672c\u5728\u54ea\u4e2aBroker\u4e0a\u5c31\u53ef\u4ee5\u4e86\uff0c\u8fd9\u4e2aBroker\uff0c\u5c31\u662f\u6211\u4eec\u8981\u627e\u7684Coordinator\u3002<\/p>\n<h3>\u300c\u6d88\u8d39\u8005\u4f4d\u79fb\uff1aConsumer Offset\u300d<\/h3>\n<p>\u6d88\u8d39\u8005\u6d88\u8d39\u8fdb\u5ea6\uff0c\u6bcf\u4e2a\u6d88\u8d39\u8005\u90fd\u6709\u81ea\u5df1\u7684\u6d88\u8d39\u8005\u4f4d\u79fb\u3002<\/p>\n<h3>\u300c\u91cd\u5e73\u8861\uff1aRebalance\u300d<\/h3>\n<p>\u6d88\u8d39\u8005\u7ec4\u5185\u67d0\u4e2a\u6d88\u8d39\u8005\u5b9e\u4f8b\u6302\u6389\u540e\uff0c\u5176\u4ed6\u6d88\u8d39\u8005\u5b9e\u4f8b\u81ea\u52a8\u91cd\u65b0\u5206\u914d\u8ba2\u9605\u4e3b\u9898\u5206\u533a\u7684\u8fc7\u7a0b\u3002<\/p>\n<p>Rebalance\u662fKafka\u6d88\u8d39\u8005\u7aef\u5b9e\u73b0\u9ad8\u53ef\u7528\u7684\u91cd\u8981\u624b\u6bb5\u3002<\/p>\n<p><strong>\u300cAR(Assigned Replicas)\u300d<\/strong>\uff1a\u5206\u533a\u4e2d\u7684\u6240\u6709\u526f\u672c\u7edf\u79f0\u4e3aAR\u3002<\/p>\n<p>\u6240\u6709\u6d88\u606f\u4f1a\u5148\u53d1\u9001\u5230leader\u526f\u672c\uff0c\u7136\u540efollower\u526f\u672c\u624d\u80fd\u4eceleader\u4e2d\u62c9\u53d6\u6d88\u606f\u8fdb\u884c\u540c\u6b65\u3002<\/p>\n<p>\u4f46\u662f\u5728\u540c\u6b65\u671f\u95f4\uff0cfollower\u5bf9\u4e8eleader\u800c\u8a00\u4f1a\u6709\u4e00\u5b9a\u7a0b\u5ea6\u7684\u6ede\u540e\uff0c\u8fd9\u4e2a\u65f6\u5019follower\u548cleader\u5e76\u975e\u5b8c\u5168\u540c\u6b65\u72b6\u6001<\/p>\n<p><strong>\u300cOSR(Out Sync Replicas)\u300d<\/strong>\uff1afollower\u526f\u672c\u4e0eleader\u526f\u672c\u6ca1\u6709\u5b8c\u5168\u540c\u6b65\u6216\u6ede\u540e\u7684\u526f\u672c\u96c6\u5408<\/p>\n<p><strong>\u300cISR(In Sync Replicas)<\/strong>\uff1a\u300cAR\u4e2d\u7684\u4e00\u4e2a\u5b50\u96c6\uff0cISR\u4e2d\u7684\u526f\u672c\u90fd\u300d\u662f\u4e0eleader\u4fdd\u6301\u5b8c\u5168\u540c\u6b65\u7684\u526f\u672c\u300d\uff0c\u5982\u679c\u67d0\u4e2a\u5728ISR\u4e2d\u7684follower\u526f\u672c\u843d\u540e\u4e8eleader\u526f\u672c\u592a\u591a\uff0c\u5219\u4f1a\u88ab\u4eceISR\u4e2d\u79fb\u9664\uff0c\u5426\u5219\u5982\u679c\u5b8c\u5168\u540c\u6b65\uff0c\u4f1a\u4eceOSR\u4e2d\u79fb\u81f3ISR\u96c6\u5408\u3002<\/p>\n<p>\u5728\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u5f53leader\u526f\u672c\u53d1\u751f\u6545\u969c\u65f6\uff0c\u53ea\u6709\u5728ISR\u96c6\u5408\u4e2d\u7684follower\u526f\u672c\u624d\u6709\u8d44\u683c\u88ab\u9009\u4e3e\u4e3a\u65b0leader\uff0c\u800cOSR\u4e2d\u7684\u526f\u672c\u6ca1\u6709\u673a\u4f1a(\u53ef\u4ee5\u901a\u8fc7unclean.leader.election.enable\u8fdb\u884c\u914d\u7f6e)<\/p>\n<p><strong>\u300cHW(High Watermark)<\/strong>\u300d\uff1a\u9ad8\u6c34\u4f4d\uff0c\u5b83\u6807\u8bc6\u4e86\u4e00\u4e2a\u7279\u5b9a\u7684\u6d88\u606f\u504f\u79fb\u91cf(offset)\uff0c\u6d88\u8d39\u8005\u53ea\u80fd\u62c9\u53d6\u5230\u8fd9\u4e2a\u6c34\u4f4d offset \u4e4b\u524d\u7684\u6d88\u606f<\/p>\n<p>\u4e0b\u56fe\u8868\u793a\u4e00\u4e2a\u65e5\u5fd7\u6587\u4ef6\uff0c\u8fd9\u4e2a\u65e5\u5fd7\u6587\u4ef6\u4e2d\u53ea\u67099\u6761\u6d88\u606f\uff0c\u7b2c\u4e00\u6761\u6d88\u606f\u7684offset(LogStartOffset)\u4e3a0\uff0c\u6700\u6709\u4e00\u6761\u6d88\u606f\u7684offset\u4e3a8\uff0coffset\u4e3a9\u7684\u6d88\u606f\u4f7f\u7528\u865a\u7ebf\u8868\u793a\u7684\uff0c\u4ee3\u8868\u4e0b\u4e00\u6761\u5f85\u5199\u5165\u7684\u6d88\u606f\u3002<\/p>\n<p>\u65e5\u5fd7\u6587\u4ef6\u7684 HW \u4e3a6\uff0c\u8868\u793a\u6d88\u8d39\u8005\u53ea\u80fd\u62c9\u53d6offset\u5728 0 \u5230 5 \u4e4b\u95f4\u7684\u6d88\u606f\uff0coffset\u4e3a6\u7684\u6d88\u606f\u5bf9\u6d88\u8d39\u8005\u800c\u8a00\u662f\u4e0d\u53ef\u89c1\u7684\u3002<\/p>\n<p style=\"text-align: center\">\n<p><strong>\u300cLEO(Log End Offset)\u300d<\/strong>\uff1a\u6807\u8bc6\u5f53\u524d\u65e5\u5fd7\u6587\u4ef6\u4e2d\u4e0b\u4e00\u6761\u5f85\u5199\u5165\u7684\u6d88\u606f\u7684offset<\/p>\n<p>\u4e0a\u56fe\u4e2doffset\u4e3a9\u7684\u4f4d\u7f6e\u5373\u4e3a\u5f53\u524d\u65e5\u5fd7\u6587\u4ef6\u7684 LEO\uff0cLEO \u7684\u5927\u5c0f\u76f8\u5f53\u4e8e\u5f53\u524d\u65e5\u5fd7\u5206\u533a\u4e2d\u6700\u540e\u4e00\u6761\u6d88\u606f\u7684offset\u503c\u52a01<\/p>\n<p>\u5206\u533a ISR \u96c6\u5408\u4e2d\u7684\u6bcf\u4e2a\u526f\u672c\u90fd\u4f1a\u7ef4\u62a4\u81ea\u8eab\u7684 LEO \uff0c\u800c ISR \u96c6\u5408\u4e2d\u6700\u5c0f\u7684 LEO \u5373\u4e3a\u5206\u533a\u7684 HW\uff0c\u5bf9\u6d88\u8d39\u8005\u800c\u8a00\u53ea\u80fd\u6d88\u8d39 HW \u4e4b\u524d\u7684\u6d88\u606f\u3002<\/p>\n<h2>\u7cfb\u7edf\u67b6\u6784<\/h2>\n<h3>\u300ckafka\u8bbe\u8ba1\u601d\u60f3\u300d<\/h3>\n<p>\u4e00\u4e2a\u6700\u57fa\u672c\u7684\u67b6\u6784\u662f\u751f\u4ea7\u8005\u53d1\u5e03\u4e00\u4e2a\u6d88\u606f\u5230Kafka\u7684\u4e00\u4e2aTopic \uff0c\u8be5Topic\u7684\u6d88\u606f\u5b58\u653e\u4e8e\u7684Broker\u4e2d\uff0c\u6d88\u8d39\u8005\u8ba2\u9605\u8fd9\u4e2aTopic\uff0c\u7136\u540e\u4eceBroker\u4e2d\u6d88\u8d39\u6d88\u606f\uff0c\u4e0b\u9762\u8fd9\u4e2a\u56fe\u53ef\u4ee5\u66f4\u76f4\u89c2\u7684\u63cf\u8ff0\u8fd9\u4e2a\u573a\u666f\uff1a<\/p>\n<p style=\"text-align: center\">\n<p><strong>\u300c\u6d88\u606f\u72b6\u6001\uff1a<\/strong>\u300d \u5728Kafka\u4e2d\uff0c\u6d88\u606f\u662f\u5426\u88ab\u6d88\u8d39\u7684\u72b6\u6001\u4fdd\u5b58\u5728Consumer\u4e2d\uff0cBroker\u4e0d\u4f1a\u5173\u5fc3\u6d88\u606f\u662f\u5426\u88ab\u6d88\u8d39\u6216\u88ab\u8c01\u6d88\u8d39\uff0cConsumer\u4f1a\u8bb0\u5f55\u4e00\u4e2aoffset\u503c(\u6307\u5411partition\u4e2d\u4e0b\u4e00\u6761\u5c06\u8981\u88ab\u6d88\u8d39\u7684\u6d88\u606f\u4f4d\u7f6e)\uff0c\u5982\u679coffset\u88ab\u9519\u8bef\u8bbe\u7f6e\u53ef\u80fd\u5bfc\u81f4\u540c\u4e00\u6761\u6d88\u606f\u88ab\u591a\u6b21\u6d88\u8d39\u6216\u8005\u6d88\u606f\u4e22\u5931\u3002<\/p>\n<p><strong>\u300c\u6d88\u606f\u6301\u4e45\u5316\uff1a\u300d<\/strong> Kafka\u4f1a\u628a\u6d88\u606f\u6301\u4e45\u5316\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u5e76\u4e14\u5177\u6709\u6781\u9ad8\u7684\u6027\u80fd\u3002<\/p>\n<p><strong>\u300c\u6279\u91cf\u53d1\u9001\uff1a\u300d<\/strong> Kafka\u652f\u6301\u4ee5\u6d88\u606f\u96c6\u5408\u4e3a\u5355\u4f4d\u8fdb\u884c\u6279\u91cf\u53d1\u9001\uff0c\u4ee5\u63d0\u9ad8\u6548\u7387\u3002<\/p>\n<p><strong>\u300cPush-and-Pull\uff1a<\/strong><strong>\u300d<\/strong> Kafka\u4e2d\u7684Producer\u548cConsumer\u91c7\u7528\u7684\u662fPush-and-Pull\u6a21\u5f0f\uff0c\u5373Producer\u5411Broker Push\u6d88\u606f\uff0cConsumer\u4eceBroker Pull\u6d88\u606f\u3002<\/p>\n<p><strong>\u300c\u5206\u533a\u673a\u5236(Partition)\uff1a\u300d<\/strong> Kafka\u7684Broker\u7aef\u652f\u6301\u6d88\u606f\u5206\u533a\uff0cProducer\u53ef\u4ee5\u51b3\u5b9a\u628a\u6d88\u606f\u53d1\u5230\u54ea\u4e2aPartition\uff0c\u5728\u4e00\u4e2aPartition\u4e2d\u6d88\u606f\u7684\u987a\u5e8f\u5c31\u662fProducer\u53d1\u9001\u6d88\u606f\u7684\u987a\u5e8f\uff0c\u4e00\u4e2aTopic\u4e2d\u7684Partition\u6570\u662f\u53ef\u914d\u7f6e\u7684\uff0cPartition\u662fKafka\u9ad8\u541e\u5410\u91cf\u7684\u91cd\u8981\u4fdd\u8bc1\u3002<\/p>\n<h3>\u300c\u7cfb\u7edf\u67b6\u6784\u300d<\/h3>\n<p style=\"text-align: center\">\n<p>\u901a\u5e38\u60c5\u51b5\u4e0b\uff0c\u4e00\u4e2akafka\u4f53\u7cfb\u67b6\u6784\u5305\u62ec\u300c\u591a\u4e2aProducer\u300d\u3001\u300c\u591a\u4e2aConsumer\u300d\u3001\u300c\u591a\u4e2abroker\u300d\u4ee5\u53ca\u300c\u4e00\u4e2aZookeeper\u96c6\u7fa4\u300d\u3002<\/p>\n<p><strong>\u300cProducer\u300d<\/strong>\uff1a\u751f\u4ea7\u8005\uff0c\u8d1f\u8d23\u5c06\u6d88\u606f\u53d1\u9001\u5230kafka\u4e2d\u3002<\/p>\n<p><strong>\u300cConsumer\u300d<\/strong>\uff1a\u6d88\u8d39\u8005\uff0c\u8d1f\u8d23\u4ecekafka\u4e2d\u62c9\u53d6\u6d88\u606f\u8fdb\u884c\u6d88\u8d39\u3002<\/p>\n<p><strong>\u300cBroker\u300d<\/strong>\uff1aKafka\u670d\u52a1\u8282\u70b9\uff0c\u4e00\u4e2a\u6216\u591a\u4e2aBroker\u7ec4\u6210\u4e86\u4e00\u4e2aKafka\u96c6\u7fa4<\/p>\n<p><strong>\u300cZookeeper\u96c6\u7fa4\u300d<\/strong>\uff1a\u8d1f\u8d23\u7ba1\u7406kafka\u96c6\u7fa4\u5143\u6570\u636e\u4ee5\u53ca\u63a7\u5236\u5668\u9009\u4e3e\u7b49\u3002<\/p>\n<h2>\u751f\u4ea7\u8005\u5206\u533a<\/h2>\n<h3>\u300c\u4e3a\u4ec0\u4e48\u5206\u533a?\u300d<\/h3>\n<p>Kafka\u7684\u6d88\u606f\u7ec4\u7ec7\u65b9\u5f0f\u5b9e\u9645\u4e0a\u662f\u4e09\u7ea7\u7ed3\u6784\uff1a\u4e3b\u9898-\u5206\u533a-\u6d88\u606f\u3002<\/p>\n<p>\u4e3b\u9898\u4e0b\u7684\u6bcf\u6761\u6d88\u606f\u53ea\u4f1a\u4fdd\u5b58\u5728\u67d0\u4e00\u4e2a\u5206\u533a\u4e2d\uff0c\u800c\u4e0d\u4f1a\u5728\u591a\u4e2a\u5206\u533a\u4e2d\u88ab\u4fdd\u5b58\u591a\u4efd\u3002<\/p>\n<p>\u5176\u5b9e\u5206\u533a\u7684\u4f5c\u7528\u5c31\u662f\u63d0\u4f9b\u8d1f\u8f7d\u5747\u8861\u7684\u80fd\u529b\uff0c\u6216\u8005\u8bf4\u5bf9\u6570\u636e\u8fdb\u884c\u5206\u533a\u7684\u4e3b\u8981\u539f\u56e0\uff0c\u5c31\u662f\u4e3a\u4e86\u5b9e\u73b0\u7cfb\u7edf\u7684\u9ad8\u4f38\u7f29\u6027(Scalability)\u3002<\/p>\n<p>\u4e0d\u540c\u7684\u5206\u533a\u80fd\u591f\u88ab\u653e\u7f6e\u5230\u4e0d\u540c\u8282\u70b9\u7684\u673a\u5668\u4e0a\uff0c\u800c\u6570\u636e\u7684\u8bfb\u5199\u64cd\u4f5c\u4e5f\u90fd\u662f\u9488\u5bf9\u5206\u533a\u8fd9\u4e2a\u7c92\u5ea6\u800c\u8fdb\u884c\u7684\uff0c\u8fd9\u6837\u6bcf\u4e2a\u8282\u70b9\u7684\u673a\u5668\u90fd\u80fd\u72ec\u7acb\u5730\u6267\u884c\u5404\u81ea\u5206\u533a\u7684\u8bfb\u5199\u8bf7\u6c42\u5904\u7406\uff0c\u5e76\u4e14\uff0c\u6211\u4eec\u8fd8\u53ef\u4ee5\u901a\u8fc7\u6dfb\u52a0\u65b0\u7684\u8282\u70b9\u673a\u5668\u6765\u589e\u52a0\u6574\u4f53\u7cfb\u7edf\u7684\u541e\u5410\u91cf\u3002<\/p>\n<h3>\u300c\u90fd\u6709\u54ea\u4e9b\u5206\u533a\u7b56\u7565?\u300d<\/h3>\n<p>\u300c\u6240\u8c13\u5206\u533a\u7b56\u7565\u662f\u51b3\u5b9a\u751f\u4ea7\u8005\u5c06\u6d88\u606f\u53d1\u9001\u5230\u54ea\u4e2a\u5206\u533a\u7684\u7b97\u6cd5\u3002\u300d<\/p>\n<p>Kafka\u4e3a\u6211\u4eec\u63d0\u4f9b\u4e86\u9ed8\u8ba4\u7684\u5206\u533a\u7b56\u7565\uff0c\u540c\u65f6\u5b83\u4e5f\u652f\u6301\u4f60\u81ea\u5b9a\u4e49\u5206\u533a\u7b56\u7565\u3002<\/p>\n<h3>\u300c\u81ea\u5b9a\u4e49\u5206\u533a\u7b56\u7565\u300d<\/h3>\n<p>\u5982\u679c\u8981\u81ea\u5b9a\u4e49\u5206\u533a\u7b56\u7565\uff0c\u4f60\u9700\u8981\u663e\u5f0f\u5730\u914d\u7f6e\u751f\u4ea7\u8005\u7aef\u7684\u53c2\u6570partitioner.class\u3002<\/p>\n<p>\u5728\u7f16\u5199\u751f\u4ea7\u8005\u7a0b\u5e8f\u65f6\uff0c\u4f60\u53ef\u4ee5\u7f16\u5199\u4e00\u4e2a\u5177\u4f53\u7684\u7c7b\u5b9e\u73b0org.apache.kafka.clients.producer.Partitioner\u63a5\u53e3\u3002<\/p>\n<p>\u8fd9\u4e2a\u63a5\u53e3\u4e5f\u5f88\u7b80\u5355\uff0c\u53ea\u5b9a\u4e49\u4e86\u4e24\u4e2a\u65b9\u6cd5\uff1apartition()\u548cclose()\uff0c\u901a\u5e38\u4f60\u53ea\u9700\u8981\u5b9e\u73b0\u6700\u91cd\u8981\u7684partition\u65b9\u6cd5\u3002<\/p>\n<p>\u6211\u4eec\u6765\u770b\u770b\u8fd9\u4e2a\u65b9\u6cd5\u7684\u65b9\u6cd5\u7b7e\u540d\uff1a<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>int<\/span><span>&nbsp;partition(String&nbsp;topic,&nbsp;Object&nbsp;<\/span><span>key<\/span><span>,&nbsp;byte[]&nbsp;keyBytes,&nbsp;Object&nbsp;value,&nbsp;byte[]&nbsp;valueBytes,&nbsp;Cluster&nbsp;cluster);&nbsp;<\/span><\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>\u8fd9\u91cc\u7684topic\u3001key\u3001keyBytes\u3001value\u548cvalueBytes\u90fd\u5c5e\u4e8e\u6d88\u606f\u6570\u636e\uff0ccluster\u5219\u662f\u96c6\u7fa4\u4fe1\u606f(\u6bd4\u5982\u5f53\u524dKafka\u96c6\u7fa4\u5171\u6709\u591a\u5c11\u4e3b\u9898\u3001\u591a\u5c11Broker\u7b49)\u3002<\/p>\n<p>Kafka\u7ed9\u4f60\u8fd9\u4e48\u591a\u4fe1\u606f\uff0c\u5c31\u662f\u5e0c\u671b\u8ba9\u4f60\u80fd\u591f\u5145\u5206\u5730\u5229\u7528\u8fd9\u4e9b\u4fe1\u606f\u5bf9\u6d88\u606f\u8fdb\u884c\u5206\u533a\uff0c\u8ba1\u7b97\u51fa\u5b83\u8981\u88ab\u53d1\u9001\u5230\u54ea\u4e2a\u5206\u533a\u4e2d\u3002<\/p>\n<p>\u53ea\u8981\u4f60\u81ea\u5df1\u7684\u5b9e\u73b0\u7c7b\u5b9a\u4e49\u597d\u4e86partition\u65b9\u6cd5\uff0c\u540c\u65f6\u8bbe\u7f6epartitioner.class\u53c2\u6570\u4e3a\u4f60\u81ea\u5df1\u5b9e\u73b0\u7c7b\u7684Full Qualified Name\uff0c\u90a3\u4e48\u751f\u4ea7\u8005\u7a0b\u5e8f\u5c31\u4f1a\u6309\u7167\u4f60\u7684\u4ee3\u7801\u903b\u8f91\u5bf9\u6d88\u606f\u8fdb\u884c\u5206\u533a\u3002<\/p>\n<h3>\u300c\u8f6e\u8be2\u7b56\u7565\u300d<\/h3>\n<p>\u4e5f\u79f0Round-robin\u7b56\u7565\uff0c\u5373\u987a\u5e8f\u5206\u914d\u3002<\/p>\n<p>\u6bd4\u5982\u4e00\u4e2a\u4e3b\u9898\u4e0b\u67093\u4e2a\u5206\u533a\uff0c\u90a3\u4e48\u7b2c\u4e00\u6761\u6d88\u606f\u88ab\u53d1\u9001\u5230\u5206\u533a0\uff0c\u7b2c\u4e8c\u6761\u88ab\u53d1\u9001\u5230\u5206\u533a1\uff0c\u7b2c\u4e09\u6761\u88ab\u53d1\u9001\u5230\u5206\u533a2\uff0c\u4ee5\u6b64\u7c7b\u63a8\u3002\u5f53\u751f\u4ea7\u7b2c4\u6761\u6d88\u606f\u65f6\u53c8\u4f1a\u91cd\u65b0\u5f00\u59cb\uff0c\u5373\u5c06\u5176\u5206\u914d\u5230\u5206\u533a0<\/p>\n<p>\u8fd9\u5c31\u662f\u6240\u8c13\u7684\u8f6e\u8be2\u7b56\u7565\u3002\u8f6e\u8be2\u7b56\u7565\u662fKafka Java\u751f\u4ea7\u8005API\u9ed8\u8ba4\u63d0\u4f9b\u7684\u5206\u533a\u7b56\u7565\u3002<\/p>\n<p>\u300c\u8f6e\u8be2\u7b56\u7565\u6709\u975e\u5e38\u4f18\u79c0\u7684\u8d1f\u8f7d\u5747\u8861\u8868\u73b0\uff0c\u5b83\u603b\u662f\u80fd\u4fdd\u8bc1\u6d88\u606f\u6700\u5927\u9650\u5ea6\u5730\u88ab\u5e73\u5747\u5206\u914d\u5230\u6240\u6709\u5206\u533a\u4e0a\uff0c\u6545\u9ed8\u8ba4\u60c5\u51b5\u4e0b\u5b83\u662f\u6700\u5408\u7406\u7684\u5206\u533a\u7b56\u7565\uff0c\u4e5f\u662f\u6211\u4eec\u6700\u5e38\u7528\u7684\u5206\u533a\u7b56\u7565\u4e4b\u4e00\u3002\u300d<\/p>\n<h3>\u300c\u968f\u673a\u7b56\u7565\u300d<\/h3>\n<p>\u4e5f\u79f0Randomness\u7b56\u7565\u3002\u6240\u8c13\u968f\u673a\u5c31\u662f\u6211\u4eec\u968f\u610f\u5730\u5c06\u6d88\u606f\u653e\u7f6e\u5230\u4efb\u610f\u4e00\u4e2a\u5206\u533a\u4e0a\u3002<\/p>\n<p>\u5982\u679c\u8981\u5b9e\u73b0\u968f\u673a\u7b56\u7565\u7248\u7684partition\u65b9\u6cd5\uff0c\u5f88\u7b80\u5355\uff0c\u53ea\u9700\u8981\u4e24\u884c\u4ee3\u7801\u5373\u53ef\uff1a<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>List&nbsp;partitions&nbsp;=&nbsp;cluster.partitionsForTopic(topic);&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span><span>return<\/span><span>&nbsp;ThreadLocalRandom.<\/span><span>current<\/span><span>().nextInt(partitions.<\/span><span>size<\/span><span>());&nbsp;<\/span><\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>\u5148\u8ba1\u7b97\u51fa\u8be5\u4e3b\u9898\u603b\u7684\u5206\u533a\u6570\uff0c\u7136\u540e\u968f\u673a\u5730\u8fd4\u56de\u4e00\u4e2a\u5c0f\u4e8e\u5b83\u7684\u6b63\u6574\u6570\u3002<\/p>\n<p>\u672c\u8d28\u4e0a\u770b\u968f\u673a\u7b56\u7565\u4e5f\u662f\u529b\u6c42\u5c06\u6570\u636e\u5747\u5300\u5730\u6253\u6563\u5230\u5404\u4e2a\u5206\u533a\uff0c\u4f46\u4ece\u5b9e\u9645\u8868\u73b0\u6765\u770b\uff0c\u5b83\u8981\u900a\u4e8e\u8f6e\u8be2\u7b56\u7565\uff0c\u6240\u4ee5\u300c\u5982\u679c\u8ffd\u6c42\u6570\u636e\u7684\u5747\u5300\u5206\u5e03\uff0c\u8fd8\u662f\u4f7f\u7528\u8f6e\u8be2\u7b56\u7565\u6bd4\u8f83\u597d\u300d\u3002\u4e8b\u5b9e\u4e0a\uff0c\u968f\u673a\u7b56\u7565\u662f\u8001\u7248\u672c\u751f\u4ea7\u8005\u4f7f\u7528\u7684\u5206\u533a\u7b56\u7565\uff0c\u5728\u65b0\u7248\u672c\u4e2d\u5df2\u7ecf\u6539\u4e3a\u8f6e\u8be2\u4e86\u3002<\/p>\n<h3>\u300c\u6309\u6d88\u606f\u952e\u4fdd\u5e8f\u7b56\u7565\u300d<\/h3>\n<p>Kafka\u5141\u8bb8\u4e3a\u6bcf\u6761\u6d88\u606f\u5b9a\u4e49\u6d88\u606f\u952e\uff0c\u7b80\u79f0\u4e3aKey\u3002<\/p>\n<p>\u8fd9\u4e2aKey\u7684\u4f5c\u7528\u975e\u5e38\u5927\uff0c\u5b83\u53ef\u4ee5\u662f\u4e00\u4e2a\u6709\u7740\u660e\u786e\u4e1a\u52a1\u542b\u4e49\u7684\u5b57\u7b26\u4e32\uff0c\u6bd4\u5982\u5ba2\u6237\u4ee3\u7801\u3001\u90e8\u95e8\u7f16\u53f7\u6216\u662f\u4e1a\u52a1ID\u7b49;\u4e5f\u53ef\u4ee5\u7528\u6765\u8868\u5f81\u6d88\u606f\u5143\u6570\u636e\u3002<\/p>\n<p>\u7279\u522b\u662f\u5728Kafka\u4e0d\u652f\u6301\u65f6\u95f4\u6233\u7684\u5e74\u4ee3\uff0c\u5728\u4e00\u4e9b\u573a\u666f\u4e2d\uff0c\u5de5\u7a0b\u5e08\u4eec\u90fd\u662f\u76f4\u63a5\u5c06\u6d88\u606f\u521b\u5efa\u65f6\u95f4\u5c01\u88c5\u8fdbKey\u91cc\u9762\u7684\u3002<\/p>\n<p>\u4e00\u65e6\u6d88\u606f\u88ab\u5b9a\u4e49\u4e86Key\uff0c\u90a3\u4e48\u4f60\u5c31\u53ef\u4ee5\u4fdd\u8bc1\u540c\u4e00\u4e2aKey\u7684\u6240\u6709\u6d88\u606f\u90fd\u8fdb\u5165\u5230\u76f8\u540c\u7684\u5206\u533a\u91cc\u9762\uff0c\u7531\u4e8e\u6bcf\u4e2a\u5206\u533a\u4e0b\u7684\u6d88\u606f\u5904\u7406\u90fd\u662f\u6709\u987a\u5e8f\u7684\uff0c\u6545\u8fd9\u4e2a\u7b56\u7565\u88ab\u79f0\u4e3a\u6309\u6d88\u606f\u952e\u4fdd\u5e8f\u7b56\u7565<\/p>\n<p>\u5b9e\u73b0\u8fd9\u4e2a\u7b56\u7565\u7684partition\u65b9\u6cd5\u540c\u6837\u7b80\u5355\uff0c\u53ea\u9700\u8981\u4e0b\u9762\u4e24\u884c\u4ee3\u7801\u5373\u53ef\uff1a<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>List&nbsp;partitions&nbsp;=&nbsp;cluster.partitionsForTopic(topic);&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span><span>return<\/span><span>&nbsp;Math.<\/span><span>abs<\/span><span>(<\/span><span>key<\/span><span>.hashCode())&nbsp;%&nbsp;partitions.<\/span><span>size<\/span><span>();&nbsp;<\/span><\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>\u524d\u9762\u63d0\u5230\u7684Kafka\u9ed8\u8ba4\u5206\u533a\u7b56\u7565\u5b9e\u9645\u4e0a\u540c\u65f6\u5b9e\u73b0\u4e86\u4e24\u79cd\u7b56\u7565\uff1a\u5982\u679c\u6307\u5b9a\u4e86Key\uff0c\u90a3\u4e48\u9ed8\u8ba4\u5b9e\u73b0\u6309\u6d88\u606f\u952e\u4fdd\u5e8f\u7b56\u7565;\u5982\u679c\u6ca1\u6709\u6307\u5b9aKey\uff0c\u5219\u4f7f\u7528\u8f6e\u8be2\u7b56\u7565\u3002<\/p>\n<h3>\u300c\u5176\u4ed6\u5206\u533a\u7b56\u7565\u300d<\/h3>\n<p>\u5176\u5b9e\u8fd8\u6709\u4e00\u79cd\u6bd4\u8f83\u5e38\u89c1\u7684\uff0c\u5373\u6240\u8c13\u7684\u57fa\u4e8e\u5730\u7406\u4f4d\u7f6e\u7684\u5206\u533a\u7b56\u7565\u3002<\/p>\n<p>\u5f53\u7136\u8fd9\u79cd\u7b56\u7565\u4e00\u822c\u53ea\u9488\u5bf9\u90a3\u4e9b\u5927\u89c4\u6a21\u7684Kafka\u96c6\u7fa4\uff0c\u7279\u522b\u662f\u8de8\u57ce\u5e02\u3001\u8de8\u56fd\u5bb6\u751a\u81f3\u662f\u8de8\u5927\u6d32\u7684\u96c6\u7fa4\u3002<\/p>\n<p>\u6211\u4eec\u53ef\u4ee5\u6839\u636eBroker\u6240\u5728\u7684IP\u5730\u5740\u5b9e\u73b0\u5b9a\u5236\u5316\u7684\u5206\u533a\u7b56\u7565\u3002\u6bd4\u5982\u4e0b\u9762\u8fd9\u6bb5\u4ee3\u7801\uff1a<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>List&nbsp;partitions&nbsp;=&nbsp;cluster.partitionsForTopic(topic);&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span><span>return<\/span><span>&nbsp;partitions.stream().filter(p&nbsp;-&gt;&nbsp;isSouth(p.leader().host())).map(PartitionInfo::partition).findAny().get();&nbsp;<\/span><\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>\u6211\u4eec\u53ef\u4ee5\u4ece\u6240\u6709\u5206\u533a\u4e2d\u627e\u51fa\u90a3\u4e9bLeader\u526f\u672c\u5728\u5357\u65b9\u7684\u6240\u6709\u5206\u533a\uff0c\u7136\u540e\u968f\u673a\u6311\u9009\u4e00\u4e2a\u8fdb\u884c\u6d88\u606f\u53d1\u9001\u3002<\/p>\n<h2>\u751f\u4ea7\u8005\u538b\u7f29\u7b97\u6cd5<\/h2>\n<h3>\u300cKafka\u662f\u5982\u4f55\u538b\u7f29\u6d88\u606f\u7684\u5462?\u300d<\/h3>\n<p>\u76ee\u524dKafka\u5171\u6709\u4e24\u5927\u7c7b\u6d88\u606f\u683c\u5f0f\uff0c\u793e\u533a\u5206\u522b\u79f0\u4e4b\u4e3aV1\u7248\u672c\u548cV2\u7248\u672c\u3002<\/p>\n<p>V2\u7248\u672c\u662fKafka 0.11.0.0\u4e2d\u6b63\u5f0f\u5f15\u5165\u7684\u3002<\/p>\n<p>\u4e0d\u8bba\u662f\u54ea\u4e2a\u7248\u672c\uff0cKafka\u7684\u6d88\u606f\u5c42\u6b21\u90fd\u5206\u4e3a\u4e24\u5c42\uff1a\u6d88\u606f\u96c6\u5408\u4ee5\u53ca\u6d88\u606f\u3002<\/p>\n<p>\u4e00\u4e2a\u6d88\u606f\u96c6\u5408\u4e2d\u5305\u542b\u82e5\u5e72\u6761\u65e5\u5fd7\u9879\uff0c\u800c\u65e5\u5fd7\u9879\u624d\u662f\u771f\u6b63\u5c01\u88c5\u6d88\u606f\u7684\u5730\u65b9\u3002<\/p>\n<p>Kafka\u5e95\u5c42\u7684\u6d88\u606f\u65e5\u5fd7\u7531\u4e00\u7cfb\u5217\u6d88\u606f\u96c6\u5408\u65e5\u5fd7\u9879\u7ec4\u6210\u3002<\/p>\n<p>Kafka\u901a\u5e38\u4e0d\u4f1a\u76f4\u63a5\u64cd\u4f5c\u5177\u4f53\u7684\u4e00\u6761\u6761\u6d88\u606f\uff0c\u5b83\u603b\u662f\u5728\u6d88\u606f\u96c6\u5408\u8fd9\u4e2a\u5c42\u9762\u4e0a\u8fdb\u884c\u5199\u5165\u64cd\u4f5c\u3002<\/p>\n<h3>\u300c\u90a3\u4e48\u793e\u533a\u5f15\u5165V2\u7248\u672c\u7684\u76ee\u7684\u662f\u4ec0\u4e48\u5462?\u300d<\/h3>\n<p>V2\u7248\u672c\u4e3b\u8981\u662f\u9488\u5bf9V1\u7248\u672c\u7684\u4e00\u4e9b\u5f0a\u7aef\u505a\u4e86\u4fee\u6b63\uff0c\u6bd4\u5982\u628a\u6d88\u606f\u7684\u516c\u5171\u90e8\u5206\u62bd\u53d6\u51fa\u6765\u653e\u5230\u5916\u5c42\u6d88\u606f\u96c6\u5408\u91cc\u9762\uff0c\u8fd9\u6837\u5c31\u4e0d\u7528\u6bcf\u6761\u6d88\u606f\u90fd\u4fdd\u5b58\u8fd9\u4e9b\u4fe1\u606f\u4e86\u3002<\/p>\n<p>\u4e3e\u4e2a\u4f8b\u5b50\uff1a\u539f\u6765\u5728V1\u7248\u672c\u4e2d\uff0c\u6bcf\u6761\u6d88\u606f\u90fd\u9700\u8981\u6267\u884cCRC\u6821\u9a8c\uff0c\u4f46\u6709\u4e9b\u60c5\u51b5\u4e0b\u6d88\u606f\u7684CRC\u503c\u662f\u4f1a\u53d1\u751f\u53d8\u5316\u7684\u3002<\/p>\n<p>\u6bd4\u5982\u5728Broker\u7aef\u53ef\u80fd\u4f1a\u5bf9\u6d88\u606f\u65f6\u95f4\u6233\u5b57\u6bb5\u8fdb\u884c\u66f4\u65b0\uff0c\u90a3\u4e48\u91cd\u65b0\u8ba1\u7b97\u4e4b\u540e\u7684CRC\u503c\u4e5f\u4f1a\u76f8\u5e94\u66f4\u65b0;\u518d\u6bd4\u5982Broker\u7aef\u5728\u6267\u884c\u6d88\u606f\u683c\u5f0f\u8f6c\u6362\u65f6(\u4e3b\u8981\u662f\u4e3a\u4e86\u517c\u5bb9\u8001\u7248\u672c\u5ba2\u6237\u7aef\u7a0b\u5e8f)\uff0c\u4e5f\u4f1a\u5e26\u6765CRC\u503c\u7684\u53d8\u5316\u3002<\/p>\n<p>\u9274\u4e8e\u8fd9\u4e9b\u60c5\u51b5\uff0c\u518d\u5bf9\u6bcf\u6761\u6d88\u606f\u90fd\u6267\u884cCRC\u6821\u9a8c\u5c31\u6709\u70b9\u6ca1\u5fc5\u8981\u4e86\uff0c\u4e0d\u4ec5\u6d6a\u8d39\u7a7a\u95f4\u8fd8\u803d\u8befCPU\u65f6\u95f4\uff0c\u56e0\u6b64\u5728V2\u7248\u672c\u4e2d\uff0c\u6d88\u606f\u7684CRC\u6821\u9a8c\u5de5\u4f5c\u5c31\u88ab\u79fb\u5230\u4e86\u6d88\u606f\u96c6\u5408\u8fd9\u4e00\u5c42\u3002<\/p>\n<p>V2\u7248\u672c\u8fd8\u6709\u4e00\u4e2a\u548c\u538b\u7f29\u606f\u606f\u76f8\u5173\u7684\u6539\u8fdb\uff0c\u5c31\u662f\u4fdd\u5b58\u538b\u7f29\u6d88\u606f\u7684\u65b9\u6cd5\u53d1\u751f\u4e86\u53d8\u5316\u3002<\/p>\n<p>\u4e4b\u524dV1\u7248\u672c\u4e2d\u4fdd\u5b58\u538b\u7f29\u6d88\u606f\u7684\u65b9\u6cd5\u662f\u628a\u591a\u6761\u6d88\u606f\u8fdb\u884c\u538b\u7f29\u7136\u540e\u4fdd\u5b58\u5230\u5916\u5c42\u6d88\u606f\u7684\u6d88\u606f\u4f53\u5b57\u6bb5\u4e2d;\u800cV2\u7248\u672c\u7684\u505a\u6cd5\u662f\u5bf9\u6574\u4e2a\u6d88\u606f\u96c6\u5408\u8fdb\u884c\u538b\u7f29\uff0c\u663e\u7136\u540e\u8005\u5e94\u8be5\u6bd4\u524d\u8005\u6709\u66f4\u597d\u7684\u538b\u7f29\u6548\u679c\u3002<\/p>\n<h3>\u300c\u4f55\u65f6\u538b\u7f29?\u300d<\/h3>\n<p>\u5728Kafka\u4e2d\uff0c\u538b\u7f29\u53ef\u80fd\u53d1\u751f\u5728\u4e24\u4e2a\u5730\u65b9\uff1a\u751f\u4ea7\u8005\u7aef\u548cBroker\u7aef\u3002<\/p>\n<p>\u751f\u4ea7\u8005\u7a0b\u5e8f\u4e2d\u914d\u7f6ecompression.type\u53c2\u6570\u5373\u8868\u793a\u542f\u7528\u6307\u5b9a\u7c7b\u578b\u7684\u538b\u7f29\u7b97\u6cd5\u3002<\/p>\n<p>\u6bd4\u5982\u4e0b\u9762\u8fd9\u6bb5\u7a0b\u5e8f\u4ee3\u7801\u5c55\u793a\u4e86\u5982\u4f55\u6784\u5efa\u4e00\u4e2a\u5f00\u542fGZIP\u7684Producer\u5bf9\u8c61\uff1a<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>Properties&nbsp;props&nbsp;=&nbsp;new&nbsp;Properties();&nbsp;&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>props.put(<span>\"bootstrap.servers\"<\/span><span>,&nbsp;<\/span><span>\"localhost:9092\"<\/span><span>);&nbsp;&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>props.put(<span>\"acks\"<\/span><span>,&nbsp;<\/span><span>\"all\"<\/span><span>);&nbsp;&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>props.put(<span>\"key.serializer\"<\/span><span>,&nbsp;<\/span><span>\"org.apache.kafka.common.serialization.StringSerializer\"<\/span><span>);&nbsp;props.put(<\/span><span>\"value.serializer\"<\/span><span>,&nbsp;<\/span><span>\"org.apache.kafka.common.serialization.StringSerializer\"<\/span><span>);&nbsp;\/\/&nbsp;\u5f00\u542fGZIP\u538b\u7f29&nbsp;&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>props.put(<span>\"compression.type\"<\/span><span>,&nbsp;<\/span><span>\"gzip\"<\/span><span>);&nbsp;&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>Producer&nbsp;producer&nbsp;=&nbsp;new&nbsp;KafkaProducer&lt;&gt;(props);&nbsp;<\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>\u8fd9\u91cc\u6bd4\u8f83\u5173\u952e\u7684\u4ee3\u7801\u884c\u662fprops.put(\u201ccompression.type\u201d, \u201cgzip\u201d)\uff0c\u5b83\u8868\u660e\u8be5Producer\u7684\u538b\u7f29\u7b97\u6cd5\u4f7f\u7528\u7684\u662fGZIP\u3002<\/p>\n<p>\u8fd9\u6837Producer\u542f\u52a8\u540e\u751f\u4ea7\u7684\u6bcf\u4e2a\u6d88\u606f\u96c6\u5408\u90fd\u662f\u7ecfGZIP\u538b\u7f29\u8fc7\u7684\uff0c\u6545\u800c\u80fd\u5f88\u597d\u5730\u8282\u7701\u7f51\u7edc\u4f20\u8f93\u5e26\u5bbd\u4ee5\u53caKafka Broker\u7aef\u7684\u78c1\u76d8\u5360\u7528\u3002<\/p>\n<p>\u6709\u4e24\u79cd\u4f8b\u5916\u60c5\u51b5\u5c31\u53ef\u80fd\u8ba9Broker\u91cd\u65b0\u538b\u7f29\u6d88\u606f\uff1a<\/p>\n<h3>\u300c\u60c5\u51b5\u4e00\uff1aBroker\u7aef\u6307\u5b9a\u4e86\u548cProducer\u7aef\u4e0d\u540c\u7684\u538b\u7f29\u7b97\u6cd5\u3002\u300d<\/h3>\n<p>\u4e00\u65e6\u4f60\u5728Broker\u7aef\u8bbe\u7f6e\u4e86\u4e0d\u540c\u7684compression.type\u503c\uff0c\u5c31\u4e00\u5b9a\u8981\u5c0f\u5fc3\u4e86\uff0c\u56e0\u4e3a\u53ef\u80fd\u4f1a\u53d1\u751f\u9884\u6599\u4e4b\u5916\u7684\u538b\u7f29\/\u89e3\u538b\u7f29\u64cd\u4f5c\uff0c\u901a\u5e38\u8868\u73b0\u4e3aBroker\u7aefCPU\u4f7f\u7528\u7387\u98d9\u5347\u3002<\/p>\n<h3>\u300c\u60c5\u51b5\u4e8c\uff1aBroker\u7aef\u53d1\u751f\u4e86\u6d88\u606f\u683c\u5f0f\u8f6c\u6362\u3002\u300d<\/h3>\n<p>\u6240\u8c13\u7684\u6d88\u606f\u683c\u5f0f\u8f6c\u6362\u4e3b\u8981\u662f\u4e3a\u4e86\u517c\u5bb9\u8001\u7248\u672c\u7684\u6d88\u8d39\u8005\u7a0b\u5e8f\u3002<\/p>\n<p>\u5728\u4e00\u4e2a\u751f\u4ea7\u73af\u5883\u4e2d\uff0cKafka\u96c6\u7fa4\u4e2d\u540c\u65f6\u4fdd\u5b58\u591a\u79cd\u7248\u672c\u7684\u6d88\u606f\u683c\u5f0f\u975e\u5e38\u5e38\u89c1\u3002<\/p>\n<p>\u4e3a\u4e86\u517c\u5bb9\u8001\u7248\u672c\u7684\u683c\u5f0f\uff0cBroker\u7aef\u4f1a\u5bf9\u65b0\u7248\u672c\u6d88\u606f\u6267\u884c\u5411\u8001\u7248\u672c\u683c\u5f0f\u7684\u8f6c\u6362\u3002<\/p>\n<p>\u8fd9\u4e2a\u8fc7\u7a0b\u4e2d\u4f1a\u6d89\u53ca\u6d88\u606f\u7684\u89e3\u538b\u7f29\u548c\u91cd\u65b0\u538b\u7f29\u3002<\/p>\n<p>\u4e00\u822c\u60c5\u51b5\u4e0b\u8fd9\u79cd\u6d88\u606f\u683c\u5f0f\u8f6c\u6362\u5bf9\u6027\u80fd\u662f\u6709\u5f88\u5927\u5f71\u54cd\u7684\uff0c\u9664\u4e86\u8fd9\u91cc\u7684\u538b\u7f29\u4e4b\u5916\uff0c\u5b83\u8fd8\u8ba9Kafka\u4e27\u5931\u4e86Zero Copy\u7279\u6027\u3002<\/p>\n<h3>\u300c\u4f55\u65f6\u89e3\u538b\u7f29?\u300d<\/h3>\n<p>\u6709\u538b\u7f29\u5fc5\u6709\u89e3\u538b\u7f29!\u901a\u5e38\u6765\u8bf4\u89e3\u538b\u7f29\u53d1\u751f\u5728\u6d88\u8d39\u8005\u7a0b\u5e8f\u4e2d\uff0c\u4e5f\u5c31\u662f\u8bf4Producer\u53d1\u9001\u538b\u7f29\u6d88\u606f\u5230Broker\u540e\uff0cBroker\u7167\u5355\u5168\u6536\u5e76\u539f\u6837\u4fdd\u5b58\u8d77\u6765\u3002\u5f53Consumer\u7a0b\u5e8f\u8bf7\u6c42\u8fd9\u90e8\u5206\u6d88\u606f\u65f6\uff0cBroker\u4f9d\u7136\u539f\u6837\u53d1\u9001\u51fa\u53bb\uff0c\u5f53\u6d88\u606f\u5230\u8fbeConsumer\u7aef\u540e\uff0c\u7531Consumer\u81ea\u884c\u89e3\u538b\u7f29\u8fd8\u539f\u6210\u4e4b\u524d\u7684\u6d88\u606f\u3002<\/p>\n<h3>\u300c\u57fa\u672c\u8fc7\u7a0b\uff1aProducer\u7aef\u538b\u7f29\u3001Broker\u7aef\u4fdd\u6301\u3001Consumer\u7aef\u89e3\u538b\u7f29\u3002\u300d<\/h3>\n<p>\u6ce8\u610f\uff1a\u9664\u4e86\u5728Consumer\u7aef\u89e3\u538b\u7f29\uff0cBroker\u7aef\u4e5f\u4f1a\u8fdb\u884c\u89e3\u538b\u7f29\u3002<\/p>\n<p>\u6bcf\u4e2a\u538b\u7f29\u8fc7\u7684\u6d88\u606f\u96c6\u5408\u5728Broker\u7aef\u5199\u5165\u65f6\u90fd\u8981\u53d1\u751f\u89e3\u538b\u7f29\u64cd\u4f5c\uff0c\u76ee\u7684\u5c31\u662f\u4e3a\u4e86\u5bf9\u6d88\u606f\u6267\u884c\u5404\u79cd\u9a8c\u8bc1\u3002<\/p>\n<p>\u6211\u4eec\u5fc5\u987b\u627f\u8ba4\u8fd9\u79cd\u89e3\u538b\u7f29\u5bf9Broker\u7aef\u6027\u80fd\u662f\u6709\u4e00\u5b9a\u5f71\u54cd\u7684\uff0c\u7279\u522b\u662f\u5bf9CPU\u7684\u4f7f\u7528\u7387\u800c\u8a00\u3002<\/p>\n<h3>\u300c\u5404\u79cd\u538b\u7f29\u7b97\u6cd5\u5bf9\u6bd4\u300d<\/h3>\n<p>\u5728Kafka 2.1.0\u7248\u672c\u4e4b\u524d\uff0cKafka\u652f\u63013\u79cd\u538b\u7f29\u7b97\u6cd5\uff1aGZIP\u3001Snappy\u548cLZ4\u3002<\/p>\n<p>\u4ece2.1.0\u5f00\u59cb\uff0cKafka\u6b63\u5f0f\u652f\u6301Zstandard\u7b97\u6cd5(\u7b80\u5199\u4e3azstd)\u3002<\/p>\n<p>\u5b83\u662fFacebook\u5f00\u6e90\u7684\u4e00\u4e2a\u538b\u7f29\u7b97\u6cd5\uff0c\u80fd\u591f\u63d0\u4f9b\u8d85\u9ad8\u7684\u538b\u7f29\u6bd4\u3002<\/p>\n<p>\u5728\u5b9e\u9645\u4f7f\u7528\u4e2d\uff0cGZIP\u3001Snappy\u3001LZ4\u548czstd\u7684\u8868\u73b0\u5404\u6709\u5343\u79cb\u3002<\/p>\n<p>\u4f46\u5bf9\u4e8eKafka\u800c\u8a00\uff0c\u5728\u541e\u5410\u91cf\u65b9\u9762\uff1aLZ4 &gt; Snappy &gt; zstd\u548cGZIP;\u800c\u5728\u538b\u7f29\u6bd4\u65b9\u9762\uff0czstd &gt; LZ4 &gt; GZIP &gt; Snappy\u3002<\/p>\n<p>\u5177\u4f53\u5230\u7269\u7406\u8d44\u6e90\uff0c\u4f7f\u7528Snappy\u7b97\u6cd5\u5360\u7528\u7684\u7f51\u7edc\u5e26\u5bbd\u6700\u591a\uff0czstd\u6700\u5c11;<\/p>\n<p>\u5728CPU\u4f7f\u7528\u7387\u65b9\u9762\uff0c\u5404\u4e2a\u7b97\u6cd5\u8868\u73b0\u5f97\u5dee\u4e0d\u591a\uff0c\u53ea\u662f\u5728\u538b\u7f29\u65f6Snappy\u7b97\u6cd5\u4f7f\u7528\u7684CPU\u8f83\u591a\u4e00\u4e9b\uff0c\u800c\u5728\u89e3\u538b\u7f29\u65f6GZIP\u7b97\u6cd5\u5219\u53ef\u80fd\u4f7f\u7528\u66f4\u591a\u7684CPU\u3002<\/p>\n<h3>\u300c\u6700\u4f73\u5b9e\u8df5\u300d<\/h3>\n<p>\u4f55\u65f6\u542f\u7528\u538b\u7f29\u662f\u6bd4\u8f83\u5408\u9002\u7684\u65f6\u673a\u5462?<\/p>\n<p>\u542f\u7528\u538b\u7f29\u7684\u4e00\u4e2a\u6761\u4ef6\u5c31\u662fProducer\u7a0b\u5e8f\u8fd0\u884c\u673a\u5668\u4e0a\u7684CPU\u8d44\u6e90\u8981\u5f88\u5145\u8db3\u3002<\/p>\n<p>\u9664\u4e86CPU\u8d44\u6e90\u5145\u8db3\u8fd9\u4e00\u6761\u4ef6\uff0c\u5982\u679c\u4f60\u7684\u73af\u5883\u4e2d\u5e26\u5bbd\u8d44\u6e90\u6709\u9650\uff0c\u90a3\u4e48\u5efa\u8bae\u4f60\u5f00\u542f\u538b\u7f29\u3002<\/p>\n<h2>\u6d88\u8d39\u8005\u7ec4<\/h2>\n<h3>\u300cConsumer Group\u662fKafka\u63d0\u4f9b\u7684\u53ef\u6269\u5c55\u4e14\u5177\u6709\u5bb9\u9519\u6027\u7684\u6d88\u8d39\u8005\u673a\u5236\u300d\u3002<\/h3>\n<p>\u65e2\u7136\u662f\u4e00\u4e2a\u7ec4\uff0c\u90a3\u4e48\u7ec4\u5185\u5fc5\u7136\u53ef\u4ee5\u6709\u591a\u4e2a\u6d88\u8d39\u8005\u6216\u6d88\u8d39\u8005\u5b9e\u4f8b\uff0c\u5b83\u4eec\u5171\u4eab\u4e00\u4e2a\u516c\u5171\u7684ID\uff0c\u8fd9\u4e2aID\u88ab\u79f0\u4e3aGroup ID\u3002<\/p>\n<p>\u7ec4\u5185\u7684\u6240\u6709\u6d88\u8d39\u8005\u534f\u8c03\u5728\u4e00\u8d77\u6765\u6d88\u8d39\u8ba2\u9605\u4e3b\u9898\u7684\u6240\u6709\u5206\u533a\u3002<\/p>\n<ul>\n<li>\u6bcf\u4e2a\u5206\u533a\u53ea\u80fd\u7531\u540c\u4e00\u4e2a\u6d88\u8d39\u8005\u7ec4\u5185\u7684\u4e00\u4e2aConsumer\u5b9e\u4f8b\u6765\u6d88\u8d39\u3002<\/li>\n<\/ul>\n<h3>\u300cConsumer Group\u4e09\u4e2a\u7279\u6027\uff1a\u300d<\/h3>\n<ol>\n<li>Consumer Group\u4e0b\u53ef\u4ee5\u6709\u4e00\u4e2a\u6216\u591a\u4e2aConsumer\u5b9e\u4f8b\uff0c\u8fd9\u91cc\u7684\u5b9e\u4f8b\u53ef\u4ee5\u662f\u4e00\u4e2a\u5355\u72ec\u7684\u8fdb\u7a0b\uff0c\u4e5f\u53ef\u4ee5\u662f\u540c\u4e00\u8fdb\u7a0b\u4e0b\u7684\u7ebf\u7a0b\u3002<\/li>\n<li>Group ID\u662f\u4e00\u4e2a\u5b57\u7b26\u4e32\uff0c\u5728\u4e00\u4e2aKafka\u96c6\u7fa4\u4e2d\uff0c\u5b83\u6807\u8bc6\u552f\u4e00\u7684\u4e00\u4e2aConsumer Group\u3002<\/li>\n<li>Consumer Group\u4e0b\u6240\u6709\u5b9e\u4f8b\u8ba2\u9605\u7684\u4e3b\u9898\u7684\u5355\u4e2a\u5206\u533a\uff0c\u53ea\u80fd\u5206\u914d\u7ed9\u7ec4\u5185\u7684\u67d0\u4e2aConsumer\u5b9e\u4f8b\u6d88\u8d39\uff0c\u8fd9\u4e2a\u5206\u533a\u5f53\u7136\u4e5f\u53ef\u4ee5\u88ab\u5176\u4ed6\u7684Group\u6d88\u8d39\u3002<\/li>\n<\/ol>\n<p>\u5f53Consumer Group\u8ba2\u9605\u4e86\u591a\u4e2a\u4e3b\u9898\u540e\uff0c\u7ec4\u5185\u7684\u6bcf\u4e2a\u5b9e\u4f8b\u4e0d\u8981\u6c42\u4e00\u5b9a\u8981\u8ba2\u9605\u4e3b\u9898\u7684\u6240\u6709\u5206\u533a\uff0c\u5b83\u53ea\u4f1a\u6d88\u8d39\u90e8\u5206\u5206\u533a\u4e2d\u7684\u6d88\u606f\u3002<\/p>\n<p>Consumer Group\u4e4b\u95f4\u5f7c\u6b64\u72ec\u7acb\uff0c\u4e92\u4e0d\u5f71\u54cd\uff0c\u5b83\u4eec\u80fd\u591f\u8ba2\u9605\u76f8\u540c\u7684\u4e00\u7ec4\u4e3b\u9898\u800c\u4e92\u4e0d\u5e72\u6d89\u3002<\/p>\n<h3>\u300cKafka\u4ec5\u4ec5\u4f7f\u7528Consumer Group\u8fd9\u4e00\u79cd\u673a\u5236\uff0c\u5374\u540c\u65f6\u5b9e\u73b0\u4e86\u4f20\u7edf\u6d88\u606f\u5f15\u64ce\u7cfb\u7edf\u7684\u4e24\u5927\u6a21\u578b\u300d\uff1a<\/h3>\n<p>\u5982\u679c\u6240\u6709\u5b9e\u4f8b\u90fd\u5c5e\u4e8e\u540c\u4e00\u4e2aGroup\uff0c\u90a3\u4e48\u5b83\u5b9e\u73b0\u7684\u5c31\u662f\u6d88\u606f\u961f\u5217\u6a21\u578b;<\/p>\n<p>\u5982\u679c\u6240\u6709\u5b9e\u4f8b\u5206\u522b\u5c5e\u4e8e\u4e0d\u540c\u7684Group\uff0c\u90a3\u4e48\u5b83\u5b9e\u73b0\u7684\u5c31\u662f\u53d1\u5e03\/\u8ba2\u9605\u6a21\u578b\u3002<\/p>\n<p>\u300c\u4e00\u4e2aGroup\u4e0b\u8be5\u6709\u591a\u5c11\u4e2aConsumer\u5b9e\u4f8b\u5462?\u300d<\/p>\n<p>\u300c\u7406\u60f3\u60c5\u51b5\u4e0b\uff0cConsumer\u5b9e\u4f8b\u7684\u6570\u91cf\u5e94\u8be5\u7b49\u4e8e\u8be5Group\u8ba2\u9605\u4e3b\u9898\u7684\u5206\u533a\u603b\u6570\u3002\u300d<\/p>\n<p>\u5047\u8bbe\u4e00\u4e2aConsumer Group\u8ba2\u9605\u4e863\u4e2a\u4e3b\u9898\uff0c\u5206\u522b\u662fA\u3001B\u3001C\uff0c\u5b83\u4eec\u7684\u5206\u533a\u6570\u4f9d\u6b21\u662f1\u30012\u30013\uff0c\u90a3\u4e48\u901a\u5e38\u60c5\u51b5\u4e0b\uff0c\u4e3a\u8be5Group\u8bbe\u7f6e6\u4e2aConsumer\u5b9e\u4f8b\u662f\u6bd4\u8f83\u7406\u60f3\u7684\u60c5\u5f62\uff0c\u56e0\u4e3a\u5b83\u80fd\u6700\u5927\u9650\u5ea6\u5730\u5b9e\u73b0\u9ad8\u4f38\u7f29\u6027\u3002<\/p>\n<h3>\u300c\u9488\u5bf9Consumer Group\uff0cKafka\u662f\u600e\u4e48\u7ba1\u7406\u4f4d\u79fb\u7684\u5462?\u300d<\/h3>\n<h3>\u300c\u4f4d\u79fbOffset\u300d<\/h3>\n<p>\u8001\u7248\u672c\u7684Consumer Group\u628a\u4f4d\u79fb\u4fdd\u5b58\u5728ZooKeeper\u4e2d\u3002<\/p>\n<p>Apache ZooKeeper\u662f\u4e00\u4e2a\u5206\u5e03\u5f0f\u7684\u534f\u8c03\u670d\u52a1\u6846\u67b6\uff0cKafka\u91cd\u5ea6\u4f9d\u8d56\u5b83\u5b9e\u73b0\u5404\u79cd\u5404\u6837\u7684\u534f\u8c03\u7ba1\u7406\u3002<\/p>\n<p>\u5c06\u4f4d\u79fb\u4fdd\u5b58\u5728ZooKeeper\u5916\u90e8\u7cfb\u7edf\u7684\u505a\u6cd5\uff0c\u6700\u663e\u800c\u6613\u89c1\u7684\u597d\u5904\u5c31\u662f\u51cf\u5c11\u4e86Kafka Broker\u7aef\u7684\u72b6\u6001\u4fdd\u5b58\u5f00\u9500\u3002<\/p>\n<p>\u4e0d\u8fc7\uff0c\u6162\u6162\u5730\u53d1\u73b0\u4e86\u4e00\u4e2a\u95ee\u9898\uff0c\u5373ZooKeeper\u8fd9\u7c7b\u5143\u6846\u67b6\u5176\u5b9e\u5e76\u4e0d\u9002\u5408\u8fdb\u884c\u9891\u7e41\u7684\u5199\u66f4\u65b0\uff0c\u800cConsumer Group\u7684\u4f4d\u79fb\u66f4\u65b0\u5374\u662f\u4e00\u4e2a\u975e\u5e38\u9891\u7e41\u7684\u64cd\u4f5c\u3002<\/p>\n<p>\u8fd9\u79cd\u5927\u541e\u5410\u91cf\u7684\u5199\u64cd\u4f5c\u4f1a\u6781\u5927\u5730\u62d6\u6162ZooKeeper\u96c6\u7fa4\u7684\u6027\u80fd\u3002<\/p>\n<p>\u4e8e\u662f\uff0c\u5728\u65b0\u7248\u672c\u7684Consumer Group\u4e2d\uff0cKafka\u793e\u533a\u91cd\u65b0\u8bbe\u8ba1\u4e86Consumer Group\u7684\u4f4d\u79fb\u7ba1\u7406\u65b9\u5f0f\uff0c\u91c7\u7528\u4e86\u5c06\u4f4d\u79fb\u4fdd\u5b58\u5728Kafka\u5185\u90e8\u4e3b\u9898\u7684\u65b9\u6cd5\u3002<\/p>\n<p>\u8fd9\u4e2a\u5185\u90e8\u4e3b\u9898\u5c31\u662f__consumer_offsets\u3002<\/p>\n<h2>\u6d88\u8d39\u8005\u7b56\u7565<\/h2>\n<h3>\u300c\u7b2c\u4e00\u79cd\u662fRound\u300d<\/h3>\n<p>\u9ed8\u8ba4\uff0c\u4e5f\u53eb\u8f6e\u5faa\uff0c\u8bf4\u7684\u662f\u5bf9\u4e8e\u540c\u4e00\u7ec4\u6d88\u8d39\u8005\u6765\u8bf4\uff0c\u4f7f\u7528\u8f6e\u8bad\u5206\u914d\u7684\u65b9\u5f0f\uff0c\u51b3\u5b9a\u6d88\u8d39\u8005\u6d88\u8d39\u7684\u5206\u533a<\/p>\n<p style=\"text-align: center\">\n<h3>\u300c\u7b2c\u4e8c\u79cd\u53eb\u505aRange\u300d<\/h3>\n<p>\u5bf9\u4e00\u4e2a\u6d88\u8d39\u8005\u7ec4\u6765\u8bf4\u51b3\u5b9a\u6d88\u8d39\u65b9\u5f0f\u662f\u4ee5\u5206\u533a\u603b\u6570\u9664\u4ee5\u6d88\u8d39\u8005\u603b\u6570\u6765\u51b3\u5b9a\uff0c\u4e00\u822c\u5982\u679c\u4e0d\u80fd\u6574\u9664\uff0c\u5f80\u5f80\u662f\u4ece\u5934\u5f00\u59cb\u5c06\u5269\u4f59\u7684\u5206\u533a\u5206\u914d\u5f00<\/p>\n<p style=\"text-align: center\">\n<h3>\u300c\u7b2c\u4e09\u79cd\u53ebSticky\u300d<\/h3>\n<p>\u662f\u57280.11.x\uff0c\u65b0\u589e\u7684\uff0c\u5b83\u548c\u524d\u9762\u4e24\u4e2a\u4e0d\u662f\u5f88\u4e00\u6837\uff0c\u5b83\u662f\u5728Range\u4e0a\u7684\u4e00\u79cd\u5347\u534e\uff0c\u4e14\u524d\u9762\u4e24\u4e2a\u5f53\u540c\u7ec4\u5185\u6709\u65b0\u7684\u6d88\u8d39\u8005\u52a0\u5165\u6216\u8005\u65e7\u7684\u6d88\u8d39\u8005\u9000\u51fa\u7684\u65f6\u5019\uff0c\u4f1a\u4ece\u65b0\u5f00\u59cb\u51b3\u5b9a\u6d88\u8d39\u8005\u6d88\u8d39\u65b9\u5f0f\uff0c\u4f46\u662fSticky\uff0c\u5728\u540c\u7ec4\u4e2d\u6709\u65b0\u7684\u65b0\u7684\u6d88\u8d39\u8005\u52a0\u5165\u6216\u8005\u65e7\u7684\u6d88\u8d39\u8005\u9000\u51fa\u65f6\uff0c\u4e0d\u4f1a\u76f4\u63a5\u5f00\u59cb\u65b0\u7684Range\u5206\u914d\uff0c\u800c\u662f\u4fdd\u7559\u73b0\u6709\u6d88\u8d39\u8005\u539f\u6765\u7684\u6d88\u8d39\u7b56\u7565\uff0c\u5c06\u9000\u51fa\u7684\u6d88\u8d39\u8005\u6240\u6d88\u8d39\u7684\u5206\u533a\u5e73\u5747\u5206\u914d\u7ed9\u73b0\u6709\u6d88\u8d39\u8005\uff0c\u65b0\u589e\u6d88\u8d39\u8005\u540c\u7406\uff0c\u540c\u5176\u4ed6\u73b0\u5b58\u6d88\u8d39\u8005\u7684\u6d88\u8d39\u7b56\u7565\u4e2d\u5206\u79bb<\/p>\n<h2>\u4f4d\u79fb\u63d0\u4ea4<\/h2>\n<p>\u5047\u8bbe\u4e00\u4e2a\u5206\u533a\u4e2d\u670910\u6761\u6d88\u606f\uff0c\u4f4d\u79fb\u5206\u522b\u662f0\u52309\u3002<\/p>\n<p>\u67d0\u4e2aConsumer\u5e94\u7528\u5df2\u6d88\u8d39\u4e865\u6761\u6d88\u606f\uff0c\u8fd9\u5c31\u8bf4\u660e\u8be5Consumer\u6d88\u8d39\u4e86\u4f4d\u79fb\u4e3a0\u52304\u76845\u6761\u6d88\u606f\uff0c\u6b64\u65f6Consumer\u7684\u4f4d\u79fb\u662f5\uff0c\u6307\u5411\u4e86\u4e0b\u4e00\u6761\u6d88\u606f\u7684\u4f4d\u79fb\u3002<\/p>\n<p>\u56e0\u4e3aConsumer\u80fd\u591f\u540c\u65f6\u6d88\u8d39\u591a\u4e2a\u5206\u533a\u7684\u6570\u636e\uff0c\u6240\u4ee5\u4f4d\u79fb\u7684\u63d0\u4ea4\u5b9e\u9645\u4e0a\u662f\u5728\u5206\u533a\u7c92\u5ea6\u4e0a\u8fdb\u884c\u7684\uff0c\u5373\u300cConsumer\u9700\u8981\u4e3a\u5206\u914d\u7ed9\u5b83\u7684\u6bcf\u4e2a\u5206\u533a\u63d0\u4ea4\u5404\u81ea\u7684\u4f4d\u79fb\u6570\u636e\u300d\u3002<\/p>\n<p>\u300c\u4f4d\u79fb\u63d0\u4ea4\u5206\u4e3a\u81ea\u52a8\u63d0\u4ea4\u548c\u624b\u52a8\u63d0\u4ea4;\u4eceConsumer\u7aef\u7684\u89d2\u5ea6\u6765\u8bf4\uff0c\u4f4d\u79fb\u63d0\u4ea4\u5206\u4e3a\u540c\u6b65\u63d0\u4ea4\u548c\u5f02\u6b65\u63d0\u4ea4\u300d\u3002<\/p>\n<p>\u5f00\u542f\u81ea\u52a8\u63d0\u4ea4\u4f4d\u79fb\u7684\u65b9\u6cd5\uff1aConsumer\u7aef\u6709\u4e2a\u53c2\u6570enable.auto.commit\uff0c\u628a\u5b83\u8bbe\u7f6e\u4e3atrue\u6216\u8005\u538b\u6839\u4e0d\u8bbe\u7f6e\u5b83\u5c31\u53ef\u4ee5\u4e86\u3002<\/p>\n<p>\u56e0\u4e3a\u5b83\u7684\u9ed8\u8ba4\u503c\u5c31\u662ftrue\uff0c\u5373Java Consumer\u9ed8\u8ba4\u5c31\u662f\u81ea\u52a8\u63d0\u4ea4\u4f4d\u79fb\u7684\u3002<\/p>\n<p>\u5982\u679c\u542f\u7528\u4e86\u81ea\u52a8\u63d0\u4ea4\uff0cConsumer\u7aef\u8fd8\u6709\u4e2a\u53c2\u6570\uff1aauto.commit.interval.ms\u3002<\/p>\n<p>\u5b83\u7684\u9ed8\u8ba4\u503c\u662f5\u79d2\uff0c\u8868\u660eKafka\u6bcf5\u79d2\u4f1a\u4e3a\u4f60\u81ea\u52a8\u63d0\u4ea4\u4e00\u6b21\u4f4d\u79fb\u3002<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>Properties&nbsp;props&nbsp;=&nbsp;new&nbsp;Properties();&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;props.put(<span>\"bootstrap.servers\"<\/span><span>,&nbsp;<\/span><span>\"localhost:9092\"<\/span><span>);&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;props.put(<span>\"group.id\"<\/span><span>,&nbsp;<\/span><span>\"test\"<\/span><span>);&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;props.put(<span>\"enable.auto.commit\"<\/span><span>,&nbsp;<\/span><span>\"true\"<\/span><span>);&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;props.put(<span>\"auto.commit.interval.ms\"<\/span><span>,&nbsp;<\/span><span>\"2000\"<\/span><span>);&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;props.put(<span>\"key.deserializer\"<\/span><span>,&nbsp;<\/span><span>\"org.apache.kafka.common.serialization.StringDeserializer\"<\/span><span>);&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;props.put(<span>\"value.deserializer\"<\/span><span>,&nbsp;<\/span><span>\"org.apache.kafka.common.serialization.StringDeserializer\"<\/span><span>);&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;KafkaConsumer&nbsp;consumer&nbsp;=&nbsp;new&nbsp;KafkaConsumer&lt;&gt;(props);&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;consumer.subscribe(Arrays.asList(<span>\"foo\"<\/span><span>,&nbsp;<\/span><span>\"bar\"<\/span><span>));&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;while&nbsp;(<span>true<\/span><span>)&nbsp;{&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;ConsumerRecords&nbsp;records&nbsp;=&nbsp;consumer.poll(100);&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span>for<\/span><span>&nbsp;(ConsumerRecord&nbsp;record&nbsp;:&nbsp;records)&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;System.<span>out<\/span><span>.printf(<\/span><span>\"offset&nbsp;=&nbsp;%d,&nbsp;key&nbsp;=&nbsp;%s,&nbsp;value&nbsp;=&nbsp;%s%n\"<\/span><span>,&nbsp;record.offset(),&nbsp;record.<\/span><span>key<\/span><span>(),&nbsp;record.value());&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;}&nbsp;<\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>\u4e0a\u9762\u7684\u7b2c3\u3001\u7b2c4\u884c\u4ee3\u7801\uff0c\u5c31\u662f\u5f00\u542f\u81ea\u52a8\u63d0\u4ea4\u4f4d\u79fb\u7684\u65b9\u6cd5\u3002<\/p>\n<p>\u5f00\u542f\u624b\u52a8\u63d0\u4ea4\u4f4d\u79fb\u7684\u65b9\u6cd5\u5c31\u662f\u8bbe\u7f6eenable.auto.commit\u4e3afalse\u3002<\/p>\n<p>\u8fd8\u9700\u8981\u8c03\u7528\u76f8\u5e94\u7684API\u624b\u52a8\u63d0\u4ea4\u4f4d\u79fb\u3002\u6700\u7b80\u5355\u7684API\u5c31\u662f\u300cKafkaConsumer#commitSync()\u300d\u3002<\/p>\n<p>\u8be5\u65b9\u6cd5\u4f1a\u63d0\u4ea4KafkaConsumer#poll()\u8fd4\u56de\u7684\u6700\u65b0\u4f4d\u79fb\u3002<\/p>\n<p>\u4ece\u540d\u5b57\u4e0a\u6765\u770b\uff0c\u5b83\u662f\u4e00\u4e2a\u540c\u6b65\u64cd\u4f5c\uff0c\u5373\u8be5\u65b9\u6cd5\u4f1a\u4e00\u76f4\u7b49\u5f85\uff0c\u76f4\u5230\u4f4d\u79fb\u88ab\u6210\u529f\u63d0\u4ea4\u624d\u4f1a\u8fd4\u56de\u3002<\/p>\n<p>\u5982\u679c\u63d0\u4ea4\u8fc7\u7a0b\u4e2d\u51fa\u73b0\u5f02\u5e38\uff0c\u8be5\u65b9\u6cd5\u4f1a\u5c06\u5f02\u5e38\u4fe1\u606f\u629b\u51fa\u3002<\/p>\n<p>\u4e0b\u9762\u8fd9\u6bb5\u4ee3\u7801\u5c55\u793a\u4e86commitSync()\u7684\u4f7f\u7528\u65b9\u6cd5\uff1a<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>while&nbsp;(<\/span><span>true<\/span><span>)&nbsp;{&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;ConsumerRecords&nbsp;records&nbsp;=&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;consumer.poll(Duration.ofSeconds(1));&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;process(records);&nbsp;\/\/&nbsp;\u5904\u7406\u6d88\u606f&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;try&nbsp;{&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;consumer.commitSync();&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;}&nbsp;catch&nbsp;(CommitFailedException&nbsp;e)&nbsp;{&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;handle(e);&nbsp;\/\/&nbsp;\u5904\u7406\u63d0\u4ea4\u5931\u8d25\u5f02\u5e38&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;}&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>}&nbsp;<\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>\u4e00\u65e6\u8bbe\u7f6e\u4e86enable.auto.commit\u4e3atrue\uff0cKafka\u4f1a\u4fdd\u8bc1\u5728\u5f00\u59cb\u8c03\u7528poll\u65b9\u6cd5\u65f6\uff0c\u63d0\u4ea4\u4e0a\u6b21poll\u8fd4\u56de\u7684\u6240\u6709\u6d88\u606f\u3002<\/p>\n<p>\u4ece\u987a\u5e8f\u4e0a\u6765\u8bf4\uff0cpoll\u65b9\u6cd5\u7684\u903b\u8f91\u662f\u5148\u63d0\u4ea4\u4e0a\u4e00\u6279\u6d88\u606f\u7684\u4f4d\u79fb\uff0c\u518d\u5904\u7406\u4e0b\u4e00\u6279\u6d88\u606f\uff0c\u56e0\u6b64\u5b83\u80fd\u4fdd\u8bc1\u4e0d\u51fa\u73b0\u6d88\u8d39\u4e22\u5931\u7684\u60c5\u51b5\u3002<\/p>\n<p>\u4f46\u81ea\u52a8\u63d0\u4ea4\u4f4d\u79fb\u7684\u4e00\u4e2a\u95ee\u9898\u5728\u4e8e\uff0c<\/p>\n<h3>\u300c\u5b83\u53ef\u80fd\u4f1a\u51fa\u73b0\u91cd\u590d\u6d88\u8d39\u300d\u3002<\/h3>\n<p>\u800c\u624b\u52a8\u63d0\u4ea4\u4f4d\u79fb\uff0c\u5b83\u7684\u597d\u5904\u5c31\u5728\u4e8e\u66f4\u52a0\u7075\u6d3b\uff0c\u4f60\u5b8c\u5168\u80fd\u591f\u628a\u63a7\u4f4d\u79fb\u63d0\u4ea4\u7684\u65f6\u673a\u548c\u9891\u7387\u3002<\/p>\n<p>\u4f46\u662f\uff0c\u5b83\u4e5f\u6709\u4e00\u4e2a\u7f3a\u9677\uff0c\u5c31\u662f\u5728\u8c03\u7528commitSync()\u65f6\uff0cConsumer\u7a0b\u5e8f\u4f1a\u5904\u4e8e\u963b\u585e\u72b6\u6001\uff0c\u76f4\u5230\u8fdc\u7aef\u7684Broker\u8fd4\u56de\u63d0\u4ea4\u7ed3\u679c\uff0c\u8fd9\u4e2a\u72b6\u6001\u624d\u4f1a\u7ed3\u675f\u3002<\/p>\n<p>\u9274\u4e8e\u8fd9\u4e2a\u95ee\u9898\uff0cKafka\u793e\u533a\u4e3a\u624b\u52a8\u63d0\u4ea4\u4f4d\u79fb\u63d0\u4f9b\u4e86\u53e6\u4e00\u4e2aAPI\u65b9\u6cd5\uff1a<\/p>\n<h3>\u300cKafkaConsumer#commitAsync()\u300d\u3002<\/h3>\n<p>\u4ece\u540d\u5b57\u4e0a\u6765\u770b\u5b83\u5c31\u4e0d\u662f\u540c\u6b65\u7684\uff0c\u800c\u662f\u4e00\u4e2a\u5f02\u6b65\u64cd\u4f5c\u3002<\/p>\n<p>\u8c03\u7528commitAsync()\u4e4b\u540e\uff0c\u5b83\u4f1a\u7acb\u5373\u8fd4\u56de\uff0c\u4e0d\u4f1a\u963b\u585e\uff0c\u56e0\u6b64\u4e0d\u4f1a\u5f71\u54cdConsumer\u5e94\u7528\u7684TPS\u3002<\/p>\n<p>\u7531\u4e8e\u5b83\u662f\u5f02\u6b65\u7684\uff0cKafka\u63d0\u4f9b\u4e86\u56de\u8c03\u51fd\u6570(callback)\uff0c\u4f9b\u4f60\u5b9e\u73b0\u63d0\u4ea4\u4e4b\u540e\u7684\u903b\u8f91\uff0c\u6bd4\u5982\u8bb0\u5f55\u65e5\u5fd7\u6216\u5904\u7406\u5f02\u5e38\u7b49\u3002<\/p>\n<p>\u4e0b\u9762\u8fd9\u6bb5\u4ee3\u7801\u5c55\u793a\u4e86\u8c03\u7528commitAsync()\u7684\u65b9\u6cd5\uff1a<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>while&nbsp;(<\/span><span>true<\/span><span>)&nbsp;{&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;ConsumerRecords&nbsp;records&nbsp;=&nbsp;&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;consumer.poll(Duration.ofSeconds(1));&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;process(records);&nbsp;\/\/&nbsp;\u5904\u7406\u6d88\u606f&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;consumer.commitAsync((offsets,&nbsp;exception)&nbsp;-&gt;&nbsp;{&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;if&nbsp;(exception&nbsp;!=&nbsp;<span>null<\/span><span>)&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;handle(exception);&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;});&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>}&nbsp;<\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>commitAsync\u7684\u95ee\u9898\u5728\u4e8e\uff0c\u51fa\u73b0\u95ee\u9898\u65f6\u5b83\u4e0d\u4f1a\u81ea\u52a8\u91cd\u8bd5\u3002<\/p>\n<p>\u663e\u7136\uff0c\u5982\u679c\u662f\u624b\u52a8\u63d0\u4ea4\uff0c\u6211\u4eec\u9700\u8981\u5c06commitSync\u548ccommitAsync\u7ec4\u5408\u4f7f\u7528\u624d\u80fd\u5230\u8fbe\u6700\u7406\u60f3\u7684\u6548\u679c\uff0c\u539f\u56e0\u6709\u4e24\u4e2a\uff1a<\/p>\n<ol>\n<li>\u6211\u4eec\u53ef\u4ee5\u5229\u7528commitSync\u7684\u81ea\u52a8\u91cd\u8bd5\u6765\u89c4\u907f\u90a3\u4e9b\u77ac\u65f6\u9519\u8bef\uff0c\u6bd4\u5982\u7f51\u7edc\u7684\u77ac\u65f6\u6296\u52a8\uff0cBroker\u7aefGC\u7b49\uff0c\u56e0\u4e3a\u8fd9\u4e9b\u95ee\u9898\u90fd\u662f\u77ed\u6682\u7684\uff0c\u81ea\u52a8\u91cd\u8bd5\u901a\u5e38\u90fd\u4f1a\u6210\u529f\u3002<\/li>\n<li>\u6211\u4eec\u4e0d\u5e0c\u671b\u7a0b\u5e8f\u603b\u5904\u4e8e\u963b\u585e\u72b6\u6001\uff0c\u5f71\u54cdTPS\u3002<\/li>\n<\/ol>\n<p>\u6211\u4eec\u6765\u770b\u4e00\u4e0b\u4e0b\u9762\u8fd9\u6bb5\u4ee3\u7801\uff0c\u5b83\u5c55\u793a\u7684\u662f\u5982\u4f55\u5c06\u4e24\u4e2aAPI\u65b9\u6cd5\u7ed3\u5408\u4f7f\u7528\u8fdb\u884c\u624b\u52a8\u63d0\u4ea4\u3002<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>&nbsp;&nbsp;&nbsp;try&nbsp;{&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;while(<span>true<\/span><span>)&nbsp;{&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;ConsumerRecords&nbsp;records&nbsp;=&nbsp;&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;consumer.poll(Duration.ofSeconds(1));&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;process(records);&nbsp;\/\/&nbsp;\u5904\u7406\u6d88\u606f&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;commitAysnc();&nbsp;\/\/&nbsp;\u4f7f\u7528\u5f02\u6b65\u63d0\u4ea4\u89c4\u907f\u963b\u585e&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;}&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>}&nbsp;catch(Exception&nbsp;e)&nbsp;{&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;handle(e);&nbsp;\/\/&nbsp;\u5904\u7406\u5f02\u5e38&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>}&nbsp;finally&nbsp;{&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;try&nbsp;{&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;consumer.commitSync();&nbsp;\/\/&nbsp;\u6700\u540e\u4e00\u6b21\u63d0\u4ea4\u4f7f\u7528\u540c\u6b65\u963b\u585e\u5f0f\u63d0\u4ea4&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;}&nbsp;finally&nbsp;{&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;consumer.<span>close<\/span><span>();&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>}&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>}&nbsp;<\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>\u8fd9\u6837\u4e00\u4e2a\u573a\u666f\uff1a\u4f60\u7684poll\u65b9\u6cd5\u8fd4\u56de\u7684\u4e0d\u662f500\u6761\u6d88\u606f\uff0c\u800c\u662f5000\u6761\u3002<\/p>\n<p>\u90a3\u4e48\uff0c\u4f60\u80af\u5b9a\u4e0d\u60f3\u628a\u8fd95000\u6761\u6d88\u606f\u90fd\u5904\u7406\u5b8c\u4e4b\u540e\u518d\u63d0\u4ea4\u4f4d\u79fb\uff0c\u56e0\u4e3a\u4e00\u65e6\u4e2d\u95f4\u51fa\u73b0\u5dee\u9519\uff0c\u4e4b\u524d\u5904\u7406\u7684\u5168\u90e8\u90fd\u8981\u91cd\u6765\u4e00\u904d\u3002<\/p>\n<p>\u6bd4\u5982\u524d\u9762\u8fd9\u4e2a5000\u6761\u6d88\u606f\u7684\u4f8b\u5b50\uff0c\u4f60\u53ef\u80fd\u5e0c\u671b\u6bcf\u5904\u7406\u5b8c100\u6761\u6d88\u606f\u5c31\u63d0\u4ea4\u4e00\u6b21\u4f4d\u79fb\uff0c\u8fd9\u6837\u80fd\u591f\u907f\u514d\u5927\u6279\u91cf\u7684\u6d88\u606f\u91cd\u65b0\u6d88\u8d39\u3002<\/p>\n<p>Kafka Consumer API\u4e3a\u624b\u52a8\u63d0\u4ea4\u63d0\u4f9b\u4e86\u8fd9\u6837\u7684\u65b9\u6cd5\uff1acommitSync(Map)\u548ccommitAsync(Map)\u3002<\/p>\n<p>\u5b83\u4eec\u7684\u53c2\u6570\u662f\u4e00\u4e2aMap\u5bf9\u8c61\uff0c\u952e\u5c31\u662fTopicPartition\uff0c\u5373\u6d88\u8d39\u7684\u5206\u533a\uff0c\u800c\u503c\u662f\u4e00\u4e2aOffsetAndMetadata\u5bf9\u8c61\uff0c\u4fdd\u5b58\u7684\u4e3b\u8981\u662f\u4f4d\u79fb\u6570\u636e\u3002<\/p>\n<ul>\n<li>\u5982\u4f55\u6bcf\u5904\u7406100\u6761\u6d88\u606f\u5c31\u63d0\u4ea4\u4e00\u6b21\u4f4d\u79fb\u5462?<\/li>\n<\/ul>\n<p>\u5728\u8fd9\u91cc\uff0c\u6211\u4ee5commitAsync\u4e3a\u4f8b\uff0c\u5c55\u793a\u4e00\u6bb5\u4ee3\u7801\uff0c\u5b9e\u9645\u4e0a\uff0ccommitSync\u7684\u8c03\u7528\u65b9\u6cd5\u548c\u5b83\u662f\u4e00\u6a21\u4e00\u6837\u7684\u3002<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>private&nbsp;Map&nbsp;offsets&nbsp;=&nbsp;new&nbsp;HashMap&lt;&gt;();&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span><span>int<\/span><span>&nbsp;<\/span><span>count<\/span><span>&nbsp;=&nbsp;0;&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>\u2026\u2026&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>while&nbsp;(<span>true<\/span><span>)&nbsp;{&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;ConsumerRecords&nbsp;records&nbsp;=&nbsp;&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;consumer.poll(Duration.ofSeconds(1));&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span>for<\/span><span>&nbsp;(ConsumerRecord&nbsp;record:&nbsp;records)&nbsp;{&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;process(record);&nbsp;&nbsp;\/\/&nbsp;\u5904\u7406\u6d88\u606f&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;offsets.put(new&nbsp;TopicPartition(record.topic(),&nbsp;record.partition()),&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;new&nbsp;OffsetAndMetadata(record.offset()&nbsp;+&nbsp;1)\uff1b&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;if\uff08<span>count<\/span><span>&nbsp;%&nbsp;100&nbsp;==&nbsp;0\uff09&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;consumer.commitAsync(offsets,&nbsp;<span>null<\/span><span>);&nbsp;\/\/&nbsp;\u56de\u8c03\u5904\u7406\u903b\u8f91\u662f<\/span><span>null<\/span><span>&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span>count<\/span><span>++;&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;}&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>}&nbsp;<\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>\u7a0b\u5e8f\u5148\u662f\u521b\u5efa\u4e86\u4e00\u4e2aMap\u5bf9\u8c61\uff0c\u7528\u4e8e\u4fdd\u5b58Consumer\u6d88\u8d39\u5904\u7406\u8fc7\u7a0b\u4e2d\u8981\u63d0\u4ea4\u7684\u5206\u533a\u4f4d\u79fb\uff0c\u4e4b\u540e\u5f00\u59cb\u9010\u6761\u5904\u7406\u6d88\u606f\uff0c\u5e76\u6784\u9020\u8981\u63d0\u4ea4\u7684\u4f4d\u79fb\u503c\u3002<\/p>\n<p>\u4ee3\u7801\u7684\u6700\u540e\u90e8\u5206\u662f\u505a\u4f4d\u79fb\u7684\u63d0\u4ea4\u3002\u8bbe\u7f6e\u4e86\u4e00\u4e2a\u8ba1\u6570\u5668\uff0c\u6bcf\u7d2f\u8ba1100\u6761\u6d88\u606f\u5c31\u7edf\u4e00\u63d0\u4ea4\u4e00\u6b21\u4f4d\u79fb\u3002<\/p>\n<p>\u4e0e\u8c03\u7528\u65e0\u53c2\u7684commitAsync\u4e0d\u540c\uff0c\u8fd9\u91cc\u8c03\u7528\u4e86\u5e26Map\u5bf9\u8c61\u53c2\u6570\u7684commitAsync\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u4f4d\u79fb\u63d0\u4ea4\u3002<\/p>\n<p>\u8fd9\u6837\uff0c\u8fd9\u6bb5\u4ee3\u7801\u5c31\u80fd\u591f\u5b9e\u73b0\u6bcf\u5904\u7406100\u6761\u6d88\u606f\u5c31\u63d0\u4ea4\u4e00\u6b21\u4f4d\u79fb\uff0c\u4e0d\u7528\u518d\u53d7poll\u65b9\u6cd5\u8fd4\u56de\u7684\u6d88\u606f\u603b\u6570\u7684\u9650\u5236\u4e86\u3002<\/p>\n<h2>\u91cd\u5e73\u8861<\/h2>\n<h3>\u300c(\u91cd\u5e73\u8861)Rebalance\u672c\u8d28\u4e0a\u662f\u4e00\u79cd\u534f\u8bae\uff0c\u89c4\u5b9a\u4e86\u4e00\u4e2aConsumer Group\u4e0b\u7684\u6240\u6709Consumer\u5982\u4f55\u8fbe\u6210\u4e00\u81f4\uff0c\u6765\u5206\u914d\u8ba2\u9605Topic\u7684\u6bcf\u4e2a\u5206\u533a\u300d\u3002<\/h3>\n<p>\u6bd4\u5982\u67d0\u4e2aGroup\u4e0b\u670920\u4e2aConsumer\u5b9e\u4f8b\uff0c\u5b83\u8ba2\u9605\u4e86\u4e00\u4e2a\u5177\u6709100\u4e2a\u5206\u533a\u7684Topic\u3002<\/p>\n<p>\u6b63\u5e38\u60c5\u51b5\u4e0b\uff0cKafka\u5e73\u5747\u4f1a\u4e3a\u6bcf\u4e2aConsumer\u5206\u914d5\u4e2a\u5206\u533a\u3002\u8fd9\u4e2a\u5206\u914d\u7684\u8fc7\u7a0b\u5c31\u53ebRebalance\u3002<\/p>\n<h3>\u300cRebalance\u7684\u89e6\u53d1\u6761\u4ef6\u67093\u4e2a\u3002\u300d<\/h3>\n<ol>\n<li>\u7ec4\u6210\u5458\u6570\u53d1\u751f\u53d8\u66f4\u3002\u6bd4\u5982\u6709\u65b0\u7684Consumer\u5b9e\u4f8b\u52a0\u5165\u7ec4\u6216\u8005\u79bb\u5f00\u7ec4\uff0c\u6216\u662f\u6709Consumer\u5b9e\u4f8b\u5d29\u6e83\u88ab\u8e22\u51fa\u7ec4\u3002<\/li>\n<li>\u8ba2\u9605\u4e3b\u9898\u6570\u53d1\u751f\u53d8\u66f4\u3002Consumer Group\u53ef\u4ee5\u4f7f\u7528\u6b63\u5219\u8868\u8fbe\u5f0f\u7684\u65b9\u5f0f\u8ba2\u9605\u4e3b\u9898\uff0c\u6bd4\u5982consumer.subscribe(Pattern.compile(\u201ct.*c\u201d))\u5c31\u8868\u660e\u8be5Group\u8ba2\u9605\u6240\u6709\u4ee5\u5b57\u6bcdt\u5f00\u5934\u3001\u5b57\u6bcdc\u7ed3\u5c3e\u7684\u4e3b\u9898\uff0c\u5728Consumer Group\u7684\u8fd0\u884c\u8fc7\u7a0b\u4e2d\uff0c\u4f60\u65b0\u521b\u5efa\u4e86\u4e00\u4e2a\u6ee1\u8db3\u8fd9\u6837\u6761\u4ef6\u7684\u4e3b\u9898\uff0c\u90a3\u4e48\u8be5Group\u5c31\u4f1a\u53d1\u751fRebalance\u3002<\/li>\n<li>\u8ba2\u9605\u4e3b\u9898\u7684\u5206\u533a\u6570\u53d1\u751f\u53d8\u66f4\u3002Kafka\u5f53\u524d\u53ea\u80fd\u5141\u8bb8\u589e\u52a0\u4e00\u4e2a\u4e3b\u9898\u7684\u5206\u533a\u6570\uff0c\u5f53\u5206\u533a\u6570\u589e\u52a0\u65f6\uff0c\u5c31\u4f1a\u89e6\u53d1\u8ba2\u9605\u8be5\u4e3b\u9898\u7684\u6240\u6709Group\u5f00\u542fRebalance\u3002<\/li>\n<\/ol>\n<p>Rebalance\u53d1\u751f\u65f6\uff0cGroup\u4e0b\u6240\u6709\u7684Consumer\u5b9e\u4f8b\u90fd\u4f1a\u534f\u8c03\u5728\u4e00\u8d77\u5171\u540c\u53c2\u4e0e\u3002<\/p>\n<h3>\u300c\u5206\u914d\u7b56\u7565\u300d<\/h3>\n<p>\u5f53\u524dKafka\u9ed8\u8ba4\u63d0\u4f9b\u4e863\u79cd\u5206\u914d\u7b56\u7565\uff0c\u6bcf\u79cd\u7b56\u7565\u90fd\u6709\u4e00\u5b9a\u7684\u4f18\u52bf\u548c\u52a3\u52bf\uff0c\u793e\u533a\u4f1a\u4e0d\u65ad\u5730\u5b8c\u5584\u8fd9\u4e9b\u7b56\u7565\uff0c\u4fdd\u8bc1\u63d0\u4f9b\u6700\u516c\u5e73\u7684\u5206\u914d\u7b56\u7565\uff0c\u5373\u6bcf\u4e2aConsumer\u5b9e\u4f8b\u90fd\u80fd\u591f\u5f97\u5230\u8f83\u4e3a\u5e73\u5747\u7684\u5206\u533a\u6570\u3002<\/p>\n<p>\u6bd4\u5982\u4e00\u4e2aGroup\u5185\u670910\u4e2aConsumer\u5b9e\u4f8b\uff0c\u8981\u6d88\u8d39100\u4e2a\u5206\u533a\uff0c\u7406\u60f3\u7684\u5206\u914d\u7b56\u7565\u81ea\u7136\u662f\u6bcf\u4e2a\u5b9e\u4f8b\u5e73\u5747\u5f97\u523010\u4e2a\u5206\u533a\u3002<\/p>\n<p>\u8fd9\u5c31\u53eb\u516c\u5e73\u7684\u5206\u914d\u7b56\u7565\u3002<\/p>\n<p>\u4e3e\u4e2a\u7b80\u5355\u7684\u4f8b\u5b50\u6765\u8bf4\u660e\u4e00\u4e0bConsumer Group\u53d1\u751fRebalance\u7684\u8fc7\u7a0b\u3002<\/p>\n<p>\u5047\u8bbe\u76ee\u524d\u67d0\u4e2aConsumer Group\u4e0b\u6709\u4e24\u4e2aConsumer\uff0c\u6bd4\u5982A\u548cB\uff0c\u5f53\u7b2c\u4e09\u4e2a\u6210\u5458C\u52a0\u5165\u65f6\uff0cKafka\u4f1a\u89e6\u53d1Rebalance\uff0c\u5e76\u6839\u636e\u9ed8\u8ba4\u7684\u5206\u914d\u7b56\u7565\u91cd\u65b0\u4e3aA\u3001B\u548cC\u5206\u914d\u5206\u533a<\/p>\n<p>Rebalance\u4e4b\u540e\u7684\u5206\u914d\u4f9d\u7136\u662f\u516c\u5e73\u7684\uff0c\u5373\u6bcf\u4e2aConsumer\u5b9e\u4f8b\u90fd\u83b7\u5f97\u4e862\u4e2a\u5206\u533a\u7684\u6d88\u8d39\u6743\u3002<\/p>\n<p>\u5728Rebalance\u8fc7\u7a0b\u4e2d\uff0c\u6240\u6709Consumer\u5b9e\u4f8b\u90fd\u4f1a\u505c\u6b62\u6d88\u8d39\uff0c\u7b49\u5f85Rebalance\u5b8c\u6210\uff0c\u8fd9\u662fRebalance\u4e3a\u4eba\u8bdf\u75c5\u7684\u4e00\u4e2a\u65b9\u9762\u3002<\/p>\n<p>\u76ee\u524dRebalance\u7684\u8bbe\u8ba1\u662f\u6240\u6709Consumer\u5b9e\u4f8b\u5171\u540c\u53c2\u4e0e\uff0c\u5168\u90e8\u91cd\u65b0\u5206\u914d\u6240\u6709\u5206\u533a\u3002<\/p>\n<h3>\u300cCoordinator\u4f1a\u5728\u4ec0\u4e48\u60c5\u51b5\u4e0b\u8ba4\u4e3a\u67d0\u4e2aConsumer\u5b9e\u4f8b\u5df2\u6302\u4ece\u800c\u8981\u9000\u7ec4\u5462?\u300d<\/h3>\n<p>\u5f53Consumer Group\u5b8c\u6210Rebalance\u4e4b\u540e\uff0c\u6bcf\u4e2aConsumer\u5b9e\u4f8b\u90fd\u4f1a\u5b9a\u671f\u5730\u5411Coordinator\u53d1\u9001\u5fc3\u8df3\u8bf7\u6c42\uff0c\u8868\u660e\u5b83\u8fd8\u5b58\u6d3b\u7740\u3002<\/p>\n<p>\u5982\u679c\u67d0\u4e2aConsumer\u5b9e\u4f8b\u4e0d\u80fd\u53ca\u65f6\u5730\u53d1\u9001\u8fd9\u4e9b\u5fc3\u8df3\u8bf7\u6c42\uff0cCoordinator\u5c31\u4f1a\u8ba4\u4e3a\u8be5Consumer\u5df2\u7ecf\u6b7b\u4e86\uff0c\u4ece\u800c\u5c06\u5176\u4eceGroup\u4e2d\u79fb\u9664\uff0c\u7136\u540e\u5f00\u542f\u65b0\u4e00\u8f6eRebalance\u3002<\/p>\n<p>Consumer\u7aef\u6709\u4e2a\u53c2\u6570\uff0c\u53ebsession.timeout.ms\u3002<\/p>\n<p>\u8be5\u53c2\u6570\u7684\u9ed8\u8ba4\u503c\u662f10\u79d2\uff0c\u5373\u5982\u679cCoordinator\u572810\u79d2\u4e4b\u5185\u6ca1\u6709\u6536\u5230Group\u4e0b\u67d0Consumer\u5b9e\u4f8b\u7684\u5fc3\u8df3\uff0c\u5b83\u5c31\u4f1a\u8ba4\u4e3a\u8fd9\u4e2aConsumer\u5b9e\u4f8b\u5df2\u7ecf\u6302\u4e86\u3002<\/p>\n<p>\u9664\u4e86\u8fd9\u4e2a\u53c2\u6570\uff0cConsumer\u8fd8\u63d0\u4f9b\u4e86\u4e00\u4e2a\u5141\u8bb8\u4f60\u63a7\u5236\u53d1\u9001\u5fc3\u8df3\u8bf7\u6c42\u9891\u7387\u7684\u53c2\u6570\uff0c\u5c31\u662fheartbeat.interval.ms\u3002<\/p>\n<p>\u8fd9\u4e2a\u503c\u8bbe\u7f6e\u5f97\u8d8a\u5c0f\uff0cConsumer\u5b9e\u4f8b\u53d1\u9001\u5fc3\u8df3\u8bf7\u6c42\u7684\u9891\u7387\u5c31\u8d8a\u9ad8\u3002<\/p>\n<p>\u9891\u7e41\u5730\u53d1\u9001\u5fc3\u8df3\u8bf7\u6c42\u4f1a\u989d\u5916\u6d88\u8017\u5e26\u5bbd\u8d44\u6e90\uff0c\u4f46\u597d\u5904\u662f\u80fd\u591f\u66f4\u52a0\u5feb\u901f\u5730\u77e5\u6653\u5f53\u524d\u662f\u5426\u5f00\u542fRebalance\uff0c\u56e0\u4e3a\uff0c\u76ee\u524dCoordinator\u901a\u77e5\u5404\u4e2aConsumer\u5b9e\u4f8b\u5f00\u542fRebalance\u7684\u65b9\u6cd5\uff0c\u5c31\u662f\u5c06REBALANCE_NEEDED\u6807\u5fd7\u5c01\u88c5\u8fdb\u5fc3\u8df3\u8bf7\u6c42\u7684\u54cd\u5e94\u4f53\u4e2d\u3002<\/p>\n<p>\u9664\u4e86\u4ee5\u4e0a\u4e24\u4e2a\u53c2\u6570\uff0cConsumer\u7aef\u8fd8\u6709\u4e00\u4e2a\u53c2\u6570\uff0c\u7528\u4e8e\u63a7\u5236Consumer\u5b9e\u9645\u6d88\u8d39\u80fd\u529b\u5bf9Rebalance\u7684\u5f71\u54cd\uff0c\u5373max.poll.interval.ms\u53c2\u6570\u3002<\/p>\n<p>\u5b83\u9650\u5b9a\u4e86Consumer\u7aef\u5e94\u7528\u7a0b\u5e8f\u4e24\u6b21\u8c03\u7528poll\u65b9\u6cd5\u7684\u6700\u5927\u65f6\u95f4\u95f4\u9694\u3002<\/p>\n<p>\u5b83\u7684\u9ed8\u8ba4\u503c\u662f5\u5206\u949f\uff0c\u8868\u793a\u4f60\u7684Consumer\u7a0b\u5e8f\u5982\u679c\u57285\u5206\u949f\u4e4b\u5185\u65e0\u6cd5\u6d88\u8d39\u5b8cpoll\u65b9\u6cd5\u8fd4\u56de\u7684\u6d88\u606f\uff0c\u90a3\u4e48Consumer\u4f1a\u4e3b\u52a8\u53d1\u8d77\u79bb\u5f00\u7ec4\u7684\u8bf7\u6c42\uff0cCoordinator\u4e5f\u4f1a\u5f00\u542f\u65b0\u4e00\u8f6eRebalance\u3002<\/p>\n<h3>\u300c\u53ef\u907f\u514dRebalance\u7684\u914d\u7f6e\u300d<\/h3>\n<p>\u7b2c\u4e00\u7c7bRebalance\u662f\u56e0\u4e3a\u672a\u80fd\u53ca\u65f6\u53d1\u9001\u5fc3\u8df3\uff0c\u5bfc\u81f4Consumer\u88ab\u8e22\u51faGroup\u800c\u5f15\u53d1\u7684<\/p>\n<p>\u56e0\u6b64\u53ef\u4ee5\u8bbe\u7f6e\u300csession.timeout.ms\u548cheartbeat.interval.ms\u300d\u7684\u503c\u3002<\/p>\n<ul>\n<li>\u8bbe\u7f6esession.timeout.ms = 6s\u3002<\/li>\n<li>\u8bbe\u7f6eheartbeat.interval.ms = 2s\u3002<\/li>\n<li>\u8981\u4fdd\u8bc1Consumer\u5b9e\u4f8b\u5728\u88ab\u5224\u5b9a\u4e3adead\u4e4b\u524d\uff0c\u80fd\u591f\u53d1\u9001\u81f3\u5c113\u8f6e\u7684\u5fc3\u8df3\u8bf7\u6c42\uff0c\u5373session.timeout.ms &gt;= 3 * heartbeat.interval.ms\u3002<\/li>\n<\/ul>\n<p>\u5c06session.timeout.ms\u8bbe\u7f6e\u62106s\u4e3b\u8981\u662f\u4e3a\u4e86\u8ba9Coordinator\u80fd\u591f\u66f4\u5feb\u5730\u5b9a\u4f4d\u5df2\u7ecf\u6302\u6389\u7684Consumer\u3002<\/p>\n<h3>\u300c\u7b2c\u4e8c\u7c7bRebalance\u662fConsumer\u6d88\u8d39\u65f6\u95f4\u8fc7\u957f\u5bfc\u81f4\u7684\u300d\u3002<\/h3>\n<p>\u4f60\u8981\u4e3a\u4f60\u7684\u4e1a\u52a1\u5904\u7406\u903b\u8f91\u7559\u4e0b\u5145\u8db3\u7684\u65f6\u95f4\uff0c\u8fd9\u6837Consumer\u5c31\u4e0d\u4f1a\u56e0\u4e3a\u5904\u7406\u8fd9\u4e9b\u6d88\u606f\u7684\u65f6\u95f4\u592a\u957f\u800c\u5f15\u53d1Rebalance\u4e86\u3002<\/p>\n<h2>ConsumerOffsets<\/h2>\n<h3>\u300cKafka\u5c06Consumer\u7684\u4f4d\u79fb\u6570\u636e\u4f5c\u4e3a\u4e00\u6761\u6761\u666e\u901a\u7684Kafka\u6d88\u606f\uff0c\u63d0\u4ea4\u5230__consumer_offsets\u4e2d\u3002\u300d<\/h3>\n<h3>\u300c__consumer_offsets\u7684\u4e3b\u8981\u4f5c\u7528\u662f\u4fdd\u5b58Kafka\u6d88\u8d39\u8005\u7684\u4f4d\u79fb\u4fe1\u606f\u3002\u300d<\/h3>\n<p>\u5b83\u8981\u6c42\u8fd9\u4e2a\u63d0\u4ea4\u8fc7\u7a0b\u4e0d\u4ec5\u8981\u5b9e\u73b0\u9ad8\u6301\u4e45\u6027\uff0c\u8fd8\u8981\u652f\u6301\u9ad8\u9891\u7684\u5199\u64cd\u4f5c\u3002<\/p>\n<p>__consumer_offsets\u4e3b\u9898\u5c31\u662f\u666e\u901a\u7684Kafka\u4e3b\u9898\u3002\u4f60\u53ef\u4ee5\u624b\u52a8\u5730\u521b\u5efa\u5b83\u3001\u4fee\u6539\u5b83\uff0c\u751a\u81f3\u662f\u5220\u9664\u5b83\u3002<\/p>\n<p>\u867d\u8bf4__consumer_offsets\u4e3b\u9898\u662f\u4e00\u4e2a\u666e\u901a\u7684Kafka\u4e3b\u9898\uff0c\u4f46\u300c\u5b83\u7684\u6d88\u606f\u683c\u5f0f\u5374\u662fKafka\u81ea\u5df1\u5b9a\u4e49\u7684\u300d\uff0c\u7528\u6237\u4e0d\u80fd\u4fee\u6539\uff0c\u4e5f\u5c31\u662f\u8bf4\u4f60\u4e0d\u80fd\u968f\u610f\u5730\u5411\u8fd9\u4e2a\u4e3b\u9898\u5199\u6d88\u606f\uff0c\u56e0\u4e3a\u4e00\u65e6\u4f60\u5199\u5165\u7684\u6d88\u606f\u4e0d\u6ee1\u8db3Kafka\u89c4\u5b9a\u7684\u683c\u5f0f\uff0c\u90a3\u4e48Kafka\u5185\u90e8\u65e0\u6cd5\u6210\u529f\u89e3\u6790\uff0c\u5c31\u4f1a\u9020\u6210Broker\u7684\u5d29\u6e83\u3002<\/p>\n<p>Kafka Consumer\u6709API\u5e2e\u4f60\u63d0\u4ea4\u4f4d\u79fb\uff0c\u4e5f\u5c31\u662f\u5411__consumer_offsets\u4e3b\u9898\u5199\u6d88\u606f\uff0c\u5343\u4e07\u4e0d\u8981\u81ea\u5df1\u5199\u4e2aProducer\u968f\u610f\u5411\u8be5\u4e3b\u9898\u53d1\u9001\u6d88\u606f\u3002<\/p>\n<p>__consumer_offsets\u67093\u79cd\u6d88\u606f\u683c\u5f0f\uff1a<\/p>\n<ol>\n<li>\u7528\u4e8e\u4fdd\u5b58Consumer Group\u4fe1\u606f\u7684\u6d88\u606f\u3002<\/li>\n<li>\u7528\u4e8e\u5220\u9664Group\u8fc7\u671f\u4f4d\u79fb\u751a\u81f3\u662f\u5220\u9664Group\u7684\u6d88\u606f\u3002<\/li>\n<li>\u4fdd\u5b58\u4e86\u4f4d\u79fb\u503c\u3002<\/li>\n<\/ol>\n<p>\u7b2c2\u79cd\u683c\u5f0f\u5b83\u6709\u4e2a\u4e13\u5c5e\u7684\u540d\u5b57\uff1atombstone\u6d88\u606f\uff0c\u5373\u5893\u7891\u6d88\u606f\uff0c\u4e5f\u79f0delete mark\uff0c\u5b83\u7684\u4e3b\u8981\u7279\u70b9\u662f\u5b83\u7684\u6d88\u606f\u4f53\u662fnull\uff0c\u5373\u7a7a\u6d88\u606f\u4f53\u3002<\/p>\n<p>\u4e00\u65e6\u67d0\u4e2aConsumer Group\u4e0b\u7684\u6240\u6709Consumer\u5b9e\u4f8b\u90fd\u505c\u6b62\u4e86\uff0c\u800c\u4e14\u5b83\u4eec\u7684\u4f4d\u79fb\u6570\u636e\u90fd\u5df2\u88ab\u5220\u9664\u65f6\uff0cKafka\u4f1a\u5411__consumer_offsets\u4e3b\u9898\u7684\u5bf9\u5e94\u5206\u533a\u5199\u5165tombstone\u6d88\u606f\uff0c\u8868\u660e\u8981\u5f7b\u5e95\u5220\u9664\u8fd9\u4e2aGroup\u7684\u4fe1\u606f\u3002<\/p>\n<p>__consumer_offsets\u662f\u600e\u4e48\u88ab\u521b\u5efa\u7684?<\/p>\n<p>\u901a\u5e38\u6765\u8bf4\uff0c\u300c\u5f53Kafka\u96c6\u7fa4\u4e2d\u7684\u7b2c\u4e00\u4e2aConsumer\u7a0b\u5e8f\u542f\u52a8\u65f6\uff0cKafka\u4f1a\u81ea\u52a8\u521b\u5efa\u4f4d\u79fb\u4e3b\u9898\u300d\u3002<\/p>\n<h3>\u300c\u9ed8\u8ba4\u8be5\u4e3b\u9898\u7684\u5206\u533a\u6570\u662f50\uff0c\u526f\u672c\u6570\u662f3\u300d\u3002<\/h3>\n<p>\u76ee\u524dKafka Consumer\u63d0\u4ea4\u4f4d\u79fb\u7684\u65b9\u5f0f\u6709\u4e24\u79cd\uff1a\u300c\u81ea\u52a8\u63d0\u4ea4\u4f4d\u79fb\u548c\u624b\u52a8\u63d0\u4ea4\u4f4d\u79fb\u3002\u300d<\/p>\n<p>Consumer\u7aef\u6709\u4e2a\u53c2\u6570\u53ebenable.auto.commit\uff0c\u5982\u679c\u503c\u662ftrue\uff0c\u5219Consumer\u5728\u540e\u53f0\u9ed8\u9ed8\u5730\u4e3a\u4f60\u5b9a\u671f\u63d0\u4ea4\u4f4d\u79fb\uff0c\u63d0\u4ea4\u95f4\u9694\u7531\u4e00\u4e2a\u4e13\u5c5e\u7684\u53c2\u6570auto.commit.interval.ms\u6765\u63a7\u5236\u3002<\/p>\n<p>\u81ea\u52a8\u63d0\u4ea4\u4f4d\u79fb\u6709\u4e00\u4e2a\u663e\u8457\u7684\u4f18\u70b9\uff0c\u5c31\u662f\u7701\u4e8b\uff0c\u4f60\u4e0d\u7528\u64cd\u5fc3\u4f4d\u79fb\u63d0\u4ea4\u7684\u4e8b\u60c5\uff0c\u5c31\u80fd\u4fdd\u8bc1\u6d88\u606f\u6d88\u8d39\u4e0d\u4f1a\u4e22\u5931\u3002<\/p>\n<p>\u4f46\u8fd9\u4e00\u70b9\u540c\u65f6\u4e5f\u662f\u7f3a\u70b9\uff0c\u4e27\u5931\u4e86\u5f88\u5927\u7684\u7075\u6d3b\u6027\u548c\u53ef\u63a7\u6027\uff0c\u4f60\u5b8c\u5168\u6ca1\u6cd5\u628a\u63a7Consumer\u7aef\u7684\u4f4d\u79fb\u7ba1\u7406\u3002<\/p>\n<p>Kafka Consumer API\u4e3a\u4f60\u63d0\u4f9b\u4e86\u4f4d\u79fb\u63d0\u4ea4\u7684\u65b9\u6cd5\uff0c\u5982consumer.commitSync\u7b49\u3002<\/p>\n<p>\u5f53\u8c03\u7528\u8fd9\u4e9b\u65b9\u6cd5\u65f6\uff0cKafka\u4f1a\u5411__consumer_offsets\u4e3b\u9898\u5199\u5165\u76f8\u5e94\u7684\u6d88\u606f\u3002<\/p>\n<p>\u5982\u679c\u4f60\u9009\u62e9\u7684\u662f\u81ea\u52a8\u63d0\u4ea4\u4f4d\u79fb\uff0c\u90a3\u4e48\u5c31\u53ef\u80fd\u5b58\u5728\u4e00\u4e2a\u95ee\u9898\uff1a\u53ea\u8981Consumer\u4e00\u76f4\u542f\u52a8\u7740\uff0c\u5b83\u5c31\u4f1a\u65e0\u9650\u671f\u5730\u5411\u4f4d\u79fb\u4e3b\u9898\u5199\u5165\u6d88\u606f\u3002<\/p>\n<h3>\u300c\u4e3e\u4e2a\u6781\u7aef\u4e00\u70b9\u7684\u4f8b\u5b50\u3002\u300d<\/h3>\n<p>\u5047\u8bbeConsumer\u5f53\u524d\u6d88\u8d39\u5230\u4e86\u67d0\u4e2a\u4e3b\u9898\u7684\u6700\u65b0\u4e00\u6761\u6d88\u606f\uff0c\u4f4d\u79fb\u662f100\uff0c\u4e4b\u540e\u8be5\u4e3b\u9898\u6ca1\u6709\u4efb\u4f55\u65b0\u6d88\u606f\u4ea7\u751f\uff0c\u6545Consumer\u65e0\u6d88\u606f\u53ef\u6d88\u8d39\u4e86\uff0c\u6240\u4ee5\u4f4d\u79fb\u6c38\u8fdc\u4fdd\u6301\u5728100\u3002<\/p>\n<p>\u7531\u4e8e\u662f\u81ea\u52a8\u63d0\u4ea4\u4f4d\u79fb\uff0c\u4f4d\u79fb\u4e3b\u9898\u4e2d\u4f1a\u4e0d\u505c\u5730\u5199\u5165\u4f4d\u79fb=100\u7684\u6d88\u606f\u3002<\/p>\n<p>\u663e\u7136Kafka\u53ea\u9700\u8981\u4fdd\u7559\u8fd9\u7c7b\u6d88\u606f\u4e2d\u7684\u6700\u65b0\u4e00\u6761\u5c31\u53ef\u4ee5\u4e86\uff0c\u4e4b\u524d\u7684\u6d88\u606f\u90fd\u662f\u53ef\u4ee5\u5220\u9664\u7684\u3002<\/p>\n<p>\u8fd9\u5c31\u8981\u6c42Kafka\u5fc5\u987b\u8981\u6709\u9488\u5bf9\u4f4d\u79fb\u4e3b\u9898\u6d88\u606f\u7279\u70b9\u7684\u6d88\u606f\u5220\u9664\u7b56\u7565\uff0c\u5426\u5219\u8fd9\u79cd\u6d88\u606f\u4f1a\u8d8a\u6765\u8d8a\u591a\uff0c\u6700\u7ec8\u6491\u7206\u6574\u4e2a\u78c1\u76d8\u3002<\/p>\n<h3>\u300cCompact\u7b56\u7565\u300d<\/h3>\n<p>Kafka\u4f7f\u7528\u300cCompact\u7b56\u7565\u300d\u6765\u5220\u9664__consumer_offsets\u4e3b\u9898\u4e2d\u7684\u8fc7\u671f\u6d88\u606f\uff0c\u907f\u514d\u8be5\u4e3b\u9898\u65e0\u9650\u671f\u81a8\u80c0\u3002<\/p>\n<p>\u6bd4\u5982\u5bf9\u4e8e\u540c\u4e00\u4e2aKey\u7684\u4e24\u6761\u6d88\u606fM1\u548cM2\uff0c\u5982\u679cM1\u7684\u53d1\u9001\u65f6\u95f4\u65e9\u4e8eM2\uff0c\u90a3\u4e48M1\u5c31\u662f\u8fc7\u671f\u6d88\u606f\u3002<\/p>\n<p>Compact\u7684\u8fc7\u7a0b\u5c31\u662f\u626b\u63cf\u65e5\u5fd7\u7684\u6240\u6709\u6d88\u606f\uff0c\u5254\u9664\u90a3\u4e9b\u8fc7\u671f\u7684\u6d88\u606f\uff0c\u7136\u540e\u628a\u5269\u4e0b\u7684\u6d88\u606f\u6574\u7406\u5728\u4e00\u8d77\u3002<\/p>\n<p>\u6211\u5728\u8fd9\u91cc\u8d34\u4e00\u5f20\u6765\u81ea\u5b98\u7f51\u7684\u56fe\u7247\uff0c\u6765\u8bf4\u660eCompact\u8fc7\u7a0b\u3002<\/p>\n<p style=\"text-align: center\">\n<p>\u56fe\u4e2d\u4f4d\u79fb\u4e3a0\u30012\u548c3\u7684\u6d88\u606f\u7684Key\u90fd\u662fK1\uff0cCompact\u4e4b\u540e\uff0c\u5206\u533a\u53ea\u9700\u8981\u4fdd\u5b58\u4f4d\u79fb\u4e3a3\u7684\u6d88\u606f\uff0c\u56e0\u4e3a\u5b83\u662f\u6700\u65b0\u53d1\u9001\u7684\u3002<\/p>\n<h3>\u300cKafka\u63d0\u4f9b\u4e86\u4e13\u95e8\u7684\u540e\u53f0\u7ebf\u7a0b\u5b9a\u671f\u5730\u5de1\u68c0\u5f85Compact\u7684\u4e3b\u9898\uff0c\u770b\u770b\u662f\u5426\u5b58\u5728\u6ee1\u8db3\u6761\u4ef6\u7684\u53ef\u5220\u9664\u6570\u636e\u300d\u3002<\/h3>\n<p>\u8fd9\u4e2a\u540e\u53f0\u7ebf\u7a0b\u53ebLog Cleaner\u3002<\/p>\n<p>\u5f88\u591a\u5b9e\u9645\u751f\u4ea7\u73af\u5883\u4e2d\u90fd\u51fa\u73b0\u8fc7\u4f4d\u79fb\u4e3b\u9898\u65e0\u9650\u81a8\u80c0\u5360\u7528\u8fc7\u591a\u78c1\u76d8\u7a7a\u95f4\u7684\u95ee\u9898\uff0c\u5982\u679c\u4f60\u7684\u73af\u5883\u4e2d\u4e5f\u6709\u8fd9\u4e2a\u95ee\u9898\uff0c\u5efa\u8bae\u4f60\u53bb\u68c0\u67e5\u4e00\u4e0bLog Cleaner\u7ebf\u7a0b\u7684\u72b6\u6001\uff0c\u901a\u5e38\u90fd\u662f\u8fd9\u4e2a\u7ebf\u7a0b\u6302\u6389\u4e86\u5bfc\u81f4\u7684\u3002<\/p>\n<h2>\u526f\u672c\u673a\u5236<\/h2>\n<p>\u6839\u636eKafka\u526f\u672c\u673a\u5236\u7684\u5b9a\u4e49\uff0c\u540c\u4e00\u4e2a\u5206\u533a\u4e0b\u7684\u6240\u6709\u526f\u672c\u4fdd\u5b58\u6709\u76f8\u540c\u7684\u6d88\u606f\u5e8f\u5217\uff0c\u8fd9\u4e9b\u526f\u672c\u5206\u6563\u4fdd\u5b58\u5728\u4e0d\u540c\u7684Broker\u4e0a\uff0c\u4ece\u800c\u80fd\u591f\u5bf9\u6297\u90e8\u5206Broker\u5b95\u673a\u5e26\u6765\u7684\u6570\u636e\u4e0d\u53ef\u7528\u3002<\/p>\n<p>\u4e0b\u9762\u5c55\u793a\u7684\u662f\u4e00\u4e2a\u67093\u53f0Broker\u7684Kafka\u96c6\u7fa4\u4e0a\u7684\u526f\u672c\u5206\u5e03\u60c5\u51b5\u3002<\/p>\n<p>\u4ece\u8fd9\u5f20\u56fe\u4e2d\uff0c\u6211\u4eec\u53ef\u4ee5\u770b\u5230\uff0c\u4e3b\u98981\u5206\u533a0\u76843\u4e2a\u526f\u672c\u5206\u6563\u57283\u53f0Broker\u4e0a\uff0c\u5176\u4ed6\u4e3b\u9898\u5206\u533a\u7684\u526f\u672c\u4e5f\u90fd\u6563\u843d\u5728\u4e0d\u540c\u7684Broker\u4e0a\uff0c\u4ece\u800c\u5b9e\u73b0\u6570\u636e\u5197\u4f59\u3002<\/p>\n<p style=\"text-align: center\">\n<h3>\u300c\u526f\u672c\u89d2\u8272\u300d<\/h3>\n<p style=\"text-align: center\">\n<p>\u5728Kafka\u4e2d\uff0c\u526f\u672c\u5206\u6210\u4e24\u7c7b\uff1a\u9886\u5bfc\u8005\u526f\u672c(Leader Replica)\u548c\u8ffd\u968f\u8005\u526f\u672c(Follower Replica)\u3002<\/p>\n<p>\u6bcf\u4e2a\u5206\u533a\u5728\u521b\u5efa\u65f6\u90fd\u8981\u9009\u4e3e\u4e00\u4e2a\u526f\u672c\uff0c\u79f0\u4e3a\u9886\u5bfc\u8005\u526f\u672c\uff0c\u5176\u4f59\u7684\u526f\u672c\u81ea\u52a8\u79f0\u4e3a\u8ffd\u968f\u8005\u526f\u672c\u3002<\/p>\n<p>\u5728Kafka\u4e2d\uff0c\u8ffd\u968f\u8005\u526f\u672c\u662f\u4e0d\u5bf9\u5916\u63d0\u4f9b\u670d\u52a1\u7684\u3002\u8fd9\u5c31\u662f\u8bf4\uff0c\u4efb\u4f55\u4e00\u4e2a\u8ffd\u968f\u8005\u526f\u672c\u90fd\u4e0d\u80fd\u54cd\u5e94\u6d88\u8d39\u8005\u548c\u751f\u4ea7\u8005\u7684\u8bfb\u5199\u8bf7\u6c42\u3002\u6240\u6709\u7684\u8bf7\u6c42\u90fd\u5fc5\u987b\u7531\u9886\u5bfc\u8005\u526f\u672c\u6765\u5904\u7406\uff0c\u6216\u8005\u8bf4\uff0c\u6240\u6709\u7684\u8bfb\u5199\u8bf7\u6c42\u90fd\u5fc5\u987b\u53d1\u5f80\u9886\u5bfc\u8005\u526f\u672c\u6240\u5728\u7684Broker\uff0c\u7531\u8be5Broker\u8d1f\u8d23\u5904\u7406\u3002<\/p>\n<p>\u8ffd\u968f\u8005\u526f\u672c\u4e0d\u5904\u7406\u5ba2\u6237\u7aef\u8bf7\u6c42\uff0c\u5b83\u552f\u4e00\u7684\u4efb\u52a1\u5c31\u662f\u4ece\u9886\u5bfc\u8005\u526f\u672c\u300c\u5f02\u6b65\u62c9\u53d6\u300d\u6d88\u606f\uff0c\u5e76\u5199\u5165\u5230\u81ea\u5df1\u7684\u63d0\u4ea4\u65e5\u5fd7\u4e2d\uff0c\u4ece\u800c\u5b9e\u73b0\u4e0e\u9886\u5bfc\u8005\u526f\u672c\u7684\u540c\u6b65\u3002<\/p>\n<p>\u5f53\u9886\u5bfc\u8005\u526f\u672c\u6302\u6389\u4e86\uff0c\u6216\u8005\u8bf4\u9886\u5bfc\u8005\u526f\u672c\u6240\u5728\u7684Broker\u5b95\u673a\u65f6\uff0cKafka\u4f9d\u6258\u4e8eZooKeeper\u63d0\u4f9b\u7684\u76d1\u63a7\u529f\u80fd\u80fd\u591f\u5b9e\u65f6\u611f\u77e5\u5230\uff0c\u5e76\u7acb\u5373\u5f00\u542f\u65b0\u4e00\u8f6e\u7684\u9886\u5bfc\u8005\u9009\u4e3e\uff0c\u4ece\u8ffd\u968f\u8005\u526f\u672c\u4e2d\u9009\u4e00\u4e2a\u4f5c\u4e3a\u65b0\u7684\u9886\u5bfc\u8005\u3002\u8001Leader\u526f\u672c\u91cd\u542f\u56de\u6765\u540e\uff0c\u53ea\u80fd\u4f5c\u4e3a\u8ffd\u968f\u8005\u526f\u672c\u52a0\u5165\u5230\u96c6\u7fa4\u4e2d\u3002<\/p>\n<p>\u5bf9\u4e8e\u5ba2\u6237\u7aef\u7528\u6237\u800c\u8a00\uff0cKafka\u7684\u8ffd\u968f\u8005\u526f\u672c\u6ca1\u6709\u4efb\u4f55\u4f5c\u7528\uff0cKafka\u4e3a\u4ec0\u4e48\u8981\u8fd9\u6837\u8bbe\u8ba1\u5462?<\/p>\n<p>\u8fd9\u79cd\u526f\u672c\u673a\u5236\u6709\u4e24\u4e2a\u65b9\u9762\u7684\u597d\u5904\u3002<\/p>\n<h3>1.\u300c\u65b9\u4fbf\u5b9e\u73b0Read-your-writes\u300d\u3002<\/h3>\n<p>\u6240\u8c13Read-your-writes\uff0c\u987e\u540d\u601d\u4e49\u5c31\u662f\uff0c\u5f53\u4f60\u4f7f\u7528\u751f\u4ea7\u8005API\u5411Kafka\u6210\u529f\u5199\u5165\u6d88\u606f\u540e\uff0c\u9a6c\u4e0a\u4f7f\u7528\u6d88\u8d39\u8005API\u53bb\u8bfb\u53d6\u521a\u624d\u751f\u4ea7\u7684\u6d88\u606f\u3002<\/p>\n<h3>2.\u300c\u65b9\u4fbf\u5b9e\u73b0\u5355\u8c03\u8bfb(Monotonic Reads)\u300d\u3002<\/h3>\n<p>\u5047\u8bbe\u5f53\u524d\u67092\u4e2a\u8ffd\u968f\u8005\u526f\u672cF1\u548cF2\uff0c\u5b83\u4eec\u5f02\u6b65\u5730\u62c9\u53d6\u9886\u5bfc\u8005\u526f\u672c\u6570\u636e\u3002\u5018\u82e5F1\u62c9\u53d6\u4e86Leader\u7684\u6700\u65b0\u6d88\u606f\u800cF2\u8fd8\u672a\u53ca\u65f6\u62c9\u53d6\uff0c\u90a3\u4e48\uff0c\u6b64\u65f6\u5982\u679c\u6709\u4e00\u4e2a\u6d88\u8d39\u8005\u5148\u4eceF1\u8bfb\u53d6\u6d88\u606f\u4e4b\u540e\u53c8\u4eceF2\u62c9\u53d6\u6d88\u606f\uff0c\u5b83\u53ef\u80fd\u4f1a\u770b\u5230\u8fd9\u6837\u7684\u73b0\u8c61\uff1a\u7b2c\u4e00\u6b21\u6d88\u8d39\u65f6\u770b\u5230\u7684\u6700\u65b0\u6d88\u606f\u5728\u7b2c\u4e8c\u6b21\u6d88\u8d39\u65f6\u4e0d\u89c1\u4e86\uff0c\u8fd9\u5c31\u4e0d\u662f\u5355\u8c03\u8bfb\u4e00\u81f4\u6027\u3002<\/p>\n<p>\u4f46\u662f\uff0c\u5982\u679c\u6240\u6709\u7684\u8bfb\u8bf7\u6c42\u90fd\u662f\u7531Leader\u6765\u5904\u7406\uff0c\u90a3\u4e48Kafka\u5c31\u5f88\u5bb9\u6613\u5b9e\u73b0\u5355\u8c03\u8bfb\u4e00\u81f4\u6027\u3002<\/p>\n<h2>ISR\u673a\u5236<\/h2>\n<p>In-sync Replicas\uff0c\u4e5f\u5c31\u662f\u6240\u8c13\u7684ISR\u526f\u672c\u96c6\u5408\u3002<\/p>\n<p>ISR\u4e2d\u7684\u526f\u672c\u90fd\u662f\u4e0eLeader\u540c\u6b65\u7684\u526f\u672c\uff0c\u76f8\u53cd\uff0c\u4e0d\u5728ISR\u4e2d\u7684\u8ffd\u968f\u8005\u526f\u672c\u5c31\u88ab\u8ba4\u4e3a\u662f\u4e0eLeader\u4e0d\u540c\u6b65\u7684\u3002<\/p>\n<ul>\n<li>\u4ec0\u4e48\u526f\u672c\u80fd\u591f\u8fdb\u5165\u5230ISR\u4e2d\u5462?<\/li>\n<\/ul>\n<p>Leader\u526f\u672c\u5929\u7136\u5c31\u5728ISR\u4e2d\u3002\u4e5f\u5c31\u662f\u8bf4\uff0c\u300cISR\u4e0d\u53ea\u662f\u8ffd\u968f\u8005\u526f\u672c\u96c6\u5408\uff0c\u5b83\u5fc5\u7136\u5305\u62ecLeader\u526f\u672c\u3002\u751a\u81f3\u5728\u67d0\u4e9b\u60c5\u51b5\u4e0b\uff0cISR\u53ea\u6709Leader\u8fd9\u4e00\u4e2a\u526f\u672c\u300d\u3002<\/p>\n<p>\u53e6\u5916\uff0c\u80fd\u591f\u8fdb\u5165\u5230ISR\u7684\u8ffd\u968f\u8005\u526f\u672c\u8981\u6ee1\u8db3\u4e00\u5b9a\u7684\u6761\u4ef6\u3002<\/p>\n<h3>\u300c\u901a\u8fc7Broker\u7aef\u53c2\u6570replica.lag.time.max.ms\u53c2\u6570\u503c\u300d\u3002<\/h3>\n<p>\u8fd9\u4e2a\u53c2\u6570\u7684\u542b\u4e49\u662fFollower\u526f\u672c\u80fd\u591f\u843d\u540eLeader\u526f\u672c\u7684\u6700\u957f\u65f6\u95f4\u95f4\u9694\uff0c\u5f53\u524d\u9ed8\u8ba4\u503c\u662f10\u79d2\u3002<\/p>\n<p>\u8fd9\u5c31\u662f\u8bf4\uff0c\u53ea\u8981\u4e00\u4e2aFollower\u526f\u672c\u843d\u540eLeader\u526f\u672c\u7684\u65f6\u95f4\u4e0d\u8fde\u7eed\u8d85\u8fc710\u79d2\uff0c\u90a3\u4e48Kafka\u5c31\u8ba4\u4e3a\u8be5Follower\u526f\u672c\u4e0eLeader\u662f\u540c\u6b65\u7684\uff0c\u5373\u4f7f\u6b64\u65f6Follower\u526f\u672c\u4e2d\u4fdd\u5b58\u7684\u6d88\u606f\u660e\u663e\u5c11\u4e8eLeader\u526f\u672c\u4e2d\u7684\u6d88\u606f\u3002<\/p>\n<p>Follower\u526f\u672c\u552f\u4e00\u7684\u5de5\u4f5c\u5c31\u662f\u4e0d\u65ad\u5730\u4eceLeader\u526f\u672c\u62c9\u53d6\u6d88\u606f\uff0c\u7136\u540e\u5199\u5165\u5230\u81ea\u5df1\u7684\u63d0\u4ea4\u65e5\u5fd7\u4e2d\u3002<\/p>\n<p>\u5018\u82e5\u8be5\u526f\u672c\u540e\u9762\u6162\u6162\u5730\u8ffd\u4e0a\u4e86Leader\u7684\u8fdb\u5ea6\uff0c\u90a3\u4e48\u5b83\u662f\u80fd\u591f\u91cd\u65b0\u88ab\u52a0\u56deISR\u7684\u3002<\/p>\n<p>ISR\u662f\u4e00\u4e2a\u52a8\u6001\u8c03\u6574\u7684\u96c6\u5408\uff0c\u800c\u975e\u9759\u6001\u4e0d\u53d8\u7684\u3002<\/p>\n<p>Unclean\u9886\u5bfc\u8005\u9009\u4e3e\u300cKafka\u628a\u6240\u6709\u4e0d\u5728ISR\u4e2d\u7684\u5b58\u6d3b\u526f\u672c\u90fd\u79f0\u4e3a\u975e\u540c\u6b65\u526f\u672c\u300d\u3002<\/p>\n<p>\u901a\u5e38\u6765\u8bf4\uff0c\u975e\u540c\u6b65\u526f\u672c\u843d\u540eLeader\u592a\u591a\uff0c\u56e0\u6b64\uff0c\u5982\u679c\u9009\u62e9\u8fd9\u4e9b\u526f\u672c\u4f5c\u4e3a\u65b0Leader\uff0c\u5c31\u53ef\u80fd\u51fa\u73b0\u6570\u636e\u7684\u4e22\u5931\u3002<\/p>\n<p>\u6bd5\u7adf\uff0c\u8fd9\u4e9b\u526f\u672c\u4e2d\u4fdd\u5b58\u7684\u6d88\u606f\u8fdc\u8fdc\u843d\u540e\u4e8e\u8001Leader\u4e2d\u7684\u6d88\u606f\u3002<\/p>\n<p>\u5728Kafka\u4e2d\uff0c\u9009\u4e3e\u8fd9\u79cd\u526f\u672c\u7684\u8fc7\u7a0b\u79f0\u4e3aUnclean\u9886\u5bfc\u8005\u9009\u4e3e\u3002<\/p>\n<h3>\u300cBroker\u7aef\u53c2\u6570unclean.leader.election.enable\u63a7\u5236\u662f\u5426\u5141\u8bb8Unclean\u9886\u5bfc\u8005\u9009\u4e3e\u300d\u3002<\/h3>\n<p>\u5f00\u542fUnclean\u9886\u5bfc\u8005\u9009\u4e3e\u53ef\u80fd\u4f1a\u9020\u6210\u6570\u636e\u4e22\u5931\uff0c\u4f46\u597d\u5904\u662f\uff0c\u5b83\u4f7f\u5f97\u5206\u533aLeader\u526f\u672c\u4e00\u76f4\u5b58\u5728\uff0c\u4e0d\u81f3\u4e8e\u505c\u6b62\u5bf9\u5916\u63d0\u4f9b\u670d\u52a1\uff0c\u56e0\u6b64\u63d0\u5347\u4e86\u9ad8\u53ef\u7528\u6027\u3002\u53cd\u4e4b\uff0c\u7981\u6b62Unclean\u9886\u5bfc\u8005\u9009\u4e3e\u7684\u597d\u5904\u5728\u4e8e\u7ef4\u62a4\u4e86\u6570\u636e\u7684\u4e00\u81f4\u6027\uff0c\u907f\u514d\u4e86\u6d88\u606f\u4e22\u5931\uff0c\u4f46\u727a\u7272\u4e86\u9ad8\u53ef\u7528\u6027\u3002<\/p>\n<h2>\u526f\u672c\u9009\u4e3e<\/h2>\n<p>\u5bf9\u4e8ekafka\u96c6\u7fa4\u4e2d\u5bf9\u4e8e\u4efb\u610f\u7684topic\u7684\u5206\u533a\u4ee5\u53ca\u526f\u672cleader\u7684\u8bbe\u5b9a\uff0c\u90fd\u9700\u8981\u8003\u8651\u5230\u96c6\u7fa4\u6574\u4f53\u7684\u8d1f\u8f7d\u80fd\u529b\u7684\u5e73\u8861\u6027\uff0c\u4f1a\u5c3d\u91cf\u5206\u914d\u6bcf\u4e00\u4e2apartition\u7684\u526f\u672cleader\u5728\u4e0d\u540c\u7684broker\u4e2d\uff0c\u8fd9\u6837\u4f1a\u907f\u514d\u591a\u4e2aleader\u5728\u540c\u4e00\u4e2abroker\uff0c\u5bfc\u81f4\u96c6\u7fa4\u4e2d\u7684broker\u8d1f\u8f7d\u4e0d\u5e73\u8861<\/p>\n<p>kafka\u5f15\u5165\u4e86\u4f18\u5148\u526f\u672c\u7684\u6982\u5ff5\uff0c\u4f18\u5148\u526f\u672c\u7684\u610f\u601d\u5728AR(\u5206\u533a\u4e2d\u7684\u6240\u6709\u526f\u672c)\u96c6\u5408\u5217\u8868\u4e2d\u7684\u7b2c\u4e00\u4e2a\u526f\u672c\uff0c\u5728\u7406\u60f3\u72b6\u6001\u4e0b\u8be5\u526f\u672c\u5c31\u662f\u8be5\u5206\u533a\u7684leader\u526f\u672c<\/p>\n<p>\u4f8b\u5982kafka\u96c6\u7fa4\u75313\u53f0broker\u7ec4\u6210\uff0c\u521b\u5efa\u4e86\u4e00\u4e2a\u540d\u4e3atopic-partitions\u7684topic\uff0c\u8bbe\u7f6epartition\u4e3a3\uff0c\u526f\u672c\u6570\u4e3a3\uff0cpartition0\u4e2dAR\u5217\u8868\u4e3a [1,2,0]\uff0c\u90a3\u4e48\u5206\u533a0\u7684\u4f18\u5148\u526f\u672c\u4e3a1<\/p>\n<p>kafka\u4f7f\u7528\u591a\u526f\u672c\u673a\u5236\u63d0\u9ad8\u53ef\u9760\u6027\uff0c\u4f46\u662f\u53ea\u6709leader\u526f\u672c\u5bf9\u5916\u63d0\u4f9b\u8bfb\u5199\u670d\u52a1\uff0cfollow\u526f\u672c\u53ea\u662f\u505a\u6d88\u606f\u540c\u6b65\u3002<\/p>\n<h3>\u300c\u5982\u679c\u4e00\u4e2a\u5206\u533a\u7684leader\u526f\u672c\u4e0d\u53ef\u7528\uff0c\u5c31\u610f\u5473\u7740\u6574\u4e2a\u5206\u533a\u4e0d\u53ef\u7528\uff0c\u6b64\u65f6\u9700\u8981\u4ecefollower\u526f\u672c\u4e2d\u9009\u4e3e\u51fa\u65b0\u7684leader\u526f\u672c\u63d0\u4f9b\u670d\u52a1\u300d\u3002<\/h3>\n<h3>\u300c\u5728\u521b\u5efa\u4e3b\u9898\u7684\u65f6\u5019\uff0c\u8be5\u5206\u533a\u7684\u4e3b\u9898\u548c\u526f\u672c\u4f1a\u5c3d\u53ef\u80fd\u7684\u5747\u5300\u53d1\u5e03\u5230kafka\u7684\u5404\u4e2abroker\u4e0a\u300d\u3002<\/h3>\n<p>\u6bd4\u5982\u6211\u4eec\u5728\u5305\u542b3\u4e2abroker\u8282\u70b9\u7684kafka\u96c6\u7fa4\u4e0a\u521b\u5efa\u4e00\u4e2a\u5206\u533a\u6570\u4e3a3\uff0c\u526f\u672c\u56e0\u5b50\u4e3a3\u7684\u4e3b\u9898topic-partitions\u65f6\uff0cleader\u526f\u672c\u4f1a\u5747\u5300\u7684\u5206\u5e03\u57283\u53f0broker\u8282\u70b9\u4e0a\u3002<\/p>\n<p style=\"text-align: center\">\n<h3>\u300c\u9488\u5bf9\u540c\u4e00\u4e2a\u5206\u533a\uff0c\u5728\u540c\u4e00\u4e2abroker\u8282\u70b9\u4e0a\u4e0d\u53ef\u80fd\u51fa\u73b0\u5b83\u7684\u591a\u4e2a\u526f\u672c\u300d\u3002<\/h3>\n<p>\u6211\u4eec\u53ef\u4ee5\u628aleader\u526f\u672c\u6240\u5728\u7684\u8282\u70b9\u53eb\u4f5c\u5206\u533a\u7684leader\u8282\u70b9\uff0c\u628afollower\u526f\u672c\u6240\u5728\u7684\u8282\u70b9\u53eb\u4f5cfollower\u8282\u70b9\u3002<\/p>\n<p>\u5728\u4e0a\u9762\u7684\u4f8b\u5b50\u4e2d\uff0c\u5206\u533a0\u7684leader\u8282\u70b9\u662fbroker1\uff0c\u5206\u533a1\u7684leader\u8282\u70b9\u662fbroker2\uff0c\u5206\u533a2\u7684leader\u8282\u70b9\u662fbroker0\u3002<\/p>\n<p>\u5f53\u5206\u533aleader\u8282\u70b9\u53d1\u751f\u6545\u969c\u65f6\uff0c\u5176\u4e2d\u7684\u4e00\u4e2afollower\u8282\u70b9\u5c31\u4f1a\u9009\u4e3e\u4e3a\u65b0\u7684leader\u8282\u70b9\u3002<\/p>\n<p>\u5f53\u539f\u6765leader\u7684\u8282\u70b9\u6062\u590d\u4e4b\u540e\uff0c\u5b83\u53ea\u80fd\u6210\u4e3a\u4e00\u4e2afollower\u8282\u70b9\uff0c\u6b64\u65f6\u5c31\u5bfc\u81f4\u4e86\u96c6\u7fa4\u8d1f\u8f7d\u4e0d\u5747\u8861\u3002<\/p>\n<p>\u6bd4\u5982\u5206\u533a1\u7684leader\u8282\u70b9broker2\u5d29\u6e83\u4e86\uff0c\u6b64\u65f6\u9009\u4e3e\u4e86\u5728broker1\u4e0a\u7684\u5206\u533a1follower\u8282\u70b9\u4f5c\u4e3a\u65b0\u7684leader\u8282\u70b9\u3002<\/p>\n<p>\u5f53broker2\u91cd\u65b0\u6062\u590d\u65f6\uff0c\u6b64\u65f6\u7684kafka\u96c6\u7fa4\u72b6\u6001\u5982\u4e0b\uff1a<\/p>\n<p style=\"text-align: center\">\n<p>\u53ef\u4ee5\u770b\u5230\uff0c\u6b64\u65f6broker1\u4e0a\u8d1f\u8f7d\u66f4\u5927\uff0c\u800cbroker2\u4e0a\u6ca1\u6709\u8d1f\u8f7d\u3002<\/p>\n<h3>\u300c\u4e3a\u4e86\u89e3\u51b3\u4e0a\u8ff0\u8d1f\u8f7d\u4e0d\u5747\u8861\u7684\u60c5\u51b5\uff0ckafka\u652f\u6301\u4e86\u4f18\u5148\u526f\u672c\u9009\u4e3e\uff0c\u4f18\u5148\u526f\u672c\u6307\u7684\u662f\u4e00\u4e2a\u5206\u533a\u6240\u5728\u7684AR\u96c6\u5408\u7684\u7b2c\u4e00\u4e2a\u526f\u672c\u300d\u3002<\/h3>\n<p>\u6bd4\u5982\u4e0a\u9762\u7684\u5206\u533a1\uff0c\u5b83\u7684AR\u96c6\u5408\u662f[2,0,1]\uff0c\u8868\u793a\u5206\u533a1\u7684\u4f18\u5148\u526f\u672c\u5c31\u662f\u5728broker2\u4e0a\u3002<\/p>\n<p>\u7406\u60f3\u60c5\u51b5\u4e0b\uff0c\u4f18\u5148\u526f\u672c\u5e94\u8be5\u5c31\u662fleader\u526f\u672c\uff0ckafka\u4fdd\u8bc1\u4e86\u4f18\u5148\u526f\u672c\u7684\u5747\u8861\u5206\u5e03\uff0c\u800c\u8fd9\u4e0ebroker\u8282\u70b9\u5b95\u673a\u4e0e\u5426\u6ca1\u6709\u5173\u7cfb\u3002<\/p>\n<p>\u300c\u4f18\u5148\u526f\u672c\u9009\u4e3e\u5c31\u662f\u5bf9\u5206\u533aleader\u526f\u672c\u8fdb\u884c\u9009\u4e3e\u7684\u65f6\u5019\uff0c\u5c3d\u53ef\u80fd\u8ba9\u4f18\u5148\u526f\u672c\u6210\u4e3aleader\u526f\u672c\u300d\uff0c\u9488\u5bf9\u4e0a\u8ff0\u7684\u60c5\u51b5\uff0c\u53ea\u8981\u518d\u89e6\u53d1\u4e00\u6b21\u4f18\u5148\u526f\u672c\u9009\u4e3e\u5c31\u80fd\u4fdd\u8bc1\u5206\u533a\u8d1f\u8f7d\u5747\u8861\u3002<\/p>\n<p>kafka\u652f\u6301\u81ea\u52a8\u4f18\u5148\u526f\u672c\u9009\u4e3e\u529f\u80fd\uff0c\u9ed8\u8ba4\u6bcf5\u5206\u949f\u89e6\u53d1\u4e00\u6b21\u4f18\u5148\u526f\u672c\u9009\u4e3e\u64cd\u4f5c\u3002<\/p>\n<h2>\u7f51\u7edc\u901a\u4fe1\u6a21\u578b<\/h2>\n<p style=\"text-align: center\">\n<p>Broker \u4e2d\u6709\u4e2aAcceptor(mainReactor)\u76d1\u542c\u65b0\u8fde\u63a5\u7684\u5230\u6765\uff0c\u4e0e\u65b0\u8fde\u63a5\u5efa\u8fde\u4e4b\u540e\u8f6e\u8be2\u9009\u62e9\u4e00\u4e2aProcessor(subReactor)\u7ba1\u7406\u8fd9\u4e2a\u8fde\u63a5\u3002<\/p>\n<p>\u800cProcessor\u4f1a\u76d1\u542c\u5176\u7ba1\u7406\u7684\u8fde\u63a5\uff0c\u5f53\u4e8b\u4ef6\u5230\u8fbe\u4e4b\u540e\uff0c\u8bfb\u53d6\u5c01\u88c5\u6210Request\uff0c\u5e76\u5c06Request\u653e\u5165\u5171\u4eab\u8bf7\u6c42\u961f\u5217\u4e2d\u3002<\/p>\n<p>\u7136\u540eIO\u7ebf\u7a0b\u6c60\u4e0d\u65ad\u7684\u4ece\u8be5\u961f\u5217\u4e2d\u53d6\u51fa\u8bf7\u6c42\uff0c\u6267\u884c\u771f\u6b63\u7684\u5904\u7406\u3002\u5904\u7406\u5b8c\u4e4b\u540e\u5c06\u54cd\u5e94\u53d1\u9001\u5230\u5bf9\u5e94\u7684Processor\u7684\u54cd\u5e94\u961f\u5217\u4e2d\uff0c\u7136\u540e\u7531Processor\u5c06Response\u8fd4\u8fd8\u7ed9\u5ba2\u6237\u7aef\u3002<\/p>\n<p>\u6bcf\u4e2alistener\u53ea\u6709\u4e00\u4e2aAcceptor\u7ebf\u7a0b\uff0c\u56e0\u4e3a\u5b83\u53ea\u662f\u4f5c\u4e3a\u65b0\u8fde\u63a5\u5efa\u8fde\u518d\u5206\u53d1\uff0c\u6ca1\u6709\u8fc7\u591a\u7684\u903b\u8f91\uff0c\u5f88\u8f7b\u91cf\u3002<\/p>\n<p>Processor \u5728Kafka\u4e2d\u79f0\u4e4b\u4e3a\u7f51\u7edc\u7ebf\u7a0b\uff0c\u9ed8\u8ba4\u7f51\u7edc\u7ebf\u7a0b\u6c60\u67093\u4e2a\u7ebf\u7a0b\uff0c\u5bf9\u5e94\u7684\u53c2\u6570\u662fnum.network.threads\uff0c\u5e76\u4e14\u53ef\u4ee5\u6839\u636e\u5b9e\u9645\u7684\u4e1a\u52a1\u52a8\u6001\u589e\u51cf\u3002<\/p>\n<p>\u8fd8\u6709\u4e2a IO \u7ebf\u7a0b\u6c60\uff0c\u5373KafkaRequestHandlerPool\uff0c\u6267\u884c\u771f\u6b63\u7684\u5904\u7406\uff0c\u5bf9\u5e94\u7684\u53c2\u6570\u662fnum.io.threads\uff0c\u9ed8\u8ba4\u503c\u662f 8\u3002<\/p>\n<p>IO\u7ebf\u7a0b\u5904\u7406\u5b8c\u4e4b\u540e\u4f1a\u5c06Response\u653e\u5165\u5bf9\u5e94\u7684Processor\u4e2d\uff0c\u7531Processor\u5c06\u54cd\u5e94\u8fd4\u8fd8\u7ed9\u5ba2\u6237\u7aef\u3002<\/p>\n<p>\u53ef\u4ee5\u770b\u5230\u7f51\u7edc\u7ebf\u7a0b\u548cIO\u7ebf\u7a0b\u4e4b\u95f4\u5229\u7528\u7684\u7ecf\u5178\u7684\u751f\u4ea7\u8005 - \u6d88\u8d39\u8005\u6a21\u5f0f\uff0c\u4e0d\u8bba\u662f\u7528\u4e8e\u5904\u7406Request\u7684\u5171\u4eab\u8bf7\u6c42\u961f\u5217\uff0c\u8fd8\u662fIO\u5904\u7406\u5b8c\u8fd4\u56de\u7684Response\u3002<\/p>\n<h2>\u5e42\u7b49\u6027<\/h2>\n<h3>\u300c\u5e42\u7b49\u6027Producer\u300d<\/h3>\n<p>\u5728Kafka\u4e2d\uff0cProducer\u9ed8\u8ba4\u4e0d\u662f\u5e42\u7b49\u6027\u7684\uff0c\u4f46\u6211\u4eec\u53ef\u4ee5\u521b\u5efa\u5e42\u7b49\u6027Producer\u3002<\/p>\n<p>\u5b83\u5176\u5b9e\u662f0.11.0.0\u7248\u672c\u5f15\u5165\u7684\u65b0\u529f\u80fd\uff0c\u5728\u6b64\u4e4b\u524d\uff0cKafka\u5411\u5206\u533a\u53d1\u9001\u6570\u636e\u65f6\uff0c\u53ef\u80fd\u4f1a\u51fa\u73b0\u540c\u4e00\u6761\u6d88\u606f\u88ab\u53d1\u9001\u4e86\u591a\u6b21\uff0c\u5bfc\u81f4\u6d88\u606f\u91cd\u590d\u7684\u60c5\u51b5\u3002<\/p>\n<p>\u57280.11\u4e4b\u540e\uff0c\u6307\u5b9aProducer\u5e42\u7b49\u6027\u7684\u65b9\u6cd5\u5f88\u7b80\u5355\uff0c\u4ec5\u9700\u8981\u8bbe\u7f6e\u4e00\u4e2a\u53c2\u6570\u5373\u53ef\uff0c\u5373<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>props.put(\u201cenable.idempotence\u201d,&nbsp;ture)\uff0c&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>\u6216props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG\uff0c&nbsp;<span>true<\/span><span>)\u3002&nbsp;<\/span><\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>enable.idempotence\u88ab\u8bbe\u7f6e\u6210true\u540e\uff0cProducer\u81ea\u52a8\u5347\u7ea7\u6210\u5e42\u7b49\u6027Producer\uff0c\u5176\u4ed6\u6240\u6709\u7684\u4ee3\u7801\u903b\u8f91\u90fd\u4e0d\u9700\u8981\u6539\u53d8\u3002<\/p>\n<p>Kafka\u81ea\u52a8\u5e2e\u4f60\u505a\u6d88\u606f\u7684\u91cd\u590d\u53bb\u91cd\u3002<\/p>\n<p>\u5e95\u5c42\u5177\u4f53\u7684\u539f\u7406\u5f88\u7b80\u5355\uff0c\u5c31\u662f\u7ecf\u5178\u7684\u7528\u7a7a\u95f4\u53bb\u6362\u65f6\u95f4\u7684\u4f18\u5316\u601d\u8def\uff0c\u5373\u5728Broker\u7aef\u591a\u4fdd\u5b58\u4e00\u4e9b\u5b57\u6bb5\u3002<\/p>\n<p>\u5f53Producer\u53d1\u9001\u4e86\u5177\u6709\u76f8\u540c\u5b57\u6bb5\u503c\u7684\u6d88\u606f\u540e\uff0cBroker\u80fd\u591f\u81ea\u52a8\u77e5\u6653\u8fd9\u4e9b\u6d88\u606f\u5df2\u7ecf\u91cd\u590d\u4e86\uff0c\u4e8e\u662f\u53ef\u4ee5\u5728\u540e\u53f0\u9ed8\u9ed8\u5730\u628a\u5b83\u4eec\u4e22\u5f03\u6389\u3002<\/p>\n<h3>\u300c\u5e42\u7b49\u6027Producer\u7684\u4f5c\u7528\u8303\u56f4\u300d<\/h3>\n<p>\u9996\u5148\uff0c\u5b83\u53ea\u80fd\u4fdd\u8bc1\u5355\u5206\u533a\u4e0a\u7684\u5e42\u7b49\u6027\uff0c\u5373\u4e00\u4e2a\u5e42\u7b49\u6027Producer\u80fd\u591f\u4fdd\u8bc1\u67d0\u4e2a\u4e3b\u9898\u7684\u4e00\u4e2a\u5206\u533a\u4e0a\u4e0d\u51fa\u73b0\u91cd\u590d\u6d88\u606f\uff0c\u5b83\u65e0\u6cd5\u5b9e\u73b0\u591a\u4e2a\u5206\u533a\u7684\u5e42\u7b49\u6027\u3002<\/p>\n<p>\u5176\u6b21\uff0c\u5b83\u53ea\u80fd\u5b9e\u73b0\u5355\u4f1a\u8bdd\u4e0a\u7684\u5e42\u7b49\u6027\uff0c\u4e0d\u80fd\u5b9e\u73b0\u8de8\u4f1a\u8bdd\u7684\u5e42\u7b49\u6027\u3002<\/p>\n<p>\u8fd9\u91cc\u7684\u4f1a\u8bdd\uff0c\u4f60\u53ef\u4ee5\u7406\u89e3\u4e3aProducer\u8fdb\u7a0b\u7684\u4e00\u6b21\u8fd0\u884c\uff0c\u5f53\u4f60\u91cd\u542f\u4e86Producer\u8fdb\u7a0b\u4e4b\u540e\uff0c\u8fd9\u79cd\u5e42\u7b49\u6027\u4fdd\u8bc1\u5c31\u4e27\u5931\u4e86\u3002<\/p>\n<h2>\u4e8b\u52a1<\/h2>\n<p>Kafka\u81ea0.11\u7248\u672c\u5f00\u59cb\u4e5f\u63d0\u4f9b\u4e86\u5bf9\u4e8b\u52a1\u7684\u652f\u6301\uff0c\u76ee\u524d\u4e3b\u8981\u662f\u5728read committed\u9694\u79bb\u7ea7\u522b\u4e0a\u505a\u4e8b\u60c5\u3002<\/p>\n<p>\u5b83\u80fd\u4fdd\u8bc1\u591a\u6761\u6d88\u606f\u539f\u5b50\u6027\u5730\u5199\u5165\u5230\u76ee\u6807\u5206\u533a\uff0c\u540c\u65f6\u4e5f\u80fd\u4fdd\u8bc1Consumer\u53ea\u80fd\u770b\u5230\u4e8b\u52a1\u6210\u529f\u63d0\u4ea4\u7684\u6d88\u606f\u3002<\/p>\n<h3>\u300c\u4e8b\u52a1\u578bProducer\u300d<\/h3>\n<p>\u4e8b\u52a1\u578bProducer\u80fd\u591f\u4fdd\u8bc1\u5c06\u6d88\u606f\u539f\u5b50\u6027\u5730\u5199\u5165\u5230\u591a\u4e2a\u5206\u533a\u4e2d\u3002<\/p>\n<p>\u8fd9\u6279\u6d88\u606f\u8981\u4e48\u5168\u90e8\u5199\u5165\u6210\u529f\uff0c\u8981\u4e48\u5168\u90e8\u5931\u8d25\uff0c\u53e6\u5916\uff0c\u4e8b\u52a1\u578bProducer\u4e5f\u4e0d\u60e7\u8fdb\u7a0b\u7684\u91cd\u542f\u3002<\/p>\n<p>Producer\u91cd\u542f\u56de\u6765\u540e\uff0cKafka\u4f9d\u7136\u4fdd\u8bc1\u5b83\u4eec\u53d1\u9001\u6d88\u606f\u7684\u7cbe\u786e\u4e00\u6b21\u5904\u7406\u3002<\/p>\n<p>\u8bbe\u7f6e\u4e8b\u52a1\u578bProducer\u7684\u65b9\u6cd5\u4e5f\u5f88\u7b80\u5355\uff0c\u6ee1\u8db3\u4e24\u4e2a\u8981\u6c42\u5373\u53ef\uff1a<\/p>\n<ul>\n<li>\u548c\u5e42\u7b49\u6027Producer\u4e00\u6837\uff0c\u5f00\u542fenable.idempotence = true\u3002<\/li>\n<li>\u8bbe\u7f6eProducer\u7aef\u53c2\u6570transactional. id\uff0c\u6700\u597d\u4e3a\u5176\u8bbe\u7f6e\u4e00\u4e2a\u6709\u610f\u4e49\u7684\u540d\u5b57\u3002<\/li>\n<\/ul>\n<p>\u6b64\u5916\uff0c\u4f60\u8fd8\u9700\u8981\u5728Producer\u4ee3\u7801\u4e2d\u505a\u4e00\u4e9b\u8c03\u6574\uff0c\u5982\u8fd9\u6bb5\u4ee3\u7801\u6240\u793a\uff1a<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>producer.initTransactions();&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>try&nbsp;{&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;producer.beginTransaction();&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;producer.send(record1);&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;producer.send(record2);&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;producer.commitTransaction();&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>}&nbsp;catch&nbsp;(KafkaException&nbsp;e)&nbsp;{&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;producer.abortTransaction();&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>}&nbsp;<\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<p>\u548c\u666e\u901aProducer\u4ee3\u7801\u76f8\u6bd4\uff0c\u4e8b\u52a1\u578bProducer\u7684\u663e\u8457\u7279\u70b9\u662f\u8c03\u7528\u4e86\u4e00\u4e9b\u4e8b\u52a1API\uff0c\u5982initTransaction\u3001beginTransaction\u3001commitTransaction\u548cabortTransaction\uff0c\u5b83\u4eec\u5206\u522b\u5bf9\u5e94\u4e8b\u52a1\u7684\u521d\u59cb\u5316\u3001\u4e8b\u52a1\u5f00\u59cb\u3001\u4e8b\u52a1\u63d0\u4ea4\u4ee5\u53ca\u4e8b\u52a1\u7ec8\u6b62\u3002<\/p>\n<p>\u8fd9\u6bb5\u4ee3\u7801\u80fd\u591f\u4fdd\u8bc1Record1\u548cRecord2\u88ab\u5f53\u4f5c\u4e00\u4e2a\u4e8b\u52a1\u7edf\u4e00\u63d0\u4ea4\u5230Kafka\uff0c\u8981\u4e48\u5b83\u4eec\u5168\u90e8\u63d0\u4ea4\u6210\u529f\uff0c\u8981\u4e48\u5168\u90e8\u5199\u5165\u5931\u8d25\u3002<\/p>\n<p>\u5b9e\u9645\u4e0a\u5373\u4f7f\u5199\u5165\u5931\u8d25\uff0cKafka\u4e5f\u4f1a\u628a\u5b83\u4eec\u5199\u5165\u5230\u5e95\u5c42\u7684\u65e5\u5fd7\u4e2d\uff0c\u4e5f\u5c31\u662f\u8bf4Consumer\u8fd8\u662f\u4f1a\u770b\u5230\u8fd9\u4e9b\u6d88\u606f\u3002<\/p>\n<p>\u6709\u4e00\u4e2aisolation.level\u53c2\u6570\uff0c\u8fd9\u4e2a\u53c2\u6570\u6709\u4e24\u4e2a\u53d6\u503c\uff1a<\/p>\n<ol>\n<li>read_uncommitted\uff1a\u8fd9\u662f\u9ed8\u8ba4\u503c\uff0c\u8868\u660eConsumer\u80fd\u591f\u8bfb\u53d6\u5230Kafka\u5199\u5165\u7684\u4efb\u4f55\u6d88\u606f\uff0c\u4e0d\u8bba\u4e8b\u52a1\u578bProducer\u63d0\u4ea4\u4e8b\u52a1\u8fd8\u662f\u7ec8\u6b62\u4e8b\u52a1\uff0c\u5176\u5199\u5165\u7684\u6d88\u606f\u90fd\u53ef\u4ee5\u8bfb\u53d6\uff0c\u5982\u679c\u4f60\u7528\u4e86\u4e8b\u52a1\u578bProducer\uff0c\u90a3\u4e48\u5bf9\u5e94\u7684Consumer\u5c31\u4e0d\u8981\u4f7f\u7528\u8fd9\u4e2a\u503c\u3002<\/li>\n<li>read_committed\uff1a\u8868\u660eConsumer\u53ea\u4f1a\u8bfb\u53d6\u4e8b\u52a1\u578bProducer\u6210\u529f\u63d0\u4ea4\u4e8b\u52a1\u5199\u5165\u7684\u6d88\u606f\uff0c\u5b83\u4e5f\u80fd\u770b\u5230\u975e\u4e8b\u52a1\u578bProducer\u5199\u5165\u7684\u6240\u6709\u6d88\u606f\u3002<\/li>\n<\/ol>\n<h2>\u62e6\u622a\u5668<\/h2>\n<h3>\u300cKafka\u62e6\u622a\u5668\u5206\u4e3a\u751f\u4ea7\u8005\u62e6\u622a\u5668\u548c\u6d88\u8d39\u8005\u62e6\u622a\u5668\u300d\u3002<\/h3>\n<p>\u751f\u4ea7\u8005\u62e6\u622a\u5668\u5141\u8bb8\u4f60\u5728\u53d1\u9001\u6d88\u606f\u524d\u4ee5\u53ca\u6d88\u606f\u63d0\u4ea4\u6210\u529f\u540e\u690d\u5165\u4f60\u7684\u62e6\u622a\u5668\u903b\u8f91;<\/p>\n<p>\u800c\u6d88\u8d39\u8005\u62e6\u622a\u5668\u652f\u6301\u5728\u6d88\u8d39\u6d88\u606f\u524d\u4ee5\u53ca\u63d0\u4ea4\u4f4d\u79fb\u540e\u7f16\u5199\u7279\u5b9a\u903b\u8f91\u3002<\/p>\n<p>\u53ef\u4ee5\u5c06\u4e00\u7ec4\u62e6\u622a\u5668\u4e32\u8fde\u6210\u4e00\u4e2a\u5927\u7684\u62e6\u622a\u5668\uff0cKafka\u4f1a\u6309\u7167\u6dfb\u52a0\u987a\u5e8f\u4f9d\u6b21\u6267\u884c\u62e6\u622a\u5668\u903b\u8f91\u3002<\/p>\n<p>\u5f53\u524dKafka\u62e6\u622a\u5668\u7684\u8bbe\u7f6e\u65b9\u6cd5\u662f\u901a\u8fc7\u53c2\u6570\u914d\u7f6e\u5b8c\u6210\u7684\uff0c\u751f\u4ea7\u8005\u548c\u6d88\u8d39\u8005\u4e24\u7aef\u6709\u4e00\u4e2a\u76f8\u540c\u7684\u53c2\u6570interceptor.classes\uff0c\u5b83\u6307\u5b9a\u7684\u662f\u4e00\u7ec4\u7c7b\u7684\u5217\u8868\uff0c\u6bcf\u4e2a\u7c7b\u5c31\u662f\u7279\u5b9a\u903b\u8f91\u7684\u62e6\u622a\u5668\u5b9e\u73b0\u7c7b\u3002<\/p>\n<pre>\n  \n \n \n <ol>\n   \n  \n  \n  <li><span><span>Properties&nbsp;props&nbsp;=&nbsp;new&nbsp;Properties();&nbsp;&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>List&nbsp;interceptors&nbsp;=&nbsp;new&nbsp;ArrayList&lt;&gt;();&nbsp;&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>interceptors.<span>add<\/span><span>(<\/span><span>\"com.yourcompany.kafkaproject.interceptors.AddTimestampInterceptor\"<\/span><span>);&nbsp;\/\/&nbsp;\u62e6\u622a\u56681&nbsp;&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>interceptors.<span>add<\/span><span>(<\/span><span>\"com.yourcompany.kafkaproject.interceptors.UpdateCounterInterceptor\"<\/span><span>);&nbsp;\/\/&nbsp;\u62e6\u622a\u56682&nbsp;&nbsp;<\/span><\/span><\/li>\n   \n  \n  \n  <li><span>props.put(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG,&nbsp;interceptors);&nbsp;&nbsp;<\/span><\/li>\n   \n  \n  \n  <li><span>\u2026\u2026&nbsp;&nbsp;<\/span><\/li>\n  \n \n \n <\/ol><\/pre>\n<ul>\n<li>\u600e\u4e48\u7f16\u5199AddTimeStampInterceptor\u548cUpdateCounterInterceptor\u7c7b\u5462?<\/li>\n<\/ul>\n<p>\u8fd9\u4e24\u4e2a\u7c7b\u4ee5\u53ca\u4f60\u81ea\u5df1\u7f16\u5199\u7684\u6240\u6709Producer\u7aef\u62e6\u622a\u5668\u5b9e\u73b0\u7c7b\u90fd\u8981\u7ee7\u627forg.apache.kafka.clients.producer.ProducerInterceptor\u63a5\u53e3\u3002<\/p>\n<p>\u8be5\u63a5\u53e3\u662fKafka\u63d0\u4f9b\u7684\uff0c\u91cc\u9762\u6709\u4e24\u4e2a\u6838\u5fc3\u7684\u65b9\u6cd5\u3002<\/p>\n<ol>\n<li>onSend\uff1a\u8be5\u65b9\u6cd5\u4f1a\u5728\u6d88\u606f\u53d1\u9001\u4e4b\u524d\u88ab\u8c03\u7528\u3002<\/li>\n<li>onAcknowledgement\uff1a\u8be5\u65b9\u6cd5\u4f1a\u5728\u6d88\u606f\u6210\u529f\u63d0\u4ea4\u6216\u53d1\u9001\u5931\u8d25\u4e4b\u540e\u88ab\u8c03\u7528\u3002onAcknowledgement\u7684\u8c03\u7528\u8981\u65e9\u4e8ecallback\u7684\u8c03\u7528\u3002\u503c\u5f97\u6ce8\u610f\u7684\u662f\uff0c\u8fd9\u4e2a\u65b9\u6cd5\u548conSend\u4e0d\u662f\u5728\u540c\u4e00\u4e2a\u7ebf\u7a0b\u4e2d\u88ab\u8c03\u7528\u7684\uff0c\u56e0\u6b64\u5982\u679c\u4f60\u5728\u8fd9\u4e24\u4e2a\u65b9\u6cd5\u4e2d\u8c03\u7528\u4e86\u67d0\u4e2a\u5171\u4eab\u53ef\u53d8\u5bf9\u8c61\uff0c\u4e00\u5b9a\u8981\u4fdd\u8bc1\u7ebf\u7a0b\u5b89\u5168\u3002<\/li>\n<\/ol>\n<p>\u540c\u7406\uff0c\u6307\u5b9a\u6d88\u8d39\u8005\u62e6\u622a\u5668\u4e5f\u662f\u540c\u6837\u7684\u65b9\u6cd5\uff0c\u53ea\u662f\u5177\u4f53\u7684\u5b9e\u73b0\u7c7b\u8981\u5b9e\u73b0org.apache.kafka.clients.consumer.ConsumerInterceptor\u63a5\u53e3\uff0c\u8fd9\u91cc\u9762\u4e5f\u6709\u4e24\u4e2a\u6838\u5fc3\u65b9\u6cd5\u3002<\/p>\n<ol>\n<li>onConsume\uff1a\u8be5\u65b9\u6cd5\u5728\u6d88\u606f\u8fd4\u56de\u7ed9Consumer\u7a0b\u5e8f\u4e4b\u524d\u8c03\u7528\u3002<\/li>\n<li>onCommit\uff1aConsumer\u5728\u63d0\u4ea4\u4f4d\u79fb\u4e4b\u540e\u8c03\u7528\u8be5\u65b9\u6cd5\u3002\u901a\u5e38\u4f60\u53ef\u4ee5\u5728\u8be5\u65b9\u6cd5\u4e2d\u505a\u4e00\u4e9b\u8bb0\u8d26\u7c7b\u7684\u52a8\u4f5c\uff0c\u6bd4\u5982\u6253\u65e5\u5fd7\u7b49\u3002<\/li>\n<\/ol>\n<p>\u4e00\u5b9a\u8981\u6ce8\u610f\u7684\u662f\uff0c\u300c\u6307\u5b9a\u62e6\u622a\u5668\u7c7b\u65f6\u8981\u6307\u5b9a\u5b83\u4eec\u7684\u5168\u9650\u5b9a\u540d\u300d\u3002<\/p>\n<p>\u901a\u4fd7\u70b9\u8bf4\u5c31\u662f\u8981\u628a\u5b8c\u6574\u5305\u540d\u4e5f\u52a0\u4e0a\uff0c\u4e0d\u8981\u53ea\u6709\u4e00\u4e2a\u7c7b\u540d\u5728\u90a3\u91cc\uff0c\u5e76\u4e14\u8fd8\u8981\u4fdd\u8bc1\u4f60\u7684Producer\u7a0b\u5e8f\u80fd\u591f\u6b63\u786e\u52a0\u8f7d\u4f60\u7684\u62e6\u622a\u5668\u7c7b\u3002<\/p>\n<h2>\u63a7\u5236\u5668<\/h2>\n<h3>\u300c\u63a7\u5236\u5668\u7ec4\u4ef6(Controller)\uff0c\u5b83\u7684\u4e3b\u8981\u4f5c\u7528\u662f\u5728Apache ZooKeeper\u7684\u5e2e\u52a9\u4e0b\u7ba1\u7406\u548c\u534f\u8c03\u6574\u4e2aKafka\u96c6\u7fa4\u300d\u3002<\/h3>\n<p>\u96c6\u7fa4\u4e2d\u4efb\u610f\u4e00\u53f0Broker\u90fd\u80fd\u5145\u5f53\u63a7\u5236\u5668\u7684\u89d2\u8272\uff0c\u4f46\u662f\uff0c\u5728\u8fd0\u884c\u8fc7\u7a0b\u4e2d\uff0c\u53ea\u80fd\u6709\u4e00\u4e2aBroker\u6210\u4e3a\u63a7\u5236\u5668\uff0c\u884c\u4f7f\u5176\u7ba1\u7406\u548c\u534f\u8c03\u7684\u804c\u8d23\u3002<\/p>\n<p>Kafka\u63a7\u5236\u5668\u5927\u91cf\u4f7f\u7528ZooKeeper\u7684Watch\u529f\u80fd\u5b9e\u73b0\u5bf9\u96c6\u7fa4\u7684\u534f\u8c03\u7ba1\u7406\u3002<\/p>\n<h3>\u300c\u63a7\u5236\u5668\u662f\u5982\u4f55\u88ab\u9009\u51fa\u6765\u7684\u300d<\/h3>\n<p>\u5b9e\u9645\u4e0a\uff0cBroker\u5728\u542f\u52a8\u65f6\uff0c\u4f1a\u5c1d\u8bd5\u53bbZooKeeper\u4e2d\u521b\u5efa\/controller\u8282\u70b9\u3002<\/p>\n<p>Kafka\u5f53\u524d\u9009\u4e3e\u63a7\u5236\u5668\u7684\u89c4\u5219\u662f\uff1a\u300c\u7b2c\u4e00\u4e2a\u6210\u529f\u521b\u5efa\/controller\u8282\u70b9\u7684Broker\u4f1a\u88ab\u6307\u5b9a\u4e3a\u63a7\u5236\u5668\u300d\u3002<\/p>\n<h3>\u300c\u63a7\u5236\u5668\u662f\u505a\u4ec0\u4e48\u7684\u300d<\/h3>\n<p>\u63a7\u5236\u5668\u7684\u804c\u8d23\u5927\u81f4\u53ef\u4ee5\u5206\u4e3a5\u79cd\uff1a<\/p>\n<h3>1.\u300c\u4e3b\u9898\u7ba1\u7406(\u521b\u5efa\u3001\u5220\u9664\u3001\u589e\u52a0\u5206\u533a)\u300d<\/h3>\n<p>\u63a7\u5236\u5668\u5e2e\u52a9\u6211\u4eec\u5b8c\u6210\u5bf9Kafka\u4e3b\u9898\u7684\u521b\u5efa\u3001\u5220\u9664\u4ee5\u53ca\u5206\u533a\u589e\u52a0\u7684\u64cd\u4f5c\u3002<\/p>\n<h3>2.\u300c\u5206\u533a\u91cd\u5206\u914d\u300d<\/h3>\n<h3>3.\u300cPreferred\u9886\u5bfc\u8005\u9009\u4e3e\u300d<\/h3>\n<p>Preferred\u9886\u5bfc\u8005\u9009\u4e3e\u4e3b\u8981\u662fKafka\u4e3a\u4e86\u907f\u514d\u90e8\u5206Broker\u8d1f\u8f7d\u8fc7\u91cd\u800c\u63d0\u4f9b\u7684\u4e00\u79cd\u6362Leader\u7684\u65b9\u6848\u3002<\/p>\n<h3>4.\u300c\u96c6\u7fa4\u6210\u5458\u7ba1\u7406(\u65b0\u589eBroker\u3001Broker\u4e3b\u52a8\u5173\u95ed\u3001Broker\u5b95\u673a)\u300d<\/h3>\n<p>\u5305\u62ec\u81ea\u52a8\u68c0\u6d4b\u65b0\u589eBroker\u3001Broker\u4e3b\u52a8\u5173\u95ed\u53ca\u88ab\u52a8\u5b95\u673a\u3002<\/p>\n<p>\u8fd9\u79cd\u81ea\u52a8\u68c0\u6d4b\u662f\u4f9d\u8d56\u4e8eWatch\u529f\u80fd\u548cZooKeeper\u4e34\u65f6\u8282\u70b9\u7ec4\u5408\u5b9e\u73b0\u7684\u3002<\/p>\n<p>\u6bd4\u5982\uff0c\u63a7\u5236\u5668\u7ec4\u4ef6\u4f1a\u5229\u7528\u300cWatch\u673a\u5236\u300d\u68c0\u67e5ZooKeeper\u7684\/brokers\/ids\u8282\u70b9\u4e0b\u7684\u5b50\u8282\u70b9\u6570\u91cf\u53d8\u66f4\u3002<\/p>\n<p>\u76ee\u524d\uff0c\u5f53\u6709\u65b0Broker\u542f\u52a8\u540e\uff0c\u5b83\u4f1a\u5728\/brokers\u4e0b\u521b\u5efa\u4e13\u5c5e\u7684znode\u8282\u70b9\u3002<\/p>\n<p>\u4e00\u65e6\u521b\u5efa\u5b8c\u6bd5\uff0cZooKeeper\u4f1a\u901a\u8fc7Watch\u673a\u5236\u5c06\u6d88\u606f\u901a\u77e5\u63a8\u9001\u7ed9\u63a7\u5236\u5668\uff0c\u8fd9\u6837\uff0c\u63a7\u5236\u5668\u5c31\u80fd\u81ea\u52a8\u5730\u611f\u77e5\u5230\u8fd9\u4e2a\u53d8\u5316\uff0c\u8fdb\u800c\u5f00\u542f\u540e\u7eed\u7684\u65b0\u589eBroker\u4f5c\u4e1a\u3002<\/p>\n<p>\u4fa6\u6d4bBroker\u5b58\u6d3b\u6027\u5219\u662f\u4f9d\u8d56\u4e8e\u521a\u521a\u63d0\u5230\u7684\u53e6\u4e00\u4e2a\u673a\u5236\uff1a\u300c\u4e34\u65f6\u8282\u70b9\u300d\u3002<\/p>\n<p>\u6bcf\u4e2aBroker\u542f\u52a8\u540e\uff0c\u4f1a\u5728\/brokers\/ids\u4e0b\u521b\u5efa\u4e00\u4e2a\u4e34\u65f6znode\u3002<\/p>\n<p>\u5f53Broker\u5b95\u673a\u6216\u4e3b\u52a8\u5173\u95ed\u540e\uff0c\u8be5Broker\u4e0eZooKeeper\u7684\u4f1a\u8bdd\u7ed3\u675f\uff0c\u8fd9\u4e2aznode\u4f1a\u88ab\u81ea\u52a8\u5220\u9664\u3002<\/p>\n<p>\u540c\u7406\uff0cZooKeeper\u7684Watch\u673a\u5236\u5c06\u8fd9\u4e00\u53d8\u66f4\u63a8\u9001\u7ed9\u63a7\u5236\u5668\uff0c\u8fd9\u6837\u63a7\u5236\u5668\u5c31\u80fd\u77e5\u9053\u6709Broker\u5173\u95ed\u6216\u5b95\u673a\u4e86\uff0c\u4ece\u800c\u8fdb\u884c\u5584\u540e\u3002<\/p>\n<h3>5.\u300c\u6570\u636e\u670d\u52a1\u300d<\/h3>\n<p>\u63a7\u5236\u5668\u4e0a\u4fdd\u5b58\u4e86\u6700\u5168\u7684\u96c6\u7fa4\u5143\u6570\u636e\u4fe1\u606f\uff0c\u5176\u4ed6\u6240\u6709Broker\u4f1a\u5b9a\u671f\u63a5\u6536\u63a7\u5236\u5668\u53d1\u6765\u7684\u5143\u6570\u636e\u66f4\u65b0\u8bf7\u6c42\uff0c\u4ece\u800c\u66f4\u65b0\u5176\u5185\u5b58\u4e2d\u7684\u7f13\u5b58\u6570\u636e\u3002<\/p>\n<p>\u300c\u63a7\u5236\u5668\u6545\u969c\u8f6c\u79fb(Failover)\u300d<\/p>\n<p>\u300c\u6545\u969c\u8f6c\u79fb\u6307\u7684\u662f\uff0c\u5f53\u8fd0\u884c\u4e2d\u7684\u63a7\u5236\u5668\u7a81\u7136\u5b95\u673a\u6216\u610f\u5916\u7ec8\u6b62\u65f6\uff0cKafka\u80fd\u591f\u5feb\u901f\u5730\u611f\u77e5\u5230\uff0c\u5e76\u7acb\u5373\u542f\u7528\u5907\u7528\u63a7\u5236\u5668\u6765\u4ee3\u66ff\u4e4b\u524d\u5931\u8d25\u7684\u63a7\u5236\u5668\u300d\u3002\u8fd9\u4e2a\u8fc7\u7a0b\u5c31\u88ab\u79f0\u4e3aFailover\uff0c\u8be5\u8fc7\u7a0b\u662f\u81ea\u52a8\u5b8c\u6210\u7684\uff0c\u65e0\u9700\u4f60\u624b\u52a8\u5e72\u9884\u3002<\/p>\n<p style=\"text-align: center\">\n<p>\u6700\u5f00\u59cb\u65f6\uff0cBroker 0\u662f\u63a7\u5236\u5668\u3002\u5f53Broker 0\u5b95\u673a\u540e\uff0cZooKeeper\u901a\u8fc7Watch\u673a\u5236\u611f\u77e5\u5230\u5e76\u5220\u9664\u4e86\/controller\u4e34\u65f6\u8282\u70b9\u3002<\/p>\n<p>\u4e4b\u540e\uff0c\u6240\u6709\u5b58\u6d3b\u7684Broker\u5f00\u59cb\u7ade\u9009\u65b0\u7684\u63a7\u5236\u5668\u8eab\u4efd\u3002<\/p>\n<p>Broker 3\u6700\u7ec8\u8d62\u5f97\u4e86\u9009\u4e3e\uff0c\u6210\u529f\u5730\u5728ZooKeeper\u4e0a\u91cd\u5efa\u4e86\/controller\u8282\u70b9\u3002<\/p>\n<p>\u4e4b\u540e\uff0cBroker 3\u4f1a\u4eceZooKeeper\u4e2d\u8bfb\u53d6\u96c6\u7fa4\u5143\u6570\u636e\u4fe1\u606f\uff0c\u5e76\u521d\u59cb\u5316\u5230\u81ea\u5df1\u7684\u7f13\u5b58\u4e2d\u3002<\/p>\n<p>\u81f3\u6b64\uff0c\u63a7\u5236\u5668\u7684Failover\u5b8c\u6210\uff0c\u53ef\u4ee5\u884c\u4f7f\u6b63\u5e38\u7684\u5de5\u4f5c\u804c\u8d23\u4e86\u3002<\/p>\n<h2>\u65e5\u5fd7\u5b58\u50a8<\/h2>\n<p>Kafka\u4e2d\u7684\u6d88\u606f\u662f\u4ee5\u4e3b\u9898\u4e3a\u57fa\u672c\u5355\u4f4d\u8fdb\u884c\u5f52\u7c7b\u7684\uff0c\u6bcf\u4e2a\u4e3b\u9898\u5728\u903b\u8f91\u4e0a\u76f8\u4e92\u72ec\u7acb\u3002<\/p>\n<p>\u6bcf\u4e2a\u4e3b\u9898\u53c8\u53ef\u4ee5\u5206\u4e3a\u4e00\u4e2a\u6216\u591a\u4e2a\u5206\u533a\uff0c\u5728\u4e0d\u8003\u8651\u526f\u672c\u7684\u60c5\u51b5\u4e0b\uff0c\u4e00\u4e2a\u5206\u533a\u4f1a\u5bf9\u5e94\u4e00\u4e2a\u65e5\u5fd7\u3002<\/p>\n<p>\u4f46\u8bbe\u8ba1\u8005\u8003\u8651\u5230\u968f\u7740\u65f6\u95f4\u63a8\u79fb\uff0c\u65e5\u5fd7\u6587\u4ef6\u4f1a\u4e0d\u65ad\u6269\u5927\uff0c\u56e0\u6b64\u4e3a\u4e86\u9632\u6b62Log\u8fc7\u5927\uff0c\u8bbe\u8ba1\u8005\u5f15\u5165\u4e86\u65e5\u5fd7\u5206\u6bb5(LogSegment)\u7684\u6982\u5ff5\uff0c\u5c06Log\u5207\u5206\u4e3a\u591a\u4e2aLogSegment\uff0c\u4fbf\u4e8e\u540e\u7eed\u7684\u6d88\u606f\u7ef4\u62a4\u548c\u6e05\u7406\u5de5\u4f5c\u3002<\/p>\n<p>\u4e0b\u56fe\u63cf\u7ed8\u4e86\u4e3b\u9898\u3001\u5206\u533a\u3001\u526f\u672c\u3001Log\u3001LogSegment\u4e94\u8005\u4e4b\u95f4\u7684\u5173\u7cfb\u3002<\/p>\n<p style=\"text-align: center\">\n<h3>\u300cLogSegment\u300d<\/h3>\n<p>\u5728Kafka\u4e2d\uff0c\u6bcf\u4e2aLog\u5bf9\u8c61\u53c8\u53ef\u4ee5\u5212\u5206\u4e3a\u591a\u4e2aLogSegment\u6587\u4ef6\uff0c\u6bcf\u4e2aLogSegment\u6587\u4ef6\u5305\u62ec\u4e00\u4e2a\u65e5\u5fd7\u6570\u636e\u6587\u4ef6\u548c\u4e24\u4e2a\u7d22\u5f15\u6587\u4ef6(\u504f\u79fb\u91cf\u7d22\u5f15\u6587\u4ef6\u548c\u6d88\u606f\u65f6\u95f4\u6233\u7d22\u5f15\u6587\u4ef6)\u3002<\/p>\n<p>\u5176\u4e2d\uff0c\u6bcf\u4e2aLogSegment\u4e2d\u7684\u65e5\u5fd7\u6570\u636e\u6587\u4ef6\u5927\u5c0f\u5747\u76f8\u7b49(\u8be5\u65e5\u5fd7\u6570\u636e\u6587\u4ef6\u7684\u5927\u5c0f\u53ef\u4ee5\u901a\u8fc7\u5728Kafka Broker\u7684config\/server.properties\u914d\u7f6e\u6587\u4ef6\u7684\u4e2d\u7684\u300clog.segment.bytes\u300d\u8fdb\u884c\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3a1G\u5927\u5c0f(1073741824\u5b57\u8282)\uff0c\u5728\u987a\u5e8f\u5199\u5165\u6d88\u606f\u65f6\u5982\u679c\u8d85\u51fa\u8be5\u8bbe\u5b9a\u7684\u9608\u503c\uff0c\u5c06\u4f1a\u521b\u5efa\u4e00\u7ec4\u65b0\u7684\u65e5\u5fd7\u6570\u636e\u548c\u7d22\u5f15\u6587\u4ef6)\u3002<\/p>\n<p style=\"text-align: center\">\n<h2>\u5e38\u7528\u53c2\u6570<\/h2>\n<h3>\u300cbroker\u7aef\u914d\u7f6e\u300d<\/h3>\n<ul>\n<li>broker.id<\/li>\n<\/ul>\n<p>\u6bcf\u4e2a kafka broker \u90fd\u6709\u4e00\u4e2a\u552f\u4e00\u7684\u6807\u8bc6\u6765\u8868\u793a\uff0c\u8fd9\u4e2a\u552f\u4e00\u7684\u6807\u8bc6\u7b26\u5373\u662f broker.id\uff0c\u5b83\u7684\u9ed8\u8ba4\u503c\u662f 0\u3002<\/p>\n<p>\u8fd9\u4e2a\u503c\u5728 kafka \u96c6\u7fa4\u4e2d\u5fc5\u987b\u662f\u552f\u4e00\u7684\uff0c\u8fd9\u4e2a\u503c\u53ef\u4ee5\u4efb\u610f\u8bbe\u5b9a\uff0c<\/p>\n<ul>\n<li>port<\/li>\n<\/ul>\n<p>\u5982\u679c\u4f7f\u7528\u914d\u7f6e\u6837\u672c\u6765\u542f\u52a8 kafka\uff0c\u5b83\u4f1a\u76d1\u542c 9092 \u7aef\u53e3\uff0c\u4fee\u6539 port \u914d\u7f6e\u53c2\u6570\u53ef\u4ee5\u628a\u5b83\u8bbe\u7f6e\u6210\u4efb\u610f\u7684\u7aef\u53e3\u3002<\/p>\n<p>\u8981\u6ce8\u610f\uff0c\u5982\u679c\u4f7f\u7528 1024 \u4ee5\u4e0b\u7684\u7aef\u53e3\uff0c\u9700\u8981\u4f7f\u7528 root \u6743\u9650\u542f\u52a8 kakfa\u3002<\/p>\n<ul>\n<li>zookeeper.connect<\/li>\n<\/ul>\n<p>\u7528\u4e8e\u4fdd\u5b58 broker \u5143\u6570\u636e\u7684 Zookeeper \u5730\u5740\u662f\u901a\u8fc7 zookeeper.connect \u6765\u6307\u5b9a\u7684\u3002<\/p>\n<p>\u6bd4\u5982\u53ef\u4ee5\u8fd9\u4e48\u6307\u5b9a localhost:2181 \u8868\u793a\u8fd9\u4e2a Zookeeper \u662f\u8fd0\u884c\u5728\u672c\u5730 2181 \u7aef\u53e3\u4e0a\u7684\u3002<\/p>\n<p>\u6211\u4eec\u4e5f\u53ef\u4ee5\u901a\u8fc7 \u6bd4\u5982\u6211\u4eec\u53ef\u4ee5\u901a\u8fc7 zk1:2181,zk2:2181,zk3:2181 \u6765\u6307\u5b9a zookeeper.connect \u7684\u591a\u4e2a\u53c2\u6570\u503c\u3002<\/p>\n<p>\u8be5\u914d\u7f6e\u53c2\u6570\u662f\u7528\u5192\u53f7\u5206\u5272\u7684\u4e00\u7ec4 hostname:port\/path \u5217\u8868\uff0c\u5176\u542b\u4e49\u5982\u4e0b<\/p>\n<ul>\n<li>hostname \u662f Zookeeper \u670d\u52a1\u5668\u7684\u673a\u5668\u540d\u6216\u8005 ip \u5730\u5740\u3002<\/li>\n<li>port \u662f Zookeeper \u5ba2\u6237\u7aef\u7684\u7aef\u53e3\u53f7<\/li>\n<li>\/path \u662f\u53ef\u9009\u62e9\u7684 Zookeeper \u8def\u5f84\uff0cKafka \u8def\u5f84\u662f\u4f7f\u7528\u4e86 chroot \u73af\u5883\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u9ed8\u8ba4\u4f7f\u7528\u8ddf\u8def\u5f84\u3002<\/li>\n<\/ul>\n<p>\u5982\u679c\u4f60\u6709\u4e24\u5957 Kafka \u96c6\u7fa4\uff0c\u5047\u8bbe\u5206\u522b\u53eb\u5b83\u4eec kafka1 \u548c kafka2\uff0c\u90a3\u4e48\u4e24\u5957\u96c6\u7fa4\u7684zookeeper.connect\u53c2\u6570\u53ef\u4ee5\u8fd9\u6837\u6307\u5b9a\uff1azk1:2181,zk2:2181,zk3:2181\/kafka1\u548czk1:2181,zk2:2181,zk3:2181\/kafka2<\/p>\n<ul>\n<li>log.dirs<\/li>\n<\/ul>\n<p>Kafka \u628a\u6240\u6709\u7684\u6d88\u606f\u90fd\u4fdd\u5b58\u5230\u78c1\u76d8\u4e0a\uff0c\u5b58\u653e\u8fd9\u4e9b\u65e5\u5fd7\u7247\u6bb5\u7684\u76ee\u5f55\u662f\u901a\u8fc7 log.dirs \u6765\u5236\u5b9a\u7684\uff0c\u5b83\u662f\u7528\u4e00\u7ec4\u9017\u53f7\u6765\u5206\u5272\u7684\u672c\u5730\u7cfb\u7edf\u8def\u5f84\uff0clog.dirs \u662f\u6ca1\u6709\u9ed8\u8ba4\u503c\u7684\uff0c\u300c\u4f60\u5fc5\u987b\u624b\u52a8\u6307\u5b9a\u4ed6\u7684\u9ed8\u8ba4\u503c\u300d\u3002<\/p>\n<p>\u5176\u5b9e\u8fd8\u6709\u4e00\u4e2a\u53c2\u6570\u662f log.dir\uff0c\u8fd9\u4e2a\u914d\u7f6e\u662f\u6ca1\u6709 s \u7684\uff0c\u9ed8\u8ba4\u60c5\u51b5\u4e0b\u53ea\u7528\u914d\u7f6e log.dirs \u5c31\u597d\u4e86\uff0c\u6bd4\u5982\u4f60\u53ef\u4ee5\u901a\u8fc7 \/home\/kafka1,\/home\/kafka2,\/home\/kafka3 \u8fd9\u6837\u6765\u914d\u7f6e\u8fd9\u4e2a\u53c2\u6570\u7684\u503c\u3002<\/p>\n<ul>\n<li>auto.create.topics.enable<\/li>\n<\/ul>\n<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0ckafka \u4f1a\u81ea\u52a8\u521b\u5efa\u4e3b\u9898<\/p>\n<p>auto.create.topics.enable\u53c2\u6570\u5efa\u8bae\u6700\u597d\u8bbe\u7f6e\u6210 false\uff0c\u5373\u4e0d\u5141\u8bb8\u81ea\u52a8\u521b\u5efa Topic\u3002<\/p>\n<h3>\u300c\u4e3b\u9898\u76f8\u5173\u914d\u7f6e\u300d<\/h3>\n<ul>\n<li>num.partitions<\/li>\n<\/ul>\n<p>num.partitions \u53c2\u6570\u6307\u5b9a\u4e86\u65b0\u521b\u5efa\u7684\u4e3b\u9898\u9700\u8981\u5305\u542b\u591a\u5c11\u4e2a\u5206\u533a\uff0c\u8be5\u53c2\u6570\u7684\u9ed8\u8ba4\u503c\u662f 1\u3002<\/p>\n<ul>\n<li>default.replication.factor<\/li>\n<\/ul>\n<p>\u8fd9\u4e2a\u53c2\u6570\u6bd4\u8f83\u7b80\u5355\uff0c\u5b83\u8868\u793a kafka\u4fdd\u5b58\u6d88\u606f\u7684\u526f\u672c\u6570\u3002<\/p>\n<ul>\n<li>log.retention.ms<\/li>\n<\/ul>\n<p>Kafka \u901a\u5e38\u6839\u636e\u65f6\u95f4\u6765\u51b3\u5b9a\u6570\u636e\u53ef\u4ee5\u4fdd\u7559\u591a\u4e45\u3002<\/p>\n<p>\u9ed8\u8ba4\u4f7f\u7528log.retention.hours\u53c2\u6570\u6765\u914d\u7f6e\u65f6\u95f4\uff0c\u9ed8\u8ba4\u662f 168 \u4e2a\u5c0f\u65f6\uff0c\u4e5f\u5c31\u662f\u4e00\u5468\u3002<\/p>\n<p>\u9664\u6b64\u4e4b\u5916\uff0c\u8fd8\u6709\u4e24\u4e2a\u53c2\u6570log.retention.minutes \u548clog.retentiion.ms \u3002<\/p>\n<p>\u8fd9\u4e09\u4e2a\u53c2\u6570\u4f5c\u7528\u662f\u4e00\u6837\u7684\uff0c\u90fd\u662f\u51b3\u5b9a\u6d88\u606f\u591a\u4e45\u4ee5\u540e\u88ab\u5220\u9664\uff0c\u63a8\u8350\u4f7f\u7528log.retention.ms\u3002<\/p>\n<ul>\n<li>message.max.bytes<\/li>\n<\/ul>\n<p>broker \u901a\u8fc7\u8bbe\u7f6e message.max.bytes \u53c2\u6570\u6765\u9650\u5236\u5355\u4e2a\u6d88\u606f\u7684\u5927\u5c0f\uff0c\u9ed8\u8ba4\u662f 1000 000\uff0c \u4e5f\u5c31\u662f 1MB\uff0c\u5982\u679c\u751f\u4ea7\u8005\u5c1d\u8bd5\u53d1\u9001\u7684\u6d88\u606f\u8d85\u8fc7\u8fd9\u4e2a\u5927\u5c0f\uff0c\u4e0d\u4ec5\u6d88\u606f\u4e0d\u4f1a\u88ab\u63a5\u6536\uff0c\u8fd8\u4f1a\u6536\u5230 broker \u8fd4\u56de\u7684\u9519\u8bef\u6d88\u606f\u3002<\/p>\n<ul>\n<li>retention.ms<\/li>\n<\/ul>\n<p>\u89c4\u5b9a\u4e86\u8be5\u4e3b\u9898\u6d88\u606f\u88ab\u4fdd\u5b58\u7684\u65f6\u5e38\uff0c\u9ed8\u8ba4\u662f7\u5929\uff0c\u5373\u8be5\u4e3b\u9898\u53ea\u80fd\u4fdd\u5b587\u5929\u7684\u6d88\u606f\uff0c\u4e00\u65e6\u8bbe\u7f6e\u4e86\u8fd9\u4e2a\u503c\uff0c\u5b83\u4f1a\u8986\u76d6\u6389 Broker \u7aef\u7684\u5168\u5c40\u53c2\u6570\u503c\u3002<\/p>\n<h2>\u6d88\u606f\u4e22\u5931\u95ee\u9898<\/h2>\n<h3>\u300c\u751f\u4ea7\u8005\u7a0b\u5e8f\u4e22\u5931\u6570\u636e\u300d<\/h3>\n<p>\u76ee\u524dKafka Producer\u662f\u5f02\u6b65\u53d1\u9001\u6d88\u606f\u7684\uff0c\u4e5f\u5c31\u662f\u8bf4\u5982\u679c\u4f60\u8c03\u7528\u7684\u662fproducer.send(msg)\u8fd9\u4e2aAPI\uff0c\u90a3\u4e48\u5b83\u901a\u5e38\u4f1a\u7acb\u5373\u8fd4\u56de\uff0c\u4f46\u6b64\u65f6\u4f60\u4e0d\u80fd\u8ba4\u4e3a\u6d88\u606f\u53d1\u9001\u5df2\u6210\u529f\u5b8c\u6210\u3002<\/p>\n<p>\u5982\u679c\u7528\u8fd9\u4e2a\u65b9\u5f0f\uff0c\u53ef\u80fd\u4f1a\u6709\u54ea\u4e9b\u56e0\u7d20\u5bfc\u81f4\u6d88\u606f\u6ca1\u6709\u53d1\u9001\u6210\u529f\u5462?<\/p>\n<p>\u5176\u5b9e\u539f\u56e0\u6709\u5f88\u591a\uff0c\u4f8b\u5982\u7f51\u7edc\u6296\u52a8\uff0c\u5bfc\u81f4\u6d88\u606f\u538b\u6839\u5c31\u6ca1\u6709\u53d1\u9001\u5230Broker\u7aef;\u6216\u8005\u6d88\u606f\u672c\u8eab\u4e0d\u5408\u683c\u5bfc\u81f4Broker\u62d2\u7edd\u63a5\u6536(\u6bd4\u5982\u6d88\u606f\u592a\u5927\u4e86\uff0c\u8d85\u8fc7\u4e86Broker\u7684\u627f\u53d7\u80fd\u529b)\u7b49\u3002<\/p>\n<p>\u5b9e\u9645\u4e0a\uff0c\u89e3\u51b3\u6b64\u95ee\u9898\u7684\u65b9\u6cd5\u975e\u5e38\u7b80\u5355\uff1aProducer\u6c38\u8fdc\u8981\u4f7f\u7528\u5e26\u6709\u56de\u8c03\u901a\u77e5\u7684\u53d1\u9001API\uff0c\u4e5f\u5c31\u662f\u8bf4\u4e0d\u8981\u4f7f\u7528producer.send(msg)\uff0c\u800c\u8981\u4f7f\u7528producer.send(msg, callback)\u3002<\/p>\n<p>\u5b83\u80fd\u51c6\u786e\u5730\u544a\u8bc9\u4f60\u6d88\u606f\u662f\u5426\u771f\u7684\u63d0\u4ea4\u6210\u529f\u4e86\u3002<\/p>\n<p>\u4e00\u65e6\u51fa\u73b0\u6d88\u606f\u63d0\u4ea4\u5931\u8d25\u7684\u60c5\u51b5\uff0c\u4f60\u5c31\u53ef\u4ee5\u6709\u9488\u5bf9\u6027\u5730\u8fdb\u884c\u5904\u7406\u3002<\/p>\n<h3>\u300c\u6d88\u8d39\u8005\u7a0b\u5e8f\u4e22\u5931\u6570\u636e\u300d<\/h3>\n<p>Consumer\u7aef\u4e22\u5931\u6570\u636e\u4e3b\u8981\u4f53\u73b0\u5728Consumer\u7aef\u8981\u6d88\u8d39\u7684\u6d88\u606f\u4e0d\u89c1\u4e86\u3002<\/p>\n<p>\u4e0b\u9762\u8fd9\u5f20\u56fe\u5b83\u6e05\u6670\u5730\u5c55\u793a\u4e86Consumer\u7aef\u7684\u4f4d\u79fb\u6570\u636e\u3002<\/p>\n<p style=\"text-align: center\">\n<p>\u6bd4\u5982\u5bf9\u4e8eConsumer A\u800c\u8a00\uff0c\u5b83\u5f53\u524d\u7684\u4f4d\u79fb\u503c\u5c31\u662f9;Consumer B\u7684\u4f4d\u79fb\u503c\u662f11\u3002<\/p>\n<p>Consumer\u7a0b\u5e8f\u4eceKafka\u83b7\u53d6\u5230\u6d88\u606f\u540e\u5f00\u542f\u4e86\u591a\u4e2a\u7ebf\u7a0b\u5f02\u6b65\u5904\u7406\u6d88\u606f\uff0c\u800cConsumer\u7a0b\u5e8f\u81ea\u52a8\u5730\u5411\u524d\u66f4\u65b0\u4f4d\u79fb\u3002<\/p>\n<p>\u5047\u5982\u5176\u4e2d\u67d0\u4e2a\u7ebf\u7a0b\u8fd0\u884c\u5931\u8d25\u4e86\uff0c\u5b83\u8d1f\u8d23\u7684\u6d88\u606f\u6ca1\u6709\u88ab\u6210\u529f\u5904\u7406\uff0c\u4f46\u4f4d\u79fb\u5df2\u7ecf\u88ab\u66f4\u65b0\u4e86\uff0c\u56e0\u6b64\u8fd9\u6761\u6d88\u606f\u5bf9\u4e8eConsumer\u800c\u8a00\u5b9e\u9645\u4e0a\u662f\u4e22\u5931\u4e86\u3002<\/p>\n<p>\u8fd9\u91cc\u7684\u5173\u952e\u5728\u4e8eConsumer\u81ea\u52a8\u63d0\u4ea4\u4f4d\u79fb\u3002<\/p>\n<p>\u8fd9\u4e2a\u95ee\u9898\u7684\u89e3\u51b3\u65b9\u6848\u4e5f\u5f88\u7b80\u5355\uff1a<\/p>\n<h3>\u300c\u5982\u679c\u662f\u591a\u7ebf\u7a0b\u5f02\u6b65\u5904\u7406\u6d88\u8d39\u6d88\u606f\uff0cConsumer\u7a0b\u5e8f\u4e0d\u8981\u5f00\u542f\u81ea\u52a8\u63d0\u4ea4\u4f4d\u79fb\uff0c\u800c\u662f\u8981\u5e94\u7528\u7a0b\u5e8f\u624b\u52a8\u63d0\u4ea4\u4f4d\u79fb\u300d\u3002<\/h3>\n<h2>\u6700\u4f73\u5b9e\u8df5<\/h2>\n<p>\u603b\u7ed3Kafka\u65e0\u6d88\u606f\u4e22\u5931\u7684\u914d\u7f6e\uff1a<\/p>\n<ol>\n<li>\u4e0d\u8981\u4f7f\u7528producer.send(msg)\uff0c\u800c\u8981\u4f7f\u7528producer.send(msg, callback)\uff0c\u4e00\u5b9a\u8981\u4f7f\u7528\u5e26\u6709\u56de\u8c03\u901a\u77e5\u7684send\u65b9\u6cd5\u3002<\/li>\n<li>\u8bbe\u7f6eacks = all\uff0cacks\u662fProducer\u7684\u4e00\u4e2a\u53c2\u6570\uff0c\u4ee3\u8868\u4e86\u4f60\u5bf9\u5df2\u63d0\u4ea4\u6d88\u606f\u7684\u5b9a\u4e49\uff0c\u5982\u679c\u8bbe\u7f6e\u6210all\uff0c\u5219\u8868\u660e\u6240\u6709\u526f\u672cBroker\u90fd\u8981\u63a5\u6536\u5230\u6d88\u606f\uff0c\u8be5\u6d88\u606f\u624d\u7b97\u662f\u5df2\u63d0\u4ea4\u3002<\/li>\n<li>\u8bbe\u7f6eretries\u4e3a\u4e00\u4e2a\u8f83\u5927\u7684\u503c\u3002\u8fd9\u91cc\u7684retries\u540c\u6837\u662fProducer\u7684\u53c2\u6570\uff0c\u5bf9\u5e94\u524d\u9762\u63d0\u5230\u7684Producer\u81ea\u52a8\u91cd\u8bd5\uff0c\u5f53\u51fa\u73b0\u7f51\u7edc\u7684\u77ac\u65f6\u6296\u52a8\u65f6\uff0c\u6d88\u606f\u53d1\u9001\u53ef\u80fd\u4f1a\u5931\u8d25\uff0c\u6b64\u65f6\u914d\u7f6e\u4e86retries &gt; 0\u7684Producer\u80fd\u591f\u81ea\u52a8\u91cd\u8bd5\u6d88\u606f\u53d1\u9001\uff0c\u907f\u514d\u6d88\u606f\u4e22\u5931\u3002<\/li>\n<li>\u8bbe\u7f6eunclean.leader.election.enable = false\uff0c\u8fd9\u662fBroker\u7aef\u7684\u53c2\u6570\uff0c\u5b83\u63a7\u5236\u7684\u662f\u54ea\u4e9bBroker\u6709\u8d44\u683c\u7ade\u9009\u5206\u533a\u7684Leader\uff0c\u5982\u679c\u4e00\u4e2aBroker\u843d\u540e\u539f\u5148\u7684Leader\u592a\u591a\uff0c\u90a3\u4e48\u5b83\u4e00\u65e6\u6210\u4e3a\u65b0\u7684Leader\uff0c\u5fc5\u7136\u4f1a\u9020\u6210\u6d88\u606f\u7684\u4e22\u5931\uff0c\u6545\u4e00\u822c\u90fd\u8981\u5c06\u8be5\u53c2\u6570\u8bbe\u7f6e\u6210false\uff0c\u5373\u4e0d\u5141\u8bb8\u8fd9\u79cd\u60c5\u51b5\u7684\u53d1\u751f\u3002<\/li>\n<li>\u8bbe\u7f6ereplication.factor &gt;= 3\uff0c\u8fd9\u4e5f\u662fBroker\u7aef\u7684\u53c2\u6570\uff0c\u5c06\u6d88\u606f\u591a\u4fdd\u5b58\u51e0\u4efd\uff0c\u76ee\u524d\u9632\u6b62\u6d88\u606f\u4e22\u5931\u7684\u4e3b\u8981\u673a\u5236\u5c31\u662f\u5197\u4f59\u3002<\/li>\n<li>\u8bbe\u7f6emin.insync.replicas &gt; 1\uff0c\u8fd9\u4f9d\u7136\u662fBroker\u7aef\u53c2\u6570\uff0c\u63a7\u5236\u7684\u662f\u6d88\u606f\u81f3\u5c11\u8981\u88ab\u5199\u5165\u5230\u591a\u5c11\u4e2a\u526f\u672c\u624d\u7b97\u662f\u5df2\u63d0\u4ea4\uff0c\u8bbe\u7f6e\u6210\u5927\u4e8e1\u53ef\u4ee5\u63d0\u5347\u6d88\u606f\u6301\u4e45\u6027\uff0c\u5728\u5b9e\u9645\u73af\u5883\u4e2d\u5343\u4e07\u4e0d\u8981\u4f7f\u7528\u9ed8\u8ba4\u503c1\u3002<\/li>\n<li>\u786e\u4fddreplication.factor &gt; min.insync.replicas\uff0c\u5982\u679c\u4e24\u8005\u76f8\u7b49\uff0c\u90a3\u4e48\u53ea\u8981\u6709\u4e00\u4e2a\u526f\u672c\u6302\u673a\uff0c\u6574\u4e2a\u5206\u533a\u5c31\u65e0\u6cd5\u6b63\u5e38\u5de5\u4f5c\u4e86\uff0c\u6211\u4eec\u4e0d\u4ec5\u8981\u6539\u5584\u6d88\u606f\u7684\u6301\u4e45\u6027\uff0c\u9632\u6b62\u6570\u636e\u4e22\u5931\uff0c\u8fd8\u8981\u5728\u4e0d\u964d\u4f4e\u53ef\u7528\u6027\u7684\u57fa\u7840\u4e0a\u5b8c\u6210\uff0c\u63a8\u8350\u8bbe\u7f6e\u6210replication.factor = min.insync.replicas + 1\u3002<\/li>\n<\/ol>\n<p>\u786e\u4fdd\u6d88\u606f\u6d88\u8d39\u5b8c\u6210\u518d\u63d0\u4ea4\uff0cConsumer\u7aef\u6709\u4e2a\u53c2\u6570enable.auto.commit\uff0c\u6700\u597d\u628a\u5b83\u8bbe\u7f6e\u6210false\uff0c\u5e76\u91c7\u7528\u624b\u52a8\u63d0\u4ea4\u4f4d\u79fb\u7684\u65b9\u5f0f\u3002<\/p>\n<h2>\u91cd\u590d\u6d88\u8d39\u95ee\u9898<\/h2>\n<h3>\u300c\u6d88\u8d39\u91cd\u590d\u7684\u573a\u666f\u300d<\/h3>\n<p>\u5728enable.auto.commit \u9ed8\u8ba4\u503ctrue\u60c5\u51b5\u4e0b\uff0c\u51fa\u73b0\u91cd\u590d\u6d88\u8d39\u7684\u573a\u666f\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a<\/p>\n<ul>\n<li>consumer \u5728\u6d88\u8d39\u8fc7\u7a0b\u4e2d\uff0c\u5e94\u7528\u8fdb\u7a0b\u88ab\u5f3a\u5236kill\u6389\u6216\u53d1\u751f\u5f02\u5e38\u9000\u51fa\u3002<\/li>\n<\/ul>\n<p>\u4f8b\u5982\u5728\u4e00\u6b21poll 500\u6761\u6d88\u606f\u540e\uff0c\u6d88\u8d39\u5230200\u6761\u65f6\uff0c\u8fdb\u7a0b\u88ab\u5f3a\u5236kill\u6d88\u8d39\u5230offset\u672a\u63d0\u4ea4\uff0c\u6216\u51fa\u73b0\u5f02\u5e38\u9000\u51fa\u5bfc\u81f4\u6d88\u8d39\u5230offset\u672a\u63d0\u4ea4\u3002<\/p>\n<p>\u4e0b\u6b21\u91cd\u542f\u65f6\uff0c\u4f9d\u7136\u4f1a\u91cd\u65b0\u62c9\u53d6500\u6d88\u606f\uff0c\u9020\u6210\u4e4b\u524d\u6d88\u8d39\u5230200\u6761\u6d88\u606f\u91cd\u590d\u6d88\u8d39\u4e86\u4e24\u6b21\u3002<\/p>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a\u5728\u53d1\u751f\u5f02\u5e38\u65f6\u6b63\u786e\u5904\u7406\u672a\u63d0\u4ea4\u7684offset<\/p>\n<p>\u300c\u6d88\u8d39\u8005\u6d88\u8d39\u65f6\u95f4\u8fc7\u957f\u300d<\/p>\n<p>max.poll.interval.ms\u53c2\u6570\u5b9a\u4e49\u4e86\u4e24\u6b21poll\u7684\u6700\u5927\u95f4\u9694\uff0c\u5b83\u7684\u9ed8\u8ba4\u503c\u662f 5 \u5206\u949f\uff0c\u8868\u793a\u4f60\u7684 Consumer \u7a0b\u5e8f\u5982\u679c\u5728 5 \u5206\u949f\u4e4b\u5185\u65e0\u6cd5\u6d88\u8d39\u5b8c poll \u65b9\u6cd5\u8fd4\u56de\u7684\u6d88\u606f\uff0c\u90a3\u4e48 Consumer \u4f1a\u4e3b\u52a8\u53d1\u8d77\u79bb\u5f00\u7ec4\u7684\u8bf7\u6c42\uff0cCoordinator \u4e5f\u4f1a\u5f00\u542f\u65b0\u4e00\u8f6e Rebalance\u3002<\/p>\n<p>\u4e3e\u4f8b\uff1a\u5355\u6b21\u62c9\u53d611\u6761\u6d88\u606f\uff0c\u6bcf\u6761\u6d88\u606f\u8017\u65f630s\uff0c11\u6761\u6d88\u606f\u8017\u65f65\u5206\u949f30\u79d2\uff0c\u7531\u4e8emax.poll.interval.ms \u9ed8\u8ba4\u503c5\u5206\u949f\uff0c\u6240\u4ee5\u6d88\u8d39\u8005\u65e0\u6cd5\u57285\u5206\u949f\u5185\u6d88\u8d39\u5b8c\uff0cconsumer\u4f1a\u79bb\u5f00\u7ec4\uff0c\u5bfc\u81f4rebalance\u3002<\/p>\n<p>\u5728\u6d88\u8d39\u5b8c11\u6761\u6d88\u606f\u540e\uff0cconsumer\u4f1a\u91cd\u65b0\u8fde\u63a5broker\uff0c\u518d\u6b21rebalance\uff0c\u56e0\u4e3a\u4e0a\u6b21\u6d88\u8d39\u7684offset\u672a\u63d0\u4ea4\uff0c\u518d\u6b21\u62c9\u53d6\u7684\u6d88\u606f\u662f\u4e4b\u524d\u6d88\u8d39\u8fc7\u7684\u6d88\u606f\uff0c\u9020\u6210\u91cd\u590d\u6d88\u8d39\u3002<\/p>\n<h3>\u300c\u89e3\u51b3\u65b9\u6848\uff1a\u300d<\/h3>\n<p>1\u3001\u63d0\u9ad8\u6d88\u8d39\u80fd\u529b\uff0c\u63d0\u9ad8\u5355\u6761\u6d88\u606f\u7684\u5904\u7406\u901f\u5ea6;\u6839\u636e\u5b9e\u9645\u573a\u666f\u53ef\u8bb2max.poll.interval.ms\u503c\u8bbe\u7f6e\u5927\u4e00\u70b9\uff0c\u907f\u514d\u4e0d\u5fc5\u8981\u7684rebalance;\u53ef\u9002\u5f53\u51cf\u5c0fmax.poll.records\u7684\u503c\uff0c\u9ed8\u8ba4\u503c\u662f500\uff0c\u53ef\u6839\u636e\u5b9e\u9645\u6d88\u606f\u901f\u7387\u9002\u5f53\u8c03\u5c0f\u3002<\/p>\n<p>2\u3001\u751f\u6210\u6d88\u606f\u65f6\uff0c\u53ef\u52a0\u5165\u552f\u4e00\u6807\u8bc6\u7b26\u5982\u6d88\u606fid\uff0c\u5728\u6d88\u8d39\u7aef\uff0c\u4fdd\u5b58\u6700\u8fd1\u76841000\u6761\u6d88\u606fid\u5b58\u5165\u5230redis\u6216mysql\u4e2d\uff0c\u6d88\u8d39\u7684\u6d88\u606f\u65f6\u901a\u8fc7\u524d\u7f6e\u53bb\u91cd\u3002<\/p>\n<h2>\u6d88\u606f\u987a\u5e8f\u95ee\u9898<\/h2>\n<p>\u6211\u4eec\u90fd\u77e5\u9053kafka\u7684topic\u662f\u65e0\u5e8f\u7684\uff0c\u4f46\u662f\u4e00\u4e2atopic\u5305\u542b\u591a\u4e2apartition\uff0c\u6bcf\u4e2apartition\u5185\u90e8\u662f\u6709\u5e8f\u7684<\/p>\n<p style=\"text-align: center\">\n<h3>\u300c\u4e71\u5e8f\u573a\u666f1\u300d<\/h3>\n<p>\u56e0\u4e3a\u4e00\u4e2atopic\u53ef\u4ee5\u6709\u591a\u4e2apartition\uff0ckafka\u53ea\u80fd\u4fdd\u8bc1partition\u5185\u90e8\u6709\u5e8f<\/p>\n<h3>\u300c\u89e3\u51b3\u65b9\u6848\u300d<\/h3>\n<p>1\u3001\u53ef\u4ee5\u8bbe\u7f6etopic\uff0c\u6709\u4e14\u53ea\u6709\u4e00\u4e2apartition<\/p>\n<p>2\u3001\u6839\u636e\u4e1a\u52a1\u9700\u8981\uff0c\u9700\u8981\u987a\u5e8f\u7684 \u6307\u5b9a\u4e3a\u540c\u4e00\u4e2apartition<\/p>\n<p>3\u3001\u6839\u636e\u4e1a\u52a1\u9700\u8981\uff0c\u6bd4\u5982\u540c\u4e00\u4e2a\u8ba2\u5355\uff0c\u4f7f\u7528\u540c\u4e00\u4e2akey\uff0c\u53ef\u4ee5\u4fdd\u8bc1\u5206\u914d\u5230\u540c\u4e00\u4e2apartition\u4e0a<\/p>\n<h3>\u300c\u4e71\u5e8f\u573a\u666f2\u300d<\/h3>\n<p>\u5bf9\u4e8e\u540c\u4e00\u4e1a\u52a1\u8fdb\u5165\u4e86\u540c\u4e00\u4e2a\u6d88\u8d39\u8005\u7ec4\u4e4b\u540e\uff0c\u7528\u4e86\u591a\u7ebf\u7a0b\u6765\u5904\u7406\u6d88\u606f\uff0c\u4f1a\u5bfc\u81f4\u6d88\u606f\u7684\u4e71\u5e8f<\/p>\n<h3>\u300c\u89e3\u51b3\u65b9\u6848\u300d<\/h3>\n<p>\u6d88\u8d39\u8005\u5185\u90e8\u6839\u636e\u7ebf\u7a0b\u6570\u91cf\u521b\u5efa\u7b49\u91cf\u7684\u5185\u5b58\u961f\u5217\uff0c\u5bf9\u4e8e\u9700\u8981\u987a\u5e8f\u7684\u4e00\u7cfb\u5217\u4e1a\u52a1\u6570\u636e\uff0c\u6839\u636ekey\u6216\u8005\u4e1a\u52a1\u6570\u636e\uff0c\u653e\u5230\u540c\u4e00\u4e2a\u5185\u5b58\u961f\u5217\u4e2d\uff0c\u7136\u540e\u7ebf\u7a0b\u4ece\u5bf9\u5e94\u7684\u5185\u5b58\u961f\u5217\u4e2d\u53d6\u51fa\u5e76\u64cd\u4f5c<\/p>\n<p style=\"text-align: center\">\n<h3>\u300c\u901a\u8fc7\u8bbe\u7f6e\u76f8\u540ckey\u6765\u4fdd\u8bc1\u6d88\u606f\u6709\u5e8f\u6027\uff0c\u4f1a\u6709\u4e00\u70b9\u7f3a\u9677\uff1a\u300d<\/h3>\n<p>\u4f8b\u5982\u6d88\u606f\u53d1\u9001\u8bbe\u7f6e\u4e86\u91cd\u8bd5\u673a\u5236\uff0c\u5e76\u4e14\u5f02\u6b65\u53d1\u9001\uff0c\u6d88\u606fA\u548cB\u8bbe\u7f6e\u76f8\u540c\u7684key\uff0c\u4e1a\u52a1\u4e0aA\u5148\u53d1\uff0cB\u540e\u53d1\uff0c\u7531\u4e8e\u7f51\u7edc\u6216\u8005\u5176\u4ed6\u539f\u56e0A\u53d1\u9001\u5931\u8d25\uff0cB\u53d1\u9001\u6210\u529f;A\u7531\u4e8e\u53d1\u9001\u5931\u8d25\u5c31\u4f1a\u91cd\u8bd5\u4e14\u91cd\u8bd5\u6210\u529f\uff0c\u8fd9\u65f6\u5019\u6d88\u606f\u987a\u5e8fB\u5728\u524dA\u5728\u540e\uff0c\u4e0e\u4e1a\u52a1\u53d1\u9001\u987a\u5e8f\u4e0d\u4e00\u81f4\uff0c\u5982\u679c\u9700\u8981\u89e3\u51b3\u8fd9\u4e2a\u95ee\u9898\uff0c\u9700\u8981\u8bbe\u7f6e\u53c2\u6570max.in.flight.requests.per.connection=1\uff0c\u5176\u542b\u4e49\u662f\u9650\u5236\u5ba2\u6237\u7aef\u5728\u5355\u4e2a\u8fde\u63a5\u4e0a\u80fd\u591f\u53d1\u9001\u7684\u672a\u54cd\u5e94\u8bf7\u6c42\u7684\u4e2a\u6570\uff0c\u8bbe\u7f6e\u6b64\u503c\u662f1\u8868\u793akafka broker\u5728\u54cd\u5e94\u8bf7\u6c42\u4e4b\u524dclient\u4e0d\u80fd\u518d\u5411\u540c\u4e00\u4e2abroker\u53d1\u9001\u8bf7\u6c42\uff0c\u8fd9\u4e2a\u53c2\u6570\u9ed8\u8ba4\u503c\u662f5<\/p>\n<ul>\n<li>\u5b98\u65b9\u6587\u6863\u8bf4\u660e\uff0c\u8fd9\u4e2a\u53c2\u6570\u5982\u679c\u5927\u4e8e1\uff0c\u7531\u4e8e\u91cd\u8bd5\u6d88\u606f\u987a\u5e8f\u53ef\u80fd\u91cd\u6392<\/li>\n<\/ul>\n<h2>\u9ad8\u6027\u80fd\u539f\u56e0<\/h2>\n<h3>\u300c\u987a\u5e8f\u8bfb\u5199\u300d<\/h3>\n<p>kafka\u7684\u6d88\u606f\u662f\u4e0d\u65ad\u8ffd\u52a0\u5230\u6587\u4ef6\u4e2d\u7684\uff0c\u8fd9\u4e2a\u7279\u6027\u4f7fkafka\u53ef\u4ee5\u5145\u5206\u5229\u7528\u78c1\u76d8\u7684\u987a\u5e8f\u8bfb\u5199\u6027\u80fd<\/p>\n<p>\u987a\u5e8f\u8bfb\u5199\u4e0d\u9700\u8981\u786c\u76d8\u78c1\u5934\u7684\u5bfb\u9053\u65f6\u95f4\uff0c\u53ea\u9700\u5f88\u5c11\u7684\u6247\u533a\u65cb\u8f6c\u65f6\u95f4\uff0c\u6240\u4ee5\u901f\u5ea6\u8fdc\u5feb\u4e8e\u968f\u673a\u8bfb\u5199<\/p>\n<p>Kafka \u53ef\u4ee5\u914d\u7f6e\u5f02\u6b65\u5237\u76d8\uff0c\u4e0d\u5f00\u542f\u540c\u6b65\u5237\u76d8\uff0c\u5f02\u6b65\u5237\u76d8\u4e0d\u9700\u8981\u7b49\u5199\u5165\u78c1\u76d8\u540e\u8fd4\u56de\u6d88\u606f\u6295\u9012\u7684 ACK\uff0c\u6240\u4ee5\u5b83\u63d0\u9ad8\u4e86\u6d88\u606f\u53d1\u9001\u7684\u541e\u5410\u91cf\uff0c\u964d\u4f4e\u4e86\u8bf7\u6c42\u7684\u5ef6\u65f6<\/p>\n<h3>\u300c\u96f6\u62f7\u8d1d\u300d<\/h3>\n<p>\u4f20\u7edf\u7684 IO \u6d41\u7a0b\uff0c\u9700\u8981\u5148\u628a\u6570\u636e\u62f7\u8d1d\u5230\u5185\u6838\u7f13\u51b2\u533a\uff0c\u518d\u4ece\u5185\u6838\u7f13\u51b2\u62f7\u8d1d\u5230\u7528\u6237\u7a7a\u95f4\uff0c\u5e94\u7528\u7a0b\u5e8f\u5904\u7406\u5b8c\u6210\u4ee5\u540e\uff0c\u518d\u62f7\u8d1d\u56de\u5185\u6838\u7f13\u51b2\u533a<\/p>\n<p>\u8fd9\u4e2a\u8fc7\u7a0b\u4e2d\u53d1\u751f\u4e86\u591a\u6b21\u6570\u636e\u62f7\u8d1d<\/p>\n<p>\u4e3a\u4e86\u51cf\u5c11\u4e0d\u5fc5\u8981\u7684\u62f7\u8d1d\uff0cKafka \u4f9d\u8d56 Linux \u5185\u6838\u63d0\u4f9b\u7684 Sendfile \u7cfb\u7edf\u8c03\u7528<\/p>\n<p>\u5728 Sendfile \u65b9\u6cd5\u4e2d\uff0c\u6570\u636e\u5728\u5185\u6838\u7f13\u51b2\u533a\u5b8c\u6210\u8f93\u5165\u548c\u8f93\u51fa\uff0c\u4e0d\u9700\u8981\u62f7\u8d1d\u5230\u7528\u6237\u7a7a\u95f4\u5904\u7406\uff0c\u8fd9\u4e5f\u5c31\u907f\u514d\u4e86\u91cd\u590d\u7684\u6570\u636e\u62f7\u8d1d<\/p>\n<p>\u5728\u5177\u4f53\u7684\u64cd\u4f5c\u4e2d\uff0cKafka \u628a\u6240\u6709\u7684\u6d88\u606f\u90fd\u5b58\u653e\u5728\u5355\u72ec\u7684\u6587\u4ef6\u91cc\uff0c\u5728\u6d88\u606f\u6295\u9012\u65f6\u76f4\u63a5\u901a\u8fc7 Sendfile \u65b9\u6cd5\u53d1\u9001\u6587\u4ef6\uff0c\u51cf\u5c11\u4e86\u4e0a\u4e0b\u6587\u5207\u6362\uff0c\u56e0\u6b64\u5927\u5927\u63d0\u9ad8\u4e86\u6027\u80fd<\/p>\n<h3>\u300cMMAP\u6280\u672f\u300d<\/h3>\n<p>\u9664\u4e86 Sendfile \u4e4b\u5916\uff0c\u8fd8\u6709\u4e00\u79cd\u96f6\u62f7\u8d1d\u7684\u5b9e\u73b0\u6280\u672f\uff0c\u5373 Memory Mapped Files<\/p>\n<p>Kafka \u4f7f\u7528 Memory Mapped Files \u5b8c\u6210\u5185\u5b58\u6620\u5c04\uff0cMemory Mapped Files \u5bf9\u6587\u4ef6\u7684\u64cd\u4f5c\u4e0d\u662f write\/read\uff0c\u800c\u662f\u76f4\u63a5\u5bf9\u5185\u5b58\u5730\u5740\u7684\u64cd\u4f5c\uff0c\u5982\u679c\u662f\u8c03\u7528\u6587\u4ef6\u7684 read \u64cd\u4f5c\uff0c\u5219\u628a\u6570\u636e\u5148\u8bfb\u53d6\u5230\u5185\u6838\u7a7a\u95f4\u4e2d\uff0c\u7136\u540e\u518d\u590d\u5236\u5230\u7528\u6237\u7a7a\u95f4\uff0c\u4f46 MMAP\u53ef\u4ee5\u5c06\u6587\u4ef6\u76f4\u63a5\u6620\u5c04\u5230\u7528\u6237\u6001\u7684\u5185\u5b58\u7a7a\u95f4\uff0c\u7701\u53bb\u4e86\u7528\u6237\u7a7a\u95f4\u5230\u5185\u6838\u7a7a\u95f4\u590d\u5236\u7684\u5f00\u9500<\/p>\n<p>Producer\u751f\u4ea7\u7684\u6570\u636e\u6301\u4e45\u5316\u5230broker\uff0c\u91c7\u7528mmap\u6587\u4ef6\u6620\u5c04\uff0c\u5b9e\u73b0\u987a\u5e8f\u7684\u5feb\u901f\u5199\u5165<\/p>\n<p>Customer\u4ecebroker\u8bfb\u53d6\u6570\u636e\uff0c\u91c7\u7528sendfile\uff0c\u5c06\u78c1\u76d8\u6587\u4ef6\u8bfb\u5230OS\u5185\u6838\u7f13\u51b2\u533a\u540e\uff0c\u76f4\u63a5\u8f6c\u5230socket buffer\u8fdb\u884c\u7f51\u7edc\u53d1\u9001\u3002<\/p>\n<h3>\u300c\u6279\u91cf\u53d1\u9001\u8bfb\u53d6\u300d<\/h3>\n<p>Kafka \u7684\u6279\u91cf\u5305\u62ec\u6279\u91cf\u5199\u5165\u3001\u6279\u91cf\u53d1\u5e03\u7b49\u3002\u5b83\u5728\u6d88\u606f\u6295\u9012\u65f6\u4f1a\u5c06\u6d88\u606f\u7f13\u5b58\u8d77\u6765\uff0c\u7136\u540e\u6279\u91cf\u53d1\u9001<\/p>\n<p>\u540c\u6837\uff0c\u6d88\u8d39\u7aef\u5728\u6d88\u8d39\u6d88\u606f\u65f6\uff0c\u4e5f\u4e0d\u662f\u4e00\u6761\u4e00\u6761\u5904\u7406\u7684\uff0c\u800c\u662f\u6279\u91cf\u8fdb\u884c\u62c9\u53d6\uff0c\u63d0\u9ad8\u4e86\u6d88\u606f\u7684\u5904\u7406\u901f\u5ea6<\/p>\n<h3>\u300c\u6570\u636e\u538b\u7f29\u300d<\/h3>\n<p>Kafka\u8fd8\u652f\u6301\u5bf9\u6d88\u606f\u96c6\u5408\u8fdb\u884c\u538b\u7f29\uff0cProducer\u53ef\u4ee5\u901a\u8fc7GZIP\u6216Snappy\u683c\u5f0f\u5bf9\u6d88\u606f\u96c6\u5408\u8fdb\u884c\u538b\u7f29<\/p>\n<p>\u538b\u7f29\u7684\u597d\u5904\u5c31\u662f\u51cf\u5c11\u4f20\u8f93\u7684\u6570\u636e\u91cf\uff0c\u51cf\u8f7b\u5bf9\u7f51\u7edc\u4f20\u8f93\u7684\u538b\u529b<\/p>\n<p>Producer\u538b\u7f29\u4e4b\u540e\uff0c\u5728Consumer\u9700\u8fdb\u884c\u89e3\u538b\uff0c\u867d\u7136\u589e\u52a0\u4e86CPU\u7684\u5de5\u4f5c\uff0c\u4f46\u5728\u5bf9\u5927\u6570\u636e\u5904\u7406\u4e0a\uff0c\u74f6\u9888\u5728\u7f51\u7edc\u4e0a\u800c\u4e0d\u662fCPU\uff0c\u6240\u4ee5\u8fd9\u4e2a\u6210\u672c\u5f88\u503c\u5f97<\/p>\n<h3>\u300c\u5206\u533a\u673a\u5236\u300d<\/h3>\n<p>kafka\u4e2d\u7684topic\u4e2d\u7684\u5185\u5bb9\u53ef\u4ee5\u88ab\u5206\u4e3a\u591apartition\u5b58\u5728\uff0c\u6bcf\u4e2apartition\u53c8\u5206\u4e3a\u591a\u4e2a\u6bb5segment\uff0c\u6240\u4ee5\u6bcf\u6b21\u64cd\u4f5c\u90fd\u662f\u9488\u5bf9\u4e00\u5c0f\u90e8\u5206\u505a\u64cd\u4f5c\uff0c\u5f88\u8f7b\u4fbf\uff0c\u5e76\u4e14\u589e\u52a0\u5e76\u884c\u64cd\u4f5c\u7684\u80fd\u529b<\/p>\n<h2>\u5e38\u89c1\u9762\u8bd5\u9898<\/h2>\n<h3>\u300cKafka\u662fPush\u8fd8\u662fPull\u6a21\u5f0f?\u300d<\/h3>\n<p>Kafka\u6700\u521d\u8003\u8651\u7684\u95ee\u9898\u662f\uff0ccustomer\u5e94\u8be5\u4ecebrokes\u62c9\u53d6\u6d88\u606f\u8fd8\u662fbrokers\u5c06\u6d88\u606f\u63a8\u9001\u5230consumer\u3002<\/p>\n<p>\u5728\u8fd9\u65b9\u9762\uff0cKafka\u9075\u5faa\u4e86\u4e00\u79cd\u5927\u90e8\u5206\u6d88\u606f\u7cfb\u7edf\u5171\u540c\u7684\u4f20\u7edf\u7684\u8bbe\u8ba1\uff1aproducer\u5c06\u6d88\u606f\u63a8\u9001\u5230broker\uff0cconsumer\u4ecebroker\u62c9\u53d6\u6d88\u606f\u3002<\/p>\n<p>push\u6a21\u5f0f\u7531broker\u51b3\u5b9a\u6d88\u606f\u63a8\u9001\u7684\u901f\u7387\uff0c\u5bf9\u4e8e\u4e0d\u540c\u6d88\u8d39\u901f\u7387\u7684consumer\u5c31\u4e0d\u592a\u597d\u5904\u7406\u4e86\u3002<\/p>\n<p>\u6d88\u606f\u7cfb\u7edf\u90fd\u81f4\u529b\u4e8e\u8ba9consumer\u4ee5\u6700\u5927\u7684\u901f\u7387\u6700\u5feb\u901f\u7684\u6d88\u8d39\u6d88\u606f\uff0cpush\u6a21\u5f0f\u4e0b\uff0c\u5f53broker\u63a8\u9001\u7684\u901f\u7387\u8fdc\u5927\u4e8econsumer\u6d88\u8d39\u7684\u901f\u7387\u65f6\uff0cconsumer\u6050\u6015\u5c31\u8981\u5d29\u6e83\u4e86\u3002<\/p>\n<ul>\n<li>Kafka\u4e2d\u7684Producer\u548cConsumer\u91c7\u7528\u7684\u662fPush-and-Pull\u6a21\u5f0f\uff0c\u5373Producer\u5411Broker Push\u6d88\u606f\uff0cConsumer\u4eceBroker Pull\u6d88\u606f\u3002<\/li>\n<\/ul>\n<p>Pull\u6a21\u5f0f\u7684\u4e00\u4e2a\u597d\u5904\u662fconsumer\u53ef\u4ee5\u81ea\u4e3b\u51b3\u5b9a\u662f\u5426\u6279\u91cf\u7684\u4ecebroker\u62c9\u53d6\u6570\u636e\u3002<\/p>\n<p>Pull\u6709\u4e2a\u7f3a\u70b9\u662f\uff0c\u5982\u679cbroker\u6ca1\u6709\u53ef\u4f9b\u6d88\u8d39\u7684\u6d88\u606f\uff0c\u5c06\u5bfc\u81f4consumer\u4e0d\u65ad\u5728\u5faa\u73af\u4e2d\u8f6e\u8be2\uff0c\u76f4\u5230\u65b0\u6d88\u606f\u5230\u8fbe\u3002<\/p>\n<h3>\u300cKafka\u5982\u4f55\u4fdd\u8bc1\u9ad8\u53ef\u7528?\u300d<\/h3>\n<p>\u9762\u8bd5\u9898\uff1aKafka\u5982\u4f55\u4fdd\u8bc1\u9ad8\u53ef\u7528?\u6709\u56fe\u6709\u771f\u76f8<\/p>\n<h3>\u300cKafk\u7684\u4f7f\u7528\u573a\u666f\u300d<\/h3>\n<p>\u4e1a\u754cKafka\u5b9e\u9645\u5e94\u7528\u573a\u666f<\/p>\n<ul>\n<li>\u5f02\u6b65\u901a\u4fe1<\/li>\n<\/ul>\n<p>\u6d88\u606f\u4e2d\u95f4\u4ef6\u5728\u5f02\u6b65\u901a\u4fe1\u4e2d\u7528\u7684\u6700\u591a\uff0c\u5f88\u591a\u4e1a\u52a1\u6d41\u7a0b\u4e2d\uff0c\u5982\u679c\u6240\u6709\u6b65\u9aa4\u90fd\u540c\u6b65\u8fdb\u884c\u53ef\u80fd\u4f1a\u5bfc\u81f4\u6838\u5fc3\u6d41\u7a0b\u8017\u65f6\u975e\u5e38\u957f\uff0c\u66f4\u91cd\u8981\u7684\u662f\u6240\u6709\u6b65\u9aa4\u90fd\u540c\u6b65\u8fdb\u884c\u4e00\u65e6\u975e\u6838\u5fc3\u6b65\u9aa4\u5931\u8d25\u4f1a\u5bfc\u81f4\u6838\u5fc3\u6d41\u7a0b\u6574\u4f53\u5931\u8d25\uff0c\u56e0\u6b64\u5728\u5f88\u591a\u4e1a\u52a1\u6d41\u7a0b\u4e2dKafka\u5c31\u5145\u5f53\u4e86\u5f02\u6b65\u901a\u4fe1\u89d2\u8272\u3002<\/p>\n<ul>\n<li>\u65e5\u5fd7\u540c\u6b65<\/li>\n<\/ul>\n<p>\u5927\u89c4\u6a21\u5206\u5e03\u5f0f\u7cfb\u7edf\u4e2d\u7684\u673a\u5668\u975e\u5e38\u591a\u800c\u4e14\u5206\u6563\u5728\u4e0d\u540c\u673a\u623f\u4e2d\uff0c\u5206\u5e03\u5f0f\u7cfb\u7edf\u5e26\u6765\u7684\u4e00\u4e2a\u660e\u663e\u95ee\u9898\u5c31\u662f\u4e1a\u52a1\u65e5\u5fd7\u7684\u67e5\u770b\u3001\u8ffd\u8e2a\u548c\u5206\u6790\u7b49\u884c\u4e3a\u53d8\u5f97\u5341\u5206\u56f0\u96be\uff0c\u5bf9\u4e8e\u96c6\u7fa4\u89c4\u6a21\u5728\u767e\u53f0\u4ee5\u4e0a\u7684\u7cfb\u7edf\uff0c\u67e5\u8be2\u7ebf\u4e0a\u65e5\u5fd7\u5f88\u6050\u6016\u3002<\/p>\n<p>\u4e3a\u4e86\u5e94\u5bf9\u8fd9\u79cd\u573a\u666f\u7edf\u4e00\u65e5\u5fd7\u7cfb\u7edf\u5e94\u8fd0\u800c\u751f\uff0c\u65e5\u5fd7\u6570\u636e\u90fd\u662f\u6d77\u91cf\u6570\u636e\uff0c\u901a\u5e38\u4e3a\u4e86\u4e0d\u7ed9\u7cfb\u7edf\u5e26\u6765\u989d\u5916\u8d1f\u62c5\u4e00\u822c\u4f1a\u91c7\u7528\u5f02\u6b65\u4e0a\u62a5\uff0c\u8fd9\u91ccKafka\u4ee5\u5176\u9ad8\u541e\u5410\u91cf\u5728\u65e5\u5fd7\u5904\u7406\u4e2d\u5f97\u5230\u4e86\u5f88\u597d\u7684\u5e94\u7528\u3002<\/p>\n<ul>\n<li>\u5b9e\u65f6\u8ba1\u7b97<\/li>\n<\/ul>\n<p>\u968f\u7740\u636e\u91cf\u7684\u589e\u52a0\uff0c\u79bb\u7ebf\u7684\u8ba1\u7b97\u4f1a\u8d8a\u6765\u8d8a\u6162\uff0c\u96be\u4ee5\u6ee1\u8db3\u7528\u6237\u5728\u67d0\u4e9b\u573a\u666f\u4e0b\u7684\u5b9e\u65f6\u6027\u8981\u6c42\uff0c\u56e0\u6b64\u5f88\u591a\u89e3\u51b3\u65b9\u6848\u4e2d\u5f15\u5165\u4e86\u5b9e\u65f6\u8ba1\u7b97\u3002<\/p>\n<p>\u5f88\u591a\u65f6\u5019\uff0c\u5373\u4f7f\u662f\u6d77\u91cf\u6570\u636e\uff0c\u6211\u4eec\u4e5f\u5e0c\u671b\u5373\u65f6\u53bb\u67e5\u770b\u4e00\u4e9b\u6570\u636e\u6307\u6807\uff0c\u5b9e\u65f6\u6d41\u8ba1\u7b97\u5e94\u8fd0\u800c\u751f\u3002<\/p>\n<p>\u5b9e\u65f6\u6d41\u8ba1\u7b97\u6709\u4e24\u4e2a\u7279\u70b9\uff0c\u4e00\u4e2a\u662f\u5b9e\u65f6\uff0c\u968f\u65f6\u53ef\u4ee5\u770b\u6570\u636e;\u53e6\u4e00\u4e2a\u662f\u6d41\u3002<\/p>\n<p style=\"padding: 0px 0px 15px;margin: 0px;font-size: 16px;text-align: justify;line-height: 30px\">\u3010\u7f16\u8f91\u63a8\u8350\u3011<\/p>\n<ol>\n<li>\u9e3f\u8499\u5b98\u65b9\u6218\u7565\u5408\u4f5c\u5171\u5efa\u2014\u2014HarmonyOS\u6280\u672f\u793e\u533a<\/li>\n<li>\u7167\u6284\u4e0d\u7ffb\u8f66\uff1a\u6297\u4f4f\u5343\u4e07\u6d41\u91cf\u7684\u5927\u578b\u5206\u5e03\u5f0f\u7cfb\u7edf\u67b6\u6784\u8bbe\u8ba1<\/li>\n<li>2021\u5e74\u4e94\u5927\u5f00\u6e90\u5f0f\u6e38\u620f\u5316\u5de5\u5177<\/li>\n<li>\u6570\u5b57\u5316\u8f6c\u578b\u7684\u4e03\u5927\u70ed\u95e8\u8d8b\u52bf\u548c\u4e09\u5927\u6e10\u51b7\u8d8b\u52bf<\/li>\n<li>Windows 11\u65b0\u9884\u89c8\u724822449\u63a8\u9001\uff1a\u542f\u52a8\u5f15\u5bfc\u52a8\u753b\u53d8\u6837\u4e86<\/li>\n<li>\u4ec0\u4e48\u60c5\u51b5\uff1f\u6e38\u620f\u73a9\u5bb6\u5927\u89c4\u6a21\u9000\u56deWindows 7\u7cfb\u7edf\uff1aWindows 10\u66b4\u8dcc<\/li>\n<\/ol>\n<p>&nbsp;<\/p>\n<p><span><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u4e00\u7bc7\u5e26\u7ed9\u4f60Kafka\u6838\u5fc3\u77e5\u8bc6\u603b\u7ed3\uff01 \u4f5c\u8005\uff1a\u65e5\u5e38\u52a0\u6cb9\u7ad9 2021-09-06 08:31:11 \u5f00\u53d1 \u67b6\u6784 Ka [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":223332,"comment_status":"closed","ping_status":"","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[55],"tags":[],"class_list":["post-223331","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-cloud-phone"],"_links":{"self":[{"href":"https:\/\/idc.net\/help\/wp-json\/wp\/v2\/posts\/223331","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/idc.net\/help\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/idc.net\/help\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/idc.net\/help\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/idc.net\/help\/wp-json\/wp\/v2\/comments?post=223331"}],"version-history":[{"count":0,"href":"https:\/\/idc.net\/help\/wp-json\/wp\/v2\/posts\/223331\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/idc.net\/help\/wp-json\/wp\/v2\/media\/223332"}],"wp:attachment":[{"href":"https:\/\/idc.net\/help\/wp-json\/wp\/v2\/media?parent=223331"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/idc.net\/help\/wp-json\/wp\/v2\/categories?post=223331"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/idc.net\/help\/wp-json\/wp\/v2\/tags?post=223331"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}