{"id":16376,"date":"2024-08-08T14:19:59","date_gmt":"2024-08-08T06:19:59","guid":{"rendered":"http:\/\/139.9.1.231\/?p=16376"},"modified":"2024-09-18T19:37:23","modified_gmt":"2024-09-18T11:37:23","slug":"nccl","status":"publish","type":"post","link":"http:\/\/139.9.1.231\/index.php\/2024\/08\/08\/nccl\/","title":{"rendered":"NCCL&#8211;\u591a\u5361\u8bad\u7ec3\u540e\u7aef[\u6301\u7eed\u8865\u5145]"},"content":{"rendered":"\n<p class=\"has-text-align-center has-dark-gray-color has-text-color\"><strong>\u672c\u6587\u4e3b\u8981\u8bb0\u5f55\u548c\u5b66\u4e60pytorch\u540e\u7aefNCCL\u76f8\u5173\u7684\u77e5\u8bc6\u70b9\uff0c\u4e3a\u540e\u7eed\u5927\u6a21\u578b\u8bad\u7ec3\u6253\u597d\u57fa\u7840<\/strong><\/p>\n\n\n\n\n\n<p>         <a href=\"https:\/\/docs.nvidia.com\/deeplearning\/nccl\/user-guide\/docs\/overview.html\" target=\"_blank\" rel=\"noreferrer noopener\">https:\/\/docs.nvidia.com\/deeplearning\/nccl\/user-guide\/docs\/overview.html<\/a><\/p>\n\n\n\n<p class=\"has-text-align-center\"><a href=\"https:\/\/developer.nvidia.com\/nccl\">https:\/\/developer.nvidia.com\/nccl<\/a><\/p>\n\n\n\n<p>&#8220;<strong>NCCL<\/strong>&#8221; \u4ee3\u8868 &#8220;<strong>N<\/strong>VIDIA&nbsp;<strong>C<\/strong>ollective&nbsp;<strong>C<\/strong>ommunications&nbsp;<strong>L<\/strong>ibrary&#8221;\uff0c&#8221;<strong>NVIDIA \u96c6\u4f53\u901a\u4fe1\u5e93<\/strong>&#8220;\uff0c\u5b83\u662f\u4e00\u79cd\u7531&nbsp;NVIDIA&nbsp;\u5f00\u53d1\u7684\u7528\u4e8e<strong>\u9ad8\u6027\u80fd\u8ba1\u7b97<\/strong>\u7684<strong>\u901a\u4fe1<\/strong>\u5e93\u3002NCCL \u4e13\u95e8\u8bbe\u8ba1\u7528\u4e8e<strong>\u52a0\u901f GPU \u7fa4\u96c6\u4e4b\u95f4\u7684\u901a\u4fe1<\/strong>\uff0c\u4ee5\u4fbf\u5728<strong>\u5e76\u884c\u8ba1\u7b97<\/strong>\u548c<strong>\u6df1\u5ea6\u5b66\u4e60<\/strong>\u7b49\u9886\u57df\u4e2d\u63d0\u4f9b\u66f4\u597d\u7684\u6027\u80fd\u3002<\/p>\n\n\n\n<p>NVIDIA \u96c6\u5408\u901a\u4fe1\u5e93 (NCCL) \u53ef\u5b9e\u73b0\u9488\u5bf9 NVIDIA GPU \u548c\u7f51\u7edc\u8fdb\u884c\u6027\u80fd\u4f18\u5316\u7684\u591a GPU \u548c\u591a\u8282\u70b9\u901a\u4fe1\u57fa\u5143\u3002NCCL \u63d0\u4f9b\u4e86 all-gather\u3001all-reduce\u3001broadcast\u3001reduce\u3001reduce-scatter\u3001point-to-point send \u548c receive \u7b49\u4f8b\u7a0b\uff0c\u8fd9\u4e9b\u4f8b\u7a0b\u5747\u7ecf\u8fc7\u4f18\u5316\uff0c\u53ef\u901a\u8fc7\u8282\u70b9\u5185\u7684 PCIe \u548c NVLink \u9ad8\u901f\u4e92\u8054\u4ee5\u53ca\u8282\u70b9\u95f4\u7684 NVIDIA Mellanox \u7f51\u7edc\u5b9e\u73b0\u9ad8\u5e26\u5bbd\u548c\u4f4e\u5ef6\u8fdf\u3002<\/p>\n\n\n\n<h2>NCCL\u76f8\u5173\u73af\u5883\u53d8\u91cf\u8bf4\u660e \uff1a<\/h2>\n\n\n\n<p>\u3010https:\/\/docs.nvidia.com\/deeplearning\/nccl\/user-guide\/docs\/usage.html\u3011<\/p>\n\n\n\n<ol><li><strong>NCCL_TIMEOUT:\u8bbe\u7f6e\u96c6\u5408\u64cd\u4f5c\u8d85\u65f6\u9608\u503c,\u5355\u4f4d\u6beb\u79d2\uff1b\u5982\u679c\u5e38\u89c1\u8d85\u65f6\u9519\u8bef,\u9002\u5f53\u589e\u5927\u8be5\u503c,\u4f46\u4e0d\u80fd\u592a\u5927<\/strong>\u3002<code>NCCL_TIMEOUT<\/code> \u73af\u5883\u53d8\u91cf\u7528\u4e8e\u8bbe\u7f6e NCCL \u96c6\u4f53\u901a\u4fe1\u64cd\u4f5c\u7684\u8d85\u65f6\u65f6\u95f4\u3002\u901a\u8fc7\u8c03\u6574\u8fd9\u4e2a\u503c\uff0c\u4f60\u53ef\u4ee5\u66f4\u597d\u5730\u5904\u7406\u7f51\u7edc\u5ef6\u8fdf\u548c\u4e0d\u7a33\u5b9a\u7684\u95ee\u9898\uff0c\u786e\u4fdd NCCL \u901a\u4fe1\u7684\u7a33\u5b9a\u6027\u548c\u53ef\u9760\u6027\u3002\u5982\u679c\u5728\u96c6\u4f53\u901a\u4fe1\u8fc7\u7a0b\u4e2d\u9047\u5230\u8d85\u65f6\u95ee\u9898\uff0c\u53ef\u4ee5\u5c1d\u8bd5\u8c03\u6574\u6b64\u73af\u5883\u53d8\u91cf\u4ee5\u89e3\u51b3\u95ee\u9898\u3002<\/li><\/ol>\n\n\n\n<p><strong>\u8bbe\u7f6e\u8d85\u65f6\u65f6\u95f4<\/strong>:<\/p>\n\n\n\n<ul><li><code>NCCL_TIMEOUT<\/code> \u7528\u4e8e\u5b9a\u4e49 NCCL \u96c6\u4f53\u901a\u4fe1\u64cd\u4f5c\u7684\u8d85\u65f6\u65f6\u95f4\u3002\u8d85\u65f6\u65f6\u95f4\u662f NCCL \u5728\u6267\u884c\u64cd\u4f5c\u65f6\u7b49\u5f85\u54cd\u5e94\u7684\u6700\u957f\u65f6\u95f4\uff0c\u8d85\u51fa\u6b64\u65f6\u95f4\u5c06\u89e6\u53d1\u8d85\u65f6\u9519\u8bef\u3002<\/li><\/ul>\n\n\n\n<p><strong>\u89e3\u51b3\u7f51\u7edc\u95ee\u9898<\/strong>:<\/p>\n\n\n\n<ul><li>\u5728\u9ad8\u6027\u80fd\u8ba1\u7b97\u548c\u5927\u89c4\u6a21\u5206\u5e03\u5f0f\u8bad\u7ec3\u4e2d\uff0c\u7f51\u7edc\u5ef6\u8fdf\u6216\u4e0d\u7a33\u5b9a\u53ef\u80fd\u5bfc\u81f4\u96c6\u4f53\u901a\u4fe1\u64cd\u4f5c\u8d85\u65f6\u3002\u8bbe\u7f6e\u5408\u9002\u7684 <code>NCCL_TIMEOUT<\/code> \u53ef\u4ee5\u5e2e\u52a9\u8c03\u8282\u5bb9\u9519\u8bbe\u7f6e\uff0c\u907f\u514d\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u56e0\u8d85\u65f6\u9519\u8bef\u800c\u4e2d\u65ad\u3002<\/li><\/ul>\n\n\n\n<p><strong>\u6027\u80fd\u8c03\u4f18<\/strong>:<\/p>\n\n\n\n<ul><li>\u6839\u636e\u4f60\u7684\u96c6\u7fa4\u914d\u7f6e\u548c\u7f51\u7edc\u72b6\u51b5\uff0c\u9002\u5f53\u8c03\u6574 <code>NCCL_TIMEOUT<\/code> \u53ef\u4ee5\u5e2e\u52a9\u4f18\u5316\u901a\u4fe1\u6027\u80fd\u548c\u7a33\u5b9a\u6027\u3002<\/li><\/ul>\n\n\n\n<ol><li>NCCL_ALGO:\u9009\u62e9\u96c6\u5408\u901a\u4fe1\u7b97\u6cd5,\u5982Ring, Tree\uff1b\u4e0d\u540c\u62d3\u6251\u9002\u5408\u4e0d\u540c\u7b97\u6cd5,\u6d4b\u8bd5\u9009\u66f4\u4f18\u7b97\u6cd5<\/li><li>NCCL_CHUNK_SIZE:\u5b9a\u4e49\u73af\u5f62\u4f20\u8f93\u7f13\u51b2\u533a\u5927\u5c0f\uff1b\u5408\u7406\u8bbe\u7f6e\u53ef\u63d0\u901f,\u4f46\u4e5f\u4f1a\u589e\u52a0\u5185\u5b58\u6d88\u8017<\/li><li>NCCL_DEBUG:\u6253\u5f00NCCL\u8c03\u8bd5\u65e5\u5fd7\uff1b\u51fa\u73b0\u95ee\u9898\u65f6\u6253\u5f00\u8c03\u8bd5,\u4f46\u4f1a\u964d\u4f4e\u901f\u5ea6,\u4e0d\u8981\u5728\u751f\u4ea7\u73af\u5883\u4f7f\u7528<\/li><li>NCCL_DEBUG_FILE\u8bbe\u7f6e\u4e00\u4e2a\u6587\u4ef6\u5730\u5740\uff0c\u53d8\u91cf\u7528\u4e8e\u5c06NCCL\u7684\u8c03\u8bd5\u65e5\u5fd7\u8f93\u51fa\u5230\u6587\u4ef6\u4e2d\u3002\u6709\u52a9\u4e8e\u8c03\u8bd5nccl\u3002<\/li><li>NCCL_P2P_LEVEL:\u8bbe\u7f6e\u70b9\u5bf9\u70b9\u901a\u4fe1\u4f18\u5316\u7ea7\u522b\uff1b\u589e\u52a0\u8be5\u503c\u53ef\u51cf\u5c11P2P\u6b21\u6570,\u63d0\u9ad8\u67d0\u4e9b\u64cd\u4f5c\u6548\u7387<\/li><li><strong>NCCL_P2P_DISABLE:\u7981\u7528\u70b9\u5bf9\u70b9\u901a\u4fe1,\u5f3a\u5236\u4f7f\u7528\u96c6\u5408\u901a\u4fe1\u3002<\/strong>\u5728\u67d0\u4e9b\u60c5\u51b5\u4e0b\uff0cP2P \u901a\u4fe1\u53ef\u80fd\u4f1a\u5bfc\u81f4\u6027\u80fd\u95ee\u9898\u6216\u51fa\u73b0\u9519\u8bef\u3002\u7981\u7528 P2P \u901a\u4fe1\u53ef\u4ee5\u5e2e\u52a9\u89e3\u51b3\u8fd9\u4e9b\u95ee\u9898\u3002\u5982\u679c\u4f60\u9047\u5230\u4e0e P2P \u901a\u4fe1\u76f8\u5173\u7684\u9519\u8bef\u6216\u4e0d\u7a33\u5b9a\u6027\uff0c\u7981\u7528 P2P \u53ef\u80fd\u6709\u52a9\u4e8e\u6062\u590d\u7cfb\u7edf\u7684\u7a33\u5b9a\u6027\u3002<\/li><li>NCCL_PXN_DISABLE\uff1a\u7981\u7528\u4f7f\u7528\u975e\u672c\u5730 NIC \u7684\u8282\u70b9\u95f4\u901a\u4fe1\uff0c\u4f7f\u7528 NVLink \u548c\u4e00\u4e2a\u4e2d\u95f4 GPU\u3002\u5efa\u8bae\u8bbe\u7f6e\u62101\u3002\u5728PyTorch\u4e2d\u8fdb\u884c\u8de8\u8282\u70b9all-to-all\u901a\u4fe1\u65f6\uff0c\u5982\u679c\u8be5\u73af\u5883\u53d8\u91cf\u662f0\u4f1a\u51fa\u73b0\u5f02\u5e38\u3002<\/li><li>NCCL_SOCKET_IFNAME:\u9009\u62e9\u7f51\u7edc\u63a5\u53e3\u3002<\/li><li>NCCL_SOCKET_NTHREADS \u589e\u52a0\u5b83\u7684\u6570\u91cf\u53ef\u4ee5\u63d0\u9ad8socker\u4f20\u8f93\u7684\u6548\u7387\uff0c\u4f46\u662f\u4f1a\u589e\u52a0CPU\u7684\u8d1f\u62c5<\/li><li>NCCL_NET_GDR_LEVEL:\u8bbe\u7f6eGPUDirect RDMA\u7684\u4f7f\u7528\u7ea7\u522b\u3002<\/li><li>NCCL_MAX_NRINGS:\u5b9a\u4e49\u652f\u6301\u7684\u6700\u5927NCCL\u73af\u8def\u6570\u3002<\/li><li>NCCL_MIN_NRINGS:\u5b9a\u4e49\u6700\u5c0f\u73af\u8def\u6570\u3002<\/li><li>NCCL_BUFFSIZE:\u8bbe\u7f6escratch\u7a7a\u95f4\u5927\u5c0f\u3002<\/li><li>NCCL_BUFFLE_SIZE \u7f13\u5b58\u6570\u636e\u91cf\uff0c\u7f13\u5b58\u8d8a\u5927\u4e00\u6b21ring\u4f20\u8f93\u7684\u6570\u636e\u5c31\u8d8a\u5927\u81ea\u7136\u5bf9\u5e26\u5bbd\u7684\u538b\u529b\u6700\u5927\uff0c\u4f46\u662f\u76f8\u5e94\u7684\u603b\u5ef6\u8fdf\u6b21\u6570\u4f1a\u5c11\u3002\u9ed8\u8ba4\u503c\u662f4M\uff084194304\uff09\uff0c\u6ce8\u610f\u8bbe\u7f6e\u7684\u65f6\u5019\u4f7f\u7528bytes\uff08\u5b57\u8282\u5927\u5c0f\uff09<\/li><li>NCCL_NTHREADS:\u8bbe\u7f6eNCCL\u5185\u90e8\u4f7f\u7528\u7684\u7ebf\u7a0b\u6570\u3002<\/li><li>NCCL_VERSION:\u663e\u793aNCCL\u7248\u672c\u4fe1\u606f\u3002<\/li><li>NCCL_MAX\/MIN_NCHANNELS \u6700\u5c0f\u548c\u6700\u5927\u7684rings\uff0crings\u8d8a\u591a\u5bf9GPU\u7684\u663e\u5b58\u3001\u5e26\u5bbd\u7684\u538b\u529b\u90fd\u8d8a\u5927\uff0c\u4e5f\u4f1a\u5f71\u54cd\u8ba1\u7b97\u6027\u80fd<\/li><li>NCCL_CHECKS_DISABLE \u5728\u6bcf\u6b21\u96c6\u5408\u901a\u4fe1\u8fdb\u884c\u524d\u5bf9\u53c2\u6570\u68c0\u9a8c\u6821\u5bf9\uff0c\u8fd9\u4f1a\u589e\u52a0\u5ef6\u8fdf\u65f6\u95f4\uff0c\u5728\u751f\u4ea7\u73af\u5883\u4e2d\u53ef\u4ee5\u8bbe\u4e3a1.\u9ed8\u8ba4\u662f0<\/li><li>NCCL_CHECK_POINTERS \u5728\u6bcf\u6b21\u96c6\u5408\u901a\u4fe1\u8fdb\u884c\u524d\u5bf9CUDA\u5185\u5b58 \u6307\u9488\u8fdb\u884c\u6821\u9a8c\uff0c\u8fd9\u4f1a\u589e\u52a0\u5ef6\u8fdf\u65f6\u95f4\uff0c\u5728\u751f\u4ea7\u73af\u5883\u4e2d\u53ef\u4ee5\u8bbe\u4e3a1.\u9ed8\u8ba4\u662f0<\/li><li>NCCL_NET_GDR_LEVEL GDR\u89e6\u53d1\u7684\u6761\u4ef6\uff0c\u9ed8\u8ba4\u662f\u5f53GPU\u548cNIC\u6302\u8f7d\u4e00\u4e2aswith\u4e0a\u9762\u65f6\u4f7f\u7528GDR<\/li><li>NCCL_IGNORE_CPU_AFFINITY \u5ffd\u7565CPU\u4e0e\u5e94\u7528\u7684\u4eb2\u548c\u6027\u4f7f\u7528GPU\u4e0enic\u7684\u4eb2\u548c\u6027\u4e3a\u4e3b<\/li><li><strong>NCCL_IB_DISABLE:\u7981\u7528InfiniBand\u4f20\u8f93\u3002<\/strong><\/li><\/ol>\n\n\n\n<p><strong>\u7981\u7528 InfiniBand<\/strong>: \u8bbe\u7f6e <code>NCCL_IB_DISABLE=1<\/code> \u4f1a\u7981\u7528 NCCL \u5728 InfiniBand \u8bbe\u5907\u4e0a\u7684\u4f7f\u7528\u3002\u8fd9\u610f\u5473\u7740 NCCL \u5c06\u4e0d\u4f1a\u5229\u7528 InfiniBand \u7f51\u7edc\u8fdb\u884c\u6570\u636e\u4f20\u8f93\uff0c\u800c\u662f\u56de\u9000\u5230\u5176\u4ed6\u7f51\u7edc\u63a5\u53e3\uff08\u4f8b\u5982\u4ee5\u592a\u7f51\u6216\u5176\u4ed6\u7f51\u7edc\u63a5\u53e3\uff09\u3002<\/p>\n\n\n\n<p><strong>\u8c03\u8bd5\u548c\u517c\u5bb9\u6027<\/strong>: \u7981\u7528 InfiniBand \u53ef\u80fd\u7528\u4e8e\u8c03\u8bd5\u76ee\u7684\uff0c\u6216\u5728\u7cfb\u7edf\u4e2d InfiniBand \u7f51\u7edc\u51fa\u73b0\u95ee\u9898\u65f6\u56de\u9000\u5230\u5176\u4ed6\u7f51\u7edc\u63a5\u53e3\u3002\u5982\u679c\u4f60\u9047\u5230\u4e0e InfiniBand \u76f8\u5173\u7684\u9519\u8bef\u6216\u517c\u5bb9\u6027\u95ee\u9898\uff0c\u7981\u7528 InfiniBand \u53ef\u80fd\u6709\u52a9\u4e8e\u89e3\u51b3\u8fd9\u4e9b\u95ee\u9898\u3002<\/p>\n\n\n\n<ol><li>NCCL_IB_HCA \u4ee3\u8868IB\u4f7f\u7528\u7684\u8bbe\u5907\uff1aMellanox mlx5\u7cfb\u5217\u7684HCA\u8bbe\u5907NCCL_IB_HCA=mlx5 \u4f1a\u9ed8\u8ba4\u8f6e\u8be2\u6240\u6709\u7684\u8bbe\u5907\u3002NCCL_IB_HCA=mlx5_0:1 \u6307\u5b9a\u5176\u4e2d\u4e00\u53f0\u8bbe\u5907\u3002<\/li><li><strong>NCCL_IB_TIMEOUT&nbsp;\u6539\u53d8\u91cf\u7528\u4e8e\u63a7\u5236InfiniBand Verbs\u8d85\u65f6\u3002\u53d6\u503c\u8303\u56f41-22\u3002\u8d85\u65f6\u65f6\u95f4\u7684\u8ba1\u7b97\u516c\u5f0f\u4e3a4.096\u5fae\u79d2 * 2 ^ timeout\uff0c\u6b63\u786e\u7684\u503c\u53d6\u51b3\u4e8e\u7f51\u7edc\u7684\u5927\u5c0f\u3002\u589e\u52a0\u8be5\u503c\u53ef\u4ee5\u5728\u975e\u5e38\u5927\u7684\u7f51\u7edc\u4e0a\u63d0\u4f9b\u5e2e\u52a9\uff0c\u4f8b\u5982 NCCL\u5728\u8c03\u7528ibv_poll_cq\u65f6\u51fa\u73b0\u9519\u8bef12\u65f6\u3002\u5efa\u8bae\u5728\u5927\u6a21\u578b\u8bad\u7ec3\u4efb\u52a1\u4e2d\u8bbe\u7f6e\u6210\u6700\u5927\u503c22\uff0c\u53ef\u4ee5\u51cf\u5c11\u4e0d\u5c11nccl timeout\u5f02\u5e38\u3002\u8bbe\u7f6e\u8d85\u65f6\u65f6\u95f4<\/strong>: <code>NCCL_IB_TIMEOUT<\/code> \u7528\u4e8e\u63a7\u5236 InfiniBand \u7f51\u7edc\u64cd\u4f5c\u7684\u8d85\u65f6\u65f6\u95f4\u3002\u901a\u8fc7\u8c03\u6574\u8fd9\u4e2a\u503c\uff0c\u4f60\u53ef\u4ee5\u63a7\u5236 NCCL \u5728\u9047\u5230\u901a\u4fe1\u5ef6\u8fdf\u6216\u7f51\u7edc\u95ee\u9898\u65f6\u7684\u5bb9\u5fcd\u5ea6\u3002<strong>\u89e3\u51b3\u7f51\u7edc\u95ee\u9898<\/strong>: \u5728\u9ad8\u6027\u80fd\u8ba1\u7b97\u548c\u5927\u89c4\u6a21\u5206\u5e03\u5f0f\u8bad\u7ec3\u4e2d\uff0c\u7f51\u7edc\u5ef6\u8fdf\u6216\u4e0d\u7a33\u5b9a\u53ef\u80fd\u5bfc\u81f4\u8d85\u65f6\u9519\u8bef\u3002\u8c03\u6574 <code>NCCL_IB_TIMEOUT<\/code> \u53ef\u4ee5\u5e2e\u52a9\u4f60\u5728\u9047\u5230\u7f51\u7edc\u95ee\u9898\u65f6\u66f4\u597d\u5730\u8c03\u8282\u8d85\u65f6\u8bbe\u7f6e\uff0c\u907f\u514d\u8bad\u7ec3\u8fc7\u7a0b\u88ab\u4e2d\u65ad\u3002<\/li><\/ol>\n\n\n\n<ol><li>NCCL_IB_RETRY_CNT\u53d8\u91cf\u63a7\u5236 InfiniBand \u7684\u91cd\u8bd5\u6b21\u6570\u3002\u5efa\u8bae\u5728\u5927\u6a21\u578b\u8bad\u7ec3\u4efb\u52a1\u4e2d\u8bbe\u7f6e\u621013\uff0c\u5c3d\u53ef\u80fd\u591a\u91cd\u8bd5\u3002<\/li><li>NCCL_DEBUG_FILE\u8bbe\u7f6e\u4e00\u4e2a\u6587\u4ef6\u5730\u5740\uff0c\u53d8\u91cf\u7528\u4e8e\u5c06NCCL\u7684\u8c03\u8bd5\u65e5\u5fd7\u8f93\u51fa\u5230\u6587\u4ef6\u4e2d\u3002\u6709\u52a9\u4e8e\u8c03\u8bd5nccl\u3002<\/li><li>NCCL_IB_PCI_RELAXED_ORDERING\u542f\u7528 IB Verbs \u4f20\u8f93\u7684Relaxed Ordering\u3002Relaxed Ordering\u53ef\u4ee5\u6781\u5927\u5730\u63d0\u9ad8\u865a\u62df\u5316\u73af\u5883\u4e0b InfiniBand \u7f51\u7edc\u7684\u6027\u80fd\u3002\u8bbe\u7f6e\u4e3a 2\uff0c\u5982\u679c\u53ef\u7528\uff0c\u81ea\u52a8\u4f7f\u7528Relaxed Ordering\u3002\u8bbe\u7f6e\u4e3a 1\uff0c\u5f3a\u5236\u4f7f\u7528Relaxed Ordering\uff0c\u5982\u679c\u4e0d\u53ef\u7528\u5219\u5931\u8d25\u3002\u8bbe\u7f6e\u4e3a 0\uff0c\u7981\u7528\u4f7f\u7528Relaxed Ordering\u3002\u9ed8\u8ba4\u503c\u4e3a 2\u3002\u5efa\u8bae\u503c\u4e3a1<\/li><\/ol>\n","protected":false},"excerpt":{"rendered":"<p>\u672c\u6587\u4e3b\u8981\u8bb0\u5f55\u548c\u5b66\u4e60pytorch\u540e\u7aefNCCL\u76f8\u5173\u7684\u77e5\u8bc6\u70b9\uff0c\u4e3a\u540e\u7eed\u5927\u6a21\u578b\u8bad\u7ec3\u6253\u597d\u57fa\u7840 https:\/\/docs. &hellip; <a href=\"http:\/\/139.9.1.231\/index.php\/2024\/08\/08\/nccl\/\" class=\"more-link\">\u7ee7\u7eed\u9605\u8bfb<span class=\"screen-reader-text\">NCCL&#8211;\u591a\u5361\u8bad\u7ec3\u540e\u7aef[\u6301\u7eed\u8865\u5145]<\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[11,39],"tags":[],"_links":{"self":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/16376"}],"collection":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/comments?post=16376"}],"version-history":[{"count":16,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/16376\/revisions"}],"predecessor-version":[{"id":19057,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/16376\/revisions\/19057"}],"wp:attachment":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/media?parent=16376"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/categories?post=16376"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/tags?post=16376"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}