{"id":28,"date":"2025-08-16T13:33:00","date_gmt":"2025-08-16T05:33:00","guid":{"rendered":"https:\/\/dioho.me\/?p=28"},"modified":"2025-09-29T22:48:40","modified_gmt":"2025-09-29T14:48:40","slug":"%e7%ba%afcpu%e6%8e%a8%e7%90%86gpt-oss-120b%e6%80%a7%e8%83%bd%e6%b5%8b%e8%af%95","status":"publish","type":"post","link":"https:\/\/dioho.me\/index.php\/2025\/08\/16\/%e7%ba%afcpu%e6%8e%a8%e7%90%86gpt-oss-120b%e6%80%a7%e8%83%bd%e6%b5%8b%e8%af%95\/","title":{"rendered":"\u7eafCPU\u63a8\u7406GPT-OSS-120b\u6027\u80fd\u6d4b\u8bd5"},"content":{"rendered":"\n
#\u8f6f\u786c\u4ef6\u73af\u5883\nUbuntu 24.04\n\u53cc\u8def8259CL\nDDR4 2400 LRDIMM 768G\nNUMA\u8282\u70b9\uff1a\u5173\u95ed <\/code><\/pre>\n\n\n\n
GPT-OSS-120b\u662fOpenAI \u7684\u5f00\u6e90\u6743\u91cd\uff08open-weight\uff09MoE \u63a8\u7406\u6a21\u578b\uff0c36 \u5c42\u3001\u603b\u53c2\u7ea6 116.8B\uff0c\u6bcf token \u6fc0\u6d3b\u53c2\u6570\u7ea6 5.13B<\/strong>\uff0cMoE \u6743\u91cd\u7528 MXFP4\uff084.25 bit\uff09<\/strong> \u91cf\u5316\uff0c\u6574\u6a21\u6743\u91cd\uff08checkpoint\uff09\u7ea6 60.8 GiB<\/strong>\uff1bMoE \u91c7\u7528 128 \u4e13\u5bb6\u3001top-4 \u8def\u7531<\/strong>\uff0c\u6ce8\u610f\u529b\u4e3a 64 Q heads \/ 8 KV heads\uff08GQA\uff09\uff0chead_dim=64<\/strong>\uff0c\u6700\u957f\u4e0a\u4e0b\u6587 128k<\/strong>\u3002<\/p>\n\n\n\n
\u5355token\u7406\u8bba\u901f\u5ea6\u8ba1\u7b97\uff1a<\/p>\n\n\n\n
\u6743\u91cd\u8bfb\u6d41\u91cf\uff08\u4e0e\u4e0a\u4e0b\u6587 L \u65e0\u5173\uff09\uff1a<\/p>\n\n\n\n
\u6fc0\u6d3b\u53c2\u6570\u7ea6 5.13B \/ token<\/strong>\uff0c\u53ef\u7531\u4e0b\u5f0f\u8fd1\u4f3c\u5206\u89e3\uff1a<\/p>\n\n\n\n
MoE-MLP\uff1a\u603b 114.71B\uff0ctop-4\/128 \u2192 \u6fc0\u6d3b 114.71B\u00d7(4\/128)=3.5847B\uff1bMXFP4=0.53125 B\/param \u2192 \u22481.904 GB\/token<\/p>\n\n\n\n
Attention \u53c2\u6570\uff1a0.96B \u00d7 2 B \u2248 1.920 GB\/token\uff08\u6309 bf16\/fp16 \u4f30\uff09<\/p>\n\n\n\n
Unembed\uff1a(1.16B\/2) \u00d7 2 B \u2248 1.160 GB\/token<\/p>\n\n\n\n
\u5408\u8ba1\u6743\u91cd\u8bfb \u2248 4.98 GB\/token\uff08\u22484.64 GiB\/token\uff09\u3002<\/p>\n\n\n\n
\u4f46\u662fCascade Lake\u4e0d\u652f\u6301FP16\/BF16\uff0c\u53ea\u80fd\u4ee5FP32\u5b58\uff0c\u56e0\u6b64 \u2248 8.064 GB\/token<\/strong><\/p>\n\n\n\n
KV Cache \u8bfb\/\u5199\uff08\u4e0e\u4e0a\u4e0b\u6587 L \u7ebf\u6027\u76f8\u5173\uff09\uff1a<\/p>\n\n\n\n
\u6bcf\u5c42\u6bcf\u4e2a token \u7684 K\u3001V \u5411\u91cf\uff1a2 * H_kv * d = 2*8*64 = 1024<\/code> \u5143\u7d20\uff0c\u6309 2 B\/elem \u2192 2048 B\/\u5c42\u00b7token\u3002<\/p>\n\n\n\n
\u5bf9\u4e8e\u5168\u5c40\u5c42\uff0818 \u5c42\uff09\uff1a\u8bfb 2048 * L<\/code>\/\u5c42\uff1b\u7a97\u53e3\u5c42\uff0818 \u5c42, \u7a97\u53e3=128\uff09\uff1a\u8bfb 2048 * 128<\/code>\/\u5c42\uff1b<\/p>\n\n\n\n
\u5199\u5165\uff08\u6240\u6709 36 \u5c42\uff09\uff1a\u6bcf\u65b0 token \u5199\u5165\u5f53\u524d\u7684 K\u3001V\uff1a36 * 2048 = 73,728 B \u2248 72 KiB\/token<\/code>\uff08\u76f8\u5bf9\u8bfb\u6d41\u91cf\u53ef\u5ffd\u7565\uff09\u3002<\/p>\n\n\n\n
\u4e0a\u4e0b\u6587\u548c\u5e26\u5bbd\u7684\u5173\u7cfb<\/p>\n\n\n\n
\u4e0a\u4e0b\u6587 L<\/td>FP16 
\u6743\u91cd\u8bfb (GB)<\/td>		FP16 KV (GB)<\/td>FP16\u5408\u8ba1 (GB\/token)<\/td>FP32 KV (GB)<\/td> FP32
 \u6743\u91cd\u8bfb (GB)<\/td>		FP32\u5408\u8ba1 (GB\/token)<\/td><\/tr>
8k<\/td>4.984<\/td>0.307<\/td>5.291<\/td>0.6136<\/td>8.064<\/td>8.678<\/td><\/tr>
32k<\/td>4.984<\/td>1.213<\/td>6.197<\/td>2.4255<\/td>8.064<\/td>10.490<\/td><\/tr>
128k<\/td>4.984<\/td>4.837<\/td>9.821<\/td>9.6733<\/td>8.064<\/td>17.737<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n
1\u3001\u73af\u5883\u51c6\u5907\uff1a<\/p>\n\n\n\n
sudo apt update\nsudo apt install -y git build-essential cmake ninja-build ccache\nsudo apt install -y libcurl4-openssl-dev\ngit clone https:\/\/github.com\/ggml-org\/llama.cpp.git\ncd llama.cpp<\/code><\/pre>\n\n\n\n
2\u3001\u7f16\u8bd1llama.cpp<\/p>\n\n\n\n
cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \\\n  -DGGML_NATIVE=OFF \\\n  -DGGML_AVX=ON -DGGML_AVX2=ON \\\n  -DGGML_F16C=ON -DGGML_FMA=ON \\\n  -DGGML_AVX512=ON \\\n  -DGGML_AVX512_VNNI=ON \\\n  -DGGML_AVX512_VBMI=OFF \\\n  -DGGML_AVX512_BF16=OFF\ncmake --build build -j\n#\u4e8c\u4ee3Cascade Lake \u589e\u4e86 AVX512_VNNI\uff1bBF16\u662fCooper Lake\u624d\u6709\uff0cVBMI\u4eceIce Lake\u624d\u6709\u3002\n#Cooper Lake\u548cIce Lake\u90fd\u662f\u4e09\u4ee34189\u81f3\u5f3a\uff0c\u533a\u522b\u5c31\u662fCooper Lake\u4ecd\u7136\u662f14nm\uff0cIce Lake\u624d\u662f\u771f10nm\u4e09\u4ee3xeon\uff0c\u4ef7\u683c\u4e0d\u5982\u76f4\u63a5\u4e0aEmerald Rapids\u4e94\u4ee3xeon<\/code><\/pre>\n\n\n\n
3\u3001\u8fd0\u884c\u63a8\u7406\u811a\u672c<\/p>\n\n\n\n
#\u8bbe\u7f6e\u6a21\u578b\u76ee\u5f55\nMODEL=\"$HOME\/\u684c\u9762\/gpt-oss-120b-GGUF\/gpt-oss-120b-MXFP4-00001-of-00002.gguf\"\n#\u5728llama.cpp\u7f16\u8bd1\u76ee\u5f55\u8fd0\u884c\n.\/build\/bin\/llama-cli -m \"$MODEL\" \\\n  -i \\\n  -c 4096 \\\n  -b 256 \\\n  -t $(nproc) \\\n  --simple-io\n#\u7ebf\u7a0b\u6570 -t\uff1a\u5148\u7528 $(nproc) \u8dd1\u4e00\u8f6e\uff1b\u53cc\u8def 8259CL \u5f80\u5f80 96 \u7ebf\u7a0b\uff0c\u82e5\u53d1\u73b0\u541e\u5410\u4e0a\u4e0d\u53bb\uff08\u8c03\u5ea6\u5f00\u9500\uff09\uff0c\u518d\u8bd5\u7269\u7406\u6838\u6570\uff08\u6bd4\u5982 48\uff09\u3002\n#\u4e0a\u4e0b\u6587 -c \u4e0e \u6279\u5927\u5c0f -b\uff1a120B \u5185\u5b58\u3001\u8ba1\u7b97\u90fd\u5f88\u91cd\u3002-c 4096\/-b 256 \u662f\u7a33\u59a5\u8d77\u6b65\u503c\uff1bprefill \u60f3\u66f4\u5feb\u53ef\u9010\u6b65\u628a -b \u8c03\u5927\uff08512\u3001768\uff09\uff0c\u4f46\u5185\u5b58\u4f1a\u6da8\u3002<\/code><\/pre>\n\n\n\n
\"\"<\/div><\/figure>\n\n\n\n
96\u7ebf\u7a0b\u5168\u90e8\u8dd1\u6ee1\uff0c\u63a8\u7406\u901f\u5ea61\u79d23\u5b57\u5de6\u53f3\uff0c\u5c5e\u4e8e\u52c9\u5f3a\u53ef\u4ee5\u4f7f\u7528\u7ea7\u522b\uff0c120b\u4e0d\u6dfb\u52a0\u8054\u7f51\u672c\u8eab\u4e0d\u662f\u806a\u660e\uff0c\u4f46\u662f\u6bd4deepseek 671b\u52c9\u5f3a2.3token\/s\u5f3a\u592a\u591a\u4e86<\/p>\n\n\n\n
\n
\"\"<\/div><\/figure>\n<\/figure>\n\n\n\n
\n
\"\"<\/div><\/figure>\n<\/figure>\n\n\n\n
4\u3001\u6027\u80fd\u8c03\u4f18<\/p>\n\n\n\n
\u90a3\u4e48\u7ebf\u7a0b\u6570\u8d8a\u591a\u8d8a\u597d\u5417\uff1f\u5c3d\u7ba196\u7ebf\u7a0b\u6ee1\u8f7d\u770b\u4e0a\u53bb\u5f88\u5389\u5bb3\uff0c\u4f46\u662f\u6838\u5fc3\u8ba1\u7b97\u6027\u80fd\u5176\u5b9e\u4e0d\u662f\u975e\u5e38\u91cd\u8981\uff0c\u5185\u5b58\u5e26\u5bbd\u662f\u7b2c\u4e00\u8981\u7d20\u3002\u7eafcpu\u65b9\u6848\u5185\u5b58\u5e26\u5bbd\u662f\u6781\u5927\u74f6\u9888\uff0c\u6211\u4eec\u7c97\u7565\u8ba1\u7b97\u4e00\u4e0b\u5355\u8def\u5e26\u5bbd\uff0c\u56e0\u4e3aCascade Lake\u4e3a6\u901a\u9053DDR4\uff0c\u56e0\u6b6464x6x2400\/8\/1024=112.5GB\/s\uff0c\u53cc\u8def\u8fdc\u8fdc\u8fbe\u4e0d\u5230x2\u6c34\u5e73\uff0c\u8981\u53d7\u5230UPI\u603b\u7ebf\u9650\u5236\uff0c\u6b64\u901f\u5ea6\u5582\u6ee1\u53cc\u8def\u662f\u5b8c\u5168\u4e0d\u591f\u7684\uff0c\u4ece\u5b9e\u9645\u63a8\u7406\u901f\u5ea6\u7ed3\u679c\u6765\u770b\u5c31\u53ef\u89c1\u4e00\u6591\u3002\u53e6\u5916avx512\u529f\u8017\u8fc7\u5927\uff0c\u964d\u9891\u6781\u4e3a\u660e\u663e\uff0c\u56e0\u6b64\u6211\u4eec\u91cd\u65b0\u7f16\u8bd1\u4e86AVX2\u7248\u672c\u3002\u4ece96\u7ebf\u7a0b\u76845.32t\/s\u7684\u7ed3\u679c\u6765\u770b\uff0c\u8fdc\u8fdc\u6ca1\u6709\u8fbe\u5230\u7eafcpu\u63a8\u7406\u7684\u4e0a\u9650\uff0c\u56e0\u6b64\u6211\u4eec\u5c06\u7ebf\u7a0b\u9650\u5236\u4e3a15\uff0c\u5173\u95ed\u8d85\u7ebf\u7a0b\uff0c\u4e0a\u4e0b\u6587\u589e\u52a0\u52308192\uff088k\uff09\uff0c\u7ed1\u5b9acpu0\uff0c\u5f00\u542fnuma\uff0c\u8fd0\u884c\u4ee5\u4e0b\u63a8\u7406\u547d\u4ee4\uff1a<\/p>\n\n\n\n
MODEL=\"$HOME\/\u684c\u9762\/gpt-oss-120b-GGUF\/gpt-oss-120b-MXFP4-00001-of-00002.gguf\"\nCPUSET_PHYS=$(lscpu -e=CPU,CORE,SOCKET | awk '$3==0{if(!( $2 in seen)){printf (n++?\",\":\"\") $1; seen[$2]=1}}')\ntaskset -c \"$CPUSET_PHYS\" .\/build-avx2\/bin\/llama-cli   -m \"$MODEL\" -i -c 8192 -b 256 -t 15 --simple-io<\/code><\/pre>\n\n\n\n
\"\"<\/div><\/figure>\n\n\n\n
\u6211\u4eec\u53ef\u4ee5\u770b\u5230\uff0c\u6b64\u65f6\u63a8\u7406\u901f\u5ea6\u9ad8\u8fbe12.5t\/s\uff0c\u540a\u625396\u7ebf\u7a0b\u76845t\/s\uff0c\u63a5\u8fd1\u7f51\u9875\u7248deepseek\u6c34\u5e73\u3002<\/p>\n\n\n\n
<\/p>\n","protected":false},"excerpt":{"rendered":"
GPT-OSS-120b\u662fOpenAI \u7684\u5f00\u6e90\u6743\u91cd\uff08open-weight\uff09MoE \u63a8\u7406\u6a21\u578b\uff0c36 \u5c42\u3001\u603b\u53c2\u7ea6 […]<\/p>\n","protected":false},"author":5,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[6],"tags":[],"class_list":["post-28","post","type-post","status-publish","format-standard","hentry","category-6"],"_links":{"self":[{"href":"https:\/\/dioho.me\/index.php\/wp-json\/wp\/v2\/posts\/28","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/dioho.me\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/dioho.me\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/dioho.me\/index.php\/wp-json\/wp\/v2\/users\/5"}],"replies":[{"embeddable":true,"href":"https:\/\/dioho.me\/index.php\/wp-json\/wp\/v2\/comments?post=28"}],"version-history":[{"count":19,"href":"https:\/\/dioho.me\/index.php\/wp-json\/wp\/v2\/posts\/28\/revisions"}],"predecessor-version":[{"id":69,"href":"https:\/\/dioho.me\/index.php\/wp-json\/wp\/v2\/posts\/28\/revisions\/69"}],"wp:attachment":[{"href":"https:\/\/dioho.me\/index.php\/wp-json\/wp\/v2\/media?parent=28"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/dioho.me\/index.php\/wp-json\/wp\/v2\/categories?post=28"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/dioho.me\/index.php\/wp-json\/wp\/v2\/tags?post=28"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}