{"id":4969,"date":"2023-08-15T09:57:42","date_gmt":"2023-08-15T00:57:42","guid":{"rendered":"http:\/\/www.sougetu.net\/?p=4969"},"modified":"2023-08-15T09:57:45","modified_gmt":"2023-08-15T00:57:45","slug":"rinna4b%e3%82%92ctranslate2%e3%81%a7%e8%bb%bd%e3%81%8f%e3%81%97%e3%81%a6windows%e3%81%a7%e5%8b%95%e3%81%8b%e3%81%97%e3%81%9f%e3%80%82","status":"publish","type":"post","link":"https:\/\/www.sougetu.net\/?p=4969","title":{"rendered":"Rinna4B\u3092Ctranslate2\u3067\u8efd\u304f\u3057\u3066Windows\u3067\u52d5\u304b\u3057\u305f\u3002"},"content":{"rendered":"\n<p>\u4e00\u6628\u65e5\u3001\u6628\u65e5\u3068\u9811\u5f35\u3063\u3066\u305f\u300cJapanese StableLM Alpha\u300d\u306a\u306e\u3060\u3051\u3069\u3001\u3068\u308a\u3042\u3048\u305a\u4ed6\u306e\u304b\u3089\u52d5\u304b\u3057\u3066\u307f\u3066\u3001\u5b9f\u7e3e\u3092\u4f5c\u308b\u4e8b\u306b\u3057\u305f\u3002\u305d\u306e\u3046\u3061\u3001\u5206\u304b\u3089\u306a\u304b\u3063\u305f\u3068\u3053\u308d\u3082\u6cbb\u305b\u308b\u3088\u3046\u306b\u306a\u308b\u3060\u308d\u3046\u3002<\/p>\n<p>\u305d\u3093\u306a\u308f\u3051\u3067\u4eca\u671d\u306f\u300crinna\/bilingual-gpt-neox-4b-instruction-ppo\u300d\u3092Ctranslate2\u3067\u91cf\u5b50\u5316\u3057\u3066\u52d5\u304b\u3059\u4e8b\u306b\u30c1\u30e3\u30ec\u30f3\u30b8\u3002<\/p>\n<ul>\n<li><a href=\"https:\/\/zenn.dev\/kazuph\/articles\/84f5debf015e76\" target=\"_blank\" rel=\"noopener\"><span>Rinna\u306eppo\u30e2\u30c7\u30eb\u3092CTranslate2\u3067\u9ad8\u901f\u306b\u52d5\u304b\u3059<\/span><\/a><\/li>\n<li><a href=\"https:\/\/note.com\/if001\/n\/n07faad488175\" target=\"_blank\" rel=\"noopener\">CTranslate2\u3067rinna instruction\u3092quantize\u3057\u3066\u52d5\u304b\u3059<\/a><\/li>\n<\/ul>\n<p>\u4e3b\u306b\u3053\u3061\u3089\u3092\u53c2\u7167\u306b\u3002\u4ed6\u306b\u3082\u8272\u3005\u898b\u3066\u305f\u306e\u3060\u3051\u3069\u8a33\u304c\u5206\u304b\u3089\u306a\u304f\u306a\u3063\u3066\u3057\u307e\u3063\u305f\u3002<\/p>\n<h2>\u524d\u63d0\u6761\u4ef6<\/h2>\n<ul>\n<li>windows11<\/li>\n<li>Python3.10.10<\/li>\n<li>\u4eee\u60f3\u74b0\u5883\u306fpyenv + venv\u3067\u4f5c\u6210<\/li>\n<\/ul>\n<p><\/p>\n<p><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code><strong># \u74b0\u5883\u4f5c\u6210<\/strong>\r\npyenv local 3.10.10\r\npython -m venv .venv\r\n.venv\/Scripts\/activate\r\npython -V\r\npip install --upgrade pip\r\npython -m pip install --upgrade pip\n<\/code><\/pre>\n\n\n\n<pre class=\"wp-block-code\"><code><strong>#torch\u306e\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb<\/strong>\r\npip install torch torchvision torchaudio --index-url https:\/\/download.pytorch.org\/whl\/cu118\r\n\r\n<strong># ctranslate2\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb<\/strong>\r\npip install ctranslate2\r\npip install sentencepiece transformers\r\npip install protobuf\r\n<\/code><\/pre>\n\n\n\n<pre class=\"wp-block-code\"><code># \u5909\u63db\r\nct2-transformers-converter --model rinna\/japanese-gpt-neox-3.6b-instruction-ppo --quantization bfloat16 --force --output_dir rinna-ppo-bf16\r\n<\/code><\/pre>\n\n\n\n<p>\u5909\u63db\u306f\u3001bfloat16\u306b\u3057\u3066\u307f\u305f\u3002\u3053\u306e\u72b6\u614b\u3067\u52d5\u304b\u3057\u3066\u3082\uff16\uff27\u7a0b\u5ea6\u3002\u5909\u63db\u3067\u4f7f\u7528\u3057\u305f\u30e1\u30a4\u30f3\u30e1\u30e2\u30ea\u306f\uff12\uff10G\u7a0b\u5ea6\u3060\u3063\u305f\u304b\u306a\uff1f\uff08\u3046\u308d\u899a\u3048\uff09\u5fdc\u7b54\u3082\u5341\u5206\u65e9\u304f\u3001\u9045\u5ef6\u306f\u6c17\u306b\u306a\u3089\u306a\u304b\u3063\u305f\u3002<\/p>\n\n\n\n<p>\u6700\u7d42\u7684\u306b\u52d5\u304b\u3057\u305f\u30b3\u30fc\u30c9\u306f\u4ee5\u4e0b\u306e\u901a\u308a\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>import ctranslate2\r\nimport transformers\r\nimport torch\r\nimport os\r\n\r\nos.environ&#91;\"CUDA_VISIBLE_DEVICES\"] = \"0\" #GPU\u3092\u4f7f\u7528\u3059\u308b\u5834\u5408\u306f\u30b3\u30e1\u30f3\u30c8\u30a2\u30a6\u30c8\u3092\u5916\u3059\r\nmodel='rinna-ppo-int8' #directory of the model\r\n\r\nppo = \"rinna\/japanese-gpt-neox-3.6b-instruction-ppo\"\r\ngenerator = ctranslate2.Generator(model, device=\"auto\")\r\ntokenizer = transformers.AutoTokenizer.from_pretrained(ppo, use_fast=False)\r\n\r\n\r\n# \u30d7\u30ed\u30f3\u30d7\u30c8\u3092\u4f5c\u6210\u3059\u308b\r\ndef prompt(msg):\r\n    p = &#91;\r\n        {\"speaker\": \"\u30e6\u30fc\u30b6\u30fc\", \"text\": msg},\r\n    ]\r\n    p = &#91;f\"{uttr&#91;'speaker']}: {uttr&#91;'text']}\" for uttr in p]\r\n    p = \"&lt;NL>\".join(p)\r\n    p = p + \"&lt;NL>\" + \"\u30b7\u30b9\u30c6\u30e0: \"\r\n    # print(p)\r\n    return p\r\n\r\n# \u8fd4\u4fe1\u3092\u4f5c\u6210\u3059\u308b\r\ndef reply(msg):\r\n    p = prompt(msg)\r\n    tokens = tokenizer.convert_ids_to_tokens(\r\n        tokenizer.encode(\r\n            p,\r\n            add_special_tokens=False,\r\n        )\r\n    )\r\n\r\n    results = generator.generate_batch(\r\n        &#91;tokens],\r\n        max_length=256,\r\n        sampling_topk=10,\r\n        sampling_temperature=0.9,\r\n        include_prompt_in_result=False,\r\n    )\r\n\r\n    text = tokenizer.decode(results&#91;0].sequences_ids&#91;0])\r\n    print(\"\u30b7\u30b9\u30c6\u30e0(ppo-ct2): \" + text + \"\\n\")\r\n    return text\r\n\r\n\r\nif __name__ == \"__main__\":\r\n    # import readline\r\n    while True:\r\n        msg = input(\"\u30e6\u30fc\u30b6\u30fc: \")\r\n        reply(msg)\r\n<\/code><\/pre>\n\n\n\n<p><\/p>\n\n\n\n<figure class=\"wp-block-image size-full is-resized\"><a href=\"https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/a4712f733e33fc076034cd256a68ecbf.png\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/a4712f733e33fc076034cd256a68ecbf.png\" alt=\"\" class=\"wp-image-4970\" width=\"840\" height=\"614\" srcset=\"https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/a4712f733e33fc076034cd256a68ecbf.png 1022w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/a4712f733e33fc076034cd256a68ecbf-300x220.png 300w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/a4712f733e33fc076034cd256a68ecbf-768x562.png 768w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/a4712f733e33fc076034cd256a68ecbf-150x110.png 150w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/a4712f733e33fc076034cd256a68ecbf-205x150.png 205w\" sizes=\"auto, (max-width: 840px) 100vw, 840px\" \/><\/a><\/figure>\n\n\n\n\n\n<figure class=\"wp-block-image size-large\"><a href=\"https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/2f7dd2cf07481a22a5df81ac33181f25.png\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"414\" src=\"https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/2f7dd2cf07481a22a5df81ac33181f25-1024x414.png\" alt=\"\" class=\"wp-image-4971\" srcset=\"https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/2f7dd2cf07481a22a5df81ac33181f25-1024x414.png 1024w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/2f7dd2cf07481a22a5df81ac33181f25-300x121.png 300w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/2f7dd2cf07481a22a5df81ac33181f25-768x310.png 768w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/2f7dd2cf07481a22a5df81ac33181f25-150x61.png 150w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/2f7dd2cf07481a22a5df81ac33181f25-250x101.png 250w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/2f7dd2cf07481a22a5df81ac33181f25.png 1445w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/a><\/figure>\n\n\n\n<p>\u6b21\u306bint8\u3067\u3082\u5b9f\u884c\u3057\u3066\u307f\u305f\u3002VRAM\u306f4G\u307b\u3069\u304b\uff1f\u5fdc\u7b54\u901f\u5ea6\u306f\u3001\u3053\u308c\u3082\u6c17\u306b\u306a\u3089\u306a\u304b\u3063\u305f\u3002\u5fdc\u7b54\u7cbe\u5ea6\u3082\u9ad8\u3044\u3002<\/p>\n\n\n\n<p><\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><a href=\"https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/ba331492908cfbcdb0adb270666541ea.png\"><img loading=\"lazy\" decoding=\"async\" width=\"1015\" height=\"791\" src=\"https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/ba331492908cfbcdb0adb270666541ea.png\" alt=\"\" class=\"wp-image-4973\" srcset=\"https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/ba331492908cfbcdb0adb270666541ea.png 1015w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/ba331492908cfbcdb0adb270666541ea-300x234.png 300w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/ba331492908cfbcdb0adb270666541ea-768x599.png 768w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/ba331492908cfbcdb0adb270666541ea-150x117.png 150w, https:\/\/www.sougetu.net\/wp-content\/uploads\/2023\/08\/ba331492908cfbcdb0adb270666541ea-192x150.png 192w\" sizes=\"auto, (max-width: 1015px) 100vw, 1015px\" \/><\/a><\/figure>\n","protected":false},"excerpt":{"rendered":"<p>\u5909\u63db\u306f\u3001bfloat16\u306b\u3057\u3066\u307f\u305f\u3002\u3053\u306e\u72b6\u614b\u3067\u52d5\u304b\u3057\u3066\u3082\uff16\uff27\u7a0b\u5ea6\u3002\u5909\u63db\u3067\u4f7f\u7528\u3057\u305f\u30e1\u30a4\u30f3\u30e1\u30e2\u30ea\u306f\uff12\uff10G\u7a0b\u5ea6\u3060\u3063\u305f\u304b\u306a\uff1f\uff08\u3046\u308d\u899a\u3048\uff09\u5fdc\u7b54\u3082\u5341\u5206\u65e9\u304f\u3001\u9045\u5ef6\u306f\u6c17\u306b\u306a\u3089\u306a\u304b\u3063\u305f\u3002 \u6700\u7d42\u7684\u306b\u52d5\u304b\u3057\u305f\u30b3\u30fc\u30c9\u306f\u4ee5\u4e0b\u306e\u901a\u308a\u3002 \u6b21\u306bint8 \u2026 <a class=\"continue-reading-link\" href=\"https:\/\/www.sougetu.net\/?p=4969\"> Continue reading <span class=\"meta-nav\">&rarr; <\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[8],"tags":[],"class_list":["post-4969","post","type-post","status-publish","format-standard","hentry","category-tech-computer-program"],"jetpack_featured_media_url":"","jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/www.sougetu.net\/index.php?rest_route=\/wp\/v2\/posts\/4969","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.sougetu.net\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.sougetu.net\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.sougetu.net\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.sougetu.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=4969"}],"version-history":[{"count":2,"href":"https:\/\/www.sougetu.net\/index.php?rest_route=\/wp\/v2\/posts\/4969\/revisions"}],"predecessor-version":[{"id":4974,"href":"https:\/\/www.sougetu.net\/index.php?rest_route=\/wp\/v2\/posts\/4969\/revisions\/4974"}],"wp:attachment":[{"href":"https:\/\/www.sougetu.net\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=4969"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.sougetu.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=4969"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.sougetu.net\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=4969"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}