Win環境にFlexGenインストール

python --version

cd "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\bin"
curl -O http://www.winimage.com/zLibDll/zlib123dllx64.zip
call powershell -command "Expand-Archive zlib123dllx64.zip"
copy zlib123dllx64\dll_x64\zlibwapi.dll .

cd C:\
git clone https://github.com/FMInference/FlexGen.git
cd C:\FlexGen
python -m venv env
env\Scripts\activate.bat

python -m pip install -U pip setuptools
pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117

pip3 install -e .

python3 -m flexgen.flex_opt --model facebook/opt-1.3b

Python -m flexgen.flex_opt --model EleutherAI/gpt-j-6B

(env) C:\FlexGen>Python -m flexgen.flex_opt --model EleutherAI/gpt-j-6B
Traceback (most recent call last):
  File "C:\Program Files\Python37\lib\runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "C:\Program Files\Python37\lib\runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "C:\FlexGen\flexgen\flex_opt.py", line 1307, in <module>
    run_flexgen(args)
  File "C:\FlexGen\flexgen\flex_opt.py", line 1189, in run_flexgen
    opt_config = get_opt_config(args.model)
  File "C:\FlexGen\flexgen\opt_config.py", line 116, in get_opt_config
    raise ValueError(f"Invalid model name: {name}")
ValueError: Invalid model name: gpt-j-6b

"""Run a chatbot with FlexGen and OPT models."""
import argparse

from transformers import AutoTokenizer
from flexgen.flex_opt import (Policy, OptLM, TorchDevice, TorchDisk, TorchMixedDevice,
    CompressionConfig, Env, Task, get_opt_config)


def main(args):
    # Initialize environment
    gpu = TorchDevice("cuda:0")
    cpu = TorchDevice("cpu")
    disk = TorchDisk(args.offload_dir)
    env = Env(gpu=gpu, cpu=cpu, disk=disk, mixed=TorchMixedDevice([gpu, cpu, disk]))

    # Offloading policy
    policy = Policy(1, 1,
                    args.percent[0], args.percent[1],
                    args.percent[2], args.percent[3],
                    args.percent[4], args.percent[5],
                    overlap=True, sep_layer=True, pin_weight=True,
                    cpu_cache_compute=False, attn_sparsity=1.0,
                    compress_weight=args.compress_weight,
                    comp_weight_config=CompressionConfig(
                        num_bits=4, group_size=64,
                        group_dim=0, symmetric=False),
                    compress_cache=args.compress_cache,
                    comp_cache_config=CompressionConfig(
                        num_bits=4, group_size=64,
                        group_dim=2, symmetric=False))

    # Model
    tokenizer = AutoTokenizer.from_pretrained("facebook/opt-1.3b", padding_side="left")
    tokenizer.add_bos_token = False
    stop = tokenizer("\n").input_ids[0]

    print("Initialize...")
    opt_config = get_opt_config(args.model)
    model = OptLM(opt_config, env, args.path, policy)
    model.init_all_weights()

    context = (
        "AI：こんにちは。私はAIです。.\n"
        "あなた: こんにちは。よろしくお願いします\n"
        "AI: はい、よろしくお願いします.\n"
        "あなた: 私はこれからいろんな質問をあなたにします。あなたはそれに答えてください\n"
        "AI: わかりました。私は質問に答えます.\n"
    )

    # Chat
    print(context, end="")
    while True:
        inp = input("あなた: ")
        if not inp:
            print("exit...")
            break

        context += "Human: " + inp + "\n"
        inputs = tokenizer([context])
        output_ids = model.generate(
            inputs.input_ids,
            do_sample=True,
            temperature=0.7,
            max_new_tokens=96,
            stop=stop)
        outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0]
        try:
            index = outputs.index("\n", len(context))
        except ValueError:
            outputs += "\n"
            index = outputs.index("\n", len(context))
        
        outputs = outputs[:index + 1]
        print(outputs[len(context):], end="")
        context = outputs

    # TODO: optimize the performance by reducing redundant computation.

    # Shutdown
    model.delete_all_weights()
    disk.close_copy_threads()


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--model", type=str, default="facebook/opt-1.3b",
        help="The model name.")
    parser.add_argument("--path", type=str, default="~/opt_weights",
        help="The path to the model weights. If there are no cached weights, "
             "FlexGen will automatically download them from HuggingFace.")
    parser.add_argument("--offload-dir", type=str, default="~/flexgen_offload_dir",
        help="The directory to offload tensors. ")
    parser.add_argument("--percent", nargs="+", type=int,
        default=[100, 0, 100, 0, 100, 0],
        help="Six numbers. They are "
         "the percentage of weight on GPU, "
         "the percentage of weight on CPU, "
         "the percentage of attention cache on GPU, "
         "the percentage of attention cache on CPU, "
         "the percentage of activations on GPU, "
         "the percentage of activations on CPU")
    parser.add_argument("--compress-weight", action="store_true",
        help="Whether to compress weight.")
    parser.add_argument("--compress-cache", action="store_true",
        help="Whether to compress cache.")
    args = parser.parse_args()

    assert len(args.percent) == 6

    main(args)

python C:\FlexGen\apps\chatbot_ja.py