olive-recipes/deepseek-ai-DeepSeek-R1-Distill-Qwen-1.5B/aitk/deepseek_qnn_gpu_config.json at main · microsoft/olive-recipes · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
{
    "input_model": {
        "type": "HfModel",
        "model_path": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
    },
    "systems": {
        "target_system": {
            "type": "LocalSystem",
            "accelerators": [
                {
                    "device": "gpu",
                    "execution_providers": [
                        "QNNExecutionProvider"
                    ]
                }
            ]
        }
    },
    "data_configs": [
        {
            "name": "wikitext2_train_joined",
            "type": "HuggingfaceContainer",
            "load_dataset_config": {
                "data_name": "wikitext",
                "subset": "wikitext-2-raw-v1",
                "split": "train"
            },
            "pre_process_data_config": {
                "strategy": "join",
                "add_special_tokens": false,
                "max_seq_len": 4096,
                "max_samples": 128
            }
        },
        {
            "name": "wikitext2_train_act",
            "type": "HuggingfaceContainer",
            "load_dataset_config": {
                "data_name": "wikitext",
                "subset": "wikitext-2-raw-v1",
                "split": "train"
            },
            "pre_process_data_config": {
                "strategy": "line-by-line",
                "add_special_tokens": true,
                "max_samples": 128,
                "max_seq_len": 512
            }
        }
    ],
    "passes": {
        "q": {
            "type": "QuaRot"
        },
        "g": {
            "type": "GptqQuantizer",
            "sym": true,
            "group_size": 32,
            "desc_act": false,
            "data_config": "wikitext2_train_joined"
        },
        "mb": {
            "type": "ModelBuilder",
            "precision": "int4",
            "int4_block_size": 32,
            "int4_accuracy_level": 4,
            "int4_op_types_to_quantize": [
                "MatMul",
                "Gemm"
            ]
        },
        "gs": {
            "type": "GraphSurgeries",
            "surgeries": [
                {
                    "surgeon": "RemoveRopeMultiCache"
                },
                {
                    "surgeon": "AttentionMaskToSequenceLengths"
                }
            ],
            "save_as_external_data": true
        },
        "st": {
            "type": "StaticLLM",
            "batch_size": 1,
            "context_length": 1
        }
    },
    "target": "target_system",
    "log_severity_level": 1,
    "output_dir": "model/deepseek",
    "cache_dir": "cache",
    "no_artifacts": false,
    "evaluate_input_model": false
}