U
    di]c                     @   s^  d Z ddlmZ ddlZddlmZmZmZmZ ddl	m
Z
 ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ eeZdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd dd ddd!dddd"d#d$d%d&d'ddd(d)dd*d+d,d-d&d'dd(d)d.d/d0	d,d-d&d'd1d2d(d)d.d/d3
ddddddddddddddd4d%d5d*d6d7d8d9d:d;d<d=ddd;d<ddddddddd>d?Zd@dAdBdCdDdEdFdGdHdIdJdKdLdM
dCdDdEdFdGdHdIdJdKdLdM
dCdDdEdFddHdIdJdKdLdM
dCdDdEdFddHdIdJdKdLdNdOdPdCdDdEdFddHdIdJdKdLdM
dQdRdSdTdUdCdDdEdFddHdIdJdKdLdM
dVdFdWdLdXdYdCdDdEdFddIdJdZdLd[	dVd\d]d^dWdXd_dDdCdFdEdIdJd`dadbZdcdddedfdgdQdRdSdTdhdi
djd@dQdRdSdTdkdlZdmdn ZG dodp dpZG dqdr dreZG dsdt dteZG dudv dveZG dwdx dxeZ eeeee e e e e dy	Z!edzd{d|Z"dS )}z
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
    )arrayN)	Tokenizerdecodersnormalizerspre_tokenizers)BPE   
AddedToken)GPT2ConverterLlamaConverterQwen2Converter)logging)tqdmzmodel.embed_tokenszmodel.layerszmlp.up_projzmlp.down_projzmlp.gate_projZpost_attention_layernormZinput_layernormzself_attn.q_projzself_attn.v_projzself_attn.k_projzself_attn.o_projzlm_head.weightz
model.norm)
token_embdblkffn_upffn_downffn_gateffn_norm	attn_normattn_qattn_vattn_kattn_outputoutput.weightoutput_normzmlp.expertszmlp.shared_expert.up_projzmlp.shared_expert.down_projzmlp.gate.weightzmlp.shared_expert.gate_projzmlp.shared_expert_gate)r   r   Zffn_up_expsZffn_up_shexpZffn_down_expsZffn_down_shexpr   zffn_gate_inp.weightZffn_gate_expsZffn_gate_shexpZffn_gate_inp_shexpr   r   r   r   r   r   r   zmlp.gate_up_projzself_attn.qkv_proj)r   r   r   r   r   r   r   attn_qkvr   r   r   z"transformer.word_embeddings.weightz%transformer.word_embeddings_layernormztransformer.hzmlp.dense_h_to_4hzmlp.dense_4h_to_hzself_attention.query_key_valuezself_attention.denseztransformer.ln_f)ztoken_embd.weightZtoken_embd_normr   r   r   r   r   r   r   r   r   Zword_embeddingshz	.lm_head.Zln_f)	r   r   r   r   r   r   r   .output.r   z.ln_mlp.Zln_attn)
r   r   r   r   z.attn_norm.Zattn_norm_2r   r   r   r   ztransformer.wteztransformer.wpeZln_1zattn.c_attnzattn.c_proj.weightzattn.c_proj.biasZln_2zmlp.c_fcz
mlp.c_proj)r   r   Zposition_embdr   r   r   zattn_output.weightzattn_output.biasr   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   )llamamistralqwen2qwen2moephi3bloomZfalcon7bZ	falcon40bstablelmgpt2
starcoder2Z
model_typeZ_model_name_or_path)architecturenameZmax_position_embeddingsZnum_hidden_layersZintermediate_sizeZhidden_sizeZhead_dimZ
rope_thetaZnum_attention_headsZnum_key_value_headsZrms_norm_eps
vocab_size)
context_lengthblock_countfeed_forward_lengthembedding_lengthrope.dimension_countrope.freq_baseattention.head_countattention.head_count_kv attention.layer_norm_rms_epsilonr+   Znum_expertsZnum_experts_per_tok)r,   r-   r.   r/   r0   r1   r2   r3   r4   r+   Zexpert_countZexpert_used_countbos_token_ideos_token_idunk_token_idpad_token_id)ggml.bos_token_idggml.eos_token_idggml.unknown_token_idggml.padding_token_idZn_layerZn_headZlayer_norm_epsilon)r-   r/   r2   r+   attention.layer_norm_epsilonZlayer_norm_eps)	r,   r-   r.   r/   r0   r2   r3   r=   r+   Zn_ctxZn_embdr.   )r-   r,   r/   r.   r2   r=   Znorm_epsilon)r-   r,   r/   r.   r2   r3   r=   )Zgeneralr    r!   r"   r#   falcon	tokenizerr$   r%   r&   r'   r(   tokenizer_typetokensscores
token_typemergesadd_prefix_space)

ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesr9   r:   r;   r<   zggml.add_space_prefixchat_template)rG   rF   r9   r:   r;   r<   )r?   Ztokenizer_configc                 C   s   t |ts|g}t|dkr*|d }d }n|d dkr>td|\}}|dkr\t| d } n^|dkrrt| d } nH|dkrt| d } n2|dkrtd	t|  	 } n|d
krt
| |} | S )N   r   	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   rH   r            
      )      )   )   B)rI   )
isinstancelistlen
ValueErrorintfloatboolr   tobytesdecode_gguf_parse_value)_valueZ	data_typeZarray_data_type r_   B/tmp/pip-unpacked-wheel-bm_b0l5e/transformers/integrations/ggml.pyr]   X  s&    

r]   c                   @   s   e Zd Zdd ZdS )GGUFTokenizerSkeletonc                    s  |  D ]\}}t| || qt| ds&t| dr>t| dsFtd| j}| j  fddt|D td g }t	  D ]z\}}g }t
dt|D ]>}	|d |	 ||	d   }
}|
|kr||kr||
||f qt|fd	d
dd}|| qt|dd
 dd}dd |D }|| _n8dd | jD | _t| ds^dd t
t| jD | _t| dspg | _t| dsd | _t| dr| jd kr| j| _d S )NrD   rA   rB   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.c                    s   i | ]\}}| | qS r_   r_   ).0it)rB   r_   r`   
<dictcomp>|  s      z2GGUFTokenizerSkeleton.__init__.<locals>.<dictcomp>z:Merges were not in checkpoint, building merges on the fly.rH   c                    s    | d   | d  fS )Nr   rH   r_   )x)vocabr_   r`   <lambda>      z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>T)keyreversec                 S   s   | d S )Nr   r_   )valr_   r_   r`   rh     ri   c                 S   s   g | ]}|d  |d fqS )r   rH   r_   )rb   rl   r_   r_   r`   
<listcomp>  s     z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>c                 S   s   g | ]}t |d qS ) )tuplesplit)rb   merger_   r_   r`   rm     s     c                 S   s   g | ]}d qS Nr_   )rb   _r_   r_   r`   rm     s     added_tokensr7   unknown_token_id)itemssetattrhasattrrW   rA   rB   	enumerateloggerwarningr   rangerV   appendsortedextendrD   rt   r7   ru   )selfZdict_kvrA   rD   rq   Zpiece_scorelocalindexZpiece_lZpiece_rr_   )rB   rg   r`   __init__q  s@    
zGGUFTokenizerSkeleton.__init__N)__name__
__module____qualname__r   r_   r_   r_   r`   ra   p  s   ra   c                   @   s<   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd ZdS )GGUFLlamaConverterc                 C   s0   t || _| j| _i | _t| jdddk| _d S )Nr@   r    )ra   protooriginal_tokenizeradditional_kwargsgetattris_llama_3_tokenizerr   tokenizer_dictr_   r_   r`   r     s    
zGGUFLlamaConverter.__init__c                 C   s   t t|j|jS rr   rU   ziprA   rB   r   r   r_   r_   r`   rg     s    zGGUFLlamaConverter.vocabc                 C   s   |j S rr   rD   r   r_   r_   r`   rD     s    zGGUFLlamaConverter.mergesc                 C   s  |  | j}| | j}dd t|D }|jd k	r@|j|j nd }t|dd d k	r`|j|j nd }t|dd d k	r|j|j nd }tt	|||ddd}g }	t
| jds|d k	r|	t|ddd	 |d k	r|	t|ddd	 |d k	rH|	t|ddd	 nDtt| jjd
kd }
|
D ]"}|	t| jj| ddd	 q$t|	dkr`||	 t| jjdkr|dd | jjD  || jd< || jd< || jd< | jrd | jd< d| jd< d| jd< d| j_|S )Nc                 S   s   i | ]\}\}}||qS r_   r_   rb   rc   wordZ_scorer_   r_   r`   re     s    
  z0GGUFLlamaConverter.tokenizer.<locals>.<dictcomp>r5   r6   T)	unk_tokenZfuse_unkZbyte_fallbackrC   F
normalizedspecialrJ   r   c                 S   s   g | ]}t |d d dqS )Fr   r	   )rb   Zadded_tokenr_   r_   r`   rm     s     z0GGUFLlamaConverter.tokenizer.<locals>.<listcomp>r   	eos_token	bos_tokenrE   Zclean_up_tokenization_spaceslegacy)rg   r   rD   ry   r7   rA   r   r5   r   r   rx   r}   r
   npwherer   rC   rV   add_special_tokensrt   Z
add_tokensr   r   r   r   )r   r   vocab_scoresrD   	bpe_vocabr   r   r   r?   Zspecial_tokensZspecial_tokens_idxidxr_   r_   r`   r?     sR      

 






zGGUFLlamaConverter.tokenizerc                 C   sX   t  t  t ddg}| jr6|t jddddg7 }|rN|t jdddg7 }t |S )N   ▁rn   FTrE   Ztrim_offsetsZ	use_regexrH   contentleft)r   ByteFallbackFuseReplacer   	ByteLevelStripSequencer   replacementrE   sequencer_   r_   r`   decoder  s    
zGGUFLlamaConverter.decoderc                 C   s   |  | j}| | j}|d k	r&||_d}d}t| jdrB| jj}| ||}|d k	r\||_| |||_|  }|r|||_| j	rt
jdddd|_tg |_|S )Nr   TrE   Fr   )r?   r   
normalizerrx   r   rE   pre_tokenizerr   post_processorr   r   r   r   r   )r   r?   r   r   rE   r   r   r_   r_   r`   	converted  s.      zGGUFLlamaConverter.convertedN)	r   r   r   r   rg   rD   r?   r   r   r_   r_   r_   r`   r     s   :r   c                       s*   e Zd Zdd Zed fddZ  ZS )GGUFQwen2Converterc                 C   s   t || _i | _d S rr   ra   r   r   r   r_   r_   r`   r     s    
zGGUFQwen2Converter.__init__returnc              	      s^   dd t | jjD }| jj}t ||}|tddddtddddtddddg |S )	Nc                 S   s   i | ]\}}||qS r_   r_   rb   rc   r   r_   r_   r`   re     s      z0GGUFQwen2Converter.converted.<locals>.<dictcomp><|endoftext|>FTr   z<|im_start|>z
<|im_end|>)ry   r   rA   rD   superr   r   r
   r   rg   rD   r?   	__class__r_   r`   r     s    zGGUFQwen2Converter.convertedr   r   r   r   r   r   __classcell__r_   r_   r   r`   r     s   r   c                   @   sB   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZedddZ	dS )GGUFPhi3Converterc                 C   s   t || _| j| _i | _d S rr   )ra   r   r   r   r   r_   r_   r`   r   )  s    
zGGUFPhi3Converter.__init__c                 C   s   t t|j|jS rr   r   r   r_   r_   r`   rg   .  s    zGGUFPhi3Converter.vocabc                 C   s   |j S rr   r   r   r_   r_   r`   rD   1  s    zGGUFPhi3Converter.mergesc                 C   sv  |  | j}| | j}dd t|D }tt||}|tddddddtddddtd	dddd
tddddd
tddddd
tddddd
tddddd
tddddd
tddddd
tddddd
tddddd
tddddd
g |jd k	r|j	|j nd | j
d< |jd k	r$|j	|j nd | j
d< |jd k	rF|j	|j nd | j
d< |jd k	rh|j	|j nd | j
d< |S )Nc                 S   s   i | ]\}\}}||qS r_   r_   r   r_   r_   r`   re   7  s    
  z/GGUFPhi3Converter.tokenizer.<locals>.<dictcomp>z</s>TF)rstriplstripr   r   r   r   z<|assistant|>)r   r   r   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>r   r   r   Z	pad_token)rg   r   rD   ry   r   r   r   r
   r7   rA   r   r6   r5   r8   )r   r   r   rD   r   r?   r_   r_   r`   r?   4  s8    zGGUFPhi3Converter.tokenizerc                 C   s<   t  t  t |dg}|r2|t jdddg7 }t |S )Nrn   rH   r   )r   r   r   r   r   r   r   r_   r_   r`   r   [  s    
zGGUFPhi3Converter.decoderr   c                 C   s:   |  | j}d}d}t| jdr(| jj}| |||_|S )Nr   TrE   )r?   r   rx   r   rE   r   )r   r?   r   rE   r_   r_   r`   r   f  s    zGGUFPhi3Converter.convertedN)
r   r   r   r   rg   rD   r?   r   r   r   r_   r_   r_   r`   r   (  s   'r   c                       s*   e Zd Zdd Zed fddZ  ZS )GGUFGPTConverterc                 C   s   t || _i | _d S rr   r   r   r_   r_   r`   r   t  s    
zGGUFGPTConverter.__init__r   c                    s0   dd t | jjD }| jj}t ||}|S )Nc                 S   s   i | ]\}}||qS r_   r_   r   r_   r_   r`   re   y  s      z.GGUFGPTConverter.converted.<locals>.<dictcomp>)ry   r   rA   rD   r   r   r   r   r_   r`   r   x  s    zGGUFGPTConverter.convertedr   r_   r_   r   r`   r   s  s   r   )	r    r"   Z	qwen2_moer$   r%   r>   r&   r'   r(   r   c                 C   s"   | }t | |}| }||jfS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )GGUF_TO_FAST_CONVERTERSr   r   )r)   r   Ztokenizer_class_name	converterZfast_tokenizerr_   r_   r`   convert_gguf_tokenizer  s    r   )#__doc__r   Znumpyr   Z
tokenizersr   r   r   r   Ztokenizers.modelsr    r
   Zconvert_slow_tokenizerr   r   r   utilsr   Zutils.loggingr   Z
get_loggerr   rz   ZGGUF_TENSOR_MAPPINGZGGUF_CONFIG_MAPPINGZGGUF_TOKENIZER_MAPPINGr]   ra   r   r   r   r   r   r   r_   r_   r_   r`   <module>   sZ  
  #	~+yK