U
    diN                     @   s
  d dl mZ esdekr&ddlmZ nd dlZzd dlZW n e	k
rV   d dlZY nX dd Z
dd	 Zd
d Zdd ZG dd deZG dd deZee G dd deZee G dd deZee G dd deZee dd Zdd ZG dd deZee G dd deZee d dlZd dl Z d dl Z d dl!Z!d d l"m#Z# d d!l"m$Z$ d"d# Z%d$d% Z&ej'Z(ej'Z)e*ed&ej+ e*ed&ej+ ej,e_-ej.e_/d'D ]Z0e&ee0 qe%e e%e e%e eZ1eZ2dd(l3m4Z4 G d)d* d*eZ5dS )+    )version_info.   )_sentencepieceNc                 C   sD   zd| j   }W n tjk
r,   d}Y nX d| jj| jj|f S )Nz	proxy of  z<%s.%s; %s >)this__repr____builtin__	Exception	__class__
__module____name__)selfZstrthis r   :/tmp/pip-unpacked-wheel-dnuolidd/sentencepiece/__init__.py
_swig_repr   s
    
r   c                    s    fdd}|S )Nc                    sh   |dkr | || nN|dkr,| j | n8t| |rXttt| |trX | || ntd|  d S )Nr   thisownz(You cannot add instance attributes to %s)r   ownhasattr
isinstancegetattrtypepropertyAttributeErrorr   namevaluesetr   r   set_instance_attr   s    zE_swig_setattr_nondynamic_instance_variable.<locals>.set_instance_attrr   )r   r   r   r   r   *_swig_setattr_nondynamic_instance_variable   s    	r    c                    s    fdd}|S )Nc                    s8   t | |r(tt| |ts( | || ntd|  d S )Nz%You cannot add class attributes to %s)r   r   r   r   r   )clsr   r   r   r   r   set_class_attr)   s    z?_swig_setattr_nondynamic_class_variable.<locals>.set_class_attrr   )r   r"   r   r   r   '_swig_setattr_nondynamic_class_variable(   s    r#   c                    s    fdd}|S )zlClass decorator for adding a metaclass to a SWIG wrapped class - a slimmed down version of six.add_metaclassc                    s    | j | j| j S N)r   	__bases____dict__copy)r!   	metaclassr   r   wrapper3   s    z$_swig_add_metaclass.<locals>.wrapperr   )r)   r*   r   r(   r   _swig_add_metaclass1   s    r+   c                   @   s   e Zd ZdZeejZdS )_SwigNonDynamicMetazKMeta class to enforce nondynamic attributes (no new attributes) for a classN)r   r   __qualname____doc__r#   r   __setattr__r   r   r   r   r,   8   s   r,   c                   @   s   e Zd Zedd dd ddZeZdd Zej	Z
dd	 Zd
d Zdd Zdd Zdd Zdd Zdd ZeeZeeZeeZeeZeeZeeZeeZdd Zdd Zdd ZeZdS )1ImmutableSentencePieceText_ImmutableSentencePiecec                 C   s
   | j  S r$   r   r   xr   r   r   <lambda>>       z:ImmutableSentencePieceText_ImmutableSentencePiece.<lambda>c                 C   s   | j |S r$   r1   r3   vr   r   r   r4   >   r5   The membership flagdocc                 C   s   t | t   d S r$   )r   Z:ImmutableSentencePieceText_ImmutableSentencePiece_swiginitZ5new_ImmutableSentencePieceText_ImmutableSentencePiecer   r   r   r   __init__A   s    z:ImmutableSentencePieceText_ImmutableSentencePiece.__init__c                 C   s
   t | S r$   )r   Z8ImmutableSentencePieceText_ImmutableSentencePiece__piecer;   r   r   r   _pieceE   s    z8ImmutableSentencePieceText_ImmutableSentencePiece._piecec                 C   s
   t | S r$   )r   Z:ImmutableSentencePieceText_ImmutableSentencePiece__surfacer;   r   r   r   _surfaceH   s    z:ImmutableSentencePieceText_ImmutableSentencePiece._surfacec                 C   s
   t | S r$   )r   Z5ImmutableSentencePieceText_ImmutableSentencePiece__idr;   r   r   r   _idK   s    z5ImmutableSentencePieceText_ImmutableSentencePiece._idc                 C   s
   t | S r$   )r   Z8ImmutableSentencePieceText_ImmutableSentencePiece__beginr;   r   r   r   _beginN   s    z8ImmutableSentencePieceText_ImmutableSentencePiece._beginc                 C   s
   t | S r$   )r   Z6ImmutableSentencePieceText_ImmutableSentencePiece__endr;   r   r   r   _endQ   s    z6ImmutableSentencePieceText_ImmutableSentencePiece._endc                 C   s
   t | S r$   )r   ZCImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytesr;   r   r   r   _surface_as_bytesT   s    zCImmutableSentencePieceText_ImmutableSentencePiece._surface_as_bytesc                 C   s
   t | S r$   )r   ZAImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytesr;   r   r   r   _piece_as_bytesW   s    zAImmutableSentencePieceText_ImmutableSentencePiece._piece_as_bytesc                 C   s   d | j| j| j| j| jS )Nz3piece: "{}"
id: {}
surface: "{}"
begin: {}
end: {}
)formatpieceidsurfacebeginendr;   r   r   r   __str__b   s       z9ImmutableSentencePieceText_ImmutableSentencePiece.__str__c                 C   s<   | j |j ko:| j|jko:| j|jko:| j|jko:| j|jkS r$   )rE   rF   rG   rH   rI   r   otherr   r   r   __eq__j   s    z8ImmutableSentencePieceText_ImmutableSentencePiece.__eq__c                 C   s   t t| S r$   )hashstrr;   r   r   r   __hash__m   s    z:ImmutableSentencePieceText_ImmutableSentencePiece.__hash__N)r   r   r-   r   r   r   r   r<   r   Z8delete_ImmutableSentencePieceText_ImmutableSentencePiece__swig_destroy__r=   r>   r?   r@   rA   rB   rC   rE   Zpiece_as_bytesrG   Zsurface_as_bytesrF   rH   rI   rJ   rM   rP   r   r   r   r   r0   =   s,   r0   c                   @   s   e Zd Zedd dd ddZeZdd Zej	Z
dd	 Zd
d Zdd Zdd Zdd Zdd ZeeZeeZeeZG dd dZedd Zdd Zdd Zdd ZeZdS )ImmutableSentencePieceTextc                 C   s
   | j  S r$   r1   r2   r   r   r   r4   v   r5   z#ImmutableSentencePieceText.<lambda>c                 C   s   | j |S r$   r1   r6   r   r   r   r4   v   r5   r8   r9   c                 C   s   t | t   d S r$   )r   Z#ImmutableSentencePieceText_swiginitZnew_ImmutableSentencePieceTextr;   r   r   r   r<   y   s    z#ImmutableSentencePieceText.__init__c                 C   s
   t | S r$   )r   Z'ImmutableSentencePieceText__pieces_sizer;   r   r   r   _pieces_size}   s    z'ImmutableSentencePieceText._pieces_sizec                 C   s   t | |S r$   )r   Z"ImmutableSentencePieceText__piecesr   indexr   r   r   _pieces   s    z"ImmutableSentencePieceText._piecesc                 C   s
   t | S r$   )r   Z ImmutableSentencePieceText__textr;   r   r   r   _text   s    z ImmutableSentencePieceText._textc                 C   s
   t | S r$   )r   Z!ImmutableSentencePieceText__scorer;   r   r   r   _score   s    z!ImmutableSentencePieceText._scorec                 C   s
   t | S r$   )r   Z,ImmutableSentencePieceText_SerializeAsStringr;   r   r   r   SerializeAsString   s    z,ImmutableSentencePieceText.SerializeAsStringc                 C   s
   t | S r$   )r   Z)ImmutableSentencePieceText__text_as_bytesr;   r   r   r   _text_as_bytes   s    z)ImmutableSentencePieceText._text_as_bytesc                   @   s0   e Zd Zdd Zdd Zdd Zdd ZeZd	S )
z9ImmutableSentencePieceText.ImmutableSentencePieceIteratorc                 C   s   || _ | j  | _d S r$   )protorS   lenr   r[   r   r   r   r<      s    zBImmutableSentencePieceText.ImmutableSentencePieceIterator.__init__c                 C   s   | j S r$   r\   r;   r   r   r   __len__   s    zAImmutableSentencePieceText.ImmutableSentencePieceIterator.__len__c                    sj   t |tr2 fddt jD |j|j|j S |dk rD| j }|dk sV| jkr^td j	|S )Nc                    s   g | ]} j |qS r   )r[   rV   .0ir;   r   r   
<listcomp>   s     zYImmutableSentencePieceText.ImmutableSentencePieceIterator.__getitem__.<locals>.<listcomp>r   zpiece index is out of range)
r   sliceranger\   startstopstep
IndexErrorr[   rV   rT   r   r;   r   __getitem__   s    
(
zEImmutableSentencePieceText.ImmutableSentencePieceIterator.__getitem__c                 C   s   d dd | D S )N
c                 S   s   g | ]}d  t|qS zpieces {{
{}}}rD   rO   ra   r3   r   r   r   rc      s     zUImmutableSentencePieceText.ImmutableSentencePieceIterator.__str__.<locals>.<listcomp>joinr;   r   r   r   rJ      s    zAImmutableSentencePieceText.ImmutableSentencePieceIterator.__str__Nr   r   r-   r<   r_   rj   rJ   r   r   r   r   r   ImmutableSentencePieceIterator   s
   	rr   c                 C   s
   t | S r$   )rR   rr   r;   r   r   r   pieces   s    z!ImmutableSentencePieceText.piecesc                 C   s   |   |  kS r$   rY   rK   r   r   r   rM      s    z!ImmutableSentencePieceText.__eq__c                 C   s   t |  S r$   rN   rY   r;   r   r   r   rP      s    z#ImmutableSentencePieceText.__hash__c                 C   s$   d | j| jddd | jD S )Nztext: "{}"
score: {}
{}rk   c                 S   s   g | ]}d  t|qS rl   rm   rn   r   r   r   rc      s     z6ImmutableSentencePieceText.__str__.<locals>.<listcomp>)rD   textscorerp   rs   r;   r   r   r   rJ      s
     z"ImmutableSentencePieceText.__str__N)r   r   r-   r   r   r   r   r<   r   Z!delete_ImmutableSentencePieceTextrQ   rS   rV   rW   rX   rY   rZ   rv   Ztext_as_bytesrw   rr   rs   rM   rP   rJ   r   r   r   r   rR   u   s(   
rR   c                   @   s   e Zd Zedd dd ddZeZdd Zej	Z
dd	 Zd
d Zdd ZG dd dZedd Zdd Zdd Zdd ZeZdS )ImmutableNBestSentencePieceTextc                 C   s
   | j  S r$   r1   r2   r   r   r   r4      r5   z(ImmutableNBestSentencePieceText.<lambda>c                 C   s   | j |S r$   r1   r6   r   r   r   r4      r5   r8   r9   c                 C   s   t | t   d S r$   )r   Z(ImmutableNBestSentencePieceText_swiginitZ#new_ImmutableNBestSentencePieceTextr;   r   r   r   r<      s    z(ImmutableNBestSentencePieceText.__init__c                 C   s
   t | S r$   )r   Z,ImmutableNBestSentencePieceText__nbests_sizer;   r   r   r   _nbests_size   s    z,ImmutableNBestSentencePieceText._nbests_sizec                 C   s   t | |S r$   )r   Z'ImmutableNBestSentencePieceText__nbestsrT   r   r   r   _nbests   s    z'ImmutableNBestSentencePieceText._nbestsc                 C   s
   t | S r$   )r   Z1ImmutableNBestSentencePieceText_SerializeAsStringr;   r   r   r   rY      s    z1ImmutableNBestSentencePieceText.SerializeAsStringc                   @   s0   e Zd Zdd Zdd Zdd Zdd ZeZd	S )
zBImmutableNBestSentencePieceText.ImmutableSentencePieceTextIteratorc                 C   s   || _ | j  | _d S r$   )r[   ry   r\   r]   r   r   r   r<      s    zKImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__init__c                 C   s   | j S r$   r^   r;   r   r   r   r_      s    zJImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__len__c                    sj   t |tr2 fddt jD |j|j|j S |dk rD| j }|dk sV| jkr^td j	|S )Nc                    s   g | ]} j |qS r   )r[   rz   r`   r;   r   r   rc      s     zbImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__getitem__.<locals>.<listcomp>r   znbests index is out of range)
r   rd   re   r\   rf   rg   rh   ri   r[   rz   rT   r   r;   r   rj      s    
(
zNImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__getitem__c                 C   s   d dd | D S )Nrk   c                 S   s   g | ]}d  t|qS znbests {{
{}}}rm   rn   r   r   r   rc      s     z^ImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__str__.<locals>.<listcomp>ro   r;   r   r   r   rJ      s    zJImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__str__Nrq   r   r   r   r   "ImmutableSentencePieceTextIterator   s
   	r|   c                 C   s
   t | S r$   )rx   r|   r;   r   r   r   nbests   s    z&ImmutableNBestSentencePieceText.nbestsc                 C   s   |   |  kS r$   rt   rK   r   r   r   rM      s    z&ImmutableNBestSentencePieceText.__eq__c                 C   s   t |  S r$   ru   r;   r   r   r   rP      s    z(ImmutableNBestSentencePieceText.__hash__c                 C   s   d dd | jD S )Nrk   c                 S   s   g | ]}d  t|qS r{   rm   rn   r   r   r   rc      s     z;ImmutableNBestSentencePieceText.__str__.<locals>.<listcomp>)rp   r}   r;   r   r   r   rJ      s    z'ImmutableNBestSentencePieceText.__str__N)r   r   r-   r   r   r   r   r<   r   Z&delete_ImmutableNBestSentencePieceTextrQ   ry   rz   rY   r|   r}   rM   rP   rJ   r   r   r   r   rx      s   
rx   c                   @   sn  e Zd Zedd dd ddZeZdd Zej	Z
dd	 Zd
d Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0d1 Zd2d3 Z d4d5 Z!d6d7 Z"d8d9 Z#d:d; Z$d<d= Z%d>d? Z&d@dA Z'dBdC Z(dDdE Z)dFdG Z*dHdI Z+dJdK Z,dLdM Z-dNdO Z.dPdQ Z/dRdS Z0dTdU Z1dVdW Z2dXdY Z3dZd[ Z4d\d] Z5d^d_ Z6d`da Z7dbdc Z8ddde Z9dfdg Z:dhdi Z;djdk Z<dldm Z=dndo Z>dpdq Z?drds Z@dtdu ZAdvdw ZBdxdxeCdydydydydydzd{dzfd|d}ZDdd~dZEdd ZFdd ZGdd ZHdd ZIdddZJdddZKdddZLdddZMdddZNdddZOdddZPdddZQdddZRdddZSdddZTdddZUdddZVdddZWeXdxfddZYeXfddZZeXfddZ[dddZ\dddZ]dddZ^dddZ_dddZdddZ`dd Zadd Zbdd Zcdd Zddd Zedd ZfddĄ ZgdddƄZhdxS )SentencePieceProcessorc                 C   s
   | j  S r$   r1   r2   r   r   r   r4      r5   zSentencePieceProcessor.<lambda>c                 C   s   | j |S r$   r1   r6   r   r   r   r4      r5   r8   r9   c                 C   s   t | t   d S r$   )r   ZSentencePieceProcessor_swiginitZnew_SentencePieceProcessorr;   r   r   r   r<      s    zSentencePieceProcessor.__init__c                 C   s   t | |S r$   )r   Z.SentencePieceProcessor_LoadFromSerializedProtor   Z
serializedr   r   r   LoadFromSerializedProto   s    z.SentencePieceProcessor.LoadFromSerializedProtoc                 C   s   t | |S r$   )r   Z,SentencePieceProcessor_SetEncodeExtraOptionsr   Zextra_optionr   r   r   SetEncodeExtraOptions  s    z,SentencePieceProcessor.SetEncodeExtraOptionsc                 C   s   t | |S r$   )r   Z,SentencePieceProcessor_SetDecodeExtraOptionsr   r   r   r   SetDecodeExtraOptions  s    z,SentencePieceProcessor.SetDecodeExtraOptionsc                 C   s   t | |S r$   )r   Z$SentencePieceProcessor_SetVocabulary)r   Zvalid_vocabr   r   r   SetVocabulary  s    z$SentencePieceProcessor.SetVocabularyc                 C   s
   t | S r$   )r   Z&SentencePieceProcessor_ResetVocabularyr;   r   r   r   ResetVocabulary  s    z&SentencePieceProcessor.ResetVocabularyc                 C   s   t | ||S r$   )r   Z%SentencePieceProcessor_LoadVocabulary)r   filename	thresholdr   r   r   LoadVocabulary  s    z%SentencePieceProcessor.LoadVocabularyc                 G   s   t j| f| S r$   )r   Z'SentencePieceProcessor_CalculateEntropyr   argsr   r   r   CalculateEntropy  s    z'SentencePieceProcessor.CalculateEntropyc                 C   s
   t | S r$   )r   Z#SentencePieceProcessor_GetPieceSizer;   r   r   r   GetPieceSize  s    z#SentencePieceProcessor.GetPieceSizec                 C   s   t | |S r$   )r   Z SentencePieceProcessor_PieceToIdr   rE   r   r   r   	PieceToId  s    z SentencePieceProcessor.PieceToIdc                 C   s   t | |S r$   )r   Z SentencePieceProcessor_IdToPiecer   rF   r   r   r   	IdToPiece  s    z SentencePieceProcessor.IdToPiecec                 C   s   t | |S r$   )r   ZSentencePieceProcessor_GetScorer   r   r   r   GetScore  s    zSentencePieceProcessor.GetScorec                 C   s   t | |S r$   )r   Z SentencePieceProcessor_IsUnknownr   r   r   r   	IsUnknown   s    z SentencePieceProcessor.IsUnknownc                 C   s   t | |S r$   )r   Z SentencePieceProcessor_IsControlr   r   r   r   	IsControl#  s    z SentencePieceProcessor.IsControlc                 C   s   t | |S r$   )r   ZSentencePieceProcessor_IsUnusedr   r   r   r   IsUnused&  s    zSentencePieceProcessor.IsUnusedc                 C   s   t | |S r$   )r   ZSentencePieceProcessor_IsByter   r   r   r   IsByte)  s    zSentencePieceProcessor.IsBytec                 C   s
   t | S r$   )r   ZSentencePieceProcessor_unk_idr;   r   r   r   unk_id,  s    zSentencePieceProcessor.unk_idc                 C   s
   t | S r$   )r   ZSentencePieceProcessor_bos_idr;   r   r   r   bos_id/  s    zSentencePieceProcessor.bos_idc                 C   s
   t | S r$   )r   ZSentencePieceProcessor_eos_idr;   r   r   r   eos_id2  s    zSentencePieceProcessor.eos_idc                 C   s
   t | S r$   )r   ZSentencePieceProcessor_pad_idr;   r   r   r   pad_id5  s    zSentencePieceProcessor.pad_idc                 C   s
   t | S r$   )r   Z-SentencePieceProcessor_serialized_model_protor;   r   r   r   serialized_model_proto8  s    z-SentencePieceProcessor.serialized_model_protoc                 C   s   t | |S r$   )r   Z#SentencePieceProcessor_LoadFromFiler   argr   r   r   LoadFromFile;  s    z#SentencePieceProcessor.LoadFromFilec	           	      C   s   t | ||||||||	S r$   )r   Z#SentencePieceProcessor__EncodeAsIds	r   rv   enable_sampling
nbest_sizealphaadd_bosadd_eosreverseemit_unk_piecer   r   r   _EncodeAsIds>  s    z#SentencePieceProcessor._EncodeAsIdsc	           	      C   s   t | ||||||||	S r$   )r   Z&SentencePieceProcessor__EncodeAsPiecesr   r   r   r   _EncodeAsPiecesA  s    z&SentencePieceProcessor._EncodeAsPiecesc	           	      C   s   t | ||||||||	S r$   )r   Z/SentencePieceProcessor__EncodeAsSerializedProtor   r   r   r   _EncodeAsSerializedProtoD  s    z/SentencePieceProcessor._EncodeAsSerializedProtoc	           	      C   s   t | ||||||||	S r$   )r   Z.SentencePieceProcessor__EncodeAsImmutableProtor   r   r   r   _EncodeAsImmutableProtoG  s    z.SentencePieceProcessor._EncodeAsImmutableProtoc
           
      C   s   t | |||||||||	
S r$   )r   Z(SentencePieceProcessor__EncodeAsIdsBatch
r   insnum_threadsr   r   r   r   r   r   r   r   r   r   _EncodeAsIdsBatchJ  s    z(SentencePieceProcessor._EncodeAsIdsBatchc
           
      C   s   t | |||||||||	
S r$   )r   Z+SentencePieceProcessor__EncodeAsPiecesBatchr   r   r   r   _EncodeAsPiecesBatchM  s    z+SentencePieceProcessor._EncodeAsPiecesBatchc
           
      C   s   t | |||||||||	
S r$   )r   Z4SentencePieceProcessor__EncodeAsSerializedProtoBatchr   r   r   r   _EncodeAsSerializedProtoBatchP  s    z4SentencePieceProcessor._EncodeAsSerializedProtoBatchc
           
      C   s   t | |||||||||	
S r$   )r   Z3SentencePieceProcessor__EncodeAsImmutableProtoBatchr   r   r   r   _EncodeAsImmutableProtoBatchS  s    z3SentencePieceProcessor._EncodeAsImmutableProtoBatchc                 C   s   t | |S r$   )r   Z!SentencePieceProcessor__DecodeIdsr   idsr   r   r   
_DecodeIdsV  s    z!SentencePieceProcessor._DecodeIdsc                 C   s   t | |S r$   )r   Z(SentencePieceProcessor__DecodeIdsAsBytesr   r   r   r   _DecodeIdsAsBytesY  s    z(SentencePieceProcessor._DecodeIdsAsBytesc                 C   s   t | |S r$   )r   Z$SentencePieceProcessor__DecodePiecesr   rs   r   r   r   _DecodePieces\  s    z$SentencePieceProcessor._DecodePiecesc                 C   s   t | |S r$   )r   Z2SentencePieceProcessor__DecodeIdsAsSerializedProtor   r   r   r   _DecodeIdsAsSerializedProto_  s    z2SentencePieceProcessor._DecodeIdsAsSerializedProtoc                 C   s   t | |S r$   )r   Z5SentencePieceProcessor__DecodePiecesAsSerializedProtor   r   r   r   _DecodePiecesAsSerializedProtob  s    z5SentencePieceProcessor._DecodePiecesAsSerializedProtoc                 C   s   t | |S r$   )r   Z1SentencePieceProcessor__DecodeIdsAsImmutableProtor   r   r   r   _DecodeIdsAsImmutableProtoe  s    z1SentencePieceProcessor._DecodeIdsAsImmutableProtoc                 C   s   t | |S r$   )r   Z4SentencePieceProcessor__DecodePiecesAsImmutableProtor   r   r   r   _DecodePiecesAsImmutableProtoh  s    z4SentencePieceProcessor._DecodePiecesAsImmutableProtoc                 C   s   t | ||S r$   )r   Z&SentencePieceProcessor__DecodeIdsBatchr   r   r   r   r   r   _DecodeIdsBatchk  s    z&SentencePieceProcessor._DecodeIdsBatchc                 C   s   t | ||S r$   )r   Z-SentencePieceProcessor__DecodeIdsAsBytesBatchr   r   r   r   _DecodeIdsAsBytesBatchn  s    z-SentencePieceProcessor._DecodeIdsAsBytesBatchc                 C   s   t | ||S r$   )r   Z7SentencePieceProcessor__DecodeIdsAsSerializedProtoBatchr   r   r   r    _DecodeIdsAsSerializedProtoBatchq  s    z7SentencePieceProcessor._DecodeIdsAsSerializedProtoBatchc                 C   s   t | ||S r$   )r   Z6SentencePieceProcessor__DecodeIdsAsImmutableProtoBatchr   r   r   r   _DecodeIdsAsImmutableProtoBatcht  s    z6SentencePieceProcessor._DecodeIdsAsImmutableProtoBatchc                 C   s   t | ||S r$   )r   Z)SentencePieceProcessor__DecodePiecesBatchr   r   r   r   _DecodePiecesBatchw  s    z)SentencePieceProcessor._DecodePiecesBatchc                 C   s   t | ||S r$   )r   Z:SentencePieceProcessor__DecodePiecesAsSerializedProtoBatchr   r   r   r   #_DecodePiecesAsSerializedProtoBatchz  s    z:SentencePieceProcessor._DecodePiecesAsSerializedProtoBatchc                 C   s   t | ||S r$   )r   Z9SentencePieceProcessor__DecodePiecesAsImmutableProtoBatchr   r   r   r   "_DecodePiecesAsImmutableProtoBatch}  s    z9SentencePieceProcessor._DecodePiecesAsImmutableProtoBatchc              	   C   s   t | ||||||S r$   )r   Z(SentencePieceProcessor__NBestEncodeAsIdsr   rv   r   r   r   r   r   r   r   r   _NBestEncodeAsIds  s    z(SentencePieceProcessor._NBestEncodeAsIdsc              	   C   s   t | ||||||S r$   )r   Z+SentencePieceProcessor__NBestEncodeAsPiecesr   r   r   r   _NBestEncodeAsPieces  s    z+SentencePieceProcessor._NBestEncodeAsPiecesc              	   C   s   t | ||||||S r$   )r   Z4SentencePieceProcessor__NBestEncodeAsSerializedProtor   r   r   r   _NBestEncodeAsSerializedProto  s    z4SentencePieceProcessor._NBestEncodeAsSerializedProtoc              	   C   s   t | ||||||S r$   )r   Z3SentencePieceProcessor__NBestEncodeAsImmutableProtor   r   r   r   _NBestEncodeAsImmutableProto  s    z3SentencePieceProcessor._NBestEncodeAsImmutableProtoc
           
      C   s   t | |||||||||	
S r$   )r   Z1SentencePieceProcessor__SampleEncodeAndScoreAsIds
r   rv   num_samplesr   worinclude_bestr   r   r   r   r   r   r   _SampleEncodeAndScoreAsIds  s    z1SentencePieceProcessor._SampleEncodeAndScoreAsIdsc
           
      C   s   t | |||||||||	
S r$   )r   Z4SentencePieceProcessor__SampleEncodeAndScoreAsPiecesr   r   r   r   _SampleEncodeAndScoreAsPieces  s    z4SentencePieceProcessor._SampleEncodeAndScoreAsPiecesc
           
      C   s   t | |||||||||	
S r$   )r   Z=SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProtor   r   r   r   &_SampleEncodeAndScoreAsSerializedProto  s    z=SentencePieceProcessor._SampleEncodeAndScoreAsSerializedProtoc
           
      C   s   t | |||||||||	
S r$   )r   Z<SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProtor   r   r   r   %_SampleEncodeAndScoreAsImmutableProto  s    z<SentencePieceProcessor._SampleEncodeAndScoreAsImmutableProtoc                 C   s   t | |S r$   )r   Z!SentencePieceProcessor__Normalizer   rv   r   r   r   
_Normalize  s    z!SentencePieceProcessor._Normalizec                 C   s   t | |S r$   )r   Z,SentencePieceProcessor__NormalizeWithOffsetsr   r   r   r   _NormalizeWithOffsets  s    z,SentencePieceProcessor._NormalizeWithOffsetsc                 C   s   t | ||S r$   )r   Z(SentencePieceProcessor__CalculateEntropy)r   rv   r   r   r   r   _CalculateEntropy  s    z(SentencePieceProcessor._CalculateEntropyc                 C   s   t | |||S r$   )r   Z-SentencePieceProcessor__CalculateEntropyBatch)r   r   r   r   r   r   r   _CalculateEntropyBatch  s    z-SentencePieceProcessor._CalculateEntropyBatchc                 C   s   t | |S r$   )r   Z.SentencePieceProcessor__OverrideNormalizerSpecr   r   r   r   _OverrideNormalizerSpec  s    z.SentencePieceProcessor._OverrideNormalizerSpecNFg?c                 C   sX   t |  || _|| _|| _|| _|| _|| _|	| _|
| _|| _	|sF|rT| j
||d dS )a  Initialzie sentencepieceProcessor.

      Args:
        model_file: The sentencepiece model file path.
        model_proto: The sentencepiece model serialized proto.
        out_type: output type. int or str.
        add_bos: Add <s> to the result (Default = false)
        add_eos: Add </s> to the result (Default = false) <s>/</s> is added after
          reversing (if enabled).
        reverse: Reverses the tokenized sequence (Default = false)
        emit_unk_piece: Emits the unk literal string (Default = false)
        nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout.
                    nbest_size = {0,1}: No sampling is performed.
                    nbest_size > 1: samples from the nbest_size results.
                    nbest_size < 0: assuming that nbest_size is infinite and samples
                      from the all hypothesis (lattice) using
                      forward-filtering-and-backward-sampling algorithm.
        alpha: Soothing parameter for unigram sampling, and dropout probability of
               merge operations for BPE-dropout.
        num_threads: number of threads in batch processing (Default = -1, auto-detected)
      )
model_filemodel_protoN)$_sentencepiece_processor_init_native	_out_type_add_bos_add_eos_reverse_emit_unk_piece_enable_sampling_nbest_size_alpha_num_threadsLoad)r   r   r   out_typer   r   r   r   r   r   r   r   r   r   r   Init  s    "zSentencePieceProcessor.Initc                 C   s  |dkr| j }|dkr| j}|dkr*| j}|dkr8| j}|dkrF| j}|dkrT| j}|dkrb| j}|	dkrp| j}	|
dkr~| j}
|dkr|dks|dks|dks|	dkrt	d|
dkst
|
tk	rt	dt
|tkrp|tkr| ||
|||	||||	S |tkr| ||
|||	||||	S |dks2|dkrL| ||
|||	||||	S |d	krp| ||
|||	||||	S |tkr| ||||	||||S |tkr| ||||	||||S |dks|dkr| ||||	||||S |d	kr| ||||	||||S t	d
|dS )a~  Encode text input to segmented ids or tokens.

        Args:
        input: input string. accepsts list of string.
        out_type: output type. int or str.
        add_bos: Add <s> to the result (Default = false)
        add_eos: Add </s> to the result (Default = false) <s>/</s> is added after
                 reversing (if enabled).
        reverse: Reverses the tokenized sequence (Default = false)
        emit_unk_piece: Emits the unk literal string (Default = false)
        nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout.
                    nbest_size = {0,1}: No sampling is performed.
                    nbest_size > 1: samples from the nbest_size results.
                    nbest_size < 0: assuming that nbest_size is infinite and samples
                    from the all hypothesis (lattice) using
                    forward-filtering-and-backward-sampling algorithm.
        alpha: Soothing parameter for unigram sampling, and merge probability for
               BPE-dropout (probablity 'p' in BPE-dropout paper).
        num_threads: the number of threads used in the batch processing (Default = -1).
      NTr   r   a  When enable_sampling is True, We must specify "nbest_size > 1" or "nbest_size = -1", and "alpha". "nbest_size" is enabled only on unigram mode ignored in BPE-dropout. when "nbest_size = -1" , this method samples from all candidates on the lattice instead of nbest segmentations.num_threads must be intserialized_protor[   immutable_protozunknown out_type={})r   r   r   r   r   r   r   r   r   RuntimeErrorr   intlistr   rO   r   r   r   r   r   r   r   rD   )r   inputr   r   r   r   r   r   r   r   r   r   r   r   Encode  s         
        
    

    

    
    

    zSentencePieceProcessor.Encodec                 K   s   | j f |td|S Nr   r   r   rO   r   r   kwargsr   r   r   EncodeAsPieces5  s    z%SentencePieceProcessor.EncodeAsPiecesc                 K   s   | j f |td|S r   r   r   r   r   r   r   EncodeAsIds9  s    z"SentencePieceProcessor.EncodeAsIdsc                 K   s   | j f |dd|S )Nr   r   r   r   r   r   r   EncodeAsSerializedProto=  s    z.SentencePieceProcessor.EncodeAsSerializedProtoc                 K   s   | j f |dd|S )Nr   r   r   r   r   r   r   EncodeAsImmutableProtoA  s    z-SentencePieceProcessor.EncodeAsImmutableProtoc                 K   s   | j f |||tdd|S NTr   r   r   r   r   r   r   r   r   r   r   r   r   r   SampleEncodeAsPiecesE  s     z+SentencePieceProcessor.SampleEncodeAsPiecesc                 K   s   | j f |||tdd|S r   r   r   r   r   r   SampleEncodeAsIdsJ  s     z(SentencePieceProcessor.SampleEncodeAsIdsc                 K   s   | j f |||ddd|S )Nr   Tr   r   r   r   r   r   SampleEncodeAsSerializedProtoO  s     z4SentencePieceProcessor.SampleEncodeAsSerializedProtoc                 K   s   | j f |||ddd|S )Nr   Tr   r   r   r   r   r   SampleEncodeAsImmutableProtoT  s     z3SentencePieceProcessor.SampleEncodeAsImmutableProtoc                    s   dkrj dkrjdkr*jdkr8jdkrFjdkrTjdkr`dfdd t|tkr fdd|D S  |S )a  NBestEncode text input to segmented ids or tokens.

        Args:
        input: input string. accepsts list of string.
        out_type: output type. int or str.
        add_bos: Add <s> to the result (Default = false)
        add_eos: Add </s> to the result (Default = false) <s>/</s> is added after reversing (if enabled).
        reverse: Reverses the tokenized sequence (Default = false)
        emit_unk_piece: Emits the unk literal string (Default = false)
        nbest_size: nbest size
      Nr   r   c                    s   t kr|  S tkr8|  S dksHdkr\|  S dkrx|  S tdd S )Nr   r[   r   zunknown out_type)r   r   rO   r   r   r   r   rv   )r   r   r   r   r   r   r   r   r   _encode}  s:                z3SentencePieceProcessor.NBestEncode.<locals>._encodec                    s   g | ]} |qS r   r   ra   nr  r   r   rc     s     z6SentencePieceProcessor.NBestEncode.<locals>.<listcomp>)r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   )r  r   r   r   r   r   r   r   r   NBestEncodeY  s$    z"SentencePieceProcessor.NBestEncodec                 K   s   | j f ||td|S Nr   r   r   )r  rO   r   r   r   r   r   r   r   NBestEncodeAsPieces  s
    
z*SentencePieceProcessor.NBestEncodeAsPiecesc                 K   s   | j f ||td|S r  )r  r   r  r   r   r   NBestEncodeAsIds  s
    
z'SentencePieceProcessor.NBestEncodeAsIdsc                 K   s   | j f ||dd|S )Nr   r  r  r  r   r   r   NBestEncodeAsSerializedProto  s
    
z3SentencePieceProcessor.NBestEncodeAsSerializedProtoc                 K   s   | j f ||dd|S )Nr   r  r  r  r   r   r   NBestEncodeAsImmutableProto  s
    
z2SentencePieceProcessor.NBestEncodeAsImmutableProtoc              
      s   dkr	j dkr	jdkr*	jdkr8	jdkrF	jdkrRddkr^d
dkrjd
dkrvddkrtdr
std	
f
dd	 t|tkr҇ fd
d|D S  |S )a   SampleEncodeAndScore text input to segmented ids or tokens.

        Args:
        input: input string. accepsts list of string.
        out_type: output type. int or str or 'serialized_proto' or 'immutable_proto'
        add_bos: Add <s> to the result (Default = false)
        add_eos: Add </s> to the result (Default = false) <s>/</s> is added after reversing (if enabled).
        reverse: Reverses the tokenized sequence (Default = false)
        emit_unk_piece: Emits the unk literal string (Default = false)
        num_samples: How many samples to return (Default = 1)
        alpha: inverse temperature for sampling
        wor: whether to sample without replacement (Default = false)
        include_best: whether to include the best tokenization, requires wor=True (Default = false)
      Nr   g      ?Fr   znum_examples must be positivez8When include_best is True, We must specify "wor = True".c                    s   t kr"| 	 	S tkrD| 	 	S dksTdkrn| 	 	S dkr| 	 	S tdd S )Nr   r[   r   zunknown output type)r   r   rO   r   r   r   r   r   )
r   r   r   r   r   r   r   r   r   r   r   r   r    s:                z<SentencePieceProcessor.SampleEncodeAndScore.<locals>._encodec                    s   g | ]} |qS r   r   r  r  r   r   rc     s     z?SentencePieceProcessor.SampleEncodeAndScore.<locals>.<listcomp>)r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   )r  r   r   r   r   r   r   r   r   r   r   r   SampleEncodeAndScore  s4    z+SentencePieceProcessor.SampleEncodeAndScorec                 K   s   | j f |||td|S Nr   r   r   r   )r  rO   r   r   r   r   r   r   r   r   SampleEncodeAndScoreAsPieces  s
    z3SentencePieceProcessor.SampleEncodeAndScoreAsPiecesc                 K   s   | j f |||td|S r  )r  r   r  r   r   r   SampleEncodeAndScoreAsIds  s
    z0SentencePieceProcessor.SampleEncodeAndScoreAsIdsc                 K   s   | j f |||dd|S )Nr   r  r  r  r   r   r   %SampleEncodeAndScoreAsSerializedProto  s
    z<SentencePieceProcessor.SampleEncodeAndScoreAsSerializedProtoc                 K   s   | j f |||dd|S )Nr   r  r  r  r   r   r   $SampleEncodeAndScoreAsImmutableProto  s
    z;SentencePieceProcessor.SampleEncodeAndScoreAsImmutableProtoc                 C   s  |dkr| j }|dks"t|tk	r*td|s2dS |tkrt|tkrT| |gS t|tkrl| |gS t|tkrt|dkst|d tkr| |S t|d tkr| |S t|d tkrt|d dkst|d d tkr| 	||S t|d d tkr| 
||S |tkrt|tkrB| |gS t|tkr\| |gS t|tkrt|dkst|d tkr| |S t|d tkr| |S t|d tkrt|d dkst|d d tkr| ||S t|d d tkr| 
||S |dkrt|tkr<| |gS t|tkrV| |gS t|tkrt|dkst|d tkr| |S t|d tkr| |S t|d tkrt|d dkst|d d tkr| ||S t|d d tkr| ||S |dkrt|tkr6| |gS t|tkrP| |gS t|tkrt|dks~t|d tkr| |S t|d tkr| |S t|d tkrt|d dkst|d d tkr| ||S t|d d tkr| ||S tddS )zDecode processed id or token sequences.

      Args:
        out_type: output type. str, bytes or 'serialized_proto' or 'immutable_proto' (Default = str)
        num_threads: the number of threads used in the batch processing (Default = -1).
      Nr   r   r   r   r   zunknown output or input type)r   r   r   r   rO   r   r   r   r\   r   r   bytesr   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   Decode  s    


$
 

(
 

(
 

(zSentencePieceProcessor.Decodec                 K   s   | j f ||d|S r   r  r   r   r   r   r   r   r   DecodePiecesg  s    z#SentencePieceProcessor.DecodePiecesc                 K   s   | j f ||d|S r   r  r  r   r   r   	DecodeIdsk  s    z SentencePieceProcessor.DecodeIdsr   c                 K   s   | j f ||d|S r   r  r  r   r   r   DecodePiecesAsSerializedProtoo  s    z4SentencePieceProcessor.DecodePiecesAsSerializedProtoc                 K   s   | j f ||d|S r   r  r  r   r   r   DecodeIdsAsSerializedProtos  s    z1SentencePieceProcessor.DecodeIdsAsSerializedProtor   c                 K   s   | j f ||d|S r   r  r  r   r   r   DecodePiecesAsImmutableProtow  s    z3SentencePieceProcessor.DecodePiecesAsImmutableProtoc                 K   s   | j f ||d|S r   r  r  r   r   r   DecodeIdsAsImmutableProto{  s    z0SentencePieceProcessor.DecodeIdsAsImmutableProtoc                 C   sP   t |tkrD|dkr| j}|dks.t |tk	r6td| |||S | ||S )zCalculate sentence entropyNr   )r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r     s    c                    s4   fdd t |tkr, fdd|D S  |S )Nc                    s   r  | S  | S r$   r   r   r   r   with_offsetsr   r   
_normalize  s    
z4SentencePieceProcessor.Normalize.<locals>._normalizec                    s   g | ]} |qS r   r   rn   r$  r   r   rc     s     z4SentencePieceProcessor.Normalize.<locals>.<listcomp>r   r   r   r   r#  r   r$  r   r#  r   	Normalize  s    z SentencePieceProcessor.Normalizec                 K   s,   i }|  D ]\}}t|||< q| |S r$   )itemsrO   r   )r   r   
new_kwargskeyr   r   r   r   OverrideNormalizerSpec  s    z-SentencePieceProcessor.OverrideNormalizerSpecc                 C   s   |   S r$   r   r;   r   r   r   
piece_size  s    z!SentencePieceProcessor.piece_sizec                 C   s   |   S r$   r.  r;   r   r   r   
vocab_size  s    z!SentencePieceProcessor.vocab_sizec                 C   s   |   S r$   r   r;   r   r   r   __getstate__  s    z#SentencePieceProcessor.__getstate__c                 C   s   |    | | d S r$   r<   r   r   r   r   r   r   __setstate__  s    z#SentencePieceProcessor.__setstate__c                 C   s   |   S r$   r.  r;   r   r   r   r_     s    zSentencePieceProcessor.__len__c                 C   s
   |  |S r$   )r   r   r   r   r   rj     s    z"SentencePieceProcessor.__getitem__c                 C   s(   |r|rt d|r| |S | |S )a  Overwride SentencePieceProcessor.Load to support both model_file and model_proto.

      Args:
        model_file: The sentencepiece model file path.
        model_proto: The sentencepiece model serialized proto. Either `model_file`
          or `model_proto` must be set.
      z-model_file and model_proto must be exclusive.)r   r   r   )r   r   r   r   r   r   r     s
    
zSentencePieceProcessor.Load)	NNNNNNNNN)NN)NN)NN)NN)NNNNNN)N)N)N)N)	NNNNNNNNN)NN)NN)NN)NN)r   )r   )r   )r   )N)N)NN)ir   r   r-   r   r   r   r   r<   r   Zdelete_SentencePieceProcessorrQ   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r	  r
  r  r  r  r  r  r  r  rO   r  r  r  r  r  r  r   r)  r-  r/  r0  r2  r5  r_   rj   r   r   r   r   r   r~      s  
2         
^



      
:



         
M



_






r~   c                 C   s
   t | S r$   )r   SetRandomGeneratorSeed)seedr   r   r   r6    s    r6  c                 C   s
   t | S r$   )r   SetMinLogLevel)r7   r   r   r   r8    s    r8  c                   @   s   e Zd Zedd dd ddZdd ZeZedd	 Z	ed
d Z
edd Zedd Zedd ZedddZedddZdS )SentencePieceTrainerc                 C   s
   | j  S r$   r1   r2   r   r   r   r4     r5   zSentencePieceTrainer.<lambda>c                 C   s   | j |S r$   r1   r6   r   r   r   r4     r5   r8   r9   c                 O   s   t dd S )NzNo constructor defined)r   )r   r   r   r   r   r   r<     s    zSentencePieceTrainer.__init__c                 C   s
   t | S r$   )r   Z%SentencePieceTrainer__TrainFromString)r   r   r   r   _TrainFromString  s    z%SentencePieceTrainer._TrainFromStringc                 C   s
   t | S r$   )r   Z"SentencePieceTrainer__TrainFromMapr   r   r   r   _TrainFromMap  s    z"SentencePieceTrainer._TrainFromMapc                 C   s   t | |S r$   )r   Z#SentencePieceTrainer__TrainFromMap2r   iterr   r   r   _TrainFromMap2  s    z#SentencePieceTrainer._TrainFromMap2c                 C   s
   t | S r$   )r   Z#SentencePieceTrainer__TrainFromMap3r;  r   r   r   _TrainFromMap3  s    z#SentencePieceTrainer._TrainFromMap3c                 C   s   t | |S r$   )r   Z#SentencePieceTrainer__TrainFromMap4r=  r   r   r   _TrainFromMap4  s    z#SentencePieceTrainer._TrainFromMap4Nc           	      K   s   | dk	rt | tkrt| S dd }d}d}i }| D ]0\}}|dkrP|}q:|dkr^|}q:||||< q:|r|rt||}n
t|}|| n|rt||S t	|S dS )zDTrain Sentencepiece model. Accept both kwargs and legacy string arg.Nc                 S   s^   t | tkrRtjd dkr"t }nt }tj|dd}|dd | D  |	 S t
| S dS )zEncode value to CSV..r      r   )lineterminatorc                 S   s   g | ]}t |qS r   )rO   )ra   r7   r   r   r   rc     s     z@SentencePieceTrainer._Train.<locals>._encode.<locals>.<listcomp>N)r   r   sysr   StringIOBytesIOcsvwriterwriterowgetvaluerO   )r   frH  r   r   r   r    s    z,SentencePieceTrainer._Train.<locals>._encode)sentence_iteratorZsentence_reader)model_writer)
r   rO   r9  r:  r*  rA  r@  writer?  r<  )	r   r   r  rL  rM  r+  r,  r   r   r   r   r   _Train  s.    


zSentencePieceTrainer._Trainc              	   K   s.   t |d tjf d| i| W 5 Q R X d S )N)ostreamr   )
_LogStreamr9  rO  )r   Z	logstreamr   r   r   r   Train  s    zSentencePieceTrainer.Train)N)NN)r   r   r-   r   r   r<   r   r   staticmethodr:  r<  r?  r@  rA  rO  rR  r   r   r   r   r9    s"   




,r9  c                   @   s   e Zd Zedd dd ddZeZdd Zej	Z
dd	 Zd
d Zdd Zdd Zdd Zdd Zdd Zdd Zd"ddZd#ddZdd Zd d! ZdS )$SentencePieceNormalizerc                 C   s
   | j  S r$   r1   r2   r   r   r   r4     r5   z SentencePieceNormalizer.<lambda>c                 C   s   | j |S r$   r1   r6   r   r   r   r4     r5   r8   r9   c                 C   s   t | t   d S r$   )r   Z SentencePieceNormalizer_swiginitZnew_SentencePieceNormalizerr;   r   r   r   r<      s    z SentencePieceNormalizer.__init__c                 C   s   t | |S r$   )r   Z/SentencePieceNormalizer_LoadFromSerializedProtor   r   r   r   r   $  s    z/SentencePieceNormalizer.LoadFromSerializedProtoc                 C   s   t | |S r$   )r   Z'SentencePieceNormalizer_LoadFromRuleTSV)r   r   r   r   r   LoadFromRuleTSV'  s    z'SentencePieceNormalizer.LoadFromRuleTSVc                 C   s   t | |S r$   )r   Z(SentencePieceNormalizer_LoadFromRuleName)r   r   r   r   r   LoadFromRuleName*  s    z(SentencePieceNormalizer.LoadFromRuleNamec                 C   s
   t | S r$   )r   Z.SentencePieceNormalizer_serialized_model_protor;   r   r   r   r   -  s    z.SentencePieceNormalizer.serialized_model_protoc                 C   s   t | |S r$   )r   Z$SentencePieceNormalizer_LoadFromFiler   r   r   r   r   0  s    z$SentencePieceNormalizer.LoadFromFilec                 C   s   t | |S r$   )r   Z"SentencePieceNormalizer__Normalizer   r   r   r   r   3  s    z"SentencePieceNormalizer._Normalizec                 C   s   t | |S r$   )r   Z-SentencePieceNormalizer__NormalizeWithOffsetsr   r   r   r   r   6  s    z-SentencePieceNormalizer._NormalizeWithOffsetsc                 C   s   t | ||S r$   )r   Z&SentencePieceNormalizer__SetProtoFieldr   r   r   r   _SetProtoField9  s    z&SentencePieceNormalizer._SetProtoFieldNFc           	      C   s|   t |  |r| |}n8|r(| |}n(|r8| |}n|rH| |}ntd|rx| d| | d| | d| dS )a  Initialzie sentencePieceNormalizer.

      Args:
        model_file: The sentencepiece model file path.
        model_proto: The sentencepiece model serialized proto.
        rule_tsv: The normalization rule file in TSV format.
        rule_name: Pre-defined normalization name.
        add_dummy_prefix: add dummy prefix.
        escape_whitespaces: escape whitespaces.
        remove_extra_whitespaces: remove extra whitespaces.
      zno model is specifiedadd_dummy_prefixescape_whitespacesremove_extra_whitespacesN)%_sentencepiece_normalizer_init_nativer   r   rU  rV  r   rW  )	r   r   r   Zrule_tsvZ	rule_namerX  rY  rZ  statusr   r   r   r   <  s    zSentencePieceNormalizer.Initc                    s4   fdd t |tkr, fdd|D S  |S )Nc                    s   r  | S  | S r$   r!  r   r"  r   r   r$  c  s    
z5SentencePieceNormalizer.Normalize.<locals>._normalizec                    s   g | ]} |qS r   r   rn   r%  r   r   rc   i  s     z5SentencePieceNormalizer.Normalize.<locals>.<listcomp>r&  r'  r   r(  r   r)  b  s    z!SentencePieceNormalizer.Normalizec                 C   s   |   S r$   r1  r;   r   r   r   r2  m  s    z$SentencePieceNormalizer.__getstate__c                 C   s   |    | | d S r$   r3  r4  r   r   r   r5  q  s    z$SentencePieceNormalizer.__setstate__)NNNNFFF)N)r   r   r-   r   r   r   r   r<   r   Zdelete_SentencePieceNormalizerrQ   r   rU  rV  r   r   r   r   rW  r   r)  r2  r5  r   r   r   r   rT    s.          
&
rT  )rE  )rF  c                 C   sh   i }| j  D ]6\}}td|rtdd| dd}|||< q| D ]\}}t| || qNdS )z1Added snake_cased method from CammelCased method.z^[A-Z]+z(?<!^)(?=[A-Z])_Zn_bestZnbestN)r&   r*  rematchsublowerreplacesetattr)	classnameZ	snake_mapkr7   Zsnaker   r   r   _add_snake_case  s     
rf  c                    s4   t | |dfdd  fdd}t| || dS )z4Enables batch request for the method classname.name.Nc                    s2   t |tkr(|dk s ||  kr(td | |S )Nr   zpiece id is out of range.)r   r   r/  ri   )r7   r  )funcr   r   _func  s     z_batchnize.<locals>._funcc                    s.   t |tkr  fdd|D S  |S d S )Nc                    s   g | ]} |qS r   r   r  )rh  r   r   r   rc     s     z5_batchnize.<locals>._batched_func.<locals>.<listcomp>r&  r   )rh  r;   r   _batched_func  s    z!_batchnize.<locals>._batched_func)r   rc  )rd  r   ri  r   )rh  rg  r   
_batchnize  s    rj  r<   )r   r   r   r   r   r   r   )__version__c                   @   s&   e Zd ZdddZdd Zdd ZdS )	rQ  Nc                 C   s    || _ | j d k	rtj | _d S r$   )rP  rD  stderrfilenoorig_stream_fileno)r   rP  r   r   r   r<     s    
z_LogStream.__init__c                 C   s0   | j d k	r,t| j| _t| j  | j d S r$   )rP  osduprn  orig_stream_dupdup2rm  r;   r   r   r   	__enter__  s    
z_LogStream.__enter__c                 C   s@   | j d k	r<t| j t| j| j t| j | j   d S r$   )rP  ro  closern  rr  rq  )r   r   r   	tracebackr   r   r   __exit__  s
    
z_LogStream.__exit__)N)r   r   r-   r<   rs  rv  r   r   r   r   rQ    s   
rQ  )6rD  r   Z_swig_python_version_info__package__r   r   r   builtinsr	   ImportErrorr   r    r#   r+   r   r,   objectr0   Z>ImmutableSentencePieceText_ImmutableSentencePiece_swigregisterrR   Z'ImmutableSentencePieceText_swigregisterrx   Z,ImmutableNBestSentencePieceText_swigregisterr~   Z#SentencePieceProcessor_swigregisterr6  r8  r9  Z!SentencePieceTrainer_swigregisterrT  Z$SentencePieceNormalizer_swigregisterr^  rG  ro  iorE  rF  rf  rj  r<   r   r[  rc  r   r   ZTokenizer  Z
DetokenizemZset_random_generator_seedZset_min_log_level_versionrk  rQ  r   r   r   r   <module>   sl   	7
H
8
     S
O
[
