U
    diQ:                     @   s  d dl mZ d dlmZmZ dd Zdd Zdd Zd	d
 Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zd_ddZdd Zdd Zeeeeee	e
eeeeeeed ZG d!d" d"Zed#d$d%gd&gd'd(ed)d*d+gd,gd-d.gd(ed/d0gd1d2gd3d(ed4d5d6gd7gd8d(ed9d:d;gd&d,gd<d(ed=d>gd7gd?d(ed@gd1gdAd(edBdCdDgd,d2gdEd(edFdGdHgd&gdId(edJdKdLgd2dMidNd(edOgd7gdPd(gZdQdR Zd`dUdVZdaedWdXdYZdbdZd[Zdcd]d^ZdSS )d   )BASE_PYTHON_TOOLS)InterpreterErrorevaluatec                 C   s   d|  d| dS )NzThis is the classification of z along . )textlabelsr   r   F/tmp/pip-unpacked-wheel-bm_b0l5e/transformers/agents/evaluate_agent.py
classifier   s    r
   c                 C   s   d|  d| d| dS )NzThis is the translation of  from z to r   r   )r   Zsrc_langZtgt_langr   r   r	   
translator   s    r   c                 C   s   d|  dS )Nz!This is actually a sound reading r   r   r   r   r   r	   speaker   s    r   c                 C   s$   d| krt d|  dd|  dS )NZsoundz	`audio` (z) is not a sound.z"This is the transcribed text from r   
ValueError)audior   r   r	   transcriber"   s    r   c                 C   s   d|  dS )Nz'This is actually an image representing r   r   )promptr   r   r	   image_generator(   s    r   c                 C   s$   d| krt d|  dd|  dS )Nimage	`image` () is not an image.zThis is a description of r   r   )r   r   r   r	   image_captioner,   s    r   c                 C   s*   d| krt d|  dd|  d| dS )Nr   r   r   zThis is a transformation of z according to r   r   r   r   r   r   r	   image_transformer2   s    r   c                 C   s   d| d|  dS )NThis is the answer to r   r   r   )r   questionr   r   r	   question_answerer8   s    r   c                 C   s*   d| krt d|  dd| d|  dS )Nr   r   r   r   r   r   r   r   r   r   r   r	   image_qa<   s    r   c                 C   s   d|  dS )NzThis is the content of r   r   )urlr   r   r	   text_downloaderB   s    r!   c                 C   s   d|  dS )NzThis is a summary of r   r   r   r   r   r	   
summarizerF   s    r"      c                 C   s
   d|  S )NzA video of r   )r   secondsr   r   r	   video_generatorJ   s    r%   c                 C   s   d| d|  dS )Nr   z from the document r   r   r   r   r   r	   document_qaN   s    r&   c                 C   s   d| d|  S )NzThis is the mask of z in r   r   r   r   r	   image_segmenterR   s    r'   )Ztext_classifierr   Ztext_readerr"   r   r   r   r   Ztext_qar!   r   r%   r&   r'   c                   @   s   e Zd ZdZdd ZdS )Problema#  
    A class regrouping all the information to solve a problem on which we will evaluate agents.

    Args:
        task (`str` ou `list[str]`):
            One or several descriptions of the task to perform. If a list, it should contain variations on the
            phrasing, but for the same task.
        inputs (`list[str]` or `dict[str, str]`):
            The inputs that will be fed to the tools. For this testing environment, only strings are accepted as
            values. Pass along a dictionary when you want to specify the values of each inputs, or just the list of
            inputs expected (the value used will be `<<input_name>>` in this case).
        answer (`str` or `list[str]`):
            The theoretical answer (or list of possible valid answers) to the problem, as code.
    c                 C   s   || _ || _|| _d S Ntaskinputsanswer)selfr+   r,   r-   r   r   r	   __init__x   s    zProblem.__init__N)__name__
__module____qualname____doc__r/   r   r   r   r	   r(   h   s   r(   z:Is the following `text` (in Spanish) positive or negative?zEIs the text in the variable `text` (in Spanish) positive or negative?z`Translate the following `text` from Spanish to English then tell me if its positive or negative.r   zjtext_classifier(translator(text, src_lang="Spanish", tgt_lang="English"), labels=["positive", "negative"])r*   z+Tell me out loud what the `image` contains.z(Describe the following `image` out loud.zDFind what is in the picture stored in `image` then read it out loud.r   z#text_reader(image_captioner(image))z>text_reader(image_qa(image, question='What is in the image?'))zkGenerate an image from the text given in `text_input`. Then transform it according to the text in `prompt`.zeUse the following `text_input` to generate an image, then transform it by using the text in `prompt`.Z
text_inputr   z6image_transformer(image_generator(text_input), prompt)zTDownload the content of `url`, summarize it then generate an image from its content.z<Use a summary of the web page at `url` to generate an image.zXSummarize the content of the web page at `url`, and use the result to generate an image.r    z1image_generator(summarizer(text_downloader(url)))zUTransform the following `image` using the prompt in `text`. The prompt is in Spanish.zNUse the text prompt in `text` (in Spanish) to transform the following `image`.z]Translate the `text` from Spanish to English then use it to transform the picture in `image`.zRimage_transformer(image, translator(text, src_lang='Spanish', tgt_lang='English'))zHDownload the content of `url`, summarize it then read it out loud to me.z+Read me a summary of the web page at `url`.z-text_reader(summarizer(text_downloader(url)))z6Generate an image from the text given in `text_input`.zimage_generator(text_input)z2Replace the beaver in the `image` by the `prompt`.z7Transform the `image` so that it contains the `prompt`.z'Use `prompt` to transform this `image`.z image_transformer(image, prompt)zmProvide me the summary of the `text`, then read it to me before transcribing it and translating it in French.zXSummarize `text`, read it out loud then transcribe the audio and translate it in French.zURead me a summary of the `text` out loud. Transcribe this and translate it in French.z]translator(transcriber(text_reader(summarizer(text))), src_lang='English', tgt_lang='French')z Generate a video of the `prompt`zAnimate a `prompt`z%Make me a short video using `prompt`.zA lobster swimmingz%video_generator('A lobster swimming')zDownload the following file `url`, summarize it in a few words and generate a video from it.Fetch the file at this `url`, summarize it, and create an animation out of it.z1video_generator(summarizer(text_downloader(url)))c                    s   t |ts fddtD S t | tr\t| D ]*\}||  kr0fddtD   S q0t| D ]&\}| |krffddtD   S qf fddtD S )Nc                    s   h | ]}| kr|qS r   r   .0namecode_answerr   r	   	<setcomp>   s      z(get_theoretical_tools.<locals>.<setcomp>c                    s   h | ]}| kr|qS r   r   r4   one_coder   r	   r9      s      c                    s   h | ]}| kr|qS r   r   r4   r:   r   r	   r9      s      c                    s   h | ]}| d  kr|qS )    r   r4   r7   r   r	   r9      s      )
isinstancelist
TEST_TOOLSdictzipvalues)agent_answertheoretical_answerr8   Z
one_answerr   )r8   r;   r	   get_theoretical_tools   s    

rE   NFc           	   
   C   s   t  }t D ]\}}|| kr"q|||< qt|tr@| }n|d k	rVdd |D }|d k	rj|| n|}zt| ||W S  tk
r } zt	| W Y S d }~X Y n4 t
k
r } z|rt| W Y d S d }~X Y nX d S )Nc                 S   s   i | ]}|d | dqS )z<<z>>r   )r5   inpr   r   r	   
<dictcomp>   s      z!evaluate_code.<locals>.<dictcomp>)r   copyr?   itemsr=   r@   updater   r   str	Exceptionprint)	coder,   stateverboseZreturn_interpretor_errorZtoolsr6   Ztooler   r   r	   evaluate_code   s(    


rR   rP   c                    s   |rt |   t tr n g |  kr:|r6t d dS t| trnt fdd|  D rn|rjt d dS |rzt d dS d S )	NzPerfect!r   c                 3   s   | ]}| kV  qd S r)   r   )r5   vrD   r   r	   	<genexpr>  s     zscore_code.<locals>.<genexpr>z Almsot perfect, result in state!g      ?z.Result is not the right one but code executed.g333333?)rM   r=   r>   r@   anyrB   )rC   rD   rP   r   rU   r	   
score_code  s    
$rX   c                    s   fddt D }t|||}||kr0d}d }n<t|| }	t|| }
tddd|	  d|
  }||d} fddt D }||krd}d }n<t|| }	t|| }
tddd|	  d|
  }||d}t|||d}|dk r ||d	}nd }|||f|||ffS )
Nc                    s    h | ]}d | d  kr|qS )`r   r4   rN   r   r	   r9     s      z&evaluate_one_result.<locals>.<setcomp>g      ?r<   g      ?)Zselected_toolstheoretical_toolsc                    s   h | ]}| kr|qS r   r   r4   rZ   r   r	   r9   .  s      rS   )Zcode_producedZ
evaluationrD   )r?   rE   lenmaxrX   )rN   rC   rD   r-   rP   Ztools_in_coder[   tool_selection_scoretool_selection_errorsmissing_toolsunexpected_toolstool_used_scoretool_used_errorsZscorecode_errorsr   rZ   r	   evaluate_one_result  s:    re      c              	      s  t  j }|t tkrHt t| }t |t }td| d| dg }g }ttD ]N\}	}
t|
jt	r|
|
j |
|	gt|
j  qX||
j ||	 qXd}d}d}|ri }i }i }tdt||D ]j}t|| t|}||| } fdd|D }t|D ]*\}	}t|||	   |rPtd||	  d||	  d	  j|d
d}t|j|d}tjt	rfddjD }ntjj}t|||j|d\}}||d 7 }||d 7 }||d 7 }|r|d dk	r |d |||	 < |d dk	r|d |||	 < |d dk	r|d |||	 < qqd|t|  d|t|  d|t|  d}|r||||fS |S dS )z
    Evaluates a new agent on all `EVALUATION_TASKS`.

    Example:

    ```py
    agent = NewOpenAiAgent(model="text-davinci-003", api_key=your_api_key)
    bads = new_evaluate_agent(agent)
    for bad in bads:
        print(bad)
    ```
    z?Fix the test tools in the evaluate_agent module. Tools mising: z. Extra tools: r   r<   c                    s   g | ]} j |d dqS )T)Zreturn_generated_code)run)r5   r+   )agentr   r	   
<listcomp>v  s     z"evaluate_agent.<locals>.<listcomp>z	====Task z====

zAnswer:)Zsplit_tokenrS   c                    s   g | ]}t | jqS r   )rR   r,   )r5   r-   )problemr   r	   ri     s     r   r#   Nd   )ztool selection scoreztool used scorez
code score)setZtoolboxkeysr?   r   	enumerateEVALUATION_TASKSr=   r+   r>   extendr\   appendrangeminrM   Zextract_actionrR   r,   r-   re   )rh   Z
batch_sizerP   Zreturn_errorsZagent_toolsr`   ra   Z
eval_tasksZeval_idxidxZpbr^   rb   Z
code_scorer_   rc   rd   Z	start_idxZend_idxZbatch_tasksresultsresultrN   rC   rD   Zscoreserrorsr   )rh   rk   r	   evaluate_agentI  sv        
ry   )r#   )NNFF)F)F)rf   FF)Zagentsr   Zpython_interpreterr   r   r
   r   r   r   r   r   r   r   r   r!   r"   r%   r&   r'   r?   r(   rp   rE   rR   boolrX   re   ry   r   r   r   r	   <module>   s   
					a

+