o
    hR:                     @   s  d dl mZ d dlmZmZ dd Zdd Zdd Zd	d
 Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd ZdQddZdd Zdd Zeeeeee	e
eeeeeeed ZG d!d" d"Zeg d#d$gd%d&eg d'd(gd)d*gd&ed+d,gd-d.gd/d&eg d0d1gd2d&eg d3d$d(gd4d&ed5d6gd1gd7d&ed8gd-gd9d&eg d:d(d.gd;d&eg d<d$gd=d&eg d>d.d?id@d&edAgd1gdBd&gZdCdD ZdRdGdHZdSdIefdJdKZdSdLdMZdTdOdPZdES )U   )BASE_PYTHON_TOOLS)InterpreterErrorevaluatec                 C   s   d|  d| dS )NzThis is the classification of z along . )textlabelsr   r   v/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/transformers/agents/evaluate_agent.py
classifier      r
   c                 C   s   d|  d| d| dS )NzThis is the translation of  from z to r   r   )r   src_langtgt_langr   r   r	   
translator   s   r   c                 C      d|  dS )Nz!This is actually a sound reading r   r   r   r   r   r	   speaker      r   c                 C   $   d| vrt d|  dd|  dS )Nsoundz	`audio` (z) is not a sound.z"This is the transcribed text from r   
ValueError)audior   r   r	   transcriber"      r   c                 C   r   )Nz'This is actually an image representing r   r   )promptr   r   r	   image_generator(   r   r   c                 C   r   )Nimage	`image` () is not an image.zThis is a description of r   r   )r   r   r   r	   image_captioner,   r   r    c                 C   s*   d| vrt d|  dd|  d| dS )Nr   r   r   zThis is a transformation of z according to r   r   r   r   r   r   r	   image_transformer2      r"   c                 C      d| d|  dS )NThis is the answer to r   r   r   )r   questionr   r   r	   question_answerer8   r   r'   c                 C   s*   d| vrt d|  dd| d|  dS )Nr   r   r   r%   r   r   r   r   r&   r   r   r	   image_qa<   r#   r)   c                 C   r   )NzThis is the content of r   r   )urlr   r   r	   text_downloaderB   r   r+   c                 C   r   )NzThis is a summary of r   r   r   r   r   r	   
summarizerF   r   r,      c                 C   s
   d|  S )NzA video of r   )r   secondsr   r   r	   video_generatorJ   s   
r/   c                 C   r$   )Nr%   z from the document r   r   r(   r   r   r	   document_qaN   r   r0   c                 C   s   d| d|  S )NzThis is the mask of z in r   r!   r   r   r	   image_segmenterR   s   r1   )text_classifierr   text_readerr,   r   r   r    r"   text_qar+   r)   r/   r0   r1   c                   @   s   e Zd ZdZdd ZdS )Problema#  
    A class regrouping all the information to solve a problem on which we will evaluate agents.

    Args:
        task (`str` ou `list[str]`):
            One or several descriptions of the task to perform. If a list, it should contain variations on the
            phrasing, but for the same task.
        inputs (`list[str]` or `dict[str, str]`):
            The inputs that will be fed to the tools. For this testing environment, only strings are accepted as
            values. Pass along a dictionary when you want to specify the values of each inputs, or just the list of
            inputs expected (the value used will be `<<input_name>>` in this case).
        answer (`str` or `list[str]`):
            The theoretical answer (or list of possible valid answers) to the problem, as code.
    c                 C   s   || _ || _|| _d S Ntaskinputsanswer)selfr8   r9   r:   r   r   r	   __init__x   s   
zProblem.__init__N)__name__
__module____qualname____doc__r<   r   r   r   r	   r5   h   s    r5   )z:Is the following `text` (in Spanish) positive or negative?zEIs the text in the variable `text` (in Spanish) positive or negative?z`Translate the following `text` from Spanish to English then tell me if its positive or negative.r   zjtext_classifier(translator(text, src_lang="Spanish", tgt_lang="English"), labels=["positive", "negative"])r7   )z+Tell me out loud what the `image` contains.z(Describe the following `image` out loud.zDFind what is in the picture stored in `image` then read it out loud.r   z#text_reader(image_captioner(image))z>text_reader(image_qa(image, question='What is in the image?'))zkGenerate an image from the text given in `text_input`. Then transform it according to the text in `prompt`.zeUse the following `text_input` to generate an image, then transform it by using the text in `prompt`.
text_inputr   z6image_transformer(image_generator(text_input), prompt))zTDownload the content of `url`, summarize it then generate an image from its content.z<Use a summary of the web page at `url` to generate an image.zXSummarize the content of the web page at `url`, and use the result to generate an image.r*   z1image_generator(summarizer(text_downloader(url))))zUTransform the following `image` using the prompt in `text`. The prompt is in Spanish.zNUse the text prompt in `text` (in Spanish) to transform the following `image`.z]Translate the `text` from Spanish to English then use it to transform the picture in `image`.zRimage_transformer(image, translator(text, src_lang='Spanish', tgt_lang='English'))zHDownload the content of `url`, summarize it then read it out loud to me.z+Read me a summary of the web page at `url`.z-text_reader(summarizer(text_downloader(url)))z6Generate an image from the text given in `text_input`.zimage_generator(text_input))z2Replace the beaver in the `image` by the `prompt`.z7Transform the `image` so that it contains the `prompt`.z'Use `prompt` to transform this `image`.z image_transformer(image, prompt))zmProvide me the summary of the `text`, then read it to me before transcribing it and translating it in French.zXSummarize `text`, read it out loud then transcribe the audio and translate it in French.zURead me a summary of the `text` out loud. Transcribe this and translate it in French.z]translator(transcriber(text_reader(summarizer(text))), src_lang='English', tgt_lang='French'))z Generate a video of the `prompt`zAnimate a `prompt`z%Make me a short video using `prompt`.zA lobster swimmingz%video_generator('A lobster swimming')zDownload the following file `url`, summarize it in a few words and generate a video from it.Fetch the file at this `url`, summarize it, and create an animation out of it.z1video_generator(summarizer(text_downloader(url)))c                    s   t |ts fddtD S t | tr.t| D ]\}||  v r-fddtD   S qt| D ]\}| |krFfddtD   S q3 fddtD S )Nc                       h | ]}| v r|qS r   r   .0namecode_answerr   r	   	<setcomp>       z(get_theoretical_tools.<locals>.<setcomp>c                    rB   r   r   rC   one_coder   r	   rH      rI   c                    rB   r   r   rC   rJ   r   r	   rH      rI   c                    s   h | ]
}| d  v r|qS )    r   rC   rF   r   r	   rH      s    )
isinstancelist
TEST_TOOLSdictzipvalues)agent_answertheoretical_answerrG   
one_answerr   )rG   rK   r	   get_theoretical_tools   s   

rV   NFc           	   
   C   s   t  }t D ]\}}|| vrq|||< qt|tr | }n|d ur+dd |D }|d ur5|| n|}zt| ||W S  tyT } z
t	|W  Y d }~S d }~w t
yl } z|rat| W Y d }~d S d }~ww )Nc                 S   s   i | ]	}|d | dqS )z<<z>>r   )rD   inpr   r   r	   
<dictcomp>       z!evaluate_code.<locals>.<dictcomp>)r   copyrO   itemsrM   rP   updater   r   str	Exceptionprint)	coder9   stateverbosereturn_interpretor_errortoolsrE   tooler   r   r	   evaluate_code   s.   


rg   rb   c                    s~   |rt |   t tr n g |  v r|rt d dS t| tr7t fdd|  D r7|r5t d dS |r=t d dS )	NzPerfect!r   c                 3   s    | ]}| v V  qd S r6   r   )rD   vrT   r   r	   	<genexpr>  s    zscore_code.<locals>.<genexpr>z Almost perfect, result in state!g      ?z.Result is not the right one but code executed.g333333?)r_   rM   rN   rP   anyrR   )rS   rT   rb   r   ri   r	   
score_code  s   
$rl   c                    s   fddt D }t|||}||krd}d }nt|| }	t|| }
tddd|	  d|
  }||d} fddt D }||krHd}d }nt|| }	t|| }
tddd|	  d|
  }||d}t|||d}|dk rx ||d	}nd }|||f|||ffS )
Nc                    s    h | ]}d | d  v r|qS )`r   rC   r`   r   r	   rH     s     z&evaluate_one_result.<locals>.<setcomp>g      ?rL   g      ?)selected_toolstheoretical_toolsc                    rB   r   r   rC   rn   r   r	   rH   .  rI   rb   )code_produced
evaluationrT   )rO   rV   lenmaxrl   )r`   rS   rT   r:   rb   tools_in_coderp   tool_selection_scoretool_selection_errorsmissing_toolsunexpected_toolstool_used_scoretool_used_errorsscorecode_errorsr   rn   r	   evaluate_one_result  s:   r      c              	      s|  t  j }|t tkr$t t| }t |t }td| d| dg }g }ttD ]'\}	}
t|
jt	rH|
|
j |
|	gt|
j  q,||
j ||	 q,d}d}d}|rbi }i }i }tdt||D ]}t|| t|}||| } fdd|D }t|D ]\}	}t|||	   |rtd||	  d||	  d	  j|d
d}t|j|d}tjt	rŇfddjD }ntjj}t|||j|d\}}||d 7 }||d 7 }||d 7 }|r|d dur|d |||	 < |d dur
|d |||	 < |d dur|d |||	 < qqjd|t|  d|t|  d|t|  d}|r<||||fS |S )z
    Evaluates a new agent on all `EVALUATION_TASKS`.

    Example:

    ```py
    agent = NewOpenAiAgent(model="text-davinci-003", api_key=your_api_key)
    bads = new_evaluate_agent(agent)
    for bad in bads:
        print(bad)
    ```
    z@Fix the test tools in the evaluate_agent module. Tools missing: z. Extra tools: r   rL   c                    s   g | ]	} j |d dqS )T)return_generated_code)run)rD   r8   )agentr   r	   
<listcomp>v  rY   z"evaluate_agent.<locals>.<listcomp>z	====Task z====

zAnswer:)split_tokenrq   c                    s   g | ]}t | jqS r   )rg   r9   )rD   r:   )problemr   r	   r     rI   r   r-   Nd   )ztool selection scoreztool used scorez
code score)settoolboxkeysrO   r   	enumerateEVALUATION_TASKSrM   r8   rN   extendrt   appendrangeminr_   extract_actionrg   r9   r:   r   )r   
batch_sizerb   return_errorsagent_toolsry   rz   
eval_taskseval_idxidxpbrw   r{   
code_scorerx   r|   r~   	start_idxend_idxbatch_tasksresultsresultr`   rS   rT   scoreserrorsr   )r   r   r	   evaluate_agentI  sr   
r   )r-   )NNFF)F)r   FF)agentsr   python_interpreterr   r   r
   r   r   r   r   r    r"   r'   r)   r+   r,   r/   r0   r1   rO   r5   r   rV   rg   boolrl   r   r   r   r   r   r	   <module>   s   
					a

+