U diO@s2ddlZddlZddlZddlZddlZddlmZmZmZddl m Z ddl m Z ddl m Z ddlmZddlmZmZmZmZmZmZmZmZmZmZmZddlZedeZed eZd d Ze ee!gefd d dZ"ddej#ej#ddee!ee!fe!eegefe$ej%dddZ&GdddeZ'dS)N)ArgumentDefaultsHelpFormatterArgumentParserArgumentTypeError)copy)Enum)isclassPath) AnyCallableDictIterableListLiteralNewTypeOptionalTupleUnionget_type_hints DataClass DataClassTypecCsBt|tr|S|dkrdS|dkr.dStd|ddS)N)yestruety1T)nofalsefn0FzTruthy value expected: got zJ but expected one of yes/no, true/false, t/f, y/n, 1/0 (case insensitive).) isinstanceboollowerr)vr%=/tmp/pip-unpacked-wheel-bm_b0l5e/transformers/hf_argparser.pystring_to_bool#s    r')choicesreturncsdd|DfddS)aN Creates a mapping function from each choices string representation to the actual value. Used to support multiple value types for a single argument. Args: choices (list): List of choices. Returns: Callable[[str], Any]: Mapping function from string representation to actual value for each choice. cSsi|]}t||qSr%)str).0choicer%r%r& ;sz-make_choice_type_function..cs ||S)N)get)argZ str_to_choicer%r&<z+make_choice_type_function..r%)r(r%r0r&make_choice_type_function0s r3)aliaseshelpdefaultdefault_factorymetadata)r4r5r6r7r8r)cKsD|dkr i}|dk r||d<|dk r,||d<tjf|||d|S)aArgument helper enabling a concise syntax to create dataclass fields for parsing with `HfArgumentParser`. Example comparing the use of `HfArg` and `dataclasses.field`: ``` @dataclass class Args: regular_arg: str = dataclasses.field(default="Huggingface", metadata={"aliases": ["--example", "-e"], "help": "This syntax could be better!"}) hf_arg: str = HfArg(default="Huggingface", aliases=["--example", "-e"], help="What a nice syntax!") ``` Args: aliases (Union[str, List[str]], optional): Single string or list of strings of aliases to pass on to argparse, e.g. `aliases=["--example", "-e"]`. Defaults to None. help (str, optional): Help string to pass on to argparse that can be displayed with --help. Defaults to None. default (Any, optional): Default value for the argument. If not default or default_factory is specified, the argument is required. Defaults to dataclasses.MISSING. default_factory (Callable[[], Any], optional): The default_factory is a 0-argument function called to initialize a field's value. It is useful to provide default values for mutable types, e.g. lists: `default_factory=list`. Mutually exclusive with `default=`. Defaults to dataclasses.MISSING. metadata (dict, optional): Further metadata to pass on to `dataclasses.field`. Defaults to None. Returns: Field: A `dataclasses.Field` with the desired properties. Nr4r5)r8r6r7) dataclassesfield)r4r5r6r7r8kwargsr%r%r&HfArg?s$r<cseZdZUdZeeed<eeeefdfdd Ze e e j dddZ ed d d ZdeedfdddZdeeefeeedfdddZdeeejfeeedfdddZdeeejfeeedfdddZZS) HfArgumentParsera This subclass of `argparse.ArgumentParser` uses type hints on dataclasses to generate arguments. The class is designed to play well with the native argparse. In particular, you can add more (non-dataclass backed) arguments to the parser after initialization and you'll get the output back after parsing as an additional namespace. Optional: To create sub argument groups use the `_argument_group_name` attribute in the dataclass. dataclass_types)r>c sRd|krt|d<tjf|t|r.|g}t||_|jD]}||q>dS)a% Args: dataclass_types: Dataclass type, or list of dataclass types for which we will "fill" instances with the parsed args. kwargs (`Dict[str, Any]`, *optional*): Passed to `argparse.ArgumentParser()` in the regular way. formatter_classN)rsuper__init__r9Z is_dataclasslistr>_add_dataclass_arguments)selfr>r;dtype __class__r%r&rAys    zHfArgumentParser.__init__)parserr:cCsd|jg}d|jkr2|d|jdd|j}t|jtrPtd| dg}t|trl|g}t |jd|j}|t kst t drxt|t jrxt|jjkrt|jjdkstd|jjkrtd |jd td|jjkr,|jjd tkr |jjd n |jjd |_t |jd|j}nLt|jjkrxtd|jjd rZ|jjd n |jjd |_t |jd|j}i}|tkst|jtrt|jtr|tkr|jj|d <ndd|jD|d <t|d |d<|jtjk r|j|d<nd|d<n.|jtks |jttkrt|}t|d<|jtksV|jdk r2|jtjk r2|jtjkrhdn|j}||d<d|d<d|d<nt|rt|tr|jjd |d<d|d<|jtjk r||d<n|jtjkr2d|d<nH|j|d<|jtjk r|j|d<n$|jtjk r*||d<nd|d<|j||||jdkr|jtksj|jttkrd|d<|jd|jd|jddfd|jd|dS)Nz--_-zpUnresolved type detected, which should have been done with the help of `typing.get_type_hints` method by defaultr4 __origin__ UnionTypezOnly `Union[X, NoneType]` (i.e., `Optional[X]`) is allowed for `Union` because the argument parser only supports one type per argument. Problem encountered in field 'z'.rr(cSsg|] }|jqSr%)value)r+xr%r%r& sz;HfArgumentParser._parse_dataclass_field..typer6TrequiredF?nargsconst+z--no_z--no- store_false)actiondest) nameappendreplacer8rr!rRr* RuntimeErrorpopgetattrrhasattrtypesrL__args__len ValueErrorr"r issubclassrr3r6r9MISSINGrr'rrBr7 add_argument)rHr:Z long_optionsr;r4Z origin_typeZ bool_kwargsr6r%r%r&_parse_dataclass_fields     "  ,*&   &    ( z'HfArgumentParser._parse_dataclass_field)rEc Cst|dr||j}n|}z t|}Wntk rLtd|dYnttk r}zVtjdddkrdt |krd t t tjdd}td|d |d |W5d}~XYnXt |D]$}|jsq||j|_|||qdS) N_argument_group_namezType resolution failed for z. Try declaring the class in global scope or removing line of `from __future__ import annotations` which opts in Postponed Evaluation of Annotations (PEP 563)rM) z!unsupported operand type(s) for |.rkz on Python a6. Try removing line of `from __future__ import annotations` which opts in union types as `X | Y` (PEP 604) via Postponed Evaluation of Annotations (PEP 563). To support Python versions that lower than 3.10, you need to use `typing.Union[X, Y]` instead of `X | Y` and `typing.Optional[X]` instead of `X | None`.)raadd_argument_grouprjr NameErrorr^ TypeErrorsys version_infor*joinmapr9fieldsinitr[rRri)rDrErHZ type_hintsexpython_versionr:r%r%r&rCs.    z)HfArgumentParser._add_dataclass_argumentsNFT.)r)cs|s|s|rttjrg}|r.|t|n(|rVttjrV|ttjdd|rt}|j|tdd|j |d\}}t | | dd} | r| dd | Dg} |D]} | r| | 7} q|dk r| |n| tjd d}|j |d\} } g}|jD]^}d d t|Dfd dt | D}D]}t| |qF|f|}||qt| jdkr|| |r|| fS| rtd| |SdS)a' Parse command-line args into instances of the specified dataclass types. This relies on argparse's `ArgumentParser.parse_known_args`. See the doc at: docs.python.org/3.7/library/argparse.html#argparse.ArgumentParser.parse_args Args: args: List of strings to parse. The default is taken from sys.argv. (same as argparse.ArgumentParser) return_remaining_strings: If true, also return a list of remaining argument strings. look_for_args_file: If true, will look for a ".args" file with the same base name as the entry point script for this process, and will append its potential content to the command line args. args_filename: If not None, will uses this file instead of the ".args" file specified in the previous argument. args_file_flag: If not None, will look for a file in the command-line args specified with this flag. The flag can be specified multiple times and precedence is determined by the order (last one wins). Returns: Tuple consisting of: - the dataclass instances in the same order as they were passed to the initializer.abspath - if applicable, an additional namespace for more (non-dataclass backed) arguments added to the parser after initialization. - The potential list of remaining argument strings. (same as argparse.ArgumentParser.parse_known_args) rz.argsr\)rRrY)argsrJNcSsg|] }t|qSr%r)r+pr%r%r&rQOsz@HfArgumentParser.parse_args_into_dataclasses..rNcSsh|]}|jr|jqSr%rvr[r+rr%r%r& \sz?HfArgumentParser.parse_args_into_dataclasses..csi|]\}}|kr||qSr%r%r+kr$keysr%r&r-]sz@HfArgumentParser.parse_args_into_dataclasses..z?Some specified arguments are not used by the HfArgumentParser: )rdrqargvr\r with_suffixrrhr*parse_known_argsvarsr.lstripextendexists read_textsplitr>r9ruitemsdelattr__dict__re)rDryZreturn_remaining_stringsZlook_for_args_fileZ args_filenameZargs_file_flagZ args_filesZargs_file_parsercfgZcmd_args_file_pathsZ file_args args_file namespaceremaining_argsoutputsrEinputsrobjr%rr&parse_args_into_dataclassessD%"    z,HfArgumentParser.parse_args_into_dataclasses)ryallow_extra_keysr)cst|}g}|jD]P}ddt|Dfdd|D}|||f|}||q|s|rtdt |t |S)a< Alternative helper method that does not use `argparse` at all, instead uses a dict and populating the dataclass types. Args: args (`dict`): dict containing config values allow_extra_keys (`bool`, *optional*, defaults to `False`): Defaults to False. If False, will raise an exception if the dict contains keys that are not parsed. Returns: Tuple consisting of: - the dataclass instances in the same order as they were passed to the initializer. cSsh|]}|jr|jqSr%r{r|r%r%r&r}sz.HfArgumentParser.parse_dict..csi|]\}}|kr||qSr%r%r~rr%r&r-sz/HfArgumentParser.parse_dict..z0Some keys are not used by the HfArgumentParser: ) setrr>r9rurdifference_updater\resortedtuple)rDryrZ unused_keysrrErrr%rr& parse_dictms    zHfArgumentParser.parse_dict) json_filerr)c Cs@tt|dd}t|}W5QRX|j||d}t|S)at Alternative helper method that does not use `argparse` at all, instead loading a json file and populating the dataclass types. Args: json_file (`str` or `os.PathLike`): File name of the json file to parse allow_extra_keys (`bool`, *optional*, defaults to `False`): Defaults to False. If False, will raise an exception if the json file contains keys that are not parsed. Returns: Tuple consisting of: - the dataclass instances in the same order as they were passed to the initializer. zutf-8)encodingr)openr jsonloadsreadrr)rDrrZopen_json_filedatarr%r%r&parse_json_filesz HfArgumentParser.parse_json_file) yaml_filerr)cCs$|jtt||d}t|S)at Alternative helper method that does not use `argparse` at all, instead loading a yaml file and populating the dataclass types. Args: yaml_file (`str` or `os.PathLike`): File name of the yaml file to parse allow_extra_keys (`bool`, *optional*, defaults to `False`): Defaults to False. If False, will raise an exception if the json file contains keys that are not parsed. Returns: Tuple consisting of: - the dataclass instances in the same order as they were passed to the initializer. r)ryamlZ safe_loadr rr)rDrrrr%r%r&parse_yaml_filesz HfArgumentParser.parse_yaml_file)NFTNN)F)F)F)__name__ __module__ __qualname____doc__r r__annotations__rrA staticmethodrr9FieldrirCrrrr r*r r"rosPathLikerr __classcell__r%r%rFr&r=ns6  i$ V$    r=)(r9rrrqrbargparserrrrenumrinspectrpathlibr typingr r r r rrrrrrrrrrr'rBr*r3rgdictrr<r=r%r%r%r&s:    4     /