o Zhv@sxddlZddlZddlZddlmZmZmZddlmZddl m Z m Z m Z m Z mZmZmZmZmZddlmZmZmZddlmZddlmZdd lmZmZdd lmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&dd l'm(Z(e rrdd l)m*Z*e$ryddl+Z+e#rdd l,m-Z.ddl,m/Z0e%1e2Z3edddZ4dZ5eGdddZ6GdddZ7Gddde7Z8dedde9ddfddZ:dS)N)asdict dataclass is_dataclass)Path) TYPE_CHECKINGAnyDictListOptionalTypeTypeVarUnionget_args) CONFIG_NAMEPYTORCH_WEIGHTS_NAMESAFETENSORS_SINGLE_FILE)hf_hub_download)HfApi) ModelCard ModelCardData)EntryNotFoundErrorHfHubHTTPErrorSoftTemporaryDirectory is_jsonableis_safetensors_availableis_torch_availableloggingvalidate_hf_hub_args)_deprecate_arguments)DataclassInstance) load_model) save_modelT ModelHubMixin)bounda --- # For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 # Doc / guide: https://huggingface.co/docs/hub/model-cards {{ card_data }} --- This model has been pushed to the Hub using **{{ library_name }}**: - Repo: {{ repo_url | default("[More Information Needed]", true) }} - Docs: {{ docs_url | default("[More Information Needed]", true) }} c@sReZdZUdZeeed<dZeeeed<dZ eeed<dZ eeed<dS) MixinInfoN library_nametagsrepo_urldocs_url) __name__ __module__ __qualname__r'r str__annotations__r(r r)r*r0r0P/var/www/html/lang_env/lib/python3.10/site-packages/huggingface_hub/hub_mixin.pyr&6s r&cseZdZUdZdZeeedfed<e ed<e e e j fed<e e efed<eed<ddddd d ee d eee d ee d ee ddf fddZd=fdd Zdddddee efdeeedfdee dedee f ddZdeddfddZeedddddddddeedee efd ed!ed"ee d#eee efd$eee efd%ed&ee defd'd(Zedeed)e d&ee d$eee efd ed"ee d!ed%ed#eee efdefd*d+Zed,d-gd.d/edd0ddddddddd1 de deeedfd2e d3ed#ee d4ee d5eed6eeee e fd7eeee e fd8eeee e fd-ee de fd9d:Zdefd;d<ZZ S)>r$aE A generic mixin to integrate ANY machine learning framework with the Hub. To integrate your framework, your model class must inherit from this class. Custom logic for saving/loading models have to be overwritten in [`_from_pretrained`] and [`_save_pretrained`]. [`PyTorchModelHubMixin`] is a good example of mixin integration with the Hub. Check out our [integration guide](../guides/integrations) for more instructions. When inheriting from [`ModelHubMixin`], you can define class-level attributes. These attributes are not passed to `__init__` but to the class definition itself. This is useful to define metadata about the library integrating [`ModelHubMixin`]. Args: library_name (`str`, *optional*): Name of the library integrating ModelHubMixin. Used to generate model card. tags (`List[str]`, *optional*): Tags to be added to the model card. Used to generate model card. repo_url (`str`, *optional*): URL of the library repository. Used to generate model card. docs_url (`str`, *optional*): URL of the library documentation. Used to generate model card. Example: ```python >>> from huggingface_hub import ModelHubMixin # Inherit from ModelHubMixin >>> class MyCustomModel( ... ModelHubMixin, ... library_name="my-library", ... tags=["x-custom-tag"], ... repo_url="https://github.com/huggingface/my-cool-library", ... docs_url="https://huggingface.co/docs/my-cool-library", ... # ^ optional metadata to generate model card ... ): ... def __init__(self, size: int = 512, device: str = "cpu"): ... # define how to initialize your model ... super().__init__() ... ... ... ... def _save_pretrained(self, save_directory: Path) -> None: ... # define how to serialize your model ... ... ... ... @classmethod ... def from_pretrained( ... cls: Type[T], ... pretrained_model_name_or_path: Union[str, Path], ... *, ... force_download: bool = False, ... resume_download: bool = False, ... proxies: Optional[Dict] = None, ... token: Optional[Union[str, bool]] = None, ... cache_dir: Optional[Union[str, Path]] = None, ... local_files_only: bool = False, ... revision: Optional[str] = None, ... **model_kwargs, ... ) -> T: ... # define how to deserialize your model ... ... >>> model = MyCustomModel(size=256, device="gpu") # Save model weights to local directory >>> model.save_pretrained("my-awesome-model") # Push model weights to the Hub >>> model.push_to_hub("my-awesome-model") # Download and initialize weights from the Hub >>> reloaded_model = MyCustomModel.from_pretrained("username/my-awesome-model") >>> reloaded_model._hub_mixin_config {"size": 256, "device": "gpu"} # Model card has been correctly populated >>> from huggingface_hub import ModelCard >>> card = ModelCard.load("username/my-awesome-model") >>> card.data.tags ["x-custom-tag", "pytorch_model_hub_mixin", "model_hub_mixin"] >>> card.data.library_name "my-library" ``` Nr _hub_mixin_config_hub_mixin_info_hub_mixin_init_parameters"_hub_mixin_jsonable_default_values_hub_mixin_inject_configr'r(r)r*r'r(r)r*returncspt|pg}|dt||||d|_tt|jj |_ dd|j D|_ dt|j j v|_dS)zIInspect __init__ signature only once when subclassing + handle modelcard.Zmodel_hub_mixinr7cSs.i|]}|jtjjurt|jr|j|jqSr0)defaultinspect Parameteremptyrname.0paramr0r0r1 s z3ModelHubMixin.__init_subclass__..configN)super__init_subclass__appendr&r3dictr: signature__init__ parametersr4valuesr5_from_pretrainedr6)clsr'r(r)r* __class__r0r1rDs  zModelHubMixin.__init_subclass__cst|}|jdur |Siddtt|jdd|D|}t|dr0|d|_|Si|jdd| D}| di|d}t |t rS| ||ikrZ||_|S)aACreate a new instance of the class and handle config. 3 cases: - If `self._hub_mixin_config` is already set, do nothing. - If `config` is passed as a dataclass, set it as `self._hub_mixin_config`. - Otherwise, build `self._hub_mixin_config` from default values and passed values. NcSsi|]\}}||qSr0r0r?keyvaluer0r0r1rAsz)ModelHubMixin.__new__..rrBcSsi|] \}}t|r||qSr0)rrOr0r0r1rAs)rC__new__r2ziplistr4rgetr5itemspop isinstancerFupdate)rLargskwargsinstanceZ passed_valuesZ init_configZprovided_configrMr0r1rRs8        zModelHubMixin.__new__F)rBrepo_id push_to_hubsave_directoryrBr]r^c Kst|}|jddd|t}|jdd|||dur!|j}|dur>t|r-t|}|s>t j |ddd}| ||d}|sO| |d|rn|} |dur]|| d<|durd|j}|jd d |i| SdS) aE Save weights in local directory. Args: save_directory (`str` or `Path`): Path to directory in which the model weights and configuration will be saved. config (`dict` or `DataclassInstance`, *optional*): Model configuration specified as a key/value dictionary or a dataclass instance. push_to_hub (`bool`, *optional*, defaults to `False`): Whether or not to push your model to the Huggingface Hub after saving it. repo_id (`str`, *optional*): ID of your repository on the Hub. Used only if `push_to_hub=True`. Will default to the folder name if not provided. kwargs: Additional key word arguments passed along to the [`~ModelHubMixin.push_to_hub`] method. T)parentsexist_ok) missing_okN) sort_keysindentz README.mdrBr]r0)rmkdirrunlink_save_pretrainedr2rrexistsjsondumps write_textgenerate_model_cardsavecopyr=r^) selfr_rBr]r^Zpush_to_hub_kwargsZ config_pathZ config_strZmodel_card_pathr[r0r0r1save_pretraineds0   zModelHubMixin.save_pretrainedcCt)aD Overwrite this method in subclass to define how to save your model. Check out our [integration guide](../guides/integrations) for instructions. Args: save_directory (`str` or `Path`): Path to directory in which the model weights and configuration will be saved. NotImplementedError)rpr_r0r0r1rh+s zModelHubMixin._save_pretrained)force_downloadresume_downloadproxiestoken cache_dirlocal_files_onlyrevisionrLpretrained_model_name_or_pathrurvrwrxryrzr{c KsZt|} d} tj| r*tt| vrtj| t} n=ttdt |  n.zt | t|||||||d } Wnt yW} zt tdt| WYd} ~ nd} ~ wwd} | durt| ddd }t|} Wdn1svwY|jD]}|j| vr|j| vr| |j| |j<qd|jvr|jdj}|tjjurnt|rt|| } nt|D] }t|rt|| } nq| | d<t|r|jD]}|| vr|| vr| || |<qntd d |jDr| D] \}}|| vr|| |<q|jr| | d<|jd t| |||||||d | }| dur+t |d ddifvr+| |_!|S)a Download a model from the Huggingface Hub and instantiate it. Args: pretrained_model_name_or_path (`str`, `Path`): - Either the `model_id` (string) of a model hosted on the Hub, e.g. `bigscience/bloom`. - Or a path to a `directory` containing model weights saved using [`~transformers.PreTrainedModel.save_pretrained`], e.g., `../path/to/my_model_directory/`. revision (`str`, *optional*): Revision of the model on the Hub. Can be a branch name, a git tag or any commit id. Defaults to the latest commit on `main` branch. force_download (`bool`, *optional*, defaults to `False`): Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding the existing cache. resume_download (`bool`, *optional*, defaults to `False`): Whether to delete incompletely received files. Will attempt to resume the download if such a file exists. proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on every request. token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. By default, it will use the token cached when running `huggingface-cli login`. cache_dir (`str`, `Path`, *optional*): Path to the folder where cached files are stored. local_files_only (`bool`, *optional*, defaults to `False`): If `True`, avoid downloading the file and return the path to the local cached file if it exists. model_kwargs (`Dict`, *optional*): Additional kwargs to pass to the model during initialization. Nz not found in r]filenamer{ryrurwrvrxrzz# not found on the HuggingFace Hub: rzutf-8)encodingrBcss|] }|jtjjkVqdS)N)kindr:r; VAR_KEYWORDr>r0r0r1 sz0ModelHubMixin.from_pretrained..)model_idr{ryrurwrvrzrxr2r0)"r.ospathisdirrlistdirjoinloggerwarningrresolverrinfoopenrjloadr4rJr= annotationr:r;r<r_load_dataclassr__dataclass_fields__anyrVr6rKgetattrr2)rLr|rurvrwrxryrzr{ model_kwargsr config_fileerBfr@Zconfig_annotationZ_sub_annotationrPrQr\r0r0r1from_pretrained6s,   $            zModelHubMixin.from_pretrainedrc Krr)aOverwrite this method in subclass to define how to load your model from pretrained. Use [`hf_hub_download`] or [`snapshot_download`] to download files from the Hub before loading them. Most args taken as input can be directly passed to those 2 methods. If needed, you can add more arguments to this method using "model_kwargs". For example [`PyTorchModelHubMixin._from_pretrained`] takes as input a `map_location` parameter to set on which device the model should be loaded. Check out our [integration guide](../guides/integrations) for more instructions. Args: model_id (`str`): ID of the model to load from the Huggingface Hub (e.g. `bigscience/bloom`). revision (`str`, *optional*): Revision of the model on the Hub. Can be a branch name, a git tag or any commit id. Defaults to the latest commit on `main` branch. force_download (`bool`, *optional*, defaults to `False`): Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding the existing cache. resume_download (`bool`, *optional*, defaults to `False`): Whether to delete incompletely received files. Will attempt to resume the download if such a file exists. proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint (e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`). token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. By default, it will use the token cached when running `huggingface-cli login`. cache_dir (`str`, `Path`, *optional*): Path to the folder where cached files are stored. local_files_only (`bool`, *optional*, defaults to `False`): If `True`, avoid downloading the file and return the path to the local cached file if it exists. model_kwargs: Additional keyword arguments passed along to the [`~ModelHubMixin._from_pretrained`] method. rs) rLrr{ryrurwrvrzrxrr0r0r1rKs/zModelHubMixin._from_pretrainedz0.23.0 api_endpointz/Use `HF_ENDPOINT` environment variable instead.)versionZdeprecated_argsZcustom_messagez!Push model using huggingface_hub.) rBcommit_messageprivaterxbranch create_prallow_patternsignore_patternsdelete_patternsrrrrrrrrc  Cs~t| |d} | j||ddj}t#} t| |}|j||d| j|d|||||| | d WdS1s8wYdS)a Upload model checkpoint to the Hub. Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be pushed to the hub. Use `delete_patterns` to delete existing remote files in the same commit. See [`upload_folder`] reference for more details. Args: repo_id (`str`): ID of the repository to push to (example: `"username/my-model"`). config (`dict` or `DataclassInstance`, *optional*): Model configuration specified as a key/value dictionary or a dataclass instance. commit_message (`str`, *optional*): Message to commit while pushing. private (`bool`, *optional*, defaults to `False`): Whether the repository created should be private. api_endpoint (`str`, *optional*): The API endpoint to use when pushing the model to the hub. token (`str`, *optional*): The token to use as HTTP bearer authorization for remote files. By default, it will use the token cached when running `huggingface-cli login`. branch (`str`, *optional*): The git branch on which to push the model. This defaults to `"main"`. create_pr (`boolean`, *optional*): Whether or not to create a Pull Request from `branch` with that commit. Defaults to `False`. allow_patterns (`List[str]` or `str`, *optional*): If provided, only files matching at least one pattern are pushed. ignore_patterns (`List[str]` or `str`, *optional*): If provided, files matching any of the patterns are not pushed. delete_patterns (`List[str]` or `str`, *optional*): If provided, remote files matching any of the patterns will be deleted from the repo. Returns: The url of the commit of your model in the given repository. )ZendpointrxT)r]rra)rBmodel) r]Z repo_typeZ folder_pathrr{rrrrN)rZ create_repor]rrrqZ upload_folder)rpr]rBrrrxrrrrrrapitmpZ saved_pathr0r0r1r^s" 9 $zModelHubMixin.push_to_hubcOs"tjtdit|jtd}|S)N)Z card_dataZ template_strr0)rZ from_templaterrr3DEFAULT_MODEL_CARD)rprZr[Zcardr0r0r1rm4s z!ModelHubMixin.generate_model_card)r8r$)!r+r,r-__doc__r2r r rFr/r&rr.r:r;rboolr rDrRrrqrh classmethodrr r#rrKrr^rrm __classcell__r0r0rMr1r$>s T 7  >        0    FcseZdZdZdddeeeddffddZdeddfd d Z e d d d dedeedee eefde dee de de de ee dfdede fddZe dededede def ddZe dededede def ddZZS) PyTorchModelHubMixina Implementation of [`ModelHubMixin`] to provide model Hub upload/download capabilities to PyTorch models. The model is set in evaluation mode by default using `model.eval()` (dropout modules are deactivated). To train the model, you should first set it back in training mode with `model.train()`. Example: ```python >>> import torch >>> import torch.nn as nn >>> from huggingface_hub import PyTorchModelHubMixin >>> class MyModel( ... nn.Module, ... PyTorchModelHubMixin, ... library_name="keras-nlp", ... repo_url="https://github.com/keras-team/keras-nlp", ... docs_url="https://keras.io/keras_nlp/", ... # ^ optional metadata to generate model card ... ): ... def __init__(self, hidden_size: int = 512, vocab_size: int = 30000, output_size: int = 4): ... super().__init__() ... self.param = nn.Parameter(torch.rand(hidden_size, vocab_size)) ... self.linear = nn.Linear(output_size, vocab_size) ... def forward(self, x): ... return self.linear(x + self.param) >>> model = MyModel(hidden_size=256) # Save model weights to local directory >>> model.save_pretrained("my-awesome-model") # Push model weights to the Hub >>> model.push_to_hub("my-awesome-model") # Download and initialize weights from the Hub >>> model = MyModel.from_pretrained("username/my-awesome-model") >>> model.hidden_size 256 ``` N)r(r(r8cs,|pg}|d||d<tj|i|S)NZpytorch_model_hub_mixinr()rErCrD)rLr(rZr[rMr0r1rDgs z&PyTorchModelHubMixin.__init_subclass__r_cCs*t|dr|jn|}t|t|tdS)z7Save weights from a Pytorch model to a local directory.moduleN)hasattrrsave_model_as_safetensorr.r)rpr_Z model_to_saver0r0r1rhmsz%PyTorchModelHubMixin._save_pretrainedcpuF) map_locationstrictrr{ryrurwrvrzrxrrc Ks|di| } tj|r tdtj|t} || | | | Szt|t|||||||d } || | | | WStyTt|t |||||||d } | | | | | YSw)z.Nr0)rV)rrr0rr1rs r);r:rjr dataclassesrrrpathlibrtypingrrrr r r r r r constantsrrrZ file_downloadrZhf_apirZrepocardrrutilsrrrrrrrrZutils._deprecationrZ _typeshedr rZsafetensors.torchr!rr"rZ get_loggerr+rr#rr&r$rrFrr0r0r0r1s> ,  (