o  hgs%@s UddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlmZddlmZddlmZddlmZddlmZddlmZmZddlmZdd lmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&dd l'm(Z(ddl)Z)ddl*Z)dd l+m,Z,dd l-m.Z.dd l)m/Z/m0Z0ddl1m2Z2ddl3m4Z4m5Z5ddl6m7Z7ddl8m9Z9e9rddl:m;Z;ddlm?Z?ddl@mAZAddlBmCZCmDZDmEZEddlFmGZGmHZHmIZIddlJmKZKmLZLddlMmNZNmOZOddlPmQZQddlRmSZSddlTmUZUddlVmWZWmXZXddlYmZZZdd l[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbdd!lcmdZdmeZedd"lfmgZgdd#lhmiZidd$ljmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd%lmZmZdd&lmZmZmZmZdd'lmZmZe jd(d)Ze jd*d)Ze~rdd+lmZmZdd,lmZdd-lmZmZmZmZmZmZmZe.ej.d.Zee.d/krdd0lmZer dd1lmZdd2lmZdd3lmZeOrddlZeeZd4ad5ad5ae)jZd6d7Zd8d9ZerGddlmm)Zdd:lmZe.eɡe.d;kZnd5ZerSddd?Ze0jje0jje0jje0jje0jje0jje0jje0jje0jje0jje0jje0jje0jje0jjd@ZedAdBZedCdDZedEdFZdGdHZdIe&e0jdJffdKdLZdIe&e0jdJffdMdNZdIe&e0jdJffdOdPZdQdRZdSdTZddUdVZe)je)je)je)je)je)je)je)je)je)je)jdW ZedXre)jedY<edZre)jed[<e)jed\<e)jed]<edXre)jedY<e)jed^< 5 _ 4dd`e&ee jfdaedbe!e&ee)jfdcefdddeZdfdgZdhe)j/diefdjdkZddme0jfdndoZdpe e"edqeee)j/fdie#e e"ee effdrdsZdpe e"edqeee)j/fdie#e e"ee"effdtduZ  ddvd>dwedxe)j/dye!e jdze!eedie&ee!e)jff d{d|Zdvd>dwedhe)j/fd}d~Ze)       5   ddvd>dqedede edeeefde!ede!ede!ede!ede!edze!eededye!e jde!e ede!ddie#e!ee!eff ddZddede!ediefddZde!e&ee jfdede!ede!edededededede!eeefdede!e&eefde dede!edie#e!e ee!eff ddZ de!e&ee)jefde!e ede?de!edqe!edcedie#e?e!e)je!e)jffddZ dvd>de!e&eefde!edze!eede!e)jdye!e jdiefddZ dvd>de ede ededze!eededie#e ee effddZ dvd>dqe!ede!e ededeeefdaedcedie#e ee e#eefffddZGdddeZGddJdJZGdd>d>e0jeeEexeGZezeje_ejjdurVejjjdvdddej_Gddde0jZGddde0jZGddde0jZeGdddewZGddÄde0jZGddńde0jZddve0jdedie0jfddȄZddʄZddvedefdd΄ZddЄZGdd҄deZeZ ee!d<dS)N) defaultdict)MutableMapping)contextmanager) dataclass)Enum)partialwraps)Thread) AnyCallableDictListOptionalSetTupleTypeTypeVarUnion) is_zipfile)"split_torch_state_dict_into_shards)version)Tensornn) constraints)CrossEntropyLossIdentity) checkpoint)is_torchao_available)Int4WeightOnlyConfig)get_activation)PretrainedConfig)custom_object_save) CompileConfigGenerationConfigGenerationMixin)PeftAdapterMixindeepspeed_configis_deepspeed_zero3_enabled)find_tied_parametersinit_empty_weights)!_load_state_dict_into_zero3_modelis_deepspeed_available)flash_attention_forward)flex_attention_forward)sdpa_attention_forward)SUPPORTED_TP_STYLESshard_and_distribute_module) LOSS_MAPPING)Conv1Dapply_chunking_to_forward find_pruneable_heads_and_indicesid_tensor_storageprune_conv1d_layer prune_layerprune_linear_layer)AutoHfQuantizer HfQuantizer)get_module_from_name)auto_conversion)$ADAPTER_SAFE_WEIGHTS_NAMEADAPTER_WEIGHTS_NAME CONFIG_NAME DUMMY_INPUTSFLAX_WEIGHTS_NAMESAFE_WEIGHTS_INDEX_NAMESAFE_WEIGHTS_NAMETF2_WEIGHTS_NAMETF_WEIGHTS_NAMEWEIGHTS_INDEX_NAME WEIGHTS_NAMEContextManagers ModelOutputPushToHubMixin cached_file copy_func download_urlextract_commit_hashhas_fileis_accelerate_availableis_bitsandbytes_availableis_flash_attn_2_availableis_offline_modeis_optimum_availableis_peft_available is_remote_urlis_safetensors_availableis_torch_flex_attn_availableis_torch_greater_or_equalis_torch_mlu_availableis_torch_npu_availableis_torch_sdpa_availableis_torch_xla_availableloggingreplace_return_docstrings strtobool)create_and_tag_model_cardget_checkpoint_shard_files)ENV_VARS_TRUE_VALUESis_sagemaker_mp_enabledis_torch_fx_proxyis_torchdynamo_compiling)BitsAndBytesConfigQuantizationMethod XLA_USE_BF160XLA_DOWNCAST_BF16)dispatch_modelinfer_auto_device_map)add_hook_to_module)$check_tied_parameters_on_same_deviceextract_model_from_parallelget_balanced_memoryget_max_memoryload_offloaded_weightsoffload_weightsave_offload_index accelerate0.31)get_state_dict_from_offload) safe_open) load_file) save_fileTFcCs@tjotjottjdddkottjdddkS)NACCELERATE_USE_FSDPFalserFSDP_CPU_RAM_EFFICIENT_LOADING)torch distributed is_availableis_initializedraosenvirongetrro/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/transformers/modeling_utils.pyis_fsdp_enableds rcCs*tjotjottjdddkS)N LOCAL_RANKr)rrrrintrrrrrrris_local_dist_rank_0s r __version__z1.10)find_adapter_config_fileSpecificPreTrainedModelTypePreTrainedModel)bound)uniform_normal_ trunc_normal_ constant_xavier_uniform_xavier_normal_kaiming_uniform_kaiming_normal_uniformnormalxavier_uniform xavier_normalkaiming_uniformkaiming_normalc cst}dadd}tD] \}}ttjj||q zdVW|atD] \}}ttjj||q%dS|atD] \}}ttjj||q:w)ze Context manager to globally disable weight initialization to speed up loading large models. Fc_sdSNr)argskwargsrrr _skip_initsz#no_init_weights.._skip_initN) _init_weightsTORCH_INIT_FUNCTIONSitemssetattrrrinit)old_init_weightsrname init_funcrrrno_init_weightss rccdazdVWdadSdawNTF) _is_quantizedrrrrset_quantized_state rccrr)_is_ds_init_calledrrrrset_zero3_staterrcstfdd}|S)z Decorator to restore the default torch dtype at the end of the function. Serves as a backup in case calling the function raises an error after the function has changed the default dtype but before it could restore it. c s0t}z |i|Wt|St|wr)rget_default_dtypeset_default_dtype)rr old_dtypefuncrr_wrappersz-restore_default_torch_dtype.._wrapper)r)rrrrrrestore_default_torch_dtype sr parameterModuleUtilsMixinc Cbzt|jWSty0dtjdtttt ffdd}|j |d}t|}|djYSw)NmodulereturncSdd|jD}|S)NcS"g|] \}}t|r||fqSrr is_tensor.0kvrrr %"zHget_parameter_device..find_tensor_attributes..__dict__rrtuplesrrrfind_tensor_attributes$z4get_parameter_device..find_tensor_attributesget_members_fnr) next parametersdevice StopIterationrModuler rstrr_named_membersrrgen first_tuplerrrget_parameter_devices   rc Cr)z` Returns the first parameter dtype (can be non-floating) or asserts if none were found. rrcSr)NcSrrrrrrrr7rzMget_first_parameter_dtype..find_tensor_attributes..rrrrrr6rz9get_first_parameter_dtype..find_tensor_attributesrr) rrdtyperrrr rrrrrrrrget_first_parameter_dtype-s   rcCsd}|D]7}|j}|r=ttvrtrtjSttvr8tr8|jtj kr-tjS|jtj kr8tj S|jSq|durD|Sdt j dttttffdd}|j|d}d}|D]}|}|dro|djSq^|dury|djS|D]}|j}|r|jSq}|S)zz Returns the first found floating dtype in parameters if there is one, otherwise returns the last dtype it found. NrrcSr)NcSrrrrrrrrZrzGget_parameter_dtype..find_tensor_attributes..rrrrrrYrz3get_parameter_dtype..find_tensor_attributesrr)rris_floating_pointrjrdr^rbfloat16rlfloatdoublefloat32rrr rrrrbuffers)r last_dtypetrr last_tupletuplerrrget_parameter_dtype?s>             rcCs(|D] }|r|jSqtd)z_ Returns the first found floating dtype in `state_dict` or asserts if none were found. z5couldn't find any floating point dtypes in state_dict)valuesrr ValueError state_dictrrrrget_state_dict_float_dtypeps  rcCs.|D] }|r|jSqt|jS)zt Returns the first found floating dtype in `state_dict` if there is one, otherwise returns the first dtype. )rrrrrrrrget_state_dict_dtype{s  rcs*tj|t}tj|t}tj|}tj|}|s9|r!ts9tr(ttfntf}tdd|d|dd} |rS|rOtrEd} nt d|dn|sSd} | rW|n|} t | d d d  } t | } Wd n1spwYt t| d } | d |fddD}fddD}|rt|dkst|dkrd|jj}t|dkrddd|D}|d|d7}t|dkrddd|D}|d|d7}t|| rtnttj ddd}| D]}|tj||}|j|dd~tqtjjj||S)a This is the same as [`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict) but for a sharded checkpoint. This load is performed efficiently: each checkpoint shard is loaded one by one in RAM and deleted after being loaded in the model. Args: model (`torch.nn.Module`): The model in which to load the checkpoint. folder (`str` or `os.PathLike`): A path to a folder containing the sharded checkpoint. strict (`bool`, *optional*, defaults to `True`): Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint. prefer_safe (`bool`, *optional*, defaults to `False`): If both safetensors and PyTorch save files are present in checkpoint and `prefer_safe` is True, the safetensors files will be loaded. Otherwise, PyTorch files are always loaded when possible. Returns: `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields - `missing_keys` is a list of str containing the missing keys - `unexpected_keys` is a list of str containing the unexpected keys zCan't find a checkpoint index ( or z) in .FTz"Cannot load sharded checkpoint at z+ safely since safetensors is not installed!rutf-8encodingN weight_mapcg|]}|vr|qSrrrkey) loaded_keysrrrz+load_sharded_checkpoint..crrrr) model_keysrrrrr#Error(s) in loading state_dict for ,cSg|]}d|dqS"rrrrrrrrz Missing key(s): cSrrrrrrrrrcpu map_location weights_only)strict) rpathjoinrGrCisfilerXrloggerwarningopenjsonloadlistsetrkeysrlen __class____name__ RuntimeErrorsafe_load_filerrload_state_dictgccollectrmodulesr_IncompatibleKeys)modelfolderr  prefer_safe index_filesafe_index_file index_presentsafe_index_present filenames load_safe load_indexfindex shard_files missing_keysunexpected_keys error_messagestr_missing_keysstr_unexpected_keysloader shard_filerr)rrrload_sharded_checkpointsV          r3) BOOLU8I8I16F16BF16I32F32F64I64F8_E4M32.1.0r>z2.3.0U16U32U64F8_E5M2rcheckpoint_file is_quantizedrrc Cs|drrtrrt|dd\}|}|dur&|ddvr&td|di}|D]4}||}|t vr>t |} nt d ||d krYt j || | d d ||<q,||||<q,|WdS1smwYzK|durtrt jrt jd kstrts|sd }nd }i} t|tr|d krtt jtdkrt|rddi} t j|f||d| WSty } z?z#t|}|ddkrtdt d|d| 1swYWntt fytd|d|dwWYd} ~ dSd} ~ ww)zg Reads a `safetensor` or a `.bin` checkpoint file. We load the checkpoint on "cpu" by default. .safetensorspt frameworkNformat)rGtfflaxmlxz"The safetensors archive passed at zf does not contain the valid metadata. Make sure you save your model with the `save_pretrained` method.z)Cannot load safetensors of unknown dtype meta)sizerrrrr?mmapTrrzYou seem to have cloned a repository without having git-lfs installed. Please install git-lfs and run `git lfs install` followed by `git lfs pull` in the folder you cloned.zUnable to locate the file z_ which is necessary to load this pretrained model. Make sure you have saved the model properly.z9Unable to load weights from pytorch checkpoint file for 'z' at 'zZ'. If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True.) endswithrXrzmetadatarOSErrorr get_slice get_dtypestr_to_torch_dtyperrempty get_shape get_tensorr(rrget_rankrr isinstancerrparserrr ExceptionrreadUnicodeDecodeError) rDrErrr)rSrrk_dtyper extra_argserrrrs         rcsht|}i}|D]'\}dkrt|}n fdd|D}||r-d|_q ||<q |S)z Sets the `_is_hf_initialized` flag in all submodules of a given model when all its weights are in the loaded state dict. csh|] }d|qSrrr module_namerr Tz-set_initialized_submodules..T)r named_modulesrissubset_is_hf_initialized)rstate_dict_keysnot_initialized_submodulesr module_keysrrfrset_initialized_submodulesHs  rptensorrcCs2|r|dd|}|S|}|S)Nr)nelementviewdata_ptr element_size)rqstoprrr_end_ptr\s rwrdrcsg}t|dddurfdd|jD}||t|dddur0fdd|jD}|||D]\}}rAd|n|}|t||dq4|S)N_tied_weights_keysc"g|] }r d|n|qSrerrprefixrrrhrz)_get_tied_weight_keys.._dynamic_tied_weights_keyscryrerrrzrrrkrrrz)getattrrxextendr|named_children_get_tied_weight_keys)rr{tied_weight_keysnamesr submodule local_prefixrrzrres  rtensorsrcCsg}|D]V}t|dkr||qg}|D]}||}||t||fq||d\}}} || h|ddD]\} } }| |krP||hn|d|| }q@qg} g} |D]}t|dkrq| |qa| |qa| | fS)Nrrr)rappendrtrwsortaddpop)rrfiltered_tensorssharedareasrrq_ last_stop last_namestartrvdisjoint_tensorsshared_tensorsrrr_find_disjointss2     rc Csg}g}|D]9}t|dkrqtt}|D]}||}|j|t|f}|||qt|dkr:||q||q||fS)Nrr) r collectionsrrrrtrwrr) rrr identicalrrrrqarearrr_find_identicals     rr param_name empty_paramkeep_in_fp32_regex hf_quantizerc Csz||}Wn ty'}z|dur!|jjtjkr!WYd}~dS|d}~wwttd}d}|o6|jtj k} |jj rV| sV|durJ| |rJtj }n |durS|j j}n|j}|duo]||fS)N)TN float8_e4m3fn)get_parameter_or_bufferr^quantization_config quant_methodriHQQhasattrrrrrsearchrconfig_pre_quantization_dtype is_contiguous) rrrrr old_paramrcis_torch_e4m3fn_available casting_dtypeis_param_float8_e4m3fnrrr_infer_parameter_dtypes$   rcCs&t||\}}|j||iddddS)zKCast a single parameter `param_name` into the `model`, with value `tensor`.FT)r assignN)r<r)rrrqr param_typerrr_load_parameter_into_modelsrr2 expected_keysreverse_renaming_mapping device_mapdisk_offload_folderdisk_offload_indexcpu_offload_foldercpu_offload_indexis_safetensorsr- device_mesh(torch.distributed.device_mesh.DeviceMeshc" Csd}|dur*|dddur*|ddtdfvr*t|dtjr&|djn|d}|dur>dddt|ddD}| du}|oM| jj t j t j fv}| d oU| }d}|rat|d |d }|D]\}}||vroqe|r{||}||}n||}t|||| | \}}|durt||||||ttjd |qe|d }|dur||}|r|}|durd}nt||}|st|d||}|dkr| st||||}qe|dkr| durt|||| } qe|r| jr| j||||||dstr t rdnd}t!||||qe| "|||||| ts%t#rht$||\}}t%||}d} tr=t s=d} i}!t&|drQ|j'j(j)dkrQd|!d<t*||j+| fi|!|j,}t-|||qe|duru|.ddd|| fS)aLoad parameters from `meta_state_dict` into the model. The parameters of the `meta_state_dict` are on the meta device in order to easily infer the shapes and dtypes that they will have. Then proper parameters are then loaded from `shard_file`, which is the actual state dict file on disk. This function takes care of correctly casting dtypes, devices, and sharding tensors in case of tensor parallelism. rNrd|cSg|]}t|qSrreescaperrrrrz4_load_state_dict_into_meta_model..T)reverserFrG)rIrRANK.z doesn't have any device set.disk) param_devicerrNweight Int8ParamsF requires_grad)/rrrr\r*r sortedrrrrirBITS_AND_BYTESrRrzrrUtorr1rrr contiguousrrrgroupru requires_parameters_quantizationcheck_quantized_paramrrrcreate_quantized_paramr(r<r}rrrrtypedatarr__exit__)"rrr2rrrrrrrrrrr-r tensor_devicedevice_map_regexrE is_hqq_or_bnbis_meta_state_dict file_pointerrrserialized_param_nameparam to_contiguousrr module_layerrrvalueparam_to val_kwargsrrr _load_state_dict_into_meta_models"           "  r weights_namevariantcCs0|dur|dd\}}|d|d|}|S)Nrr)rsplit)rrr rrrr _add_variantSsrpretrained_model_name_or_path subfolder gguf_filefrom_tf from_flaxuse_safetensors cache_dirforce_downloadproxieslocal_files_onlytoken user_agentrevision commit_hashcCsZd}|dura|durat|}tj|}|r`|r5tjtj||tdr5tj||td}n|rMtjtj||trMtj||t}n|retjtj||tretj||t}n|durtjtj||t t |rtj||t t |}n|durtjtj||t t |rtj||t t |}d}n|stjtj||t t |rtj||t t |}n|stjtj||t t |rtj||t t |}d}nb|stjtj||tdstjtj||trtdt t |d|d|s0tjtj||tr0tdt t |d|d|rAtdt t |d|d tdt t |d t t |d td tdd td|d tjtj||rr|}d}ntjtj||dr|std |dd tj||d}d}nt|r|}t|}n|rt}n|rt}n|durt t |}nt t |}zR||| | | | | |dd|d }t||fi|}|dur;|t t |kr;t|t t |fi|}|durd}n>|r-| dkrt|fi|\}} }| |d<|dur,t|dt t |d t t |dnt t |}t||fi|}|dur[|t t |kr[t|t t |fi|}|dur[d}| sts|dur|t t fvr|rst nt }| | | || d}||| | |dd|d|}t||fi|stt|fddi|ddnq| | | || d}t|tfi|rt|dt t |dt|tfi|rt|dt t |d|durt|t fi|rt|dt t |d|dt|dt t |d t t |d td td td Wn0tytyF}ztd|d|dt t |d td td td |d}~ww|rUtd||}n0td|d|n$|rtj|rn|}n||| | | | | |dd|d }t||fi|}d}|rt||||| | | | | ||d \}}||fS|dur|gnd}||fS) zGet all the checkpoint filenames based on `pretrained_model_name_or_path`, and optional metadata if the checkpoints are sharded. This function will download the data if necesary. FN.indexTzError no file named z found in directory zf but there is a file for TensorFlow weights. Use `from_tf=True` to load this model from those weights.zb but there is a file for Flax weights. Use `from_flax=True` to load this model from those weights.rz, rz$We found a TensorFlow checkpoint at z:, please set from_tf to True to load from this checkpoint.) rrrrrrrr _raise_exceptions_for_gated_repo%_raise_exceptions_for_missing_entries _commit_hashmainrz& does not appear to have a file named z and thus cannot be loaded with `safetensors`. Please make sure that the model has been saved with `safe_serialization=True` or do not set `use_safetensors=True`.)rrrrr)rrrrrrrrignore_errors_during_conversionzThread-auto_conversion)targetrrrz) but there is a file without the variant z;. Use `variant=None` to load this model from those weights.zCan't load the model for 'z'. If you were trying to load it from 'https://huggingface.co/models', make sure you don't have a local directory with the same name. Otherwise, make sure 'z=' is the correct path to a directory containing a file named zloading weights file z from cache at ) rrrrrrrrr)rrr isdirr r rFrErBrrDrCrHrGEnvironmentErrorrrWrNrLr=rTrPr rr^r inforc)rrrrrrrrrrrrrrr is_shardedis_local archive_filefilenameresolved_archive_filecached_file_kwargssafe_weights_namehas_file_kwargsrcsharded_metadatacheckpoint_filesrrr_get_resolved_checkpoint_filesZs6                             r torch_dtyperrrcCsd}|du}|durt|tro|dkrQt|dr*|jdur*|j}td|dn|r5d|vr5|d}n|dur>t|}n t|dd|d }t|}td ntt|rnt t|}||_|j D] } t || } || _qcndt|tj r||_|j D] } t || } || _q}nJt|t r|D]\} } t|| rt || } t| ts| nt t| } | | _q|d }t|ts|nt t|}||_|durtj}ntd |||}nt}||_|j D] } t || } || _q|||fS) aFind the correct `torch_dtype` to use based on provided arguments. Also update the `config` based on the inferred dtype. We do the following: 1. If torch_dtype is not None, we use that dtype 2. If torch_dtype is "auto", we auto-detect dtype from the loaded state_dict, by checking its first weights entry that is of a floating type - we assume all floating dtype weights are of the same dtype we also may have config.torch_dtype available, but we won't rely on it till v5 NautorzWill use torch_dtype=z$ as defined in model's config objectrrrNrzSince the `torch_dtype` attribute can't be found in model's config object, will use torch_dtype={torch_dtype} as derived from model's weightsrdz`torch_dtype` can be one of: `torch.dtype`, `'auto'`, a string of a valid `torch.dtype` or a `dict` with valid `torch_dtype` for each sub-config in composite configs, but received )r\rrrr rrrrr} sub_configsrrdictrrrr_set_default_torch_dtyper)clsrrrrrr dtype_origrsub_config_key sub_configr curr_dtyper default_dtyperrr_get_torch_dtypesr                 r max_memoryc s6t|tri}|dur||||dur&|fdd|D|}|dur1||}||}d|i} dtt j vrG|| d<n t |dkrRt d|dkret|f||d k|d | }nt|}|durr||}|| d <t |fd |i| }|dur|j|d |S|durt|} t| ||S)zCompute the final `device_map` to use if we passed a value in ['auto', 'balanced', 'balanced_low_0', 'sequential']. Otherwise, we check for any device inconsistencies in the device_map. Ncs"i|] \}}|r|tjqSr)rrrrrrrrr rz#_get_device_map..no_split_module_classesspecial_dtypesrzThis model has some weights that should be kept in higher precision, you need to upgrade `accelerate` to properly deal with them (`pip install --upgrade accelerate`). sequentialbalanced_low_0)rlow_zerorrr)r)r\rupdateget_special_dtypes_updatenamed_parametersadjust_target_dtype_get_no_split_modulesinspect signaturernrrr rrrrsadjust_max_memoryvalidate_environmentr)rp) rrrrrrr target_dtypeno_split_modulesdevice_map_kwargs tied_paramsrrr_get_device_mapsR        r'original_checkpoint_keyscheckpoint_keys'loading_base_model_from_task_state_dictcs|jt|}|dur||||}tt|t|}t|t|} |r8fdd|D} | | dd|D} t| | } t dd| D} | rYdd| D} t |} | D]"fd d|Dt d krt t krfd d|D}q_|dur| ||}| || } |jdur|jD] fd d|D}q|jdur|jD] fd d| D} q|| fS)zFind missing keys (keys that are part of the model parameters but were NOT found in the loaded state dict keys) and unexpected keys (keys found in the loaded state dict keys, but that are NOT part of the model parameters) Ncs g|] }|ds|qSre startswithrrzrrr: z5_find_missing_and_unexpected_keys..cSsh|]\}}|qSrr)rnrrrrrh?sz4_find_missing_and_unexpected_keys..css|]}|dVqdS)rotary_emb.inv_freqN)rR)rbufferrrr Dz4_find_missing_and_unexpected_keys..cSsg|]}d|vr|qS)r/rrrrrrFrcsg|]}|vr|qSrrr)rrrrJrrcrrrr)missing_in_grouprrrLrc g|] }t|dur|qSrrrrpatternrrrUr-cr4rr5rr6rrrYr-)base_model_prefixrrrupdate_expected_keysrrr named_buffersanyr)rupdate_missing_keysupdate_unexpected_keys_keys_to_ignore_on_load_missing"_keys_to_ignore_on_load_unexpected)r rr(r)r*rrrr,r-task_specific_keys model_buffershas_inv_freq_buffersr&r)rr3r7r{r!_find_missing_and_unexpected_keys"s<       rCignore_mismatched_sizeskeys_to_rename_mappingc s|sggfS|dur dg}|}g}g} |D]W} | dkr%t| |d|d}fdd|D} | D]9} | |vrm| | j|| jkrm| | jddkr[| | d || ksm|| | | | j|| jfq4q|| fS) a  Find potential shape mismatch between the different state dicts and the model parameters, but only if `ignore_mismatched_sizes` is True. Otherwise, return immediately and any shape mismatch that may exist will be raised later on. This avoids checking every parameter in advance, as shape mismatch are extremely rare in practice. If we want to ignore them however, we do need to check in advance as we need to know which parameters we need to move back from meta to cpu, and initialize correctly. Indeed, as our model initialization takes place at the module level, and not the weight level, in the case of a sharded checkpoint we cannot correctly initialize the weights according to `model._init_weights()` if we perform this check on each state dict at loading time (after the first loaded checkpoint, there are no way to initialize only the mismatched weights if any, without overwriting the previously loaded weights as well because all the module will be initialized, not only the weights that are mismatched). NrdrNrErrc"i|] \}}|vr||qSrrrrErrrrz)_find_mismatched_keys..rrr)rrrrshapenumelr) rrrrDrErErmodel_state_dictmismatched_keysmismatched_shapesr2new_state_dictrrrHr_find_mismatched_keys^s,   rOc@seZdZUded<ded<dS)PipelineParallelrinputsroutputsN)r __module__ __qualname____annotations__rrrrrPs  rPc @s<eZdZdZeddZeddZddZdd Ze d e j fd d Z e d e j fd dZ de d e fddZed-ddZ d.de deede j de jd e f ddZ d/dee deded e fdd Zd!d"Zd0d#ed$ed efd%d&Zd'eeee j effd efd(d)Z *d1d'eeee j effd$ed efd+d,ZdS)2rzH A few utilities for `torch.nn.Modules`, to be used as a mixin. cOsFzddl}Wn tytdw|t}|}|j|_dS)NrFYou need to install psutil (pip install psutil) to use memory tracing.)psutil ImportErrorProcessrgetpid memory_inforssmem_rss_pre_forward)rrrrWprocessmemrrr_hook_rss_memory_pre_forwards  z-ModuleUtilsMixin._hook_rss_memory_pre_forwardcOslzddl}Wn tytdw|t}|}|j|_|j|j}|t |dr0|j nd|_ dS)NrrV mem_rss_diff) rWrXrYrrZr[r\mem_rss_post_forwardr]rra)rrrrWr^r_rarrr_hook_rss_memory_post_forwards   z.ModuleUtilsMixin._hook_rss_memory_post_forwardcCs2|D]}||j||jq|dS)a% Add a memory hook before and after each sub-module forward pass to record increase in memory consumption. Increase in memory consumption is stored in a `mem_rss_diff` attribute for each module and can be reset to zero with `model.reset_memory_hooks_state()`. N)rregister_forward_pre_hookr`register_forward_hookrcreset_memory_hooks_stateselfrrrradd_memory_hookss   z!ModuleUtilsMixin.add_memory_hookscCs$|D] }d|_d|_d|_qdS)z Reset the `mem_rss_diff` attribute of each module (see [`~modeling_utils.ModuleUtilsMixin.add_memory_hooks`]). rN)rrarbr]rgrrrrfs z)ModuleUtilsMixin.reset_memory_hooks_statercCt|S)z `torch.device`: The device on which the module is (assuming that all the module parameters are on the same device). )rrhrrrrzModuleUtilsMixin.devicecCrj)zw `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype). )rrkrrrrszModuleUtilsMixin.dtypeencoder_attention_maskcCst|dkr|dddddddf}|dkr&|ddddddf}|j|jd}d|t|jj}|S)z Invert an attention mask (e.g., switches 0. and 1.). Args: encoder_attention_mask (`torch.Tensor`): An attention mask. Returns: `torch.Tensor`: The inverted attention mask. Nrr?)dimrrrfinfomin)rhrmencoder_extended_attention_maskrrrinvert_attention_masks  z&ModuleUtilsMixin.invert_attention_maskNc Cs|dur tdtn|j}|\}}tj||d}|ddddf||d|ddddfk}||j}|j d|j dkr]|j d|j d}tj tj |||f||jd|gdd}|dddddddf|ddddddf}|S)NNThe `device` argument is deprecated and will be removed in v5 of Transformers.)rrrrraxis) warningswarn FutureWarningrrarangerepeatrrrIcatones) input_shapeattention_maskr batch_size seq_lengthseq_ids causal_maskprefix_seq_lenextended_attention_maskrrr*create_extended_attention_mask_for_decoders&. 4z;ModuleUtilsMixin.create_extended_attention_mask_for_decoderrrrrcCs|dur|j}|dkr|jjs|durtdt|dkr0|dddddddf}n+|dkrO|jjrBt|||}n|ddddddf}n t d|d|j d|j |d}d |t |j}|S) a Makes broadcastable attention and causal masks so that future and masked tokens are ignored. Arguments: attention_mask (`torch.Tensor`): Mask with ones indicating tokens to attend to, zeros for tokens to ignore. input_shape (`Tuple[int]`): The shape of the input to the model. Returns: `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`. Nrrvrnz!Wrong shape for input_ids (shape z) or attention_mask (shape )rorp)rrqr is_decoderrzr{r|rrrrIrrrrrs)rhrrrrrrrrget_extended_attention_masks*   z,ModuleUtilsMixin.get_extended_attention_maskF head_masknum_hidden_layersis_attention_chunkedcCs8|dur|||}|dur|d}|Sdg|}|S)a Prepare the head mask if needed. Args: head_mask (`torch.Tensor` with shape `[num_heads]` or `[num_hidden_layers x num_heads]`, *optional*): The mask indicating if we should keep the heads or not (1.0 for keep, 0.0 for discard). num_hidden_layers (`int`): The number of hidden layers in the model. is_attention_chunked (`bool`, *optional*, defaults to `False`): Whether or not the attentions scores are computed by chunks or not. Returns: `torch.Tensor` with shape `[num_hidden_layers x batch x num_heads x seq_length x seq_length]` or list with `[None]` for each layer. NTr)_convert_head_mask_to_5d unsqueeze)rhrrrrrr get_head_maskFs   zModuleUtilsMixin.get_head_maskcCs|dkr|dddd}||dddd}n|dkr/|ddd}|dks>Jd||j|jd}|S)zD-> [num_hidden_layers x batch x num_heads x seq_length x seq_length]rrrrzhead_mask.dim != 5, instead ro)rqrexpandrr)rhrrrrrras  z)ModuleUtilsMixin._convert_head_mask_to_5donly_trainableexclude_embeddingsc s|rdd|Dfdd|D}nt|}g}t|dd}|r3tr/ddl}ntd|D]9}|js<|sn|rgt ||j j rgt |d rO| }n t |d rY|jj}nd }||d |q5||q5t|S) a Get number of (optionally, trainable or non-embeddings) parameters in the module. Args: only_trainable (`bool`, *optional*, defaults to `False`): Whether or not to return only the number of trainable parameters exclude_embeddings (`bool`, *optional*, defaults to `False`): Whether or not to return only the number of non-embeddings parameters Returns: `int`: The number of parameters. cSs&g|]\}}t|tjr|dqS).weight)r\r Embedding)rr module_typerrrr|s z3ModuleUtilsMixin.num_parameters..csg|] \}}|vr|qSrr)rrrembedding_param_namesrrrsis_loaded_in_4bitFrNzbitsandbytes is not installed but it seems that the model has been loaded in 4bit precision, something went wrong make sure to install bitsandbytes with `pip install bitsandbytes`. You also need a GPU. ru quant_storagerr)rjrrrr}rR bitsandbytesrrr\r Params4bitrruritemsizerrJsum) rhrrtotal_parameters total_numelrbnbr num_bytesrrrnum_parametersls8         zModuleUtilsMixin.num_parameters input_dictcCsJt|dsi|_|j|vr||jSd|jvr#tdd|jd<dS)z Helper function to estimate the total number of tokens from the model inputs. Args: inputs (`dict`): The model inputs. Returns: `int`: The total number of tokens. warnings_issuedestimate_tokenszdCould not estimate the number of tokens of the input, floating-point operations will not be computedTr)rrmain_input_namerJr r)rhrrrrrs    z ModuleUtilsMixin.estimate_tokensTcCsd|||j|dS)a Get number of (optionally, non-embeddings) floating-point operations for the forward and backward passes of a batch with this transformer model. Default approximation neglects the quadratic dependency on the number of tokens (valid if `12 * d_model << sequence_length`) as laid out in [this paper](https://arxiv.org/pdf/2001.08361.pdf) section 2.1. Should be overridden for transformers with parameter re-use e.g. Albert or Universal Transformers, or if doing long-range modeling with very high sequence lengths. Args: batch_size (`int`): The batch size for the forward pass. sequence_length (`int`): The number of tokens in each line of the batch. exclude_embeddings (`bool`, *optional*, defaults to `True`): Whether or not to count embedding and softmax operations. Returns: `int`: The number of floating-point operations. )r)rr)rhrrrrrfloating_point_opssz#ModuleUtilsMixin.floating_point_opsrNNFFFT)rrSrT__doc__ staticmethodr`rcrirfpropertyrrrrrurrrrrrboolrrrr rrr rrrrrrrs`       5  $6c!sXeZdZdZdZdZdZdZdZdZ dZ dZ dZ dZ dZdZdZdZdZdZdZdZdZdZdZdZdZdZedeeej ffddZ!edefd d Z"d e#ffd d Z$ddZ%ddZ&ddZ'de(e)eefddfddZ*e+e,ddZ-e+    dde.de/ej0de/e(eeee1ffde.fddZ2e+d ej0dej0fd!d"Z3ede4j5fd#d$Z6e+de.fd%d&Z7e+    dde/ej0de/e(eeee1ffde.d'e.de#f d(d)Z8e+dd'e.de#fd*d+Z9e+dd'e.de#fd,d-Z:d.d/Z;d0d1Zde4j5fd7d8Z?d9d:Z@d;d<ZAd=d>ZBeCd?e4j5d@e4j5dAedBefdCdDZDdEdFZEdefdGdHZF   ddIe/e1dJe/e1dKe.de4jGfdLdMZHddNdOZI   ddPe4jGdIe/e1dJe/e1dKe.de4jGf dQdRZJ   ddSe4jKdIe/e1dTe/e.dKe.de4jKf dUdVZLdWdXZM ddYdZZNd[d\ZOd]d^ZPd_e1fd`daZQde(e4jGeRe4jGffdbdcZSdddeZTdfee1e)e1ffdgdhZUddidjZVdeWfdke.dleXfdmdnZYdodpZZede.fdqdrZ[ddej\ddsddddf dte(ee]j^fdue.dve/e_dweXdxe.dye(e1efdze.d{e/ed|e/e(ee.fd}e.fd~dZ`eaebjcfddZcdddZdeaej4j5jefddZeeaej4j5jffddZffddZgfddZhe+de.de.fddZie+e,dddddddddd dejekde/e(ee]j^fd e/e(e#ee]j^fde/e(ee]j^fde.de.de.d|e/e(ee.fdede/e.de.dekfddZleCdedeRee.ffddZm   dde)ede/eeefde.de.fddZneCdeRee.ffddZoddZpe+           ddddve/ede/e)ede/ede.de/ede/ede/ede/e.d e/ej0de/eqde/erjsde/dde/eeefde.fddZte+ddZue+ddZvdddZwe+dddZxdddZyddZzddÄZ{eddńZ|eddDŽZ}eddɄZ~e~jddɄZ~defdd̈́Ze+ddτZde)ede)ed e/ej0de/eqddf ddӄZde)ede.de.ddfddքZdefddلZZS)ra Base class for all models. [`PreTrainedModel`] takes care of storing the configuration of the models and handles methods for loading, downloading and saving models as well as a few methods common to all models to: - resize the input embeddings, - prune heads in the self-attention heads. Class attributes (overridden by derived classes): - **config_class** ([`PretrainedConfig`]) -- A subclass of [`PretrainedConfig`] to use as configuration class for this model architecture. - **load_tf_weights** (`Callable`) -- A python *method* for loading a TensorFlow checkpoint in a PyTorch model, taking as arguments: - **model** ([`PreTrainedModel`]) -- An instance of the model on which to load the TensorFlow checkpoint. - **config** ([`PreTrainedConfig`]) -- An instance of the configuration associated to the model. - **path** (`str`) -- A path to the TensorFlow checkpoint. - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model. - **is_parallelizable** (`bool`) -- A flag indicating whether this model supports model parallelization. - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for NLP models, `pixel_values` for vision models and `input_values` for speech models). Nrd input_idsFrcCsdttiS)z^ `Dict[str, torch.Tensor]`: Dummy inputs to do a forward pass in the network. r)rrqrArkrrr dummy_inputs3zPreTrainedModel.dummy_inputscCdS)z@ :str: Identifies that this is a PyTorch model. rGrrkrrrrI:szPreTrainedModel.frameworkrcstt|tstd|jjd|jjdt|dds3t|dr'|j nt }|j ||dd}||_ |jj}|tvr\dd td }t||jj}t|d krZ|d }nd}||_|j|_i|_|rot|nd|_t|jj|_|jp~g|_dS) NzParameter config in `zt(config)` should be an instance of class `PretrainedConfig`. To create a model from a pretrained model use `model = z(.from_pretrained(PRETRAINED_MODEL_NAME)`_attn_implementation_autosetFr)rcheck_device_map(rrr)super__init__r\r!rrrr}rrrr_autoset_attn_implementationrr2r rfindallr loss_type name_or_pathr can_generater$from_model_configgeneration_configcopy_keep_in_fp32_modules_no_split_modules)rhrrQrrr loss_groupsrrrrAs0      zPreTrainedModel.__init__csN|||jdur@dd|D}t}|D]}|dd|dDq|jD]}||vr?t|d|jj q.|j j durL|j j nd|_ |j jdur[|j j ni|_|D]\}t|dd}r|jfd d | Dqb|jdurtd r|jD]\}}|tvrtd |d tqdSdSdS)z A method executed at the end of each Transformer model initialization, to execute code that needs the model's modules properly initialized (such as weight initialization). NcSs h|] \}}t|dkr|qSrrrrrrrhnr-z,PreTrainedModel.post_init..cSs g|] }|s|dvr|qS))rbias) isnumericrrrrrrsr-z-PreTrainedModel.post_init..rzV was specified in the `_keep_in_fp32_modules` list, but is not part of the modules in _tp_planc i|] \}}d||qSrerrrrrrr-z-PreTrainedModel.post_init..z2.3z"Unsupported tensor parallel style z. Supported styles are ) init_weights._backward_compatibility_gradient_checkpointingrrrrsplitrrrrbase_model_pp_planr_pp_planbase_model_tp_planrrr}rrZr0)rhall_parametersunique_module_namesrrplanrrrrr post_initds@  "zPreTrainedModel.post_initcCs&t|dd}|durtd||S)z Potentially dequantize the model in case it has been quantized by a quantization method that support dequantization. rNz?You need to first quantize your model in order to dequantize it)r}r dequantize)rhrrrrrs  zPreTrainedModel.dequantizecCs4|jrt|jddr|t|jddSdSdS)Ngradient_checkpointingF)supports_gradient_checkpointingr}rgradient_checkpointing_enabledelattrrkrrrrsz>PreTrainedModel._backward_compatibility_gradient_checkpointingtagscCsDt|tr|g}|jdurg|_|D] }||jvr|j|qdS)a\ Add custom tags into the model that gets pushed to the Hugging Face Hub. Will not overwrite existing tags in the model. Args: tags (`Union[List[str], str]`): The desired tags to inject in the model Examples: ```python from transformers import AutoModel model = AutoModel.from_pretrained("google-bert/bert-base-cased") model.add_model_tags(["custom", "custom-bert"]) # Push the model to your namespace with the name "my-custom-bert". model.push_to_hub("my-custom-bert") ``` N)r\r model_tagsr)rhrtagrrradd_model_tagss    zPreTrainedModel.add_model_tagsc Ks|d|j}t|trtt|}|dd}d}|dur"||}t|}|j dur0|j }nd}|d||_ t|ddsH|j ||d|d}t r|t s|ts|tdtjjtd tg}t|||fi|}Wdn1svwYn||fi|}|durt||S) z All context managers that the model should be initialized under go here. Args: torch_dtype (`torch.dtype`, *optional*): Override the default `torch.dtype` and load the model under this dtype. ruse_flash_attention_2FNattn_implementationr)rrr@Detected DeepSpeed ZeRO-3: activating zero.init() for this modelconfig_dict_or_path)rrr\rr}rr rdeepcopy_attn_implementation_internal_attn_implementationrr(rrr r deepspeedzeroInitr'rrIr) r rrrrr r init_contextsrrrr _from_configs<          zPreTrainedModel._from_configTrrrrc Csd}t|drQ|jdurQ|jdkr|rtd|jdt|jtsN|jdgtvrNd|jd}|jr:|d 7}|j rA|d 7}|j rH|d 7}t|d |j}|j D]}t ||} t|tsd|n||d} | durq| | _qV|r|td d|_|jdkr|j|||d|dn`|dkr|j|dd}nT|dvrts|j||durdndd}tjjdur|jdkrtjdkrttjtdkrtdtjjdn|tvr||_n t|trd|_nd|_d|_|S)az Automatically checks and dispatches to a default attention implementation. In order of priority: 1. An implementation specified in `config._attn_implementation` (due for example to the argument attn_implementation="sdpa" in from_pretrained). 2. DEPRECATED: if use_flash_attention_2 is set to `True` and `flash_attn` is available, flash attention. (`LlamaFlashAttention` for example) 3. SDPA implementation, if available and supported by the model type. (`LlamaSdpaAttention` for example) 4. The default model's implementation otherwise (`LlamaAttention` for example) . Nrflash_attention_2zBoth attn_implementation="z" and `use_flash_attention_2=True` were used when loading the model, which are not compatible. We recommend to just use `attn_implementation="flash_attention_2"` when loading the model.eagerz Specified `attn_implementation="zt"` is not supported. The only possible arguments are `attn_implementation="eager"` (manual attention implementation)zT, `"attn_implementation=flash_attention_2"` (implementation using flash attention 2)zf, `"attn_implementation=sdpa"` (implementation using torch.nn.functional.scaled_dot_product_attention)zV, `"attn_implementation=flex_attention"` (implementation using torch's flex_attention)rzThe model was loaded with use_flash_attention_2=True, which is deprecated and may be removed in a future release. Please use `attn_implementation="flash_attention_2"` instead.F)rrhard_check_onlyrflex_attentionT)r)Nsdparrz2.4.1zUsing the `SDPA` attention implementation on multi-gpu setup with ROCM may lead to performance issues due to the FA backend. Disabling it to use alternative backends.)rrrrr\rALL_ATTENTION_FUNCTIONS valid_keys_supports_flash_attn_2_supports_sdpa_supports_flex_attnrrr}rr  warning_once_check_and_enable_flash_attn_2_check_and_enable_flex_attnr^_check_and_enable_sdparrhipcuda device_countr]rbackendsenable_flash_sdpr) r rrrrrrequested_attn_implementationmessagerr curr_attn_implementationrrrrs          z,PreTrainedModel._autoset_attn_implementationrcCsN|jstd|jd|dtd|jd|dt}t||S)a Change the default dtype and return the previous one. This is needed when wanting to instantiate the model under specific dtype. Args: dtype (`torch.dtype`): a floating dtype to set to. Returns: `torch.dtype`: the original `dtype` that can be used to restore `torch.set_default_dtype(dtype)` if it was modified. If it wasn't, returns `None`. Note `set_default_dtype` currently only works with floating-point types and asserts if for example, `torch.int64` is passed. So if a non-float `dtype` is passed this functions will throw an exception. zCan't instantiate z model under dtype=z' since it is not a floating point dtypezInstantiating z model under default dtype r)rrrr rrrr)r rr rrrr as z(PreTrainedModel._set_default_torch_dtypecCst||j|S)z@ `torch.nn.Module`: The main body of the model. )r}r8rkrrr base_model|rzPreTrainedModel.base_modelcCsldt|jvr dS|jD]}t|dsq dt|vr!|r!dSq dt|jvr4t|jddSdS)a Returns whether this model can generate sequences with `.generate()` from the `GenerationMixin`. Under the hood, on classes where this function returns True, some generation-specific changes are triggered: for instance, the model instance will have a populated `generation_config` attribute. Returns: `bool`: Whether this model can generate sequences with `.generate()`. r%Trru: has generative capabilities, as `prepare_inputs_for_generation` is explicitly overwritten. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions. - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception). - If you are not the owner of the model architecture class, please contact the model code owner to update it.F)r __bases__rrprepare_inputs_for_generationr rr)r baserrrrs     zPreTrainedModel.can_generaterc Cs|jst|jd|jdtsd}d}tjddur6tr-|s&d|_ t d|St |d |t tj d}tj jrk|t d krVt |d |d |tjsbt|d t |d|tj jr|t dkrt |d|d |t |d|t|dd} | rtd|durt dn|dur|tjtjfvrt d|jd|d|r|durtdjjdvrtjrt dn(trt dntd|r|durt|trd|vsd|vrtd|sd|_ |S) a9 Checks the availability of Flash Attention 2 and compatibility with the current model. If all checks pass and `hard_check_only` is False, the method will set the config attribute `attn_implementation` to "flash_attention_2" so that the model can initialize the correct attention module. z does not support Flash Attention 2.0 yet. Please request to add support where the model is hosted, on its model hub page: https://huggingface.co/zk/discussions/new or in the Transformers GitHub repo: https://github.com/huggingface/transformers/issues/newzVFlashAttention2 has been toggled on, but it cannot be used due to the following error:zPlease refer to the documentation of https://huggingface.co/docs/transformers/perf_infer_gpu_one#flashattention-2 to install Flash Attention 2. flash_attnNrz+Detect using FlashAttention2 on Ascend NPU.z3 the package flash_attn seems to be not installed. r?zY you need flash_attn package version to be greater or equal than 2.1.0. Detected version z. z\ Flash Attention 2 is not available on CPU. Please make sure torch can access a CUDA device.z% Flash Attention 2 is not available. z2.0.4z you need flash_attn package version to be greater or equal than 2.0.4. Make sure to have that version installed - detected version use_bettertransformerFzFlash Attention 2 and BetterTransformer API are not compatible. Please make sure to disable BetterTransformers by doing model.reverse_bettertransformer()zwYou are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviourzcFlash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in z is aG. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)`r)rmluzYou are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.zYou are attempting to use Flash Attention 2.0 with a model not initialized on MLU. Make sure to move the model to MLU after initializing it on CPU with `model.to('mlu')`.a-You are attempting to use Flash Attention 2.0 with a model not initialized on GPU and with no GPU available. This is not supported yet. Please make sure to have access to a GPU and either initialise the model on a GPU by passing a device_map or initialising the model on CPU and then moving it to GPU.rrzYou are attempting to use Flash Attention 2.0 with a model dispatched on CPU or disk. This is not supported. Please make sure to initialise the model on a GPU by passing a device_map that contains only GPU devices as keys.)rrr _name_or_pathrS importlibutil find_specr\rr rrXrr]rSrrrrr}rfloat16rrXrrr[r\rr) r rrrrrprefaceinstall_messageflash_attention_version_is_bettertransformerrrrrs    z.PreTrainedModel._check_and_enable_flash_attn_2cCsZ|r|js t|jdtstdtr|js|St|dd}|r&|S|s+d|_|S)a  Checks the availability of SDPA for a given model. If all checks pass and `hard_check_only` is False, the method will set the config attribute `_attn_implementation` to "sdpa" so that the model can initialize the correct attention module. a does not support an attention implementation through torch.nn.functional.scaled_dot_product_attention yet. Please request the support for this architecture: https://github.com/huggingface/transformers/issues/28005. If you believe this error is a bug, please open an issue in Transformers GitHub repository and load your model with the argument `attn_implementation="eager"` meanwhile. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="eager")`zSPyTorch SDPA requirements in Transformers are not met. Please install torch>=2.1.1.r Fr)rrrr]rXr}r)r rrrrrrr s"   z&PreTrainedModel._check_and_enable_sdpacCsF|r|js t|jdtstdtr|js|S|s!d|_|S)a Checks the availability of Flex Attention for a given model. If all checks pass and `hard_check_only` is False, the method will set the config attribute `_attn_implementation` to "flex_attention" so that the model can initialize the correct attention module. a does not support an attention implementation through torch's flex_attention. Please request the support for this architecture: https://github.com/huggingface/transformers/issues/34809. If you believe this error is a bug, please open an issue in Transformers GitHub repository and load your model with the argument `attn_implementation="eager"` meanwhile. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="eager")`z]PyTorch Flex Attention requirements in Transformers are not met. Please install torch>=2.5.0.r)rrrrYrXr)r rrrrrr3 s  z+PreTrainedModel._check_and_enable_flex_attncCsdd}|||_dS)z Enables the gradients for the input embeddings. This is useful for fine-tuning adapter weights while keeping the model weights fixed. cSs|ddSNT)requires_grad_)rinputoutputrrrmake_inputs_require_gradsV szMPreTrainedModel.enable_input_require_grads..make_inputs_require_gradsN)get_input_embeddingsre_require_grads_hook)rhrrrrenable_input_require_gradsP sz*PreTrainedModel.enable_input_require_gradscCs|jdS)z4 Removes the `_require_grads_hook`. N)rremoverkrrrdisable_input_require_grads[ sz+PreTrainedModel.disable_input_require_gradscCs"t||j|}||ur|St)z Returns the model's input embeddings. Returns: `nn.Module`: A torch module mapping vocabulary to hidden states. )r}r8rNotImplementedError)rhrrrrra sz$PreTrainedModel.get_input_embeddingsrcCs(t||j|}||ur||dSt)z Set model's input embeddings. Args: value (`nn.Module`): A module mapping vocabulary to hidden states. N)r}r8set_input_embeddingsr")rhrrrrrr#n sz$PreTrainedModel.set_input_embeddingscCr)z Returns the model's output embeddings. Returns: `nn.Module`: A torch module mapping hidden states to vocabulary. Nrrkrrrget_output_embeddings{ z%PreTrainedModel.get_output_embeddingscCr)a] Initialize the weights. This method should be overridden by derived class and is the only initialization method that will be called when loading a checkpoint using `from_pretrained`. Any attempt to initialize outside of this function will be useless as the torch.nn.init function are all replaced with skip. Nrrgrrrr r%zPreTrainedModel._init_weightscCs$t|ddrdS||d|_dS)zM Initialize the weights if they are not already initialized. rlFNT)r}rrlrgrrr_initialize_weights s   z#PreTrainedModel._initialize_weightscCst|jjddddr|}|dur|||t|jddrCt|jddrCt||jr5t||j}||j |j |jd}||_ | D] }t|d rR| qGdS) z Tie the weights between the input embeddings and the output embeddings. If the `torchscript` flag is set in the configuration, can't handle parameter sharing so we are cloning the weights instead. Tdecodertie_word_embeddingsNis_encoder_decoderFtie_encoder_decoderencoder _tie_weights)r}rget_text_configr$_tie_or_clone_weightsrrr8_tie_encoder_decoder_weightsr,r(r|rr-)rhoutput_embeddings tied_weightsrrrr tie_weights s     zPreTrainedModel.tie_weightsr,r(r8base_encoder_namec sg}g|j|jkrt|jd|jd   d dtjdtjdtdtd ttf fd d |||||t|dkrGtd |S)N and zZ are not equal. In this case make sure that all encoder weights are correctly initialized.rrddecoder_pointerencoder_pointerrgr4uninitialized_encoder_weightsc st|tjr t|tjsJ|d|dt|drLt|ds"J|j|_||dt|drJt|ds.tie_encoder_to_decoder_recursively..rizMax depth of recursive function `tie_encoder_to_decoder` reached. It seems that there is a circular dependency between two or more `nn.Modules` of your model.r9r)depthtotal_encoder_nametotal_decoder_name)r\rrrrrr_modulesrrrisdigitrrrrr r)r6r7rgr4r8r;r=r<encoder_modulesdecoder_modulesall_encoder_weightsencoder_layer_posrr encoder_name decoder_name"tie_encoder_to_decoder_recursivelyr2rfrrG sh        zXPreTrainedModel._tie_encoder_decoder_weights..tie_encoder_to_decoder_recursivelyz;The following encoder weights were not tied to the decoder )rrdrd) rr rrrrr rr)r,r(r8r4r8rrFrr0 s8  D  z,PreTrainedModel._tie_encoder_decoder_weightscCs|jjrt|j|_n|j|_t|dddur3tj|j j d|jj d|j j dfdd|j _ t |drCt |drE|j |_dSdSdS)zPTie or clone module weights depending of whether we are using TorchScript or notrNrconstant out_featuresnum_embeddings)r torchscriptr Parameterrcloner} functionalpadrrrIrrJrI)rhr1input_embeddingsrrrr/ s z%PreTrainedModel._tie_or_clone_weightscCst}|g}t|dkrB|d}|jj|vr= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities won't be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Embedding`: Pointer to the input tokens Embeddings Module of the model. Nr modifier_rankr) _resize_token_embeddingsrrr(rrGatheredParametersrrIrr. vocab_sizer3)rhrSrTrU model_embedsrErZrrrresize_token_embeddings< s$   z'PreTrainedModel.resize_token_embeddingsc CsR|}|||||}t|dr|j}t|||jj}||||t|do0|j du}|dur`t rZ|sZt j j |jdd|jjd}Wdn1sTwYn|jjd}|dur|jjddjs|} t| tjjr|j| ||d} n|j| ||d} t| dr| j}t| || jj} | | || |S)N_hf_hookrrVrTr')rU)r_get_resized_embeddingsrr]rorrrr#rr(rrrYrIr$rr.r)r\rrr_get_resized_lm_headset_output_embeddings) rhrSrTrUold_embeddingsnew_embeddingshookold_embeddings_requires_gradrE old_lm_head new_lm_headold_lm_head_requires_gradrrrrXu s@           z(PreTrainedModel._resize_token_embeddingsrac Csl|dur&t|tstd|d|dur|jjd}||d||}n td|d|dur5|St|do>|jdu}t re|set j j |jdd |j \}}Wdn1s_wYn|j \}}||krut su|St|tjstd t|d tjd tjd tj|||jj|jjd}||kr|s||nA||kr|rtd||} t r|st j j |jgdd |||||| Wdn1swYn |||||| t||} t r)|s)|j|jg} t j j | dd |jjd| ddf|jjd| ddf<Wdn 1s#wYn|jjd| ddf|jjd| ddf<t r|s|j|jg} t j j | dd 6|j|_|jjjd|_|jdurz|d|jkrd|_Wd|SWd|SWd|S1swY|S|jj|j_|jjjd|_|jdur|d|jkrd|_|S)a Build a resized Embedding Module from a provided token Embedding Module. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end Args: old_embeddings (`torch.nn.Embedding`): Old embeddings to be resized. new_num_tokens (`int`, *optional*): New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just returns a pointer to the input tokens `torch.nn.Embedding` module of the model without doing anything. pad_to_multiple_of (`int`, *optional*): If set will pad the embedding matrix to a multiple of the provided value. If `new_num_tokens` is set to `None` will just pad the embedding to a multiple of `pad_to_multiple_of`. This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities will not be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Embedding`: Pointer to the resized Embedding Module or the old Embedding Module if `new_num_tokens` is `None` Nz5Asking to pad the embedding matrix to a multiple of `z@`, which is not and integer. Please make sure to pass an integerrrzYou are resizing the embedding layer without providing a `pad_to_multiple_of` parameter. This means that the new embedding dimension will be a.. This might induce some performance reduction as *Tensor Cores* will not be available. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tcrrVzOld embeddings are of type , which is not an instance of zj. You should either use a different resize function or make sure that `old_embeddings` are an instance of rrwzThe new embeddings will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`)r\rrrrIr rrrr(rrrYrOrr TypeErrorrrrrr(_init_added_embeddings_weights_with_meanrsrrJ padding_idx) rhrarSrTrUrEold_num_tokensold_embedding_dimrbadded_num_tokensr.paramsrrrr^ s+              .,     z'PreTrainedModel._get_resized_embeddingsre transposedc Cs||dur|St|do|jdu}tr?|s?tjj|jdd|s&|jn|j\}}Wdn1s9wYn|sF|jn|j\}}||krXtsX|St |t j spt dt |dt j dt j d|sv||fn||f}|jdu} t j || |jj|jjd} ||kr|s|| n_||kr|rtd ||} tr|s|jg} | r| |jg7} tjj| dd||| ||| || r||| | Wdn1swYn||| ||| || r||| | t||} tr3|s3|j|j| j| jg} tjj| d d|| || || Wd| S1s,wY| S|| || || | S) a Build a resized Linear Module from a provided old Linear Module. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end Args: old_lm_head (`torch.nn.Linear`): Old lm head liner layer to be resized. new_num_tokens (`int`, *optional*): New number of tokens in the linear matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just returns a pointer to the input tokens `torch.nn.Linear` module of the model without doing anything. transposed (`bool`, *optional*, defaults to `False`): Whether `old_lm_head` is transposed or not. If True `old_lm_head.size()` is `lm_head_dim, vocab_size` else `vocab_size, lm_head_dim`. mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities will not be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Linear`: Pointer to the resized Linear Module or the old Linear Module if `new_num_tokens` is `None` NrrVz#Old language model head is of type rhzg. You should either use a different resize function or make sure that `old_lm_head` are an instance of r)rrraThe new lm_head weights will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`r)rrr(rrrYrrOrr\rLinearrirrrrrr r%_init_added_lm_head_weights_with_mean"_init_added_lm_head_bias_with_meanrs!_copy_lm_head_original_to_resized)rhrerSrprUrErlold_lm_head_dimnew_lm_head_shapehas_new_lm_head_biasrfrnronum_tokens_to_copyrrrr_3 s#                z$PreTrainedModel._get_resized_lm_headc Cs|jjtj}tj|dd}||}|j||} d} tj | |  } | rMtj j j || | d} | j|fd|jj|jjd|dddf<dS|dddf|d|jj|jjd|dddf<dS)Nrrx& .>)covariance_matrix) sample_shaperr)rrrrrmeanTrpositive_definitecheckall distributionsmultivariate_normalMultivariateNormalsamplerr~) rhrarbrmrlrnold_embeddings_weightmean_embeddingsold_centered_embeddings covarianceepsilonis_covariance_psd distributionrrrrj s"  z8PreTrainedModel._init_added_embeddings_weights_with_meancCsZ|r|jjj|j_|jjj|j_|||||||r+|jjj|j_|jjj|j_dSdSr)rrr}rj)rhrerfrurlrnrprrrrr s  z5PreTrainedModel._init_added_lm_head_weights_with_meancCsVtj|jjdtjd}tj|jjddtj}|jjd|dj|d|ddS)Nr)ryrrxrry)r|std)rr|rrrrrr)rhrerfrn bias_meanbias_stdrrrrs s&z2PreTrainedModel._init_added_lm_head_bias_with_meancCs|s|jjd|ddf|jjd|ddf<n|jjddd|f|jjddd|f<|rA|jjd||jjd|<dSdSr)rrr)rhrfrerxrprwrrrrt s ., z1PreTrainedModel._copy_lm_head_original_to_resizednew_num_position_embeddingscC$td|jd|jd|jjd)Nz4`resize_position_embeddings` is not implemented for B`. To implement it, you should overwrite this method in the class  in `modeling_.py`r"rrS)rhrrrrresize_position_embeddings  z*PreTrainedModel.resize_position_embeddingscCr)Nz1`get_position_embeddings` is not implemented for rrrrrkrrrget_position_embeddings rz'PreTrainedModel.get_position_embeddingscCs6|jjr ||jjtr||j|dSdS)z If needed prunes and maybe initializes weights. If using a custom `PreTrainedModel`, you need to implement any initialization logic in `_init_weights`. N)r pruned_heads prune_headsrapplyr&r3rkrrrr s   zPreTrainedModel.init_weightsheads_to_prunecCsN|D]\}}t|jj|gt|B}t||jj|<q|j|dS)a Prunes heads of the base model. Arguments: heads_to_prune (`Dict[int, List[int]]`): Dictionary with keys being selected layer indices (`int`) and associated values being the list of heads to prune in said layer (list of `int`). For instance {1: [0, 2], 2: [2, 3]} will prune heads 0 and 2 on layer 1 and heads 2 and 3 on layer 2. N)rrrrrrr _prune_heads)rhrlayerheads union_headsrrrr s zPreTrainedModel.prune_headscCs|js t|jjd|durddi}tjtfi|}dt|j j v}|s0|j d|dn| t|j ddt dt|d d rK|dSdS) az Activates gradient checkpointing for the current model. Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint activations". We pass the `__call__` method of the modules instead of `forward` because `__call__` attaches all the hooks of the module. https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2 Args: gradient_checkpointing_kwargs (dict, *optional*): Additional keyword arguments passed along to the `torch.utils.checkpoint.checkpoint` function. z) does not support gradient checkpointing.N use_reentrantTr)enablegradient_checkpointing_funcrVYou are using an old version of the checkpointing format that is deprecated (We will also silently ignore `gradient_checkpointing_kwargs` in case you passed it).Please update to the new format on your modeling file. To use the new format, you need to completely remove the definition of the method `_set_gradient_checkpointing` in your model._hf_peft_config_loadedF)rrrr functoolsrrrr _set_gradient_checkpointingrrr rr}r)rhgradient_checkpointing_kwargsr_is_using_old_formatrrrr s  z-PreTrainedModel.gradient_checkpointing_enablerrcCs`d}t|dr||_||_d}|D]}t|dr"||_||_d}q|s.t|jjddS)NFrTz is not compatible with gradient checkpointing. Make sure all the architecture support it by setting a boolean attribute `gradient_checkpointing` to modules of the model that uses checkpointing.)r_gradient_checkpointing_funcrrrrr)rhrris_gradient_checkpointing_setrrrrrH s     z+PreTrainedModel._set_gradient_checkpointingcCsd|jr$dt|jjv}|s|jddntd|t|jddt |ddr0| dSdS)z Deactivates gradient checkpointing for the current model. Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint activations". rF)rrrrN) rrr rrr rrrr}r!)rhrrrrgradient_checkpointing_disable^ s  z.PreTrainedModel.gradient_checkpointing_disablecCstdd|DS)z Whether gradient checkpointing is activated for this model or not. Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint activations". css |] }t|do |jVqdS)rN)rr)rmrrrr1} sz.)r;rrkrrris_gradient_checkpointingu sz)PreTrainedModel.is_gradient_checkpointing5GBsave_directoryis_main_processr save_function push_to_hubmax_shard_sizesafe_serializationrrsave_peft_formatc H s| dd} | dd} | dur tdt| durtd| } | dur(| | d<t|dd}t|d d}|duoBt|toB|j|d }|durU|sU|sUtd |j j d d | vrctd| d }|rlt slt dt j|r}td|ddSt j|dd|r| dd}| d|t jjd}|j|fi| }||}t|}t|}t|dd|j_|jjg|j_d|j_|j durt!|||jd|rc|s|j"}|#r t$|dkr td|dt%|&D]\}}t'|j(||t'|j|dq|j)||#r|j()||rct*d|j+|d}| rFt*di}|&D] \}}||d |<q7|}|,}t$|dkrUtd!|d}|j-|}|)|i}|durt.|d"rt$t/|j01dkrd#|j01vsd$|j01vrtd%|2D]\}d&krq|3} | D] }||d|<qq|3}t4rt5j6j7j8D] \}!}"|!|}q|j9dur|j9D] }#|#|:vr||#=q|;|}|rt<=t>}$|&D] \}%t|%t?j@r |$tA|%Bq|$tC|%Bqt.|d"r4tD|}&|&r1|&dfd'd(|$&D}'n i}'n d)d(|$&D}'tE|}(g})t/}*|'1D]5}+|(dur~d},tF|+D]&tGfd*d+|(D}-|-r||vr||,d7},|,t$|+kr||*HqWqJtI|'1|\}.}/|/D] |J|<qtK|.|\}.}0|0D]"}1|1L|*}2|2D]|=q|1M|*}3t$|3dkr|)B|3q|.r|)Bt/|.t$|)dkrtNd,|)d-|s|rtOntP}4tQ|4|}4n|rtRntS}4|4Td.d/Td0d1}5tU||5|d2}6d}7|6jVr|6jW|6jXd3}7t Y|D]H}8t jZ||8}9|4Td.d&Td0d&}:|8Td.d&Td0d&};t[\d4}<|8]|:r]t j|9r]|8|6j^:vr]|r]|<_|;dur]t `|9q|6j^&}=|rntajb|=d5d6}=|=D]o\}>}?i}@|?D]}%||%c|@|%<||%=qx|rtdtefd7krt d8tdd9tgh|@d&}A|@D]}B|Ai|Bd&krq||B}tj||B|A}Aq|A}@~Atkl|rtm|@t jZ||>d:d;id<qp||@t jZ||>qp~|7durt jZ||4}Ct*d=|CnJ|rtnnto}Dt jZ|tQ|D|}Dtp|Dd>d?d@}Etqjr|7dAddBdC}F|Es|FWdn 1s)wYt*dD|dEt$|6j^dF|Dd|rbtt||ju| | dG}G|Gvt jZ|dH|jw||||| dIdSdS)Ja Save a model and its configuration file to a directory, so that it can be re-loaded using the [`~PreTrainedModel.from_pretrained`] class method. Arguments: save_directory (`str` or `os.PathLike`): Directory to which to save. Will be created if it doesn't exist. is_main_process (`bool`, *optional*, defaults to `True`): Whether the process calling this is the main process or not. Useful when in distributed training like TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on the main process to avoid race conditions. state_dict (nested dictionary of `torch.Tensor`): The state dictionary of the model to save. Will default to `self.state_dict()`, but can be used to only save parts of the model or if special precautions need to be taken when recovering the state dictionary of a model (like when using model parallelism). save_function (`Callable`): The function to use to save the state dictionary. Useful on distributed training like TPUs when one need to replace `torch.save` by another method. push_to_hub (`bool`, *optional*, defaults to `False`): Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). max_shard_size (`int` or `str`, *optional*, defaults to `"5GB"`): The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`). We default it to 5GB in order for models to be able to run easily on free-tier google colab instances without CPU OOM issues. If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard which will be bigger than `max_shard_size`. safe_serialization (`bool`, *optional*, defaults to `True`): Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). variant (`str`, *optional*): If specified, weights are saved in the format pytorch_model..bin. token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). save_peft_format (`bool`, *optional*, defaults to `True`): For backward compatibility with PEFT library, in case adapter weights are attached to the model, all keys of the state dict of adapters needs to be pre-pended with `base_model.model`. Advanced users can disable this behaviours by setting `save_peft_format` to `False`. kwargs (`Dict[str, Any]`, *optional*): Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. use_auth_tokenNignore_metadata_errorsFrThe `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.V`token` and `use_auth_token` are both specified. Please set only the argument `token`.rrr)rzThe model is quantized with z and is not serializable - check out the warnings from the logger on the traceback to understand the reason why the quantized model is not serializable. save_configze`save_config` is deprecated and will be removed in v5 of Transformers. Use `is_main_process` instead.zR`safe_serialization` requires the `safetensors library: `pip install safetensors`.zProvided path (z#) should be a directory, not a fileTexist_okcommit_messagerepo_idrrrrrzHMoving the following attributes in the config to the generation config: z. You are seeing this warning because you've set generation parameters in the model config, as opposed to in the generation config.zhDetected adapters on the model, saving the model in the PEFT format, only adapter weights will be saved.)rzTo match the expected format of the PEFT library, all keys of the state dict of adapters will be pre-pended with `base_model.model`.zbase_model.model.zMultiple active adapters detected, saving multiple active adapters is not supported yet. You can save adapters separately one by one by iteratively calling `model.set_adapter(adapter_name)` then `model.save_pretrained(...)` hf_device_maprrz|Attempting to save a model with offloaded modules. Ensure that unallocated cpu memory exceeds the `shard_size` (5GB default)rdcs,i|]\}}tfdd|Dr||qS)c3s|]}|vVqdSrrr tied_namesrrr1n sz=PreTrainedModel.save_pretrained...)r;rptrrrrrrm s z3PreTrainedModel.save_pretrained..cSs"i|] \}}t|dkr||qS)rrrrrrrs rc3s|] }t|VqdSrr5)rpatrrrr1 sz2PreTrainedModel.save_pretrained..z8The weights trying to be saved contained shared tensors z that are mismatching the transformers base configuration. Try saving using `safe_serialization=False` or remove this tensor sharing.z.binz {suffix}.binrFz{suffix}.safetensors)filename_patternr)rSrz(.*?)-\d{5}-of-\d{5}zSaving checkpoint shardsdescrxzxYou need accelerate version to be greater or equal than 0.31 to save models with offloaded parameters. Detected version z<. Please upgrade accelerate with `pip install -U accelerate`rJrG)rSzModel weights saved in wrrr)indent sort_keys z:The model is bigger than the maximum size per checkpoint (z) and is going to be split in z^ checkpoint shards. You can find where each parameters has been saved in the index located at )rrz README.md)rr)xrrzr{r|rr}r\r;is_serializablerrrXrXrr r r errormakedirsrsep _create_repo_get_files_timestamps unwrap_modelrrrrrr architecturesr _auto_classr"&_get_non_default_generation_parametersrr UserWarningrrrsave_pretrainedrget_adapter_state_dictactive_adapters peft_configrrrrrjrIS_SAGEMAKER_MP_POST_1_10smpstatemodule_managertranslate_functions_keys_to_ignore_on_saver_fix_state_dict_keys_on_saverrrrrr6ridr)rrr;rrrMr intersection differencerrDrHrr>r?replacerrrStensor_to_filenamelistdirr rcompiler,filename_to_tensors fullmatchr r_tqdmraccelerate_versionrr]rfromkeysrryrrsafe_save_filerCrGrrdumpswriterbrsave_upload_modified_files)Hrhrrrrrrrrrrrrrrrquantization_serializablerrfiles_timestamps model_to_savermisplaced_generation_parametersr param_valuepeft_state_dictrractive_adaptercurrent_peft_config module_maprmodule_state_dict smp_to_hfr ignore_keyptrsrqr& shared_ptrsrx error_namesto_delete_namesrfoundmatches_pattern shared_namesdisjoint_namesidentical_namesinamesknownunknownrrstate_dict_splitr*r full_filenameweights_no_suffixfilename_no_suffixregrr2rshardshard_state_dictrgpath_to_weightssave_index_filer)content model_cardr)rrrr s ?                                                  zPreTrainedModel.save_pretrainedcsj|jdur|jng}|dg}t|tr|g}|D] }||vr%||q|r,||d<tj|i|S)Nr)rrr\rrrr)rhrrr tags_kwargsrrrrrs   zPreTrainedModel.push_to_hubcCs<tdd|D}|rtdd|D}||}|S)a Get the memory footprint of a model. This will return the memory footprint of the current model in bytes. Useful to benchmark the memory footprint of the current model and design some tests. Solution inspired from the PyTorch discussions: https://discuss.pytorch.org/t/gpu-memory-that-model-uses/56822/2 Arguments: return_buffers (`bool`, *optional*, defaults to `True`): Whether to return the size of the buffer tensors in the computation of the memory footprint. Buffers are tensors that do not require gradients and not registered as parameters. E.g. mean and std in batch norm layers. Please see: https://discuss.pytorch.org/t/what-pytorch-means-by-buffers/120266/2 cSg|] }||qSrrrru)rrrrrr/z8PreTrainedModel.get_memory_footprint..cSrrr)rbufrrrr1r)rrr)rhreturn_buffersr_mem_bufsrrrget_memory_footprint#s  z$PreTrainedModel.get_memory_footprintcst|ddtjkr tdt|ddtjkr9t|ddr tdttjdtdkr7td|j d dSt j |i|S) Nquantization_methodz2`.cuda` is not supported for HQQ-quantized models.is_loaded_in_8bitFzCalling `cuda()` is not supported for `8-bit` quantized models. Please use the model as it is, since the model has already been set to the correct devices.r0.43.2zCalling `cuda()` is not supported for `4-bit` quantized models with the installed version of bitsandbytes. The current device is `L`. If you intended to move the model, please install bitsandbytes >= 0.43.2.) r}rirrrrr]rrSrrr)rhrrrrrr5s zPreTrainedModel.cudacsd|v}|s|D] }t|tjrd}nqt|ddtjkr"td|r1t|ddtjkr1tdt|ddtjkrb|r@tdt|ddrJtd t t j d t d kratd |j d nt|ddtjkrq|rqtdtj|i|S)NrTr z0`.to` is not supported for HQQ-quantized models.zBCasting a Quark quantized model to a new `dtype` is not supported.zYou cannot cast a bitsandbytes model in a new `dtype`. Make sure to load the model using `from_pretrained` using the desired `dtype` by passing the correct `torch_dtype` argument.r!Fz`.to` is not supported for `8-bit` bitsandbytes models. Please use the model as it is, since the model has already been set to the correct devices and casted to the correct `dtype`.rr"zCalling `to()` is not supported for `4-bit` quantized models with the installed version of bitsandbytes. The current device is `r#zYou cannot cast a GPTQ model in a new `dtype`. Make sure to load the model using `from_pretrained` using the desired `dtype` by passing the correct `torch_dtype` argument.)r\rrr}rirrQUARKrrr]rrSrGPTQrr)rhrrdtype_present_in_argsargrrrrHsB  zPreTrainedModel.toc t|ddr tdtj|S)NrEFz`.half()` is not supported for quantized model. Please use the model as it is, since the model has already been casted to the correct `dtype`.)r}rrhalfrhrrrrr)t  zPreTrainedModel.halfcr()NrEFz`.float()` is not supported for quantized model. Please use the model as it is, since the model has already been casted to the correct `dtype`.)r}rrrr*rrrr~r+zPreTrainedModel.floatrErcCsjtr-tg}|s |s td|tjjtdt g|S|r+|t t g|Stt g}|S)Nrr) r(rr rr~rrrr'rr*r)r rErrrrrget_init_contexts  z PreTrainedModel.get_init_contextr) rrrDrrrrrrr rrrDrrrrrc POsb | dd} | dd}| dd}| dd}| dd}| dd}| d d}| d d}| d d}| d d}| d d}| dd}| dd}| dd}| dd}| dd}| dd}| dd}| dd}| dd} | di}!| dd}"| dd}#| dd}$| dd}%| dd}&| dd}'| d d}(| d!d}(| d"d}(| d#d$}(| d%d}(| dur|dus|%durtd&|&dur|&d'krtd(|&d)|&dur|durtd*|d'kr|&durttjd+d,rd'}&d}d})|&durtd-std.tj j }*tj szzGttjd/}+ttjd+},|*d0krJtj jd1|+|,d2d3tjttjd4n|*d5krfttjd6d,r[d7nd8}-tj j|-|+|,d9Wntyy}.ztd:|.d}.~.ww|*d5krdntj}/t|*|/}0|/dur|/d,krd,dl}1ttjd;|1_ttjd;|1_|0}tj },tj |0j |,f})|durtd|!vr||!d><| durtsd} |%durtstd?|dur t |t!st"|t#|||||||dddd@ }2t$|2|}nt%|dd}t&rk|!dAd}3|3dur=t'|f|||||dB|!}3|3durjtj()|3rjt|3dCdDdE}4|}3t*+|4dF}Wdn 1sewYnd}3t |tjryd|i}n8t |t,r|dGvrz dt|i}Wn#t-ytdH|d)wt |tr|d,krtdId|i}|durt.rtdJtstdK|s|r|durtdLdMdN| /D}5i|5||dO}5t0j1d|5d$dP| \}} t23dQ||B }6dRdS|dT}7|dur ||7dU<t4r|st25dVd$}t |t!sM|dur(|n|}8|j6j7|8f|d$|||||||%||dW | \}}9d|9vrL|9dnt89|}| dXd}:|:dur`|:|_:| }9t;|d};|;rst<=|j>ssd};|;s{|dur|;rt||_>n||_>t|;dY}jEd[r|jEjF|7d\<n|jE|7d\<|%dur||>du}?|| | \}}}E||_P|Q|@tR}Ft89|}t%|dpds|jS||#||dq}tT|F||g| Ri|9}GWdn 1swY|GU|)dur,|GjVs,|jWdur,|XjWdur,tYdr|GjZ}d}H|Gj[durR|tj\ksDt%||||||<|H|)|'| dx\}G}J}K}L}M}N|GU|Gh|Gir|$durt25dy|Gjj1|$k|G_jn0|Gir |dur ztlj7|f|||||||||dz | |G_jWntmy t25d{Ynw|durx|)durx|||M|d|}Od}tnotpjqvr)|Gjr|Od}<d~tnotpjqvrC|jEtsjtkrCd$|Od~<|jEtsjukrht |tGrhd5|Hvsdd^|Hvrhd$|Od<tvsxt.sxtp|Gfi|O| To test a pull request you made on the Hub, you can pass `revision="refs/pr/"`. attn_implementation (`str`, *optional*): The attention implementation to use in the model (if relevant). Can be any of `"eager"` (manual implementation of the attention), `"sdpa"` (using [`F.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html)), or `"flash_attention_2"` (using [Dao-AILab/flash-attention](https://github.com/Dao-AILab/flash-attention)). By default, if available, SDPA will be used for torch>=2.1.1. The default is otherwise the manual `"eager"` implementation. > Parameters for big model inference torch_dtype (`str` or `torch.dtype`, *optional*): Override the default `torch.dtype` and load the model under a specific `dtype`. The different options are: 1. `torch.float16` or `torch.bfloat16` or `torch.float`: load in a specified `dtype`, ignoring the model's `config.torch_dtype` if one exists. If not specified - the model will get loaded in `torch.float` (fp32). 2. `"auto"` - A `torch_dtype` entry in the `config.json` file of the model will be attempted to be used. If this entry isn't found then next check the `dtype` of the first weight in the checkpoint that's of a floating point type and use that as `dtype`. This will load the model using the `dtype` it was saved in at the end of the training. It can't be used as an indicator of how the model was trained. Since it could be trained in one of half precision dtypes, but saved in fp32. 3. A string that is a valid `torch.dtype`. E.g. "float32" loads the model in `torch.float32`, "float16" loads in `torch.float16` etc. For some models the `dtype` they were trained in is unknown - you may try to check the model's paper or reach out to the authors and ask them to add this information to the model's card and to insert the `torch_dtype` entry in `config.json` on the hub. device_map (`str` or `Dict[str, Union[int, str, torch.device]]` or `int` or `torch.device`, *optional*): A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the same device. If we only pass the device (*e.g.*, `"cpu"`, `"cuda:1"`, `"mps"`, or a GPU ordinal rank like `1`) on which the model will be allocated, the device map will map the entire model to this device. Passing `device_map = 0` means put the whole model on GPU 0. To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For more information about each option see [designing a device map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). max_memory (`Dict`, *optional*): A dictionary device identifier to maximum memory if using `device_map`. Will default to the maximum memory available for each GPU and the available CPU RAM if unset. tp_plan (`str`, *optional*): A torch tensor parallel plan, see [here](https://pytorch.org/tutorials/intermediate/TP_tutorial.html). Currently, it only accepts `tp_plan="auto"` to use predefined plan based on the model. Note that if you use it, you should launch your script accordingly with `torchrun [args] script.py`. This will be much faster than using a `device_map`, but has limitations. offload_folder (`str` or `os.PathLike`, *optional*): If the `device_map` contains any value `"disk"`, the folder where we will offload weights. offload_state_dict (`bool`, *optional*): If `True`, will temporarily offload the CPU state dict to the hard drive to avoid getting out of CPU RAM if the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` when there is some disk offload. offload_buffers (`bool`, *optional*): Whether or not to offload the buffers with the model parameters. quantization_config (`Union[QuantizationConfigMixin,Dict]`, *optional*): A dictionary of configuration parameters or a QuantizationConfigMixin object for quantization (e.g bitsandbytes, gptq). There may be other quantization-related kwargs, including `load_in_4bit` and `load_in_8bit`, which are parsed by QuantizationConfigParser. Supported only for bitsandbytes quantizations and not preferred. consider inserting all such arguments into quantization_config instead. subfolder (`str`, *optional*, defaults to `""`): In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can specify the folder name here. variant (`str`, *optional*): If specified load weights from `variant` filename, *e.g.* pytorch_model..bin. `variant` is ignored when using `from_tf` or `from_flax`. use_safetensors (`bool`, *optional*, defaults to `None`): Whether or not to use `safetensors` checkpoints. Defaults to `None`. If not specified and `safetensors` is not installed, it will be set to `False`. weights_only (`bool`, *optional*, defaults to `True`): Indicates whether unpickler should be restricted to loading only tensors, primitive types, dictionaries and any types added via torch.serialization.add_safe_globals(). When set to False, we can load wrapper tensor subclass weights. key_mapping (`Dict[str, str], *optional*): A potential mapping of the weight names if using a model on the Hub which is compatible to a Transformers architecture, but was not converted accordingly. kwargs (remaining dictionary of keyword arguments, *optional*): Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., `output_attentions=True`). Behaves differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with `config`, `**kwargs` will be directly passed to the underlying model's `__init__` method (we assume all relevant updates to the configuration have already been done) - If a configuration is not provided, `kwargs` will be first passed to the configuration class initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that corresponds to a configuration attribute will be used to override said attribute with the supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's `__init__` function. Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to use this method in a firewalled environment. Examples: ```python >>> from transformers import BertConfig, BertModel >>> # Download model and configuration from huggingface.co and cache. >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased") >>> # Model was saved using *save_pretrained('./test/saved_model/')* (for example purposes, not runnable). >>> model = BertModel.from_pretrained("./test/saved_model/") >>> # Update configuration during loading. >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased", output_attentions=True) >>> assert model.config.output_attentions == True >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower, for example purposes, not runnable). >>> config = BertConfig.from_json_file("./tf_model/my_tf_model_config.json") >>> model = BertModel.from_pretrained("./tf_model/my_tf_checkpoint.ckpt.index", from_tf=True, config=config) >>> # Loading from a Flax checkpoint file instead of a PyTorch model (slower) >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased", from_flax=True) ``` rNrFrroutput_loading_infor_from_pipeline _from_autorrroffload_folderoffload_state_dictoffload_buffers load_in_8bit load_in_4bitrrrdrradapter_kwargs adapter_namedefaultrrrtp_plan key_mappingresume_downloadtrust_remote_codemirror _fast_initTlow_cpu_mem_usagezq`state_dict` cannot be passed together with a model name or a `gguf_file`. Use one of the two loading strategies.rz-tp_plan supports 'auto' only for now but got rzY`tp_plan` and `device_map` are mutually exclusive. Choose either one for parallelization. WORLD_SIZErz2.5z3tensor parallel is only supported for `torch>=2.5`.rrncclzenv://)rank world_size init_methodrrCCL_WORKER_COUNTcclgloo)rArBzWe tried to initialize torch.distributed for you, but it failed, makesure you init torch distributed in your script to use `tp_plan='auto'`rrrrzIaccelerate is required when loading a GGUF file `pip install accelerate`.) rrrrrrrrr'_raise_exceptions_for_connection_errors_adapter_model_path)rrrrrrrrbase_model_name_or_path)rbalancedrrzWhen passing device_map as a string, the value needs to be a device name (e.g. cpu, cuda:0) or 'auto', 'balanced', 'balanced_low_0', 'sequential' but found znYou can't pass device_map as a negative int. If you want to put the model on the cpu, pass device_map = 'cpu' z?DeepSpeed Zero-3 is not compatible with passing a `device_map`.ziUsing a `device_map` or `tp_plan` requires `accelerate`. You can install it with `pip install accelerate`zwYou can't pass `load_in_4bit`or `load_in_8bit` as a kwarg when passing `quantization_config` argument at the same time.cSs&i|]\}}|ttjvr||qSr)rr rhrrrrrr8s&z3PreTrainedModel.from_pretrained..)r4r3) config_dictreturn_unused_kwargszThe `load_in_4bit` and `load_in_8bit` arguments are deprecated and will be removed in the future versions. Please, pass a `BitsAndBytesConfig` object in `quantization_config` argument instead.rpytorch) file_typerIfrom_auto_classusing_pipelinez+Offline mode: forcing local_files_only=True) rrLrrrrrrrr/r.r) pre_quantized)rrrrrrquantzYou cannot combine Quantization and loading a model from a GGUF file, try again by making sure you did not passed a `quantization_config` or that you did not load a quantized model from the Hub.rzxOne or more modules is configured to be mapped to disk. Disk offload is not supported for models loaded from GGUF files.)rrrrrrrrrrrrrrrrFrGrHrJrKzAA TensorFlow safetensors file is being loaded in a PyTorch model.rLz;A Flax safetensors file is being loaded in a PyTorch model.rMzTIncompatible safetensors file. File metadata is not ['pt', 'tf', 'flax', 'mlx'] but r)load_gguf_checkpointrN)return_tensors model_to_loadrr)rrrz0This model does not have a tensor parallel plan.use_keep_in_fp32_modulesrcSsg|]}d|dqS)z((^|\.)z($|\.))r)rrrrrr rz3PreTrainedModel.from_pretrained..)rrkeep_in_fp32_modulesr) rDrrrr1rrrrr9rz\The user-defined `generation_config` will be used to override the default generation config.) rrrrrrrr/r.zZGeneration config file not found, using a generation config created from the model config.)r offload_dir offload_indexr2 skip_keys force_hooksr)r6rr5)r,r-rL error_msgsr)zrrrrrrrZrr_C_get_acceleratorrrrinit_process_groupr set_devicer^current_devicersysrdevnullstdoutstderrget_world_sizeinit_device_meshrzr{r|rXrQr\r!rLr@rOr}rVrr r rrrrr(rrh from_dictr rrTr config_classfrom_pretrainedrrrrr:supports_quant_methodrmerge_quantization_configs from_configr"update_torch_dtypeupdate_device_mapupdate_tp_planrrrrrrRrzrSmodeling_gguf_pytorch_utilsrSrrr,rrrIr3supports_tp_planrr.r"rrrrrr preprocess_modelrrr' _load_from_tf_load_from_flaxr_load_pretrained_modelevalrrto_dictr$rTrr rmr_skip_keys_device_placementrir FBGEMM_FP8rpostprocess_modelr load_adapter)Pr rrrrDrrrrrr model_argsrrrrrr-r from_pipelinerOrrrr0r1r2r3r4rrrrr5r6rrrr8r9rr device_typerArB cpu_backendrcr* tp_devicerbresolved_config_filerHr)rKfrom_ptr config_path model_kwargskwarg_attn_imprQrrrrrE is_from_filerSrS dummy_modelr model_init_contextrr loading_infor,r-rLrYr\r%rrrrjsn a                               (                                                                     zPreTrainedModel.from_pretrainedrcCs|dr |dddfS|dr|dddfSttjjdr?|dr.|dddfS|d r;|d d dfS|d fS|drL|dddfS|d rY|d d dfS|d fS) zaReplace legacy parameter names with their modern equivalents. E.g. beta -> bias, gamma -> weight.LayerNorm.betazLayerNorm.biasTLayerNorm.gammazLayerNorm.weight weight_normweight_gz!parametrizations.weight.original0weight_vz!parametrizations.weight.original1F)rRrrrutilsparametrizationsrrrr_fix_state_dict_key_on_loads      z+PreTrainedModel._fix_state_dict_key_on_loadr)r9r*'loading_task_model_from_base_state_dictcCs4|j}|d}i}i}|D]^} || \} } |dur5|D]\} } t| | | \} }|dkr4d} nq|r?d|| g} n|rO| |sGq| t|d} | || <| rl| dra| | f|d<q| drl| | f|d<q|rd|j j d}|d 7}| D]\}} |d |d | d 7}q|d 7}t ||S)a& Compute a mapping between the serialized keys on disk `checkpoint_keys`, and the keys that the model that we are loading expects. This is the single entry point for key renaming that will be used during loading. Log if any parameters have been renamed. rNrTrrzA pretrained model of type `z` zrcontains parameters that have been renamed internally (a few are listed below but more are present in the model): z* `z` -> `z` znIf you are using a model from the Hub, consider submitting a PR to adjust these weights and help future users.)r8rrrsubnr r,rrRrrrr  info_once)rhr)r9r*rr{_prefix renamed_keyskey_renaming_mappingrnew_key has_changedr7 replacement n_replace warning_msgold_keyrrr_get_key_renaming_mappingsD      z)PreTrainedModel._get_key_renaming_mappingcCs|dfS)z Similar to `_fix_state_dict_key_on_load` allows to define hook for state dict key renaming on model save. Do nothing by default, but can be overridden in particular models. Frrrrr_fix_state_dict_key_on_saverlz+PreTrainedModel._fix_state_dict_key_on_savecsfdd|DS)z Similar to `_fix_state_dict_keys_on_load` allows to define hook for state dict key renaming on model save. Apply `_fix_state_dict_key_on_save` to all keys in `state_dict`. cs i|] \}}|d|qSr)rrrrrkrrrr-z@PreTrainedModel._fix_state_dict_keys_on_save..)r)rhrrrkrrsz,PreTrainedModel._fix_state_dict_keys_on_saverrrrr1rrrrc4sr| du}|o | jjtjk}|o| jjtjtjfv}|dur"|d}n|dur-t|}n tt|dd|d}|jdt dkrSt fdd|Dnd}t dkr`t |nd}| of|}|ok| }| ||||t }t|||||| |\}}t||||||\}fd d Dt }|||| | ||||| dur|D]\}}| |r|jtj|_q|}|rt|}fd d Dt }|durfd d |D}fd d|D t|t  fdd|Drtddd Dd}d} g}!|durd| vr| dur@d} |durLtj|dd|duoW|dd}|durd|sdtd|rt||| duryt | !ddnd|durt"#||d}"n/tj$j%&|d'tj$j%ddfdd |dD}"fdd |"D}"t(||"}!fdd |"D} ni} d}#d}$| rt)*}#i}$|durt |dkrt+j,|d d!}n|durdg}t|}%| dur| -||%|}%|dur |s t||%}&t.||&| durd"nd#d$g}'|D]}(|(|!vr-q$d%})|(drB|sBt/r?|rBd})n/|durq| durq| jjtj0krq| jj1d&vsct2| jj1t3rqt4d'd| Dd})|(dkr~t|(||)|d(}fd)d |D}t/r|s|'t5||7}'n t6rt7s|rt8|||(|%||| |#|$| || || d*\} }$~q$| durt | dkr|r|j|s| D]}*t9:tj$&||*d+tj$&|d|*d+q҇fd,d | D} |st;| |d} | rt<||$|#t9=|#| dur| >||}| durqt| d}+|?D]},|,j4|+kr=|,|+|,_q/|rqfd-d |D}-|-D]\}}t@|||| \}.}/tA|||+|||/|.tjBd.| qQt |'dkrd/&|'}0d0|0vr|0d17}0tCd2|jDjEd3|0t |dkr|jFjGdurgn|jFjG}1|jDjE|1vrtHjIntHjJ}2|2d4|d5|jDjEd6|d7|jDjEd8|jDjEd9 n tHJd:|jDjEd;t |dkrtHId<|jDjEd=|d>|d?nt dkr tHJd@|jDjEdA|dB|jDjEdCt dkr1dD&dEdtK|D}3tHId<|jDjEd=|dF|3d?|||| |'fS)GNall_checkpoint_keysrrNrrc3s|]}|VqdSrr+rsrzrrr1$r2z9PreTrainedModel._load_pretrained_model..Fcsi|] \}}|vr||qSrrr)rLrrrIsz:PreTrainedModel._load_pretrained_model..cs"i|] \}}||tdqSrrrrrrrarcs0i|]\}}|r|tdn||qSr)r,rrrrrres0csg|] }|s|qSrr+rrrrrgriz:PreTrainedModel._load_pretrained_model..c3s |] }|vo |vVqdSrrr)base_model_expected_keystask_specific_expected_keysrrr1is zjThe state dictionary of the model you are trying to load is corrupted. Are you sure it was properly saved?cSsi|]\}}||qSrrrrrrrrrrTrrFzThe current `device_map` had weights offloaded to the disk. Please provide an `offload_folder` for them. Alternatively, make sure you have `safetensors` installed if the model you are using offers the weights in this format.ztorch.rdrrcrGrrrrrrrs rcs i|] \}}|tj|qSr)rr r r)r rrrr-cs.i|]\}}|dkr|||dqS)r)safetensors_file weight_namerr)rrfile)param_device_mapreverse_key_renaming_mapping str_dtyperrrs rzLoading checkpoint shardsrr)factorr)int4_weight_only autoquantcSsg|]}|dvr|qS)rrr)rdrrrrrrFcrGrrrrrrrr) rrrrrrrrr-rz.datcrrerrrzrrrr-cs i|] \}}|s||qSrr+)rrrrzrrrs rz z size mismatchz_ You may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method.rz: z(Some weights of the model checkpoint at z! were not used when initializing z: z, - This IS expected if you are initializing z from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model). - This IS NOT expected if you are initializing z from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).z9All model checkpoint weights were used when initializing z. zSome weights of z3 were not initialized from the model checkpoint at z and are newly initialized: zo You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.zAll the weights of z/ were initialized from the model checkpoint at zf. If your task is similar to the task the model of the checkpoint was trained on, you can already use z* for predictions without further training.rc Ss,g|]\}\}}d|d|d|dqS)z- z: found shape z in the checkpoint and z in the model instantiatedr)rrshape1shape2rrrrRs z= and are newly initialized because the shapes did not match: )Lrrrirrrrrr8rr;rrrrCrOr#_move_missing_keys_from_meta_to_cpu_initialize_missing_keysrrrrrrr}rrrrrRexpand_device_maprrrrr rr rget_disk_only_shard_filestempfilemkdtempr_rr9caching_allocator_warmupr(TORCHAO quant_typer\rrr+rrrshutilmovervrtrmtree!update_missing_keys_after_loadingrrr1rrrrrrr rrzip)4r rrrrrDrrrr1rrrrr9rrEis_hqqrr(has_prefix_moduleexpects_prefix_modulerr*r)r,r-rMrrrUis_offloaded_safetensorsrdisk_only_shard_filesrrrrexpanded_device_mapr\r2rrrr0parameters_to_initializerr error_msgarchswarnermismatched_warningr) rrr rrLrr{rrrrrvs   &             $                            z&PreTrainedModel._load_pretrained_modelcCs~|ddr||||ddd}d}||fSzddlm}|||dddd\}}W||fSty>tdw) Nrrir)$load_tf2_checkpoint_in_pytorch_modelT)allow_missing_keysr-zLoading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed. Please see https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.)rRload_tf_weightsmodeling_tf_pytorch_utilsrrXr r)r rrrrrrrrrt`s     zPreTrainedModel._load_from_tfcCs<zddlm}|||d}W|Stytdw)Nr)%load_flax_checkpoint_in_pytorch_modelrzLoading a Flax model in PyTorch, requires both PyTorch and Flax to be installed. Please see https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation instructions.)modeling_flax_pytorch_utilsrrXr r)r rrrrrrruws  zPreTrainedModel._load_from_flaxc Csdd|D}|dd|D}g}|D]8\}}|r3|jd}||r0|t|dn|}n|rFt|dkrCd|j|gn|j}||vrO||q|S)NcSs$h|]}d|dddqS)rNr)r rrrrrrhs$z>PreTrainedModel.retrieve_modules_from_names..cSs<h|]}t|dkr|drd|dddqS)rrrN)rr?r rrrrrrhs<rr)unionrjr8r,rr r) rhr add_prefix remove_prefixroretrieved_modulesrrrrrrretrieve_modules_from_namess   " z+PreTrainedModel.retrieve_modules_from_names AutoModelcCsDt|ts|j}ddlmm}t||st|d||_dS)a Register this class with a given auto class. This should only be used for custom models as the ones in the library are already mapped with an auto class. This API is experimental and may have some slight breaking changes in the next releases. Args: auto_class (`str` or `type`, *optional*, defaults to `"AutoModel"`): The auto class to register this new model with. rNz is not a valid auto class.) r\rrtransformers.models.automodelsrrrr)r  auto_class auto_modulerrrregister_for_auto_classs   z'PreTrainedModel.register_for_auto_classcCTtstdddlm}t|tdkrtd|dddlm}||S)a( Converts the model to use [PyTorch's native attention implementation](https://pytorch.org/docs/stable/generated/torch.nn.MultiheadAttention.html), integrated to Transformers through [Optimum library](https://huggingface.co/docs/optimum/bettertransformer/overview). Only a subset of all Transformers models are supported. PyTorch's attention fastpath allows to speed up inference through kernel fusions and the use of [nested tensors](https://pytorch.org/docs/stable/nested.html). Detailed benchmarks can be found in [this blog post](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2). Returns: [`PreTrainedModel`]: The model converted to BetterTransformer. =1.7.0 to use Better Transformer. The version was found.BetterTransformer) rUrXoptimum.versionrrr]optimum.bettertransformerr transformrhoptimum_versionrrrrto_bettertransformers    z$PreTrainedModel.to_bettertransformercCr)a Reverts the transformation from [`~PreTrainedModel.to_bettertransformer`] so that the original modeling is used, for example in order to save the model. Returns: [`PreTrainedModel`]: The model converted back to the original modeling. rrrrrrr) rUrXrrrr]rrrrrrrreverse_bettertransformers    z)PreTrainedModel.reverse_bettertransformerc Cst|s tjs trdS|dus|jjdurdS|jj|ddddgfvrtd}|jjdur8|jj|jjksT|jjdurF|jj|jjksT|jj durm|jj |jjkrm|d|jjd|jjd|jjd|jj d 7}t |dSdS) zv Shows a one-time warning if the input_ids appear to contain padding and no attention mask was given. NrrzWe strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked.z5 You may ignore this warning if your `pad_token_id` (z&) is identical to the `bos_token_id` (z), `eos_token_id` (z), or the `sep_token_id` (z ), and your input is not padded.) rfrjit is_tracingrgr pad_token_id bos_token_id eos_token_id sep_token_idr r)rhrr warn_stringrrr%warn_if_padding_and_no_attention_masks*  z5PreTrainedModel.warn_if_padding_and_no_attention_maskcCs(|jdurdSt|jdddurdSdS)zJ Returns whether the model has a tensor parallelism plan. NTrF)rr}rrkrrrrr s z PreTrainedModel.supports_tp_plancCs(|jdurdSt|jdddurdSdS)NTrF)rr}rrkrrrsupports_pp_plans z PreTrainedModel.supports_pp_plancCsJt|dr|jSt|dd}|dus|tvr!td|dd}t|S)N_loss_functionrz `loss_type=zY` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. ForCausalLM)rrr}r2r r)rhrrrr loss_function!s   zPreTrainedModel.loss_functioncCs ||_dSr)r)rhrrrrr0s compile_configcCsbd|jjvr |jSt|jdt}t|drt|d||kr.||_tj |jfi| |_ |j S)aReturn a `torch.compile`'d version of `self.__call__`. This is useful to dynamically choose between non-compiled/compiled `forward` during inference, especially to switch between prefill (where we don't want to use compiled version to avoid recomputing the graph with new shapes) and iterative decoding (where we want the speed-ups of compiled version with static shapes).llama4r_compiled_call_last_compile_config) r model_type__call__r}rr#rrrrrxr)rhrdefault_configrrrget_compiled_call4s z!PreTrainedModel.get_compiled_callcCs|jSr)_supports_attention_backend)r rrris_backend_compatibleEsz%PreTrainedModel.is_backend_compatibler,r-c Cs|du}tr%ts%|s%|D]\}}tj||dd}t|||qdS|} |D]8}| |}|jtdkrctj||dd}|rRt|ddsR|j |||idsYt|||q+| |||d| |q+dS)zMove the missing keys (keys that are part of the model parameters, but were NOT found in the loaded state dicts) back from meta device to cpu. Nr)rrrNrF)rrr) rrrr empty_likerrrr}rr) rhr,r-rrrErrrrKrrrrIs*  z3PreTrainedModel._move_missing_keys_from_meta_to_cpurcCs|s/t||}t|jjdddr.|jjddjr.|}|dur.t|dr+|jdur.d|_nt| }t rl|slt t t jdd|D}tjj|dd ||jWddS1sewYdS||jdS) aInitialize the missing keys (keys that are part of the model parameters, but were NOT found in the loaded state dicts), according to `_initialize_weights`. Indeed, since the corresponding weights are missing from the state dict, they will not be replaced and need to be initialized correctly (i.e. weight initialization distribution). Also take care of setting the `_is_hf_initialized` flag for keys that are not missing. Tr'r)Nrcss|] }|jddVqdS)F)recurseN)r)rrrrrr1s  z;PreTrainedModel._initialize_missing_keys..rrV)rprrr.r)r$rrlrrjr(rr itertoolschain from_iterablerrrrYrr&)rhrrDrErnr1not_initialized_parametersrrrrls0    "z(PreTrainedModel._initialize_missing_keysrcCsPz||WStyYnwz||WStyYnwtd|d)a Return the parameter or buffer given by `target` if it exists, otherwise throw an error. This combines `get_parameter()` and `get_buffer()` in a single handy function. Note that it only work if `target` is a leaf of the model. `z&` is neither a parameter nor a buffer.) get_parameterAttributeError get_buffer)rhrrrrrs    z'PreTrainedModel.get_parameter_or_buffer)FNNT)NNTFr)NNTr)NFTrr)NFF) FNNNNNNNNNTr)r)rr)rrSrTrrir8rrrrryrr>r?rrxis_parallelizabler _is_statefulrrr_supports_cache_class_supports_static_cache_supports_quantized_cacherrrrr rrrrrIr!rrrrrr r classmethodrrrrrrrr rrrrrrrrr!rr#r$rr&r3rr0r/rrr\rXr^rqr_rjrrrsrtrrrrrrrr rrrrrPathLikerrrrKrrrrr)rr,rrrjrrrrr;rPatternrvrtrurrrrrrrrrsetterr#rrrrr __classcell__rrrrrs  #'   9 h& j       Y 9+  z   *          +       <       e     #      # 'rz model file)object object_class object_filescsJeZdZdZdeffdd Z d dejdeejdejfd d Z Z S) PoolerStartLogitsz Compute SQuAD start logits from sequence hidden states. Args: config ([`PretrainedConfig`]): The config used by the model, will be used to grab the `hidden_size` of the model. rcstt|jd|_dS)Nr)rrrrq hidden_sizedenserhrrrrrs zPoolerStartLogits.__init__N hidden_statesp_maskrcCsV||d}|dur)t|tjkr|d|d|}|S|d|d|}|S)a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): The final hidden states of the model. p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*): Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token should be masked. Returns: `torch.FloatTensor`: The start logits for SQuAD. rNrꌠ9Y>)F)r squeezerrr)rhr"r#xrrrforwardszPoolerStartLogits.forwardr) rrSrTrr!rr FloatTensorrr(rrrrrrsrc sbeZdZdZdeffdd Z   d dejdeejdeej d eejd ejf d d Z Z S)PoolerEndLogitsz Compute SQuAD end logits from sequence hidden states. Args: config ([`PretrainedConfig`]): The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps` to use. rcsRtt|jd|j|_t|_tj|j|j d|_t|jd|_ dS)Nr)epsr) rrrrqrdense_0Tanh activation LayerNormlayer_norm_epsdense_1r!rrrrs  zPoolerEndLogits.__init__Nr" start_statesstart_positionsr#rcCs|dus |dus Jd|dur4|jdd\}}|ddddfdd|}|d|}|d|d}|tj||gdd}||}||}|| d}|durst |tj kri|d|d|}|S|d|d|}|S) a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): The final hidden states of the model. start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*): The hidden states of the first tokens for the labeled span. start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): The position of the first token for the labeled span. p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*): Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token should be masked. One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides `start_states`. Returns: `torch.FloatTensor`: The end logits for SQuAD. N7One of start_states, start_positions should be not Nonerrrqrr$r%) rIrgatherr,rrr.r/r1r&rr)rhr"r2r3r#slenhszr'rrrr(s$   zPoolerEndLogits.forwardNNN rrSrTrr!rrr)r LongTensorr(rrrrrr*s"  r*c s\eZdZdZfddZ   d dejdeejdeejdeejd ejf d d Z Z S) PoolerAnswerClassz Compute SQuAD 2.0 answer class from classification and start tokens hidden states. Args: config ([`PretrainedConfig`]): The config used by the model, will be used to grab the `hidden_size` of the model. csBtt|jd|j|_t|_tj|jddd|_dS)NrrF)r) rrrrqrr,r-r.r1r!rrrrs  zPoolerAnswerClass.__init__Nr"r2r3 cls_indexrcCs|jd}|dus|dusJd|dur,|ddddfdd|}|d|d}|durH|ddddfdd|}|d|d}n |dddddf}|tj||gdd}||}||d}|S)a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): The final hidden states of the model. start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*): The hidden states of the first tokens for the labeled span. start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): The position of the first token for the labeled span. cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Position of the CLS token for each sentence in the batch. If `None`, takes the last token. One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides `start_states`. Returns: `torch.FloatTensor`: The SQuAD 2.0 answer class. rNr4rr5) rIrr6r&r,rrr.r1)rhr"r2r3r=r8cls_token_stater'rrrr(%s  zPoolerAnswerClass.forwardr9) rrSrTrrrr)rr;r(rrrrrr<s"  r<c@s~eZdZUdZdZeejed<dZ eejed<dZ eej ed<dZ eejed<dZ eej ed<dZeejed<dS) SquadHeadOutputa Base class for outputs of question answering models using a [`~modeling_utils.SQuADHead`]. Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned if both `start_positions` and `end_positions` are provided): Classification loss as the sum of start token, end token (and is_impossible if provided) classification losses. start_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided): Log probabilities for the top config.start_n_top start token possibilities (beam-search). start_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided): Indices for the top config.start_n_top start token possibilities (beam-search). end_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided): Log probabilities for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search). end_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided): Indices for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search). cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided): Log probabilities for the `is_impossible` label of the answers. Nlossstart_top_log_probsstart_top_indexend_top_log_probs end_top_index cls_logits)rrSrTrr@rrr)rUrArBr;rCrDrErrrrr?Ws r?cseZdZdZfddZeeed      ddej de ej d e ej d e ej d e ej d e ej d e de eeej ffddZZS) SQuADHeadz A SQuAD head inspired by XLNet. Args: config ([`PretrainedConfig`]): The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps` to use. cs<t|j|_|j|_t||_t||_t||_ dSr) rr start_n_top end_n_topr start_logitsr* end_logitsr< answer_classr!rrrrs   zSQuADHead.__init__) output_typeriNFr"r3 end_positionsr= is_impossibler# return_dictrcCs|j||d}|durk|durk||||fD]} | dur&| dkr&| dq|j|||d} t} | ||} | | |} | | d}|dura|dura|j|||d}t}|||}||d7}|rht|d S|fS| \}}}tj j |dd }t j ||jdd \}}|ddd|}t |d |}|dd|dd}|d|}|dur|dnd}|j|||d } tj j | dd }t j ||jdd \}}|d|j|j}|d|j|j}t d ||}|j|||d}|s|||||fSt|||||dS)a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): Final hidden states of the model on the sequence tokens. start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Positions of the first token for the labeled span. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Positions of the last token for the labeled span. cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Position of the CLS token for each sentence in the batch. If `None`, takes the last token. is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Whether the question has a possible answer in the paragraph or not. p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*): Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token should be masked. return_dict (`bool`, *optional*, defaults to `False`): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Returns: )r#Nrr)r3r#r)r3r=g?)r@r5r)r2r#z blh,bl->bh)r2r=)rArBrCrDrE)rIrqsqueeze_rJrrKrBCEWithLogitsLossr?rOrNsoftmaxrtopkrGrrr6 expand_asrHrseinsum)rhr"r3rMr=rNr#rOrIr'rJloss_fct start_lossend_loss total_lossrE loss_fct_clscls_lossbszr7r8start_log_probsrArBstart_top_index_expr2hidden_states_expanded end_log_probsrCrDrrrr(s^         zSQuADHead.forward)NNNNNF)rrSrTrrr`r?r!rr)rr;rrrr(rrrrrrFvs6  rFcsJeZdZdZdeffdd Z d dejdeej dejfd d Z Z S) SequenceSummarya Compute a single vector summary of a sequence hidden states. Args: config ([`PretrainedConfig`]): The config used by the model. Relevant arguments in the config class of the model are (refer to the actual config class of your model for the default values it uses): - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are: - `"last"` -- Take the last token hidden state (like XLNet) - `"first"` -- Take the first token hidden state (like Bert) - `"mean"` -- Take the mean of all tokens hidden states - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2) - `"attn"` -- Not implemented now, use multi-head attention - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction. - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes (otherwise to `config.hidden_size`). - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output, another string or `None` will add no activation. - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation. - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation. rcstt|dd|_|jdkrtt|_t|dr;|jr;t|dr0|j r0|j dkr0|j }n|j }t |j ||_t|dd}|rGt|nt|_t|_t|dr`|jdkr`t |j|_t|_t|d rw|jdkryt |j|_dSdSdS) N summary_typelastattnsummary_use_projsummary_proj_to_labelsrsummary_activationsummary_first_dropoutsummary_last_dropout)rrr}rbr"rsummaryrrerf num_labelsrrrqr r. first_dropoutrhDropout last_dropoutri)rhr num_classesactivation_stringrrrrs&   zSequenceSummary.__init__Nr"r=rcCs|jdkr|dddf}ne|jdkr|dddf}nW|jdkr(|jdd}nK|jd krl|durItj|d ddddf|jd dtjd }n|dd}|d |d| df}| d | d }n|jdkrst | |}||}||}||}|S)ak Compute a single vector summary of a sequence hidden states. Args: hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`): The hidden states of the last layer. cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*): Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token. Returns: `torch.FloatTensor`: The summary of the sequence hidden states. rcNrfirstrr|rr5r=.rro)rrd)rbr|r full_likerIlongrrrqrOr6r&r"rlrjr.rn)rhr"r=rrrrr("s.     "     zSequenceSummary.forwardrr:rrrrrasra recursivecCsNtri}|rtdstd||d<t|fi|St|dr%t|jS|S)a Recursively unwraps a model from potential containers (as used in distributed training). Args: model (`torch.nn.Module`): The model to unwrap. recursive (`bool`, *optional*, defaults to `False`): Whether to recursively extract all cases of `module.module` from `model` as well as unwrap child sublayers recursively, not just the top-level distributed containers. z0.29.0zsSetting `recursive=True` to `unwrap_model` requires `accelerate` v0.29.0. Please upgrade your version of acceleratertr)rQrrqrrr)rrtrrrrrNs   rcs4i}|D]\|fdd|Dq|S)zT Expand a device map to return the correspondence parameter name to device. cs2i|]}|ks|dsdkr|qS)rrdr+)rprrrrrss2z%expand_device_map..)rr)r param_namesnew_device_maprrvrrls rrrcCs>dd|D}t|sdStr$tjr$tddd|j Dnd}t dd}|D]4\}}| |}t |j|} |dur\td d |} | || rYtjnd } ||| 7<q0|D]3\}} |jd kr|jdurz|jntj} tj| d } t| td| } tj| |tj|dd}qidS)aIThis function warm-ups the caching allocator based on the size of the model tensors that will reside on each device. It allows to have one large call to Malloc, instead of recursively calling it later when loading the model, which is actually the loading speed botteneck. Calling this function allows to cut the model loading time by a very large margin. A few facts related to loading speed (taking into account the use of this function): - When loading a model the first time, it is usually slower than the subsequent times, because the OS is very likely to cache the different state dicts (if enough ressources/RAM are available) - Trying to force the OS to cache the files in advance (by e.g. accessing a small portion of them) is really hard, and not a good idea in general as this is low level OS optimizations that depend on ressource usage anyway - As of 18/03/2025, loading a Llama 70B model with TP takes ~1 min without file cache, and ~13s with full file cache. The baseline, i.e. only loading the tensor shards on device and adjusting dtype (i.e. copying them) is ~5s with full cache. These numbers are reported for TP on 4 H100 GPUs. - It is useless to pre-allocate more than the model size in this function (i.e. using an `allocation_factor` > 1) as cudaMalloc is not a bottleneck at all anymore - Loading speed bottleneck is now almost only tensor copy (i.e. changing the dtype) and moving the tensors to the devices. However, we cannot really improve on those aspects obviously, as the data needs to be moved/copied in the end. cSs$i|]\}}|dvr|t|qSr)rr)rrrrrrrsz,caching_allocator_warmup..NrcSrrr)rrrrrrrz,caching_allocator_warmup..cSr)Nrrrrrrsz*caching_allocator_warmup..z\.\d+\.z.*.rrrgffffff?F)rrr)rr_torch_distributed_availablerrrrrr rrrmathprodrIrusubrrfrr*rra mem_get_inforsrrXr)rrraccelerator_device_map tp_plan_regextotal_byte_countrrrparam_byte_count generic_name byte_countr* device_memoryrrrrrxs4   rcCstt}|D]-\}}t|dkr-||vr-d|ddd}t|dkr-||vs||||q dd|DS)zT Returns the list of shard files containing only weights offloaded to disk. rrNrcSs"g|] \}}t|dhkr|qS)r)r)rfnamedevicesrrrrrz-get_disk_only_shard_files..)rrrrrr rr)rr files_contentrrrrrrs rc@steZdZdZeeedZddZddZ ddZ d d Z d d Z d dZ ededefddZdeefddZdS)AttentionInterfacea_ Dict-like object keeping track of allowed attention functions. You can easily add a new attention function with a call to `register()`. If a model needs to locally overwrite an existing attention function, say `sdpa`, it needs to declare a new instance of this class inside the `modeling_.py`, and declare it on that instance. )rrrcCs i|_dSr_local_mappingrkrrrrs zAttentionInterface.__init__cCs||jvr |j|S|j|Sr)r_global_mappingrhrrrr __getitem__s   zAttentionInterface.__getitem__cC|j||idSr)rr)rhrrrrr __setitem__zAttentionInterface.__setitem__cCs |j|=dSrrrrrr __delitem__ zAttentionInterface.__delitem__cCsti|j|jSr)iterrrrkrrr__iter__rzAttentionInterface.__iter__cCst|j|jBSr)rrrrrkrrr__len__szAttentionInterface.__len__rrcCrr)rr)r rrrrrregisterrzAttentionInterface.registerrcCs t|Sr)rrrkrrrrrzAttentionInterface.valid_keysN)rrSrTrr-r.r/rrrrrrrrrr rr rrrrrrs rr)TT)FrT)rdr) NNNNNNFNNNrr)r("rrrrimportlib.metadatarrr rr{rrrrrzrcollections.abcr contextlibr dataclassesrenumrrr threadingr typingr r r r rrrrrrzipfilerrtorch.distributed.tensorhuggingface_hubr packagingrrrtorch.distributionsrtorch.nnrrtorch.utils.checkpointrtransformers.utilsrtorchao.quantizationr activationsr configuration_utilsr!dynamic_module_utilsr" generationr#r$r% integrationsr&r'r(integrations.accelerater)r*integrations.deepspeedr+r,integrations.flash_attentionr-integrations.flex_attentionr.integrations.sdpa_attentionr/integrations.tensor_parallelr0r1loss.loss_utilsr2 pytorch_utilsr3r4r5r6r7r8r9 quantizersr:r;quantizers.quantizers_utilsr<safetensors_conversionr=rr>r?r@rArBrCrDrErFrGrHrIrJrKrLrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_r`ra utils.hubrbrcutils.import_utilsrdrerfrgutils.quantization_configrhrirrupperrjrlrwrmrnaccelerate.hooksroaccelerate.utilsrprqrrrsrtrurvr]rSraccelerate.utils.modelingry safetensorsrzsafetensors.torchr{rr|rr get_loggerrr rrrrrrzrr!smdistributed.modelparallel.torch modelparallelrsmdistributed.modelparallelr SMP_VERSIONrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr3ruint8int8int16rrint32rfloat64int64rrWuint16uint32uint64 float8_e5m2rrrrrprrwrrrrrrrno_gradrrrrrr'rCrOrPrrrrrJrr*r<r?rFrarrrrrrrUrrrrs       0              $  & $               1 U           Q >>               ;  P   ? <   8:l&EAuc  6 /