o  hn @sdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z mZddlmZddlmZddlmZmZmZmZmZmZddlZddlZddlmZm Z dd lm!Z"dd l#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*e*+e,Z-e)rddl.Z.d Z/z'ej0"d Z1d Z2ej"3e1ej"3e/kZ4ddl5Z5e5j67ddurd Z8ndZ8Wnej0j9e:e;efydZ1dZ2dZ4dZ8Ynwej?@ddupej?@dduZAereArzej0"dZBe-CdeBdWn,ej0j9yzej0"dZBe-CdeBdWnej0j9ydZAYnwYnwddlmDZDddlEmFZFmGZGddlHmIZImJZJmKZKddlLmMZMddl#mNZNmOZOddZPddZQddZRd d!ZSd"d#ZTd$d%ZUd&d'ZVd(d)ZWd*d+ZXd,d-ZYd.d/ZZd0d1Z[d2d3Z\d4d5Z]d6d7Z^d8d9Z_d:d;Z`debd?ecd@eJfdAdBZdd>ebd?ecd@eJfdCdDZed>ebd?ecd@eJfdEdFZfd>ebd?ecd@eJfdGdHZgdIdJZhdKdLZiGdMdNdNeGZjdOedPecfdQdRZkGdSdTdTeceZlGdUdVdVeGZmGdWdXdXeGZnGdYdZdZeGZoGd[d\d\eGZpGd]d^d^epZqGd_d`d`erZsGdadbdbeGZtGdcddddeGZuGdedfdfeGZvGdgdhdheGZwGdidjdjeGZxGdkdldleGZyeoenepetejemeueveqewexeydm ZzdndoZ{dS)pz+ Integrations with other Python libraries. N)asdictfields)Enum)Path) TYPE_CHECKINGAnyDictLiteralOptionalUnion)PreTrainedModelTFPreTrainedModel) __version__)PushToHubMixin flatten_dictis_datasets_availableis_pandas_availableis_tf_availableis_torch_availableloggingz3.43.2comet_mlTz comet.api_keyFneptunezneptune-clientzNeptune version z available.zNeptune-client version ) modelcard)ProgressCallbackTrainerCallback)PREFIX_CHECKPOINT_DIRBestRunIntervalStrategy) ParallelMode)ENV_VARS_TRUE_VALUESis_torch_xla_availablecCs2tddtvrtddStjdduS)NWANDB_DISABLEDzUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).Fwandb) osgetenvupperr loggerwarning importlibutil find_specr-r-/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/transformers/integrations/integration_utils.pyis_wandb_availableas r/cCtjdduS)Nclearmlr*r+r,r-r-r-r.is_clearml_availablelr3cCsdtdddkrtddStdurdStdur%tdtttdSt dur0tddSdS) N COMET_MODEr#DISABLEDzUsing the `COMET_MODE=DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).Fzcomet_ml version %s is installed, but version %s or higher is required. Please update comet_ml to the latest version to enable Comet logging with pip install 'comet-ml>=%s'.a#comet_ml is installed but the Comet API Key is not configured. Please set the `COMET_API_KEY` environment variable to enable Comet logging. Check out the documentation for other ways of configuring it: https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#set-the-api-keyT) r%r&r'r(r)_is_comet_installed_is_comet_recent_enough_comet_version_MIN_COMET_VERSION_is_comet_configuredr-r-r-r.is_comet_availableps*r<cCs tjdduptjdduS)N tensorboard tensorboardXr2r-r-r-r.is_tensorboard_availables r?cCr0)Noptunar2r-r-r-r.is_optuna_availabler4rAcCr0)Nrayr2r-r-r-r.is_ray_availabler4rCcCtsdStjdduS)NFzray.tune)rCr*r+r,r-r-r-r.is_ray_tune_availablerEcCr0)Nsigoptr2r-r-r-r.is_sigopt_availabler4rHcCs8tjddur dStjddurdStjdduS)NazuremlFz azureml.corezazureml.core.runr2r-r-r-r.is_azureml_availables rJcCs(tdddkr dStjdduS)NDISABLE_MLFLOW_INTEGRATIONFALSETRUEFmlflow)r%r&r'r*r+r,r-r-r-r.is_mlflow_availablesrOcCsdtjdtjdfvS)NdagshubrNr2r-r-r-r.is_dagshub_availablesrQcCstSN) _has_neptuner-r-r-r.is_neptune_availablesrTcCr0)N codecarbonr2r-r-r-r.is_codecarbon_availabler4rVcCr0)Nflytekitr2r-r-r-r.is_flytekit_availabler4rXcCrD)NFzflytekitplugins.deck)rXr*r+r,r-r-r-r. is_flyte_deck_standard_availablerFrYcCr0)Ndvcliver2r-r-r-r.is_dvclive_availabler4r[cCr0)Nswanlabr2r-r-r-r.is_swanlab_availabler4r]cCsntrddl}t||jjr|jStrt|tr|Str%t|tr%|St r/t|tr/|St d|j )NrzUnknown type for trial ) rAr@ isinstancetrial BaseTrialparamsrEdictrHr/ RuntimeError __class__)r_r@r-r-r. hp_paramss   ren_trials directionreturnc s\ddlddlmjjdkrqddjffdd }|dd}|dd}|d d }t|tr6|nd}|dur>dn|}j d||d |} | j |||||d | sg| j } t t| j| j| jS| j} d d| DSt|D]6} d_dg} jjtjkrtdtjj| ddjd| ddtdddur}|_qudS)Nr)release_memoryr_csd}|rt|D]}|trtj||}q d_jjdkrMjj t j kr,t d |j|j|j}|g}tjj|ddj||dnj||dtdddurf}|_jj\__jjS)NConly support DDP optuna HPO for ParallelMode.DISTRIBUTED currently.rsrcresume_from_checkpointr_ objective)r%listdir startswithrpathjoinrpargs world_size parallel_moder DISTRIBUTEDrchp_spacer_ FixedTrialranumbertorch distributedbroadcast_object_listtraingetattrevaluatecompute_objective model_wrappedmodel acceleratorclear)r_checkpoint_dir checkpointsubdir fixed_trialtrial_main_rank_listmetricsr@ritrainerr-r. _objectives,     z(run_hp_search_optuna.._objectivetimeoutn_jobsrjgc_after_trialF)rg directions)rfrrrcSs"g|] }tt|j|j|jqSr-)rstrr{valuesra).0bestr-r-r. s"z(run_hp_search_optuna..rkrlrnrprRr-) r@accelerate.utils.memoryriru process_indexTrialpopr^list create_studyoptimize_is_multi_objective best_trialrrr{valuera best_trialsrangerprwrrxrcr|r}r~rrrr)rrfrgkwargsrrrrrstudyrrirrr-rr.run_hp_search_optunas8       rc sddlddldtffdd }|jjs&ddlm}td|dd |_| t }d|_ d |vrYd d i|d <|j j dkrDd |d d <d|j j dkrMdnd}td|d|d d d}||j _d|vrwddlm} | dgd|d<d|vrddlm} m} m} m} t|d| | | | fr|j jr|j jtjkrtdjt|dj dj!j"||dt#$fdd}t%drƈj&|_&j!j'|f|(d|d|}|j)d|dd |j j*d!}t+|j,|j-d|j.|}|dur|/||S)"Nrr_csDzddlm}||r|tWn tyYnwd|_j}|rSd|_| }t t | t d}|j||dWdn1sMwYn|j|dt|dddur|}|||_||jddt}|j|d jj|}jj||d WddS1swYdSdS) Nr)NotebookProgressCallbackrp*rn)r_T)rpdone)r)r)transformers.utils.notebookr pop_callback add_callbackrModuleNotFoundErrorrprget_checkpoint as_directorynextrglobras_posixrrrupdatetempfileTemporaryDirectory_tune_save_checkpoint Checkpointfrom_directoryreport)r_ local_trainerrrrcheckpoint_pathrtemp_checkpoint_dir)rBr-r.r*s8          "z%run_hp_search_ray.._objectiver )TrainerMemoryTrackerzMemory tracking for your Trainer is currently enabled. Automatically disabling the memory tracker since the memory tracker is not serializable.T)skip_memory_metricsresources_per_trialcpurjgpuz1 CPUz and 1 GPUr#zgNo `resources_per_trial` arg was passed into `hyperparameter_search`. Setting it to a default value of z for each trial.progress_reporter) CLIReporterrp)metric_columns scheduler) ASHASchedulerHyperBandForBOHBMedianStoppingRulePopulationBasedTrainingaYou are using {cls} as a scheduler but you haven't enabled evaluation during training. This means your trials will not report intermediate results to Ray Tune, and can thus not be stopped early or used to exploit other trials parameters. If this is what you want, do not use {cls}. If you would like to use {cls}, make sure you pass `do_eval=True` and `eval_strategy='steps'` in the Trainer `args`.)cls)rcsbtr*ddl}tj|jd}tj d|}tj |}|t j |j <|j||i|S)a" Wrapper around `tune.with_parameters` to ensure datasets_modules are loaded on each Actor. Without this, an ImportError will be thrown. See https://github.com/huggingface/transformers/issues/11565. Assumes that `_objective`, defined above, is a function. rNz __init__.pydatasets_modules)r datasets.loadr%rsrtloadinit_dynamic_modulesr*r+spec_from_file_locationmodule_from_specsysmodulesnameloader exec_module)rurdatasetsdynamic_modules_pathspecr) trainabler-r. dynamic_modules_import_trainables    z;run_hp_search_ray..dynamic_modules_import_trainable __mixins__)config num_samples)metricmodescope)0rB ray.trainrb_memory_trackerr trainer_utilsrr(r)rTensorBoardCallbackrrun_gpuinfoget_n_gpuray.tunerray.tune.schedulersrrrrr^do_eval eval_strategyrNOrcformattype__name__tunewith_parameters functoolswrapshasattrrrunryget_best_trial ray_scopertrial_id last_resultrr)rrfrgrrr _tb_writer resource_msggpus_per_trialrrrrrranalysisrbest_runr-)rBrr.run_hp_search_ray&sr&          rc Ksddl}|jjdkrtjddkr|d|jdd|dd|dd gd |d }t d |j | D]b}|Vd|_ |jjd krj|jjtjkrQtd ||jtjjt|jdd|jddn|jd|jdt|dddur|}|||_ |d|j Wdn1swYq8t| d}t!|j |j"dj#|j$} | Sddlm%} | } |&dd} | dur| '| | (j)d|dd|dd gd |dd}t d |j |j*j+|j,krs| (|j -)} d|_ |jjd kr'|jjtjkrtd || tjjt|jdd|jddn|jd| dt|dddurA|}|||_ d|j dg}| (|j .j)| j |d}t d| j d|j d| (|j /}|j*j+|j,kst| (|j 0/1d}t!|j |j#|j$} | St2|D]Z}d|_ tt|j}|jjtjkrtd tjj|ddt3t4|}t5|6D]\}}|dkrt7|j||q|jddt|dddur|}|||_ qdS)NrrGz8.0.0 huggingfacezhuggingface-tuneofflinerpr)rrpstrategyrj)rr parametersrparallel_bandwidthbudgetz6created experiment: https://app.sigopt.com/experiment/zConly support DDP Sigopt HPO for ParallelMode.DISTRIBUTED currently.rl)rorn) Connectionproxies)rrrrobservation_budgetprojectrr) suggestionrz"[suggestion_id, observation_id]: [, ] local_rank)8rGrurr*metadataversion set_projectcreate_experimentryr(ridlooprprvrwrrxrc_hp_search_setuprr|r}r~pickledumpsrrrr log_metricr get_best_runsrrr assignmentsr r set_proxies experimentscreateprogressobservation_countr  suggestions observationsfetchbest_assignments iterate_pagesrloadsbytesritemssetattr)rrfrgrrG experimentrrrrr connr rrobsrargs_main_rankrukeyrr-r-r.run_hp_search_sigopts      *           r2c  s^ddlm}|s tdddld}jjD] }t|tr"d}nq|s+tdgj _ dddd| d d}| d d}| d d} | d d} | d d d} | d d<| d d <| rk| | d <fdd} |sj | || d}nddl| rj| j|td|j|| |dtddd|S)Nr )r/z8This function needs wandb installed: `pip install wandb`rFTr$)run_idrphyperparameterssweep_idr rentityrz eval/lossgoalcsjrjn}|jj_|jidj}d_jdt |ddt dddurP } |_t |}|vrPtdd|d}ddurod krdjdk}n d krojdk}|swddur|jd<jd<t|d <jS) N)rr_itemsrnrpzProvided metric zU not found. This might result in unexpected sweeps charts. The available metrics are Fr3minimizemaximizer4)rinitrstate trial_namerrrprvarsrrr rewrite_logsr(r)keysrrb)rrrformat_metrics best_scorerrgrrr$r-r.r+s6      z'run_hp_search_wandb.._objective)r r6zwandb sweep id - )functioncountr3rpr4) integrationsr/ ImportErrorr$callback_handler callbacksr^ WandbCallbackrru report_torrysweep wandb.envenv set_entityrr(ragentr) rrfrgrr/reporting_to_wandbcallbackr5r rr6 sweep_configrr-rCr.run_hp_search_wandbsD              !  rTcCsg}tr ts |dtr|dtr|dtr%|dtr-|dtr5|dtr=|dtrE|dt rM|d t rU|d t r]|d |S) Nazure_mlrrPrZrNrr=r$rUr1r\) rJrOappendr<rQr[rTr?r/rVr3r])rFr-r-r.$get_available_reporting_integrations[s0            rWcCs~i}d}t|}d}t|}|D]*\}}||r&||d||d<q||r6||d||d<q||d|<q|S)Neval_test_eval/ztest/ztrain/)lenr+rr)dnew_d eval_prefixeval_prefix_len test_prefixtest_prefix_lenkvr-r-r.r?vs  r?c@s>eZdZdZd ddZd ddZddZd d d Zd d ZdS)rz A [`TrainerCallback`] that sends the logs to [TensorBoard](https://www.tensorflow.org/tensorboard). Args: tb_writer (`SummaryWriter`, *optional*): The writer to use. Will instantiate one if not set. Nc Cst}|s td|r8z ddlm}||_Wn$ty7z ddlm}||_Wn ty4d|_YnwYnwd|_||_dS)NzuTensorBoardCallback requires tensorboard to be installed. Either update your PyTorch version or install tensorboardX.r) SummaryWriter)r?rctorch.utils.tensorboardrd_SummaryWriterrGr> tb_writer)selfrghas_tensorboardrdr-r-r.__init__s(        zTensorBoardCallback.__init__cCs*|p|j}|jdur|j|d|_dSdS)N)log_dir) logging_dirrfrg)rhrurkr-r-r._init_summary_writers  z(TensorBoardCallback._init_summary_writerc Ks|jsdSd}|jr|j}|durtj|j|}|jdur$||||jdurR|j d| d|vrT|d}t |drV|j durX|j }|j d|dSdSdSdSdS)Nrurr model_config) is_world_process_zerois_hyper_param_searchr=r%rsrtrlrgrmadd_textto_json_stringrr) rhrur<controlrrkr=rmodel_config_jsonr-r-r.on_train_begins&    z"TensorBoardCallback.on_train_beginc Ks|jsdS|jdur|||jdurYt|}|D]5\}}t|ttfr1|j|||j qt|t r@|j |||j qt d|dt|d|dq|jdSdS)N)Trainer is attempting to log a value of " " of type for key "zn" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.)rorgrmr?r+r^intfloat add_scalar global_steprrqr(r)rflushrhrur<rslogsrrbrcr-r-r.on_logs,    zTensorBoardCallback.on_logcKs|jr |jd|_dSdSrR)rgcloserhrur<rsrr-r-r. on_train_end  z TensorBoardCallback.on_train_endrR) r __module__ __qualname____doc__rjrmrurrr-r-r-r.rs    rr output_dircst|ddit|trt|dn4tr*t|tr*fdd}|j|dn/trQt|tj j t frat |drit|dWddSWddSWddSWddSWddSWddS1stwYdS)Nz/model_architecture.txtzw+filecst|ddS)Nr)print)sfr-r. print_to_filer4z6save_model_architecture_to_file..print_to_file)print_fn base_model) openr^r rrrsummaryrr|nnModulerr)rrrr-rr.save_model_architecture_to_files.     "rc@sDeZdZdZdZdZdZedefddZ e de ddfd d Z d S) WandbLogModelz)Enum of possible log model values in W&B.rendfalserhcCs|tjtjfvS)zYCheck if the value corresponds to a state where the `WANDB_LOG_MODEL` setting is enabled.)r CHECKPOINTENDrhr-r-r. is_enabledszWandbLogModel.is_enabledrcCsVt|tstdt||tvrtdtddt d|dt jS) Nz>Expecting to have a string `WANDB_LOG_MODEL` setting, but got zSetting `WANDB_LOG_MODEL` as WANDB_LOG_MODELzn is deprecated and will be removed in version 5 of transformers. Use one of `'end'` or `'checkpoint'` instead.zSetting `WANDB_LOG_MODEL` from z to `end` insteadz6Received unrecognized `WANDB_LOG_MODEL` setting value=z ; so disabling `WANDB_LOG_MODEL`)r^r ValueErrorrr'r DeprecationWarningr%r&r(rrrr)rL)rrr-r-r. _missing_s   zWandbLogModel._missing_N) rrrrrrrLpropertyboolr classmethodrrr-r-r-r.rsrc@NeZdZdZddZddZdddZdd d Zdd d Zd dZ ddZ dS)rJz{ A [`TrainerCallback`] that logs metrics, media, model checkpoints to [Weight and Biases](https://www.wandb.com/). cCs@t}|s td|rddl}||_d|_ttdd|_dS)NzFWandbCallback requires wandb to be installed. Run `pip install wandb`.rFrr) r/rcr$_wandb _initializedrr%r& _log_model)rh has_wandbr$r-r-r.rjszWandbCallback.__init__c KsN|jdurdSd|_ddlm}|jrtdi|}t|dr=|j dur=t |j t r2|j n|j }i||}t|drR|j durR|j }id|i|}|j } i} | dure| | d<|j| d <n|jdur}|j| d<|j|jkr}|jjd d d |jjdur|jjd*d tddi| |jj j|ddt|jddr|jd|jjddddtdd} ts| dvr|jj|| td|jd|jjjddz ||jj d<WntytdYn|ytdYnw|j j!rt"#} |jdus|j|jkrd|jjj$nd|jjj%} |jj&| d t|dr0|j nd|jj 'ddd!d"}t(|| t)| *dD](}|+rp|j,|j%d#d$}|-|.Wdn 1skwYqI|jjj/|d%gd&d'|jj0d(}t1j2d)|7_2WddS1swYdSdSdS)+u Setup the optional Weights & Biases (*wandb*) integration. One can subclass and override this method to customize the setup if needed. Find more information [here](https://docs.wandb.ai/guides/integrations/huggingface). You can also override the following environment variables: Environment: - **WANDB_LOG_MODEL** (`str`, *optional*, defaults to `"false"`): Whether to log model and checkpoints during training. Can be `"end"`, `"checkpoint"` or `"false"`. If set to `"end"`, the model will be uploaded at the end of training. If set to `"checkpoint"`, the checkpoint will be uploaded every `args.save_steps` . If set to `"false"`, the model will not be uploaded. Use along with [`~transformers.TrainingArguments.load_best_model_at_end`] to upload best model. Setting `WANDB_LOG_MODEL` as `bool` will be deprecated in version 5 of 🤗 Transformers. - **WANDB_WATCH** (`str`, *optional* defaults to `"false"`): Can be `"gradients"`, `"all"`, `"parameters"`, or `"false"`. Set to `"all"` to log gradients and parameters. - **WANDB_PROJECT** (`str`, *optional*, defaults to `"huggingface"`): Set this to a custom string to store results in a different project. - **WANDB_DISABLED** (`bool`, *optional*, defaults to `False`): Whether to disable wandb entirely. Set `WANDB_DISABLED=true` to disable. NTr) ConfigErrorz`Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"r peft_configrgroupzThe `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.F)repeatr  WANDB_PROJECTr)allow_val_change define_metrictrain/global_stepr) step_metric step_sync WANDB_WATCHr)allr gradientsd)loglog_freqtransformers_trainer)codemodel/num_parameterszZCould not log the number of model parameters in Weights & Biases due to an AttributeError.zbA ConfigError was raised whilst setting the number of model parameters in Weights & Biases config.model-r)rnnum_parameters initial_modelrrrwbrraliasesz[Visualize in Weights & Biases]() r-)3rrwandb.sdk.lib.config_utilrror(rto_dictrrr^rbrr=run_namertermwarnrr;r%r&rrrr!watchmax logging_steps_labelrAttributeErrorr)rrrrrrArtifactrrrris_filenew_filewrite read_bytes log_artifactget_urlrAUTOGENERATED_TRAINER_COMMENT)rhrur<rrWandbConfigError combined_dictrnrr= init_args _watch_modeltemp_dir model_namemodel_artifactrfabadge_markdownr-r-r.setups                     $:zWandbCallback.setupNcKsR|jdurdS|j}|r|jd|_d|_|js'|j|||fi|dSdSNF)rrpfinishrrr)rhrur<rsrr hp_searchr-r-r.rus  zWandbCallback.on_train_beginc Ks|jdurdS|jjr|jr|jrddlm}||||dgd}t} | | |j s:ddt |jj  Dnd|j|jd|jd |jjd i} d | d <td |jdusa|j|jkrid |jjjnd |jjj} t|| |jj| d| d} t| dD]%} | r| j| jdd}| | !Wdn1swYq|jjj"| d gdWddS1swYdSdSdSdS)Nr Trainerfakerurprocessing_class eval_datasetcS,i|]\}}t|tjr|ds||qS_r^numbersNumberrrrrbrcr-r-r.  z.WandbCallback.on_train_end..rZztrain/total_flossrT final_modelzLogging model artifacts. ...rrrrrrr)#rrrrrorrrr save_modelload_best_model_at_endrbrr+metric_for_best_model best_metric total_flosrrr(rrrrrrrrrrrrrrr)rhrur<rsrrrr fake_trainerrrrartifactrrr-r-r.rsB      " zWandbCallback.on_train_endc  sgd|jdur dS|js|||||jrJ|D]\}}|vr+||jjj|<qfdd|D} t| } |ji| d|j idSdS)N train_runtimetrain_samples_per_secondtrain_steps_per_second train_lossrci|] \}}|vr||qSr-r-rsingle_value_scalarsr-r.rz(WandbCallback.on_log..r) rrrror+rrr?rr| rhrur<rsrrrrbrcnon_scalar_logsr-rr.rs zWandbCallback.on_logc Ks|jtjkry|jr{|jr}ddt|jjD}|jj d|d<d|j }t j |j|}td|d|jdusD|j|jkrLd|jjjnd|jjj}|jj|d|d } | ||jj| d t|jd d |j gd dSdSdSdS)NcSrrrrr-r-r.rrz)WandbCallback.on_save..r checkpoint- Logging checkpoint artifacts in z. ...rrrepoch_r checkpoint_global_step_r)rrrrrorbrrr+rrr|r%rsrtrr(rrrrrradd_dirrroundepoch) rhrur<rsrcheckpoint_metadatackpt_dir artifact_pathcheckpoint_namerr-r-r.on_saves$   zWandbCallback.on_savecKsJ|jdurdS|js|j||fi||jr#t|}|j|dSdSrR)rrrror?rrhrur<rsrrr-r-r. on_predicts zWandbCallback.on_predictrRNN rrrrrjrrurrrrr-r-r-r.rJ s ~ ' rJc@sDeZdZdZddZddZdddZdd d Zd d Zd dZ dS) CometCallbackz_ A [`TrainerCallback`] that sends the logs to [Comet ML](https://www.comet.com/site/). cCs<tdustdurtdtdtdd|_d|_d|_dS)NFz!CometCallback requires comet-ml>=z- to be installed. Run `pip install comet-ml>=z`.)r7r8rcr:r _log_assets _experimentrr-r-r.rjs zCometCallback.__init__cCsd|_tdd}|dvrd|_|jrtd}d}d}|durE|}|dkr-d}n|dkr4d }n|d vr;|}n |rEtd |dS|j rT|durRtd |d }ddl }|j durk|j |j krk|j |j d} n| } |j||| d|_|jj|ddd|i} t|dr|jdur|j} | | d<t|dr|jdur|j} | | d<|jj| dddd|j rt|dd} t|dd}|jj| |ddSdSdS)a Setup the optional Comet integration. Environment: - **COMET_MODE** (`str`, *optional*, default to `get_or_create`): Control whether to create and log to a new Comet experiment or append to an existing experiment. It accepts the following values: * `get_or_create`: Decides automatically depending if `COMET_EXPERIMENT_KEY` is set and whether an Experiment with that key already exists or not. * `create`: Always create a new Comet Experiment. * `get`: Always try to append to an Existing Comet Experiment. Requires `COMET_EXPERIMENT_KEY` to be set. * `ONLINE`: **deprecated**, used to create an online Experiment. Use `COMET_START_ONLINE=1` instead. * `OFFLINE`: **deprecated**, used to created an offline Experiment. Use `COMET_START_ONLINE=0` instead. * `DISABLED`: **deprecated**, used to disable Comet logging. Use the `--report_to` flag to control the integrations used for logging result instead. - **COMET_PROJECT_NAME** (`str`, *optional*): Comet project name for experiments. - **COMET_LOG_ASSETS** (`str`, *optional*, defaults to `TRUE`): Whether or not to log training assets (tf event logs, checkpoints, etc), to Comet. Can be `TRUE`, or `FALSE`. For a number of configurable items in the environment, see [here](https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#explore-comet-configuration-options). TCOMET_LOG_ASSETSrL>1rMr5NonlinerF)r get_or_creater!z:Invalid COMET_MODE env value %r, Comet logging is disabledzjHyperparameter Search is enabled, forcing the creation of new experimetns, COMET_MODE value %r is ignoredr!r)r)rrexperiment_config transformers) frameworkrurrmanual)r sourceflatten_nestedr= trial_params)optimization_idr)rr%r&r'rrolowerr(r)rprrrExperimentConfigstartr!__internal_api__set_model_graph__rrrr __internal_api__log_parameters__rlog_optimization)rhrur<r log_assetscomet_old_moderrrrrarnrr%optimization_paramsr-r-r.rs`      4zCometCallback.setupNcK|js ||||dSdSrRrrrhrur<rsrrr-r-r.rukzCometCallback.on_train_begincKsP|js |||||jr$|jdur&t|}|jj||j|jdddSdSdS)Nrstepr r rrrorr?__internal_api__log_metrics__r|r )rhrur<rsrrrrewritten_logsr-r-r.ros   zCometCallback.on_logcKsh|jr.|jr0|jdur!|jdur!td|jj|jdd|jd|j r2|j d|_dSdSdSdS)NTz(Logging checkpoints. This may take time.) recursive log_file_namer4F) rrorrr(rlog_asset_folderrr|rpcleanrr-r-r.rys        zCometCallback.on_train_endcKsR|js |j||dd|jr%|jdur't|}|jj||j|jdddSdSdS)N)rrr3r5)rhrur<rsrrrewritten_metricsr-r-r.rs  zCometCallback.on_predictrRr) rrrrrjrrurrrr-r-r-r.rs \  rc@s,eZdZdZd ddZddZd ddZdS) AzureMLCallbackzh A [`TrainerCallback`] that sends the logs to [AzureML](https://pypi.org/project/azureml-sdk/). NcCststd||_dS)NzPAzureMLCallback requires azureml to be installed. Run `pip install azureml-sdk`.)rJrc azureml_run)rhr>r-r-r.rjs zAzureMLCallback.__init__cKs2ddlm}|jdur|jr||_dSdSdS)NrRun)azureml.core.runr@r>ro get_context)rhrur<rsrr@r-r-r. on_init_ends zAzureMLCallback.on_init_endcKsJ|jr|jr!|D]\}}t|ttfr|jj|||dq dSdSdS)N) description)r>ror+r^ryrzrr~r-r-r.rs zAzureMLCallback.on_logrR)rrrrrjrCrr-r-r-r.r=s  r=c@sLeZdZdZddZddZdddZdd d Zd d Zd dZ ddZ dS)MLflowCallbackz A [`TrainerCallback`] that sends the logs to [MLflow](https://www.mlflow.org/). Can be disabled by setting environment variable `DISABLE_MLFLOW_INTEGRATION = TRUE`. cCsJtstdddl}|jjj|_|jjj|_d|_ d|_ d|_ ||_ dS)NzIMLflowCallback requires mlflow to be installed. Run `pip install mlflow`.rF) rOrcrNutils validationMAX_PARAM_VAL_LENGTH_MAX_PARAM_VAL_LENGTHMAX_PARAMS_TAGS_PER_BATCH_MAX_PARAMS_TAGS_PER_BATCHr _auto_end_run_log_artifacts_ml_flow)rhrNr-r-r.rjs   zMLflowCallback.__init__c Cstddtv|_tddtv|_tdd|_tdd|_tddtv|_tdd|_ td d|_ t j |jjt j d k|_td |jd |jd |jd|j|jr}|js|jr~|j|jtd|jntdn td|j|jdus|js|j r|jr|j|j|jj|j|jdtd|jjjd|_|}t|dr|j dur|j }i||}|jrt!|n|}t"|#D]\}}t$t%||j&krt'd|d|d||=qt"|#}|j r6|j (r6t)|j } | t$|kr6tdt$|d| d|d| }t*dt$||j+D])} |jrX|jj,t-|| | |j+ddq?|j,t-|| | |j+q?tdd} | r}t./| } |j0| d|_1dS)a Setup the optional MLflow integration. Environment: - **HF_MLFLOW_LOG_ARTIFACTS** (`str`, *optional*): Whether to use MLflow `.log_artifact()` facility to log artifacts. This only makes sense if logging to a remote server, e.g. s3 or GCS. If set to `True` or *1*, will copy each saved checkpoint on each save in [`TrainingArguments`]'s `output_dir` to the local or remote artifact storage. Using it without a remote storage will just copy the files to your artifact location. - **MLFLOW_TRACKING_URI** (`str`, *optional*): Whether to store runs at a specific path or remote server. Unset by default, which skips setting the tracking URI entirely. - **MLFLOW_EXPERIMENT_NAME** (`str`, *optional*, defaults to `None`): Whether to use an MLflow experiment_name under which to launch the run. Default to `None` which will point to the `Default` experiment in MLflow. Otherwise, it is a case sensitive name of the experiment to be activated. If an experiment with this name does not exist, a new experiment with this name is created. - **MLFLOW_TAGS** (`str`, *optional*): A string dump of a dictionary of key/value pair to be added to the MLflow run as tags. Example: `os.environ['MLFLOW_TAGS']='{"release.candidate": "RC1", "release.version": "2.2.0"}'`. - **MLFLOW_NESTED_RUN** (`str`, *optional*): Whether to use MLflow nested runs. If set to `True` or *1*, will create a nested run inside the current run. - **MLFLOW_RUN_ID** (`str`, *optional*): Allow to reattach to an existing run which can be useful when resuming training from a checkpoint. When `MLFLOW_RUN_ID` environment variable is set, `start_run` attempts to resume a run with the specified run ID and other parameters are ignored. - **MLFLOW_FLATTEN_PARAMS** (`str`, *optional*, defaults to `False`): Whether to flatten the parameters dictionary before logging. - **MLFLOW_MAX_LOG_PARAMS** (`int`, *optional*): Set the maximum number of parameters to log in the run. HF_MLFLOW_LOG_ARTIFACTSrLMLFLOW_NESTED_RUNMLFLOW_TRACKING_URINMLFLOW_EXPERIMENT_NAMEMLFLOW_FLATTEN_PARAMS MLFLOW_RUN_IDMLFLOW_MAX_LOG_PARAMSz2.8.0zMLflow experiment_name=z , run_name=z , nested=z, tracking_uri=zMLflow tracking URI is set to zdEnvironment variable `MLFLOW_TRACKING_URI` is not provided and therefore will not be explicitly set.)rnestedzMLflow run started with run_id=Trrvz " for key "z" as a parameter. MLflow's log_param() only accepts values no longer than 250 characters so we dropped this attribute. You can use `MLFLOW_FLATTEN_PARAMS` environment variable to flatten the parameters and avoid this message.z.Reducing the number of parameters to log from z to .rF) synchronous MLFLOW_TAGS)2r%r&r'r rM _nested_run _tracking_uri_experiment_name_flatten_params_run_id_max_log_params packagingrparserNr _async_logr(debugrrois_tracking_uri_setset_tracking_uriget_tracking_uri active_runset_experiment start_runrr3rLrrrrrr+r[rrIr)isdigitryrrK log_paramsrbjsonr)set_tagsr) rhrur<rrrnrrcombined_dict_itemsmax_log_paramsr mlflow_tagsr-r-r.rsv        "    zMLflowCallback.setupNcKr/rRr0r1r-r-r.ru&r2zMLflowCallback.on_train_beginc Ks|js |||||jrbi}|D]4\}} t| ttfr#| ||<qt| tjr6| dkr6| ||<qt d| dt | d|dq|jrW|jj||jdddS|jj||jddSdS) Nrjrvrwrxzc" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.F)rr4rX)rr4)rrror+r^ryrzr|Tensornumelitemr(r)rrbrN log_metricsr|) rhrur<rsrrrrrbrcr-r-r.r*s  zMLflowCallback.on_logcKs:|jr|jr|jr|jr|jdSdSdSdSdSrR)rrorLrNrgend_runrr-r-r.r?s zMLflowCallback.on_train_endcKsp|jr2|jr4|jr6d|j}tj|j|}t d|d|j j j |d|i|j j ddSdSdSdS)Nrrz. This may take time. model_path) artifacts python_model)rrorMr|r%rsrtrr(rrNpyfunc log_model PythonModelrhrur<rsrrrr-r-r.rDs   zMLflowCallback.on_savecCs@|jrtt|jddr|jdur|jdSdSdSdS)Nrg)rLcallablerrNrgrurr-r-r.__del__OszMLflowCallback.__del__rR) rrrrrjrrurrrr~r-r-r-r.rEs k  rEcs4eZdZdZfddZfddZddZZS)DagsHubCallbackzh A [`TrainerCallback`] that logs to [DagsHub](https://dagshub.com/). Extends [`MLflowCallback`] cs.tts tdddlm}||_dS)NzLDagsHubCallback requires dagshub to be installed. Run `pip install dagshub`.r)Repo)superrjrQrGdagshub.uploadr)rhrrdr-r.rj_s   zDagsHubCallback.__init__cstddtv|_tdpd|_td|_|j|jtj d|jtj ddd td p6dd |_ t d |_ |jd urHt dtj|i|d S)z Setup the DagsHub's Logging integration. Environment: - **HF_DAGSHUB_LOG_ARTIFACTS** (`str`, *optional*): Whether to save the data and model artifacts for the experiment. Default to `False`. HF_DAGSHUB_LOG_ARTIFACTSrLHF_DAGSHUB_MODEL_NAMEmainrQrWrBRANCH)ownerrbranchrwNzpDagsHubCallback requires the `MLFLOW_TRACKING_URI` environment variable to be set. Did you run `dagshub.init()`?)r%r&r'r log_artifactsrremotersplitsepreporrsrcrr)rhrurrr-r.rhs     zDagsHubCallback.setupcKsP|jr&t|ddrt|jjtj|j d|j t |j |j dSdS)Ntrain_dataloaderz dataset.pt)rrr|saverdatasetr%rsrtrr directoryrr rr-r-r.rs  zDagsHubCallback.on_train_end)rrrrrjrr __classcell__r-r-rr.rZs   rcseZdZfddZZS)NeptuneMissingConfigurationcstddS)NaA ------ Unsupported ---- We were not able to create new runs. You provided a custom Neptune run to `NeptuneCallback` with the `run` argument. For the integration to work fully, provide your `api_token` and `project` by saving them as environment variables or passing them to the callback. )rrjrrr-r.rjsz$NeptuneMissingConfiguration.__init__)rrrrjrr-r-rr.rsrc@s<eZdZdZdZdZdZdZdZdhZ dddd dd dd d e e d e e de e de de de e f ddZ ddZddZddZddZddZeddZed d!Zd"d#Zd$d%Zd&d'Zd(d)Zd*e d+e fd,d-Zd.d/Zd?d0d1Zd2d3Zd4d5Zd6d7Zd?d8d9Z e!d:d;Z"d?dZ%dS)@NeptuneCallbackaTrainerCallback that sends the logs to [Neptune](https://app.neptune.ai). Args: api_token (`str`, *optional*): Neptune API token obtained upon registration. You can leave this argument out if you have saved your token to the `NEPTUNE_API_TOKEN` environment variable (strongly recommended). See full setup instructions in the [docs](https://docs.neptune.ai/setup/installation). project (`str`, *optional*): Name of an existing Neptune project, in the form "workspace-name/project-name". You can find and copy the name in Neptune from the project settings -> Properties. If None (default), the value of the `NEPTUNE_PROJECT` environment variable is used. name (`str`, *optional*): Custom name for the run. base_namespace (`str`, *optional*, defaults to "finetuning"): In the Neptune run, the root namespace that will contain all of the metadata logged by the callback. log_parameters (`bool`, *optional*, defaults to `True`): If True, logs all Trainer arguments and model parameters provided by the Trainer. log_checkpoints (`str`, *optional*): If "same", uploads checkpoints whenever they are saved by the Trainer. If "last", uploads only the most recently saved checkpoint. If "best", uploads the best checkpoint (among the ones saved by the Trainer). If `None`, does not upload checkpoints. run (`Run`, *optional*): Pass a Neptune run object if you want to continue logging to an existing run. Read more about resuming runs in the [docs](https://docs.neptune.ai/logging/to_existing_object). **neptune_run_kwargs (*optional*): Additional keyword arguments to be passed directly to the [`neptune.init_run()`](https://docs.neptune.ai/api/neptune#init_run) function when a new run is created. For instructions and examples, see the [Transformers integration guide](https://docs.neptune.ai/integrations/transformers) in the Neptune documentation. z%source_code/integrations/transformersmodel_parametersr_r$trainer_parametersz train/epochN finetuningT) api_tokenr rbase_namespacerlog_parameterslog_checkpointsrr rrrrc Ks^tstdzddlm} ddlm} Wnty*ddlm} ddlm} Ynw| d|t t df| d|t t df| d|t t df| d|t | d || t df| d |t | d |t t df||_ ||_ ||_||_d|_d |_d|_d |_|||d ||_d|_|jdu|_d|_|jdvrd|j|_d|_dSd|_d |_dS)NzwNeptuneCallback requires the Neptune client library to be installed. To install the library, run `pip install neptune`.rr?) verify_typerr rrrrrF)rr r>rlastz checkpoints/T checkpoints)rTrrr@neptune.internal.utilsrrGneptune.new.internal.utils#neptune.new.metadata_containers.runrrr_base_namespace_path_log_parameters_log_checkpoints _initial_run_run_is_monitoring_runr^_force_reset_monitoring_run_init_run_kwargs_volatile_checkpoints_dir_should_upload_checkpoint_recent_checkpoint_path_target_checkpoints_namespace*_should_clean_recently_uploaded_checkpoint) rhrr rrrrrneptune_run_kwargsr@rr-r-r.rjsF          zNeptuneCallback.__init__cCs"|jr|j|`d|_dSdSrR)rstoprr-r-r._stop_run_if_existss   z#NeptuneCallback._stop_run_if_existsc Kszddlm}ddlm}m}Wnty'ddlm}ddlm}m}Ynw|z| }| |j |di||_ |j d |_WdS||fy[}zt|d}~ww)Nr)init_run)NeptuneMissingApiTokenException"NeptuneMissingProjectNameExceptionsys/idr-)rrneptune.exceptionsrrrG neptune.newneptune.new.exceptionsrcopyrrrr&r^r)rhadditional_neptune_kwargsrrr run_paramser-r-r._initialize_runs"    zNeptuneCallback._initialize_runcCs(|j|_d|_|jd|_d|_dS)NTr)rrrr&r^rr-r-r._use_initial_run s z NeptuneCallback._use_initial_runcCsb|jdur |dS|js|jrdS|jr(|js(|js(|j|jdd|_dS|d|_dS)N)with_idTF)rrrrrrr^rr-r-r._ensure_run_with_monitorings     z+NeptuneCallback._ensure_run_with_monitoringcCs@|jdur |dS|js|j|jdddddd|_dSdS)NF)rcapture_stdoutcapture_stderrcapture_hardware_metricscapture_traceback)rrrrr^rrr-r-r.'_ensure_at_least_run_without_monitorings   z7NeptuneCallback._ensure_at_least_run_without_monitoringcCs|jdur ||jSrR)rrrr-r-r.r+s zNeptuneCallback.runcCs |j|jSrR)rrrr-r-r._metadata_namespace1s z#NeptuneCallback._metadata_namespacecCst|jtj<dSrR)rrrintegration_version_keyrr-r-r._log_integration_version5r4z(NeptuneCallback._log_integration_versioncCs||jtj<dSrR)to_sanitized_dictrrtrainer_parameters_key)rhrur-r-r._log_trainer_parameters8sz'NeptuneCallback._log_trainer_parameterscCsJddlm}|rt|dr!|jdur#||j|jtj<dSdSdSdS)Nr)stringify_unsupportedr) neptune.utilsrrrrrrmodel_parameters_key)rhrrr-r-r._log_model_parameters;s  z%NeptuneCallback._log_model_parameterscCsR|rt|dr|j|jtj<|r#t|dr%|jdur'|j|jtj<dSdSdSdS)Nr=r$)rr=rrtrial_name_keyr$trial_params_key)rhr<r-r-r."_log_hyper_param_search_parametersCs  z2NeptuneCallback._log_hyper_param_search_parameterssource_directoryrc Cstj||}}|jdurMtj|j|}z|ddtjj}tj||}t|||}Wnt yL}zt d |WYd}~nd}~ww|j |j||jrh|jdurh|j |j|j||_dS)Nz..r#zpNeptuneCallback was unable to made a copy of checkpoint due to I/O exception: '{}'. Could fail trying to upload.)r%rsrtrreplacelstriprshutilcopytreeIOErrorr(r)rrr upload_filesrr delete_files) rhrr target_path relative_pathconsistent_checkpoint_path cpkt_path copy_pathrr-r-r._log_model_checkpointJs&   z%NeptuneCallback._log_model_checkpointcKsHd|_|jr|js|jdurtj|_|jdkr |js"tddSdS)NrzWTo save the best model checkpoint, the load_best_model_at_end argument must be enabled.) rroverwrite_output_dirsave_total_limitrrrrrrr-r-r.rCbs  zNeptuneCallback.on_init_endcKsR|jsdS|d|_||jr|||||jr'||dSdS)NT) rorrrrrrrprr1r-r-r.rujs  zNeptuneCallback.on_train_begincKs |dSrR)rrr-r-r.rys zNeptuneCallback.on_train_endcCs&|jdur tj|jdd|dS)NT) ignore_errors)rrrmtreerrr-r-r.r~|s  zNeptuneCallback.__del__cKs$|jr||jd|jdSdS)Nr)rrrr|rr-r-r.rzNeptuneCallback.on_savec Ks`|jdkr.|j}|dsd|}||}|jrtjntj}|jdup*|||j|_ dSdS)NrrX) rrrrrgreater_is_betternpgreaterlessrr) rhrur<rsrrbest_metric_name metric_valueoperatorr-r-r. on_evaluates    zNeptuneCallback.on_evaluatecCs*|jjD] }t||r|jSqtd)Nz6The trainer doesn't have a NeptuneCallback configured.)rHrIr^r Exception)rrrRr-r-r.get_runs   zNeptuneCallback.get_runrcKsj|jsdS|dur1t|D]#\}}t|ttfr0|tjvr%||j|<q|j|j ||j dqdSdS)Nr4) ror?r+r^ryrzr flat_metricsrrr|)rhrur<rsrrrrr-r-r.rs  zNeptuneCallback.on_logrR)&rrrrrrrrrrr rrrjrrrrrrrrrrrrrrCrurr~rrrrrrzrr-r-r-r.rsf 7      rc@s2eZdZdZddZddZd ddZd d ZdS) CodeCarbonCallbackzI A [`TrainerCallback`] that tracks the CO2 emission of training. cCs6tstdtjjrtdddl}||_d|_dS)NzWCodeCarbonCallback requires `codecarbon` to be installed. Run `pip install codecarbon`.aZCodeCarbonCallback requires `codecarbon` package, which is not compatible with AMD ROCm (https://github.com/mlco2/codecarbon/pull/490). When using the Trainer, please specify the `report_to` argument (https://huggingface.co/docs/transformers/v4.39.3/en/main_classes/trainer#transformers.TrainingArguments.report_to) to disable CodeCarbonCallback.r)rVrcr|rhiprU _codecarbontracker)rhrUr-r-r.rjs zCodeCarbonCallback.__init__cKs.|jdur|jr|jj|jd|_dSdSdS)N)r)ris_local_process_zerorEmissionsTrackerrrr-r-r.rCszCodeCarbonCallback.on_init_endNcK"|jr |jr|jdSdSdSrR)rrr(r1r-r-r.ru z!CodeCarbonCallback.on_train_begincKrrR)rrrrr-r-r.rrzCodeCarbonCallback.on_train_endrR)rrrrrjrCrurr-r-r-r.rs   rc@sxeZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd d ZddZdddZddZdddZddZddZd S)ClearMLCallbacka A [`TrainerCallback`] that sends the logs to [ClearML](https://clear.ml/). Environment: - **CLEARML_PROJECT** (`str`, *optional*, defaults to `HuggingFace Transformers`): ClearML project name. - **CLEARML_TASK** (`str`, *optional*, defaults to `Trainer`): ClearML task name. - **CLEARML_LOG_MODEL** (`bool`, *optional*, defaults to `False`): Whether to log models as artifacts during training. r# TransformerszModel Configuration_ignore_hparams_ui_overrides_"_ignore_model_config_ui_overrides_z%The configuration of model number {}.zNote that, when cloning this task and running it remotely, the configuration might be applied to another model instead of this one. To avoid this, initialize the task externally by calling `Task.init` before the `ClearMLCallback` is instantiated.rFNcCs:tr ddl}||_ntdd|_d|_d|_g|_dS)NrzNClearMLCallback requires 'clearml' to be installed. Run `pip install clearml`.F)r3r1_clearmlrcr _clearml_taskr_checkpoints_saved)rhr1r-r-r.rjs zClearMLCallback.__init__c Ks|jdurdS|jr dStjd7_tjd7_tjdkr!dndttjt_|jrat d|j durtj durO|jj rH|jj rLdt_ ndt_ |jj r||jj r||jj |_ tdtjsjdnd td hv|_t d n.|jj jtd d td dddddd|_ tdd td hv|_dt_t dd|_tjtj}|dtj}|jj r||||j j|dtddn|j j|ddds|j ||n ||tjtjt|dddurc|dtj}tj !tj} tjtjkr| dtj"7} |jj r4|j j|dtdd|j j#tj$tj|j%&| ddS|j j|dddsO|j%'|j (tj$tj|_%dS|j j#tj$tj|j%&| ddSdSdS)Nrjr#rz"Automatic ClearML logging enabled.FTCLEARML_LOG_MODELrLrMz)External ClearML Task has been connected.CLEARML_PROJECTzHuggingFace Transformers CLEARML_TASKr)r=pytorch) project_name task_nameauto_connect_frameworks output_uriz"ClearML Task has been initialized./zIf True, ignore Transformers hyperparameters overrides done in the UI/backend when running remotely. Otherwise, the overrides will be applied when running remotely)rr value_typerD)defaultcastr zIf True, ignore Transformers model configuration overrides done in the UI/backend when running remotely. Otherwise, the overrides will be applied when running remotely)r config_dictrD))rrr_train_run_counter_model_connect_counterr log_suffixror(rr_should_close_on_train_endTaskrunning_locally current_taskr%r&_task_created_in_callbackr'r unionrr;_hparams_section_ignore_hparams_overrides_copy_training_args_as_hparams set_parameterr get_parameterconnectr_ignoge_model_config_overrides_model_config_descriptionr_model_config_description_noteset_configuration_object_model_config_sectionrr from_dict get_configuration_object_as_dict) rhrur<rrrsuffixed_hparams_sectionignore_hparams_config_sectionignore_model_config_section configuration_object_descriptionr-r-r.rs                     4zClearMLCallback.setupcKsF|jdurdSg|_|jrd|_|js!|j||||fi|dSdSr)rrrprrrhrur<rsrrrr-r-r.ruYs zClearMLCallback.on_train_begincKstjr |jdt_dSdS)Nr)rrrrrrr-r-r.rbrzClearMLCallback.on_train_endc KsJ|jdurdS|js|j||||fi||jrd}t|} d} t| } gd} |D]u\} }t|ttfr| | vrJ|j j | t j |dq-| |rd|j jdt j | | d||jdq-| | r~|j jdt j | | d||jdq-|j jdt j | ||jdq-td |d t|d | d q-dSdS) NrXrY)rrrrrr reval)titleseriesr iterationtestrrvrwrxzn" as a scalar. This invocation of ClearML logger's report_scalar() is incorrect so we dropped this attribute.)rrrror[r+r^ryrzr get_loggerreport_single_valuerrrr report_scalarr|r(r)r)rhrur<rsrrrrr^r_r`rarrbrcr-r-r.rgs^           zClearMLCallback.on_logc KsH|jr|jr|jrd|j}tj|j|}|tj }t d|d|j j |j|d}|j|j|d|j|||jdd|j||jr|jt|jkrz|j jjj|jddddd Wn ty} zt d |jdj| WYd} ~ dSd} ~ ww|jd d|_|jr|jt|jksQdSdSdSdSdSdSdS) NrzLogging checkpoint artifact `z`. This may take some time.)taskrF) weights_pathtarget_filenamer0auto_delete_filerT)delete_weights_fileforceraise_on_errorszVCould not remove checkpoint `{}` after going over the `save_total_limit`. Error is: {}rj)rrror|r%rsrtrrrr(rr OutputModelr update_weights_packagerrVrr[rModelremoverr)rr) rhrur<rsrrrr output_modelrr-r-r.rsF       zClearMLCallback.on_savecsJfddtD}dd|jjj|D}|jjj||ddS)Ncs.i|]}|jr|jds|jt|jqS)_token)r;rendswithr)rfield training_argsr-r.rs zBClearMLCallback._copy_training_args_as_hparams..cSsi|] \}}t||qSr-)rrr-r-r.rs)prefix) rr utilities proxy_objectflatten_dictionaryr+r _argumentscopy_from_dict)rhrErFas_dict flat_dictr-rDr.rs z.ClearMLCallback._copy_training_args_as_hparamsr)NNN)rrrrrrr%rr!r"r#rrrrrjrrurrrrr-r-r-r.rs*  c 5 rcs<eZdZdZd dedeffdd ZddZd d ZZS) FlyteCallbacka^A [`TrainerCallback`] that sends the logs to [Flyte](https://flyte.org/). NOTE: This callback only works within a Flyte task. Args: save_log_history (`bool`, *optional*, defaults to `True`): When set to True, the training logs are saved as a Flyte Deck. sync_checkpoints (`bool`, *optional*, defaults to `True`): When set to True, checkpoints are synced with Flyte and can be used to resume training in the case of an interruption. Example: ```python # Note: This example skips over some setup steps for brevity. from flytekit import current_context, task @task def train_hf_transformer(): cp = current_context().checkpoint trainer = Trainer(..., callbacks=[FlyteCallback()]) output = trainer.train(resume_from_checkpoint=cp.restore()) ``` Tsave_log_historysync_checkpointscsXtts tdtrtstdd}ddlm }|j |_ ||_ ||_ dS)NzLFlyteCallback requires flytekit to be installed. Run `pip install flytekit`.zSyncing log history requires both flytekitplugins-deck-standard and pandas to be installed. Run `pip install flytekitplugins-deck-standard pandas` to enable this feature.Fr)current_context)rrjrXrGrYrr(r)rWrQrcprOrP)rhrOrPrQrr-r.rjs     zFlyteCallback.__init__cKsR|jr%|jr'd|j}tj|j|}td|d|j |dSdSdS)NrzSyncing checkpoint in z to Flyte. This may take time.) rPror|r%rsrtrr(rrRrr|r-r-r.rs  zFlyteCallback.on_savec KsL|jr$ddl}ddlm}ddlm}||j}|d||dSdS)Nr)Deck) TableRendererz Log History) rOpandasrWrSflytekitplugins.deck.rendererrT DataFrame log_historyto_html) rhrur<rsrpdrSrTlog_history_dfr-r-r.rs   zFlyteCallback.on_train_end)TT) rrrrrrjrrrr-r-rr.rNs rNc@sheZdZdZ  ddeedeeedeffddZ dd Z dd d Z dd d Z ddZ ddZdS)DVCLiveCallbackaW A [`TrainerCallback`] that sends the logs to [DVCLive](https://www.dvc.org/doc/dvclive). Use the environment variables below in `setup` to configure the integration. To customize this callback beyond those environment variables, see [here](https://dvc.org/doc/dvclive/ml-frameworks/huggingface). Args: live (`dvclive.Live`, *optional*, defaults to `None`): Optional Live instance. If None, a new instance will be created using **kwargs. log_model (Union[Literal["all"], bool], *optional*, defaults to `None`): Whether to use `dvclive.Live.log_artifact()` to log checkpoints created by [`Trainer`]. If set to `True`, the final checkpoint is logged at the end of training. If set to `"all"`, the entire [`TrainingArguments`]'s `output_dir` is logged at each checkpoint. NliverzrcKststdddlm}d|_d|_t||r||_n |dur)td|jd||_|jdurMt dd}| t vrBd |_dS| d krOd |_dSdSdS) NzLDVCLiveCallback requires dvclive to be installed. Run `pip install dvclive`.rLiveFz Found class z for live, expected dvclive.LiveHF_DVCLIVE_LOG_MODELrLTr)r[rcrZr_rr]r^rdrr%r&r'r r&)rhr]rzrr_ log_model_envr-r-r.rjs$        zDVCLiveCallback.__init__cCs>ddlm}d|_|jr|js||_|j|dSdS)a@ Setup the optional DVCLive integration. To customize this callback beyond the environment variables below, see [here](https://dvc.org/doc/dvclive/ml-frameworks/huggingface). Environment: - **HF_DVCLIVE_LOG_MODEL** (`str`, *optional*): Whether to use `dvclive.Live.log_artifact()` to log checkpoints created by [`Trainer`]. If set to `True` or *1*, the final checkpoint is logged at the end of training. If set to `all`, the entire [`TrainingArguments`]'s `output_dir` is logged at each checkpoint. rr^TN)rZr_rror]rkr)rhrur<rr_r-r-r.r.s zDVCLiveCallback.setupcKr/rRr0r1r-r-r.ruAr2zDVCLiveCallback.on_train_beginc Ks|js |||||jrJddlm}ddlm}|D]%\} } || r1|j || d| qt d| dt | d| dq|j dSdS) Nr)Metric)standardize_metric_namezdvclive.huggingfacervrwrxzh" as a scalar. This invocation of DVCLive's Live.log_metric() is incorrect so we dropped this attribute.)rrro dvclive.plotsrb dvclive.utilsrcr+ could_logr]rr(r)r next_step) rhrur<rsrrrrbrcr1rr-r-r.rEs&   zDVCLiveCallback.on_logcKs4|jdkr|jr|jr|j|jdSdSdSdS)Nr)rrror]rrrr-r-r.rXs zDVCLiveCallback.on_savec Ks|jrE|jrGddlm}|jdur>|||d|ddgd}|jr%dnd }tj |j |}| ||j j ||ddd |j dSdSdS) NrrTrrrrrr)rrr)rrotransformers.trainerrrrrr%rsrtrrr]rr) rhrur<rsrrrrrr-r-r.r\s    zDVCLiveCallback.on_train_endrrR)rrrrr rr r rrjrrurrrr-r-r-r.r\s    r\c@r)SwanLabCallbackzn A [`TrainerCallback`] that logs metrics, media, model checkpoints to [SwanLab](https://swanlab.cn/). cCs4tstdddl}||_d|_tdd|_dS)NzLSwanLabCallback requires swanlab to be installed. Run `pip install swanlab`.rFSWANLAB_LOG_MODEL)r]rcr\_swanlabrr%r&r)rhr\r-r-r.rjss zSwanLabCallback.__init__c Ksd|_|jrtdi|}t|dr/|jdur/t|jtr$|jn|j}i||}t|drD|j durD|j }id|i|}|j }i} |durX|j d|| d<n |j durb|j | d<t dd| d <|jdurz|jjdi| d |jjd <|jj|z3|jjd |it|jd kst|jdkr|\} } |jjd| i|jjd| iWntytdYnw|jdurtdd|jjjjd} tjd| 7_dSdSdS)a Setup the optional SwanLab (*swanlab*) integration. One can subclass and override this method to customize the setup if needed. Find more information [here](https://docs.swanlab.cn/guide_cloud/integration/integration-huggingface-transformers.html). You can also override the following environment variables. Find more information about environment variables [here](https://docs.swanlab.cn/en/api/environment-variable.html#environment-variables) Environment: - **SWANLAB_API_KEY** (`str`, *optional*, defaults to `None`): Cloud API Key. During login, this environment variable is checked first. If it doesn't exist, the system checks if the user is already logged in. If not, the login process is initiated. - If a string is passed to the login interface, this environment variable is ignored. - If the user is already logged in, this environment variable takes precedence over locally stored login information. - **SWANLAB_PROJECT** (`str`, *optional*, defaults to `None`): Set this to a custom string to store results in a different project. If not specified, the name of the current running directory is used. - **SWANLAB_LOG_DIR** (`str`, *optional*, defaults to `swanlog`): This environment variable specifies the storage path for log files when running in local mode. By default, logs are saved in a folder named swanlog under the working directory. - **SWANLAB_MODE** (`Literal["local", "cloud", "disabled"]`, *optional*, defaults to `cloud`): SwanLab's parsing mode, which involves callbacks registered by the operator. Currently, there are three modes: local, cloud, and disabled. Note: Case-sensitive. Find more information [here](https://docs.swanlab.cn/en/api/py-init.html#swanlab-init) - **SWANLAB_LOG_MODEL** (`str`, *optional*, defaults to `None`): SwanLab does not currently support the save mode functionality.This feature will be available in a future release - **SWANLAB_WEB_HOST** (`str`, *optional*, defaults to `None`): Web address for the SwanLab cloud environment for private version (its free) - **SWANLAB_API_HOST** (`str`, *optional*, defaults to `None`): API address for the SwanLab cloud environment for private version (its free) TzYAutomatic SwanLab logging enabled, to disable set os.environ["SWANLAB_MODE"] = "disabled"rNr-experiment_nameSWANLAB_PROJECTr u🤗transformers FRAMEWORKmodel_num_parameters PeftModelPeftMixedModelpeft_model_trainable_paramspeft_model_all_paramzQCould not log the number of model parameters in SwanLab due to an AttributeError.sSwanLab does not currently support the save mode functionality. This feature will be available in a future release.z[Visualize in SwanLab](rrr-)rror(rrrrr^rbrr=rr%r&rkrr;rrrrget_nb_trainable_parametersrrr)publiccloudexp_urlrr) rhrur<rrrrnrr=rtrainable_params all_paramrr-r-r.r|sX+          'zSwanLabCallback.setupNcKs$|js|j|||fi|dSdSrRr0r1r-r-r.rurzSwanLabCallback.on_train_begincK0|jdur|jr|jrtddSdSdSdSNrurrror(r)r,r-r-r.r  zSwanLabCallback.on_train_endc  sgd|js|||||jrM|D]\}}|vr+|jjd||i|jdqfdd|D} t| } |jji| d|ji|jddSdS)Nrz single_value/rcrr-r-rrr-r.rrz*SwanLabCallback.on_log..r)rrror+rkrr|r?rr-rr.rs$zSwanLabCallback.on_logcKr|r}r~rr-r-r.rrzSwanLabCallback.on_savecKs<|js |j||fi||jrt|}|j|dSdSrR)rrror?rkrrr-r-r.r s zSwanLabCallback.on_predictrRrrr-r-r-r.rins a   ri) rUrrNrr=r$rUr1rPflyterZr\cCsv|durgSt|trd|krgSd|krt}n|g}|D]}|tvr3t|ddtdqdd|DS)Nnonerz is not supported, only rz are supported.cSsg|]}t|qSr-)INTEGRATION_TO_CALLBACK)r integrationr-r-r.r, sz7get_reporting_integration_callbacks..)r^rrWrrrtr@)rKrr-r-r.#get_reporting_integration_callbacks s r)|rrimportlib.metadatar*importlib.utilrlrr%rrrr dataclassesrrenumrpathlibrtypingrrrr r r numpyrpackaging.versionr`r#r rrrrFrrrrrrrr2rr(r|r:rr9r7rar8rr get_configr;PackageNotFoundErrorrGr TypeErrorrKeyErrorr+r,rS_neptune_versionrrtrainer_callbackrrrrrrrErr r!r/r3r<r?rArCrErHrJrOrQrTrVrXrYr[r]reryrrrr2rTrWr?rrrrJrr=rErrrrrrrNr\rirrr-r-r-r.s     $        !>bMXw31 "~?i