o hL&@s*ddlZddlmZddlmZddlmZddlZddlm m Z ddl m m Z ddlmZddlmZddlmZddl mZddlmZd gZeeZejGd d d ZGd d d ejZGdddZddZddZ ddZ!ej"deede#ddfddZ$ddZ%Gdd d Z&dS)N) defaultdict)Lock)Optional)Tensor)RRef)functional_optim_mapDistributedOptimizerc@seZdZdeddfddZdS)_ScriptLocalOptimizerInterfaceautograd_ctx_idreturnNcCsdSN)selfr rru/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/torch/distributed/optim/optimizer.pystep$sz#_ScriptLocalOptimizerInterface.step)__name__ __module__ __qualname__intrrrrrr "sr cs6eZdZeZfddZejdefddZ Z S)_ScriptLocalOptimizercs8tdd|D|_||jg|Ri||_dS)NcSg|]}|qSr local_value.0rrefrrr 0z2_ScriptLocalOptimizer.__init__..)super__init__ _local_paramsoptimr optim_clslocal_params_rrefargskwargs __class__rrr .s z_ScriptLocalOptimizer.__init__r cs.t|fdd|jD}|j|dS)Ncs g|] }|vr |ndqSr r)rpall_local_gradsrrr7sz._ScriptLocalOptimizer.step..) dist_autograd get_gradientsr!r"r)rr gradsrr+rr3s  z_ScriptLocalOptimizer.step) rrrr compile_lockr jitexportrr __classcell__rrr(rr(s  rc@s"eZdZeZddZddZdS)_LocalOptimizercOs.dd|D|_||jg|Ri||_dS)NcSrrrrrrrrLrz,_LocalOptimizer.__init__..)r!r"r#rrrr Ksz_LocalOptimizer.__init__cCsXt|}tj|D]\}}||_q |jWddS1s%wYdSr )r-r.r4 global_lockitemsgradr"r)rr r,paramr7rrrrOs  "z_LocalOptimizer.stepN)rrrrr5r rrrrrr4As r4cOstt||g|Ri|Sr )rpcrr4)r$r%r&r'rrr_new_local_optimizerXsr:cC|}||dSr rrlocal_optim_rrefr local_optimrrr_local_optimizer_step\sr@cOsXt||g|Ri|}tjt|}t|tWdS1s%wYdSr )rr0r1scriptr9rr )r$r%r&r'r" script_optimrrr_new_script_local_optimizerbs   $rCr>r r cCr;r r<r=rrr_script_local_optimizer_stepjsrDc Csfd}g}|D]$}z ||Wqty*}z |||}WYd}~qd}~ww|dur1||Sr )appendwait Exception)rpc_futs exceptionresultsfuterrr _wait_for_allrs rMc@s eZdZdZddZddZdS)r ah DistributedOptimizer takes remote references to parameters scattered across workers and applies the given optimizer locally for each parameter. This class uses :meth:`~torch.distributed.autograd.get_gradients` in order to retrieve the gradients for specific parameters. Concurrent calls to :meth:`~torch.distributed.optim.DistributedOptimizer.step`, either from the same or different clients, will be serialized on each worker -- as each worker's optimizer can only work on one set of gradients at a time. However, there is no guarantee that the full forward-backward-optimizer sequence will execute for one client at a time. This means that the gradients being applied may not correspond to the latest forward pass executed on a given worker. Also, there is no guaranteed ordering across workers. `DistributedOptimizer` creates the local optimizer with TorchScript enabled by default, so that optimizer updates are not blocked by the Python Global Interpreter Lock (GIL) in the case of multithreaded training (e.g. Distributed Model Parallel). This feature is currently enabled for most optimizers. You can also follow `the recipe`__ in PyTorch tutorials to enable TorchScript support for your own custom optimizers. Args: optimizer_class (optim.Optimizer): the class of optimizer to instantiate on each worker. params_rref (list[RRef]): list of RRefs to local or remote parameters to optimize. args: arguments to pass to the optimizer constructor on each worker. kwargs: arguments to pass to the optimizer constructor on each worker. Example:: >>> # xdoctest: +SKIP("distributed") >>> import torch.distributed.autograd as dist_autograd >>> import torch.distributed.rpc as rpc >>> from torch import optim >>> from torch.distributed.optim import DistributedOptimizer >>> >>> with dist_autograd.context() as context_id: >>> # Forward pass. >>> rref1 = rpc.remote("worker1", torch.add, args=(torch.ones(2), 3)) >>> rref2 = rpc.remote("worker1", torch.add, args=(torch.ones(2), 1)) >>> loss = rref1.to_here() + rref2.to_here() >>> >>> # Backward pass. >>> dist_autograd.backward(context_id, [loss.sum()]) >>> >>> # Optimizer. >>> dist_optim = DistributedOptimizer( >>> optim.SGD, >>> [rref1, rref2], >>> lr=0.05, >>> ) >>> dist_optim.step(context_id) __ https://github.com/pytorch/tutorials/pull/1465 c Ostjdtt}|D] }|||q |tvr&tj j r&t |}n|}||k|_ |j r3t }ntd|t}g} |D]\} } tj| ||| f||d} | | qAt| |_dS)Nz,torch.distributed.optim.DistributedOptimizera$Creating the optimizer %s without TorchScript support, this might result in slow computation time in multithreading environment(i.e. Distributed Model Parallel training on CPU) due to the Python's Global Interpreter Lock (GIL). Please file an issue if you need this optimizer in TorchScript. )r&r')torch_C_log_api_usage_oncerlistownerrErr1_state_enabledgetis_functional_optimrCloggerwarningr:r6r9 rpc_asyncrMremote_optimizers) roptimizer_class params_rrefr&r'per_worker_params_rrefr8 optim_ctoroptimizer_new_funcremote_optim_futsworker param_rrefsremote_optim_rref_futrrrr s2     zDistributedOptimizer.__init__cs:t|jr tntfdd|jD}t|dS)a% Performs a single optimization step. This will call :meth:`torch.optim.Optimizer.step` on each worker containing parameters to be optimized, and will block until all workers return. The provided ``context_id`` will be used to retrieve the corresponding :class:`~torch.distributed.autograd.context` that contains the gradients that should be applied to the parameters. Args: context_id: the autograd context id for which we should run the optimizer step. cs$g|]}tj||fdqS))r&)r9rYrR)r optimizer context_idoptimizer_step_funcrrrsz-DistributedOptimizer.step..N)r-_is_valid_contextrVrDr@rZrM)rrfrHrrerrs   zDistributedOptimizer.stepN)rrr__doc__r rrrrrr s; %)'logging collectionsr threadingrtypingrrNtorch.distributed.autograd distributedautogradr-torch.distributed.rpcr9 torch.jitr1torch.nnnnrrutilsr__all__ getLoggerrrW interfacer Modulerr4r:r@rCrArrDrMr rrrrs>