o hv @sUddlZddlZddlZddlZddlZddlmZmZddlm Z m Z m Z m Z m Z ddlmZmZmZmZmZmZmZmZddlmZddlmZmZddlZddlmZddlm Z dd l!m"Z"erkdd l#m$Z$gd Z%e&e'Z(Gd d d ej)Z*deej+dfde*fddZ,GdddeZ-dej+de-fddZ.deej+e ej+fddfddZ/GdddZ0Gddde0Z1ej2de3de dfdd Z4Gd!d"d"e"Z5d#d$d%e ej+d&eee eej+gdfeej+gdffd'ed(de"fd)d*Z6d+a7e8e9d,<e:e;e;e;fZdej+de=fd1d2Z?Gd3d4d4Z@Gd5d6d6e0ZAGd7d8d8e ZBGd9d:d:ZCej2de eCddffd;d<ZDd=e eej+e-fdegdffd>d?ZEd=e eej+e-fd@edAede:ej+dBffdCdDZFdS)EN) defaultdictdeque) GeneratorIterableIteratorMutableMappingSequence)AnyCallablecastLiteral NamedTupleOptional TYPE_CHECKINGUnion) TypeAlias)WeakKeyDictionaryWeakValueDictionary)Variable)TorchDispatchMode)RemovableHandle) OpOverload) saved_tensors_hooks save_on_cpudisable_saved_tensors_hooksregister_multi_grad_hookallow_mutation_on_saved_tensorsNode GradientEdgeget_gradient_edgeincrement_versionc@seZdZejdefddZeejdeee de fdffddZ ejde fddZ eejdeefd d Zejd ejdd fd dZejdedefdefddZejdedefdefddZededefddZd S)rreturncCt)a6Return the name. Example:: >>> import torch >>> a = torch.tensor([0., 0., 0.], requires_grad=True) >>> b = a.clone() >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) >>> print(b.grad_fn.name()) CloneBackward0 NotImplementedErrorselfr'h/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/torch/autograd/graph.pyname0s z Node.name.cCr"Nr#r%r'r'r(next_functions?zNode.next_functionscCr")zReturn the metadata.r#r%r'r'r(metadataDr,z Node.metadatacCr"r*r#r%r'r'r(_input_metadataIr,zNode._input_metadatatensorNcCr"r*r#)r&r/r'r'r(_register_hook_dictNszNode._register_hook_dictfncCr")aRegister a backward hook. The hook will be called every time a gradient with respect to the Node is computed. The hook should have the following signature:: hook(grad_inputs: Tuple[Tensor], grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None The hook should not modify its argument, but it can optionally return a new gradient which will be used in place of :attr:`grad_inputs`. This function returns a handle with a method ``handle.remove()`` that removes the hook from the module. .. note:: See :ref:`backward-hooks-execution` for more information on how when this hook is executed, and how its execution is ordered relative to other hooks. .. note:: In the rare case where the hook is registered while the Node has already begun execution, there is no longer any guarantee on :attr:`grad_outputs` content (it might be as usual or empty depending on other factors). The hook can still optionally return a new gradient to be used in place of :attr:`grad_inputs` independent of :attr:`grad_outputs`. Example:: >>> import torch >>> a = torch.tensor([0., 0., 0.], requires_grad=True) >>> b = a.clone() >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) >>> handle = b.grad_fn.register_hook(lambda gI, gO: (gO[0] * 2,)) >>> b.sum().backward(retain_graph=True) >>> print(a.grad) tensor([2., 2., 2.]) >>> handle.remove() # Removes the hook >>> a.grad = None >>> b.sum().backward(retain_graph=True) >>> print(a.grad) tensor([1., 1., 1.]) r#r&r1r'r'r( register_hookRs+zNode.register_hookcCr")aRegister a backward pre-hook. The hook will be called every time a gradient with respect to the Node is computed. The hook should have the following signature:: hook(grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None The hook should not modify its argument, but it can optionally return a new gradient which will be used in place of :attr:`grad_outputs`. This function returns a handle with a method ``handle.remove()`` that removes the hook from the module. .. note:: See :ref:`backward-hooks-execution` for more information on how when this hook is executed, and how its execution is ordered relative to other hooks. Example:: >>> a = torch.tensor([0., 0., 0.], requires_grad=True) >>> b = a.clone() >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) >>> handle = b.grad_fn.register_prehook(lambda gI: (gI[0] * 2,)) >>> b.sum().backward(retain_graph=True) >>> print(a.grad) tensor([2., 2., 2.]) >>> handle.remove() >>> a.grad = None >>> b.sum().backward(retain_graph=True) >>> print(a.grad) tensor([1., 1., 1.]) r#r2r'r'r(register_prehooks"zNode.register_prehooksubclasscCs>|tur|dur|ttjj|jdust|tjjj rdSt S)NT) rgetattrtorch_C _functions__name__ issubclassautogradfunctionBackwardCFunctionNotImplemented)clsr5r'r'r(__subclasshook__szNode.__subclasshook__)r: __module__ __qualname__abcabstractmethodstrr)propertytuplerintr+dictr-listr r.r7Tensorr0r rr3r4 classmethodtypeboolrAr'r'r'r(r/s&&,#rtrr!cCsvt|tr|jS|jr0|jdur0t||jjdd}Wdn1s*wYn|j}|dus9J|SNr) isinstancernode requires_gradgrad_fnr7 enable_gradview_asr+)rPrSr'r'r(_get_grad_fn_or_grad_accs   rXc@s"eZdZUdZeed<eed<dS)rzObject representing a given gradient edge within the autograd graph. To get the gradient edge where a given Tensor gradient will be computed, you can do ``edge = autograd.graph.get_gradient_edge(tensor)``. rS output_nrN)r:rBrC__doc__r__annotations__rIr'r'r'r(rs  r/cCs"|jstdt|}t||jS)zGet the gradient edge for computing the gradient of the given Tensor. In particular, it is equivalent to call ``g = autograd.grad(loss, input)`` and ``g = autograd.grad(loss, get_gradient_edge(input))``. zXIt is not possible to get the gradient edge for a Tensor that does not require gradients)rT RuntimeErrorrXrrY)r/rUr'r'r(rs  rcCs"t|tjr |f}tj|dS)ayUpdate autograd metadata tracking whether the given Tensor was modified in place. This is to enable more accurate error checking within the autograd engine. It is already done automatically by PyTorch functions and within custom Function when mark_dirty() is called appropriately so you only need to call this explicitly if you are doing inplace operation on the Tensor data in a way that Pytorch doesn't know about. For example a custom kernel that reads the Tensor data_ptr and modifies the memory inplace based on this pointer. Can accept either a tensor, or a list of tensors. Note that incrementing the version counter multiple times for a single inplace operation is not problematic. Note that if you pass in tensor constructed under torch.inference_mode(), we will not bump its version counter (because your tensor does not have one). N)rRr7rLr8_increment_version)r/r'r'r(r s r c@sZeZdZdZdeejgefdeegejfddfddZd dd Z d e ddfd d Z dS)ra Context-manager that sets a pair of pack / unpack hooks for saved tensors. Use this context-manager to define how intermediary results of an operation should be packed before saving, and unpacked on retrieval. In that context, the ``pack_hook`` function will be called everytime an operation saves a tensor for backward (this includes intermediary results saved using :func:`~torch.autograd.function._ContextMethodMixin.save_for_backward` but also those recorded by a PyTorch-defined operation). The output of ``pack_hook`` is then stored in the computation graph instead of the original tensor. The ``unpack_hook`` is called when the saved tensor needs to be accessed, namely when executing :func:`torch.Tensor.backward()` or :func:`torch.autograd.grad()`. It takes as argument the *packed* object returned by ``pack_hook`` and should return a tensor which has the same content as the original tensor (passed as input to the corresponding ``pack_hook``). The hooks should have the following signatures: pack_hook(tensor: Tensor) -> Any unpack_hook(Any) -> Tensor where the return value of ``pack_hook`` is a valid input to ``unpack_hook``. In general, you want ``unpack_hook(pack_hook(t))`` to be equal to ``t`` in terms of value, size, dtype and device. Example:: >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) >>> def pack_hook(x): ... print("Packing", x) ... return x >>> >>> def unpack_hook(x): ... print("Unpacking", x) ... return x >>> >>> a = torch.ones(5, requires_grad=True) >>> b = torch.ones(5, requires_grad=True) * 2 >>> with torch.autograd.graph.saved_tensors_hooks(pack_hook, unpack_hook): ... y = a * b Packing tensor([1., 1., 1., 1., 1.], requires_grad=True) Packing tensor([2., 2., 2., 2., 2.], grad_fn=) >>> y.sum().backward() Unpacking tensor([1., 1., 1., 1., 1.], requires_grad=True) Unpacking tensor([2., 2., 2., 2., 2.], grad_fn=) .. warning :: Performing an inplace operation on the input to either hooks may lead to undefined behavior. .. warning :: Only one pair of hooks is allowed at a time. When recursively nesting this context-manager, only the inner-most pair of hooks will be applied. pack_hook unpack_hookr!NcCs||_||_dSr*)r^r_)r&r^r_r'r'r(__init__,s zsaved_tensors_hooks.__init__cCstjj|j|jdSr*)r7r8 _autograd!_push_saved_tensors_default_hooksr^r_r%r'r'r( __enter__4szsaved_tensors_hooks.__enter__argscGstjjdSr*)r7r8ra _pop_saved_tensors_default_hooks)r&rdr'r'r(__exit__9szsaved_tensors_hooks.__exit__r!N) r:rBrCrZr r7rLr r`rcobjectrfr'r'r'r(rs= rcs0eZdZdZd dededdffdd ZZS) razContext manager under which tensors saved by the forward pass will be stored on cpu, then retrieved for backward. When performing operations within this context manager, intermediary results saved in the graph during the forward pass will be moved to CPU, then copied back to the original device when needed for the backward pass. If the graph was already on CPU, no tensor copy is performed. Use this context-manager to trade compute for GPU memory usage (e.g. when your model doesn't fit in GPU memory during training). Args: pin_memory (bool): If ``True`` tensors will be saved to CPU pinned memory during packing and copied to GPU asynchronously during unpacking. Defaults to ``False``. Also see :ref:`cuda-memory-pinning`. Example:: >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) >>> a = torch.randn(5, requires_grad=True, device="cuda") >>> b = torch.randn(5, requires_grad=True, device="cuda") >>> c = torch.randn(5, requires_grad=True, device="cuda") >>> >>> def f(a, b, c): ... prod_1 = a * b # a and b are saved on GPU ... with torch.autograd.graph.save_on_cpu(): ... prod_2 = prod_1 * c # prod_1 and c are saved on CPU ... y = prod_2 * a # prod_2 and a are saved on GPU ... return y >>> >>> y = f(a, b, c) >>> del a, b, c # for illustration only >>> # the content of a, b, and prod_2 are still alive on GPU >>> # the content of prod_1 and c only live on CPU >>> y.sum().backward() # all CPU tensors are moved back to GPU, for backward >>> # all intermediary tensors are released (deleted) after the call to backward Fcuda pin_memory device_typer!Ncsjtt|tjdtjdttjtjfffdd }dttjtjfdtjffdd }t||dS)Nr/r!csLs |j|fStj||j|jo|j d}| ||j|fS)N)dtypelayoutrj) devicecpur7emptysizerlrm is_available is_sparsecopy_)r/packed device_modulerjr'r( pack_to_cpuis  z)save_on_cpu.__init__..pack_to_cpurucs|\}}|j|dS)N) non_blocking)to)rurnr/)rjr'r(unpack_from_cpuusz-save_on_cpu.__init__..unpack_from_cpu)r6r7rirLrHrnsuperr`)r&rjrkrxr{ __class__rvr(r`fs&$ zsave_on_cpu.__init__)Fri)r:rBrCrZrOrFr` __classcell__r'r'r}r(r=s$(r error_message)NNNc cszd}z&tjj}tjj|dVW|dur!tjjdStjj|dS|dur5tjjwtjj|w)aContext-manager that disables the saved tensors default hooks feature. Useful for if you are creating a feature that does not work with saved tensors default hooks. Args: error_message (str): When saved tensors default hooks are used when they have been are disabled, a RuntimeError with this error message gets raised. Example:: >>> # xdoctest: +SKIP(failing) >>> message = "saved tensors default hooks are disabled" >>> with torch.autograd.graph.disable_saved_tensors_hooks(message): ... # Raises RuntimeError: saved tensors default hooks are disabled ... with torch.autograd.graph.save_on_cpu(): ... pass N)r7r8ra/_saved_tensors_hooks_get_disabled_error_message_saved_tensors_hooks_disable_saved_tensors_hooks_enable)rmaybe_prev_messager'r'r(r|s rc@sreZdZUeedfed<deedfddfddZdddZdeedffd d Zd eedfddfd d Z dS) _MultiHandle.handlesr!NcC ||_dSr*r)r&rr'r'r(r` z_MultiHandle.__init__cCs|jD]}|qdSr*)rremove)r&handler'r'r(rs  z_MultiHandle.removecCs|jSr*rr%r'r'r( __getstate__sz_MultiHandle.__getstate__statecCrr*r)r&rr'r'r( __setstate__rz_MultiHandle.__setstate__rg) r:rBrCrHrr[r`rrrr'r'r'r(rs  rall)modetensorsr1rranycs d}t||vrtd|d||dkrQidittt|t|dtdtt j gdfffdd t fd d t |D}t|S|d krt tt j gdftttd t j ddffdd  t fdd |D}t|S)aMRegister a multi-grad backward hook. There are two supported modes: ``"all"`` and ``"any"``. Under the ``"all"`` mode, the hook will be called after gradients with respect to every tensor in :attr:`tensors` have been computed. If a tensor is in :attr:`tensors` but is not part of the graph, or if a tensor is not needed to compute the gradients for any ``inputs`` specified for the current ``.backward()`` or ``.grad()`` call, this tensor will be ignored and the hook will not wait for its gradient to be computed. After every non-ignored tensor's gradient has been computed, :attr:`fn` will be called with those gradients. ``None`` will be passed for tensors that did not have their gradients computed. Under the ``"any"`` mode, the hook will be called after the first gradient with respect to a tensor in :attr:`tensors` has been computed. The hook will be called with that gradient as its argument. The hook should not modify its arguments. This function returns a handle with a method ``handle.remove()`` that removes the hook. .. note:: See :ref:`backward-hooks-execution` for more information on how when this hook is executed, and how its execution is ordered relative to other hooks. Example:: >>> import torch >>> >>> a = torch.rand(2, 3, requires_grad=True) >>> b = torch.rand(2, 3, requires_grad=True) >>> c = a * b >>> d = a * b >>> >>> def fn(grads): ... print([g is not None for g in grads]) ... >>> torch.autograd.graph.register_multi_grad_hook((a, b, c, d), fn) >>> >>> c.sum().backward(retain_graph=True) [True, True, True, False] >>> c.sum().backward(inputs=(a,), retain_graph=True) [True, False, True, False] >>> rzExpects mode to be one of z but got rNidxr!c s*dtjddffdd }|S)Ngradr!cstj}|dks Jd|d|<|dg|<"||d}|<|dkr=tttjjWdn1sGwY||<dusXJ|dkr{ttt t tj gdf||=|=dSdS)N6expected this hook to be called inside a backward callr) r7r8_current_graph_task_idgetsummap_will_engine_execute_noder r rrrL)rid curr_count)buffercountr1grad_fnsr len_tensorslocknb_callsr'r( inner_hooks,       zDregister_multi_grad_hook..get_inner_hook..inner_hook)r7rL)rr)rrr1rrrr)rr(get_inner_hooks&z0register_multi_grad_hook..get_inner_hookc3s"|] \}}||VqdSr*)r3).0irP)rr'r( s z+register_multi_grad_hook..rrcsdtj}|dks Jd|d}|<Wdn1s#wY|r,dS|dS)NrrT)r7r8r)rrprev)r1rran_hookr'r( wrapped_fns  z,register_multi_grad_hook..wrapped_fnc3s |] }|jr|VqdSr*)rTr3)rr/)rr'r(r's ) threadingLock ValueErrorrKrrXlenrIr r7rLrH enumerater rrO functoolswrapsr)rr1rsupported_modesrr') rrr1rrrrrrrr(rs08.  rF(_allow_mutation_on_saved_tensors_enabled_TID_SIDcCs8t|tjjjtjjjfrd}n|}t|||j fSrQ) rRr7 _subclasses fake_tensor FakeTensorfunctional_tensorFunctionalTensordata_ptrr_versionr/rr'r'r(_get_tidDsrcCs2t|tjjjtjjjfrd}n|}||jfSrQ) rRr7rrrrrrrrr'r'r(_get_sidSs rc@s eZdZdS)_HandleN)r:rBrCr'r'r'r(rbsrcseZdZdfdd ZZS)_swap_with_clonedctx_AllowMutationOnSavedContextr!NcsBdtjdtffdd }dtdtjffdd }t||dS)Nr/r!csZt|}t|}d}j|||jvr&t}|j|<|j|<|Sj|}|Sr*)rr sid_to_tidaddtid_to_weakhandleroriginal)r/tidsidrrr'r(r^hs    z-_swap_with_cloned.__init__..pack_hookrcsHd}tsJ||jvrj|}|S|jvsJ|j|}|S)NzvTrying to backward outside of the 'allow_mutation_on_saved_tensors' contextin which the graph was originally recorded.)rclonedr)r error_msgresrr'r(r_{s    z/_swap_with_cloned.__init__..unpack_hook)r7rLrr|r`)r&rr^r_r}rr(r`gs z_swap_with_cloned.__init__rrr!N)r:rBrCr`rr'r'r}r(rfsrc @sReZdZdddZ  ddd d eed eed fd ee eefdef ddZ dS)_CloneArgBeforeMutateModerrr!NcCrr*r)r&rr'r'r(r`rz"_CloneArgBeforeMutateMode.__init__r'funcrtypesrd.kwargsc s|pi}dtjddffdd }t|jjD]0\}}|jdurF|jjrF|jr-||dqt||t r@||D]}||q8q|||q||i|S)NrPr!csvt|}t|}j}||jvr7|j|D]#}||jvrq|j|}||jvr(q|j||j|<|j|=qdSdSr*)rrrrrrrclone)rPrrrrr%r'r( maybe_clones     zA_CloneArgBeforeMutateMode.__torch_dispatch__..maybe_cloneout) r7rLr_schema arguments alias_infois_writeis_outrRrK) r&rrrdrrrargrPr'r%r(__torch_dispatch__s   z,_CloneArgBeforeMutateMode.__torch_dispatch__r)r'N) r:rBrCr`rrNrHr rrJrr'r'r'r(rs  rc@s eZdZdddZdddZdS)rr!NcCs&t|_t|_t|_tt|_dSr*)rrrrrrsetrr%r'r'r(r`sz%_AllowMutationOnSavedContext.__init__cCs,|j|j|j|jdSr*)rclearrrrr%r'r'r(rs   z"_AllowMutationOnSavedContext.clearrg)r:rBrCr`rr'r'r'r(rs rc cst}t|@t|#ztrtdda|VW|dan|dawWdn1s3wYWddSWddS1sKwYdS)aContext manager under which mutating tensors saved for backward is allowed. Under this context manager, tensors saved for backward are cloned on mutation, so the original version can still be used during backward. Normally, mutating a tensor saved for backward will result in an error raised when it's used during backward. To ensure the correct behavior, both the forward and backward should be run under the same context manager. Returns: An _AllowMutationOnSavedContext object storing the state managed by this context manager. This object can be useful for debugging purposes. The state managed by the context manager is automatically cleared upon exiting. Example:: >>> import torch >>> with torch.autograd.graph.allow_mutation_on_saved_tensors(): ... # forward ... a = torch.ones(2, 3, requires_grad=True) ... b = a.clone() ... out = (b**2).sum() ... b.sin_() ... # backward ... out.sum().backward() ... tensor([[0.8415, 0.8415, 0.8415], [0.8415, 0.8415, 0.8415]], grad_fn=) z9allow_mutation_on_saved_tensors contexts cannot be nestedTFN)rrrrr\rrr'r'r(rs#Pr t_outputscsttt|}dttdttfdd}dttjdtfdddt ttjddffd d fd d ||Ddfd d }|S)Nrootsr!css|sdSt}t}|D]}|dur||||q |rF|}|jD]\}}||vs3|dur4q'||||q'|V|s dSdSr*)rrrappendpopleftr+)rseenqrSr1_r'r'r( iter_graphs&     z:_register_logging_hooks_on_whole_graph..iter_graphrPcSs<ddlm}|dur dS||jddtt|jdS)Nr) dtype_abbrsNone[z, ])$torch.testing._internal.common_utilsrrljoinrrFshape)rPrr'r'r(fmts $z3_register_logging_hooks_on_whole_graph..fmt grad_outputscsHtj}ddfdd|Dd}d|d|}t|dS)Nr,c3s|]}|VqdSr*r')rrPrr'r(r"szJ_register_logging_hooks_on_whole_graph..prehook..rz Executing: z with grad_outputs: )r7r8_current_autograd_noderlogdebug)rrSgrad_outputs_strlog_strrr'r(prehook s  z7_register_logging_hooks_on_whole_graph..prehookcsg|]}|qSr')r4)rrS)rr'r( &sz:_register_logging_hooks_on_whole_graph..csD]}|qdSr*)r)rrr'r(unregister_hooks(s z@_register_logging_hooks_on_whole_graph..unregister_hooksrg) rKrrXrrrr7rLrFr)rrrrr')rrrr(&_register_logging_hooks_on_whole_graphs rrdr.cOsRttjk}|r t|}ztjj|g|Ri|W|r!|SS|r(|wwr*)rgetEffectiveLevelloggingDEBUGrr_execution_engine run_backward)rrdrattach_logging_hooksrr'r'r(_engine_run_backward/s r)GrD contextlibrrr collectionsrrcollections.abcrrrrrtypingr r r r r rrrtyping_extensionsrweakrefrrr7torch.autograd.variablertorch.utils._python_dispatchrtorch.utils.hooksr torch._opsr__all__ getLoggerr:rABCrrLrXrrr rrcontextmanagerrFrrrrrOr[rHrIrrrrrrrrrrrr'r'r'r(s (        "O?#  %4  1  0