o h" @sbddlZddlZddlZddlZddlmZmZddlmZddl Z ddl m Z m Z ddl m Z ddlmZddlmZdd lmZgd Ze je jd Zd&d e d e dede fddZededd&d e d e dede fddZGdddeZGdddeZGdddeZGdddeZGdddeZ Gd d!d!e Z!Gd"d#d#e Z"Gd$d%d%e Z#dS)'N)Optionaloverload) deprecated)_VFTensor)init) Parameter)PackedSequence)Module)RNNBaseRNNLSTMGRU RNNCellBaseRNNCellLSTMCellGRUCell)RNN_TANHRNN_RELUtensor permutationdimreturncCs |||SN) index_selectrrrrh/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/torch/nn/modules/rnn.py_apply_permutation$s rz]`apply_permutation` is deprecated, please use `tensor.index_select(dim, permutation)` instead)categorycCs t|||Srrrrrrapply_permutation(s r"cseZdZUdZgdZdgZeed<eed<eed<eed<e ed<e ed <e ed <e ed <eed <       d@dedededede d e d e d e d eddffdd Z ddZ fddZ dAddZdBfdd ZdAddZd ed!eeddfd"d#Zd ed!eedeeeeffd$d%Z &dCd'ed(eeeefd)eddfd*d+Zd,d-Zd ed.ed!eefd/d0Zd'ed1eefd2d3Zdefd4d5Zd6d7Zd8d9Zfd:d;Zedeee fdd?Z"Z#S)Dr aBase class for RNN modules (RNN, LSTM, GRU). Implements aspects of RNNs shared by the RNN, LSTM, and GRU classes, such as module initialization and utility methods for parameter storage management. .. note:: The forward method is not implemented by the RNNBase class. .. note:: LSTM and GRU classes override some methods implemented by RNNBase. ) mode input_size hidden_size num_layersbias batch_firstdropout bidirectional proj_size all_weightsr#r$r%r&r'r(r)r*r+r TFrNrc s$| | d} t||_||_||_||_||_||_t||_ ||_ | |_ g|_ |r.dnd} t |tjrJd|kr@dkrJntdt |trNtd|dkra|dkratd|d|t |tsptdt|j|dkrxtd |dkrtd | dkrtd | |krtd |d krd|}n|dkrd|}n|dkr|}n |dkr|}ntd|g|_g|_t|D]ljt| D]}| dkr| n|}dkr|n|| }ttj||ffi| }ttj||ffi| }ttj|fi| }ttj|fi| }d}|j dkr!|r||||f}n"||f}nttj| |ffi| }|r9|||||f}n|||f}|dkrEdndddg}|rT|ddg7}|j dkr_|dg7}fdd|D}t||D] \}}t|||qn|j ||j!|qq|"|#dS)Ndevicedtyper rzbdropout should be a number in range [0, 1] representing the probability of an element being zeroedzdropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=z and num_layers=z(hidden_size should be of type int, got: z%hidden_size must be greater than zeroz$num_layers must be greater than zerozEproj_size should be a positive integer or zero to disable projectionsz,proj_size has to be smaller than hidden_sizerrrrzUnrecognized RNN mode: r_reverseweight_ih_l{}{}weight_hh_l{}{} bias_ih_l{}{} bias_hh_l{}{}weight_hr_l{}{}cg|]}|qSrformat.0xlayersuffixrr z$RNNBase.__init__..)$super__init__r#r$r%r&r'r(floatr)r*r+_flat_weight_refs isinstancenumbersNumberbool ValueErrorwarningswarnint TypeErrortype__name___flat_weights_names _all_weightsrangertorchemptyzipsetattrextendappend_init_flat_weightsreset_parameters)selfr#r$r%r&r'r(r)r*r+r/r0factory_kwargsnum_directions gate_size directionreal_hidden_sizelayer_input_sizew_ihw_hhb_ihb_hh layer_paramsw_hr param_namesnameparam __class__rArrGTs                  , zRNNBase.__init__cs4fddjD_ddjD_dS)Nc$g|]}t|rt|ndqSrhasattrgetattrr?wnr`rrrDz.RNNBase._init_flat_weights..cS"g|] }|dur t|ndqSrweakrefrefr?wrrrrD)rU _flat_weightsrIflatten_parametersrxrrxrr^s  zRNNBase._init_flat_weightscs<t|dr||jvr|j|}||j|<t||dS)NrU)rtrUindexrrF __setattr__)r`attrvalueidxrprrrs  zRNNBase.__setattr__cCst|jt|jkr dS|jD] }t|tsdSq|jd}|j}|jD]}t|tr;|j|kr;|jr;tjj |s>dSq%dd|jD}t|t|jkrRdStj |_ddl mm m}t6tr|jrsdnd}|jdkr~|d7}t|j||j||j|j|j|j|jt|j Wdn1swYWddSWddS1swYdS)zReset parameter data pointer so that they can use faster code paths. Right now, this works only if the module is on the GPU and cuDNN is enabled. Otherwise, it's a no-op. NrcSsh|]}|qSr)data_ptr)r?prrr sz-RNNBase.flatten_parameters..r2r1r )lenrrUrJrr0is_cudarXbackendscudnn is_acceptablecuda device_oftorch.backends.cudnn.rnnrnnno_grad_use_cudnn_rnn_flatten_weightr'r+_cudnn_rnn_flatten_weightr$get_cudnn_moder#r%r&r(rMr*)r`rfirst_fwr0fwunique_data_ptrsr num_weightsrrrrs\          "zRNNBase.flatten_parameterscs g|_t||}||Sr)rIrF_applyr^)r`fnrecurseretrprrrszRNNBase._applycC@|jdkr dt|jnd}|D] }t|| |qdSNrg?r%mathsqrt parametersruniform_r`stdvweightrrrr_& zRNNBase.reset_parametersinput batch_sizescCstjs"|j|jdjkr"tjs"td|jdjd|j|dur(dnd}||krs  z RNNBase.get_expected_hidden_sizeExpected hidden size {}, got {}hxrmsgcCs(||krt||t|dSr)rrr=list)r`rrrrrrcheck_hidden_sizeTs zRNNBase.check_hidden_sizecCs\d}t|j|jD]"\}}t||rt||nd}|dur+|dur+||ur+d}|Sq |S)NFT)rZrIrUrtru)r`weights_changedr}rnrrrr_weights_have_changed]szRNNBase._weights_have_changedhiddencCs(||||||}|||dSr)rrr)r`rrrrrrrcheck_forward_argshs  zRNNBase.check_forward_argsrcCs|dur|St||Srr!r`rrrrrpermute_hiddenps zRNNBase.permute_hiddencCsd}|jdkr |d7}|jdkr|d7}|jdur|d7}|jdur&|d 7}|jdkr/|d 7}|jdur8|d 7}|jd i|jS) N{input_size}, {hidden_size}rz, proj_size={proj_size}r z, num_layers={num_layers}T , bias={bias}Fz, batch_first={batch_first}z, dropout={dropout}z, bidirectional={bidirectional}r)r+r&r'r(r)r*r=__dict__r`srrr extra_reprus      zRNNBase.extra_reprcCs&tjs|r|dSdSdSr)rXrrrr^rxrrr_update_flat_weightss  zRNNBase._update_flat_weightscCs||j}|d=|S)NrI)rrcopy)r`staterrr __getstate__s zRNNBase.__getstate__cst|d|vr|d_d|vrd_tjddts͈j}jr(dnd}g_g_t |D]t |D]}|dkrBdndgd}fd d |D}j rjdkrij|g7_j |q:j|dd g7_j |dd q:jdkrj|ddg|d dg7_j |dd|d dgq:j|ddg7_j |ddq:q4fd d jD_ dd j D_ dS)Nr,r+rr1r r4r5)r6r7r8r9r:cr;rr<r>rArrrDrEz(RNNBase.__setstate__..r2rcrrrrsrvrxrrrDrycSrzrr{r~rrrrDr)rF __setstate__rVr+rJstrr&r*rUrWr'r\rrI)r`dr&rbrdweightsrp)rBr`rCrrsF      &  zRNNBase.__setstate__csfddjDS)Ncsg|] }fdd|DqS)csg|]}t|qSr)ru)r?rrxrrrDsz2RNNBase.all_weights...r)r?rrxrrrDsz'RNNBase.all_weights..)rVrxrrxrr,s zRNNBase.all_weightscs.t}|jdd|_|jdd|_|Sr)rF_replicate_for_data_parallelrrU)r`replicarprrrs z$RNNBase._replicate_for_data_parallelr TFr-FrNNrN)T)r)$rT __module__ __qualname____doc__ __constants____jit_unused_properties__r__annotations__rQrMrHrGr^rrrr_rrrtuplerrrrrrrrrpropertyrrr,r __classcell__rrrprr 0s      { ;         3r cseZdZdZe        dded ed ed ed ed edededdfddZ eddZ fddZ ee j j dde dee dee e ffddZee j j ddedee deee ffddZdddZZS)r a__init__(input_size,hidden_size,num_layers=1,nonlinearity='tanh',bias=True,batch_first=False,dropout=0.0,bidirectional=False,device=None,dtype=None) Apply a multi-layer Elman RNN with :math:`\tanh` or :math:`\text{ReLU}` non-linearity to an input sequence. For each element in the input sequence, each layer computes the following function: .. math:: h_t = \tanh(x_t W_{ih}^T + b_{ih} + h_{t-1}W_{hh}^T + b_{hh}) where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the input at time `t`, and :math:`h_{(t-1)}` is the hidden state of the previous layer at time `t-1` or the initial hidden state at time `0`. If :attr:`nonlinearity` is ``'relu'``, then :math:`\text{ReLU}` is used instead of :math:`\tanh`. .. code-block:: python # Efficient implementation equivalent to the following with bidirectional=False def forward(x, hx=None): if batch_first: x = x.transpose(0, 1) seq_len, batch_size, _ = x.size() if hx is None: hx = torch.zeros(num_layers, batch_size, hidden_size) h_t_minus_1 = hx h_t = hx output = [] for t in range(seq_len): for layer in range(num_layers): h_t[layer] = torch.tanh( x[t] @ weight_ih[layer].T + bias_ih[layer] + h_t_minus_1[layer] @ weight_hh[layer].T + bias_hh[layer] ) output.append(h_t[-1]) h_t_minus_1 = h_t output = torch.stack(output) if batch_first: output = output.transpose(0, 1) return output, h_t Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would mean stacking two RNNs together to form a `stacked RNN`, with the second RNN taking in outputs of the first RNN and computing the final results. Default: 1 nonlinearity: The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. Default: ``'tanh'`` bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` batch_first: If ``True``, then the input and output tensors are provided as `(batch, seq, feature)` instead of `(seq, batch, feature)`. Note that this does not apply to hidden or cell states. See the Inputs/Outputs sections below for details. Default: ``False`` dropout: If non-zero, introduces a `Dropout` layer on the outputs of each RNN layer except the last layer, with dropout probability equal to :attr:`dropout`. Default: 0 bidirectional: If ``True``, becomes a bidirectional RNN. Default: ``False`` Inputs: input, hx * **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. The input can also be a packed variable length sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or :func:`torch.nn.utils.rnn.pack_sequence` for details. * **hx**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{out})` containing the initial hidden state for the input sequence batch. Defaults to zeros if not provided. where: .. math:: \begin{aligned} N ={} & \text{batch size} \\ L ={} & \text{sequence length} \\ D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ H_{in} ={} & \text{input\_size} \\ H_{out} ={} & \text{hidden\_size} \end{aligned} Outputs: output, h_n * **output**: tensor of shape :math:`(L, D * H_{out})` for unbatched input, :math:`(L, N, D * H_{out})` when ``batch_first=False`` or :math:`(N, L, D * H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the RNN, for each `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, the output will also be a packed sequence. * **h_n**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{out})` containing the final hidden state for each element in the batch. Attributes: weight_ih_l[k]: the learnable input-hidden weights of the k-th layer, of shape `(hidden_size, input_size)` for `k = 0`. Otherwise, the shape is `(hidden_size, num_directions * hidden_size)` weight_hh_l[k]: the learnable hidden-hidden weights of the k-th layer, of shape `(hidden_size, hidden_size)` bias_ih_l[k]: the learnable input-hidden bias of the k-th layer, of shape `(hidden_size)` bias_hh_l[k]: the learnable hidden-hidden bias of the k-th layer, of shape `(hidden_size)` .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` .. note:: For bidirectional RNNs, forward and backward are directions 0 and 1 respectively. Example of splitting the output layers when ``batch_first=False``: ``output.view(seq_len, batch, num_directions, hidden_size)``. .. note:: ``batch_first`` argument is ignored for unbatched inputs. .. include:: ../cudnn_rnn_determinism.rst .. include:: ../cudnn_persistent_rnn.rst Examples:: >>> rnn = nn.RNN(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> output, hn = rnn(input, h0) r tanhTFr-Nr$r%r& nonlinearityr'r(r)r*rc CdSrr) r`r$r%r&rr'r(r)r*r/r0rrrrGVz RNN.__init__cOrrrr`argskwargsrrrrGfcsd|vrtdt|dkr |d|_|dd|dd}n|dd|_|jdkr/d}n|jdkr7d }n td |jd tj|g|Ri|dS) Nr+=proj_size argument is only supported for LSTM, not RNN or GRUr3r2rrrrelurzUnknown nonlinearity 'z '. Select from 'tanh' or 'relu'.)rNrrpoprFrG)r`rrr#rprrrGjs      rrcCrrrr`rrrrrforward~z RNN.forwardcCrrrrrrrrrc Cs||jr dnd}|}t|tr8|\}}}}|d}|dur1tj|j|||j|j|j d}n| ||}nd}| dvrJt d| d| dk} |j rUdnd} | sx|| }|durw| dkrrtd | d |d}n|dur| dkrtd | d |j r|dn|d}d}d}|durtj|j|||j|j|j d}n| ||}|dusJ|||||jd ks|jd ksJ|dur|jd krt|||j|j|j|j|j|j|j } nBt|||j|j|j|j|j|j|j } n-|jd krt||||j|j|j|j|j|j } nt||||j|j|j|j|j|j } | d} | d} t|trOt| |||}|| | |fS| s\| | } | d} | | | |fS)Nr1r rr0r/r1r3z(RNN: Expected input to be 2D or 3D, got zD tensor insteadr37For unbatched 2-D input, hx should also be 2-D but got -D tensor5For batched 3-D input, hx should also be 3-D but got rr)rr*rJr rXzerosr&r%r0r/rrrNr( unsqueezerrrr#rrnn_tanhrr'r)trainingrnn_relusqueeze)r`rrrb orig_inputrsorted_indicesunsorted_indicesmax_batch_size is_batched batch_dimresultoutputr output_packedrrrrs                  )r rTFr-FNNr)rTrrrrrQrrMrHrGrX _jit_internal_overload_methodrrrrr rrrrprr sn      r csneZdZdZe        d$ded ed ed ed ed edededdfddZeddZfddZde de e de eeeffddZ de de e e fde e fddZ de e e fde e de e e ffddZeejj d%de de e e e fde e e e e fffd d!Zeejj d%dede e e e fde ee e e fffd"d!Zd%d#d!ZZS)&ra/%__init__(input_size,hidden_size,num_layers=1,bias=True,batch_first=False,dropout=0.0,bidirectional=False,proj_size=0,device=None,dtype=None) Apply a multi-layer long short-term memory (LSTM) RNN to an input sequence. For each element in the input sequence, each layer computes the following function: .. math:: \begin{array}{ll} \\ i_t = \sigma(W_{ii} x_t + b_{ii} + W_{hi} h_{t-1} + b_{hi}) \\ f_t = \sigma(W_{if} x_t + b_{if} + W_{hf} h_{t-1} + b_{hf}) \\ g_t = \tanh(W_{ig} x_t + b_{ig} + W_{hg} h_{t-1} + b_{hg}) \\ o_t = \sigma(W_{io} x_t + b_{io} + W_{ho} h_{t-1} + b_{ho}) \\ c_t = f_t \odot c_{t-1} + i_t \odot g_t \\ h_t = o_t \odot \tanh(c_t) \\ \end{array} where :math:`h_t` is the hidden state at time `t`, :math:`c_t` is the cell state at time `t`, :math:`x_t` is the input at time `t`, :math:`h_{t-1}` is the hidden state of the layer at time `t-1` or the initial hidden state at time `0`, and :math:`i_t`, :math:`f_t`, :math:`g_t`, :math:`o_t` are the input, forget, cell, and output gates, respectively. :math:`\sigma` is the sigmoid function, and :math:`\odot` is the Hadamard product. In a multilayer LSTM, the input :math:`x^{(l)}_t` of the :math:`l` -th layer (:math:`l \ge 2`) is the hidden state :math:`h^{(l-1)}_t` of the previous layer multiplied by dropout :math:`\delta^{(l-1)}_t` where each :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is :math:`0` with probability :attr:`dropout`. If ``proj_size > 0`` is specified, LSTM with projections will be used. This changes the LSTM cell in the following way. First, the dimension of :math:`h_t` will be changed from ``hidden_size`` to ``proj_size`` (dimensions of :math:`W_{hi}` will be changed accordingly). Second, the output hidden state of each layer will be multiplied by a learnable projection matrix: :math:`h_t = W_{hr}h_t`. Note that as a consequence of this, the output of LSTM network will be of different shape as well. See Inputs/Outputs sections below for exact dimensions of all variables. You can find more details in https://arxiv.org/abs/1402.1128. Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would mean stacking two LSTMs together to form a `stacked LSTM`, with the second LSTM taking in outputs of the first LSTM and computing the final results. Default: 1 bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` batch_first: If ``True``, then the input and output tensors are provided as `(batch, seq, feature)` instead of `(seq, batch, feature)`. Note that this does not apply to hidden or cell states. See the Inputs/Outputs sections below for details. Default: ``False`` dropout: If non-zero, introduces a `Dropout` layer on the outputs of each LSTM layer except the last layer, with dropout probability equal to :attr:`dropout`. Default: 0 bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` proj_size: If ``> 0``, will use LSTM with projections of corresponding size. Default: 0 Inputs: input, (h_0, c_0) * **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. The input can also be a packed variable length sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or :func:`torch.nn.utils.rnn.pack_sequence` for details. * **h_0**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{out})` containing the initial hidden state for each element in the input sequence. Defaults to zeros if (h_0, c_0) is not provided. * **c_0**: tensor of shape :math:`(D * \text{num\_layers}, H_{cell})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{cell})` containing the initial cell state for each element in the input sequence. Defaults to zeros if (h_0, c_0) is not provided. where: .. math:: \begin{aligned} N ={} & \text{batch size} \\ L ={} & \text{sequence length} \\ D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ H_{in} ={} & \text{input\_size} \\ H_{cell} ={} & \text{hidden\_size} \\ H_{out} ={} & \text{proj\_size if } \text{proj\_size}>0 \text{ otherwise hidden\_size} \\ \end{aligned} Outputs: output, (h_n, c_n) * **output**: tensor of shape :math:`(L, D * H_{out})` for unbatched input, :math:`(L, N, D * H_{out})` when ``batch_first=False`` or :math:`(N, L, D * H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the LSTM, for each `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, the output will also be a packed sequence. When ``bidirectional=True``, `output` will contain a concatenation of the forward and reverse hidden states at each time step in the sequence. * **h_n**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{out})` containing the final hidden state for each element in the sequence. When ``bidirectional=True``, `h_n` will contain a concatenation of the final forward and reverse hidden states, respectively. * **c_n**: tensor of shape :math:`(D * \text{num\_layers}, H_{cell})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{cell})` containing the final cell state for each element in the sequence. When ``bidirectional=True``, `c_n` will contain a concatenation of the final forward and reverse cell states, respectively. Attributes: weight_ih_l[k] : the learnable input-hidden weights of the :math:`\text{k}^{th}` layer `(W_ii|W_if|W_ig|W_io)`, of shape `(4*hidden_size, input_size)` for `k = 0`. Otherwise, the shape is `(4*hidden_size, num_directions * hidden_size)`. If ``proj_size > 0`` was specified, the shape will be `(4*hidden_size, num_directions * proj_size)` for `k > 0` weight_hh_l[k] : the learnable hidden-hidden weights of the :math:`\text{k}^{th}` layer `(W_hi|W_hf|W_hg|W_ho)`, of shape `(4*hidden_size, hidden_size)`. If ``proj_size > 0`` was specified, the shape will be `(4*hidden_size, proj_size)`. bias_ih_l[k] : the learnable input-hidden bias of the :math:`\text{k}^{th}` layer `(b_ii|b_if|b_ig|b_io)`, of shape `(4*hidden_size)` bias_hh_l[k] : the learnable hidden-hidden bias of the :math:`\text{k}^{th}` layer `(b_hi|b_hf|b_hg|b_ho)`, of shape `(4*hidden_size)` weight_hr_l[k] : the learnable projection weights of the :math:`\text{k}^{th}` layer of shape `(proj_size, hidden_size)`. Only present when ``proj_size > 0`` was specified. weight_ih_l[k]_reverse: Analogous to `weight_ih_l[k]` for the reverse direction. Only present when ``bidirectional=True``. weight_hh_l[k]_reverse: Analogous to `weight_hh_l[k]` for the reverse direction. Only present when ``bidirectional=True``. bias_ih_l[k]_reverse: Analogous to `bias_ih_l[k]` for the reverse direction. Only present when ``bidirectional=True``. bias_hh_l[k]_reverse: Analogous to `bias_hh_l[k]` for the reverse direction. Only present when ``bidirectional=True``. weight_hr_l[k]_reverse: Analogous to `weight_hr_l[k]` for the reverse direction. Only present when ``bidirectional=True`` and ``proj_size > 0`` was specified. .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` .. note:: For bidirectional LSTMs, forward and backward are directions 0 and 1 respectively. Example of splitting the output layers when ``batch_first=False``: ``output.view(seq_len, batch, num_directions, hidden_size)``. .. note:: For bidirectional LSTMs, `h_n` is not equivalent to the last element of `output`; the former contains the final forward and reverse hidden states, while the latter contains the final forward hidden state and the initial reverse hidden state. .. note:: ``batch_first`` argument is ignored for unbatched inputs. .. note:: ``proj_size`` should be smaller than ``hidden_size``. .. include:: ../cudnn_rnn_determinism.rst .. include:: ../cudnn_persistent_rnn.rst Examples:: >>> rnn = nn.LSTM(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> c0 = torch.randn(2, 3, 20) >>> output, (hn, cn) = rnn(input, (h0, c0)) r TFr-rNr$r%r&r'r(r)r*r+rc Crrr) r`r$r%r&r'r(r)r*r+r/r0rrrrGrz LSTM.__init__cOrrrrrrrrGrcstjdg|Ri|dS)NrrFrGrrprrrGsrrcCsT|dur t|d}n |jr|dn|d}|jrdnd}|j|||jf}|Sr)rQr(rr*r&r%rrrrget_expected_cell_sizeszLSTM.get_expected_cell_sizercCsD|||||d|||d||d|||ddS)Nrz"Expected hidden[0] size {}, got {}r z"Expected hidden[1] size {}, got {})rrrr)r`rrrrrrrs   zLSTM.check_forward_argsrrcCs(|dur|St|d|t|d|fS)Nrr r!rrrrrs zLSTM.permute_hiddencCrrrrrrrrrz LSTM.forwardcCrrrrrrrr rc Cs||}d}|jr dnd}|jdkr|jn|j}t|trX|\}}}}|d} |durQtj|j|| ||j |j d} tj|j|| |j|j |j d} | | f}n| ||}n| dvrht d| d| dk} |jrsdnd} | s||| }|jr|dn|d} d}d}|durtj|j|| ||j |j d} tj|j|| |j|j |j d} | | f}||||ni| r|d dks|d dkrd |d d |d d }t|n4|d dks|d dkr d |d d |d d }t||dd|ddf}||||| ||}|dur?t|||j|j|j|j|j|j|j }nt||||j|j|j|j|j|j }|d}|dd}t|trqt||||}|| ||fS| s|| }|dd|ddf}|| ||fS) Nr1r rrrz)LSTM: Expected input to be 2D or 3D, got D insteadr3z=For batched 3-D input, hx and cx should also be 3-D but got (z-D, z -D) tensorsz?For unbatched 2-D input, hx and cx should also be 2-D but got ()rr*r+r%rJr rXrr&r0r/rrrNr(rrrrrlstmrr'r)rr)r`rrrrrbrerrrh_zerosc_zerosrrrrrrrrrrrs         "        rr)rTrrrrrQrMrHrGrrrrrrrXrrrr rrrrprrs!            rcseZdZdZe       ddeded ed ed ed ed eddfddZeddZfddZee j j dde de e dee e ffddZee j j ddede e deee ffddZdddZZS)ra__init__(input_size,hidden_size,num_layers=1,bias=True,batch_first=False,dropout=0.0,bidirectional=False,device=None,dtype=None) Apply a multi-layer gated recurrent unit (GRU) RNN to an input sequence. For each element in the input sequence, each layer computes the following function: .. math:: \begin{array}{ll} r_t = \sigma(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\ z_t = \sigma(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\ n_t = \tanh(W_{in} x_t + b_{in} + r_t \odot (W_{hn} h_{(t-1)}+ b_{hn})) \\ h_t = (1 - z_t) \odot n_t + z_t \odot h_{(t-1)} \end{array} where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the input at time `t`, :math:`h_{(t-1)}` is the hidden state of the layer at time `t-1` or the initial hidden state at time `0`, and :math:`r_t`, :math:`z_t`, :math:`n_t` are the reset, update, and new gates, respectively. :math:`\sigma` is the sigmoid function, and :math:`\odot` is the Hadamard product. In a multilayer GRU, the input :math:`x^{(l)}_t` of the :math:`l` -th layer (:math:`l \ge 2`) is the hidden state :math:`h^{(l-1)}_t` of the previous layer multiplied by dropout :math:`\delta^{(l-1)}_t` where each :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is :math:`0` with probability :attr:`dropout`. Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would mean stacking two GRUs together to form a `stacked GRU`, with the second GRU taking in outputs of the first GRU and computing the final results. Default: 1 bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` batch_first: If ``True``, then the input and output tensors are provided as `(batch, seq, feature)` instead of `(seq, batch, feature)`. Note that this does not apply to hidden or cell states. See the Inputs/Outputs sections below for details. Default: ``False`` dropout: If non-zero, introduces a `Dropout` layer on the outputs of each GRU layer except the last layer, with dropout probability equal to :attr:`dropout`. Default: 0 bidirectional: If ``True``, becomes a bidirectional GRU. Default: ``False`` Inputs: input, h_0 * **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. The input can also be a packed variable length sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or :func:`torch.nn.utils.rnn.pack_sequence` for details. * **h_0**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` or :math:`(D * \text{num\_layers}, N, H_{out})` containing the initial hidden state for the input sequence. Defaults to zeros if not provided. where: .. math:: \begin{aligned} N ={} & \text{batch size} \\ L ={} & \text{sequence length} \\ D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ H_{in} ={} & \text{input\_size} \\ H_{out} ={} & \text{hidden\_size} \end{aligned} Outputs: output, h_n * **output**: tensor of shape :math:`(L, D * H_{out})` for unbatched input, :math:`(L, N, D * H_{out})` when ``batch_first=False`` or :math:`(N, L, D * H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the GRU, for each `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, the output will also be a packed sequence. * **h_n**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` or :math:`(D * \text{num\_layers}, N, H_{out})` containing the final hidden state for the input sequence. Attributes: weight_ih_l[k] : the learnable input-hidden weights of the :math:`\text{k}^{th}` layer (W_ir|W_iz|W_in), of shape `(3*hidden_size, input_size)` for `k = 0`. Otherwise, the shape is `(3*hidden_size, num_directions * hidden_size)` weight_hh_l[k] : the learnable hidden-hidden weights of the :math:`\text{k}^{th}` layer (W_hr|W_hz|W_hn), of shape `(3*hidden_size, hidden_size)` bias_ih_l[k] : the learnable input-hidden bias of the :math:`\text{k}^{th}` layer (b_ir|b_iz|b_in), of shape `(3*hidden_size)` bias_hh_l[k] : the learnable hidden-hidden bias of the :math:`\text{k}^{th}` layer (b_hr|b_hz|b_hn), of shape `(3*hidden_size)` .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` .. note:: For bidirectional GRUs, forward and backward are directions 0 and 1 respectively. Example of splitting the output layers when ``batch_first=False``: ``output.view(seq_len, batch, num_directions, hidden_size)``. .. note:: ``batch_first`` argument is ignored for unbatched inputs. .. note:: The calculation of new gate :math:`n_t` subtly differs from the original paper and other frameworks. In the original implementation, the Hadamard product :math:`(\odot)` between :math:`r_t` and the previous hidden state :math:`h_{(t-1)}` is done before the multiplication with the weight matrix `W` and addition of bias: .. math:: \begin{aligned} n_t = \tanh(W_{in} x_t + b_{in} + W_{hn} ( r_t \odot h_{(t-1)} ) + b_{hn}) \end{aligned} This is in contrast to PyTorch implementation, which is done after :math:`W_{hn} h_{(t-1)}` .. math:: \begin{aligned} n_t = \tanh(W_{in} x_t + b_{in} + r_t \odot (W_{hn} h_{(t-1)}+ b_{hn})) \end{aligned} This implementation differs on purpose for efficiency. .. include:: ../cudnn_persistent_rnn.rst Examples:: >>> rnn = nn.GRU(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> output, hn = rnn(input, h0) r TFr-Nr$r%r&r'r(r)r*rc Crrr) r`r$r%r&r'r(r)r*r/r0rrrrG s z GRU.__init__cOrrrrrrrrGrcs.d|vrtdtjdg|Ri|dS)Nr+rr)rNrFrGrrprrrGs rrcCrrrrrrrr&rz GRU.forwardcCrrrrrrrr-rc CsH||}t|tr8|\}}}}|d}|dur1|jrdnd}tj|j|||j|j|j d}n| ||}nd}| dvrJt d| d| dk} |j rUdnd} | sx|| }|durw| dkrrtd | d |d}n|dur| dkrtd | d |j r|dn|d}d}d}|dur|jrdnd}tj|j|||j|j|j d}n| ||}|||||durt|||j|j|j|j|j|j|j } nt||||j|j|j|j|j|j } | d} | d} t|trt| |||}|| | |fS| s| | } | d} | | | |fS) Nrr1r rrz(GRU: Expected input to be 2D or 3D, got rr3rrr)rrJr r*rXrr&r%r0r/rrrNr(rrrrrgrurr'r)rr)r`rrrrrrrrbrrrrrrrrrr4s              )r TFr-FNNr)rTrrrrrQrMrHrGrXrrrrrrr rrrrprrsh     rc seZdZUgdZeed<eed<eed<eed<eed<  ddedededed df fd d Zd e fd d Z dddZ Z S)r)r$r%r'r$r%r' weight_ih weight_hhN num_chunksrcs||d}t||_||_||_ttj|||ffi||_ttj|||ffi||_ |rRttj||fi||_ ttj||fi||_ n | dd| dd| dS)Nr.bias_ihbias_hh)rFrGr$r%r'rrXrYr r rrregister_parameterr_)r`r$r%r'rr/r0rarprrrGs*    zRNNCellBase.__init__cCsNd}d|jvr|jdur|d7}d|jvr|jdkr|d7}|jdi|jS) Nrr'Trrrz, nonlinearity={nonlinearity}r)rr'rr=rrrrrs zRNNCellBase.extra_reprcCrrrrrrrr_rzRNNCellBase.reset_parameters)NNr) rTrrrrQrrMrrGrrr_rrrrprrs,  !rc sneZdZUdZgdZeed<    ddeded eded df fd d Z dd e de e d e fddZ Z S)rarAn Elman RNN cell with tanh or ReLU non-linearity. .. math:: h' = \tanh(W_{ih} x + b_{ih} + W_{hh} h + b_{hh}) If :attr:`nonlinearity` is `'relu'`, then ReLU is used in place of tanh. Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` nonlinearity: The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. Default: ``'tanh'`` Inputs: input, hidden - **input**: tensor containing input features - **hidden**: tensor containing the initial hidden state Defaults to zero if not provided. Outputs: h' - **h'** of shape `(batch, hidden_size)`: tensor containing the next hidden state for each element in the batch Shape: - input: :math:`(N, H_{in})` or :math:`(H_{in})` tensor containing input features where :math:`H_{in}` = `input_size`. - hidden: :math:`(N, H_{out})` or :math:`(H_{out})` tensor containing the initial hidden state where :math:`H_{out}` = `hidden_size`. Defaults to zero if not provided. - output: :math:`(N, H_{out})` or :math:`(H_{out})` tensor containing the next hidden state. Attributes: weight_ih: the learnable input-hidden weights, of shape `(hidden_size, input_size)` weight_hh: the learnable hidden-hidden weights, of shape `(hidden_size, hidden_size)` bias_ih: the learnable input-hidden bias, of shape `(hidden_size)` bias_hh: the learnable hidden-hidden bias, of shape `(hidden_size)` .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: >>> rnn = nn.RNNCell(10, 20) >>> input = torch.randn(6, 3, 10) >>> hx = torch.randn(3, 20) >>> output = [] >>> for i in range(6): ... hx = rnn(input[i], hx) ... output.append(hx) )r$r%r'rrTrNr$r%r'rcs0||d}tj|||fddi|||_dS)Nr.rr )rFrGr)r`r$r%r'rr/r0rarprrrG s  zRNNCell.__init__rrcCs|dvrtd|d|dur$|dvr$td|d|dk}|s1|d}|durEtj|d|j|j|jd}n |sL|dn|}|j dkrbt |||j |j |j|j}n|j d krvt |||j |j |j|j}n |}td |j |s|d}|S) Nr r1z,RNNCell: Expected input to be 1D or 2D, got rz-RNNCell: Expected hidden to be 1D or 2D, got r1rrrrzUnknown nonlinearity: )rrNrrXrrr%r0r/rr rnn_tanh_cellr r rr rnn_relu_cellrrr`rrrrrrrrsN       zRNNCell.forward)TrNNr)rTrrrrrrrQrMrGrrrrrrrprrs( 6$ rc sheZdZdZ   ddedededdffdd Z dd ed ee eefde eeffd d Z Z S)ra2 A long short-term memory (LSTM) cell. .. math:: \begin{array}{ll} i = \sigma(W_{ii} x + b_{ii} + W_{hi} h + b_{hi}) \\ f = \sigma(W_{if} x + b_{if} + W_{hf} h + b_{hf}) \\ g = \tanh(W_{ig} x + b_{ig} + W_{hg} h + b_{hg}) \\ o = \sigma(W_{io} x + b_{io} + W_{ho} h + b_{ho}) \\ c' = f \odot c + i \odot g \\ h' = o \odot \tanh(c') \\ \end{array} where :math:`\sigma` is the sigmoid function, and :math:`\odot` is the Hadamard product. Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` Inputs: input, (h_0, c_0) - **input** of shape `(batch, input_size)` or `(input_size)`: tensor containing input features - **h_0** of shape `(batch, hidden_size)` or `(hidden_size)`: tensor containing the initial hidden state - **c_0** of shape `(batch, hidden_size)` or `(hidden_size)`: tensor containing the initial cell state If `(h_0, c_0)` is not provided, both **h_0** and **c_0** default to zero. Outputs: (h_1, c_1) - **h_1** of shape `(batch, hidden_size)` or `(hidden_size)`: tensor containing the next hidden state - **c_1** of shape `(batch, hidden_size)` or `(hidden_size)`: tensor containing the next cell state Attributes: weight_ih: the learnable input-hidden weights, of shape `(4*hidden_size, input_size)` weight_hh: the learnable hidden-hidden weights, of shape `(4*hidden_size, hidden_size)` bias_ih: the learnable input-hidden bias, of shape `(4*hidden_size)` bias_hh: the learnable hidden-hidden bias, of shape `(4*hidden_size)` .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. Examples:: >>> rnn = nn.LSTMCell(10, 20) # (input_size, hidden_size) >>> input = torch.randn(2, 3, 10) # (time_steps, batch, input_size) >>> hx = torch.randn(3, 20) # (batch, hidden_size) >>> cx = torch.randn(3, 20) >>> output = [] >>> for i in range(input.size()[0]): ... hx, cx = rnn(input[i], (hx, cx)) ... output.append(hx) >>> output = torch.stack(output, dim=0) TNr$r%r'rc*||d}tj|||fddi|dS)Nr.rr2rr`r$r%r'r/r0rarprrrG  zLSTMCell.__init__rrcCs|dvrtd|d|dur0t|D]\}}|dvr/td|d|dq|dk}|s=|d}|durUtj|d|j|j|j d}||f}n|se|dd|d dfn|}t |||j |j |j|j}|s|dd|d df}|S) Nrz-LSTMCell: Expected input to be 1D or 2D, got rzLSTMCell: Expected hx[z] to be 1D or 2D, got r1rrr )rrN enumeraterrXrrr%r0r/r lstm_cellr r rrr)r`rrrrrrrrrrrs>     $ zLSTMCell.forwardTNNr) rTrrrrQrMrGrrrrrrrrprrJs,?  rc sVeZdZdZ   ddedededdffdd Zdd ed eedefd d Z Z S)raz A gated recurrent unit (GRU) cell. .. math:: \begin{array}{ll} r = \sigma(W_{ir} x + b_{ir} + W_{hr} h + b_{hr}) \\ z = \sigma(W_{iz} x + b_{iz} + W_{hz} h + b_{hz}) \\ n = \tanh(W_{in} x + b_{in} + r \odot (W_{hn} h + b_{hn})) \\ h' = (1 - z) \odot n + z \odot h \end{array} where :math:`\sigma` is the sigmoid function, and :math:`\odot` is the Hadamard product. Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` Inputs: input, hidden - **input** : tensor containing input features - **hidden** : tensor containing the initial hidden state for each element in the batch. Defaults to zero if not provided. Outputs: h' - **h'** : tensor containing the next hidden state for each element in the batch Shape: - input: :math:`(N, H_{in})` or :math:`(H_{in})` tensor containing input features where :math:`H_{in}` = `input_size`. - hidden: :math:`(N, H_{out})` or :math:`(H_{out})` tensor containing the initial hidden state where :math:`H_{out}` = `hidden_size`. Defaults to zero if not provided. - output: :math:`(N, H_{out})` or :math:`(H_{out})` tensor containing the next hidden state. Attributes: weight_ih: the learnable input-hidden weights, of shape `(3*hidden_size, input_size)` weight_hh: the learnable hidden-hidden weights, of shape `(3*hidden_size, hidden_size)` bias_ih: the learnable input-hidden bias, of shape `(3*hidden_size)` bias_hh: the learnable hidden-hidden bias, of shape `(3*hidden_size)` .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. Examples:: >>> rnn = nn.GRUCell(10, 20) >>> input = torch.randn(6, 3, 10) >>> hx = torch.randn(3, 20) >>> output = [] >>> for i in range(6): ... hx = rnn(input[i], hx) ... output.append(hx) TNr$r%r'rcr)Nr.rr3rrrprrrGrzGRUCell.__init__rrcCs|dvrtd|d|dur$|dvr$td|d|dk}|s1|d}|durEtj|d|j|j|jd}n |sL|dn|}t |||j |j |j |j}|sc|d}|S)Nrz,GRUCell: Expected input to be 1D or 2D, got rz-GRUCell: Expected hidden to be 1D or 2D, got r1rr)rrNrrXrrr%r0r/rgru_cellr r rrrrrrrrs6    zGRUCell.forwardrr) rTrrrrQrMrGrrrrrrrprrsA$ r)r )$rrKrOr|typingrrtyping_extensionsrrXrrtorch.nnrtorch.nn.parameterrtorch.nn.utils.rnnr moduler __all__rr _rnn_implsrQr FutureWarningr"r r rrrrrrrrrrsN      (Hq:wn