o  h0@sddlmZddlmZmZmZmZerddlZddlmZer(ddl m Z er/ddl Z e eZGdddejjZGd d d ejZ     dd d Z     dddZdS))ACT2FN)is_accelerate_availableis_fbgemm_gpu_availableis_torch_availableloggingN)nn)init_empty_weightsc*eZdZejffdd ZddZZS)FbgemmFp8Linearcst|||||_||_tjtj||ftjd|_ tjtj|df|d|_ |j dtjdgtj ddd|rKtjtj|j|d|_ dSd|_ dS)Ndtypeinput_scale_ubF persistent)super__init__ in_features out_featurestorchr Parameterzeros float8_e4m3fnweight weight_scaleregister_bufferfloatbias)selfrrr weight_dtype __class__x/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/transformers/integrations/fbgemm_fp8.pyr!s zFbgemmFp8Linear.__init__cCsg|jdddR}tjjj|d|jd|jd\}}|j tj }tjjj ||j ||dd}|j dur?||j n|}| |j}||}~~|S)N)scale_ubTuse_fast_accum)shaperopsfbgemmquantize_fp8_per_rowview contiguousrrtofloat32f8f8bf16_rowwiserrdevicereshape)rx output_shape x_quantizedx_scaleweight_scale_float32outputr#r#r$forward/s    zFbgemmFp8Linear.forward__name__ __module__ __qualname__rr0rr: __classcell__r#r#r!r$r sr cr )FbgemmFp8Llama4TextExpertscst|j|_|j|_|j|_|j|_t|j|_ t j t j |j|jd|jft jd|_t j t j |jd|jdft jd|_t j t j |j|j|jft jd|_t j t j |j|jdft jd|_|jdt j dgt jddddS)Nrr rrFr)rrnum_local_experts num_expertsintermediate_size hidden_size expert_dimr hidden_actact_fnrrrrr gate_up_projr0gate_up_proj_scale down_projdown_proj_scalerr)rconfigr r!r#r$rHs&  "z#FbgemmFp8Llama4TextExperts.__init__c Cs||jd|j}d}t|}t|jD]}||}|d|j}tjj |||j \}}|j j dd} |j tj} tjjj||j |ddd| || |dd| dddd} tjjj||j |dd| d|| |d| ddddd} | || } tjj | ||j \}}|jtj}tjjj||j|dd|||dddd}|||<q||j}|d|jS)z Args: hidden_states (torch.Tensor): (batch_size * token_num, hidden_size) Returns: torch.Tensor: (batch_size * token_num, hidden_size) r%NrrrTr')r-rBrDr empty_likeranger3r*r+r,rrHr)rIr/r0r1 transposer.rGrKrJr2)r hidden_states num_tokens next_statesi expert_hiddenexpert_hidden_reshapedexpert_quantized expert_scalesharded_expert_dimgate_up_proj_scale_float32gateup activatedactivated_quantizedactivated_scaledown_proj_scale_float32 expert_outputr#r#r$r:`sP   z"FbgemmFp8Llama4TextExperts.forwardr;r#r#r!r$r@Gsr@Fc sddl}|dur g}|D]\} } || t| tjrp| |vrpd|tfdd|Dsptdd$| j } | j } t | | | j du|j | <d}|j | dWdn1s]wYtj|jgtjd |j | _| jjd kr| |vrd|tfd d|Dstddd||d d d<t|j|j | <Wdn1swYtj|jgtjd |j | _tt| dkrt| |||||||d\} }|dq||fS)z Private method that wraps the recursion for module replacement. Returns the converted model and a boolean that indicates if the conversion has been successfull or not. rN.c3$|] }|dvp |kVqdSraNr#.0keycurrent_key_name_strr#r$  z2_replace_with_fbgemm_fp8_linear..T)include_buffersFr Llama4TextExpertsc3rbrcr#rdrgr#r$rirjz\d+*z.down_proj_scale)has_been_replaced pre_quantizedrLtp_planr%)renamed_childrenappend isinstancerLinearjoinanyr rrr r_modulesrequires_grad_rtensoractivation_scale_ubrrr"r<subr@ text_configlenlistchildren_replace_with_fbgemm_fp8_linearpop)modelmodules_to_not_convertcurrent_key_namequantization_configrnrorLrprqnamemodulerr_r#rgr$rsh            rc Cs`|durdgn|}|jdur||jtt|}t|||||||d\}}|s.td|S)a A helper function to replace all `torch.nn.Linear` modules by `FbgemmFp8Linear` modules. This will enable running your models using high performance fp8 kernel from FBGEMM library. The function will be run recursively and replace all `torch.nn.Linear` modules except for the `lm_head` that should be kept as a `torch.nn.Linear` module. The replacement is done under `init_empty_weights` context manager so no CPU/GPU memory is required to run this function. Each weight will be quantized along the channel. Parameters: model (`torch.nn.Module`): Input model or `torch.nn.Module` as the function is run recursively. modules_to_not_convert (`List[`str`]`, *optional*, defaults to `["lm_head"]`): Names of the modules to not convert in `FP8Linear`. In practice we keep the `lm_head` in full precision for numerical stability reasons. current_key_name (`List[`str`]`, *optional*): An array to track the current key of the recursion. This is used to check whether the current key (part of it) is not in the list of modules to not convert (for instances modules that are offloaded to `cpu` or `disk`). Nlm_head)rorLrpzYou are loading your model using FP8 quantization but no linear modules were found in your model. Please double check your model architecture, or submit an issue on github if you think this is a bug.)rextendrsetrloggerwarning)rrrrrorLrprnr#r#r$replace_with_fbgemm_fp8_linears$     r)NNNFFNN)NNNFNN) activationsrutilsrrrrrr accelerater fbgemm_gpu.experimental.gen_ai fbgemm_gpu get_loggerr<rrur Moduler@rrr#r#r#r$s6    'X O