o n hO@sddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZejejeZejedgZdd Zed d Zd d ZGdddeZddZddZGdddeZGdddeZdS)N)Path)_build)get_cache_manager) GPUTarget) GPUDriverincludec sddl}|dkr dSddlddlm}m}m}mmm}Gfdddj } |||||||}z dj }Wn t yLYdSw|g|_||_dd} fd d } ||| | rut| SdS) NrLinux)c_charc_intc_size_tc_void_pc_char_pPOINTERcseZdZdfdfgZdS)z8_find_already_mmapped_dylib_on_linux..DlPhdrInfo dlpi_addr dlpi_nameN)__name__ __module__ __qualname___fields_)r r rn/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/triton/backends/amd/driver.py DlPhdrInfosrz libc.so.6ics@|jj}tt|}|jvr||tt|dSdS)Nrr) contentsrrosfsdecodenamememmoveminlen)infosizedatarp)ctypeslib_namemax_path_lengthrrcallback2s  z6_find_already_mmapped_dylib_on_linux..callback)platformsystemr$r r r r r r Structure CFUNCTYPECDLLdl_iterate_phdr Exceptionargtypesrestypecreate_string_bufferrr string_at) r%r(r r r rr callback_tr-pathr'r)r r r$r%r&r$_find_already_mmapped_dylib_on_linuxs(     r5csdtd}|r |rtj|r|Std|dt}|r8tj|r.|Std|dg}ddl}|}| }|j rN|g|}|D]}tj |dd }tj|re|S| |qPtd }|r| d D]}tj |} tj| r| S| | qwtd d g} fdd| D} | D]} tj| r| S| | qtj d} tj| r| S| | tdd|)Nzlibamdhip64.soTRITON_LIBHIP_PATHzTRITON_LIBHIP_PATH 'z' does not point to a valid zmemory mapped 'z'' in process does not point to a valid rtorchlibLD_LIBRARY_PATH:z/sbin/ldconfigz-pcs&g|]}|r|dqS))stripendswithsplit).0liner%rr ps&z2_get_path_to_hip_runtime_dylib..z/opt/rocm/lib/zcannot locate z after attempted paths )rgetenvr=r4exists RuntimeErrorr5sitegetsitepackagesgetusersitepackagesENABLE_USER_SITEjoinappendr> subprocess check_outputdecode splitlines)env_libhip_path mmapped_pathpathsrF site_packages user_siter4env_ld_library_pathdflibslocsloccommon_install_pathrrAr_get_path_to_hip_runtime_dylib@sP            r\c Cs&t|d}t|}||d}|durztT}tj |d}t |d }| |Wdn1s;wYt |||gtg}t |d}|j||ddd}Wdn1sfwYWdn1suwYddl} | j||} | j| } | j| | S) Nzutf-8z.sozmain.cwrbT)binaryr)hashlibsha256encode hexdigestrget_filetempfileTemporaryDirectoryrr4rJopenwriter include_dirputreadimportlib.utilutilspec_from_file_locationmodule_from_specloader exec_module) srcrkeycache cache_pathtmpdirsrc_pathrWso importlibspecmodrrrcompile_module_from_srcs(      r|cs$eZdZfddZddZZS)HIPUtilscs"t|dstt|||_|jS)Ninstance)hasattrsuperr}__new__r~)cls __class__rrrs zHIPUtils.__new__cCsHt}ttjtd}|d|d}t|d}|j |_ |j |_ dS)Nzdriver.cz/*py_libhip_search_path*/r hip_utils) r\rrr4rJdirname read_textreplacer| load_binaryget_device_properties)self libhip_pathrrr{rrr__init__s   zHIPUtils.__init__)rrrrr __classcell__rrrrr}s r}cCs8|ddkrdSddddddd d dd d d d d d d|S)Nr*hipDeviceptr_tint32_tint8_tint16_tint64_tuint32_tuint8_tuint16_tuint64_tfloatdouble)i1i8i16i32i64u1u8u16u32u64fp16bf16fp32f32fp64r)tyrrr ty_to_cpps( rc sfddfddfdddfdd |D}d |}d t|}ttt|d }d d t|D}t|dkrUdddd| Dnd}ddd| D}g}| D] \}} | ddkr}| d|dqi| dkr| d|qit } tt t|} dd | D} | dd| dt|dkrd|nddd| d|d|dd fd!d | Dd"|d#|d$d%d&d | Dd't|dkrdd|ndd(} | S))Ncst|tr dt|S|S)N,) isinstancetuplerJmap)sig)_serialize_signaturerrrs z+make_launcher.._serialize_signaturecsJt|trdt|}d|dS|ddkrdS|dvr!dSt|S)Nr[]rrz PyObject* constexprrrrJrrrval_extracted_typerrrs   z&make_launcher.._extracted_typec sft|trdt|}d|dS|ddkrdS|dvr!dSdd d d d d dddddd t|S)N()rrOrrWrVlbhiLBHIK) rrlongrrrrrrrrrr format_ofrrrs*    z make_launcher..format_ofrcsg|]}|qSrr)r?rrrrrBsz!make_launcher.. piiiKKOOOOrcSi|]\}}||qSrr)r?rsrrr z!make_launcher..r, css|] \}}d|VqdS)z&_argNrr?rrrrr sz make_launcher..css.|]\}}|dkrt|d|VqdS)rz argN)rrrrrrs,rptr_infoz.dev_ptrr_argcSs"g|] \}}|dkrd|qS)rz&argrrrrrrBs"z&global_scratcha; #define __HIP_PLATFORM_AMD__ #include #include #include #include #include // The list of paths to search for the HIP runtime library. The caller Python // code should substitute the search path placeholder. static const char *hipLibSearchPaths[] = {"a"}; // The list of HIP dynamic library symbols and their signature we are interested // in this file. #define HIP_SYMBOL_LIST(FOR_EACH_ERR_FN, FOR_EACH_STR_FN) \ FOR_EACH_STR_FN(hipGetErrorString, hipError_t hipError) \ FOR_EACH_ERR_FN(hipModuleLaunchKernel, hipFunction_t f, \ unsigned int gridDimX, unsigned int gridDimY, \ unsigned int gridDimZ, unsigned int blockDimX, \ unsigned int blockDimY, unsigned int blockDimZ, \ unsigned int sharedMemBytes, hipStream_t stream, \ void **kernelParams, void **extra) \ FOR_EACH_ERR_FN(hipModuleLaunchCooperativeKernel, hipFunction_t f, \ unsigned int gridDimX, unsigned int gridDimY, \ unsigned int gridDimZ, unsigned int blockDimX, \ unsigned int blockDimY, unsigned int blockDimZ, \ unsigned int sharedMemBytes, hipStream_t stream, \ void **kernelParams, void **extra) \ FOR_EACH_ERR_FN(hipPointerGetAttribute, void *data, \ hipPointer_attribute attribute, hipDeviceptr_t ptr) // The HIP symbol table for holding resolved dynamic library symbols. struct HIPSymbolTable { #define DEFINE_EACH_ERR_FIELD(hipSymbolName, ...) \ hipError_t (*hipSymbolName)(__VA_ARGS__); #define DEFINE_EACH_STR_FIELD(hipSymbolName, ...) \ const char *(*hipSymbolName)(__VA_ARGS__); HIP_SYMBOL_LIST(DEFINE_EACH_ERR_FIELD, DEFINE_EACH_STR_FIELD) }; static struct HIPSymbolTable hipSymbolTable; bool initSymbolTable() { // Use the HIP runtime library loaded into the existing process if it exits. void *lib = dlopen("libamdhip64.so", RTLD_NOLOAD); if (lib) { // printf("[triton] chosen loaded libamdhip64.so in the process\n"); } // Otherwise, go through the list of search paths to dlopen the first HIP // driver library. if (!lib) { int n = sizeof(hipLibSearchPaths) / sizeof(hipLibSearchPaths[0]); for (int i = 0; i < n; ++i) { void *handle = dlopen(hipLibSearchPaths[i], RTLD_LAZY | RTLD_LOCAL); if (handle) { lib = handle; // printf("[triton] chosen %s\n", hipLibSearchPaths[i]); } } } if (!lib) { PyErr_SetString(PyExc_RuntimeError, "cannot open libamdhip64.so"); return false; } // Resolve all symbols we are interested in. dlerror(); // Clear existing errors const char *error = NULL; #define QUERY_EACH_FN(hipSymbolName, ...) \ *(void **)&hipSymbolTable.hipSymbolName = dlsym(lib, #hipSymbolName); \ error = dlerror(); \ if (error) { \ PyErr_SetString(PyExc_RuntimeError, \ "cannot query " #hipSymbolName " from libamdhip64.so"); \ dlclose(lib); \ return false; \ } HIP_SYMBOL_LIST(QUERY_EACH_FN, QUERY_EACH_FN) return true; } static inline void gpuAssert(hipError_t code, const char *file, int line) { if (code != HIP_SUCCESS) { const char* prefix = "Triton Error [HIP]: "; const char* str = hipSymbolTable.hipGetErrorString(code); char err[1024] = {0}; snprintf(err, 1024, "%s Code: %d, Messsage: %s", prefix, code, str ); PyErr_SetString(PyExc_RuntimeError, err); } } #define HIP_CHECK(ans) { gpuAssert((ans), __FILE__, __LINE__); } static void _launch(int gridX, int gridY, int gridZ, int num_warps, int num_ctas, int launch_cooperative_grid, int clusterDimX, int clusterDimY, int clusterDimZ, int shared_memory, hipStream_t stream, hipFunction_t functionzc) { // printf("_launch hip kernel\n"); hipDeviceptr_t global_scratch = 0; void *params[] = { z }; if (gridX*gridY*gridZ > 0 && launch_cooperative_grid) { HIP_CHECK(hipSymbolTable.hipModuleLaunchCooperativeKernel(function, gridX, gridY, gridZ, z*num_warps, 1, 1, shared_memory, stream, params, 0)); return; } if (gridX*gridY*gridZ > 0) { HIP_CHECK(hipSymbolTable.hipModuleLaunchKernel(function, gridX, gridY, gridZ, a*num_warps, 1, 1, shared_memory, stream, params, 0)); } } typedef struct _DevicePtrInfo { hipDeviceptr_t dev_ptr; bool valid; } DevicePtrInfo; static inline DevicePtrInfo getPointer(PyObject *obj, int idx) { DevicePtrInfo ptr_info; ptr_info.dev_ptr = 0; ptr_info.valid = true; if (PyLong_Check(obj)) { ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(obj); return ptr_info; } if (obj == Py_None) { // valid nullptr return ptr_info; } PyObject *ptr = PyObject_GetAttrString(obj, "data_ptr"); if(ptr){ PyObject *empty_tuple = PyTuple_New(0); PyObject *ret = PyObject_Call(ptr, empty_tuple, NULL); Py_DECREF(empty_tuple); Py_DECREF(ptr); if (!PyLong_Check(ret)) { PyErr_SetString(PyExc_TypeError, "data_ptr method of Pointer object must return 64-bit int"); ptr_info.valid = false; return ptr_info; } ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(ret); if(!ptr_info.dev_ptr) return ptr_info; uint64_t dev_ptr; hipError_t status = hipSymbolTable.hipPointerGetAttribute(&dev_ptr, HIP_POINTER_ATTRIBUTE_DEVICE_POINTER, ptr_info.dev_ptr); if (status == hipErrorInvalidValue) { PyErr_Format(PyExc_ValueError, "Pointer argument (at %d) cannot be accessed from Triton (cpu tensor?)", idx); ptr_info.valid = false; } ptr_info.dev_ptr = (hipDeviceptr_t)dev_ptr; Py_DECREF(ret); return ptr_info; } PyErr_SetString(PyExc_TypeError, "Pointer argument must be either uint64 or have data_ptr method"); return ptr_info; } static PyObject* launch(PyObject* self, PyObject* args) { // printf("launch\n"); int gridX, gridY, gridZ; uint64_t _stream; uint64_t _function; int launch_cooperative_grid; PyObject *launch_enter_hook = NULL; PyObject *launch_exit_hook = NULL; PyObject *kernel_metadata = NULL; PyObject *launch_metadata = NULL;  cs$g|]\}}|d|dqS)z _arg; rrrrrrBs$z if(!PyArg_ParseTuple(args, "a", &launch_cooperative_grid, &gridX, &gridY, &gridZ, &_stream, &_function, &kernel_metadata, &launch_metadata, &launch_enter_hook, &launch_exit_hook a=)) { return NULL; } // extract kernel metadata int num_warps, num_ctas, shared_memory, clusterDimX, clusterDimY, clusterDimZ; if (!PyArg_ParseTuple(kernel_metadata, "iiiiii", &num_warps, &num_ctas, &shared_memory, &clusterDimX, &clusterDimY, &clusterDimZ)) { return NULL; } // extract launch metadata if (launch_enter_hook != Py_None){ PyObject* args = Py_BuildValue("(O)", launch_metadata); PyObject* ret = PyObject_CallObject(launch_enter_hook, args); Py_DECREF(args); if (!ret) return NULL; } // raise exception asap rc Ss>g|]\}}|ddkrd|d|d|d|d ndqS)rrzDevicePtrInfo ptr_infoz = getPointer(_argrz); if (!ptr_infoz.valid) return NULL;rrrrrrrBs>z; _launch(gridX, gridY, gridZ, num_warps, num_ctas, launch_cooperative_grid, clusterDimX, clusterDimY, clusterDimZ, shared_memory, (hipStream_t)_stream, (hipFunction_t)_functionan); if(launch_exit_hook != Py_None){ PyObject* args = Py_BuildValue("(O)", launch_metadata); PyObject* ret = PyObject_CallObject(launch_exit_hook, args); Py_DECREF(args); if (!ret) return NULL; } if(PyErr_Occurred()) { return NULL; } // return None Py_INCREF(Py_None); return Py_None; } static PyMethodDef ModuleMethods[] = { {"launch", launch, METH_VARARGS, "Entry point for all kernels with this signature"}, {NULL, NULL, 0, NULL} // sentinel }; static struct PyModuleDef ModuleDef = { PyModuleDef_HEAD_INIT, "__triton_launcher", NULL, //documentation -1, //size ModuleMethods }; PyMODINIT_FUNC PyInit___triton_launcher(void) { if (!initSymbolTable()) { return NULL; } PyObject *m = PyModule_Create(&ModuleDef); if(m == NULL) { return NULL; } PyModule_AddFunctions(m, ModuleMethods); return m; } ) rJvaluesrlistfilterboolr> enumerateritemsrKr\range) constants signature warp_size args_formatformat args_list arg_declsinternal_args_listrrrparamsrrr)rrrr make_launchersl   ,   cfhl)*-ABmrc@seZdZddZddZdS) HIPLaunchercsxtdrjnt}fddfdd|D}ddjD}t|||jtd}|j|_|j |_ dS)Nrcst|tr jj|fS|SN)rstrfn arg_namesindex)x)rrrrsz&HIPLauncher.__init__..csi|] \}}||qSrrr?idxvalue)arg_idxrrrsz(HIPLauncher.__init__..cSrrrrrrrrr__triton_launcher) rrdictrrrrr|launchlaunch_cooperative_grid)rrrmetadatarrr{r)rrrrrs   zHIPLauncher.__init__cGs|j|jg|RdSr)rr)rargsrrr__call__szHIPLauncher.__call__N)rrrrrrrrrrs rcsXeZdZfddZddZeddZddZd d Zd d Z d dZ ddZ Z S) HIPDrivercstt|_t|_dSr)rrr}utilsr launcher_cls)rrrrrs  zHIPDriver.__init__cCsddl}|jS)Nr)r7cudarr7rrrget_device_interfaceszHIPDriver.get_device_interfacecCs,z ddl}|jjduWStyYdSw)NrF)r7versionhip ImportError)r7rrr is_actives  zHIPDriver.is_activecCs:|}|j|}|d}|d}td|dd|S)NarchwarpSizerr:r)get_current_devicerrrr>)rdevicedevice_propertiesrrrrrget_current_target s  zHIPDriver.get_current_targetcCsddl}|d|S)Nrr)r7rrrrrrget_active_torch_devicesz!HIPDriver.get_active_torch_devicecCsddlm}|S)Nr)do_bench)triton.testingr )rr rrrget_benchmarkers zHIPDriver.get_benchmarkercCs&ddl}d}|jt|d|jddS)Nrir)dtyper)r7emptyint)rr7 cache_sizerrrget_empty_cache_for_benchmarksz'HIPDriver.get_empty_cache_for_benchmarkcCs |dSr)zero_)rrtrrr clear_cache!s zHIPDriver.clear_cache) rrrrr staticmethodrr r r rrrrrrrrs  r) functoolsrr`rLrepathlibrtriton.runtime.buildrtriton.runtime.cachertriton.backends.compilerrtriton.backends.driverrr4rrealpath__file__rJrir5 lru_cacher\r|objectr}rrrrrrrrs.     0 >-