gOY zdZddlZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z ddl Z ddlmZddlmZmZmZddlmZmZmZddlmZmZejeZdZdZd Z d Z!d Z"eZ#eZ$d %Z&e'e&D]\Z(Z)e*ej+ee)e(d Z,dZ-dZ.e j/Z0Gdde1Z2Gdde1Z3dZ4ddddddddddd Z5dZ6dZ7dZ8dZ9dZ:Gdd e1Z;d!Zd$Z?d%Z@d&ZAd'ZBd(ZCd)ZDd*ZEd+ZFd,ZGejHd-ZId.ZJd/ZKd0ZLdS)1z( This is a direct translation of nvvm.h N)c_void_pc_intPOINTERc_char_pc_size_tbyrefc_char)ir) NvvmErrorNvvmSupportError NvvmWarning) get_libdeviceopen_libdevice open_cudalib)cgutilsconfiga NVVM_SUCCESS NVVM_ERROR_OUT_OF_MEMORY NVVM_ERROR_PROGRAM_CREATION_FAILURE NVVM_ERROR_IR_VERSION_MISMATCH NVVM_ERROR_INVALID_INPUT NVVM_ERROR_INVALID_PROGRAM NVVM_ERROR_INVALID_IR NVVM_ERROR_INVALID_OPTION NVVM_ERROR_NO_MODULE_IN_PROGRAM NVVM_ERROR_COMPILATION ze-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64ze-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-i128:128:128-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64cF tdS#t$rYdSwxYw)z( Return if libNVVM is available TF)NVVMr M/granary/edev/quantum/lib/python3.11/site-packages/numba/cuda/cudadrv/nvvm.py is_availabler<s; t uus   ceZdZdZeeeeefeeefeeefeeee efeeee efeeeeefeeee feeefeeee feeefeeeeeeeeefeeeeefd Z dZ dZ dZ edZedZdZd Zd d ZdS) rzProcess-wide singleton. ) nvvmVersionnvvmCreateProgramnvvmDestroyProgramnvvmAddModuleToProgramnvvmLazyAddModuleToProgramnvvmCompileProgramnvvmGetCompiledResultSizenvvmGetCompiledResultnvvmGetProgramLogSizenvvmGetProgramLog nvvmIRVersionnvvmVerifyProgramNct5|jt|x|_} t d|_n-#t $r }d|_d}t||zd}~wwxYw|j D]G\}}t|j|}|d|_ |dd|_ t|||Hdddn #1swxYwY|jS)Nnvvmz;libNVVM cannot be found. Do `conda install cudatoolkit`: %srr ) _nvvm_lock_NVVM__INSTANCEobject__new__rdriverOSErrorr _PROTOTYPESitemsgetattrrestypeargtypessetattr)clsinsteerrmsgnameprotofuncs rr/z NVVM.__new__s@  . .~%(.s(;(;;7".v"6"6DKK777%)CN2F*6A:666 7$(#3#9#9#;#;..KD%"4;55D#(8DL$)!""IDMD$----! . . . . . . . . . . . . . . .$~s5)C!AC! A1A,,A11A$C!!C%(C%c|}|d|_|d|_|d|_|d|_t |_dS)Nrr r)get_ir_version_majorIR_minorIR _majorDbg _minorDbgget_supported_ccs_supported_ccs)self ir_versionss r__init__z NVVM.__init__sS))++ #A #A $Q$Q/11rcB|j|jfdkrtStS)N)r )rBrC_datalayout_original_datalayout_i128rHs r data_layoutzNVVM.data_layouts" M4= )F 2 2' '# #rc|jSN)rGrOs r supported_ccszNVVM.supported_ccss ""rct}t}|t|t|}||d|j|jfS)NzFailed to get version.)rrr check_errorvalue)rHmajorminorerrs r get_versionzNVVM.get_versions[uU||U5\\:: 6777{EK''rcjt}t}t}t}|t|t|t|t|}||d|j|j|j|jfS)NzFailed to get IR version.)rr(rrUrV)rHmajorIRminorIRmajorDbgminorDbgrYs rrAzNVVM.get_ir_versions''''7777  ww!&x%//CC 9:::}gmX^X^KKrFc|rDt|t|}|r%t|tjddS|dS)Nr )r RESULT_CODE_NAMESprintsysexit)rHerrormsgrdexcs rrUzNVVM.check_errorsR  C!25!9::C c     r)F)__name__ __module__ __qualname____doc__ nvvm_resultrr nvvm_programrrr2r-r/rJpropertyrPrSrZrArUrrrrrKs $WWU^^WWU^^D*77<+@+@A +GGL,A,AB x8#E x8'E uggh.?.?A wwx'8'8&:#.|X!F#.|WWX=N=N!O*<B&wwu~~wwu~~!'%..''%..:*<%gh//1c33KlJ*222$$X$ ##X#(((LLLrrc8eZdZdZdZdZdZdZdZdZ dS) CompilationUnitct|_t|_|jt |j}|j|ddS)NzFailed to create CU)rr0rm_handlerrrU)rHrYs rrJzCompilationUnit.__init__sVff #~~ k++E$,,?,?@@ %:;;;;;rct}|t|j}||dddS)NzFailed to destroy CUT)rd)rr rrrrU)rHr0rYs r__del__zCompilationUnit.__del__sJ''dl(;(;<<3 6TBBBBBrc|j|j|t|d}|j|ddS)z Add a module level NVVM IR to a compilation unit. - The buffer should contain an NVVM module IR either in the bitcode representation (LLVM3.0) or in the text representation. NFailed to add module)r0r!rrlenrUrHbufferrYs r add_modulezCompilationUnit.add_modulesM k00v14VdDD %;<<<<=". - Booleans passed as option values will be converted to integers. - Options which take no value (such as `-gen-lto`) should have a value of `None` passed in and will be emitted in the form "-". For documentation on NVVM compilation options, see the CUDA Toolkit Documentation: https://docs.nvidia.com/cuda/libnvvm-api/index.html#_CPPv418nvvmCompileProgram11nvvmProgramiPPKc c|dd}|d|St|trt|}d|d|S)N_-=)replace isinstanceboolint)kvs rstringify_optionz1CompilationUnit.compile..stringify_optionsT #s##Ay1ww!T"" FFq;;1;; rc.g|]\}}||Srr).0rrrs r z+CompilationUnit.compile..s+FFFda##Aq))FFFrcRg|]$}t|d%S)utf8)rencode)rxs rrz+CompilationUnit.compile.. s<-?-?-?12.6ahhv6F6F-G-G-?-?-?rzFailed to verify zFailed to compile z&Failed to get size of compiled result.zFailed to get compiled result.)categoryN)r3rrwr0r)rr _try_errorr#rr$rr rVr%get_loglogwarningswarnr)rHoptionsc_optsrYreslenptxbufrs @rcompilezCompilationUnit.compiles(   GFFFgmmooFFFS\\)-?-?6=-?-?-?@k++DL#g,,OO 1222k,,T\3w<<PP 2333k33DL%--PP EFFF6<'**k// fEE =>>><<>> 8 : M$([ 9 9 9 9aaayrcj|j||d|dS)N )r0rUr)rHrYrfs rrzCompilationUnit._try_error%s3 dllnnn%EFFFFFrct}|j|jt |}|j|d|jdkrkt|jz}|j|j|}|j|d|j dSdS)Nz#Failed to get compilation log size.r zFailed to get compilation log.r) rr0r&rrrrUrVr r'decode)rHrrYlogbufs rrzCompilationUnit.get_log(sk// eFmmLL %JKKK .SsD111R2////////////rc2g|]}|tjk|Sr)rCUDA_DEFAULT_PTX_CC)rrs rrz(ccs_supported_by_ctk..Xs/;;;Rv999999r) CTK_SUPPORTEDtupleCOMPUTE_CAPABILITIESKeyError) ctk_versionrrs @@rccs_supported_by_ctkrOs <&{311111#711122 2 <<<;;#7;;;<< < < <=%d.%d)r )rrSr enumerate)myccrSrfirs rfind_closest_archrusFF(M $Qs###=)) , ,2 ::III $YYAvv+.2Ri9&s+++%QU++++  rc`tjr tj}nt||f}d|zS)z1Matches with the closest architecture option z compute_%d%d)r FORCE_CUDA_CCr)rWrXarchs rget_arch_optionrs51# %00 D  rz~Missing libdevice file. Please ensure you have package cudatoolkit >= 11.0 Install package by: conda install cudatoolkit ceZdZdZdZdZdS) LibDeviceNc|j5tttt |_|j|_dSrR)_cache_r RuntimeErrorMISSING_LIBDEVICE_FILE_MSGrbcrOs rrJzLibDevice.__init__s= < &"#=>>>)++DL,rc|jSrR)rrOs rgetz LibDevice.gets wr)rhrirjrrJrrrrrrs7Grrz %cas_success = cmpxchg volatile {Ti}* %iptr, {Ti} %old, {Ti} %new monotonic monotonic %cas = extractvalue {{ {Ti}, i1 }} %cas_success, 0 a define internal {T} @___numba_atomic_{T}_{FUNC}({T}* %ptr, {T} %val) alwaysinline {{ entry: %iptr = bitcast {T}* %ptr to {Ti}* %old2 = load volatile {Ti}, {Ti}* %iptr br label %attempt attempt: %old = phi {Ti} [ %old2, %entry ], [ %cas, %attempt ] %dold = bitcast {Ti} %old to {T} %dnew = {OP} {T} %dold, %val %new = bitcast {T} %dnew to {Ti} {CAS} %repeat = icmp ne {Ti} %cas, %old br i1 %repeat, label %attempt, label %done done: %result = bitcast {Ti} %old to {T} ret {T} %result }} a define internal {T} @___numba_atomic_{Tu}_inc({T}* %iptr, {T} %val) alwaysinline {{ entry: %old2 = load volatile {T}, {T}* %iptr br label %attempt attempt: %old = phi {T} [ %old2, %entry ], [ %cas, %attempt ] %bndchk = icmp ult {T} %old, %val %inc = add {T} %old, 1 %new = select i1 %bndchk, {T} %inc, {T} 0 {CAS} %repeat = icmp ne {T} %cas, %old br i1 %repeat, label %attempt, label %done done: ret {T} %old }} a define internal {T} @___numba_atomic_{Tu}_dec({T}* %iptr, {T} %val) alwaysinline {{ entry: %old2 = load volatile {T}, {T}* %iptr br label %attempt attempt: %old = phi {T} [ %old2, %entry ], [ %cas, %attempt ] %dec = add {T} %old, -1 %bndchk = icmp ult {T} %dec, %val %new = select i1 %bndchk, {T} %dec, {T} %val {CAS} %repeat = icmp ne {T} %cas, %old br i1 %repeat, label %attempt, label %done done: ret {T} %old }} a define internal {T} @___numba_atomic_{T}_{NAN}{FUNC}({T}* %ptr, {T} %val) alwaysinline {{ entry: %ptrval = load volatile {T}, {T}* %ptr ; Return early when: ; - For nanmin / nanmax when val is a NaN ; - For min / max when val or ptr is a NaN %early_return = fcmp uno {T} %val, %{PTR_OR_VAL}val br i1 %early_return, label %done, label %lt_check lt_check: %dold = phi {T} [ %ptrval, %entry ], [ %dcas, %attempt ] ; Continue attempts if dold less or greater than val (depending on whether min or max) ; or if dold is NaN (for nanmin / nanmax) %cmp = fcmp {OP} {T} %dold, %val br i1 %cmp, label %attempt, label %done attempt: ; Attempt to swap in the value %old = bitcast {T} %dold to {Ti} %iptr = bitcast {T}* %ptr to {Ti}* %new = bitcast {T} %val to {Ti} {CAS} %dcas = bitcast {Ti} %cas to {T} br label %lt_check done: ret {T} %ptrval }} c8t|S)NTi)cas_nvvmformatrs rir_casrs ??b? ! !!rc ft||||t|}tjdi|S)N)TrOPFUNCCASr)dictrir_numba_atomic_binary_templater)rrrrparamss rir_numba_atomic_binaryr!s7 A"$F2JJ ? ? ?F * 1 ; ;F ; ;;rc jt||||||t|}tjdi|S)N)rrNANr PTR_OR_VALrrr)rrir_numba_atomic_minmax_templater)rrrrrrrs rir_numba_atomic_minmaxr&sD A"#"---F + 1 ; ;F ; ;;rcVt||t|SN)rTur)ir_numba_atomic_inc_templaterrrrs rir_numba_atomic_incr-# ' . .rvayy . I IIrcVt||t|Sr)ir_numba_atomic_dec_templaterrrs rir_numba_atomic_decr1rrcdtddddfdtdd d d fd tddd d fd tddfdtddfdtdd ddddfdtddddddfdtdd ddddfdtddddddfdtdd ddddfdtddddddfd tdd dd!ddfd"tdddd!ddfd#g}|D]\}}|||}t |}|S)$NzIdeclare double @"___numba_atomic_double_add"(double* %".1", double %".2")doublei64faddadd)rrrrzEdeclare float @"___numba_atomic_float_sub"(float* %".1", float %".2")floati32fsubsubzIdeclare double @"___numba_atomic_double_sub"(double* %".1", double %".2")z=declare i64 @"___numba_atomic_u64_inc"(i64* %".1", i64 %".2")u64rz=declare i64 @"___numba_atomic_u64_dec"(i64* %".1", i64 %".2")zEdeclare float @"___numba_atomic_float_max"(float* %".1", float %".2")rznnan oltptrmax)rrrrrrzIdeclare double @"___numba_atomic_double_max"(double* %".1", double %".2")zEdeclare float @"___numba_atomic_float_min"(float* %".1", float %".2")znnan ogtrzIdeclare double @"___numba_atomic_double_min"(double* %".1", double %".2")zHdeclare float @"___numba_atomic_float_nanmax"(float* %".1", float %".2")nanultzLdeclare double @"___numba_atomic_double_nanmax"(double* %".1", double %".2")zHdeclare float @"___numba_atomic_float_nanmin"(float* %".1", float %".2")ugtzLdeclare double @"___numba_atomic_double_nanmin"(double* %".1", double %".2"))immargr)rrrrrllvm140_to_70_ir)llvmir replacementsdeclfns r llvm_replacer5s  T (ue L L L N P 'eU K K K M T (ue L L L N H u / / / 1 H u / / / 1 P 'e +0u > > > ? U (u"+0u > > > ? Q 'e +0u > > > ? U (u"+0u > > > ? T 'e5+-E ; ; ; < X (u%E+-E ; ; ; < T 'e5+-E ; ; ; < X (u%E+-E ; ; ; < G$LL!**bb)) f % %F Mrc t|tr|g}|ddr|dddddt }t }|D]9}t |}||d:| | |j di|S)NfastmathFT)ftzfmaprec_div prec_sqrtrr) rstrpopupdaterprrrzrr|rr)roptscu libdevicemods r llvm_to_ptxrds&# xx E""           B I**3 cjj(())))y}}''' 2:    rz"^attributes #\d+ = \{ ([\w\s]+)\ }cg}|jD]}|drt|}|d}dd|D}||d|}||d|S)z, Convert LLVM 14.0 IR for LLVM 7.0. z attributes #r  c3&K|] }|dk|V dS) willreturnNr)ras r z#llvm140_to_70_ir..s,CC1l1B1BQ1B1B1B1BCCrr) splitlines startswithre_attributes_defmatchgroupsplitjoinrappend)r buflinemattrss rrr~s C  ??> * * 3!''--AGGAJJ$$&&EHHCCCCCCCE<< E22D 4 99S>>rcD|j}tj|d}tjtjdd}||||f}t j|d}|||j ddS)Nkernel r znvvm.annotationsnoinline) moduler MetaDataStringConstantIntType add_metadatarget_or_insert_named_metadatar attributesdiscard)lfuncrmdstrmdvaluemdnmds rset_cuda_kernelr:s ,C  c8 , ,Ek"*R..!,,G  5%1 2 2B  .s4F G GCGGBKKK Z(((((rctjdfdtD}||}|d|dS)zAdd NVVM IR version to moduler+c&g|] }|Srr)rrrs rrz"add_ir_version..s!;;;a33q66;;;rznvvmir.versionN)r r0rrAr1add_named_metadata)rrImd_verrs @radd_ir_versionr?sk *R..C;;;;466#8#8#:#:;;;K   k * *F+V44444r)Mrkloggingrercrctypesrrrrrrr threadingllvmliter rer r rlibsrrr numba.corerr getLoggerrhloggerADDRSPACE_GENERICADDRSPACE_GLOBALADDRSPACE_SHAREDADDRSPACE_CONSTANTADDRSPACE_LOCALrmrlr"rarrrr7modulesrMrNrLockr,r.rrprrrrFrrrrrrrrrrrrrrrrrrrr:r?rrrrPs ;;;;;;;;;;==========&&&&&&&&  8 $ $    EGG I' ( ())DAq GCK !1a((((;7   Y^   {{{{{6{{{|kkkkkfkkk\             < < <2D!!!         #, ( (#@"""<<< <<<JJJJJJ,,,^.BJDEE$ ) ) )55555r