diff --git a/sdk/include/xdk/memaccess.h b/sdk/include/xdk/memaccess.h new file mode 100644 index 00000000000..a9036207f41 --- /dev/null +++ b/sdk/include/xdk/memaccess.h @@ -0,0 +1,728 @@ +$if(0) +#pragma once + +#include +$endif() + +#define KeMemoryBarrierWithoutFence() _ReadWriteBarrier() + +#if defined(_M_IX86) || defined(_M_AMD64) +#define PreFetchCacheLine(l, a) _mm_prefetch((char const *) a, l) +#define PF_TEMPORAL_LEVEL_1 _MM_HINT_T0 +#define PF_TEMPORAL_LEVEL_2 _MM_HINT_T1 +#define PF_TEMPORAL_LEVEL_3 _MM_HINT_T2 +#define PF_NON_TEMPORAL_LEVEL_ALL _MM_HINT_NTA +#define _AcquireBarrier() +#define _ReleaseBarrier() +#elif defined(_M_ARM) || defined(_M_ARM64) +#define PreFetchCacheLine(l,a) __prefetch((const void *) (a)) +#define PrefetchForWrite(p) __prefetch((const void *) (p)) +#define PF_TEMPORAL_LEVEL_1 0 +#define PF_TEMPORAL_LEVEL_2 1 +#define PF_TEMPORAL_LEVEL_3 2 +#define PF_NON_TEMPORAL_LEVEL_ALL 3 +#define ReadForWriteAccess(p) (*(p)) +#endif + +#if !defined(RC_INVOKED) + +#if defined(_M_IX86) +__forceinline +void +MemoryBarrier ( + void) +{ + long Barrier; + _InterlockedOr(&Barrier, 0); +} +#define PrefetchForWrite(p) +#define ReadForWriteAccess(p) (*(p)) +#elif defined(_M_AMD64) +#define MemoryBarrier __faststorefence +#define PrefetchForWrite(p) _m_prefetchw(p) +#define ReadForWriteAccess(p) (_m_prefetchw(p), *(p)) +#elif defined(_M_ARM) +# define MemoryBarrier() __dmb(_ARM_BARRIER_SY) +# define _AcquireBarrier() __dmb(_ARM_BARRIER_ISH) +# define _ReleaseBarrier() __dmb(_ARM_BARRIER_ISH) +# define _DataSynchronizationBarrier() __dsb(_ARM_BARRIER_SY) +# define _InstructionSynchronizationBarrier() __isb(_ARM_BARRIER_SY) +#elif defined(_M_ARM64) +# define MemoryBarrier() __dmb(_ARM64_BARRIER_SY) +# define _AcquireBarrier() __dmb(_ARM64_BARRIER_ISH) +# define _ReleaseBarrier() __dmb(_ARM64_BARRIER_ISH) +# define _DataSynchronizationBarrier() __dsb(_ARM64_BARRIER_SY) +# define _InstructionSynchronizationBarrier() __isb(_ARM64_BARRIER_SY) +#else +#error Unsupported architecture +#endif /* _M_ARM */ + +#if defined(_M_IX86) || defined(_M_AMD64) +#define __iso_volatile_load8(p) (*(volatile char*)(p)) +#define __iso_volatile_load16(p) (*(volatile short*)(p)) +#define __iso_volatile_load32(p) (*(volatile int*)(p)) +#define __iso_volatile_load64(p) (*(volatile __int64*)(p)) +#define __iso_volatile_store8(p,v) (*(volatile char*)(p) = (v)) +#define __iso_volatile_store16(p,v) (*(volatile short*)(p) = (v)) +#define __iso_volatile_store32(p,v) (*(volatile int*)(p) = (v)) +#define __iso_volatile_store64(p,v) (*(volatile __int64*)(p) = (v)) +#endif + +__forceinline +char +ReadRaw8 ( + _In_ _Interlocked_operand_ char const volatile *Source) +{ + return *(char *)Source; +} + +__forceinline +void +WriteRaw8 ( + _Out_ _Interlocked_operand_ char volatile *Destination, + _In_ char Value) +{ + *(char *)Destination = Value; +} + +__forceinline +short +ReadRaw16 ( + _In_ _Interlocked_operand_ short const volatile *Source) +{ + return *(short *)Source; +} + +__forceinline +void +WriteRaw16 ( + _Out_ _Interlocked_operand_ short volatile *Destination, + _In_ short Value) +{ + *(short *)Destination = Value; +} + +__forceinline +long +ReadRaw ( + _In_ _Interlocked_operand_ long const volatile *Source) +{ + return *(long *)Source; +} + +__forceinline +void +WriteRaw ( + _Out_ _Interlocked_operand_ long volatile *Destination, + _In_ long Value) +{ + *(long *)Destination = Value; +} + +__forceinline +__int64 +ReadRaw64 ( + _In_ _Interlocked_operand_ __int64 const volatile *Source) +{ + return *(__int64 *)Source; +} + +__forceinline +void +WriteRaw64 ( + _Out_ _Interlocked_operand_ __int64 volatile *Destination, + _In_ __int64 Value) +{ + *(__int64 *)Destination = Value; +} + +__forceinline +char +ReadNoFence8 ( + _In_ _Interlocked_operand_ char const volatile *Source) +{ + return __iso_volatile_load8(Source); +} + +__forceinline +void +WriteNoFence8 ( + _Out_ _Interlocked_operand_ char volatile *Destination, + _In_ char Value) +{ + __iso_volatile_store8(Destination, Value); +} + +__forceinline +short +ReadNoFence16 ( + _In_ _Interlocked_operand_ short const volatile *Source) +{ + return __iso_volatile_load16(Source); +} + +__forceinline +void +WriteNoFence16 ( + _Out_ _Interlocked_operand_ short volatile *Destination, + _In_ short Value) +{ + __iso_volatile_store16(Destination, Value); +} + +__forceinline +long +ReadNoFence ( + _In_ _Interlocked_operand_ long const volatile *Source) +{ + return __iso_volatile_load32((const volatile int*)Source); +} + +__forceinline +void +WriteNoFence ( + _Out_ _Interlocked_operand_ long volatile *Destination, + _In_ long Value) +{ + __iso_volatile_store32((volatile int*)Destination, Value); +} + +__forceinline +__int64 +ReadNoFence64 ( + _In_ _Interlocked_operand_ __int64 const volatile *Source) +{ + return __iso_volatile_load64(Source); +} + +__forceinline +void +WriteNoFence64 ( + _Out_ _Interlocked_operand_ __int64 volatile *Destination, + _In_ __int64 Value) +{ + __iso_volatile_store64(Destination, Value); +} + + +__forceinline +char +ReadAcquire8 ( + _In_ _Interlocked_operand_ char const volatile *Source) +{ + char Value = __iso_volatile_load8(Source); + _AcquireBarrier(); + return Value; +} + +__forceinline +void +WriteRelease8 ( + _Out_ _Interlocked_operand_ char volatile *Destination, + _In_ char Value) +{ + _ReleaseBarrier(); + __iso_volatile_store8(Destination, Value); +} + +__forceinline +short +ReadAcquire16 ( + _In_ _Interlocked_operand_ short const volatile *Source) +{ + short Value = __iso_volatile_load16(Source); + _AcquireBarrier(); + return Value; +} + +__forceinline +void +WriteRelease16 ( + _Out_ _Interlocked_operand_ short volatile *Destination, + _In_ short Value) +{ + _ReleaseBarrier(); + __iso_volatile_store16(Destination, Value); +} + +__forceinline +long +ReadAcquire ( + _In_ _Interlocked_operand_ long const volatile *Source) +{ + long Value = __iso_volatile_load32((const volatile int*)Source); + _AcquireBarrier(); + return Value; +} + +__forceinline +void +WriteRelease ( + _Out_ _Interlocked_operand_ long volatile *Destination, + _In_ long Value) +{ + _ReleaseBarrier(); + __iso_volatile_store32((volatile int*)Destination, Value); +} + +__forceinline +__int64 +ReadAcquire64 ( + _In_ _Interlocked_operand_ __int64 const volatile *Source) +{ + __int64 Value = __iso_volatile_load64(Source); + _AcquireBarrier(); + return Value; +} + +__forceinline +void +WriteRelease64 ( + _Out_ _Interlocked_operand_ __int64 volatile *Destination, + _In_ __int64 Value) +{ + _ReleaseBarrier(); + __iso_volatile_store64(Destination, Value); +} + + +__forceinline +unsigned char +ReadUCharAcquire ( + _In_ _Interlocked_operand_ unsigned char const volatile *Source) +{ + return (unsigned char)ReadAcquire8((char*)Source); +} + +__forceinline +unsigned char +ReadUCharNoFence ( + _In_ _Interlocked_operand_ unsigned char const volatile *Source) +{ + return (unsigned char)ReadNoFence8((char*)Source); +} + +__forceinline +unsigned char +ReadUCharRaw ( + _In_ _Interlocked_operand_ unsigned char const volatile *Source) +{ + return (unsigned char)ReadRaw8((char*)Source); +} + +__forceinline +void +WriteUCharRelease ( + _Out_ _Interlocked_operand_ unsigned char volatile *Destination, + _In_ unsigned char Value) +{ + WriteRelease8((char*)Destination, (char)Value); +} + +__forceinline +void +WriteUCharNoFence ( + _Out_ _Interlocked_operand_ unsigned char volatile *Destination, + _In_ unsigned char Value) +{ + WriteNoFence8((char*)Destination, (char)Value); +} + +__forceinline +void +WriteUCharRaw ( + _Out_ _Interlocked_operand_ unsigned char volatile *Destination, + _In_ unsigned char Value) +{ + WriteRaw8((char*)Destination, (char)Value); +} + +__forceinline +BOOLEAN +ReadBooleanAcquire ( + _In_ _Interlocked_operand_ BOOLEAN const volatile *Source) +{ + return (BOOLEAN)ReadAcquire8((char*)Source); +} + +__forceinline +unsigned char +ReadBooleanNoFence ( + _In_ _Interlocked_operand_ BOOLEAN const volatile *Source) +{ + return (BOOLEAN)ReadNoFence8((char*)Source); +} + +__forceinline +void +WriteBooleanRelease ( + _Out_ _Interlocked_operand_ BOOLEAN volatile *Destination, + _In_ BOOLEAN Value) +{ + WriteRelease8((char*)Destination, (char)Value); +} + +__forceinline +void +WriteBooleanNoFence ( + _Out_ _Interlocked_operand_ BOOLEAN volatile *Destination, + _In_ BOOLEAN Value) +{ + WriteNoFence8((char*)Destination, (char)Value); +} + +__forceinline +unsigned short +ReadUShortAcquire ( + _In_ _Interlocked_operand_ unsigned short const volatile *Source) +{ + return (unsigned short)ReadAcquire16((short*)Source); +} + +__forceinline +unsigned short +ReadUShortNoFence ( + _In_ _Interlocked_operand_ unsigned short const volatile *Source) +{ + return (unsigned short)ReadNoFence16((short*)Source); +} + +__forceinline +unsigned short +ReadUShortRaw ( + _In_ _Interlocked_operand_ unsigned short const volatile *Source) +{ + return (unsigned short)ReadRaw16((short*)Source); +} + +__forceinline +void +WriteUShortRelease ( + _Out_ _Interlocked_operand_ unsigned short volatile *Destination, + _In_ unsigned short Value) +{ + WriteRelease16((short*)Destination, (short)Value); +} + +__forceinline +void +WriteUShortNoFence ( + _Out_ _Interlocked_operand_ unsigned short volatile *Destination, + _In_ unsigned short Value) +{ + WriteNoFence16((short*)Destination, (short)Value); +} + +__forceinline +void +WriteUShortRaw ( + _Out_ _Interlocked_operand_ unsigned short volatile *Destination, + _In_ unsigned short Value) +{ + WriteRaw16((short*)Destination, (short)Value); +} + +__forceinline +unsigned long +ReadULongAcquire ( + _In_ _Interlocked_operand_ unsigned long const volatile *Source) +{ + return (unsigned long)ReadAcquire((long*)Source); +} + +__forceinline +unsigned long +ReadULongNoFence ( + _In_ _Interlocked_operand_ unsigned long const volatile *Source) +{ + return (unsigned long)ReadNoFence((long*)Source); +} + +__forceinline +unsigned long +ReadULongRaw ( + _In_ _Interlocked_operand_ unsigned long const volatile *Source) +{ + return (unsigned long)ReadRaw((long*)Source); +} + +__forceinline +void +WriteULongRelease ( + _Out_ _Interlocked_operand_ unsigned long volatile *Destination, + _In_ unsigned long Value) +{ + WriteRelease((long*)Destination, (long)Value); +} + +__forceinline +void +WriteULongNoFence ( + _Out_ _Interlocked_operand_ unsigned long volatile *Destination, + _In_ unsigned long Value) +{ + WriteNoFence((long*)Destination, (long)Value); +} + +__forceinline +void +WriteULongRaw ( + _Out_ _Interlocked_operand_ unsigned long volatile *Destination, + _In_ unsigned long Value) +{ + WriteRaw((long*)Destination, (long)Value); +} + +__forceinline +unsigned __int64 +ReadULong64Acquire ( + _In_ _Interlocked_operand_ unsigned __int64 const volatile *Source) +{ + return (unsigned __int64)ReadAcquire64((__int64*)Source); +} + +__forceinline +unsigned __int64 +ReadULong64NoFence ( + _In_ _Interlocked_operand_ unsigned __int64 const volatile *Source) +{ + return (unsigned __int64)ReadNoFence64((__int64*)Source); +} + +__forceinline +unsigned __int64 +ReadULong64Raw ( + _In_ _Interlocked_operand_ unsigned __int64 const volatile *Source) +{ + return (unsigned __int64)ReadRaw64((__int64*)Source); +} + +__forceinline +void +WriteULong64Release ( + _Out_ _Interlocked_operand_ unsigned __int64 volatile *Destination, + _In_ unsigned __int64 Value) +{ + WriteRelease64((__int64*)Destination, (__int64)Value); +} + +__forceinline +void +WriteULong64NoFence ( + _Out_ _Interlocked_operand_ unsigned __int64 volatile *Destination, + _In_ unsigned __int64 Value) +{ + WriteNoFence64((__int64*)Destination, (__int64)Value); +} + +__forceinline +void +WriteULong64Raw ( + _Out_ _Interlocked_operand_ unsigned __int64 volatile *Destination, + _In_ unsigned __int64 Value) +{ + WriteRaw64((__int64*)Destination, (__int64)Value); +} + +#ifdef _WIN64 + +__forceinline +void* +ReadPointerAcquire ( + _In_ _Interlocked_operand_ void* const volatile *Source) +{ + return (void*)ReadAcquire64((__int64*)Source); +} + +__forceinline +void* +ReadPointerNoFence ( + _In_ _Interlocked_operand_ void* const volatile *Source) +{ + return (void*)ReadNoFence64((__int64*)Source); +} + +__forceinline +void* +ReadPointerRaw ( + _In_ _Interlocked_operand_ void* const volatile *Source) +{ + return (void*)ReadRaw64((__int64*)Source); +} + +__forceinline +void +WritePointerRelease ( + _Out_ _Interlocked_operand_ void* volatile *Destination, + _In_ void* Value) +{ + WriteRelease64((__int64*)Destination, (__int64)Value); +} + +__forceinline +void +WritePointerNoFence ( + _Out_ _Interlocked_operand_ void* volatile *Destination, + _In_ void* Value) +{ + WriteNoFence64((__int64*)Destination, (__int64)Value); +} + +__forceinline +void +WritePointerRaw ( + _Out_ _Interlocked_operand_ void* volatile *Destination, + _In_ void* Value) +{ + WriteRaw64((__int64*)Destination, (__int64)Value); +} + +#define ReadLongPtrAcquire ReadAcquire64 +#define ReadLongPtrNoFence ReadNoFence64 +#define ReadLongPtrRaw ReadRaw64 +#define WriteLongPtrRelease WriteRelease64 +#define WriteLongPtrNoFence WriteNoFence64 +#define WriteLongPtrRaw WriteRaw64 +#define ReadULongPtrAcquire ReadULong64Acquire +#define ReadULongPtrNoFence ReadULong64NoFence +#define ReadULongPtrRaw ReadULong64Raw +#define WriteULongPtrRelease WriteULong64Release +#define WriteULongPtrNoFence WriteULong64NoFence +#define WriteULongPtrRaw WriteULong64Raw + +#else // _WIN64 + +__forceinline +void* +ReadPointerAcquire ( + _In_ _Interlocked_operand_ void* const volatile *Source) +{ + return (void*)ReadAcquire((long*)Source); +} + +__forceinline +void* +ReadPointerNoFence ( + _In_ _Interlocked_operand_ void* const volatile *Source) +{ + return (void*)ReadNoFence((long*)Source); +} + +__forceinline +void* +ReadPointerRaw ( + _In_ _Interlocked_operand_ void* const volatile *Source) +{ + return (void*)ReadRaw((long*)Source); +} + +__forceinline +void +WritePointerRelease ( + _Out_ _Interlocked_operand_ void* volatile *Destination, + _In_ void* Value) +{ + WriteRelease((long*)Destination, (long)Value); +} + +__forceinline +void +WritePointerNoFence ( + _Out_ _Interlocked_operand_ void* volatile *Destination, + _In_opt_ void* Value) +{ + WriteNoFence((long*)Destination, (long)Value); +} + +__forceinline +void +WritePointerRaw ( + _Out_ _Interlocked_operand_ void* volatile *Destination, + _In_opt_ void* Value) +{ + WriteRaw((long*)Destination, (long)Value); +} + +#define ReadLongPtrAcquire ReadAcquire +#define ReadLongPtrNoFence ReadNoFence +#define ReadLongPtrRaw ReadRaw +#define WriteLongPtrRelease WriteRelease +#define WriteLongPtrNoFence WriteNoFence +#define WriteLongPtrRaw WriteRaw +#define ReadULongPtrAcquire ReadULongAcquire +#define ReadULongPtrNoFence ReadULongNoFence +#define ReadULongPtrRaw ReadULongRaw +#define WriteULongPtrRelease WriteULongRelease +#define WriteULongPtrNoFence WriteULongNoFence +#define WriteULongPtrRaw WriteULongRaw + +#endif // _WIN64 + +#define ReadSizeTAcquire ReadULongPtrAcquire +#define ReadSizeTNoFence ReadULongPtrNoFence +#define ReadSizeTRaw ReadULongPtrRaw +#define WriteSizeTRelease WriteULongPtrRelease +#define WriteSizeTNoFence WriteULongPtrNoFence +#define WriteSizeTRaw WriteULongPtrRaw + +/* Overloaded functions for C++ */ +#if defined(__cplusplus) +extern "C++" { + + template + __forceinline + T + ReadRaw ( + _In_ _Interlocked_operand_ T const volatile *Source) + { + return *(T*)Source; + } + + template + __forceinline + void + WriteRaw ( + _Out_ _Interlocked_operand_ T volatile *Destination, + _In_ T Value) + { + *(T*)Destination = Value; + } + + template + __forceinline + T + ReadNoFence ( + _In_ _Interlocked_operand_ T const volatile *Source) + { + switch (sizeof(T)) + { + case 1: return (T)ReadNoFence8((char const volatile *)Source); + case 2: return (T)ReadNoFence16((short const volatile *)Source); + case 4: return (T)ReadNoFence((long const volatile *)Source); + case 8: return (T)ReadNoFence64((__int64 const volatile *)Source); + } + } + + template + __forceinline + void + WriteNoFence ( + _Out_ _Interlocked_operand_ T volatile *Destination, + _In_ T Value) + { + switch (sizeof(T)) + { + case 1: WriteNoFence8((char volatile *)Destination, (char)Value); + case 2: WriteNoFence16((short volatile *)Destination, (short)Value); + case 4: WriteNoFence((long volatile *)Destination, (long)Value); + case 8: WriteNoFence64((__int64 volatile *)Destination, (__int64)Value); + } + } + +} // extern "C++" +#endif // __cplusplus + +#endif // !defined(RC_INVOKED) + +#undef _AcquireBarrier +#undef _ReleaseBarrier diff --git a/sdk/include/xdk/wdm.template.h b/sdk/include/xdk/wdm.template.h index 956bd6ce2ac..7031dec880b 100644 --- a/sdk/include/xdk/wdm.template.h +++ b/sdk/include/xdk/wdm.template.h @@ -240,6 +240,7 @@ inline int IsEqualGUIDAligned(REFGUID guid1, REFGUID guid2) $define (_WDMDDK_) +$include (memaccess.h) $include (interlocked.h) $include (rtltypes.h) $include (ketypes.h) diff --git a/sdk/include/xdk/winnt.template.h b/sdk/include/xdk/winnt.template.h index 0da46e9285b..4334b32a049 100644 --- a/sdk/include/xdk/winnt.template.h +++ b/sdk/include/xdk/winnt.template.h @@ -71,6 +71,7 @@ $define(ULONG=DWORD) $define(USHORT=WORD) $define(UCHAR=BYTE) $include(ntbasedef.h) +$include(memaccess.h) $include(interlocked.h) $include(ketypes.h) $include(extypes.h) diff --git a/sdk/include/xdk/winnt_old.h b/sdk/include/xdk/winnt_old.h index e618e6347cf..58ff1c0763f 100644 --- a/sdk/include/xdk/winnt_old.h +++ b/sdk/include/xdk/winnt_old.h @@ -4572,55 +4572,6 @@ FORCEINLINE PVOID GetFiberData(VOID) return *((PVOID *)GetCurrentFiber()); } -/* TODO: Other architectures than X86 */ -#if defined(_M_IX86) -#define PF_TEMPORAL_LEVEL_1 -#define PF_NON_TEMPORAL_LEVEL_ALL -#define PreFetchCacheLine(l, a) -#elif defined (_M_AMD64) -#define PreFetchCacheLine(l, a) -#elif defined(_M_PPC) -#define PreFetchCacheLine(l, a) -#elif defined(_M_ARM) -#define PreFetchCacheLine(l, a) -#elif defined(_M_ARM64) -#define PreFetchCacheLine(l, a) -#else -#error Unknown architecture -#endif - -/* TODO: Other architectures than X86 */ -#if defined(_M_IX86) -#if defined(_MSC_VER) -FORCEINLINE -VOID -MemoryBarrier(VOID) -{ - LONG Barrier; - __asm { xchg Barrier, eax } -} -#else -FORCEINLINE -VOID -MemoryBarrier(VOID) -{ - LONG Barrier, *Dummy = &Barrier; - (VOID)Dummy; - __asm__ __volatile__("xchgl %%eax, %[Barrier]" : : [Barrier] "m" (Barrier) : "memory"); -} -#endif -#elif defined (_M_AMD64) -#define MemoryBarrier __faststorefence -#elif defined(_M_PPC) -#define MemoryBarrier() -#elif defined(_M_ARM) -#define MemoryBarrier() -#elif defined(_M_ARM64) -#define MemoryBarrier() -#else -#error Unknown architecture -#endif - #if defined(_M_IX86) || defined(_M_AMD64) #define YieldProcessor _mm_pause