13 #include "kmp_atomic.h" 16 typedef unsigned char uchar;
17 typedef unsigned short ushort;
565 #ifndef KMP_GOMP_COMPAT 566 int __kmp_atomic_mode = 1;
568 int __kmp_atomic_mode = 2;
574 kmp_atomic_lock_t __kmp_atomic_lock;
576 kmp_atomic_lock_t __kmp_atomic_lock_1i;
578 kmp_atomic_lock_t __kmp_atomic_lock_2i;
580 kmp_atomic_lock_t __kmp_atomic_lock_4i;
582 kmp_atomic_lock_t __kmp_atomic_lock_4r;
584 kmp_atomic_lock_t __kmp_atomic_lock_8i;
586 kmp_atomic_lock_t __kmp_atomic_lock_8r;
588 kmp_atomic_lock_t __kmp_atomic_lock_8c;
590 kmp_atomic_lock_t __kmp_atomic_lock_10r;
592 kmp_atomic_lock_t __kmp_atomic_lock_16r;
594 kmp_atomic_lock_t __kmp_atomic_lock_16c;
596 kmp_atomic_lock_t __kmp_atomic_lock_20c;
598 kmp_atomic_lock_t __kmp_atomic_lock_32c;
605 #define KMP_ATOMIC_VOLATILE volatile 607 #if (KMP_ARCH_X86) && KMP_HAVE_QUAD 609 static inline void operator+=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
612 static inline void operator-=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
615 static inline void operator*=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
618 static inline void operator/=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
621 static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
622 return lhs.q < rhs.q;
624 static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
625 return lhs.q > rhs.q;
628 static inline void operator+=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
631 static inline void operator-=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
634 static inline void operator*=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
637 static inline void operator/=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
640 static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
641 return lhs.q < rhs.q;
643 static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
644 return lhs.q > rhs.q;
647 static inline void operator+=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
650 static inline void operator-=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
653 static inline void operator*=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
656 static inline void operator/=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
660 static inline void operator+=(kmp_cmplx128_a16_t &lhs,
661 kmp_cmplx128_a16_t &rhs) {
664 static inline void operator-=(kmp_cmplx128_a16_t &lhs,
665 kmp_cmplx128_a16_t &rhs) {
668 static inline void operator*=(kmp_cmplx128_a16_t &lhs,
669 kmp_cmplx128_a16_t &rhs) {
672 static inline void operator/=(kmp_cmplx128_a16_t &lhs,
673 kmp_cmplx128_a16_t &rhs) {
677 #endif // (KMP_ARCH_X86) && KMP_HAVE_QUAD 684 #define KMP_CHECK_GTID \ 685 if (gtid == KMP_GTID_UNKNOWN) { \ 686 gtid = __kmp_entry_gtid(); \ 687 } // check and get gtid when needed 694 #define ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 695 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 696 TYPE *lhs, TYPE rhs) { \ 697 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 698 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 702 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat 703 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char 704 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short 705 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int 706 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float 707 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int 708 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double 709 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex 710 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double 711 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad 712 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex 713 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex 714 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex 722 #define OP_CRITICAL(OP, LCK_ID) \ 723 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 727 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 751 #ifdef KMP_GOMP_COMPAT 752 #define OP_GOMP_CRITICAL(OP, FLAG) \ 753 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 755 OP_CRITICAL(OP, 0); \ 759 #define OP_GOMP_CRITICAL(OP, FLAG) 763 #define KMP_DO_PAUSE _mm_delay_32(1) 765 #define KMP_DO_PAUSE KMP_CPU_PAUSE() 773 #define OP_CMPXCHG(TYPE, BITS, OP) \ 775 TYPE old_value, new_value; \ 776 old_value = *(TYPE volatile *)lhs; \ 777 new_value = old_value OP rhs; \ 778 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 779 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 780 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 783 old_value = *(TYPE volatile *)lhs; \ 784 new_value = old_value OP rhs; \ 795 #define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 799 kmp_int##BITS *vvv; \ 801 struct _sss old_value, new_value; \ 802 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \ 803 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \ 804 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ 805 new_value.cmp = old_value.cmp OP rhs; \ 806 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 807 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \ 808 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \ 811 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ 812 new_value.cmp = old_value.cmp OP rhs; \ 816 #endif // USE_CMPXCHG_FIX 818 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 822 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 824 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 825 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 827 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 830 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 832 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 833 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 834 OP_CMPXCHG(TYPE, BITS, OP) \ 839 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \ 841 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 842 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 843 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 846 #endif // USE_CMPXCHG_FIX 851 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 853 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 854 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 855 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 857 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 860 OP_CRITICAL(OP## =, LCK_ID) \ 864 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 866 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 867 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 868 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 869 OP_CMPXCHG(TYPE, BITS, OP) \ 872 OP_CRITICAL(OP## =, LCK_ID) \ 878 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \ 880 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 881 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 882 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 883 OP_CMPXCHG(TYPE, BITS, OP) \ 886 OP_CRITICAL(OP## =, LCK_ID) \ 890 #endif // USE_CMPXCHG_FIX 894 ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3,
896 ATOMIC_FIXED_ADD(fixed4, sub, kmp_int32, 32, -, 4i, 3,
899 ATOMIC_CMPXCHG(float4, add, kmp_real32, 32, +, 4r, 3,
901 ATOMIC_CMPXCHG(float4, sub, kmp_real32, 32, -, 4r, 3,
905 ATOMIC_FIXED_ADD(fixed8, add, kmp_int64, 64, +, 8i, 7,
907 ATOMIC_FIXED_ADD(fixed8, sub, kmp_int64, 64, -, 8i, 7,
910 ATOMIC_CMPXCHG(float8, add, kmp_real64, 64, +, 8r, 7,
912 ATOMIC_CMPXCHG(float8, sub, kmp_real64, 64, -, 8r, 7,
930 ATOMIC_CMPXCHG(fixed1, add, kmp_int8, 8, +, 1i, 0,
932 ATOMIC_CMPXCHG(fixed1, andb, kmp_int8, 8, &, 1i, 0,
934 ATOMIC_CMPXCHG(fixed1, div, kmp_int8, 8, /, 1i, 0,
936 ATOMIC_CMPXCHG(fixed1u, div, kmp_uint8, 8, /, 1i, 0,
938 ATOMIC_CMPXCHG(fixed1, mul, kmp_int8, 8, *, 1i, 0,
940 ATOMIC_CMPXCHG(fixed1, orb, kmp_int8, 8, |, 1i, 0,
942 ATOMIC_CMPXCHG(fixed1, shl, kmp_int8, 8, <<, 1i, 0,
944 ATOMIC_CMPXCHG(fixed1, shr, kmp_int8, 8, >>, 1i, 0,
946 ATOMIC_CMPXCHG(fixed1u, shr, kmp_uint8, 8, >>, 1i, 0,
948 ATOMIC_CMPXCHG(fixed1, sub, kmp_int8, 8, -, 1i, 0,
950 ATOMIC_CMPXCHG(fixed1, xor, kmp_int8, 8, ^, 1i, 0,
952 ATOMIC_CMPXCHG(fixed2, add, kmp_int16, 16, +, 2i, 1,
954 ATOMIC_CMPXCHG(fixed2, andb, kmp_int16, 16, &, 2i, 1,
956 ATOMIC_CMPXCHG(fixed2, div, kmp_int16, 16, /, 2i, 1,
958 ATOMIC_CMPXCHG(fixed2u, div, kmp_uint16, 16, /, 2i, 1,
960 ATOMIC_CMPXCHG(fixed2, mul, kmp_int16, 16, *, 2i, 1,
962 ATOMIC_CMPXCHG(fixed2, orb, kmp_int16, 16, |, 2i, 1,
964 ATOMIC_CMPXCHG(fixed2, shl, kmp_int16, 16, <<, 2i, 1,
966 ATOMIC_CMPXCHG(fixed2, shr, kmp_int16, 16, >>, 2i, 1,
968 ATOMIC_CMPXCHG(fixed2u, shr, kmp_uint16, 16, >>, 2i, 1,
970 ATOMIC_CMPXCHG(fixed2, sub, kmp_int16, 16, -, 2i, 1,
972 ATOMIC_CMPXCHG(fixed2, xor, kmp_int16, 16, ^, 2i, 1,
974 ATOMIC_CMPXCHG(fixed4, andb, kmp_int32, 32, &, 4i, 3,
976 ATOMIC_CMPXCHG(fixed4, div, kmp_int32, 32, /, 4i, 3,
978 ATOMIC_CMPXCHG(fixed4u, div, kmp_uint32, 32, /, 4i, 3,
980 ATOMIC_CMPXCHG(fixed4, mul, kmp_int32, 32, *, 4i, 3,
982 ATOMIC_CMPXCHG(fixed4, orb, kmp_int32, 32, |, 4i, 3,
984 ATOMIC_CMPXCHG(fixed4, shl, kmp_int32, 32, <<, 4i, 3,
986 ATOMIC_CMPXCHG(fixed4, shr, kmp_int32, 32, >>, 4i, 3,
988 ATOMIC_CMPXCHG(fixed4u, shr, kmp_uint32, 32, >>, 4i, 3,
990 ATOMIC_CMPXCHG(fixed4, xor, kmp_int32, 32, ^, 4i, 3,
992 ATOMIC_CMPXCHG(fixed8, andb, kmp_int64, 64, &, 8i, 7,
994 ATOMIC_CMPXCHG(fixed8, div, kmp_int64, 64, /, 8i, 7,
996 ATOMIC_CMPXCHG(fixed8u, div, kmp_uint64, 64, /, 8i, 7,
998 ATOMIC_CMPXCHG(fixed8, mul, kmp_int64, 64, *, 8i, 7,
1000 ATOMIC_CMPXCHG(fixed8, orb, kmp_int64, 64, |, 8i, 7,
1002 ATOMIC_CMPXCHG(fixed8, shl, kmp_int64, 64, <<, 8i, 7,
1004 ATOMIC_CMPXCHG(fixed8, shr, kmp_int64, 64, >>, 8i, 7,
1006 ATOMIC_CMPXCHG(fixed8u, shr, kmp_uint64, 64, >>, 8i, 7,
1008 ATOMIC_CMPXCHG(fixed8, xor, kmp_int64, 64, ^, 8i, 7,
1010 ATOMIC_CMPXCHG(float4, div, kmp_real32, 32, /, 4r, 3,
1012 ATOMIC_CMPXCHG(float4, mul, kmp_real32, 32, *, 4r, 3,
1014 ATOMIC_CMPXCHG(float8, div, kmp_real64, 64, /, 8r, 7,
1016 ATOMIC_CMPXCHG(float8, mul, kmp_real64, 64, *, 8r, 7,
1026 #define ATOMIC_CRIT_L(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1027 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1028 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1029 OP_CRITICAL(= *lhs OP, LCK_ID) \ 1032 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1036 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \ 1037 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1038 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1039 OP_CMPXCHG(TYPE, BITS, OP) \ 1045 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \ 1046 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1047 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1048 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1049 OP_CMPXCHG(TYPE, BITS, OP) \ 1052 OP_CRITICAL(= *lhs OP, LCK_ID) \ 1057 ATOMIC_CMPX_L(fixed1, andl,
char, 8, &&, 1i, 0,
1059 ATOMIC_CMPX_L(fixed1, orl,
char, 8, ||, 1i, 0,
1061 ATOMIC_CMPX_L(fixed2, andl,
short, 16, &&, 2i, 1,
1063 ATOMIC_CMPX_L(fixed2, orl,
short, 16, ||, 2i, 1,
1065 ATOMIC_CMPX_L(fixed4, andl, kmp_int32, 32, &&, 4i, 3,
1067 ATOMIC_CMPX_L(fixed4, orl, kmp_int32, 32, ||, 4i, 3,
1069 ATOMIC_CMPX_L(fixed8, andl, kmp_int64, 64, &&, 8i, 7,
1071 ATOMIC_CMPX_L(fixed8, orl, kmp_int64, 64, ||, 8i, 7,
1083 #define MIN_MAX_CRITSECT(OP, LCK_ID) \ 1084 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1086 if (*lhs OP rhs) { \ 1089 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1092 #ifdef KMP_GOMP_COMPAT 1093 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) \ 1094 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1096 MIN_MAX_CRITSECT(OP, 0); \ 1100 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) 1104 #define MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1106 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1109 old_value = temp_val; \ 1110 while (old_value OP rhs && \ 1111 !KMP_COMPARE_AND_STORE_ACQ##BITS( \ 1112 (kmp_int##BITS *)lhs, \ 1113 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 1114 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \ 1117 old_value = temp_val; \ 1123 #define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1124 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1125 if (*lhs OP rhs) { \ 1126 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1127 MIN_MAX_CRITSECT(OP, LCK_ID) \ 1131 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1135 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1137 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1138 if (*lhs OP rhs) { \ 1139 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1140 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1147 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1149 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1150 if (*lhs OP rhs) { \ 1151 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1152 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1153 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1156 MIN_MAX_CRITSECT(OP, LCK_ID) \ 1162 MIN_MAX_COMPXCHG(fixed1, max,
char, 8, <, 1i, 0,
1164 MIN_MAX_COMPXCHG(fixed1, min,
char, 8, >, 1i, 0,
1166 MIN_MAX_COMPXCHG(fixed2, max,
short, 16, <, 2i, 1,
1168 MIN_MAX_COMPXCHG(fixed2, min,
short, 16, >, 2i, 1,
1170 MIN_MAX_COMPXCHG(fixed4, max, kmp_int32, 32, <, 4i, 3,
1172 MIN_MAX_COMPXCHG(fixed4, min, kmp_int32, 32, >, 4i, 3,
1174 MIN_MAX_COMPXCHG(fixed8, max, kmp_int64, 64, <, 8i, 7,
1176 MIN_MAX_COMPXCHG(fixed8, min, kmp_int64, 64, >, 8i, 7,
1178 MIN_MAX_COMPXCHG(float4, max, kmp_real32, 32, <, 4r, 3,
1180 MIN_MAX_COMPXCHG(float4, min, kmp_real32, 32, >, 4r, 3,
1182 MIN_MAX_COMPXCHG(float8, max, kmp_real64, 64, <, 8r, 7,
1184 MIN_MAX_COMPXCHG(float8, min, kmp_real64, 64, >, 8r, 7,
1187 MIN_MAX_CRITICAL(float16, max, QUAD_LEGACY, <, 16r,
1189 MIN_MAX_CRITICAL(float16, min, QUAD_LEGACY, >, 16r,
1192 MIN_MAX_CRITICAL(float16, max_a16, Quad_a16_t, <, 16r,
1194 MIN_MAX_CRITICAL(float16, min_a16, Quad_a16_t, >, 16r,
1196 #endif // (KMP_ARCH_X86) 1197 #endif // KMP_HAVE_QUAD 1201 #define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1202 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1203 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \ 1204 OP_CRITICAL(^= ~, LCK_ID) \ 1208 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1211 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1213 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1214 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \ 1215 OP_CMPXCHG(TYPE, BITS, OP) \ 1221 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1223 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1224 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \ 1225 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1226 OP_CMPXCHG(TYPE, BITS, OP) \ 1229 OP_CRITICAL(^= ~, LCK_ID) \ 1234 ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0,
1236 ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1,
1238 ATOMIC_CMPXCHG(fixed4, neqv, kmp_int32, 32, ^, 4i, 3,
1240 ATOMIC_CMPXCHG(fixed8, neqv, kmp_int64, 64, ^, 8i, 7,
1242 ATOMIC_CMPX_EQV(fixed1, eqv, kmp_int8, 8, ^~, 1i, 0,
1244 ATOMIC_CMPX_EQV(fixed2, eqv, kmp_int16, 16, ^~, 2i, 1,
1246 ATOMIC_CMPX_EQV(fixed4, eqv, kmp_int32, 32, ^~, 4i, 3,
1248 ATOMIC_CMPX_EQV(fixed8, eqv, kmp_int64, 64, ^~, 8i, 7,
1257 #define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1258 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1259 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1260 OP_CRITICAL(OP## =, LCK_ID) \ 1265 ATOMIC_CRITICAL(float10, add,
long double, +, 10r,
1267 ATOMIC_CRITICAL(float10, sub,
long double, -, 10r,
1269 ATOMIC_CRITICAL(float10, mul,
long double, *, 10r,
1271 ATOMIC_CRITICAL(float10, div,
long double, /, 10r,
1275 ATOMIC_CRITICAL(float16, add, QUAD_LEGACY, +, 16r,
1277 ATOMIC_CRITICAL(float16, sub, QUAD_LEGACY, -, 16r,
1279 ATOMIC_CRITICAL(float16, mul, QUAD_LEGACY, *, 16r,
1281 ATOMIC_CRITICAL(float16, div, QUAD_LEGACY, /, 16r,
1284 ATOMIC_CRITICAL(float16, add_a16, Quad_a16_t, +, 16r,
1286 ATOMIC_CRITICAL(float16, sub_a16, Quad_a16_t, -, 16r,
1288 ATOMIC_CRITICAL(float16, mul_a16, Quad_a16_t, *, 16r,
1290 ATOMIC_CRITICAL(float16, div_a16, Quad_a16_t, /, 16r,
1292 #endif // (KMP_ARCH_X86) 1293 #endif // KMP_HAVE_QUAD 1298 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, add, kmp_cmplx32, 64, +, 8c, 7,
1300 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7,
1302 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7,
1304 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, div, kmp_cmplx32, 64, /, 8c, 7,
1308 ATOMIC_CRITICAL(cmplx4, add, kmp_cmplx32, +, 8c, 1)
1309 ATOMIC_CRITICAL(cmplx4, sub, kmp_cmplx32, -, 8c, 1)
1310 ATOMIC_CRITICAL(cmplx4, mul, kmp_cmplx32, *, 8c, 1)
1311 ATOMIC_CRITICAL(cmplx4, div, kmp_cmplx32, /, 8c, 1)
1312 #endif // USE_CMPXCHG_FIX 1314 ATOMIC_CRITICAL(cmplx8, add, kmp_cmplx64, +, 16c, 1)
1315 ATOMIC_CRITICAL(cmplx8, sub, kmp_cmplx64, -, 16c, 1)
1316 ATOMIC_CRITICAL(cmplx8, mul, kmp_cmplx64, *, 16c, 1)
1317 ATOMIC_CRITICAL(cmplx8, div, kmp_cmplx64, /, 16c, 1)
1318 ATOMIC_CRITICAL(cmplx10, add, kmp_cmplx80, +, 20c,
1320 ATOMIC_CRITICAL(cmplx10, sub, kmp_cmplx80, -, 20c,
1322 ATOMIC_CRITICAL(cmplx10, mul, kmp_cmplx80, *, 20c,
1324 ATOMIC_CRITICAL(cmplx10, div, kmp_cmplx80, /, 20c,
1327 ATOMIC_CRITICAL(cmplx16, add, CPLX128_LEG, +, 32c,
1329 ATOMIC_CRITICAL(cmplx16, sub, CPLX128_LEG, -, 32c,
1331 ATOMIC_CRITICAL(cmplx16, mul, CPLX128_LEG, *, 32c,
1333 ATOMIC_CRITICAL(cmplx16, div, CPLX128_LEG, /, 32c,
1336 ATOMIC_CRITICAL(cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c,
1338 ATOMIC_CRITICAL(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1340 ATOMIC_CRITICAL(cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c,
1342 ATOMIC_CRITICAL(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1344 #endif // (KMP_ARCH_X86) 1345 #endif // KMP_HAVE_QUAD 1349 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1357 #define OP_CRITICAL_REV(OP, LCK_ID) \ 1358 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1360 (*lhs) = (rhs)OP(*lhs); \ 1362 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1364 #ifdef KMP_GOMP_COMPAT 1365 #define OP_GOMP_CRITICAL_REV(OP, FLAG) \ 1366 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1368 OP_CRITICAL_REV(OP, 0); \ 1372 #define OP_GOMP_CRITICAL_REV(OP, FLAG) 1380 #define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 1381 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev(ident_t *id_ref, int gtid, \ 1382 TYPE *lhs, TYPE rhs) { \ 1383 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1384 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid)); 1393 #define OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1395 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1396 TYPE old_value, new_value; \ 1398 old_value = temp_val; \ 1399 new_value = rhs OP old_value; \ 1400 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 1401 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 1402 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 1406 old_value = temp_val; \ 1407 new_value = rhs OP old_value; \ 1412 #define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \ 1413 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \ 1414 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ 1415 OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1432 ATOMIC_CMPXCHG_REV(fixed1, div, kmp_int8, 8, /, 1i,
1434 ATOMIC_CMPXCHG_REV(fixed1u, div, kmp_uint8, 8, /, 1i,
1436 ATOMIC_CMPXCHG_REV(fixed1, shl, kmp_int8, 8, <<, 1i,
1438 ATOMIC_CMPXCHG_REV(fixed1, shr, kmp_int8, 8, >>, 1i,
1440 ATOMIC_CMPXCHG_REV(fixed1u, shr, kmp_uint8, 8, >>, 1i,
1442 ATOMIC_CMPXCHG_REV(fixed1, sub, kmp_int8, 8, -, 1i,
1445 ATOMIC_CMPXCHG_REV(fixed2, div, kmp_int16, 16, /, 2i,
1447 ATOMIC_CMPXCHG_REV(fixed2u, div, kmp_uint16, 16, /, 2i,
1449 ATOMIC_CMPXCHG_REV(fixed2, shl, kmp_int16, 16, <<, 2i,
1451 ATOMIC_CMPXCHG_REV(fixed2, shr, kmp_int16, 16, >>, 2i,
1453 ATOMIC_CMPXCHG_REV(fixed2u, shr, kmp_uint16, 16, >>, 2i,
1455 ATOMIC_CMPXCHG_REV(fixed2, sub, kmp_int16, 16, -, 2i,
1458 ATOMIC_CMPXCHG_REV(fixed4, div, kmp_int32, 32, /, 4i,
1460 ATOMIC_CMPXCHG_REV(fixed4u, div, kmp_uint32, 32, /, 4i,
1462 ATOMIC_CMPXCHG_REV(fixed4, shl, kmp_int32, 32, <<, 4i,
1464 ATOMIC_CMPXCHG_REV(fixed4, shr, kmp_int32, 32, >>, 4i,
1466 ATOMIC_CMPXCHG_REV(fixed4u, shr, kmp_uint32, 32, >>, 4i,
1468 ATOMIC_CMPXCHG_REV(fixed4, sub, kmp_int32, 32, -, 4i,
1471 ATOMIC_CMPXCHG_REV(fixed8, div, kmp_int64, 64, /, 8i,
1473 ATOMIC_CMPXCHG_REV(fixed8u, div, kmp_uint64, 64, /, 8i,
1475 ATOMIC_CMPXCHG_REV(fixed8, shl, kmp_int64, 64, <<, 8i,
1477 ATOMIC_CMPXCHG_REV(fixed8, shr, kmp_int64, 64, >>, 8i,
1479 ATOMIC_CMPXCHG_REV(fixed8u, shr, kmp_uint64, 64, >>, 8i,
1481 ATOMIC_CMPXCHG_REV(fixed8, sub, kmp_int64, 64, -, 8i,
1484 ATOMIC_CMPXCHG_REV(float4, div, kmp_real32, 32, /, 4r,
1486 ATOMIC_CMPXCHG_REV(float4, sub, kmp_real32, 32, -, 4r,
1489 ATOMIC_CMPXCHG_REV(float8, div, kmp_real64, 64, /, 8r,
1491 ATOMIC_CMPXCHG_REV(float8, sub, kmp_real64, 64, -, 8r,
1501 #define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1502 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \ 1503 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ 1504 OP_CRITICAL_REV(OP, LCK_ID) \ 1509 ATOMIC_CRITICAL_REV(float10, sub,
long double, -, 10r,
1511 ATOMIC_CRITICAL_REV(float10, div,
long double, /, 10r,
1515 ATOMIC_CRITICAL_REV(float16, sub, QUAD_LEGACY, -, 16r,
1517 ATOMIC_CRITICAL_REV(float16, div, QUAD_LEGACY, /, 16r,
1520 ATOMIC_CRITICAL_REV(float16, sub_a16, Quad_a16_t, -, 16r,
1522 ATOMIC_CRITICAL_REV(float16, div_a16, Quad_a16_t, /, 16r,
1524 #endif // KMP_ARCH_X86 1525 #endif // KMP_HAVE_QUAD 1528 ATOMIC_CRITICAL_REV(cmplx4, sub, kmp_cmplx32, -, 8c,
1530 ATOMIC_CRITICAL_REV(cmplx4, div, kmp_cmplx32, /, 8c,
1532 ATOMIC_CRITICAL_REV(cmplx8, sub, kmp_cmplx64, -, 16c,
1534 ATOMIC_CRITICAL_REV(cmplx8, div, kmp_cmplx64, /, 16c,
1536 ATOMIC_CRITICAL_REV(cmplx10, sub, kmp_cmplx80, -, 20c,
1538 ATOMIC_CRITICAL_REV(cmplx10, div, kmp_cmplx80, /, 20c,
1541 ATOMIC_CRITICAL_REV(cmplx16, sub, CPLX128_LEG, -, 32c,
1543 ATOMIC_CRITICAL_REV(cmplx16, div, CPLX128_LEG, /, 32c,
1546 ATOMIC_CRITICAL_REV(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1548 ATOMIC_CRITICAL_REV(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1550 #endif // KMP_ARCH_X86 1551 #endif // KMP_HAVE_QUAD 1553 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 1567 #define ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1568 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \ 1569 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs) { \ 1570 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1572 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \ 1576 #define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \ 1578 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1579 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1580 OP_CRITICAL(OP## =, LCK_ID) \ 1584 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1587 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1588 LCK_ID, MASK, GOMP_FLAG) \ 1589 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1590 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1591 OP_CMPXCHG(TYPE, BITS, OP) \ 1597 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1598 LCK_ID, MASK, GOMP_FLAG) \ 1599 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1600 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1601 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1602 OP_CMPXCHG(TYPE, BITS, OP) \ 1605 OP_CRITICAL(OP## =, LCK_ID) \ 1611 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1613 #define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 1614 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 1615 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1616 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ 1617 OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1619 #define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 1620 LCK_ID, GOMP_FLAG) \ 1621 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1622 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ 1623 OP_CRITICAL_REV(OP, LCK_ID) \ 1628 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0,
1630 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0,
1632 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1,
1634 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1,
1636 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3,
1638 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3,
1640 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7,
1642 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7,
1644 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3,
1646 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3,
1648 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3,
1650 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3,
1656 ATOMIC_CMPXCHG_MIX(fixed1,
char, add, 8, +, fp, _Quad, 1i, 0,
1658 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, add, 8, +, fp, _Quad, 1i, 0,
1660 ATOMIC_CMPXCHG_MIX(fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0,
1662 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, sub, 8, -, fp, _Quad, 1i, 0,
1664 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0,
1666 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, mul, 8, *, fp, _Quad, 1i, 0,
1668 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, fp, _Quad, 1i, 0,
1670 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0,
1673 ATOMIC_CMPXCHG_MIX(fixed2,
short, add, 16, +, fp, _Quad, 2i, 1,
1675 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, add, 16, +, fp, _Quad, 2i, 1,
1677 ATOMIC_CMPXCHG_MIX(fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1,
1679 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, sub, 16, -, fp, _Quad, 2i, 1,
1681 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1,
1683 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, mul, 16, *, fp, _Quad, 2i, 1,
1685 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, fp, _Quad, 2i, 1,
1687 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1,
1690 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3,
1692 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3,
1694 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3,
1696 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3,
1698 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3,
1700 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3,
1702 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3,
1704 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3,
1707 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7,
1709 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7,
1711 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7,
1713 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7,
1715 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7,
1717 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7,
1719 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7,
1721 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7,
1724 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3,
1726 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3,
1728 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3,
1730 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3,
1733 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7,
1735 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7,
1737 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7,
1739 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7,
1742 ATOMIC_CRITICAL_FP(float10,
long double, add, +, fp, _Quad, 10r,
1744 ATOMIC_CRITICAL_FP(float10,
long double, sub, -, fp, _Quad, 10r,
1746 ATOMIC_CRITICAL_FP(float10,
long double, mul, *, fp, _Quad, 10r,
1748 ATOMIC_CRITICAL_FP(float10,
long double, div, /, fp, _Quad, 10r,
1751 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1753 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, sub_rev, 8, -, fp, _Quad, 1i, 0,
1755 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, sub_rev, 8, -, fp, _Quad, 1i, 0,
1757 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, div_rev, 8, /, fp, _Quad, 1i, 0,
1759 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, div_rev, 8, /, fp, _Quad, 1i, 0,
1762 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, sub_rev, 16, -, fp, _Quad, 2i, 1,
1764 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, sub_rev, 16, -, fp, _Quad, 2i, 1,
1766 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, div_rev, 16, /, fp, _Quad, 2i, 1,
1768 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, div_rev, 16, /, fp, _Quad, 2i, 1,
1771 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1773 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1775 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, div_rev, 32, /, fp, _Quad, 4i, 3,
1777 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3,
1780 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1782 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1784 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, div_rev, 64, /, fp, _Quad, 8i, 7,
1786 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7,
1789 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3,
1791 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3,
1794 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7,
1796 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7,
1799 ATOMIC_CRITICAL_REV_FP(float10,
long double, sub_rev, -, fp, _Quad, 10r,
1801 ATOMIC_CRITICAL_REV_FP(float10,
long double, div_rev, /, fp, _Quad, 10r,
1805 #endif // KMP_HAVE_QUAD 1807 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1812 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1813 LCK_ID, MASK, GOMP_FLAG) \ 1814 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1815 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1816 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 1820 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1821 LCK_ID, MASK, GOMP_FLAG) \ 1822 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1823 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1824 OP_CMPXCHG(TYPE, BITS, OP) \ 1826 #endif // USE_CMPXCHG_FIX 1830 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1831 LCK_ID, MASK, GOMP_FLAG) \ 1832 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1833 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1834 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1835 OP_CMPXCHG(TYPE, BITS, OP) \ 1838 OP_CRITICAL(OP## =, LCK_ID) \ 1843 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c,
1845 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c,
1847 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c,
1849 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c,
1853 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1864 #define ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 1865 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 1867 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1868 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 1879 #define OP_CMPXCHG_READ(TYPE, BITS, OP) \ 1881 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1884 kmp_int##BITS i_val; \ 1886 union f_i_union old_value; \ 1888 old_value.f_val = temp_val; \ 1889 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( \ 1890 (kmp_int##BITS *)loc, \ 1891 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val, \ 1892 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val); \ 1893 new_value = old_value.f_val; \ 1903 #define OP_CRITICAL_READ(OP, LCK_ID) \ 1904 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1906 new_value = (*loc); \ 1908 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1911 #ifdef KMP_GOMP_COMPAT 1912 #define OP_GOMP_CRITICAL_READ(OP, FLAG) \ 1913 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1915 OP_CRITICAL_READ(OP, 0); \ 1919 #define OP_GOMP_CRITICAL_READ(OP, FLAG) 1923 #define ATOMIC_FIXED_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 1924 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 1926 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 1927 new_value = KMP_TEST_THEN_ADD##BITS(loc, OP 0); \ 1931 #define ATOMIC_CMPXCHG_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 1932 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 1934 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 1935 OP_CMPXCHG_READ(TYPE, BITS, OP) \ 1943 #define ATOMIC_CRITICAL_READ(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1944 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 1946 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 1947 OP_CRITICAL_READ(OP, LCK_ID) \ 1955 #if (KMP_OS_WINDOWS) 1957 #define OP_CRITICAL_READ_WRK(OP, LCK_ID) \ 1958 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1962 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1964 #ifdef KMP_GOMP_COMPAT 1965 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) \ 1966 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1968 OP_CRITICAL_READ_WRK(OP, 0); \ 1971 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) 1974 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \ 1975 void __kmpc_atomic_##TYPE_ID##_##OP_ID(TYPE *out, ident_t *id_ref, int gtid, \ 1977 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1978 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 1981 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1982 ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \ 1983 OP_GOMP_CRITICAL_READ_WRK(OP## =, GOMP_FLAG) \ 1984 OP_CRITICAL_READ_WRK(OP, LCK_ID) \ 1987 #endif // KMP_OS_WINDOWS 1991 ATOMIC_FIXED_READ(fixed4, rd, kmp_int32, 32, +, 0)
1992 ATOMIC_FIXED_READ(fixed8, rd, kmp_int64, 64, +,
1994 ATOMIC_CMPXCHG_READ(float4, rd, kmp_real32, 32, +,
1996 ATOMIC_CMPXCHG_READ(float8, rd, kmp_real64, 64, +,
2000 ATOMIC_CMPXCHG_READ(fixed1, rd, kmp_int8, 8, +,
2002 ATOMIC_CMPXCHG_READ(fixed2, rd, kmp_int16, 16, +,
2005 ATOMIC_CRITICAL_READ(float10, rd,
long double, +, 10r,
2008 ATOMIC_CRITICAL_READ(float16, rd, QUAD_LEGACY, +, 16r,
2010 #endif // KMP_HAVE_QUAD 2013 #if (KMP_OS_WINDOWS) 2014 ATOMIC_CRITICAL_READ_WRK(cmplx4, rd, kmp_cmplx32, +, 8c,
2017 ATOMIC_CRITICAL_READ(cmplx4, rd, kmp_cmplx32, +, 8c,
2019 #endif // (KMP_OS_WINDOWS) 2020 ATOMIC_CRITICAL_READ(cmplx8, rd, kmp_cmplx64, +, 16c,
2022 ATOMIC_CRITICAL_READ(cmplx10, rd, kmp_cmplx80, +, 20c,
2025 ATOMIC_CRITICAL_READ(cmplx16, rd, CPLX128_LEG, +, 32c,
2028 ATOMIC_CRITICAL_READ(float16, a16_rd, Quad_a16_t, +, 16r,
2030 ATOMIC_CRITICAL_READ(cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c,
2032 #endif // (KMP_ARCH_X86) 2033 #endif // KMP_HAVE_QUAD 2038 #define ATOMIC_XCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2039 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2040 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2041 KMP_XCHG_FIXED##BITS(lhs, rhs); \ 2044 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2045 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2046 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2047 KMP_XCHG_REAL##BITS(lhs, rhs); \ 2057 #define OP_CMPXCHG_WR(TYPE, BITS, OP) \ 2059 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2060 TYPE old_value, new_value; \ 2062 old_value = temp_val; \ 2064 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2065 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2066 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2070 old_value = temp_val; \ 2076 #define ATOMIC_CMPXCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2077 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2078 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2079 OP_CMPXCHG_WR(TYPE, BITS, OP) \ 2088 #define ATOMIC_CRITICAL_WR(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2089 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2090 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2091 OP_CRITICAL(OP, LCK_ID) \ 2095 ATOMIC_XCHG_WR(fixed1, wr, kmp_int8, 8, =,
2097 ATOMIC_XCHG_WR(fixed2, wr, kmp_int16, 16, =,
2099 ATOMIC_XCHG_WR(fixed4, wr, kmp_int32, 32, =,
2102 ATOMIC_CMPXCHG_WR(fixed8, wr, kmp_int64, 64, =,
2105 ATOMIC_XCHG_WR(fixed8, wr, kmp_int64, 64, =,
2107 #endif // (KMP_ARCH_X86) 2109 ATOMIC_XCHG_FLOAT_WR(float4, wr, kmp_real32, 32, =,
2112 ATOMIC_CMPXCHG_WR(float8, wr, kmp_real64, 64, =,
2115 ATOMIC_XCHG_FLOAT_WR(float8, wr, kmp_real64, 64, =,
2117 #endif // (KMP_ARCH_X86) 2119 ATOMIC_CRITICAL_WR(float10, wr,
long double, =, 10r,
2122 ATOMIC_CRITICAL_WR(float16, wr, QUAD_LEGACY, =, 16r,
2124 #endif // KMP_HAVE_QUAD 2125 ATOMIC_CRITICAL_WR(cmplx4, wr, kmp_cmplx32, =, 8c, 1)
2126 ATOMIC_CRITICAL_WR(cmplx8, wr, kmp_cmplx64, =, 16c,
2128 ATOMIC_CRITICAL_WR(cmplx10, wr, kmp_cmplx80, =, 20c,
2131 ATOMIC_CRITICAL_WR(cmplx16, wr, CPLX128_LEG, =, 32c,
2134 ATOMIC_CRITICAL_WR(float16, a16_wr, Quad_a16_t, =, 16r,
2136 ATOMIC_CRITICAL_WR(cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c,
2138 #endif // (KMP_ARCH_X86) 2139 #endif // KMP_HAVE_QUAD 2149 #define ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 2150 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 2151 TYPE *lhs, TYPE rhs, int flag) { \ 2152 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2153 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 2161 #define OP_CRITICAL_CPT(OP, LCK_ID) \ 2162 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2166 new_value = (*lhs); \ 2168 new_value = (*lhs); \ 2172 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2176 #ifdef KMP_GOMP_COMPAT 2177 #define OP_GOMP_CRITICAL_CPT(OP, FLAG) \ 2178 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2180 OP_CRITICAL_CPT(OP## =, 0); \ 2183 #define OP_GOMP_CRITICAL_CPT(OP, FLAG) 2193 #define OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2195 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2196 TYPE old_value, new_value; \ 2198 old_value = temp_val; \ 2199 new_value = old_value OP rhs; \ 2200 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2201 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2202 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2206 old_value = temp_val; \ 2207 new_value = old_value OP rhs; \ 2216 #define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2217 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2219 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ 2220 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2224 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2225 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2226 TYPE old_value, new_value; \ 2227 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ 2229 old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 2231 return old_value OP rhs; \ 2237 ATOMIC_FIXED_ADD_CPT(fixed4, add_cpt, kmp_int32, 32, +,
2239 ATOMIC_FIXED_ADD_CPT(fixed4, sub_cpt, kmp_int32, 32, -,
2241 ATOMIC_FIXED_ADD_CPT(fixed8, add_cpt, kmp_int64, 64, +,
2243 ATOMIC_FIXED_ADD_CPT(fixed8, sub_cpt, kmp_int64, 64, -,
2246 ATOMIC_CMPXCHG_CPT(float4, add_cpt, kmp_real32, 32, +,
2248 ATOMIC_CMPXCHG_CPT(float4, sub_cpt, kmp_real32, 32, -,
2250 ATOMIC_CMPXCHG_CPT(float8, add_cpt, kmp_real64, 64, +,
2252 ATOMIC_CMPXCHG_CPT(float8, sub_cpt, kmp_real64, 64, -,
2267 ATOMIC_CMPXCHG_CPT(fixed1, add_cpt, kmp_int8, 8, +,
2269 ATOMIC_CMPXCHG_CPT(fixed1, andb_cpt, kmp_int8, 8, &,
2271 ATOMIC_CMPXCHG_CPT(fixed1, div_cpt, kmp_int8, 8, /,
2273 ATOMIC_CMPXCHG_CPT(fixed1u, div_cpt, kmp_uint8, 8, /,
2275 ATOMIC_CMPXCHG_CPT(fixed1, mul_cpt, kmp_int8, 8, *,
2277 ATOMIC_CMPXCHG_CPT(fixed1, orb_cpt, kmp_int8, 8, |,
2279 ATOMIC_CMPXCHG_CPT(fixed1, shl_cpt, kmp_int8, 8, <<,
2281 ATOMIC_CMPXCHG_CPT(fixed1, shr_cpt, kmp_int8, 8, >>,
2283 ATOMIC_CMPXCHG_CPT(fixed1u, shr_cpt, kmp_uint8, 8, >>,
2285 ATOMIC_CMPXCHG_CPT(fixed1, sub_cpt, kmp_int8, 8, -,
2287 ATOMIC_CMPXCHG_CPT(fixed1, xor_cpt, kmp_int8, 8, ^,
2289 ATOMIC_CMPXCHG_CPT(fixed2, add_cpt, kmp_int16, 16, +,
2291 ATOMIC_CMPXCHG_CPT(fixed2, andb_cpt, kmp_int16, 16, &,
2293 ATOMIC_CMPXCHG_CPT(fixed2, div_cpt, kmp_int16, 16, /,
2295 ATOMIC_CMPXCHG_CPT(fixed2u, div_cpt, kmp_uint16, 16, /,
2297 ATOMIC_CMPXCHG_CPT(fixed2, mul_cpt, kmp_int16, 16, *,
2299 ATOMIC_CMPXCHG_CPT(fixed2, orb_cpt, kmp_int16, 16, |,
2301 ATOMIC_CMPXCHG_CPT(fixed2, shl_cpt, kmp_int16, 16, <<,
2303 ATOMIC_CMPXCHG_CPT(fixed2, shr_cpt, kmp_int16, 16, >>,
2305 ATOMIC_CMPXCHG_CPT(fixed2u, shr_cpt, kmp_uint16, 16, >>,
2307 ATOMIC_CMPXCHG_CPT(fixed2, sub_cpt, kmp_int16, 16, -,
2309 ATOMIC_CMPXCHG_CPT(fixed2, xor_cpt, kmp_int16, 16, ^,
2311 ATOMIC_CMPXCHG_CPT(fixed4, andb_cpt, kmp_int32, 32, &,
2313 ATOMIC_CMPXCHG_CPT(fixed4, div_cpt, kmp_int32, 32, /,
2315 ATOMIC_CMPXCHG_CPT(fixed4u, div_cpt, kmp_uint32, 32, /,
2317 ATOMIC_CMPXCHG_CPT(fixed4, mul_cpt, kmp_int32, 32, *,
2319 ATOMIC_CMPXCHG_CPT(fixed4, orb_cpt, kmp_int32, 32, |,
2321 ATOMIC_CMPXCHG_CPT(fixed4, shl_cpt, kmp_int32, 32, <<,
2323 ATOMIC_CMPXCHG_CPT(fixed4, shr_cpt, kmp_int32, 32, >>,
2325 ATOMIC_CMPXCHG_CPT(fixed4u, shr_cpt, kmp_uint32, 32, >>,
2327 ATOMIC_CMPXCHG_CPT(fixed4, xor_cpt, kmp_int32, 32, ^,
2329 ATOMIC_CMPXCHG_CPT(fixed8, andb_cpt, kmp_int64, 64, &,
2331 ATOMIC_CMPXCHG_CPT(fixed8, div_cpt, kmp_int64, 64, /,
2333 ATOMIC_CMPXCHG_CPT(fixed8u, div_cpt, kmp_uint64, 64, /,
2335 ATOMIC_CMPXCHG_CPT(fixed8, mul_cpt, kmp_int64, 64, *,
2337 ATOMIC_CMPXCHG_CPT(fixed8, orb_cpt, kmp_int64, 64, |,
2339 ATOMIC_CMPXCHG_CPT(fixed8, shl_cpt, kmp_int64, 64, <<,
2341 ATOMIC_CMPXCHG_CPT(fixed8, shr_cpt, kmp_int64, 64, >>,
2343 ATOMIC_CMPXCHG_CPT(fixed8u, shr_cpt, kmp_uint64, 64, >>,
2345 ATOMIC_CMPXCHG_CPT(fixed8, xor_cpt, kmp_int64, 64, ^,
2347 ATOMIC_CMPXCHG_CPT(float4, div_cpt, kmp_real32, 32, /,
2349 ATOMIC_CMPXCHG_CPT(float4, mul_cpt, kmp_real32, 32, *,
2351 ATOMIC_CMPXCHG_CPT(float8, div_cpt, kmp_real64, 64, /,
2353 ATOMIC_CMPXCHG_CPT(float8, mul_cpt, kmp_real64, 64, *,
2365 #define ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2366 TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \ 2367 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs, int flag) { \ 2368 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2370 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \ 2374 #define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 2375 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 2376 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2378 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ 2379 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2383 #define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 2384 LCK_ID, GOMP_FLAG) \ 2385 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2387 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ 2388 OP_CRITICAL_CPT(OP## =, LCK_ID) \ 2391 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, add_cpt, 8, +, fp, _Quad, 1i, 0,
2393 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0,
2395 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2397 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2399 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2401 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2403 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, div_cpt, 8, /, fp, _Quad, 1i, 0,
2405 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, div_cpt, 8, /, fp, _Quad, 1i, 0,
2408 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, add_cpt, 16, +, fp, _Quad, 2i, 1,
2410 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, add_cpt, 16, +, fp, _Quad, 2i, 1,
2412 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2414 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2416 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2418 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2420 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, div_cpt, 16, /, fp, _Quad, 2i, 1,
2422 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, div_cpt, 16, /, fp, _Quad, 2i, 1,
2425 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2427 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2429 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2431 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2433 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2435 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2437 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2439 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2442 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2444 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2446 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2448 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2450 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2452 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2454 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2456 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2459 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3,
2461 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3,
2463 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3,
2465 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3,
2468 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7,
2470 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7,
2472 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7,
2474 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7,
2477 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, add_cpt, +, fp, _Quad, 10r,
2479 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, sub_cpt, -, fp, _Quad, 10r,
2481 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, mul_cpt, *, fp, _Quad, 10r,
2483 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, div_cpt, /, fp, _Quad, 10r,
2486 #endif // KMP_HAVE_QUAD 2497 #define OP_CRITICAL_L_CPT(OP, LCK_ID) \ 2498 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2503 new_value = (*lhs); \ 2505 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 2508 #ifdef KMP_GOMP_COMPAT 2509 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) \ 2510 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2512 OP_CRITICAL_L_CPT(OP, 0); \ 2516 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) 2521 #define ATOMIC_CMPX_L_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2522 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2524 OP_GOMP_CRITICAL_L_CPT(= *lhs OP, GOMP_FLAG) \ 2525 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2528 ATOMIC_CMPX_L_CPT(fixed1, andl_cpt,
char, 8, &&,
2530 ATOMIC_CMPX_L_CPT(fixed1, orl_cpt,
char, 8, ||,
2532 ATOMIC_CMPX_L_CPT(fixed2, andl_cpt,
short, 16, &&,
2534 ATOMIC_CMPX_L_CPT(fixed2, orl_cpt,
short, 16, ||,
2536 ATOMIC_CMPX_L_CPT(fixed4, andl_cpt, kmp_int32, 32, &&,
2538 ATOMIC_CMPX_L_CPT(fixed4, orl_cpt, kmp_int32, 32, ||,
2540 ATOMIC_CMPX_L_CPT(fixed8, andl_cpt, kmp_int64, 64, &&,
2542 ATOMIC_CMPX_L_CPT(fixed8, orl_cpt, kmp_int64, 64, ||,
2554 #define MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \ 2555 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2557 if (*lhs OP rhs) { \ 2563 new_value = old_value; \ 2567 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2571 #ifdef KMP_GOMP_COMPAT 2572 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) \ 2573 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2575 MIN_MAX_CRITSECT_CPT(OP, 0); \ 2578 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) 2582 #define MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \ 2584 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2587 old_value = temp_val; \ 2588 while (old_value OP rhs && \ 2589 !KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2590 (kmp_int##BITS *)lhs, \ 2591 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2592 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \ 2595 old_value = temp_val; \ 2605 #define MIN_MAX_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2606 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2607 TYPE new_value, old_value; \ 2608 if (*lhs OP rhs) { \ 2609 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \ 2610 MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \ 2615 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2616 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2617 TYPE new_value, old_value; \ 2618 if (*lhs OP rhs) { \ 2619 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \ 2620 MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \ 2625 MIN_MAX_COMPXCHG_CPT(fixed1, max_cpt,
char, 8, <,
2627 MIN_MAX_COMPXCHG_CPT(fixed1, min_cpt,
char, 8, >,
2629 MIN_MAX_COMPXCHG_CPT(fixed2, max_cpt,
short, 16, <,
2631 MIN_MAX_COMPXCHG_CPT(fixed2, min_cpt,
short, 16, >,
2633 MIN_MAX_COMPXCHG_CPT(fixed4, max_cpt, kmp_int32, 32, <,
2635 MIN_MAX_COMPXCHG_CPT(fixed4, min_cpt, kmp_int32, 32, >,
2637 MIN_MAX_COMPXCHG_CPT(fixed8, max_cpt, kmp_int64, 64, <,
2639 MIN_MAX_COMPXCHG_CPT(fixed8, min_cpt, kmp_int64, 64, >,
2641 MIN_MAX_COMPXCHG_CPT(float4, max_cpt, kmp_real32, 32, <,
2643 MIN_MAX_COMPXCHG_CPT(float4, min_cpt, kmp_real32, 32, >,
2645 MIN_MAX_COMPXCHG_CPT(float8, max_cpt, kmp_real64, 64, <,
2647 MIN_MAX_COMPXCHG_CPT(float8, min_cpt, kmp_real64, 64, >,
2650 MIN_MAX_CRITICAL_CPT(float16, max_cpt, QUAD_LEGACY, <, 16r,
2652 MIN_MAX_CRITICAL_CPT(float16, min_cpt, QUAD_LEGACY, >, 16r,
2655 MIN_MAX_CRITICAL_CPT(float16, max_a16_cpt, Quad_a16_t, <, 16r,
2657 MIN_MAX_CRITICAL_CPT(float16, min_a16_cpt, Quad_a16_t, >, 16r,
2659 #endif // (KMP_ARCH_X86) 2660 #endif // KMP_HAVE_QUAD 2663 #ifdef KMP_GOMP_COMPAT 2664 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) \ 2665 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2667 OP_CRITICAL_CPT(OP, 0); \ 2670 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) 2673 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2674 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2676 OP_GOMP_CRITICAL_EQV_CPT(^= ~, GOMP_FLAG) \ 2677 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2682 ATOMIC_CMPXCHG_CPT(fixed1, neqv_cpt, kmp_int8, 8, ^,
2684 ATOMIC_CMPXCHG_CPT(fixed2, neqv_cpt, kmp_int16, 16, ^,
2686 ATOMIC_CMPXCHG_CPT(fixed4, neqv_cpt, kmp_int32, 32, ^,
2688 ATOMIC_CMPXCHG_CPT(fixed8, neqv_cpt, kmp_int64, 64, ^,
2690 ATOMIC_CMPX_EQV_CPT(fixed1, eqv_cpt, kmp_int8, 8, ^~,
2692 ATOMIC_CMPX_EQV_CPT(fixed2, eqv_cpt, kmp_int16, 16, ^~,
2694 ATOMIC_CMPX_EQV_CPT(fixed4, eqv_cpt, kmp_int32, 32, ^~,
2696 ATOMIC_CMPX_EQV_CPT(fixed8, eqv_cpt, kmp_int64, 64, ^~,
2705 #define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2706 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2708 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ 2709 OP_CRITICAL_CPT(OP## =, LCK_ID) \ 2715 #define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \ 2716 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2726 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2730 #ifdef KMP_GOMP_COMPAT 2731 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) \ 2732 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2734 OP_CRITICAL_CPT_WRK(OP## =, 0); \ 2737 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) 2741 #define ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 2742 void __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, TYPE *lhs, \ 2743 TYPE rhs, TYPE *out, int flag) { \ 2744 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2745 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 2748 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2749 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 2750 OP_GOMP_CRITICAL_CPT_WRK(OP, GOMP_FLAG) \ 2751 OP_CRITICAL_CPT_WRK(OP## =, LCK_ID) \ 2757 ATOMIC_CRITICAL_CPT(float10, add_cpt,
long double, +, 10r,
2759 ATOMIC_CRITICAL_CPT(float10, sub_cpt,
long double, -, 10r,
2761 ATOMIC_CRITICAL_CPT(float10, mul_cpt,
long double, *, 10r,
2763 ATOMIC_CRITICAL_CPT(float10, div_cpt,
long double, /, 10r,
2767 ATOMIC_CRITICAL_CPT(float16, add_cpt, QUAD_LEGACY, +, 16r,
2769 ATOMIC_CRITICAL_CPT(float16, sub_cpt, QUAD_LEGACY, -, 16r,
2771 ATOMIC_CRITICAL_CPT(float16, mul_cpt, QUAD_LEGACY, *, 16r,
2773 ATOMIC_CRITICAL_CPT(float16, div_cpt, QUAD_LEGACY, /, 16r,
2776 ATOMIC_CRITICAL_CPT(float16, add_a16_cpt, Quad_a16_t, +, 16r,
2778 ATOMIC_CRITICAL_CPT(float16, sub_a16_cpt, Quad_a16_t, -, 16r,
2780 ATOMIC_CRITICAL_CPT(float16, mul_a16_cpt, Quad_a16_t, *, 16r,
2782 ATOMIC_CRITICAL_CPT(float16, div_a16_cpt, Quad_a16_t, /, 16r,
2784 #endif // (KMP_ARCH_X86) 2785 #endif // KMP_HAVE_QUAD 2790 ATOMIC_CRITICAL_CPT_WRK(cmplx4, add_cpt, kmp_cmplx32, +, 8c,
2792 ATOMIC_CRITICAL_CPT_WRK(cmplx4, sub_cpt, kmp_cmplx32, -, 8c,
2794 ATOMIC_CRITICAL_CPT_WRK(cmplx4, mul_cpt, kmp_cmplx32, *, 8c,
2796 ATOMIC_CRITICAL_CPT_WRK(cmplx4, div_cpt, kmp_cmplx32, /, 8c,
2799 ATOMIC_CRITICAL_CPT(cmplx8, add_cpt, kmp_cmplx64, +, 16c,
2801 ATOMIC_CRITICAL_CPT(cmplx8, sub_cpt, kmp_cmplx64, -, 16c,
2803 ATOMIC_CRITICAL_CPT(cmplx8, mul_cpt, kmp_cmplx64, *, 16c,
2805 ATOMIC_CRITICAL_CPT(cmplx8, div_cpt, kmp_cmplx64, /, 16c,
2807 ATOMIC_CRITICAL_CPT(cmplx10, add_cpt, kmp_cmplx80, +, 20c,
2809 ATOMIC_CRITICAL_CPT(cmplx10, sub_cpt, kmp_cmplx80, -, 20c,
2811 ATOMIC_CRITICAL_CPT(cmplx10, mul_cpt, kmp_cmplx80, *, 20c,
2813 ATOMIC_CRITICAL_CPT(cmplx10, div_cpt, kmp_cmplx80, /, 20c,
2816 ATOMIC_CRITICAL_CPT(cmplx16, add_cpt, CPLX128_LEG, +, 32c,
2818 ATOMIC_CRITICAL_CPT(cmplx16, sub_cpt, CPLX128_LEG, -, 32c,
2820 ATOMIC_CRITICAL_CPT(cmplx16, mul_cpt, CPLX128_LEG, *, 32c,
2822 ATOMIC_CRITICAL_CPT(cmplx16, div_cpt, CPLX128_LEG, /, 32c,
2825 ATOMIC_CRITICAL_CPT(cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c,
2827 ATOMIC_CRITICAL_CPT(cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c,
2829 ATOMIC_CRITICAL_CPT(cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c,
2831 ATOMIC_CRITICAL_CPT(cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c,
2833 #endif // (KMP_ARCH_X86) 2834 #endif // KMP_HAVE_QUAD 2846 #define OP_CRITICAL_CPT_REV(OP, LCK_ID) \ 2847 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2851 (*lhs) = (rhs)OP(*lhs); \ 2852 new_value = (*lhs); \ 2854 new_value = (*lhs); \ 2855 (*lhs) = (rhs)OP(*lhs); \ 2857 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2861 #ifdef KMP_GOMP_COMPAT 2862 #define OP_GOMP_CRITICAL_CPT_REV(OP, FLAG) \ 2863 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2865 OP_CRITICAL_CPT_REV(OP, 0); \ 2868 #define OP_GOMP_CRITICAL_CPT_REV(OP, FLAG) 2878 #define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 2880 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2881 TYPE old_value, new_value; \ 2883 old_value = temp_val; \ 2884 new_value = rhs OP old_value; \ 2885 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2886 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2887 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2891 old_value = temp_val; \ 2892 new_value = rhs OP old_value; \ 2901 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2902 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2904 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \ 2905 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 2908 ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev, kmp_int8, 8, /,
2910 ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev, kmp_uint8, 8, /,
2912 ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev, kmp_int8, 8, <<,
2914 ATOMIC_CMPXCHG_CPT_REV(fixed1, shr_cpt_rev, kmp_int8, 8, >>,
2916 ATOMIC_CMPXCHG_CPT_REV(fixed1u, shr_cpt_rev, kmp_uint8, 8, >>,
2918 ATOMIC_CMPXCHG_CPT_REV(fixed1, sub_cpt_rev, kmp_int8, 8, -,
2920 ATOMIC_CMPXCHG_CPT_REV(fixed2, div_cpt_rev, kmp_int16, 16, /,
2922 ATOMIC_CMPXCHG_CPT_REV(fixed2u, div_cpt_rev, kmp_uint16, 16, /,
2924 ATOMIC_CMPXCHG_CPT_REV(fixed2, shl_cpt_rev, kmp_int16, 16, <<,
2926 ATOMIC_CMPXCHG_CPT_REV(fixed2, shr_cpt_rev, kmp_int16, 16, >>,
2928 ATOMIC_CMPXCHG_CPT_REV(fixed2u, shr_cpt_rev, kmp_uint16, 16, >>,
2930 ATOMIC_CMPXCHG_CPT_REV(fixed2, sub_cpt_rev, kmp_int16, 16, -,
2932 ATOMIC_CMPXCHG_CPT_REV(fixed4, div_cpt_rev, kmp_int32, 32, /,
2934 ATOMIC_CMPXCHG_CPT_REV(fixed4u, div_cpt_rev, kmp_uint32, 32, /,
2936 ATOMIC_CMPXCHG_CPT_REV(fixed4, shl_cpt_rev, kmp_int32, 32, <<,
2938 ATOMIC_CMPXCHG_CPT_REV(fixed4, shr_cpt_rev, kmp_int32, 32, >>,
2940 ATOMIC_CMPXCHG_CPT_REV(fixed4u, shr_cpt_rev, kmp_uint32, 32, >>,
2942 ATOMIC_CMPXCHG_CPT_REV(fixed4, sub_cpt_rev, kmp_int32, 32, -,
2944 ATOMIC_CMPXCHG_CPT_REV(fixed8, div_cpt_rev, kmp_int64, 64, /,
2946 ATOMIC_CMPXCHG_CPT_REV(fixed8u, div_cpt_rev, kmp_uint64, 64, /,
2948 ATOMIC_CMPXCHG_CPT_REV(fixed8, shl_cpt_rev, kmp_int64, 64, <<,
2950 ATOMIC_CMPXCHG_CPT_REV(fixed8, shr_cpt_rev, kmp_int64, 64, >>,
2952 ATOMIC_CMPXCHG_CPT_REV(fixed8u, shr_cpt_rev, kmp_uint64, 64, >>,
2954 ATOMIC_CMPXCHG_CPT_REV(fixed8, sub_cpt_rev, kmp_int64, 64, -,
2956 ATOMIC_CMPXCHG_CPT_REV(float4, div_cpt_rev, kmp_real32, 32, /,
2958 ATOMIC_CMPXCHG_CPT_REV(float4, sub_cpt_rev, kmp_real32, 32, -,
2960 ATOMIC_CMPXCHG_CPT_REV(float8, div_cpt_rev, kmp_real64, 64, /,
2962 ATOMIC_CMPXCHG_CPT_REV(float8, sub_cpt_rev, kmp_real64, 64, -,
2972 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2973 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2976 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \ 2977 OP_CRITICAL_CPT_REV(OP, LCK_ID) \ 2982 ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev,
long double, -, 10r,
2984 ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev,
long double, /, 10r,
2988 ATOMIC_CRITICAL_CPT_REV(float16, sub_cpt_rev, QUAD_LEGACY, -, 16r,
2990 ATOMIC_CRITICAL_CPT_REV(float16, div_cpt_rev, QUAD_LEGACY, /, 16r,
2993 ATOMIC_CRITICAL_CPT_REV(float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r,
2995 ATOMIC_CRITICAL_CPT_REV(float16, div_a16_cpt_rev, Quad_a16_t, /, 16r,
2997 #endif // (KMP_ARCH_X86) 2998 #endif // KMP_HAVE_QUAD 3005 #define OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \ 3006 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3009 (*lhs) = (rhs)OP(*lhs); \ 3013 (*lhs) = (rhs)OP(*lhs); \ 3016 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3020 #ifdef KMP_GOMP_COMPAT 3021 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) \ 3022 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3024 OP_CRITICAL_CPT_REV_WRK(OP, 0); \ 3027 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) 3031 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, \ 3033 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 3034 OP_GOMP_CRITICAL_CPT_REV_WRK(OP, GOMP_FLAG) \ 3035 OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \ 3041 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c,
3043 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c,
3046 ATOMIC_CRITICAL_CPT_REV(cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c,
3048 ATOMIC_CRITICAL_CPT_REV(cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c,
3050 ATOMIC_CRITICAL_CPT_REV(cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c,
3052 ATOMIC_CRITICAL_CPT_REV(cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c,
3055 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c,
3057 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c,
3060 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c,
3062 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c,
3064 #endif // (KMP_ARCH_X86) 3065 #endif // KMP_HAVE_QUAD 3076 #define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 3077 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 3078 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 3080 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \ 3081 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 3085 #define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 3086 LCK_ID, GOMP_FLAG) \ 3087 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 3089 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \ 3090 OP_CRITICAL_CPT_REV(OP, LCK_ID) \ 3093 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3095 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3097 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3099 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3102 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1,
3104 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, sub_cpt_rev, 16, -, fp, _Quad, 2i,
3107 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, div_cpt_rev, 16, /, fp, _Quad, 2i, 1,
3109 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, div_cpt_rev, 16, /, fp, _Quad, 2i,
3113 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4i,
3115 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad,
3117 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4i,
3119 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad,
3122 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8i,
3125 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad,
3128 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8i,
3131 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad,
3135 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, sub_cpt_rev, 32, -, fp, _Quad,
3138 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, div_cpt_rev, 32, /, fp, _Quad,
3142 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, sub_cpt_rev, 64, -, fp, _Quad,
3145 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, div_cpt_rev, 64, /, fp, _Quad,
3149 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, sub_cpt_rev, -, fp, _Quad,
3151 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, div_cpt_rev, /, fp, _Quad,
3154 #endif // KMP_HAVE_QUAD 3158 #define ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3159 TYPE __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \ 3161 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 3162 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid)); 3164 #define CRITICAL_SWP(LCK_ID) \ 3165 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3167 old_value = (*lhs); \ 3170 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3174 #ifdef KMP_GOMP_COMPAT 3175 #define GOMP_CRITICAL_SWP(FLAG) \ 3176 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3181 #define GOMP_CRITICAL_SWP(FLAG) 3184 #define ATOMIC_XCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3185 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3187 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3188 old_value = KMP_XCHG_FIXED##BITS(lhs, rhs); \ 3192 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3193 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3195 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3196 old_value = KMP_XCHG_REAL##BITS(lhs, rhs); \ 3201 #define CMPXCHG_SWP(TYPE, BITS) \ 3203 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 3204 TYPE old_value, new_value; \ 3206 old_value = temp_val; \ 3208 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 3209 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 3210 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 3214 old_value = temp_val; \ 3221 #define ATOMIC_CMPXCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3222 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3224 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3225 CMPXCHG_SWP(TYPE, BITS) \ 3228 ATOMIC_XCHG_SWP(fixed1, kmp_int8, 8, KMP_ARCH_X86)
3229 ATOMIC_XCHG_SWP(fixed2, kmp_int16, 16, KMP_ARCH_X86)
3230 ATOMIC_XCHG_SWP(fixed4, kmp_int32, 32, KMP_ARCH_X86)
3232 ATOMIC_XCHG_FLOAT_SWP(float4, kmp_real32, 32,
3236 ATOMIC_CMPXCHG_SWP(fixed8, kmp_int64, 64,
3238 ATOMIC_CMPXCHG_SWP(float8, kmp_real64, 64,
3241 ATOMIC_XCHG_SWP(fixed8, kmp_int64, 64, KMP_ARCH_X86)
3242 ATOMIC_XCHG_FLOAT_SWP(float8, kmp_real64, 64,
3244 #endif // (KMP_ARCH_X86) 3249 #define ATOMIC_CRITICAL_SWP(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \ 3250 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3252 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3253 CRITICAL_SWP(LCK_ID) \ 3261 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \ 3262 void __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \ 3263 TYPE rhs, TYPE *out) { \ 3264 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 3265 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid)); 3267 #define CRITICAL_SWP_WRK(LCK_ID) \ 3268 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3273 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3277 #ifdef KMP_GOMP_COMPAT 3278 #define GOMP_CRITICAL_SWP_WRK(FLAG) \ 3279 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3281 CRITICAL_SWP_WRK(0); \ 3284 #define GOMP_CRITICAL_SWP_WRK(FLAG) 3288 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \ 3289 ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \ 3291 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \ 3292 CRITICAL_SWP_WRK(LCK_ID) \ 3296 ATOMIC_CRITICAL_SWP(float10,
long double, 10r, 1)
3298 ATOMIC_CRITICAL_SWP(float16, QUAD_LEGACY, 16r, 1)
3299 #endif // KMP_HAVE_QUAD 3301 ATOMIC_CRITICAL_SWP_WRK(cmplx4, kmp_cmplx32, 8c, 1)
3306 ATOMIC_CRITICAL_SWP(cmplx8, kmp_cmplx64, 16c, 1)
3307 ATOMIC_CRITICAL_SWP(cmplx10, kmp_cmplx80, 20c, 1)
3309 ATOMIC_CRITICAL_SWP(cmplx16, CPLX128_LEG, 32c, 1)
3311 ATOMIC_CRITICAL_SWP(float16_a16, Quad_a16_t, 16r,
3313 ATOMIC_CRITICAL_SWP(cmplx16_a16, kmp_cmplx128_a16_t, 32c,
3315 #endif // (KMP_ARCH_X86) 3316 #endif // KMP_HAVE_QUAD 3320 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 3327 void __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3328 void (*f)(
void *,
void *,
void *)) {
3329 KMP_DEBUG_ASSERT(__kmp_init_serial);
3332 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3338 kmp_int8 old_value, new_value;
3340 old_value = *(kmp_int8 *)lhs;
3341 (*f)(&new_value, &old_value, rhs);
3344 while (!KMP_COMPARE_AND_STORE_ACQ8((kmp_int8 *)lhs, *(kmp_int8 *)&old_value,
3345 *(kmp_int8 *)&new_value)) {
3348 old_value = *(kmp_int8 *)lhs;
3349 (*f)(&new_value, &old_value, rhs);
3356 #ifdef KMP_GOMP_COMPAT 3357 if (__kmp_atomic_mode == 2) {
3358 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3361 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3363 (*f)(lhs, lhs, rhs);
3365 #ifdef KMP_GOMP_COMPAT 3366 if (__kmp_atomic_mode == 2) {
3367 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3370 __kmp_release_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3374 void __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3375 void (*f)(
void *,
void *,
void *)) {
3377 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3379 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3382 !((kmp_uintptr_t)lhs & 0x1)
3385 kmp_int16 old_value, new_value;
3387 old_value = *(kmp_int16 *)lhs;
3388 (*f)(&new_value, &old_value, rhs);
3391 while (!KMP_COMPARE_AND_STORE_ACQ16(
3392 (kmp_int16 *)lhs, *(kmp_int16 *)&old_value, *(kmp_int16 *)&new_value)) {
3395 old_value = *(kmp_int16 *)lhs;
3396 (*f)(&new_value, &old_value, rhs);
3403 #ifdef KMP_GOMP_COMPAT 3404 if (__kmp_atomic_mode == 2) {
3405 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3408 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3410 (*f)(lhs, lhs, rhs);
3412 #ifdef KMP_GOMP_COMPAT 3413 if (__kmp_atomic_mode == 2) {
3414 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3417 __kmp_release_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3421 void __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3422 void (*f)(
void *,
void *,
void *)) {
3423 KMP_DEBUG_ASSERT(__kmp_init_serial);
3428 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
3431 !((kmp_uintptr_t)lhs & 0x3)
3434 kmp_int32 old_value, new_value;
3436 old_value = *(kmp_int32 *)lhs;
3437 (*f)(&new_value, &old_value, rhs);
3440 while (!KMP_COMPARE_AND_STORE_ACQ32(
3441 (kmp_int32 *)lhs, *(kmp_int32 *)&old_value, *(kmp_int32 *)&new_value)) {
3444 old_value = *(kmp_int32 *)lhs;
3445 (*f)(&new_value, &old_value, rhs);
3453 #ifdef KMP_GOMP_COMPAT 3454 if (__kmp_atomic_mode == 2) {
3455 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3458 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3460 (*f)(lhs, lhs, rhs);
3462 #ifdef KMP_GOMP_COMPAT 3463 if (__kmp_atomic_mode == 2) {
3464 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3467 __kmp_release_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3471 void __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3472 void (*f)(
void *,
void *,
void *)) {
3473 KMP_DEBUG_ASSERT(__kmp_init_serial);
3476 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3478 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3481 !((kmp_uintptr_t)lhs & 0x7)
3484 kmp_int64 old_value, new_value;
3486 old_value = *(kmp_int64 *)lhs;
3487 (*f)(&new_value, &old_value, rhs);
3489 while (!KMP_COMPARE_AND_STORE_ACQ64(
3490 (kmp_int64 *)lhs, *(kmp_int64 *)&old_value, *(kmp_int64 *)&new_value)) {
3493 old_value = *(kmp_int64 *)lhs;
3494 (*f)(&new_value, &old_value, rhs);
3502 #ifdef KMP_GOMP_COMPAT 3503 if (__kmp_atomic_mode == 2) {
3504 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3507 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3509 (*f)(lhs, lhs, rhs);
3511 #ifdef KMP_GOMP_COMPAT 3512 if (__kmp_atomic_mode == 2) {
3513 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3516 __kmp_release_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3520 void __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3521 void (*f)(
void *,
void *,
void *)) {
3522 KMP_DEBUG_ASSERT(__kmp_init_serial);
3524 #ifdef KMP_GOMP_COMPAT 3525 if (__kmp_atomic_mode == 2) {
3526 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3529 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3531 (*f)(lhs, lhs, rhs);
3533 #ifdef KMP_GOMP_COMPAT 3534 if (__kmp_atomic_mode == 2) {
3535 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3538 __kmp_release_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3541 void __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3542 void (*f)(
void *,
void *,
void *)) {
3543 KMP_DEBUG_ASSERT(__kmp_init_serial);
3545 #ifdef KMP_GOMP_COMPAT 3546 if (__kmp_atomic_mode == 2) {
3547 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3550 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3552 (*f)(lhs, lhs, rhs);
3554 #ifdef KMP_GOMP_COMPAT 3555 if (__kmp_atomic_mode == 2) {
3556 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3559 __kmp_release_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3562 void __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3563 void (*f)(
void *,
void *,
void *)) {
3564 KMP_DEBUG_ASSERT(__kmp_init_serial);
3566 #ifdef KMP_GOMP_COMPAT 3567 if (__kmp_atomic_mode == 2) {
3568 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3571 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3573 (*f)(lhs, lhs, rhs);
3575 #ifdef KMP_GOMP_COMPAT 3576 if (__kmp_atomic_mode == 2) {
3577 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3580 __kmp_release_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3583 void __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3584 void (*f)(
void *,
void *,
void *)) {
3585 KMP_DEBUG_ASSERT(__kmp_init_serial);
3587 #ifdef KMP_GOMP_COMPAT 3588 if (__kmp_atomic_mode == 2) {
3589 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3592 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3594 (*f)(lhs, lhs, rhs);
3596 #ifdef KMP_GOMP_COMPAT 3597 if (__kmp_atomic_mode == 2) {
3598 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3601 __kmp_release_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3607 void __kmpc_atomic_start(
void) {
3608 int gtid = __kmp_entry_gtid();
3609 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
3610 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3613 void __kmpc_atomic_end(
void) {
3614 int gtid = __kmp_get_gtid();
3615 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
3616 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);